Merge pull request #231 from mrubanov/master

Check memory when performing resource demand loads

com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/DefaultBuilderResourceLoadStrategy.java

@@ -18,7 +18,7 @@
 
 
 /**
- * Determine when to stop loading resources during build.
+ * Determine when to stop loading resources during the build.
  */
 @SuppressWarnings("nls")
 // CHECKSTYLE:CONSTANTS-OFF
@@ -27,10 +27,10 @@ public class DefaultBuilderResourceLoadStrategy implements IBuilderResourceLoadS
   /** Class-wide logger. */
   private static final Logger LOGGER = Logger.getLogger(DefaultBuilderResourceLoadStrategy.class);
 
-  /** Default value for target free memory in megabytes. */
+  /** Property for setting target free memory value. (in megabytes). */
   private static final String TARGET_FREE_MEMORY_PROPERTY = "com.avaloq.tools.ddk.xtext.builder.targetFreeMemory"; //$NON-NLS-1$
 
-  /** Property for setting minimum free memory setting. */
+  /** Property for setting minimum free memory value. (in megabytes) */
   private static final String MIN_FREE_MEMORY_PROPERTY = "com.avaloq.tools.ddk.xtext.builder.minFreeMemory"; //$NON-NLS-1$
 
   /** Property for setting the maximal cluster size. */
@@ -51,16 +51,16 @@ public class DefaultBuilderResourceLoadStrategy implements IBuilderResourceLoadS
   /** Default value for MINIMUM_FREE_MEMORY. */
   private static final long DEFAULT_MIN_FREE_MEMORY = 100 * ONE_MEGABYTE;
 
-  /** Cluster will always end if memory falls bellow this threshold. */
+  /** Cluster will always end if memory falls below this threshold. */
   private static final long MINIMUM_FREE_MEMORY = getMinimumFreeMemory();
 
-  /** Attempt to cap cluster when memory reaches this limit, as long as we processed more than the minimum cluster size. */
+  /** Attempt to cap a cluster when memory reaches this limit, as long as we processed more than the minimum cluster size. */
   private static final long TARGET_FREE_MEMORY = getTargetFreeMemory();
 
   /** Cluster size. */
   @Inject(optional = true)
   @Named("org.eclipse.xtext.builder.clustering.ClusteringBuilderState.clusterSize")
-  private int clusterSize = 20; // NOPMD Cannot be static because of the way Guice injection works.
+  private int clusterSize = 50; // NOPMD Cannot be static because of the way Guice injection works.
 
   /** The Java runtime; needed to get access to memory usage data. */
   private static final Runtime RUNTIME = Runtime.getRuntime();
@@ -100,7 +100,6 @@ public void setClusterSize(final int clusterSize) {
     this.clusterSize = clusterSize;
   }
 
-  /** {@inheritDoc} */
   @Override
   public boolean mayProcessAnotherResource(final ResourceSet resourceSet, final int alreadyProcessed) {
     if (alreadyProcessed == 0) {
@@ -115,13 +114,13 @@ public boolean mayProcessAnotherResource(final ResourceSet resourceSet, final in
       return false;
     }
 
-    final long freeMemory = RUNTIME.freeMemory();
-    if (freeMemory < MINIMUM_FREE_MEMORY) {
+    if (isMemoryLimitReached()) {
       return false;
     } else if (alreadyProcessed < clusterSize) {
       return true; // Process at least up to cluster size
     }
 
+    final long freeMemory = RUNTIME.freeMemory();
     if (freeMemory < TARGET_FREE_MEMORY) {
       // Running GC here might free memory, but would not significantly speed up processing.
       // We may use slightly smaller clusters than strictly necessary without GC, though.
@@ -134,4 +133,10 @@ public boolean mayProcessAnotherResource(final ResourceSet resourceSet, final in
     }
     return true; // Keep on loading resources into this resource set
   }
+
+  @Override
+  public boolean isMemoryLimitReached() {
+    return RUNTIME.freeMemory() < MINIMUM_FREE_MEMORY;
+  }
+
 }

com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/DemandLoadFailedException.java

@@ -0,0 +1,21 @@
+/*******************************************************************************
+ * Copyright (c) 2016 Avaloq Evolution AG and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *     Avaloq Evolution AG - initial API and implementation
+ *******************************************************************************/
+
+package com.avaloq.tools.ddk.xtext.builder;
+
+/**
+ * Exception thrown when a resource set fails to demand-load a resource because of not having enough memory.
+ */
+public class DemandLoadFailedException extends RuntimeException {
+
+  private static final long serialVersionUID = 8162994321500072190L;
+
+}

com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/IBuilderResourceLoadStrategy.java

@@ -22,7 +22,7 @@
 public interface IBuilderResourceLoadStrategy {
 
   /**
-   * Determine whether the builder may, in its second phase, process yet another resource.
+   * Determine whether the builder may process yet another resource.
    *
    * @param resourceSet
    *          The builder's resource set, containing all currently loaded resources.
@@ -32,4 +32,11 @@ public interface IBuilderResourceLoadStrategy {
    */
   boolean mayProcessAnotherResource(ResourceSet resourceSet, int alreadyProcessed);
 
+  /**
+   * Checks whether the builder has reached the minimal amount of free memory, which prevents further progress without clearing the resource set.
+   *
+   * @return {@code true} if the described condition is met, {@code false} otherwise
+   */
+  boolean isMemoryLimitReached();
+
 }

com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/Messages.java

@@ -15,7 +15,9 @@
 
 //CHECKSTYLE:OFF
 public class Messages extends NLS {
+
   private static final String BUNDLE_NAME = "com.avaloq.tools.ddk.xtext.builder.messages"; //$NON-NLS-1$
+
   public static String MonitoredClusteringBuilderState_CANNOT_LOAD_RESOURCE;
   public static String MonitoredClusteringBuilderState_NO_MORE_RESOURCES;
   public static String MonitoredClusteringBuilderState_PHASE_ONE_DONE;

com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/MonitoredClusteringBuilderState.java

@@ -852,11 +852,14 @@ protected boolean recordDeltaAsNew(final Delta newDelta) {
    *          The progress monitor used for user feedback
    * @return the list of {@link URI}s of loaded resources to be processed in the second phase
    */
-  private List<URI> writeResources(final Collection<URI> toWrite, final BuildData buildData, final IResourceDescriptions oldState, final CurrentDescriptions newState, final IProgressMonitor monitor) {
+  private List<URI> writeResources(final Collection<URI> toWrite, final BuildData buildData, final IResourceDescriptions oldState, final CurrentDescriptions newState, final IProgressMonitor monitor) { // NOPMD
     ResourceSet resourceSet = buildData.getResourceSet();
     IProject currentProject = getBuiltProject(buildData);
-    List<URI> toBuild = Lists.newLinkedList();
+    List<URI> toBuild = Lists.newArrayList();
+
     IResourceLoader.LoadOperation loadOperation = null;
+    URI resourceToRetry = null;
+
     try {
       int resourcesToWriteSize = toWrite.size();
       int index = 1;
@@ -866,16 +869,23 @@ private List<URI> writeResources(final Collection<URI> toWrite, final BuildData
 
       // Not using the loadingStrategy here; seems to work fine with a reasonable clusterSize (20 by default), even with
       // large resources and "scarce" memory (say, about 500MB).
-      while (loadOperation.hasNext()) {
+      while (loadOperation.hasNext() || resourceToRetry != null) {
         if (monitor.isCanceled()) {
           loadOperation.cancel();
           throw new OperationCanceledException();
         }
+
         URI uri = null;
         Resource resource = null;
         try {
-          resource = addResource(loadOperation.next().getResource(), resourceSet);
+          if (resourceToRetry == null) {
+            resource = addResource(loadOperation.next().getResource(), resourceSet);
+          } else {
+            resource = resourceSet.getResource(resourceToRetry, true);
+            resourceToRetry = null;
+          }
           uri = resource.getURI();
+
           final Object[] bindings = {Integer.valueOf(index), Integer.valueOf(resourcesToWriteSize), uri.fileExtension(), URI.decode(uri.lastSegment())};
           monitor.subTask(NLS.bind(Messages.MonitoredClusteringBuilderState_WRITE_ONE_DESCRIPTION, bindings));
           traceSet.started(ResourceIndexingEvent.class, uri);
@@ -891,6 +901,11 @@ private List<URI> writeResources(final Collection<URI> toWrite, final BuildData
             newState.register(intermediateDelta);
             toBuild.add(uri);
           }
+        } catch (final DemandLoadFailedException e) {
+          // Demand load failed because of memory shortage, save the uri to process again in the next cluster
+          // Resource set will be cleared at the end of this loop iteration
+          resourceToRetry = uri;
+          LOGGER.info("Demand load failed during resource indexing: " + resourceToRetry); //$NON-NLS-1$
         } catch (final WrappedException ex) {
           pollForCancellation(monitor);
           if (uri == null && ex instanceof LoadOperationException) { // NOPMD
@@ -925,10 +940,13 @@ private List<URI> writeResources(final Collection<URI> toWrite, final BuildData
           monitor.worked(1);
         }
 
-        if (!loadingStrategy.mayProcessAnotherResource(resourceSet, resourceSet.getResources().size())) {
+        if (!loadingStrategy.mayProcessAnotherResource(resourceSet, resourceSet.getResources().size()) || resourceToRetry != null) {
           clearResourceSet(resourceSet);
         }
-        index++;
+
+        if (resourceToRetry == null) {
+          index++;
+        }
       }
     } finally {
       if (loadOperation != null) {
@@ -938,17 +956,32 @@ private List<URI> writeResources(final Collection<URI> toWrite, final BuildData
     return toBuild;
   }
 
-  /** {@inheritDoc} */
+  /**
+   * Writes resources descriptions to the database.
+   *
+   * @param buildData
+   *          builder's data, must not be {@code null}
+   * @param oldState
+   *          represents old index state, must not be {@code null}
+   * @param newState
+   *          represents the new index state, must not be {@code null}
+   * @param newData
+   *          new resource descriptions, must not be {@code null}
+   * @param monitor
+   *          progress monitor, must not be {@code null}
+   */
   protected void writeNewResourceDescriptions(final BuildData buildData, final IResourceDescriptions oldState, final CurrentDescriptions newState, final ResourceDescriptionsData newData, final IProgressMonitor monitor) {
     final List<List<URI>> toWriteGroups = phaseOneBuildSorter.sort(buildData.getToBeUpdated(), oldState);
-    final List<URI> toBuild = Lists.newLinkedList();
     ResourceSet resourceSet = buildData.getResourceSet();
     BuildPhases.setIndexing(resourceSet, true);
+
     int totalSize = 0;
     for (List<URI> group : toWriteGroups) {
       totalSize = totalSize + group.size();
     }
+
     final SubMonitor subMonitor = SubMonitor.convert(monitor, Messages.MonitoredClusteringBuilderState_WRITE_DESCRIPTIONS, totalSize);
+    final List<URI> toBuild = Lists.newArrayListWithCapacity(totalSize);
 
     try {
       traceSet.started(BuildIndexingEvent.class);