Skip to content

Commit

Permalink
Revert the demand load memory check
Browse files Browse the repository at this point in the history
  • Loading branch information
Maksim Rubanov authored and Maksim Rubanov committed Apr 9, 2019
1 parent 9d86abc commit 353f741
Show file tree
Hide file tree
Showing 5 changed files with 18 additions and 86 deletions.
23 changes: 9 additions & 14 deletions ...xt.builder/src/com/avaloq/tools/ddk/xtext/builder/DefaultBuilderResourceLoadStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@


/**
* Determine when to stop loading resources during the build.
* Determine when to stop loading resources during build.
*/
@SuppressWarnings("nls")
// CHECKSTYLE:CONSTANTS-OFF
Expand All @@ -27,10 +27,10 @@ public class DefaultBuilderResourceLoadStrategy implements IBuilderResourceLoadS
/** Class-wide logger. */
private static final Logger LOGGER = Logger.getLogger(DefaultBuilderResourceLoadStrategy.class);

/** Property for setting target free memory value. (in megabytes). */
/** Default value for target free memory in megabytes. */
private static final String TARGET_FREE_MEMORY_PROPERTY = "com.avaloq.tools.ddk.xtext.builder.targetFreeMemory"; //$NON-NLS-1$

/** Property for setting minimum free memory value. (in megabytes) */
/** Property for setting minimum free memory setting. */
private static final String MIN_FREE_MEMORY_PROPERTY = "com.avaloq.tools.ddk.xtext.builder.minFreeMemory"; //$NON-NLS-1$

/** Property for setting the maximal cluster size. */
Expand All @@ -51,16 +51,16 @@ public class DefaultBuilderResourceLoadStrategy implements IBuilderResourceLoadS
/** Default value for MINIMUM_FREE_MEMORY. */
private static final long DEFAULT_MIN_FREE_MEMORY = 100 * ONE_MEGABYTE;

/** Cluster will always end if memory falls below this threshold. */
/** Cluster will always end if memory falls bellow this threshold. */
private static final long MINIMUM_FREE_MEMORY = getMinimumFreeMemory();

/** Attempt to cap a cluster when memory reaches this limit, as long as we processed more than the minimum cluster size. */
/** Attempt to cap cluster when memory reaches this limit, as long as we processed more than the minimum cluster size. */
private static final long TARGET_FREE_MEMORY = getTargetFreeMemory();

/** Cluster size. */
@Inject(optional = true)
@Named("org.eclipse.xtext.builder.clustering.ClusteringBuilderState.clusterSize")
private int clusterSize = 50; // NOPMD Cannot be static because of the way Guice injection works.
private int clusterSize = 20; // NOPMD Cannot be static because of the way Guice injection works.

/** The Java runtime; needed to get access to memory usage data. */
private static final Runtime RUNTIME = Runtime.getRuntime();
Expand Down Expand Up @@ -100,6 +100,7 @@ public void setClusterSize(final int clusterSize) {
this.clusterSize = clusterSize;
}

/** {@inheritDoc} */
@Override
public boolean mayProcessAnotherResource(final ResourceSet resourceSet, final int alreadyProcessed) {
if (alreadyProcessed == 0) {
Expand All @@ -114,13 +115,13 @@ public boolean mayProcessAnotherResource(final ResourceSet resourceSet, final in
return false;
}

if (isMemoryLimitReached()) {
final long freeMemory = RUNTIME.freeMemory();
if (freeMemory < MINIMUM_FREE_MEMORY) {
return false;
} else if (alreadyProcessed < clusterSize) {
return true; // Process at least up to cluster size
}

final long freeMemory = RUNTIME.freeMemory();
if (freeMemory < TARGET_FREE_MEMORY) {
// Running GC here might free memory, but would not significantly speed up processing.
// We may use slightly smaller clusters than strictly necessary without GC, though.
Expand All @@ -133,10 +134,4 @@ public boolean mayProcessAnotherResource(final ResourceSet resourceSet, final in
}
return true; // Keep on loading resources into this resource set
}

@Override
public boolean isMemoryLimitReached() {
return RUNTIME.freeMemory() < MINIMUM_FREE_MEMORY;
}

}
21 changes: 0 additions & 21 deletions ...s.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/DemandLoadFailedException.java
View file

This file was deleted.

9 changes: 1 addition & 8 deletions ...dk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/IBuilderResourceLoadStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
public interface IBuilderResourceLoadStrategy {

/**
* Determine whether the builder may process yet another resource.
* Determine whether the builder may, in its second phase, process yet another resource.
*
* @param resourceSet
* The builder's resource set, containing all currently loaded resources.
Expand All @@ -32,11 +32,4 @@ public interface IBuilderResourceLoadStrategy {
*/
boolean mayProcessAnotherResource(ResourceSet resourceSet, int alreadyProcessed);

/**
* Checks whether the builder has reached the minimal amount of free memory, which prevents further progress without clearing the resource set.
*
* @return {@code true} if the described condition is met, {@code false} otherwise
*/
boolean isMemoryLimitReached();

}
2 changes: 0 additions & 2 deletions com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/Messages.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,7 @@

//CHECKSTYLE:OFF
public class Messages extends NLS {

private static final String BUNDLE_NAME = "com.avaloq.tools.ddk.xtext.builder.messages"; //$NON-NLS-1$

public static String MonitoredClusteringBuilderState_CANNOT_LOAD_RESOURCE;
public static String MonitoredClusteringBuilderState_NO_MORE_RESOURCES;
public static String MonitoredClusteringBuilderState_PHASE_ONE_DONE;
Expand Down
49 changes: 8 additions & 41 deletions ...xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/MonitoredClusteringBuilderState.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -852,14 +852,11 @@ protected boolean recordDeltaAsNew(final Delta newDelta) {
* The progress monitor used for user feedback
* @return the list of {@link URI}s of loaded resources to be processed in the second phase
*/
private List<URI> writeResources(final Collection<URI> toWrite, final BuildData buildData, final IResourceDescriptions oldState, final CurrentDescriptions newState, final IProgressMonitor monitor) { // NOPMD
private List<URI> writeResources(final Collection<URI> toWrite, final BuildData buildData, final IResourceDescriptions oldState, final CurrentDescriptions newState, final IProgressMonitor monitor) {
ResourceSet resourceSet = buildData.getResourceSet();
IProject currentProject = getBuiltProject(buildData);
List<URI> toBuild = Lists.newArrayList();

List<URI> toBuild = Lists.newLinkedList();
IResourceLoader.LoadOperation loadOperation = null;
URI resourceToRetry = null;

try {
int resourcesToWriteSize = toWrite.size();
int index = 1;
Expand All @@ -869,23 +866,16 @@ private List<URI> writeResources(final Collection<URI> toWrite, final BuildData

// Not using the loadingStrategy here; seems to work fine with a reasonable clusterSize (20 by default), even with
// large resources and "scarce" memory (say, about 500MB).
while (loadOperation.hasNext() || resourceToRetry != null) {
while (loadOperation.hasNext()) {
if (monitor.isCanceled()) {
loadOperation.cancel();
throw new OperationCanceledException();
}

URI uri = null;
Resource resource = null;
try {
if (resourceToRetry == null) {
resource = addResource(loadOperation.next().getResource(), resourceSet);
} else {
resource = resourceSet.getResource(resourceToRetry, true);
resourceToRetry = null;
}
resource = addResource(loadOperation.next().getResource(), resourceSet);
uri = resource.getURI();

final Object[] bindings = {Integer.valueOf(index), Integer.valueOf(resourcesToWriteSize), uri.fileExtension(), URI.decode(uri.lastSegment())};
monitor.subTask(NLS.bind(Messages.MonitoredClusteringBuilderState_WRITE_ONE_DESCRIPTION, bindings));
traceSet.started(ResourceIndexingEvent.class, uri);
Expand All @@ -901,11 +891,6 @@ private List<URI> writeResources(final Collection<URI> toWrite, final BuildData
newState.register(intermediateDelta);
toBuild.add(uri);
}
} catch (final DemandLoadFailedException e) {
// Demand load failed because of memory shortage, save the uri to process again in the next cluster
// Resource set will be cleared at the end of this loop iteration
resourceToRetry = uri;
LOGGER.info("Demand load failed during resource indexing: " + resourceToRetry); //$NON-NLS-1$
} catch (final WrappedException ex) {
pollForCancellation(monitor);
if (uri == null && ex instanceof LoadOperationException) { // NOPMD
Expand Down Expand Up @@ -940,13 +925,10 @@ private List<URI> writeResources(final Collection<URI> toWrite, final BuildData
monitor.worked(1);
}

if (!loadingStrategy.mayProcessAnotherResource(resourceSet, resourceSet.getResources().size()) || resourceToRetry != null) {
if (!loadingStrategy.mayProcessAnotherResource(resourceSet, resourceSet.getResources().size())) {
clearResourceSet(resourceSet);
}

if (resourceToRetry == null) {
index++;
}
index++;
}
} finally {
if (loadOperation != null) {
Expand All @@ -956,32 +938,17 @@ private List<URI> writeResources(final Collection<URI> toWrite, final BuildData
return toBuild;
}

/**
* Writes resources descriptions to the database.
*
* @param buildData
* builder's data, must not be {@code null}
* @param oldState
* represents old index state, must not be {@code null}
* @param newState
* represents the new index state, must not be {@code null}
* @param newData
* new resource descriptions, must not be {@code null}
* @param monitor
* progress monitor, must not be {@code null}
*/
/** {@inheritDoc} */
protected void writeNewResourceDescriptions(final BuildData buildData, final IResourceDescriptions oldState, final CurrentDescriptions newState, final ResourceDescriptionsData newData, final IProgressMonitor monitor) {
final List<List<URI>> toWriteGroups = phaseOneBuildSorter.sort(buildData.getToBeUpdated(), oldState);
final List<URI> toBuild = Lists.newLinkedList();
ResourceSet resourceSet = buildData.getResourceSet();
BuildPhases.setIndexing(resourceSet, true);

int totalSize = 0;
for (List<URI> group : toWriteGroups) {
totalSize = totalSize + group.size();
}

final SubMonitor subMonitor = SubMonitor.convert(monitor, Messages.MonitoredClusteringBuilderState_WRITE_DESCRIPTIONS, totalSize);
final List<URI> toBuild = Lists.newArrayListWithCapacity(totalSize);

try {
traceSet.started(BuildIndexingEvent.class);
Expand Down

0 comments on commit 353f741

Please sign in to comment.