Skip to content

Commit

Permalink
option to write results in parallel is used only for writing to the db
Browse files Browse the repository at this point in the history
  • Loading branch information
Cristian Goina committed Apr 1, 2024
1 parent 4927c02 commit deadaee
Showing 1 changed file with 5 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -125,9 +125,10 @@ static class ColorDepthSearchArgs extends AbstractColorDepthMatchArgs {
@Parameter(names = {"--write-batch-size"}, description = "If this is set the results will be written in batches of this size")
int writeBatchSize = 0;

@Parameter(names = {"--parallel-result-writing"},
description = "If set, parallelize writing batch results", arity = 0)
boolean parallelizeResultWriting = false;
@Parameter(names = {"--parallel-write-results"},
description = "If set, result batches are written concurrently. This option is used only when the results destination is the database",
arity = 0)
boolean parallelWriteResults = false;

@Parameter(names = {"--use-spark"}, arity = 0,
description = "If set, use spark to run color depth search process")
Expand Down Expand Up @@ -271,7 +272,7 @@ private <M extends AbstractNeuronEntity, T extends AbstractNeuronEntity> void ru
(Runtime.getRuntime().totalMemory() / _1M));
if (args.writeBatchSize > 0) {
Stream<Map.Entry<Integer, List<CDMatchEntity<M, T>>>> cdsResultsPartitionedStream;
if (args.parallelizeResultWriting) {
if (args.parallelWriteResults && args.commonArgs.resultsStorage == StorageType.DB) {
cdsResultsPartitionedStream = ItemsHandling.partitionCollection(cdsResults, args.writeBatchSize)
.entrySet()
.parallelStream();
Expand Down

0 comments on commit deadaee

Please sign in to comment.