From e8cfc78631bc2dd8634892666877694228530d6f Mon Sep 17 00:00:00 2001 From: Jan Kieseler Date: Tue, 23 Jan 2024 15:03:21 +0100 Subject: [PATCH] some fixes --- src/djcdata/DataCollection.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/djcdata/DataCollection.py b/src/djcdata/DataCollection.py index d3bcb69..f4a5da4 100644 --- a/src/djcdata/DataCollection.py +++ b/src/djcdata/DataCollection.py @@ -6,6 +6,7 @@ from . import TrainData +from .compiled import trainDataGenerator from .dataPipeline import TrainDataGenerator import tempfile import pickle @@ -375,8 +376,6 @@ def createDataFromSource( def __writeData(self, sample, outputDir): sw=stopwatch() td=self.dataclass() - - fileTimeOut(sample,120) #once available copy to ram sbasename = os.path.basename(sample) newname = sbasename[:sbasename.rfind('.')]+'.djctd' @@ -391,7 +390,7 @@ def __writeData(self, sample, outputDir): if not self.batch_mode: self.writeToFile(outputDir+'/snapshot.djcdc') - def _writeData_async(self,index,woq,wrlck,outputDir): + def _writeData_async(self,index,woq,wrlck,outputDir,tempstoragepath): logger.info('async started') @@ -473,7 +472,7 @@ def _writeData_async_andCollect(self, startindex, outputDir): processrunning=[] processfinished=[] for i in range(startindex,len(self.sourceList)): - processes.append(Process(target=self._writeData_async, args=(i,wo_queue,writelock, outputDir) ) ) + processes.append(Process(target=self._writeData_async, args=(i,wo_queue,writelock, outputDir, tempstoragepath) ) ) processrunning.append(False) processfinished.append(False)