Clone and install requirements for CPPlugins (#91)

* Clone the plugins repo * Update Dockerfile * Add plugins directory to CP call * Add plugins to config * Add USE_PLUGINS * Fix -i bug in batchfiles, make plugins follow an input variable * Fix formatting of cmd lines * Space before --plugins * Rebase plugins use (#109) * Add userdata for each LaunchSpecification (#102) Co-authored-by: Ubuntu <[email protected]> * Batch files should be able to have input files too Co-authored-by: Matthew Chess <[email protected]> Co-authored-by: Ubuntu <[email protected]> Co-authored-by: Matthew Chess <[email protected]> Co-authored-by: Ubuntu <[email protected]>
DistributedScience · Nov 3, 2020 · 338e7c6 · 338e7c6
1 parent 2520c42
commit 338e7c6
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 2 deletions.
diff --git a/config.py b/config.py
@@ -37,3 +37,6 @@
 CHECK_IF_DONE_BOOL = 'True'  #True or False- should it check if there are a certain number of non-empty files and delete the job if yes?
 EXPECTED_NUMBER_FILES = 7    #What is the number of files that trigger skipping a job?
 MIN_FILE_SIZE_BYTES = 1      #What is the minimal number of bytes an object should be to "count"?
+
+# PLUGINS
+USE_PLUGINS = 'True'
diff --git a/fabfile.py b/fabfile.py
@@ -158,6 +158,10 @@ def generate_task_definition():
 	{
 	    "name": "MIN_FILE_SIZE_BYTES",
 	    "value": str(MIN_FILE_SIZE_BYTES)
+	},
+	{
+	    "name": "USE_PLUGINS",
+	    "value": str(USE_PLUGINS)
 	}
     ]
     return task_definition

diff --git a/worker/Dockerfile b/worker/Dockerfile
@@ -5,6 +5,7 @@
 # This wraps the cellprofiler docker registry
 #
 
+
 FROM cellprofiler/cellprofiler:3.1.9
 
 # Install S3FS 
@@ -56,6 +57,12 @@ COPY instance-monitor.py .
 COPY run-worker.sh .
 RUN chmod 755 run-worker.sh
 
+RUN git clone https://github.com/CellProfiler/CellProfiler-plugins.git
+WORKDIR /home/ubuntu/CellProfiler-plugins
+RUN \
+    pip install -r requirements.txt
+
+WORKDIR /home/ubuntu
 ENTRYPOINT ["./run-worker.sh"]
 CMD [""]
 
diff --git a/worker/cp-worker.py b/worker/cp-worker.py
@@ -17,6 +17,7 @@
 
 DATA_ROOT = '/home/ubuntu/bucket'
 LOCAL_OUTPUT = '/home/ubuntu/local_output'
+PLUGIN_DIR = '/home/ubuntu/CellProfiler-plugins'
 QUEUE_URL = os.environ['SQS_QUEUE_URL']
 AWS_BUCKET = os.environ['AWS_BUCKET']
 LOG_GROUP_NAME= os.environ['LOG_GROUP_NAME']
@@ -26,6 +27,10 @@
     MIN_FILE_SIZE_BYTES = 1
 else:
     MIN_FILE_SIZE_BYTES = int(os.environ['MIN_FILE_SIZE_BYTES'])
+if 'USE_PLUGINS' not in os.environ:
+    USE_PLUGINS = 'False'
+else:
+    USE_PLUGINS = os.environ['USE_PLUGINS']
 
 #################################
 # CLASS TO HANDLE THE SQS QUEUE
@@ -129,7 +134,7 @@ def runCellProfiler(message):
     remoteOut= os.path.join(message['output'],metadataID)
     replaceValues = {'PL':message['pipeline'], 'OUT':localOut, 'FL':message['data_file'],
 			'DATA': DATA_ROOT, 'Metadata': message['Metadata'], 'IN': message['input'], 
-			'MetadataID':metadataID }
+			'MetadataID':metadataID, 'PLUGINS':PLUGIN_DIR }
     # See if this is a message you've already handled, if you've so chosen
     if CHECK_IF_DONE_BOOL.upper() == 'TRUE':
         try:
@@ -159,9 +164,12 @@ def runCellProfiler(message):
     cpDone = localOut + '/cp.is.done'
     if message['pipeline'][-3:]!='.h5':
         cmd = cmdstem + '-p %(DATA)s/%(PL)s -i %(DATA)s/%(IN)s -o %(OUT)s -d ' + cpDone
-        cmd += ' --data-file=%(DATA)s/%(FL)s -g %(Metadata)s'
+        cmd += ' --data-file=%(DATA)s/%(FL)s '
+        cmd += '-g %(Metadata)s'
     else:
         cmd = cmdstem + '-p %(DATA)s/%(PL)s -i %(DATA)s/%(IN)s -o %(OUT)s -d ' + cpDone + ' --data-file=%(DATA)s/%(FL)s -g %(Metadata)s'
+    if USE_PLUGINS == 'True':
+        cmd += ' --plugins-directory=%(PLUGINS)s'
     cmd = cmd % replaceValues
     print('Running', cmd)
     logger.info(cmd)