update cellprofiler-plugins organization (#172)

* update cellprofiler-plugins organization

* more granular,updated  plugins options

* proper truthiness

* make old new, make new old

---------

Co-authored-by: Beth Cimini <[email protected]>

config.py

@@ -54,8 +54,8 @@
 ALWAYS_CONTINUE = 'False'     # Whether or not to run CellProfiler with the --always-continue flag, which will keep CellProfiler from crashing if it errors
 
 # PLUGINS
-USE_PLUGINS = 'False'        # True to use any plugin from CellProfiler-plugins repo
-UPDATE_PLUGINS = 'False'     # True to download updates from CellProfiler-plugins repo
-PLUGINS_COMMIT = ''          # What commit or version tag do you want to check out?
-INSTALL_REQUIREMENTS = 'False' # True to install REQUIREMENTS_FILE defined below. Requirements should have all plugin dependencies.
-REQUIREMENTS_FILE = ''       # Path within the CellProfiler-plugins repo to a requirements file
+USE_PLUGINS = 'False'          # True to use any plugin from CellProfiler-plugins repo
+UPDATE_PLUGINS = 'False'       # True to download updates from CellProfiler-plugins repo
+PLUGINS_COMMIT = 'False'       # What commit or version tag do you want to check out? If not, set to False.
+INSTALL_REQUIREMENTS = 'False' # True to install REQUIREMENTS defined below. Requirements should have all plugin dependencies.
+REQUIREMENTS = ''       # Flag to use with install (current) or path within the CellProfiler-plugins repo to a requirements file (deprecated).

documentation/DCP-documentation/step_1_configuration.md

@@ -121,13 +121,27 @@ We suggest using this setting in conjunction with a small number of JOB_RETRIES.
 
 ### PLUGINS
 * **USE_PLUGINS:** Whether or not you will be using external plugins from the CellProfiler-plugins repository.
+When True, passes the `--plugins-directory` flag to CellProfiler.
+Defaults to the current v1.0 `CellProfiler-plugins/active_plugins` location for plugins but will revert to the historical location of plugins in the `CellProfiler-plugins` root directory if the `active_plugins` folder is not present.
 * **UPDATE_PLUGINS:** Whether or not to update the plugins repository before use.
 (i.e. run `git fetch --all` on CellProfiler-plugins)
-* **PLUGINS_COMMIT:** What commit or version tag to check out.
+* **PLUGINS_COMMIT:** If desired, what commit or version tag to check out.
 Used in the `git checkout PLUGINS_COMMIT` command in CellProfiler-plugins.
+If you do not want to checkout a specific commit, set to False.
 * **INSTALL_REQUIREMENTS:** Whether or not to install requirements associate with plugins.
-* **REQUIREMENTS_FILE:** A path within the CellProfiler-plugins repository to the requirements file you would like to install.
-Used in the `pip install -r REQUIREMENTS_FILE` command.
+Not all plugins require additional requirement installation.
+See [CellProfiler-plugins Documentation](https://plugins.cellprofiler.org/using_plugins.html) for more information on requirements.
+* **REQUIREMENTS:** For current v1.0 CellProfiler-plugins, a flag that will be passed to the install command (e.g. `cellpose`).
+See [CellProfiler-plugins Documentation](https://plugins.cellprofiler.org/using_plugins.html) for more information on supported flags.
+For deprecated versions of CellProfiler-plugins before v1.0, pass a path within the CellProfiler-plugins repository to the requirements file you would like to install that will be used in the `pip install -r REQUIREMENTS_FILE` command.
+
+The [CellProfiler/Distributed-CellProfiler Docker](https://hub.docker.com/r/cellprofiler/distributed-cellprofiler/tags) 2.0.0_4.2.4 and older have a clone of the CellProfiler-plugins repository with deprecated organization in them.
+If you would like to continue using this clone, set `USE_PLUGINS = 'True'` and `UPDATE_PLUGINS = 'False'`.
+Note that if you do need to install requirements with the deprecated organization, pass the path to the requirements file within the CellProfiler-plugins repository as `REQUIREMENTS`.
+If you would like to update the CellProfiler-plugins repository with up-to-date plugins and new structure while using the CellProfiler/Distributed-CellProfiler Docker 2.0.0_4.2.4 and older, set `UPDATE_PLUGINS = 'True'`.
+
+[CellProfiler/Distributed-CellProfiler Dockers](https://hub.docker.com/r/cellprofiler/distributed-cellprofiler/tags) newer than 2.0.0_4.2.4 have current CellProfiler-plugins repository organization.
+If you need to use deprecated plugin organization you can access previous commits or version tags by passing them as `PLUGINS_COMMIT`.
 
 ***
 

run.py

@@ -18,6 +18,7 @@
 CREATE_DASHBOARD = 'False'
 CLEAN_DASHBOARD = 'False'
 AUTO_MONITOR = 'False'
+REQUIREMENTS_FILE = False
 ALWAYS_CONTINUE = 'False'
 JOB_RETRIES = 10
 
@@ -26,6 +27,8 @@
 # Back compatability with old config requirements
 if ':' in SQS_DEAD_LETTER_QUEUE:
     SQS_DEAD_LETTER_QUEUE = SQS_DEAD_LETTER_QUEUE.rsplit(':',1)[1]
+if REQUIREMENTS_FILE:
+    REQUIREMENTS = REQUIREMENTS_FILE
 
 WAIT_TIME = 60
 MONITOR_TIME = 60
@@ -151,12 +154,12 @@ def generate_task_definition(AWS_PROFILE):
                 'name': 'UPLOAD_FLAGS',
                 'value': UPLOAD_FLAGS
             }]
-    if UPDATE_PLUGINS.lower()=='true':
+    if USE_PLUGINS.lower()=='true':
         task_definition["containerDefinitions"][0]["environment"] += [
             {"name": "UPDATE_PLUGINS", "value": str(UPDATE_PLUGINS)},
             {"name": "PLUGINS_COMMIT", "value": str(PLUGINS_COMMIT)},
             {"name": "INSTALL_REQUIREMENTS", "value": str(INSTALL_REQUIREMENTS)},
-            {"name": "REQUIREMENTS_FILE", "value": str(REQUIREMENTS_FILE)},
+            {"name": "REQUIREMENTS", "value": str(REQUIREMENTS)},
         ]
     return task_definition, taskRoleArn
 

worker/cp-worker.py

@@ -12,7 +12,7 @@
 
 DATA_ROOT = '/home/ubuntu/bucket'
 LOCAL_OUTPUT = '/home/ubuntu/local_output'
-PLUGIN_DIR = '/home/ubuntu/CellProfiler-plugins'
+PLUGIN_DIR = '/home/ubuntu/CellProfiler-plugins/active_plugins'
 QUEUE_URL = os.environ['SQS_QUEUE_URL']
 AWS_BUCKET = os.environ['AWS_BUCKET']
 if 'SOURCE_BUCKET' not in os.environ:
@@ -279,7 +279,11 @@ def runCellProfiler(message):
     else:
         printandlog("Didn't recognize input file",logger)
     if USE_PLUGINS.lower() == 'true':
-        cmd += f' --plugins-directory={PLUGIN_DIR}'
+        if os.path.isdir(PLUGIN_DIR):
+            cmd += f' --plugins-directory={PLUGIN_DIR}'
+        else:
+            printandlog("Can't find current 'active_plugins' folder, defaulting to plugins root directory.",logger)
+            cmd += " --plugins-directory='/home/ubuntu/CellProfiler-plugins/'"
     if ALWAYS_CONTINUE.lower() == 'true':
         cmd +='  --always-continue'
     print(f'Running {cmd}')

worker/run-worker.sh

@@ -36,17 +36,35 @@ aws cloudwatch put-metric-alarm --alarm-name ${APP_NAME}_${MY_INSTANCE_ID} --ala
 python3.8 instance-monitor.py &
 
 # 5. UPDATE AND/OR INSTALL PLUGINS
-if [[ ${UPDATE_PLUGINS} == 'True' ]]; then
+if [[ ${USE_PLUGINS} == 'True' ]]; then
+  if [[ ${UPDATE_PLUGINS} == 'True' ]]; then
+    echo "Updating CellProfiler-plugins."
     cd CellProfiler-plugins
     git fetch --all
-    git checkout ${PLUGINS_COMMIT} || echo "No such commit, branch, or version; failing here." & exit 1
     cd ..
-fi 
-if [[ ${INSTALL_REQUIREMENTS} == 'True' ]]; then
+  fi
+  if [[ ${PLUGINS_COMMIT} != 'False' ]]; then
+    echo "Checking out specific CellProfiler-plugins commit."
     cd CellProfiler-plugins
-    pip install -r ${REQUIREMENTS_FILE} || echo "Requirements file not present or install failed; failing here." & exit 1
+    git checkout ${PLUGINS_COMMIT} || echo "No such commit, branch, or version; failing here." & exit 1
     cd ..
-fi 
+  fi 
+  if [[ ${INSTALL_REQUIREMENTS} == 'True' ]]; then
+    cd CellProfiler-plugins
+    if [[ -z "$REQUIREMENTS" ]]; then
+      REQUIREMENTS = $REQUIREMENTS_FILE
+    fi
+    if [[ -d "active_plugins" ]]; then
+      echo "Installing CellProfiler-plugins requirements."
+      pip install -e . ${REQUIREMENTS} || echo "Requirements install failed." & exit 1
+      cd ..
+    else 
+      echo "Detected deprecated CellProfiler-plugins repo organization. Installing requirements."
+      pip install -r ${REQUIREMENTS} || echo "Requirements file not present or install failed; failing here." & exit 1
+      cd ..
+    fi
+  fi
+fi
 
 # 6. RUN CP WORKERS
 for((k=0; k<$DOCKER_CORES; k++)); do