diff --git a/config.py b/config.py index abfac4b..db80ca2 100644 --- a/config.py +++ b/config.py @@ -54,8 +54,8 @@ ALWAYS_CONTINUE = 'False' # Whether or not to run CellProfiler with the --always-continue flag, which will keep CellProfiler from crashing if it errors # PLUGINS -USE_PLUGINS = 'False' # True to use any plugin from CellProfiler-plugins repo -UPDATE_PLUGINS = 'False' # True to download updates from CellProfiler-plugins repo -PLUGINS_COMMIT = '' # What commit or version tag do you want to check out? -INSTALL_REQUIREMENTS = 'False' # True to install REQUIREMENTS_FILE defined below. Requirements should have all plugin dependencies. -REQUIREMENTS_FILE = '' # Path within the CellProfiler-plugins repo to a requirements file +USE_PLUGINS = 'False' # True to use any plugin from CellProfiler-plugins repo +UPDATE_PLUGINS = 'False' # True to download updates from CellProfiler-plugins repo +PLUGINS_COMMIT = 'False' # What commit or version tag do you want to check out? If not, set to False. +INSTALL_REQUIREMENTS = 'False' # True to install REQUIREMENTS defined below. Requirements should have all plugin dependencies. +REQUIREMENTS = '' # Flag to use with install (current) or path within the CellProfiler-plugins repo to a requirements file (deprecated). diff --git a/documentation/DCP-documentation/step_1_configuration.md b/documentation/DCP-documentation/step_1_configuration.md index 62aadcb..fc2aa65 100644 --- a/documentation/DCP-documentation/step_1_configuration.md +++ b/documentation/DCP-documentation/step_1_configuration.md @@ -121,13 +121,27 @@ We suggest using this setting in conjunction with a small number of JOB_RETRIES. ### PLUGINS * **USE_PLUGINS:** Whether or not you will be using external plugins from the CellProfiler-plugins repository. +When True, passes the `--plugins-directory` flag to CellProfiler. +Defaults to the current v1.0 `CellProfiler-plugins/active_plugins` location for plugins but will revert to the historical location of plugins in the `CellProfiler-plugins` root directory if the `active_plugins` folder is not present. * **UPDATE_PLUGINS:** Whether or not to update the plugins repository before use. (i.e. run `git fetch --all` on CellProfiler-plugins) -* **PLUGINS_COMMIT:** What commit or version tag to check out. +* **PLUGINS_COMMIT:** If desired, what commit or version tag to check out. Used in the `git checkout PLUGINS_COMMIT` command in CellProfiler-plugins. +If you do not want to checkout a specific commit, set to False. * **INSTALL_REQUIREMENTS:** Whether or not to install requirements associate with plugins. -* **REQUIREMENTS_FILE:** A path within the CellProfiler-plugins repository to the requirements file you would like to install. -Used in the `pip install -r REQUIREMENTS_FILE` command. +Not all plugins require additional requirement installation. +See [CellProfiler-plugins Documentation](https://plugins.cellprofiler.org/using_plugins.html) for more information on requirements. +* **REQUIREMENTS:** For current v1.0 CellProfiler-plugins, a flag that will be passed to the install command (e.g. `cellpose`). +See [CellProfiler-plugins Documentation](https://plugins.cellprofiler.org/using_plugins.html) for more information on supported flags. +For deprecated versions of CellProfiler-plugins before v1.0, pass a path within the CellProfiler-plugins repository to the requirements file you would like to install that will be used in the `pip install -r REQUIREMENTS_FILE` command. + +The [CellProfiler/Distributed-CellProfiler Docker](https://hub.docker.com/r/cellprofiler/distributed-cellprofiler/tags) 2.0.0_4.2.4 and older have a clone of the CellProfiler-plugins repository with deprecated organization in them. +If you would like to continue using this clone, set `USE_PLUGINS = 'True'` and `UPDATE_PLUGINS = 'False'`. +Note that if you do need to install requirements with the deprecated organization, pass the path to the requirements file within the CellProfiler-plugins repository as `REQUIREMENTS`. +If you would like to update the CellProfiler-plugins repository with up-to-date plugins and new structure while using the CellProfiler/Distributed-CellProfiler Docker 2.0.0_4.2.4 and older, set `UPDATE_PLUGINS = 'True'`. + +[CellProfiler/Distributed-CellProfiler Dockers](https://hub.docker.com/r/cellprofiler/distributed-cellprofiler/tags) newer than 2.0.0_4.2.4 have current CellProfiler-plugins repository organization. +If you need to use deprecated plugin organization you can access previous commits or version tags by passing them as `PLUGINS_COMMIT`. *** diff --git a/run.py b/run.py index 0acc22e..32ec467 100644 --- a/run.py +++ b/run.py @@ -18,6 +18,7 @@ CREATE_DASHBOARD = 'False' CLEAN_DASHBOARD = 'False' AUTO_MONITOR = 'False' +REQUIREMENTS_FILE = False ALWAYS_CONTINUE = 'False' JOB_RETRIES = 10 @@ -26,6 +27,8 @@ # Back compatability with old config requirements if ':' in SQS_DEAD_LETTER_QUEUE: SQS_DEAD_LETTER_QUEUE = SQS_DEAD_LETTER_QUEUE.rsplit(':',1)[1] +if REQUIREMENTS_FILE: + REQUIREMENTS = REQUIREMENTS_FILE WAIT_TIME = 60 MONITOR_TIME = 60 @@ -151,12 +154,12 @@ def generate_task_definition(AWS_PROFILE): 'name': 'UPLOAD_FLAGS', 'value': UPLOAD_FLAGS }] - if UPDATE_PLUGINS.lower()=='true': + if USE_PLUGINS.lower()=='true': task_definition["containerDefinitions"][0]["environment"] += [ {"name": "UPDATE_PLUGINS", "value": str(UPDATE_PLUGINS)}, {"name": "PLUGINS_COMMIT", "value": str(PLUGINS_COMMIT)}, {"name": "INSTALL_REQUIREMENTS", "value": str(INSTALL_REQUIREMENTS)}, - {"name": "REQUIREMENTS_FILE", "value": str(REQUIREMENTS_FILE)}, + {"name": "REQUIREMENTS", "value": str(REQUIREMENTS)}, ] return task_definition, taskRoleArn diff --git a/worker/cp-worker.py b/worker/cp-worker.py index edb93e8..9f3683c 100644 --- a/worker/cp-worker.py +++ b/worker/cp-worker.py @@ -12,7 +12,7 @@ DATA_ROOT = '/home/ubuntu/bucket' LOCAL_OUTPUT = '/home/ubuntu/local_output' -PLUGIN_DIR = '/home/ubuntu/CellProfiler-plugins' +PLUGIN_DIR = '/home/ubuntu/CellProfiler-plugins/active_plugins' QUEUE_URL = os.environ['SQS_QUEUE_URL'] AWS_BUCKET = os.environ['AWS_BUCKET'] if 'SOURCE_BUCKET' not in os.environ: @@ -279,7 +279,11 @@ def runCellProfiler(message): else: printandlog("Didn't recognize input file",logger) if USE_PLUGINS.lower() == 'true': - cmd += f' --plugins-directory={PLUGIN_DIR}' + if os.path.isdir(PLUGIN_DIR): + cmd += f' --plugins-directory={PLUGIN_DIR}' + else: + printandlog("Can't find current 'active_plugins' folder, defaulting to plugins root directory.",logger) + cmd += " --plugins-directory='/home/ubuntu/CellProfiler-plugins/'" if ALWAYS_CONTINUE.lower() == 'true': cmd +=' --always-continue' print(f'Running {cmd}') diff --git a/worker/run-worker.sh b/worker/run-worker.sh index 9aef692..be6e733 100644 --- a/worker/run-worker.sh +++ b/worker/run-worker.sh @@ -36,17 +36,35 @@ aws cloudwatch put-metric-alarm --alarm-name ${APP_NAME}_${MY_INSTANCE_ID} --ala python3.8 instance-monitor.py & # 5. UPDATE AND/OR INSTALL PLUGINS -if [[ ${UPDATE_PLUGINS} == 'True' ]]; then +if [[ ${USE_PLUGINS} == 'True' ]]; then + if [[ ${UPDATE_PLUGINS} == 'True' ]]; then + echo "Updating CellProfiler-plugins." cd CellProfiler-plugins git fetch --all - git checkout ${PLUGINS_COMMIT} || echo "No such commit, branch, or version; failing here." & exit 1 cd .. -fi -if [[ ${INSTALL_REQUIREMENTS} == 'True' ]]; then + fi + if [[ ${PLUGINS_COMMIT} != 'False' ]]; then + echo "Checking out specific CellProfiler-plugins commit." cd CellProfiler-plugins - pip install -r ${REQUIREMENTS_FILE} || echo "Requirements file not present or install failed; failing here." & exit 1 + git checkout ${PLUGINS_COMMIT} || echo "No such commit, branch, or version; failing here." & exit 1 cd .. -fi + fi + if [[ ${INSTALL_REQUIREMENTS} == 'True' ]]; then + cd CellProfiler-plugins + if [[ -z "$REQUIREMENTS" ]]; then + REQUIREMENTS = $REQUIREMENTS_FILE + fi + if [[ -d "active_plugins" ]]; then + echo "Installing CellProfiler-plugins requirements." + pip install -e . ${REQUIREMENTS} || echo "Requirements install failed." & exit 1 + cd .. + else + echo "Detected deprecated CellProfiler-plugins repo organization. Installing requirements." + pip install -r ${REQUIREMENTS} || echo "Requirements file not present or install failed; failing here." & exit 1 + cd .. + fi + fi +fi # 6. RUN CP WORKERS for((k=0; k<$DOCKER_CORES; k++)); do