feat!: don't run Celery workers in dev mode

Tutor's importing * from devstack.py[1] for the development settings,
and that means that we aren't using Celery workers at all in dev mode
(see [2]).  This makes it so they don't start in dev mode, thus saving
everyboding a significant chunk of RAM.

[1] https://github.com/overhangio/tutor/blob/master/tutor/templates/apps/openedx/settings/lms/development.py#L3
[2] https://github.com/openedx/edx-platform/blob/master/lms/envs/devstack.py#L35

To do so, we rely on Docker compose profiles[3].

[3] https://docs.docker.com/compose/profiles/

BREAKING CHANGE: the `COMPOSE_PROJECT_STARTED` hook signature had to be changed to
accomodate profile selection.

changelog.d/20240416_162659_arbrandes_no_workers_in_dev.md

@@ -0,0 +1,2 @@
+- 💥[Feature] Use Docker compose profiles to control services. (by @arbrandes) -->
+- [Fix] Don't start Celery workers in dev mode, as they're never used. (by @arbrandes) -->

tutor/commands/compose.py

@@ -23,8 +23,9 @@
 
 
 class ComposeTaskRunner(BaseComposeTaskRunner):
-    def __init__(self, root: str, config: Config):
+    def __init__(self, root: str, config: Config, profile: str):
         super().__init__(root, config)
+        self.profile = profile
         self.project_name = ""
         self.docker_compose_files: list[str] = []
         self.docker_compose_job_files: list[str] = []
@@ -37,14 +38,19 @@ def docker_compose(self, *command: str) -> int:
             # Note that we don't trigger the action on "run". That's because we
             # don't want to trigger the action for every initialization script.
             hooks.Actions.COMPOSE_PROJECT_STARTED.do(
-                self.root, self.config, self.project_name
+                self.root, self.config, self.profile, self.project_name
             )
         args = []
         for docker_compose_path in self.docker_compose_files:
             if os.path.exists(docker_compose_path):
                 args += ["-f", docker_compose_path]
         return utils.docker_compose(
-            *args, "--project-name", self.project_name, *command
+            *args,
+            "--profile",
+            self.profile,
+            "--project-name",
+            self.project_name,
+            *command,
         )
 
     def run_task(self, service: str, command: str) -> int:

tutor/commands/dev.py

@@ -11,11 +11,11 @@
 
 
 class DevTaskRunner(compose.ComposeTaskRunner):
-    def __init__(self, root: str, config: Config):
+    def __init__(self, root: str, config: Config, profile: str):
         """
         Load docker-compose files from dev/ and local/
         """
-        super().__init__(root, config)
+        super().__init__(root, config, profile)
         self.project_name = get_typed(self.config, "DEV_PROJECT_NAME", str)
         self.docker_compose_files += [
             tutor_env.pathjoin(self.root, "local", "docker-compose.yml"),
@@ -35,7 +35,7 @@ class DevContext(compose.BaseComposeContext):
     NAME = "dev"
 
     def job_runner(self, config: Config) -> DevTaskRunner:
-        return DevTaskRunner(self.root, config)
+        return DevTaskRunner(self.root, config, self.NAME)
 
 
 @click.group(help="Run Open edX locally with development settings")
@@ -45,12 +45,14 @@ def dev(context: click.Context) -> None:
 
 
 @hooks.Actions.COMPOSE_PROJECT_STARTED.add()
-def _stop_on_local_start(root: str, config: Config, project_name: str) -> None:
+def _stop_on_local_start(
+    root: str, config: Config, profile: str, project_name: str
+) -> None:
     """
     Stop the dev platform as soon as a platform with a different project name is
     started.
     """
-    runner = DevTaskRunner(root, config)
+    runner = DevTaskRunner(root, config, profile)
     if project_name != runner.project_name:
         runner.docker_compose("stop")
 

tutor/commands/local.py

@@ -9,11 +9,11 @@
 
 
 class LocalTaskRunner(compose.ComposeTaskRunner):
-    def __init__(self, root: str, config: Config):
+    def __init__(self, root: str, config: Config, profile: str):
         """
         Load docker-compose files from local/.
         """
-        super().__init__(root, config)
+        super().__init__(root, config, profile)
         self.project_name = get_typed(self.config, "LOCAL_PROJECT_NAME", str)
         self.docker_compose_files += [
             tutor_env.pathjoin(self.root, "local", "docker-compose.yml"),
@@ -31,7 +31,7 @@ class LocalContext(compose.BaseComposeContext):
     NAME = "local"
 
     def job_runner(self, config: Config) -> LocalTaskRunner:
-        return LocalTaskRunner(self.root, config)
+        return LocalTaskRunner(self.root, config, self.NAME)
 
 
 @click.group(help="Run Open edX locally with docker-compose")
@@ -41,12 +41,14 @@ def local(context: click.Context) -> None:
 
 
 @hooks.Actions.COMPOSE_PROJECT_STARTED.add()
-def _stop_on_dev_start(root: str, config: Config, project_name: str) -> None:
+def _stop_on_dev_start(
+    root: str, config: Config, profile: str, project_name: str
+) -> None:
     """
     Stop the local platform as soon as a platform with a different project name is
     started.
     """
-    runner = LocalTaskRunner(root, config)
+    runner = LocalTaskRunner(root, config, profile)
     if project_name != runner.project_name:
         runner.docker_compose("stop")
 

tutor/hooks/catalog.py

@@ -39,7 +39,7 @@ def your_action():
     instance, to add a callback to the :py:data:`COMPOSE_PROJECT_STARTED` action::
 
         @hooks.Actions.COMPOSE_PROJECT_STARTED.add():
-        def run_this_on_start(root, config, name):
+        def run_this_on_start(root, config, profile, name):
             print(root, config["LMS_HOST", name])
 
     Your callback function will then be called whenever the ``COMPOSE_PROJECT_STARTED.do`` method
@@ -54,8 +54,9 @@ def run_this_on_start(root, config, name):
     #:
     #: :parameter str root: project root.
     #: :parameter dict config: project configuration.
+    #: :parameter str profile: docker-compose profile.
     #: :parameter str name: docker-compose project name.
-    COMPOSE_PROJECT_STARTED: Action[[str, Config, str]] = Action()
+    COMPOSE_PROJECT_STARTED: Action[[str, Config, str, str]] = Action()
 
     #: Triggered after all interactive questions have been asked.
     #: You should use this action if you want to add new questions.

tutor/templates/local/docker-compose.yml

@@ -170,6 +170,8 @@ services:
       {%- endfor %}
     depends_on:
       - lms
+    profiles:
+      - local
 
   cms-worker:
     image: {{ DOCKER_IMAGE_OPENEDX }}
@@ -189,5 +191,7 @@ services:
       {%- endfor %}
     depends_on:
       - cms
+    profiles:
+      - local
 
   {{ patch("local-docker-compose-services")|indent(2) }}