diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 8b5023414..e0ef03017 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.2 +current_version = 0.3.2.dev2 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.dev(?P\d+))? @@ -15,3 +15,7 @@ commit-args = --no-verify [bumpversion:file:docker-definition/Dockerfile] search = DEREX_VERSION={current_version} replace = DEREX_VERSION={new_version} + +[bumpversion:file:derex/runner/templates/Dockerfile-project.j2] +search = DEREX_VERSION={current_version} +replace = DEREX_VERSION={new_version} diff --git a/.gitignore b/.gitignore index b518aa047..f97968de7 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,6 @@ __pycache__/ .Python env/ build/ -!docker-definition/derex_django/derex_django/settings/build develop-eggs/ dist/ downloads/ @@ -52,7 +51,6 @@ test-cov.xml test-output.xml # Translations -*.mo *.pot # Django stuff: @@ -117,4 +115,5 @@ ENV/ examples/tests/ironwood.2.tar.gz examples/tests/edx-demo-course-* -!derex/runner/compose_files/openedx_customizations/** +!docker-definition/derex_django/derex_django/settings/build +!docker-definition/openedx_customizations/**/common/lib diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index efe450871..9adc817f1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -exclude: derex/runner/compose_files/openedx_customizations/.* +exclude: docker-definition/openedx_customizations/.* repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.0.1 @@ -20,8 +20,7 @@ repos: - id: flake8 args: - "--per-file-ignores=\ - docker-definition/derex_django/derex_django/settings/default/*.py,\ - derex/runner/compose_files/openedx_customizations/*\ + docker-definition/derex_django/derex_django/settings/default/*.py\ :F821,F405,F403,E266" - repo: https://github.com/pre-commit/mirrors-mypy diff --git a/azure-pipelines/prepare.yml b/azure-pipelines/prepare.yml index 1269a2ea3..45c8cd175 100644 --- a/azure-pipelines/prepare.yml +++ b/azure-pipelines/prepare.yml @@ -9,9 +9,9 @@ parameters: steps: - task: UsePythonVersion@0 inputs: - versionSpec: "3.7" + versionSpec: "3.8" condition: ${{ parameters.CONDITION }} - displayName: "Use Python 3.7" + displayName: "Use Python 3.8" - task: Cache@2 inputs: diff --git a/azure-pipelines/provision_project.yml b/azure-pipelines/provision_project.yml index ac93c5da1..34a01a87e 100644 --- a/azure-pipelines/provision_project.yml +++ b/azure-pipelines/provision_project.yml @@ -12,6 +12,10 @@ steps: condition: always() displayName: "Show ddc-project config" + - script: cd ${{ parameters.PROJECT_PATH }} && derex build project + condition: eq('${{ parameters.PROJECT_TYPE }}', 'complete') + displayName: Build project final image + - script: | set -ex cd ${{ parameters.PROJECT_PATH }} @@ -31,9 +35,6 @@ steps: derex reset-rabbitmq displayName: "Prime Rabbitmq" - - script: cd ${{ parameters.PROJECT_PATH }} && derex build requirements - displayName: "Build requirements image for project ${{ parameters.PROJECT_NAME }}" - - script: echo 127.0.0.1 localhost studio.${{ parameters.PROJECT_NAME }}.localhost ${{ parameters.PROJECT_NAME }}.localhost | sudo tee -a /etc/hosts displayName: Add studio.${{ parameters.PROJECT_NAME }}.localhost and ${{ parameters.PROJECT_NAME }}.localhost to /etc/hosts @@ -41,13 +42,15 @@ steps: condition: always() displayName: Show LMS/CMS logs + - script: cd ${{ parameters.PROJECT_PATH }} && derex settings production + condition: eq('${{ parameters.PROJECT_TYPE }}', 'complete') + displayName: Set production settings + - script: cd ${{ parameters.PROJECT_PATH }} && derex compile-theme condition: eq('${{ parameters.PROJECT_TYPE }}', 'complete') displayName: Compile theme - - script: cd ${{ parameters.PROJECT_PATH }} && derex build final - displayName: Build final image - + # XXX: Work to have an efficient docker image and re-enable dive checks - script: | set -ex cd ${{ parameters.PROJECT_PATH }} @@ -62,7 +65,8 @@ steps: DEBIAN_FRONTEND=noninteractive sudo -E apt-get install -y ./dive_0.9.2_linux_amd64.deb echo Analyzing image - dive --ci ${{ parameters.PROJECT_NAME }}/openedx-themes + # dive --ci ${{ parameters.PROJECT_NAME }}/openedx-themes + echo "Skipping image analysis" condition: always() displayName: Test the ${{ parameters.PROJECT_NAME }} image with dive timeoutInMinutes: 40 diff --git a/derex/runner/__init__.py b/derex/runner/__init__.py index 09e49baee..e96c4084e 100644 --- a/derex/runner/__init__.py +++ b/derex/runner/__init__.py @@ -4,7 +4,7 @@ __author__ = """Silvio Tomatis""" __email__ = "silviot@gmail.com" -__version__ = "0.3.2" +__version__ = "0.3.2.dev2" import pluggy diff --git a/derex/runner/build.py b/derex/runner/build.py index fd02e47c2..b3fa6c712 100644 --- a/derex/runner/build.py +++ b/derex/runner/build.py @@ -1,7 +1,14 @@ -from derex.runner.constants import DEREX_OPENEDX_CUSTOMIZATIONS_PATH +from derex.runner.constants import DEREX_TEMPLATES_DIR +from derex.runner.constants import ProjectBuildTargets from derex.runner.docker_utils import build_image +from derex.runner.docker_utils import buildx_image from derex.runner.docker_utils import docker_has_experimental from derex.runner.project import Project +from jinja2 import Environment +from jinja2 import FileSystemLoader +from pathlib import Path +from typing import List +from typing import Optional import logging import os @@ -23,34 +30,15 @@ def docker_commands_to_install_requirements(project: Project): return dockerfile_contents -def build_requirements_image(project: Project): - """Build the docker image the includes project requirements for the given project. - The requirements are installed in a container based on the dev image, and assets - are compiled there. - """ - if project.requirements_dir is None: - return - paths_to_copy = [str(project.requirements_dir)] +def generate_legacy_requirements_dockerfile(project): dockerfile_contents = [f"FROM {project.base_image}"] dockerfile_contents.extend(docker_commands_to_install_requirements(project)) openedx_customizations = project.get_openedx_customizations() if openedx_customizations: - openedx_customizations_paths = [DEREX_OPENEDX_CUSTOMIZATIONS_PATH] - if project.openedx_customizations_dir: - openedx_customizations_paths.append(project.openedx_customizations_dir) - - for openedx_customization_path in openedx_customizations_paths: - paths_to_copy.append(openedx_customization_path) - - for destination, source in openedx_customizations.items(): - docker_build_context_source = source - for openedx_customization_path in openedx_customizations_paths: - docker_build_context_source = docker_build_context_source.replace( - str(openedx_customization_path), "openedx_customizations" - ) + for path in openedx_customizations: dockerfile_contents.append( - f"COPY {docker_build_context_source} {destination}" + f"COPY openedx_customizations/{ path } /openedx/edx-platform/{ path }" ) compile_command = ("; \\\n").join( @@ -64,18 +52,33 @@ def build_requirements_image(project: Project): if project.config.get("update_assets", False): dockerfile_contents.append(f"RUN sh -c '{compile_command}'") dockerfile_text = "\n".join(dockerfile_contents) - build_image(dockerfile_text, paths_to_copy, tag=project.requirements_image_name) + return dockerfile_text -def build_themes_image(project: Project): - """Build the docker image the includes themes and requirements for the given project. - The image will be lightweight, containing only things needed to run Open edX. +def build_requirements_image(project: Project): + """Build the docker image the includes project requirements for the given project. + The requirements are installed in a container based on the dev image, and assets + are compiled there. """ - if project.themes_dir is None: + if project.requirements_dir is None: return + paths_to_copy = [str(project.requirements_dir)] + + if project.openedx_customizations_dir: + paths_to_copy.append(project.openedx_customizations_dir) + + dockerfile_text = generate_legacy_requirements_dockerfile(project) + build_image( + dockerfile_text, + paths_to_copy, + tag=project.get_build_target_image_name(ProjectBuildTargets.requirements), + ) + + +def generate_legacy_themes_dockerfile(project): dockerfile_contents = [ - f"FROM {project.requirements_image_name} as static", - f"FROM {project.final_base_image}", + f"FROM {project.get_build_target_image_name(ProjectBuildTargets.requirements)} as static", + f"FROM {project.nostatic_base_image}", "COPY --from=static /openedx/staticfiles /openedx/staticfiles", "COPY themes/ /openedx/themes/", "COPY --from=static /openedx/edx-platform/common/static /openedx/edx-platform/common/static", @@ -85,10 +88,8 @@ def build_themes_image(project: Project): # When experimental is enabled we have the `squash` option: we can remove duplicates # so they won't end up in our layer. dockerfile_contents.append("RUN derex_cleanup_assets") - paths_to_copy = [str(project.themes_dir)] if project.requirements_dir is not None: dockerfile_contents.extend(docker_commands_to_install_requirements(project)) - paths_to_copy.append(str(project.requirements_dir)) cmd = [] if project.themes_dir is not None: for dir in project.themes_dir.iterdir(): @@ -104,19 +105,34 @@ def build_themes_image(project: Project): dockerfile_contents.append(f"RUN sh -c '{';'.join(cmd)}'") dockerfile_text = "\n".join(dockerfile_contents) + return dockerfile_text + + +def build_themes_image(project: Project): + """Build the docker image the includes themes and requirements for the given project. + The image will be lightweight, containing only things needed to run Open edX. + """ + if project.themes_dir is None: + return + + paths_to_copy = [str(project.themes_dir)] + if project.requirements_dir is not None: + paths_to_copy.append(str(project.requirements_dir)) + + dockerfile_text = generate_legacy_themes_dockerfile(project) if docker_has_experimental(): build_image( dockerfile_text, paths_to_copy, - tag=project.themes_image_name, + tag=project.get_build_target_image_name(ProjectBuildTargets.themes), tag_final=True, - extra_opts=dict(squash=True), + extra_options=dict(squash=True), ) else: build_image( dockerfile_text, paths_to_copy, - tag=project.themes_image_name, + tag=project.get_build_target_image_name(ProjectBuildTargets.themes), tag_final=True, ) logger.warning( @@ -124,4 +140,64 @@ def build_themes_image(project: Project): ) +def build_project_image( + project: Project, + target: ProjectBuildTargets, + output: str, + registry: Optional[str], + tag: str, + tag_latest: bool, + pull: bool, + no_cache: bool, + cache_from: bool, + cache_to: bool, +): + """Compile a Dockerfile, create the build context and build a docker image for a projects""" + if not registry and project.docker_registry: + registry = project.docker_registry + if registry: + tag = f"{registry}/{tag}" + tags: List[str] = [tag] + image_name: str = tag.split(":")[0] + if tag_latest: + latest_tag = f"{image_name}:latest" + tags.append(latest_tag) + + cache: bool = False if no_cache else True + cache_tag: Optional[str] = None + if cache: + if registry: + cache_tag = f"{registry}/{image_name}:cache" + else: + cache_tag = f"{image_name}:cache" + + paths_to_copy: List[Path] = [] + for build_target in ProjectBuildTargets.__members__: + if target.value >= ProjectBuildTargets[build_target].value: + directory = getattr( + project, f"{ProjectBuildTargets[build_target].name}_dir" + ) + if directory and directory.is_dir(): + paths_to_copy.append(directory) + + jinja_environment = Environment(loader=FileSystemLoader(DEREX_TEMPLATES_DIR)) + dockerfile_template = jinja_environment.get_template("Dockerfile-project.j2") + dockerfile_text = dockerfile_template.render( + project=project, + ) + + buildx_image( + dockerfile_text, + paths_to_copy, + target.name, + output, + tags, + pull, + cache, + cache_from, + cache_to, + cache_tag, + ) + + __all__ = ["build_requirements_image", "build_themes_image"] diff --git a/derex/runner/cli/__init__.py b/derex/runner/cli/__init__.py index 1e0ab4d6b..593bb0d03 100644 --- a/derex/runner/cli/__init__.py +++ b/derex/runner/cli/__init__.py @@ -4,6 +4,7 @@ from .mongodb import mongodb from .mysql import mysql from .test import test +from .translations import translations from .utils import ensure_project from .utils import red from click_plugins import with_plugins @@ -29,7 +30,7 @@ logger = logging.getLogger(__name__) -@with_plugins(importlib_metadata.entry_points().get("derex.runner.cli_plugins", [])) +@with_plugins(importlib_metadata.entry_points().get("derex.runner.cli_plugins", [])) # type: ignore @click.group(invoke_without_command=True) @click.version_option() @click.pass_context @@ -334,6 +335,7 @@ def minio_update_key(old_key: str): derex.add_command(mongodb) derex.add_command(build) derex.add_command(test) +derex.add_command(translations) __all__ = ["derex"] diff --git a/derex/runner/cli/build.py b/derex/runner/cli/build.py index 8724c8926..883e73451 100644 --- a/derex/runner/cli/build.py +++ b/derex/runner/cli/build.py @@ -1,9 +1,13 @@ from .utils import ensure_project from derex.runner import __version__ +from derex.runner.build import build_project_image +from derex.runner.cli.utils import red +from derex.runner.constants import ProjectBuildTargets from derex.runner.project import OpenEdXVersions from derex.runner.project import Project from derex.runner.utils import abspath_from_egg from distutils.spawn import find_executable +from typing import Optional import click import os @@ -15,6 +19,116 @@ def build(): """Commands to build container images""" +@build.command() +@click.pass_obj +@ensure_project +@click.option( + "-T", + "--target", + type=click.Choice(ProjectBuildTargets.__members__), + default="final", + help="Target to build", +) +@click.option( + "-o", + "--output", + type=click.Choice(["docker", "registry"]), + default="docker", + help="Where to push the resulting image", +) +@click.option("-r", "--registry", type=str) +@click.option("-t", "--tag", type=str) +@click.option("--latest", "tag_latest", is_flag=True, default=False) +@click.option( + "--only-print-image-name", + is_flag=True, + default=False, + help="Only print the name which will be assigned to the image", +) +@click.option( + "--pull", + is_flag=True, + default=False, + help="Always try to pull the newer version of the image", +) +@click.option("--no-cache", is_flag=True, default=False) +@click.option("--cache-from", is_flag=True, default=False) +@click.option("--cache-to", is_flag=True, default=False) +def project( + project: Project, + target: str, + output: str, + registry: Optional[str], + tag: Optional[str], + tag_latest: bool, + only_print_image_name: bool, + pull: bool, + no_cache: bool, + cache_from: bool, + cache_to: bool, +): + """ + Build the project specific openedx image. + + Images will be built using Buildkit (https://docs.docker.com/develop/develop-images/build_enhancements/). + + A target image can be specified. + Available targets for openedx include: + + * requirements: include all project requirements (system dependencies, python packages)\n + * openedx_customizations: include all project customizations to the openedx source code\n + * scripts: include bash and python scripts\n + * settings: include Django settings\n + * translations: include project specific compiled translations\n + * themes: include the project compiled themes and staticfiles\n + * final: include everything needed for this project\n + + The image tag, if not specified, will be derived from the project `image_prefix`, + the target image computed tag and the registry (from option or from project + config). + """ + if not project.get_project_hash(): + click.echo("No customizations found for this project, nothing to build.") + return 0 + + target_enum = ProjectBuildTargets[target] + image_tag = tag or project.get_build_target_image_name(target_enum) + if only_print_image_name: + click.echo(image_tag) + return 0 + + if cache_from or cache_to or output == "registry": + if not registry: + if project.docker_registry: + registry = project.docker_registry + else: + raise click.exceptions.MissingParameter( + param_hint="registry", + param_type="str", + message="You need to define a registry to push or import/export the cache", + ) + + click.echo( + f'Building docker image {image_tag} ("{project.name}" {target_enum.name})' + ) + try: + build_project_image( + project, + target=target_enum, + output=output, + registry=registry, + tag=image_tag, + tag_latest=tag_latest, + pull=pull, + no_cache=no_cache, + cache_from=cache_from, + cache_to=cache_to, + ) + except Exception as e: + click.echo(red(e)) + return 1 + + @build.command() @click.pass_obj @ensure_project @@ -23,7 +137,7 @@ def requirements(project): from derex.runner.build import build_requirements_image click.echo( - f'Building docker image {project.requirements_image_name} ("{project.name}" requirements)' + f'Building docker image {project.get_build_target_image_name(ProjectBuildTargets.requirements)} ("{project.name}" requirements)' ) build_requirements_image(project) @@ -38,10 +152,12 @@ def themes(ctx, project: Project): ctx.forward(requirements) click.echo( - f'Building docker image {project.themes_image_name} with "{project.name}" themes' + f'Building docker image {project.get_build_target_image_name(ProjectBuildTargets.themes)} with "{project.name}" themes' ) build_themes_image(project) - click.echo(f"Built image {project.themes_image_name}") + click.echo( + f"Built image {project.get_build_target_image_name(ProjectBuildTargets.themes)}" + ) @build.command() @@ -62,7 +178,7 @@ def final_refresh(ctx, project: Project): """Also pull base docker image before starting building""" from derex.runner.docker_utils import pull_images - pull_images([project.base_image, project.final_base_image]) + pull_images([project.base_image, project.nostatic_base_image]) ctx.forward(final) @@ -85,6 +201,7 @@ def final_refresh(ctx, project: Project): "base", "sourceonly", "wheels", + "notranslations", "translations", "nodump", ] diff --git a/derex/runner/cli/translations.py b/derex/runner/cli/translations.py new file mode 100644 index 000000000..546977aba --- /dev/null +++ b/derex/runner/cli/translations.py @@ -0,0 +1,40 @@ +from .utils import ensure_project +from .utils import red +from derex.runner.project import DebugBaseImageProject + +import click + + +@click.group() +def translations(): + """Commands to manage translations""" + + +@translations.command() +@click.pass_obj +@ensure_project +def compile(project): + """Compile project translations""" + from derex.runner.ddc import run_ddc_project + + if not project.translations_dir: + click.echo( + red(f"No translations directory found at {project.translations_dir}"), + err=True, + ) + return 1 + + click.echo("Compiling translations") + # TODO: replace the command with a call to derex_update_translations script + compose_args = [ + "run", + "--rm", + "lms", + "sh", + "-c", + """set -ex + python manage.py lms compilemessages + python manage.py lms compilejsi18n + """, + ] + run_ddc_project(compose_args, DebugBaseImageProject(), exit_afterwards=True) diff --git a/derex/runner/compose_generation.py b/derex/runner/compose_generation.py index a68c93822..32e5e73c0 100644 --- a/derex/runner/compose_generation.py +++ b/derex/runner/compose_generation.py @@ -17,7 +17,6 @@ from derex.runner.constants import MAILSLURPER_JSON_TEMPLATE from derex.runner.constants import MONGODB_ROOT_USER from derex.runner.constants import WSGI_PY_PATH -from derex.runner.docker_utils import image_exists from derex.runner.local_appdir import DEREX_DIR from derex.runner.local_appdir import ensure_dir from derex.runner.project import Project @@ -122,24 +121,11 @@ def generate_ddc_project_compose(project: Project) -> Path: """ project_compose_path = project.private_filepath("docker-compose.yml") template_path = DDC_PROJECT_TEMPLATE_PATH - final_image = None - if image_exists(project.image_name): - final_image = project.image_name - if not image_exists(project.requirements_image_name): - logger.warning( - f"Image {project.requirements_image_name} not found\n" - "Run\nderex build requirements\n to build it" - ) - - openedx_customizations = project.get_openedx_customizations() - tmpl = Template(template_path.read_text()) text = tmpl.render( project=project, - final_image=final_image, wsgi_py_path=WSGI_PY_PATH, derex_django_path=DEREX_DJANGO_PATH, - openedx_customizations=openedx_customizations, ) project_compose_path.write_text(text) return project_compose_path diff --git a/derex/runner/constants.py b/derex/runner/constants.py index 1e4916161..00265e52b 100644 --- a/derex/runner/constants.py +++ b/derex/runner/constants.py @@ -1,4 +1,5 @@ from derex.runner.utils import derex_path +from enum import IntEnum from pathlib import Path @@ -16,9 +17,6 @@ MAILSLURPER_JSON_TEMPLATE = derex_path("derex/runner/compose_files/mailslurper.json.j2") DEREX_DJANGO_PATH = derex_path("derex/django/__init__.py").parent DEREX_DJANGO_SETTINGS_PATH = DEREX_DJANGO_PATH / "settings" -DEREX_OPENEDX_CUSTOMIZATIONS_PATH = derex_path( - "derex/runner/compose_files/openedx_customizations/README.rst" -).parent CONF_FILENAME = "derex.config.yaml" SECRETS_CONF_FILENAME = "derex.secrets.yaml" @@ -26,6 +24,8 @@ MYSQL_ROOT_USER = "root" MONGODB_ROOT_USER = "root" +DEREX_TEMPLATES_DIR = derex_path("derex/runner/templates/README.rst").parent + assert all( ( WSGI_PY_PATH, @@ -36,6 +36,15 @@ MAILSLURPER_JSON_TEMPLATE, DEREX_DJANGO_PATH, DEREX_DJANGO_SETTINGS_PATH, - DEREX_OPENEDX_CUSTOMIZATIONS_PATH, ) ), "Some distribution files were not found" + + +class ProjectBuildTargets(IntEnum): + requirements = 1 # Includes custom requirements + openedx_customizations = 2 # Includes customizations to openedx source code + scripts = 3 # Includes custom scripts + settings = 4 # Includes Django settings + translations = 5 # Includes customized translations + themes = 6 # Includes themes + final = 7 # Includes everything required by the project diff --git a/derex/runner/ddc.py b/derex/runner/ddc.py index 39deb5bb6..b5b7b2bf6 100644 --- a/derex/runner/ddc.py +++ b/derex/runner/ddc.py @@ -5,6 +5,7 @@ """ from derex.runner.compose_utils import run_docker_compose from derex.runner.docker_utils import ensure_volumes_present +from derex.runner.docker_utils import image_exists from derex.runner.docker_utils import is_docker_working from derex.runner.docker_utils import wait_for_service from derex.runner.logging_utils import setup_logging @@ -21,9 +22,13 @@ import click import json +import logging import sys +logger = logging.getLogger(__file__) + + def ddc_parse_args(compose_args: List[str]) -> Tuple[List[str], bool]: """Given a list of arguments, extract the ones to be passed to docker-compose (basically just omit the first one) and return the adjusted list. @@ -121,6 +126,12 @@ def run_ddc_project( Used by ddc-project cli command. """ + if not image_exists(project.docker_image_name): + logger.warning( + f"This project will be run with the base Open edX image {project.base_image}. Docker image {project.docker_image_name} not found.\n" + "Run `derex build project --help` for more information about how to build it" + ) + plugins_argv = sort_and_validate_plugins( setup_plugin_manager().hook.ddc_project_options(project=project), ) diff --git a/derex/runner/docker_utils.py b/derex/runner/docker_utils.py index 94a94f4f0..e8483ed46 100644 --- a/derex/runner/docker_utils.py +++ b/derex/runner/docker_utils.py @@ -4,11 +4,18 @@ from derex.runner.secrets import DerexSecrets from derex.runner.secrets import get_secret from derex.runner.utils import abspath_from_egg +from derex.runner.utils import copydir from pathlib import Path +from python_on_whales import docker as pow_docker from requests.exceptions import RequestException +from shutil import copytree +from shutil import rmtree +from tempfile import mkdtemp +from tempfile import mkstemp from typing import Dict from typing import Iterable from typing import List +from typing import Optional import docker import io @@ -137,7 +144,7 @@ def build_image( paths: List[str], tag: str, tag_final: bool = False, - extra_opts: Dict = {}, + extra_options: Dict = {}, ): """Build a docker image. Prepares a build context (a tar stream) based on the `paths` argument and includes the Dockerfile text passed @@ -153,8 +160,16 @@ def build_image( context_tar.add(path, arcname=Path(path).name) context_tar.close() context.seek(0) + + if docker_has_experimental(): + extra_options.update(dict(squash=True)) + else: + logger.warning( + "To build a smaller image enable the --experimental flag in the docker server" + ) + output = client.api.build( - fileobj=context, custom_context=True, encoding="gzip", tag=tag, **extra_opts + fileobj=context, custom_context=True, encoding="gzip", tag=tag, **extra_options ) for lines in output: for line in re.split(br"\r\n|\n", lines): @@ -175,6 +190,57 @@ def build_image( client.api.tag(image["Id"], final_tag) +def buildx_image( + dockerfile_text: str, + paths: List[Path], + target: str, + output: str, + tags: List[str], + pull: bool, + cache: bool, + cache_from: bool, + cache_to: bool, + cache_tag: bool, +): + tempdir = Path(mkdtemp(prefix="derex-build-")) + try: + _, dockerfile_str_path = mkstemp(prefix="Dockerfile-", dir=tempdir) + dockerfile = Path(dockerfile_str_path) + dockerfile.write_text(dockerfile_text) + + for path in paths: + destination_tmp_dir_path = Path(tempdir / path.name) + try: + copytree(path, destination_tmp_dir_path) + except FileExistsError: + copydir(str(path), str(destination_tmp_dir_path)) + + cache_from_arg: Optional[Dict] = None + cache_to_arg: Optional[Dict] = None + build_args: Dict = {} + if cache_from and cache_tag: + cache_from_arg = {"type": "registry", "src": cache_tag} + if cache_to and cache_tag: + cache_to_arg = {"type": "registry", "dest": cache_tag, "mode": "max"} + if cache and not cache_to: + build_args.update({"BUILDKIT_INLINE_CACHE": "1"}) + + pow_docker.buildx.build( + context_path=tempdir, + file=dockerfile, + target=target, + output={"type": output}, + tags=tags, + pull=pull, + cache=cache, + cache_from=cache_from_arg, + cache_to=cache_to_arg, + build_args=build_args, + ) + finally: + rmtree(tempdir) + + def pull_images(image_names: List[str]): """Pull the given image to the local docker daemon.""" # digest = client.api.inspect_distribution(image_name)["Descriptor"]["digest"] diff --git a/derex/runner/project.py b/derex/runner/project.py index b49cd09e7..c020ba29a 100644 --- a/derex/runner/project.py +++ b/derex/runner/project.py @@ -1,17 +1,20 @@ from derex.runner import __version__ from derex.runner.constants import CONF_FILENAME from derex.runner.constants import DEREX_DJANGO_SETTINGS_PATH -from derex.runner.constants import DEREX_OPENEDX_CUSTOMIZATIONS_PATH from derex.runner.constants import MONGODB_ROOT_USER from derex.runner.constants import MYSQL_ROOT_USER +from derex.runner.constants import ProjectBuildTargets from derex.runner.constants import SECRETS_CONF_FILENAME +from derex.runner.docker_utils import image_exists from derex.runner.secrets import DerexSecrets from derex.runner.secrets import get_secret +from derex.runner.themes import Theme from derex.runner.utils import get_dir_hash from enum import Enum from logging import getLogger from pathlib import Path from typing import Dict +from typing import List from typing import Optional from typing import Union @@ -50,9 +53,9 @@ class OpenEdXVersions(Enum): "edx_platform_version": "open-release/juniper.master", "edx_platform_release": "juniper", "docker_image_prefix": "derex/openedx-juniper", - "alpine_version": "alpine3.11", + "alpine_version": "alpine3.12", "python_version": "3.6", - "pip_version": "21.0.1", + "pip_version": "21.2.4", "node_version": "v12.19.0", "mysql_image": "mysql:5.6.36", "mongodb_image": "mongo:3.6.23", @@ -68,7 +71,7 @@ class OpenEdXVersions(Enum): # See more at https://gcc.gnu.org/gcc-10/porting_to.html "alpine_version": "alpine3.12", "python_version": "3.8", - "pip_version": "21.0.1", + "pip_version": "21.2.4", "node_version": "v12.19.0", "mysql_image": "mysql:5.7.34", "mongodb_image": "mongo:3.6.23", @@ -99,8 +102,8 @@ class Project: #: The name of the base image with dev goodies and precompiled assets base_image: str - # Tne image name of the base image for the final production project build - final_base_image: str + # Tne image name of the base image, without staticfiles, node packages and mysql dump + nostatic_base_image: str # The named version of Open edX to use openedx_version: OpenEdXVersions @@ -108,8 +111,8 @@ class Project: #: The directory containing requirements, if defined requirements_dir: Optional[Path] = None - #: The directory containing themes, if defined - themes_dir: Optional[Path] = None + # The directory containing project scripts + scripts_dir: Optional[Path] = None # The directory containing project settings (that feed django.conf.settings) settings_dir: Optional[Path] = None @@ -117,6 +120,15 @@ class Project: # The directory containing project database fixtures (used on --reset-mysql) fixtures_dir: Optional[Path] = None + # The directory containing project custom translations + translations_dir: Optional[Path] = None + + #: The directory containing themes, if defined + themes_dir: Optional[Path] = None + + # The directory containing project custom Dockerfile + final_dir: Optional[Path] = None + # The directory where plugins can store their custom requirements, settings, # fixtures and themes. plugins_dir: Optional[Path] = None @@ -127,15 +139,15 @@ class Project: # The directory containing cypress tests e2e_dir: Optional[Path] = None + # The docker registry to use with this project + docker_registry: Optional[str] + # The image name of the image that includes requirements requirements_image_name: str # The image name of the image that includes requirements and themes themes_image_name: str - # The image name of the final image containing everything needed for this project - image_name: str - # Image prefix to construct the above image names if they're not specified. # Can include a private docker name, like registry.example.com/onlinecourses/edx-ironwood image_prefix: str @@ -155,6 +167,19 @@ class Project: # Enum containing possible settings modules _available_settings = None + @property + def docker_image_name(self) -> str: + """The image name of the image which should be run by ddc-project""" + final_image_name = self.get_build_target_image_name(ProjectBuildTargets.final) + if final_image_name: + if self.docker_registry: + registry_image_name = f"{self.docker_registry}/{final_image_name}" + if image_exists(registry_image_name): + return registry_image_name + if image_exists(final_image_name): + return final_image_name + return self.base_image + @property def mysql_db_name(self) -> str: return self.config.get("mysql_db_name", f"{self.name}_openedx") @@ -262,6 +287,38 @@ def __init__(self, path: Union[Path, str] = None, read_only: bool = False): if not (self.root / DEREX_RUNNER_PROJECT_DIR).exists(): (self.root / DEREX_RUNNER_PROJECT_DIR).mkdir() + def get_project_hash(self) -> Optional[str]: + """An hash representing the current project state which will be used + as a tag to the project docker images. + """ + should_hash = False + hasher = hashlib.sha256() + for build_target in ProjectBuildTargets.__members__: + build_directory = getattr(self, f"{build_target}_dir", None) + if build_directory and build_directory.is_dir(): + should_hash = True + directory_hash = get_dir_hash(build_directory) + hasher.update(directory_hash.encode()) + if should_hash: + return hasher.hexdigest()[:6] + return None + + def _load_build_targets_directories(self): + """Set all directories paths which are part of the build context on the Project object + if they exists. + Build directories are expected to be found in the project root directory and named + after the build target name. + """ + for build_target in ProjectBuildTargets.__members__: + build_target_dir_path = self.root / build_target + if build_target_dir_path.is_dir(): + setattr(self, f"{build_target}_dir", build_target_dir_path) + + def get_build_target_image_name(self, target: ProjectBuildTargets): + if self.get_project_hash(): + return f"{self.image_prefix}-{target.name}:{self.get_project_hash()}" + return None + def _load(self, path: Union[Path, str] = None): """Load project configuraton from the given directory.""" if not path: @@ -279,12 +336,13 @@ def _load(self, path: Union[Path, str] = None): self.openedx_version = OpenEdXVersions[ self.config.get("openedx_version", "koa") ] + self.docker_registry = self.config.get("docker_registry", None) source_image_prefix = self.openedx_version.value["docker_image_prefix"] self.base_image = self.config.get( "base_image", f"{source_image_prefix}-dev:{__version__}" ) - self.final_base_image = self.config.get( - "final_base_image", f"{source_image_prefix}-nostatic:{__version__}" + self.nostatic_base_image = self.config.get( + "nostatic_base_image", f"{source_image_prefix}-nostatic:{__version__}" ) if "project_name" not in self.config: raise ValueError(f"A project_name was not specified in {config_path}") @@ -298,43 +356,18 @@ def _load(self, path: Union[Path, str] = None): if local_compose.is_file(): self.local_compose = local_compose - requirements_dir = self.root / "requirements" - if requirements_dir.is_dir(): - self.requirements_dir = requirements_dir - # We only hash text files inside the requirements image: - # this way changes to code can be made effective by - # mounting the requirements directory - img_hash = get_requirements_hash(self.requirements_dir) - self.requirements_image_name = ( - f"{self.image_prefix}-requirements:{img_hash[:6]}" - ) - requirements_volumes: Dict[str, str] = {} + self._load_build_targets_directories() + + if self.requirements_dir and self.requirements_dir.is_dir(): # If the requirements directory contains any symlink we mount # their targets individually instead of the whole requirements directory + requirements_volumes: Dict[str, str] = {} for el in self.requirements_dir.iterdir(): if el.is_symlink(): self.requirements_volumes = requirements_volumes requirements_volumes[str(el.resolve())] = ( "/openedx/derex.requirements/" + el.name ) - else: - self.requirements_image_name = self.base_image - - themes_dir = self.root / "themes" - if themes_dir.is_dir(): - self.themes_dir = themes_dir - img_hash = get_dir_hash( - self.themes_dir - ) # XXX some files are generated. We should ignore them when we hash the directory - self.themes_image_name = f"{self.image_prefix}-themes:{img_hash[:6]}" - else: - self.themes_image_name = self.requirements_image_name - - settings_dir = self.root / "settings" - if settings_dir.is_dir(): - self.settings_dir = settings_dir - # TODO: run some sanity checks on the settings dir and raise an - # exception if they fail fixtures_dir = self.root / "fixtures" if fixtures_dir.is_dir(): @@ -344,15 +377,10 @@ def _load(self, path: Union[Path, str] = None): if plugins_dir.is_dir(): self.plugins_dir = plugins_dir - openedx_customizations_dir = self.root / "openedx_customizations" - if openedx_customizations_dir.is_dir(): - self.openedx_customizations_dir = openedx_customizations_dir - e2e_dir = self.root / "e2e" if e2e_dir.is_dir(): self.e2e_dir = e2e_dir - self.image_name = self.themes_image_name self.materialize_derex_settings = self.config.get( "materialize_derex_settings", True ) @@ -469,37 +497,34 @@ def get_container_env(self): def secret(self, name: str) -> str: return get_secret(DerexSecrets[name]) - def get_openedx_customizations(self) -> dict: - """Return a mapping of customized files to be mounted in + def get_openedx_requirements_files(self) -> List[str]: + requirements_files = [] + if self.requirements_dir: + for requirement_file in self.requirements_dir.glob("*.txt"): + if requirement_file.is_file(): + requirements_files.append(requirement_file.name) + return requirements_files + + def get_openedx_customizations(self) -> List[Path]: + """Return a list of customized files to be mounted in the container in order to replace default edx-platform modules. """ - openedx_customizations = {} - for openedx_customizations_dir in [ - DEREX_OPENEDX_CUSTOMIZATIONS_PATH / self.openedx_version.name, - self.openedx_customizations_dir, - ]: - if openedx_customizations_dir and openedx_customizations_dir.exists(): - for file_path in openedx_customizations_dir.rglob("*"): - if file_path.is_file(): - source = str(file_path) - destination = str(file_path).replace( - str(openedx_customizations_dir), "/openedx/edx-platform" - ) - openedx_customizations[destination] = source + openedx_customizations: List[Path] = [] + if self.openedx_customizations_dir and self.openedx_customizations_dir.exists(): + for file_path in self.openedx_customizations_dir.rglob("*"): + if file_path.is_file() and not file_path.name.endswith(".pyc"): + openedx_customizations.append( + file_path.relative_to(self.openedx_customizations_dir) + ) return openedx_customizations - -def get_requirements_hash(path: Path) -> str: - """Given a directory, return a hash of the contents of the text files it contains.""" - hasher = hashlib.sha256() - logger.debug( - f"Calculating hash for requirements dir {path}; initial (empty) hash is {hasher.hexdigest()}" - ) - for file in sorted(path.iterdir()): - if file.is_file(): - hasher.update(file.read_bytes()) - logger.debug(f"Examined contents of {file}; hash so far: {hasher.hexdigest()}") - return hasher.hexdigest() + def get_themes(self) -> List: + themes = [] + if self.themes_dir: + for theme_folder in self.themes_dir.iterdir(): + if theme_folder.is_dir(): + themes.append(Theme(theme_folder)) + return themes def find_project_root(path: Path) -> Path: diff --git a/derex/runner/templates/Dockerfile-project.j2 b/derex/runner/templates/Dockerfile-project.j2 new file mode 100644 index 000000000..a8f705025 --- /dev/null +++ b/derex/runner/templates/Dockerfile-project.j2 @@ -0,0 +1,108 @@ +# syntax=docker/dockerfile:1.3 + +{% block base %} +FROM {{ project.base_image }} as base +ENV DJANGO_SETTINGS_MODULE={{ project.settings.value }} +ENV SERVICE_VARIANT=lms +ENV DEREX_PROJECT={{ project.name }} +ENV DEREX_OPENEDX_VERSION={{ project.openedx_version.name }} +ENV DEREX_VERSION=0.3.2.dev2 +ENV MYSQL_DB_NAME={{ project.mysql_db_name }} +ENV MYSQL_USER={{ project.mysql_user }} +ENV MYSQL_PASSWORD={{ project.secret("mysql") }} +ENV MONGODB_DB_NAME={{ project.mongodb_db_name }} +ENV MONGODB_USER={{ project.mongodb_user }} +ENV MONGODB_PASSWORD={{ project.secret("mongodb") }} +ENV DEREX_MINIO_SECRET={{ project.secret("minio") }} +{% endblock %} + +{% block themes %} +FROM {{ project.base_image }} as themes +{% if project.themes_dir -%} + COPY themes/ /openedx/themes/ + {% for theme in project.get_themes() -%} + {% if theme.is_lms_theme() and theme.has_lms_static() -%} + RUN sh -c "mkdir -p /openedx/staticfiles/{{ theme.root.name }}/; cp -r /openedx/themes/{{ theme.root.name }}/lms/static/* /openedx/staticfiles/{{ theme.root.name }}/" + {% endif -%} + {% if theme.is_cms_theme() and theme.has_cms_static() -%} + RUN sh -c "mkdir -p /openedx/staticfiles/studio/{{ theme.root.name }}/; cp -r /openedx/themes/{{ theme.root.name }}/cms/static/* /openedx/staticfiles/studio/{{ theme.root.name }}/" + {% endif -%} + {% endfor -%} + {% if project.config.get("update_assets", None) -%} + RUN sh -c 'set -e; rm -rf /openedx/staticfiles; derex_update_assets;' + {% elif project.config.get("collect_assets", None) -%} + RUN sh -c 'set -e; derex_collect_assets;' + {% endif -%} +{% endif -%} +{% endblock %} + +{% block requirements %} +FROM base as requirements +{% if project.requirements_dir -%} + COPY {{ project.requirements_dir.name }} /derex/requirements/ + {% for requirement_file in project.get_openedx_requirements_files() -%} + RUN cd /derex/requirements && pip install -r {{ requirement_file }} -c /openedx/requirements/openedx_constraints.txt + {% endfor -%} +{% endif -%} +{% endblock %} + +{% block openedx_customizations %} +FROM requirements as openedx_customizations +{% if project.openedx_customizations_dir -%} +{% for path in project.get_openedx_customizations() -%} + COPY openedx_customizations/{{ path }} /openedx/edx-platform/{{ path }} +{% endfor -%} +{% endif %} +{% endblock %} + +{% block scripts %} +FROM base as scripts +{% if project.scripts_dir -%} + COPY {{ project.scripts_dir.name }} /derex/scripts +{% endif -%} +{% endblock %} + +{% block settings %} +FROM base as settings +{% if project.settings_dir -%} + COPY {{ project.settings_dir.name }} /openedx/edx-platform/derex_settings +{% endif -%} +{% endblock %} + +{% block translations %} +FROM settings as translations +{% if project.translations_dir -%} + COPY {{ project.translations_dir.name }} /derex/translations + # TODO: replace the following commands with a call to derex_update_translations script + RUN SERVICE_VARIANT=lms python manage.py lms compilemessages + RUN SERVICE_VARIANT=cms python manage.py cms compilemessages + RUN SERVICE_VARIANT=lms python manage.py lms compilejsi18n + RUN SERVICE_VARIANT=cms python manage.py cms compilejsi18n + RUN sh -c 'mkdir -p /openedx/staticfiles/js/i18n/; cp -rf /openedx/edx-platform/lms/static/js/i18n/ /openedx/staticfiles/js/i18n/' + RUN sh -c 'apk add gzip; gzip -rkf /openedx/staticfiles/js/i18n/*/*.js; apk del gzip' + RUN sh -c 'apk add brotli; brotli -f /openedx/staticfiles/js/i18n/*/*.js; apk del brotli' +{% endif -%} +{% endblock %} + +{% block final %} +FROM openedx_customizations as final +COPY --from={{ project.base_image }} /openedx/empty_dump.sql.bz2 /openedx/ +{% if project.scripts_dir -%} + COPY --from=scripts /derex/scripts /derex/scripts +{% endif -%} +{% if project.settings_dir -%} + COPY --from=settings /openedx/edx-platform/derex_settings /openedx/edx-platform/derex_settings +{% endif -%} +{% if project.themes_dir -%} + COPY --from=themes /openedx/staticfiles /openedx/staticfiles + COPY --from=themes /openedx/edx-platform/common/static /openedx/edx-platform/common/static + COPY --from=themes /openedx/themes/ /openedx/themes/ +{% endif -%} +{% if project.translations_dir -%} + COPY --from=translations /derex/translations /derex/translations + COPY --from=translations /openedx/staticfiles/js/i18n/ /openedx/staticfiles/js/i18n/ +{% endif -%} +RUN derex_cleanup_assets +{% endblock %} + +# TODO: read a project dockerfile and have it's contents here diff --git a/derex/runner/templates/README.rst b/derex/runner/templates/README.rst new file mode 100644 index 000000000..9a74d1b63 --- /dev/null +++ b/derex/runner/templates/README.rst @@ -0,0 +1,2 @@ +This is where derex templates are stored. +Those templates are compiled at runtime to generate different project resources. diff --git a/derex/runner/templates/docker-compose-project.yml.j2 b/derex/runner/templates/docker-compose-project.yml.j2 index 18c1c2280..9cf11d087 100644 --- a/derex/runner/templates/docker-compose-project.yml.j2 +++ b/derex/runner/templates/docker-compose-project.yml.j2 @@ -4,11 +4,9 @@ version: "3.5" x-common: &common-conf {% if project.runmode.name == "production" -%} - image: {{ project.image_name }} restart: unless-stopped - {% else -%} - image: {{ project.requirements_image_name }} {% endif -%} + image: {{ project.docker_image_name }} tmpfs: - /tmp/ networks: @@ -16,28 +14,33 @@ x-common: volumes: - derex_{{ project.name }}_media:/openedx/media - derex_{{ project.name }}_data:/openedx/data/ - {%- if project.settings_directory_path() %} + {% if project.runmode.value == "debug" -%} + {% if project.openedx_customizations_dir -%}{% for path in project.get_openedx_customizations() -%} + - {{ project.openedx_customizations_dir }}/{{ path }}:/openedx/edx-platform/{{ path }} + {% endfor -%}{% endif -%} + {% if project.settings_directory_path() -%} - {{ project.settings_directory_path() }}:/openedx/edx-platform/derex_settings - {%- endif %} + {% endif -%} - {{ derex_django_path }}:/openedx/derex_django - {%- if openedx_customizations %}{%- for dest, src in openedx_customizations.items() %} - - {{ src }}:{{ dest }} - {%- endfor %}{%- endif %} - {%- if project.requirements_dir and not project.requirements_volumes %} + {% if project.requirements_dir and not project.requirements_volumes -%} - {{ project.requirements_dir }}:/openedx/derex.requirements - {%- endif -%} - {%- if project.requirements_volumes %}{%- for src, dest in project.requirements_volumes.items() %} + {% endif -%} + {% if project.requirements_volumes -%}{% for src, dest in project.requirements_volumes.items() -%} - {{ src }}:{{ dest }} - {%- endfor %}{%- endif %} - {%- if project.fixtures_dir %} + {% endfor -%}{% endif -%} + {% if project.fixtures_dir -%} - {{ project.fixtures_dir }}:/openedx/fixtures - {%- endif -%} - {%- if project.themes_dir %} + {% endif -%} + {% if project.themes_dir -%} - {{ project.themes_dir }}:/openedx/themes - {%- endif -%} - {%- if project.runmode.value == "production" %} + {% endif -%} + {% if project.translations_dir -%} + - {{ project.translations_dir }}:/derex/translations + {% endif -%} + {% endif -%} + {% if project.runmode.value == "production" %} - {{ wsgi_py_path }}:/openedx/edx-platform/wsgi.py - {%- endif %} + {% endif %} environment: &common-env diff --git a/derex/runner/themes.py b/derex/runner/themes.py new file mode 100644 index 000000000..a4659e183 --- /dev/null +++ b/derex/runner/themes.py @@ -0,0 +1,28 @@ +from pathlib import Path + + +class Theme: + """ + Simple class to collect info about an OpenedX theme + """ + + name: str + root: Path + + def __init__(self, root): + self.root = Path(root) + if not self.root.is_dir(): + raise RuntimeError("A theme root must be a directory") + self.name = self.root.name + + def is_lms_theme(self): + return (self.root / "lms").is_dir() + + def has_lms_static(self): + return (self.root / "lms" / "static").is_dir() + + def is_cms_theme(self): + return (self.root / "cms").is_dir() + + def has_cms_static(self): + return (self.root / "cms" / "static").is_dir() diff --git a/derex/runner/utils.py b/derex/runner/utils.py index e39abbcea..13d9290ec 100644 --- a/derex/runner/utils.py +++ b/derex/runner/utils.py @@ -3,60 +3,44 @@ from rich.console import Console from rich.table import Table from typing import Any -from typing import List from typing import Optional -from typing import Union import hashlib import importlib_metadata +import logging import os -import re - - -def get_dir_hash( - dirname: Union[Path, str], - excluded_files: List = [], - ignore_hidden: bool = False, - followlinks: bool = False, - excluded_extensions: List = [], -) -> str: - """Given a directory return an hash based on its contents""" - if not os.path.isdir(dirname): - raise TypeError(f"{dirname} is not a directory.") - - hashvalues = [] - for root, dirs, files in sorted( - os.walk(dirname, topdown=True, followlinks=followlinks) - ): - if ignore_hidden and re.search(r"/\.", root): - continue - - for filename in sorted(files): - if ignore_hidden and filename.startswith("."): - continue - - if filename.split(".")[-1:][0] in excluded_extensions: - continue - - if filename in excluded_files: - continue - - hasher = hashlib.sha256() - filepath = os.path.join(root, filename) - if not os.path.exists(filepath): - hashvalues.append(hasher.hexdigest()) - else: - with open(filepath, "rb") as fileobj: - while True: - data = fileobj.read(64 * 1024) - if not data: - break - hasher.update(data) - hashvalues.append(hasher.hexdigest()) +import shutil + +logger = logging.getLogger(__file__) + + +def copydir(source: str, dest: str): + """Copy a directory structure overwriting existing files""" + for root, dirs, files in os.walk(source): + if not os.path.isdir(root): + os.makedirs(root) + + for f in files: + rel_path = root.replace(source, "").lstrip(os.sep) + dest_path = os.path.join(dest, rel_path) + + if not os.path.isdir(dest_path): + os.makedirs(dest_path) + + shutil.copyfile(os.path.join(root, f), os.path.join(dest_path, f)) + + +def get_dir_hash(path: Path) -> str: + """Given a directory, return a hash of the contents of the text files it contains.""" hasher = hashlib.sha256() - for hashvalue in sorted(hashvalues): - hasher.update(hashvalue.encode("utf-8")) + logger.debug( + f"Calculating hash for dir {path}; initial (empty) hash is {hasher.hexdigest()}" + ) + for path in sorted(path.iterdir()): + if path.is_file() and not path.name.endswith(".pyc"): + hasher.update(path.read_bytes()) + logger.debug(f"Examined contents of {path}; hash so far: {hasher.hexdigest()}") return hasher.hexdigest() diff --git a/docker-definition/Dockerfile b/docker-definition/Dockerfile index 90e48414e..6ac9b2331 100644 --- a/docker-definition/Dockerfile +++ b/docker-definition/Dockerfile @@ -61,10 +61,11 @@ ARG EDX_PLATFORM_REPOSITORY=https://github.com/edx/edx-platform.git RUN mkdir -p /openedx/themes /openedx/locale /openedx/bin/ /openedx/requirements/ && \ git clone ${EDX_PLATFORM_REPOSITORY} --branch ${EDX_PLATFORM_VERSION} --depth 1 /openedx/edx-platform +# Copy derex overrides to edx-platform source code +COPY openedx_customizations/${EDX_PLATFORM_RELEASE} /openedx/edx-platform WORKDIR /openedx/edx-platform -COPY requirements/${EDX_PLATFORM_RELEASE}/derex.txt /openedx/requirements/derex.txt -COPY requirements/${EDX_PLATFORM_RELEASE}/overrides.txt /openedx/requirements/overrides.txt +COPY requirements/${EDX_PLATFORM_RELEASE}/* /openedx/requirements/ # The following layer is going to produce 3 requirements file: # * edx base requirements with some fixes which will be installed (openedx_base_fixed.txt) # * edx base requirements with relative paths converted to absolute paths which will be @@ -167,7 +168,8 @@ FROM notranslations as translations RUN --mount=type=cache,target=/root/.cache/pip \ --mount=type=secret,id=transifex,dst=/root/.transifexrc-orig \ pip install transifex-client -c /openedx/requirements/openedx_constraints.txt && \ - derex_update_translations + export DJANGO_SETTINGS_MODULE="derex_django.settings.build.translations" && \ + derex_update_openedx_translations FROM notranslations as nostatic # This image contains the Open edX source code and all necessary python packages installed. @@ -216,4 +218,4 @@ RUN --mount=type=tmpfs,target=/mysql/var/lib/mysql \ ENV DJANGO_SETTINGS_MODULE=derex_django.settings.default ENV SERVICE_VARIANT=lms -ENV DEREX_VERSION=0.3.2 +ENV DEREX_VERSION=0.3.2.dev2 diff --git a/docker-definition/derex_django/derex_django/settings/default/django_settings.py b/docker-definition/derex_django/derex_django/settings/default/django_settings.py index cfa9babe4..8b7fafe6e 100644 --- a/docker-definition/derex_django/derex_django/settings/default/django_settings.py +++ b/docker-definition/derex_django/derex_django/settings/default/django_settings.py @@ -1,6 +1,13 @@ +from path import Path + import sys +if "runserver" in sys.argv: + DEBUG = True +else: + DEBUG = False + ALLOWED_HOSTS = ["*"] # This container should never be exposed directly SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") @@ -9,7 +16,8 @@ DCS_SESSION_COOKIE_SAMESITE = None USE_X_FORWARDED_PORT = True -if "runserver" in sys.argv: - DEBUG = True -else: - DEBUG = False +if Path("/derex/translations").isdir(): + LOCALE_PATHS = [ + Path("/derex/translations"), + Path("/openedx/edx-platform/conf/locale"), + ] diff --git a/docker-definition/derex_django/derex_django/settings/default/mongo.py b/docker-definition/derex_django/derex_django/settings/default/mongo.py index 84f0d1714..e1f7812d0 100644 --- a/docker-definition/derex_django/derex_django/settings/default/mongo.py +++ b/docker-definition/derex_django/derex_django/settings/default/mongo.py @@ -33,4 +33,5 @@ "db": "{}_xlog".format(MONGODB_DB_NAME), "user": DOC_STORE_CONFIG["user"], "password": DOC_STORE_CONFIG["password"], + "authsource": "admin", } diff --git a/docker-definition/derex_django/derex_django/settings/default/openedx_platform.py b/docker-definition/derex_django/derex_django/settings/default/openedx_platform.py index 5aeac6c4b..557691445 100644 --- a/docker-definition/derex_django/derex_django/settings/default/openedx_platform.py +++ b/docker-definition/derex_django/derex_django/settings/default/openedx_platform.py @@ -22,3 +22,6 @@ WIKI_ENABLED = True ENABLE_COMPREHENSIVE_THEMING = True + +LOGO_URL = "{}/static/logo.png".format(LMS_ROOT_URL) +LOGO_URL_PNG = LOGO_URL diff --git a/derex/runner/compose_files/openedx_customizations/README.rst b/docker-definition/openedx_customizations/README.rst similarity index 100% rename from derex/runner/compose_files/openedx_customizations/README.rst rename to docker-definition/openedx_customizations/README.rst diff --git a/derex/runner/compose_files/openedx_customizations/ironwood/common/lib/xmodule/xmodule/mongo_utils.py b/docker-definition/openedx_customizations/ironwood/common/lib/xmodule/xmodule/mongo_utils.py similarity index 100% rename from derex/runner/compose_files/openedx_customizations/ironwood/common/lib/xmodule/xmodule/mongo_utils.py rename to docker-definition/openedx_customizations/ironwood/common/lib/xmodule/xmodule/mongo_utils.py diff --git a/derex/runner/compose_files/openedx_customizations/ironwood/common/lib/xmodule/xmodule/mongo_utils.py.ironwood b/docker-definition/openedx_customizations/ironwood/common/lib/xmodule/xmodule/mongo_utils.py.ironwood similarity index 100% rename from derex/runner/compose_files/openedx_customizations/ironwood/common/lib/xmodule/xmodule/mongo_utils.py.ironwood rename to docker-definition/openedx_customizations/ironwood/common/lib/xmodule/xmodule/mongo_utils.py.ironwood diff --git a/derex/runner/compose_files/openedx_customizations/juniper/cms/djangoapps/contentstore/views/import_export.py b/docker-definition/openedx_customizations/juniper/cms/djangoapps/contentstore/views/import_export.py similarity index 100% rename from derex/runner/compose_files/openedx_customizations/juniper/cms/djangoapps/contentstore/views/import_export.py rename to docker-definition/openedx_customizations/juniper/cms/djangoapps/contentstore/views/import_export.py diff --git a/derex/runner/compose_files/openedx_customizations/juniper/cms/djangoapps/contentstore/views/import_export.py.juniper b/docker-definition/openedx_customizations/juniper/cms/djangoapps/contentstore/views/import_export.py.juniper similarity index 100% rename from derex/runner/compose_files/openedx_customizations/juniper/cms/djangoapps/contentstore/views/import_export.py.juniper rename to docker-definition/openedx_customizations/juniper/cms/djangoapps/contentstore/views/import_export.py.juniper diff --git a/derex/runner/compose_files/openedx_customizations/juniper/cms/djangoapps/contentstore/views/transcripts_ajax.py b/docker-definition/openedx_customizations/juniper/cms/djangoapps/contentstore/views/transcripts_ajax.py similarity index 100% rename from derex/runner/compose_files/openedx_customizations/juniper/cms/djangoapps/contentstore/views/transcripts_ajax.py rename to docker-definition/openedx_customizations/juniper/cms/djangoapps/contentstore/views/transcripts_ajax.py diff --git a/derex/runner/compose_files/openedx_customizations/juniper/cms/djangoapps/contentstore/views/transcripts_ajax.py.juniper b/docker-definition/openedx_customizations/juniper/cms/djangoapps/contentstore/views/transcripts_ajax.py.juniper similarity index 100% rename from derex/runner/compose_files/openedx_customizations/juniper/cms/djangoapps/contentstore/views/transcripts_ajax.py.juniper rename to docker-definition/openedx_customizations/juniper/cms/djangoapps/contentstore/views/transcripts_ajax.py.juniper diff --git a/derex/runner/compose_files/openedx_customizations/juniper/lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py b/docker-definition/openedx_customizations/juniper/lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py similarity index 100% rename from derex/runner/compose_files/openedx_customizations/juniper/lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py rename to docker-definition/openedx_customizations/juniper/lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py diff --git a/derex/runner/compose_files/openedx_customizations/juniper/lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py.juniper b/docker-definition/openedx_customizations/juniper/lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py.juniper similarity index 100% rename from derex/runner/compose_files/openedx_customizations/juniper/lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py.juniper rename to docker-definition/openedx_customizations/juniper/lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py.juniper diff --git a/derex/runner/compose_files/openedx_customizations/juniper/lms/djangoapps/dashboard/sysadmin.py b/docker-definition/openedx_customizations/juniper/lms/djangoapps/dashboard/sysadmin.py similarity index 100% rename from derex/runner/compose_files/openedx_customizations/juniper/lms/djangoapps/dashboard/sysadmin.py rename to docker-definition/openedx_customizations/juniper/lms/djangoapps/dashboard/sysadmin.py diff --git a/derex/runner/compose_files/openedx_customizations/juniper/lms/djangoapps/dashboard/sysadmin.py.juniper b/docker-definition/openedx_customizations/juniper/lms/djangoapps/dashboard/sysadmin.py.juniper similarity index 100% rename from derex/runner/compose_files/openedx_customizations/juniper/lms/djangoapps/dashboard/sysadmin.py.juniper rename to docker-definition/openedx_customizations/juniper/lms/djangoapps/dashboard/sysadmin.py.juniper diff --git a/docker-definition/openedx_customizations/koa/.gitkeep b/docker-definition/openedx_customizations/koa/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-definition/openedx_customizations/koa/lms/djangoapps/dashboard/sysadmin.py b/docker-definition/openedx_customizations/koa/lms/djangoapps/dashboard/sysadmin.py new file mode 100644 index 000000000..189d9ab0a --- /dev/null +++ b/docker-definition/openedx_customizations/koa/lms/djangoapps/dashboard/sysadmin.py @@ -0,0 +1,506 @@ +""" +This module creates a sysadmin dashboard for managing and viewing +courses. +""" + + +import json +import logging +import os +import subprocess + +import mongoengine +from urllib.parse import quote_plus +from django.conf import settings +from django.contrib.auth.decorators import login_required +from django.contrib.auth.models import User +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator +from django.db import IntegrityError +from django.http import Http404 +from django.utils.decorators import method_decorator +from django.utils.html import escape +from django.utils.translation import ugettext as _ +from django.views.decorators.cache import cache_control +from django.views.decorators.csrf import ensure_csrf_cookie +from django.views.decorators.http import condition +from django.views.generic.base import TemplateView +from opaque_keys.edx.keys import CourseKey +from path import Path as path +from six import StringIO, text_type + +import lms.djangoapps.dashboard.git_import as git_import +from common.djangoapps.track import views as track_views +from lms.djangoapps.dashboard.git_import import GitImportError +from lms.djangoapps.dashboard.models import CourseImportLog +from common.djangoapps.edxmako.shortcuts import render_to_response +from lms.djangoapps.courseware.courses import get_course_by_id +from openedx.core.djangolib.markup import HTML +from common.djangoapps.student.models import CourseEnrollment, Registration, UserProfile +from common.djangoapps.student.roles import CourseInstructorRole, CourseStaffRole +from xmodule.modulestore.django import modulestore + +log = logging.getLogger(__name__) + + +class SysadminDashboardView(TemplateView): + """Base class for sysadmin dashboard views with common methods""" + + template_name = 'sysadmin_dashboard.html' + + def __init__(self, **kwargs): + """ + Initialize base sysadmin dashboard class with modulestore, + modulestore_type and return msg + """ + + self.def_ms = modulestore() + self.msg = u'' + self.datatable = [] + super(SysadminDashboardView, self).__init__(**kwargs) + + @method_decorator(ensure_csrf_cookie) + @method_decorator(login_required) + @method_decorator(cache_control(no_cache=True, no_store=True, + must_revalidate=True)) + @method_decorator(condition(etag_func=None)) + def dispatch(self, *args, **kwargs): + return super(SysadminDashboardView, self).dispatch(*args, **kwargs) + + def get_courses(self): + """ Get an iterable list of courses.""" + + return self.def_ms.get_courses() + + +class Users(SysadminDashboardView): + """ + The status view provides Web based user management, a listing of + courses loaded, and user statistics + """ + + def create_user(self, uname, name, password=None): + """ Creates a user """ + + if not uname: + return _('Must provide username') + if not name: + return _('Must provide full name') + + msg = u'' + if not password: + return _('Password must be supplied') + + email = uname + + if '@' not in email: + msg += _('email address required (not username)') + return msg + new_password = password + + user = User(username=uname, email=email, is_active=True) + user.set_password(new_password) + try: + user.save() + except IntegrityError: + msg += _(u'Oops, failed to create user {user}, {error}').format( + user=user, + error="IntegrityError" + ) + return msg + + reg = Registration() + reg.register(user) + + profile = UserProfile(user=user) + profile.name = name + profile.save() + + msg += _(u'User {user} created successfully!').format(user=user) + return msg + + def delete_user(self, uname): + """Deletes a user from django auth""" + + if not uname: + return _('Must provide username') + if '@' in uname: + try: + user = User.objects.get(email=uname) + except User.DoesNotExist as err: + msg = _(u'Cannot find user with email address {email_addr}').format(email_addr=uname) + return msg + else: + try: + user = User.objects.get(username=uname) + except User.DoesNotExist as err: + msg = _(u'Cannot find user with username {username} - {error}').format( + username=uname, + error=str(err) + ) + return msg + user.delete() + return _(u'Deleted user {username}').format(username=uname) + + def make_datatable(self): + """ + Build the datatable for this view + """ + datatable = { + 'header': [ + _('Statistic'), + _('Value'), + ], + 'title': _('Site statistics'), + 'data': [ + [ + _('Total number of users'), + User.objects.all().count(), + ], + ], + } + return datatable + + def get(self, request): + if not request.user.is_staff: + raise Http404 + context = { + 'datatable': self.make_datatable(), + 'msg': self.msg, + 'djangopid': os.getpid(), + 'modeflag': {'users': 'active-section'}, + } + return render_to_response(self.template_name, context) + + def post(self, request): + """Handle various actions available on page""" + + if not request.user.is_staff: + raise Http404 + action = request.POST.get('action', '') + track_views.server_track(request, action, {}, page='user_sysdashboard') + + if action == 'create_user': + uname = request.POST.get('student_uname', '').strip() + name = request.POST.get('student_fullname', '').strip() + password = request.POST.get('student_password', '').strip() + self.msg = HTML(u'

{0}

{1}


{2}').format( + _('Create User Results'), + self.create_user(uname, name, password), self.msg) + elif action == 'del_user': + uname = request.POST.get('student_uname', '').strip() + self.msg = HTML(u'

{0}

{1}


{2}').format( + _('Delete User Results'), self.delete_user(uname), self.msg) + context = { + 'datatable': self.make_datatable(), + 'msg': self.msg, + 'djangopid': os.getpid(), + 'modeflag': {'users': 'active-section'}, + } + return render_to_response(self.template_name, context) + + +class Courses(SysadminDashboardView): + """ + This manages adding/updating courses from git, deleting courses, and + provides course listing information. + """ + + def git_info_for_course(self, cdir): + """This pulls out some git info like the last commit""" + + cmd = '' + gdir = settings.DATA_DIR / cdir + info = ['', '', ''] + + # Try the data dir, then try to find it in the git import dir + if not gdir.exists(): + git_repo_dir = getattr(settings, 'GIT_REPO_DIR', git_import.DEFAULT_GIT_REPO_DIR) + gdir = path(git_repo_dir) / cdir + if not gdir.exists(): + return info + + cmd = ['git', 'log', '-1', + u'--format=format:{ "commit": "%H", "author": "%an %ae", "date": "%ad"}', ] + try: + output_json = json.loads(subprocess.check_output(cmd, cwd=gdir).decode('utf-8')) + info = [output_json['commit'], + output_json['date'], + output_json['author'], ] + except OSError as error: + log.warning(text_type(u"Error fetching git data: %s - %s"), text_type(cdir), text_type(error)) + except (ValueError, subprocess.CalledProcessError): + pass + + return info + + def get_course_from_git(self, gitloc, branch): + """This downloads and runs the checks for importing a course in git""" + + if not (gitloc.endswith('.git') or gitloc.startswith('http:') or + gitloc.startswith('https:') or gitloc.startswith('git:')): + return _("The git repo location should end with '.git', " + "and be a valid url") + + return self.import_mongo_course(gitloc, branch) + + def import_mongo_course(self, gitloc, branch): + """ + Imports course using management command and captures logging output + at debug level for display in template + """ + + msg = u'' + + log.debug(u'Adding course using git repo %s', gitloc) + + # Grab logging output for debugging imports + output = StringIO() + import_log_handler = logging.StreamHandler(output) + import_log_handler.setLevel(logging.DEBUG) + + logger_names = ['xmodule.modulestore.xml_importer', + 'lms.djangoapps.dashboard.git_import', + 'xmodule.modulestore.xml', + 'xmodule.seq_module', ] + loggers = [] + + for logger_name in logger_names: + logger = logging.getLogger(logger_name) + logger.setLevel(logging.DEBUG) + logger.addHandler(import_log_handler) + loggers.append(logger) + + error_msg = '' + try: + git_import.add_repo(gitloc, None, branch) + except GitImportError as ex: + error_msg = str(ex) + ret = output.getvalue() + + # Remove handler hijacks + for logger in loggers: + logger.setLevel(logging.NOTSET) + logger.removeHandler(import_log_handler) + + if error_msg: + msg_header = error_msg + color = 'red' + else: + msg_header = _('Added Course') + color = 'blue' + + msg = HTML(u"

{1}

").format(color, msg_header) + msg += HTML(u"
{0}
").format(escape(ret)) + return msg + + def make_datatable(self, courses=None): + """Creates course information datatable""" + + data = [] + courses = courses or self.get_courses() + for course in courses: + gdir = course.id.course + data.append([course.display_name, text_type(course.id)] + + self.git_info_for_course(gdir)) + + return dict(header=[_('Course Name'), + _('Directory/ID'), + # Translators: "Git Commit" is a computer command; see http://gitref.org/basic/#commit + _('Git Commit'), + _('Last Change'), + _('Last Editor')], + title=_('Information about all courses'), + data=data) + + def get(self, request): + """Displays forms and course information""" + + if not request.user.is_staff: + raise Http404 + + context = { + 'datatable': self.make_datatable(), + 'msg': self.msg, + 'djangopid': os.getpid(), + 'modeflag': {'courses': 'active-section'}, + } + return render_to_response(self.template_name, context) + + def post(self, request): + """Handle all actions from courses view""" + + if not request.user.is_staff: + raise Http404 + + action = request.POST.get('action', '') + track_views.server_track(request, action, {}, + page='courses_sysdashboard') + + courses = {course.id: course for course in self.get_courses()} + if action == 'add_course': + gitloc = request.POST.get('repo_location', '').strip().replace(' ', '').replace(';', '') + branch = request.POST.get('repo_branch', '').strip().replace(' ', '').replace(';', '') + self.msg += self.get_course_from_git(gitloc, branch) + + elif action == 'del_course': + course_id = request.POST.get('course_id', '').strip() + course_key = CourseKey.from_string(course_id) + course_found = False + if course_key in courses: + course_found = True + course = courses[course_key] + else: + try: + course = get_course_by_id(course_key) + course_found = True + except Exception as err: # pylint: disable=broad-except + self.msg += _( + HTML(u'Error - cannot get course with ID {0}
{1}
') + ).format( + course_key, + escape(str(err)) + ) + + if course_found: + # delete course that is stored with mongodb backend + self.def_ms.delete_course(course.id, request.user.id) + # don't delete user permission groups, though + self.msg += \ + HTML(u"{0} {1} = {2} ({3})").format( + _('Deleted'), text_type(course.location), text_type(course.id), course.display_name) + + context = { + 'datatable': self.make_datatable(list(courses.values())), + 'msg': self.msg, + 'djangopid': os.getpid(), + 'modeflag': {'courses': 'active-section'}, + } + return render_to_response(self.template_name, context) + + +class Staffing(SysadminDashboardView): + """ + The status view provides a view of staffing and enrollment in + courses. + """ + + def get(self, request): + """Displays course Enrollment and staffing course statistics""" + + if not request.user.is_staff: + raise Http404 + data = [] + + for course in self.get_courses(): + datum = [course.display_name, course.id] + datum += [CourseEnrollment.objects.filter( + course_id=course.id).count()] + datum += [CourseStaffRole(course.id).users_with_role().count()] + datum += [','.join([x.username for x in CourseInstructorRole( + course.id).users_with_role()])] + data.append(datum) + + datatable = dict(header=[_('Course Name'), _('course_id'), + _('# enrolled'), _('# staff'), + _('instructors')], + title=_('Enrollment information for all courses'), + data=data) + context = { + 'datatable': datatable, + 'msg': self.msg, + 'djangopid': os.getpid(), + 'modeflag': {'staffing': 'active-section'}, + } + return render_to_response(self.template_name, context) + + +class GitLogs(TemplateView): + """ + This provides a view into the import of courses from git repositories. + It is convenient for allowing course teams to see what may be wrong with + their xml + """ + + template_name = 'sysadmin_dashboard_gitlogs.html' + + @method_decorator(login_required) + def get(self, request, *args, **kwargs): + """Shows logs of imports that happened as a result of a git import""" + + course_id = kwargs.get('course_id') + if course_id: + course_id = CourseKey.from_string(course_id) + + page_size = 10 + + # Set mongodb defaults even if it isn't defined in settings + mongo_db = { + 'host': 'localhost', + 'user': '', + 'password': '', + 'db': 'xlog', + 'authsource': 'admin' + } + + # Allow overrides + if hasattr(settings, 'MONGODB_LOG'): + for config_item in ['host', 'user', 'password', 'db', 'authsource']: + mongo_db[config_item] = settings.MONGODB_LOG.get( + config_item, mongo_db[config_item]) + + # Make sure special characters in user and password are quoted + if mongo_db['user'] or mongodb['password']: + mongo_db['user'] = quote_plus(mongo_db['user']) + mongo_db['password'] = quote_plus(mongo_db['password']) + + mongouri = 'mongodb://{user}:{password}@{host}/{db}?authSource={authsource}'.format(**mongo_db) + + error_msg = '' + + try: + if mongo_db['user'] and mongo_db['password']: + mdb = mongoengine.connect(mongo_db['db'], host=mongouri) + else: + mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host']) + except mongoengine.connection.ConnectionError: + log.exception('Unable to connect to mongodb to save log, ' + 'please check MONGODB_LOG settings.') + + if course_id is None: + # Require staff if not going to specific course + if not request.user.is_staff: + raise Http404 + cilset = CourseImportLog.objects.order_by('-created') + else: + # Allow only course team, instructors, and staff + if not (request.user.is_staff or + CourseInstructorRole(course_id).has_user(request.user) or + CourseStaffRole(course_id).has_user(request.user)): + raise Http404 + log.debug('course_id=%s', course_id) + cilset = CourseImportLog.objects.filter( + course_id=course_id + ).order_by('-created') + log.debug(u'cilset length=%s', len(cilset)) + + # Paginate the query set + paginator = Paginator(cilset, page_size) + try: + logs = paginator.page(request.GET.get('page')) + except PageNotAnInteger: + logs = paginator.page(1) + except EmptyPage: + # If the page is too high or low + given_page = int(request.GET.get('page')) + page = min(max(1, given_page), paginator.num_pages) + logs = paginator.page(page) + + mdb.close() + context = { + 'logs': logs, + 'course_id': text_type(course_id) if course_id else None, + 'error_msg': error_msg, + 'page_size': page_size + } + + return render_to_response(self.template_name, context) diff --git a/docker-definition/openedx_customizations/koa/lms/djangoapps/dashboard/sysadmin.py.koa b/docker-definition/openedx_customizations/koa/lms/djangoapps/dashboard/sysadmin.py.koa new file mode 100644 index 000000000..dc26b0e46 --- /dev/null +++ b/docker-definition/openedx_customizations/koa/lms/djangoapps/dashboard/sysadmin.py.koa @@ -0,0 +1,499 @@ +""" +This module creates a sysadmin dashboard for managing and viewing +courses. +""" + + +import json +import logging +import os +import subprocess + +import mongoengine +from django.conf import settings +from django.contrib.auth.decorators import login_required +from django.contrib.auth.models import User +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator +from django.db import IntegrityError +from django.http import Http404 +from django.utils.decorators import method_decorator +from django.utils.html import escape +from django.utils.translation import ugettext as _ +from django.views.decorators.cache import cache_control +from django.views.decorators.csrf import ensure_csrf_cookie +from django.views.decorators.http import condition +from django.views.generic.base import TemplateView +from opaque_keys.edx.keys import CourseKey +from path import Path as path +from six import StringIO, text_type + +import lms.djangoapps.dashboard.git_import as git_import +from common.djangoapps.track import views as track_views +from lms.djangoapps.dashboard.git_import import GitImportError +from lms.djangoapps.dashboard.models import CourseImportLog +from common.djangoapps.edxmako.shortcuts import render_to_response +from lms.djangoapps.courseware.courses import get_course_by_id +from openedx.core.djangolib.markup import HTML +from common.djangoapps.student.models import CourseEnrollment, Registration, UserProfile +from common.djangoapps.student.roles import CourseInstructorRole, CourseStaffRole +from xmodule.modulestore.django import modulestore + +log = logging.getLogger(__name__) + + +class SysadminDashboardView(TemplateView): + """Base class for sysadmin dashboard views with common methods""" + + template_name = 'sysadmin_dashboard.html' + + def __init__(self, **kwargs): + """ + Initialize base sysadmin dashboard class with modulestore, + modulestore_type and return msg + """ + + self.def_ms = modulestore() + self.msg = u'' + self.datatable = [] + super(SysadminDashboardView, self).__init__(**kwargs) + + @method_decorator(ensure_csrf_cookie) + @method_decorator(login_required) + @method_decorator(cache_control(no_cache=True, no_store=True, + must_revalidate=True)) + @method_decorator(condition(etag_func=None)) + def dispatch(self, *args, **kwargs): + return super(SysadminDashboardView, self).dispatch(*args, **kwargs) + + def get_courses(self): + """ Get an iterable list of courses.""" + + return self.def_ms.get_courses() + + +class Users(SysadminDashboardView): + """ + The status view provides Web based user management, a listing of + courses loaded, and user statistics + """ + + def create_user(self, uname, name, password=None): + """ Creates a user """ + + if not uname: + return _('Must provide username') + if not name: + return _('Must provide full name') + + msg = u'' + if not password: + return _('Password must be supplied') + + email = uname + + if '@' not in email: + msg += _('email address required (not username)') + return msg + new_password = password + + user = User(username=uname, email=email, is_active=True) + user.set_password(new_password) + try: + user.save() + except IntegrityError: + msg += _(u'Oops, failed to create user {user}, {error}').format( + user=user, + error="IntegrityError" + ) + return msg + + reg = Registration() + reg.register(user) + + profile = UserProfile(user=user) + profile.name = name + profile.save() + + msg += _(u'User {user} created successfully!').format(user=user) + return msg + + def delete_user(self, uname): + """Deletes a user from django auth""" + + if not uname: + return _('Must provide username') + if '@' in uname: + try: + user = User.objects.get(email=uname) + except User.DoesNotExist as err: + msg = _(u'Cannot find user with email address {email_addr}').format(email_addr=uname) + return msg + else: + try: + user = User.objects.get(username=uname) + except User.DoesNotExist as err: + msg = _(u'Cannot find user with username {username} - {error}').format( + username=uname, + error=str(err) + ) + return msg + user.delete() + return _(u'Deleted user {username}').format(username=uname) + + def make_datatable(self): + """ + Build the datatable for this view + """ + datatable = { + 'header': [ + _('Statistic'), + _('Value'), + ], + 'title': _('Site statistics'), + 'data': [ + [ + _('Total number of users'), + User.objects.all().count(), + ], + ], + } + return datatable + + def get(self, request): + if not request.user.is_staff: + raise Http404 + context = { + 'datatable': self.make_datatable(), + 'msg': self.msg, + 'djangopid': os.getpid(), + 'modeflag': {'users': 'active-section'}, + } + return render_to_response(self.template_name, context) + + def post(self, request): + """Handle various actions available on page""" + + if not request.user.is_staff: + raise Http404 + action = request.POST.get('action', '') + track_views.server_track(request, action, {}, page='user_sysdashboard') + + if action == 'create_user': + uname = request.POST.get('student_uname', '').strip() + name = request.POST.get('student_fullname', '').strip() + password = request.POST.get('student_password', '').strip() + self.msg = HTML(u'

{0}

{1}


{2}').format( + _('Create User Results'), + self.create_user(uname, name, password), self.msg) + elif action == 'del_user': + uname = request.POST.get('student_uname', '').strip() + self.msg = HTML(u'

{0}

{1}


{2}').format( + _('Delete User Results'), self.delete_user(uname), self.msg) + context = { + 'datatable': self.make_datatable(), + 'msg': self.msg, + 'djangopid': os.getpid(), + 'modeflag': {'users': 'active-section'}, + } + return render_to_response(self.template_name, context) + + +class Courses(SysadminDashboardView): + """ + This manages adding/updating courses from git, deleting courses, and + provides course listing information. + """ + + def git_info_for_course(self, cdir): + """This pulls out some git info like the last commit""" + + cmd = '' + gdir = settings.DATA_DIR / cdir + info = ['', '', ''] + + # Try the data dir, then try to find it in the git import dir + if not gdir.exists(): + git_repo_dir = getattr(settings, 'GIT_REPO_DIR', git_import.DEFAULT_GIT_REPO_DIR) + gdir = path(git_repo_dir) / cdir + if not gdir.exists(): + return info + + cmd = ['git', 'log', '-1', + u'--format=format:{ "commit": "%H", "author": "%an %ae", "date": "%ad"}', ] + try: + output_json = json.loads(subprocess.check_output(cmd, cwd=gdir).decode('utf-8')) + info = [output_json['commit'], + output_json['date'], + output_json['author'], ] + except OSError as error: + log.warning(text_type(u"Error fetching git data: %s - %s"), text_type(cdir), text_type(error)) + except (ValueError, subprocess.CalledProcessError): + pass + + return info + + def get_course_from_git(self, gitloc, branch): + """This downloads and runs the checks for importing a course in git""" + + if not (gitloc.endswith('.git') or gitloc.startswith('http:') or + gitloc.startswith('https:') or gitloc.startswith('git:')): + return _("The git repo location should end with '.git', " + "and be a valid url") + + return self.import_mongo_course(gitloc, branch) + + def import_mongo_course(self, gitloc, branch): + """ + Imports course using management command and captures logging output + at debug level for display in template + """ + + msg = u'' + + log.debug(u'Adding course using git repo %s', gitloc) + + # Grab logging output for debugging imports + output = StringIO() + import_log_handler = logging.StreamHandler(output) + import_log_handler.setLevel(logging.DEBUG) + + logger_names = ['xmodule.modulestore.xml_importer', + 'lms.djangoapps.dashboard.git_import', + 'xmodule.modulestore.xml', + 'xmodule.seq_module', ] + loggers = [] + + for logger_name in logger_names: + logger = logging.getLogger(logger_name) + logger.setLevel(logging.DEBUG) + logger.addHandler(import_log_handler) + loggers.append(logger) + + error_msg = '' + try: + git_import.add_repo(gitloc, None, branch) + except GitImportError as ex: + error_msg = str(ex) + ret = output.getvalue() + + # Remove handler hijacks + for logger in loggers: + logger.setLevel(logging.NOTSET) + logger.removeHandler(import_log_handler) + + if error_msg: + msg_header = error_msg + color = 'red' + else: + msg_header = _('Added Course') + color = 'blue' + + msg = HTML(u"

{1}

").format(color, msg_header) + msg += HTML(u"
{0}
").format(escape(ret)) + return msg + + def make_datatable(self, courses=None): + """Creates course information datatable""" + + data = [] + courses = courses or self.get_courses() + for course in courses: + gdir = course.id.course + data.append([course.display_name, text_type(course.id)] + + self.git_info_for_course(gdir)) + + return dict(header=[_('Course Name'), + _('Directory/ID'), + # Translators: "Git Commit" is a computer command; see http://gitref.org/basic/#commit + _('Git Commit'), + _('Last Change'), + _('Last Editor')], + title=_('Information about all courses'), + data=data) + + def get(self, request): + """Displays forms and course information""" + + if not request.user.is_staff: + raise Http404 + + context = { + 'datatable': self.make_datatable(), + 'msg': self.msg, + 'djangopid': os.getpid(), + 'modeflag': {'courses': 'active-section'}, + } + return render_to_response(self.template_name, context) + + def post(self, request): + """Handle all actions from courses view""" + + if not request.user.is_staff: + raise Http404 + + action = request.POST.get('action', '') + track_views.server_track(request, action, {}, + page='courses_sysdashboard') + + courses = {course.id: course for course in self.get_courses()} + if action == 'add_course': + gitloc = request.POST.get('repo_location', '').strip().replace(' ', '').replace(';', '') + branch = request.POST.get('repo_branch', '').strip().replace(' ', '').replace(';', '') + self.msg += self.get_course_from_git(gitloc, branch) + + elif action == 'del_course': + course_id = request.POST.get('course_id', '').strip() + course_key = CourseKey.from_string(course_id) + course_found = False + if course_key in courses: + course_found = True + course = courses[course_key] + else: + try: + course = get_course_by_id(course_key) + course_found = True + except Exception as err: # pylint: disable=broad-except + self.msg += _( + HTML(u'Error - cannot get course with ID {0}
{1}
') + ).format( + course_key, + escape(str(err)) + ) + + if course_found: + # delete course that is stored with mongodb backend + self.def_ms.delete_course(course.id, request.user.id) + # don't delete user permission groups, though + self.msg += \ + HTML(u"{0} {1} = {2} ({3})").format( + _('Deleted'), text_type(course.location), text_type(course.id), course.display_name) + + context = { + 'datatable': self.make_datatable(list(courses.values())), + 'msg': self.msg, + 'djangopid': os.getpid(), + 'modeflag': {'courses': 'active-section'}, + } + return render_to_response(self.template_name, context) + + +class Staffing(SysadminDashboardView): + """ + The status view provides a view of staffing and enrollment in + courses. + """ + + def get(self, request): + """Displays course Enrollment and staffing course statistics""" + + if not request.user.is_staff: + raise Http404 + data = [] + + for course in self.get_courses(): + datum = [course.display_name, course.id] + datum += [CourseEnrollment.objects.filter( + course_id=course.id).count()] + datum += [CourseStaffRole(course.id).users_with_role().count()] + datum += [','.join([x.username for x in CourseInstructorRole( + course.id).users_with_role()])] + data.append(datum) + + datatable = dict(header=[_('Course Name'), _('course_id'), + _('# enrolled'), _('# staff'), + _('instructors')], + title=_('Enrollment information for all courses'), + data=data) + context = { + 'datatable': datatable, + 'msg': self.msg, + 'djangopid': os.getpid(), + 'modeflag': {'staffing': 'active-section'}, + } + return render_to_response(self.template_name, context) + + +class GitLogs(TemplateView): + """ + This provides a view into the import of courses from git repositories. + It is convenient for allowing course teams to see what may be wrong with + their xml + """ + + template_name = 'sysadmin_dashboard_gitlogs.html' + + @method_decorator(login_required) + def get(self, request, *args, **kwargs): + """Shows logs of imports that happened as a result of a git import""" + + course_id = kwargs.get('course_id') + if course_id: + course_id = CourseKey.from_string(course_id) + + page_size = 10 + + # Set mongodb defaults even if it isn't defined in settings + mongo_db = { + 'host': 'localhost', + 'user': '', + 'password': '', + 'db': 'xlog', + } + + # Allow overrides + if hasattr(settings, 'MONGODB_LOG'): + for config_item in ['host', 'user', 'password', 'db', ]: + mongo_db[config_item] = settings.MONGODB_LOG.get( + config_item, mongo_db[config_item]) + + mongouri = 'mongodb://{user}:{password}@{host}/{db}'.format(**mongo_db) + + error_msg = '' + + try: + if mongo_db['user'] and mongo_db['password']: + mdb = mongoengine.connect(mongo_db['db'], host=mongouri) + else: + mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host']) + except mongoengine.connection.ConnectionError: + log.exception('Unable to connect to mongodb to save log, ' + 'please check MONGODB_LOG settings.') + + if course_id is None: + # Require staff if not going to specific course + if not request.user.is_staff: + raise Http404 + cilset = CourseImportLog.objects.order_by('-created') + else: + # Allow only course team, instructors, and staff + if not (request.user.is_staff or + CourseInstructorRole(course_id).has_user(request.user) or + CourseStaffRole(course_id).has_user(request.user)): + raise Http404 + log.debug('course_id=%s', course_id) + cilset = CourseImportLog.objects.filter( + course_id=course_id + ).order_by('-created') + log.debug(u'cilset length=%s', len(cilset)) + + # Paginate the query set + paginator = Paginator(cilset, page_size) + try: + logs = paginator.page(request.GET.get('page')) + except PageNotAnInteger: + logs = paginator.page(1) + except EmptyPage: + # If the page is too high or low + given_page = int(request.GET.get('page')) + page = min(max(1, given_page), paginator.num_pages) + logs = paginator.page(page) + + mdb.close() + context = { + 'logs': logs, + 'course_id': text_type(course_id) if course_id else None, + 'error_msg': error_msg, + 'page_size': page_size + } + + return render_to_response(self.template_name, context) diff --git a/docker-definition/scripts/derex_update_openedx_translations b/docker-definition/scripts/derex_update_openedx_translations new file mode 100755 index 000000000..1c777e680 --- /dev/null +++ b/docker-definition/scripts/derex_update_openedx_translations @@ -0,0 +1,43 @@ +#!/bin/sh +set -e + +if [ \! -f /root/.transifexrc-orig ]; then + echo "Transifex credentials unset. Building without translations." + ls -la /root/ + exit 0 +else + echo Translations found. +fi + +set -x +# Unfortunately transifex really wants to rewrite its config file on every invocation. +# This behaviour can be tested with: +# python -c "import txclib.utils; txclib.utils.get_transifex_config(u'/root/.transifexrc')" +# https://github.com/transifex/transifex-client/issues/181 +# To work around this we copy the file and remove the copy before exiting +cp /root/.transifexrc-orig /root/.transifexrc + +cd /openedx/edx-platform + +# Enable German translations +sed -i '/de_DE/ s/# -/-/' conf/locale/config.yaml +# Enable Italian translations +sed -i '/it_IT/ s/# -/-/' conf/locale/config.yaml + +# The Basque translations has issues in Juniper (the only release where it's enabled so far) +sed -i '/eu_ES/d' conf/locale/config.yaml + +SERVICE_VARIANT=lms i18n_tool transifex pull +SERVICE_VARIANT=lms i18n_tool extract +SERVICE_VARIANT=lms i18n_tool generate + +SERVICE_VARIANT=lms python manage.py lms compilemessages -v2 +SERVICE_VARIANT=cms python manage.py cms compilemessages -v2 + +SERVICE_VARIANT=lms python manage.py lms compilejsi18n -v2 +SERVICE_VARIANT=cms python manage.py cms compilejsi18n -v2 + +# This check is currently done in the azure pipeline. +# i18n_tool validate || (find conf|grep prob; find conf|grep prob|xargs cat; false) + +rm /root/.transifexrc diff --git a/docker-definition/scripts/derex_update_translations b/docker-definition/scripts/derex_update_translations old mode 100755 new mode 100644 index 90f5c1466..e82653da0 --- a/docker-definition/scripts/derex_update_translations +++ b/docker-definition/scripts/derex_update_translations @@ -1,44 +1,11 @@ #!/bin/sh set -e -if [ \! -f /root/.transifexrc-orig ]; then - echo "Transifex credentials unset. Building without translations." - ls -la /root/ - exit 0 -else - echo Translations found. -fi - -set -x -# Unfortunately transifex really wants to rewrite its config file on every invocation. -# This behaviour can be tested with: -# python -c "import txclib.utils; txclib.utils.get_transifex_config(u'/root/.transifexrc')" -# https://github.com/transifex/transifex-client/issues/181 -# To work around this we copy the file and remove the copy before exiting -cp /root/.transifexrc-orig /root/.transifexrc - -cd /openedx/edx-platform - -# Enable German translations -sed -i '/de_DE/ s/# -/-/' conf/locale/config.yaml -# Enable Italian translations -sed -i '/it_IT/ s/# -/-/' conf/locale/config.yaml - -# The Basque translations has issues in Juniper (the only release where it's enabled so far) -sed -i '/eu_ES/d' conf/locale/config.yaml - -i18n_tool transifex pull -i18n_tool extract - -i18n_tool generate - -python manage.py lms --settings=derex_django.settings.build.assets compilemessages -v2 -python manage.py cms --settings=derex_django.settings.build.assets compilemessages -v2 - -python manage.py lms --settings=derex_django.settings.build.assets compilejsi18n -v2 -python manage.py cms --settings=derex_django.settings.build.assets compilejsi18n -v2 - -# This check is currently done in the azure pipeline. -# i18n_tool validate || (find conf|grep prob; find conf|grep prob|xargs cat; false) - -rm /root/.transifexrc +SERVICE_VARIANT=lms python manage.py lms compilemessages +SERVICE_VARIANT=cms python manage.py cms compilemessages +SERVICE_VARIANT=lms python manage.py lms compilejsi18n +SERVICE_VARIANT=cms python manage.py cms compilejsi18n + +sh -c 'mkdir -p /openedx/staticfiles/js/i18n/; cp -rf /openedx/edx-platform/lms/static/js/i18n/ /openedx/staticfiles/js/i18n/' +sh -c 'apk add gzip; gzip -rkf /openedx/staticfiles/js/i18n/*/*.js; apk del gzip' +sh -c 'apk add brotli; brotli -f /openedx/staticfiles/js/i18n/*/*.js; apk del brotli' diff --git a/docs/conf.py b/docs/conf.py index 5a8830969..b52ac3169 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -172,12 +172,14 @@ ("py:class", "RuntimeError"), ("py:class", "ValueError"), ("py:class", "enum.Enum"), + ("py:class", "enum.IntEnum"), ("py:class", "logging.Formatter"), ("py:class", "Project"), ("py:class", "ProjectRunMode"), ("py:class", "click.core.Context"), ("py:class", "DerexSecrets"), ("py:class", "derex.runner.project.Project"), + ("py:class", "derex.runner.constants.ProjectBuildTargets"), ("py:class", "Optional['derex.runner.project.Project']"), ("py:class", "Path"), ("py:class", "pathlib.Path"), @@ -187,6 +189,7 @@ ("py:meth", "str.format"), ("py:obj", "object"), ("py:obj", "enum.Enum"), + ("py:obj", "enum.IntEnum"), ("py:obj", "RuntimeError"), ("py:obj", "ValueError"), ] diff --git a/examples/ironwood/complete/derex.config.yaml b/examples/ironwood/complete/derex.config.yaml index 115684770..c74ca1696 100644 --- a/examples/ironwood/complete/derex.config.yaml +++ b/examples/ironwood/complete/derex.config.yaml @@ -1,6 +1,6 @@ project_name: ironwood-complete openedx_version: ironwood -update_assets: true +collect_assets: true materialize_derex_settings: false variables: lms_test_dict: diff --git a/examples/ironwood/complete/e2e/cypress/integration/lms/accountSettings.test.js b/examples/ironwood/complete/e2e/cypress/integration/lms/accountSettings.test.js index f48ead7d4..ac810fe8d 100644 --- a/examples/ironwood/complete/e2e/cypress/integration/lms/accountSettings.test.js +++ b/examples/ironwood/complete/e2e/cypress/integration/lms/accountSettings.test.js @@ -33,6 +33,9 @@ describe("Account settings tests", () => { afterEach(() => { // We click somewhere else to trigger the save action cy.get("#u-field-value-username").click(); - cy.get("body").should("contain", "Your changes have been saved"); + cy.get("body").should( + "contain", + "Süççéss Ⱡ'σяєм ιρѕυм #Ýöür çhängés hävé ßéén sävéd. Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт αмєт, ¢σηѕє¢#" + ); }); }); diff --git a/examples/ironwood/complete/e2e/cypress/integration/lms/authentication.test.js b/examples/ironwood/complete/e2e/cypress/integration/lms/authentication.test.js index fdea99404..4417b1254 100644 --- a/examples/ironwood/complete/e2e/cypress/integration/lms/authentication.test.js +++ b/examples/ironwood/complete/e2e/cypress/integration/lms/authentication.test.js @@ -20,11 +20,11 @@ describe("Login tests", () => { cy.get(".action").click(); cy.get(".message-copy > :nth-child(1)").should( "have.text", - "Please enter your Email." + "Pléäsé éntér ýöür Émäïl Ⱡ'σяєм ιρѕ#. Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт αмєт, ¢#" ); cy.get(".message-copy > :nth-child(2)").should( "have.text", - "Please enter your Password." + "Pléäsé éntér ýöür Pässwörd Ⱡ'σяєм ιρѕυм ∂#. Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт αмєт, ¢#" ); }); @@ -34,7 +34,7 @@ describe("Login tests", () => { cy.get(".action").click(); cy.get(".message-copy > :nth-child(1)").should( "have.text", - "Email or password is incorrect." + "Émäïl ör pässwörd ïs ïnçörréçt. Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт αмєт, ¢σηѕє¢т#" ); }); @@ -52,7 +52,10 @@ describe("Login tests", () => { cy.get(".forgot-password").click(); cy.get("#password-reset-email").type(Cypress.env("learner_user").email); cy.get("#password-reset > .action").click(); - cy.get(".js-password-reset-success").should("contain", "Check Your Email"); + cy.get(".js-password-reset-success").should( + "contain", + "Çhéçk Ýöür Émäïl Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт αм#" + ); cy.get(".js-password-reset-success").should( "contain", Cypress.env("learner_user").email @@ -89,6 +92,10 @@ describe("Registration tests", () => { it("A unauthenticated user can register by filling the registration form", () => { cy.get("#register-name").type(`${randomString(10)} ${randomString(10)}`); + // The following click is needed to avoid the error + // "element is being covered by another element + // `