diff --git a/.github/workflows/publish_packages.yml b/.github/workflows/publish_packages.yml new file mode 100644 index 0000000..bfe437d --- /dev/null +++ b/.github/workflows/publish_packages.yml @@ -0,0 +1,45 @@ +name: Publish Docker image + +on: + release: + types: [published] + +jobs: + push_to_registries: + name: Push Docker image to multiple registries + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - name: Check out the repo + uses: actions/checkout@v4 + + - name: Log in to Docker Hub + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.TOKEN_FOR_GHCR }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: | + ebivariation/eva-sub-cli + ghcr.io/ebivariation/eva-sub-cli + + - name: Build and push Docker images + uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 + with: + context: docker + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 141b351..9dca22e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,8 +25,6 @@ jobs: pip install flake8 pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi python setup.py install - # build docker image - docker build -t eva_sub_cli -f docker/Dockerfile . - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names diff --git a/cli/docker_validator.py b/cli/docker_validator.py index ddd38f1..f6e128c 100644 --- a/cli/docker_validator.py +++ b/cli/docker_validator.py @@ -1,9 +1,12 @@ import argparse import csv import os +import re import subprocess import time +from ebi_eva_common_pyutils.command_utils import run_command_with_output + from cli import ETC_DIR from cli.reporter import Reporter from ebi_eva_common_pyutils.logger import logging_config @@ -11,50 +14,24 @@ logger = logging_config.get_logger(__name__) docker_path = 'docker' -container_image = 'eva_sub_cli' +container_image = 'ebivariation/eva-sub-cli' +container_tag = 'v0.0.1.dev0' container_validation_dir = '/opt/vcf_validation' container_validation_output_dir = '/opt/vcf_validation/vcf_validation_output' container_etc_dir = '/opt/cli/etc' -def run_command_with_output(command_description, command, return_process_output=True, - log_error_stream_to_output=False): - process_output = "" - - logger.info(f"Starting process: {command_description}") - logger.info(f"Running command: {command}") - - stdout = subprocess.PIPE - # Some utilities output non-error messages to error stream. This is a workaround for that - stderr = subprocess.STDOUT if log_error_stream_to_output else subprocess.PIPE - with subprocess.Popen(command, stdout=stdout, stderr=stderr, bufsize=1, universal_newlines=True, - shell=True) as process: - for line in iter(process.stdout.readline, ''): - line = str(line).rstrip() - logger.info(line) - if return_process_output: - process_output += line + "\n" - if not log_error_stream_to_output: - for line in iter(process.stderr.readline, ''): - line = str(line).rstrip() - logger.error(line) - if process.returncode != 0: - logger.error(f"{command_description} - failed! Refer to the error messages for details.") - raise subprocess.CalledProcessError(process.returncode, process.args) - else: - logger.info(f"{command_description} - completed successfully") - if return_process_output: - return process_output - class DockerValidator(Reporter): - def __init__(self, mapping_file, output_dir, metadata_json=None, metadata_xlsx=None, - container_name=container_image, docker_path='docker', submission_config=None): + def __init__(self, mapping_file, output_dir, metadata_json=None, + metadata_xlsx=None, container_name=None, docker_path='docker', submission_config=None): self.docker_path = docker_path self.mapping_file = mapping_file self.metadata_json = metadata_json self.metadata_xlsx = metadata_xlsx self.container_name = container_name + if self.container_name is None: + self.container_name = container_image.split('/')[1] + '.' + container_tag self.spreadsheet2json_conf = os.path.join(ETC_DIR, "spreadsheet2json_conf.yaml") super().__init__(self._find_vcf_file(), output_dir, submission_config=submission_config) @@ -154,7 +131,7 @@ def verify_docker_is_installed(self): raise RuntimeError(f"Please make sure docker ({self.docker_path}) is installed and available on the path") def verify_container_is_running(self): - container_run_cmd_ouptut = run_command_with_output("check if container is running", f"{self.docker_path} ps") + container_run_cmd_ouptut = run_command_with_output("check if container is running", f"{self.docker_path} ps", return_process_output=True) if container_run_cmd_ouptut is not None and self.container_name in container_run_cmd_ouptut: logger.info(f"Container ({self.container_name}) is running") return True @@ -166,6 +143,7 @@ def verify_container_is_stopped(self): container_stop_cmd_output = run_command_with_output( "check if container is stopped", f"{self.docker_path} ps -a" + , return_process_output=True ) if container_stop_cmd_output is not None and self.container_name in container_stop_cmd_output: logger.info(f"Container ({self.container_name}) is in stop state") @@ -187,9 +165,10 @@ def try_restarting_container(self): def verify_image_available_locally(self): container_images_cmd_ouptut = run_command_with_output( "Check if validator image is present", - f"{self.docker_path} images" + f"{self.docker_path} images", + return_process_output=True ) - if container_images_cmd_ouptut is not None and container_image in container_images_cmd_ouptut: + if container_images_cmd_ouptut is not None and re.search(container_image + r'\s+' + container_tag, container_images_cmd_ouptut): logger.info(f"Container ({container_image}) image is available locally") return True else: @@ -201,7 +180,7 @@ def run_container(self): try: run_command_with_output( "Try running container", - f"{self.docker_path} run -it --rm -d --name {self.container_name} {container_image}" + f"{self.docker_path} run -it --rm -d --name {self.container_name} {container_image}:{container_tag}" ) # stopping execution to give some time to container to get up and running time.sleep(5) @@ -215,18 +194,19 @@ def stop_running_container(self): if not self.verify_container_is_stopped(): run_command_with_output( "Stop the running container", - f"{self.docker_path} stop --name {self.container_name}" + f"{self.docker_path} stop {self.container_name}" ) def download_container_image(self): logger.info(f"Pulling container ({container_image}) image") try: - run_command_with_output("pull container image", f"{self.docker_path} pull {container_image}") - if not self.run_container(): - raise RuntimeError(f"Container ({self.container_name}) could not be started") + run_command_with_output("pull container image", f"{self.docker_path} pull {container_image}:{container_tag}") except subprocess.CalledProcessError as ex: logger.error(ex) raise RuntimeError(f"Cannot pull container ({container_image}) image") + # Give the pull command some time to complete + time.sleep(5) + self.run_container() def verify_docker_env(self): self.verify_docker_is_installed() @@ -289,4 +269,4 @@ def _copy(file_description, file_path): validator = DockerValidator(args.vcf_files_mapping, args.output_dir, args.metadata_json, args.metadata_xlsx, docker_container_name, docker_path) validator.validate() - validator.create_reports() \ No newline at end of file + validator.create_reports()