diff --git a/cli/auth.py b/cli/auth.py index 37d136b..c15db82 100644 --- a/cli/auth.py +++ b/cli/auth.py @@ -27,8 +27,9 @@ def token(self): 'scope': 'openid' } response = requests.post(self.device_authorization_url, data=payload) - response_json = response.json() + response.raise_for_status() + response_json = response.json() device_code = response_json['device_code'] user_code = response_json['user_code'] verification_uri = response_json['verification_uri'] diff --git a/cli/submit.py b/cli/submit.py index ee2fc6a..c93927f 100644 --- a/cli/submit.py +++ b/cli/submit.py @@ -1,23 +1,27 @@ #!/usr/bin/env python import os +from urllib.parse import urljoin import requests import yaml -from ebi_eva_common_pyutils.logger import logging_config + +from ebi_eva_common_pyutils.logger import AppLogger +from retry import retry from cli.auth import get_auth -logger = logging_config.get_logger(__name__) SUB_CLI_CONFIG_FILE = ".eva-sub-cli-config.yml" SUB_CLI_CONFIG_KEY_SUBMISSION_ID = "submission_id" SUB_CLI_CONFIG_KEY_SUBMISSION_UPLOAD_URL = "submission_upload_url" SUBMISSION_INITIATE_URL = "http://www.ebi.ac.uk/eva/v1/submission/initiate" -class StudySubmitter: - def __init__(self, submission_initiate_url=SUBMISSION_INITIATE_URL): +class StudySubmitter(AppLogger): + def __init__(self, vcf_files, metadata_file, submission_initiate_url=SUBMISSION_INITIATE_URL): self.auth = get_auth() self.submission_initiate_url = submission_initiate_url + self.vcf_files = vcf_files + self.metadata_file = metadata_file def create_submission_config_file(self, submission_dir, submission_id, submission_upload_url): submission_config_file = os.path.join(submission_dir, SUB_CLI_CONFIG_FILE) @@ -38,11 +42,20 @@ def get_submission_id_and_upload_url(self, submission_dir): else: raise FileNotFoundError(f'Could not upload. No config file found for the submission in {submission_dir}.') - # TODO - def upload_submission(self, submission_dir, submission_id=None, submission_upload_url=None): - if not submission_id or not submission_upload_url: + def upload_submission(self, submission_dir, submission_upload_url=None): + if not submission_upload_url: submission_id, submission_upload_url = self.get_submission_id_and_upload_url(submission_dir) - pass + for f in self.vcf_files: + self.upload_file(submission_upload_url, f) + self.upload_file(submission_upload_url, self.metadata_file) + + @retry(tries=5, delay=10, backoff=5) + def upload_file(self, submission_upload_url, input_file): + base_name = os.path.basename(input_file) + self.info(f'Transfer {base_name} to EVA FTP') + r = requests.put(urljoin(submission_upload_url, base_name), data=open(input_file, 'rb')) + r.raise_for_status() + self.info(f'Upload of {base_name} completed') def verify_submission_dir(self, submission_dir): if not os.path.exists(submission_dir): @@ -50,7 +63,6 @@ def verify_submission_dir(self, submission_dir): if not os.access(submission_dir, os.W_OK): raise Exception(f"The directory '{submission_dir}' does not have write permissions.") - def submit(self, submission_dir): self.verify_submission_dir(submission_dir) response = requests.post(self.submission_initiate_url, @@ -58,6 +70,6 @@ def submit(self, submission_dir): 'Authorization': 'Bearer ' + self.auth.token}) response.raise_for_status() response_json = response.json() - logger.info("Submission ID {} received!!".format(response_json["submissionId"])) + self.info("Submission ID {} received!!".format(response_json["submissionId"])) self.create_submission_config_file(submission_dir, response_json["submissionId"], response_json["uploadUrl"]) - self.upload_submission(submission_dir, response_json["submissionId"], response_json["uploadUrl"]) + self.upload_submission(submission_dir, response_json["uploadUrl"]) diff --git a/tests/test_submit.py b/tests/test_submit.py index d3bdac1..97336a5 100644 --- a/tests/test_submit.py +++ b/tests/test_submit.py @@ -2,7 +2,7 @@ import os import shutil import unittest -from unittest.mock import MagicMock, patch, Mock, PropertyMock +from unittest.mock import MagicMock, patch, Mock import yaml @@ -13,12 +13,15 @@ class TestSubmit(unittest.TestCase): + resource_dir = os.path.join(os.path.dirname(__file__), 'resources') def setUp(self) -> None: self.token = 'a token' with patch('cli.submit.get_auth', return_value=Mock(token=self.token)): - self.submitter = StudySubmitter() - self.test_sub_dir = os.path.join(os.path.dirname(__file__), 'resources', 'test_sub_dir') + vcf_files = [os.path.join(self.resource_dir, 'vcf_files', 'example2.vcf.gz')] + metadata_file = os.path.join(self.resource_dir, 'EVA_Submission_template.V1.1.4.xlsx') + self.submitter = StudySubmitter(vcf_files=vcf_files, metadata_file=metadata_file) + self.test_sub_dir = os.path.join(os.path.dirname(__file__), 'resources', 'test_sub_dir') shutil.rmtree(self.test_sub_dir, ignore_errors=True) def tearDown(self) -> None: @@ -33,13 +36,31 @@ def test_submit(self): 'uploadUrl': 'directory to use for upload'}) # Set the side_effect attribute to return different responses - with patch('cli.submit.requests.post', return_value=mock_submit_response) as mock_post: - with patch('cli.submit.StudySubmitter.create_submission_config_file'): + with patch('cli.submit.requests.post', return_value=mock_submit_response) as mock_post,\ + patch.object(StudySubmitter, 'create_submission_config_file'), \ + patch.object(StudySubmitter, 'upload_submission'): self.submitter.submit('test_submission_directory') mock_post.assert_called_once_with('http://www.ebi.ac.uk/eva/v1/submission/initiate', headers={'Accept': 'application/hal+json', 'Authorization': 'Bearer a token'}) - # TODO: Check that upload_submission was called with submission id + def test_submit_with_config(self): + mock_submit_response = MagicMock() + mock_submit_response.status_code = 200 + mock_submit_response.json.return_value = { + "submissionId": "mock_submission_id", + "uploadUrl": "directory to use for upload", + } + with patch('cli.submit.requests.post', return_value=mock_submit_response) as mock_post,\ + patch.object(StudySubmitter, 'upload_submission'): + self.submitter.submit(self.test_sub_dir) + + assert os.path.exists(self.test_sub_dir) + sub_config_file = os.path.join(self.test_sub_dir, SUB_CLI_CONFIG_FILE) + assert os.path.exists(sub_config_file) + with (open(sub_config_file, 'r') as f): + sub_config_data = yaml.safe_load(f) + assert sub_config_data[SUB_CLI_CONFIG_KEY_SUBMISSION_ID] == "mock_submission_id" + assert sub_config_data[SUB_CLI_CONFIG_KEY_SUBMISSION_UPLOAD_URL] == "directory to use for upload" def test_verify_submission_dir(self): self.submitter.verify_submission_dir(self.test_sub_dir) @@ -66,25 +87,23 @@ def test_get_submission_id_and_upload_url(self): assert submission_id == 1234 assert upload_url == "/sub/upload/url" - - - def test_submit(self): + def test_upload_submission(self): mock_submit_response = MagicMock() mock_submit_response.status_code = 200 - mock_submit_response.json.return_value = { - "submissionId": "mock_submission_id", - "uploadUrl": "directory to use for upload", - } - with patch('cli.submit.requests.post', return_value=mock_submit_response) as mock_post: - self.submitter.submit(self.test_sub_dir) - - assert os.path.exists(self.test_sub_dir) - sub_config_file = os.path.join(self.test_sub_dir, SUB_CLI_CONFIG_FILE) - assert os.path.exists(sub_config_file) - with (open(sub_config_file, 'r') as f): - sub_config_data = yaml.safe_load(f) - assert sub_config_data[SUB_CLI_CONFIG_KEY_SUBMISSION_ID] == "mock_submission_id" - assert sub_config_data[SUB_CLI_CONFIG_KEY_SUBMISSION_UPLOAD_URL] == "directory to use for upload" - + test_url = 'http://example.com/' + with patch.object(StudySubmitter, 'upload_file') as mock_upload_file: + self.submitter.upload_submission(submission_upload_url=test_url, submission_dir=self.test_sub_dir) + for vcf_file in self.submitter.vcf_files: + mock_upload_file.assert_any_call(test_url, vcf_file) + mock_upload_file.assert_called_with(test_url, self.submitter.metadata_file) + + def test_upload_file(self): + resource_dir = os.path.join(os.path.dirname(__file__), 'resources') + test_url = 'http://example.com/' + with patch('cli.submit.requests.put') as mock_put: + file_to_upload = os.path.join(resource_dir, 'EVA_Submission_template.V1.1.4.xlsx') + self.submitter.upload_file(submission_upload_url=test_url, input_file=file_to_upload) + assert mock_put.mock_calls[0][1][0] == test_url + os.path.basename(file_to_upload) + # Cannot test the content of the upload as opening the same file twice give different object