From a14e151f5dc748830ab92dd71aeaf209f3152669 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Thu, 20 Feb 2020 13:07:17 -0800 Subject: [PATCH 1/6] use `context.temp_queue` to download artifacts This will break if `context.temp_queue` is None. --- src/scriptworker/artifacts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scriptworker/artifacts.py b/src/scriptworker/artifacts.py index 471202f4..a63838ab 100644 --- a/src/scriptworker/artifacts.py +++ b/src/scriptworker/artifacts.py @@ -206,9 +206,9 @@ def get_artifact_url(context, task_id, path): """ if path.startswith("public/"): - url = context.queue.buildUrl("getLatestArtifact", task_id, path) + url = context.temp_queue.buildUrl("getLatestArtifact", task_id, path) else: - url = context.queue.buildSignedUrl( + url = context.temp_queue.buildSignedUrl( "getLatestArtifact", task_id, path, From b9e86d8df9336668f207a2bc5aae71a954da0c1d Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Thu, 20 Feb 2020 13:07:53 -0800 Subject: [PATCH 2/6] `context.temp_queue` should return `context.queue` if not set --- src/scriptworker/context.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/scriptworker/context.py b/src/scriptworker/context.py index 9c2dda7c..67d66255 100644 --- a/src/scriptworker/context.py +++ b/src/scriptworker/context.py @@ -53,7 +53,7 @@ class Context(object): queue = None session = None task = None - temp_queue = None + _temp_queue = None running_tasks = None _credentials = None _claim_task = None # This assumes a single task per worker. @@ -169,7 +169,10 @@ def temp_credentials(self): @temp_credentials.setter def temp_credentials(self, credentials): self._temp_credentials = credentials - self.temp_queue = self.create_queue(self.temp_credentials) + if credentials is not None: + self._temp_queue = self.create_queue(self.temp_credentials) + else: + self._temp_queue = None def write_json(self, path, contents, message): """Write json to disk. @@ -215,6 +218,24 @@ def event_loop(self): def event_loop(self, event_loop): self._event_loop = event_loop + @property + def temp_queue(self): + """dict: The queue for the current task. + + These will have different sets of scopes than the worker queue. + + """ + if self._temp_queue: + return self._temp_queue + else: + return self.queue + + @credentials.setter + def credentials(self, creds): + self._credentials = creds + self.queue = self.create_queue(self.credentials) + self.credentials_timestamp = arrow.utcnow().timestamp + async def populate_projects(self, force=False): """Download the ``projects.yml`` file and populate ``self.projects``. From 18b3afe380a4b3016da5d59caec990123b951072 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Thu, 20 Feb 2020 13:41:56 -0800 Subject: [PATCH 3/6] fix tests --- src/scriptworker/context.py | 8 ++++++-- tests/test_artifacts.py | 8 ++++---- tests/test_integration.py | 6 +++--- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/scriptworker/context.py b/src/scriptworker/context.py index 67d66255..7a5f8880 100644 --- a/src/scriptworker/context.py +++ b/src/scriptworker/context.py @@ -170,9 +170,9 @@ def temp_credentials(self): def temp_credentials(self, credentials): self._temp_credentials = credentials if credentials is not None: - self._temp_queue = self.create_queue(self.temp_credentials) + self.temp_queue = self.create_queue(self.temp_credentials) else: - self._temp_queue = None + self.temp_queue = None def write_json(self, path, contents, message): """Write json to disk. @@ -230,6 +230,10 @@ def temp_queue(self): else: return self.queue + @temp_queue.setter + def temp_queue(self, queue): + self._temp_queue = queue + @credentials.setter def credentials(self, creds): self._credentials = creds diff --git a/tests/test_artifacts.py b/tests/test_artifacts.py index 6a99f891..0ff5da59 100644 --- a/tests/test_artifacts.py +++ b/tests/test_artifacts.py @@ -182,10 +182,10 @@ def buildSignedUrl(*args, **kwargs): return expected context = mock.MagicMock() - context.queue = mock.MagicMock() - context.queue.options = {"baseUrl": "https://netloc/"} - context.queue.buildUrl = buildUrl - context.queue.buildSignedUrl = buildSignedUrl + context.temp_queue = mock.MagicMock() + context.temp_queue.options = {"baseUrl": "https://netloc/"} + context.temp_queue.buildUrl = buildUrl + context.temp_queue.buildSignedUrl = buildSignedUrl assert get_artifact_url(context, "x", path) == expected diff --git a/tests/test_integration.py b/tests/test_integration.py index 0e53fe98..743e2fd9 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -143,8 +143,8 @@ async def get_temp_creds_context(config_override=None): yield context -async def create_task(context, task_id, task_group_id): - payload = integration_create_task_payload(context.config, task_group_id) +async def create_task(context, task_id, task_group_id, **kwargs): + payload = integration_create_task_payload(context.config, task_group_id, **kwargs) return await context.queue.createTask(task_id, payload) @@ -327,7 +327,7 @@ async def test_private_artifacts(context_function): task_group_id = task_id = slugid.nice() override = {"task_script": ("bash", "-c", ">&2 echo")} async with context_function(override) as context: - result = await create_task(context, task_id, task_group_id) + result = await create_task(context, task_id, task_group_id, scopes=["queue:get-artifact:SampleArtifacts/_/X.txt"]) assert result["status"]["state"] == "pending" path = os.path.join(context.config["artifact_dir"], "SampleArtifacts/_/X.txt") utils.makedirs(os.path.dirname(path)) From a88383100a4d0179e61a590377bf7b7712b50ca4 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Fri, 21 Feb 2020 11:43:16 -0800 Subject: [PATCH 4/6] fix errant paste --- src/scriptworker/context.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/scriptworker/context.py b/src/scriptworker/context.py index 7a5f8880..445179e3 100644 --- a/src/scriptworker/context.py +++ b/src/scriptworker/context.py @@ -234,12 +234,6 @@ def temp_queue(self): def temp_queue(self, queue): self._temp_queue = queue - @credentials.setter - def credentials(self, creds): - self._credentials = creds - self.queue = self.create_queue(self.credentials) - self.credentials_timestamp = arrow.utcnow().timestamp - async def populate_projects(self, force=False): """Download the ``projects.yml`` file and populate ``self.projects``. From 78b9f7a75211ac262fc22f6c2e4e9560d21971b3 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Fri, 21 Feb 2020 12:01:08 -0800 Subject: [PATCH 5/6] coverage fixes --- tests/test_context.py | 6 ++++++ tox.ini | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_context.py b/tests/test_context.py index 4cdc33a4..f452b684 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -84,6 +84,12 @@ def test_temp_queue(rw_context, mocker): assert taskcluster.aio.Queue.called_once_with( options={"rootUrl": rw_context.config["taskcluster_root_url"], "credentials": rw_context.temp_credentials}, session=rw_context.session ) + assert rw_context._temp_queue is not None + rw_context.temp_queue = None + fake_queue = mocker.MagicMock() + rw_context.queue = fake_queue + assert rw_context._temp_queue is None + assert rw_context.temp_queue is fake_queue @pytest.mark.asyncio diff --git a/tox.ini b/tox.ini index ade7e9ad..7f98f209 100644 --- a/tox.ini +++ b/tox.ini @@ -45,7 +45,7 @@ depends = [testenv:report] skip_install = true commands = coverage report -m -depends = py37 +depends = py38 parallel_show_output = true [testenv:coveralls] From 38e142660e892542028331bf95c6119316984cd8 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Mon, 24 Feb 2020 08:46:37 -0800 Subject: [PATCH 6/6] 32.0.0 --- HISTORY.rst | 7 +++++++ src/scriptworker/version.py | 2 +- version.json | 6 +++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index baed370c..2f47d317 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,6 +4,13 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to `Semantic Versioning `__. +[32.0.0] - 2020-02-24 +--------------------- + +Changed +~~~~~~~ +- Scriptworker now uses ``context.temp_queue`` to download artifacts. + [31.1.0] - 2020-02-21 --------------------- diff --git a/src/scriptworker/version.py b/src/scriptworker/version.py index 9f90e101..7b51ba4a 100755 --- a/src/scriptworker/version.py +++ b/src/scriptworker/version.py @@ -54,7 +54,7 @@ def get_version_string(version: Union[ShortVerType, LongVerType]) -> str: # 1}}} # Semantic versioning 2.0.0 http://semver.org/ -__version__ = (31, 1, 0) +__version__ = (32, 0, 0) __version_string__ = get_version_string(__version__) diff --git a/version.json b/version.json index a28aba1c..c116947a 100644 --- a/version.json +++ b/version.json @@ -1,8 +1,8 @@ { "version":[ - 31, - 1, + 32, + 0, 0 ], - "version_string":"31.1.0" + "version_string":"32.0.0" }