From 6b3af3c2a95602e23153eb95171d268bd7c3189b Mon Sep 17 00:00:00 2001 From: Vishal Date: Mon, 18 Dec 2023 14:14:35 +0530 Subject: [PATCH] [TDL-24569] New Stream inclusion: subtasks (#56) * added full table sync for subtasks * updated base test * removed pylint warning * more pylint fixes * updated comments * full table to incremental for subtasks * removed unused import * fixed pylint issue * updated subtask schema * updated start date test * updated bookmark test * updated base tests * updated bookmark test * updated start date test * updated readme and changelog * removed logger set to debug * removed unsed import --------- Co-authored-by: Nitin Gaikwad --- CHANGELOG.md | 3 + README.md | 1 + setup.py | 4 +- tap_asana/schemas/subtasks.json | 916 ++++++++++++++++++++++++++++++++ tap_asana/streams/__init__.py | 1 + tap_asana/streams/subtasks.py | 87 +++ tests/base.py | 8 +- tests/test_bookmarks.py | 12 +- tests/test_start_date.py | 14 +- 9 files changed, 1040 insertions(+), 6 deletions(-) create mode 100644 tap_asana/schemas/subtasks.json create mode 100644 tap_asana/streams/subtasks.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fb9e854..00e249c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 2.3.0 + * New Stream Inclusion: Subtasks [#56](https://github.com/singer-io/tap-asana/pull/56) + ## 2.2.0 * Below are the changes [#48](https://github.com/singer-io/tap-asana/pull/48) * Upgraded the asana-python SDK diff --git a/README.md b/README.md index 567ead9..0b3f847 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ This tap: - [Projects](https://developers.asana.com/docs/projects) - [Sections](https://developers.asana.com/docs/sections) - [Stories](https://developers.asana.com/docs/stories) + - [Subtasks](https://developers.asana.com/reference/getsubtasksfortask) - [Tags](https://developers.asana.com/docs/tags) - [Tasks](https://developers.asana.com/docs/tasks) - [Teams](https://developers.asana.com/docs/teams) diff --git a/setup.py b/setup.py index 90a246b..e474f20 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-asana", - version="2.2.0", + version="2.3.0", description="Singer.io tap for extracting Asana data", author="Stitch", url="http://github.com/singer-io/tap-asana", @@ -28,7 +28,7 @@ tap-asana=tap_asana:main """, packages=["tap_asana"], - package_data = { + package_data={ "schemas": ["tap_asana/schemas/*.json"] }, include_package_data=True, diff --git a/tap_asana/schemas/subtasks.json b/tap_asana/schemas/subtasks.json new file mode 100644 index 0000000..29436b2 --- /dev/null +++ b/tap_asana/schemas/subtasks.json @@ -0,0 +1,916 @@ +{ + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "gid": { + "type": [ + "null", + "string" + ] + }, + "resource_type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "assignee_status": { + "type": [ + "null", + "string" + ] + }, + "completed": { + "type": [ + "null", + "boolean" + ] + }, + "completed_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "completed_by": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "gid": { + "type": [ + "null", + "string" + ] + }, + "resource_type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + } + } + }, + "created_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "dependencies": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "gid": { + "type": [ + "null", + "string" + ] + }, + "resource_type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "dependents": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "gid": { + "type": [ + "null", + "string" + ] + }, + "resource_type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "due_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "due_on": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "hearted": { + "type": [ + "null", + "boolean" + ] + }, + "hearts": { + "anyOf": [ + { + "type": "array", + "items": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "gid": { + "type": ["null", "string"] + }, + "user": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "gid": { + "type": [ + "null", + "string" + ] + }, + "resource_type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + }, + { + "type": "null" + } + ] + }, + "is_rendered_as_separator": { + "type": [ + "null", + "boolean" + ] + }, + "html_notes": { + "type": [ + "null", + "string" + ] + }, + "liked": { + "type": [ + "null", + "boolean" + ] + }, + "likes": { + "anyOf": [ + { + "type": "array", + "items": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "gid": { + "type": ["null", "string"] + }, + "user": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "gid": { + "type": [ + "null", + "string" + ] + }, + "resource_type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + }, + { + "type": "null" + } + ] + }, + "memberships": { + "anyOf": [ + { + "type": "array", + "items": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "project": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "gid": { + "type": [ + "null", + "string" + ] + }, + "resource_type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + } + } + }, + "section": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "gid": { + "type": [ + "null", + "string" + ] + }, + "resource_type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + }, + { + "type": "null" + } + ] + }, + "modified_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "notes": { + "type": [ + "null", + "string" + ] + }, + "num_hearts": { + "type": [ + "null", + "integer" + ] + }, + "num_likes": { + "type": [ + "null", + "integer" + ] + }, + "num_subtasks": { + "type": [ + "null", + "integer" + ] + }, + "resource_subtype": { + "type": [ + "null", + "string" + ] + }, + "start_on": { + "type": [ + "null", + "string" + ] + }, + "assignee": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "gid": { + "type": [ + "null", + "string" + ] + }, + "resource_type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + } + } + }, + "custom_fields": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "gid": { + "type": [ + "null", + "string" + ] + }, + "resource_type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "resource_subtype": { + "type": [ + "null", + "string" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + }, + "enum_options": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "gid": { + "type": [ + "null", + "string" + ] + }, + "resource_type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "enabled": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "enum_value": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "gid": { + "type": [ + "null", + "string" + ] + }, + "resource_type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "enabled": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + } + } + }, + "enabled": { + "type": [ + "null", + "boolean" + ] + }, + "text_value": { + "type": [ + "null", + "string" + ] + }, + "number_value": { + "type": [ + "null", + "number" + ] + }, + "description": { + "type": [ + "null", + "string" + ] + }, + "precision": { + "type": [ + "null", + "integer" + ] + }, + "is_global_to_workspace": { + "type": [ + "null", + "boolean" + ] + }, + "has_notifications_enabled": { + "type": [ + "null", + "boolean" + ] + }, + "created_by": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "gid": { + "type": [ + "null", + "string" + ] + }, + "resource_type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + } + } + }, + "currency_code": { + "type": [ + "null", + "string" + ] + }, + "custom_label": { + "type": [ + "null", + "string" + ] + }, + "custom_label_position": { + "type": [ + "null", + "string" + ] + }, + "date_value": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "date": { + "type": [ + "null", + "string" + ] + }, + "date_time": { + "type": [ + "null", + "string" + ], + "format": "date-time" + } + } + }, + "display_value": { + "type": [ + "null", + "string" + ] + }, + "format": { + "type": [ + "null", + "string" + ] + }, + "multi_enum_values": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "gid": { + "type": [ + "null", + "string" + ] + }, + "resource_type": { + "type": [ + "null", + "string" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "enabled": { + "type": [ + "null", + "boolean" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "people_value": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "gid": { + "type": [ + "null", + "string" + ] + }, + "resource_type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + } + }, + "followers": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "gid": { + "type": [ + "null", + "string" + ] + }, + "resource_type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "parent": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "gid": { + "type": [ + "null", + "string" + ] + }, + "resource_type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "resource_subtype": { + "type": [ + "null", + "string" + ] + } + } + }, + "permalink_url": { + "type": [ + "null", + "string" + ] + }, + "projects": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "gid": { + "type": [ + "null", + "string" + ] + }, + "resource_type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "tags": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "gid": { + "type": [ + "null", + "string" + ] + }, + "resource_type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "workspace": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "gid": { + "type": [ + "null", + "string" + ] + }, + "resource_type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + } + } + }, + "start_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + } + } +} diff --git a/tap_asana/streams/__init__.py b/tap_asana/streams/__init__.py index f321cd1..ffc839f 100644 --- a/tap_asana/streams/__init__.py +++ b/tap_asana/streams/__init__.py @@ -2,6 +2,7 @@ import tap_asana.streams.projects import tap_asana.streams.sections import tap_asana.streams.stories +import tap_asana.streams.subtasks import tap_asana.streams.tags import tap_asana.streams.tasks import tap_asana.streams.teams diff --git a/tap_asana/streams/subtasks.py b/tap_asana/streams/subtasks.py new file mode 100644 index 0000000..8268155 --- /dev/null +++ b/tap_asana/streams/subtasks.py @@ -0,0 +1,87 @@ +# pylint:disable=duplicate-code +import singer +from tap_asana.context import Context +from tap_asana.streams.base import Stream + +LOGGER = singer.get_logger() + + + +class SubTasks(Stream): + name = "subtasks" + replication_key = "modified_at" + replication_method = "INCREMENTAL" + fields = [ + "gid", + "resource_type", + "name", + "approval_status", + "assignee_status", + "completed", + "completed_at", + "completed_by", + "created_at", + "dependencies", + "dependents", + "due_at", + "due_on", + "external", + "hearted", + "hearts", + "html_notes", + "is_rendered_as_separator", + "liked", + "likes", + "memberships", + "modified_at", + "notes", + "num_hearts", + "num_likes", + "num_subtasks", + "resource_subtype", + "start_on", + "assignee", + "custom_fields", + "followers", + "parent", + "permalink_url", + "projects", + "tags", + "workspace", + "start_at", + "assignee_section" + ] + + def get_objects(self): + """Get stream object""" + # list of project ids + project_ids = [] + opt_fields = ",".join(self.fields) + bookmark = self.get_bookmark() + session_bookmark = bookmark + for workspace in self.call_api("workspaces"): + for project in self.call_api("projects", workspace=workspace["gid"]): + project_ids.append(project["gid"]) + + for indx, project_id in enumerate(project_ids, 1): + LOGGER.info("Fetching Subtasks for project: %s/%s", indx, len(project_ids)) + tasks_list = self.call_api("tasks", project=project_id, opt_fields=opt_fields) + for task in tasks_list: + for subt in self.fetch_children(task, opt_fields): + session_bookmark = self.get_updated_session_bookmark( + session_bookmark, subt[self.replication_key] + ) + if self.is_bookmark_old(subt[self.replication_key]): + yield subt + self.update_bookmark(session_bookmark) + + def fetch_children(self, p_task, opt_fields): + subtasks_children = [] + resource = getattr(Context.asana.client, "tasks") + subtasks = list(resource.get_subtasks_for_task(p_task.get("gid"), opt_fields=opt_fields)) + for s_task in subtasks: + subtasks_children.extend(self.fetch_children(s_task, opt_fields)) + return subtasks + subtasks_children + + +Context.stream_objects["subtasks"] = SubTasks diff --git a/tests/base.py b/tests/base.py index 3baf486..5a33c05 100644 --- a/tests/base.py +++ b/tests/base.py @@ -21,7 +21,7 @@ class AsanaBase(unittest.TestCase): BOOKMARK_FOMAT = "%Y-%m-%dT%H:%M:%S.%f" first_start_date = '2019-01-01T00:00:00Z' - second_start_date = '2020-08-15T00:00:00Z' + second_start_date = '2023-08-15T00:00:00Z' def tap_name(self): return "tap-asana" @@ -105,6 +105,12 @@ def expected_metadata(self): self.REPLICATION_KEYS: {"modified_at"}, self.OBEYS_START_DATE: True }, + "subtasks": { + self.PRIMARY_KEYS: {"gid"}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {"modified_at"}, + self.OBEYS_START_DATE: True + }, "teams": { self.PRIMARY_KEYS: {"gid"}, self.REPLICATION_METHOD: self.FULL_TABLE, diff --git a/tests/test_bookmarks.py b/tests/test_bookmarks.py index 22f8bba..2910cda 100644 --- a/tests/test_bookmarks.py +++ b/tests/test_bookmarks.py @@ -12,6 +12,13 @@ def name(self): return "tap_tester_asana_bookmarks_test" def test_run(self): + # running sync with multiple date timestamps accross different streams due to differences in bookmark values + self.run_test("2021-11-09T00:00:00Z", "2023-11-10T00:00:00Z", {"projects",}) + self.run_test("2023-11-28T00:00:00Z", "2023-11-30T00:00:00Z", {"subtasks",}) + self.run_test("2019-01-28T00:00:00Z", "2023-11-30T00:00:00Z", self.expected_streams() - {"subtasks","projects"}) + + + def run_test(self, start_date_1, start_date_2, streams): """ Testing that the bookmarking for the tap works as expected - Verify for each incremental stream you can do a sync which records bookmarks @@ -24,7 +31,10 @@ def test_run(self): conn_id = connections.ensure_connection(self) runner.run_check_mode(self, conn_id) - expected_streams = self.expected_streams() + self.first_start_date = start_date_1 + self.second_start_date = start_date_2 + self.streams = streams + expected_streams = streams found_catalogs = self.run_and_verify_check_mode(conn_id) self.select_found_catalogs(conn_id, found_catalogs, diff --git a/tests/test_start_date.py b/tests/test_start_date.py index 050fefa..f4f428d 100644 --- a/tests/test_start_date.py +++ b/tests/test_start_date.py @@ -11,6 +11,12 @@ def name(self): return "tap_tester_asana_start_date_test" def test_run(self): + # running sync with multiple date timestamps accross different streams due to differences in bookmark values + self.run_test("2021-11-09T00:00:00Z", "2023-11-10T00:00:00Z", {"projects",}) + self.run_test("2023-11-28T00:00:00Z", "2023-11-30T00:00:00Z", {"subtasks",}) + self.run_test("2019-01-28T00:00:00Z", "2023-11-30T00:00:00Z", self.expected_streams() - {"subtasks","projects"}) + + def run_test(self, start_date_1, start_date_2, streams): """ Testing that the tap respects the start date - INCREMENTAL @@ -26,6 +32,12 @@ def test_run(self): records. """ + self.first_start_date = start_date_1 + self.second_start_date = start_date_2 + self.streams = streams + expected_streams = streams + + start_date_1_epoch = self.dt_to_ts(self.first_start_date, self.START_DATE_FORMAT) start_date_2_epoch = self.dt_to_ts(self.second_start_date, self.START_DATE_FORMAT) @@ -39,8 +51,6 @@ def test_run(self): # First Sync ########################################################################## - expected_streams = self.expected_streams() - conn_id_1 = connections.ensure_connection(self, original_properties=False) runner.run_check_mode(self, conn_id_1)