From 08bbe37f99a50f020860065537d8fb2cd22ce41c Mon Sep 17 00:00:00 2001 From: devketanpro Date: Fri, 10 Nov 2023 17:31:46 +0530 Subject: [PATCH 01/10] udpate renditions settings in NINJS formatter and parser --- superdesk/io/feed_parsers/ninjs.py | 8 +++++--- superdesk/publish/formatters/ninjs_formatter.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/superdesk/io/feed_parsers/ninjs.py b/superdesk/io/feed_parsers/ninjs.py index 58511cf25f..d5bc549ffa 100644 --- a/superdesk/io/feed_parsers/ninjs.py +++ b/superdesk/io/feed_parsers/ninjs.py @@ -120,9 +120,11 @@ def _transform_from_ninjs(self, ninjs): associated_item["versioncreated"] = self.datetime(associated_item["versioncreated"]) item["associations"][key] = deepcopy(associated_item) - if ninjs.get("renditions", {}).get("baseImage"): - item["renditions"] = {"baseImage": {"href": ninjs.get("renditions", {}).get("original", {}).get("href")}} - + renditions = ninjs.get("renditions", {}) + if renditions.get("baseImage") or renditions.get("original"): + href = ninjs.get("renditions", {}).get("original", {}).get("href") + item["renditions"] = {"baseImage": {"href": href}, + "original": {"href": href}} if ninjs.get("located"): item["dateline"] = {"located": {"city": ninjs.get("located")}} diff --git a/superdesk/publish/formatters/ninjs_formatter.py b/superdesk/publish/formatters/ninjs_formatter.py index 78c8a1449a..e6afc7e2d7 100644 --- a/superdesk/publish/formatters/ninjs_formatter.py +++ b/superdesk/publish/formatters/ninjs_formatter.py @@ -142,7 +142,7 @@ def __init__(self): self.format_type = "ninjs" self.can_preview = True self.can_export = True - self.internal_renditions = ["original"] + self.internal_renditions = ["original", "baseImage"] def format(self, article, subscriber, codes=None): try: From c6c1a3255cad7204fcc310d932049745e5d9d5e8 Mon Sep 17 00:00:00 2001 From: devketanpro Date: Fri, 10 Nov 2023 17:58:37 +0530 Subject: [PATCH 02/10] fix black --- superdesk/io/feed_parsers/ninjs.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/superdesk/io/feed_parsers/ninjs.py b/superdesk/io/feed_parsers/ninjs.py index d5bc549ffa..d98f750147 100644 --- a/superdesk/io/feed_parsers/ninjs.py +++ b/superdesk/io/feed_parsers/ninjs.py @@ -122,9 +122,8 @@ def _transform_from_ninjs(self, ninjs): renditions = ninjs.get("renditions", {}) if renditions.get("baseImage") or renditions.get("original"): - href = ninjs.get("renditions", {}).get("original", {}).get("href") - item["renditions"] = {"baseImage": {"href": href}, - "original": {"href": href}} + href = ninjs.get("renditions", {}).get("original", {}).get("href") + item["renditions"] = {"baseImage": {"href": href}, "original": {"href": href}} if ninjs.get("located"): item["dateline"] = {"located": {"city": ninjs.get("located")}} From 3eb029f9311be15119250db6f6eb2bce3ce8ea63 Mon Sep 17 00:00:00 2001 From: devketanpro Date: Thu, 16 Nov 2023 11:47:49 +0530 Subject: [PATCH 03/10] add new renditions config --- superdesk/default_settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/superdesk/default_settings.py b/superdesk/default_settings.py index d7b7fcd239..c6994b9cf3 100644 --- a/superdesk/default_settings.py +++ b/superdesk/default_settings.py @@ -581,6 +581,7 @@ def local_to_utc_hour(hour): }, } +NINJS_COMMON_RENDITIONS = list(RENDITIONS["picture"].keys()) #: BCRYPT work factor BCRYPT_GENSALT_WORK_FACTOR = 12 From d0640cdf1a7700833eaf31347809c31392ecf81d Mon Sep 17 00:00:00 2001 From: devketanpro Date: Thu, 16 Nov 2023 11:50:11 +0530 Subject: [PATCH 04/10] format and populate data based on new config --- superdesk/io/feed_parsers/ninjs.py | 7 +++---- superdesk/publish/formatters/ninjs_formatter.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/superdesk/io/feed_parsers/ninjs.py b/superdesk/io/feed_parsers/ninjs.py index d98f750147..544ef6af9a 100644 --- a/superdesk/io/feed_parsers/ninjs.py +++ b/superdesk/io/feed_parsers/ninjs.py @@ -120,10 +120,9 @@ def _transform_from_ninjs(self, ninjs): associated_item["versioncreated"] = self.datetime(associated_item["versioncreated"]) item["associations"][key] = deepcopy(associated_item) - renditions = ninjs.get("renditions", {}) - if renditions.get("baseImage") or renditions.get("original"): - href = ninjs.get("renditions", {}).get("original", {}).get("href") - item["renditions"] = {"baseImage": {"href": href}, "original": {"href": href}} + if ninjs.get("renditions"): + item["renditions"] = ninjs["renditions"] + if ninjs.get("located"): item["dateline"] = {"located": {"city": ninjs.get("located")}} diff --git a/superdesk/publish/formatters/ninjs_formatter.py b/superdesk/publish/formatters/ninjs_formatter.py index e6afc7e2d7..c66157016e 100644 --- a/superdesk/publish/formatters/ninjs_formatter.py +++ b/superdesk/publish/formatters/ninjs_formatter.py @@ -142,7 +142,7 @@ def __init__(self): self.format_type = "ninjs" self.can_preview = True self.can_export = True - self.internal_renditions = ["original", "baseImage"] + self.internal_renditions = app.config.get("NINJS_COMMON_RENDITIONS", []) + ['original'] def format(self, article, subscriber, codes=None): try: From 2c73492a2755b953e603dfe0ad0b99e38495880a Mon Sep 17 00:00:00 2001 From: devketanpro Date: Thu, 16 Nov 2023 11:50:37 +0530 Subject: [PATCH 05/10] reformat code via black --- superdesk/publish/formatters/ninjs_formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superdesk/publish/formatters/ninjs_formatter.py b/superdesk/publish/formatters/ninjs_formatter.py index c66157016e..010dcc904b 100644 --- a/superdesk/publish/formatters/ninjs_formatter.py +++ b/superdesk/publish/formatters/ninjs_formatter.py @@ -142,7 +142,7 @@ def __init__(self): self.format_type = "ninjs" self.can_preview = True self.can_export = True - self.internal_renditions = app.config.get("NINJS_COMMON_RENDITIONS", []) + ['original'] + self.internal_renditions = app.config.get("NINJS_COMMON_RENDITIONS", []) + ["original"] def format(self, article, subscriber, codes=None): try: From fbdf9478fac1c523682968cb9430f607aa19b370 Mon Sep 17 00:00:00 2001 From: devketanpro Date: Thu, 16 Nov 2023 16:26:55 +0530 Subject: [PATCH 06/10] refactore rendition parsing --- superdesk/io/feed_parsers/ninjs.py | 38 +++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/superdesk/io/feed_parsers/ninjs.py b/superdesk/io/feed_parsers/ninjs.py index 544ef6af9a..2278eac699 100644 --- a/superdesk/io/feed_parsers/ninjs.py +++ b/superdesk/io/feed_parsers/ninjs.py @@ -121,7 +121,7 @@ def _transform_from_ninjs(self, ninjs): item["associations"][key] = deepcopy(associated_item) if ninjs.get("renditions"): - item["renditions"] = ninjs["renditions"] + item["renditions"] = self.parse_renditions(ninjs["renditions"]) if ninjs.get("located"): item["dateline"] = {"located": {"city": ninjs.get("located")}} @@ -146,6 +146,42 @@ def _transform_from_ninjs(self, ninjs): return item + def parse_renditions(self, renditions): + rend = [] + for rendition_name, rendition_data in renditions.items(): + parsed_rendition = {} + + # Parse href + href = rendition_data.get("href", "") + if isinstance(href, str) and href: + parsed_rendition["href"] = href + + # Parse width and height + width = rendition_data.get("width") + height = rendition_data.get("height") + if isinstance(width, int) and isinstance(height, int): + parsed_rendition["width"] = width + parsed_rendition["height"] = height + + # Parse mimetype + mimetype = rendition_data.get("mimetype", "") + if isinstance(mimetype, str) and mimetype: + parsed_rendition["mimetype"] = mimetype + + # Parse poi + poi = rendition_data.get("poi", {}) + if isinstance(poi, dict) and "x" in poi and "y" in poi: + parsed_rendition["poi"] = {"x": poi["x"], "y": poi["y"]} + + # Parse media + media = rendition_data.get("media", "") + if isinstance(media, str) and media: + parsed_rendition["media"] = media + + if parsed_rendition: + rend.append({rendition_name: parsed_rendition}) + return rend + def _format_qcodes(self, items): subjects = [] for item in items: From e79edcfa31c29868da046f99680cb17081aa2272 Mon Sep 17 00:00:00 2001 From: devketanpro Date: Thu, 16 Nov 2023 17:07:29 +0530 Subject: [PATCH 07/10] fix pytest --- tests/publish/formatters/ninjs_formatter_test.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/publish/formatters/ninjs_formatter_test.py b/tests/publish/formatters/ninjs_formatter_test.py index b6ecea8c33..586a05272b 100644 --- a/tests/publish/formatters/ninjs_formatter_test.py +++ b/tests/publish/formatters/ninjs_formatter_test.py @@ -187,6 +187,12 @@ def test_picture_formatter(self): "href": "https://one-api.aap.com.au/api/v3/Assets/20150723001158606583/Original/download", "mimetype": "image/jpeg", }, + "viewImage": { + "height": 401, + "href": "http://localhost:5000/api/upload/55b032041d41c8d278d21b6f/raw?_schema=http", + "mimetype": "image/jpeg", + "width": 640, + }, }, "headline": "AMAZING PICTURE", "pubstatus": "usable", @@ -203,7 +209,7 @@ def test_picture_formatter(self): "embargoed": embargoed.isoformat(), } self.assertEqual(expected, json.loads(doc)) - self.assertNotIn("viewImage", json.loads(doc).get("renditions")) + self.assertIn("viewImage", json.loads(doc).get("renditions")) def test_composite_formatter(self): article = { From 8a50458ca3d8412898837f996c1f4e1cf78f5594 Mon Sep 17 00:00:00 2001 From: devketanpro Date: Thu, 16 Nov 2023 17:10:08 +0530 Subject: [PATCH 08/10] change type --- superdesk/io/feed_parsers/ninjs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/superdesk/io/feed_parsers/ninjs.py b/superdesk/io/feed_parsers/ninjs.py index 2278eac699..b69543a5e0 100644 --- a/superdesk/io/feed_parsers/ninjs.py +++ b/superdesk/io/feed_parsers/ninjs.py @@ -147,7 +147,7 @@ def _transform_from_ninjs(self, ninjs): return item def parse_renditions(self, renditions): - rend = [] + rend = {} for rendition_name, rendition_data in renditions.items(): parsed_rendition = {} @@ -179,7 +179,7 @@ def parse_renditions(self, renditions): parsed_rendition["media"] = media if parsed_rendition: - rend.append({rendition_name: parsed_rendition}) + rend.update({rendition_name: parsed_rendition}) return rend def _format_qcodes(self, items): From 3929271672a5f08afba2fbf72c16215ebdfbdc0b Mon Sep 17 00:00:00 2001 From: devketanpro Date: Thu, 16 Nov 2023 18:35:18 +0530 Subject: [PATCH 09/10] minor change and update tests --- features/ingest.feature | 144 +++++++++++++++++++---------- superdesk/io/feed_parsers/ninjs.py | 5 +- 2 files changed, 96 insertions(+), 53 deletions(-) diff --git a/features/ingest.feature b/features/ingest.feature index d0211f4761..2c3b3ec778 100644 --- a/features/ingest.feature +++ b/features/ingest.feature @@ -477,18 +477,29 @@ Feature: Fetch From Ingest "type":"picture", "state":"ingested", "renditions":{ - "baseImage":{ - "width":1400 - }, - "thumbnail":{ - - }, - "original":{ - + "original": { + "poi": { + "y": 1193, + "x": 2128 + }, + "href": "http://content.reuters.com/auth-server/content/tag_reuters.com_2014_newsml_LYNXMPEA6F0MT_2/tag_reuters.com_2014_binary_LYNXMPEA6F0MT-BASEIMAGE?auth_token=fake_token", + "mimetype": "image/jpeg", + "media": "599e31ec1d41c833e0b8a830", + "width": 3489, + "height": 2296 }, - "viewImage":{ + "baseImage": { + "poi": { + "y": 478, + "x": 854 + }, + "href": "http://content.reuters.com/auth-server/content/tag_reuters.com_2014_newsml_LYNXMPEA6F0MT_2/tag_reuters.com_2014_binary_LYNXMPEA6F0MT-BASEIMAGE?auth_token=fake_token", + "mimetype": "image/jpeg", + "media": "599e31ed1d41c833e0b8a840", + "width": 1400, + "height": 921 } - } + } }, { "type":"text", @@ -497,11 +508,28 @@ Feature: Fetch From Ingest "featuremedia":{ "state": "ingested", "renditions":{ - "baseImage":{ - "width":1400 - }, - "original":{ - } + "original": { + "poi": { + "y": 1193, + "x": 2128 + }, + "href": "http://content.reuters.com/auth-server/content/tag_reuters.com_2014_newsml_LYNXMPEA6F0MT_2/tag_reuters.com_2014_binary_LYNXMPEA6F0MT-BASEIMAGE?auth_token=fake_token", + "mimetype": "image/jpeg", + "media": "599e31ec1d41c833e0b8a830", + "width": 3489, + "height": 2296 + }, + "baseImage": { + "poi": { + "y": 478, + "x": 854 + }, + "href": "http://content.reuters.com/auth-server/content/tag_reuters.com_2014_newsml_LYNXMPEA6F0MT_2/tag_reuters.com_2014_binary_LYNXMPEA6F0MT-BASEIMAGE?auth_token=fake_token", + "mimetype": "image/jpeg", + "media": "599e31ed1d41c833e0b8a840", + "width": 1400, + "height": 921 + } } } } @@ -524,19 +552,22 @@ Feature: Fetch From Ingest "type":"picture", "state":"ingested", "headline": "Polizeihunde sollen effizienter trainiert werden – Klone sollen die Lösung sein.", - "renditions":{ - "baseImage":{ - "width":1400 + "renditions": { + "original": { + "media": "Polizeihund-China-2.jpg", + "href": "https://img.futurezone.de/img/science/origs216742511/239706555-w1280-h960-q85/Polizeihund-China-2.jpg", + "mimetype": "image\/jpeg", + "height": 853, + "width": 1280 }, - "thumbnail":{ - - }, - "original":{ - - }, - "viewImage":{ + "baseImage": { + "media": "Polizeihund-China-2.jpg", + "href": "https://img.futurezone.de/img/science/origs216742511/239706555-w1280-h960-q85/Polizeihund-China-2.jpg", + "mimetype": "image\/jpeg", + "height": 853, + "width": 1280 } - } + } }, { "type":"picture", @@ -549,13 +580,22 @@ Feature: Fetch From Ingest "associations":{ "featuremedia":{ "state": "ingested", - "renditions":{ - "baseImage":{ - "width":1400 - }, - "original":{ - } - } + "renditions": { + "original": { + "media": "Polizeihund-China-2.jpg", + "href": "https://img.futurezone.de/img/science/origs216742511/239706555-w1280-h960-q85/Polizeihund-China-2.jpg", + "mimetype": "image\/jpeg", + "height": 853, + "width": 1280 + }, + "baseImage": { + "media": "Polizeihund-China-2.jpg", + "href": "https://img.futurezone.de/img/science/origs216742511/239706555-w1280-h960-q85/Polizeihund-China-2.jpg", + "mimetype": "image\/jpeg", + "height": 853, + "width": 1280 + } + } } } } @@ -591,25 +631,27 @@ Feature: Fetch From Ingest "headline":"German Air Force Museum", "state":"ingested", "renditions":{ - "baseImage" : { - "width" : 1400, - "height" : 1074, - "mimetype" : "image/jpeg" - }, - "thumbnail" : { - "width":156, - "height":120, - "mimetype" : "image/jpeg" - }, - "original" : { - "width" : 800, - "height" : 614, - "mimetype" : "image/jpeg" + "baseImage": { + "poi": { + "y": 382, + "x": 784 + }, + "media": "599faa351d41c8fdc696d904", + "width": 1400, + "height": 934, + "mimetype": "image/jpeg", + "href": "http://localhost:5000/api/upload-raw/599faa351d41c8fdc696d904?_schema=http" }, - "viewImage":{ - "width" : 640, - "height" : 491, - "mimetype" : "image/jpeg" + "original": { + "poi": { + "y": 1646, + "x": 3368 + }, + "media": "599faa331d41c8fdc696d8b4", + "width": 6016, + "height": 4016, + "mimetype": "image/jpeg", + "href": "http://content.reuters.com/auth-server/content/tag_reuters.com_2014_newsml_LYNXMPEA6F0MT_2/tag_reuters.com_2014_binary_LYNXMPEA6F0MT-BASEIMAGE?auth_token=fake_token" } } } diff --git a/superdesk/io/feed_parsers/ninjs.py b/superdesk/io/feed_parsers/ninjs.py index b69543a5e0..a9fb4bb3e9 100644 --- a/superdesk/io/feed_parsers/ninjs.py +++ b/superdesk/io/feed_parsers/ninjs.py @@ -159,8 +159,9 @@ def parse_renditions(self, renditions): # Parse width and height width = rendition_data.get("width") height = rendition_data.get("height") - if isinstance(width, int) and isinstance(height, int): + if isinstance(width, int): parsed_rendition["width"] = width + if isinstance(height, int): parsed_rendition["height"] = height # Parse mimetype @@ -179,7 +180,7 @@ def parse_renditions(self, renditions): parsed_rendition["media"] = media if parsed_rendition: - rend.update({rendition_name: parsed_rendition}) + rend[rendition_name] = parsed_rendition return rend def _format_qcodes(self, items): From fa2d24855e2d479e72c8ba6b4f681e757b94aaf1 Mon Sep 17 00:00:00 2001 From: devketanpro Date: Thu, 16 Nov 2023 19:13:32 +0530 Subject: [PATCH 10/10] fix tests and addressed comment --- features/ingest.feature | 2 +- superdesk/io/feed_parsers/ninjs.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/features/ingest.feature b/features/ingest.feature index 2c3b3ec778..c25ce8fa5d 100644 --- a/features/ingest.feature +++ b/features/ingest.feature @@ -542,7 +542,7 @@ Feature: Fetch From Ingest @provider Scenario: Ingest ninjs with embedded items Given empty "ingest" - When we fetch from "ninjs" ingest "ninjs4.json" (mocking with "ninjs4_mock.json") + When we fetch from "ninjs" ingest "ninjs4.json" And we get "/ingest" Then we get existing resource """ diff --git a/superdesk/io/feed_parsers/ninjs.py b/superdesk/io/feed_parsers/ninjs.py index a9fb4bb3e9..80a8295375 100644 --- a/superdesk/io/feed_parsers/ninjs.py +++ b/superdesk/io/feed_parsers/ninjs.py @@ -157,11 +157,10 @@ def parse_renditions(self, renditions): parsed_rendition["href"] = href # Parse width and height - width = rendition_data.get("width") - height = rendition_data.get("height") - if isinstance(width, int): + width = rendition_data.get("width", None) + height = rendition_data.get("height", None) + if isinstance(width, int) and isinstance(height, int): parsed_rendition["width"] = width - if isinstance(height, int): parsed_rendition["height"] = height # Parse mimetype