Merge remote-tracking branch 'upstream/develop' into develop

.github/workflows/nose-tests.yml

@@ -7,8 +7,9 @@ jobs:
     runs-on: ubuntu-latest
 
     strategy:
+      fail-fast: false
       matrix:
-        python-version: ['3.8', '3.10']
+        python-version: ['3.8', '3.10', '3.12']
 
     steps:
     - uses: actions/checkout@v3
@@ -27,7 +28,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ['3.8', '3.10']
+        python-version: ['3.8', '3.10', '3.12']
 
     steps:
     - uses: actions/checkout@v3
@@ -43,7 +44,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ['3.8', '3.10']
+        python-version: ['3.8', '3.10', '3.12']
 
     steps:
     - uses: actions/checkout@v3
@@ -64,7 +65,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ['3.8', '3.10']
+        python-version: ['3.8', '3.10', '3.12']
 
     steps:
     - uses: actions/checkout@v3
@@ -87,7 +88,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ['3.8', '3.10']
+        python-version: ['3.8', '3.10', '3.12']
 
     steps:
     - uses: actions/checkout@v3

apps/desks.py

@@ -27,7 +27,6 @@
 from superdesk.activity import add_activity, ACTIVITY_UPDATE
 from superdesk.metadata.item import FAMILY_ID, ITEM_STATE, CONTENT_STATE
 from eve.utils import ParsedRequest
-from superdesk.utils import ListCursor
 from flask_babel import _, lazy_gettext
 
 
@@ -103,6 +102,9 @@ def init_app(app) -> None:
     endpoint_name = "desk_overview"
     service = OverviewService(endpoint_name, backend=superdesk.get_backend())
     OverviewResource(endpoint_name, app=app, service=service)
+    endpoint_name = "desk_users"
+    service = DeskUsersService(endpoint_name, backend=superdesk.get_backend())
+    DeskUsersResource(endpoint_name, app=app, service=service)
 
 
 superdesk.privilege(
@@ -371,6 +373,42 @@ def get_by_user(self, user_id):
         return list(self.get(req=None, lookup={"user_id": user_id}))
 
 
+class DeskUsersResource(Resource):
+    url = 'desks/<regex("[a-f0-9]{24}"):desk_id>/users'
+    resource_title = "desk_users"
+    datasource = {
+        "source": "users",
+        "default_sort": [("username", 1)],
+        "projection": {
+            "username": 1,
+            "first_name": 1,
+            "last_name": 1,
+            "display_name": 1,
+            "email": 1,
+            "picture_url": 1,
+            "avatar": 1,
+            "avatar_renditions": 1,
+            "role": 1,
+            "last_activity_at": 1,
+            "sign_off": 1,
+        },
+    }
+    resource_methods = ["GET"]
+
+
+class DeskUsersService(BaseService):
+    def get(self, req, lookup):
+        desk_id = lookup.pop("desk_id", None)
+        desks_service = superdesk.get_resource_service("desks")
+        if desk_id:
+            desk = desks_service.find_one(req=None, _id=ObjectId(desk_id))
+            if desk and desk.get("members"):
+                lookup["_id"] = {"$in": [member["user"] for member in desk.get("members", [])]}
+            else:
+                lookup["_id"] = ""  # return empty result
+        return super().get(req, lookup)
+
+
 class SluglineDesksResource(Resource):
     url = 'desks/<regex("[a-f0-9]{24}"):desk_id>/sluglines'
     datasource = {

apps/export/resource.py

@@ -12,6 +12,7 @@ class ExportResource(Resource):
         "item_ids": {"type": "list", "required": True},
         "format_type": {"type": "string", "required": True},
         "validate": {"type": "boolean", "required": False},
+        "inline": {"type": "boolean", "required": False, "default": False},
         "failures": {
             "type": "integer",
             "readonly": True,
@@ -20,5 +21,9 @@ class ExportResource(Resource):
             "type": "string",
             "nullable": True,
         },
+        "export": {
+            "type": "dict",
+            "readonly": True,
+        },
     }
     privileges = {"POST": "archive"}

apps/export/service.py

@@ -2,13 +2,13 @@
 import re
 from superdesk.services import BaseService
 from superdesk import get_resource_service
-from superdesk.errors import SuperdeskApiError
+from superdesk.errors import FormatterError, SuperdeskApiError
 from superdesk.publish.formatters import get_all_formatters
 from superdesk.utils import get_random_string
 from superdesk.validation import ValidationError
 from io import BytesIO
 from zipfile import ZipFile
-from flask import current_app as app
+from flask import current_app as app, json
 from flask_babel import _
 
 logger = logging.getLogger(__name__)
@@ -22,50 +22,60 @@ def create(self, docs, **kwargs):
         validate = doc.get("validate", False)
         archive_service = get_resource_service("archive")
 
+        items = {}
         unsuccessful_exports = 0
-        try:
-            in_memory_zip = BytesIO()
-            with ZipFile(in_memory_zip, "a") as zip:
-                for item_id in doc.get("item_ids"):
-                    item = archive_service.find_one(req=None, _id=item_id)
-                    if item:
-                        try:
-                            if validate:
-                                self._validate_for_publish(item)
-
-                            contents = formatter.export(item)
-                            # Remove invalid filename chars (for windows OS) and create the filename
-                            filename = (
-                                re.sub(r'[\\/*?:"<>|]', "", item.get("slugline", ""))
-                                + "_"
-                                + str(item.get("unique_id"))
-                                + ".txt"
-                            )
-                            zip.writestr(filename, contents.encode("UTF-8"))
-                        except ValidationError:
-                            unsuccessful_exports += 1
-                    else:
+
+        for item_id in doc.get("item_ids"):
+            item = archive_service.find_one(req=None, _id=item_id)
+            if item:
+                if validate:
+                    try:
+                        self._validate_for_publish(item)
+                    except ValidationError:
                         unsuccessful_exports += 1
+                        continue
 
-            url = None
-            # Store the zip file on media_storage
-            # only if at least one item is formatted successfully
-            if unsuccessful_exports < len(doc.get("item_ids")):
-                zip_id = app.media.put(
-                    in_memory_zip.getvalue(),
-                    filename="export_{}.zip".format(get_random_string()),
-                    content_type="application/zip",
-                    folder="temp",
+                try:
+                    contents = formatter.export(item)
+                except FormatterError as e:
+                    logger.exception(e)
+                    unsuccessful_exports += 1
+                    continue
+
+                # Remove invalid filename chars (for windows OS) and create the filename
+                filename = (
+                    re.sub(r'[\\/*?:"<>|]', "", item.get("slugline", "")) + "_" + str(item.get("unique_id")) + ".txt"
                 )
-                url = app.media.url_for_download(zip_id, "application/zip")
 
-            doc["url"] = url
-            doc["failures"] = unsuccessful_exports
+                items[item_id] = filename, contents
+
+        doc["failures"] = unsuccessful_exports
+
+        if not items:
             return [len(docs)]
-        except Exception as ex:
-            raise SuperdeskApiError.badRequestError(
-                _("Error creating export zip file. Try again please."), exception=ex
+
+        if doc.get("inline"):
+            doc["export"] = {}
+            for item_id, (filename, contents) in items.items():
+                try:
+                    doc["export"][item_id] = json.loads(contents)
+                except ValueError:
+                    doc["export"][item_id] = contents
+        else:
+            in_memory_zip = BytesIO()
+            with ZipFile(in_memory_zip, "a") as zip:
+                for item_id, (filename, contents) in items.items():
+                    zip.writestr(filename, contents.encode("UTF-8"))
+
+            zip_id = app.media.put(
+                in_memory_zip.getvalue(),
+                filename="export_{}.zip".format(get_random_string()),
+                content_type="application/zip",
+                folder="temp",
             )
+            doc["url"] = app.media.url_for_download(zip_id, "application/zip")
+
+        return [len(docs)]
 
     def _validate_for_publish(self, doc):
         """Validates the given story for publish action"""

apps/macros/macro_register.py

@@ -11,7 +11,6 @@
 import inspect
 import importlib
 import sys
-import imp
 import os
 
 from flask import current_app
@@ -24,15 +23,15 @@ def load_macros(app=None):
     load_module(module)
 
 
-def load_module(module):
+def load_module(module: str):
     """Loads the given module
 
     If the module is loaded before it will reload it
 
     :param module: name of he module
     """
     try:
-        imp.reload(sys.modules[module])
+        importlib.reload(sys.modules[module])
     except (AttributeError, KeyError):
         try:
             importlib.import_module(module)

apps/publish/content/common.py

@@ -727,7 +727,14 @@ def _set_updates_for_media_items(self, doc, updates):
 
         for key in DEFAULT_SCHEMA.keys():
             if doc.get(key):
-                updates[key] = doc[key]
+                if (
+                    doc.get("_current_version")
+                    and updates.get("_current_version")
+                    and doc["_current_version"] <= updates["_current_version"]
+                ):  # media item was not changed outside, only populate missing data
+                    updates.setdefault(key, doc[key])
+                else:  # media item could be updated outside, so update all fields
+                    updates[key] = doc[key]
 
     def _refresh_associated_items(self, original, skip_related=False):
         """Refreshes associated items with the latest version. Any further updates made to basic metadata done after
@@ -920,7 +927,7 @@ def _inherit_publish_schedule(self, original, updates, associated_item):
 
     def _update_picture_metadata(self, updates, original, updated):
         renditions = updated.get("renditions") or {}
-        mapping = app.config.get("PHOTO_METADATA_MAPPING")
+        mapping = app.config.get("PICTURE_METADATA_MAPPING")
         if not mapping or not renditions:
             return
         try:

apps/publish/publish_service_tests.py

@@ -9,7 +9,6 @@
 # at https://www.sourcefabric.org/superdesk/license
 
 from bson import ObjectId
-from nose.tools import assert_raises
 
 from apps.publish import init_app
 from superdesk.errors import PublishQueueError
@@ -128,7 +127,7 @@ def mock_transmit(*args):
             publish_service = PublishService()
             publish_service._transmit = mock_transmit
 
-            with assert_raises(PublishQueueError):
+            with self.assertRaises(PublishQueueError):
                 publish_service.transmit(self.queue_items[0])
 
             subscriber = self.app.data.find_one("subscribers", None)

content_api/commands/remove_expired_items.py

@@ -74,7 +74,7 @@ def _remove_expired_items(self, expiry_datetime, expiry_days):
         :param int expiry_days: The number of days an item will be active
         """
         logger.info("{} Starting to remove expired items.".format(self.log_msg))
-        items_service = get_resource_service("items")
+        items_service = get_resource_service("capi_items_internal")
 
         num_items_removed = 0
         for expired_items in items_service.get_expired_items(

content_api/items/__init__.py

@@ -9,8 +9,8 @@
 # at https://www.sourcefabric.org/superdesk/license
 
 import superdesk
-from .service import ItemsService
-from .resource import ItemsResource
+from .service import InternalItemsService, ItemsService
+from .resource import InternalItemsResource, ItemsResource
 
 
 def init_app(app) -> None:
@@ -22,3 +22,6 @@ def init_app(app) -> None:
     endpoint_name = "items"
     service = ItemsService(endpoint_name, backend=superdesk.get_backend())
     ItemsResource(endpoint_name, app=app, service=service)
+
+    internal_service = InternalItemsService("capi_items_internal", backend=superdesk.get_backend())
+    InternalItemsResource("capi_items_internal", app=app, service=internal_service)

content_api/items/resource.py

@@ -108,6 +108,7 @@ class ItemsResource(Resource):
         "search_backend": "elastic",
         "elastic_filter": {"bool": {"must_not": {"term": {"type": "composite"}}}},
         "default_sort": [("versioncreated", -1)],
+        "source": "items",
     }
 
     mongo_indexes = {
@@ -120,3 +121,9 @@ class ItemsResource(Resource):
     versioning = True
     mongo_prefix = MONGO_PREFIX
     elastic_prefix = ELASTIC_PREFIX
+
+
+class InternalItemsResource(ItemsResource):
+    internal = True
+    mongo_indexes = {}
+    versioning = False