Merge pull request #1998 from Sefaria/new-post

New post

helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml

@@ -222,6 +222,9 @@ data:
     SENTINEL_HEADLESS_URL = os.getenv("SENTINEL_HEADLESS_URL") 
     SENTINEL_TRANSPORT_OPTS = json.loads(os.getenv("SENTINEL_TRANSPORT_OPTS", "{}"))
     SENTINEL_PASSWORD = os.getenv("SENTINEL_PASSWORD")
+    CELERY_ENABLED = os.getenv("CELERY_ENABLED").lower() == "true"
+    
+    SLACK_URL = os.getenv("SLACK_URL")
 
     MOBILE_APP_KEY = os.getenv("MOBILE_APP_KEY")
 

helm-chart/sefaria-project/templates/configmap/local-settings.yaml

@@ -37,3 +37,4 @@ data:
   SENTINEL_HEADLESS_URL: {{ .Values.tasks.redis.sentinelURL }}
   SENTINEL_TRANSPORT_OPTS: {{ .Values.tasks.redis.transportOptions | toJson | quote }}
   {{- end }}
+  CELERY_ENABLED: "{{ .Values.tasks.enabled }}"

helm-chart/sefaria-project/templates/rollout/task.yaml

@@ -118,6 +118,9 @@ spec:
         - name: elastic-cert
           mountPath: /etc/ssl/certs/elastic
           readOnly: true
+        - mountPath: /varnish-secret
+          name: varnish-secret
+          readOnly: true
       volumes:
         - name:  local-settings
           configMap:
@@ -129,6 +132,9 @@ spec:
           secret:
             secretName: {{ template "sefaria.secrets.elasticCertificate" . }}
             optional: true
+        - name: varnish-secret
+          secret:
+            secretName: {{ template "sefaria.secrets.varnish" . }}
         - name: client-secret
           secret:
             secretName: {{ template "sefaria.secrets.googleClient" . }} # needs to be checked if it's a reference object or the data object we created.

helm-chart/sefaria-project/templates/rollout/web.yaml

@@ -116,6 +116,11 @@ spec:
                 fieldPath: spec.nodeName
           - name: OTEL_RESOURCE_ATTRIBUTES
             value: k8s.container.name=app,k8s.deployment.name={{ .Values.deployEnv }}-web,k8s.namespace.name={{ .Release.Namespace }},k8s.node.name=$(OTEL_RESOURCE_ATTRIBUTES_NODE_NAME),k8s.pod.name=$(OTEL_RESOURCE_ATTRIBUTES_POD_NAME)
+          - name: SLACK_URL
+            valueFrom:
+              secretKeyRef:
+                name: { { template "sefaria.secrets.slackWebhook" . } }
+                key: slack-webhook
           {{- end }}
         envFrom:
           {{- if .Values.tasks.enabled }}

reader/views.py

@@ -15,12 +15,13 @@
 import os
 import re
 import uuid
+from dataclasses import asdict
 
 from rest_framework.decorators import api_view, permission_classes
 from rest_framework.permissions import IsAuthenticated
 from django.template.loader import render_to_string
 from django.shortcuts import render, redirect
-from django.http import Http404, QueryDict
+from django.http import Http404, QueryDict, HttpResponse
 from django.contrib.auth.decorators import login_required
 from django.contrib.admin.views.decorators import staff_member_required
 from django.utils.encoding import iri_to_uri
@@ -42,8 +43,9 @@
 from sefaria.model.following import general_follow_recommendations
 from sefaria.model.trend import user_stats_data, site_stats_data
 from sefaria.client.wrapper import format_object_for_client, format_note_object_for_client, get_notes, get_links
-from sefaria.client.util import jsonResponse
+from sefaria.client.util import jsonResponse, celeryResponse
 from sefaria.history import text_history, get_maximal_collapsed_activity, top_contributors, text_at_revision, record_version_deletion, record_index_deletion
+from sefaria.sefaria_tasks_interace.history_change import LinkChange, VersionChange
 from sefaria.sheets import get_sheets_for_ref, get_sheet_for_panel, annotate_user_links, trending_topics
 from sefaria.utils.util import text_preview, short_to_long_lang_code, epoch_time
 from sefaria.utils.hebrew import hebrew_term, has_hebrew
@@ -80,6 +82,7 @@
 from babel import Locale
 from sefaria.helper.topic import update_topic
 from sefaria.helper.category import update_order_of_category_children, check_term
+from sefaria.helper.texts.tasks import save_version, save_changes, save_link
 
 if USE_VARNISH:
     from sefaria.system.varnish.wrapper import invalidate_ref, invalidate_linked
@@ -1538,6 +1541,56 @@ def protected_post(request):
 
     return jsonResponse({"error": "Unsupported HTTP method."}, callback=request.GET.get("callback", None))
 
+
+@catch_error_as_json
+@csrf_exempt
+def complete_version_api(request):
+
+    def internal_do_post():
+        skip_links = bool(int(request.POST.get("skip_links", 0)))
+        count_after = int(request.POST.get("count_after", 0))
+        version_change = VersionChange(raw_version=data, uid=request.user.id, method=method, patch=patch, count_after=count_after, skip_links=skip_links)
+        task_title = f'Version Post: {data["title"]} / {data["versionTitle"]} / {data["language"]}'
+        return save_changes([asdict(version_change)], save_version, method, task_title)
+
+    if request.method == "POST":
+        patch = False
+    elif request.method == 'PATCH':
+        patch = True
+    else:
+        return jsonResponse({"error": "Unsupported HTTP method."}, callback=request.GET.get("callback", None))
+
+    body_unicode = request.body.decode('utf-8')
+    body_data = urllib.parse.parse_qs(body_unicode)
+    json_data = body_data.get('json')[0]
+    if not json_data:
+        return jsonResponse({"error": "Missing 'json' parameter in post data."})
+    data = json.loads(json_data)
+
+    title = data.get('title')
+    if not title:
+        return jsonResponse({"error": "Missing title in 'json' parameter."})
+
+    try:
+        index = library.get_index(title.replace('_', ' '))
+    except BookNameError:
+        return jsonResponse({"error": f"No index named: {title}"})
+
+    if not request.user.is_authenticated:
+        key = body_data.get('apikey')[0]
+        if not key:
+            return jsonResponse({"error": "You must be logged in or use an API key to save texts."})
+        apikey = db.apikeys.find_one({"key": key})
+        method = 'API'
+        if not apikey:
+            return jsonResponse({"error": "Unrecognized API key."})
+    else:
+        method = None
+        internal_do_post = csrf_protect(internal_do_post())
+
+    return internal_do_post()
+
+
 @catch_error_as_json
 @csrf_exempt
 def social_image_api(request, tref):
@@ -1921,18 +1974,20 @@ def links_api(request, link_id_or_ref=None):
     #TODO: can we distinguish between a link_id (mongo id) for POSTs and a ref for GETs?
     """
 
-    def _internal_do_post(request, link, uid, **kwargs):
-        func = tracker.update if "_id" in link else tracker.add
-        # use the correct function if params indicate this is a note save
-        # func = save_note if "type" in j and j["type"] == "note" else save_link
-        #obj = func(apikey["uid"], model.Link, link, **kwargs)
-        obj = func(uid, Link, link, **kwargs)
-        try:
-            if USE_VARNISH:
-                revarnish_link(obj)
-        except Exception as e:
-            logger.error(e)
-        return format_object_for_client(obj)
+    def _internal_do_post(request, obj, uid, method, skip_check, override_preciselink):
+        responses = []
+        if isinstance(obj, dict):
+            obj = [obj]
+        links = []
+        for l, link in enumerate(obj):
+            if skip_check:
+                link["_skip_lang_check"] = True
+            if override_preciselink:
+                link["_override_preciselink"] = True
+            links.append(asdict(LinkChange(raw_link=link, uid=uid, method=method)))
+
+        task_title = f'Links Post. First link: {obj[0]["refs"][0]}-{obj[0]["refs"][1]}'
+        return save_changes(links, save_link, method, task_title)
 
     def _internal_do_delete(request, link_id_or_ref, uid):
         obj = tracker.delete(uid, Link, link_id_or_ref, callback=revarnish_link)
@@ -1958,12 +2013,12 @@ def _internal_do_delete(request, link_id_or_ref, uid):
         if not apikey:
             return jsonResponse({"error": "Unrecognized API key."})
         uid = apikey["uid"]
-        kwargs = {"method": "API"}
+        method = "API"
         user = User.objects.get(id=apikey["uid"])
     else:
         user = request.user
         uid = request.user.id
-        kwargs = {}
+        method = None
         _internal_do_post = csrf_protect(_internal_do_post)
         _internal_do_delete = staff_member_required(csrf_protect(_internal_do_delete))
 
@@ -1975,30 +2030,7 @@ def _internal_do_delete(request, link_id_or_ref, uid):
         j = json.loads(j)
         skip_check = request.GET.get("skip_lang_check", 0)
         override_preciselink = request.GET.get("override_preciselink", 0)
-        if isinstance(j, list):
-            res = []
-            for i in j:
-                try:
-                    if skip_check:
-                        i["_skip_lang_check"] = True
-                    if override_preciselink:
-                        i["_override_preciselink"] = True
-                    retval = _internal_do_post(request, i, uid, **kwargs)
-                    res.append({"status": "ok. Link: {} | {} Saved".format(retval["ref"], retval["anchorRef"])})
-                except Exception as e:
-                    res.append({"error": "Link: {} | {} Error: {}".format(i["refs"][0], i["refs"][1], str(e))})
-
-            try:
-                res_slice = request.GET.get("truncate_response", None)
-                if res_slice:
-                    res_slice = int(res_slice)
-            except Exception as e:
-                res_slice = None
-            return jsonResponse(res[:res_slice])
-        else:
-            if skip_check:
-                j["_skip_lang_check"] = True
-            return jsonResponse(_internal_do_post(request, j, uid, **kwargs))
+        return _internal_do_post(request, j, uid, method, skip_check, override_preciselink)
 
     if request.method == "DELETE":
         if not link_id_or_ref:
@@ -3085,15 +3117,16 @@ def topics_list_api(request):
 @staff_member_required
 def generate_topic_prompts_api(request, slug: str):
     if request.method == "POST":
+        task_ids = []
         from sefaria.helper.llm.tasks import generate_and_save_topic_prompts
         from sefaria.helper.llm.topic_prompt import get_ref_context_hints_by_lang
         topic = Topic.init(slug)
         post_body = json.loads(request.body)
         ref_topic_links = post_body.get('ref_topic_links')
         for lang, ref__context_hints in get_ref_context_hints_by_lang(ref_topic_links).items():
             orefs, context_hints = zip(*ref__context_hints)
-            generate_and_save_topic_prompts(lang, topic, orefs, context_hints)
-        return jsonResponse({"acknowledged": True}, status=202)
+            task_ids.append(generate_and_save_topic_prompts(lang, topic, orefs, context_hints).id)
+        return celeryResponse(task_ids)
     return jsonResponse({"error": "This API only accepts POST requests."})
 
 

scripts/move_draft_text.py

@@ -85,48 +85,12 @@ def do_copy(self):
             self.post_terms_from_schema()
             self._handle_categories()
             self._make_post_request_to_server(self._prepare_index_api_call(idx_title), idx_contents)
-        content_nodes = self._index_obj.nodes.get_leaf_nodes()
+
         for ver in self._version_objs:
             found_non_empty_content = False
             print(ver.versionTitle.encode('utf-8'))
-            flags = {}
-            for flag in ver.optional_attrs:
-                if hasattr(ver, flag):
-                    flags[flag] = getattr(ver, flag, None)
-            for node_num, node in enumerate(content_nodes,1):
-                print(node.full_title(force_update=True))
-                text = JaggedTextArray(ver.content_node(node)).array()
-                version_payload = {
-                        "versionTitle": ver.versionTitle,
-                        "versionSource": ver.versionSource,
-                        "language": ver.language,
-                        "text": text
-                }
-                if len(text) > 0:
-                    # only bother posting nodes that have content.
-                    found_non_empty_content = True
-                    if node_num == len(content_nodes):
-                        # try:
-                        self._make_post_request_to_server(self._prepare_text_api_call(node.full_title(force_update=True), count_after=True), version_payload)
-                        # except:
-                        #     pass
-                    else:
-                        self._make_post_request_to_server(self._prepare_text_api_call(
-                            node.full_title(force_update=True)), version_payload)
-            if not found_non_empty_content:
-                # post the last node again with dummy text, to make sure an actual version db object is created
-                # then post again to clear the dummy text
-                dummy_text = "This is a dummy text"
-                empty = ""
-                for _ in range(node.depth):
-                    dummy_text = [dummy_text]
-                    empty = [empty]
-                version_payload['text'] = dummy_text
-                self._make_post_request_to_server(self._prepare_text_api_call(node.full_title()), version_payload)
-                version_payload['text'] = empty
-                self._make_post_request_to_server(self._prepare_text_api_call(node.full_title()), version_payload)
-            if flags:
-                self._make_post_request_to_server(self._prepare_version_attrs_api_call(ver.title, ver.language, ver.versionTitle), flags)
+            self._make_post_request_to_server(self._prepare_text_api_call(), ver.contents(), params={'count_after': 1})
+
         if self._post_links and len(self._linkset) > 0:
             if self._post_links_step <= 0 or self._post_links_step > len(self._linkset):
                 self._post_links_step = len(self._linkset)
@@ -168,17 +132,15 @@ def _upload_term(self, name):
     def _prepare_index_api_call(self, index_title):
         return 'api/v2/raw/index/{}'.format(index_title.replace(" ", "_"))
 
-    def _prepare_text_api_call(self, terminal_ref, count_after=False):
-        return 'api/texts/{}?count_after={}&index_after=0'.format(urllib.parse.quote(terminal_ref.replace(" ", "_").encode('utf-8')), int(count_after))
-
-    def _prepare_version_attrs_api_call(self, title, lang, vtitle):
-        return "api/version/flags/{}/{}/{}".format(urllib.parse.quote(title), urllib.parse.quote(lang), urllib.parse.quote(vtitle))
+    def _prepare_text_api_call(self):
+        return 'api/versions/'
 
     def _prepare_links_api_call(self):
         return "api/links/"
 
-    def _make_post_request_to_server(self, url, payload):
-        full_url = "{}/{}".format(self._dest_server, url)
+    def _make_post_request_to_server(self, url, payload, params=None):
+        params = params or {}
+        full_url = f"{self._dest_server}/{url}?{urllib.parse.urlencode(params)}"
         jpayload = json.dumps(payload)
         values = {'json': jpayload, 'apikey': self._apikey}
         data = urllib.parse.urlencode(values).encode('utf-8')
@@ -204,7 +166,7 @@ def _make_post_request_to_server(self, url, payload):
     parser.add_argument("-v", "--versionlist", help="pipe separated version list: lang:versionTitle. To copy all versions, simply input 'all'")
     parser.add_argument("-k", "--apikey", help="non default api key", default=SEFARIA_BOT_API_KEY)
     parser.add_argument("-d", "--destination_server", help="override destination server", default='http://eph.sefaria.org')
-    parser.add_argument("-l", "--links", default=0, type=int, help="Enter '1' to move manual links on this text as well, '2' to move auto links")
+    parser.add_argument("-l", "--links", default=0, type=int, help="Enter '1' to move only manual links, '2' to move auto links on this text as well")
     parser.add_argument("-s", "--step", default=-1, type=int, help="Enter step size for link posting.  Size of 400 means links are posted 400 at a time.")
     args = parser.parse_args()
     print(args)

sefaria/celery_setup/generate_config.py

@@ -37,7 +37,7 @@ def add_db_num_to_url(url, port, db_num):
 
 
 def add_password_to_url(url, password):
-    if len(password) == 0:
+    if not password:
         return url
     return re.sub(r'((?:redis|sentinel)://)', fr'\1:{password}@', url)
 

sefaria/client/util.py

@@ -39,6 +39,12 @@ def jsonpResponse(data, callback, status=200):
     return HttpResponse("%s(%s)" % (callback, json.dumps(data, ensure_ascii=False)), content_type="application/javascript; charset=utf-8", charset="utf-8", status=status)
 
 
+def celeryResponse(task_id: str, sub_task_ids: list[str] = None):
+    data = {'task_id': task_id}
+    if sub_task_ids:
+        data['sub_task_ids'] = sub_task_ids
+    return jsonResponse(data, status=202)
+
 def send_email(subject, message_html, from_email, to_email):
     msg = EmailMultiAlternatives(subject, message_html, "Sefaria <[email protected]>", [to_email], reply_to=[from_email])
     msg.send()

sefaria/helper/slack/send_message.py

@@ -0,0 +1,20 @@
+import requests
+from sefaria.settings import SLACK_URL
+
+
+def send_message(channel, username, pretext, text, fallback=None, icon_emoji=':robot_face:', color="#a30200"):
+    post_object = {
+            "icon_emoji": icon_emoji,
+            "username": username,
+            "channel": channel,
+            "attachments": [
+                {
+                    "fallback": fallback or pretext,
+                    "color": color,
+                    "pretext": pretext,
+                    "text": text
+                }
+            ]
+        }
+
+    requests.post(SLACK_URL, json=post_object)