Merge branch 'master' into chore/sc-28332/inconsistent-loading-indica…

…tors-in-about

api/api_errors.py

@@ -0,0 +1,16 @@
+"""
+Classes for API errors
+"""
+from sefaria.client.util import jsonResponse
+
+
+class APIInvalidInputException(Exception):
+    """
+    When data in an invalid format is passed to an API
+    """
+    def __init__(self, message):
+        super().__init__(message)
+        self.message = message
+
+    def to_json_response(self):
+        return jsonResponse({"invalid_input_error": self.message}, status=400)

api/views.py

@@ -1,6 +1,7 @@
 from sefaria.model import *
 from sefaria.model.text_reuqest_adapter import TextRequestAdapter
 from sefaria.client.util import jsonResponse
+from sefaria.system.exceptions import InputError, ComplexBookLevelRefError
 from django.views import View
 from .api_warnings import *
 
@@ -53,6 +54,12 @@ def get(self, request, *args, **kwargs):
         if return_format not in self.RETURN_FORMATS:
             return jsonResponse({'error': f'return_format should be one of those formats: {self.RETURN_FORMATS}.'}, status=400)
         text_manager = TextRequestAdapter(self.oref, versions_params, fill_in_missing_segments, return_format)
-        data = text_manager.get_versions_for_query()
-        data = self._handle_warnings(data)
+
+        try:
+            data = text_manager.get_versions_for_query()
+            data = self._handle_warnings(data)
+
+        except Exception as e:
+            return jsonResponse({'error': str(e)}, status=400)
+
         return jsonResponse(data)

build/linker/Dockerfile

@@ -38,7 +38,7 @@ RUN apt-get update -qq \
  && rm -rf /var/lib/apt/lists/*
 
 # fix issues with shared objects
-RUN ls /usr/local/cuda-11.6/targets/x86_64-linux/lib/* | xargs -I{} ln -s {} /usr/lib/x86_64-linux-gnu/ \
+RUN ls /usr/local/cuda-11.4/targets/x86_64-linux/lib/* | xargs -I{} ln -s {} /usr/lib/x86_64-linux-gnu/ \
  && ln -s libcuda.so /usr/lib/x86_64-linux-gnu/libcuda.so.1 \
  && ln -s libnvidia-ml.so /usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1
 

docs/openAPI.json

@@ -4461,6 +4461,24 @@
         "tags": [
           "Topic"
         ],
+        "parameters": [
+          {
+              "examples": {
+                  "Return all topics": {
+                      "value": "limit=0"
+                  },
+                  "Return 20 topics": {
+                      "value": "limit=20"
+                  }
+              },
+              "name": "limit",
+              "description": "This parameter limits the number of topics returned. The default is `1000`. If `limit=0` then all topics will be returned.",
+              "schema": {
+                  "type": "integer"
+              },
+              "in": "query"
+          }
+      ],
         "responses": {
           "200": {
             "content": {

helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml

@@ -222,6 +222,9 @@ data:
     SENTINEL_HEADLESS_URL = os.getenv("SENTINEL_HEADLESS_URL") 
     SENTINEL_TRANSPORT_OPTS = json.loads(os.getenv("SENTINEL_TRANSPORT_OPTS", "{}"))
     SENTINEL_PASSWORD = os.getenv("SENTINEL_PASSWORD")
+    CELERY_ENABLED = os.getenv("CELERY_ENABLED").lower() == "true"
+    
+    SLACK_URL = os.getenv("SLACK_URL")
 
     MOBILE_APP_KEY = os.getenv("MOBILE_APP_KEY")
 

helm-chart/sefaria-project/templates/configmap/local-settings.yaml

@@ -37,3 +37,4 @@ data:
   SENTINEL_HEADLESS_URL: {{ .Values.tasks.redis.sentinelURL }}
   SENTINEL_TRANSPORT_OPTS: {{ .Values.tasks.redis.transportOptions | toJson | quote }}
   {{- end }}
+  CELERY_ENABLED: "{{ .Values.tasks.enabled }}"

helm-chart/sefaria-project/templates/cronjob/reindex-elasticsearch.yaml

@@ -28,7 +28,7 @@ spec:
             image: "{{ .Values.web.containerImage.imageRegistry }}:{{ .Values.web.containerImage.tag }}"
             resources:
               limits:
-                memory: 9Gi
+                memory: 10Gi
               requests:
                 memory: 7Gi
             env:

helm-chart/sefaria-project/templates/rollout/task.yaml

@@ -79,6 +79,11 @@ spec:
             value: "varnish-{{ .Values.deployEnv }}-{{ .Release.Revision }}"
           - name: HELM_REVISION
             value: "{{ .Release.Revision }}"
+          - name: SLACK_URL
+            valueFrom:
+              secretKeyRef:
+                name: { { template "sefaria.secrets.slackWebhook" . } }
+                key: slack-webhook
         envFrom:
           {{- if .Values.tasks.enabled }}
           - secretRef:
@@ -118,6 +123,9 @@ spec:
         - name: elastic-cert
           mountPath: /etc/ssl/certs/elastic
           readOnly: true
+        - mountPath: /varnish-secret
+          name: varnish-secret
+          readOnly: true
       volumes:
         - name:  local-settings
           configMap:
@@ -129,6 +137,9 @@ spec:
           secret:
             secretName: {{ template "sefaria.secrets.elasticCertificate" . }}
             optional: true
+        - name: varnish-secret
+          secret:
+            secretName: {{ template "sefaria.secrets.varnish" . }}
         - name: client-secret
           secret:
             secretName: {{ template "sefaria.secrets.googleClient" . }} # needs to be checked if it's a reference object or the data object we created.

helm-chart/sefaria-project/templates/rollout/web.yaml

@@ -117,6 +117,11 @@ spec:
           - name: OTEL_RESOURCE_ATTRIBUTES
             value: k8s.container.name=app,k8s.deployment.name={{ .Values.deployEnv }}-web,k8s.namespace.name={{ .Release.Namespace }},k8s.node.name=$(OTEL_RESOURCE_ATTRIBUTES_NODE_NAME),k8s.pod.name=$(OTEL_RESOURCE_ATTRIBUTES_POD_NAME)
           {{- end }}
+          - name: SLACK_URL
+            valueFrom:
+              secretKeyRef:
+                name: {{ template "sefaria.secrets.slackWebhook" . }}
+                key: slack-webhook
         envFrom:
           {{- if .Values.tasks.enabled }}
           - secretRef:

helm-chart/sefaria-project/values.yaml

@@ -104,10 +104,11 @@ linker:
   enabled: false
   # key-pair values to load into web pod environment.  Takes precedence over global localsettings
   model_paths:
-    # en: future_path
+    # english model supports people and refs
+    en: "gs://sefaria-ml-models/en_ner_model.tar.gz"
     he: "gs://sefaria-ml-models/ref_model.tar.gz"
   part_model_paths:
-    # en: future_path
+    en: "gs://sefaria-ml-models/en_subref_model.tar.gz"
     he: "gs://sefaria-ml-models/subref_model.tar.gz"
   localsettings:
   # APP: web

reader/views.py

@@ -15,12 +15,13 @@
 import os
 import re
 import uuid
+from dataclasses import asdict
 
 from rest_framework.decorators import api_view, permission_classes
 from rest_framework.permissions import IsAuthenticated
 from django.template.loader import render_to_string
 from django.shortcuts import render, redirect
-from django.http import Http404, QueryDict
+from django.http import Http404, QueryDict, HttpResponse
 from django.contrib.auth.decorators import login_required
 from django.contrib.admin.views.decorators import staff_member_required
 from django.utils.encoding import iri_to_uri
@@ -42,8 +43,9 @@
 from sefaria.model.following import general_follow_recommendations
 from sefaria.model.trend import user_stats_data, site_stats_data
 from sefaria.client.wrapper import format_object_for_client, format_note_object_for_client, get_notes, get_links
-from sefaria.client.util import jsonResponse
+from sefaria.client.util import jsonResponse, celeryResponse
 from sefaria.history import text_history, get_maximal_collapsed_activity, top_contributors, text_at_revision, record_version_deletion, record_index_deletion
+from sefaria.sefaria_tasks_interace.history_change import LinkChange, VersionChange
 from sefaria.sheets import get_sheets_for_ref, get_sheet_for_panel, annotate_user_links, trending_topics
 from sefaria.utils.util import text_preview, short_to_long_lang_code, epoch_time
 from sefaria.utils.hebrew import hebrew_term, has_hebrew
@@ -53,7 +55,7 @@
 from sefaria.site.site_settings import SITE_SETTINGS
 from sefaria.system.multiserver.coordinator import server_coordinator
 from sefaria.system.decorators import catch_error_as_json, sanitize_get_params, json_response_decorator
-from sefaria.system.exceptions import InputError, PartialRefInputError, BookNameError, NoVersionFoundError, DictionaryEntryNotFoundError
+from sefaria.system.exceptions import InputError, PartialRefInputError, BookNameError, NoVersionFoundError, DictionaryEntryNotFoundError, ComplexBookLevelRefError
 from sefaria.system.cache import django_cache
 from sefaria.system.database import db
 from sefaria.helper.search import get_query_obj
@@ -80,6 +82,7 @@
 from babel import Locale
 from sefaria.helper.topic import update_topic
 from sefaria.helper.category import update_order_of_category_children, check_term
+from sefaria.helper.texts.tasks import save_version, save_changes, save_link
 
 if USE_VARNISH:
     from sefaria.system.varnish.wrapper import invalidate_ref, invalidate_linked
@@ -1448,8 +1451,11 @@ def _get_text(oref, versionEn=versionEn, versionHe=versionHe, commentary=comment
             return text
 
         if not multiple or abs(multiple) == 1:
-            text = _get_text(oref, versionEn=versionEn, versionHe=versionHe, commentary=commentary, context=context, pad=pad,
-                             alts=alts, wrapLinks=wrapLinks, layer_name=layer_name)
+            try:
+                text = _get_text(oref, versionEn=versionEn, versionHe=versionHe, commentary=commentary, context=context, pad=pad,
+                                 alts=alts, wrapLinks=wrapLinks, layer_name=layer_name)
+            except Exception as e:
+                return jsonResponse({'error': str(e)}, status=400)
             return jsonResponse(text, cb)
         else:
             # Return list of many sections
@@ -1538,6 +1544,56 @@ def protected_post(request):
 
     return jsonResponse({"error": "Unsupported HTTP method."}, callback=request.GET.get("callback", None))
 
+
+@catch_error_as_json
+@csrf_exempt
+def complete_version_api(request):
+
+    def internal_do_post():
+        skip_links = bool(int(request.POST.get("skip_links", 0)))
+        count_after = int(request.POST.get("count_after", 0))
+        version_change = VersionChange(raw_version=data, uid=request.user.id, method=method, patch=patch, count_after=count_after, skip_links=skip_links)
+        task_title = f'Version Post: {data["title"]} / {data["versionTitle"]} / {data["language"]}'
+        return save_changes([asdict(version_change)], save_version, method, task_title)
+
+    if request.method == "POST":
+        patch = False
+    elif request.method == 'PATCH':
+        patch = True
+    else:
+        return jsonResponse({"error": "Unsupported HTTP method."}, callback=request.GET.get("callback", None))
+
+    body_unicode = request.body.decode('utf-8')
+    body_data = urllib.parse.parse_qs(body_unicode)
+    json_data = body_data.get('json')[0]
+    if not json_data:
+        return jsonResponse({"error": "Missing 'json' parameter in post data."})
+    data = json.loads(json_data)
+
+    title = data.get('title')
+    if not title:
+        return jsonResponse({"error": "Missing title in 'json' parameter."})
+
+    try:
+        index = library.get_index(title.replace('_', ' '))
+    except BookNameError:
+        return jsonResponse({"error": f"No index named: {title}"})
+
+    if not request.user.is_authenticated:
+        key = body_data.get('apikey')[0]
+        if not key:
+            return jsonResponse({"error": "You must be logged in or use an API key to save texts."})
+        apikey = db.apikeys.find_one({"key": key})
+        method = 'API'
+        if not apikey:
+            return jsonResponse({"error": "Unrecognized API key."})
+    else:
+        method = None
+        internal_do_post = csrf_protect(internal_do_post())
+
+    return internal_do_post()
+
+
 @catch_error_as_json
 @csrf_exempt
 def social_image_api(request, tref):
@@ -1921,18 +1977,20 @@ def links_api(request, link_id_or_ref=None):
     #TODO: can we distinguish between a link_id (mongo id) for POSTs and a ref for GETs?
     """
 
-    def _internal_do_post(request, link, uid, **kwargs):
-        func = tracker.update if "_id" in link else tracker.add
-        # use the correct function if params indicate this is a note save
-        # func = save_note if "type" in j and j["type"] == "note" else save_link
-        #obj = func(apikey["uid"], model.Link, link, **kwargs)
-        obj = func(uid, Link, link, **kwargs)
-        try:
-            if USE_VARNISH:
-                revarnish_link(obj)
-        except Exception as e:
-            logger.error(e)
-        return format_object_for_client(obj)
+    def _internal_do_post(request, obj, uid, method, skip_check, override_preciselink):
+        responses = []
+        if isinstance(obj, dict):
+            obj = [obj]
+        links = []
+        for l, link in enumerate(obj):
+            if skip_check:
+                link["_skip_lang_check"] = True
+            if override_preciselink:
+                link["_override_preciselink"] = True
+            links.append(asdict(LinkChange(raw_link=link, uid=uid, method=method)))
+
+        task_title = f'Links Post. First link: {obj[0]["refs"][0]}-{obj[0]["refs"][1]}'
+        return save_changes(links, save_link, method, task_title)
 
     def _internal_do_delete(request, link_id_or_ref, uid):
         obj = tracker.delete(uid, Link, link_id_or_ref, callback=revarnish_link)
@@ -1958,12 +2016,12 @@ def _internal_do_delete(request, link_id_or_ref, uid):
         if not apikey:
             return jsonResponse({"error": "Unrecognized API key."})
         uid = apikey["uid"]
-        kwargs = {"method": "API"}
+        method = "API"
         user = User.objects.get(id=apikey["uid"])
     else:
         user = request.user
         uid = request.user.id
-        kwargs = {}
+        method = None
         _internal_do_post = csrf_protect(_internal_do_post)
         _internal_do_delete = staff_member_required(csrf_protect(_internal_do_delete))
 
@@ -1975,30 +2033,7 @@ def _internal_do_delete(request, link_id_or_ref, uid):
         j = json.loads(j)
         skip_check = request.GET.get("skip_lang_check", 0)
         override_preciselink = request.GET.get("override_preciselink", 0)
-        if isinstance(j, list):
-            res = []
-            for i in j:
-                try:
-                    if skip_check:
-                        i["_skip_lang_check"] = True
-                    if override_preciselink:
-                        i["_override_preciselink"] = True
-                    retval = _internal_do_post(request, i, uid, **kwargs)
-                    res.append({"status": "ok. Link: {} | {} Saved".format(retval["ref"], retval["anchorRef"])})
-                except Exception as e:
-                    res.append({"error": "Link: {} | {} Error: {}".format(i["refs"][0], i["refs"][1], str(e))})
-
-            try:
-                res_slice = request.GET.get("truncate_response", None)
-                if res_slice:
-                    res_slice = int(res_slice)
-            except Exception as e:
-                res_slice = None
-            return jsonResponse(res[:res_slice])
-        else:
-            if skip_check:
-                j["_skip_lang_check"] = True
-            return jsonResponse(_internal_do_post(request, j, uid, **kwargs))
+        return _internal_do_post(request, j, uid, method, skip_check, override_preciselink)
 
     if request.method == "DELETE":
         if not link_id_or_ref:
@@ -3046,7 +3081,7 @@ def topic_page(request, topic, test_version=None):
     props = {
         "initialMenu": "topics",
         "initialTopic": topic,
-        "initialTab": urllib.parse.unquote(request.GET.get('tab', 'sources')),
+        "initialTab": urllib.parse.unquote(request.GET.get('tab', 'notable-sources')),
         "initialTopicSort": urllib.parse.unquote(request.GET.get('sort', 'Relevance')),
         "initialTopicTitle": {
             "en": topic_obj.get_primary_title('en'),
@@ -3085,15 +3120,16 @@ def topics_list_api(request):
 @staff_member_required
 def generate_topic_prompts_api(request, slug: str):
     if request.method == "POST":
+        task_ids = []
         from sefaria.helper.llm.tasks import generate_and_save_topic_prompts
         from sefaria.helper.llm.topic_prompt import get_ref_context_hints_by_lang
         topic = Topic.init(slug)
         post_body = json.loads(request.body)
         ref_topic_links = post_body.get('ref_topic_links')
         for lang, ref__context_hints in get_ref_context_hints_by_lang(ref_topic_links).items():
             orefs, context_hints = zip(*ref__context_hints)
-            generate_and_save_topic_prompts(lang, topic, orefs, context_hints)
-        return jsonResponse({"acknowledged": True}, status=202)
+            task_ids.append(generate_and_save_topic_prompts(lang, topic, orefs, context_hints).id)
+        return celeryResponse(task_ids)
     return jsonResponse({"error": "This API only accepts POST requests."})