diff --git a/api/api_errors.py b/api/api_errors.py new file mode 100644 index 0000000000..aac047b48d --- /dev/null +++ b/api/api_errors.py @@ -0,0 +1,16 @@ +""" +Classes for API errors +""" +from sefaria.client.util import jsonResponse + + +class APIInvalidInputException(Exception): + """ + When data in an invalid format is passed to an API + """ + def __init__(self, message): + super().__init__(message) + self.message = message + + def to_json_response(self): + return jsonResponse({"invalid_input_error": self.message}, status=400) diff --git a/api/views.py b/api/views.py index bd1525390a..73d4dffa67 100644 --- a/api/views.py +++ b/api/views.py @@ -1,6 +1,7 @@ from sefaria.model import * from sefaria.model.text_reuqest_adapter import TextRequestAdapter from sefaria.client.util import jsonResponse +from sefaria.system.exceptions import InputError, ComplexBookLevelRefError from django.views import View from .api_warnings import * @@ -53,6 +54,12 @@ def get(self, request, *args, **kwargs): if return_format not in self.RETURN_FORMATS: return jsonResponse({'error': f'return_format should be one of those formats: {self.RETURN_FORMATS}.'}, status=400) text_manager = TextRequestAdapter(self.oref, versions_params, fill_in_missing_segments, return_format) - data = text_manager.get_versions_for_query() - data = self._handle_warnings(data) + + try: + data = text_manager.get_versions_for_query() + data = self._handle_warnings(data) + + except Exception as e: + return jsonResponse({'error': str(e)}, status=400) + return jsonResponse(data) diff --git a/build/linker/Dockerfile b/build/linker/Dockerfile index 4ed62d2af1..4be52f7b59 100644 --- a/build/linker/Dockerfile +++ b/build/linker/Dockerfile @@ -38,7 +38,7 @@ RUN apt-get update -qq \ && rm -rf /var/lib/apt/lists/* # fix issues with shared objects -RUN ls /usr/local/cuda-11.6/targets/x86_64-linux/lib/* | xargs -I{} ln -s {} /usr/lib/x86_64-linux-gnu/ \ +RUN ls /usr/local/cuda-11.4/targets/x86_64-linux/lib/* | xargs -I{} ln -s {} /usr/lib/x86_64-linux-gnu/ \ && ln -s libcuda.so /usr/lib/x86_64-linux-gnu/libcuda.so.1 \ && ln -s libnvidia-ml.so /usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1 diff --git a/docs/openAPI.json b/docs/openAPI.json index 2d8c71d9ae..213320032b 100644 --- a/docs/openAPI.json +++ b/docs/openAPI.json @@ -4461,6 +4461,24 @@ "tags": [ "Topic" ], + "parameters": [ + { + "examples": { + "Return all topics": { + "value": "limit=0" + }, + "Return 20 topics": { + "value": "limit=20" + } + }, + "name": "limit", + "description": "This parameter limits the number of topics returned. The default is `1000`. If `limit=0` then all topics will be returned.", + "schema": { + "type": "integer" + }, + "in": "query" + } + ], "responses": { "200": { "content": { diff --git a/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml b/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml index ee9d51ff01..d12b5ee7f1 100644 --- a/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml +++ b/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml @@ -222,6 +222,9 @@ data: SENTINEL_HEADLESS_URL = os.getenv("SENTINEL_HEADLESS_URL") SENTINEL_TRANSPORT_OPTS = json.loads(os.getenv("SENTINEL_TRANSPORT_OPTS", "{}")) SENTINEL_PASSWORD = os.getenv("SENTINEL_PASSWORD") + CELERY_ENABLED = os.getenv("CELERY_ENABLED").lower() == "true" + + SLACK_URL = os.getenv("SLACK_URL") MOBILE_APP_KEY = os.getenv("MOBILE_APP_KEY") diff --git a/helm-chart/sefaria-project/templates/configmap/local-settings.yaml b/helm-chart/sefaria-project/templates/configmap/local-settings.yaml index 7c3cf997b0..9f2ba22e56 100644 --- a/helm-chart/sefaria-project/templates/configmap/local-settings.yaml +++ b/helm-chart/sefaria-project/templates/configmap/local-settings.yaml @@ -37,3 +37,4 @@ data: SENTINEL_HEADLESS_URL: {{ .Values.tasks.redis.sentinelURL }} SENTINEL_TRANSPORT_OPTS: {{ .Values.tasks.redis.transportOptions | toJson | quote }} {{- end }} + CELERY_ENABLED: "{{ .Values.tasks.enabled }}" diff --git a/helm-chart/sefaria-project/templates/cronjob/reindex-elasticsearch.yaml b/helm-chart/sefaria-project/templates/cronjob/reindex-elasticsearch.yaml index 7577a0a95e..af5a5ecd1f 100644 --- a/helm-chart/sefaria-project/templates/cronjob/reindex-elasticsearch.yaml +++ b/helm-chart/sefaria-project/templates/cronjob/reindex-elasticsearch.yaml @@ -28,7 +28,7 @@ spec: image: "{{ .Values.web.containerImage.imageRegistry }}:{{ .Values.web.containerImage.tag }}" resources: limits: - memory: 9Gi + memory: 10Gi requests: memory: 7Gi env: diff --git a/helm-chart/sefaria-project/templates/rollout/task.yaml b/helm-chart/sefaria-project/templates/rollout/task.yaml index 4169f418af..3f9a233e59 100644 --- a/helm-chart/sefaria-project/templates/rollout/task.yaml +++ b/helm-chart/sefaria-project/templates/rollout/task.yaml @@ -79,6 +79,11 @@ spec: value: "varnish-{{ .Values.deployEnv }}-{{ .Release.Revision }}" - name: HELM_REVISION value: "{{ .Release.Revision }}" + - name: SLACK_URL + valueFrom: + secretKeyRef: + name: { { template "sefaria.secrets.slackWebhook" . } } + key: slack-webhook envFrom: {{- if .Values.tasks.enabled }} - secretRef: @@ -118,6 +123,9 @@ spec: - name: elastic-cert mountPath: /etc/ssl/certs/elastic readOnly: true + - mountPath: /varnish-secret + name: varnish-secret + readOnly: true volumes: - name: local-settings configMap: @@ -129,6 +137,9 @@ spec: secret: secretName: {{ template "sefaria.secrets.elasticCertificate" . }} optional: true + - name: varnish-secret + secret: + secretName: {{ template "sefaria.secrets.varnish" . }} - name: client-secret secret: secretName: {{ template "sefaria.secrets.googleClient" . }} # needs to be checked if it's a reference object or the data object we created. diff --git a/helm-chart/sefaria-project/templates/rollout/web.yaml b/helm-chart/sefaria-project/templates/rollout/web.yaml index 1264c78ba5..8b5837a08c 100644 --- a/helm-chart/sefaria-project/templates/rollout/web.yaml +++ b/helm-chart/sefaria-project/templates/rollout/web.yaml @@ -117,6 +117,11 @@ spec: - name: OTEL_RESOURCE_ATTRIBUTES value: k8s.container.name=app,k8s.deployment.name={{ .Values.deployEnv }}-web,k8s.namespace.name={{ .Release.Namespace }},k8s.node.name=$(OTEL_RESOURCE_ATTRIBUTES_NODE_NAME),k8s.pod.name=$(OTEL_RESOURCE_ATTRIBUTES_POD_NAME) {{- end }} + - name: SLACK_URL + valueFrom: + secretKeyRef: + name: {{ template "sefaria.secrets.slackWebhook" . }} + key: slack-webhook envFrom: {{- if .Values.tasks.enabled }} - secretRef: diff --git a/helm-chart/sefaria-project/values.yaml b/helm-chart/sefaria-project/values.yaml index 040e37a7ee..6f87339448 100644 --- a/helm-chart/sefaria-project/values.yaml +++ b/helm-chart/sefaria-project/values.yaml @@ -104,10 +104,11 @@ linker: enabled: false # key-pair values to load into web pod environment. Takes precedence over global localsettings model_paths: - # en: future_path + # english model supports people and refs + en: "gs://sefaria-ml-models/en_ner_model.tar.gz" he: "gs://sefaria-ml-models/ref_model.tar.gz" part_model_paths: - # en: future_path + en: "gs://sefaria-ml-models/en_subref_model.tar.gz" he: "gs://sefaria-ml-models/subref_model.tar.gz" localsettings: # APP: web diff --git a/reader/views.py b/reader/views.py index a8ed36a120..b8eab6c121 100644 --- a/reader/views.py +++ b/reader/views.py @@ -15,12 +15,13 @@ import os import re import uuid +from dataclasses import asdict from rest_framework.decorators import api_view, permission_classes from rest_framework.permissions import IsAuthenticated from django.template.loader import render_to_string from django.shortcuts import render, redirect -from django.http import Http404, QueryDict +from django.http import Http404, QueryDict, HttpResponse from django.contrib.auth.decorators import login_required from django.contrib.admin.views.decorators import staff_member_required from django.utils.encoding import iri_to_uri @@ -42,8 +43,9 @@ from sefaria.model.following import general_follow_recommendations from sefaria.model.trend import user_stats_data, site_stats_data from sefaria.client.wrapper import format_object_for_client, format_note_object_for_client, get_notes, get_links -from sefaria.client.util import jsonResponse +from sefaria.client.util import jsonResponse, celeryResponse from sefaria.history import text_history, get_maximal_collapsed_activity, top_contributors, text_at_revision, record_version_deletion, record_index_deletion +from sefaria.sefaria_tasks_interace.history_change import LinkChange, VersionChange from sefaria.sheets import get_sheets_for_ref, get_sheet_for_panel, annotate_user_links, trending_topics from sefaria.utils.util import text_preview, short_to_long_lang_code, epoch_time from sefaria.utils.hebrew import hebrew_term, has_hebrew @@ -53,7 +55,7 @@ from sefaria.site.site_settings import SITE_SETTINGS from sefaria.system.multiserver.coordinator import server_coordinator from sefaria.system.decorators import catch_error_as_json, sanitize_get_params, json_response_decorator -from sefaria.system.exceptions import InputError, PartialRefInputError, BookNameError, NoVersionFoundError, DictionaryEntryNotFoundError +from sefaria.system.exceptions import InputError, PartialRefInputError, BookNameError, NoVersionFoundError, DictionaryEntryNotFoundError, ComplexBookLevelRefError from sefaria.system.cache import django_cache from sefaria.system.database import db from sefaria.helper.search import get_query_obj @@ -80,6 +82,7 @@ from babel import Locale from sefaria.helper.topic import update_topic from sefaria.helper.category import update_order_of_category_children, check_term +from sefaria.helper.texts.tasks import save_version, save_changes, save_link if USE_VARNISH: from sefaria.system.varnish.wrapper import invalidate_ref, invalidate_linked @@ -1448,8 +1451,11 @@ def _get_text(oref, versionEn=versionEn, versionHe=versionHe, commentary=comment return text if not multiple or abs(multiple) == 1: - text = _get_text(oref, versionEn=versionEn, versionHe=versionHe, commentary=commentary, context=context, pad=pad, - alts=alts, wrapLinks=wrapLinks, layer_name=layer_name) + try: + text = _get_text(oref, versionEn=versionEn, versionHe=versionHe, commentary=commentary, context=context, pad=pad, + alts=alts, wrapLinks=wrapLinks, layer_name=layer_name) + except Exception as e: + return jsonResponse({'error': str(e)}, status=400) return jsonResponse(text, cb) else: # Return list of many sections @@ -1538,6 +1544,56 @@ def protected_post(request): return jsonResponse({"error": "Unsupported HTTP method."}, callback=request.GET.get("callback", None)) + +@catch_error_as_json +@csrf_exempt +def complete_version_api(request): + + def internal_do_post(): + skip_links = bool(int(request.POST.get("skip_links", 0))) + count_after = int(request.POST.get("count_after", 0)) + version_change = VersionChange(raw_version=data, uid=request.user.id, method=method, patch=patch, count_after=count_after, skip_links=skip_links) + task_title = f'Version Post: {data["title"]} / {data["versionTitle"]} / {data["language"]}' + return save_changes([asdict(version_change)], save_version, method, task_title) + + if request.method == "POST": + patch = False + elif request.method == 'PATCH': + patch = True + else: + return jsonResponse({"error": "Unsupported HTTP method."}, callback=request.GET.get("callback", None)) + + body_unicode = request.body.decode('utf-8') + body_data = urllib.parse.parse_qs(body_unicode) + json_data = body_data.get('json')[0] + if not json_data: + return jsonResponse({"error": "Missing 'json' parameter in post data."}) + data = json.loads(json_data) + + title = data.get('title') + if not title: + return jsonResponse({"error": "Missing title in 'json' parameter."}) + + try: + index = library.get_index(title.replace('_', ' ')) + except BookNameError: + return jsonResponse({"error": f"No index named: {title}"}) + + if not request.user.is_authenticated: + key = body_data.get('apikey')[0] + if not key: + return jsonResponse({"error": "You must be logged in or use an API key to save texts."}) + apikey = db.apikeys.find_one({"key": key}) + method = 'API' + if not apikey: + return jsonResponse({"error": "Unrecognized API key."}) + else: + method = None + internal_do_post = csrf_protect(internal_do_post()) + + return internal_do_post() + + @catch_error_as_json @csrf_exempt def social_image_api(request, tref): @@ -1921,18 +1977,20 @@ def links_api(request, link_id_or_ref=None): #TODO: can we distinguish between a link_id (mongo id) for POSTs and a ref for GETs? """ - def _internal_do_post(request, link, uid, **kwargs): - func = tracker.update if "_id" in link else tracker.add - # use the correct function if params indicate this is a note save - # func = save_note if "type" in j and j["type"] == "note" else save_link - #obj = func(apikey["uid"], model.Link, link, **kwargs) - obj = func(uid, Link, link, **kwargs) - try: - if USE_VARNISH: - revarnish_link(obj) - except Exception as e: - logger.error(e) - return format_object_for_client(obj) + def _internal_do_post(request, obj, uid, method, skip_check, override_preciselink): + responses = [] + if isinstance(obj, dict): + obj = [obj] + links = [] + for l, link in enumerate(obj): + if skip_check: + link["_skip_lang_check"] = True + if override_preciselink: + link["_override_preciselink"] = True + links.append(asdict(LinkChange(raw_link=link, uid=uid, method=method))) + + task_title = f'Links Post. First link: {obj[0]["refs"][0]}-{obj[0]["refs"][1]}' + return save_changes(links, save_link, method, task_title) def _internal_do_delete(request, link_id_or_ref, uid): obj = tracker.delete(uid, Link, link_id_or_ref, callback=revarnish_link) @@ -1958,12 +2016,12 @@ def _internal_do_delete(request, link_id_or_ref, uid): if not apikey: return jsonResponse({"error": "Unrecognized API key."}) uid = apikey["uid"] - kwargs = {"method": "API"} + method = "API" user = User.objects.get(id=apikey["uid"]) else: user = request.user uid = request.user.id - kwargs = {} + method = None _internal_do_post = csrf_protect(_internal_do_post) _internal_do_delete = staff_member_required(csrf_protect(_internal_do_delete)) @@ -1975,30 +2033,7 @@ def _internal_do_delete(request, link_id_or_ref, uid): j = json.loads(j) skip_check = request.GET.get("skip_lang_check", 0) override_preciselink = request.GET.get("override_preciselink", 0) - if isinstance(j, list): - res = [] - for i in j: - try: - if skip_check: - i["_skip_lang_check"] = True - if override_preciselink: - i["_override_preciselink"] = True - retval = _internal_do_post(request, i, uid, **kwargs) - res.append({"status": "ok. Link: {} | {} Saved".format(retval["ref"], retval["anchorRef"])}) - except Exception as e: - res.append({"error": "Link: {} | {} Error: {}".format(i["refs"][0], i["refs"][1], str(e))}) - - try: - res_slice = request.GET.get("truncate_response", None) - if res_slice: - res_slice = int(res_slice) - except Exception as e: - res_slice = None - return jsonResponse(res[:res_slice]) - else: - if skip_check: - j["_skip_lang_check"] = True - return jsonResponse(_internal_do_post(request, j, uid, **kwargs)) + return _internal_do_post(request, j, uid, method, skip_check, override_preciselink) if request.method == "DELETE": if not link_id_or_ref: @@ -3046,7 +3081,7 @@ def topic_page(request, topic, test_version=None): props = { "initialMenu": "topics", "initialTopic": topic, - "initialTab": urllib.parse.unquote(request.GET.get('tab', 'sources')), + "initialTab": urllib.parse.unquote(request.GET.get('tab', 'notable-sources')), "initialTopicSort": urllib.parse.unquote(request.GET.get('sort', 'Relevance')), "initialTopicTitle": { "en": topic_obj.get_primary_title('en'), @@ -3085,6 +3120,7 @@ def topics_list_api(request): @staff_member_required def generate_topic_prompts_api(request, slug: str): if request.method == "POST": + task_ids = [] from sefaria.helper.llm.tasks import generate_and_save_topic_prompts from sefaria.helper.llm.topic_prompt import get_ref_context_hints_by_lang topic = Topic.init(slug) @@ -3092,8 +3128,8 @@ def generate_topic_prompts_api(request, slug: str): ref_topic_links = post_body.get('ref_topic_links') for lang, ref__context_hints in get_ref_context_hints_by_lang(ref_topic_links).items(): orefs, context_hints = zip(*ref__context_hints) - generate_and_save_topic_prompts(lang, topic, orefs, context_hints) - return jsonResponse({"acknowledged": True}, status=202) + task_ids.append(generate_and_save_topic_prompts(lang, topic, orefs, context_hints).id) + return celeryResponse(task_ids) return jsonResponse({"error": "This API only accepts POST requests."}) diff --git a/scripts/move_draft_text.py b/scripts/move_draft_text.py index ab07284200..3425c975d3 100644 --- a/scripts/move_draft_text.py +++ b/scripts/move_draft_text.py @@ -85,48 +85,12 @@ def do_copy(self): self.post_terms_from_schema() self._handle_categories() self._make_post_request_to_server(self._prepare_index_api_call(idx_title), idx_contents) - content_nodes = self._index_obj.nodes.get_leaf_nodes() + for ver in self._version_objs: found_non_empty_content = False print(ver.versionTitle.encode('utf-8')) - flags = {} - for flag in ver.optional_attrs: - if hasattr(ver, flag): - flags[flag] = getattr(ver, flag, None) - for node_num, node in enumerate(content_nodes,1): - print(node.full_title(force_update=True)) - text = JaggedTextArray(ver.content_node(node)).array() - version_payload = { - "versionTitle": ver.versionTitle, - "versionSource": ver.versionSource, - "language": ver.language, - "text": text - } - if len(text) > 0: - # only bother posting nodes that have content. - found_non_empty_content = True - if node_num == len(content_nodes): - # try: - self._make_post_request_to_server(self._prepare_text_api_call(node.full_title(force_update=True), count_after=True), version_payload) - # except: - # pass - else: - self._make_post_request_to_server(self._prepare_text_api_call( - node.full_title(force_update=True)), version_payload) - if not found_non_empty_content: - # post the last node again with dummy text, to make sure an actual version db object is created - # then post again to clear the dummy text - dummy_text = "This is a dummy text" - empty = "" - for _ in range(node.depth): - dummy_text = [dummy_text] - empty = [empty] - version_payload['text'] = dummy_text - self._make_post_request_to_server(self._prepare_text_api_call(node.full_title()), version_payload) - version_payload['text'] = empty - self._make_post_request_to_server(self._prepare_text_api_call(node.full_title()), version_payload) - if flags: - self._make_post_request_to_server(self._prepare_version_attrs_api_call(ver.title, ver.language, ver.versionTitle), flags) + self._make_post_request_to_server(self._prepare_text_api_call(), ver.contents(), params={'count_after': 1}) + if self._post_links and len(self._linkset) > 0: if self._post_links_step <= 0 or self._post_links_step > len(self._linkset): self._post_links_step = len(self._linkset) @@ -168,17 +132,15 @@ def _upload_term(self, name): def _prepare_index_api_call(self, index_title): return 'api/v2/raw/index/{}'.format(index_title.replace(" ", "_")) - def _prepare_text_api_call(self, terminal_ref, count_after=False): - return 'api/texts/{}?count_after={}&index_after=0'.format(urllib.parse.quote(terminal_ref.replace(" ", "_").encode('utf-8')), int(count_after)) - - def _prepare_version_attrs_api_call(self, title, lang, vtitle): - return "api/version/flags/{}/{}/{}".format(urllib.parse.quote(title), urllib.parse.quote(lang), urllib.parse.quote(vtitle)) + def _prepare_text_api_call(self): + return 'api/versions/' def _prepare_links_api_call(self): return "api/links/" - def _make_post_request_to_server(self, url, payload): - full_url = "{}/{}".format(self._dest_server, url) + def _make_post_request_to_server(self, url, payload, params=None): + params = params or {} + full_url = f"{self._dest_server}/{url}?{urllib.parse.urlencode(params)}" jpayload = json.dumps(payload) values = {'json': jpayload, 'apikey': self._apikey} data = urllib.parse.urlencode(values).encode('utf-8') @@ -204,7 +166,7 @@ def _make_post_request_to_server(self, url, payload): parser.add_argument("-v", "--versionlist", help="pipe separated version list: lang:versionTitle. To copy all versions, simply input 'all'") parser.add_argument("-k", "--apikey", help="non default api key", default=SEFARIA_BOT_API_KEY) parser.add_argument("-d", "--destination_server", help="override destination server", default='http://eph.sefaria.org') - parser.add_argument("-l", "--links", default=0, type=int, help="Enter '1' to move manual links on this text as well, '2' to move auto links") + parser.add_argument("-l", "--links", default=0, type=int, help="Enter '1' to move only manual links, '2' to move auto links on this text as well") parser.add_argument("-s", "--step", default=-1, type=int, help="Enter step size for link posting. Size of 400 means links are posted 400 at a time.") args = parser.parse_args() print(args) diff --git a/sefaria/celery_setup/generate_config.py b/sefaria/celery_setup/generate_config.py index e297b99e69..876249b834 100644 --- a/sefaria/celery_setup/generate_config.py +++ b/sefaria/celery_setup/generate_config.py @@ -37,7 +37,7 @@ def add_db_num_to_url(url, port, db_num): def add_password_to_url(url, password): - if len(password) == 0: + if not password: return url return re.sub(r'((?:redis|sentinel)://)', fr'\1:{password}@', url) diff --git a/sefaria/client/util.py b/sefaria/client/util.py index 38a4014fad..2119f88ba0 100644 --- a/sefaria/client/util.py +++ b/sefaria/client/util.py @@ -39,6 +39,12 @@ def jsonpResponse(data, callback, status=200): return HttpResponse("%s(%s)" % (callback, json.dumps(data, ensure_ascii=False)), content_type="application/javascript; charset=utf-8", charset="utf-8", status=status) +def celeryResponse(task_id: str, sub_task_ids: list[str] = None): + data = {'task_id': task_id} + if sub_task_ids: + data['sub_task_ids'] = sub_task_ids + return jsonResponse(data, status=202) + def send_email(subject, message_html, from_email, to_email): msg = EmailMultiAlternatives(subject, message_html, "Sefaria ", [to_email], reply_to=[from_email]) msg.send() diff --git a/sefaria/export.py b/sefaria/export.py index 444fdc6278..77de185715 100644 --- a/sefaria/export.py +++ b/sefaria/export.py @@ -726,7 +726,11 @@ def _import_versions_from_csv(rows, columns, user_id): from sefaria.tracker import modify_bulk_text index_title = rows[0][columns[0]] # assume the same index title for all - index_node = Ref(index_title).index_node + index = Index().load({'title': index_title}) + if index: + index_node = index.nodes + else: + raise InputError(f'No book with primary title "{index_title}"') action = "edit" diff --git a/sefaria/helper/linker.py b/sefaria/helper/linker.py index 0d0d57d57f..9451fd4392 100644 --- a/sefaria/helper/linker.py +++ b/sefaria/helper/linker.py @@ -2,15 +2,55 @@ import json import spacy import structlog +from cerberus import Validator from sefaria.model.linker.ref_part import TermContext, RefPartType from sefaria.model.linker.ref_resolver import PossiblyAmbigResolvedRef from sefaria.model import text, library from sefaria.model.webpage import WebPage from sefaria.system.cache import django_cache -from typing import List, Union, Optional, Tuple +from api.api_errors import APIInvalidInputException +from typing import List, Optional, Tuple logger = structlog.get_logger(__name__) +FIND_REFS_POST_SCHEMA = { + "text": { + "type": "dict", + "required": True, + "schema": { + "title": {"type": "string", "required": True}, + "body": {"type": "string", "required": True}, + }, + }, + "metaDataForTracking": { + "type": "dict", + "required": False, + "schema": { + "url": {"type": "string", "required": False}, + "description": {"type": "string", "required": False}, + "title": {"type": "string", "required": False}, + }, + }, + "lang": { + "type": "string", + "allowed": ["he", "en"], + "required": False, + }, + "version_preferences_by_corpus": { + "type": "dict", + "required": False, + "keysrules": {"type": "string"}, + "valuesrules": { + "type": "dict", + "schema": { + "type": "string", + "keysrules": {"type": "string"}, + "valuesrules": {"type": "string"}, + }, + }, + }, +} + def load_spacy_model(path: str) -> spacy.Language: import re, tarfile @@ -23,7 +63,7 @@ def load_spacy_model(path: str) -> spacy.Language: if path.startswith("gs://"): # file is located in Google Cloud - # file is expected to be a tar.gz of the model folder + # file is expected to be a tar.gz of the contents of the model folder (not the folder itself) match = re.match(r"gs://([^/]+)/(.+)$", path) bucket_name = match.group(1) blob_name = match.group(2) @@ -64,13 +104,12 @@ class _FindRefsText: body: str lang: str - # def __post_init__(self): - # from sefaria.utils.hebrew import is_mostly_hebrew - # self.lang = 'he' if is_mostly_hebrew(self.body) else 'en' - def _unpack_find_refs_request(request): + validator = Validator(FIND_REFS_POST_SCHEMA) post_body = json.loads(request.body) + if not validator.validate(post_body): + raise APIInvalidInputException(validator.errors) meta_data = post_body.get('metaDataForTracking') return _create_find_refs_text(post_body), _create_find_refs_options(request.GET, post_body), meta_data @@ -101,10 +140,7 @@ def _add_webpage_hit_for_url(url): @django_cache(cache_type="persistent") def _make_find_refs_response_with_cache(request_text: _FindRefsText, options: _FindRefsTextOptions, meta_data: dict) -> dict: - if request_text.lang == 'he': - response = _make_find_refs_response_linker_v3(request_text, options) - else: - response = _make_find_refs_response_linker_v2(request_text, options) + response = _make_find_refs_response_linker_v3(request_text, options) if meta_data: _, webpage = WebPage.add_or_update_from_linker({ diff --git a/sefaria/helper/llm/topic_prompt.py b/sefaria/helper/llm/topic_prompt.py index 607ccc06f6..fba0b9e033 100644 --- a/sefaria/helper/llm/topic_prompt.py +++ b/sefaria/helper/llm/topic_prompt.py @@ -56,8 +56,11 @@ def _get_context_ref(segment_oref: Ref) -> Optional[Ref]: if segment_oref.primary_category == "Tanakh": return segment_oref.section_ref() elif segment_oref.index.get_primary_corpus() == "Bavli": - passage = Passage.containing_segment(segment_oref) - return passage.ref() + try: + passage = Passage.containing_segment(segment_oref) + return passage.ref() + except: + return None return None diff --git a/sefaria/helper/slack/__init__.py b/sefaria/helper/slack/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sefaria/helper/slack/send_message.py b/sefaria/helper/slack/send_message.py new file mode 100644 index 0000000000..e60a0af0ed --- /dev/null +++ b/sefaria/helper/slack/send_message.py @@ -0,0 +1,20 @@ +import requests +from sefaria.settings import SLACK_URL + + +def send_message(channel, username, pretext, text, fallback=None, icon_emoji=':robot_face:', color="#a30200"): + post_object = { + "icon_emoji": icon_emoji, + "username": username, + "channel": channel, + "attachments": [ + { + "fallback": fallback or pretext, + "color": color, + "pretext": pretext, + "text": text + } + ] + } + + requests.post(SLACK_URL, json=post_object) diff --git a/sefaria/helper/tests/linker_test.py b/sefaria/helper/tests/linker_test.py index e4afc30574..8779429e77 100644 --- a/sefaria/helper/tests/linker_test.py +++ b/sefaria/helper/tests/linker_test.py @@ -10,6 +10,7 @@ from sefaria.model.text import Ref, TextChunk from sefaria.model.webpage import WebPage from sefaria.settings import ENABLE_LINKER +from api.api_errors import APIInvalidInputException if not ENABLE_LINKER: pytest.skip("Linker not enabled", allow_module_level=True) @@ -80,6 +81,12 @@ def mock_request_post_data_without_meta_data(mock_request_post_data: dict) -> di return mock_request_post_data +@pytest.fixture +def mock_request_invalid_post_data(mock_request_post_data: dict) -> dict: + mock_request_post_data['text'] = 'plain text' + return mock_request_post_data + + def make_mock_request(post_data: dict) -> WSGIRequest: factory = RequestFactory() request = factory.post('/api/find-refs', data=json.dumps(post_data), content_type='application/json') @@ -109,6 +116,11 @@ def mock_request_without_meta_data(mock_request_post_data_without_meta_data: dic return make_mock_request(mock_request_post_data_without_meta_data) +@pytest.fixture +def mock_request_invalid(mock_request_invalid_post_data: dict) -> WSGIRequest: + return make_mock_request(mock_request_invalid_post_data) + + @pytest.fixture def mock_webpage() -> WebPage: # Note, the path of WebPage matches the path of the import we want to patch @@ -162,6 +174,13 @@ def test_make_find_refs_response_without_meta_data(self, mock_request_without_me mock_webpage.add_hit.assert_not_called() mock_webpage.save.assert_not_called() + def test_make_find_refs_response_invalid_post_data(self, mock_request_invalid: dict, + mock_webpage: Mock): + with pytest.raises(APIInvalidInputException) as exc_info: + response = linker.make_find_refs_response(mock_request_invalid) + # assert that the 'text' field had a validation error + assert 'text' in exc_info.value.args[0] + class TestUnpackFindRefsRequest: def test_unpack_find_refs_request(self, mock_request: WSGIRequest): @@ -198,8 +217,8 @@ def mock_get_linker(self, spacy_model: spacy.Language): with patch.object(library, 'get_linker') as mock_get_linker: mock_linker = Mock() mock_get_linker.return_value = mock_linker - mock_linker.link.return_value = LinkedDoc('', [], []) - mock_linker.link_by_paragraph.return_value = LinkedDoc('', [], []) + mock_linker.link.return_value = LinkedDoc('', [], [], []) + mock_linker.link_by_paragraph.return_value = LinkedDoc('', [], [], []) yield mock_get_linker def test_make_find_refs_response_linker_v3(self, mock_get_linker: WSGIRequest, diff --git a/sefaria/helper/texts/tasks.py b/sefaria/helper/texts/tasks.py new file mode 100644 index 0000000000..b5696d8a1c --- /dev/null +++ b/sefaria/helper/texts/tasks.py @@ -0,0 +1,89 @@ +import traceback +import uuid +import structlog +import django +from celery import chord +from collections import Counter +from sefaria.client.util import celeryResponse, jsonResponse +from sefaria.system.exceptions import DuplicateRecordError + +django.setup() +from sefaria.model import * +import sefaria.tracker as tracker +from sefaria.client.wrapper import format_object_for_client +from sefaria.settings import CELERY_QUEUES, CELERY_ENABLED +from sefaria.celery_setup.app import app +from sefaria.settings import USE_VARNISH +from sefaria.helper.slack.send_message import send_message +if USE_VARNISH: + from sefaria.system.varnish.wrapper import invalidate_ref + +logger = structlog.get_logger(__name__) + + +def should_run_with_celery(from_api): + return CELERY_ENABLED and from_api + +def save_changes(changes, func, method, task_title=''): + if should_run_with_celery(method == 'API'): + main_task_id = str(uuid.uuid4()) + tasks = [save_change.s(func.__name__, c).set(queue=CELERY_QUEUES['tasks']) for c in changes] + job = chord(tasks, inform.s(main_task_id=main_task_id, task_title=task_title).set(queue=CELERY_QUEUES['tasks']))(task_id=main_task_id) + tasks_ids = [task.id for task in job.parent.results] + return celeryResponse(job.id, tasks_ids) + else: + results = [] + for change in changes: + try: + func(change) + except Exception as e: + results.append({'error': f'Object: {change}. Error: {e}'}) + else: + results.append({'status': 'ok'}) + return jsonResponse(results) + +@app.task(name="web.save_change", acks_late=True, ignore_result=True) +def save_change(func_name, raw_history_change): + function_names = {'save_link': save_link, 'save_version': save_version} + func = function_names[func_name] + try: + func(raw_history_change) + return 'Success' + except Exception as e: + logger.error(f'''Error: + change: {raw_history_change} + {traceback.format_exc()}''') + if isinstance(e, DuplicateRecordError): + return 'DuplicateRecordError' + else: + return repr(e) + +@app.task(name="web.inform", acks_late=True) +def inform(results, main_task_id, task_title): + title = f'{task_title} (celery main task id {main_task_id})' + results = '\n'.join([f'{k}: {v}.' for k, v in Counter(results).items()]) + send_message('#engineering-signal', 'Text Upload', title, results, icon_emoji=':leafy_green:') + +def save_link(raw_link_change: dict): + link = raw_link_change['raw_link'] + uid = raw_link_change['uid'] + kwargs = {} + if raw_link_change['method'] == 'API': + kwargs['method'] = raw_link_change['method'] + func = tracker.update if "_id" in link else tracker.add + # use the correct function if params indicate this is a note save + obj = func(uid, Link, link, **kwargs) + try: + if USE_VARNISH: + for ref in link.refs: + invalidate_ref(Ref(ref), purge=True) + except Exception as e: + logger.error(e) + return format_object_for_client(obj) + +def save_version(raw_version_change: dict): + version = raw_version_change['raw_version'] + uid = raw_version_change['uid'] + patch = raw_version_change['patch'] + kwargs = {'skip_links': raw_version_change['skip_links'], 'count_after': raw_version_change['count_after']} + tracker.modify_version(uid, version, patch, **kwargs) diff --git a/sefaria/local_settings_example.py b/sefaria/local_settings_example.py index 3517e77eb9..7240dc4392 100644 --- a/sefaria/local_settings_example.py +++ b/sefaria/local_settings_example.py @@ -270,8 +270,12 @@ CELERY_REDIS_BROKER_DB_NUM = 2 CELERY_REDIS_RESULT_BACKEND_DB_NUM = 3 CELERY_QUEUES = {} +CELERY_ENABLED = False # END Celery +#Slack +SLACK_URL = '' + # Key which identifies the Sefaria app as opposed to a user # using our API outside of the app. Mainly for registration MOBILE_APP_KEY = "MOBILE_APP_KEY" diff --git a/sefaria/model/category.py b/sefaria/model/category.py index 1cea152560..6a849ab69d 100644 --- a/sefaria/model/category.py +++ b/sefaria/model/category.py @@ -9,9 +9,10 @@ from . import schema as schema from . import text as text from . import collection as collection +from .linker.has_match_template import HasMatchTemplates -class Category(abstract.AbstractMongoRecord, schema.AbstractTitledOrTermedObject): +class Category(abstract.AbstractMongoRecord, schema.AbstractTitledOrTermedObject, HasMatchTemplates): collection = 'category' history_noun = "category" @@ -30,6 +31,7 @@ class Category(abstract.AbstractMongoRecord, schema.AbstractTitledOrTermedObject "isPrimary", "searchRoot", "order", + "match_templates", ] def __str__(self): diff --git a/sefaria/model/linker/category_resolver.py b/sefaria/model/linker/category_resolver.py new file mode 100644 index 0000000000..2287fb1fbe --- /dev/null +++ b/sefaria/model/linker/category_resolver.py @@ -0,0 +1,45 @@ +from collections import defaultdict +from sefaria.model.category import Category +from sefaria.model.linker.ref_part import RawRef + + +class ResolvedCategory: + + def __init__(self, raw_ref: RawRef, categories: list[Category]) -> None: + self.raw_entity = raw_ref + self.categories = categories + + @property + def is_ambiguous(self): + return len(self.categories) != 1 + + @property + def resolution_failed(self): + return len(self.categories) == 0 + + +class CategoryMatcher: + + def __init__(self, lang: str, category_registry: list[Category]) -> None: + self._title_to_cat: dict[str, list[Category]] = defaultdict(list) + for cat in category_registry: + for match_template in cat.get_match_templates(): + for term in match_template.get_terms(): + for title in term.get_titles(lang): + self._title_to_cat[title] += [cat] + + def match(self, raw_ref: RawRef) -> list[Category]: + return self._title_to_cat[raw_ref.text] + + +class CategoryResolver: + + def __init__(self, category_matcher: CategoryMatcher) -> None: + self._matcher = category_matcher + + def bulk_resolve(self, raw_refs: list[RawRef]) -> list[ResolvedCategory]: + resolved = [] + for raw_ref in raw_refs: + matched_categories = self._matcher.match(raw_ref) + resolved += [ResolvedCategory(raw_ref, matched_categories)] + return resolved diff --git a/sefaria/model/linker/has_match_template.py b/sefaria/model/linker/has_match_template.py new file mode 100644 index 0000000000..1f15c04879 --- /dev/null +++ b/sefaria/model/linker/has_match_template.py @@ -0,0 +1,19 @@ +class HasMatchTemplates: + MATCH_TEMPLATE_ALONE_SCOPES = {'any', 'alone'} + + def get_match_templates(self): + from sefaria.model.linker.match_template import MatchTemplate + for raw_match_template in getattr(self, 'match_templates', []): + yield MatchTemplate(**raw_match_template) + + def get_match_template_trie(self, lang: str): + from sefaria.model.linker.match_template import MatchTemplateTrie + return MatchTemplateTrie(lang, nodes=[self], scope='combined') + + def has_scope_alone_match_template(self): + """ + @return: True if `self` has any match template that has scope = "alone" OR scope = "any" + """ + return any(template.scope in self.MATCH_TEMPLATE_ALONE_SCOPES for template in self.get_match_templates()) + + diff --git a/sefaria/model/linker/linker.py b/sefaria/model/linker/linker.py index f496780dd0..69dd08d6b7 100644 --- a/sefaria/model/linker/linker.py +++ b/sefaria/model/linker/linker.py @@ -3,28 +3,31 @@ from tqdm import tqdm from sefaria.model.text import Ref from sefaria.model.linker.ref_part import RawRef, RawNamedEntity, span_inds -from sefaria.model.linker.ref_resolver import RefResolver, ResolutionThoroughness, PossiblyAmbigResolvedRef +from sefaria.model.linker.ref_resolver import RefResolver, ResolutionThoroughness, PossiblyAmbigResolvedRef, ResolvedRef from sefaria.model.linker.named_entity_resolver import NamedEntityResolver, ResolvedNamedEntity from sefaria.model.linker.named_entity_recognizer import NamedEntityRecognizer +from sefaria.model.linker.category_resolver import CategoryResolver, ResolvedCategory @dataclasses.dataclass class LinkedDoc: text: str - resolved_refs: List[PossiblyAmbigResolvedRef] - resolved_named_entities: List[ResolvedNamedEntity] + resolved_refs: list[PossiblyAmbigResolvedRef] + resolved_named_entities: list[ResolvedNamedEntity] + resolved_categories: list[ResolvedCategory] @property - def all_resolved(self) -> List[Union[PossiblyAmbigResolvedRef, ResolvedNamedEntity]]: - return self.resolved_refs + self.resolved_named_entities + def all_resolved(self) -> List[Union[PossiblyAmbigResolvedRef, ResolvedNamedEntity, ResolvedCategory]]: + return self.resolved_refs + self.resolved_named_entities + self.resolved_categories class Linker: - def __init__(self, ner: NamedEntityRecognizer, ref_resolver: RefResolver, ne_resolver: NamedEntityResolver, ): + def __init__(self, ner: NamedEntityRecognizer, ref_resolver: RefResolver, ne_resolver: NamedEntityResolver, cat_resolver: CategoryResolver): self._ner = ner self._ref_resolver = ref_resolver self._ne_resolver = ne_resolver + self._cat_resolver = cat_resolver def bulk_link(self, inputs: List[str], book_context_refs: Optional[List[Optional[Ref]]] = None, with_failures=False, verbose=False, thoroughness=ResolutionThoroughness.NORMAL, type_filter='all') -> List[LinkedDoc]: @@ -47,12 +50,14 @@ def bulk_link(self, inputs: List[str], book_context_refs: Optional[List[Optional iterable = self._get_bulk_link_iterable(inputs, all_named_entities, book_context_refs, verbose) for input_str, book_context_ref, inner_named_entities in iterable: raw_refs, named_entities = self._partition_raw_refs_and_named_entities(inner_named_entities) - resolved_refs, resolved_named_entities = [], [] + resolved_refs, resolved_named_entities, resolved_cats = [], [], [] if type_filter in {'all', 'citation'}: - resolved_refs = self._ref_resolver.bulk_resolve(raw_refs, book_context_ref, with_failures, thoroughness, reset_ibids=False) + resolved_refs, resolved_cats = self._bulk_resolve_refs_and_cats(raw_refs, book_context_ref, thoroughness, False) if type_filter in {'all', 'named entity'}: - resolved_named_entities = self._ne_resolver.bulk_resolve(named_entities, with_failures) - docs += [LinkedDoc(input_str, resolved_refs, resolved_named_entities)] + resolved_named_entities = self._ne_resolver.bulk_resolve(named_entities) + if not with_failures: + resolved_refs, resolved_named_entities, resolved_cats = self._remove_failures(resolved_refs, resolved_named_entities, resolved_cats) + docs += [LinkedDoc(input_str, resolved_refs, resolved_named_entities, resolved_cats)] named_entity_list_list = [[rr.raw_entity for rr in doc.all_resolved] for doc in docs] self._ner.bulk_map_normal_output_to_original_input(inputs, named_entity_list_list) @@ -70,16 +75,18 @@ def link(self, input_str: str, book_context_ref: Optional[Ref] = None, with_fail @return: """ raw_refs, named_entities = self._ner.recognize(input_str) - resolved_refs, resolved_named_entities = [], [] + resolved_refs, resolved_named_entities, resolved_cats = [], [], [] if type_filter in {'all', 'citation'}: - resolved_refs = self._ref_resolver.bulk_resolve(raw_refs, book_context_ref, with_failures, thoroughness) + resolved_refs, resolved_cats = self._bulk_resolve_refs_and_cats(raw_refs, book_context_ref, thoroughness) if type_filter in {'all', 'named entity'}: - resolved_named_entities = self._ne_resolver.bulk_resolve(named_entities, with_failures) - doc = LinkedDoc(input_str, resolved_refs, resolved_named_entities) + resolved_named_entities = self._ne_resolver.bulk_resolve(named_entities) + if not with_failures: + resolved_refs, resolved_named_entities, resolved_cats = self._remove_failures(resolved_refs, resolved_named_entities, resolved_cats) + doc = LinkedDoc(input_str, resolved_refs, resolved_named_entities, resolved_cats) self._ner.map_normal_output_to_original_input(input_str, [x.raw_entity for x in doc.all_resolved]) return doc - def link_by_paragraph(self, input_str: str, book_context_ref: Ref, *link_args, **link_kwargs) -> LinkedDoc: + def link_by_paragraph(self, input_str: str, book_context_ref: Optional[Ref] = None, *link_args, **link_kwargs) -> LinkedDoc: """ Similar to `link()` except model is run on each paragraph individually (via a bulk operation) This better mimics the way the underlying ML models were trained and tends to lead to better results @@ -96,11 +103,13 @@ def link_by_paragraph(self, input_str: str, book_context_ref: Ref, *link_args, * linked_docs = self.bulk_link(inputs, [book_context_ref]*len(inputs), *link_args, **link_kwargs) resolved_refs = [] resolved_named_entities = [] + resolved_categories = [] full_spacy_doc = self._ner.named_entity_model.make_doc(input_str) offset = 0 for curr_input, linked_doc in zip(inputs, linked_docs): resolved_refs += linked_doc.resolved_refs resolved_named_entities += linked_doc.resolved_named_entities + resolved_categories += linked_doc.resolved_categories for resolved in linked_doc.all_resolved: named_entity = resolved.raw_entity @@ -111,7 +120,7 @@ def link_by_paragraph(self, input_str: str, book_context_ref: Ref, *link_args, * named_entity.align_parts_to_new_doc(full_spacy_doc, raw_ref_offset) curr_token_count = len(self._ner.named_entity_model.make_doc(curr_input)) offset += curr_token_count+1 # +1 for newline token - return LinkedDoc(input_str, resolved_refs, resolved_named_entities) + return LinkedDoc(input_str, resolved_refs, resolved_named_entities, resolved_categories) def get_ner(self) -> NamedEntityRecognizer: return self._ner @@ -123,6 +132,23 @@ def reset_ibid_history(self) -> None: """ self._ref_resolver.reset_ibid_history() + def _bulk_resolve_refs_and_cats(self, raw_refs, book_context_ref, thoroughness, reset_ibids=True) -> (list[ResolvedRef], list[ResolvedCategory]): + """ + First match categories, then resolve refs for anything that didn't match a category + This prevents situations where a category is parsed as a ref using ibid (e.g. Talmud with context Berakhot 2a) + @param raw_refs: + @param book_context_ref: + @param thoroughness: + @param reset_ibids: + @return: + """ + possibly_resolved_cats = self._cat_resolver.bulk_resolve(raw_refs) + unresolved_raw_refs = [r.raw_entity for r in filter(lambda r: r.resolution_failed, possibly_resolved_cats)] + resolved_cats = list(filter(lambda r: not r.resolution_failed, possibly_resolved_cats)) + #try to resolve only unresolved categories (unresolved_raw_refs) as refs: + resolved_refs = self._ref_resolver.bulk_resolve(unresolved_raw_refs, book_context_ref, thoroughness, reset_ibids=reset_ibids) + return resolved_refs, resolved_cats + @staticmethod def _partition_raw_refs_and_named_entities(raw_refs_and_named_entities: List[RawNamedEntity]) \ -> Tuple[List[RawRef], List[RawNamedEntity]]: @@ -138,3 +164,10 @@ def _get_bulk_link_iterable(inputs: List[str], all_named_entities: List[List[Raw if verbose: iterable = tqdm(iterable, total=len(book_context_refs)) return iterable + + @staticmethod + def _remove_failures(*args): + out = [] + for arg in args: + out.append(list(filter(lambda x: not x.resolution_failed, arg))) + return out diff --git a/sefaria/model/linker/named_entity_resolver.py b/sefaria/model/linker/named_entity_resolver.py index 4e40b128cd..e3856b8f14 100644 --- a/sefaria/model/linker/named_entity_resolver.py +++ b/sefaria/model/linker/named_entity_resolver.py @@ -26,6 +26,10 @@ def topic(self): def is_ambiguous(self): return len(self.topics) != 1 + @property + def resolution_failed(self): + return len(self.topics) == 0 + class TitleGenerator: @@ -134,12 +138,11 @@ class NamedEntityResolver: def __init__(self, topic_matcher: TopicMatcher): self._topic_matcher = topic_matcher - def bulk_resolve(self, raw_named_entities: List[RawNamedEntity], with_failures=False) -> List[ResolvedNamedEntity]: + def bulk_resolve(self, raw_named_entities: List[RawNamedEntity]) -> List[ResolvedNamedEntity]: resolved = [] for named_entity in raw_named_entities: matched_topics = self._topic_matcher.match(named_entity) - if len(matched_topics) > 0 or with_failures: - resolved += [ResolvedNamedEntity(named_entity, matched_topics)] + resolved += [ResolvedNamedEntity(named_entity, matched_topics)] return resolved diff --git a/sefaria/model/linker/ref_resolver.py b/sefaria/model/linker/ref_resolver.py index 1ae7f54637..206b9e78f9 100644 --- a/sefaria/model/linker/ref_resolver.py +++ b/sefaria/model/linker/ref_resolver.py @@ -1,13 +1,12 @@ from collections import defaultdict from typing import List, Union, Dict, Optional, Tuple, Iterable, Set -from functools import reduce from enum import IntEnum, Enum from sefaria.system.exceptions import InputError from sefaria.model import abstract as abst from sefaria.model import text from sefaria.model import schema from sefaria.model.linker.ref_part import RawRef, RawRefPart, SpanOrToken, span_inds, RefPartType, SectionContext, ContextPart, TermContext -from sefaria.model.linker.referenceable_book_node import ReferenceableBookNode +from sefaria.model.linker.referenceable_book_node import ReferenceableBookNode, NamedReferenceableBookNode from sefaria.model.linker.match_template import MatchTemplateTrie, LEAF_TRIE_ENTRY from sefaria.model.linker.resolved_ref_refiner_factory import resolved_ref_refiner_factory import structlog @@ -108,6 +107,10 @@ def merge_parts(self, other: 'ResolvedRef') -> None: self.resolved_parts = [part] + self.resolved_parts else: self.resolved_parts += [part] + if not self.ref: + # self may reference an AltStructNode and therefore doesn't have a ref. + # Use ref from other which is expected to be equivalent or more specific + self.ref = other.ref def get_resolved_parts(self, include: Iterable[type] = None, exclude: Iterable[type] = None) -> List[RawRefPart]: """ @@ -138,6 +141,36 @@ def count_by_part_type(parts) -> Dict[RefPartType, int]: def get_node_children(self): return self.node.get_children(self.ref) + def contains(self, other: 'ResolvedRef') -> bool: + """ + Does `self` contain `other`. If `self.ref` and `other.ref` aren't None, this is just ref comparison. + Otherwise, see if the schema/altstruct node that back `self` contains `other`'s node. + Note this function is a bit confusing. It works like this: + - If `self.ref` and `other.ref` are None, we compare the nodes themselves to see if self is an ancestor of other + - If `self.ref` is None and `other.ref` isn't, we check that `other.ref` is contained in at least one of `self`'s children (`self` may be an AltStructNode in which case it has no Ref) + - If `self.ref` isn't None and `other_ref` is None, we check that `self.ref` contains all of `other`'s children (`other` may be an AltStructNode in which case it has no Ref) + - If `self.ref` and `other.ref` are both defined, we can use Ref.contains() + @param other: + @return: + """ + if not other.node or not self.node: + return False + if other.ref and self.ref: + return self.ref.contains(other.ref) + try: + if other.ref is None: + if self.ref is None: + return self.node.is_ancestor_of(other.node) + # other is alt struct and self has a ref + # check that every leaf node is contained by self's ref + return all([self.ref.contains(leaf_ref) for leaf_ref in other.node.leaf_refs()]) + # self is alt struct and other has a ref + # check if any leaf node contains other's ref + return any([leaf_ref.contains(other.ref) for leaf_ref in self.node.leaf_refs()]) + except NotImplementedError: + return False + + @property def order_key(self): """ @@ -152,6 +185,10 @@ def order_key(self): num_context_parts_matched = self.num_resolved(include={ContextPart}) return len(explicit_matched), num_context_parts_matched + @property + def resolution_failed(self) -> bool: + return self.ref is None and self.node is None + class AmbiguousResolvedRef: """ @@ -170,6 +207,10 @@ def pretty_text(self): # assumption is first resolved refs pretty_text is good enough return self.resolved_raw_refs[0].pretty_text + @property + def resolution_failed(self) -> bool: + return False + PossiblyAmbigResolvedRef = Union[ResolvedRef, AmbiguousResolvedRef] @@ -209,23 +250,12 @@ def match_terms(self, ref_parts: List[RawRefPart]) -> List[schema.NonUniqueTerm] class IbidHistory: - ignored_term_slugs = ['torah', 'talmud', 'gemara', 'mishnah', 'midrash'] - def __init__(self, last_n_titles: int = 3, last_n_refs: int = 3): self.last_n_titles = last_n_titles self.last_n_refs = last_n_refs self._last_refs: List[text.Ref] = [] self._last_titles: List[str] = [] self._title_ref_map: Dict[str, text.Ref] = {} - self._ignored_titles: Set[str] = self._get_ignored_titles() - - @classmethod - def _get_ignored_titles(cls) -> Set[str]: - terms = [schema.NonUniqueTerm.init(slug) for slug in cls.ignored_term_slugs] - return reduce(lambda a, b: a | set(b), [term.get_titles() for term in terms], set()) - - def should_ignore_text(self, text) -> bool: - return text in self._ignored_titles def _get_last_refs(self) -> List[text.Ref]: return self._last_refs @@ -266,12 +296,11 @@ def reset_ibid_history(self): self._ibid_history = IbidHistory() def bulk_resolve(self, raw_refs: List[RawRef], book_context_ref: Optional[text.Ref] = None, - with_failures=False, thoroughness=ResolutionThoroughness.NORMAL, reset_ibids=True) -> List[PossiblyAmbigResolvedRef]: + thoroughness=ResolutionThoroughness.NORMAL, reset_ibids=True) -> List[PossiblyAmbigResolvedRef]: """ Main function for resolving refs in text. Given a list of RawRefs, returns ResolvedRefs for each @param raw_refs: @param book_context_ref: - @param with_failures: @param thoroughness: how thorough should the search be. More thorough == slower. Currently "normal" will avoid searching for DH matches at book level and avoid filtering empty refs @param reset_ibids: If true, reset ibid history before resolving @return: @@ -281,25 +310,25 @@ def bulk_resolve(self, raw_refs: List[RawRef], book_context_ref: Optional[text.R self.reset_ibid_history() resolved = [] for raw_ref in raw_refs: - temp_resolved = self._resolve_raw_ref_and_update_ibid_history(raw_ref, book_context_ref, with_failures) + temp_resolved = self._resolve_raw_ref_and_update_ibid_history(raw_ref, book_context_ref) resolved += temp_resolved return resolved - def _resolve_raw_ref_and_update_ibid_history(self, raw_ref: RawRef, book_context_ref: text.Ref, with_failures=False) -> List[PossiblyAmbigResolvedRef]: + def _resolve_raw_ref_and_update_ibid_history(self, raw_ref: RawRef, book_context_ref: text.Ref) -> List[PossiblyAmbigResolvedRef]: temp_resolved = self.resolve_raw_ref(book_context_ref, raw_ref) self._update_ibid_history(raw_ref, temp_resolved) - if len(temp_resolved) == 0 and with_failures: + if len(temp_resolved) == 0: return [ResolvedRef(raw_ref, [], None, None, context_ref=book_context_ref)] return temp_resolved def _update_ibid_history(self, raw_ref: RawRef, temp_resolved: List[PossiblyAmbigResolvedRef]): - if self._ibid_history.should_ignore_text(raw_ref.text): - return if len(temp_resolved) == 0: self.reset_ibid_history() - elif any(r.is_ambiguous for r in temp_resolved): + elif any(r.is_ambiguous for r in temp_resolved) or temp_resolved[-1].ref is None: # can't be sure about future ibid inferences # TODO can probably salvage parts of history if matches are ambiguous within one book + # if ref is None, match is likely to AltStructNode + # TODO this node still has useful info. Try to salvage it. self.reset_ibid_history() else: self._ibid_history.last_refs = temp_resolved[-1].ref @@ -349,7 +378,7 @@ def resolve_raw_ref(self, book_context_ref: Optional[text.Ref], raw_ref: RawRef) unrefined_matches = self.get_unrefined_ref_part_matches(book_context_ref, temp_raw_ref) if is_non_cts: # filter unrefined matches to matches that resolved previously - resolved_titles = {r.ref.index.title for r in resolved_list} + resolved_titles = {r.ref.index.title for r in resolved_list if not r.is_ambiguous} unrefined_matches = list(filter(lambda x: x.ref.index.title in resolved_titles, unrefined_matches)) # resolution will start at context_ref.sections - len(ref parts). rough heuristic for match in unrefined_matches: @@ -480,7 +509,9 @@ def refine_ref_part_matches(self, book_context_ref: Optional[text.Ref], matches: # combine if len(context_full_matches) > 0: - context_free_matches = list(filter(lambda x: x.ref.normal() not in refs_matched, context_free_matches)) + # assumption is we don't want refs that used context at book level, then didn't get refined more when considering context free + # BUT did get refined more when considering context + context_free_matches = list(filter(lambda x: not (x.num_resolved(include={ContextPart}) > 0 and x.ref.normal() in refs_matched), context_free_matches)) temp_matches += context_free_matches + context_full_matches return ResolvedRefPruner.prune_refined_ref_part_matches(self._thoroughness, temp_matches) @@ -597,7 +628,7 @@ def __init__(self): def prune_unrefined_ref_part_matches(ref_part_matches: List[ResolvedRef]) -> List[ResolvedRef]: index_match_map = defaultdict(list) for match in ref_part_matches: - key = match.node.ref().normal() + key = match.node.unique_key() index_match_map[key] += [match] pruned_matches = [] for match_list in index_match_map.values(): @@ -689,17 +720,17 @@ def is_match_correct(match: ResolvedRef) -> bool: @staticmethod def remove_superfluous_matches(thoroughness: ResolutionThoroughness, resolved_refs: List[ResolvedRef]) -> List[ResolvedRef]: # make matches with refs that are essentially equivalent (i.e. refs cover same span) actually equivalent - resolved_refs.sort(key=lambda x: x.ref and x.ref.order_id()) + resolved_refs.sort(key=lambda x: x.ref.order_id() if x.ref else "ZZZ") for i, r in enumerate(resolved_refs[:-1]): next_r = resolved_refs[i+1] - if r.ref.contains(next_r.ref) and next_r.ref.contains(r.ref): + if r.contains(next_r) and next_r.contains(r): next_r.ref = r.ref # make unique resolved_refs = list({r.ref: r for r in resolved_refs}.values()) if thoroughness >= ResolutionThoroughness.HIGH or len(resolved_refs) > 1: # remove matches that have empty refs - resolved_refs = list(filter(lambda x: not x.ref.is_empty(), resolved_refs)) + resolved_refs = list(filter(lambda x: x.ref and not x.ref.is_empty(), resolved_refs)) return resolved_refs @staticmethod @@ -751,21 +782,31 @@ def _merge_subset_matches(resolved_refs: List[ResolvedRef]) -> List[ResolvedRef] Merge matches where one ref is contained in another ref E.g. if matchA.ref == Ref("Genesis 1") and matchB.ref == Ref("Genesis 1:1"), matchA will be deleted and its parts will be appended to matchB's parts """ - resolved_refs.sort(key=lambda x: "N/A" if x.ref is None else x.ref.order_id()) + def get_sort_key(resolved_ref: ResolvedRef) -> str: + if resolved_ref.ref is None: + if resolved_ref.node is None: + return "N/A" + elif hasattr(resolved_ref.node, "ref_order_id"): + return resolved_ref.node.ref_order_id() + else: + return "N/A" + else: + return resolved_ref.ref.order_id() + resolved_refs.sort(key=get_sort_key) merged_resolved_refs = [] next_merged = False for imatch, match in enumerate(resolved_refs[:-1]): next_match = resolved_refs[imatch+1] - if match.is_ambiguous or match.ref is None or next_match.ref is None or next_merged: + if match.is_ambiguous or match.node is None or next_match.node is None or next_merged: merged_resolved_refs += [match] next_merged = False continue - if match.ref.index.title != next_match.ref.index.title: + if match.ref and next_match.ref and match.ref.index.title != next_match.ref.index.title: # optimization, the easiest cases to check for merged_resolved_refs += [match] - elif match.ref.contains(next_match.ref): + elif match.contains(next_match): next_match.merge_parts(match) - elif next_match.ref.contains(match.ref): + elif next_match.contains(match): # unfortunately Ref.order_id() doesn't consistently put larger refs before smaller ones # e.g. Tosafot on Berakhot 2 precedes Tosafot on Berakhot Chapter 1... # check if next match actually contains this match diff --git a/sefaria/model/linker/referenceable_book_node.py b/sefaria/model/linker/referenceable_book_node.py index 70a05bf212..b9c6560cd6 100644 --- a/sefaria/model/linker/referenceable_book_node.py +++ b/sefaria/model/linker/referenceable_book_node.py @@ -85,28 +85,69 @@ def is_default(self) -> bool: def referenceable(self) -> bool: return True + def is_ancestor_of(self, other: 'ReferenceableBookNode') -> bool: + other_node = other._get_titled_tree_node() + self_node = self._get_titled_tree_node() + return self_node.is_ancestor_of(other_node) -class NamedReferenceableBookNode(ReferenceableBookNode): + def _get_titled_tree_node(self) -> schema.TitledTreeNode: + raise NotImplementedError - def __init__(self, titled_tree_node_or_index: Union[schema.TitledTreeNode, text.Index]): - self._titled_tree_node_or_index = titled_tree_node_or_index - self._titled_tree_node = titled_tree_node_or_index - if isinstance(titled_tree_node_or_index, text.Index): - self._titled_tree_node = titled_tree_node_or_index.nodes + def leaf_refs(self) -> list[text.Ref]: + """ + Get the Refs for the ReferenceableBookNode leaf nodes from `self` + @return: + """ + raise NotImplementedError + + +class IndexNodeReferenceableBookNode(ReferenceableBookNode): + """ + ReferenceableBookNode backed by node in an Index (either SchemaNode or AltStructNode) + """ + + def __init__(self, titled_tree_node: schema.TitledTreeNode): + self._titled_tree_node = titled_tree_node @property def referenceable(self): return getattr(self._titled_tree_node, 'referenceable', not self.is_default()) - def is_default(self): - return self._titled_tree_node.is_default() + def _get_titled_tree_node(self) -> schema.TitledTreeNode: + return self._titled_tree_node - def get_numeric_equivalent(self): - return getattr(self._titled_tree_node, "numeric_equivalent", None) + def is_default(self): + return self._titled_tree_node.is_default() and self._titled_tree_node.parent is not None def ref(self) -> text.Ref: return self._titled_tree_node.ref() + def unique_key(self) -> str: + return self.ref().normal() + + def ref_order_id(self) -> str: + if isinstance(self._titled_tree_node, schema.AltStructNode): + leaves = self._titled_tree_node.get_leaf_nodes() + # assume leaves are contiguous. If this is wrong, will be disproven later in the function + if len(leaves) == 0: + return "N/A" + approx_ref = leaves[0].ref().to(leaves[-1].ref()) + return approx_ref.order_id() + return self.ref().order_id() + + +class NamedReferenceableBookNode(IndexNodeReferenceableBookNode): + + def __init__(self, titled_tree_node_or_index: Union[schema.TitledTreeNode, text.Index]): + self._titled_tree_node_or_index = titled_tree_node_or_index + titled_tree_node = titled_tree_node_or_index + if isinstance(titled_tree_node_or_index, text.Index): + titled_tree_node = titled_tree_node_or_index.nodes + super().__init__(titled_tree_node) + + def get_numeric_equivalent(self): + return getattr(self._titled_tree_node, "numeric_equivalent", None) + @staticmethod def _is_array_map_referenceable(node: schema.ArrayMapNode) -> bool: if not getattr(node, "isMapReferenceable", True): @@ -168,21 +209,22 @@ def get_children(self, *args, **kwargs) -> List[ReferenceableBookNode]: def ref_part_title_trie(self, *args, **kwargs): return self._titled_tree_node.get_match_template_trie(*args, **kwargs) + def leaf_refs(self) -> list[text.Ref]: + return [n.ref() for n in self._get_titled_tree_node().get_leaf_nodes()] + -class NumberedReferenceableBookNode(ReferenceableBookNode): +class NumberedReferenceableBookNode(IndexNodeReferenceableBookNode): def __init__(self, ja_node: schema.NumberedTitledTreeNode): - self._ja_node = ja_node + super().__init__(ja_node) + self._ja_node: schema.NumberedTitledTreeNode = ja_node @property def referenceable(self): return getattr(self._ja_node, 'referenceable', True) - def is_default(self): - return self._ja_node.is_default() and self._ja_node.parent is not None - - def ref(self): - return self._ja_node.ref() + def leaf_refs(self) -> list[text.Ref]: + return [self.ref()] def possible_subrefs(self, lang: str, initial_ref: text.Ref, section_str: str, fromSections=None) -> Tuple[List[text.Ref], List[bool]]: try: @@ -229,6 +271,7 @@ def _get_serialized_node(self) -> dict: serial, next_referenceable_depth = insert_amud_node_values(serial) serial['depth'] -= next_referenceable_depth serial['default'] = False # any JA node that has been modified should lose 'default' flag + serial['parent'] = self._ja_node if serial['depth'] == 0: raise ValueError("Can't serialize JaggedArray of depth 0") serial = truncate_serialized_node_to_depth(serial, next_referenceable_depth) @@ -324,7 +367,10 @@ def __get_section_with_offset(self, i: int, node: schema.ArrayMapNode) -> int: return section def ref(self): - return self._ref + raise NotImplementedError(f'{self.__class__} does not have a single ref.') + + def leaf_refs(self) -> list[text.Ref]: + return list(self._section_ref_map.values()) def possible_subrefs(self, lang: str, initial_ref: text.Ref, section_str: str, fromSections=None) -> Tuple[List[text.Ref], List[bool]]: try: diff --git a/sefaria/model/linker/resolved_ref_refiner.py b/sefaria/model/linker/resolved_ref_refiner.py index f71d40cb27..39f77c3984 100644 --- a/sefaria/model/linker/resolved_ref_refiner.py +++ b/sefaria/model/linker/resolved_ref_refiner.py @@ -76,7 +76,7 @@ def __refine_context_full(self) -> List['ResolvedRef']: return [self._clone_resolved_ref(resolved_parts=self._get_resolved_parts(), node=self.node, ref=refined_ref)] def __refine_context_free(self, lang: str, fromSections=None) -> List['ResolvedRef']: - if self.node is None: + if self.node is None or not isinstance(self.node, NumberedReferenceableBookNode): return [] possible_subrefs, can_match_out_of_order_list = self.node.possible_subrefs(lang, self.resolved_ref.ref, self.part_to_match.text, fromSections) refined_refs = [] diff --git a/sefaria/model/linker/tests/category_matcher_test.py b/sefaria/model/linker/tests/category_matcher_test.py new file mode 100644 index 0000000000..a88db7408c --- /dev/null +++ b/sefaria/model/linker/tests/category_matcher_test.py @@ -0,0 +1,58 @@ +import pytest +from unittest.mock import Mock +from sefaria.model.category import Category +from sefaria.model.linker.category_resolver import CategoryMatcher + + +def make_title(text, lang): + return {"text": text, "lang": lang} + + +@pytest.fixture +def mock_category(): + return Category({ + "titles": [make_title("Title1", "en"), make_title("Title2", "en")] + }) + + +@pytest.fixture +def mock_category_2(): + return Category({ + "titles": [make_title("Title1", "en"), make_title("Title4", "en")] + }) + + +@pytest.fixture +def mock_raw_ref(): + raw_ref = Mock() + raw_ref.text = "Title2" + return raw_ref + + +@pytest.fixture +def category_matcher(mock_category, mock_category_2): + return CategoryMatcher(lang="en", category_registry=[mock_category, mock_category_2]) + + +def test_match_single_title(category_matcher, mock_raw_ref, mock_category): + # Test matching for a valid title in mock_raw_ref + matched_categories = category_matcher.match(mock_raw_ref) + assert len(matched_categories) == 1 + assert mock_category in matched_categories + + +def test_match_no_match(category_matcher, mock_raw_ref): + # Test case where the raw_ref title does not match any category + mock_raw_ref.text = "NonexistentTitle" + matched_categories = category_matcher.match(mock_raw_ref) + assert matched_categories == [] + + +def test_match_multiple_titles(category_matcher, mock_raw_ref, mock_category, mock_category_2): + # Test case where multiple categories match the same title + mock_raw_ref.text = "Title1" + + matched_categories = category_matcher.match(mock_raw_ref) + assert len(matched_categories) == 2 + assert mock_category in matched_categories + assert mock_category_2 in matched_categories diff --git a/sefaria/model/linker/tests/linker_test.py b/sefaria/model/linker/tests/linker_test.py index 60f8f2f9e2..6dd6a224a9 100644 --- a/sefaria/model/linker/tests/linker_test.py +++ b/sefaria/model/linker/tests/linker_test.py @@ -47,6 +47,14 @@ def test_resolved_raw_ref_clone(): [crrd(['@ספר בראשית', '#פסוק א', '#פרק יג']), ("Genesis 13:1",)], # sections out of order [crrd(['@שמות', '#א', '#ב']), ("Exodus 1:2",)], # used to also match Exodus 2:1 b/c would allow mixing integer parts + # Roman numerals + [crrd(['@Job', '#III', '#5'], lang='en'), ("Job 3:5",)], + [crrd(['@Job', '#ix', '#5'], lang='en'), ("Job 9:5",)], + [crrd(['@Job', '#IV .', '#5'], lang='en'), ("Job 4:5",)], + [crrd(['@Job', '#xli.', '#5'], lang='en'), ("Job 41:5",)], + [crrd(['@Job', '#CIV', '#5'], lang='en'), tuple()], # too high + [crrd(['@Job', '#iiii', '#5'], lang='en'), tuple()], # invalid roman numeral + # Amud split into two parts [crrd(['@בבלי', '@יבמות', '#סא', '#א']), ("Yevamot 61a",)], [crrd(["@תוספות", "@פסחים", "#קו", "#א"]), ("Tosafot on Pesachim 106a",)], # amud for commentary that has DH @@ -141,6 +149,7 @@ def test_resolved_raw_ref_clone(): [crrd(['#פרק ז'], prev_trefs=["II Kings 17:31"]), ["II Kings 7"]], [crrd(['@ערוך השולחן', '#תצג'], prev_trefs=["Arukh HaShulchan, Orach Chaim 400"]), ["Arukh HaShulchan, Orach Chaim 493"]], # ibid named part that's not root [crrd(['@רש"י', '&שם'], prev_trefs=["Genesis 25:9", "Rashi on Genesis 21:20"]), ["Rashi on Genesis 21:20", "Rashi on Genesis 25:9"]], # ambiguous ibid + [crrd(["@Job"], lang='en', prev_trefs=['Job 1:1']), ("Job",)], # don't use ibid context if there's a match that uses all input # Relative (e.g. Lekaman) [crrd(["@תוס'", "<לקמן", "#ד ע\"ב", "*ד\"ה דאר\"י"], "Gilyon HaShas on Berakhot 2a:2"), ("Tosafot on Berakhot 4b:6:1",)], # likaman + abbrev in DH @@ -198,7 +207,7 @@ def test_resolved_raw_ref_clone(): [crrd(['@זהר חדש', '@בראשית']), ['Zohar Chadash, Bereshit']], [crrd(['@מסכת', '@סופרים', '#ב', '#ג']), ['Tractate Soferim 2:3']], - [crrd(['@אדר"נ', '#ב', '#ג']), ["Avot D'Rabbi Natan 2:3"]], + [crrd(['@אדר"נ', '#ב', '#ג']), ["Avot DeRabbi Natan 2:3"]], [crrd(['@פרק השלום', '#ג']), ["Tractate Derekh Eretz Zuta, Section on Peace 3"]], [crrd(['@ד"א זוטא', '@פרק השלום', '#ג']), ["Tractate Derekh Eretz Zuta, Section on Peace 3"]], [crrd(['@ספר החינוך', '@לך לך', '#ב']), ['Sefer HaChinukh 2']], @@ -292,24 +301,28 @@ class TestResolveRawRef: pass + @pytest.mark.parametrize(('context_tref', 'input_str', 'lang', 'expected_trefs', 'expected_pretty_texts'), [ + ["Berakhot 2a", 'It says in the Talmud, "Don\'t steal" which implies it\'s bad to steal.', 'en', tuple(), tuple()], # Don't match Talmud using Berakhot 2a as ibid context + [None, 'It says in the Torah, "Don\'t steal" which implies it\'s bad to steal.', 'en', tuple(), tuple()], [None, """גמ' שמזונותן עליך. עיין ביצה (דף טו ע"ב רש"י ד"ה שמא יפשע:)""", 'he', ("Rashi on Beitzah 15b:8:1",), ['ביצה (דף טו ע"ב רש"י ד"ה שמא יפשע:)']], [None, """שם אלא ביתך ל"ל. ע' מנחות מד ע"א תד"ה טלית:""", 'he', ("Tosafot on Menachot 44a:12:1",), ['מנחות מד ע"א תד"ה טלית']], [None, """גמ' במה מחנכין. עי' מנחות דף עח ע"א תוס' ד"ה אחת:""", 'he',("Tosafot on Menachot 78a:10:1",), ['''מנחות דף עח ע"א תוס' ד"ה אחת''']], - [None, """cf. Ex. 9:6,5""", 'en', ("Exodus 9:6", "Exodus 9:5"), ['Ex. 9:6', '5']], + [None, """cf. Ex. 9:6,12:8""", 'en', ("Exodus 9:6", "Exodus 12:8"), ['Ex. 9:6', '12:8']], ["Gilyon HaShas on Berakhot 25b:1", 'רש"י תמורה כח ע"ב ד"ה נעבד שהוא מותר. זה רש"י מאוד יפה.', 'he', ("Rashi on Temurah 28b:4:2",), ['רש"י תמורה כח ע"ב ד"ה נעבד שהוא מותר']], + [None, "See Genesis 1:1. It says in the Torah, \"Don't steal\". It also says in 1:3 \"Let there be light\".", "en", ("Genesis 1:1", "Genesis 1:3"), ("Genesis 1:1", "1:3")], ]) def test_full_pipeline_ref_resolver(context_tref, input_str, lang, expected_trefs, expected_pretty_texts): context_oref = context_tref and Ref(context_tref) linker = library.get_linker(lang) doc = linker.link(input_str, context_oref, type_filter='citation') resolved = doc.resolved_refs - assert len(resolved) == len(expected_trefs) resolved_orefs = sorted(reduce(lambda a, b: a + b, [[match.ref] if not match.is_ambiguous else [inner_match.ref for inner_match in match.resolved_raw_refs] for match in resolved], []), key=lambda x: x.normal()) if len(expected_trefs) != len(resolved_orefs): print(f"Found {len(resolved_orefs)} refs instead of {len(expected_trefs)}") for matched_oref in resolved_orefs: print("-", matched_oref.normal()) + assert len(resolved) == len(expected_trefs) for expected_tref, matched_oref in zip(sorted(expected_trefs, key=lambda x: x), resolved_orefs): assert matched_oref == Ref(expected_tref) for match, expected_pretty_text in zip(resolved, expected_pretty_texts): diff --git a/sefaria/model/linker/tests/named_entity_resolver_tests.py b/sefaria/model/linker/tests/named_entity_resolver_tests.py index c207c7f8f4..4d40df088b 100644 --- a/sefaria/model/linker/tests/named_entity_resolver_tests.py +++ b/sefaria/model/linker/tests/named_entity_resolver_tests.py @@ -1,5 +1,5 @@ import pytest -from sefaria.model.linker.named_entity_resolver import NamedEntityTitleGenerator, PersonTitleGenerator +from sefaria.model.linker.named_entity_resolver import PersonTitleGenerator @pytest.mark.parametrize(('title', 'expected_output'), [ diff --git a/sefaria/model/linker/tests/resolved_category_test.py b/sefaria/model/linker/tests/resolved_category_test.py new file mode 100644 index 0000000000..a832b74a2d --- /dev/null +++ b/sefaria/model/linker/tests/resolved_category_test.py @@ -0,0 +1,37 @@ +import pytest +from unittest.mock import Mock +from sefaria.model.linker.category_resolver import ResolvedCategory + + +@pytest.fixture +def mock_raw_ref(): + return Mock() + + +@pytest.fixture +def mock_category(): + return Mock() + + +def test_is_ambiguous_with_single_category(mock_raw_ref, mock_category): + # Test when there is exactly one category + resolved_category = ResolvedCategory(mock_raw_ref, [mock_category]) + assert not resolved_category.is_ambiguous + + +def test_is_ambiguous_with_multiple_categories(mock_raw_ref, mock_category): + # Test when there are multiple categories + resolved_category = ResolvedCategory(mock_raw_ref, [mock_category, mock_category]) + assert resolved_category.is_ambiguous + + +def test_resolution_failed_with_no_categories(mock_raw_ref): + # Test when there are no categories + resolved_category = ResolvedCategory(mock_raw_ref, []) + assert resolved_category.resolution_failed + + +def test_resolution_not_failed_with_categories(mock_raw_ref, mock_category): + # Test when there are one or more categories + resolved_category = ResolvedCategory(mock_raw_ref, [mock_category]) + assert not resolved_category.resolution_failed diff --git a/sefaria/model/linker/tests/resolved_ref.py b/sefaria/model/linker/tests/resolved_ref.py new file mode 100644 index 0000000000..6b7f531fa3 --- /dev/null +++ b/sefaria/model/linker/tests/resolved_ref.py @@ -0,0 +1,54 @@ +import pytest +from unittest.mock import Mock +from sefaria.model.linker.referenceable_book_node import ReferenceableBookNode, NumberedReferenceableBookNode, NamedReferenceableBookNode, MapReferenceableBookNode +from sefaria.model.linker.ref_resolver import ResolvedRef +from sefaria.model.text import Ref + + +def make_num_node(ref: Ref, depth=0) -> ReferenceableBookNode: + ja_node = ref.index_node + if ja_node.has_default_child(): + ja_node = ja_node.get_default_child() + node = NumberedReferenceableBookNode(ja_node) + for _ in range(depth): + node = node.get_children()[0] + return node + + +def make_named_node(title: str, node_path: list[str], is_alt_struct_path: bool): + index = Ref(title).index + if is_alt_struct_path: + # first element in path is alt struct name + node = index.get_alt_structure(node_path[0]) + node_path = node_path[1:] + else: + node = index.nodes + while len(node_path) > 0: + node = next((child for child in node.children if child.get_primary_title('en') == node_path[0]), None) + if node is None: + raise ValueError(f'Could not find node with title "{node_path[0]}".') + node_path = node_path[1:] + return NamedReferenceableBookNode(node) + + +zohar_volume1_intro_node = MapReferenceableBookNode(Ref('Zohar').index.get_alt_structure("Daf").children[0].children[0]) +zohar_first_daf_node = zohar_volume1_intro_node.get_children()[0] + + +@pytest.mark.parametrize(('node_a', 'node_b', 'self_tref', 'other_tref', 'is_contained'), [ + [make_num_node(Ref('Genesis')), make_num_node(Ref('Genesis'), 1), None, None, True], # Generic pasuk node is contained in generic perek node + [make_num_node(Ref('Genesis')), make_num_node(Ref('Genesis'), 1), "Genesis 1", "Genesis 1:2", True], # Specific pasuk ref is contained in specific perek ref + [make_num_node(Ref('Genesis')), make_num_node(Ref('Genesis'), 1), "Genesis 2", "Genesis 1:2", False], # case where specific pasuk isn't contained in specific perek + [make_named_node('Sefer HaChinukh', ['Parasha', 'Lech Lecha'], True), make_num_node(Ref("Sefer HaChinukh")), None, "Sefer HaChinukh, 2", True], # ref is contained in alt struct node (which has no ref) + [make_num_node(Ref("Sefer HaChinukh")), make_named_node('Sefer HaChinukh', ['Parasha', 'Lech Lecha'], True), "Sefer HaChinukh, 2", None, True], # alt struct node with only one ref is contained in that ref + [make_num_node(Ref("Sefer HaChinukh")), make_named_node('Sefer HaChinukh', ['Parasha', 'Bo'], True), "Sefer HaChinukh, 4", None, False], # alt struct node with multiple refs is not contained in a single one of refs + [make_named_node('Sefer HaChinukh', ['Parasha', 'Lech Lecha'], True), make_num_node(Ref("Sefer HaChinukh")), None, "Sefer HaChinukh, 3", False], # ref outside of alt struct node isn't contained in it + [zohar_volume1_intro_node, zohar_first_daf_node, None, 'Zohar, Volume I, Introduction 1b', True], # zohar altStruct ref + [zohar_first_daf_node, zohar_volume1_intro_node, 'Zohar, Volume I, Introduction 1b', None, False], # zohar altStruct ref +]) +def test_contains(node_a: ReferenceableBookNode, node_b: ReferenceableBookNode, self_tref: str, other_tref: str, is_contained: bool): + self_oref = self_tref and Ref(self_tref) + other_oref = other_tref and Ref(other_tref) + rr_a = ResolvedRef(Mock(), Mock(), node_a, self_oref) + rr_b = ResolvedRef(Mock(), Mock(), node_b, other_oref) + assert rr_a.contains(rr_b) == is_contained diff --git a/sefaria/model/schema.py b/sefaria/model/schema.py index ca075ab46c..12b9af2b60 100644 --- a/sefaria/model/schema.py +++ b/sefaria/model/schema.py @@ -14,6 +14,7 @@ from . import abstract as abst from sefaria.system.database import db from sefaria.model.lexicon import LexiconEntrySet +from sefaria.model.linker.has_match_template import HasMatchTemplates from sefaria.system.exceptions import InputError, IndexSchemaError, DictionaryEntryNotFoundError, SheetNotFoundError from sefaria.utils.hebrew import decode_hebrew_numeral, encode_small_hebrew_numeral, encode_hebrew_numeral, encode_hebrew_daf, hebrew_term, sanitize from sefaria.utils.talmud import daf_to_section @@ -671,7 +672,7 @@ def get_child_order(self, child): return self.all_children().index(child) + 1 -class TitledTreeNode(TreeNode, AbstractTitledOrTermedObject): +class TitledTreeNode(TreeNode, AbstractTitledOrTermedObject, HasMatchTemplates): """ A tree node that has a collection of titles - as contained in a TitleGroup instance. In this class, node titles, terms, 'default', and combined titles are handled. @@ -680,7 +681,6 @@ class TitledTreeNode(TreeNode, AbstractTitledOrTermedObject): after_title_delimiter_re = r"(?:[,.:\s]|(?:to|\u05d5?\u05d1?(?:\u05e1\u05d5\u05e3|\u05e8\u05d9\u05e9)))+" # should be an arg? \r\n are for html matches after_address_delimiter_ref = r"[,.:\s]+" title_separators = [", "] - MATCH_TEMPLATE_ALONE_SCOPES = {'any', 'alone'} def __init__(self, serial=None, **kwargs): super(TitledTreeNode, self).__init__(serial, **kwargs) @@ -840,10 +840,6 @@ def add_title(self, text, lang, primary=False, replace_primary=False, presentati """ return self.title_group.add_title(text, lang, primary, replace_primary, presentation) - def get_match_template_trie(self, lang: str): - from .linker.match_template import MatchTemplateTrie - return MatchTemplateTrie(lang, nodes=[self], scope='combined') - def validate(self): super(TitledTreeNode, self).validate() @@ -896,17 +892,6 @@ def serialize(self, **kwargs): d["heTitle"] = self.title_group.primary_title("he") return d - def get_match_templates(self): - from .linker.match_template import MatchTemplate - for raw_match_template in getattr(self, 'match_templates', []): - yield MatchTemplate(**raw_match_template) - - def has_scope_alone_match_template(self): - """ - @return: True if `self` has any match template that has scope = "alone" OR scope = "any" - """ - return any(template.scope in self.MATCH_TEMPLATE_ALONE_SCOPES for template in self.get_match_templates()) - def get_referenceable_alone_nodes(self): """ Currently almost exact copy of function with same name in Index @@ -1489,7 +1474,11 @@ def all_children(self): return self.traverse_to_list(lambda n, i: list(n.all_children()) if n.is_virtual else [n])[1:] def __eq__(self, other): - return self.address() == other.address() + try: + return self.address() == other.address() + except AttributeError: + # in case `other` isn't a SchemaNode + return False def __ne__(self, other): return not self.__eq__(other) @@ -2093,7 +2082,10 @@ def is_special_case(self, s): def to_numeric_possibilities(self, lang, s, **kwargs): if s in self.special_cases: return self.special_cases[s] - return [self.toNumber(lang, s)] + try: + return [self.toNumber(lang, s)] + except ValueError: + return [] @classmethod def can_match_out_of_order(cls, lang, s): @@ -2137,7 +2129,7 @@ def get_all_possible_sections_from_string(cls, lang, s, fromSections=None, strip section_str = curr_s else: strict = SuperClass not in {AddressAmud, AddressTalmud} # HACK: AddressTalmud doesn't inherit from AddressInteger so it relies on flexibility of not matching "Daf" - regex_str = addr.regex(lang, strict=strict, group_id='section') + "$" # must match entire string + regex_str = addr.regex(lang, strict=strict, group_id='section', with_roman_numerals=True) + "$" # must match entire string if regex_str is None: continue reg = regex.compile(regex_str, regex.VERBOSE) match = reg.match(curr_s) @@ -2580,7 +2572,11 @@ def _core_regex(self, lang, group_id=None, **kwargs): reg = r"(" if lang == "en": - reg += r"\d+)" + if kwargs.get('with_roman_numerals', False): + # any char valid in roman numerals (I, V, X, L, C, D, M) + optional trailing period + reg += r"(?:\d+|[ivxlcdmIVXLCDM]+(?:\s?\.)?))" + else: + reg += r"\d+)" elif lang == "he": reg += self.hebrew_number_regex() + r")" @@ -2592,6 +2588,19 @@ def toNumber(self, lang, s, **kwargs): elif lang == "he": return decode_hebrew_numeral(s) + def to_numeric_possibilities(self, lang, s, **kwargs): + import roman + from roman import InvalidRomanNumeralError + + possibilities = super().to_numeric_possibilities(lang, s, **kwargs) + if lang == "en": + try: + s = re.sub(r"\.$", "", s).strip() # remove trailing period + possibilities.append(roman.fromRoman(s.upper())) + except InvalidRomanNumeralError as e: + pass + return possibilities + @classmethod def can_match_out_of_order(cls, lang, s): """ @@ -2657,7 +2666,7 @@ class AddressPasuk(AddressInteger): "en": r"""(?:(?:([Vv](erses?|[vs]?\.)|[Pp]ass?u[kq]))?\s*)""", # the internal ? is a hack to allow a non match, even if 'strict' "he": r"""(?:\u05d1?(?: # optional ב in front (?:\u05e4\u05b8?\u05bc?\u05e1\u05d5\u05bc?\u05e7(?:\u05d9\u05dd)?\s*)| #pasuk spelled out, with a space after - (?:\u05e4\u05e1(?:['\u2018\u2019\u05f3])\s+) + (?:\u05e4\u05e1['\u2018\u2019\u05f3]?\s+) ))""" } diff --git a/sefaria/model/text.py b/sefaria/model/text.py index b8b254a74d..6e088c9589 100644 --- a/sefaria/model/text.py +++ b/sefaria/model/text.py @@ -25,7 +25,7 @@ import sefaria.system.cache as scache from sefaria.system.cache import in_memory_cache from sefaria.system.exceptions import InputError, BookNameError, PartialRefInputError, IndexSchemaError, \ - NoVersionFoundError, DictionaryEntryNotFoundError, MissingKeyError + NoVersionFoundError, DictionaryEntryNotFoundError, MissingKeyError, ComplexBookLevelRefError from sefaria.utils.hebrew import has_hebrew, is_all_hebrew, hebrew_term from sefaria.utils.util import list_depth, truncate_string from sefaria.datatype.jagged_array import JaggedTextArray, JaggedArray @@ -259,7 +259,7 @@ def annotate_schema_with_content_counts(self, schema): Returns the `schema` dictionary with each node annotated with section lengths info from version_state. """ - vstate = self.versionState() + vstate = self.versionState() def simplify_version_state(vstate_node): return aggregate_available_texts(vstate_node["_all"]["availableTexts"]) @@ -1412,7 +1412,7 @@ def _sanitize(self): pass def get_index(self): - return library.get_index(self.title) + return Index().load({'title': self.title}) def first_section_ref(self): """ @@ -1700,7 +1700,7 @@ def __init__(self, oref, lang, vtitle, merge_versions=False, versions=None): elif oref.has_default_child(): #use default child: self.oref = oref.default_child_ref() else: - raise InputError("Can not get TextRange at this level, please provide a more precise reference") + raise ComplexBookLevelRefError(book_ref=oref.normal()) self.lang = lang self.vtitle = vtitle self.merge_versions = merge_versions @@ -2434,7 +2434,7 @@ def __init__(self, oref, context=1, commentary=True, version=None, lang=None, self._alts = [] if not isinstance(oref.index_node, JaggedArrayNode) and not oref.index_node.is_virtual: - raise InputError("Can not get TextFamily at this level, please provide a more precise reference") + raise InputError("Unable to find text for that ref") for i in range(0, context): oref = oref.context_ref() @@ -4000,8 +4000,10 @@ def padded_ref(self): except AttributeError: # This is a schema node, try to get a default child if self.has_default_child(): return self.default_child_ref().padded_ref() + elif self.is_book_level(): + raise ComplexBookLevelRefError(book_ref=self.normal()) else: - raise InputError("Can not pad a schema node ref") + raise InputError("Cannot pad a schema node ref.") d = self._core_dict() if self.is_talmud(): @@ -5733,7 +5735,8 @@ def build_linker(self, lang: str): named_entity_resolver = self._build_named_entity_resolver(lang) ref_resolver = self._build_ref_resolver(lang) named_entity_recognizer = self._build_named_entity_recognizer(lang) - self._linker_by_lang[lang] = Linker(named_entity_recognizer, ref_resolver, named_entity_resolver) + cat_resolver = self._build_category_resolver(lang) + self._linker_by_lang[lang] = Linker(named_entity_recognizer, ref_resolver, named_entity_resolver, cat_resolver) return self._linker_by_lang[lang] @staticmethod @@ -5757,6 +5760,12 @@ def _build_named_entity_recognizer(lang: str): load_spacy_model(RAW_REF_PART_MODEL_BY_LANG_FILEPATH[lang]) ) + def _build_category_resolver(self, lang: str): + from sefaria.model.category import CategorySet, Category + from .linker.category_resolver import CategoryResolver, CategoryMatcher + categories: list[Category] = CategorySet({"match_templates": {"$exists": True}}).array() + return CategoryResolver(CategoryMatcher(lang, categories)) + def _build_ref_resolver(self, lang: str): from .linker.match_template import MatchTemplateTrie from .linker.ref_resolver import RefResolver, TermMatcher diff --git a/sefaria/search.py b/sefaria/search.py index 0cf7baa37b..020963f82a 100644 --- a/sefaria/search.py +++ b/sefaria/search.py @@ -106,6 +106,14 @@ def unicode_number(u): n += ord(u[i]) return n +def make_sheet_topics(sheet): + topics = [] + for t in sheet.get('topics', []): + topic_obj = Topic.init(t['slug']) + if not topic_obj: + continue + topics += [topic_obj] + return topics def index_sheet(index_name, id): """ @@ -116,14 +124,7 @@ def index_sheet(index_name, id): if not sheet: return False pud = public_user_data(sheet["owner"]) - tag_terms_simple = make_sheet_tags(sheet) - tags = [t["en"] for t in tag_terms_simple] - topics = [] - for t in sheet.get('topics', []): - topic_obj = Topic.init(t['slug']) - if not topic_obj: - continue - topics += [topic_obj] + topics = make_sheet_topics(sheet) collections = CollectionSet({"sheets": id, "listed": True}) collection_names = [c.name for c in collections] try: @@ -135,7 +136,6 @@ def index_sheet(index_name, id): "owner_image": pud["imageUrl"], "profile_url": pud["profileUrl"], "version": "Source Sheet by " + user_link(sheet["owner"]), - "tags": tags, "topic_slugs": [topic_obj.slug for topic_obj in topics], "topics_en": [topic_obj.get_primary_title('en') for topic_obj in topics], "topics_he": [topic_obj.get_primary_title('he') for topic_obj in topics], @@ -156,26 +156,6 @@ def index_sheet(index_name, id): print(e) return False - -def make_sheet_tags(sheet): - def get_primary_title(lang, titles): - return [t for t in titles if t.get("primary") and t.get("lang", "") == lang][0]["text"] - - tags = sheet.get('tags', []) - tag_terms = [(Term().load({'name': t}) or Term().load_by_title(t)) for t in tags] - tag_terms_simple = [ - { - 'en': tags[iterm], # save as en even if it's Hebrew - 'he': '' - } if term is None else - { - 'en': get_primary_title('en', term.titles), - 'he': get_primary_title('he', term.titles) - } for iterm, term in enumerate(tag_terms) - ] - #tags_en, tags_he = zip(*tag_terms_simple.values()) - return tag_terms_simple - def make_sheet_text(sheet, pud): """ Returns a plain text representation of the content of sheet. @@ -186,8 +166,11 @@ def make_sheet_text(sheet, pud): if pud.get("name"): text += "\nBy: " + pud["name"] text += "\n" - if sheet.get("tags"): - text += " [" + ", ".join(sheet["tags"]) + "]\n" + if sheet.get("topics"): + topics = make_sheet_topics(sheet) + topics_en = [topic_obj.get_primary_title('en') for topic_obj in topics] + topics_he = [topic_obj.get_primary_title('he') for topic_obj in topics] + text += " [" + ", ".join(topics_en+topics_he) + "]\n" for s in sheet["sources"]: text += source_text(s) + " " @@ -521,7 +504,9 @@ def index_all(cls, index_name, debug=False, for_es=True, action=None): total_versions = len(versions) versions = None # release RAM for title, vlist in list(versions_by_index.items()): - cls.curr_index = vlist[0].get_index() if len(vlist) > 0 else None + if len(vlist) == 0: + continue + cls.curr_index = vlist[0].get_index() if for_es: cls._bulk_actions = [] try: diff --git a/sefaria/sefaria_tasks_interace/history_change.py b/sefaria/sefaria_tasks_interace/history_change.py new file mode 100644 index 0000000000..3c91df8dcf --- /dev/null +++ b/sefaria/sefaria_tasks_interace/history_change.py @@ -0,0 +1,17 @@ +from dataclasses import dataclass + +@dataclass +class AbstractHistoryChange: + uid: int + method: str # ("API" or "Site") + +@dataclass +class LinkChange(AbstractHistoryChange): + raw_link: dict + +@dataclass +class VersionChange(AbstractHistoryChange): + raw_version: dict + patch: bool + skip_links: bool + count_after: int diff --git a/sefaria/system/exceptions.py b/sefaria/system/exceptions.py index 72d3493f78..de010b46da 100644 --- a/sefaria/system/exceptions.py +++ b/sefaria/system/exceptions.py @@ -99,3 +99,14 @@ def __init__(self, method): self.method = method self.message = f"'{method}' is not a valid HTTP API method." super().__init__(self.message) + + +class ComplexBookLevelRefError(InputError): + def __init__(self, book_ref): + self.book_ref = book_ref + self.message = (f"You passed '{book_ref}', please pass a more specific ref for this book, and try again. " + f"'{book_ref}' is a \'complex\' book-level ref. We only support book-level " + f"refs in cases of texts with a 'simple' structure. To learn more about the " + f"structure of a text on Sefaria, " + f"see: https://developers.sefaria.org/docs/the-schema-of-a-simple-text") + super().__init__(self.message) diff --git a/sefaria/tracker.py b/sefaria/tracker.py index 7e68968ea7..e5cd32daf3 100644 --- a/sefaria/tracker.py +++ b/sefaria/tracker.py @@ -3,6 +3,8 @@ Accepts change requests for model objects, passes the changes to the models, and records the changes in history """ +from functools import reduce + import structlog logger = structlog.get_logger(__name__) @@ -91,6 +93,53 @@ def populate_change_map(old_text, en_tref, he_tref, _): return error_map +def modify_version(user: int, version_dict: dict, patch=True, **kwargs): + + def modify_node(jagged_array_node): + address = jagged_array_node.address()[1:] # first element is the index name + new_content = reduce(lambda x, y: x.get(y, {}), address, version_dict['chapter']) + if is_version_new: + old_content = [] + else: + old_content = reduce(lambda x, y: x.setdefault(y, {}), address, version.chapter) or [] + if (patch and new_content == {}) or old_content == new_content: + return + new_content = new_content or [] + if not is_version_new: + if address: + reduce(lambda x, y: x[y], address[:-1], version.chapter)[address[-1]] = new_content + else: + version.chapter = new_content + action = 'add' if not old_content else 'edit' + changing_texts.append({'action': action, 'oref': jagged_array_node.ref(), 'old_text': old_content, 'curr_text': new_content}) + + index_title = version_dict['title'] + lang = version_dict['language'] + version_title = version_dict['versionTitle'] + version = model.Version().load({'title': index_title, 'versionTitle': version_title, 'language': lang}) + changing_texts = [] + if version: + is_version_new = False + if not patch: + for attr in model.Version.required_attrs + model.Version.optional_attrs: + if hasattr(version, attr) and attr != 'chapter': + delattr(version, attr) + for key, value in version_dict.items(): + if key == 'chapter': + continue + else: + setattr(version, key, value) + else: + is_version_new = True + version = model.Version(version_dict) + model.Ref(index_title).index_node.visit_content(modify_node) + version.save() + + for change in changing_texts: + post_modify_text(user, change['action'], change['oref'], lang, version_title, change['old_text'], change['curr_text'], version._id, **kwargs) + count_segments(version.get_index()) + + def post_modify_text(user, action, oref, lang, vtitle, old_text, curr_text, version_id, **kwargs) -> None: model.log_text(user, action, oref, lang, vtitle, old_text, curr_text, **kwargs) if USE_VARNISH: diff --git a/sefaria/urls.py b/sefaria/urls.py index 928606f4e0..b73ae60733 100644 --- a/sefaria/urls.py +++ b/sefaria/urls.py @@ -150,6 +150,7 @@ url(r'^api/texts/modify-bulk/(?P.+)$', reader_views.modify_bulk_text_api), url(r'^api/texts/(?P<tref>.+)/(?P<lang>\w\w)/(?P<version>.+)$', reader_views.old_text_versions_api_redirect), url(r'^api/texts/(?P<tref>.+)$', reader_views.texts_api), + url(r'^api/versions/?$', reader_views.complete_version_api), url(r'^api/v3/texts/(?P<tref>.+)$', api_views.Text.as_view()), url(r'^api/index/?$', reader_views.table_of_contents_api), url(r'^api/opensearch-suggestions/?$', reader_views.opensearch_suggestions_api), diff --git a/sefaria/views.py b/sefaria/views.py index 8778a273b7..7c0f70078c 100644 --- a/sefaria/views.py +++ b/sefaria/views.py @@ -47,6 +47,7 @@ from sefaria.datatype.jagged_array import JaggedTextArray # noinspection PyUnresolvedReferences from sefaria.system.exceptions import InputError, NoVersionFoundError +from api.api_errors import APIInvalidInputException from sefaria.system.database import db from sefaria.system.decorators import catch_error_as_http from sefaria.utils.hebrew import has_hebrew, strip_nikkud @@ -339,7 +340,10 @@ def find_refs_report_api(request): @api_view(["POST"]) def find_refs_api(request): from sefaria.helper.linker import make_find_refs_response - return jsonResponse(make_find_refs_response(request)) + try: + return jsonResponse(make_find_refs_response(request)) + except APIInvalidInputException as e: + return e.to_json_response() @api_view(["GET"]) @@ -419,7 +423,7 @@ def bundle_many_texts(refs, useTextFamily=False, as_sized_string=False, min_char 'url': oref.url() } else: - he_tc = model.TextChunk(oref, "he", actual_lang=translation_language_preference, vtitle=hebrew_version) + he_tc = model.TextChunk(oref, "he", vtitle=hebrew_version) en_tc = model.TextChunk(oref, "en", actual_lang=translation_language_preference, vtitle=english_version) if hebrew_version and he_tc.is_empty(): raise NoVersionFoundError(f"{oref.normal()} does not have the Hebrew version: {hebrew_version}") @@ -470,7 +474,7 @@ def bulktext_api(request, refs): g = lambda x: request.GET.get(x, None) min_char = int(g("minChar")) if g("minChar") else None max_char = int(g("maxChar")) if g("maxChar") else None - res = bundle_many_texts(refs, g("useTextFamily"), g("asSizedString"), min_char, max_char, g("transLangPref"), g("ven"), g("vhe")) + res = bundle_many_texts(refs, int(g("useTextFamily")), g("asSizedString"), min_char, max_char, g("transLangPref"), g("ven"), g("vhe")) resp = jsonResponse(res, cb) return resp diff --git a/static/css/s2.css b/static/css/s2.css index dcadef5221..7401943378 100644 --- a/static/css/s2.css +++ b/static/css/s2.css @@ -2312,7 +2312,7 @@ div.interfaceLinks-row a { flex-direction: column; } .topicPanel .mainColumn { - padding: 0 40px; + padding: 0 40px 80px; } @media (max-width: 450px) { .topicPanel .mainColumn, @@ -2367,7 +2367,8 @@ div.interfaceLinks-row a { --hebrew-font: var(--hebrew-sans-serif-font-family); } .topicPanel .mainColumn .story { - padding: 0 0 20px 0; + padding: 0; + margin-bottom: 20px; } .topicPanel .mainColumn .storySheetListItem { padding: 0 0 30px 0; @@ -2405,8 +2406,17 @@ div.interfaceLinks-row a { font-size: 29px; } .topicPanel .story .storyTitle { - font-size: 24px; + --english-font: var(--english-sans-serif-font-family); + --hebrew-font: var(--hebrew-sans-serif-font-family); + font-size: 16px; + font-weight: 600; +} + +.topicPanel .story .storyTitle .int-he { + font-size: 18px; + font-weight: 600; } + .topicPanel h1 { --english-font: var(--english-serif-font-family); --hebrew-font: var(--hebrew-serif-font-family); @@ -2511,7 +2521,8 @@ div.interfaceLinks-row a { height: auto; max-width: calc(66.67vw); max-height: calc(66.67vw); - margin-bottom: 10px; + margin: 0 auto 10px; + display: block; } .topicImage{ padding-left: 0; @@ -7096,6 +7107,11 @@ But not to use a display block directive that might break continuous mode for ot flex: 1 1 auto; } +.toolsButton ::before { + position: relative; + top: 3px; +} + .toolsSecondaryButton { color: var(--dark-grey); --english-font: var(--english-sans-serif-font-family); @@ -7962,6 +7978,15 @@ a .button:hover { flex-wrap: nowrap; width: 100%; } +.headerWithAdminButtonsContainer .pencilEditorButton{ + margin-top: 8px; + margin-bottom: 5px; + margin-inline-end: 6px; + cursor: pointer; +} +.button.extraSmall.reviewState{ + margin-inline-end: 7px; +} .button.extraSmall.reviewState.reviewed { background-color: #5D956F; @@ -10924,6 +10949,16 @@ cursor: pointer; margin-inline-end: 30px; cursor: pointer; } + +.tab-view .tab-list .tab.popover { + margin-inline-end: 0; + margin-inline-start: 10px; +} + +.interface-hebrew .tab-view .tab-list .tab.popover{ + transform: scale(1.3); +} + .tab-view .tab-list .tab a { color: inherit; } @@ -10933,9 +10968,6 @@ cursor: pointer; .tab-view .tab-list .active .tab { border-bottom: 4px solid #CCC; } -.interface-hebrew .tab-view .tab-list .tab { - margin: 0 0 0 30px; -} .tab-view .tab-list .tab img { width: 18px; height: 18px; @@ -10953,20 +10985,26 @@ cursor: pointer; } .tab-view .tab-list .tab.filter, .tab-view.largeTabs .tab-list .tab.filter { - font-size: 16px; margin: 0; - padding: 6px 9px; - border: 1px solid #EDEDEC; - background-color: #EDEDEC; - border-radius: 6px; } + +.interface-hebrew .singlePanel .tab-view.largeTabs .tab-list .tab.filter { + margin-top: 3px; +} + +.tab-view .tab-list .tab.popover { + position: relative; + display: inline-block; +} + + .tab-view .tab-list .tab.filter.open { background-color: inherit; } .tab-view .tab-list .tab.filter img { margin: 0 0 1px 6px; - width: 12px; - height: 12px; + width: 18px; + height: 18px; } .interface-hebrew .tab-view .tab-list .tab.filter img { margin: 0 6px 1px 0; @@ -10990,19 +11028,109 @@ cursor: pointer; .tab-view.largeTabs .tab-list .tab { font-size: 22px; } +.singlePanel .tab-view.largeTabs .tab-list .tab:not(.popover) { + font-size: 16px; +} .tab-view.largeTabs .tab-list .active .tab { border-bottom: 4px solid var(--light-grey); } + +.singlePanel .tab-view.largeTabs .tab-list .active .tab { + padding-bottom: 18px; +} + +.langSelectPopover { + position: absolute; + top: 100%; + right: 0; + background-color: #fff; + border-radius: 6px; + box-shadow: 0px 1px 3px 0px rgba(0, 0, 0, 0.25); + text-align: start; + width: 266px; + padding-bottom: 15px; +} + +.interface-hebrew .singlePanel .langSelectPopover { + right: -233px; +} +.langSelectPopover .langHeader { + font-weight: 500; + font-size: 16px; + padding: 15px; + margin-bottom: 15px; + border-bottom: 1px solid var(--light-grey); +} + +.langSelectPopover .radioChoice.active { + background-color: var(--sefaria-blue); + color: #fff; +} + +.langSelectPopover .radioChoice { + display: flex; + justify-content: space-between; + align-items: center; + font-size: 14px; + color: #000; + margin-inline: 15px; + background-color: var(--lighter-grey); + cursor: pointer; +} + +.langSelectPopover .radioChoice:nth-of-type(2) { + border-radius: 6px 6px 0px 0px; +} + +.langSelectPopover .radioChoice:nth-of-type(4) { + border-radius: 0px 0px 6px 6px; +} + +.langSelectPopover .radioChoice label { + flex-grow: 1; + padding: 10px; + cursor: pointer; +} + +.langSelectPopover .radioChoice input[type=radio] { + appearance: none; + background-color: #fff; + width: 20px; + height: 20px; + border: 2px solid var(--medium-grey); + border-radius: 20px; + display: inline-grid; + place-content: center; + margin-top: 0; + margin-inline-end: 10px; + + } + +/* Styles the radio select as a checkbox */ +.langSelectPopover .radioChoice input[type=radio]::before { + content: ""; + width: 10px; + height: 10px; + transform: scale(0); + transform-origin: bottom left; + background-color: var(--sefaria-blue); + clip-path: polygon(13% 50%, 34% 66%, 81% 2%, 100% 18%, 39% 100%, 0 71%); +} + +.langSelectPopover .radioChoice input[type=radio]:checked::before { + transform: scale(1); +} + +.langSelectPopover .radioChoice input[type=radio]:checked{ + background-color: #fff; + border: 0; +} + @media (max-width: 540px) { .profile-page .tab-view .tab .tabIcon { display: none; } - .interface-hebrew .tab-view .tab-list .justifyright { - margin: initial; - } - .tab-view .tab-list .justifyright { - margin: initial; - } + .tab-view .tab-list{ flex-wrap: wrap; } @@ -11402,6 +11530,19 @@ cursor: pointer; color: white; background-color: #18345d; } +.resourcesLink.studyCompanion { + margin-inline-start: 10px; +} +@media screen and (max-width: 900px) { + .resourcesLink.studyCompanion { + margin-inline-start: 0; + } +} +@media screen and (max-width: 900px) { + .resourcesLink.studyCompanion { + margin-inline-start: 0; + } +} .resourcesLink.blue img { filter: invert(1); opacity: 1; @@ -11737,8 +11878,28 @@ body .homeFeedWrapper .content { color: #666; } .story .learningPrompt { - padding: 10px 0 20px; + padding: 20px 0 20px; } +.story details .learningPrompt { + background-color: var(--highlight-blue-light); + margin-inline-start: -20px; + padding-inline-start: 20px; + padding-inline-end: 20px; + border-top: 1px solid var(--light-grey); +} + +.story.topicPassageStory { + box-shadow: 0px 1px 3px rgba(0, 0, 0, 0.25); +} + +.story.topicPassageStory .storyBody { + padding-inline-end: 20px; + padding-top: 20px; +} +.story.topicPassageStory .contentText.subHeading { + padding-bottom: 20px; +} + .mainColumn .story .storyTitleBlock { clear: both; } @@ -11782,12 +11943,71 @@ body .homeFeedWrapper .content { float: right; } +.story details .storyBody { + margin-top: 0; +} + +.story details > summary { + cursor: pointer; + list-style: none; +} +.story details > summary::-webkit-details-marker { + display: none; +} + +.story details > summary .topicStoryDescBlock { + background-color: var(--lightest-grey); + margin-inline-start: -20px; + padding: 20px; + display: flex; + justify-content: space-between; +} + +/* extremely hacky, but required unless there's a major refactor of CategoryHeader logic */ +.story details > summary .topicStoryDescBlock > span:nth-child(1) { + flex: 1 +} + + +.story details > summary .storyTitleBlock { + background: url('/static/icons/arrow-down-bold.svg') no-repeat transparent; + background-size: 14px; + background-position-y: center; + margin: 0; + padding: 0px 0px 1px 24px; + vertical-align: middle; +} + +.interface-hebrew .story details > summary .storyTitleBlock { + background-position-x: right; + padding: 0px 24px 1px 0px; +} + +.story details[open] > summary .storyTitleBlock { + background: url('/static/icons/arrow-up-bold.svg') no-repeat transparent; + background-size: 14px; + background-position-y: center; +} + +.interface-hebrew .story details[open] > summary .storyTitleBlock { + background-position-x: right; +} + + .story .storyBody { clear: both; margin: 10px 0; text-align: justify; font-size: 18px; } +/*HACK to make English text of sources in topic pages ltr*/ +.interface-hebrew .storyBody { + direction: ltr; + margin-top: 10px; + margin-right: -17px; + margin-bottom: 10px; + margin-left: 26px; +} .story .storySheetListItem > div.storyBody.sheetSummary > * { color: var(--dark-grey); font-family: var(--english-sans-serif-font-family); @@ -13344,10 +13564,6 @@ span.ref-link-color-3 {color: blue} } -.productsDevBox p { - margin-top: 0; -} - .productsDevBox a::after { content: " ›"; color: var(--commentary-blue); diff --git a/static/font-awesome/css/font-awesome.css b/static/font-awesome/css/font-awesome.css index a0b879fa00..9120713168 100644 --- a/static/font-awesome/css/font-awesome.css +++ b/static/font-awesome/css/font-awesome.css @@ -627,6 +627,10 @@ .fa-twitter:before { content: "\f099"; } +.fa-X:before { + content: "𝕏"; + font-size: larger; +} .fa-facebook-f:before, .fa-facebook:before { content: "\f09a"; diff --git a/static/icons/editing-pencil.svg b/static/icons/editing-pencil.svg new file mode 100644 index 0000000000..59d40b3175 --- /dev/null +++ b/static/icons/editing-pencil.svg @@ -0,0 +1,18 @@ +<svg width="36" height="36" viewBox="0 0 36 36" fill="none" xmlns="http://www.w3.org/2000/svg"> +<g filter="url(#filter0_d_3926_542)"> +<rect x="3" y="2" width="30" height="30" rx="6" fill="#18345D"/> +<path d="M23.4956 9.23893L25.7613 11.504L13.8606 23.4048L11.0242 23.9764L11.5966 21.139L23.4956 9.23893ZM23.4956 7L10.1384 20.3576L9 26L14.642 24.8623L28 11.504L23.4956 7V7Z" fill="white"/> +</g> +<defs> +<filter id="filter0_d_3926_542" x="0" y="0" width="36" height="36" filterUnits="userSpaceOnUse" color-interpolation-filters="sRGB"> +<feFlood flood-opacity="0" result="BackgroundImageFix"/> +<feColorMatrix in="SourceAlpha" type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0" result="hardAlpha"/> +<feOffset dy="1"/> +<feGaussianBlur stdDeviation="1.5"/> +<feComposite in2="hardAlpha" operator="out"/> +<feColorMatrix type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0.25 0"/> +<feBlend mode="normal" in2="BackgroundImageFix" result="effect1_dropShadow_3926_542"/> +<feBlend mode="normal" in="SourceGraphic" in2="effect1_dropShadow_3926_542" result="shape"/> +</filter> +</defs> +</svg> diff --git a/static/icons/email-newsletter.svg b/static/icons/email-newsletter.svg new file mode 100644 index 0000000000..1887110e41 --- /dev/null +++ b/static/icons/email-newsletter.svg @@ -0,0 +1,5 @@ +<svg width="24" height="22" viewBox="0 0 24 22" fill="none" xmlns="http://www.w3.org/2000/svg"> +<path fill-rule="evenodd" clip-rule="evenodd" d="M12 0C12.5098 0 12.9231 0.413274 12.9231 0.923073V8.67688C12.9231 9.18668 12.5098 9.59996 12 9.59996C11.4902 9.59996 11.0769 9.18668 11.0769 8.67688V0.923073C11.0769 0.413274 11.4902 0 12 0ZM6.10352e-05 6.46151C6.10352e-05 5.95171 0.413335 5.53844 0.923134 5.53844H5.35388C5.86368 5.53844 6.27696 5.95171 6.27696 6.46151C6.27696 6.97131 5.86368 7.38458 5.35388 7.38458H1.84621V19.9384H22.1538V7.38458H18.6461C18.1363 7.38458 17.7231 6.97131 17.7231 6.46151C17.7231 5.95171 18.1363 5.53844 18.6461 5.53844H23.0769C23.5867 5.53844 24 5.95171 24 6.46151V20.8614C24 21.3712 23.5867 21.7845 23.0769 21.7845H0.923134C0.413335 21.7845 6.10352e-05 21.3712 6.10352e-05 20.8614V6.46151Z" fill="#000000"/> +<path fill-rule="evenodd" clip-rule="evenodd" d="M8.02421 4.7011C8.38469 4.34062 8.96915 4.34062 9.32963 4.7011L12 7.37145L14.6703 4.7011C15.0308 4.34062 15.6153 4.34062 15.9758 4.7011C16.3362 5.06158 16.3362 5.64604 15.9758 6.00652L12.6527 9.32959C12.4796 9.5027 12.2448 9.59995 12 9.59995C11.7552 9.59995 11.5204 9.5027 11.3473 9.32959L8.02421 6.00652C7.66372 5.64604 7.66372 5.06158 8.02421 4.7011Z" fill="#000000"/> +<path fill-rule="evenodd" clip-rule="evenodd" d="M0.184669 5.90767C0.490548 5.49983 1.06913 5.41717 1.47697 5.72305L12 13.6153L22.523 5.72305C22.9309 5.41717 23.5095 5.49983 23.8153 5.90767C24.1212 6.31551 24.0386 6.89409 23.6307 7.19997L12.5538 15.5076C12.2256 15.7538 11.7744 15.7538 11.4462 15.5076L0.369283 7.19997C-0.0385557 6.89409 -0.121211 6.31551 0.184669 5.90767Z" fill="#000000"/> +</svg> diff --git a/static/icons/filter.svg b/static/icons/filter.svg new file mode 100644 index 0000000000..2322da7242 --- /dev/null +++ b/static/icons/filter.svg @@ -0,0 +1,3 @@ +<svg width="17" height="18" viewBox="0 0 17 18" fill="none" xmlns="http://www.w3.org/2000/svg"> +<path d="M0 0L6.75 11.3205V15.75L9.75 18V11.3205L16.5 0H0ZM13.8593 1.5L11.6235 5.25H4.8765L2.64075 1.5H13.8593Z" fill="#666666"/> +</svg> diff --git a/static/js/AboutBox.jsx b/static/js/AboutBox.jsx index 791b77e42c..1ede90b49b 100644 --- a/static/js/AboutBox.jsx +++ b/static/js/AboutBox.jsx @@ -5,7 +5,7 @@ import VersionBlock, {VersionsBlocksList} from './VersionBlock/VersionBlock'; import Component from 'react-class'; import {InterfaceText} from "./Misc"; import {ContentText} from "./ContentText"; -import { Modules } from './NavSidebar'; +import { SidebarModules } from './NavSidebar'; class AboutBox extends Component { @@ -232,8 +232,8 @@ class AboutBox extends Component { (<div>{versionSectionEn}{versionSectionHe}{alternateSectionHe}</div>) : (<div>{versionSectionHe}{versionSectionEn}{alternateSectionHe}</div>) } - <Modules type={"RelatedTopics"} props={{title: this.props.title}} /> - { !isDictionary ? <Modules type={"DownloadVersions"} props={{sref: this.props.title}} /> : null} + <SidebarModules type={"RelatedTopics"} props={{title: this.props.title}} /> + { !isDictionary ? <SidebarModules type={"DownloadVersions"} props={{sref: this.props.title}} /> : null} </section> ); } diff --git a/static/js/BookPage.jsx b/static/js/BookPage.jsx index 4c87581a41..60060856e8 100644 --- a/static/js/BookPage.jsx +++ b/static/js/BookPage.jsx @@ -19,7 +19,7 @@ import React, { useState, useRef } from 'react'; import ReactDOM from 'react-dom'; import $ from './sefaria/sefariaJquery'; import Sefaria from './sefaria/sefaria'; -import { NavSidebar, Modules } from './NavSidebar'; +import { NavSidebar, SidebarModules } from './NavSidebar'; import DictionarySearch from './DictionarySearch'; import VersionBlock from './VersionBlock/VersionBlock'; import ExtendedNotes from './ExtendedNotes'; @@ -272,7 +272,7 @@ class BookPage extends Component { {this.props.multiPanel ? null : <div className="about"> - <Modules type={"AboutText"} props={{index: this.state.indexDetails, hideTitle: true}} /> + <SidebarModules type={"AboutText"} props={{index: this.state.indexDetails, hideTitle: true}} /> </div>} <TabView @@ -303,7 +303,7 @@ class BookPage extends Component { } </div> {this.isBookToc() && ! this.props.compare ? - <NavSidebar modules={sidebarModules} /> : null} + <NavSidebar sidebarModules={sidebarModules} /> : null} </div> {this.isBookToc() && ! this.props.compare ? <Footer /> : null} diff --git a/static/js/CalendarsPage.jsx b/static/js/CalendarsPage.jsx index c6d96a8f5f..78198f0719 100644 --- a/static/js/CalendarsPage.jsx +++ b/static/js/CalendarsPage.jsx @@ -6,7 +6,7 @@ import React, { useState } from 'react'; import classNames from 'classnames'; import Sefaria from './sefaria/sefaria'; import $ from './sefaria/sefariaJquery'; -import { NavSidebar, Modules }from './NavSidebar'; +import { NavSidebar, SidebarModules }from './NavSidebar'; import Footer from './Footer'; import Component from 'react-class'; @@ -28,7 +28,7 @@ const CalendarsPage = ({multiPanel, initialWidth}) => { const weeklyListings = makeListings(weeklyCalendars); const about = multiPanel ? null : - <Modules type={"AboutLearningSchedules"} /> + <SidebarModules type={"AboutLearningSchedules"} /> const sidebarModules = [ multiPanel ? {type: "AboutLearningSchedules"} : {type: null}, @@ -56,7 +56,7 @@ const CalendarsPage = ({multiPanel, initialWidth}) => { <ResponsiveNBox content={weeklyListings} initialWidth={initialWidth} /> </div> </div> - <NavSidebar modules={sidebarModules} /> + <NavSidebar sidebarModules={sidebarModules} /> </div> <Footer /> </div> diff --git a/static/js/CollectionPage.jsx b/static/js/CollectionPage.jsx index 49adfa7131..5fb3bfe07a 100644 --- a/static/js/CollectionPage.jsx +++ b/static/js/CollectionPage.jsx @@ -294,7 +294,7 @@ class CollectionPage extends Component { <div className="contentInner"> {content} </div> - <NavSidebar modules={sidebarModules} /> + <NavSidebar sidebarModules={sidebarModules} /> </div> <Footer /> </div> diff --git a/static/js/CommunityPage.jsx b/static/js/CommunityPage.jsx index 8510b13093..f199ec2c3f 100644 --- a/static/js/CommunityPage.jsx +++ b/static/js/CommunityPage.jsx @@ -4,7 +4,7 @@ import $ from './sefaria/sefariaJquery'; import Sefaria from './sefaria/sefaria'; import PropTypes from 'prop-types'; import classNames from 'classnames'; -import { NavSidebar, Modules } from './NavSidebar'; +import { NavSidebar, SidebarModules } from './NavSidebar'; import Footer from'./Footer'; import { InterfaceText, @@ -62,7 +62,7 @@ const CommunityPage = ({multiPanel, toggleSignUpModal, initialWidth}) => { <RecentlyPublished multiPanel={multiPanel} toggleSignUpModal={toggleSignUpModal} /> </div> - <NavSidebar modules={sidebarModules} /> + <NavSidebar sidebarModules={sidebarModules} /> </div> <Footer /> </div> @@ -103,7 +103,7 @@ const RecentlyPublished = ({multiPanel, toggleSignUpModal}) => { recentSheets.map(s => <FeaturedSheet sheet={s} showDate={true} toggleSignUpModal={toggleSignUpModal} />); const joinTheConversation = ( <div className="navBlock"> - <Modules type={"JoinTheConversation"} props={{wide:multiPanel}} /> + <SidebarModules type={"JoinTheConversation"} props={{wide:multiPanel}} /> </div> ); if (recentSheets) { diff --git a/static/js/ConnectionsPanel.jsx b/static/js/ConnectionsPanel.jsx index b874731285..039b3eadf3 100644 --- a/static/js/ConnectionsPanel.jsx +++ b/static/js/ConnectionsPanel.jsx @@ -1480,7 +1480,7 @@ class ShareBox extends Component { </ConnectionsPanelSection> <ConnectionsPanelSection title="More Options"> <ToolsButton en="Share on Facebook" he="פייסבוק" icon="facebook-official" onClick={shareFacebook} /> - <ToolsButton en="Share on Twitter" he="טוויטר" icon="twitter" onClick={shareTwitter} /> + <ToolsButton en="Share on X" he="X" icon="X" onClick={shareTwitter} /> <ToolsButton en="Share by Email" he="אימייל" icon="envelope-o" onClick={shareEmail} /> </ConnectionsPanelSection> </div>); diff --git a/static/js/ContentText.jsx b/static/js/ContentText.jsx index 234979da76..2e0c14049d 100644 --- a/static/js/ContentText.jsx +++ b/static/js/ContentText.jsx @@ -15,7 +15,7 @@ const ContentText = (props) => { * order to return the bilingual langauage elements in (as opposed to the unguaranteed order by default). */ const langAndContentItems = _filterContentTextByLang(props); - return langAndContentItems.map(item => <ContentSpan lang={item[0]} content={item[1]} isHTML={!!props.html} markdown={props.markdown}/>); + return langAndContentItems.map(item => <ContentSpan key={item[0]} lang={item[0]} content={item[1]} isHTML={!!props.html} markdown={props.markdown}/>); }; const VersionContent = (props) => { diff --git a/static/js/Misc.jsx b/static/js/Misc.jsx index 735b24afd3..bc988e2ae8 100644 --- a/static/js/Misc.jsx +++ b/static/js/Misc.jsx @@ -23,7 +23,7 @@ import Cookies from "js-cookie"; import {EditTextInfo} from "./BookPage"; import ReactMarkdown from 'react-markdown'; import TrackG4 from "./sefaria/trackG4"; -import { ReaderApp } from './ReaderApp'; +import { ReaderApp } from './ReaderApp'; /** * Component meant to simply denote a language specific string to go inside an InterfaceText element @@ -500,6 +500,7 @@ const FilterableList = ({ name="filterableListInput" value={filter} onChange={e => setFilter(e.target.value)} + data-anl-event="search:input" /> </div> <div className="filter-sort-wrapper"> @@ -511,6 +512,17 @@ const FilterableList = ({ key={option} className={classNames({'sans-serif': 1, 'sort-option': 1, noselect: 1, active: sortOption === option})} onClick={() => setSort(option)} + tabIndex="0" + onKeyDown={(e) => { + if (e.key === 'Enter') { + e.preventDefault(); + e.target.click(); + } + }} + data-anl-event="sort_by:click" + data-anl-batch={JSON.stringify({ + text: option, from: sortOption, to: option, + })} > <InterfaceText context="FilterableList">{option}</InterfaceText> </span> @@ -1050,7 +1062,7 @@ class ToggleOption extends Component { const TopicToCategorySlug = function(topic, category=null) { //helper function for AdminEditor if (!category) { - category = Sefaria.topicTocCategory(topic.slug); + category = Sefaria.displayTopicTocCategory(topic.slug); } let initCatSlug = category ? category.slug : "Main Menu"; //category topics won't be found using topicTocCategory, // so all category topics initialized to "Main Menu" @@ -1082,6 +1094,7 @@ const AllAdminButtons = ({ buttonOptions, buttonIDs, adminClasses }) => { const [buttonText, toggleAddingTopics] = buttonOptions[key]; return ( <AdminEditorButton + key={`${buttonText}|${i}`} text={buttonText} top={top} bottom={bottom} @@ -1143,6 +1156,15 @@ const CategoryHeader = ({children, type, data = [], toggleButtonIDs = ["subcate } return <span className={wrapper}><span onMouseEnter={() => setHiddenButtons()}>{children}</span><span>{adminButtonsSpan}</span></span>; } + + +//Pencil-shaped button to open the ref-link (source) editor +const PencilSourceEditor = ({topic, text, classes}) => { + const [addSource, toggleAddSource] = useEditToggle(); + return addSource ? <SourceEditor topic={topic} origData={text} close={toggleAddSource}/> : + <img className={classes} id={"editTopic"} onClick={toggleAddSource} src={"/static/icons/editing-pencil.svg"}/>; +} + const ReorderEditorWrapper = ({toggle, type, data}) => { /* Wrapper for ReorderEditor that can reorder topics, categories, and sources. It is only used for reordering topics and categories at the @@ -1161,9 +1183,11 @@ const ReorderEditorWrapper = ({toggle, type, data}) => { } const _createURLs = (type, data) => { if (reorderingSources) { + const urlObj = new URL(window.location.href); + const tabName = urlObj.searchParams.get('tab'); return { url: `/api/source/reorder?topic=${data.slug}&lang=${Sefaria.interfaceLang}`, - redirect: `/topics/${data.slug}`, + redirect: `/topics/${data.slug}?sort=Relevance&tab=${tabName}`, origItems: _filterAndSortRefs(data.tabs?.sources?.refs) || [], } } @@ -1497,31 +1521,60 @@ const ToolTipped = ({ altText, classes, style, onClick, children }) => { </div> )}; +const AiLearnMoreLink = ({lang}) => { + const text = lang === 'english' ? 'Learn More' : 'לפרטים נוספים'; + return ( + <a href={"/sheets/583824?lang=bi"} data-anl-event="learn_more_click:click" data-anl-text="learn_more"> + {text} + </a> + ); +}; + +const AiFeedbackLink = ({lang}) => { + const text = lang === 'english' ? 'Feedback' : 'כתבו לנו'; + return ( + <a href={"https://sefaria.formstack.com/forms/ai_feedback_form"} data-anl-event="feedback_click:click" data-anl-text="feedback"> + {text} + </a> + ); +} + const AiInfoTooltip = () => { const [showMessage, setShowMessage] = useState(false); const aiInfoIcon = ( - <img className="ai-info-icon" src="/static/icons/ai-info.svg" alt="AI Info Icon" onMouseEnter={() => setShowMessage(true)} onMouseLeave={() => setShowMessage(false)}/> + <img + className="ai-info-icon" + data-anl-event="ai_marker_hover:mouseover" + src="/static/icons/ai-info.svg" + alt="AI Info Icon" onMouseEnter={() => setShowMessage(true)} + onMouseLeave={() => setShowMessage(false)} + /> ); - const aiMessage = ( - <div className="ai-info-messages-box" onMouseEnter={() => setShowMessage(true)} onMouseLeave={() => setShowMessage(false)}> - <div className="ai-info-first-message"> - <InterfaceText> - <EnglishText>Some of the text on this page has been AI generated and reviewed by our editors. <a href={"/sheets/583824?lang=bi"}>Learn more.</a></EnglishText> - <HebrewText>חלק מהטקסטים בדף זה נוצרו על ידי בינה מלאכותית ועברו הגהה על ידי צוות העורכים שלנו.  - <a href={"/sheets/583824?lang=bi"}>לפרטים נוספים</a></HebrewText> - </InterfaceText> + const aiMessage = ( + <div className="ai-info-messages-box" onMouseEnter={() => setShowMessage(true)} onMouseLeave={() => setShowMessage(false)}> + <div className="ai-info-first-message"> + <InterfaceText> + <EnglishText>Some of the text on this page has been AI generated. +  <AiLearnMoreLink lang="english" /> + </EnglishText> + <HebrewText>חלק מהטקסטים בדף זה נוצרו על ידי בינה מלאכותית.  + <AiLearnMoreLink lang="hebrew" /> + </HebrewText> + </InterfaceText> - </div> - <hr className="ai-info-messages-hr" /> - <div className="ai-info-last-message"> - <InterfaceText><EnglishText><a href={"https://sefaria.formstack.com/forms/ai_feedback_form"}>Feedback</a></EnglishText> - <HebrewText><a href={"https://sefaria.formstack.com/forms/ai_feedback_form"}>כתבו לנו</a></HebrewText> - </InterfaceText> - </div> </div> - ); + <hr className="ai-info-messages-hr" /> + <div className="ai-info-last-message"> + <InterfaceText><EnglishText><AiFeedbackLink lang="english" /></EnglishText> + <HebrewText><AiFeedbackLink lang="hebrew" /></HebrewText> + </InterfaceText> + </div> + </div> + ); return ( - <div className="ai-info-tooltip"> + <div className="ai-info-tooltip" + data-anl-feature_name="ai_marker" + > {aiInfoIcon} <div className={`ai-message ${(showMessage) ? 'visible' : ''}`}> {aiMessage} @@ -2078,7 +2131,7 @@ function OnInView({ children, onVisible }) { * `onVisible` callback function that will be called when given component(s) are visible within the viewport * Ex. <OnInView onVisible={handleImageIsVisible}><img src="..." /></OnInView> */ - const elementRef = useRef(); + const elementRef = useRef(); useEffect(() => { const observer = new IntersectionObserver( @@ -3281,8 +3334,8 @@ return Sefaria._v(caption) || Sefaria._('Illustrative image'); const ImageWithCaption = ({photoLink, caption }) => { return ( <div> - <img class="imageWithCaptionPhoto" src={photoLink} alt={getImgAltText(caption)}/> - <div class="imageCaption"> + <img className="imageWithCaptionPhoto" src={photoLink} alt={getImgAltText(caption)}/> + <div className="imageCaption"> <InterfaceText text={caption} /> </div> </div>); @@ -3310,7 +3363,7 @@ const handleAnalyticsOnMarkdown = (e, gtag_fxn, rank, product, cta, label, link_ let parent = target; let outmost = e.currentTarget; let text = ""; - + while (parent) { if(parent.nodeName === 'A'){ linkTarget = parent; @@ -3337,6 +3390,87 @@ const handleAnalyticsOnMarkdown = (e, gtag_fxn, rank, product, cta, label, link_ } +const LangRadioButton = ({buttonTitle, lang, buttonId, handleLangChange}) => { + + return ( + <div + className={classNames({ active: lang === buttonId, radioChoice: 1 })} + data-anl-event="lang_toggle_select:click" + data-anl-text={buttonId} + data-anl-to={buttonId} + > + <label htmlFor={buttonId}> + <InterfaceText>{buttonTitle}</InterfaceText> + </label> + <input + type="radio" + id={buttonId} + name="options" + value={buttonId} + checked={lang === buttonId} + onChange={handleLangChange} + /> + </div> + ); +}; +const LangSelectInterface = ({callback, defaultVal, closeInterface}) => { + const [lang, setLang] = useState(defaultVal); + const buttonData = [ + { buttonTitle: "Source Language", buttonId: "source" }, + { buttonTitle: "Translation", buttonId: "translation" }, + { buttonTitle: "Source with Translation", buttonId: "sourcewtrans" } +]; + + const handleLangChange = (event) => { + setLang(event.target.value); + callback(event.target.value); + closeInterface(); + }; + + useEffect(()=>{ + document.querySelector('.langSelectPopover').focus() + },[]) + + return ( + <div className="langSelectPopover" + tabIndex="0" + data-anl-batch={JSON.stringify({ + feature_name: "source lang toggled", + text: lang, + from: lang, + })} + onClick={(e) => { + e.stopPropagation(); + e.nativeEvent.stopImmediatePropagation(); + } + } + + // HACK to prevent the option menu to close once an option is selected (which is technically blurring this element) + onBlur={(e) => { + setTimeout(() => { + if (!document.querySelector('.langSelectPopover').contains(document.activeElement)) { + closeInterface(); + } + }, 50); + } + } + > + <div className="langHeader"><InterfaceText>Source Language</InterfaceText></div> + {buttonData.map((button, index) => ( + <LangRadioButton + key={button.buttonId} + buttonTitle={button.buttonTitle} + lang={lang} + buttonId={button.buttonId} + handleLangChange={handleLangChange} + /> + ))} + </div> + ); + +} + + export { AppStoreButton, CategoryHeader, @@ -3403,6 +3537,8 @@ export { TitleVariants, OnInView, TopicPictureUploader, - ImageWithCaption, - handleAnalyticsOnMarkdown + ImageWithCaption, + handleAnalyticsOnMarkdown, + LangSelectInterface, + PencilSourceEditor }; diff --git a/static/js/NavSidebar.jsx b/static/js/NavSidebar.jsx index 1aaadeaee4..38d0355549 100644 --- a/static/js/NavSidebar.jsx +++ b/static/js/NavSidebar.jsx @@ -7,10 +7,10 @@ import {InterfaceText, ProfileListing, Dropdown} from './Misc'; import { Promotions } from './Promotions' import {SignUpModalKind} from "./sefaria/signupModalContent"; -const NavSidebar = ({modules}) => { +const NavSidebar = ({sidebarModules}) => { return <div className="navSidebar sans-serif"> - {modules.map((m, i) => - <Modules + {sidebarModules.map((m, i) => + <SidebarModules type={m.type} props={m.props || {}} key={i} /> @@ -19,7 +19,7 @@ const NavSidebar = ({modules}) => { }; -const Modules = ({type, props}) => { +const SidebarModules = ({type, props}) => { // Choose the appropriate module component to render by `type` const moduleTypes = { "AboutSefaria": AboutSefaria, @@ -56,20 +56,21 @@ const Modules = ({type, props}) => { "PortalOrganization": PortalOrganization, "PortalNewsletter": PortalNewsletter, "RecentlyViewed": RecentlyViewed, + "StudyCompanion": StudyCompanion, }; if (!type) { return null; } - const ModuleType = moduleTypes[type]; - return <ModuleType {...props} /> + const SidebarModuleType = moduleTypes[type]; + return <SidebarModuleType {...props} /> }; -const Module = ({children, blue, wide}) => { +const SidebarModule = ({children, blue, wide}) => { const classes = classNames({navSidebarModule: 1, "sans-serif": 1, blue, wide}); return <div className={classes}>{children}</div> }; -const ModuleTitle = ({children, en, he, h1}) => { +const SidebarModuleTitle = ({children, en, he, h1}) => { const content = children ? <InterfaceText>{children}</InterfaceText> : <InterfaceText text={{en, he}} />; @@ -81,10 +82,10 @@ const ModuleTitle = ({children, en, he, h1}) => { const TitledText = ({enTitle, heTitle, enText, heText}) => { - return <Module> - <ModuleTitle en={enTitle} he={heTitle} /> + return <SidebarModule> + <SidebarModuleTitle en={enTitle} he={heTitle} /> <InterfaceText markdown={{en: enText, he: heText}} /> - </Module> + </SidebarModule> }; const RecentlyViewedItem = ({oref}) => { @@ -135,31 +136,46 @@ const RecentlyViewed = ({toggleSignUpModal, mobile}) => { } const allHistoryPhrase = mobile ? "All History" : "All history "; const recentlyViewedList = <RecentlyViewedList items={recentlyViewedItems}/>; - return <Module> + return <SidebarModule> <div className="recentlyViewed"> <div id="header"> - <ModuleTitle h1={true}>Recently Viewed</ModuleTitle> + <SidebarModuleTitle h1={true}>Recently Viewed</SidebarModuleTitle> {!mobile && recentlyViewedList} <a href="/texts/history" id="history" onClick={handleAllHistory}><InterfaceText>{allHistoryPhrase}</InterfaceText></a> </div> {mobile && recentlyViewedList} </div> - </Module>; + </SidebarModule>; } const Promo = () => - <Module> + <SidebarModule> <Promotions adType="sidebar"/> - </Module> + </SidebarModule> ; +const StudyCompanion = () => ( + <SidebarModule> + <SidebarModuleTitle>Study Companion</SidebarModuleTitle> + <div><InterfaceText>Get the Weekly Parashah Study Companion in your inbox.</InterfaceText></div> + <a className="button small" + data-anl-event="select_promotion:click|view_promotion:scrollIntoView" + data-anl-promotion_name="Parashah Email Signup - Topic TOC" + href="https://learn.sefaria.org/weekly-parashah/"> + <img src="/static/icons/email-newsletter.svg" alt="Sign up for our weekly parashah study companion"/> + <InterfaceText>Sign Up</InterfaceText> + </a> + </SidebarModule> +) + + const AboutSefaria = ({hideTitle}) => ( - <Module> - {!hideTitle ? - <ModuleTitle h1={true}>A Living Library of Torah</ModuleTitle> : null } - <InterfaceText> - <EnglishText> - Sefaria is home to 3,000 years of Jewish texts. We are a nonprofit organization offering free access to texts, translations, + <SidebarModule> + {!hideTitle ? + <SidebarModuleTitle h1={true}>A Living Library of Torah</SidebarModuleTitle> : null} + <InterfaceText> + <EnglishText> + Sefaria is home to 3,000 years of Jewish texts. We are a nonprofit organization offering free access to texts, translations, and commentaries so that everyone can participate in the ongoing process of studying, interpreting, and creating Torah. </EnglishText> <HebrewText> @@ -193,7 +209,7 @@ const AboutSefaria = ({hideTitle}) => ( </HebrewText> </InterfaceText> } - </Module> + </SidebarModule> ); @@ -214,9 +230,9 @@ const AboutTranslatedText = ({translationsSlug}) => { "yi": {title: "א לעבעדיקע ביבליאטעק פון תורה", body: "אין ספֿריאַ איז אַ היים פֿון 3,000 יאָר ייִדישע טעקסטן. מיר זענען אַ נאַן-נוץ אָרגאַניזאַציע וואָס אָפפערס פריי אַקסעס צו טעקסטן, איבערזעצונגען און קאָמענטאַרן אַזוי אַז אַלעמען קענען אָנטייל נעמען אין די אָנגאָינג פּראָצעס פון לערנען, ינטערפּריטיישאַן און שאפן תורה."} } return ( - <Module> - <ModuleTitle h1={true}>{translationLookup[translationsSlug] ? - translationLookup[translationsSlug]["title"] : "A Living Library of Torah"}</ModuleTitle> + <SidebarModule> + <SidebarModuleTitle h1={true}>{translationLookup[translationsSlug] ? + translationLookup[translationsSlug]["title"] : "A Living Library of Torah"}</SidebarModuleTitle> { translationLookup[translationsSlug] ? translationLookup[translationsSlug]["body"] : <InterfaceText> @@ -231,13 +247,13 @@ const AboutTranslatedText = ({translationsSlug}) => { </HebrewText> </InterfaceText> } - </Module> + </SidebarModule> ); } const Resources = () => ( - <Module> + <SidebarModule> <h3><InterfaceText context="ResourcesModule">Resources</InterfaceText></h3> <div className="linkList"> <IconLink text="Mobile Apps" url="/mobile" icon="mobile.svg" /> @@ -248,42 +264,42 @@ const Resources = () => ( <IconLink text="Torah Tab" url="/torah-tab" icon="torah-tab.svg" /> <IconLink text="Help" url="/help" icon="help.svg" /> </div> - </Module> + </SidebarModule> ); const TheJewishLibrary = ({hideTitle}) => ( - <Module> + <SidebarModule> {!hideTitle ? - <ModuleTitle>The Jewish Library</ModuleTitle> : null} + <SidebarModuleTitle>The Jewish Library</SidebarModuleTitle> : null} <InterfaceText>The tradition of Torah texts is a vast, interconnected network that forms a conversation across space and time. The five books of the Torah form its foundation, and each generation of later texts functions as a commentary on those that came before it.</InterfaceText> - </Module> + </SidebarModule> ); const SupportSefaria = ({blue}) => ( - <Module blue={blue}> - <ModuleTitle>Support Sefaria</ModuleTitle> + <SidebarModule blue={blue}> + <SidebarModuleTitle>Support Sefaria</SidebarModuleTitle> <InterfaceText>Sefaria is an open source, nonprofit project. Support us by making a tax-deductible donation.</InterfaceText> <br /> <DonateLink classes={"button small" + (blue ? " white" : "")} source={"NavSidebar-SupportSefaria"}> <img src="/static/img/heart.png" alt="donation icon" /> <InterfaceText>Make a Donation</InterfaceText> </DonateLink> - </Module> + </SidebarModule> ); const SponsorADay = () => ( - <Module> - <ModuleTitle>Sponsor A Day of Learning</ModuleTitle> + <SidebarModule> + <SidebarModuleTitle>Sponsor A Day of Learning</SidebarModuleTitle> <InterfaceText>With your help, we can add more texts and translations to the library, develop new tools for learning, and keep Sefaria accessible for Torah study anytime, anywhere.</InterfaceText> <br /> <DonateLink classes={"button small"} link={"dayOfLearning"} source={"NavSidebar-SponsorADay"}> <img src="/static/img/heart.png" alt="donation icon" /> <InterfaceText>Sponsor A Day</InterfaceText> </DonateLink> - </Module> + </SidebarModule> ); @@ -298,10 +314,10 @@ const AboutTextCategory = ({cats}) => { } return ( - <Module> + <SidebarModule> <h3><InterfaceText text={{en: enTitle, he: heTitle}} /></h3> <InterfaceText markdown={{en: tocObject.enDesc, he: tocObject.heDesc}} /> - </Module> + </SidebarModule> ); }; @@ -327,9 +343,9 @@ const AboutText = ({index, hideTitle}) => { if (!authors.length && !composed && !description) { return null; } return ( - <Module> + <SidebarModule> {hideTitle ? null : - <ModuleTitle>About This Text</ModuleTitle>} + <SidebarModuleTitle>About This Text</SidebarModuleTitle>} { composed || authors.length ? <div className="aboutTextMetadata"> @@ -355,7 +371,7 @@ const AboutText = ({index, hideTitle}) => { {description ? <InterfaceText markdown={{en: enDesc, he: heDesc}}/> : null} - </Module> + </SidebarModule> ); }; @@ -415,8 +431,8 @@ const DafLink = () => { } const Translations = () => { - return (<Module> - <ModuleTitle>Translations</ModuleTitle> + return (<SidebarModule> + <SidebarModuleTitle>Translations</SidebarModuleTitle> <InterfaceText> <EnglishText> Access key works from the library in several languages. @@ -426,14 +442,14 @@ const Translations = () => { </HebrewText> </InterfaceText> <TranslationLinks /> - </Module>) + </SidebarModule>) } const LearningSchedules = () => { return ( - <Module> - <ModuleTitle>Learning Schedules</ModuleTitle> + <SidebarModule> + <SidebarModuleTitle>Learning Schedules</SidebarModuleTitle> <div className="readingsSection"> <span className="readingsSectionTitle"> <InterfaceText>Weekly Torah Portion</InterfaceText>: <ParashahName /> @@ -458,15 +474,15 @@ const LearningSchedules = () => { <HebrewText>לוחות לימוד נוספים ›</HebrewText> </InterfaceText> </a> - </Module> + </SidebarModule> ); }; const WeeklyTorahPortion = () => { return ( - <Module> - <ModuleTitle>Weekly Torah Portion</ModuleTitle> + <SidebarModule> + <SidebarModuleTitle>Weekly Torah Portion</SidebarModuleTitle> <div className="readingsSection"> <span className="readingsSectionTitle"> <ParashahName /> @@ -485,22 +501,22 @@ const WeeklyTorahPortion = () => { <HebrewText>פרשות השבוע ›</HebrewText> </InterfaceText> </a> - </Module> + </SidebarModule> ); }; const DafYomi = () => { return ( - <Module> - <ModuleTitle>Daily Learning</ModuleTitle> + <SidebarModule> + <SidebarModuleTitle>Daily Learning</SidebarModuleTitle> <div className="readingsSection"> <span className="readingsSectionTitle"> <InterfaceText >Daf Yomi</InterfaceText> </span> <DafLink /> </div> - </Module> + </SidebarModule> ); }; @@ -535,8 +551,8 @@ const Visualizations = ({categories}) => { if (links.length == 0) { return null; } return ( - <Module> - <ModuleTitle>Visualizations</ModuleTitle> + <SidebarModule> + <SidebarModuleTitle>Visualizations</SidebarModuleTitle> <InterfaceText>Explore interconnections among texts with our interactive visualizations.</InterfaceText> <div className="linkList"> {links.map((link, i) => @@ -552,15 +568,15 @@ const Visualizations = ({categories}) => { <HebrewText>תרשימים גרפיים נוספים ›</HebrewText> </InterfaceText> </a> - </Module> + </SidebarModule> ); }; const AboutTopics = ({hideTitle}) => ( - <Module> + <SidebarModule> {hideTitle ? null : - <ModuleTitle>About Topics</ModuleTitle> } + <SidebarModuleTitle>About Topics</SidebarModuleTitle> } <InterfaceText> <HebrewText> דפי הנושא מציגים מקורות נבחרים מארון הספרים היהודי עבור אלפי נושאים. ניתן לדפדף לפי קטגוריה או לחפש לפי נושא ספציפי, ובסרגל הצד מוצגים הנושאים הפופולריים ביותר ואלה הקשורים אליהם. הקליקו ושוטטו בין הנושאים השונים כדי ללמוד עוד. @@ -569,19 +585,19 @@ const AboutTopics = ({hideTitle}) => ( Topics Pages present a curated selection of various genres of sources on thousands of chosen subjects. You can browse by category, search for something specific, or view the most popular topics — and related topics — on the sidebar. Explore and click through to learn more. </EnglishText> </InterfaceText> - </Module> + </SidebarModule> ); const TrendingTopics = () => ( - <Module> - <ModuleTitle>Trending Topics</ModuleTitle> + <SidebarModule> + <SidebarModuleTitle>Trending Topics</SidebarModuleTitle> {Sefaria.trendingTopics.map((topic, i) => <div className="navSidebarLink ref serif" key={i}> <a href={"/topics/" + topic.slug}><InterfaceText text={{en: topic.en, he: topic.he}}/></a> </div> )} - </Module> + </SidebarModule> ); @@ -594,8 +610,8 @@ const RelatedTopics = ({title}) => { Sefaria.getIndexDetails(title).then(data => setTopics(data.relatedTopics)); },[title]); return (topics.length ? - <Module> - <ModuleTitle>Related Topics</ModuleTitle> + <SidebarModule> + <SidebarModuleTitle>Related Topics</SidebarModuleTitle> {shownTopics.map((topic, i) => <div className="navSidebarLink ref serif" key={i}> <a href={"/topics/" + topic.slug}><InterfaceText text={{en: topic.title.en, he: topic.title.he}}/></a> @@ -605,7 +621,7 @@ const RelatedTopics = ({title}) => { <a className="moreLink" onClick={()=>{setShowMore(true);}}> <InterfaceText>More</InterfaceText> </a> : null} - </Module> : null + </SidebarModule> : null ); }; @@ -614,9 +630,9 @@ const JoinTheConversation = ({wide}) => { if (!Sefaria.multiPanel) { return null; } // Don't advertise create sheets on mobile (yet) return ( - <Module wide={wide}> + <SidebarModule wide={wide}> <div> - <ModuleTitle>Join the Conversation</ModuleTitle> + <SidebarModuleTitle>Join the Conversation</SidebarModuleTitle> <InterfaceText>Combine sources from our library with your own comments, questions, images, and videos.</InterfaceText> </div> <div> @@ -625,16 +641,16 @@ const JoinTheConversation = ({wide}) => { <InterfaceText>Make a Sheet</InterfaceText> </a> </div> - </Module> + </SidebarModule> ); }; const JoinTheCommunity = ({wide}) => { return ( - <Module wide={wide}> + <SidebarModule wide={wide}> <div> - <ModuleTitle>Join the Conversation</ModuleTitle> + <SidebarModuleTitle>Join the Conversation</SidebarModuleTitle> <InterfaceText>People around the world use Sefaria to create and share Torah resources. You're invited to add your voice.</InterfaceText> </div> <div> @@ -643,14 +659,14 @@ const JoinTheCommunity = ({wide}) => { <InterfaceText>Explore the Community</InterfaceText> </a> </div> - </Module> + </SidebarModule> ); }; const GetTheApp = () => ( - <Module> - <ModuleTitle>Get the Mobile App</ModuleTitle> + <SidebarModule> + <SidebarModuleTitle>Get the Mobile App</SidebarModuleTitle> <InterfaceText>Access the Jewish library anywhere and anytime with the</InterfaceText> <a href="/mobile" className="inTextLink"><InterfaceText>Sefaria mobile app.</InterfaceText></a> <br /> <AppStoreButton @@ -663,7 +679,7 @@ const GetTheApp = () => ( platform='android' altText={Sefaria._("Sefaria app on Android")} /> - </Module> + </SidebarModule> ); @@ -671,8 +687,8 @@ const StayConnected = () => { // TODO: remove? looks like we are not using this const fbURL = Sefaria.interfaceLang == "hebrew" ? "https://www.facebook.com/sefaria.org.il" : "https://www.facebook.com/sefaria.org"; return ( - <Module> - <ModuleTitle>Stay Connected</ModuleTitle> + <SidebarModule> + <SidebarModuleTitle>Stay Connected</SidebarModuleTitle> <InterfaceText>Get updates on new texts, learning resources, features, and more.</InterfaceText> <br /> <NewsletterSignUpForm context="sidebar" /> @@ -687,14 +703,14 @@ const StayConnected = () => { // TODO: remove? looks like we are not using this <img src="/static/icons/youtube.svg" alt={Sefaria._("Sefaria on YouTube")} /> </a> - </Module> + </SidebarModule> ); }; const AboutLearningSchedules = () => ( - <Module> - <ModuleTitle h1={true}>Learning Schedules</ModuleTitle> + <SidebarModule> + <SidebarModuleTitle h1={true}>Learning Schedules</SidebarModuleTitle> <InterfaceText> <EnglishText> Since biblical times, the Torah has been divided into sections which are read each week on a set yearly calendar. @@ -705,14 +721,14 @@ const AboutLearningSchedules = () => ( בעקבות המנהג הזה התפתחו לאורך השנים סדרי לימוד תקופתיים רבים נוספים, ובעזרתם יכולות קהילות וקבוצות של לומדים ללמוד יחד טקסטים שלמים. </HebrewText> </InterfaceText> - </Module> + </SidebarModule> ); const AboutCollections = ({hideTitle}) => ( - <Module> + <SidebarModule> {hideTitle ? null : - <ModuleTitle h1={true}>About Collections</ModuleTitle>} + <SidebarModuleTitle h1={true}>About Collections</SidebarModuleTitle>} <InterfaceText> <EnglishText>Collections are user generated bundles of sheets which can be used privately, shared with friends, or made public on Sefaria.</EnglishText> <HebrewText>אסופות הן מקבצים של דפי מקורות שנוצרו על ידי משתמשי האתר. הן ניתנות לשימוש פרטי, לצורך שיתוף עם אחרים או לשימוש ציבורי באתר ספריא.</HebrewText> @@ -724,13 +740,13 @@ const AboutCollections = ({hideTitle}) => ( <InterfaceText>Create a Collection</InterfaceText> </a> </div>} - </Module> + </SidebarModule> ); const ExploreCollections = () => ( - <Module> - <ModuleTitle>Collections</ModuleTitle> + <SidebarModule> + <SidebarModuleTitle>Collections</SidebarModuleTitle> <InterfaceText>Organizations, communities and individuals around the world curate and share collections of sheets for you to explore.</InterfaceText> <div> <a className="button small white" href="/collections"> @@ -738,31 +754,31 @@ const ExploreCollections = () => ( <InterfaceText>Explore Collections</InterfaceText> </a> </div> - </Module> + </SidebarModule> ); const WhoToFollow = ({toggleSignUpModal}) => ( - <Module> - <ModuleTitle>Who to Follow</ModuleTitle> + <SidebarModule> + <SidebarModuleTitle>Who to Follow</SidebarModuleTitle> {Sefaria.followRecommendations.map(user => <ProfileListing {...user} key={user.uid} toggleSignUpModal={toggleSignUpModal} />)} - </Module> + </SidebarModule> ); const Image = ({url}) => ( - <Module> + <SidebarModule> <img className="imageModuleImage" src={url} /> - </Module> + </SidebarModule> ); const Wrapper = ({title, content}) => ( - <Module> - {title ? <ModuleTitle>{title}</ModuleTitle> : null} + <SidebarModule> + {title ? <SidebarModuleTitle>{title}</SidebarModuleTitle> : null} {content} - </Module> + </SidebarModule> ); @@ -830,8 +846,8 @@ const DownloadVersions = ({sref}) => { }, [sref]); return( - <Module> - <ModuleTitle>Download Text</ModuleTitle> + <SidebarModule> + <SidebarModuleTitle>Download Text</SidebarModuleTitle> <div className="downloadTextModule sans-serif"> <Dropdown name="dlVersionName" @@ -862,51 +878,51 @@ const DownloadVersions = ({sref}) => { /> <a className={`button fillWidth${isReady ? "" : " disabled"}`} onClick={handleClick} href={versionDlLink()} download>{Sefaria._("Download")}</a> </div> - </Module> + </SidebarModule> ); }; const PortalAbout = ({title, description, image_uri, image_caption}) => { return( - <Module> - <ModuleTitle en={title.en} he={title.he} /> + <SidebarModule> + <SidebarModuleTitle en={title.en} he={title.he} /> <div className="portalTopicImageWrapper"> <ImageWithCaption photoLink={image_uri} caption={image_caption} /> </div> <InterfaceText markdown={{en: description.en, he: description.he}} /> - </Module> + </SidebarModule> ) }; const PortalMobile = ({title, description, android_link, ios_link}) => { return( - <Module> + <SidebarModule> <div className="portalMobile"> - <ModuleTitle en={title.en} he={title.he} /> + <SidebarModuleTitle en={title.en} he={title.he} /> {description && <InterfaceText markdown={{en: description.en, he: description.he}} />} <AppStoreButton href={ios_link} platform={'ios'} altText='Steinsaltz app on iOS' /> <AppStoreButton href={android_link} platform={'android'} altText='Steinsaltz app on Android' /> </div> - </Module> + </SidebarModule> ) }; const PortalOrganization = ({title, description}) => { return( - <Module> - <ModuleTitle en={title.en} he={title.he} /> + <SidebarModule> + <SidebarModuleTitle en={title.en} he={title.he} /> {description && <InterfaceText markdown={{en: description.en, he: description.he}} />} - </Module> + </SidebarModule> ) }; const PortalNewsletter = ({title, description}) => { - let titleElement = <ModuleTitle en={title.en} he={title.he} />; + let titleElement = <SidebarModuleTitle en={title.en} he={title.he} />; return( - <Module> + <SidebarModule> {titleElement} <InterfaceText markdown={{en: description.en, he: description.he}} /> <NewsletterSignUpForm @@ -914,13 +930,13 @@ const PortalNewsletter = ({title, description}) => { emailPlaceholder={{en: "Email Address", he: "כתובת מייל"}} subscribe={Sefaria.subscribeSefariaAndSteinsaltzNewsletter} /> - </Module> + </SidebarModule> ) }; export { NavSidebar, - Modules, + SidebarModules, RecentlyViewed }; diff --git a/static/js/NotificationsPanel.jsx b/static/js/NotificationsPanel.jsx index 27abe785d4..567ba1e7a4 100644 --- a/static/js/NotificationsPanel.jsx +++ b/static/js/NotificationsPanel.jsx @@ -90,7 +90,7 @@ class NotificationsPanel extends Component { notifications : <LoginPrompt fullPanel={true} /> } </div> - <NavSidebar modules={sidebarModules} /> + <NavSidebar sidebarModules={sidebarModules} /> </div> <Footer /> </div> diff --git a/static/js/PublicCollectionsPage.jsx b/static/js/PublicCollectionsPage.jsx index 5d71fd7b16..cb91d48041 100644 --- a/static/js/PublicCollectionsPage.jsx +++ b/static/js/PublicCollectionsPage.jsx @@ -4,7 +4,7 @@ import classNames from 'classnames'; import Footer from './Footer'; import Sefaria from './sefaria/sefaria'; import Component from 'react-class'; -import { NavSidebar, Modules } from './NavSidebar'; +import { NavSidebar, SidebarModules } from './NavSidebar'; import { InterfaceText, LoadingMessage, @@ -70,7 +70,7 @@ const PublicCollectionsPage = ({multiPanel, initialWidth}) => { </h1> {multiPanel ? null : - <Modules type={"AboutCollections"} props={{hideTitle: true}} /> } + <SidebarModules type={"AboutCollections"} props={{hideTitle: true}} /> } <div className="collectionsList"> { !!collectionsList ? @@ -89,7 +89,7 @@ const PublicCollectionsPage = ({multiPanel, initialWidth}) => { : <LoadingMessage /> } </div> </div> - <NavSidebar modules={sidebarModules} /> + <NavSidebar sidebarModules={sidebarModules} /> </div> <Footer /> </div> diff --git a/static/js/ReaderApp.jsx b/static/js/ReaderApp.jsx index a0d1657eaa..2218baf842 100644 --- a/static/js/ReaderApp.jsx +++ b/static/js/ReaderApp.jsx @@ -1010,15 +1010,24 @@ toggleSignUpModal(modalContentKind = SignUpModalKind.Default) { showHighlight: true, currentlyVisibleRef: refs, } - this.replacePanel(n-1, ref, currVersions, new_opts); + this.replacePanel(n-1, ref, currVersions, new_opts, false); } getHTMLLinkParentOfEventTarget(event){ //get the lowest level parent element of an event target that is an HTML link tag. Or Null. - let target = event.target, - parent = target, - outmost = event.currentTarget; + return this.getEventTargetByCondition(event, element => element.nodeName === "A"); + } + getEventTargetByCondition(event, condition, eventTarget=null) { + /** + * Searches the parents of an event target for an element to meets a certain condition + * `condition` is a function of form condition(element) => bool. + * If `eventTarget` is passed, it will be used as the starting point of the search instead of `event.target` + * Returns the first element in parent hierarchy where `condition` returns true + * If no element returns true, returns null. + */ + let parent = eventTarget || event.target; + const outmost = event.currentTarget; while (parent) { - if(parent.nodeName === 'A'){ + if(condition(parent)){ return parent } else if (parent.parentNode === outmost) { @@ -1039,6 +1048,14 @@ toggleSignUpModal(modalContentKind = SignUpModalKind.Default) { } } } + handleAppClick(event) { + if (linkTarget) { + this.handleInAppLinkClick(event); + } + if (this.eventIsAnalyticsEvent(event)) { + this.handleAnalyticsEvent(event); + } + } handleInAppLinkClick(e) { //Allow global navigation handling in app via link elements // If a default has been prevented, assume a custom handler is already in place @@ -1530,9 +1547,9 @@ toggleSignUpModal(modalContentKind = SignUpModalKind.Default) { openPanelAtEnd(ref, currVersions) { this.openPanelAt(this.state.panels.length+1, ref, currVersions); } - replacePanel(n, ref, currVersions, options) { + replacePanel(n, ref, currVersions, options, convertCommentaryRefToBaseRef=true) { // Opens a text in in place of the panel currently open at `n`. - this.openPanelAt(n, ref, currVersions, options, true); + this.openPanelAt(n, ref, currVersions, options, true, convertCommentaryRefToBaseRef); } openComparePanel(n, connectAfter) { const comparePanel = this.makePanelState({ diff --git a/static/js/ReaderPanel.jsx b/static/js/ReaderPanel.jsx index abff7d53c0..8197774615 100644 --- a/static/js/ReaderPanel.jsx +++ b/static/js/ReaderPanel.jsx @@ -111,17 +111,19 @@ class ReaderPanel extends Component { // Because it's called in the constructor, assume state isnt necessarily defined and pass // variables mode and menuOpen manually let contentLangOverride = originalLanguage; - if (["topics", "allTopics", "calendars", "community", "collection" ].includes(menuOpen)) { // "story_editor", - // Always bilingual for English interface, always Hebrew for Hebrew interface - contentLangOverride = (Sefaria.interfaceLang === "english") ? "bilingual" : "hebrew"; - - } else if (mode === "Connections" || !!menuOpen){ + if (["topics"].includes(menuOpen)) { + contentLangOverride = "bilingual"; + } + else if (mode === "Connections" || !!menuOpen){ // Always Hebrew for Hebrew interface, treat bilingual as English for English interface contentLangOverride = (Sefaria.interfaceLang === "hebrew") ? "hebrew" : ((originalLanguage === "bilingual") ? "english" : originalLanguage); - } return contentLangOverride; } + getContentLanguageOverrideStateful() { + // same as getContentLanguageOverride() but relies on values in this.state + return this.getContentLanguageOverride(this.state.settings.language, this.state.mode, this.state.menuOpen); + } clonePanel(panel) { // Todo: Move the multiple instances of this out to a utils file return Sefaria.util.clone(panel); @@ -615,6 +617,30 @@ class ReaderPanel extends Component { const highlighted = this.getHighlightedByScrollPos(); highlighted.click(); } + getPanelType() { + const {menuOpen, tab} = this.state; + if (menuOpen === "topics") { + return `${menuOpen}_${tab}`; + } + } + getPanelName() { + const {menuOpen, navigationTopic, navigationTopicCategory} = this.state; + if (menuOpen === "topics") { + return navigationTopicCategory || navigationTopic; + } + } + getPanelNumber() { + // TODO update for commentary panels + return this.props.panelPosition+1; + } + getAnalyticsData() { + return { + panel_type: this.getPanelType(), + panel_number: this.getPanelNumber(), + content_lang: this.getContentLanguageOverrideStateful(), + panel_name: this.getPanelName(), + }; + } render() { if (this.state.error) { return ( @@ -636,7 +662,7 @@ class ReaderPanel extends Component { let items = []; let menu = null; - const contextContentLang = {"language": this.getContentLanguageOverride(this.state.settings.language, this.state.mode, this.state.menuOpen)}; + const contextContentLang = {"language": this.getContentLanguageOverrideStateful()}; if (this.state.mode === "Text" || this.state.mode === "TextAndConnections") { const oref = Sefaria.parseRef(this.state.refs[0]); @@ -1081,7 +1107,8 @@ class ReaderPanel extends Component { ); return ( <ContentLanguageContext.Provider value={contextContentLang}> - <div ref={this.readerContentRef} className={classes} onKeyDown={this.handleKeyPress} role="region" id={"panel-"+this.props.panelPosition}> + <div ref={this.readerContentRef} className={classes} onKeyDown={this.handleKeyPress} role="region" + id={"panel-"+this.props.panelPosition} data-anl-batch={JSON.stringify(this.getAnalyticsData())}> {hideReaderControls ? null : <ReaderControls showBaseText={this.showBaseText} diff --git a/static/js/SearchResultList.jsx b/static/js/SearchResultList.jsx index d2b6f65aaa..2083dce806 100644 --- a/static/js/SearchResultList.jsx +++ b/static/js/SearchResultList.jsx @@ -183,7 +183,7 @@ class SearchResultList extends Component { numSheets: 0, url: "/topics/" + topic.key } - const typeObj = Sefaria.topicTocCategory(topic.key); + const typeObj = Sefaria.displayTopicTocCategory(topic.key); if (!typeObj) { searchTopic.topicCat = "Topics"; searchTopic.heTopicCat = Sefaria.hebrewTranslation("Topics"); diff --git a/static/js/SourceEditor.jsx b/static/js/SourceEditor.jsx index 1755c74efe..6efa15b9bc 100644 --- a/static/js/SourceEditor.jsx +++ b/static/js/SourceEditor.jsx @@ -53,8 +53,10 @@ const SourceEditor = ({topic, close, origData={}}) => { interface_lang: Sefaria.interfaceLang, description: {"title": data.enTitle, "prompt": data.prompt, "ai_context": data.ai_context, "review_state": "edited"}, } + const currentUrlObj = new URL(window.location.href); + const tabName = currentUrlObj.searchParams.get('tab'); Sefaria.postRefTopicLink(refInUrl, payload) - .then(() => window.location.href = `/topics/${topic}`) + .then(() => window.location.href = `/topics/${topic}?sort=Relevance&tab=${tabName}`) .finally(() => setSavingStatus(false)); } diff --git a/static/js/StaticPages.jsx b/static/js/StaticPages.jsx index f3c4563dd7..cdb0ff5ec4 100644 --- a/static/js/StaticPages.jsx +++ b/static/js/StaticPages.jsx @@ -1437,8 +1437,8 @@ const DonatePage = () => ( heText="" enButtonText="Donate Now" heButtonText="" - enButtonUrl="https://donate.sefaria.org/english?c_src=waystogive" - heButtonUrl="https://donate.sefaria.org/he?c_src=waystogive" + enButtonUrl="https://donate.sefaria.org/give/451346/#!/donation/checkout?c_src=ways-to-give" + heButtonUrl="https://donate.sefaria.org/give/468442/#!/donation/checkout?c_src=ways-to-give" borderColor="#004E5F" />, <FeatureBox @@ -1448,8 +1448,8 @@ const DonatePage = () => ( heText="" enButtonText="Join the Sustainers" heButtonText="" - enButtonUrl="https://donate.sefaria.org/sustainers?c_src=waystogive" - heButtonUrl="https://donate.sefaria.org/sustainershe?c_src=waystogive" + enButtonUrl="https://donate.sefaria.org/give/457760/#!/donation/checkout?c_src=waystogive" + heButtonUrl="https://donate.sefaria.org/give/478929/#!/donation/checkout?c_src=waystogive" borderColor="#97B386" />, <FeatureBox @@ -1459,8 +1459,8 @@ const DonatePage = () => ( heText="" enButtonText="Sponsor a Day of Learning" heButtonText="" - enButtonUrl="https://donate.sefaria.org/sponsor?c_src=waystogive" - heButtonUrl="https://donate.sefaria.org/sponsorhe?c_src=waystogive" + enButtonUrl="https://donate.sefaria.org/campaign/sponsor-a-day-of-learning/c460961?c_src=waystogive" + heButtonUrl="https://donate.sefaria.org/campaign/sponsor-a-day-of-learning-hebrew/c479003?c_src=waystogive" borderColor="#4B71B7" />, <FeatureBox @@ -1493,7 +1493,7 @@ const DonatePage = () => ( <HeaderWithColorAccentBlockAndText enTitle="Donate Online" heTitle="" - enText="<p>Make a donation by <strong>credit card, PayPal, GooglePay, ApplePay, Venmo, or bank transfer</strong> on our <a href='http://donate.sefaria.org/english'>main donation page</a>.</p>" + enText="<p>Make a donation by <strong>credit card, PayPal, GooglePay, ApplePay, Venmo, or bank transfer</strong> on our <a href='https://donate.sefaria.org/give/451346/#!/donation/checkout?c_src=waystogive'>main donation page</a>.</p>" heText="" colorBar="#AB4E66" />, @@ -1566,7 +1566,7 @@ const DonatePage = () => ( <Accordian enTitle="Can I make a gift to support a specific program or initiative?" heTitle="" - enText="<p>Our online giving page does not support restricted gifts. You can sponsor a day of learning <a href='https://donate.sefaria.org/sponsor'>here</a>. If you would like to sponsor a text or support a specific Sefaria program, please email Samantha Shokin, Grant Writer and Development Associate, at <a href='mailto:samantha@sefaria.org'>samantha@sefaria.org</a> for more information.</p>" + enText="<p>Our online giving page does not support restricted gifts. You can sponsor a day of learning <a href='https://donate.sefaria.org/campaign/sponsor-a-day-of-learning/c460961?c_src=waystogive'>here</a>. If you would like to sponsor a text or support a specific Sefaria program, please email Samantha Shokin, Grant Writer and Development Associate, at <a href='mailto:samantha@sefaria.org'>samantha@sefaria.org</a> for more information.</p>" heText="" colorBar="#B8D4D3" /> @@ -1605,7 +1605,7 @@ const DonatePage = () => ( <Accordian enTitle="Can I still donate from outside the USA?" heTitle="" - enText="<p>Yes! Donors outside of the USA may make a gift online – via credit card, PayPal, GooglePay, ApplePay, Venmo, and bank transfer – <a href='https://donate.sefaria.org/english'>on this page</a> On this page you can modify your currency. You can also <a href='https://sefaria.formstack.com/forms/wire_request'>make a wire transfer</a>.</p>" + enText="<p>Yes! Donors outside of the USA may make a gift online – via credit card, PayPal, GooglePay, ApplePay, Venmo, and bank transfer – <a href='https://donate.sefaria.org/give/451346/#!/donation/checkout?c_src=waystogive'>on this page</a> On this page you can modify your currency. You can also <a href='https://sefaria.formstack.com/forms/wire_request'>make a wire transfer</a>.</p>" heText="" colorBar="#7F85A9" /> @@ -3138,21 +3138,21 @@ const JobsPage = memo(() => { // The static content on the page inviting users to browse our "powered-by" products const DevBox = () => { return ( - <div className='productsDevBox'> - <p className='productsDevHeader'> + <aside className='productsDevBox'> + <div className='productsDevHeader'> <InterfaceText text={{en: "Powered by Sefaria" , he:"פרויקטים מכח ספריא" }} /> - </p> - <p> + </div> + <div> <InterfaceText> <HebrewText> - נסו את המוצרים שמפתחי תוכנה וידידי ספריא מרחבי העולם בנו עבורכם! <a href="www.example.com">גלו את הפרויקטים</a> + נסו את המוצרים שמפתחי תוכנה וידידי ספריא מרחבי העולם בנו עבורכם! <a href="https://developers.sefaria.org/docs/powered-by-sefaria">גלו את הפרויקטים</a> </HebrewText> <EnglishText> - Check out the products our software developer friends from around the world have been building for you! <a href="www.example.com">Explore</a> + Check out the products our software developer friends from around the world have been building for you! <a href="https://developers.sefaria.org/docs/powered-by-sefaria">Explore</a> </EnglishText> </InterfaceText> - </p> - </div> + </div> + </aside> ); }; @@ -3439,18 +3439,18 @@ const ProductsPage = memo(() => { <h1 className="mobileAboutHeader"> <span className="int-en">Sefaria's Products</span> - <span className="int-he">מוצרים של בספריא</span> + <span className="int-he">המוצרים של ספריא</span> </h1> - <div className='productsFlexWrapper'> - {products && products.length > 0 ? ( - <> - {initialProducts} - {/* <DevBox /> */} - {remainingProducts} - </> - ) : null} - </div> - </> + <div className='productsFlexWrapper'> + {products && products.length > 0 ? ( + <> + {initialProducts} + <DevBox /> + {remainingProducts} + </> + ) : null} + </div> + </> ); }); diff --git a/static/js/Story.jsx b/static/js/Story.jsx index 8aaacc0d89..693cae58dd 100644 --- a/static/js/Story.jsx +++ b/static/js/Story.jsx @@ -11,9 +11,10 @@ import { InterfaceText, EnglishText, HebrewText, - CategoryHeader, + CategoryHeader, PencilSourceEditor, } from './Misc'; import {ContentText} from "./ContentText"; +import {SourceEditor} from "./SourceEditor"; // Much of Stories was removed November 2022. // It remains because some of the Components are re-used in other areas of the site. @@ -72,15 +73,39 @@ SheetListStory.propTypes = { *****************************/ // todo: if we don't want the monopoly card effect, this component isn't needed. // style={{"borderColor": cardColor || "#18345D"}}> -const StoryFrame = ({cls, cardColor, children}) => ( +const StoryFrame = ({cls, children}) => { +return ( <div className={'story ' + cls}> - {children} + {children} </div> -); +)}; StoryFrame.propTypes = { cls: PropTypes.string, // Story type as class name - cardColor: PropTypes.string }; +const SummarizedStoryFrame = ({cls, title, tref, children}) => { +return ( + <StoryFrame cls={cls}> + <details + data-anl-event="package_toggle:toggle|package_viewed:scrollIntoView" + data-anl-feature_name="prompt" + data-anl-text={title.en} + > + <summary> + <ColorBarBox tref={tref}> + <TopicStoryDescBlock title={title} tref={tref}/> + </ColorBarBox> + </summary> + {children} + </details> + </StoryFrame> +)}; +SummarizedStoryFrame.propTypes = { + cls: PropTypes.string, // Story type as class name + title: PropTypes.object, + tref: PropTypes.string, +}; + + const StoryTypeBlock = ({en, he}) => ( @@ -129,7 +154,60 @@ StorySheetList.propTypes = { toggleSignUpModal: PropTypes.func }; +const TopicStoryDescBlock = ({title, tref}) => + ( + <div className="topicStoryDescBlock"> + <StoryTitleBlock en={title.en} he={title.he}></StoryTitleBlock> + <div>{Sefaria._(Sefaria.index(Sefaria.parseRef(tref).index).primary_category).toUpperCase()}</div> + </div> +) + +const TopicTextPassage = ({text, topic, bodyTextIsLink=false, langPref, displayDescription, isAdmin, hideLanguageMissingSources=false}) => { + if (!text.ref) { + return null; + } + const langKey = Sefaria.interfaceLang === 'english' ? 'en' : 'he'; + const isCurated = text.descriptions?.[langKey]?.title ?? false; + const url = `/${Sefaria.normRef(text.ref)}${Sefaria.util.getUrlVersionsParams(text.versions) ? `?${Sefaria.util.getUrlVersionsParams(text.versions)}` : ''}`; + const heOnly = !text.en; + const enOnly = !text.he; + const overrideLanguage = (enOnly || heOnly) ? (heOnly ? "hebrew" : "english") : langPref; + let innerContent = <ContentText html={{en: text.en, he: text.he}} overrideLanguage={overrideLanguage} + bilingualOrder={["he", "en"]}/>; + const content = bodyTextIsLink ? <a href={url} style={{textDecoration: 'none'}} data-anl-event="clickto_reader:click">{innerContent}</a> : innerContent; + const isIntroducedSource = isCurated && displayDescription + const StoryFrameComp = isIntroducedSource ? SummarizedStoryFrame : StoryFrame + const hideThisLanguageMissingSource = (heOnly && (langPref == 'english') && hideLanguageMissingSources) || (enOnly && (langPref == 'hebrew') && hideLanguageMissingSources); + return ( + !hideThisLanguageMissingSource && + <StoryFrameComp cls="topicPassageStory" tref={text.ref} title={{'en': text.descriptions?.en?.title, 'he': text.descriptions?.he?.title}}> + {isIntroducedSource ? + <ColorBarBox tref={text.ref}> + + <div className={"systemText learningPrompt"}> + <InterfaceText + text={{"en": text.descriptions?.en?.prompt, "he": text.descriptions?.he?.prompt}}/> + </div> + </ColorBarBox> + + : null + } + <ColorBarBox tref={text.ref}> + <StoryBodyBlock> + {content} + </StoryBodyBlock> + <div className={"headerWithAdminButtonsContainer"}> + <div className={"headerWithAdminButtons"} data-anl-event="clickto_reader:click"> + <SimpleLinkedBlock classes={"contentText subHeading"} en={text.ref} he={text.heRef} url={url}/> + </div> + {isAdmin && <ReviewStateIndicator topic={topic} topicLink={text}/>} + {isAdmin && <PencilSourceEditor topic={topic} text={text} classes={"pencilEditorButton"}/>} + </div> + </ColorBarBox> + </StoryFrameComp> + ); +}; const reviewStateToClassNameMap = { "reviewed": "reviewed", "not reviewed": "notReviewed", @@ -142,6 +220,7 @@ const reviewStateToDisplayedTextMap = { } const ReviewStateIndicator = ({topic, topicLink}) => { + if (!topicLink.descriptions){ return null; } const [reviewStateByLang, markReviewed] = useReviewState(topic, topicLink); if (!Sefaria.is_moderator){ return null; } const langComponentMap = {he: HebrewText, en: EnglishText}; @@ -193,51 +272,12 @@ const useReviewState = (topic, topicLink) => { return [reviewStateByLang, markReviewed]; } -const IntroducedTextPassage = ({text, topic, afterSave, toggleSignUpModal, bodyTextIsLink=false}) => { - if (!text.ref) { return null; } - const versions = text.versions || {} - const params = Sefaria.util.getUrlVersionsParams(versions); - const url = "/" + Sefaria.normRef(text.ref) + (params ? "?" + params : ""); - const heOnly = !text.en; - const enOnly = !text.he; - const overrideLanguage = (enOnly || heOnly) ? (heOnly ? "hebrew" : "english") : null; - let innerContent = <ContentText html={{en: text.en, he: text.he}} overrideLanguage={overrideLanguage} bilingualOrder={["he", "en"]} />; - const content = bodyTextIsLink ? <a href={url} style={{ textDecoration: 'none' }}>{innerContent}</a> : innerContent; - - return ( - <StoryFrame cls="introducedTextPassageStory"> - <div className={"headerWithAdminButtonsContainer"}> - <CategoryHeader type="sources" data={[topic, text]} toggleButtonIDs={["edit"]}> - <StoryTitleBlock en={text.descriptions?.en?.title} he={text.descriptions?.he?.title}/> - </CategoryHeader> - <ReviewStateIndicator topic={topic} topicLink={text}/> - </div> - <div className={"systemText learningPrompt"}> - <InterfaceText text={{"en": text.descriptions?.en?.prompt, "he": text.descriptions?.he?.prompt}} /> - </div> - <SaveLine - dref={text.ref} - versions={versions} - toggleSignUpModal={toggleSignUpModal} - classes={"storyTitleWrapper"} - afterChildren={afterSave || null} > - <SimpleLinkedBlock classes={"contentText subHeading"} en={text.ref} he={text.heRef} url={url}/> - </SaveLine> - <ColorBarBox tref={text.ref}> - <StoryBodyBlock> - {content} - </StoryBodyBlock> - </ColorBarBox> - </StoryFrame> - ); -}; -IntroducedTextPassage.propTypes = { - intros: PropTypes.object, - text: textPropType, - afterSave: PropTypes.object, - toggleSignUpModal: PropTypes.func +TopicTextPassage.propTypes = { + text: textPropType, }; + + const TextPassage = ({text, topic, afterSave, toggleSignUpModal, bodyTextIsLink=false}) => { if (!text.ref) { return null; } const versions = text.versions || {} @@ -275,7 +315,6 @@ TextPassage.propTypes = { toggleSignUpModal: PropTypes.func }; - const SheetBlock = ({sheet, compact, cozy, smallfonts, afterSave, toggleSignUpModal}) => { const historyObject = { ref: "Sheet " + sheet.sheet_id, @@ -290,17 +329,17 @@ const SheetBlock = ({sheet, compact, cozy, smallfonts, afterSave, toggleSignUpMo historyObject={historyObject} afterChildren={afterSave || null} toggleSignUpModal={toggleSignUpModal}> - <SimpleLinkedBlock + <SimpleLinkedBlock en={sheet.sheet_title} he={sheet.sheet_title} url={"/sheets/" + sheet.sheet_id} classes={"sheetTitle storyTitle"}/> </SaveLine> - {(sheet.sheet_summary && !(compact || cozy)) ? + {(sheet.sheet_summary && !(compact || cozy)) ? <SimpleInterfaceBlock classes={"storyBody"} en={sheet.sheet_summary} he={sheet.sheet_summary}/> : null} - + {cozy ? null : <ProfileListing uid={sheet.publisher_id} @@ -350,5 +389,5 @@ export { SheetBlock, StorySheetList, TextPassage, - IntroducedTextPassage + TopicTextPassage, }; diff --git a/static/js/TextCategoryPage.jsx b/static/js/TextCategoryPage.jsx index c8bcb4b9bb..a8e48de4e2 100644 --- a/static/js/TextCategoryPage.jsx +++ b/static/js/TextCategoryPage.jsx @@ -101,7 +101,7 @@ const TextCategoryPage = ({category, categories, setCategories, toggleLanguage, initialWidth={initialWidth} nestLevel={nestLevel} /> </div> - {!compare ? <NavSidebar modules={sidebarModules} /> : null} + {!compare ? <NavSidebar sidebarModules={sidebarModules} /> : null} </div> {footer} </div> diff --git a/static/js/TextRange.jsx b/static/js/TextRange.jsx index 777f34ada4..ce18332a3d 100644 --- a/static/js/TextRange.jsx +++ b/static/js/TextRange.jsx @@ -309,8 +309,7 @@ class TextRange extends Component { } const formatEnAsPoetry = data && data.formatEnAsPoetry const formatHeAsPoetry = data && data.formatHeAsPoetry - - const showNumberLabel = data && data.categories && + const showNumberLabel = data && data.categories && !data.ref.startsWith("Guide for the Perplexed") && data.categories[0] !== "Liturgy" && data.categories[0] !== "Reference"; diff --git a/static/js/TextsPage.jsx b/static/js/TextsPage.jsx index 53ef228dfb..8d0ce960cc 100644 --- a/static/js/TextsPage.jsx +++ b/static/js/TextsPage.jsx @@ -3,7 +3,7 @@ import PropTypes from 'prop-types'; import classNames from 'classnames'; import Sefaria from './sefaria/sefaria'; import $ from './sefaria/sefariaJquery'; -import { NavSidebar, Modules, RecentlyViewed } from './NavSidebar'; +import { NavSidebar, SidebarModules, RecentlyViewed } from './NavSidebar'; import TextCategoryPage from './TextCategoryPage'; import Footer from './Footer'; import ComparePanelHeader from './ComparePanelHeader'; @@ -79,7 +79,7 @@ const TextsPage = ({categories, settings, setCategories, onCompareBack, openSear </div> const about = compare || multiPanel ? null : - <Modules type={"AboutSefaria"} props={{hideTitle: true}}/>; + <SidebarModules type={"AboutSefaria"} props={{hideTitle: true}}/>; const dedication = Sefaria._siteSettings.TORAH_SPECIFIC && !compare ? <Dedication /> : null; @@ -112,7 +112,7 @@ const TextsPage = ({categories, settings, setCategories, onCompareBack, openSear { !multiPanel && <RecentlyViewed toggleSignUpModal={toggleSignUpModal} mobile={true}/>} { categoryListings } </div> - {!compare ? <NavSidebar modules={sidebarModules} /> : null} + {!compare ? <NavSidebar sidebarModules={sidebarModules} /> : null} </div> {footer} </div> diff --git a/static/js/TopicPage.jsx b/static/js/TopicPage.jsx index 0d25a0dac8..198b974f22 100644 --- a/static/js/TopicPage.jsx +++ b/static/js/TopicPage.jsx @@ -10,8 +10,7 @@ import {TopicEditor} from './TopicEditor'; import {AdminEditorButton, useEditToggle} from './AdminEditor'; import { SheetBlock, - TextPassage, - IntroducedTextPassage + TopicTextPassage, } from './Story'; import { TabView, @@ -26,7 +25,8 @@ import { CategoryHeader, ImageWithCaption, EnglishText, - HebrewText + HebrewText, + LangSelectInterface, } from './Misc'; import {ContentText} from "./ContentText"; @@ -159,8 +159,19 @@ const hasPrompts = (description) => { */ return description?.title?.length && (Sefaria.is_moderator || description?.published !== false); } - -const refRenderWrapper = (toggleSignUpModal, topicData, topicTestVersion) => item => { +const adminRefRenderWrapper = (toggleSignUpModal, topicData, topicTestVersion, langPref) => refRenderWrapper(toggleSignUpModal, topicData, topicTestVersion, langPref, true, true, false); +const notableSourcesRefRenderWrapper = (toggleSignUpModal, topicData, topicTestVersion, langPref) => refRenderWrapper(toggleSignUpModal, topicData, topicTestVersion, langPref, false, true, true); +const allSourcesRefRenderWrapper = (toggleSignUpModal, topicData, topicTestVersion, langPref) => refRenderWrapper(toggleSignUpModal, topicData, topicTestVersion, langPref, false, false, true); + +const _extractAnalyticsDataFromRef = ref => { + const title = Sefaria.parseRef(ref).index; + return { + item_id: ref, + content_id: title, + content_type: Sefaria.index(title)?.categories?.join('|'), + }; +}; +const refRenderWrapper = (toggleSignUpModal, topicData, topicTestVersion, langPref, isAdmin, displayDescription, hideLanguageMissingSources) => (item, index) => { const text = item[1]; const topicTitle = topicData && topicData.primaryTitle; const langKey = Sefaria.interfaceLang === 'english' ? 'en' : 'he'; @@ -176,19 +187,22 @@ const refRenderWrapper = (toggleSignUpModal, topicData, topicTestVersion) => ite </ToolTipped> ); - - // When running a test, topicTestVersion is respected. - // const Passage = (topicTestVersion && hasPrompts) ? IntroducedTextPassage : TextPassage; - const Passage = hasPrompts(text?.descriptions?.[langKey]) ? IntroducedTextPassage : TextPassage; return ( - <Passage - key={item[0]} - topic={topicData.slug} - text={text} - afterSave={afterSave} - toggleSignUpModal={toggleSignUpModal} - bodyTextIsLink= {true} - /> + <div key={item[0]} data-anl-batch={JSON.stringify({ + position: index, + ai: doesLinkHaveAiContent(langKey, text) ? 'ai' : 'human', + ..._extractAnalyticsDataFromRef(text.ref), + })}> + <TopicTextPassage + topic={topicData.slug} + text={text} + bodyTextIsLink= {true} + langPref={langPref} + isAdmin={isAdmin} + displayDescription={displayDescription} + hideLanguageMissingSources={hideLanguageMissingSources} + /> + </div> ); }; @@ -247,12 +261,15 @@ const TopicCategory = ({topic, topicTitle, setTopic, setNavTopic, compare, initi ); }); - const sidebarModules = [ + let sidebarModules = [ {type: "AboutTopics"}, {type: "Promo"}, {type: "TrendingTopics"}, {type: "SponsorADay"}, ]; + if (topic === "torah-portions" && Sefaria.interfaceLang === "english") { + sidebarModules.splice(1, 0, {type: "StudyCompanion"}); + } return ( <div className="readerNavMenu noLangToggleInHebrew"> @@ -268,7 +285,7 @@ const TopicCategory = ({topic, topicTitle, setTopic, setNavTopic, compare, initi <ResponsiveNBox content={topicBlocks} initialWidth={initialWidth} /> </div> </div> - <NavSidebar modules={sidebarModules} /> + <NavSidebar sidebarModules={sidebarModules} /> </div> <Footer /> </div> @@ -302,6 +319,10 @@ const TopicSponsorship = ({topic_slug}) => { "parashat-vaetchanan": { "en": "Shabbat Nachamu learning is dedicated in memory of Jerome L. Stern, Yehuda Leib ben David Shmuel, z\"l.", "he": "הלימוד לשבת נחמו מוקדש לזכרו של ג'רום ל. שטרן, יהודה לייב בן דוד שמואל ז\"ל." + }, + "parashat-vzot-haberachah": { + "en": "Parashat VeZot HaBerakhah is dedicated to the victims of the October 7th, 2023, terrorist attack in Israel.", + "he": "פרשת ״וזאת הברכה״ מוקדשת לקורבנות מתקפת הטרור והטבח הנורא שאירע ב-7 באוקטובר 2023." } }; @@ -315,13 +336,13 @@ const TopicSponsorship = ({topic_slug}) => { ); } -const isLinkPublished = (lang, link) => {return link.descriptions?.[lang]?.published !== false;} +const isLinkPublished = (lang, link) => link.descriptions?.[lang]?.published !== false; + +const doesLinkHaveAiContent = (lang, link) => link.descriptions?.[lang]?.ai_title?.length > 0 && isLinkPublished(lang, link); const getLinksWithAiContent = (refTopicLinks = []) => { const lang = Sefaria.interfaceLang === "english" ? 'en' : 'he'; - return refTopicLinks.filter(link => { - return link.descriptions?.[lang]?.ai_title?.length > 0 && isLinkPublished(lang, link) - }); + return refTopicLinks.filter(link => doesLinkHaveAiContent(lang, link)); }; const getLinksToGenerate = (refTopicLinks = []) => { @@ -386,7 +407,7 @@ const getTopicHeaderAdminActionButtons = (topicSlug, refTopicLinks) => { const TopicHeader = ({ topic, topicData, topicTitle, multiPanel, isCat, setNavTopic, openDisplaySettings, openSearch, topicImage }) => { const { en, he } = !!topicData && topicData.primaryTitle ? topicData.primaryTitle : {en: "Loading...", he: "טוען..."}; - const category = !!topicData ? Sefaria.topicTocCategory(topicData.slug) : null; + const category = !!topicData ? Sefaria.displayTopicTocCategory(topicData.slug) : null; const tpTopImg = !multiPanel && topicImage ? <TopicImage photoLink={topicImage.image_uri} caption={topicImage.image_caption}/> : null; const actionButtons = getTopicHeaderAdminActionButtons(topic, topicData.refs?.about?.refs); const hasAiContentLinks = getLinksWithAiContent(topicData.refs?.about?.refs).length != 0; @@ -422,30 +443,36 @@ return ( : null} {tpTopImg} {topicData && topicData.ref ? - <a href={`/${topicData.ref.url}`} className="resourcesLink button blue"> - <img src="/static/icons/book-icon-black.svg" alt="Book Icon" /> - <span className="int-en">{ topicData.parasha ? Sefaria._('Read the Portion') : topicData.ref.en }</span> - <span className="int-he">{ topicData.parasha ? Sefaria._('Read the Portion') : norm_hebrew_ref(topicData.ref.he) }</span> - </a> - : null} - {topicData?.indexes?.length ? - <div> - <div className="sectionTitleText authorIndexTitle"><InterfaceText>Works on Sefaria</InterfaceText></div> - <div className="authorIndexList"> - {topicData.indexes.map(({url, title, description}) => <AuthorIndexItem key={url} url={url} title={title} description={description}/>)} - </div> - </div> - : null} + <div> + <a href={`/${topicData.ref.url}`} className="resourcesLink button blue"> + <img src="/static/icons/book-icon-black.svg" alt="Book Icon"/> + <span className="int-en">{topicData.parasha ? Sefaria._('Read the Portion') : topicData.ref.en}</span> + <span className="int-he">{topicData.parasha ? Sefaria._('Read the Portion') : norm_hebrew_ref(topicData.ref.he)}</span> + </a> + {Sefaria.interfaceLang === "english" && + <a className="resourcesLink button blue studyCompanion" + href="https://learn.sefaria.org/weekly-parashah/" + data-anl-event="select_promotion:click|view_promotion:scrollIntoView" + data-anl-promotion_name="Parashah Email Signup - Parashah Page" + > + <img src="/static/icons/email-newsletter.svg" alt="Sign up for our weekly parashah study companion"/> + <InterfaceText>Get the Free Study Companion</InterfaceText> + </a>} + </div> + : null} </div> -);} +); +} -const AuthorIndexItem = ({url, title, description}) => { - return ( - <div className="authorIndex" > - <a href={url} className="navBlockTitle"> - <ContentText text={title} defaultToInterfaceOnBilingual /> - </a> - <div className="navBlockDescription"> +const AuthorIndexItem = ({ + url, title, description +}) => { + return ( + <div className="authorIndex"> + <a href={url} className="navBlockTitle"> + <ContentText text={title} defaultToInterfaceOnBilingual/> + </a> + <div className="navBlockDescription"> <ContentText text={description} defaultToInterfaceOnBilingual /> </div> </div> @@ -463,13 +490,29 @@ const useTabDisplayData = (translationLanguagePreference) => { sortFunc: refSort, renderWrapper: refRenderWrapper, }, + { + key: 'admin', + fetcher: fetchBulkText.bind(null, translationLanguagePreference), + sortOptions: ['Relevance', 'Chronological'], + filterFunc: refFilter, + sortFunc: refSort, + renderWrapper: adminRefRenderWrapper, + }, + { + key: 'notable-sources', + fetcher: fetchBulkText.bind(null, translationLanguagePreference), + sortOptions: ['Relevance', 'Chronological'], + filterFunc: refFilter, + sortFunc: refSort, + renderWrapper: notableSourcesRefRenderWrapper, + }, { key: 'sources', fetcher: fetchBulkText.bind(null, translationLanguagePreference), sortOptions: ['Relevance', 'Chronological'], filterFunc: refFilter, sortFunc: refSort, - renderWrapper: refRenderWrapper, + renderWrapper: allSourcesRefRenderWrapper, }, { key: 'sheets', @@ -490,19 +533,27 @@ const PortalNavSideBar = ({portal, entriesToDisplayList}) => { "organization": "PortalOrganization", "newsletter": "PortalNewsletter" } - const modules = []; + const sidebarModules = []; for (let key of entriesToDisplayList) { if (!portal[key]) { continue; } - modules.push({ + sidebarModules.push({ type: portalModuleTypeMap[key], props: portal[key], }); } return( - <NavSidebar modules={modules} /> + <NavSidebar sidebarModules={sidebarModules} /> ) }; +const getTopicPageAnalyticsData = (slug, langPref) => { + return { + project: "topics", + content_lang: langPref || "bilingual", + panel_category: Sefaria.topicTocCategories(slug)?.map(({slug}) => slug)?.join('|'), + }; +}; + const TopicPage = ({ tab, topic, topicTitle, setTopic, setNavTopic, openTopics, multiPanel, showBaseText, navHome, toggleSignUpModal, openDisplaySettings, setTab, openSearch, translationLanguagePreference, versionPref, @@ -513,7 +564,9 @@ const TopicPage = ({ const [loadedData, setLoadedData] = useState(topicData ? Object.entries(topicData.tabs).reduce((obj, [key, tabObj]) => { obj[key] = tabObj.loadedData; return obj; }, {}) : {}); const [refsToFetchByTab, setRefsToFetchByTab] = useState({}); const [parashaData, setParashaData] = useState(null); + const [langPref, setLangPref] = useState(Sefaria.interfaceLang); const [showFilterHeader, setShowFilterHeader] = useState(false); + const [showLangSelectInterface, setShowLangSelectInterface] = useState(false); const [portal, setPortal] = useState(null); const tabDisplayData = useTabDisplayData(translationLanguagePreference, versionPref); const topicImage = topicData.image; @@ -547,9 +600,25 @@ const TopicPage = ({ } }, [topic]); + useEffect( ()=> { + // hack to redirect to temporary sheet content on topics page for those topics that only have sheet content. +if (!Sefaria.is_moderator && !topicData.isLoading && Object.keys(topicData.tabs).length == 0 && topicData.subclass != "author"){ + const interfaceIsHe = Sefaria.interfaceLang === "hebrew"; + const interfaceLang = interfaceIsHe ? 'he' : 'en'; + const coInterfaceLang = interfaceIsHe ? 'en' : 'he'; + const topicPathLang = topicTitle[interfaceLang] ? interfaceLang : coInterfaceLang + const topicPath = topicTitle[topicPathLang] + const redirectUrl = `${document.location.origin}/search?q=${topicPath}&tab=sheet&tvar=1&tsort=relevance&stopics_${topicPathLang}Filters=${topicPath}&svar=1&ssort=relevance` + window.location.replace(redirectUrl); + } + },[topicData]) + // Set up tabs and register incremental load hooks const displayTabs = []; - let onClickFilterIndex = 2; + let onClickFilterIndex = 3; + let onClickLangToggleIndex = 2; + let authorWorksAdded = false; + for (let tabObj of tabDisplayData) { const { key } = tabObj; useIncrementalLoad( @@ -563,6 +632,16 @@ const TopicPage = ({ }), topic ); + + + if (topicData?.indexes?.length && !authorWorksAdded) { + displayTabs.push({ + title: {en: "Works on Sefaria", he: Sefaria.translation('hebrew', "Works on Sefaria")}, + id: 'author-works-on-sefaria', + }); + authorWorksAdded = true + } + if (topicData?.tabs?.[key]) { displayTabs.push({ title: topicData.tabs[key].title, @@ -570,18 +649,34 @@ const TopicPage = ({ }); } } - if (displayTabs.length) { + if (displayTabs.length && tab!="notable-sources" && tab!="author-works-on-sefaria") { displayTabs.push({ title: { - en: "Filter", - he: Sefaria._("Filter") + en: "", + he: "" }, id: 'filter', - icon: `/static/icons/arrow-${showFilterHeader ? 'up' : 'down'}-bold.svg`, + icon: `/static/icons/filter.svg`, justifyright: true }); onClickFilterIndex = displayTabs.length - 1; } + + if (displayTabs.length && tab!="author-works-on-sefaria") { + displayTabs.push({ + title: { + en: "A", + he: Sefaria._("A") + }, + id: 'langToggle', + popover: true, + justifyright: tab==="notable-sources" + }); + onClickLangToggleIndex = displayTabs.length - 1; + } + + + const classStr = classNames({topicPanel: 1, readerNavMenu: 1}); let sidebar = null; if (topicData) { @@ -592,7 +687,7 @@ const TopicPage = ({ } } else { sidebar = ( - <div className="sideColumn"> + <div className="sideColumn" data-anl-panel_type="topics" data-anl-panel_number={0}> <TopicSideColumn key={topic} slug={topic} @@ -612,7 +707,33 @@ const TopicPage = ({ ); } } - return <div className={classStr}> + + const handleLangSelectInterfaceChange = (selection) => { + if (selection === "source") {setLangPref("hebrew")} + else if (selection === "translation") {setLangPref("english")} + else setLangPref(null); + } + + const getCurrentLang = () => { + if (langPref === "hebrew") {return "source"} + else if (langPref === "english") {return "translation"} + else {return "sourcewtrans"} + } + + const handleKeyDown = (e) => { + if (e.key === 'Enter') { + e.preventDefault(); + e.target.click(); + } + } + + const currentLang = getCurrentLang() + + return ( + <div + className={classStr} + data-anl-batch={JSON.stringify(getTopicPageAnalyticsData(topic, langPref))} + > <div className="content noOverflowX" ref={scrollableElement}> <div className="columnLayout"> <div className="mainColumn storyFeedInner"> @@ -623,14 +744,25 @@ const TopicPage = ({ setTab={setTab} tabs={displayTabs} renderTab={t => ( - <div className={classNames({tab: 1, noselect: 1, filter: t.justifyright, open: t.justifyright && showFilterHeader})}> - <InterfaceText text={t.title} /> - { t.icon ? <img src={t.icon} alt={`${t.title.en} icon`} /> : null } + <div tabIndex="0" onKeyDown={(e)=>handleKeyDown(e)} className={classNames({tab: 1, noselect: 1, popover: t.popover , filter: t.justifyright, open: t.justifyright && showFilterHeader})}> + <div data-anl-event={t.popover && "lang_toggle_click:click"}><InterfaceText text={t.title} /></div> + { t.icon ? <img src={t.icon} alt={`${t.title.en} icon`} data-anl-event="filter:click" data-anl-text={topicSort}/> : null } + {t.popover && showLangSelectInterface ? <LangSelectInterface defaultVal={currentLang} callback={(result) => handleLangSelectInterfaceChange(result)} closeInterface={()=>{setShowLangSelectInterface(false)}}/> : null} </div> )} containerClasses={"largeTabs"} - onClickArray={{[onClickFilterIndex]: ()=>setShowFilterHeader(!showFilterHeader)}} + onClickArray={{ + [onClickFilterIndex]: ()=>setShowFilterHeader(!showFilterHeader), + [onClickLangToggleIndex]: ()=>{setShowLangSelectInterface(!showLangSelectInterface)} + }} > + + {topicData?.indexes?.length ? ( + <div className="authorIndexList"> + {topicData.indexes.map(({url, title, description}) => <AuthorIndexItem key={url} url={url} title={title} description={description}/>)} + </div> + ) : null } + { tabDisplayData.map(tabObj => { const { key, sortOptions, filterFunc, sortFunc, renderWrapper } = tabObj; @@ -650,7 +782,7 @@ const TopicPage = ({ topicData._refsDisplayedByTab[key] = data.length; }} initialRenderSize={(topicData._refsDisplayedByTab && topicData._refsDisplayedByTab[key]) || 0} - renderItem={renderWrapper(toggleSignUpModal, topicData, topicTestVersion)} + renderItem={renderWrapper(toggleSignUpModal, topicData, topicTestVersion, langPref)} onSetTopicSort={onSetTopicSort} topicSort={topicSort} /> @@ -664,7 +796,8 @@ const TopicPage = ({ </div> <Footer /> </div> - </div>; + </div> + ); }; TopicPage.propTypes = { tab: PropTypes.string, @@ -686,10 +819,17 @@ const TopicPageTab = ({ data, renderItem, classes, sortOptions, sortFunc, filterFunc, showFilterHeader, scrollableElement, onDisplayedDataChange, initialRenderSize, onSetTopicSort, topicSort }) => { + useEffect(()=>{ + const details = document.querySelector(".story.topicPassageStory details"); + if (details) { + details.setAttribute("open", ""); + } + },[data]) + return ( <div className="topicTabContents"> {!!data ? - <div className={classes}> + <div className={classes} data-anl-feature_name="source_filter"> <FilterableList pageSize={20} scrollableElement={scrollableElement} @@ -714,22 +854,28 @@ const TopicPageTab = ({ const TopicLink = ({topic, topicTitle, onClick, isTransliteration, isCategory}) => ( - <Link className="relatedTopic" href={`/topics/${isCategory ? 'category/' : ''}${topic}`} - onClick={onClick.bind(null, topic, topicTitle)} key={topic} - title={topicTitle.en} - > - <InterfaceText text={{en:topicTitle.en, he:topicTitle.he}}/> - </Link> + <div data-anl-event="related_click:click" data-anl-batch={ + JSON.stringify({ + text: topicTitle.en, + feature_name: "related topic", + }) + }> + <Link className="relatedTopic" href={`/topics/${isCategory ? 'category/' : ''}${topic}`} + onClick={onClick.bind(null, topic, topicTitle)} key={topic} + title={topicTitle.en} + > + <InterfaceText text={{en:topicTitle.en, he:topicTitle.he}}/> + </Link> + </div> ); TopicLink.propTypes = { topic: PropTypes.string.isRequired, - clearAndSetTopic: PropTypes.func.isRequired, isTransliteration: PropTypes.object, }; const TopicSideColumn = ({ slug, links, clearAndSetTopic, parashaData, tref, setNavTopic, timePeriod, properties, topicTitle, multiPanel, topicImage }) => { - const category = Sefaria.topicTocCategory(slug); + const category = Sefaria.displayTopicTocCategory(slug); const linkTypeArray = links ? Object.values(links).filter(linkType => !!linkType && linkType.shouldDisplay && linkType.links.filter(l => l.shouldDisplay !== false).length > 0) : []; if (linkTypeArray.length === 0) { linkTypeArray.push({ @@ -782,12 +928,13 @@ const TopicSideColumn = ({ slug, links, clearAndSetTopic, parashaData, tref, set return b.order.tfidf - a.order.tfidf; }) .map(l => - TopicLink({ - topic:l.topic, topicTitle: l.title, - onClick: l.isCategory ? setNavTopic : clearAndSetTopic, - isTransliteration: l.titleIsTransliteration, - isCategory: l.isCategory - }) + (<TopicLink + key={l.topic} + topic={l.topic} topicTitle={l.title} + onClick={l.isCategory ? setNavTopic : clearAndSetTopic} + isTransliteration={l.titleIsTransliteration} + isCategory={l.isCategory} + />) ) } </TopicSideSection> @@ -795,11 +942,33 @@ const TopicSideColumn = ({ slug, links, clearAndSetTopic, parashaData, tref, set }) : null ); + + + const LinkToSheetsSearchComponent = () => { + + let searchUrlEn = `/search?q=${topicTitle.en}&tab=sheet&tvar=1&tsort=relevance&stopics_enFilters=${topicTitle.en}&svar=1&ssort=relevance`; + let searchUrlHe = `/search?q=${topicTitle.he}&tab=sheet&tvar=1&tsort=relevance&stopics_heFilters=${topicTitle.he}&svar=1&ssort=relevance`; + return ( + <TopicSideSection title={{ en: "Sheets", he: "דפי מקורות" }}> + <InterfaceText> + <EnglishText> + <a href={searchUrlEn}>Related Sheets</a> + </EnglishText> + <HebrewText> + <a href={searchUrlHe}>דפי מקורות קשורים</a> + </HebrewText> + </InterfaceText> + </TopicSideSection> + ); + }; + + return ( <div className={"topicSideColumn"}> { readingsComponent } { topicMetaData } { linksComponent } + <LinkToSheetsSearchComponent/> </div> ) } @@ -811,6 +980,7 @@ TopicSideColumn.propTypes = { const TopicSideSection = ({ title, children, hasMore }) => { const [showMore, setShowMore] = useState(false); + const moreText = showMore ? "Less" : "More"; return ( <div key={title.en} className="link-section"> <h2> @@ -823,8 +993,16 @@ const TopicSideSection = ({ title, children, hasMore }) => { { hasMore ? ( - <div className="sideColumnMore sans-serif" onClick={() => setShowMore(!showMore)}> - <InterfaceText>{ showMore ? "Less" : "More" }</InterfaceText> + <div + className="sideColumnMore sans-serif" + onClick={() => setShowMore(!showMore)} + data-anl-event="related_click:click" + data-anl-batch={JSON.stringify({ + text: moreText, + feature_name: "more", + })} + > + <InterfaceText>{ moreText }</InterfaceText> </div> ) : null @@ -834,9 +1012,9 @@ const TopicSideSection = ({ title, children, hasMore }) => { } const TopicImage = ({photoLink, caption }) => { - + return ( - <div class="topicImage"> + <div className="topicImage"> <ImageWithCaption photoLink={photoLink} caption={caption} /> </div>); } @@ -926,11 +1104,22 @@ const TopicMetaData = ({ topicTitle, timePeriod, multiPanel, topicImage, propert url = propObj.url.en || propObj.url.he; } if (!url) { return null; } + const en_text = propObj.title + (urlExists ? "" : " (Hebrew)"); + const he_text = Sefaria._(propObj.title) + (urlExists ? "" : ` (${Sefaria._("English")})`); return ( - <SimpleLinkedBlock - key={url} en={propObj.title + (urlExists ? "" : " (Hebrew)")} he={Sefaria._(propObj.title) + (urlExists ? "" : ` (${Sefaria._("English")})`)} - url={url} aclasses={"systemText topicMetaData"} openInNewTab - /> + <div + key={url} + data-anl-event="related_click:click" + data-anl-batch={JSON.stringify({ + text: en_text, + feature_name: "description link learn more", + })} + > + <SimpleLinkedBlock + en={en_text} he={he_text} + url={url} aclasses={"systemText topicMetaData"} openInNewTab + /> + </div> ); }) } diff --git a/static/js/TopicPageAll.jsx b/static/js/TopicPageAll.jsx index d732d7c0cd..9a45177c09 100644 --- a/static/js/TopicPageAll.jsx +++ b/static/js/TopicPageAll.jsx @@ -119,7 +119,7 @@ class TopicPageAll extends Component { } </div> </div> - <NavSidebar modules={sidebarModules} /> + <NavSidebar sidebarModules={sidebarModules} /> </div> <Footer /> </div> diff --git a/static/js/TopicsPage.jsx b/static/js/TopicsPage.jsx index b79ab2c52c..c1b2b4fdfd 100644 --- a/static/js/TopicsPage.jsx +++ b/static/js/TopicsPage.jsx @@ -7,7 +7,7 @@ import PropTypes from 'prop-types'; import classNames from 'classnames'; import Sefaria from './sefaria/sefaria'; import $ from './sefaria/sefariaJquery'; -import { NavSidebar, Modules } from './NavSidebar'; +import { NavSidebar, SidebarModules } from './NavSidebar'; import Footer from './Footer'; import {CategoryHeader} from "./Misc"; import Component from 'react-class'; @@ -45,7 +45,7 @@ const TopicsPage = ({setNavTopic, multiPanel, initialWidth}) => { ); const about = multiPanel ? null : - <Modules type={"AboutTopics"} props={{hideTitle: true}} />; + <SidebarModules type={"AboutTopics"} props={{hideTitle: true}} />; const sidebarModules = [ multiPanel ? {type: "AboutTopics"} : {type: null}, @@ -69,7 +69,7 @@ const TopicsPage = ({setNavTopic, multiPanel, initialWidth}) => { { about } { categoryListings } </div> - <NavSidebar modules={sidebarModules} /> + <NavSidebar sidebarModules={sidebarModules} /> </div> <Footer /> </div> diff --git a/static/js/TranslationsPage.jsx b/static/js/TranslationsPage.jsx index 0b74aa7ea5..7afe428cd9 100644 --- a/static/js/TranslationsPage.jsx +++ b/static/js/TranslationsPage.jsx @@ -2,7 +2,7 @@ import { useState, useEffect } from "react"; import Sefaria from "./sefaria/sefaria"; import classNames from 'classnames'; import {InterfaceText, TabView} from './Misc'; -import { NavSidebar, Modules } from './NavSidebar'; +import { NavSidebar, SidebarModules } from './NavSidebar'; import Footer from './Footer'; @@ -89,7 +89,7 @@ const TranslationsPage = ({translationsSlug}) => { </> </TabView>} </div> - <NavSidebar modules={sidebarModules} /> + <NavSidebar sidebarModules={sidebarModules} /> </div> <Footer /> </div> diff --git a/static/js/UserHistoryPanel.jsx b/static/js/UserHistoryPanel.jsx index c6aa7c30c8..1e6931c346 100644 --- a/static/js/UserHistoryPanel.jsx +++ b/static/js/UserHistoryPanel.jsx @@ -64,7 +64,7 @@ const UserHistoryPanel = ({menuOpen, toggleLanguage, openDisplaySettings, openNa toggleSignUpModal={toggleSignUpModal} key={menuOpen}/> </div> - <NavSidebar modules={sidebarModules} /> + <NavSidebar sidebarModules={sidebarModules} /> </div> {footer} </div> diff --git a/static/js/UserProfile.jsx b/static/js/UserProfile.jsx index 424acacd51..5476e6ab7d 100644 --- a/static/js/UserProfile.jsx +++ b/static/js/UserProfile.jsx @@ -569,7 +569,7 @@ const ProfileSummary = ({ profile:p, follow, openFollowers, openFollowing, toggl // we only store twitter handles so twitter needs to be hardcoded <span> { - socialList.map(s => (<a key={s} className="social-icon" target="_blank" href={(s == 'twitter' ? 'https://twitter.com/' : '') + p[s]}><img src={`/static/img/${s}.svg`} /></a>)) + socialList.map(s => (<a key={s} className="social-icon" target="_blank" href={(s === 'twitter' ? 'https://twitter.com/' : s === 'youtube' ? 'https://www.youtube.com/' : '') + p[s]}><img src={`/static/img/${s}.svg`} /></a>)) } </span> ); diff --git a/static/js/analyticsEventTracker.js b/static/js/analyticsEventTracker.js new file mode 100644 index 0000000000..5e0a43b925 --- /dev/null +++ b/static/js/analyticsEventTracker.js @@ -0,0 +1,271 @@ +const AnalyticsEventTracker = (function() { + const VALID_ANALYTICS_FIELDS = new Set([ + 'project', 'panel_type', 'panel_number', 'item_id', 'version', 'content_lang', + 'content_id', 'content_type', 'panel_name', 'panel_category', 'position', 'ai', + 'text', 'experiment', 'feature_name', 'from', 'to', 'action', 'engagement_value', + 'engagement_type', 'logged_in', 'site_lang', 'traffic_type', 'promotion_name' + ]); + const EVENT_ATTR = 'data-anl-event'; + const FIELD_ATTR_PREFIX = 'data-anl-'; + const BATCH_ATTR = 'data-anl-batch'; + const SCROLL_INTO_VIEW_SELECTOR ='[data-anl-event*=":scrollIntoView"]'; + const TOGGLE_NODE_SELECTOR = 'details'; + const NON_BUBBLING_EVENT_DELEGATES = { + 'bubblingToggle': 'toggle', + } + let scrollIntoViewObserver = null; + + function _filterInvalidAnalyticsKeys(obj) { + const invalid_keys = Object.keys(obj).filter( + key => !VALID_ANALYTICS_FIELDS.has(key) + ); + if (invalid_keys.length > 0) { + for (let key of invalid_keys) { + console.warn("Invalid analytics key:", key); + } + return Object.fromEntries(Object.entries(obj).filter(([k, v]) => !invalid_keys.includes(k))); + } + return obj; + } + + function _parseEventAttr(value) { + /** + * the value of `data-anl-event` is of the form `<event_name1>:<event_type1>|<event_name2>:<event_type2>...` + * Returns this data parsed into a list of objects with each object having keys `name` and `type`. + */ + if (!value) { return [{name: "", type: ""}]; } + return value.split("|").map(_parseSingleEventAttr); + } + + function _parseSingleEventAttr(value) { + /** + * the value of a single event in `data-anl-event` is of the form `<event name>:<event type>` + * Returns this data parsed into an object with keys `name` and `type`. + */ + const [eventName, eventType] = value.split(':'); + if (!eventName?.length) { + console.warn("Event name is invalid for `data-anl-event` value:", value); + } + if (!eventType?.length) { + console.warn("Event type is invalid for `data-anl-event` value:", value); + } + return {name:eventName, type:eventType}; + } + + function _getEventTargetByCondition(event, condition, eventTarget=null) { + /** + * Searches the parents of an event target for an element to meets a certain condition + * `condition` is a function of form condition(element) => bool. + * If `eventTarget` is passed, it will be used as the starting point of the search instead of `event.target` + * Returns the first element in parent hierarchy where `condition` returns true + * If no element returns true, returns null. + */ + let parent = eventTarget || event.target; + const outmost = event.currentTarget; + while (parent) { + if(condition(parent)){ + return parent + } + else if (parent.parentNode === outmost) { + return null; + } + parent = parent.parentNode; + } + } + + function _getAnalyticsEventArray(event) { + /** + * Return an array of objects of form {name, type} if this JS event should be treated as an analytics event + * Looks for a parent of e.target that has the attribute `data-anl-event` + * If this JS event doesn't match a registered analytics event, return `null` + */ + const eventType = _delegatedEventTypeToOriginal(event.type); + const element = _getEventTargetByCondition( + event, + element => { + const value = element.getAttribute(EVENT_ATTR); + return _parseEventAttr(value).some(({ type }) => type === eventType); + } + ); + if (!element) { return null; } + return _parseEventAttr(element.getAttribute(EVENT_ATTR)).filter(({ type }) => type === eventType); + } + + function _getAnlDataFromElement(element) { + if (!element) { return {}; } + return Array.from(element.attributes).reduce((attrsAggregated, currAttr) => { + const attrName = currAttr.name; + if (attrName === EVENT_ATTR) { + + } else if (attrName.startsWith(FIELD_ATTR_PREFIX)) { + if (attrName === BATCH_ATTR) { + attrsAggregated = {...attrsAggregated, ...JSON.parse(currAttr.value)}; + } else { + const anlFieldName = attrName.replace(FIELD_ATTR_PREFIX, ''); + attrsAggregated[anlFieldName] = currAttr.value; + } + } + return attrsAggregated; + }, {}); + } + + function _mergeObjectsWithoutOverwrite(a, b) { + /** + * merges a into b but doesn't overwrite fields in b that already exist + */ + for (let key in a) { + if (!(key in b)) { + b[key] = a[key]; + } + } + return b; + } + + function _delegatedEventTypeToOriginal(eventType) { + return NON_BUBBLING_EVENT_DELEGATES?.[eventType] || eventType; + } + + function _getDerivedData(event) { + /** + * Return data that can be derived directly from `event` + */ + const eventType = _delegatedEventTypeToOriginal(event.type); + if (eventType === "toggle") { + return { + from: event.target.open ? "closed" : "open", + to: event.target.open ? "open" : "closed" + }; + } else if (event.target.tagName === 'DETAILS') { + return { + from: event.target.open ? "open" : "closed", + }; + } else if (eventType === "input") { + return { + text: event.target.value, + }; + } + return {}; + } + + function _handleAnalyticsEvent(event) { + const anlEventArray = _getAnalyticsEventArray(event); + if (!anlEventArray) { return; } + let anlEventData = {}; + let currElem = null; + do { + currElem = _getEventTargetByCondition( + event, + element => Object.keys(_getAnlDataFromElement(element)).length > 0, + currElem?.parentNode + ); + const currAnlEventData = _getAnlDataFromElement(currElem); + // make sure that analytics fields that are defined lower down aren't overwritten by ones defined higher in the DOM tree + anlEventData = _mergeObjectsWithoutOverwrite(currAnlEventData, anlEventData); + } while (currElem?.parentNode); + + anlEventData = {...anlEventData, ..._getDerivedData(event)}; + anlEventData = _filterInvalidAnalyticsKeys(anlEventData); + + anlEventArray.forEach(anlEvent => { + gtag("event", anlEvent.name, anlEventData); + }); + } + + function _addToggleListener(element) { + const delegatedToggle = Object.keys(NON_BUBBLING_EVENT_DELEGATES)[0]; + element.addEventListener('toggle', function(event) { + const bubblingToggle = new CustomEvent(delegatedToggle, { + bubbles: true, + detail: { originalEvent: event } + }); + event.target.dispatchEvent(bubblingToggle); + }); + } + + function _runFunctionOnSelector(root, selector, fn) { + /** + * Runs `fn(node)` on any node that matches `selector`. + * Search includes `root` and all its children + */ + if (root.matches?.(selector)) { fn(root); } + root.querySelectorAll(selector).forEach(fn); + } + + function _observeForScrollIntoView(node) { + scrollIntoViewObserver.observe(node); + } + + function _runFunctionsOnAddedNodes(selector, functions) { + /** + * Runs `functions` on all nodes added to `selector` + */ + // Run on existing elements + functions.forEach(fn => fn(document)); + + const observer = new MutationObserver(mutations => { + mutations.forEach(mutation => { + mutation.addedNodes.forEach(node => { + if (node.nodeType !== Node.ELEMENT_NODE) { return; } + functions.forEach(fn => fn(node)); + }); + }); + }); + + const elements = document.querySelectorAll(selector); + elements.forEach(element => { + observer.observe(element, { childList: true, subtree: true }); + }) + } + + function _observeScrollIntoViewEvent(selector) { + const eventType = 'scrollIntoView'; + + // scrollIntoViewObserver is global so that we can add elements that need to be observed as they are created + scrollIntoViewObserver = new IntersectionObserver((entries, observer) => { + entries.forEach(entry => { + if (entry.isIntersecting) { + const scrollIntoViewEvent = new CustomEvent(eventType, {bubbles: true}); + entry.target.dispatchEvent(scrollIntoViewEvent); + observer.unobserve(entry.target); // Stop observing the node + } + }); + }); + + return eventType; + } + + /** + * Public interface is in return value + */ + return { + attach: function(selector, eventTypes) { + /** + * Listens for analytics events on any element that matches `selector`. + * `eventTypes` is a list of JS event types to listen for + * Any element that is a child of `selector` that has the `data-anl-event` attribute will trigger + * an analytics event. The event type needs to be in `eventTypes` and needs to match the value of `data-anl-event` + * E.g. if a child of selector is <div data-anl-event="click">...</div>, any click on this div will trigger an + * analytics event + * The data sent for the event is aggregated from all `data-anl-<field>` attributes on any parent of the event + * target (including the target itself). `<field>` needs to be a valid analytics field as specified in + * `VALID_ANALYTICS_FIELDS` above. + */ + const elements = document.querySelectorAll(selector); + + const onMutationFns = []; + for (let eventType of eventTypes) { + if (eventType === 'toggle') { + onMutationFns.push(node => _runFunctionOnSelector(node, TOGGLE_NODE_SELECTOR, _addToggleListener)); + eventType = Object.keys(NON_BUBBLING_EVENT_DELEGATES)[0]; + } else if (eventType === 'scrollIntoView') { + onMutationFns.push(node => _runFunctionOnSelector(node, SCROLL_INTO_VIEW_SELECTOR, _observeForScrollIntoView)); + eventType = _observeScrollIntoViewEvent(selector); + } + elements.forEach(element => { + element.addEventListener(eventType, _handleAnalyticsEvent); + }); + } + _runFunctionsOnAddedNodes(selector, onMutationFns); + } + }; +})(); diff --git a/static/js/linker.v3/main.js b/static/js/linker.v3/main.js index 700de691c4..ed2b5cc8f4 100644 --- a/static/js/linker.v3/main.js +++ b/static/js/linker.v3/main.js @@ -130,9 +130,10 @@ import {LinkExcluder} from "./excluder"; return startIndex; } - function getNumWordsAround(linkObj, text, numWordsAround) { + function getNumWordsAround(linkObj, text, numWordsAround, dir) { /** * gets text with `numWordsAround` number of words surrounding text in linkObj. Words are broken by any white space. + * dir can be either 'before', 'after' or 'both' to indicate in which direction to expand * returns: { * text: str with numWordsAround * startChar: int, start index of linkObj text within numWordsAround text @@ -142,9 +143,9 @@ import {LinkExcluder} from "./excluder"; if (numWordsAround === 0) { return { text: linkObj.text, startChar: 0 }; } - const newEndChar = getNthWhiteSpaceIndex(text, numWordsAround, endChar); + const newEndChar = dir === 'before' ? endChar : getNthWhiteSpaceIndex(text, numWordsAround, endChar); const textRev = [...text].reverse().join(""); - const newStartChar = text.length - getNthWhiteSpaceIndex(textRev, numWordsAround, text.length - startChar); + const newStartChar = dir === 'after' ? startChar : text.length - getNthWhiteSpaceIndex(textRev, numWordsAround, text.length - startChar); const wordsAroundText = escapeRegExp(text.substring(newStartChar, newEndChar)); // findAndReplaceDOMText and Readability deal with element boundaries differently // in order to more flexibly find these boundaries, we treat all whitespace the same @@ -220,38 +221,66 @@ import {LinkExcluder} from "./excluder"; return firstOccurrenceLength >= maxSearchLength && firstOccurrenceLength > linkObj.text.length; } - function wrapRef(linkObj, normalizedText, refData, iLinkObj, resultsKey, maxNumWordsAround = 10, maxSearchLength = 30) { + function findUniqueOccurrencesByDir(linkObj, normalizedText, dir, maxNumWordsAround = 10, maxSearchLength = 30) { /** - * wraps linkObj.text with an atag. In the case linkObj.text appears multiple times on the page, - * increases search scope to ensure we wrap the correct instance of linkObj.text - * linkObj: object representing a link, as returned by find-refs API - * normalizedText: normalized text of webpage (i.e. webpage text returned from Readability and then put through some normalization) - * refData: refData field as returned from find-refs API - * iLinkObj: index of linkObj in results list - * maxNumWordsAround: maximum number of words around linkObj.text to search to try to find its unique occurrence. - * maxSearchLength: if searchText is beyond this length, we assume the string uniquely identifies the citation. Even if there are multiple occurrences, we assume they can be wrapped with the same citation. + * See docs for `findUniqueOccurrences` + * This function specifically searches for occurrences using context in a specific direction, `dir` + * `dir` can be 'both', 'before' or 'after'. 'both' means we add context in both directions. + * 'before' means we only add context before `linkObj` and 'after' means only after */ - if (!ns.debug && (linkObj.linkFailed || linkObj.refs.length > 1)) { return; } - const urls = linkObj.refs && linkObj.refs.map(ref => refData[ref].url); document.normalize(); let occurrences = []; let numWordsAround = 0; let searchText = linkObj.text; let linkStartChar = 0; // start index of link text within searchText - const excluder = new LinkExcluder(ns.excludeFromLinking, ns.excludeFromTracking); while ((numWordsAround === 0 || occurrences.length > 1) && numWordsAround < maxNumWordsAround) { // see https://flaviocopes.com/javascript-destructure-object-to-existing-variable/ - ({ text: searchText, startChar: linkStartChar } = getNumWordsAround(linkObj, normalizedText, numWordsAround)); + ({ text: searchText, startChar: linkStartChar } = getNumWordsAround(linkObj, normalizedText, numWordsAround, dir)); occurrences = findOccurrences(searchText); numWordsAround += 1; if (isMatchedTextUniqueEnough(occurrences, linkObj, maxSearchLength)) { break; } } if (occurrences.length !== 1 && !isMatchedTextUniqueEnough(occurrences, linkObj, maxSearchLength)) { if (ns.debug) { - console.log("MISSED", numWordsAround, occurrences.length, linkObj); + console.log("MISSED", numWordsAround, occurrences.length, searchText, linkObj); } - return; + occurrences = []; + } + return [occurrences, linkStartChar]; + } + + function findUniqueOccurrences(linkObj, normalizedText, maxNumWordsAround = 10, maxSearchLength = 30) { + /** + * Find unique occurrences of `linkObj.text` in the DOM. In the case where there are multiple occurrences, + * uses `normalizedText` to increase the search context around `linkObj.text` until there are only "unique enough" occurrences. + * Context will be increased until `maxNumWordsAround`. E.g. if `maxNumWordsAround = 10` then 10 words of context are used from either side of `linkObj.text` + * "unique enough" occurrences means that either: + * a) there's only one occurrence of `linkObj.text` when including context + * b) there are multiple occurrences, but they are deemed to be "unique enough". A match is "unique enough" if it is longer than `maxSearchLength`. This length includes the extra context + */ + let [occurrences, linkStartChar] = [[], 0]; + for (let dir of ['both', 'before', 'after']) { + [occurrences, linkStartChar] = findUniqueOccurrencesByDir(linkObj, normalizedText, dir, maxNumWordsAround, maxSearchLength); + if (occurrences.length > 0) { break; } } + return [occurrences, linkStartChar]; + } + + function wrapRef(linkObj, normalizedText, refData, iLinkObj, resultsKey, maxNumWordsAround = 10, maxSearchLength = 30) { + /** + * wraps linkObj.text with an atag. In the case linkObj.text appears multiple times on the page, + * increases search scope to ensure we wrap the correct instance of linkObj.text + * linkObj: object representing a link, as returned by find-refs API + * normalizedText: normalized text of webpage (i.e. webpage text returned from Readability and then put through some normalization) + * refData: refData field as returned from find-refs API + * iLinkObj: index of linkObj in results list + * maxNumWordsAround: maximum number of words around linkObj.text to search to try to find its unique occurrence. + * maxSearchLength: if searchText is beyond this length, we assume the string uniquely identifies the citation. Even if there are multiple occurrences, we assume they can be wrapped with the same citation. + */ + if (!ns.debug && (linkObj.linkFailed || linkObj.refs.length > 1)) { return; } + const urls = linkObj.refs && linkObj.refs.map(ref => refData[ref].url); + const excluder = new LinkExcluder(ns.excludeFromLinking, ns.excludeFromTracking); + const [occurrences, linkStartChar] = findUniqueOccurrences(linkObj, normalizedText, maxNumWordsAround, maxSearchLength); const globalLinkStarts = occurrences.map(([start, end]) => linkStartChar + start); findAndReplaceDOMText(document, { find: linkObj.text, @@ -412,7 +441,8 @@ import {LinkExcluder} from "./excluder"; return new Promise((resolve, reject) => { fetch(getFindRefsUrl(), { method: 'POST', - body: JSON.stringify(postData) + body: JSON.stringify(postData), + headers: {'Content-Type': 'application/json'}, }) .then(handleApiResponse) .then(resp => resolve(resp)); diff --git a/static/js/sefaria/sefaria.js b/static/js/sefaria/sefaria.js index 520af3be3a..aae230fedd 100644 --- a/static/js/sefaria/sefaria.js +++ b/static/js/sefaria/sefaria.js @@ -330,12 +330,12 @@ Sefaria = extend(Sefaria, { } }, /** - * Helps the BookPage toc translate the given integer to the correctly formatted display string for the section given the varying address types. + * Helps the BookPage toc translate the given integer to the correctly formatted display string for the section given the varying address types. * @param {string} addressType - The address type of the schema being requested * @param {number} i - The numeric section string from the database * @param {number} offset - If needed, an offest to allow section addresses that do not start counting with 0 - * @returns {[string,string]} Section string in both languages. - */ + * @returns {[string,string]} Section string in both languages. + */ getSectionStringByAddressType: function(addressType, i, offset=0) { let section = i + offset; let enSection, heSection; @@ -343,11 +343,11 @@ Sefaria = extend(Sefaria, { enSection = Sefaria.hebrew.intToDaf(section); heSection = Sefaria.hebrew.encodeHebrewDaf(enSection); } else if (addressType === "Year") { - enSection = section + 1241; + enSection = section + 1241; heSection = Sefaria.hebrew.encodeHebrewNumeral(section+1); heSection = heSection.slice(0,-1) + '"' + heSection.slice(-1); } else if (addressType === "Folio") { - enSection = Sefaria.hebrew.intToFolio(section); + enSection = Sefaria.hebrew.intToFolio(section); heSection = Sefaria.hebrew.encodeHebrewFolio(enSection); } else { enSection = section + 1; @@ -1625,12 +1625,12 @@ Sefaria = extend(Sefaria, { // First sort by predefined "top" const hebrewTopByCategory = { "Tanakh": ["Rashi", "Ibn Ezra", "Ramban", "Sforno"], - "Talmud": ["Rashi", "Tosafot"], + "Talmud": ["Rashi", "Rashbam", "Tosafot"], "Mishnah": ["Bartenura", "Rambam", "Ikar Tosafot Yom Tov", "Yachin", "Boaz"] }; const englishTopByCategory = { "Tanakh": ["Rashi", "Ibn Ezra", "Ramban", "Sforno"], - "Talmud": ["Rashi", "Tosafot"], + "Talmud": ["Rashi", "Rashbam", "Tosafot"], "Mishnah": ["Bartenura", "English Explanation of Mishnah", "Rambam", "Ikar Tosafot Yom Tov", "Yachin", "Boaz"] }; const top = (byHebrew ? hebrewTopByCategory[category] : englishTopByCategory[category]) || []; @@ -2669,7 +2669,7 @@ _media: {}, 'authors': ['popular-writing-of'], }, getTopic: function(slug, {annotated=true, with_html=false}={}) { - const cat = Sefaria.topicTocCategory(slug); + const cat = Sefaria.displayTopicTocCategory(slug); let ref_link_type_filters = ['about', 'popular-writing-of'] // overwrite ref_link_type_filters with predefined list. currently used to hide "Sources" and "Sheets" on author pages. if (!!cat && !!Sefaria._CAT_REF_LINK_TYPE_FILTER_MAP[cat.slug]) { @@ -2689,22 +2689,25 @@ _media: {}, return slug + (annotated ? "-a" : "") + (with_html ? "-h" : ""); }, processTopicsData: function(data) { + const lang = Sefaria.interfaceLang == "hebrew" ? 'he' : 'en' if (!data) { return null; } if (!data.refs) { return data; } const tabs = {}; for (let [linkTypeSlug, linkTypeObj] of Object.entries(data.refs)) { for (let refObj of linkTypeObj.refs) { + // sheets are no longer displayed on topic pages + if (refObj.is_sheet) { continue; } let tabKey = linkTypeSlug; if (tabKey === 'about') { - tabKey = refObj.is_sheet ? 'sheets' : 'sources'; + tabKey = (refObj.descriptions?.[lang]?.title || refObj.descriptions?.[lang]?.prompt) ? 'notable-sources' : 'sources'; } if (!tabs[tabKey]) { let { title } = linkTypeObj; - if (tabKey === 'sheets') { - title = {en: 'Sheets', he: Sefaria._('Sheets')}; + if (tabKey === 'notable-sources') { + title = {en: 'Notable Sources', he: Sefaria.translation('hebrew', 'Notable Sources')}; } if (tabKey === 'sources') { - title = {en: 'Sources', he: Sefaria._('Sources')}; + title = {en: 'Sources', he: Sefaria.translation('hebrew', 'Sources')}; } tabs[tabKey] = { refMap: {}, @@ -2712,20 +2715,36 @@ _media: {}, shouldDisplay: linkTypeObj.shouldDisplay, }; } - const ref = refObj.is_sheet ? parseInt(refObj.ref.replace('Sheet ', '')) : refObj.ref; if (refObj.order) { refObj.order = {...refObj.order, availableLangs: refObj?.order?.availableLangs || [], numDatasource: refObj?.order?.numDatasource || 1, tfidf: refObj?.order?.tfidf || 0, pr: refObj?.order?.pr || 0, curatedPrimacy: {he: refObj?.order?.curatedPrimacy?.he || 0, en: refObj?.order?.curatedPrimacy?.en || 0}}} - tabs[tabKey].refMap[refObj.ref] = {ref, order: refObj.order, dataSources: refObj.dataSources, descriptions: refObj.descriptions}; + tabs[tabKey].refMap[refObj.ref] = {ref: refObj.ref, order: refObj.order, dataSources: refObj.dataSources, descriptions: refObj.descriptions}; } } for (let tabObj of Object.values(tabs)) { tabObj.refs = Object.values(tabObj.refMap); delete tabObj.refMap; } + + if (tabs["notable-sources"]) { + if (!tabs.sources) { + tabs.sources = {refMap: {}, shouldDisplay: true, refs: []}; + } + tabs.sources.title = {en: 'All Sources', he: Sefaria.translation('hebrew', 'All Sources')}; + //turn "sources" tab into 'super-set', containing all refs from all tabs: + const allRefs = [...tabs["notable-sources"].refs, ...tabs.sources.refs]; + tabs.sources.refs = allRefs; + } + if (Sefaria.is_moderator){ + tabs["admin"] = {...tabs["sources"]}; + tabs["admin"].title = {en: 'Admin', he: Sefaria.translation('hebrew', "Admin")}; + + } + + data.tabs = tabs; return data; }, @@ -2767,12 +2786,15 @@ _media: {}, }, _initTopicTocCategoryReducer: function(a,c) { if (!c.children) { - a[c.slug] = c.parent; - return a; + a[c.slug] = c.parents; + return a; + } + if (!c.parents) { + c.parents = []; } for (let sub_c of c.children) { - sub_c.parent = { en: c.en, he: c.he, slug: c.slug }; - Sefaria._initTopicTocCategoryReducer(a, sub_c); + sub_c.parents = c.parents.concat({ en: c.en, he: c.he, slug: c.slug }); + Sefaria._initTopicTocCategoryReducer(a, sub_c); } return a; }, @@ -2784,11 +2806,14 @@ _media: {}, } return this._topicTocPages[key] }, - topicTocCategory: function(slug) { + topicTocCategories: function(slug) { // return category english and hebrew for slug if (!this._topicTocCategory) { this._initTopicTocCategory(); } return this._topicTocCategory[slug]; }, + displayTopicTocCategory: function(slug) { + return this.topicTocCategories(slug)?.at(-1); + }, _topicTocCategoryTitles: null, _initTopicTocCategoryTitles: function() { this._topicTocCategoryTitles = this.topic_toc.reduce(this._initTopicTocCategoryTitlesReducer, {}); diff --git a/static/js/sefaria/strings.js b/static/js/sefaria/strings.js index 752f95cdf1..e99b29c75f 100644 --- a/static/js/sefaria/strings.js +++ b/static/js/sefaria/strings.js @@ -21,6 +21,7 @@ const Strings = { "My Source Sheets" : "דפי המקורות שלי", "Public Source Sheets":"דפי מקורות פומביים", "Log in": "התחברות", + "A": "א", "Sign up": "להרשמה", "Sign Up": "להרשמה", @@ -93,6 +94,7 @@ const Strings = { "Title must be provided": "יש לספק כותרת", "Unfortunately, there may have been an error saving this topic information": "הודעת שגיאה: נראה כי חלה שגיאה במהלך שמירת הנתונים", "Something went wrong. Sorry!": "מצטערים, משהו השתבש", + "Admin" : "עריכה", // Topics Tool @@ -256,8 +258,13 @@ const Strings = { 'Select': 'בחירה', 'Currently Selected': 'נוכחי', "Merged from": "נוצר ממיזוג", - "Source" : "מקור", "Sources": "מקורות", + "Notable Sources": "מקורות מרכזיים", + "All Sources": "כל המקורות", + "Source" : "מקור", + "Translation": "תרגום", + "Source with Translation": "מקור עם תרגום", + "Source Language": "שפת המקורות", "Digitization" : "דיגיטציה", "License" : "רשיון", "Revision History" : "היסטורית עריכה", diff --git a/templates/_sidebar.html b/templates/_sidebar.html index 0de08cb2b1..87dfc42cce 100644 --- a/templates/_sidebar.html +++ b/templates/_sidebar.html @@ -15,7 +15,7 @@ <h1 class="int-he">{{heTitle}}</h1> <span class="int-he">משרות פנויות בספריא<i class="fa fa-chevron-left"></i></span></a></li> <li><a id="products" href="/products"> <span class="int-en">Sefaria's Products<i class="fa fa-chevron-right"></i></span> - <span class="int-he">מוצרים של ספריא<i class="fa fa-chevron-left"></i></span></a></li> + <span class="int-he">המוצרים של ספריא<i class="fa fa-chevron-left"></i></span></a></li> <li><a id="supporters" href="/supporters"> <span class="int-en">Our Supporters<i class="fa fa-chevron-right"></i></span> <span class="int-he">התומכים שלנו<i class="fa fa-chevron-left"></i></span></a></li> diff --git a/templates/base.html b/templates/base.html index 154d9dfe8e..f74fd0df90 100644 --- a/templates/base.html +++ b/templates/base.html @@ -221,6 +221,7 @@ <script src="https://cdnjs.cloudflare.com/ajax/libs/jqueryui/1.12.1/jquery-ui.js"></script> <script src="{% static 'js/lib/keyboard.js' %}"></script> + <script src="{% static 'js/analyticsEventTracker.js' %}"></script> <script src="/data.{{ last_cached_short }}.js"></script> <script> @@ -260,6 +261,9 @@ gtag('config', '{{ GOOGLE_GTAG }}', { 'user_id': DJANGO_VARS.props ? DJANGO_VARS.props._uid : null }); + + <!-- attach analyticsEventTracker --> + AnalyticsEventTracker.attach("#s2, #staticContentWrapper", ['click', 'scrollIntoView', 'toggle', 'mouseover', 'input']); </script> <!-- End Google tag --> {% endif %} diff --git a/templates/static/he/ways-to-give.html b/templates/static/he/ways-to-give.html index 1101c7639c..6eb6051dc2 100644 --- a/templates/static/he/ways-to-give.html +++ b/templates/static/he/ways-to-give.html @@ -42,7 +42,7 @@ <h1> </span> </p> <div class="single-button center"> - <a class="button big blue control-elem" role="button" href="https://donate.sefaria.org/he" target="_blank"> + <a class="button big blue control-elem" role="button" href="https://donate.sefaria.org/give/468442/#!/donation/checkout?c_src=waystogive" target="_blank"> <span class="int-he">לתרומה</span> </a> </div> @@ -56,7 +56,7 @@ <h2> <section> <p> <span class="int-he"> - הצטרפות <a href="https://donate.sefaria.org/sustainershe" target="_blank">לקהילת התומכים של ספריא על ידי תרומה חודשית</a> + הצטרפות <a href="https://donate.sefaria.org/give/478929/#!/donation/checkout?c_src=waystogive" target="_blank">לקהילת התומכים של ספריא על ידי תרומה חודשית</a> </span> </p> <p class="givingOpportunitiesDesc"> @@ -69,7 +69,7 @@ <h2> <section> <p> <span class="int-he"> - <a href="https://donate.sefaria.org/sponsorhe" target="_blank">הקדשה אישית ליום, שבוע או חודש של לימוד</a> + <a href="https://donate.sefaria.org/campaign/sponsor-a-day-of-learning-hebrew/c479003?c_src=waystogive" target="_blank">הקדשה אישית ליום, שבוע או חודש של לימוד</a> </span> </p> <p class="givingOpportunitiesDesc"> @@ -92,7 +92,7 @@ <h3> </h3> <p> <span class="int-he"> - תרומה דרך <a href="https://donate.sefaria.org/he" target="_blank">עמוד התרומות הראשי שלנו</a> הפועל באמצעות <bdi>Stripe</bdi>. + תרומה דרך <a href="https://donate.sefaria.org/give/468442/#!/donation/checkout?c_src=waystogive" target="_blank">עמוד התרומות הראשי שלנו</a> הפועל באמצעות <bdi>Stripe</bdi>. </span> </p> </section> diff --git a/templates/static/products.html b/templates/static/products.html index a0d0cabed8..9aa51be2c2 100644 --- a/templates/static/products.html +++ b/templates/static/products.html @@ -18,7 +18,7 @@ {% block content %} <div class="superAbout"> {% if not request.user_agent.is_mobile %} - {% include '_sidebar.html' with whichPage='products' title="Products" heTitle="מוצרים" %} + {% include '_sidebar.html' with whichPage='products' title="Products" heTitle="המוצרים של ספריא" %} {% endif %} <main id="productsPage"> <div id="productsPageContent">