From 549cd93298d846e72e29183d6bedb5194b99b2c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matja=C5=BE=20Horvat?= Date: Fri, 10 Nov 2023 00:16:34 +0100 Subject: [PATCH] Optimize and fix the /insights page (#3007) * Fix #2980: Prevent bots from checking the /insights page * Optimize ProjectLocaleInsightsSnapshot Admin view/editing page * Refactor data gathering for the /insights page 1. Fix #2979: Optimize /insights page. To render the charts on the Insights pages, we use periodically gathered ProjectLocaleInsightsSnapshot data. That works fine on Project, Locale and ProjectLocale pages, but not on the /insights page which shows data across the entire app. The amount of data requested from the DB quickly becomes big and the request starts timing out. In this patch, we gather data directly from the ActionLog as hinted at in #2938. 2. Fix #2999: Show insights for all months. 3. Calculate monthly averages rather than averages of averages. --- pontoon/base/templates/robots.txt | 1 + pontoon/insights/admin.py | 1 + pontoon/insights/utils.py | 95 ++++++++++++++++++++----------- pontoon/insights/views.py | 2 +- 4 files changed, 66 insertions(+), 33 deletions(-) diff --git a/pontoon/base/templates/robots.txt b/pontoon/base/templates/robots.txt index e70c6458fe..386a59c8db 100644 --- a/pontoon/base/templates/robots.txt +++ b/pontoon/base/templates/robots.txt @@ -2,3 +2,4 @@ User-agent: * Disallow: /a/ Disallow: /admin/ Disallow: /contributors/ +Disallow: /*insights/ diff --git a/pontoon/insights/admin.py b/pontoon/insights/admin.py index 363509b5d9..eac551e469 100644 --- a/pontoon/insights/admin.py +++ b/pontoon/insights/admin.py @@ -52,6 +52,7 @@ class ProjectLocaleInsightsSnapshotAdmin(admin.ModelAdmin): "completion", "unreviewed_strings", ) + readonly_fields = ("project_locale",) admin.site.register(LocaleInsightsSnapshot, LocaleInsightsSnapshotAdmin) diff --git a/pontoon/insights/utils.py b/pontoon/insights/utils.py index 4d6a4a59c3..8fa3b075b5 100644 --- a/pontoon/insights/utils.py +++ b/pontoon/insights/utils.py @@ -1,11 +1,12 @@ import json -import statistics from datetime import datetime, timedelta from dateutil.relativedelta import relativedelta +from django.contrib.auth.models import User from django.db.models.functions import TruncMonth from django.db.models import Avg, Count, Q, Sum +from pontoon.actionlog.models import ActionLog from pontoon.base.utils import convert_to_unix_time from pontoon.insights.models import ( LocaleInsightsSnapshot, @@ -365,25 +366,43 @@ def get_insights(locale=None, project=None): def get_global_pretranslation_quality(category, id): start_date = get_insight_start_date() - snapshots = ProjectLocaleInsightsSnapshot.objects.filter( - created_at__gte=start_date, - project_locale__pretranslation_enabled=True, - ) - insights = ( - snapshots + sync_user = User.objects.get(email="pontoon-sync@example.com").pk + + pretranslation_users = User.objects.filter( + email__in=[ + "pontoon-tm@example.com", + "pontoon-gt@example.com", + ] + ).values_list("pk", flat=True) + + actions = ( + ActionLog.objects.filter( + created_at__gte=start_date, + action_type__in=["translation:approved", "translation:rejected"], + translation__user__in=pretranslation_users, + ) + .exclude(performed_by=sync_user) # Truncate to month and add to select list .annotate(month=TruncMonth("created_at")) # Group By month and locale - .values("month", f"project_locale__{category}") - # Select the avg/sum of the grouping - .annotate(pretranslations_approved_sum=Sum("pretranslations_approved")) - .annotate(pretranslations_rejected_sum=Sum("pretranslations_rejected")) + .values("month", f"translation__{category}") + # Select the sum of the grouping + .annotate( + pretranslations_approved_sum=Count( + "id", filter=Q(action_type="translation:approved") + ) + ) + .annotate( + pretranslations_rejected_sum=Count( + "id", filter=Q(action_type="translation:rejected") + ) + ) # Select month and values .values( "month", - f"project_locale__{category}__{id}", - f"project_locale__{category}__name", + f"translation__{category}__{id}", + f"translation__{category}__name", "pretranslations_approved_sum", "pretranslations_rejected_sum", ) @@ -393,13 +412,27 @@ def get_global_pretranslation_quality(category, id): data = { "all": { "name": "All", - "approval_rate": [], + "approval_rate": [None] * 12, } } - for insight in insights: - key = insight[f"project_locale__{category}__{id}"] - name = insight[f"project_locale__{category}__name"] + approved = "pretranslations_approved_sum" + rejected = "pretranslations_rejected_sum" + + totals = [ + { + approved: 0, + rejected: 0, + } + for x in range(0, 12) + ] + + for action in actions: + key = action[f"translation__{category}__{id}"] + name = action[f"translation__{category}__name"] + month_index = relativedelta( + action["month"].replace(tzinfo=None), start_date + ).months if category == "locale": name = f"{name} ยท {key}" @@ -408,24 +441,22 @@ def get_global_pretranslation_quality(category, id): key, { "name": name, - "approval_rate": [], + "approval_rate": [None] * 12, }, ) - - item["approval_rate"].append(get_approval_rate(insight)) - - # Monthly total across the entire category - category_approval_rates = [value["approval_rate"] for value in data.values()] - - for month in zip(*category_approval_rates[1:]): - not_none = [x for x in month if x is not None] - if not_none: - total = statistics.mean(not_none) - else: - total = None - data["all"]["approval_rate"].append(total) + item["approval_rate"][month_index] = get_approval_rate(action) + totals[month_index][approved] += action[approved] + totals[month_index][rejected] += action[rejected] + + # Monthly totals across the entire category + total_approval_rates = data["all"]["approval_rate"] + for idx, _ in enumerate(total_approval_rates): + total_approval_rates[idx] = get_approval_rate(totals[idx]) + total_approval_rates = [ + get_approval_rate(totals[idx]) for idx, _ in enumerate(total_approval_rates) + ] return { - "dates": list({convert_to_unix_time(x["month"]) for x in insights}), + "dates": list({convert_to_unix_time(x["month"]) for x in actions}), "dataset": json.dumps([v for _, v in data.items()]), } diff --git a/pontoon/insights/views.py b/pontoon/insights/views.py index f1157a1c19..3fbe8a3324 100644 --- a/pontoon/insights/views.py +++ b/pontoon/insights/views.py @@ -26,7 +26,7 @@ def insights(request): "locale", "code" ), "project_pretranslation_quality": get_global_pretranslation_quality( - "project", "slug" + "entity__resource__project", "slug" ), }, )