Refactor stats, removing aggregate fields from project, projectlocale…

… & locale (mozilla#3536)

Based on unscientific local testing, the rendering speed benefits from eagerly populating the aggregated stats into Project, ProjectLocale & Locale don't exist, compared to collating the data on the fly from TranslatedResource objects. Therefore, we should not be doing this complex extra work. This significantly simplifies the logic around stats updates, and makes the stats much easier to reason about.

The AggregatedStats class no longer extends Model, and it's mostly a crutch for reducing the risks in this change; hence its move from base/models/ to base/. With it, fields like total_strings and strings_with_errors will continue to work for the Django models from which the corresponding DB fields were removed.

pontoon/administration/tests/test_views.py

@@ -6,7 +6,6 @@
 from pontoon.administration.views import _create_or_update_translated_resources
 from pontoon.base.models import (
     Entity,
-    Locale,
     Project,
     ProjectLocale,
     Resource,
@@ -188,8 +187,8 @@ def test_manage_project_strings_translated_resource(client_superuser):
     assert project.total_strings == strings_count * locales_count
 
     for loc in locales:
-        locale = Locale.objects.get(id=loc.id)
-        assert locale.total_strings == strings_count
+        pl = ProjectLocale.objects.get(locale=loc, project=project)
+        assert pl.total_strings == strings_count
 
 
 @pytest.mark.django_db

pontoon/api/schema.py

@@ -35,6 +35,13 @@ class Meta:
 
 
 class ProjectLocale(DjangoObjectType, Stats):
+    total_strings = graphene.Int()
+    approved_strings = graphene.Int()
+    pretranslated_strings = graphene.Int()
+    strings_with_errors = graphene.Int()
+    strings_with_warnings = graphene.Int()
+    unreviewed_strings = graphene.Int()
+
     class Meta:
         model = ProjectLocaleModel
         fields = (
@@ -50,6 +57,13 @@ class Meta:
 
 
 class Project(DjangoObjectType, Stats):
+    total_strings = graphene.Int()
+    approved_strings = graphene.Int()
+    pretranslated_strings = graphene.Int()
+    strings_with_errors = graphene.Int()
+    strings_with_warnings = graphene.Int()
+    unreviewed_strings = graphene.Int()
+
     class Meta:
         convert_choices_to_enum = False
         model = ProjectModel
@@ -84,6 +98,13 @@ def resolve_tags(obj, info):
 
 
 class Locale(DjangoObjectType, Stats):
+    total_strings = graphene.Int()
+    approved_strings = graphene.Int()
+    pretranslated_strings = graphene.Int()
+    strings_with_errors = graphene.Int()
+    strings_with_warnings = graphene.Int()
+    unreviewed_strings = graphene.Int()
+
     class Meta:
         model = LocaleModel
         fields = (

pontoon/api/tests/test_schema.py

@@ -158,7 +158,8 @@ def test_project_localizations(client):
             project(slug: "pontoon-intro") {
                 localizations {
                     locale {
-                        name
+                        name,
+                        stringsWithErrors
                     }
                 }
             }
@@ -169,7 +170,13 @@ def test_project_localizations(client):
 
     assert response.status_code == 200
     assert response.json() == {
-        "data": {"project": {"localizations": [{"locale": {"name": "English"}}]}}
+        "data": {
+            "project": {
+                "localizations": [
+                    {"locale": {"name": "English", "stringsWithErrors": 0}}
+                ]
+            }
+        }
     }
 
 

pontoon/base/aggregated_stats.py

@@ -0,0 +1,92 @@
+import math
+
+from functools import cached_property
+
+
+class AggregatedStats:
+    aggregated_stats_query: object
+    """
+    Must be set by the child class as a QuerySet of TranslatedResource objects.
+
+    Should include any filters leaving out disabled or system projects.
+    """
+
+    @cached_property
+    def _stats(self) -> dict[str, int]:
+        return self.aggregated_stats_query.string_stats(
+            count_disabled=True, count_system_projects=True
+        )
+
+    @property
+    def total_strings(self) -> int:
+        return self._stats["total"]
+
+    @property
+    def approved_strings(self) -> int:
+        return self._stats["approved"]
+
+    @property
+    def pretranslated_strings(self) -> int:
+        return self._stats["pretranslated"]
+
+    @property
+    def strings_with_errors(self) -> int:
+        return self._stats["errors"]
+
+    @property
+    def strings_with_warnings(self) -> int:
+        return self._stats["warnings"]
+
+    @property
+    def unreviewed_strings(self) -> int:
+        return self._stats["unreviewed"]
+
+    @property
+    def missing_strings(self):
+        return (
+            self.total_strings
+            - self.approved_strings
+            - self.pretranslated_strings
+            - self.strings_with_errors
+            - self.strings_with_warnings
+        )
+
+
+def get_completed_percent(obj):
+    if not obj.total_strings:
+        return 0
+    completed_strings = (
+        obj.approved_strings + obj.pretranslated_strings + obj.strings_with_warnings
+    )
+    return completed_strings / obj.total_strings * 100
+
+
+def get_chart_dict(obj: "AggregatedStats"):
+    """Get chart data dictionary"""
+    if ts := obj.total_strings:
+        return {
+            "total": ts,
+            "approved": obj.approved_strings,
+            "pretranslated": obj.pretranslated_strings,
+            "errors": obj.strings_with_errors,
+            "warnings": obj.strings_with_warnings,
+            "unreviewed": obj.unreviewed_strings,
+            "approved_share": round(obj.approved_strings / ts * 100),
+            "pretranslated_share": round(obj.pretranslated_strings / ts * 100),
+            "errors_share": round(obj.strings_with_errors / ts * 100),
+            "warnings_share": round(obj.strings_with_warnings / ts * 100),
+            "unreviewed_share": round(obj.unreviewed_strings / ts * 100),
+            "completion_percent": int(math.floor(get_completed_percent(obj))),
+        }
+
+
+def get_top_instances(qs):
+    """
+    Get top instances in the queryset.
+    """
+    return {
+        "most_strings": sorted(qs, key=lambda x: x.total_strings)[-1],
+        "most_translations": sorted(qs, key=lambda x: x.approved_strings)[-1],
+        "most_suggestions": sorted(qs, key=lambda x: x.unreviewed_strings)[-1],
+        "most_missing": sorted(qs, key=lambda x: x.missing_strings)[-1],
+    }

pontoon/base/management/commands/calculate_stats.py

@@ -4,7 +4,7 @@
 from django.db.models import Count
 
 from pontoon.base.models import Project
-from pontoon.sync.core.stats import update_locale_stats, update_stats
+from pontoon.sync.core.stats import update_stats
 
 
 log = logging.getLogger(__name__)
@@ -34,7 +34,6 @@ def handle(self, *args, **options):
 
         log.info(f"Calculating stats for {len(projects)} projects...")
         for project in projects:
-            update_stats(project, update_locales=False)
-        update_locale_stats()
+            update_stats(project)
 
         log.info("Calculating stats complete for all projects.")

pontoon/base/migrations/0072_remove_locale_approved_strings_and_more.py

@@ -0,0 +1,84 @@
+# Generated by Django 4.2.17 on 2025-01-17 19:02
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("base", "0071_alter_repository_type"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="locale",
+            name="approved_strings",
+        ),
+        migrations.RemoveField(
+            model_name="locale",
+            name="pretranslated_strings",
+        ),
+        migrations.RemoveField(
+            model_name="locale",
+            name="strings_with_errors",
+        ),
+        migrations.RemoveField(
+            model_name="locale",
+            name="strings_with_warnings",
+        ),
+        migrations.RemoveField(
+            model_name="locale",
+            name="total_strings",
+        ),
+        migrations.RemoveField(
+            model_name="locale",
+            name="unreviewed_strings",
+        ),
+        migrations.RemoveField(
+            model_name="project",
+            name="approved_strings",
+        ),
+        migrations.RemoveField(
+            model_name="project",
+            name="pretranslated_strings",
+        ),
+        migrations.RemoveField(
+            model_name="project",
+            name="strings_with_errors",
+        ),
+        migrations.RemoveField(
+            model_name="project",
+            name="strings_with_warnings",
+        ),
+        migrations.RemoveField(
+            model_name="project",
+            name="total_strings",
+        ),
+        migrations.RemoveField(
+            model_name="project",
+            name="unreviewed_strings",
+        ),
+        migrations.RemoveField(
+            model_name="projectlocale",
+            name="approved_strings",
+        ),
+        migrations.RemoveField(
+            model_name="projectlocale",
+            name="pretranslated_strings",
+        ),
+        migrations.RemoveField(
+            model_name="projectlocale",
+            name="strings_with_errors",
+        ),
+        migrations.RemoveField(
+            model_name="projectlocale",
+            name="strings_with_warnings",
+        ),
+        migrations.RemoveField(
+            model_name="projectlocale",
+            name="total_strings",
+        ),
+        migrations.RemoveField(
+            model_name="projectlocale",
+            name="unreviewed_strings",
+        ),
+    ]

pontoon/base/models/__init__.py

@@ -1,4 +1,3 @@
-from pontoon.base.models.aggregated_stats import AggregatedStats
 from pontoon.base.models.changed_entity_locale import ChangedEntityLocale
 from pontoon.base.models.comment import Comment
 from pontoon.base.models.entity import Entity, get_word_count
@@ -18,7 +17,6 @@
 
 
 __all__ = [
-    "AggregatedStats",
     "ChangedEntityLocale",
     "Comment",
     "Entity",