Optimize tags dashboards (mozilla#3101)

The functionality should remain identical, but the views should be significantly faster. In the case of Firefox Tag dashboards, the load times were in the 20-30 second range (or timing out), now these pages take max. 2-3 seconds to load.

The patch also:
* adds an extra DB index, which significantly impacts Tags tab performance
* drops the unused code (which represents the vast majority of changes in this patch)
* moves remaining files from tags/utils to tags/admin

pontoon/base/migrations/0053_alter_translation_index_together.py

@@ -0,0 +1,27 @@
+# Generated by Django 3.2.15 on 2024-02-20 10:51
+
+from django.conf import settings
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ("base", "0052_rename_logged_in_users"),
+    ]
+
+    operations = [
+        migrations.AlterIndexTogether(
+            name="translation",
+            index_together={
+                ("locale", "user", "entity"),
+                ("entity", "user", "approved", "pretranslated"),
+                ("entity", "locale", "fuzzy"),
+                ("date", "locale"),
+                ("entity", "locale", "approved"),
+                ("entity", "locale", "pretranslated"),
+                ("approved_date", "locale"),
+            },
+        ),
+    ]

pontoon/base/models.py

@@ -531,6 +531,25 @@ class AggregatedStats(models.Model):
     class Meta:
         abstract = True
 
+    @classmethod
+    def get_chart_dict(cls, obj):
+        """Get chart data dictionary"""
+        if obj.total_strings:
+            return {
+                "total_strings": obj.total_strings,
+                "approved_strings": obj.approved_strings,
+                "pretranslated_strings": obj.pretranslated_strings,
+                "strings_with_errors": obj.strings_with_errors,
+                "strings_with_warnings": obj.strings_with_warnings,
+                "unreviewed_strings": obj.unreviewed_strings,
+                "approved_share": round(obj.approved_percent),
+                "pretranslated_share": round(obj.pretranslated_percent),
+                "errors_share": round(obj.errors_percent),
+                "warnings_share": round(obj.warnings_percent),
+                "unreviewed_share": round(obj.unreviewed_percent),
+                "completion_percent": int(math.floor(obj.completed_percent)),
+            }
+
     @classmethod
     def get_stats_sum(cls, qs):
         """
@@ -1837,25 +1856,6 @@ def get_chart(cls, self, extra=None):
 
         return chart
 
-    @classmethod
-    def get_chart_dict(cls, obj):
-        """Get chart data dictionary"""
-        if obj.total_strings:
-            return {
-                "total_strings": obj.total_strings,
-                "approved_strings": obj.approved_strings,
-                "pretranslated_strings": obj.pretranslated_strings,
-                "strings_with_errors": obj.strings_with_errors,
-                "strings_with_warnings": obj.strings_with_warnings,
-                "unreviewed_strings": obj.unreviewed_strings,
-                "approved_share": round(obj.approved_percent),
-                "pretranslated_share": round(obj.pretranslated_percent),
-                "errors_share": round(obj.errors_percent),
-                "warnings_share": round(obj.warnings_percent),
-                "unreviewed_share": round(obj.unreviewed_percent),
-                "completion_percent": int(math.floor(obj.completed_percent)),
-            }
-
     def aggregate_stats(self):
         TranslatedResource.objects.filter(
             resource__project=self.project,
@@ -3284,6 +3284,7 @@ class Meta:
             ("entity", "locale", "fuzzy"),
             ("locale", "user", "entity"),
             ("date", "locale"),
+            ("approved_date", "locale"),
         )
         constraints = [
             models.UniqueConstraint(

pontoon/localizations/views.py

@@ -1,5 +1,4 @@
 import math
-from operator import attrgetter
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 from django.db.models import Q
@@ -20,7 +19,7 @@
 )
 from pontoon.contributors.views import ContributorsMixin
 from pontoon.insights.utils import get_insights
-from pontoon.tags.utils import TagsTool
+from pontoon.tags.utils import Tags
 
 
 def localization(request, code, slug):
@@ -159,13 +158,7 @@ def ajax_tags(request, code, slug):
     if not project.tags_enabled:
         raise Http404
 
-    tags_tool = TagsTool(
-        locales=[locale],
-        projects=[project],
-        priority=True,
-    )
-
-    tags = sorted(tags_tool, key=attrgetter("priority"), reverse=True)
+    tags = Tags(project=project, locale=locale).get()
 
     return render(
         request,

pontoon/projects/templates/projects/includes/teams.html

@@ -19,10 +19,12 @@
   {% set locale_projects = project.available_locales_list() %}
 
   {% for locale in locales %}
+    {% if not tag %}
     {% set main_link = url('pontoon.projects.project', project.slug) %}
     {% set chart_link = url('pontoon.translate', locale.code, project.slug, 'all-resources') %}
     {% set latest_activity = locale.get_latest_activity(project) %}
     {% set chart = locale.get_chart(project) %}
+    {% endif %}
     {% if locale.code in locale_projects %}
       {% set class = 'limited' %}
       {% if chart %}
@@ -35,6 +37,8 @@
     {% if tag %}
       {% set main_link = url('pontoon.projects.project', project.slug) + '?tag=' + tag.slug %}
       {% set chart_link = url('pontoon.translate', locale.code, project.slug, 'all-resources') + '?tag=' + tag.slug %}
+      {% set latest_activity = locale.latest_activity %}
+      {% set chart = locale.chart %}
       {% if chart %}
         {% set main_link = chart_link %}
       {% endif %}

pontoon/projects/views.py

@@ -1,5 +1,4 @@
 import uuid
-from operator import attrgetter
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.contrib.contenttypes.models import ContentType
@@ -19,7 +18,7 @@
 from pontoon.contributors.views import ContributorsMixin
 from pontoon.insights.utils import get_insights
 from pontoon.projects import forms
-from pontoon.tags.utils import TagsTool
+from pontoon.tags.utils import Tags
 
 
 def projects(request):
@@ -109,12 +108,7 @@ def ajax_tags(request, slug):
     if not project.tags_enabled:
         raise Http404
 
-    tags_tool = TagsTool(
-        projects=[project],
-        priority=True,
-    )
-
-    tags = sorted(tags_tool, key=attrgetter("priority"), reverse=True)
+    tags = Tags(project=project).get()
 
     return render(
         request,

pontoon/tags/admin/__init__.py

@@ -0,0 +1,10 @@
+from .resources import TagsResourcesTool
+from .tags import TagsTool
+from .tag import TagTool
+
+
+__all__ = (
+    "TagsResourcesTool",
+    "TagsTool",
+    "TagTool",
+)

pontoon/tags/utils/base.py → pontoon/tags/admin/base.py

@@ -1,6 +1,5 @@
 from collections import OrderedDict
 
-from django.db.models import Q
 from django.utils.functional import cached_property
 
 from pontoon.base.models import (
@@ -137,50 +136,3 @@ def translation_manager(self):
     @property
     def tr_manager(self):
         return TranslatedResource.objects
-
-
-class TagsTRTool(TagsDataTool):
-    """Data Tool from the perspective of TranslatedResources"""
-
-    clone_kwargs = TagsDataTool.clone_kwargs + ("annotations", "groupby")
-
-    @property
-    def data_manager(self):
-        return self.tr_manager
-
-    def filter_locales(self, trs):
-        return trs.filter(locale__in=self.locales) if self.locales else trs
-
-    def filter_path(self, trs):
-        return (
-            trs.filter(resource__path__contains=self.path).distinct()
-            if self.path
-            else trs
-        )
-
-    def filter_projects(self, trs):
-        return trs.filter(resource__project__in=self.projects) if self.projects else trs
-
-    def filter_tag(self, trs):
-        """Filters on tag.slug and tag.priority"""
-
-        q = Q()
-        if not self.slug:
-            # if slug is not specified, then just remove all resources
-            # that have no tag
-            q &= ~Q(resource__tag__isnull=True)
-
-        if self.slug:
-            q &= Q(resource__tag__slug__contains=self.slug)
-
-        if self.priority is not None:
-            if self.priority is False:
-                # if priority is False, exclude tags with priority
-                q &= Q(resource__tag__priority__isnull=True)
-            elif self.priority is True:
-                # if priority is True show only tags with priority
-                q &= Q(resource__tag__priority__isnull=False)
-            elif isinstance(self.priority, int):
-                # if priority is an int, filter on that priority
-                q &= Q(resource__tag__priority=self.priority)
-        return trs.filter(q)

pontoon/tags/admin/forms.py

@@ -2,7 +2,7 @@
 from django.utils.functional import cached_property
 
 from pontoon.base.models import Resource
-from pontoon.tags.utils import TagsTool
+from .tags import TagsTool
 
 
 class LinkTagResourcesAdminForm(forms.Form):

pontoon/tags/utils/resources.py → pontoon/tags/admin/resources.py

@@ -1,8 +1,7 @@
 from django.db.models import Q
 
-from pontoon.tags.exceptions import InvalidProjectError
-
 from .base import TagsDataTool
+from .exceptions import InvalidProjectError
 
 
 class TagsResourcesTool(TagsDataTool):

pontoon/tags/utils/tag.py → pontoon/tags/admin/tag.py

@@ -1,9 +1,7 @@
 from django.utils.functional import cached_property
 
-from .tagged import Tagged, TaggedLocale
 
-
-class TagTool(Tagged):
+class TagTool:
     """This wraps ``Tag`` model kwargs providing an API for
     efficient retrieval of related information, eg tagged ``Resources``,
     ``Locales`` and ``Projects``, and methods for managing linked
@@ -29,15 +27,6 @@ def linked_resources(self):
         """``Resources`` that are linked to this ``Tag``"""
         return self.resource_tool.get_linked_resources(self.slug).order_by("path")
 
-    @property
-    def locale_stats(self):
-        return self.tags_tool.stat_tool(slug=self.slug, groupby="locale").data
-
-    @cached_property
-    def locale_latest(self):
-        """A cached property containing latest locale changes"""
-        return self.tags_tool.translation_tool(slug=self.slug, groupby="locale").data
-
     @cached_property
     def object(self):
         """Returns the ``Tag`` model object for this tag.
@@ -61,19 +50,6 @@ def resource_tool(self):
             else self.tags_tool.resource_tool
         )
 
-    def iter_locales(self, project=None):
-        """Iterate ``Locales`` that are associated with this tag
-        (given any filtering in ``self.tags_tool``)
-
-        yields a ``TaggedLocale`` that can be used to get eg chart data
-        """
-        for locale in self.locale_stats:
-            yield TaggedLocale(
-                project=project,
-                latest_translation=self.locale_latest.get(locale["locale"]),
-                **locale
-            )
-
     def link_resources(self, resources):
         """Link Resources to this tag, raises an Error if the tag's
         Project is set, and the requested resource is not in  that Project

pontoon/tags/utils/tags.py → pontoon/tags/admin/tags.py

@@ -4,9 +4,7 @@
 
 from .base import Clonable
 from .resources import TagsResourcesTool
-from .stats import TagsStatsTool
 from .tag import TagTool
-from .translations import TagsLatestTranslationsTool
 
 
 class TagsTool(Clonable):
@@ -17,19 +15,11 @@ class TagsTool(Clonable):
 
     tag_class = TagTool
     resources_class = TagsResourcesTool
-    translations_class = TagsLatestTranslationsTool
-    stats_class = TagsStatsTool
     clone_kwargs = ("locales", "projects", "priority", "path", "slug")
 
     def __getitem__(self, k):
         return self(slug=k)
 
-    def __iter__(self):
-        return self.iter_tags(self.stat_tool.data)
-
-    def __len__(self):
-        return len(self.stat_tool.data)
-
     @property
     def tag_manager(self):
         return Tag.objects
@@ -40,22 +30,6 @@ def resource_tool(self):
             projects=self.projects, locales=self.locales, slug=self.slug, path=self.path
         )
 
-    @cached_property
-    def stat_tool(self):
-        return self.stats_class(
-            slug=self.slug,
-            locales=self.locales,
-            projects=self.projects,
-            priority=self.priority,
-            path=self.path,
-        )
-
-    @cached_property
-    def translation_tool(self):
-        return self.translations_class(
-            slug=self.slug, locales=self.locales, projects=self.projects
-        )
-
     def get(self, slug=None):
         """Get the first tag by iterating self, or by slug if set"""
         if slug is None:
@@ -70,11 +44,3 @@ def get_tags(self, slug=None):
         if slug:
             return tags.filter(slug=slug)
         return tags
-
-    def iter_tags(self, tags):
-        """Iterate associated tags, and create TagTool objects for
-        each, adding latest translation data
-        """
-        for tag in tags:
-            latest_translation = self.translation_tool.data.get(tag["resource__tag"])
-            yield self.tag_class(self, latest_translation=latest_translation, **tag)

pontoon/tags/templates/tags/tag.html

@@ -54,7 +54,7 @@ <h1>
     </ul>
 
     {{ HeadingInfo.progress_chart() }}
-    {{ HeadingInfo.progress_chart_legend(tag) }}
+    {{ HeadingInfo.progress_chart_legend(tag.chart) }}
   </div>
 </section>
 {% endblock %}

pontoon/tags/tests/utils/test_base.py → pontoon/tags/tests/admin/test_base.py

@@ -8,8 +8,8 @@
     Translation,
 )
 
+from pontoon.tags.admin.base import Clonable, TagsDataTool
 from pontoon.tags.models import Tag
-from pontoon.tags.utils.base import Clonable, TagsDataTool
 
 
 def test_util_clonable():