diff --git a/pontoon/administration/tests/test_views.py b/pontoon/administration/tests/test_views.py index f05604b411..948f2c79fa 100644 --- a/pontoon/administration/tests/test_views.py +++ b/pontoon/administration/tests/test_views.py @@ -6,7 +6,6 @@ from pontoon.administration.views import _create_or_update_translated_resources from pontoon.base.models import ( Entity, - Locale, Project, ProjectLocale, Resource, @@ -188,8 +187,8 @@ def test_manage_project_strings_translated_resource(client_superuser): assert project.total_strings == strings_count * locales_count for loc in locales: - locale = Locale.objects.get(id=loc.id) - assert locale.total_strings == strings_count + pl = ProjectLocale.objects.get(locale=loc, project=project) + assert pl.total_strings == strings_count @pytest.mark.django_db diff --git a/pontoon/api/schema.py b/pontoon/api/schema.py index d89283ea3c..54e096f247 100644 --- a/pontoon/api/schema.py +++ b/pontoon/api/schema.py @@ -29,6 +29,13 @@ class Meta: class ProjectLocale(DjangoObjectType, Stats): + total_strings = graphene.Int() + approved_strings = graphene.Int() + pretranslated_strings = graphene.Int() + strings_with_errors = graphene.Int() + strings_with_warnings = graphene.Int() + unreviewed_strings = graphene.Int() + class Meta: model = ProjectLocaleModel fields = ( @@ -44,6 +51,13 @@ class Meta: class Project(DjangoObjectType, Stats): + total_strings = graphene.Int() + approved_strings = graphene.Int() + pretranslated_strings = graphene.Int() + strings_with_errors = graphene.Int() + strings_with_warnings = graphene.Int() + unreviewed_strings = graphene.Int() + class Meta: convert_choices_to_enum = False model = ProjectModel @@ -78,6 +92,13 @@ def resolve_tags(obj, info): class Locale(DjangoObjectType, Stats): + total_strings = graphene.Int() + approved_strings = graphene.Int() + pretranslated_strings = graphene.Int() + strings_with_errors = graphene.Int() + strings_with_warnings = graphene.Int() + unreviewed_strings = graphene.Int() + class Meta: model = LocaleModel fields = ( diff --git a/pontoon/api/tests/test_schema.py b/pontoon/api/tests/test_schema.py index b8403b2937..024a94bf1c 100644 --- a/pontoon/api/tests/test_schema.py +++ b/pontoon/api/tests/test_schema.py @@ -136,7 +136,8 @@ def test_project_localizations(client): project(slug: "pontoon-intro") { localizations { locale { - name + name, + stringsWithErrors } } } @@ -147,7 +148,13 @@ def test_project_localizations(client): assert response.status_code == 200 assert response.json() == { - "data": {"project": {"localizations": [{"locale": {"name": "English"}}]}} + "data": { + "project": { + "localizations": [ + {"locale": {"name": "English", "stringsWithErrors": 0}} + ] + } + } } diff --git a/pontoon/base/aggregated_stats.py b/pontoon/base/aggregated_stats.py new file mode 100644 index 0000000000..de1c9424e9 --- /dev/null +++ b/pontoon/base/aggregated_stats.py @@ -0,0 +1,92 @@ +import math + +from functools import cached_property + + +class AggregatedStats: + aggregated_stats_query: object + """ + Must be set by the child class as a QuerySet of TranslatedResource objects. + + Should include any filters leaving out disabled or system projects. + """ + + @cached_property + def _stats(self) -> dict[str, int]: + return self.aggregated_stats_query.string_stats( + count_disabled=True, count_system_projects=True + ) + + @property + def total_strings(self) -> int: + return self._stats["total"] + + @property + def approved_strings(self) -> int: + return self._stats["approved"] + + @property + def pretranslated_strings(self) -> int: + return self._stats["pretranslated"] + + @property + def strings_with_errors(self) -> int: + return self._stats["errors"] + + @property + def strings_with_warnings(self) -> int: + return self._stats["warnings"] + + @property + def unreviewed_strings(self) -> int: + return self._stats["unreviewed"] + + @property + def missing_strings(self): + return ( + self.total_strings + - self.approved_strings + - self.pretranslated_strings + - self.strings_with_errors + - self.strings_with_warnings + ) + + +def get_completed_percent(obj): + if not obj.total_strings: + return 0 + completed_strings = ( + obj.approved_strings + obj.pretranslated_strings + obj.strings_with_warnings + ) + return completed_strings / obj.total_strings * 100 + + +def get_chart_dict(obj: "AggregatedStats"): + """Get chart data dictionary""" + if ts := obj.total_strings: + return { + "total": ts, + "approved": obj.approved_strings, + "pretranslated": obj.pretranslated_strings, + "errors": obj.strings_with_errors, + "warnings": obj.strings_with_warnings, + "unreviewed": obj.unreviewed_strings, + "approved_share": round(obj.approved_strings / ts * 100), + "pretranslated_share": round(obj.pretranslated_strings / ts * 100), + "errors_share": round(obj.strings_with_errors / ts * 100), + "warnings_share": round(obj.strings_with_warnings / ts * 100), + "unreviewed_share": round(obj.unreviewed_strings / ts * 100), + "completion_percent": int(math.floor(get_completed_percent(obj))), + } + + +def get_top_instances(qs): + """ + Get top instances in the queryset. + """ + return { + "most_strings": sorted(qs, key=lambda x: x.total_strings)[-1], + "most_translations": sorted(qs, key=lambda x: x.approved_strings)[-1], + "most_suggestions": sorted(qs, key=lambda x: x.unreviewed_strings)[-1], + "most_missing": sorted(qs, key=lambda x: x.missing_strings)[-1], + } diff --git a/pontoon/base/management/commands/calculate_stats.py b/pontoon/base/management/commands/calculate_stats.py index 04e4e933f4..0e7040a38e 100644 --- a/pontoon/base/management/commands/calculate_stats.py +++ b/pontoon/base/management/commands/calculate_stats.py @@ -4,7 +4,7 @@ from django.db.models import Count from pontoon.base.models import Project -from pontoon.sync.core.stats import update_locale_stats, update_stats +from pontoon.sync.core.stats import update_stats log = logging.getLogger(__name__) @@ -34,7 +34,6 @@ def handle(self, *args, **options): log.info(f"Calculating stats for {len(projects)} projects...") for project in projects: - update_stats(project, update_locales=False) - update_locale_stats() + update_stats(project) log.info("Calculating stats complete for all projects.") diff --git a/pontoon/base/migrations/0072_remove_locale_approved_strings_and_more.py b/pontoon/base/migrations/0072_remove_locale_approved_strings_and_more.py new file mode 100644 index 0000000000..ba61ef48ec --- /dev/null +++ b/pontoon/base/migrations/0072_remove_locale_approved_strings_and_more.py @@ -0,0 +1,84 @@ +# Generated by Django 4.2.17 on 2025-01-17 19:02 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("base", "0071_alter_repository_type"), + ] + + operations = [ + migrations.RemoveField( + model_name="locale", + name="approved_strings", + ), + migrations.RemoveField( + model_name="locale", + name="pretranslated_strings", + ), + migrations.RemoveField( + model_name="locale", + name="strings_with_errors", + ), + migrations.RemoveField( + model_name="locale", + name="strings_with_warnings", + ), + migrations.RemoveField( + model_name="locale", + name="total_strings", + ), + migrations.RemoveField( + model_name="locale", + name="unreviewed_strings", + ), + migrations.RemoveField( + model_name="project", + name="approved_strings", + ), + migrations.RemoveField( + model_name="project", + name="pretranslated_strings", + ), + migrations.RemoveField( + model_name="project", + name="strings_with_errors", + ), + migrations.RemoveField( + model_name="project", + name="strings_with_warnings", + ), + migrations.RemoveField( + model_name="project", + name="total_strings", + ), + migrations.RemoveField( + model_name="project", + name="unreviewed_strings", + ), + migrations.RemoveField( + model_name="projectlocale", + name="approved_strings", + ), + migrations.RemoveField( + model_name="projectlocale", + name="pretranslated_strings", + ), + migrations.RemoveField( + model_name="projectlocale", + name="strings_with_errors", + ), + migrations.RemoveField( + model_name="projectlocale", + name="strings_with_warnings", + ), + migrations.RemoveField( + model_name="projectlocale", + name="total_strings", + ), + migrations.RemoveField( + model_name="projectlocale", + name="unreviewed_strings", + ), + ] diff --git a/pontoon/base/models/__init__.py b/pontoon/base/models/__init__.py index f095bdf610..affb06ecd0 100644 --- a/pontoon/base/models/__init__.py +++ b/pontoon/base/models/__init__.py @@ -1,4 +1,3 @@ -from pontoon.base.models.aggregated_stats import AggregatedStats from pontoon.base.models.changed_entity_locale import ChangedEntityLocale from pontoon.base.models.comment import Comment from pontoon.base.models.entity import Entity, get_word_count @@ -18,7 +17,6 @@ __all__ = [ - "AggregatedStats", "ChangedEntityLocale", "Comment", "Entity", diff --git a/pontoon/base/models/aggregated_stats.py b/pontoon/base/models/aggregated_stats.py deleted file mode 100644 index ac3dac3f66..0000000000 --- a/pontoon/base/models/aggregated_stats.py +++ /dev/null @@ -1,141 +0,0 @@ -import math - -from django.db import models -from django.db.models import F - - -class AggregatedStats(models.Model): - total_strings = models.PositiveIntegerField(default=0) - approved_strings = models.PositiveIntegerField(default=0) - pretranslated_strings = models.PositiveIntegerField(default=0) - strings_with_errors = models.PositiveIntegerField(default=0) - strings_with_warnings = models.PositiveIntegerField(default=0) - unreviewed_strings = models.PositiveIntegerField(default=0) - - class Meta: - abstract = True - - @classmethod - def get_chart_dict(cls, obj): - """Get chart data dictionary""" - if obj.total_strings: - return { - "total_strings": obj.total_strings, - "approved_strings": obj.approved_strings, - "pretranslated_strings": obj.pretranslated_strings, - "strings_with_errors": obj.strings_with_errors, - "strings_with_warnings": obj.strings_with_warnings, - "unreviewed_strings": obj.unreviewed_strings, - "approved_share": round(obj.approved_percent), - "pretranslated_share": round(obj.pretranslated_percent), - "errors_share": round(obj.errors_percent), - "warnings_share": round(obj.warnings_percent), - "unreviewed_share": round(obj.unreviewed_percent), - "completion_percent": int(math.floor(obj.completed_percent)), - } - - @classmethod - def get_stats_sum(cls, qs): - """ - Get sum of stats for all items in the queryset. - """ - return { - "total_strings": sum(x.total_strings for x in qs), - "approved_strings": sum(x.approved_strings for x in qs), - "pretranslated_strings": sum(x.pretranslated_strings for x in qs), - "strings_with_errors": sum(x.strings_with_errors for x in qs), - "strings_with_warnings": sum(x.strings_with_warnings for x in qs), - "unreviewed_strings": sum(x.unreviewed_strings for x in qs), - } - - @classmethod - def get_top_instances(cls, qs): - """ - Get top instances in the queryset. - """ - return { - "most_strings": sorted(qs, key=lambda x: x.total_strings)[-1], - "most_translations": sorted(qs, key=lambda x: x.approved_strings)[-1], - "most_suggestions": sorted(qs, key=lambda x: x.unreviewed_strings)[-1], - "most_missing": sorted(qs, key=lambda x: x.missing_strings)[-1], - } - - def adjust_stats( - self, - total_strings_diff, - approved_strings_diff, - pretranslated_strings_diff, - strings_with_errors_diff, - strings_with_warnings_diff, - unreviewed_strings_diff, - ): - self.total_strings = F("total_strings") + total_strings_diff - self.approved_strings = F("approved_strings") + approved_strings_diff - self.pretranslated_strings = ( - F("pretranslated_strings") + pretranslated_strings_diff - ) - self.strings_with_errors = F("strings_with_errors") + strings_with_errors_diff - self.strings_with_warnings = ( - F("strings_with_warnings") + strings_with_warnings_diff - ) - self.unreviewed_strings = F("unreviewed_strings") + unreviewed_strings_diff - - self.save( - update_fields=[ - "total_strings", - "approved_strings", - "pretranslated_strings", - "strings_with_errors", - "strings_with_warnings", - "unreviewed_strings", - ] - ) - - @property - def missing_strings(self): - return ( - self.total_strings - - self.approved_strings - - self.pretranslated_strings - - self.strings_with_errors - - self.strings_with_warnings - ) - - @property - def completed_strings(self): - return ( - self.approved_strings - + self.pretranslated_strings - + self.strings_with_warnings - ) - - @property - def complete(self): - return self.total_strings == self.completed_strings - - @property - def completed_percent(self): - return self.percent_of_total(self.completed_strings) - - @property - def approved_percent(self): - return self.percent_of_total(self.approved_strings) - - @property - def pretranslated_percent(self): - return self.percent_of_total(self.pretranslated_strings) - - @property - def errors_percent(self): - return self.percent_of_total(self.strings_with_errors) - - @property - def warnings_percent(self): - return self.percent_of_total(self.strings_with_warnings) - - @property - def unreviewed_percent(self): - return self.percent_of_total(self.unreviewed_strings) - - def percent_of_total(self, n): - return n / self.total_strings * 100 if self.total_strings else 0 diff --git a/pontoon/base/models/entity.py b/pontoon/base/models/entity.py index 9d0b83ed87..374ccd81c3 100644 --- a/pontoon/base/models/entity.py +++ b/pontoon/base/models/entity.py @@ -495,13 +495,12 @@ def save(self, *args, **kwargs): self.word_count = get_word_count(self.string) super().save(*args, **kwargs) - def get_stats(self, locale): + def get_stats(self, locale) -> dict[str, int]: """ Get stats for a single (entity, locale) pair. :arg Locale locale: filter translations for this locale. - :return: a dictionary with stats for an Entity, all keys are suffixed with `_diff` to - make them easier to pass into adjust_all_stats. + :return: a dictionary with stats for the Entity+Locale """ approved = 0 pretranslated = 0 @@ -526,12 +525,11 @@ def get_stats(self, locale): unreviewed += 1 return { - "total_strings_diff": 0, - "approved_strings_diff": approved, - "pretranslated_strings_diff": pretranslated, - "strings_with_errors_diff": errors, - "strings_with_warnings_diff": warnings, - "unreviewed_strings_diff": unreviewed, + "approved": approved, + "pretranslated": pretranslated, + "errors": errors, + "warnings": warnings, + "unreviewed": unreviewed, } def has_changed(self, locale): diff --git a/pontoon/base/models/locale.py b/pontoon/base/models/locale.py index cb5d8914ac..73aaae2fe9 100644 --- a/pontoon/base/models/locale.py +++ b/pontoon/base/models/locale.py @@ -7,9 +7,9 @@ from django.contrib.auth.models import Group from django.core.exceptions import ValidationError from django.db import models -from django.db.models import F, Prefetch +from django.db.models import Prefetch -from pontoon.base.models.aggregated_stats import AggregatedStats +from pontoon.base.aggregated_stats import AggregatedStats log = logging.getLogger(__name__) @@ -79,20 +79,19 @@ def prefetch_project_locale(self, project): ) ) - def get_stats_sum(self): - """ - Get sum of stats for all items in the queryset. - """ - return AggregatedStats.get_stats_sum(self) - def get_top_instances(self): - """ - Get top instances in the queryset. - """ - return AggregatedStats.get_top_instances(self) +class Locale(models.Model, AggregatedStats): + @property + def aggregated_stats_query(self): + from pontoon.base.models.translated_resource import TranslatedResource + return TranslatedResource.objects.filter( + locale=self, + resource__project__disabled=False, + resource__project__system_project=False, + resource__project__visibility="public", + ) -class Locale(AggregatedStats): code = models.CharField(max_length=20, unique=True) google_translate_code = models.CharField( @@ -372,83 +371,6 @@ def get_chart(self, project=None): return ProjectLocale.get_chart(self, project) - def aggregate_stats(self): - from pontoon.base.models.project import Project - from pontoon.base.models.translated_resource import TranslatedResource - - TranslatedResource.objects.filter( - resource__project__disabled=False, - resource__project__system_project=False, - resource__project__visibility=Project.Visibility.PUBLIC, - resource__entities__obsolete=False, - locale=self, - ).distinct().aggregate_stats(self) - - def stats(self): - """Get locale stats used in All Resources part.""" - return [ - { - "title": "all-resources", - "resource__path": [], - # FIXME rename as total_strings - "resource__total_strings": self.total_strings, - "pretranslated_strings": self.pretranslated_strings, - "strings_with_errors": self.strings_with_errors, - "strings_with_warnings": self.strings_with_warnings, - "unreviewed_strings": self.unreviewed_strings, - "approved_strings": self.approved_strings, - } - ] - - def parts_stats(self, project): - """Get locale-project paths with stats.""" - from pontoon.base.models.project_locale import ProjectLocale - from pontoon.base.models.translated_resource import TranslatedResource - - def get_details(parts): - return parts.order_by("title").values( - "title", - "resource__path", - "resource__deadline", - # FIXME rename as total_strings - "resource__total_strings", - "pretranslated_strings", - "strings_with_errors", - "strings_with_warnings", - "unreviewed_strings", - "approved_strings", - ) - - translatedresources = TranslatedResource.objects.filter( - resource__project=project, resource__entities__obsolete=False, locale=self - ).distinct() - details = list( - get_details( - translatedresources.annotate( - resource__total_strings=F("total_strings"), - title=F("resource__path"), - ) - ) - ) - - all_resources = ProjectLocale.objects.get(project=project, locale=self) - details.append( - { - "title": "all-resources", - "resource__path": [], - "resource__deadline": [], - # FIXME rename as total_strings - "resource__total_strings": all_resources.total_strings, - "pretranslated_strings": all_resources.pretranslated_strings, - "strings_with_errors": all_resources.strings_with_errors, - "strings_with_warnings": all_resources.strings_with_warnings, - "unreviewed_strings": all_resources.unreviewed_strings, - "approved_strings": all_resources.approved_strings, - } - ) - - return details - def save(self, *args, **kwargs): old = Locale.objects.get(pk=self.pk) if self.pk else None super().save(*args, **kwargs) diff --git a/pontoon/base/models/project.py b/pontoon/base/models/project.py index ca90e6c262..cffdbf53a6 100644 --- a/pontoon/base/models/project.py +++ b/pontoon/base/models/project.py @@ -8,7 +8,7 @@ from django.db.models.manager import BaseManager from django.utils import timezone -from pontoon.base.models.aggregated_stats import AggregatedStats +from pontoon.base.aggregated_stats import AggregatedStats from pontoon.base.models.locale import Locale @@ -86,20 +86,14 @@ def prefetch_project_locale(self, locale): ) ) - def get_stats_sum(self): - """ - Get sum of stats for all items in the queryset. - """ - return AggregatedStats.get_stats_sum(self) - def get_top_instances(self): - """ - Get top instances in the queryset. - """ - return AggregatedStats.get_top_instances(self) +class Project(models.Model, AggregatedStats): + @property + def aggregated_stats_query(self): + from pontoon.base.models.translated_resource import TranslatedResource + return TranslatedResource.objects.filter(resource__project=self) -class Project(AggregatedStats): name = models.CharField(max_length=128, unique=True) slug = models.SlugField(unique=True) locales = models.ManyToManyField(Locale, through="ProjectLocale") @@ -239,33 +233,15 @@ def serialize(self): } def save(self, *args, **kwargs): - """ - When project disabled status changes, update denormalized stats - for all project locales. - """ - disabled_changed = False - visibility_changed = False - if self.pk is not None: try: - original = Project.objects.get(pk=self.pk) - if self.visibility != original.visibility: - visibility_changed = True - if self.disabled != original.disabled: - disabled_changed = True - if self.disabled: - self.date_disabled = timezone.now() - else: - self.date_disabled = None + if self.disabled != Project.objects.get(pk=self.pk).disabled: + self.date_disabled = timezone.now() if self.disabled else None except Project.DoesNotExist: pass super().save(*args, **kwargs) - if disabled_changed or visibility_changed: - for locale in self.locales.all(): - locale.aggregate_stats() - @property def checkout_path(self): """Path where this project's VCS checkouts are located.""" @@ -281,13 +257,6 @@ def get_chart(self, locale=None): return ProjectLocale.get_chart(self, locale) - def aggregate_stats(self): - from pontoon.base.models.translated_resource import TranslatedResource - - TranslatedResource.objects.filter( - resource__project=self, resource__entities__obsolete=False - ).distinct().aggregate_stats(self) - @property def avg_string_count(self): return int(self.total_strings / self.enabled_locales) diff --git a/pontoon/base/models/project_locale.py b/pontoon/base/models/project_locale.py index 0ae6e9baca..36ceab48e2 100644 --- a/pontoon/base/models/project_locale.py +++ b/pontoon/base/models/project_locale.py @@ -1,24 +1,13 @@ from django.contrib.auth.models import Group from django.db import models -from django.db.models import Sum from pontoon.base import utils -from pontoon.base.models.aggregated_stats import AggregatedStats +from pontoon.base.aggregated_stats import AggregatedStats, get_chart_dict from pontoon.base.models.locale import Locale from pontoon.base.models.project import Project class ProjectLocaleQuerySet(models.QuerySet): - def aggregated_stats(self): - return self.aggregate( - total_strings=Sum("total_strings"), - approved_strings=Sum("approved_strings"), - pretranslated_strings=Sum("pretranslated_strings"), - strings_with_errors=Sum("strings_with_errors"), - strings_with_warnings=Sum("strings_with_warnings"), - unreviewed_strings=Sum("unreviewed_strings"), - ) - def visible_for(self, user): """ Filter project locales by the visibility of their projects. @@ -41,9 +30,17 @@ def visible(self): ).distinct() -class ProjectLocale(AggregatedStats): +class ProjectLocale(models.Model, AggregatedStats): """Link between a project and a locale that is active for it.""" + @property + def aggregated_stats_query(self): + from pontoon.base.models.translated_resource import TranslatedResource + + return TranslatedResource.objects.filter( + locale=self.locale, resource__project=self.project + ) + project = models.ForeignKey(Project, models.CASCADE, related_name="project_locale") locale = models.ForeignKey(Locale, models.CASCADE, related_name="project_locale") readonly = models.BooleanField(default=False) @@ -129,10 +126,10 @@ def get_chart(cls, self, extra=None): if getattr(self, "fetched_project_locale", None): if self.fetched_project_locale: - chart = cls.get_chart_dict(self.fetched_project_locale[0]) + chart = get_chart_dict(self.fetched_project_locale[0]) elif extra is None: - chart = cls.get_chart_dict(self) + chart = get_chart_dict(self) else: project = self if isinstance(self, Project) else extra @@ -142,16 +139,6 @@ def get_chart(cls, self, extra=None): ) if project_locale is not None: - chart = cls.get_chart_dict(project_locale) + chart = get_chart_dict(project_locale) return chart - - def aggregate_stats(self): - from pontoon.base.models.translated_resource import TranslatedResource - - TranslatedResource.objects.filter( - resource__project=self.project, - resource__project__disabled=False, - resource__entities__obsolete=False, - locale=self.locale, - ).distinct().aggregate_stats(self) diff --git a/pontoon/base/models/translated_resource.py b/pontoon/base/models/translated_resource.py index 3f6fa84009..0f82cbc3c7 100644 --- a/pontoon/base/models/translated_resource.py +++ b/pontoon/base/models/translated_resource.py @@ -1,149 +1,77 @@ import logging -from typing import Any - from django.db import models -from django.db.models import Q, Sum +from django.db.models import F, Q, Sum -from pontoon.base import utils -from pontoon.base.models.aggregated_stats import AggregatedStats -from pontoon.base.models.entity import Entity -from pontoon.base.models.locale import Locale -from pontoon.base.models.project import Project -from pontoon.base.models.project_locale import ProjectLocale -from pontoon.base.models.resource import Resource -from pontoon.base.models.translation import Translation +from .entity import Entity +from .locale import Locale +from .project import Project +from .resource import Resource +from .translation import Translation +from .user import User log = logging.getLogger(__name__) class TranslatedResourceQuerySet(models.QuerySet): - def aggregated_stats(self): - return self.aggregate( - total=Sum("total_strings"), - approved=Sum("approved_strings"), - pretranslated=Sum("pretranslated_strings"), - errors=Sum("strings_with_errors"), - warnings=Sum("strings_with_warnings"), - unreviewed=Sum("unreviewed_strings"), + def string_stats( + self, + user: User | None = None, + *, + count_disabled: bool = False, + count_system_projects: bool = False, + ) -> dict[str, int]: + query = self + if not count_disabled: + query = query.filter(resource__project__disabled=False) + if not count_system_projects: + query = query.filter(resource__project__system_project=False) + if user is None or not user.is_superuser: + query = query.filter(resource__project__visibility="public") + return query.aggregate( + total=Sum("total_strings", default=0), + approved=Sum("approved_strings", default=0), + pretranslated=Sum("pretranslated_strings", default=0), + errors=Sum("strings_with_errors", default=0), + warnings=Sum("strings_with_warnings", default=0), + unreviewed=Sum("unreviewed_strings", default=0), ) - def aggregate_stats(self, instance): - aggregated_stats = self.aggregated_stats() - - instance.total_strings = aggregated_stats["total"] or 0 - instance.approved_strings = aggregated_stats["approved"] or 0 - instance.pretranslated_strings = aggregated_stats["pretranslated"] or 0 - instance.strings_with_errors = aggregated_stats["errors"] or 0 - instance.strings_with_warnings = aggregated_stats["warnings"] or 0 - instance.unreviewed_strings = aggregated_stats["unreviewed"] or 0 - - instance.save( - update_fields=[ - "total_strings", - "approved_strings", - "pretranslated_strings", - "strings_with_errors", - "strings_with_warnings", - "unreviewed_strings", - ] - ) - - def stats(self, project, paths, locale): + def query_stats(self, project: Project, paths: list[str], locale: Locale): """ Returns statistics for the given project, paths and locale. """ - translated_resources = self.filter( - locale=locale, - resource__project__disabled=False, - ) - + query = self.filter(locale=locale) if project.slug == "all-projects": - translated_resources = translated_resources.filter( - resource__project__system_project=False, - resource__project__visibility=Project.Visibility.PUBLIC, - ) + return query.string_stats() else: - translated_resources = translated_resources.filter( - resource__project=project, - ) - + query = query.filter(resource__project=project) if paths: - translated_resources = translated_resources.filter( - resource__path__in=paths, - ) - - return translated_resources.aggregated_stats() - - def update_stats(self): - """ - Update stats on a list of TranslatedResource. - """ - - def _log(n: int, thing: str): - things = thing if n == 1 else f"{thing}s" - log.debug(f"update_stats: {n} {things}") - - fields = [ - "total_strings", - "approved_strings", - "pretranslated_strings", - "strings_with_errors", - "strings_with_warnings", - "unreviewed_strings", - ] + query = query.filter(resource__path__in=paths) + return query.string_stats(count_system_projects=True) + def calculate_stats(self): self = self.prefetch_related("resource__project", "locale") for translated_resource in self: translated_resource.calculate_stats(save=False) - TranslatedResource.objects.bulk_update(self, fields=fields) - _log(len(self), "translated resource") - - projectlocale_count = 0 - for projectlocale in ProjectLocale.objects.filter( - project__resources__translatedresources__in=self, - locale__translatedresources__in=self, - ).distinct(): - projectlocale.aggregate_stats() - projectlocale_count += 1 - _log(projectlocale_count, "projectlocale") - - project_count = 0 - for project in Project.objects.filter( - resources__translatedresources__in=self, - ).distinct(): - stats: dict[str, Any] = ProjectLocale.objects.filter( - project=project - ).aggregated_stats() - project.total_strings = stats["total_strings"] or 0 - project.approved_strings = stats["approved_strings"] or 0 - project.pretranslated_strings = stats["pretranslated_strings"] or 0 - project.strings_with_errors = stats["strings_with_errors"] or 0 - project.strings_with_warnings = stats["strings_with_warnings"] or 0 - project.unreviewed_strings = stats["unreviewed_strings"] or 0 - project.save(update_fields=fields) - project_count += 1 - _log(project_count, "project") + TranslatedResource.objects.bulk_update( + self, + fields=[ + "total_strings", + "approved_strings", + "pretranslated_strings", + "strings_with_errors", + "strings_with_warnings", + "unreviewed_strings", + ], + ) - locales = Locale.objects.filter(translatedresources__in=self).distinct() - for locale in locales: - stats: dict[str, Any] = ProjectLocale.objects.filter( - locale=locale, - project__system_project=False, - project__visibility=Project.Visibility.PUBLIC, - ).aggregated_stats() - locale.total_strings = stats["total_strings"] or 0 - locale.approved_strings = stats["approved_strings"] or 0 - locale.pretranslated_strings = stats["pretranslated_strings"] or 0 - locale.strings_with_errors = stats["strings_with_errors"] or 0 - locale.strings_with_warnings = stats["strings_with_warnings"] or 0 - locale.unreviewed_strings = stats["unreviewed_strings"] or 0 - Locale.objects.bulk_update(locales, fields=fields) - _log(len(locales), "locale") + n = len(self) + log.debug(f"update_stats: {n} translated resource{'' if n == 1 else 's'}") -class TranslatedResource(AggregatedStats): +class TranslatedResource(models.Model): """ Resource representation for a specific locale. """ @@ -155,6 +83,13 @@ class TranslatedResource(AggregatedStats): Locale, models.CASCADE, related_name="translatedresources" ) + total_strings = models.PositiveIntegerField(default=0) + approved_strings = models.PositiveIntegerField(default=0) + pretranslated_strings = models.PositiveIntegerField(default=0) + strings_with_errors = models.PositiveIntegerField(default=0) + strings_with_warnings = models.PositiveIntegerField(default=0) + unreviewed_strings = models.PositiveIntegerField(default=0) + #: Most recent translation approved or created for this translated #: resource. latest_translation = models.ForeignKey( @@ -170,28 +105,6 @@ class TranslatedResource(AggregatedStats): class Meta: unique_together = (("locale", "resource"),) - def adjust_all_stats(self, *args, **kwargs): - project = self.resource.project - locale = self.locale - - project_locale = utils.get_object_or_none( - ProjectLocale, - project=project, - locale=locale, - ) - - self.adjust_stats(*args, **kwargs) - project.adjust_stats(*args, **kwargs) - - if ( - not project.system_project - and project.visibility == Project.Visibility.PUBLIC - ): - locale.adjust_stats(*args, **kwargs) - - if project_locale: - project_locale.adjust_stats(*args, **kwargs) - def count_total_strings(self): entities = Entity.objects.filter(resource=self.resource, obsolete=False) total = entities.count() @@ -200,10 +113,43 @@ def count_total_strings(self): total += (self.locale.nplurals - 1) * plural_count return total + def adjust_stats( + self, before: dict[str, int], after: dict[str, int], tr_created: bool + ): + if tr_created: + self.total_strings = self.count_total_strings() + self.approved_strings = ( + F("approved_strings") + after["approved"] - before["approved"] + ) + self.pretranslated_strings = ( + F("pretranslated_strings") + + after["pretranslated"] + - before["pretranslated"] + ) + self.strings_with_errors = ( + F("strings_with_errors") + after["errors"] - before["errors"] + ) + self.strings_with_warnings = ( + F("strings_with_warnings") + after["warnings"] - before["warnings"] + ) + self.unreviewed_strings = ( + F("unreviewed_strings") + after["unreviewed"] - before["unreviewed"] + ) + self.save( + update_fields=[ + "total_strings", + "approved_strings", + "pretranslated_strings", + "strings_with_errors", + "strings_with_warnings", + "unreviewed_strings", + ] + ) + def calculate_stats(self, save=True): """Update stats, including denormalized ones.""" - total = self.count_total_strings() + self.total_strings = self.count_total_strings() translations = Translation.objects.filter( entity__resource=self.resource, @@ -211,19 +157,19 @@ def calculate_stats(self, save=True): locale=self.locale, ) - approved = translations.filter( + self.approved_strings = translations.filter( approved=True, errors__isnull=True, warnings__isnull=True, ).count() - pretranslated = translations.filter( + self.pretranslated_strings = translations.filter( pretranslated=True, errors__isnull=True, warnings__isnull=True, ).count() - errors = ( + self.strings_with_errors = ( translations.filter( Q( Q(Q(approved=True) | Q(pretranslated=True) | Q(fuzzy=True)) @@ -234,7 +180,7 @@ def calculate_stats(self, save=True): .count() ) - warnings = ( + self.strings_with_warnings = ( translations.filter( Q( Q(Q(approved=True) | Q(pretranslated=True) | Q(fuzzy=True)) @@ -245,36 +191,21 @@ def calculate_stats(self, save=True): .count() ) - unreviewed = translations.filter( + self.unreviewed_strings = translations.filter( approved=False, rejected=False, pretranslated=False, fuzzy=False, ).count() - if not save: - self.total_strings = total - self.approved_strings = approved - self.pretranslated_strings = pretranslated - self.strings_with_errors = errors - self.strings_with_warnings = warnings - self.unreviewed_strings = unreviewed - - return False - - # Calculate diffs to reduce DB queries - total_strings_diff = total - self.total_strings - approved_strings_diff = approved - self.approved_strings - pretranslated_strings_diff = pretranslated - self.pretranslated_strings - strings_with_errors_diff = errors - self.strings_with_errors - strings_with_warnings_diff = warnings - self.strings_with_warnings - unreviewed_strings_diff = unreviewed - self.unreviewed_strings - - self.adjust_all_stats( - total_strings_diff, - approved_strings_diff, - pretranslated_strings_diff, - strings_with_errors_diff, - strings_with_warnings_diff, - unreviewed_strings_diff, - ) + if save: + self.save( + update_fields=[ + "total_strings", + "approved_strings", + "pretranslated_strings", + "strings_with_errors", + "strings_with_warnings", + "unreviewed_strings", + ] + ) diff --git a/pontoon/base/models/translation.py b/pontoon/base/models/translation.py index 50c23464de..5cc4784c66 100644 --- a/pontoon/base/models/translation.py +++ b/pontoon/base/models/translation.py @@ -343,13 +343,7 @@ def save(self, failed_checks=None, *args, **kwargs): # Update stats AFTER changing approval status. stats_after = self.entity.get_stats(self.locale) - stats_diff = { - stat_name: stats_after[stat_name] - stats_before[stat_name] - for stat_name in stats_before - } - if created: - stats_diff["total_strings_diff"] = translatedresource.count_total_strings() - translatedresource.adjust_all_stats(**stats_diff) + translatedresource.adjust_stats(stats_before, stats_after, created) def update_latest_translation(self): """ diff --git a/pontoon/base/signals.py b/pontoon/base/signals.py index 8b61b3f6fe..beb6b862cc 100644 --- a/pontoon/base/signals.py +++ b/pontoon/base/signals.py @@ -3,7 +3,7 @@ from django.contrib.auth.models import Group, Permission, User from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ObjectDoesNotExist -from django.db.models.signals import post_delete, post_save, pre_delete, pre_save +from django.db.models.signals import post_delete, post_save, pre_save from django.dispatch import receiver from pontoon.base import errors @@ -21,42 +21,11 @@ @receiver(post_delete, sender=ProjectLocale) def project_locale_removed(sender, **kwargs): - """ - When locale is removed from a project, delete TranslatedResources - and aggregate project and locale stats. - """ project_locale = kwargs.get("instance", None) if project_locale is not None: - project = project_locale.project - locale = project_locale.locale - TranslatedResource.objects.filter( - resource__project=project, locale=locale + resource__project=project_locale.project, locale=project_locale.locale ).delete() - project.aggregate_stats() - locale.aggregate_stats() - - -@receiver(pre_delete, sender=Locale) -def locale_deleted(sender, **kwargs): - """ - Before locale is deleted, aggregate stats for all locale projects. - """ - locale = kwargs.get("instance", None) - if locale is not None: - for project in locale.project_set.all(): - project.aggregate_stats() - - -@receiver(pre_delete, sender=Project) -def project_deleted(sender, **kwargs): - """ - Before project is deleted, aggregate stats for all project locales. - """ - project = kwargs.get("instance", None) - if project is not None: - for locale in project.locales.all(): - locale.aggregate_stats() def create_group(instance, group_name, perms, name_prefix): diff --git a/pontoon/base/templates/widgets/heading_info.html b/pontoon/base/templates/widgets/heading_info.html index 50900275e9..d41dbe679c 100644 --- a/pontoon/base/templates/widgets/heading_info.html +++ b/pontoon/base/templates/widgets/heading_info.html @@ -79,36 +79,36 @@ {% endmacro %} -{% macro progress_chart_legend(obj, link=None) %} +{% macro progress_chart_legend(stats, link=None) %} @@ -116,15 +116,15 @@
{% if link %}{% endif %}

All strings

-

{{ obj.total_strings|intcomma }}

+

{{ stats.total|intcomma }}

{% if link %}{% endif %}
-
+
{% if link %}{% endif %}

Unreviewed

- {{ obj.unreviewed_strings|intcomma }} + {{ stats.unreviewed|intcomma }}

{% if link %}{% endif %}
diff --git a/pontoon/base/templates/widgets/progress_chart.html b/pontoon/base/templates/widgets/progress_chart.html index 6f8eda0870..8a0dc33ac4 100644 --- a/pontoon/base/templates/widgets/progress_chart.html +++ b/pontoon/base/templates/widgets/progress_chart.html @@ -9,51 +9,51 @@ - +
diff --git a/pontoon/base/tests/__init__.py b/pontoon/base/tests/__init__.py index 9880af4c6f..20893533d0 100644 --- a/pontoon/base/tests/__init__.py +++ b/pontoon/base/tests/__init__.py @@ -90,9 +90,7 @@ def locales(self, create, extracted, **kwargs): if extracted: for locale in extracted: - ProjectLocaleFactory.create( - project=self, locale=locale, total_strings=self.total_strings - ) + ProjectLocaleFactory.create(project=self, locale=locale) @factory.post_generation def repositories(self, create, extracted, **kwargs): diff --git a/pontoon/base/tests/models/conftest.py b/pontoon/base/tests/models/conftest.py index a15aac5232..83f215d45a 100644 --- a/pontoon/base/tests/models/conftest.py +++ b/pontoon/base/tests/models/conftest.py @@ -1,12 +1,6 @@ import pytest -from pontoon.test.factories import ( - EntityFactory, - ProjectLocaleFactory, - RepositoryFactory, - ResourceFactory, - TranslatedResourceFactory, -) +from pontoon.test.factories import RepositoryFactory @pytest.fixture @@ -27,19 +21,3 @@ def repo_hg(project_a): project=project_a, url="repo_hg0", ) - - -@pytest.fixture -def locale_parts(project_b, locale_c, locale_b): - ProjectLocaleFactory.create(project=project_b, locale=locale_c) - ProjectLocaleFactory.create(project=project_b, locale=locale_b) - resourceX = ResourceFactory.create( - project=project_b, - path="resourceX.po", - format="po", - ) - entityX = EntityFactory.create(resource=resourceX, string="entityX") - resourceX.total_strings = 1 - resourceX.save() - TranslatedResourceFactory.create(locale=locale_c, resource=resourceX) - return locale_c, locale_b, entityX diff --git a/pontoon/base/tests/models/test_locale.py b/pontoon/base/tests/models/test_locale.py index 9dcd5695a7..eb40b86dd2 100644 --- a/pontoon/base/tests/models/test_locale.py +++ b/pontoon/base/tests/models/test_locale.py @@ -3,20 +3,6 @@ import pytest from pontoon.base.models import ProjectLocale -from pontoon.test.factories import ( - EntityFactory, - LocaleFactory, - ResourceFactory, - TranslatedResourceFactory, -) - - -@pytest.fixture -def locale_c(): - return LocaleFactory( - code="nv", - name="Na'vi", - ) @pytest.mark.django_db @@ -98,53 +84,3 @@ def test_locale_managers_group(locale_a, locale_b, user_a): assert user_a.has_perm("base.can_manage_locale") is False assert user_a.has_perm("base.can_manage_locale", locale_a) is True assert user_a.has_perm("base.can_manage_locale", locale_b) is True - - -@pytest.mark.django_db -def test_locale_parts_stats_no_page_one_resource(locale_parts): - """ - Return resource paths and stats if one resource defined. - """ - locale_c, locale_b, entityX = locale_parts - project = entityX.resource.project - details = locale_c.parts_stats(project) - assert len(details) == 2 - assert details[0]["title"] == entityX.resource.path - assert details[0]["unreviewed_strings"] == 0 - - -@pytest.mark.django_db -def test_locale_parts_stats_no_page_multiple_resources(locale_parts): - """ - Return resource paths and stats for locales resources are available for. - """ - locale_c, locale_b, entityX = locale_parts - project = entityX.resource.project - resourceY = ResourceFactory.create( - total_strings=1, - project=project, - path="/other/path.po", - ) - EntityFactory.create(resource=resourceY, string="Entity Y") - TranslatedResourceFactory.create( - resource=resourceY, - locale=locale_c, - ) - TranslatedResourceFactory.create( - resource=resourceY, - locale=locale_b, - ) - - # results are sorted by title - - detailsX = locale_c.parts_stats(project) - assert [detail["title"] for detail in detailsX][:2] == sorted( - [entityX.resource.path, "/other/path.po"] - ) - assert detailsX[0]["unreviewed_strings"] == 0 - assert detailsX[1]["unreviewed_strings"] == 0 - - detailsY = locale_b.parts_stats(project) - assert len(detailsY) == 2 - assert detailsY[0]["title"] == "/other/path.po" - assert detailsY[0]["unreviewed_strings"] == 0 diff --git a/pontoon/base/tests/models/test_stats.py b/pontoon/base/tests/models/test_stats.py index fd2fffdc9b..be736fbe5f 100644 --- a/pontoon/base/tests/models/test_stats.py +++ b/pontoon/base/tests/models/test_stats.py @@ -11,7 +11,7 @@ def get_stats(translation): return TranslatedResource.objects.filter( resource=translation.entity.resource, locale=translation.locale, - ).aggregated_stats() + ).string_stats() @pytest.mark.django_db diff --git a/pontoon/base/tests/views/conftest.py b/pontoon/base/tests/views/conftest.py index f6b082e2bc..ea9a5ffcc4 100644 --- a/pontoon/base/tests/views/conftest.py +++ b/pontoon/base/tests/views/conftest.py @@ -1,7 +1,30 @@ import pytest +from pontoon.test.factories import ( + EntityFactory, + ProjectLocaleFactory, + ResourceFactory, + TranslatedResourceFactory, +) + @pytest.fixture def settings_debug(settings): """Make the settings.DEBUG for this test""" settings.DEBUG = True + + +@pytest.fixture +def locale_parts(project_b, locale_c, locale_b): + ProjectLocaleFactory.create(project=project_b, locale=locale_c) + ProjectLocaleFactory.create(project=project_b, locale=locale_b) + resourceX = ResourceFactory.create( + project=project_b, + path="resourceX.po", + format="po", + ) + entityX = EntityFactory.create(resource=resourceX, string="entityX") + resourceX.total_strings = 1 + resourceX.save() + TranslatedResourceFactory.create(locale=locale_c, resource=resourceX) + return locale_c, locale_b, entityX diff --git a/pontoon/base/tests/views/test_ajax.py b/pontoon/base/tests/views/test_ajax.py index 5e07b39c8a..c3101d7fd6 100644 --- a/pontoon/base/tests/views/test_ajax.py +++ b/pontoon/base/tests/views/test_ajax.py @@ -6,7 +6,22 @@ from django.http import Http404 -from pontoon.base.views import AjaxFormPostView, AjaxFormView +from pontoon.base.views import AjaxFormPostView, AjaxFormView, locale_project_parts +from pontoon.test.factories import ( + EntityFactory, + LocaleFactory, + ResourceFactory, + TranslatedResourceFactory, + UserFactory, +) + + +@pytest.fixture +def locale_c(): + return LocaleFactory( + code="nv", + name="Na'vi", + ) def test_view_ajax_form(rf): @@ -97,3 +112,76 @@ def test_view_ajax_form_submit_success(rf): ) assert response.status_code == 200 assert json.loads(response.content) == {"data": 23} + + +@pytest.mark.django_db +def test_locale_parts_stats_no_page_one_resource(rf, locale_parts): + """ + Return resource paths and stats if one resource defined. + """ + locale_c, _, entityX = locale_parts + project = entityX.resource.project + request = rf.get( + f"/{locale_c.code}/{project.slug}/parts", HTTP_X_REQUESTED_WITH="XMLHttpRequest" + ) + request.user = UserFactory() + response = locale_project_parts(request, locale_c.code, project.slug) + assert response.status_code == 200 + assert json.loads(response.content) == [ + { + "title": "resourceX.po", + "approved": 0, + "pretranslated": 0, + "errors": 0, + "warnings": 0, + "total": 0, + "unreviewed": 0, + }, + { + "title": "all-resources", + "approved": 0, + "pretranslated": 0, + "errors": 0, + "warnings": 0, + "total": 0, + "unreviewed": 0, + }, + ] + + +@pytest.mark.django_db +def test_locale_parts_stats_no_page_multiple_resources(rf, locale_parts): + """ + Return resource paths and stats for locales resources are available for. + """ + locale_c, locale_b, entityX = locale_parts + project = entityX.resource.project + resourceY = ResourceFactory.create( + total_strings=1, project=project, path="/other/path.po" + ) + EntityFactory.create(resource=resourceY, string="Entity Y") + TranslatedResourceFactory.create(resource=resourceY, locale=locale_b) + TranslatedResourceFactory.create(resource=resourceY, locale=locale_c) + + request_b = rf.get( + f"/{locale_b.code}/{project.slug}/parts", HTTP_X_REQUESTED_WITH="XMLHttpRequest" + ) + request_b.user = UserFactory() + response = locale_project_parts(request_b, locale_b.code, project.slug) + assert response.status_code == 200 + assert set(data["title"] for data in json.loads(response.content)) == { + "/other/path.po", + "all-resources", + } + + request_c = rf.get( + f"/{locale_c.code}/{project.slug}/parts", HTTP_X_REQUESTED_WITH="XMLHttpRequest" + ) + request_c.user = UserFactory() + response = locale_project_parts(request_c, locale_c.code, project.slug) + assert response.status_code == 200 + assert set(data["title"] for data in json.loads(response.content)) == { + entityX.resource.path, + "/other/path.po", + "all-resources", + } diff --git a/pontoon/base/views.py b/pontoon/base/views.py index 6ee819b380..27d682431f 100755 --- a/pontoon/base/views.py +++ b/pontoon/base/views.py @@ -14,7 +14,7 @@ from django.contrib.auth.models import User from django.core.paginator import EmptyPage, Paginator from django.db import transaction -from django.db.models import Prefetch, Q +from django.db.models import F, Prefetch, Q from django.http import ( Http404, HttpResponse, @@ -103,30 +103,51 @@ def locale_projects(request, locale): def locale_stats(request, locale): """Get locale stats used in All Resources part.""" locale = get_object_or_404(Locale, code=locale) - return JsonResponse(locale.stats(), safe=False) + stats = TranslatedResource.objects.filter(locale=locale).string_stats(request.user) + stats["title"] = "all-resources" + return JsonResponse([stats], safe=False) @utils.require_AJAX def locale_project_parts(request, locale, slug): """Get locale-project pages/paths with stats.""" - try: - locale = Locale.objects.get(code=locale) - except Locale.DoesNotExist as e: - return JsonResponse( - {"status": False, "message": f"Not Found: {e}"}, - status=404, - ) try: + locale = Locale.objects.get(code=locale) project = Project.objects.visible_for(request.user).get(slug=slug) - except Project.DoesNotExist as e: + tr = TranslatedResource.objects.filter( + locale=locale, resource__project=project + ).distinct() + details = list( + tr.annotate( + title=F("resource__path"), + total=F("total_strings"), + pretranslated=F("pretranslated_strings"), + errors=F("strings_with_errors"), + warnings=F("strings_with_warnings"), + unreviewed=F("unreviewed_strings"), + approved=F("approved_strings"), + ) + .order_by("title") + .values( + "title", + "total", + "pretranslated", + "errors", + "warnings", + "unreviewed", + "approved", + ) + ) + all_res_stats = tr.string_stats(request.user, count_system_projects=True) + all_res_stats["title"] = "all-resources" + details.append(all_res_stats) + return JsonResponse(details, safe=False) + except (Locale.DoesNotExist, Project.DoesNotExist) as e: return JsonResponse( {"status": False, "message": f"Not Found: {e}"}, status=404, ) - - try: - return JsonResponse(locale.parts_stats(project), safe=False) except ProjectLocale.DoesNotExist: return JsonResponse( {"status": False, "message": "Locale not enabled for selected project."}, @@ -167,7 +188,7 @@ def _get_entities_list(locale, preferred_source_locale, project, form): return JsonResponse( { "entities": Entity.map_entities(locale, preferred_source_locale, entities), - "stats": TranslatedResource.objects.stats( + "stats": TranslatedResource.objects.query_stats( project, form.cleaned_data["paths"], locale ), }, @@ -200,7 +221,7 @@ def _get_paginated_entities(locale, preferred_source_locale, project, form, enti requested_entity=requested_entity, ), "has_next": entities_page.has_next(), - "stats": TranslatedResource.objects.stats( + "stats": TranslatedResource.objects.query_stats( project, form.cleaned_data["paths"], locale ), }, diff --git a/pontoon/batch/views.py b/pontoon/batch/views.py index a2ea66c68a..e7c72ad0c5 100644 --- a/pontoon/batch/views.py +++ b/pontoon/batch/views.py @@ -122,7 +122,7 @@ def batch_edit_translations(request): ) tr_pks = [tr.pk for tr in action_status["translated_resources"]] - TranslatedResource.objects.filter(pk__in=tr_pks).update_stats() + TranslatedResource.objects.filter(pk__in=tr_pks).calculate_stats() # Mark translations as changed active_translations.bulk_mark_changed() diff --git a/pontoon/insights/models.py b/pontoon/insights/models.py index e846d67d46..e76d118592 100644 --- a/pontoon/insights/models.py +++ b/pontoon/insights/models.py @@ -3,8 +3,6 @@ from django.db import models from django.utils import timezone -from pontoon.base.models import AggregatedStats - def active_users_default(): return { @@ -14,9 +12,17 @@ def active_users_default(): } -class InsightsSnapshot(AggregatedStats, models.Model): +class InsightsSnapshot(models.Model): created_at = models.DateField(default=timezone.now) + # Aggregated stats + total_strings = models.PositiveIntegerField(default=0) + approved_strings = models.PositiveIntegerField(default=0) + pretranslated_strings = models.PositiveIntegerField(default=0) + strings_with_errors = models.PositiveIntegerField(default=0) + strings_with_warnings = models.PositiveIntegerField(default=0) + unreviewed_strings = models.PositiveIntegerField(default=0) + # Active users total_managers = models.PositiveIntegerField(default=0) total_reviewers = models.PositiveIntegerField(default=0) diff --git a/pontoon/insights/tasks.py b/pontoon/insights/tasks.py index 839b98419d..ca3056cd52 100644 --- a/pontoon/insights/tasks.py +++ b/pontoon/insights/tasks.py @@ -12,7 +12,14 @@ from django.utils import timezone from pontoon.actionlog.models import ActionLog -from pontoon.base.models import Entity, Locale, ProjectLocale, Translation +from pontoon.base.aggregated_stats import get_completed_percent +from pontoon.base.models import ( + Entity, + Locale, + ProjectLocale, + TranslatedResource, + Translation, +) from pontoon.base.utils import group_dict_by from pontoon.insights.models import ( LocaleInsightsSnapshot, @@ -209,6 +216,8 @@ def get_locale_insights_snapshot( entities_count, ): """Create LocaleInsightsSnapshot instance for the given locale and day using given data.""" + locale_stats = TranslatedResource.objects.filter(locale=locale).string_stats() + all_managers, all_reviewers = get_privileged_users_data(privileged_users) all_contributors = {c["user"] for c in contributors} @@ -276,13 +285,13 @@ def get_locale_insights_snapshot( return LocaleInsightsSnapshot( locale=locale, created_at=start_of_today, - # AggregatedStats - total_strings=locale.total_strings, - approved_strings=locale.approved_strings, - pretranslated_strings=locale.pretranslated_strings, - strings_with_errors=locale.strings_with_errors, - strings_with_warnings=locale.strings_with_warnings, - unreviewed_strings=locale.unreviewed_strings, + # Aggregated stats + total_strings=locale_stats["total"], + approved_strings=locale_stats["approved"], + pretranslated_strings=locale_stats["pretranslated"], + strings_with_errors=locale_stats["errors"], + strings_with_warnings=locale_stats["warnings"], + unreviewed_strings=locale_stats["unreviewed"], # Active users total_managers=total_managers, total_reviewers=total_reviewers, @@ -298,7 +307,7 @@ def get_locale_insights_snapshot( # Time to review pretranslations time_to_review_pretranslations=time_to_review_pretranslations, # Translation activity - completion=round(locale.completed_percent, 2), + completion=round(get_completed_percent(locale), 2), human_translations=human_translations, machinery_translations=machinery_translations, new_source_strings=entities_count, @@ -322,6 +331,10 @@ def get_project_locale_insights_snapshot( entities_count, ): """Create ProjectLocaleInsightsSnapshot instance for the given locale, project, and day using given data.""" + pl_stats = TranslatedResource.objects.filter( + locale=project_locale.locale, resource__project=project_locale.project + ).string_stats() + ( human_translations, machinery_translations, @@ -334,21 +347,21 @@ def get_project_locale_insights_snapshot( pretranslations_rejected, pretranslations_new, ) = get_activity_charts_data( - activities, locale=project_locale.locale.id, project=project_locale.project.id + activities, locale=project_locale.locale, project=project_locale.project ) return ProjectLocaleInsightsSnapshot( project_locale=project_locale, created_at=start_of_today, - # AggregatedStats - total_strings=project_locale.total_strings, - approved_strings=project_locale.approved_strings, - pretranslated_strings=project_locale.pretranslated_strings, - strings_with_errors=project_locale.strings_with_errors, - strings_with_warnings=project_locale.strings_with_warnings, - unreviewed_strings=project_locale.unreviewed_strings, + # Aggregateds stats + total_strings=pl_stats["total"], + approved_strings=pl_stats["approved"], + pretranslated_strings=pl_stats["pretranslated"], + strings_with_errors=pl_stats["errors"], + strings_with_warnings=pl_stats["warnings"], + unreviewed_strings=pl_stats["unreviewed"], # Translation activity - completion=round(project_locale.completed_percent, 2), + completion=round(get_completed_percent(project_locale), 2), human_translations=human_translations, machinery_translations=machinery_translations, new_source_strings=entities_count, diff --git a/pontoon/localizations/templates/localizations/localization.html b/pontoon/localizations/templates/localizations/localization.html index 4da643ad7d..9a19be4009 100644 --- a/pontoon/localizations/templates/localizations/localization.html +++ b/pontoon/localizations/templates/localizations/localization.html @@ -33,7 +33,7 @@

{{ HeadingInfo.progress_chart() }} - {{ HeadingInfo.progress_chart_legend(project_locale, url('pontoon.translate', locale.code, project.slug, 'all-resources')) }} + {{ HeadingInfo.progress_chart_legend(project_locale_stats, url('pontoon.translate', locale.code, project.slug, 'all-resources')) }}

{% endblock %} diff --git a/pontoon/localizations/tests/test_views.py b/pontoon/localizations/tests/test_views.py index 40e598d6a8..cc5f77fe9f 100644 --- a/pontoon/localizations/tests/test_views.py +++ b/pontoon/localizations/tests/test_views.py @@ -51,12 +51,12 @@ def test_ajax_resources(mock_render, client, project_a, locale_a): assert res.priority is None assert res.latest_activity == translation.latest_activity assert res.chart == { - "pretranslated_strings": 0, - "total_strings": 1, - "approved_strings": 0, - "unreviewed_strings": 0, - "strings_with_errors": 0, - "strings_with_warnings": 0, + "total": 1, + "pretranslated": 0, + "approved": 0, + "unreviewed": 0, + "errors": 0, + "warnings": 0, "approved_share": 0.0, "unreviewed_share": 0.0, "pretranslated_share": 0.0, diff --git a/pontoon/localizations/views.py b/pontoon/localizations/views.py index 303a1008e1..6861e8128b 100644 --- a/pontoon/localizations/views.py +++ b/pontoon/localizations/views.py @@ -6,6 +6,7 @@ from django.shortcuts import get_object_or_404, render from django.views.generic.detail import DetailView +from pontoon.base.aggregated_stats import get_chart_dict from pontoon.base.models import ( Locale, Project, @@ -36,20 +37,10 @@ def localization(request, code, slug): if isinstance(project, HttpResponseRedirect): return project - project_locale = get_object_or_404( - ProjectLocale, - locale=locale, - project=project, - ) + get_object_or_404(ProjectLocale, locale=locale, project=project) - resource_count = ( - TranslatedResource.objects.filter( - resource__project=project, - locale=locale, - resource__entities__obsolete=False, - ) - .distinct() - .count() + trans_res = TranslatedResource.objects.filter( + locale=locale, resource__project=project ) return render( @@ -58,8 +49,10 @@ def localization(request, code, slug): { "locale": locale, "project": project, - "project_locale": project_locale, - "resource_count": resource_count, + "project_locale_stats": trans_res.string_stats(count_system_projects=True), + "resource_count": trans_res.filter(resource__entities__obsolete=False) + .distinct() + .count(), "tags_count": ( project.tag_set.filter(resources__isnull=False).distinct().count() if project.tags_enabled @@ -107,7 +100,7 @@ def ajax_resources(request, code, slug): tr.latest_activity = ( tr.latest_translation.latest_activity if tr.latest_translation else None ) - tr.chart = TranslatedResource.get_chart_dict(tr) + tr.chart = get_chart_dict(tr) return render( request, diff --git a/pontoon/messaging/management/commands/send_deadline_notifications.py b/pontoon/messaging/management/commands/send_deadline_notifications.py index 5e2bc1b3e1..c6c85b9356 100644 --- a/pontoon/messaging/management/commands/send_deadline_notifications.py +++ b/pontoon/messaging/management/commands/send_deadline_notifications.py @@ -6,6 +6,7 @@ from django.core.management.base import BaseCommand from pontoon.base.models import Project +from pontoon.base.models.translated_resource import TranslatedResource class Command(BaseCommand): @@ -36,7 +37,11 @@ def handle(self, *args, **options): locales = [] for project_locale in project.project_locale.all(): - if project_locale.approved_strings < project_locale.total_strings: + pl_stats = TranslatedResource.objects.filter( + locale=project_locale.locale, + resource__project=project_locale.project, + ).string_stats() + if pl_stats["approved"] < pl_stats["total"]: locales.append(project_locale.locale) contributors = ( diff --git a/pontoon/pretranslation/pretranslate.py b/pontoon/pretranslation/pretranslate.py index 8aad40702a..7312a7056e 100644 --- a/pontoon/pretranslation/pretranslate.py +++ b/pontoon/pretranslation/pretranslate.py @@ -111,8 +111,7 @@ def get_pretranslated_data(source, locale, preserve_placeables): def update_changed_instances(tr_filter, tr_dict, translations): """ - Update the latest activity and stats for changed Locales, ProjectLocales - & TranslatedResources + Update the latest activity and stats for changed TranslatedResources """ tr_filter = tuple(tr_filter) # Combine all generated filters with an OK operator. @@ -126,7 +125,7 @@ def update_changed_instances(tr_filter, tr_dict, translations): ) ) - translatedresources.update_stats() + translatedresources.calculate_stats() for tr in translatedresources: index = tr_dict[tr.locale_resource] diff --git a/pontoon/projects/templates/projects/project.html b/pontoon/projects/templates/projects/project.html index c0ed59538b..f7aece704a 100644 --- a/pontoon/projects/templates/projects/project.html +++ b/pontoon/projects/templates/projects/project.html @@ -29,7 +29,7 @@

{{ HeadingInfo.progress_chart() }} - {{ HeadingInfo.progress_chart_legend(chart) }} + {{ HeadingInfo.progress_chart_legend(project_stats) }} {% endblock %} diff --git a/pontoon/projects/templates/projects/projects.html b/pontoon/projects/templates/projects/projects.html index 3aa2c8d3a0..c219794c7f 100644 --- a/pontoon/projects/templates/projects/projects.html +++ b/pontoon/projects/templates/projects/projects.html @@ -49,7 +49,7 @@

{{ HeadingInfo.progress_chart() }} - {{ HeadingInfo.progress_chart_legend(projects.get_stats_sum()) }} + {{ HeadingInfo.progress_chart_legend(project_stats) }} {% endblock %} diff --git a/pontoon/projects/templates/projects/widgets/project_list.html b/pontoon/projects/templates/projects/widgets/project_list.html index cd8f97e1c2..8ab9337c5d 100644 --- a/pontoon/projects/templates/projects/widgets/project_list.html +++ b/pontoon/projects/templates/projects/widgets/project_list.html @@ -39,7 +39,7 @@

{{ LatestActivity.span(latest_activity) }} - {% if project.total_strings %} + {% if chart.total %} {{ ProgressChart.span(chart, chart_link, link_parameter) }} {% else %} Not synced yet @@ -47,7 +47,7 @@

{% if request %} - {% if project.total_strings %} + {% if chart.total %} {{ project.avg_string_count|intcomma }} {% else %} Not synced yet diff --git a/pontoon/projects/views.py b/pontoon/projects/views.py index 58487272ae..522a469e96 100644 --- a/pontoon/projects/views.py +++ b/pontoon/projects/views.py @@ -6,7 +6,8 @@ from django.shortcuts import get_object_or_404, render from django.views.generic.detail import DetailView -from pontoon.base.models import Locale, Project +from pontoon.base.aggregated_stats import get_top_instances +from pontoon.base.models import Locale, Project, TranslatedResource from pontoon.base.utils import get_project_or_redirect, require_AJAX from pontoon.contributors.views import ContributorsMixin from pontoon.insights.utils import get_insights @@ -30,7 +31,13 @@ def projects(request): return render( request, "projects/projects.html", - {"projects": projects, "top_instances": projects.get_top_instances()}, + { + "projects": projects, + "project_stats": TranslatedResource.objects.all().string_stats( + request.user + ), + "top_instances": get_top_instances(projects), + }, ) @@ -43,20 +50,20 @@ def project(request, slug): return project project_locales = project.project_locale - chart = project + project_tr = TranslatedResource.objects.filter(resource__project=project) # Only include filtered teams if provided teams = request.GET.get("teams", "").split(",") filtered_locales = Locale.objects.filter(code__in=teams) if filtered_locales.exists(): project_locales = project_locales.filter(locale__in=filtered_locales) - chart = project_locales.aggregated_stats() + project_tr = project_tr.filter(locale__in=filtered_locales) return render( request, "projects/project.html", { - "chart": chart, + "project_stats": project_tr.string_stats(count_system_projects=True), "count": project_locales.count(), "project": project, "tags_count": ( diff --git a/pontoon/sync/core/stats.py b/pontoon/sync/core/stats.py index f0274a1a22..7bfe6513b4 100644 --- a/pontoon/sync/core/stats.py +++ b/pontoon/sync/core/stats.py @@ -10,7 +10,7 @@ log = logging.getLogger(__name__) -def update_stats(project: Project, *, update_locales: bool = True) -> None: +def update_stats(project: Project) -> None: """Uses raw SQL queries for performance.""" with connection.cursor() as cursor: @@ -97,119 +97,7 @@ def update_stats(project: Project, *, update_locales: bool = True) -> None: ) tr_count = cursor.rowcount - # Project locales, counted from translated resources - cursor.execute( - dedent( - """ - UPDATE base_projectlocale pl - SET - total_strings = agg.total, - approved_strings = agg.approved, - pretranslated_strings = agg.pretranslated, - strings_with_errors = agg.errors, - strings_with_warnings = agg.warnings, - unreviewed_strings = agg.unreviewed - FROM ( - SELECT - tr.locale_id AS "locale_id", - SUM(tr.total_strings) AS "total", - SUM(tr.approved_strings) AS "approved", - SUM(tr.pretranslated_strings) AS "pretranslated", - SUM(tr.strings_with_errors) AS "errors", - SUM(tr.strings_with_warnings) AS "warnings", - SUM(tr.unreviewed_strings) AS "unreviewed" - FROM "base_translatedresource" tr - INNER JOIN "base_resource" res ON (tr.resource_id = res.id) - WHERE res.project_id = %s - GROUP BY tr.locale_id - ) AS agg - WHERE agg.locale_id = pl.locale_id AND pl.project_id = %s - """ - ), - [project.id, project.id], - ) - pl_count = cursor.rowcount - - # Project, counted from project locales - cursor.execute( - dedent( - """ - UPDATE base_project proj - SET - total_strings = GREATEST(agg.total, 0), - approved_strings = GREATEST(agg.approved, 0), - pretranslated_strings = GREATEST(agg.pretranslated, 0), - strings_with_errors = GREATEST(agg.errors, 0), - strings_with_warnings = GREATEST(agg.warnings, 0), - unreviewed_strings = GREATEST(agg.unreviewed, 0) - FROM ( - SELECT - SUM(pl.total_strings) AS "total", - SUM(pl.approved_strings) AS "approved", - SUM(pl.pretranslated_strings) AS "pretranslated", - SUM(pl.strings_with_errors) AS "errors", - SUM(pl.strings_with_warnings) AS "warnings", - SUM(pl.unreviewed_strings) AS "unreviewed" - FROM "base_projectlocale" pl - WHERE pl.project_id = %s - ) AS agg - WHERE proj.id = %s - """ - ), - [project.id, project.id], - ) - - lc_count = _update_locales(cursor) if update_locales else 0 - tr_str = ( "1 translated resource" if tr_count == 1 else f"{tr_count} translated resources" ) - pl_str = "1 projectlocale" if pl_count == 1 else f"{pl_count} projectlocales" - lc_str = "1 locale" if lc_count == 1 else f"{lc_count} locales" - summary = ( - f"{tr_str} and {pl_str}" - if lc_count == 0 - else f"{tr_str}, {pl_str}, and {lc_str}" - ) - log.info(f"[{project.slug}] Updated stats for {summary}") - - -def update_locale_stats() -> None: - with connection.cursor() as cursor: - lc_count = _update_locales(cursor) - lc_str = "1 locale" if lc_count == 1 else f"{lc_count} locales" - log.info(f"Updated stats for {lc_str}") - - -def _update_locales(cursor) -> int: - # All locales, counted from project locales - cursor.execute( - dedent( - """ - UPDATE base_locale loc - SET - total_strings = agg.total, - approved_strings = agg.approved, - pretranslated_strings = agg.pretranslated, - strings_with_errors = agg.errors, - strings_with_warnings = agg.warnings, - unreviewed_strings = agg.unreviewed - FROM ( - SELECT - pl.locale_id AS "locale_id", - SUM(pl.total_strings) AS "total", - SUM(pl.approved_strings) AS "approved", - SUM(pl.pretranslated_strings) AS "pretranslated", - SUM(pl.strings_with_errors) AS "errors", - SUM(pl.strings_with_warnings) AS "warnings", - SUM(pl.unreviewed_strings) AS "unreviewed" - FROM "base_projectlocale" pl - INNER JOIN "base_project" proj ON (pl.project_id = proj.id) - WHERE NOT proj.disabled AND NOT proj.system_project AND proj.visibility = 'public' - GROUP BY pl.locale_id - ) AS agg - WHERE agg.locale_id = loc.id - """ - ) - ) - return cursor.rowcount + log.info(f"[{project.slug}] Updated stats for {tr_str}") diff --git a/pontoon/sync/tests/test_e2e.py b/pontoon/sync/tests/test_e2e.py index 8d25add176..d1a74b9164 100644 --- a/pontoon/sync/tests/test_e2e.py +++ b/pontoon/sync/tests/test_e2e.py @@ -36,12 +36,8 @@ def test_end_to_end(): ): # Database setup settings.MEDIA_ROOT = root - locale_de = LocaleFactory.create( - code="de-Test", name="Test German", total_strings=100 - ) - locale_fr = LocaleFactory.create( - code="fr-Test", name="Test French", total_strings=100 - ) + locale_de = LocaleFactory.create(code="de-Test", name="Test German") + locale_fr = LocaleFactory.create(code="fr-Test", name="Test French") repo_src = RepositoryFactory( url="http://example.com/src-repo", source_repo=True ) @@ -50,7 +46,6 @@ def test_end_to_end(): name="test-project", locales=[locale_de, locale_fr], repositories=[repo_src, repo_tgt], - total_strings=10, ) ResourceFactory.create(project=project, path="a.ftl", format="ftl") ResourceFactory.create(project=project, path="b.po", format="po") diff --git a/pontoon/sync/tests/test_entities.py b/pontoon/sync/tests/test_entities.py index d6c5776772..ce9d031589 100644 --- a/pontoon/sync/tests/test_entities.py +++ b/pontoon/sync/tests/test_entities.py @@ -195,7 +195,7 @@ def test_update_resource(): locale_map = {locale.code: locale} repo = RepositoryFactory(url="http://example.com/repo") project = ProjectFactory.create( - name="test-up", locales=[locale], repositories=[repo] + name="test-up", locales=[locale], repositories=[repo], visibility="public" ) res = {} for n in ("a", "b", "c"): @@ -270,7 +270,10 @@ def test_change_entities(): locale_map = {locale.code: locale} repo = RepositoryFactory(url="http://example.com/repo") project = ProjectFactory.create( - name="test-change", locales=[locale], repositories=[repo] + name="test-change", + locales=[locale], + repositories=[repo], + visibility="public", ) res = ResourceFactory.create( project=project, path="res.ftl", format="ftl", total_strings=3 diff --git a/pontoon/sync/tests/test_translations_from_repo.py b/pontoon/sync/tests/test_translations_from_repo.py index 3d830ace41..92ee268e4a 100644 --- a/pontoon/sync/tests/test_translations_from_repo.py +++ b/pontoon/sync/tests/test_translations_from_repo.py @@ -47,7 +47,7 @@ def test_add_ftl_translation(): name="test-add-ftl", locales=[locale], repositories=[repo], - total_strings=9, + visibility="public", ) res = {} for id in ["a", "b", "c"]: @@ -136,7 +136,7 @@ def test_add_ftl_translation(): update_stats(project) project.refresh_from_db() assert project.total_strings == 9 - assert project.approved_strings == 9 + assert project.approved_strings == 8 tm = TranslationMemoryEntry.objects.filter( entity__resource=res["c"], translation__isnull=False ).values_list("target", flat=True) @@ -157,7 +157,10 @@ def test_remove_po_target_resource(): locale_map = {locale.code: locale} repo = RepositoryFactory(url="http://example.com/repo") project = ProjectFactory.create( - name="test-rm-po", locales=[locale], repositories=[repo] + name="test-rm-po", + locales=[locale], + repositories=[repo], + visibility="public", ) res = {} for id in ["a", "b", "c"]: diff --git a/pontoon/sync/tests/test_translations_to_repo.py b/pontoon/sync/tests/test_translations_to_repo.py index 18bc81703b..0e05ce081d 100644 --- a/pontoon/sync/tests/test_translations_to_repo.py +++ b/pontoon/sync/tests/test_translations_to_repo.py @@ -33,14 +33,13 @@ def test_remove_resource(): with TemporaryDirectory() as root: # Database setup settings.MEDIA_ROOT = root - locale = LocaleFactory.create(code="fr-Test", total_strings=100) + locale = LocaleFactory.create(code="fr-Test") locale_map = {locale.code: locale} repo = RepositoryFactory(url="http://example.com/repo") project = ProjectFactory.create( name="test-rm-res", locales=[locale], repositories=[repo], - total_strings=10, ) res_a = ResourceFactory.create(project=project, path="a.ftl", format="ftl") res_b = ResourceFactory.create(project=project, path="b.po", format="po") @@ -82,14 +81,13 @@ def test_remove_entity(): with TemporaryDirectory() as root: # Database setup settings.MEDIA_ROOT = root - locale = LocaleFactory.create(code="fr-Test", total_strings=100) + locale = LocaleFactory.create(code="fr-Test") locale_map = {locale.code: locale} repo = RepositoryFactory(url="http://example.com/repo") project = ProjectFactory.create( name="test-rm-ent", locales=[locale], repositories=[repo], - total_strings=10, ) res_a = ResourceFactory.create(project=project, path="a.ftl", format="ftl") res_b = ResourceFactory.create(project=project, path="b.po", format="po") @@ -161,14 +159,13 @@ def test_add_translation(): with TemporaryDirectory() as root: # Database setup settings.MEDIA_ROOT = root - locale = LocaleFactory.create(code="fr-Test", total_strings=100) + locale = LocaleFactory.create(code="fr-Test") locale_map = {locale.code: locale} repo = RepositoryFactory(url="http://example.com/repo") project = ProjectFactory.create( name="test-add-trans", locales=[locale], repositories=[repo], - total_strings=10, ) res_a = ResourceFactory.create(project=project, path="a.ftl", format="ftl") res_b = ResourceFactory.create(project=project, path="b.po", format="po") diff --git a/pontoon/tags/templates/tags/tag.html b/pontoon/tags/templates/tags/tag.html index 04364d6c74..f3f9bd8adf 100644 --- a/pontoon/tags/templates/tags/tag.html +++ b/pontoon/tags/templates/tags/tag.html @@ -33,7 +33,7 @@

{{ HeadingInfo.details_item_priority(project.priority, title='Project priority') }} {{ HeadingInfo.details_item_priority(tag.priority, title='Tag priority') }} - {{ HeadingInfo.details_item_deadline(project.deadline, project.approved_strings == project.total_strings) }} + {{ HeadingInfo.details_item_deadline(project.deadline, tag.chart.approved == tag.chart.total) }} {{ HeadingInfo.details_item_url( title='Repository', diff --git a/pontoon/tags/tests/test_utils.py b/pontoon/tags/tests/test_utils.py index 6e5454e457..72a58e82d9 100644 --- a/pontoon/tags/tests/test_utils.py +++ b/pontoon/tags/tests/test_utils.py @@ -6,12 +6,12 @@ @pytest.fixture def chart_0(): return { - "total_strings": 0, - "approved_strings": 0, - "pretranslated_strings": 0, - "strings_with_errors": 0, - "strings_with_warnings": 0, - "unreviewed_strings": 0, + "total": 0, + "approved": 0, + "pretranslated": 0, + "errors": 0, + "warnings": 0, + "unreviewed": 0, "approved_share": 0, "pretranslated_share": 0, "errors_share": 0, @@ -43,8 +43,8 @@ def test_tags_get( assert tag.latest_activity == translation_a.latest_activity chart = chart_0 - chart["total_strings"] = 1 - chart["unreviewed_strings"] = 1 + chart["total"] = 1 + chart["unreviewed"] = 1 chart["unreviewed_share"] = 100.0 assert tag.chart == chart @@ -68,7 +68,7 @@ def test_tags_get_tag_locales( tag.latest_activity chart = chart_0 - chart["total_strings"] = 1 + chart["total"] = 1 assert tag.chart == chart locale = tag.locales.first() diff --git a/pontoon/tags/utils.py b/pontoon/tags/utils.py index 5ac58ac0f7..5eb793f7ed 100644 --- a/pontoon/tags/utils.py +++ b/pontoon/tags/utils.py @@ -1,5 +1,6 @@ from django.db.models import Max, Q, Sum +from pontoon.base.aggregated_stats import get_chart_dict from pontoon.base.models import TranslatedResource, Translation from pontoon.tags.models import Tag @@ -57,18 +58,27 @@ def chart(self, query, group_by): self.translated_resources.filter(query) .values(group_by) .annotate( - total_strings=Sum("resource__total_strings"), - approved_strings=Sum("approved_strings"), - pretranslated_strings=Sum("pretranslated_strings"), - strings_with_errors=Sum("strings_with_errors"), - strings_with_warnings=Sum("strings_with_warnings"), - unreviewed_strings=Sum("unreviewed_strings"), + # should be Sum("total_strings"), but tests fail with it + total=Sum("resource__total_strings"), + approved=Sum("approved_strings"), + pretranslated=Sum("pretranslated_strings"), + errors=Sum("strings_with_errors"), + warnings=Sum("strings_with_warnings"), + unreviewed=Sum("unreviewed_strings"), ) ) + print(list(k for k in trs[0].keys())) return { - tr[group_by]: TranslatedResource.get_chart_dict( - TranslatedResource(**{key: tr[key] for key in list(tr.keys())[1:]}) + tr[group_by]: get_chart_dict( + TranslatedResource( + total_strings=tr["total"], + approved_strings=tr["approved"], + pretranslated_strings=tr["pretranslated"], + strings_with_errors=tr["errors"], + strings_with_warnings=tr["warnings"], + unreviewed_strings=tr["unreviewed"], + ) ) for tr in trs } diff --git a/pontoon/teams/templates/teams/team.html b/pontoon/teams/templates/teams/team.html index bd9fa60d1c..e68b19293b 100644 --- a/pontoon/teams/templates/teams/team.html +++ b/pontoon/teams/templates/teams/team.html @@ -63,7 +63,7 @@

{{ HeadingInfo.progress_chart() }} - {{ HeadingInfo.progress_chart_legend(locale, url('pontoon.translate', locale.code, 'all-projects', 'all-resources')) }} + {{ HeadingInfo.progress_chart_legend(locale_stats, url('pontoon.translate', locale.code, 'all-projects', 'all-resources')) }} diff --git a/pontoon/teams/templates/teams/teams.html b/pontoon/teams/templates/teams/teams.html index b3ef2b4fb0..0334bf0003 100644 --- a/pontoon/teams/templates/teams/teams.html +++ b/pontoon/teams/templates/teams/teams.html @@ -49,7 +49,7 @@

{{ HeadingInfo.progress_chart() }} - {{ HeadingInfo.progress_chart_legend(locales.get_stats_sum()) }} + {{ HeadingInfo.progress_chart_legend(locale_stats) }} {% endblock %} diff --git a/pontoon/teams/templates/teams/widgets/team_list.html b/pontoon/teams/templates/teams/widgets/team_list.html index 16f6130f3c..155ad69b65 100644 --- a/pontoon/teams/templates/teams/widgets/team_list.html +++ b/pontoon/teams/templates/teams/widgets/team_list.html @@ -38,7 +38,7 @@

{{ LatestActivity.span(latest_activity) }} - {% if chart.total_strings %} + {% if chart.total %} {{ ProgressChart.span(chart, chart_link, link_parameter, has_param) }} {% else %} {{ not_ready_message }} diff --git a/pontoon/teams/views.py b/pontoon/teams/views.py index e4bac2c2b3..e9f38ee5dd 100644 --- a/pontoon/teams/views.py +++ b/pontoon/teams/views.py @@ -33,7 +33,14 @@ from pontoon.actionlog.models import ActionLog from pontoon.actionlog.utils import log_action from pontoon.base import forms -from pontoon.base.models import Locale, Project, TranslationMemoryEntry, User +from pontoon.base.aggregated_stats import get_top_instances +from pontoon.base.models import ( + Locale, + Project, + TranslatedResource, + TranslationMemoryEntry, + User, +) from pontoon.base.utils import get_locale_or_redirect, require_AJAX from pontoon.contributors.views import ContributorsMixin from pontoon.insights.utils import get_locale_insights @@ -59,8 +66,9 @@ def teams(request): "teams/teams.html", { "locales": locales, + "locale_stats": TranslatedResource.objects.all().string_stats(request.user), "form": form, - "top_instances": locales.get_top_instances(), + "top_instances": get_top_instances(locales), }, ) @@ -76,8 +84,14 @@ def team(request, locale): if not available_count: raise Http404 + locale_stats = TranslatedResource.objects.filter(locale=locale).string_stats( + request.user + ) + return render( - request, "teams/team.html", {"count": visible_count, "locale": locale} + request, + "teams/team.html", + {"count": visible_count, "locale": locale, "locale_stats": locale_stats}, ) diff --git a/pontoon/terminology/models.py b/pontoon/terminology/models.py index 54907d467b..80c1371084 100644 --- a/pontoon/terminology/models.py +++ b/pontoon/terminology/models.py @@ -2,40 +2,16 @@ from django.db import models -from pontoon.base.models import Entity, ProjectLocale, Resource, TranslatedResource +from pontoon.base.models import Entity, Resource, TranslatedResource def update_terminology_project_stats(): resource = Resource.objects.get(project__slug="terminology") - project = resource.project - total_strings = Entity.objects.filter(resource=resource, obsolete=False).count() - resource.total_strings = total_strings + resource.total_strings = Entity.objects.filter( + resource=resource, obsolete=False + ).count() resource.save(update_fields=["total_strings"]) - - translated_resources = list(TranslatedResource.objects.filter(resource=resource)) - - for translated_resource in translated_resources: - translated_resource.calculate_stats(save=False) - - TranslatedResource.objects.bulk_update( - translated_resources, - [ - "total_strings", - "approved_strings", - "pretranslated_strings", - "strings_with_errors", - "strings_with_warnings", - "unreviewed_strings", - ], - ) - - project.aggregate_stats() - - for locale in project.locales.all(): - locale.aggregate_stats() - - for projectlocale in ProjectLocale.objects.filter(project=project): - projectlocale.aggregate_stats() + TranslatedResource.objects.filter(resource=resource).calculate_stats() class TermQuerySet(models.QuerySet): diff --git a/pontoon/tour/migrations/0001_squashed_0001_initial.py b/pontoon/tour/migrations/0001_squashed_0001_initial.py index 4259eb0044..857c44b20b 100644 --- a/pontoon/tour/migrations/0001_squashed_0001_initial.py +++ b/pontoon/tour/migrations/0001_squashed_0001_initial.py @@ -62,12 +62,7 @@ def create_tutorial_project(apps, schema_editor): ProjectLocale = apps.get_model("base", "ProjectLocale") locales = Locale.objects.exclude(code__in=["en-US", "en"]) project_locales = [ - ProjectLocale( - project=project, - locale=locale, - total_strings=len(new_strings), - ) - for locale in locales + ProjectLocale(project=project, locale=locale) for locale in locales ] ProjectLocale.objects.bulk_create(project_locales) @@ -83,9 +78,6 @@ def create_tutorial_project(apps, schema_editor): ] TranslatedResource.objects.bulk_create(translated_resources) - project.total_strings = len(new_strings) * len(locales) - project.save(update_fields=["total_strings"]) - def remove_tutorial_project(apps, schema_editor): Project = apps.get_model("base", "Project") diff --git a/pontoon/translations/tests/test_views.py b/pontoon/translations/tests/test_views.py index f3493c86c2..64b2d6347c 100644 --- a/pontoon/translations/tests/test_views.py +++ b/pontoon/translations/tests/test_views.py @@ -81,8 +81,6 @@ def test_create_translation_force_suggestions( translation_a.active = True translation_a.save() translation_a.locale.translators_group.user_set.add(member.user) - project_locale_a.unreviewed_strings = 1 - project_locale_a.save() response = request_create_translation( member.client, diff --git a/pontoon/translations/views.py b/pontoon/translations/views.py index 5d7dd9d27f..efd928567f 100644 --- a/pontoon/translations/views.py +++ b/pontoon/translations/views.py @@ -169,7 +169,7 @@ def create_translation(request): response_data = { "status": True, "translation": translation.serialize(), - "stats": TranslatedResource.objects.stats(project, paths, locale), + "stats": TranslatedResource.objects.query_stats(project, paths, locale), } # Send Translation Champion Badge notification information @@ -317,7 +317,7 @@ def approve_translation(request): response_data = { "translation": active_translation.serialize(), - "stats": TranslatedResource.objects.stats(project, paths, locale), + "stats": TranslatedResource.objects.query_stats(project, paths, locale), } # Send Review Master Badge notification information @@ -388,7 +388,7 @@ def unapprove_translation(request): return JsonResponse( { "translation": active_translation.serialize(), - "stats": TranslatedResource.objects.stats(project, paths, locale), + "stats": TranslatedResource.objects.query_stats(project, paths, locale), } ) @@ -454,7 +454,7 @@ def reject_translation(request): response_data = { "translation": active_translation.serialize(), - "stats": TranslatedResource.objects.stats(project, paths, locale), + "stats": TranslatedResource.objects.query_stats(project, paths, locale), } # Send Review Master Badge notification information @@ -525,6 +525,6 @@ def unreject_translation(request): return JsonResponse( { "translation": active_translation.serialize(), - "stats": TranslatedResource.objects.stats(project, paths, locale), + "stats": TranslatedResource.objects.query_stats(project, paths, locale), } ) diff --git a/translate/src/api/resource.ts b/translate/src/api/resource.ts index 48d3f84ebc..1660d1d11d 100644 --- a/translate/src/api/resource.ts +++ b/translate/src/api/resource.ts @@ -2,10 +2,10 @@ import { GET } from './utils/base'; type APIResource = { readonly title: string; - readonly approved_strings: number; - readonly pretranslated_strings: number; - readonly strings_with_warnings: number; - readonly resource__total_strings: number; + readonly approved: number; + readonly pretranslated: number; + readonly warnings: number; + readonly total: number; }; export const fetchAllResources = ( diff --git a/translate/src/modules/resource/actions.ts b/translate/src/modules/resource/actions.ts index ad9ece74f0..d3e0916f7f 100644 --- a/translate/src/modules/resource/actions.ts +++ b/translate/src/modules/resource/actions.ts @@ -34,10 +34,10 @@ export const getResource = const resources = results.map((resource) => ({ path: resource.title, - approvedStrings: resource.approved_strings, - pretranslatedStrings: resource.pretranslated_strings, - stringsWithWarnings: resource.strings_with_warnings, - totalStrings: resource.resource__total_strings, + approvedStrings: resource.approved, + pretranslatedStrings: resource.pretranslated, + stringsWithWarnings: resource.warnings, + totalStrings: resource.total, })); const allResources = resources.pop();