Merge pull request #70 from rss-translator/next

feat: add tags

.dockerignore

@@ -9,13 +9,15 @@ __pycache__/
 .pdm-python
 .eggs/
 .git/
+.github/
 !.git/HEAD
 !.git/refs/heads/*
 
 venv/
 .venv/
 .venv-old/
 .docker-venv/
+.vscode/
 node_modules/
 
 docs/
@@ -30,4 +32,5 @@ docker/
 
 data/
 output/
-deploy/
+deploy/
+website/

.vscode/launch.json

@@ -0,0 +1,18 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python Debug: Django",
+            "type": "debugpy",
+            "python": "${workspaceFolder}/venv/bin/python",
+            "env": {"GEVENT_SUPPORT": "True"},
+            "request": "launch",
+            "args": [
+                "run_dev"
+            ],
+            "django": true,
+            "autoStartBrowser": false,
+            "program": "${workspaceFolder}/manage.py"
+        }
+    ]
+}

config/settings.py

@@ -58,13 +58,21 @@
     'translator.apps.TranslatorConfig',
     'core.apps.CoreConfig',
     'encrypted_model_fields', # must set FIELD_ENCRYPTION_KEY value
+    'tagulous',
 ]
 DEBUG_PLUGINS = [
     "debug_toolbar",
     'bx_django_utils',  # https://github.com/boxine/bx_django_utils
     #'huey_monitor',
 ]
 
+SERIALIZATION_MODULES = {
+    'xml':    'tagulous.serializers.xml_serializer',
+    'json':   'tagulous.serializers.json',
+    'python': 'tagulous.serializers.python',
+    'yaml':   'tagulous.serializers.pyyaml',
+}
+
 MIDDLEWARE = [
     "django.middleware.security.SecurityMiddleware",
     "whitenoise.middleware.WhiteNoiseMiddleware",

core/admin.py

@@ -7,12 +7,12 @@
 from django.urls import path
 from django.shortcuts import render,redirect
 from django.core.paginator import Paginator
-
 from django.contrib.auth.models import User, Group
 from django.urls import reverse
 from django.utils.html import format_html
 from django.utils.translation import gettext_lazy  as _
 from .models import O_Feed, T_Feed
+#from taggit.models import Tag
 from .tasks import update_original_feed, update_translated_feed
 from utils.modelAdmin_utils import CustomModelActions, get_translator_and_summary_choices, get_all_app_models, valid_icon
 
@@ -189,7 +189,7 @@ def __init__(self, *args, **kwargs):
 
     class Meta:
         model = O_Feed
-        fields = ['feed_url', 'update_frequency', 'max_posts', 'translator', 'translation_display', 'summary_engine', 'summary_detail', 'additional_prompt', 'name', 'fetch_article', 'quality']
+        fields = ['feed_url', 'update_frequency', 'max_posts', 'translator', 'translation_display', 'summary_engine', 'summary_detail', 'additional_prompt', 'fetch_article', 'quality', 'name', 'tags', ]
 
     # 重写save方法，以处理自定义字段的数据
     def save(self, commit=True):
@@ -216,9 +216,9 @@ class O_FeedAdmin(admin.ModelAdmin, CustomModelActions):
     form = O_FeedForm
     inlines = [T_FeedInline]
     list_display = ["name", "is_valid", "show_feed_url", "translated_language", "translator", "size_in_kb",
-                    "update_frequency", "last_updated", "last_pull"]
-    search_fields = ["name", "feed_url"]
-    list_filter = ["valid"]
+                    "update_frequency", "last_updated", "last_pull", "tag_list"]
+    search_fields = ["name", "feed_url", "tags__name"]
+    list_filter = ["valid","tags"]
     actions = ['o_feed_force_update', 'o_feed_export_as_opml', 'o_feed_batch_modify']
 
 
@@ -264,6 +264,12 @@ def translated_language(self, obj):
         return ", ".join(t_feed.language for t_feed in obj.t_feed_set.all())
     translated_language.short_description = _('Translated Language')
 
+    def get_queryset(self, request):
+        return super().get_queryset(request).prefetch_related('tags')
+
+    def tag_list(self, obj):
+        return ", ".join(o.name for o in obj.tags.all())
+
     def size_in_kb(self, obj):
         return int(obj.size / 1024)
 
@@ -310,7 +316,8 @@ def o_feed_batch_modify(self, request, queryset):
                 'summary_detail': 'summary_detail_value', 
                 'additional_prompt': 'additional_prompt_value',
                 'fetch_article': 'fetch_article',
-                'quality': 'quality'
+                'quality': 'quality',
+                'tags': 'tags_value'
             }
             field_types = {
                 'update_frequency': int,
@@ -322,6 +329,7 @@ def o_feed_batch_modify(self, request, queryset):
                 'quality': literal_eval        
             }
             update_fields = {}
+            tags_value = None
             for field, value_field in fields.items():
                 value = post_data.get(value_field)
                 if post_data.get(field, 'Keep') != 'Keep' and value:
@@ -334,12 +342,20 @@ def o_feed_batch_modify(self, request, queryset):
                             content_type_summary_id, object_id_summary = map(int, value.split(':'))
                             update_fields['content_type_summary_id'] = content_type_summary_id
                             update_fields['object_id_summary'] = object_id_summary
+                        case 'tags':
+                            tags_value = value.split(",")
                         case _:
                             update_fields[field] = field_types.get(field, str)(value)
 
             if update_fields:
                 queryset.update(**update_fields)
 
+            if tags_value is not None:
+                for obj in queryset:
+                    obj.tags = [*tags_value]
+                    obj.save()
+                #O_Feed.objects.bulk_update(queryset, ['tags'])??
+
             #self.message_user(request, f"Successfully modified {queryset.count()} items.")
             #return HttpResponseRedirect(request.get_full_path())
             return redirect(request.get_full_path())
@@ -348,19 +364,21 @@ def o_feed_batch_modify(self, request, queryset):
         logging.info("translator_choices: %s, summary_engine_choices: %s", translator_choices, summary_engine_choices)
         return render(request, 'admin/o_feed_batch_modify.html', context={**core_admin_site.each_context(request), 'items': queryset,'translator_choices': translator_choices, 'summary_engine_choices': summary_engine_choices})
     o_feed_batch_modify.short_description = _("Batch modification")
-
+        
 
 class T_FeedAdmin(admin.ModelAdmin, CustomModelActions):
     list_display = ["id", "feed_url", "o_feed", "status_icon", "language", "translate_title", "translate_content", "summary", "total_tokens", "total_characters", "size_in_kb", "modified"]
-    list_filter = ["status", "translate_title", "translate_content"]
-    search_fields = ["sid"]
+    list_filter = ["status", "translate_title", "translate_content", "o_feed__tags__name"]
+    search_fields = ["sid", "o_feed__tags__name", "o_feed__feed_url"]
     readonly_fields = ["status", "language", "sid", "o_feed", "total_tokens", "total_characters", "size", "modified"]
     actions = ['t_feed_force_update', 't_feed_export_as_opml', 't_feed_batch_modify']
-    def get_search_results(self, request, queryset, search_term):
-        queryset, use_distinct = super().get_search_results(request, queryset, search_term)
-        queryset |= self.model.objects.filter(o_feed__feed_url__icontains=search_term)
-        return queryset, use_distinct
-
+    # def get_search_results(self, request, queryset, search_term):
+    #     queryset, use_distinct = super().get_search_results(request, queryset, search_term)
+    #     queryset |= self.model.objects.filter(o_feed__feed_url__icontains=search_term)
+    #     return queryset, use_distinct
+
+    def get_queryset(self, request):
+        return super().get_queryset(request).prefetch_related('o_feed__tags')
 
     def size_in_kb(self, obj):
         return int(obj.size / 1024)

core/migrations/0014_tagulous_o_feed_tags_o_feed_tags.py

@@ -0,0 +1,62 @@
+# Generated by Django 5.0.6 on 2024-05-16 08:54
+
+import tagulous.models.fields
+import tagulous.models.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0013_alter_o_feed_sid_alter_t_feed_sid"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Tagulous_O_Feed_tags",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("name", models.CharField(max_length=255, unique=True)),
+                ("slug", models.SlugField()),
+                (
+                    "count",
+                    models.IntegerField(
+                        default=0,
+                        help_text="Internal counter of how many times this tag is in use",
+                    ),
+                ),
+                (
+                    "protected",
+                    models.BooleanField(
+                        default=False,
+                        help_text="Will not be deleted when the count reaches 0",
+                    ),
+                ),
+            ],
+            options={
+                "ordering": ("name",),
+                "abstract": False,
+                "unique_together": {("slug",)},
+            },
+            bases=(tagulous.models.models.BaseTagModel, models.Model),
+        ),
+        migrations.AddField(
+            model_name="o_feed",
+            name="tags",
+            field=tagulous.models.fields.TagField(
+                _set_tag_meta=True,
+                blank=True,
+                force_lowercase=True,
+                help_text="Enter a comma-separated tag string",
+                to="core.tagulous_o_feed_tags",
+            ),
+        ),
+    ]

core/models.py

@@ -8,6 +8,7 @@
 from django.db import models
 from django.utils.translation import gettext_lazy as _
 from django.core.validators import MinValueValidator, MaxValueValidator
+from tagulous.models import TagField
 
 
 class O_Feed(models.Model):
@@ -48,6 +49,7 @@ class O_Feed(models.Model):
                                        help_text=_("Level of detail of summaries of longer articles. 0: Normal, 1: Most detailed (cost more tokens)"))
 
     additional_prompt = models.TextField(_("Addtional Prompt"), default=None, blank=True, null=True, help_text=_("Addtional Prompt for translation and summary"))
+    tags = TagField(force_lowercase=True, blank=True, help_text=_("Enter a comma-separated tag string"))
 
     def __str__(self):
         return self.feed_url

core/signals.py

@@ -6,6 +6,7 @@
 from django.dispatch import receiver
 
 from .models import O_Feed, T_Feed
+#from taggit.models import TaggedItem
 
 @receiver(post_delete, sender=O_Feed)
 def delete_o_feed_xml(sender, instance, **kwargs):
@@ -20,3 +21,10 @@ def delete_t_feed_xml(sender, instance, **kwargs):
     feed_file_path = f"{settings.DATA_FOLDER}/feeds/{instance.sid}.xml"
     if os.path.exists(feed_file_path):
         os.remove(feed_file_path)
+
+#For django-taggit
+# @receiver(post_delete, sender=TaggedItem)
+# def delete_unused_tags(sender, instance, **kwargs):
+#     n_tagged = TaggedItem.objects.filter(tag_id=instance.tag_id).count()
+#     if n_tagged == 0:
+#         instance.tag.delete()

core/tasks.py

@@ -233,7 +233,7 @@ def translate_feed(
             title = entry["title"]
 
             # Translate title
-            if translate_title:
+            if translate_engine and translate_title:
                 cached = Translated_Content.is_translated(title, target_language)  # check cache db
                 translated_text = ''
                 if not cached:
@@ -274,7 +274,7 @@ def translate_feed(
                         logging.warning("Fetch original article error:%s", e)
 
             # Translate content
-            if translate_content:
+            if translate_engine and translate_content:
                 if translate_engine == None:
                     logging.warning("No translate engine")
                     continue
@@ -305,7 +305,7 @@ def translate_feed(
                     bulk_save_cache(need_cache_objs)
                     need_cache_objs = {}
 
-            if summary:
+            if summary_engine and summary:
                 if summary_engine == None:
                     logging.warning("No Summarize engine")
                     continue