Add machine translation (#304) (#320)

Co-authored-by: Armand Fardeau <[email protected]>

.env-example

@@ -42,4 +42,8 @@ FRIENDLY_SIGNUP_OVERRIDE_PASSWORDS=1
 FRIENDLY_SIGNUP_INSTANT_VALIDATION=1
 FRIENDLY_SIGNUP_HIDE_NICKNAME=1
 FRIENDLY_SIGNUP_USE_CONFIRMATION_CODES=1
-ENABLE_LETTER_OPENER=0
+ENABLE_LETTER_OPENER=0
+TRANSLATOR_ENABLED=0
+TRANSLATOR_API_KEY=
+TRANSLATOR_HOST=
+TRANSLATOR_DELAY=0

Gemfile

@@ -33,6 +33,7 @@ gem "omniauth-publik", git: "https://github.com/OpenSourcePolitics/omniauth-publ
 gem "activejob-uniqueness", require: "active_job/uniqueness/sidekiq_patch"
 gem "aws-sdk-s3", require: false
 gem "bootsnap", "~> 1.4"
+gem "deepl-rb", require: "deepl"
 gem "faker", "~> 2.14"
 gem "fog-aws"
 gem "foundation_rails_helper", git: "https://github.com/sgruhier/foundation_rails_helper.git"

Gemfile.lock

@@ -424,6 +424,7 @@ GEM
     declarative-builder (0.1.0)
       declarative-option (< 0.2.0)
     declarative-option (0.1.0)
+    deepl-rb (2.5.3)
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
     devise (4.9.2)
@@ -1001,6 +1002,7 @@ DEPENDENCIES
   decidim-spam_detection
   decidim-templates (~> 0.26.0)
   decidim-term_customizer!
+  deepl-rb
   dotenv-rails
   faker (~> 2.14)
   fog-aws

app/jobs/machine_translation_resource_job.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Decidim
+  # This job is part of the machine translation flow. This one is fired every
+  # time a `Decidim::TranslatableResource` is created or updated. If any of the
+  # attributes defines as translatable is modified, then for each of those
+  # attributes this job will schedule a `Decidim::MachineTranslationFieldsJob`.
+  class MachineTranslationResourceJob < ApplicationJob
+    queue_as :translations
+
+    # rubocop: disable Metrics/CyclomaticComplexity
+
+    # Performs the job.
+    #
+    # resource - Any kind of `Decidim::TranslatableResource` model instance
+    # previous_changes - A Hash with the set fo changes. This is intended to be
+    #   taken from `resource.previous_changes`, but we need to manually pass
+    #   them to the job because the value gets lost when serializing the
+    #   resource.
+    # source_locale - A Symbol representing the source locale for the translation
+    def perform(resource, previous_changes, source_locale)
+      return unless Decidim.machine_translation_service_klass
+
+      @resource = resource
+      @locales_to_be_translated = []
+      translatable_fields = @resource.class.translatable_fields_list.map(&:to_s)
+      translatable_fields.each do |field|
+        next unless @resource[field].is_a?(Hash) && previous_changes.keys.include?(field)
+
+        translated_locales = translated_locales_list(field)
+        remove_duplicate_translations(field, translated_locales) if @resource[field]["machine_translations"].present?
+
+        next unless default_locale_changed_or_translation_removed(previous_changes, field)
+
+        @locales_to_be_translated += pending_locales(translated_locales) if @locales_to_be_translated.blank?
+
+        @locales_to_be_translated.each do |target_locale|
+          Decidim::MachineTranslationFieldsJob.perform_later(
+            @resource,
+            field,
+            resource_field_value(
+              previous_changes,
+              field,
+              source_locale
+            ),
+            target_locale,
+            source_locale
+          )
+        end
+      end
+    end
+    # rubocop: enable Metrics/CyclomaticComplexity
+
+    def default_locale_changed_or_translation_removed(previous_changes, field)
+      default_locale = default_locale(@resource)
+      values = previous_changes[field]
+      old_value = values.first
+      new_value = values.last
+      return true unless old_value.is_a?(Hash)
+
+      return true if old_value[default_locale] != new_value[default_locale]
+
+      # In a case where the default locale is not changed
+      # but a translation of a different locale is deleted
+      # We trigger a job to translate only for that locale
+      if old_value[default_locale] == new_value[default_locale]
+        locales_present = old_value.keys
+        locales_present.each do |locale|
+          @locales_to_be_translated << locale if old_value[locale] != new_value[locale] && new_value[locale] == ""
+        end
+      end
+
+      @locales_to_be_translated.present?
+    end
+
+    def resource_field_value(previous_changes, field, source_locale)
+      values = previous_changes[field]
+      new_value = values.last
+      if new_value.is_a?(Hash)
+        locale = source_locale || default_locale(@resource)
+        return new_value[locale]
+      end
+
+      new_value
+    end
+
+    def default_locale(resource)
+      if resource.respond_to? :organization
+        resource.organization.default_locale.to_s
+      else
+        Decidim.available_locales.first.to_s
+      end
+    end
+
+    def translated_locales_list(field)
+      return nil unless @resource[field].is_a? Hash
+
+      translated_locales = []
+      existing_locales = @resource[field].keys - ["machine_translations"]
+      existing_locales.each do |locale|
+        translated_locales << locale if @resource[field][locale].present?
+      end
+
+      translated_locales
+    end
+
+    def remove_duplicate_translations(field, translated_locales)
+      machine_translated_locale = @resource[field]["machine_translations"].keys
+      unless (translated_locales & machine_translated_locale).nil?
+        (translated_locales & machine_translated_locale).each { |key| @resource[field]["machine_translations"].delete key }
+      end
+    end
+
+    def pending_locales(translated_locales)
+      available_locales = @resource.organization.available_locales.map(&:to_s) if @resource.respond_to? :organization
+      available_locales ||= Decidim.available_locales.map(&:to_s)
+      available_locales - translated_locales
+    end
+  end
+end

app/services/deepl_translator.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class DeeplTranslator
+  attr_reader :text, :source_locale, :target_locale, :resource, :field_name
+
+  def initialize(resource, field_name, text, target_locale, source_locale)
+    @resource = resource
+    @field_name = field_name
+    @text = text
+    @target_locale = target_locale
+    @source_locale = source_locale
+  end
+
+  def translate
+    translation = DeepL.translate text, source_locale, target_locale
+
+    Decidim::MachineTranslationSaveJob.perform_later(
+      resource,
+      field_name,
+      target_locale,
+      translation.text
+    )
+  end
+end

config/environments/production.rb

@@ -72,7 +72,7 @@
 
   # Use a real queuing backend for Active Job (and separate queues per environment)
   config.active_job.queue_adapter = :sidekiq
-  # see confguration for sidekiq in `config/sidekiq.yml`
+  # see configuration for sidekiq in `config/sidekiq.yml`
   # config.active_job.queue_name_prefix = "development_app_#{Rails.env}"
 
   config.action_mailer.perform_caching = false

config/i18n-tasks.yml

@@ -123,4 +123,5 @@ ignore_unused:
   - rack_attack.too_many_requests.*
   - decidim.account.destroy.success
   - decidim.account.destroy.error
+  - decidim.proposals.collaborative_drafts.new.*
 

config/initializers/decidim.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "decidim/dev/dummy_translator"
+
 Decidim.configure do |config|
   config.application_name = "OSP Agora"
   config.mailer_sender = "OSP Agora <[email protected]>"
@@ -103,6 +105,13 @@
   end
 
   config.base_uploads_path = "#{ENV["HEROKU_APP_NAME"]}/" if ENV["HEROKU_APP_NAME"].present?
+
+  # Machine Translation Configuration
+  #
+  # Enable machine translations
+  config.enable_machine_translations = Rails.application.secrets.translator[:enabled]
+  config.machine_translation_service = "DeeplTranslator"
+  config.machine_translation_delay = Rails.application.secrets.translator[:delay]
 end
 
 Decidim.module_eval do

config/initializers/deepl.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+# DeepL Translation service configuration
+DeepL.configure do |config|
+  config.auth_key = Rails.application.secrets.translator[:api_key]
+  config.host = Rails.application.secrets.translator[:host]
+end

config/locales/en.yml

@@ -145,6 +145,9 @@ en:
         edit:
           attachment_legend: "(Optional) Add an attachment"
           select_a_category: Please select a category
+        new:
+          add_file: Add file
+          edit_file: Edit file
         show:
           back: Back
           edit: Edit collaborative draftss

config/locales/fr.yml

@@ -145,6 +145,9 @@ fr:
         edit:
           attachment_legend: "(Facultatif) Ajouter une pièce jointe"
           select_a_category: Veuillez sélectionner une catégorie
+        new:
+          add_file: Ajouter le fichier
+          edit_file: Editer le fichier
         show:
           back: Retour
           edit: Modifier un brouillon collaboratif

config/secrets.yml

@@ -64,6 +64,11 @@ default: &default
     server: <%= ENV["ETHERPAD_SERVER"] %>
     api_key: <%= ENV["ETHERPAD_API_KEY"] %>
     api_version: "1.2.1"
+  translator:
+    enabled: <%= ENV.fetch("TRANSLATOR_ENABLED", "0") == "1" %>
+    delay: <%= ENV.fetch("TRANSLATOR_DELAY", "0").to_i.seconds %>
+    api_key: <%= ENV.fetch("TRANSLATOR_API_KEY", "dummy_key") %>
+    host: <%= ENV.fetch("TRANSLATOR_HOST", "https://translator.example.org") %>
 
 development:
   <<: *default

db/seeds.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "decidim/translator_configuration_helper"
 # This file should contain all the record creation needed to seed the database with its default values.
 # The data can then be loaded with the rails db:seed command (or created alongside the database with db:setup).
 #
@@ -8,6 +9,9 @@
 #   movies = Movie.create([{ name: 'Star Wars' }, { name: 'Lord of the Rings' }])
 #   Character.create(name: 'Luke', movie: movies.first)
 # You can remove the 'faker' gem if you don't want Decidim seeds.
+
+Decidim::TranslatorConfigurationHelper.able_to_seed?
+
 if ENV["HEROKU_APP_NAME"].present?
   ENV["DECIDIM_HOST"] = "#{ENV["HEROKU_APP_NAME"]}.herokuapp.com"
   ENV["SEED"] = "true"

docs/GETTING_STARTED.md

@@ -57,3 +57,21 @@ You can now start your server locally by executing `bundle exec rails s` and acc
 Testing is very important, in the decidim-app we implement existing specs from Decidim to prevent regression. Then we update specs to match customizations made in repository. 
 
 If you want to execute specs you can setup your test environment `bundle exec rake test:setup` then use Rspec `bundle exec rspec spec`
+
+### Configuration
+#### Machine translation configuration
+
+Machine translation is configured through the provider [DeepL](https://www.deepl.com) by using the gem https://github.com/wikiti/deepl-rb.
+
+In order to make it work these ENV variables need to be configured:
+
+```
+TRANSLATOR_ENABLED=0
+TRANSLATOR_API_KEY=*******
+TRANSLATOR_HOST=https://api-free.deepl.com
+```
+
+- Obtain the `TRANSLATOR_API_KEY` by creating an account at https://www.deepl.com/pro#developer
+- For `TRANSLATOR_HOST`, set it to `https://api-free.deepl.com` if using the "DeeL API Free" plan. If using the "DeepL API Pro", then set it to `https://api.deepl.com`
+
+> Note: you still need to enable machine translation at the organization settings.

lib/decidim/translator_configuration_helper.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Decidim
+  module TranslatorConfigurationHelper
+    def self.able_to_seed?
+      return true unless translator_activated?
+
+      raise "You can't seed the database with machine translations enabled unless you use a compatible backend" unless compatible_backend?
+    end
+
+    def self.compatible_backend?
+      Rails.configuration.active_job.queue_adapter != :async
+    end
+
+    def self.translator_activated?
+      Decidim.enable_machine_translations
+    end
+  end
+end

spec/jobs/decidim/machine_translation_fields_job_spec.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "deepl"
+
+module Decidim
+  describe MachineTranslationFieldsJob do
+    let(:title) { { en: "New Title" } }
+    let(:process) { build :participatory_process, title: title }
+    let(:target_locale) { "ca" }
+    let(:source_locale) { "en" }
+    let(:url) { "https://dummy_url.org" }
+    let(:translation) { double("translation", text: "Nou títol") }
+
+    before do
+      allow(Decidim).to receive(:machine_translation_service_klass).and_return(DeeplTranslator)
+      allow(DeepL).to receive(:translate).with(title[source_locale.to_sym], source_locale, target_locale).and_return(translation)
+    end
+
+    describe "When fields job is executed" do
+      before do
+        clear_enqueued_jobs
+      end
+
+      it "calls DeeplTranslator to create machine translations" do
+        expect(DeeplTranslator).to receive(:new).with(
+          process,
+          "title",
+          process["title"][source_locale],
+          target_locale,
+          source_locale
+        ).and_call_original
+
+        process.save
+
+        MachineTranslationFieldsJob.perform_now(
+          process,
+          "title",
+          process["title"][source_locale],
+          target_locale,
+          source_locale
+        )
+      end
+    end
+  end
+end