Validateur NeTEx : validation "à la demande"

apps/shared/lib/date_time_display.ex

@@ -104,6 +104,71 @@ defmodule Shared.DateTimeDisplay do
 
   def format_datetime_to_paris(nil, _, _), do: ""
 
+  @doc """
+  Formats a duration in seconds to display, according to a locale.
+
+  Supported locales: "fr" and "en".
+
+  iex> format_duration(0, "en")
+  "0 second"
+  iex> format_duration(1, "en")
+  "1 second"
+  iex> format_duration(3, "en")
+  "3 seconds"
+  iex> format_duration(60, "en")
+  "1 minute"
+  iex> format_duration(61, "en")
+  "1 minute 1 second"
+  iex> format_duration(65, "en")
+  "1 minute 5 seconds"
+  iex> format_duration(120, "en")
+  "2 minutes"
+  iex> format_duration(125, "en")
+  "2 minutes 5 seconds"
+
+  iex> format_duration(0, "fr")
+  "0 seconde"
+  iex> format_duration(1, "fr")
+  "1 seconde"
+  iex> format_duration(3, "fr")
+  "3 secondes"
+  iex> format_duration(60, "fr")
+  "1 minute"
+  iex> format_duration(61, "fr")
+  "1 minute 1 seconde"
+  iex> format_duration(65, "fr")
+  "1 minute 5 secondes"
+  iex> format_duration(120, "fr")
+  "2 minutes"
+  iex> format_duration(125, "fr")
+  "2 minutes 5 secondes"
+  """
+  @spec format_duration(non_neg_integer(), binary()) :: binary()
+  def format_duration(duration_in_seconds, locale) do
+    minutes = div(duration_in_seconds, 60)
+    seconds = rem(duration_in_seconds, 60)
+
+    cond do
+      minutes > 0 and seconds > 0 -> "#{format_minutes(minutes, locale)} #{format_seconds(seconds, locale)}"
+      minutes > 0 -> format_minutes(minutes, locale)
+      true -> format_seconds(seconds, locale)
+    end
+  end
+
+  defp format_seconds(seconds, "en"), do: format_duration_unit(seconds, "second")
+  defp format_seconds(seconds, _locale), do: format_duration_unit(seconds, "seconde")
+
+  defp format_minutes(seconds, "en"), do: format_duration_unit(seconds, "minute")
+  defp format_minutes(seconds, _locale), do: format_duration_unit(seconds, "minute")
+
+  defp format_duration_unit(duration, unit) do
+    if duration > 1 do
+      "#{duration} #{unit}s"
+    else
+      "#{duration} #{unit}"
+    end
+  end
+
   @spec convert_to_paris_time(DateTime.t() | NaiveDateTime.t()) :: DateTime.t()
   defp convert_to_paris_time(%DateTime{} = dt) do
     case Timex.Timezone.convert(dt, "Europe/Paris") do

apps/transport/lib/jobs/on_demand_validation_job.ex

@@ -15,6 +15,7 @@ defmodule Transport.Jobs.OnDemandValidationJob do
   alias Transport.DataVisualization
   alias Transport.Validators.GTFSRT
   alias Transport.Validators.GTFSTransport
+  alias Transport.Validators.NeTEx
   @download_timeout_ms 10_000
 
   @impl Oban.Worker
@@ -71,6 +72,28 @@ defmodule Transport.Jobs.OnDemandValidationJob do
     end
   end
 
+  defp perform_validation(%{"type" => "netex", "permanent_url" => url}) do
+    validator = NeTEx.validator_name()
+
+    case NeTEx.validate(url, []) do
+      {:error, msg} ->
+        %{oban_args: %{"state" => "error", "error_reason" => msg}, validator: validator}
+
+      {:ok, %{"validations" => validation, "metadata" => metadata}} ->
+        %{
+          result: validation,
+          metadata: metadata,
+          data_vis: nil,
+          validator: validator,
+          validated_data_name: url,
+          max_error: NeTEx.get_max_severity_error(validation),
+          oban_args: %{
+            "state" => "completed"
+          }
+        }
+    end
+  end
+
   defp perform_validation(%{
          "type" => "tableschema",
          "permanent_url" => url,

apps/transport/lib/transport_web/controllers/resource_controller.ex

@@ -4,7 +4,7 @@ defmodule TransportWeb.ResourceController do
   alias Transport.DataVisualization
   import Ecto.Query
 
-  import TransportWeb.ResourceView, only: [issue_type: 1, latest_validations_nb_days: 0]
+  import TransportWeb.ResourceView, only: [latest_validations_nb_days: 0]
   import TransportWeb.DatasetView, only: [availability_number_days: 0]
 
   @enabled_validators MapSet.new([
@@ -144,7 +144,7 @@ defmodule TransportWeb.ResourceController do
 
     issue_type =
       case params["issue_type"] do
-        nil -> issue_type(issues)
+        nil -> Transport.Validators.GTFSTransport.issue_type(issues)
         issue_type -> issue_type
       end
 

apps/transport/lib/transport_web/controllers/validation_controller.ex

@@ -2,7 +2,6 @@ defmodule TransportWeb.ValidationController do
   use TransportWeb, :controller
   alias DB.{MultiValidation, Repo}
   alias Transport.DataVisualization
-  import TransportWeb.ResourceView, only: [issue_type: 1]
   import Ecto.Query
 
   def validate(%Plug.Conn{} = conn, %{"upload" => %{"url" => url, "type" => "gbfs"} = params}) do
@@ -98,44 +97,57 @@ defmodule TransportWeb.ValidationController do
         unauthorized(conn)
 
       %MultiValidation{oban_args: %{"state" => "completed", "type" => "gtfs"}} = validation ->
-        current_issues = Transport.Validators.GTFSTransport.get_issues(validation.result, params)
+        validator = Transport.Validators.GTFSTransport
+        current_issues = validator.get_issues(validation.result, params)
 
         issue_type =
           case params["issue_type"] do
-            nil -> issue_type(current_issues)
+            nil -> validator.issue_type(current_issues)
             issue_type -> issue_type
           end
 
         conn
-        |> assign(:validation_id, params["id"])
-        |> assign(:other_resources, [])
-        |> assign(:issues, Scrivener.paginate(current_issues, make_pagination_config(params)))
-        |> assign(
-          :validation_summary,
-          Transport.Validators.GTFSTransport.summary(validation.result)
-        )
-        |> assign(
-          :severities_count,
-          Transport.Validators.GTFSTransport.count_by_severity(validation.result)
-        )
+        |> assign_base_validation_details(validator, validation, params, current_issues)
         |> assign(:metadata, validation.metadata.metadata)
         |> assign(:modes, validation.metadata.modes)
         |> assign(:data_vis, data_vis(validation, issue_type))
-        |> assign(:token, token)
-        |> render("show.html")
+        |> render("show_gtfs.html")
+
+      %MultiValidation{oban_args: %{"state" => "completed", "type" => "netex"}} = validation ->
+        validator = Transport.Validators.NeTEx
+        current_issues = validator.get_issues(validation.result, params)
+
+        conn
+        |> assign_base_validation_details(validator, validation, params, current_issues)
+        |> assign(:metadata, validation.metadata.metadata)
+        |> assign(:modes, [])
+        |> assign(:data_vis, nil)
+        |> render("show_netex.html")
 
       # Handles waiting for validation to complete, errors and
       # validation for schemas
       _ ->
         live_render(conn, TransportWeb.Live.OnDemandValidationLive,
           session: %{
             "validation_id" => params["id"],
+            "issue_type" => params["issue_type"],
             "current_url" => validation_path(conn, :show, params["id"], token: token)
           }
         )
     end
   end
 
+  defp assign_base_validation_details(conn, validator, validation, params, current_issues) do
+    conn
+    |> assign(:validator, validator)
+    |> assign(:validation_id, params["id"])
+    |> assign(:other_resources, [])
+    |> assign(:issues, Scrivener.paginate(current_issues, make_pagination_config(params)))
+    |> assign(:validation_summary, validator.summary(validation.result))
+    |> assign(:severities_count, validator.count_by_severity(validation.result))
+    |> assign(:token, params["token"])
+  end
+
   defp data_vis(%MultiValidation{} = validation, issue_type) do
     data_vis = validation.data_vis[issue_type]
     has_features = DataVisualization.has_features(data_vis["geojson"])
@@ -150,7 +162,7 @@ defmodule TransportWeb.ValidationController do
   defp filepath(type) do
     cond do
       type == "tableschema" -> Ecto.UUID.generate() <> ".csv"
-      type in ["jsonschema", "gtfs"] -> Ecto.UUID.generate()
+      type in ["jsonschema", "gtfs", "netex"] -> Ecto.UUID.generate()
     end
   end
 
@@ -215,6 +227,7 @@ defmodule TransportWeb.ValidationController do
     args =
       case type do
         "gtfs" -> %{"type" => "gtfs"}
+        "netex" -> %{"type" => "netex"}
         schema_name -> %{"schema_name" => schema_name, "type" => schema_type(schema_name)}
       end
 

apps/transport/lib/transport_web/live/on_demand_validation_live.ex

@@ -35,7 +35,7 @@ defmodule TransportWeb.Live.OnDemandValidationLive do
       schedule_next_update_data()
     end
 
-    if gtfs_validation_completed?(socket) do
+    if gtfs_or_netex_validation_completed?(socket) do
       redirect(socket, to: socket_value(socket, :current_url))
     else
       socket
@@ -57,10 +57,10 @@ defmodule TransportWeb.Live.OnDemandValidationLive do
     end
   end
 
-  defp gtfs_validation_completed?(socket) do
+  defp gtfs_or_netex_validation_completed?(socket) do
     case socket_value(socket, :validation) do
       %DB.MultiValidation{oban_args: oban_args} ->
-        oban_args["type"] == "gtfs" and oban_args["state"] == "completed"
+        oban_args["type"] in ["gtfs", "netex"] and oban_args["state"] == "completed"
 
       _ ->
         false

apps/transport/lib/transport_web/live/on_demand_validation_select_live.ex

@@ -53,7 +53,6 @@ defmodule TransportWeb.Live.OnDemandValidationSelectLive do
 
   def determine_input_type(type) when type in ["gbfs"], do: "link"
   def determine_input_type(type) when type in ["gtfs-rt"], do: "gtfs-rt"
-  def determine_input_type(type) when type in ["netex"], do: nil
   def determine_input_type(_), do: "file"
 
   def handle_event("form_changed", %{"upload" => params, "_target" => target}, socket) do

apps/transport/lib/transport_web/live/on_demand_validation_select_live.html.heex

@@ -38,28 +38,13 @@
             type: "url"
           ) %>
         <% end %>
-        <%= unless @input_type == "file" or @type == "netex" do %>
+        <%= unless @input_type == "file" do %>
           <%= submit(dgettext("validations", "Validate"), nodiv: true) %>
         <% end %>
       </.form>
       <p :if={@trigger_submit} class="small">
         <%= TransportWeb.Gettext.dgettext("validations", "Upload in progress") %>
       </p>
-      <div :if={@type == "netex"} class="container section section-grey" id="netex-explanations">
-        <p>
-          <%= raw(
-            TransportWeb.Gettext.dgettext("validations", "netex-siri-validator", link: "https://greenlight.itxpt.eu")
-          ) %>
-        </p>
-
-        <p>
-          <%= raw(
-            TransportWeb.Gettext.dgettext("validations", "doc-eu-formats",
-              link: "https://doc.transport.data.gouv.fr/documentation/normes-europeennes"
-            )
-          ) %>
-        </p>
-      </div>
     </div>
   </div>
 </section>

apps/transport/lib/transport_web/templates/resource/_netex_generic_issue.html.heex

@@ -0,0 +1,58 @@
+<style>
+  table.generic_issue tr.message td {
+    vertical-align: top;
+  }
+
+  table.generic_issue th:nth-child(1) {
+    width: 60%;
+  }
+
+  table.generic_issue tr.debug:hover {
+    background: revert;
+  }
+
+  table.generic_issue tr.debug td {
+    border-top: none;
+    padding-top: 0;
+  }
+
+  table.generic_issue tr.debug td summary {
+    cursor: pointer;
+  }
+
+  table.generic_issue tr.debug td pre {
+    margin-block: 0;
+  }
+
+  table.generic_issue tr.debug td code {
+    width: 100%;
+  }
+</style>
+
+<table class="table generic_issue">
+  <tr>
+    <th><%= dgettext("validations-explanations", "Message") %></th>
+    <th><%= dgettext("validations-explanations", "Location") %></th>
+  </tr>
+
+  <%= for issue <- @issues do %>
+    <tr class="message">
+      <td><%= issue["message"] %></td>
+      <td>
+        <%= if is_nil(issue["resource"]) or is_nil(issue["resource"]["filename"]) or is_nil(issue["resource"]["line"]) do %>
+          <%= dgettext("validations-explanations", "Unknown location") %>
+        <% else %>
+          <%= issue["resource"]["filename"] %>:<%= issue["resource"]["line"] %>
+        <% end %>
+      </td>
+    </tr>
+    <tr class="debug">
+      <td colspan="2">
+        <details>
+          <summary><%= dgettext("validations-explanations", "Details for debugging purposes") %></summary>
+          <pre><code><%= to_string(Jason.encode!(issue, pretty: true)) %></code></pre>
+        </details>
+      </td>
+    </tr>
+  <% end %>
+</table>

apps/transport/lib/transport_web/templates/resource/_resources_details.html.heex

@@ -30,8 +30,8 @@ stats = @metadata["stats"] %>
         <li if={networks != []}>
           <div>
             <div class="networks-list">
-              <%= dngettext("validations", "network", "networks", length(@metadata["networks"])) %> :
-              <strong><%= Enum.join(@metadata["networks"], ", ") %></strong>
+              <%= dngettext("validations", "network", "networks", length(networks)) %> :
+              <strong><%= Enum.join(networks, ", ") %></strong>
             </div>
           </div>
         </li>

apps/transport/lib/transport_web/templates/resource/_resources_details_netex.html.heex

@@ -0,0 +1,10 @@
+<% locale = get_session(@conn, :locale) %>
+<% duration =
+  if @metadata["elapsed_seconds"] < 1 do
+    dgettext("validations", "less than 1 second")
+  else
+    format_duration(@metadata["elapsed_seconds"], locale)
+  end %>
+<div>
+  <%= dgettext("validations", "Elapsed time: %{duration}.", duration: duration) %>
+</div>

apps/transport/lib/transport_web/templates/resource/_validation_summary.html.heex

@@ -3,7 +3,7 @@
     <%= for {severity, issues} <- @validation_summary do %>
       <%= if Map.get(@severities_count, severity, 0) > 0 do %>
         <div class="validation-issue">
-          <h4><%= @severities_count[severity] %> <%= severity(severity).text %></h4>
+          <h4><%= @severities_count[severity] %> <%= @validator.severity(severity).text %></h4>
           <ul>
             <%= for {key, issue} <- issues do %>
               <li>
@@ -12,7 +12,7 @@
                     "#{issue.title} (#{issue.count})",
                     to:
                       "#{current_url(@conn, %{"issue_type" => key, "token" => @token} |> Map.reject(fn {_, v} -> is_nil(v) end))}#issues",
-                    class: if(key == issue_type(@issues.entries), do: "active")
+                    class: if(key == @validator.issue_type(@issues.entries), do: "active")
                   ) %>
                 <% end %>
               </li>

apps/transport/lib/transport_web/templates/resource/gtfs_details.html.heex

@@ -108,7 +108,7 @@ associated_netex = get_associated_netex(@related_files) %>
             </nav>
             <div class="main-pane">
               <%= pagination_links(@conn, @issues, [@resource.id],
-                issue_type: issue_type(@issues.entries),
+                issue_type: Transport.Validators.GTFSTransport.issue_type(@issues.entries),
                 path: &resource_path/4,
                 action: :details
               ) %>