fsw similarity metrics

signwriting_evaluation/metrics/similarity.py

@@ -0,0 +1,108 @@
+from math import pow, sqrt, exp
+from typing import List, Tuple
+
+import numpy as np
+from scipy.optimize import linear_sum_assignment
+from signwriting.types import Sign, SignSymbol
+from signwriting.formats.fsw_to_sign import fsw_to_sign
+from base import SignWritingMetric
+
+HEX_BASE = 16
+
+
+class SignWritingSimilarityMetric(SignWritingMetric):
+    def __init__(self):
+        super().__init__("SymbolsDistancesMetric")
+        self.symbol_classes = self.symbol_classes = {
+            'handshakes': range(0x100, 0x205),
+            'contact_symbols': range(0x205, 0x221),
+            'movement_paths': range(0x221, 0x2FF),
+            'head_movement': range(0x2FF, 0x30A),
+            'facial_expressions': range(0x30A, 0x36A),
+            'etc': range(0x36A, 0x38C)
+        }
+        self.shape_weight = 24
+        self.facing_weight = 8
+        self.angle_weight = 1
+        self.parallel_weight = 24
+        self.positional_weight = 1
+        self.normalized_factor = 1/5
+        self.class_penalty = 84
+        self.max_distance = self.calculate_distance({"symbol": "S10000", "position": (0, 0)},
+                                                    {"symbol": "S38b07", "position": (999, 999)})
+
+    def euc_distance(self, first: Tuple[int, int], second: Tuple[int, int]) -> float:
+        return sqrt(pow(first[0] - second[0], 2) + pow(first[1] - second[1], 2))
+
+    def get_attributes(self, symbol: SignSymbol) -> Tuple[int, int, int, bool]:  # S12345, 123x123
+        shape = int(symbol['symbol'][1:4], HEX_BASE)  # only 123
+        facing = int(symbol['symbol'][4], HEX_BASE)  # only 4
+        angle = int(symbol['symbol'][5], HEX_BASE)  # only 5
+        parallel = facing > 2  # left or right of sign table (0 vertical, 1 horizontal)
+        return shape, facing, angle, parallel
+
+    def calculate_distance(self, pred: SignSymbol, gold: SignSymbol) -> float:
+
+        shape1, angle1, facing1, parallel1 = self.get_attributes(pred)
+        shape2, angle2, facing2, parallel2 = self.get_attributes(gold)
+        distance = (self.shape_weight * abs(shape1 - shape2) + self.facing_weight * abs(facing1 - facing2) +
+                    self.angle_weight * abs(angle1 - angle2) + self.parallel_weight * (parallel1 != parallel2) +
+                    self.positional_weight * self.euc_distance(pred["position"], gold["position"]))
+        pred_class = next((i for i, r in enumerate(self.symbol_classes.values()) if shape1 in r), None)
+        gold_class = next((i for i, r in enumerate(self.symbol_classes.values()) if shape2 in r), None)
+        distance = distance + abs(gold_class - pred_class) * self.class_penalty
+        return distance
+
+    def normalized(self, unnormalized: float) -> float:
+        return pow(unnormalized / self.max_distance, self.normalized_factor)
+
+    def symbols_score(self, pred: SignSymbol, gold: SignSymbol) -> float:
+        distance = self.calculate_distance(pred, gold)
+        normalized = self.normalized(distance)
+        return normalized
+
+    def length_acc(self, pred: Sign, gold: Sign) -> float:
+        pred = pred["symbols"]
+        gold = gold["symbols"]
+        # plus 1 for the box symbol
+        return abs(len(pred) - len(gold)) / (max(len(pred), len(gold)) + 1)
+
+    def error_rate(self, pred: Sign, gold: Sign) -> float:
+        """
+        Calculate the evaluate score for a given prediction and gold.
+        :param pred: the prediction
+        :param gold: the gold
+        :return: the FWS score
+        """
+        if not pred['symbols']:
+            return 1
+        cost_matrix = np.array(
+            [self.symbols_score(first, second) for first in pred["symbols"] for second in gold["symbols"]])
+        cost_matrix = cost_matrix.reshape(len(pred["symbols"]), -1)
+        # Find the lowest cost matching
+        row_ind, col_ind = linear_sum_assignment(cost_matrix)
+        pairs = list(zip(row_ind, col_ind))
+        # Print the matching and total cost
+        values = [cost_matrix[row, col] for row, col in pairs]
+        mean_cost = sum(values) / len(values)
+        length_error = self.length_acc(pred, gold)
+        length_weight = (1.05 / (1 + exp(-7 * length_error + 3.5))) - 0.025
+        return length_weight + mean_cost * (1 - length_weight)
+
+    def score(self, hypothesis: str, reference: str) -> float:
+        """
+        Calculate the evaluate score for a given prediction and gold.
+        :param reference:
+        :param hypothesis:
+        :param pred: the prediction
+        :param gold: the gold
+        :return: the FWS score
+        """
+        pred = fsw_to_sign(hypothesis)
+        gold = fsw_to_sign(reference)
+        return 1 - self.error_rate(pred, gold)
+
+
+
+
+

signwriting_evaluation/metrics/similarity_metrics.md

@@ -0,0 +1,40 @@
+# Evaluation metric for SignWriting
+### Introduction
+The code presented here implements a new metric for evaluating the similarity of two given phrases written in FSW.
+The goal of its creation was to replace the generic and less task-specific evaluation methods used to assess the quality of FSW transcription by default, such as
+BLEU, CHRF and more, by a signwriting-concentrated approach that knows to evaluate similarity based on the language's rules and principles (that were otherwise tested by general string comperisons)
+
+### Evaluation Method
+The evaluation method we created attempts to bridge the gap that opens when using general methods that were implemented to support strings, but not specifically signwriting strings.
+By using a method that is not aware of the field that it is used in, it cannot properly focus on the right attributes and guidelines, all the dos and don'ts that were developed when creating this language.
+
+The method we created takes into account the following important notes and is conditioned to treat them correctly, as well as other features:
+- Symbols in the FSW dictionary are organized nicely to portray their separation to types such as hand signals, motion, touch and more. The closer in the table, usually the closer they are in their representation and meaning.
+- Symbols that construct a sign can be written in different orders, yet represent the same visual output.
+- Each part of a symbol has a different meaning, and therefore different importance. Specifically: "S12345600x600" can be broken into aspects such as kind of symbol ("123"), facing direction ("4"), angle ("5"), position ("600x600") and so on for different kinds of symbols.
+
+### Main concept
+The evaluation process we developed is built on three main stages, each with its own intent and purposes:
+1. **Symbol distance function**: this function is in charge of receiving two symbols (certain parts of signs), and calculating a distance value to represent the similarity or lack of it between
+   the two according to the language and sign table. Its grade and actual factors are based on the ones stated above, custom weights we chose to define the importance of different differences and more.
+2. **Distance Normalization**: the return value of the function does have a maximum value (the distance between the two poles of the table) but is not between 0-1. Therefore, we divide by the maximum distance and add an additional step of inserting the value into a non-linear function, which makes the value more clearly representing the ratio of the distance:
+
+<p align="center">
+  <img src="https://github.com/ohadlanger/try/assets/118103585/3dab6c81-272a-48e3-8f04-9f7fed840c38" width="20%" height="20%">
+</p>
+
+$$
+f(x) = x^{\frac{1}{5}}
+$$
+
+
+3. **Matching and grading**: now that we have established our ability to quantify the similarity of two symbols, we want to utilize this ability to generalize it for whole signs (series of             continuous symbols), and hopefully understand better the resemblance and common parts of two given inputs. The way we do that is by breaking down the sign into its constructing symbols and          looking for similar parts that can be assumed to be close in meaning. We do that by iterating over all couples of the two symbols arrays and creating a distance matrix, where each [i,j] slot        represents the output of the distance function on the i'th element of the first group, and the j'th of the second. Now, we pass this matrix to a Hungarian algorithm provided by the scipy library    for matching pairs of close parts in each sign. The next step is to take the distance from all of those ideal matchings, calculate the mean, and lastly, for taking into account the fact that
+   the inputs may differ in length (and their length difference is part of their similarity), we input this difference into another non-linear function to decide a weight for the mean difference       between the inputs, according to the lengths, and so get a better representation of a grade, which is the final value we return.
+
+<p align="center">
+  <img src="https://github.com/ohadlanger/try/assets/118103585/3b706a19-a627-4b2e-bd9e-209506e81565" width="20%" height="20%">
+</p>
+
+$$
+q\left(x\right)=\left(\frac{1.05}{1+e^{-7x+3.5}}\right)-0.025
+$$