Merge pull request #55 from aalto-ui/feat/segmentation

feat(metric, segmentation): segmentation and grid metrics

.pre-commit-config.yaml

@@ -21,7 +21,7 @@ repos:
     pass_filenames: false
     args: ['--config-file=backend/mypy.ini', 'backend/']
 - repo: https://github.com/PyCQA/flake8
-  rev: 3.9.2
+  rev: 5.0.4
   hooks:
   - id: flake8
     args: ['--config=backend/.flake8', 'backend/']

backend/aim/handlers.py

@@ -42,16 +42,17 @@
 )
 from aim.exceptions import ValidationError
 from aim.models import MessageBase, MessageImage, MessageInput, MessageURL
+from aim.segmentation.model import Segmentation
 from aim.tools import Screenshot
 
 # ----------------------------------------------------------------------------
 # Metadata
 # ----------------------------------------------------------------------------
 
 __author__ = "Markku Laine"
-__date__ = "2022-04-25"
+__date__ = "2022-12-20"
 __email__ = "[email protected]"
-__version__ = "1.1"
+__version__ = "1.2"
 
 
 # ----------------------------------------------------------------------------
@@ -147,6 +148,21 @@ def on_message(self, message: Union[str, bytes]):
                 }
             )
 
+            # Execute segmentation
+            # Check if segmentation is needed
+            metrics_configurations: Dict[
+                str, Any
+            ] = utils.load_metrics_configurations()
+            metrics_conf_segmentation: List = [
+                metrics_configurations["metrics"][metric][
+                    "segmentation_required"
+                ]
+                for metric in msg.metrics.keys()
+            ]
+            gui_segments: Optional[Dict[str, Any]] = None
+            if any(metrics_conf_segmentation):
+                gui_segments = Segmentation.execute(png_image_base64)
+
             # Iterate over selected metrics and execute them one by one
             for metric in {k: v for k, v in msg.metrics.items()}:
                 logger.debug("Executing metric {}...".format(metric))
@@ -176,6 +192,7 @@ def on_message(self, message: Union[str, bytes]):
                     results: Optional[List[Union[int, float, str]]] = metric_module.Metric.execute_metric(  # type: ignore
                         png_image_base64,
                         gui_type=GUI_TYPE_DESKTOP,
+                        gui_segments=gui_segments,
                         gui_url=msg.url,
                     )
                     end_time: float = time.time()

backend/aim/metrics/interfaces.py

@@ -12,7 +12,7 @@
 
 # Standard library modules
 import abc
-from typing import Any, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 # Third-party modules
 from pydantic import HttpUrl
@@ -25,9 +25,9 @@
 # ----------------------------------------------------------------------------
 
 __author__ = "Markku Laine"
-__date__ = "2022-05-26"
+__date__ = "2022-12-20"
 __email__ = "[email protected]"
-__version__ = "1.1"
+__version__ = "1.2"
 
 
 # ----------------------------------------------------------------------------
@@ -51,6 +51,7 @@ def execute_metric(
         self,
         gui_image: str,
         gui_type: int = GUI_TYPE_DESKTOP,
+        gui_segments: Optional[Dict[str, Any]] = None,
         gui_url: Optional[HttpUrl] = None,
     ) -> Optional[List[Union[int, float, str]]]:
         """
@@ -61,6 +62,7 @@ def execute_metric(
 
         Kwargs:
             gui_type: GUI type, desktop = 0 (default), mobile = 1
+            gui_segments: GUI segments (defaults to None)
             gui_url: GUI URL (defaults to None)
 
         Returns:

backend/aim/metrics/m1/m1_png_file_size.py

@@ -51,7 +51,7 @@
 # ----------------------------------------------------------------------------
 
 # Standard library modules
-from typing import List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 # Third-party modules
 from pydantic import HttpUrl
@@ -85,6 +85,7 @@ class Metric(AIMMetricInterface):
     def execute_metric(
         gui_image: str,
         gui_type: int = GUI_TYPE_DESKTOP,
+        gui_segments: Optional[Dict[str, Any]] = None,
         gui_url: Optional[HttpUrl] = None,
     ) -> Optional[List[Union[int, float, str]]]:
         """
@@ -95,6 +96,7 @@ def execute_metric(
 
         Kwargs:
             gui_type: GUI type, desktop = 0 (default), mobile = 1
+            gui_segments: GUI segments (defaults to None)
             gui_url: GUI URL (defaults to None)
 
         Returns:

backend/aim/metrics/m10/m10_wave.py

@@ -51,7 +51,7 @@
 # Standard library modules
 import base64
 from io import BytesIO
-from typing import Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 # Third-party modules
 import numpy as np
@@ -298,6 +298,7 @@ def execute_metric(
         cls,
         gui_image: str,
         gui_type: int = GUI_TYPE_DESKTOP,
+        gui_segments: Optional[Dict[str, Any]] = None,
         gui_url: Optional[HttpUrl] = None,
     ) -> Optional[List[Union[int, float, str]]]:
         """
@@ -308,6 +309,7 @@ def execute_metric(
 
         Kwargs:
             gui_type: GUI type, desktop = 0 (default), mobile = 1
+            gui_segments: GUI segments (defaults to None)
             gui_url: GUI URL (defaults to None)
 
         Returns:

backend/aim/metrics/m11/m11_static_clusters.py

@@ -60,7 +60,7 @@
 # Standard library modules
 import base64
 from io import BytesIO
-from typing import List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 # Third-party modules
 import numpy as np
@@ -104,6 +104,7 @@ def execute_metric(
         cls,
         gui_image: str,
         gui_type: int = GUI_TYPE_DESKTOP,
+        gui_segments: Optional[Dict[str, Any]] = None,
         gui_url: Optional[HttpUrl] = None,
     ) -> Optional[List[Union[int, float, str]]]:
         """
@@ -114,6 +115,7 @@ def execute_metric(
 
         Kwargs:
             gui_type: GUI type, desktop = 0 (default), mobile = 1
+            gui_segments: GUI segments (defaults to None)
             gui_url: GUI URL (defaults to None)
 
         Returns:

backend/aim/metrics/m12/m12_dynamic_clusters.py

@@ -56,7 +56,7 @@
 # Standard library modules
 import base64
 from io import BytesIO
-from typing import List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 # Third-party modules
 import numpy as np
@@ -246,6 +246,7 @@ def execute_metric(
         cls,
         gui_image: str,
         gui_type: int = GUI_TYPE_DESKTOP,
+        gui_segments: Optional[Dict[str, Any]] = None,
         gui_url: Optional[HttpUrl] = None,
     ) -> Optional[List[Union[int, float, str]]]:
         """
@@ -256,6 +257,7 @@ def execute_metric(
 
         Kwargs:
             gui_type: GUI type, desktop = 0 (default), mobile = 1
+            gui_segments: GUI segments (defaults to None)
             gui_url: GUI URL (defaults to None)
 
         Returns:

backend/aim/metrics/m13/m13_luminance_std.py

@@ -45,7 +45,7 @@
 # Standard library modules
 import base64
 from io import BytesIO
-from typing import List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 # Third-party modules
 import numpy as np
@@ -89,6 +89,7 @@ def execute_metric(
         cls,
         gui_image: str,
         gui_type: int = GUI_TYPE_DESKTOP,
+        gui_segments: Optional[Dict[str, Any]] = None,
         gui_url: Optional[HttpUrl] = None,
     ) -> Optional[List[Union[int, float, str]]]:
         """
@@ -99,6 +100,7 @@ def execute_metric(
 
         Kwargs:
             gui_type: GUI type, desktop = 0 (default), mobile = 1
+            gui_segments: GUI segments (defaults to None)
             gui_url: GUI URL (defaults to None)
 
         Returns:

backend/aim/metrics/m14/m14_lab_avg_std.py

@@ -41,7 +41,7 @@
 # Standard library modules
 import base64
 from io import BytesIO
-from typing import List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 # Third-party modules
 import numpy as np
@@ -79,6 +79,7 @@ def execute_metric(
         cls,
         gui_image: str,
         gui_type: int = GUI_TYPE_DESKTOP,
+        gui_segments: Optional[Dict[str, Any]] = None,
         gui_url: Optional[HttpUrl] = None,
     ) -> Optional[List[Union[int, float, str]]]:
         """
@@ -89,6 +90,7 @@ def execute_metric(
 
         Kwargs:
             gui_type: GUI type, desktop = 0 (default), mobile = 1
+            gui_segments: GUI segments (defaults to None)
             gui_url: GUI URL (defaults to None)
 
         Returns:

backend/aim/metrics/m15/m15_colorfulness_hassler_susstrunk.py

@@ -46,7 +46,7 @@
 # Standard library modules
 import base64
 from io import BytesIO
-from typing import List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 # Third-party modules
 import numpy as np
@@ -86,6 +86,7 @@ def execute_metric(
         cls,
         gui_image: str,
         gui_type: int = GUI_TYPE_DESKTOP,
+        gui_segments: Optional[Dict[str, Any]] = None,
         gui_url: Optional[HttpUrl] = None,
     ) -> Optional[List[Union[int, float, str]]]:
         """
@@ -96,6 +97,7 @@ def execute_metric(
 
         Kwargs:
             gui_type: GUI type, desktop = 0 (default), mobile = 1
+            gui_segments: GUI segments (defaults to None)
             gui_url: GUI URL (defaults to None)
 
         Returns:

backend/aim/metrics/m16/m16_hsv_avg_std.py

@@ -43,7 +43,7 @@
 # Standard library modules
 import base64
 from io import BytesIO
-from typing import List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 # Third-party modules
 import numpy as np
@@ -81,6 +81,7 @@ def execute_metric(
         cls,
         gui_image: str,
         gui_type: int = GUI_TYPE_DESKTOP,
+        gui_segments: Optional[Dict[str, Any]] = None,
         gui_url: Optional[HttpUrl] = None,
     ) -> Optional[List[Union[int, float, str]]]:
         """
@@ -91,6 +92,7 @@ def execute_metric(
 
         Kwargs:
             gui_type: GUI type, desktop = 0 (default), mobile = 1
+            gui_segments: GUI segments (defaults to None)
             gui_url: GUI URL (defaults to None)
 
         Returns:

backend/aim/metrics/m17/m17_distinct_values_of_hue_saturation_and_value.py

@@ -52,7 +52,7 @@
 # Standard library modules
 import base64
 from io import BytesIO
-from typing import List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 # Third-party modules
 import numpy as np
@@ -94,6 +94,7 @@ def execute_metric(
         cls,
         gui_image: str,
         gui_type: int = GUI_TYPE_DESKTOP,
+        gui_segments: Optional[Dict[str, Any]] = None,
         gui_url: Optional[HttpUrl] = None,
     ) -> Optional[List[Union[int, float, str]]]:
         """
@@ -104,6 +105,7 @@ def execute_metric(
 
         Kwargs:
             gui_type: GUI type, desktop = 0 (default), mobile = 1
+            gui_segments: GUI segments (defaults to None)
             gui_url: GUI URL (defaults to None)
 
         Returns:

backend/aim/metrics/m18/m18_nima.py

@@ -45,7 +45,7 @@
 import collections
 import pathlib
 from io import BytesIO
-from typing import List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 # Third-party modules
 import torch
@@ -123,6 +123,7 @@ def execute_metric(
         cls,
         gui_image: str,
         gui_type: int = GUI_TYPE_DESKTOP,
+        gui_segments: Optional[Dict[str, Any]] = None,
         gui_url: Optional[HttpUrl] = None,
     ) -> Optional[List[Union[int, float, str]]]:
         """
@@ -133,6 +134,7 @@ def execute_metric(
 
         Kwargs:
             gui_type: GUI type, desktop = 0 (default), mobile = 1
+            gui_segments: GUI segments (defaults to None)
             gui_url: GUI URL (defaults to None)
 
         Returns:

backend/aim/metrics/m19/m19_distinct_rgb_values_per_dynamic_cluster.py

@@ -47,7 +47,7 @@
 # Standard library modules
 import base64
 from io import BytesIO
-from typing import List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 # Third-party modules
 from PIL import Image
@@ -84,6 +84,7 @@ def execute_metric(
         cls,
         gui_image: str,
         gui_type: int = GUI_TYPE_DESKTOP,
+        gui_segments: Optional[Dict[str, Any]] = None,
         gui_url: Optional[HttpUrl] = None,
     ) -> Optional[List[Union[int, float, str]]]:
         """
@@ -94,6 +95,7 @@ def execute_metric(
 
         Kwargs:
             gui_type: GUI type, desktop = 0 (default), mobile = 1
+            gui_segments: GUI segments (defaults to None)
             gui_url: GUI URL (defaults to None)
 
         Returns:

backend/aim/metrics/m2/m2_jpeg_file_size.py

@@ -61,7 +61,7 @@
 # ----------------------------------------------------------------------------
 
 # Standard library modules
-from typing import List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 # Third-party modules
 from pydantic import HttpUrl
@@ -100,6 +100,7 @@ def execute_metric(
         cls,
         gui_image: str,
         gui_type: int = GUI_TYPE_DESKTOP,
+        gui_segments: Optional[Dict[str, Any]] = None,
         gui_url: Optional[HttpUrl] = None,
     ) -> Optional[List[Union[int, float, str]]]:
         """
@@ -110,6 +111,7 @@ def execute_metric(
 
         Kwargs:
             gui_type: GUI type, desktop = 0 (default), mobile = 1
+            gui_segments: GUI segments (defaults to None)
             gui_url: GUI URL (defaults to None)
 
         Returns: