Support for retool compilations

- Support for retool compilations
- Simplify dictionary inheritance

CHANGES.rst

@@ -4,12 +4,18 @@
 Features
 --------
 
+- Includes initial support for ``retool`` compilations
 - Added Game Boy Advance
 - ROMPatcher now supports RomPatcher.js
 
 Fixes
 -----
 
+ROMChooser
+~~~~~~~~~~
+
+- Fixed bug where versions weren't parsed correctly
+
 ROMCleaner
 ~~~~~~~~~~
 

docs/1g1r.rst

@@ -38,6 +38,9 @@ There are also some demotions that go on. The priority is (from most to least de
 * Alternate versions
 * Demoted versions (e.g. arcade versions)
 
+Because some ROMs are compilations, if we end up with two equal scores given everything above, the single game will
+be preferred over the compilation.
+
 End result
 ----------
 

romsearch/modules/dupeparser.py

@@ -279,6 +279,19 @@ def get_retool_dupes(self, dupe_dict=None):
             for i, g in enumerate(group_titles):
                 dupe_dict[found_parent_name][g] = {"priority": priorities[i]}
 
+            # Next, check for compilations. If we have them, pull them out and optionally the title position
+            if "compilations" in retool_dupe:
+                for compilation in retool_dupe["compilations"]:
+                    comp_g = compilation["searchTerm"]
+                    title_pos = compilation.get("titlePosition", None)
+                    priority = compilation.get("priority", 1)
+
+                    dupe_dict[found_parent_name][comp_g] = {
+                        "is_compilation": True,
+                        "priority": priority,
+                        "title_pos": title_pos,
+                    }
+
         return dupe_dict, retool_dupes
 
     def download_retool_dupe(

romsearch/modules/gamefinder.py

@@ -15,6 +15,12 @@
     load_json,
 )
 
+DUPE_DEFAULT = {
+    "is_compilation": False,
+    "priority": 1,
+    "title_pos": None
+}
+
 
 def get_all_games(
     files,
@@ -35,25 +41,36 @@ def get_all_games(
     return games
 
 
-def get_priority(dupe_dict, parent_name, game_name):
-    """Get priority from a dupe dictionary"""
+def get_dupe_entry(
+    dupe_dict,
+    parent_name,
+    game_name,
+):
+    """Get dupe entry from a dupe dictionary
+
+    Args:
+        dupe_dict (dict): dupe dictionary
+        parent_name (str): parent game name
+        game_name (str): game name
+    """
 
     # First case: parent name doesn't exist in the dupe dict
     if parent_name not in dupe_dict:
-        return 1
+        return DUPE_DEFAULT
 
     # Second case: it does (potentially can be lowercase)
     dupes = [dupe.lower() for dupe in dupe_dict[parent_name]]
     reg_dupes = [dupe for dupe in dupe_dict[parent_name]]
 
     if game_name.lower() in dupes:
         found_parent_idx = dupes.index(game_name.lower())
-        priority = dupe_dict[parent_name][reg_dupes[found_parent_idx]]["priority"]
 
-        return priority
+        dupe_entry = dupe_dict[parent_name][reg_dupes[found_parent_idx]]
 
-    # Otherwise, just return 1
-    return 1
+        return dupe_entry
+
+    # Otherwise, return defaults
+    return DUPE_DEFAULT
 
 
 class GameFinder:
@@ -63,6 +80,7 @@ def __init__(
         platform,
         config_file=None,
         config=None,
+        dupe_dict=None,
         default_config=None,
         regex_config=None,
         logger=None,
@@ -78,6 +96,7 @@ def __init__(
             platform (str): Platform name
             config_file (str, optional): Path to config file. Defaults to None.
             config (dict, optional): Configuration dictionary. Defaults to None.
+            dupe_dict (dict, optional): Dupe dictionary. Defaults to None.
             default_config (dict, optional): Default configuration dictionary. Defaults to None.
             regex_config (dict, optional): Dictionary of regex config. Defaults to None.
             logger (logging.Logger, optional): Logger instance. Defaults to None.
@@ -126,6 +145,7 @@ def __init__(
         self.regex_config = regex_config
 
         # Info for dupes
+        self.dupe_dict = dupe_dict
         self.dupe_dir = config.get("dirs", {}).get("dupe_dir", None)
         self.filter_dupes = config.get("gamefinder", {}).get("filter_dupes", True)
 
@@ -289,28 +309,32 @@ def get_game_matches(
     def get_filter_dupes(self, games):
         """Parse down a list of files based on an input dupe list"""
 
-        if self.dupe_dir is None:
-            raise ValueError("dupe_dir must be specified if filtering dupes")
-
-        dupe_file = os.path.join(self.dupe_dir, f"{self.platform} (dupes).json")
-        if not os.path.exists(dupe_file):
-            self.logger.warning(f"{self.log_line_sep * self.log_line_length}")
-            self.logger.warning(
-                centred_string("No dupe files found", total_length=self.log_line_length)
+        if self.dupe_dict is None and self.dupe_dir is None:
+            raise ValueError(
+                "dupe_dict or dupe_dir must be specified if filtering dupes"
             )
-            self.logger.warning(f"{self.log_line_sep * self.log_line_length}")
-            return None
 
-        game_dict = {}
+        if self.dupe_dict is None:
+            dupe_file = os.path.join(self.dupe_dir, f"{self.platform} (dupes).json")
+            if not os.path.exists(dupe_file):
+                self.logger.warning(f"{self.log_line_sep * self.log_line_length}")
+                self.logger.warning(
+                    centred_string(
+                        "No dupe files found", total_length=self.log_line_length
+                    )
+                )
+                self.logger.warning(f"{self.log_line_sep * self.log_line_length}")
+                return None
+            self.dupe_dict = load_json(dupe_file)
 
-        dupes = load_json(dupe_file)
+        game_dict = {}
 
-        # Loop over games, and the dupes dictionary. Also pull out priority
+        # Loop over games, and the dupes dictionary. Also pull out various other important info
         for g in games:
 
             found_parent_name = get_parent_name(
                 game_name=g,
-                dupe_dict=dupes,
+                dupe_dict=self.dupe_dict,
             )
 
             found_parent_name_lower = found_parent_name.lower()
@@ -324,10 +348,15 @@ def get_filter_dupes(self, games):
                 final_parent_idx = game_dict_keys_lower.index(found_parent_name_lower)
                 final_parent_name = game_dict_keys[final_parent_idx]
 
-            priority = get_priority(
-                dupe_dict=dupes, parent_name=found_parent_name, game_name=g
+            dupe_entry = get_dupe_entry(
+                dupe_dict=self.dupe_dict,
+                parent_name=found_parent_name,
+                game_name=g,
             )
 
-            game_dict[final_parent_name][g] = {"priority": priority}
+            if g not in game_dict[final_parent_name]:
+                game_dict[final_parent_name][g] = {}
+
+            game_dict[final_parent_name][g].update(dupe_entry)
 
         return game_dict

romsearch/modules/romchooser.py

@@ -131,7 +131,7 @@ def add_versioned_score(files, rom_dict, key):
         rom_dict[f][key] = get_sanitized_version(rom_dict[f][key])
 
     versions = [version.parse(rom_dict[f][key]) for f in files]
-    versions_sorted = sorted(versions)
+    versions_sorted = np.unique(sorted(versions))
 
     file_scores_version = np.zeros(len(files))
     for i, v in enumerate(versions_sorted):
@@ -537,19 +537,20 @@ def get_best_roms(
         """Get the best ROM(s) from a list, using a scoring system"""
 
         # Positive scores
-        improved_version_score = 1
-        version_score = 1e2
-        revision_score = 1e4
-        budget_edition_score = 1e6
-        language_score = 1e8
-        region_score = 1e10
-        cheevo_score = 1e12
+        improved_version_score = 1e2
+        version_score = 1e4
+        revision_score = 1e6
+        budget_edition_score = 1e8
+        language_score = 1e10
+        region_score = 1e12
+        cheevo_score = 1e14
 
         # Negative scores
-        demoted_version_score = -1
-        alternate_version_score = -1
-        modern_version_score = -1e2
-        priority_score = -1e4
+        compilation_score = -1
+        demoted_version_score = -1e2
+        alternate_version_score = -1e2
+        modern_version_score = -1e4
+        priority_score = -1e6
 
         file_scores = np.zeros(len(files))
 
@@ -597,6 +598,11 @@ def get_best_roms(
 
         # Negative scores
 
+        # Compilation score
+        file_scores += compilation_score * np.array(
+            [rom_dict[f].get("is_compilation", False) for f in files]
+        )
+
         # Demoted version
         file_scores += demoted_version_score * np.array(
             [int(rom_dict[f]["demoted_version"]) for f in files]
@@ -612,7 +618,7 @@ def get_best_roms(
             [int(rom_dict[f]["modern_version"]) for f in files]
         )
 
-        # Priority scoring. We subtract 1 so that the highest priority has no changed
+        # Priority scoring. We subtract 1 so that the highest priority has no change
         file_scores += priority_score * (
             np.array([int(rom_dict[f]["priority"]) for f in files]) - 1
         )

romsearch/modules/romcleaner.py

@@ -9,7 +9,8 @@
     load_yml,
     setup_logger,
     load_json,
-    save_json, )
+    save_json,
+)
 
 
 class ROMCleaner:
@@ -205,7 +206,8 @@ def clean_roms(
                                 os.remove(rom_on_disk)
                                 self.logger.info(
                                     centred_string(
-                                        f"Removed {rom_short} from disk", total_length=self.log_line_length
+                                        f"Removed {rom_short} from disk",
+                                        total_length=self.log_line_length,
                                     )
                                 )
 
@@ -251,7 +253,8 @@ def clean_roms(
 
                     self.logger.info(
                         centred_string(
-                            f"Removed {d_i_to_remove} from cache", total_length=self.log_line_length
+                            f"Removed {d_i_to_remove} from cache",
+                            total_length=self.log_line_length,
                         )
                     )