Support for retool compilations

- Support for retool compilations
- Simplify dictionary inheritance
- Fix version parsing in ROMChooser

CHANGES.rst

@@ -4,12 +4,18 @@
 Features
 --------
 
+- Includes initial support for ``retool`` compilations
 - Added Game Boy Advance
 - ROMPatcher now supports RomPatcher.js
 
 Fixes
 -----
 
+ROMChooser
+~~~~~~~~~~
+
+- Fixed bug where versions weren't parsed correctly
+
 ROMCleaner
 ~~~~~~~~~~
 

docs/1g1r.rst

@@ -38,6 +38,18 @@ There are also some demotions that go on. The priority is (from most to least de
 * Alternate versions
 * Demoted versions (e.g. arcade versions)
 
+Compilations
+------------
+
+Compilation ROMs are handled somewhat differently. If a ROM is marked as part of a compilation via ``retool``, then
+these will be added to each game in the compilation. We then proceed as normal, and after filtering based on
+regions/languages, if there are still potential single-game ROMs then the compilation will be removed from that
+particular game. Otherwise, we will keep the compilations. Any compilations are then scored as above to choose
+a "best" version. This does potentially mean that these compilations can appear as the best choice for multiple games,
+but this is unlikely given the pretty stringent criteria for these compilations ROMs to be chosen.
+
+More options for handling compilations will be included in future releases.
+
 End result
 ----------
 

romsearch/configs/regex.yml

@@ -675,6 +675,10 @@ kaiser:
   pattern: "\\(Kaiser\\)"
   group: "demoted_version"
 
+ka_sheng:
+  pattern: "\\(Ka Sheng\\)"
+  group: "demoted_version"
+
 kickstarter:
   pattern: "\\(Kickstarter\\)"
   group: "demoted_version"

romsearch/modules/dupeparser.py

@@ -226,16 +226,19 @@ def get_dat_dupes(self, dupe_dict=None):
                     if parent_game_name == clone_short_name:
                         continue
 
-                    found_parent_name = get_parent_name(
+                    found_parent_names = get_parent_name(
                         game_name=parent_game_name,
                         dupe_dict=dupe_dict,
                     )
-                    if found_parent_name not in dupe_dict:
-                        dupe_dict[found_parent_name] = {}
+                    for found_parent_name in found_parent_names:
+                        if found_parent_name not in dupe_dict:
+                            dupe_dict[found_parent_name] = {}
 
-                    # Don't overwrite priority if it's already set
-                    if clone_short_name not in dupe_dict[found_parent_name]:
-                        dupe_dict[found_parent_name][clone_short_name] = {"priority": 1}
+                        # Don't overwrite priority if it's already set
+                        if clone_short_name not in dupe_dict[found_parent_name]:
+                            dupe_dict[found_parent_name][clone_short_name] = {
+                                "priority": 1
+                            }
 
         return dupe_dict
 
@@ -269,15 +272,30 @@ def get_retool_dupes(self, dupe_dict=None):
                 regex_config=self.regex_config,
             )
 
-            found_parent_name = get_parent_name(
+            found_parent_names = get_parent_name(
                 game_name=group_parsed,
                 dupe_dict=dupe_dict,
             )
-            if found_parent_name not in dupe_dict:
-                dupe_dict[found_parent_name] = {}
 
-            for i, g in enumerate(group_titles):
-                dupe_dict[found_parent_name][g] = {"priority": priorities[i]}
+            for found_parent_name in found_parent_names:
+                if found_parent_name not in dupe_dict:
+                    dupe_dict[found_parent_name] = {}
+
+                for i, g in enumerate(group_titles):
+                    dupe_dict[found_parent_name][g] = {"priority": priorities[i]}
+
+                # Next, check for compilations. If we have them, pull them out and optionally the title position
+                if "compilations" in retool_dupe:
+                    for compilation in retool_dupe["compilations"]:
+                        comp_g = compilation["searchTerm"]
+                        title_pos = compilation.get("titlePosition", None)
+                        priority = compilation.get("priority", 1)
+
+                        dupe_dict[found_parent_name][comp_g] = {
+                            "is_compilation": True,
+                            "priority": priority,
+                            "title_pos": title_pos,
+                        }
 
         return dupe_dict, retool_dupes
 

romsearch/modules/gamefinder.py

@@ -15,6 +15,8 @@
     load_json,
 )
 
+DUPE_DEFAULT = {"is_compilation": False, "priority": 1, "title_pos": None}
+
 
 def get_all_games(
     files,
@@ -35,25 +37,36 @@ def get_all_games(
     return games
 
 
-def get_priority(dupe_dict, parent_name, game_name):
-    """Get priority from a dupe dictionary"""
+def get_dupe_entry(
+    dupe_dict,
+    parent_name,
+    game_name,
+):
+    """Get dupe entry from a dupe dictionary
+
+    Args:
+        dupe_dict (dict): dupe dictionary
+        parent_name (str): parent game name
+        game_name (str): game name
+    """
 
     # First case: parent name doesn't exist in the dupe dict
     if parent_name not in dupe_dict:
-        return 1
+        return DUPE_DEFAULT
 
     # Second case: it does (potentially can be lowercase)
     dupes = [dupe.lower() for dupe in dupe_dict[parent_name]]
     reg_dupes = [dupe for dupe in dupe_dict[parent_name]]
 
     if game_name.lower() in dupes:
         found_parent_idx = dupes.index(game_name.lower())
-        priority = dupe_dict[parent_name][reg_dupes[found_parent_idx]]["priority"]
 
-        return priority
+        dupe_entry = dupe_dict[parent_name][reg_dupes[found_parent_idx]]
+
+        return dupe_entry
 
-    # Otherwise, just return 1
-    return 1
+    # Otherwise, return defaults
+    return DUPE_DEFAULT
 
 
 class GameFinder:
@@ -63,6 +76,7 @@ def __init__(
         platform,
         config_file=None,
         config=None,
+        dupe_dict=None,
         default_config=None,
         regex_config=None,
         logger=None,
@@ -78,6 +92,7 @@ def __init__(
             platform (str): Platform name
             config_file (str, optional): Path to config file. Defaults to None.
             config (dict, optional): Configuration dictionary. Defaults to None.
+            dupe_dict (dict, optional): Dupe dictionary. Defaults to None.
             default_config (dict, optional): Default configuration dictionary. Defaults to None.
             regex_config (dict, optional): Dictionary of regex config. Defaults to None.
             logger (logging.Logger, optional): Logger instance. Defaults to None.
@@ -126,6 +141,7 @@ def __init__(
         self.regex_config = regex_config
 
         # Info for dupes
+        self.dupe_dict = dupe_dict
         self.dupe_dir = config.get("dirs", {}).get("dupe_dir", None)
         self.filter_dupes = config.get("gamefinder", {}).get("filter_dupes", True)
 
@@ -289,45 +305,66 @@ def get_game_matches(
     def get_filter_dupes(self, games):
         """Parse down a list of files based on an input dupe list"""
 
-        if self.dupe_dir is None:
-            raise ValueError("dupe_dir must be specified if filtering dupes")
-
-        dupe_file = os.path.join(self.dupe_dir, f"{self.platform} (dupes).json")
-        if not os.path.exists(dupe_file):
-            self.logger.warning(f"{self.log_line_sep * self.log_line_length}")
-            self.logger.warning(
-                centred_string("No dupe files found", total_length=self.log_line_length)
+        if self.dupe_dict is None and self.dupe_dir is None:
+            raise ValueError(
+                "dupe_dict or dupe_dir must be specified if filtering dupes"
             )
-            self.logger.warning(f"{self.log_line_sep * self.log_line_length}")
-            return None
 
-        game_dict = {}
+        if self.dupe_dict is None:
+            dupe_file = os.path.join(self.dupe_dir, f"{self.platform} (dupes).json")
+            if not os.path.exists(dupe_file):
+                self.logger.warning(f"{self.log_line_sep * self.log_line_length}")
+                self.logger.warning(
+                    centred_string(
+                        "No dupe files found", total_length=self.log_line_length
+                    )
+                )
+                self.logger.warning(f"{self.log_line_sep * self.log_line_length}")
+                return None
+            self.dupe_dict = load_json(dupe_file)
 
-        dupes = load_json(dupe_file)
+        game_dict = {}
 
-        # Loop over games, and the dupes dictionary. Also pull out priority
+        # Loop over games, and the dupes dictionary. Also pull out various other important info
         for g in games:
 
-            found_parent_name = get_parent_name(
+            # Because we have compilations, these can be lists
+            found_parent_names = get_parent_name(
                 game_name=g,
-                dupe_dict=dupes,
+                dupe_dict=self.dupe_dict,
             )
 
-            found_parent_name_lower = found_parent_name.lower()
-            game_dict_keys = [key for key in game_dict.keys()]
-            game_dict_keys_lower = [key.lower() for key in game_dict.keys()]
+            for found_parent_name in found_parent_names:
 
-            if found_parent_name_lower not in game_dict_keys_lower:
-                game_dict[found_parent_name] = {}
-                final_parent_name = copy.deepcopy(found_parent_name)
-            else:
-                final_parent_idx = game_dict_keys_lower.index(found_parent_name_lower)
-                final_parent_name = game_dict_keys[final_parent_idx]
+                found_parent_name_lower = found_parent_name.lower()
+                game_dict_keys = [key for key in game_dict.keys()]
+                game_dict_keys_lower = [key.lower() for key in game_dict.keys()]
 
-            priority = get_priority(
-                dupe_dict=dupes, parent_name=found_parent_name, game_name=g
-            )
+                if found_parent_name_lower not in game_dict_keys_lower:
+                    game_dict[found_parent_name] = {}
+                    final_parent_name = copy.deepcopy(found_parent_name)
+                else:
+                    final_parent_idx = game_dict_keys_lower.index(
+                        found_parent_name_lower
+                    )
+                    final_parent_name = game_dict_keys[final_parent_idx]
+
+                dupe_entry = get_dupe_entry(
+                    dupe_dict=self.dupe_dict,
+                    parent_name=found_parent_name,
+                    game_name=g,
+                )
+
+                # We want to make sure we also don't duplicate on the names being upper/lowercase
+                g_names = [g_dict for g_dict in game_dict[final_parent_name]]
+                g_names_lower = [g_name.lower() for g_name in g_names]
+                if g.lower() in g_names_lower:
+                    g_idx = g_names_lower.index(g.lower())
+                    g = g_names[g_idx]
+
+                if g not in game_dict[final_parent_name]:
+                    game_dict[final_parent_name][g] = {}
 
-            game_dict[final_parent_name][g] = {"priority": priority}
+                game_dict[final_parent_name][g].update(dupe_entry)
 
         return game_dict

romsearch/modules/romchooser.py

@@ -131,7 +131,7 @@ def add_versioned_score(files, rom_dict, key):
         rom_dict[f][key] = get_sanitized_version(rom_dict[f][key])
 
     versions = [version.parse(rom_dict[f][key]) for f in files]
-    versions_sorted = sorted(versions)
+    versions_sorted = np.unique(sorted(versions))
 
     file_scores_version = np.zeros(len(files))
     for i, v in enumerate(versions_sorted):
@@ -203,6 +203,34 @@ def filter_by_list(
     return rom_dict
 
 
+def filter_compilations(
+    rom_dict,
+):
+    """Filter out compilations if we have them
+
+    Args:
+        rom_dict (dict): Dictionary of ROMs
+    """
+
+    compilations = []
+    non_compilations = []
+
+    for r in rom_dict:
+        if not rom_dict[r]["excluded"]:
+            if rom_dict[r].get("is_compilation", False):
+                compilations.append(r)
+            else:
+                non_compilations.append(r)
+
+    # Only remove compilations if we have singular titles
+    if len(non_compilations) > 0:
+        for comp in compilations:
+            rom_dict[comp]["excluded"] = True
+            rom_dict[comp]["excluded_reason"].append("is_compilation")
+
+    return rom_dict
+
+
 class ROMChooser:
 
     def __init__(
@@ -434,6 +462,16 @@ def run_chooser(self, rom_dict):
                 list_preferences=self.region_preferences,
             )
 
+        # If we have a split between singular titles and compilations, sort that out here
+        self.logger.debug(
+            left_aligned_string(
+                f"Potentially filtering compilations", total_length=self.log_line_length
+            )
+        )
+        rom_dict = filter_compilations(
+            rom_dict,
+        )
+
         # Remove versions we potentially don't want around
         if self.use_best_version:
             self.logger.debug(
@@ -537,19 +575,20 @@ def get_best_roms(
         """Get the best ROM(s) from a list, using a scoring system"""
 
         # Positive scores
-        improved_version_score = 1
-        version_score = 1e2
-        revision_score = 1e4
-        budget_edition_score = 1e6
-        language_score = 1e8
-        region_score = 1e10
-        cheevo_score = 1e12
+        improved_version_score = 1e2
+        version_score = 1e4
+        revision_score = 1e6
+        budget_edition_score = 1e8
+        language_score = 1e10
+        region_score = 1e12
+        cheevo_score = 1e14
 
         # Negative scores
-        demoted_version_score = -1
-        alternate_version_score = -1
-        modern_version_score = -1e2
-        priority_score = -1e4
+        compilation_score = -1
+        demoted_version_score = -1e2
+        alternate_version_score = -1e2
+        modern_version_score = -1e4
+        priority_score = -1e6
 
         file_scores = np.zeros(len(files))
 
@@ -597,6 +636,11 @@ def get_best_roms(
 
         # Negative scores
 
+        # Compilation score
+        file_scores += compilation_score * np.array(
+            [rom_dict[f].get("is_compilation", False) for f in files]
+        )
+
         # Demoted version
         file_scores += demoted_version_score * np.array(
             [int(rom_dict[f]["demoted_version"]) for f in files]
@@ -612,7 +656,7 @@ def get_best_roms(
             [int(rom_dict[f]["modern_version"]) for f in files]
         )
 
-        # Priority scoring. We subtract 1 so that the highest priority has no changed
+        # Priority scoring. We subtract 1 so that the highest priority has no change
         file_scores += priority_score * (
             np.array([int(rom_dict[f]["priority"]) for f in files]) - 1
         )