Merge pull request #66 from bbtufty/compilations

Support for retool compilations

CHANGES.rst

@@ -4,12 +4,18 @@
 Features
 --------
 
+- Includes initial support for ``retool`` compilations
 - Added Game Boy Advance
 - ROMPatcher now supports RomPatcher.js
 
 Fixes
 -----
 
+ROMChooser
+~~~~~~~~~~
+
+- Fixed bug where versions weren't parsed correctly
+
 ROMCleaner
 ~~~~~~~~~~
 

docs/1g1r.rst

@@ -38,6 +38,18 @@ There are also some demotions that go on. The priority is (from most to least de
 * Alternate versions
 * Demoted versions (e.g. arcade versions)
 
+Compilations
+------------
+
+Compilation ROMs are handled somewhat differently. If a ROM is marked as part of a compilation via ``retool``, then
+these will be added to each game in the compilation. We then proceed as normal, and after filtering based on
+regions/languages, if there are still potential single-game ROMs then the compilation will be removed from that
+particular game. Otherwise, we will keep the compilations. Any compilations are then scored as above to choose
+a "best" version. This does potentially mean that these compilations can appear as the best choice for multiple games,
+but this is unlikely given the pretty stringent criteria for these compilations ROMs to be chosen.
+
+More options for handling compilations will be included in future releases.
+
 End result
 ----------
 

docs/known_issues.rst

@@ -11,8 +11,10 @@ Known Issues
   For example, having an include of "Crash Bandicoot" for the PS1 will also grab "Crash Bash" and
   "CTR - Crash Team Racing" since at least one of their duplicates starts with "Crash Bandicoot".
 
-* Currently, the code is not aware of ``retool``'s supersets or compilations array.
+* Currently, the code is not aware of ``retool``'s supersets array, and only handles compilations to a very limited
+  degree.
 
 * Occasionally, multiple ROMs will be found with the same priority.
 
-  * This will be improved with more regex flags in future releases
+* Dupes occasionally behave oddly, as clones in dats can be specific to a ROM and not an overall game name. This will be
+  improved in a future release

romsearch/configs/regex.yml

@@ -675,6 +675,10 @@ kaiser:
   pattern: "\\(Kaiser\\)"
   group: "demoted_version"
 
+ka_sheng:
+  pattern: "\\(Ka Sheng\\)"
+  group: "demoted_version"
+
 kickstarter:
   pattern: "\\(Kickstarter\\)"
   group: "demoted_version"
@@ -769,7 +773,7 @@ ntsc:
   flags: "NOFLAG"
 
 pal:
-  pattern: "([?:-][\\s])?[(]?PAL(?: [a-zA-Z]+| 50[Hh]z)?(?:\\)?| (?=\\())"
+  pattern: "([?:-][\\s])?[(]?PAL(?: [a-zA-Z]+| 50[Hh]z)?(?:(?!-)\\)?| (?=\\())"
   flags: "NOFLAG"
 
 supervision:

romsearch/modules/dupeparser.py

@@ -226,16 +226,19 @@ def get_dat_dupes(self, dupe_dict=None):
                     if parent_game_name == clone_short_name:
                         continue
 
-                    found_parent_name = get_parent_name(
+                    found_parent_names = get_parent_name(
                         game_name=parent_game_name,
                         dupe_dict=dupe_dict,
                     )
-                    if found_parent_name not in dupe_dict:
-                        dupe_dict[found_parent_name] = {}
+                    for found_parent_name in found_parent_names:
+                        if found_parent_name not in dupe_dict:
+                            dupe_dict[found_parent_name] = {}
 
-                    # Don't overwrite priority if it's already set
-                    if clone_short_name not in dupe_dict[found_parent_name]:
-                        dupe_dict[found_parent_name][clone_short_name] = {"priority": 1}
+                        # Don't overwrite priority if it's already set
+                        if clone_short_name not in dupe_dict[found_parent_name]:
+                            dupe_dict[found_parent_name][clone_short_name] = {
+                                "priority": 1
+                            }
 
         return dupe_dict
 
@@ -248,36 +251,56 @@ def get_retool_dupes(self, dupe_dict=None):
         retool_dupes = self.get_retool_dupe_dict()
         for retool_dupe in retool_dupes:
 
-            # If we don't have titles within the dupe dict, skip
-            if "titles" not in retool_dupe:
+            # If we don't have titles or compilations within the dupe dict, skip
+            if "titles" not in retool_dupe and "compilations" not in retool_dupe:
                 continue
 
+            # Get group and parent name
             group = retool_dupe["group"]
-            group_titles = [
-                get_short_name(
-                    f["searchTerm"],
-                    default_config=self.default_config,
-                    regex_config=self.regex_config,
-                )
-                for f in retool_dupe["titles"]
-            ]
-            priorities = [f.get("priority", 1) for f in retool_dupe["titles"]]
-
             group_parsed = get_short_name(
                 group,
                 default_config=self.default_config,
                 regex_config=self.regex_config,
             )
-
-            found_parent_name = get_parent_name(
+            found_parent_names = get_parent_name(
                 game_name=group_parsed,
                 dupe_dict=dupe_dict,
             )
-            if found_parent_name not in dupe_dict:
-                dupe_dict[found_parent_name] = {}
 
-            for i, g in enumerate(group_titles):
-                dupe_dict[found_parent_name][g] = {"priority": priorities[i]}
+            # Pull out individual titles
+            if "titles" in retool_dupe:
+
+                for found_parent_name in found_parent_names:
+
+                    if found_parent_name not in dupe_dict:
+                        dupe_dict[found_parent_name] = {}
+
+                    for title in retool_dupe["titles"]:
+                        title_g = title["searchTerm"]
+                        priority = title.get("priority", 1)
+
+                        dupe_dict[found_parent_name][title_g] = {
+                            "priority": priority,
+                        }
+
+            # Next, check for compilations. If we have them, pull them out and potentially the title position
+            if "compilations" in retool_dupe:
+
+                for found_parent_name in found_parent_names:
+
+                    if found_parent_name not in dupe_dict:
+                        dupe_dict[found_parent_name] = {}
+
+                    for compilation in retool_dupe["compilations"]:
+                        comp_g = compilation["searchTerm"]
+                        title_pos = compilation.get("titlePosition", None)
+                        priority = compilation.get("priority", 1)
+
+                        dupe_dict[found_parent_name][comp_g] = {
+                            "is_compilation": True,
+                            "priority": priority,
+                            "title_pos": title_pos,
+                        }
 
         return dupe_dict, retool_dupes
 

romsearch/modules/gamefinder.py

@@ -15,6 +15,8 @@
     load_json,
 )
 
+DUPE_DEFAULT = {"is_compilation": False, "priority": 1, "title_pos": None}
+
 
 def get_all_games(
     files,
@@ -35,25 +37,36 @@ def get_all_games(
     return games
 
 
-def get_priority(dupe_dict, parent_name, game_name):
-    """Get priority from a dupe dictionary"""
+def get_dupe_entry(
+    dupe_dict,
+    parent_name,
+    game_name,
+):
+    """Get dupe entry from a dupe dictionary
+
+    Args:
+        dupe_dict (dict): dupe dictionary
+        parent_name (str): parent game name
+        game_name (str): game name
+    """
 
     # First case: parent name doesn't exist in the dupe dict
     if parent_name not in dupe_dict:
-        return 1
+        return DUPE_DEFAULT
 
     # Second case: it does (potentially can be lowercase)
     dupes = [dupe.lower() for dupe in dupe_dict[parent_name]]
     reg_dupes = [dupe for dupe in dupe_dict[parent_name]]
 
     if game_name.lower() in dupes:
         found_parent_idx = dupes.index(game_name.lower())
-        priority = dupe_dict[parent_name][reg_dupes[found_parent_idx]]["priority"]
 
-        return priority
+        dupe_entry = dupe_dict[parent_name][reg_dupes[found_parent_idx]]
+
+        return dupe_entry
 
-    # Otherwise, just return 1
-    return 1
+    # Otherwise, return defaults
+    return DUPE_DEFAULT
 
 
 class GameFinder:
@@ -63,6 +76,7 @@ def __init__(
         platform,
         config_file=None,
         config=None,
+        dupe_dict=None,
         default_config=None,
         regex_config=None,
         logger=None,
@@ -78,6 +92,7 @@ def __init__(
             platform (str): Platform name
             config_file (str, optional): Path to config file. Defaults to None.
             config (dict, optional): Configuration dictionary. Defaults to None.
+            dupe_dict (dict, optional): Dupe dictionary. Defaults to None.
             default_config (dict, optional): Default configuration dictionary. Defaults to None.
             regex_config (dict, optional): Dictionary of regex config. Defaults to None.
             logger (logging.Logger, optional): Logger instance. Defaults to None.
@@ -126,6 +141,7 @@ def __init__(
         self.regex_config = regex_config
 
         # Info for dupes
+        self.dupe_dict = dupe_dict
         self.dupe_dir = config.get("dirs", {}).get("dupe_dir", None)
         self.filter_dupes = config.get("gamefinder", {}).get("filter_dupes", True)
 
@@ -289,45 +305,66 @@ def get_game_matches(
     def get_filter_dupes(self, games):
         """Parse down a list of files based on an input dupe list"""
 
-        if self.dupe_dir is None:
-            raise ValueError("dupe_dir must be specified if filtering dupes")
-
-        dupe_file = os.path.join(self.dupe_dir, f"{self.platform} (dupes).json")
-        if not os.path.exists(dupe_file):
-            self.logger.warning(f"{self.log_line_sep * self.log_line_length}")
-            self.logger.warning(
-                centred_string("No dupe files found", total_length=self.log_line_length)
+        if self.dupe_dict is None and self.dupe_dir is None:
+            raise ValueError(
+                "dupe_dict or dupe_dir must be specified if filtering dupes"
             )
-            self.logger.warning(f"{self.log_line_sep * self.log_line_length}")
-            return None
 
-        game_dict = {}
+        if self.dupe_dict is None:
+            dupe_file = os.path.join(self.dupe_dir, f"{self.platform} (dupes).json")
+            if not os.path.exists(dupe_file):
+                self.logger.warning(f"{self.log_line_sep * self.log_line_length}")
+                self.logger.warning(
+                    centred_string(
+                        "No dupe files found", total_length=self.log_line_length
+                    )
+                )
+                self.logger.warning(f"{self.log_line_sep * self.log_line_length}")
+                return None
+            self.dupe_dict = load_json(dupe_file)
 
-        dupes = load_json(dupe_file)
+        game_dict = {}
 
-        # Loop over games, and the dupes dictionary. Also pull out priority
+        # Loop over games, and the dupes dictionary. Also pull out various other important info
         for g in games:
 
-            found_parent_name = get_parent_name(
+            # Because we have compilations, these can be lists
+            found_parent_names = get_parent_name(
                 game_name=g,
-                dupe_dict=dupes,
+                dupe_dict=self.dupe_dict,
             )
 
-            found_parent_name_lower = found_parent_name.lower()
-            game_dict_keys = [key for key in game_dict.keys()]
-            game_dict_keys_lower = [key.lower() for key in game_dict.keys()]
+            for found_parent_name in found_parent_names:
 
-            if found_parent_name_lower not in game_dict_keys_lower:
-                game_dict[found_parent_name] = {}
-                final_parent_name = copy.deepcopy(found_parent_name)
-            else:
-                final_parent_idx = game_dict_keys_lower.index(found_parent_name_lower)
-                final_parent_name = game_dict_keys[final_parent_idx]
+                found_parent_name_lower = found_parent_name.lower()
+                game_dict_keys = [key for key in game_dict.keys()]
+                game_dict_keys_lower = [key.lower() for key in game_dict.keys()]
 
-            priority = get_priority(
-                dupe_dict=dupes, parent_name=found_parent_name, game_name=g
-            )
+                if found_parent_name_lower not in game_dict_keys_lower:
+                    game_dict[found_parent_name] = {}
+                    final_parent_name = copy.deepcopy(found_parent_name)
+                else:
+                    final_parent_idx = game_dict_keys_lower.index(
+                        found_parent_name_lower
+                    )
+                    final_parent_name = game_dict_keys[final_parent_idx]
+
+                dupe_entry = get_dupe_entry(
+                    dupe_dict=self.dupe_dict,
+                    parent_name=found_parent_name,
+                    game_name=g,
+                )
+
+                # We want to make sure we also don't duplicate on the names being upper/lowercase
+                g_names = [g_dict for g_dict in game_dict[final_parent_name]]
+                g_names_lower = [g_name.lower() for g_name in g_names]
+                if g.lower() in g_names_lower:
+                    g_idx = g_names_lower.index(g.lower())
+                    g = g_names[g_idx]
+
+                if g not in game_dict[final_parent_name]:
+                    game_dict[final_parent_name][g] = {}
 
-            game_dict[final_parent_name][g] = {"priority": priority}
+                game_dict[final_parent_name][g].update(dupe_entry)
 
         return game_dict