From adb718926d2c6725533e8f3522a3f9b99d39e031 Mon Sep 17 00:00:00 2001 From: bbtufty Date: Tue, 7 May 2024 15:44:38 +0100 Subject: [PATCH] v0.0.2 Features -------- - Added Nintendo - Nintendo Entertainment System - Added tools to parse filenames or full games list out of parsed .dat files, to check for new regex terms to add Fixes ----- DupeParser ~~~~~~~~~~ - Fixed crash if "searchTerm" does not exist in the retool dupe dict - Get dupes from retool first, before dat file GameFinder ~~~~~~~~~~ - Fixed bug where if include_games was defined but not for the platform, nothing would be found - The full list of games is now sorted - Fixed bug where occasionally multiple entries due to upper/lowercase could occur ROMChooser ~~~~~~~~~~ - Revisions are now weighted more heavily than versions - Budget editions are now favoured above anything else, assuming they roll in the various revision/version changes ROMParser ~~~~~~~~~ - Fixed crash if "searchTerm" does not exist in the retool dupe dict Configs ~~~~~~~ - Added specific regex options for NES - Regions now has options for multiple rendering (e.g. UK can be UK or United Kingdom) - Grouped ``rerelease`` with ``demoted_versions`` in regex - Decoupled revisions from versions --- CHANGES.rst | 41 +++- romsearch/configs/clonelists/retool.yml | 1 + romsearch/configs/dats/no-intro.yml | 3 + romsearch/configs/defaults.yml | 119 +++++------ ...ntendo - Nintendo Entertainment System.yml | 3 + romsearch/configs/regex.yml | 184 +++++++++++++++++- romsearch/dev/__init__.py | 6 + romsearch/dev/parsing_tools.py | 71 +++++++ romsearch/modules/dupeparser.py | 12 +- romsearch/modules/gamefinder.py | 15 +- romsearch/modules/romparser.py | 4 + 11 files changed, 387 insertions(+), 72 deletions(-) create mode 100644 romsearch/configs/platforms/Nintendo - Nintendo Entertainment System.yml create mode 100644 romsearch/dev/__init__.py create mode 100644 romsearch/dev/parsing_tools.py diff --git a/CHANGES.rst b/CHANGES.rst index 82fa859..573fbf1 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,11 +1,46 @@ 0.0.2 (Unreleased) ================== -- Grouped ``rerelease`` with ``demoted_versions`` in regex -- Decoupled revisions from versions +Features +-------- + +- Added Nintendo - Nintendo Entertainment System +- Added tools to parse filenames or full games list out of parsed .dat files, to check for new regex terms to add + +Fixes +----- + +DupeParser +~~~~~~~~~~ + +- Fixed crash if "searchTerm" does not exist in the retool dupe dict +- Get dupes from retool first, before dat file + +GameFinder +~~~~~~~~~~ + +- Fixed bug where if include_games was defined but not for the platform, nothing would be found +- The full list of games is now sorted +- Fixed bug where occasionally multiple entries due to upper/lowercase could occur + +ROMChooser +~~~~~~~~~~ + - Revisions are now weighted more heavily than versions - Budget editions are now favoured above anything else, assuming they roll in the various revision/version changes -- Fixed bug in GameFinder where occasionally multiple entries due to upper/lowercase could occur + +ROMParser +~~~~~~~~~ + +- Fixed crash if "searchTerm" does not exist in the retool dupe dict + +Configs +~~~~~~~ + +- Added specific regex options for NES +- Regions now has options for multiple rendering (e.g. UK can be UK or United Kingdom) +- Grouped ``rerelease`` with ``demoted_versions`` in regex +- Decoupled revisions from versions 0.0.1 (2024-05-06) ================== diff --git a/romsearch/configs/clonelists/retool.yml b/romsearch/configs/clonelists/retool.yml index c60dead..14cd9f8 100644 --- a/romsearch/configs/clonelists/retool.yml +++ b/romsearch/configs/clonelists/retool.yml @@ -1,6 +1,7 @@ url: "https://raw.githubusercontent.com/unexpectedpanda/retool-clonelists-metadata/main/clonelists" Nintendo - GameCube: "Nintendo - GameCube (Redump).json" +Nintendo - Nintendo Entertainment System: "Nintendo - Nintendo Entertainment System (No-Intro).json" Nintendo - Super Nintendo Entertainment System: "Nintendo - Super Nintendo Entertainment System (No-Intro).json" Sony - PlayStation: "Sony - PlayStation (Redump).json" Sony - PlayStation 2: "Sony - PlayStation 2 (Redump).json" diff --git a/romsearch/configs/dats/no-intro.yml b/romsearch/configs/dats/no-intro.yml index 17ef8bc..f1d93ea 100644 --- a/romsearch/configs/dats/no-intro.yml +++ b/romsearch/configs/dats/no-intro.yml @@ -1,2 +1,5 @@ +Nintendo - Nintendo Entertainment System: + file_mapping: "Nintendo - Nintendo Entertainment System (Headered)" + Nintendo - Super Nintendo Entertainment System: file_mapping: "Nintendo - Super Nintendo Entertainment System" \ No newline at end of file diff --git a/romsearch/configs/defaults.yml b/romsearch/configs/defaults.yml index ef599fc..1e381a9 100644 --- a/romsearch/configs/defaults.yml +++ b/romsearch/configs/defaults.yml @@ -5,6 +5,7 @@ datetime_format: "%Y/%m/%d, %H:%M:%S" platforms: - "Nintendo - GameCube" + - "Nintendo - Nintendo Entertainment System" - "Nintendo - Super Nintendo Entertainment System" - "Sony - PlayStation" - "Sony - PlayStation 2" @@ -17,65 +18,65 @@ video_types: - "SECAM" regions: - - "USA" - - "World" - - "Canada" - - "Europe" - - "UK" - - "Australia" - - "New Zealand" - - "Singapore" - - "Ireland" - - "Japan" - - "Asia" - - "Thailand" - - "Spain" - - "Mexico" - - "Argentina" - - "Latin America" - - "Brazil" - - "Portugal" - - "France" - - "Belgium" - - "Netherlands" - - "Germany" - - "Austria" - - "Italy" - - "Switzerland" - - "Hong Kong" - - "China" - - "Taiwan" - - "Korea" - - "Russia" - - "Ukraine" - - "Estonia" - - "Poland" - - "Latvia" - - "Lithuania" - - "Denmark" - - "Norway" - - "Sweden" - - "Scandinavia" - - "Finland" - - "Iceland" - - "Hungary" - - "Czech" - - "Greece" - - "Macedonia" - - "India" - - "South Africa" - - "Israel" - - "Slovakia" - - "Turkey" - - "Croatia" - - "Slovenia" - - "United Arab Emirates" - - "Bulgaria" - - "Romania" - - "Albania" - - "Serbia" - - "Indonesia" - - "Unknown" + USA: "USA" + World: "World" + Canada: "Canada" + Europe: "Europe" + UK: "UK|United Kingdom" + Australia: "Australia" + New Zealand: "New Zealand" + Singapore: "Singapore" + Ireland: "Ireland" + Japan: "Japan" + Asia: "Asia" + Thailand: "Thailand" + Spain: "Spain" + Mexico: "Mexico" + Argentina: "Argentina" + Latin America: "Latin America" + Brazil: "Brazil" + Portugal: "Portugal" + France: "France" + Belgium: "Belgium" + Netherlands: "Netherlands" + Germany: "Germany" + Austria: "Austria" + Italy: "Italy" + Switzerland: "Switzerland" + Hong Kong: "Hong Kong" + China: "China" + Taiwan: "Taiwan" + Korea: "Korea" + Russia: "Russia" + Ukraine: "Ukraine" + Estonia: "Estonia" + Poland: "Poland" + Latvia: "Latvia" + Lithuania: "Lithuania" + Denmark: "Denmark" + Norway: "Norway" + Sweden: "Sweden" + Scandinavia: "Scandinavia" + Finland: "Finland" + Iceland: "Iceland" + Hungary: "Hungary" + Czech: "Czech" + Greece: "Greece" + Macedonia: "Macedonia" + India: "India" + South Africa: "South Africa" + Israel: "Israel" + Slovakia: "Slovakia" + Turkey: "Turkey" + Croatia: "Croatia" + Slovenia: "Slovenia" + United Arab Emirates: "United Arab Emirates" + Bulgaria: "Bulgaria" + Romania: "Romania" + Albania: "Albania" + Serbia: "Serbia" + Indonesia: "Indonesia" + Unknown: "Unknown" languages: English: "En(?:-[A-Z][A-Z])?" diff --git a/romsearch/configs/platforms/Nintendo - Nintendo Entertainment System.yml b/romsearch/configs/platforms/Nintendo - Nintendo Entertainment System.yml new file mode 100644 index 0000000..2430d25 --- /dev/null +++ b/romsearch/configs/platforms/Nintendo - Nintendo Entertainment System.yml @@ -0,0 +1,3 @@ +group: "No-Intro" +ftp_dir: "/No-Intro/Nintendo - Nintendo Entertainment System (Headered)" +unzip: false diff --git a/romsearch/configs/regex.yml b/romsearch/configs/regex.yml index de14fee..37901dd 100644 --- a/romsearch/configs/regex.yml +++ b/romsearch/configs/regex.yml @@ -57,6 +57,11 @@ demo: pattern: "\\((?:\\w[-.]?\\s*)*Demo(?:(?:,?\\s|-)[\\w0-9\\.]*)*\\)" group: "demos" +kiosk: + pattern: "\\((?:\\w-?\\s*)*?Kiosk,?(?:\\s\\w*?)*\\)|Kiosk Demo Disc|(PSP System|PS2) Kiosk" + group: "demos" + search_tags: false + preview: pattern: "\\(Preview\\)" group: "demos" @@ -152,6 +157,14 @@ unl: group: "unlicensed" # IMPROVED VERSIONS +c_and_e: + pattern: "\\(C&E\\)" + group: "improved_version" + +cart_present: + pattern: "\\(Cart Present\\)" + group: "improved_version" + controller_set: pattern: "\\(Controller Set\\)" group: "improved_version" @@ -171,10 +184,33 @@ enhanced: pattern: "\\(Enhanced\\)" group: "improved_version" +e_reader_edition: + pattern: "\\(e-Reader Edition\\)" + gentei: pattern: "\\((?:(?!\\(|Gentei.*?)[\\s\\S])*Gentei.*?\\)" group: "improved_version" +ideatek: + pattern: "\\(Idea-Tek\\)" + group: "improved_version" + +later: + pattern: "\\(Later\\)" + group: "improved_version" + +mega_soft: + pattern: "\\(Mega Soft\\)" + group: "improved_version" + +nina_03: + pattern: "\\(NINA-03\\)" + group: "improved_version" + +nina_06: + pattern: "\\(NINA-06\\)" + group: "improved_version" + np: pattern: "\\(NP\\)" group: "improved_version" @@ -187,6 +223,22 @@ renkaban: pattern: "\\(Renkaban\\)" group: "improved_version" +sachen: + pattern: "\\(Sachen\\)" + group: "improved_version" + +subor_keyboard: + pattern: "\\(Subor Keyboard\\)" + group: "improved_version" + +super_mega: + pattern: "\\(Super Mega\\)" + group: "improved_version" + +txc: + pattern: "\\(TXC\\)" + group: "improved_version" + # BUDGET EDITIONS artdink: @@ -251,6 +303,10 @@ value_1500: # MODERN VERSIONS +3ds_virtual_console: + pattern: "\\((?:\\w*?,?\\s)*3DS Virtual Console(?:,?\\s\\w*?)*\\)" + group: "modern_version" + battle_net: pattern: "\\(Battle\\.net\\)" group: "modern_version" @@ -263,6 +319,10 @@ castlevania_advance: pattern: "\\(Castlevania Advance Collection\\)" group: "modern_version" +capcom_classics_mini_mix: + pattern: "\\(Capcom Classics Mini Mix\\)" + group: "modern_version" + capcom_town: pattern: "\\(Capcom Town\\)" group: "modern_version" @@ -299,10 +359,34 @@ gamecube: pattern: "\\(Game[Cc]ube.*?\\)" group: "modern_version" +iam8bit: + pattern: "\\(iam8bit\\)" + group: "modern_version" + +konami_collector_series: + pattern: "\\(Konami Collector's Series\\)" + group: "modern_version" + mega_man_x_legacy: pattern: "\\(Mega Man X Legacy Collection\\)" group: "modern_version" +namco_anthology_1: + pattern: "\\(Namco Anthology 1.*?\\)" + group: "modern_version" + +namco_anthology_2: + pattern: "\\(Namco Anthology 2.*?\\)" + group: "modern_version" + +namcot_collection: + pattern: "\\(Namcot Collection.*?\\)" + group: "modern_version" + +namco_museum_archives: + pattern: "\\(Namco Museum Archives Vol [0-9]\\)" + group: "modern_version" + qubyte_classic: pattern: "\\(QUByte Classics\\)" group: "modern_version" @@ -311,6 +395,14 @@ retro_bit: pattern: "\\(Retro-Bit\\)" group: "modern_version" +retro_bit_generations: + pattern: "\\(Retro-Bit Generations\\)" + group: "modern_version" + +rockman_123: + pattern: "\\(Rockman 123\\)" + group: "modern_version" + seiken_denetsu_collection: pattern: "\\(Seiken Densetsu Collection\\)" group: "modern_version" @@ -343,11 +435,39 @@ wii: pattern: "\\(Wii\\)" group: "modern_version" +wii_virtual_console: + pattern: "\\((?:\\w*?,?\\s)*Wii Virtual Console(?:,?\\s\\w*?)*\\)" + group: "modern_version" + +wiiu_virtual_console: + pattern: "\\((?:\\w*?,?\\s)*Wii\\s?U Virtual Console(?:,?\\s\\w*?)*\\)" + group: "modern_version" + +wii_and_wiiu_virtual_console: + pattern: "\\((?:\\w*?,?\\s)*Wii Virtual Console, Wii\\s?U Virtual Console(?:,?\\s\\w*?)*\\)" + group: "modern_version" + # DEMOTED VERSIONS arcade: pattern: "\\(Arcade\\)" group: "demoted_version" +asder: + pattern: "\\(Asder\\)" + group: "demoted_version" + +caltron: + pattern: "\\(Caltron\\)" + group: "demoted_version" + +competition_cart: + pattern: "\\(Competition Cart.*?\\)" + group: "demoted_version" + +coolboy: + pattern: "\\(Coolboy\\)" + group: "demoted_version" + debug_version: pattern: "\\(Debug Version\\)" group: "demoted_version" @@ -360,14 +480,54 @@ early: pattern: "\\(Early\\)" group: "demoted_version" -competition_cart: - pattern: "\\(Competition Cart.*?\\)" +famicombox: + pattern: "\\(FamicomBox\\)" + group: "demoted_version" + +fs005: + pattern: "\\(FS005\\)" + group: "demoted_version" + +hwang_shinwei: + pattern: "\\(Hwang Shinwei\\)" + group: "demoted_version" + +jncota: + pattern: "\\(Jncota\\)" group: "demoted_version" limited_run: pattern: "\\(Limited Run Games\\)" group: "demoted_version" +mb91: + pattern: "\\(MB-91\\)" + group: "demoted_version" + +mbc3: + pattern: "\\(MBC3\\)" + group: "demoted_version" + +mbc5: + pattern: "\\(MBC5\\)" + group: "demoted_version" + +micro_genius: + pattern: "\\(Micro Genius\\)" + group: "demoted_version" + +nintendoage: + pattern: "\\(NintendoAge\\)" + group: "demoted_version" + +no_cart_present: + pattern: "\\(No Cart Present\\)" + group: "demoted_version" + +not_for_resale: + pattern: "\\(Not for Resale\\)" + group: "demoted_version" + piko_interactive: pattern: "\\(Piko Interactive\\)" group: "demote_version" @@ -386,5 +546,25 @@ shokai_seisanban: # GENERALLY IGNORE +60_pin_cart: + pattern: "\\(60 pin cart\\)" + +72_pin_cart: + pattern: "\\(72 pin cart\\)" + strictly_limited: pattern: "\\(Strictly Limited Games\\)" + +ntdec: + pattern: "\\(NTDEC\\)" + +ntsc: + pattern: "([?:-][\\s])?[(]?NTSC\\)?" + flags: "NOFLAG" + +pal: + pattern: "([?:-][\\s])?[(]?PAL(?: [a-zA-Z]+| 50[Hh]z)?(?:\\)?| (?=\\())" + flags: "NOFLAG" + +43_406_pcb: + pattern: "\\(43-406 PCB\\)" diff --git a/romsearch/dev/__init__.py b/romsearch/dev/__init__.py new file mode 100644 index 0000000..468a0d1 --- /dev/null +++ b/romsearch/dev/__init__.py @@ -0,0 +1,6 @@ +from .parsing_tools import check_regex_parsing, parse_games_from_dat + +__all__ = [ + "check_regex_parsing", + "parse_games_from_dat", +] diff --git a/romsearch/dev/parsing_tools.py b/romsearch/dev/parsing_tools.py new file mode 100644 index 0000000..fb9aea5 --- /dev/null +++ b/romsearch/dev/parsing_tools.py @@ -0,0 +1,71 @@ +import logging +import os + +import romsearch +from romsearch import GameFinder +from ..util import get_short_name, load_json, load_yml + + +def check_regex_parsing(dat_filename): + """Run a .dat file through the short name parser + + This is useful when adding a new platform, so we can see + what new terms need to be added + """ + + dat_file = load_json(dat_filename) + all_files = [f for f in dat_file] + all_files.sort() + + mod_dir = os.path.dirname(romsearch.__file__) + + default_config_file = os.path.join(mod_dir, "configs", "defaults.yml") + default_config = load_yml(default_config_file) + + default_config = default_config + + regex_config_file = os.path.join(mod_dir, "configs", "regex.yml") + regex_config = load_yml(regex_config_file) + + regex_config = regex_config + + # Parse all the names in the dat file + all_file_dict = {} + for f in all_files: + short_name = get_short_name(f, + regex_config=regex_config, + default_config=default_config, + ) + + all_file_dict[f] = { + "short_name": short_name, + "matched": False, + } + + return all_file_dict + + +def parse_games_from_dat(config_file, + platform, + ): + """Using a config file and platform, parse all the games from a dat + + This will include the dupe parsing and everything if specified in the config + """ + + config = load_yml(config_file) + + parsed_dat_dir = config.get("parsed_dat_dir", None) + + dat_filename = os.path.join(parsed_dat_dir, f"{platform} (dat parsed).json") + + all_file_dict = check_regex_parsing(dat_filename) + + finder = GameFinder(config_file=config_file, + platform=platform, + ) + finder.logger.setLevel(logging.WARNING) + + all_games = finder.run(files=all_file_dict) + + return all_games diff --git a/romsearch/modules/dupeparser.py b/romsearch/modules/dupeparser.py index 5cec03a..781804d 100644 --- a/romsearch/modules/dupeparser.py +++ b/romsearch/modules/dupeparser.py @@ -91,12 +91,13 @@ def get_dupe_dict(self): dupe_dict = {} - if self.use_dat: - self.logger.info("Gettings dupes from dat file") - dupe_dict = self.get_dat_dupes(dupe_dict) + # Prefer retool dupes first if self.use_retool: self.logger.info("Gettings dupes from retool file") dupe_dict = self.get_retool_dupes(dupe_dict) + if self.use_dat: + self.logger.info("Gettings dupes from dat file") + dupe_dict = self.get_dat_dupes(dupe_dict) dupe_dict = dict(sorted(dupe_dict.items())) @@ -174,6 +175,11 @@ def get_retool_dupes(self, dupe_dict=None): retool_dupes = self.get_retool_dupe_dict() for retool_dupe in retool_dupes: + + # If we don't have titles within the dupe dict, skip + if "titles" not in retool_dupe: + continue + group = retool_dupe["group"] group_titles = [get_short_name(f["searchTerm"], default_config=self.default_config, diff --git a/romsearch/modules/gamefinder.py b/romsearch/modules/gamefinder.py index 8f3554f..27895ac 100644 --- a/romsearch/modules/gamefinder.py +++ b/romsearch/modules/gamefinder.py @@ -87,6 +87,7 @@ def run(self, self.logger.info(create_bar(f"START GameFinder")) games_dict = self.get_game_dict(files) + games_dict = dict(sorted(games_dict.items())) self.logger.info(f"Found {len(games_dict)} games:") for g in games_dict: @@ -109,16 +110,17 @@ def get_game_dict(self, self.exclude_games, ) - for i in sorted(games_to_remove, reverse=True): - games.pop(i) + if games_to_remove is not None: + for i in sorted(games_to_remove, reverse=True): + games.pop(i) # Include only included files if self.include_games is not None: games_to_include = self.get_game_matches(games, self.include_games, ) - - games = np.asarray(games)[games_to_include] + if games_to_include is not None: + games = np.asarray(games)[games_to_include] # We need to trim down dupes here. Otherwise, the # dict is just the list we already have @@ -142,10 +144,13 @@ def get_game_matches(self, files, games_to_match): games_matched = [] if isinstance(games_to_match, dict): - games_to_match = games_to_match.get(self.platform, []) + games_to_match = games_to_match.get(self.platform, None) else: games_to_match = copy.deepcopy(games_to_match) + if games_to_match is None: + return None + games_matched.extend(games_to_match) idx = [] diff --git a/romsearch/modules/romparser.py b/romsearch/modules/romparser.py index 568f68e..ca362bf 100644 --- a/romsearch/modules/romparser.py +++ b/romsearch/modules/romparser.py @@ -221,6 +221,10 @@ def parse_retool(self, f, file_dict=None): if found_cat: continue + # If we don't have titles within the dupe dict, skip + if "titles" not in retool_dict: + continue + retool_variants = [f["searchTerm"].lower() for f in retool_dict["titles"]] if game_name.lower() in retool_variants: