diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..c6f7e4e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,57 @@ +name: Bug report +description: Submit a bug report +title: '[BUG]: ' +labels: ['bug'] +assignees: 'bbtufty' + +body: + - type: dropdown + id: version + attributes: + label: Installation + description: Are you using pip or GitHub? Which GitHub tag? + options: + - pip + - GitHub - main branch + - GitHub - dev branch + validations: + required: true + - type: textarea + id: description + attributes: + label: Describe the Bug + description: A clear and concise description of the bug. + validations: + required: true + - type: textarea + id: expected + attributes: + label: Expected Behavior + description: A clear and concise description of what you expected to happen. + - type: textarea + id: reproduce + attributes: + label: Steps to reproduce the behavior + description: If not present under all circumstances, give a step-by-step on how to reproduce the bug. + value: | + 1. + 2. + ... + - type: textarea + id: screenshots + attributes: + label: Screenshots + description: Attach any applicable screenshots that illustrate your problem. + - type: textarea + id: preferences + attributes: + label: Preference File + description: Paste your config file (likely config.yml), with any sensitive info redacted + render: yaml + - type: textarea + id: log + attributes: + label: Log + description: Attach the relevant log file(s). + validations: + required: true \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/console_request.yml b/.github/ISSUE_TEMPLATE/console_request.yml new file mode 100644 index 0000000..1abf1d2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/console_request.yml @@ -0,0 +1,15 @@ +name: Console request +description: Suggest a new console for this project +title: '[CONSOLE]: ' +labels: ['feature', 'console'] +assignees: 'bbtufty' + +body: + - type: input + id: console_name + attributes: + label: Console Name + description: What is the full name of the console you would like to add? + placeholder: ex. Nintendo Wii + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..0cdce9d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,33 @@ +name: Feature +description: Suggest a new feature for this project +title: '[FEAT]: ' +labels: ['feature'] +assignees: 'bbtufty' + +body: + - type: textarea + id: problem + attributes: + label: Problem + description: Is your feature request related to a problem? Please describe + placeholder: A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + - type: textarea + id: solution + attributes: + label: Solution + description: Describe the solution you'd like + placeholder: A clear and concise description of what you want to happen. + - type: textarea + id: alternatives + attributes: + label: Alternatives + description: Describe alternatives you've considered + placeholder: A clear and concise description of any alternative solutions or features you've considered. + - type: textarea + id: context + attributes: + label: Context + description: Additional context + placeholder: Add any other context or screenshots about the feature request here. + + diff --git a/.github/workflows/build_test.yaml b/.github/workflows/build_test.yaml new file mode 100644 index 0000000..8b5bf2f --- /dev/null +++ b/.github/workflows/build_test.yaml @@ -0,0 +1,37 @@ +name: Build Test + +on: + push: + branches: + - '*' + pull_request: + branches: + - master + +jobs: + job: + name: Build Test + runs-on: ubuntu-latest + strategy: + matrix: + # Versions listed at https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json + python-version: [ + "3.9", + "3.10", + "3.11", + "3.12", + ] + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build \ No newline at end of file diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 0000000..0692dc3 --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,36 @@ +name: Build and upload to PyPI + +on: [push, pull_request] + +jobs: + build_sdist_and_wheel: + name: Build source distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + name: Install Python + with: + python-version: "3.11" + - name: Install build + run: python -m pip install build + - name: Build sdist + run: python -m build --sdist --wheel --outdir dist/ . + - uses: actions/upload-artifact@v4 + with: + path: dist/* + + upload_pypi: + name: Upload to PyPI + needs: [build_sdist_and_wheel] + runs-on: ubuntu-latest + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') + steps: + - uses: actions/download-artifact@v4 + with: + name: artifact + path: dist + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 68bc17f..2dc53ca 100644 --- a/.gitignore +++ b/.gitignore @@ -157,4 +157,4 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..d40b065 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,30 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + builder: html + configuration: docs/conf.py + fail_on_warning: true + +# Optionally build your docs in additional formats such as PDF and ePub +formats: + - htmlzip + - pdf + +# Optionally set the version of Python and requirements required to build your docs +python: + install: + - method: pip + path: . + extra_requirements: + - docs \ No newline at end of file diff --git a/CHANGES.rst b/CHANGES.rst new file mode 100644 index 0000000..99efa7b --- /dev/null +++ b/CHANGES.rst @@ -0,0 +1,4 @@ +0.0.1 (Unreleased) +================== + +- Initial release \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..ad6d0b0 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,17 @@ +include README.md +include CHANGES.rst +include LICENSE +include pyproject.toml + +recursive-include *.pyx *.c *.pxd +recursive-include docs * +recursive-include licenses * +recursive-include cextern * +recursive-include scripts * + +prune build +prune docs/_build +prune docs/api +prune */__pycache__ + +global-exclude *.pyc *.o \ No newline at end of file diff --git a/README.md b/README.md index b6a93ec..75b428b 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,31 @@ -# romsearch -Your one-stop ROM shop +# ROMSearch + +[![](https://img.shields.io/pypi/v/romsearch.svg?label=PyPI&style=flat-square)](https://pypi.org/pypi/romsearch/) +[![](https://img.shields.io/pypi/pyversions/romsearch.svg?label=Python&color=yellow&style=flat-square)](https://pypi.org/pypi/romsearch/) +[![Docs](https://readthedocs.org/projects/romsearch/badge/?version=latest&style=flat-square)](https://romsearch.readthedocs.io/en/latest/) +[![Actions](https://img.shields.io/github/actions/workflow/status/bbtufty/romsearch/build_test.yaml?branch=main&style=flat-square)](https://github.com/bbtufty/romsearch/actions) +[![License](https://img.shields.io/badge/license-GNUv3-blue.svg?label=License&style=flat-square)](LICENSE) + +ROMSearch is designed as a simple-to-inferface with tool that will allow you to pull ROM files from some remote (or +local) location, figure out the best ROM, and move it cleanly to a folder that can imported into an emulator. ROMSearch +is supposed to be a one-shot program to get you from files online to playing games (which is what we want, right?). + +ROMSearch offers the ability to: + +* Sync from remote folder using rclone +* Parse DAT files as well as filenames for ROM information +* Remove dupes from ROM lists using DAT files as well as the excellent ``retool`` clonelists +* Moving files to a structured location, including potentially additional files that may be needed +* Discord integration so users can see the results of runs in a simple, clean way + +To get started, see the [documentation](https://romsearch.readthedocs.io/en/latest/). + +Currently, ROMSearch is in early development, and so many features may be added over time. At the moment, ROMSearch +has the capability for: + +* Nintendo - GameCube +* Nintendo - Super Nintendo Entertainment System +* Sony - PlayStation + +but be aware there may be quirks that will only become apparent over time. We encourage users to open +[issues](https://github.com/bbtufty/romsearch/issues) as and where they find them. \ No newline at end of file diff --git a/docs/1g1r.rst b/docs/1g1r.rst new file mode 100644 index 0000000..38c0214 --- /dev/null +++ b/docs/1g1r.rst @@ -0,0 +1,39 @@ +#### +1G1R +#### + +ROMSearch operates on a "one game, one ROM" (1G1R) philosophy. This means that for each game, it will find the +ROM file that it believes to be the best. The approach ROMSearch takes depends mostly on regions, languages, and +versioning, with regions being the ultimate discriminator. + +Languages +--------- + +Many ROMs are tagged with languages (En, Es, etc), that will be parsed out during the run. If a ROM does not contain +the language the user has specified (most likely En), then this will be removed from the choice. + +A lot of ROMs do not have any language tags (particularly US ones). For these, we currently do not cut them out, +but this may change in the future to associate a region with an implicit language. + +Versions +-------- + +There may be different versions of ROMs (e.g. Rev 1, v2.0, etc). For these, we will take the latest one per unique +region combination as the latest and greatest version. + +Regions +------- + +After the various cuts, there will still be a number ROMs that pass all check. We therefore do a final filter by +region (e.g. USA, Europe) to get to a final ROM. ROMSearch will choose the ROM with the highest region preference +for each game, so order is important here! + +Others +------ + +There are some other cuts that go into deciding the best ROM. These include: + +* Improved version tags (e.g. "EDC" for PS1 games) +* Removing of demos/preproduction ROMs + +The final result is that you should get the single best ROM for your preferences. Hooray! diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/changelog.rst b/docs/changelog.rst new file mode 100644 index 0000000..54be413 --- /dev/null +++ b/docs/changelog.rst @@ -0,0 +1,5 @@ +######### +Changelog +######### + +.. include:: ../CHANGES.rst \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..8ddea3e --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,50 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +import os +import sys +sys.path.insert(0, os.path.abspath('..')) + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'romsearch' +copyright = '2024, bbtufty' +author = 'bbtufty' +release = '0.0.1' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.coverage', + 'sphinx.ext.napoleon', + 'sphinx.ext.todo', + 'sphinx.ext.viewcode', + 'sphinx_automodapi.automodapi', +] + +templates_path = ['_templates'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +master_doc = 'index' + +todo_include_todos = True + +html_theme_options = { + 'collapse_navigation': False, + 'navigation_depth': 4, + 'globaltoc_collapse': False, + 'globaltoc_includehidden': False, + 'display_version': True, +} + +autoclass_content = 'both' + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'sphinx_rtd_theme' +html_static_path = [] diff --git a/docs/configs.rst b/docs/configs.rst new file mode 100644 index 0000000..96062b8 --- /dev/null +++ b/docs/configs.rst @@ -0,0 +1,18 @@ +############ +Config Files +############ + +ROMSearch is controlled primarily through a number of .yml files, which makes extending its functionality simple +without having to delve deep into the code. Here is a comprehensive list of those config files, along with arguments +for each for those who may want to extend ROMSearch's capabilities. + +.. toctree:: + :glob: + :titlesonly: + :maxdepth: 2 + + configs/config + configs/platforms + configs/dats + configs/clonelists + configs/regex \ No newline at end of file diff --git a/docs/configs/clonelists.rst b/docs/configs/clonelists.rst new file mode 100644 index 0000000..d25450e --- /dev/null +++ b/docs/configs/clonelists.rst @@ -0,0 +1,17 @@ +########## +clonelists +########## + +ROMSearch can include curated clonelists to merge into its own dupes. Currently, this is only for ``retool``, but +there is flexibility to add others in the future. + +Syntax: :: + + url: "url" # Base url to pull files from + + [platform]: [filename] # Filename for the platform-specific clonelist + +retool +====== + +.. literalinclude:: ../../romsearch/configs/clonelists/retool.yml diff --git a/docs/configs/config.rst b/docs/configs/config.rst new file mode 100644 index 0000000..5adaea5 --- /dev/null +++ b/docs/configs/config.rst @@ -0,0 +1,12 @@ +###### +config +###### + +Here we document all the options for the ``config.yml`` file that the user can supply. + +TODO + +Sample +====== + +.. literalinclude:: ../../romsearch/configs/sample_config.yml \ No newline at end of file diff --git a/docs/configs/dats.rst b/docs/configs/dats.rst new file mode 100644 index 0000000..9db1336 --- /dev/null +++ b/docs/configs/dats.rst @@ -0,0 +1,15 @@ +#### +dats +#### + +TODO + +No-Intro +======== + +.. literalinclude:: ../../romsearch/configs/dats/no-intro.yml + +Redump +====== + +.. literalinclude:: ../../romsearch/configs/dats/redump.yml diff --git a/docs/configs/platforms.rst b/docs/configs/platforms.rst new file mode 100644 index 0000000..61ef1ea --- /dev/null +++ b/docs/configs/platforms.rst @@ -0,0 +1,10 @@ +######### +platforms +######### + +TODO + +Nintendo - GameCube +=================== + +.. literalinclude:: ../../romsearch/configs/platforms/Nintendo - GameCube.yml diff --git a/docs/configs/regex.rst b/docs/configs/regex.rst new file mode 100644 index 0000000..da0d42d --- /dev/null +++ b/docs/configs/regex.rst @@ -0,0 +1,25 @@ +##### +regex +##### + +The ``regex.yml`` file controls how filenames are parsed using regex rules. + +Syntax: :: + + [name]: # Name of the group + pattern: [pattern] # Regex matching pattern + type: ["str", "list", "bool"] # OPTIONAL. How to parse this match. If "list", a list of possible values needs + to be defined in the config defaults. If "str", will pull out the string of + the regex match. If bool, if the pattern is found within the filename will be + set True, else False. Defaults to "bool" + flags: ["I", "NOFLAG"] # OPTIONAL. Flags to pass to regex. "I" indicates ignorecase, "NOFLAG" means no + flags. Defaults to "I" + group: [group] # OPTIONAL. Regex patterns can be grouped together into a single overarching group. + If not set, will not group + search_tags: [True, False] # OPTIONAL. If False, will search the whole string instead of tags within the file. + Defaults to True + +Full file +========= + +.. literalinclude:: ../../romsearch/configs/regex.yml \ No newline at end of file diff --git a/docs/configuration.rst b/docs/configuration.rst new file mode 100644 index 0000000..023ed15 --- /dev/null +++ b/docs/configuration.rst @@ -0,0 +1,52 @@ +############# +Configuration +############# + +The config.yml file +=================== + +ROMSearch is configured primarily through simple, human-readable .yml files. Having set up your config file, +you can run ROMSearch just by: :: + + from romsearch import ROMSearch + + config_file = "config.yml" + + rs = ROMSearch(config_file) + rs.run() + +Setting up a config file +======================== + +The config file is an easy way to let ROMSearch know where to look for various files. It also includes a +number of switches that may be useful in different use cases. + +**ROMSearch works in such a way that by default it will grab what it thinks is the best ROM file. You can disable +many of the ways it decides this if you want to grab multiple files** + +As a minimal example, if we wanted to grab the entire US Catalog of PlayStation games and post what you've added to a +Discord channel, the config file would look something like this: :: + + raw_dir: 'F:\Emulation\raw' + rom_dir: 'F:\Emulation\ROMs' + dat_dir: 'F:\Emulation\data\dats' + parsed_dat_dir: 'F:\Emulation\data\dats_parsed' + dupe_dir: 'F:\Emulation\data\dupes' + + platforms: + - Sony - PlayStation + + region_preferences: + - USA + + language_preferences: + - En + + romdownloader: + remote_name: 'rclone_remote' + + discord: + webhook_url: "https://discord.com/api/webhooks/webhook_url" + +ROMS will be selected and moved to F:\Emulation\ROMs\Sony - PlayStation. Pretty simple, right? There are a number of +more granular controls if you want them, for more information in the :doc:`config file documentation `. diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..8d9e885 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,27 @@ +.. romsearch documentation master file, created by + sphinx-quickstart on Tue Apr 23 11:38:48 2024. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +.. include:: intro.rst + +.. toctree:: + :titlesonly: + :maxdepth: 2 + :caption: Documentation + + installation + configuration + 1g1r + modules + configs + utils + changelog + reference_api + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000..1fb2e4c --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,12 @@ +############ +Installation +############ + +ROMSearch is pip-installable: :: + + pip install romsearch + +Or can be installed via GitHub for the latest version: :: + + git clone https://github.com/bbtufty/romsearch.git + pip install -e . diff --git a/docs/intro.rst b/docs/intro.rst new file mode 100644 index 0000000..b9e9c06 --- /dev/null +++ b/docs/intro.rst @@ -0,0 +1,42 @@ +######### +ROMSearch +######### + +.. image:: https://img.shields.io/pypi/v/romsearch.svg?label=PyPI&style=flat-square + :target: https://pypi.org/pypi/romsearch/ +.. image:: https://img.shields.io/pypi/pyversions/romsearch.svg?label=Python&color=yellow&style=flat-square + :target: https://pypi.org/pypi/romsearch/ +.. image:: https://img.shields.io/github/actions/workflow/status/bbtufty/romsearch/build_test.yaml?branch=main&style=flat-square + :target: https://github.com/bbtufty/romsearch/actions +.. image:: https://readthedocs.org/projects/romsearch/badge/?version=latest&style=flat-square + :target: https://romsearch.readthedocs.io/en/latest/ +.. image:: https://img.shields.io/badge/license-GNUv3-blue.svg?label=License&style=flat-square + +ROMSearch is designed as a simple-to-inferface with tool that will allow you to pull ROM files from some remote (or +local) location, figure out the best ROM, and move it cleanly to a folder that can imported into an emulator. ROMSearch +is supposed to be a one-shot program to get you from files online to playing games (which is what we want, right?). + +ROMSearch offers the ability to: + +* Sync from remote folder using rclone +* Parse DAT files as well as filenames for ROM information +* Remove dupes from ROM lists using DAT files as well as the excellent ``retool`` clonelists +* Moving files to a structured location, including potentially additional files that may be needed +* Discord integration so users can see the results of runs in a simple, clean way + +To get started, see the :doc:`installation ` and :doc:`configuration ` pages. For the +philosophy behind how ROMSearch chooses a ROM, see :doc:`1G1R <1g1r>`. + +Currently, ROMSearch is in early development, and so many features may be added over time. At the moment, ROMSearch +has the capability for: + +* Nintendo - GameCube +* Nintendo - Super Nintendo Entertainment System +* Sony - PlayStation + +but be aware there may be quirks that will only become apparent over time. We encourage users to open +`issues `_ as and where they find them. + +ROMSearch is also built in such a way that other platforms/filters can be added in a simple way using config files +rather than manually adding to the base code. For more details of how these work, see the various +:doc:`configs ` pages. diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..954237b --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/modules.rst b/docs/modules.rst new file mode 100644 index 0000000..f810cef --- /dev/null +++ b/docs/modules.rst @@ -0,0 +1,14 @@ +####### +Modules +####### + +ROMSearch contains a number of modules that can be switched on or off, or configured to the user's wishes. Here are +the available modules. + +.. toctree:: + :glob: + :titlesonly: + :maxdepth: 2 + + modules/romsearch + modules/romdownloader \ No newline at end of file diff --git a/docs/modules/romdownloader.rst b/docs/modules/romdownloader.rst new file mode 100644 index 0000000..f076de2 --- /dev/null +++ b/docs/modules/romdownloader.rst @@ -0,0 +1,19 @@ +############# +ROMDownloader +############# + +The ROMDownloader uses ``rclone`` to sync files from a remote (or local) location to be used in other steps. +For instructions on how to set up ``rclone``, see their documentation at `Rclone `_. + +ROMDownloader also has optional Discord integration, which will print out files downloaded or deleted at the end +of each run. + +For more details on the ROMDownloader arguments, see the :doc:`config file documentation <../configs/config>`. + +API +=== + +.. autoclass:: romsearch.ROMDownloader + :no-index: + :members: + :undoc-members: diff --git a/docs/modules/romsearch.rst b/docs/modules/romsearch.rst new file mode 100644 index 0000000..ec340f8 --- /dev/null +++ b/docs/modules/romsearch.rst @@ -0,0 +1,16 @@ +######### +ROMSearch +######### + +This is the main part that controls the various other modules. It essentially calls everything (given user preferences), +and so while it doesn't really do all that much on its own, is the interface to everything else. + +For more details on the ROMSearch arguments, see the :doc:`config file documentation <../configs/config>`. + +API +=== + +.. autoclass:: romsearch.ROMSearch + :no-index: + :members: + :undoc-members: diff --git a/docs/reference_api.rst b/docs/reference_api.rst new file mode 100644 index 0000000..dd2439c --- /dev/null +++ b/docs/reference_api.rst @@ -0,0 +1,27 @@ +############# +Reference/API +############# + +========= +ROMSearch +========= + +.. autoclass:: romsearch.ROMSearch + :members: + :undoc-members: + +============= +ROMDownloader +============= + +.. autoclass:: romsearch.ROMDownloader + :members: + :undoc-members: + +========= +Utilities +========= + +.. automodule:: romsearch.util + :members: + :undoc-members: \ No newline at end of file diff --git a/docs/utils.rst b/docs/utils.rst new file mode 100644 index 0000000..a3f03a8 --- /dev/null +++ b/docs/utils.rst @@ -0,0 +1,14 @@ +##### +Utils +##### + +This simply contains a number of useful utilities that are shared between the various ROMSearch modules. In particular, +we use centralised code for reading of files, logging, string matching using regex, and Discord posting. + +API +=== + +.. automodule:: romsearch.util + :no-index: + :members: + :undoc-members: diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c953aad --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,55 @@ +[project] + +name = "romsearch" +version = "0.0.1" +description = "One Stop ROM Shop" +readme = "README.md" +requires-python = ">=3.9" +license = {file = "LICENSE"} + +authors = [ + {name = "bbtufty"}, +] +maintainers = [ + {name = "bbtufty"}, +] + +classifiers = [ + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + "Development Status :: 3 - Alpha", + + # License + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", + + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3 :: Only", +] + +dependencies = [ + "discordwebhook", + "numpy", + "packaging", + "pathvalidate", + "xmltodict", +] + +[project.urls] +"Homepage" = "https://github.com/bbtufty/romsearch" +"Bug Reports" = "https://github.com/bbtufty/romsearch/issues" +"Source" = "https://github.com/bbtufty/romsearch" + +[build-system] +requires = [ + "setuptools>=43.0.0", + "wheel", + "setuptools_scm", +] + +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/romsearch/__init__.py b/romsearch/__init__.py new file mode 100644 index 0000000..e5aba55 --- /dev/null +++ b/romsearch/__init__.py @@ -0,0 +1,14 @@ +from importlib.metadata import version + +# Get the version +__version__ = version(__name__) + +from .modules import ROMDownloader, ROMChooser, ROMMover, ROMParser, ROMSearch + +__all__ = [ + "ROMDownloader", + "ROMChooser", + "ROMMover", + "ROMParser", + "ROMSearch", +] diff --git a/romsearch/configs/clonelists/retool.yml b/romsearch/configs/clonelists/retool.yml new file mode 100644 index 0000000..c60dead --- /dev/null +++ b/romsearch/configs/clonelists/retool.yml @@ -0,0 +1,6 @@ +url: "https://raw.githubusercontent.com/unexpectedpanda/retool-clonelists-metadata/main/clonelists" + +Nintendo - GameCube: "Nintendo - GameCube (Redump).json" +Nintendo - Super Nintendo Entertainment System: "Nintendo - Super Nintendo Entertainment System (No-Intro).json" +Sony - PlayStation: "Sony - PlayStation (Redump).json" +Sony - PlayStation 2: "Sony - PlayStation 2 (Redump).json" diff --git a/romsearch/configs/dats/no-intro.yml b/romsearch/configs/dats/no-intro.yml new file mode 100644 index 0000000..17ef8bc --- /dev/null +++ b/romsearch/configs/dats/no-intro.yml @@ -0,0 +1,2 @@ +Nintendo - Super Nintendo Entertainment System: + file_mapping: "Nintendo - Super Nintendo Entertainment System" \ No newline at end of file diff --git a/romsearch/configs/dats/redump.yml b/romsearch/configs/dats/redump.yml new file mode 100644 index 0000000..1600c33 --- /dev/null +++ b/romsearch/configs/dats/redump.yml @@ -0,0 +1,11 @@ +Nintendo - GameCube: + web_mapping: "gc" + file_mapping: "Nintendo - GameCube - Datfile" + +Sony - PlayStation: + web_mapping: "psx" + file_mapping: "Sony - PlayStation - Datfile" + +Sony - PlayStation 2: + web_mapping: "ps2" + file_mapping: "Sony - PlayStation 2 - Datfile" diff --git a/romsearch/configs/defaults.yml b/romsearch/configs/defaults.yml new file mode 100644 index 0000000..d8d9ff4 --- /dev/null +++ b/romsearch/configs/defaults.yml @@ -0,0 +1,110 @@ +default_region: "USA" +default_language: "En" + +datetime_format: "%Y/%m/%d, %H:%M:%S" + +platforms: + - "Nintendo - GameCube" + - "Nintendo - Super Nintendo Entertainment System" + - "Sony - PlayStation" + - "Sony - PlayStation 2" + +video_types: + - "NTSC" + - "PAL" + - "PAL 60Hz" + - "MPAL" + - "SECAM" + +regions: + - "USA" + - "World" + - "Canada" + - "Europe" + - "UK" + - "Australia" + - "New Zealand" + - "Singapore" + - "Ireland" + - "Japan" + - "Asia" + - "Thailand" + - "Spain" + - "Mexico" + - "Argentina" + - "Latin America" + - "Brazil" + - "Portugal" + - "France" + - "Belgium" + - "Netherlands" + - "Germany" + - "Austria" + - "Italy" + - "Switzerland" + - "Hong Kong" + - "China" + - "Taiwan" + - "Korea" + - "Russia" + - "Ukraine" + - "Estonia" + - "Poland" + - "Latvia" + - "Lithuania" + - "Denmark" + - "Norway" + - "Sweden" + - "Scandinavia" + - "Finland" + - "Iceland" + - "Hungary" + - "Czech" + - "Greece" + - "Macedonia" + - "India" + - "South Africa" + - "Israel" + - "Slovakia" + - "Turkey" + - "Croatia" + - "Slovenia" + - "United Arab Emirates" + - "Bulgaria" + - "Romania" + - "Albania" + - "Serbia" + - "Indonesia" + - "Unknown" + +languages: + - "En" + - "Ja" + - "Fr" + - "De" + - "Es" + - "It" + - "Nl" + - "Pt" + - "Sv" + - "No" + - "Da" + - "Fi" + - "Zh" + - "Pl" + +dat_categories: + - "Add-Ons" + - "Applications" + - "Audio" + - "Console" + - "Bonus Discs" + - "Coverdiscs" + - "Demos" + - "Educational" + - "Games" + - "Manual" + - "Multimedia" + - "Preproduction" + - "Promotional" + - "Video" diff --git a/romsearch/configs/platforms/Nintendo - GameCube.yml b/romsearch/configs/platforms/Nintendo - GameCube.yml new file mode 100644 index 0000000..750cdce --- /dev/null +++ b/romsearch/configs/platforms/Nintendo - GameCube.yml @@ -0,0 +1,3 @@ +group: "Redump" +ftp_dir: "/Redump/Nintendo - GameCube - NKit RVZ [zstd-19-128k]" +unzip: true diff --git a/romsearch/configs/platforms/Nintendo - Super Nintendo Entertainment System.yml b/romsearch/configs/platforms/Nintendo - Super Nintendo Entertainment System.yml new file mode 100644 index 0000000..7738b9c --- /dev/null +++ b/romsearch/configs/platforms/Nintendo - Super Nintendo Entertainment System.yml @@ -0,0 +1,3 @@ +group: "No-Intro" +ftp_dir: "/No-Intro/Nintendo - Super Nintendo Entertainment System" +unzip: false diff --git a/romsearch/configs/platforms/Sony - PlayStation 2.yml b/romsearch/configs/platforms/Sony - PlayStation 2.yml new file mode 100644 index 0000000..4dd7006 --- /dev/null +++ b/romsearch/configs/platforms/Sony - PlayStation 2.yml @@ -0,0 +1,3 @@ +group: "Redump" +ftp_dir: "/Redump/Sony - PlayStation 2" +unzip: true diff --git a/romsearch/configs/platforms/Sony - PlayStation.yml b/romsearch/configs/platforms/Sony - PlayStation.yml new file mode 100644 index 0000000..f318821 --- /dev/null +++ b/romsearch/configs/platforms/Sony - PlayStation.yml @@ -0,0 +1,7 @@ +group: "Redump" +ftp_dir: "/Redump/Sony - PlayStation" +unzip: true +improved_versions: + - "EDC" +additional_dirs: + SBI: "/Redump/Sony - PlayStation - SBI Subchannels" diff --git a/romsearch/configs/regex.yml b/romsearch/configs/regex.yml new file mode 100644 index 0000000..43cdd3a --- /dev/null +++ b/romsearch/configs/regex.yml @@ -0,0 +1,116 @@ +regions: + pattern: "\\((([regions])(,\\s?)?)*\\)" + type: "list" + flags: "NOFLAG" + +languages: + pattern: "\\((([languages])(,\\s?)?)*\\)" + type: "list" + flags: "NOFLAG" + +multi_disc: + pattern: "\\((Dis[ck]|Seite) [0-9A-Z]\\)" + +# APPLICATIONS + +bracket_program: + pattern: "\\((?:Test )?Program\\)" + group: "applications" + +non_bracket_program: + pattern: "(Check|Sample) Program" + search_tags: false + group: "applications" + +# BAD DUMP + +bad_dump: + pattern: "\\[b\\]" + search_tags: false + group: "bad_dumps" + +# CONSOLE + +bios: + pattern: "\\[BIOS\\]" + search_tags: false + group: "console" + +enhancement_chip: + pattern: "\\(Enhancement Chip\\)" + group: "console" + +# DEMOS +barai: + pattern: "\\(@barai\\)" + group: "demos" + +demo: + pattern: "\\((?:\\w[-.]?\\s*)*Demo(?:(?:,?\\s|-)[\\w0-9\\.]*)*\\)" + group: "demos" + +preview: + pattern: "\\(Preview\\)" + group: "demos" + +sample: + pattern: "\\(Sample(?:\\s[0-9]*|\\s\\d{4}-\\d{2}-\\d{2})?\\)" + group: "demos" + +taikenban: + pattern: "Taikenban" + search_tags: false + group: "demos" + +# PRE-PRODUCTION +alpha: + pattern: "\\((?:\\w*?\\s)*Alpha(?:\\s\\d+)?\\)" + group: "preproduction" + +beta: + pattern: "\\((?:\\w*?\\s)*Beta(?:\\s\\d+)?\\)" + group: "preproduction" + +prepro: + pattern: "\\((?:Pre-production|Prerelease)\\)" + group: "preproduction" + +proto: + pattern: "\\((?:\\w*?\\s)*Proto(?:type)?(?:\\s\\d+)?\\)" + group: "preproduction" + +# VERSIONS +version_no: + pattern: "\\(v[\\.0-9].*?\\)" + type: "str" + group: "version" + +revision: + pattern: "\\(R[eE][vV](?:[ -][0-9A-Z].*?)?\\)" + type: "str" + transform_pattern: "R[eE][vV][ -]([0-9A-Z].*?)?" + transform_repl: "v\\1" + group: "version" + + +# UNLICENSED +aftermarket: + pattern: "\\(Aftermarket\\)" + group: "unlicensed" + +homebrew: + pattern: "\\(Homebrew\\)" + group: "unlicensed" + +pirate: + pattern: "\\(Pirate\\)" + group: "pirate" + +unl: + pattern: "\\(Unl\\)" + group: "unlicensed" + +# IMPROVED VERSIONS +edc: + pattern: "\\(EDC\\)" + group: "improved_version" diff --git a/romsearch/configs/sample_config.yml b/romsearch/configs/sample_config.yml new file mode 100644 index 0000000..445d55d --- /dev/null +++ b/romsearch/configs/sample_config.yml @@ -0,0 +1,54 @@ +raw_dir: 'F:\Emulation\raw' +rom_dir: 'F:\Emulation\ROMs' +dat_dir: 'F:\Emulation\data\dats' +parsed_dat_dir: 'F:\Emulation\data\dats_parsed' +dupe_dir: 'F:\Emulation\data\dupes' + +run_romdownloader: true +run_datparser: true +run_dupeparser: true +run_romchooser: true +run_rommover: false + +platforms: + - Nintendo - Super Nintendo Entertainment System + +region_preferences: + - USA + +language_preferences: + - En + +include_games: + SNES: + - "Chrono Trigger" + +romsearch: + dry_run: false + +romdownloader: + dry_run: false + remote_name: 'rclone_remote' + sync_all: false + +dupeparser: + use_dat: true + use_retool: true + +gamefinder: + filter_dupes: true + +romparser: + use_dat: true + use_filename: true + +romchooser: + dry_run: false + use_best_version: true + allow_multiple_regions: false + filter_regions: true + filter_languages: true + bool_filters: "all_but_games" + +discord: + webhook_url: "https://discord.com/api/webhooks/discord_url" diff --git a/romsearch/modules/__init__.py b/romsearch/modules/__init__.py new file mode 100644 index 0000000..fd18ca3 --- /dev/null +++ b/romsearch/modules/__init__.py @@ -0,0 +1,19 @@ +from .datparser import DATParser +from .dupeparser import DupeParser +from .gamefinder import GameFinder +from .romchooser import ROMChooser +from .romdownloader import ROMDownloader +from .rommover import ROMMover +from .romparser import ROMParser +from .romsearch import ROMSearch + +__all__ = [ + "DATParser", + "DupeParser", + "GameFinder", + "ROMChooser", + "ROMDownloader", + "ROMMover", + "ROMParser", + "ROMSearch", +] diff --git a/romsearch/modules/datparser.py b/romsearch/modules/datparser.py new file mode 100644 index 0000000..e59daa7 --- /dev/null +++ b/romsearch/modules/datparser.py @@ -0,0 +1,202 @@ +import glob +import json +import os +from datetime import datetime +from urllib.request import urlopen + +import xmltodict + +import romsearch +from ..util import load_yml, setup_logger, create_bar, unzip_file, save_json + +REDUMP_URL = "http://redump.org/datfile/" + +ALLOWED_GROUPS = [ + "No-Intro", + "Redump", +] + + +def get_dat(dat_file_name, + ): + """Parse the dat file to a raw dictionary from a zip file""" + + with open(dat_file_name, "r") as f: + dat = xmltodict.parse(f.read(), attr_prefix='') + + return dat + + +def format_dat(dat): + """Format dat into a nice dictionary""" + + rom_dict = {} + rom_data = dat["datafile"]["game"] + + for rom in rom_data: + rom_dict[rom["name"]] = rom + + return rom_dict + + +class DATParser: + + def __init__(self, + config_file, + platform, + ): + """Parser for dat files from Redump or No-Intro + + For Redump dats, we can download directly from the site. + Users will have to provide their own files for No-Intro, + since there's no good way to scrape them automatically + """ + + self.config_file = config_file + config = load_yml(self.config_file) + + logger_add_dir = str(os.path.join(platform)) + + self.logger = setup_logger(log_level="info", + script_name=f"DATParser", + additional_dir=logger_add_dir, + ) + + self.dat_dir = config.get("dat_dir", None) + self.parsed_dat_dir = config.get("parsed_dat_dir", None) + + self.platform = platform + + # Read in the specific platform configuration + mod_dir = os.path.dirname(romsearch.__file__) + platform_config_file = os.path.join(mod_dir, "configs", "platforms", f"{self.platform}.yml") + self.platform_config = load_yml(platform_config_file) + + self.group = self.platform_config.get("group", None) + if self.group is None: + raise ValueError("No group name specified in platform config file") + if self.group not in ALLOWED_GROUPS: + raise ValueError(f"Group needs to be one of {ALLOWED_GROUPS}") + + # Pull out the platform specifics for the dats + dat_config_file = os.path.join(mod_dir, "configs", "dats", f"{self.group.lower()}.yml") + dat_config = load_yml(dat_config_file) + dat_config = dat_config.get(self.platform, None) + self.dat_config = dat_config + + # Set up the name for the file + self.out_file = os.path.join(self.parsed_dat_dir, f"{self.platform} (dat parsed).json") + + def run(self): + + self.logger.info(create_bar(f"START DATParser")) + + run_datparser = True + + if self.dat_dir is None: + self.logger.warning("No dat_dir defined in config file") + run_datparser = False + if self.parsed_dat_dir is None: + self.logger.warning("No parsed_dat_dir defined in config file") + run_datparser = False + if self.dat_config is None: + self.logger.warning("No platform-specific dat config in the dat configuration file") + run_datparser = False + + if run_datparser: + self.run_datparser() + + self.logger.info(create_bar(f"FINISH DATParser")) + + return True + + def run_datparser(self): + """The main meat of running the dat parser""" + + zip_file = self.get_zip_file() + if zip_file is None: + return False + + # Unzip the file if it doesn't already exist + dat_file_name = zip_file.replace(".zip", ".dat") + if not os.path.exists(dat_file_name): + unzip_file(zip_file, self.dat_dir) + + dat = get_dat(dat_file_name) + + if dat is None: + return False + + rom_dict = format_dat(dat) + + self.save_rom_dict(rom_dict) + + def get_zip_file(self): + """Get zip file from the dat directory + + If this is a Redump file, we can download the latest directly from the + site. Otherwise, you will need to download them manually + """ + + file_mapping = self.dat_config.get("file_mapping", None) + if file_mapping is None: + raise ValueError("No file mapping defined in dat config file") + + if self.group == "Redump": + self.download_latest_redump_dat() + + zip_files = glob.glob(os.path.join(self.dat_dir, f"{file_mapping}*.zip")) + zip_files.sort() + + if len(zip_files) > 1: + self.logger.info(f"Found {len(zip_files)} zip files. Will remove all but the latest (and associated dats)") + for z in zip_files[:-1]: + os.remove(z) + d = z.replace(".zip", ".dat") + if os.path.exists(d): + os.remove(d) + + zip_files = glob.glob(os.path.join(self.dat_dir, f"{file_mapping}*.zip")) + zip_files.sort() + + if len(zip_files) == 0: + self.logger.warning(f"No zip files found. " + f"You need to manually download {self.group} dat files for {self.platform}") + return None + + return zip_files[-1] + + def download_latest_redump_dat(self): + """Download Redump zip file for the platform""" + + web_mapping = self.dat_config.get("web_mapping", None) + if web_mapping is None: + raise ValueError("No web mapping defined in dat config file") + + response = urlopen(f"http://redump.org/datfile/{web_mapping}") + f = response.headers.get_filename() + + out_file = os.path.join(self.dat_dir, f) + if os.path.exists(out_file): + self.logger.info(f"{f} already downloaded, will skip") + return True + + self.logger.info(f"Downloading {f}") + if not os.path.exists(self.dat_dir): + os.makedirs(self.dat_dir) + + with open(out_file, mode="wb") as d: + d.write(response.read()) + + return True + + def save_rom_dict(self, + rom_dict, + ): + """Save the dat file parsed as a dictionary to JSON""" + + if not os.path.exists(self.parsed_dat_dir): + os.makedirs(self.parsed_dat_dir) + + out_file = os.path.join(self.out_file) + save_json(rom_dict, out_file) diff --git a/romsearch/modules/dupeparser.py b/romsearch/modules/dupeparser.py new file mode 100644 index 0000000..d1bdf77 --- /dev/null +++ b/romsearch/modules/dupeparser.py @@ -0,0 +1,231 @@ +import copy +import os + +import numpy as np +import requests + +import romsearch +from ..util import setup_logger, create_bar, load_yml, get_game_name, load_json, save_json + +ID_CLONE_KEYS = [ + "cloneof", + "cloneofid", +] + + +class DupeParser: + + def __init__(self, + config_file, + platform, + ): + """Tool for figuring out a list of dupes""" + + logger_add_dir = str(os.path.join(platform)) + + self.logger = setup_logger(log_level="info", + script_name=f"DupeParser", + additional_dir=logger_add_dir, + ) + + config = load_yml(config_file) + + self.use_dat = config.get("dupeparser", {}).get("use_dat", True) + self.use_retool = config.get("dupeparser", {}).get('use_retool', True) + + self.parsed_dat_dir = config.get("parsed_dat_dir", None) + if self.use_dat and self.parsed_dat_dir is None: + raise ValueError("Must specify parsed_dat_dir if using dat files") + + self.dupe_dir = config.get("dupe_dir", None) + if self.dupe_dir is None: + raise ValueError("dupe_dir should be specified in config file") + + self.platform = platform + + # Pull in platform config that we need + mod_dir = os.path.dirname(romsearch.__file__) + retool_config_file = os.path.join(mod_dir, "configs", "clonelists", f"retool.yml") + retool_config = load_yml(retool_config_file) + + self.retool_url = retool_config.get("url", None) + self.retool_platform_file = retool_config.get(platform, None) + + def run(self): + """Run the dupe parser""" + + if (self.retool_platform_file is None or self.retool_url is None) and self.use_retool: + self.logger.warning("retool config for the platform needs to be present if using retool") + return False + + self.logger.info(create_bar(f"START DupeParser")) + + dupe_dict = self.get_dupe_dict() + + # Save out the dupe dict + out_file = os.path.join(self.dupe_dir, f"{self.platform} (dupes).json") + save_json(dupe_dict, out_file) + + self.logger.info(create_bar(f"FINISH DupeParser")) + + return True + + def get_dupe_dict(self): + """Loop through potentially both the dat files and the retool config file to get out dupes""" + + dupe_dict = {} + + if self.use_dat: + self.logger.info("Gettings dupes from dat file") + dupe_dict = self.get_dat_dupes(dupe_dict) + if self.use_retool: + self.logger.info("Gettings dupes from retool file") + dupe_dict = self.get_retool_dupes(dupe_dict) + + # Filter out any potential duplicates in the list + for key in dupe_dict: + dupe_dict[key] = list(np.unique(dupe_dict[key])) + + dupe_dict = dict(sorted(dupe_dict.items())) + + return dupe_dict + + def get_dat_dupes(self, dupe_dict=None): + """Get dupes from the dat that we've already parsed to JSON""" + + if dupe_dict is None: + dupe_dict = {} + + json_dat = os.path.join(self.parsed_dat_dir, f"{self.platform} (dat parsed).json") + if not os.path.exists(json_dat): + self.logger.warning(f"No dat file found for {self.platform}") + return None + + self.logger.info(f"Using parsed dat file {json_dat}") + + dat_dict = load_json(json_dat) + + all_keys = list(dat_dict.keys()) + + for clone_name in dat_dict: + for id_clone_key in ID_CLONE_KEYS: + if id_clone_key in dat_dict[clone_name]: + clone_key = dat_dict[clone_name][id_clone_key] + + # If it's an ID, find that ID + if id_clone_key == "cloneofid": + + # Sometimes, IDs are missing from the dat so just move on + try: + dat_idx = np.where([dat_dict[key]["id"] == clone_key for key in dat_dict])[0][0] + except IndexError: + continue + parent_entry = dat_dict[all_keys[dat_idx]] + parent_name = parent_entry["name"] + + elif id_clone_key == "cloneof": + # TODO + raise NotImplemented("Only current implemented for cloneofid") + else: + raise ValueError(f"Only know how to parse {ID_CLONE_KEYS}") + + short_parent_name = get_game_name(parent_name) + short_clone_name = get_game_name(clone_name) + + # If the names are the same, just skip + if short_parent_name == short_clone_name: + continue + + if short_parent_name not in dupe_dict: + dupe_dict[short_parent_name] = [short_clone_name] + else: + dupe_dict[short_parent_name].append(short_clone_name) + + return dupe_dict + + def get_retool_dupes(self, dupe_dict=None): + """Get dupes from the retool curated list""" + + if dupe_dict is None: + dupe_dict = {} + + retool_dupes = self.get_retool_dupe_dict() + for retool_dupe in retool_dupes: + group = retool_dupe["group"] + group_titles = [f["searchTerm"] for f in retool_dupe["titles"]] + + # Sometimes, there's like (Disc) or whatever in the group name, + # so parse em out here + if "(" in group: + group_parsed = get_game_name(group) + else: + group_parsed = copy.deepcopy(group) + + # Do the same for all the titles + group_titles_parsed = [] + for g in group_titles: + if "(" in g: + g_parsed = get_game_name(g) + else: + g_parsed = copy.deepcopy(g) + group_titles_parsed.append(g_parsed) + + if group_parsed not in dupe_dict: + dupe_dict[group_parsed] = [] + + dupe_dict[group_parsed].extend(group_titles_parsed) + + return dupe_dict + + def download_retool_dupe(self, + out_file=None, + just_date=False, + ): + """Download the retool curated list, optionally just returning the last modified date""" + + retool_url = f"{self.retool_url}/{self.retool_platform_file}" + with requests.get(retool_url) as r: + retool_dict = r.json() + if just_date: + return retool_dict["description"]["lastUpdated"] + retool_full_file = r.text + + if out_file is None: + raise ValueError("Should specify an out_file to save the retool dupe list to") + + with open(out_file, "w", encoding="utf-8") as f: + f.write(retool_full_file) + + return True + + def get_retool_dupe_dict(self): + """Pull the retool duplicates out of the clonelist file""" + + if not os.path.exists(self.dupe_dir): + os.makedirs(self.dupe_dir) + + retool_dupe_file = os.path.join(self.parsed_dat_dir, f"{self.platform} (retool).json") + if not os.path.exists(retool_dupe_file): + + if not os.path.exists(self.parsed_dat_dir): + os.makedirs(self.parsed_dat_dir) + + self.logger.info("No retool dupe file found. Downloading") + self.download_retool_dupe(retool_dupe_file) + + retool_dupes = load_json(retool_dupe_file) + + # Check if there's a more updated file, if so download it + local_file_time = retool_dupes["description"]["lastUpdated"] + remote_file_time = self.download_retool_dupe(just_date=True) + + if not local_file_time == remote_file_time: + self.logger.info("More up-to-date dupe file found. Will download") + self.download_retool_dupe(retool_dupe_file) + + self.logger.info(f"Using retool clonelist {retool_dupe_file}") + + retool_dupes = load_json(retool_dupe_file) + retool_dupes = retool_dupes["variants"] + + return retool_dupes diff --git a/romsearch/modules/gamefinder.py b/romsearch/modules/gamefinder.py new file mode 100644 index 0000000..3913218 --- /dev/null +++ b/romsearch/modules/gamefinder.py @@ -0,0 +1,184 @@ +import copy +import os +import re + +import numpy as np + +from ..util import setup_logger, load_yml, get_game_name, create_bar, load_json + + +def get_all_games(files): + """Get all unique game names from a list of game files.""" + + games = [get_game_name(f) for f in files] + games = list(np.unique(games)) + + return games + + +class GameFinder: + + def __init__(self, + config_file, + platform, + ): + """Tool to find games within a list of files + + Will parse through files to get a unique list of games, then pull + out potential aliases and optionally remove things from user excluded list + """ + + self.logger = setup_logger(log_level="info", + script_name=f"GameFinder", + additional_dir=platform, + ) + + if platform is None: + raise ValueError("platform must be specified") + self.platform = platform + + config = load_yml(config_file) + + # Pull in specifics to include/exclude + self.include_games = config.get("include_games", None) + self.exclude_games = config.get("exclude_games", None) + + # Info for dupes + self.dupe_dir = config.get("dupe_dir", None) + self.filter_dupes = config.get("gamefinder", {}).get("filter_dupes", True) + + def run(self, + files, + ): + + self.logger.info(create_bar(f"START GameFinder")) + + games_dict = self.get_game_dict(files) + + self.logger.info(f"Found {len(games_dict)} games:") + for g in games_dict: + self.logger.info(f"{g}") + + return games_dict + + def get_game_dict(self, + files, + ): + + games = get_all_games(files) + + # Remove any excluded files + if self.exclude_games is not None: + games_to_remove = self.get_game_matches(games, + self.exclude_games, + ) + + for i in sorted(games_to_remove, reverse=True): + games.pop(i) + + # Include only included files + if self.include_games is not None: + games_to_include = self.get_game_matches(games, + self.include_games, + ) + + games = np.asarray(games)[games_to_include] + + # We need to trim down dupes here. Otherwise, the + # dict is just the list we already have + game_dict = None + if self.filter_dupes: + game_dict = self.get_filter_dupes(games) + + # If the dupe filtering has failed, then just assume everything is unique + if game_dict is None: + + game_dict = {} + + for game in games: + game_dict[game] = [game] + + return game_dict + + def get_game_matches(self, files, games_to_match): + """Get files that match an input list (games_to_match)""" + games_matched = [] + + if isinstance(games_to_match, dict): + games_to_match = games_to_match.get(self.platform, []) + else: + games_to_match = copy.deepcopy(games_to_match) + + games_matched.extend(games_to_match) + + idx = [] + for i, f in enumerate(files): + found_f = False + # Search within each item since the matches might not be exact + for game_matched in games_matched: + + if found_f: + continue + + re_find = re.findall(f"{game_matched}*", f) + + if len(re_find) > 0: + idx.append(i) + found_f = True + + return idx + + def get_filter_dupes(self, games): + """Parse down a list of files based on an input dupe list""" + + if self.dupe_dir is None: + raise ValueError("dupe_dir must be specified if filtering dupes") + + dupe_file = os.path.join(self.dupe_dir, f"{self.platform} (dupes).json") + if not os.path.exists(dupe_file): + self.logger.warning("No dupe files found") + return None + + game_dict = {} + + dupes = load_json(dupe_file) + + # Loop over games, and the dupes dictionary. Use lowercase to account + # for potential upper/lower weirdness + for g_orig in games: + + g = g_orig.lower() + + dupe_found = False + + for d_orig in dupes: + + d = d_orig.lower() + + if dupe_found: + continue + + # If we have a match, put it in here. Use lowercase to be safe + d_dupes = dupes[d_orig] + d_dupes = [dupe.lower() for dupe in d_dupes] + + if g in d_dupes: + + if d in game_dict: + game_dict[d].append(g_orig) + else: + game_dict[d] = [g_orig] + + dupe_found = True + + if not dupe_found: + if g in game_dict: + game_dict[g].append(g_orig) + else: + game_dict[g] = [g_orig] + + for g in game_dict: + game_dict[g] = list(np.unique(game_dict[g])) + game_dict[g].sort() + + return game_dict diff --git a/romsearch/modules/romchooser.py b/romsearch/modules/romchooser.py new file mode 100644 index 0000000..6095ec9 --- /dev/null +++ b/romsearch/modules/romchooser.py @@ -0,0 +1,418 @@ +import copy +import os +from collections import Counter + +import numpy as np +import packaging.version +from packaging import version + +import romsearch +from ..util import (setup_logger, + create_bar, + load_yml, + ) + +BOOL_FILTERS = [ + "add-ons", + "applications", + "audio", + "bad_dumps", + "console", + "bonus_discs", + "coverdiscs", + "demos", + "educational", + "games", + "manuals", + "multimedia", + "pirate", + "preproduction", + "promotional", + "unlicensed", + "video", +] + + +def argsort(seq): + return sorted(range(len(seq)), key=seq.__getitem__) + + +def remove_rom_dict_entries(rom_dict, + key, + remove_type="bool", + bool_remove=True, + list_preferences=None, + ): + """Remove entries from the game dict based on various rules""" + + f_to_delete = [] + for f in rom_dict: + + # Bool type, we can filter either on Trues or Falses + if remove_type == "bool": + + key_val = rom_dict[f].get(key, None) + if key_val is None: + continue + + if bool_remove: + if key_val: + f_to_delete.append(f) + else: + if not key_val: + f_to_delete.append(f) + + elif remove_type == "list": + + if list_preferences is None: + raise ValueError("list_preferences not specified") + + # If there's no information in there, assume we're OK + if len(rom_dict[f][key]) == 0: + continue + + found = False + for val in rom_dict[f][key]: + if val in list_preferences: + found = True + if not found: + f_to_delete.append(f) + + else: + raise ValueError("remove_type should be one of bool, list") + + f_to_delete = np.unique(f_to_delete) + for f in f_to_delete: + rom_dict.pop(f) + + return rom_dict + + +def get_best_version(rom_dict, + version_key="version", + ): + """Pull out all the regions we've got left, to loop over and search for versions""" + + all_regions = np.unique([",".join(rom_dict[key]["regions"]) for key in rom_dict]) + + for region in all_regions: + region_rom_dict = {key: rom_dict[key][version_key] + for key in rom_dict if ",".join(rom_dict[key]["regions"]) == region} + + # Pull out all the versions we have + all_vers = [region_rom_dict[key] for key in region_rom_dict] + # If we have anything here that doesn't have a version, set it to v0 + all_vers = [vers if vers != "" else "v0" for vers in all_vers] + + # If we have lettered versions, convert these to numbers + for i, vers in enumerate(all_vers): + try: + version.parse(vers) + except packaging.version.InvalidVersion: + all_vers[i] = f"v{ord(vers[1:])}" + + all_keys = [key for key in region_rom_dict] + + max_ver = max(all_vers, key=version.parse) + max_ver_idx = np.where(np.asarray(all_vers) == max_ver)[0] + + max_ver_key = np.asarray(all_keys)[max_ver_idx] + + keys_to_pop = [] + for key in all_keys: + if key not in max_ver_key: + keys_to_pop.append(key) + + for key in keys_to_pop: + rom_dict.pop(key) + + return rom_dict + + +def get_best_rom_per_region(rom_dict, + region_preferences, + ): + """For each individual region, get an overall best ROM""" + for reg_pref in region_preferences: + + roms = [] + + for key in rom_dict: + if reg_pref in rom_dict[key]["regions"]: + roms.append(key) + + if len(roms) > 1: + roms = get_best_roms(roms, rom_dict) + + keys_to_pop = [] + for f in rom_dict: + if f not in roms and reg_pref in rom_dict[f]["regions"]: + keys_to_pop.append(f) + + for key in keys_to_pop: + rom_dict.pop(key) + + return rom_dict + + +def get_best_improved_versions(rom_dict): + """If we have some improved files lying around per-region combination, prefer those at this point""" + + all_regions = [] + for key in rom_dict: + all_regions.extend(",".join(rom_dict[key]["regions"])) + all_regions = np.unique(all_regions) + + keys_to_pop = [] + + for region in all_regions: + + region_game_keys = [key for key in rom_dict if region in rom_dict[key]["regions"]] + + # Only filter things down if we've got multiples here + if len(region_game_keys) > 1: + found_improved = [rom_dict[key]["improved_version"] for key in region_game_keys] + + # If we've matched any here, remove those that don't match + if sum(found_improved) > 0: + for key in region_game_keys: + if not rom_dict[key]["improved_version"]: + keys_to_pop.append(key) + + for key in keys_to_pop: + rom_dict.pop(key) + + return rom_dict + + +def get_best_roms(files, rom_dict): + """Get the best ROM(s) from a list, using a scoring system""" + + improved_version_score = 1 + version_score = 10 + + file_scores = np.zeros(len(files)) + + # Take any improved files, revisions, versions. We need to parse versions + file_scores += improved_version_score * np.array([int(rom_dict[f]["improved_version"]) for f in files]) + + file_scores += version_score * add_versioned_score(files, rom_dict, "version") + + files_idx = np.where(file_scores == np.nanmax(file_scores))[0] + files = np.asarray(files)[files_idx] + + return files + + +def add_versioned_score(files, rom_dict, key): + """Get an order for versioned strings""" + + # Ensure we have a version here. If blank, set to v0 + rom_dict = copy.deepcopy(rom_dict) + for f in rom_dict: + if rom_dict[f][key] == "": + rom_dict[f][key] = "v0" + + versions = np.array([version.parse(rom_dict[f][key]) for f in files]) + versions_clean = [key for key, value in Counter(versions).most_common()] + version_vals = sorted(range(len(versions_clean)), key=versions.__getitem__) + + file_scores_version = np.zeros(len(files)) + for v_idx, v in enumerate(versions_clean): + for f_idx, f in enumerate(files): + if version.parse(rom_dict[f][key]) == v: + file_scores_version[f_idx] += version_vals[v_idx] + + return file_scores_version + + +def filter_by_list(rom_dict, + key, + key_prefs, + ): + """Find file with highest value in given list. If there are multiple matches, find the most updated one""" + + roms = [] + + found_key = False + for key_pref in key_prefs: + + if found_key: + continue + + for val in rom_dict: + if key_pref in rom_dict[val][key]: + roms.append(val) + found_key = True + + # If we have multiple matches here, define a best match using scoring system + if len(roms) > 1: + roms = get_best_roms(roms, + rom_dict, + ) + + keys_to_pop = [] + for f in rom_dict: + if f not in roms: + keys_to_pop.append(f) + + for key in keys_to_pop: + rom_dict.pop(key) + + return rom_dict + + +class ROMChooser: + + def __init__(self, + config_file, + platform, + game + ): + """ROM choose tool + + This works per-game, per-platform, so must be specified here + """ + + if platform is None: + raise ValueError("platform must be specified") + self.platform = platform + + logger_add_dir = str(os.path.join(platform, game)) + + self.logger = setup_logger(log_level="info", + script_name=f"ROMChooser", + additional_dir=logger_add_dir, + ) + + config = load_yml(config_file) + + mod_dir = os.path.dirname(romsearch.__file__) + + default_config_file = os.path.join(mod_dir, "configs", "defaults.yml") + self.default_config = load_yml(default_config_file) + + platform_config_file = os.path.join(mod_dir, "configs", "platforms", f"{platform}.yml") + self.platform_config = load_yml(platform_config_file) + + # Region preference (usually set USA for retroachievements, can also be a list to fall back to) + region_preferences = config.get("region_preferences", self.default_config["default_region"]) + if isinstance(region_preferences, str): + region_preferences = [region_preferences] + + for region_pref in region_preferences: + if region_pref not in self.default_config["regions"]: + raise ValueError(f"Regions should be any of {self.default_config['regions']}, not {region_pref}") + + self.region_preferences = region_preferences + + # Language preference (usually set En, can also be a list to fall back to) + language_preferences = config.get("language_preferences", self.default_config["default_language"]) + if isinstance(language_preferences, str): + language_preferences = [language_preferences] + + for language_pref in language_preferences: + if language_pref not in self.default_config["languages"]: + raise ValueError(f"Regions should be any of {self.default_config['languages']}, not {language_pref}") + + self.language_preferences = language_preferences + + # Various filters. First are the boolean ones + bool_filters = config.get("romchooser", {}).get("bool_filters", "all_but_games") + if "all" in bool_filters: + all_bool_filters = copy.deepcopy(BOOL_FILTERS) + if "all_but" in bool_filters: + filter_to_remove = bool_filters.split("all_but_")[-1] + all_bool_filters.remove(filter_to_remove) + bool_filters = copy.deepcopy(all_bool_filters) + + if isinstance(bool_filters, str): + all_bool_filters = [bool_filters] + else: + all_bool_filters = bool_filters + self.bool_filters = all_bool_filters + + self.filter_regions = config.get("romchooser", {}).get("filter_regions", True) + self.filter_languages = config.get("romchooser", {}).get("filter_languages", True) + self.allow_multiple_regions = config.get("romchooser", {}).get("allow_multiple_regions", False) + self.use_best_version = config.get("romchooser", {}).get("use_best_version", True) + + self.dry_run = config.get("romchooser", {}).get("dry_run", False) + + def run(self, + rom_dict): + """Run the ROM chooser""" + + self.logger.info(create_bar(f"START ROMChooser")) + + rom_dict = self.run_chooser(rom_dict) + + self.logger.info(create_bar(f"FINISH ROMChooser")) + + return rom_dict + + def run_chooser(self, + rom_dict): + """Make a ROM choice based on various factors + + This chooser works in this order: + + - Removing any demo files + - Removing any beta files + - Removing anything where the language isn't in the user preferences + (for files with no language info, this will skipped) + - Removing anything where the region isn't in the user preferences + - Get some "best version", via: + - Revision number + - Version number + - Some kind of special name to indicate an improved version + - Finally, if we only allow one region, parse down to a single region (first in the list) + """ + + for f in self.bool_filters: + self.logger.debug(f"Filtering {f}") + if f in BOOL_FILTERS: + rom_dict = remove_rom_dict_entries(rom_dict, + f, + remove_type="bool", + bool_remove=True, + ) + else: + raise ValueError(f"Unknown filter type {f}") + + # Language + if self.filter_languages: + self.logger.debug("Filtering languages") + rom_dict = remove_rom_dict_entries(rom_dict, + "languages", + remove_type="list", + list_preferences=self.language_preferences, + ) + + # Regions + if self.filter_regions: + self.logger.debug("Filtering regions") + rom_dict = remove_rom_dict_entries(rom_dict, + "regions", + remove_type="list", + list_preferences=self.region_preferences, + ) + + # Best versions (revisions/vX.X/any keys that indicated improved versions) + if self.use_best_version: + # TODO: We may also want to demote versions here (e.g. virtual console or whatever) + self.logger.debug("Getting best version") + rom_dict = get_best_version(rom_dict) + rom_dict = get_best_improved_versions(rom_dict) + rom_dict = get_best_rom_per_region(rom_dict, + self.region_preferences, + ) + + if not self.allow_multiple_regions: + self.logger.debug("Trimming down to a single region") + rom_dict = filter_by_list(rom_dict, + "regions", + self.region_preferences, + ) + + return rom_dict diff --git a/romsearch/modules/romdownloader.py b/romsearch/modules/romdownloader.py new file mode 100644 index 0000000..f84cc6d --- /dev/null +++ b/romsearch/modules/romdownloader.py @@ -0,0 +1,288 @@ +import copy +import glob +import os +import subprocess + +import romsearch +from ..util import (setup_logger, + create_bar, + load_yml, + get_file_pattern, + discord_push, +split, + ) + + +def add_rclone_filter(pattern=None, + # bracketed_pattern=None, + filter_type="include", + ): + if filter_type == "include": + filter_str = "+" + elif filter_type == "exclude": + filter_str = "-" + else: + raise ValueError("filter_type should be one of include or exclude") + + # rclone wants double curly braces which we need to escape in python strings (yum) + filter_pattern = "" + + # # Add in non-bracketed stuff (i.e. game names) at the start + # if non_bracketed_pattern is not None: + # filter_pattern += f"{{{{{non_bracketed_pattern}}}}}" + # + # filter_pattern += "*" + + if pattern is not None: + filter_pattern += f"{{{{{pattern}}}}}*" + + cmd = f' --filter "{filter_str} {filter_pattern}"' + + return cmd + +def get_tidy_files(glob_pattern): + """Get a tidy list of files from a glob pattern. + + This just strips off the leading directories to just get a filename + + Args: + glob_pattern (str): glob pattern to match + """ + + files = glob.glob(glob_pattern) + files = [os.path.split(f)[-1] for f in files] + + return files + + +class ROMDownloader: + + def __init__(self, + config_file, + platform=None, + ): + """Downloader tool via rclone + + This works per-platform, so must be specified here + """ + + if platform is None: + raise ValueError("platform must be specified") + self.platform = platform + + self.logger = setup_logger(log_level="info", + script_name=f"ROMDownloader", + additional_dir=platform, + ) + + config = load_yml(config_file) + + out_dir = config.get("raw_dir", None) + if out_dir is None: + raise ValueError("raw_dir needs to be defined in config") + self.out_dir = os.path.join(out_dir, platform) + + # Get any specific includes/excludes + include_games = config.get("include_games", None) + if isinstance(include_games, dict): + include_games = include_games.get(platform, None) + else: + include_games = copy.deepcopy(include_games) + self.include_games = include_games + + exclude_games = config.get("exclude_games", None) + if isinstance(exclude_games, dict): + exclude_games = exclude_games.get(platform, None) + else: + exclude_games = copy.deepcopy(exclude_games) + self.exclude_games = exclude_games + + remote_name = config.get("romdownloader", {}).get("remote_name", None) + if remote_name is None: + raise ValueError("remote_name must be specified in config") + self.remote_name = remote_name + + sync_all = config.get("romdownloader", {}).get("sync_all", True) + self.sync_all = sync_all + + # Read in the specific platform configuration + mod_dir = os.path.dirname(romsearch.__file__) + + platform_config_file = os.path.join(mod_dir, "configs", "platforms", f"{platform}.yml") + platform_config = load_yml(platform_config_file) + + ftp_dir = platform_config.get("ftp_dir", None) + if ftp_dir is None: + raise ValueError(f"ftp_dir should be defined in the platform config file!") + self.ftp_dir = ftp_dir + + self.platform_config = platform_config + + self.discord_url = config.get("discord", {}).get("webhook_url", None) + self.dry_run = config.get("romdownloader", {}).get("dry_run", False) + + def run(self, + ): + """Run Rclone sync tool""" + + self.logger.info(create_bar(f"START ROMDownloader")) + + start_files = get_tidy_files(os.path.join(str(self.out_dir), "*")) + + self.rclone_sync(ftp_dir=self.ftp_dir, + out_dir=self.out_dir, + ) + + end_files = get_tidy_files(os.path.join(str(self.out_dir), "*")) + + if self.discord_url is not None: + + name = f"ROMDownloader: {self.platform}" + self.post_to_discord(start_files, + end_files, + name=name + ) + + # If there are potential additional files to download, do that here + if "additional_dirs" in self.platform_config: + + for add_dir in self.platform_config["additional_dirs"]: + add_ftp_dir = self.platform_config["additional_dirs"][add_dir] + + add_out_dir = f"{self.out_dir} {add_dir}" + + start_files = get_tidy_files(os.path.join(str(add_out_dir), "*")) + + self.rclone_sync(ftp_dir=add_ftp_dir, + out_dir=add_out_dir, + ) + + end_files = get_tidy_files(os.path.join(str(add_out_dir), "*")) + + if self.discord_url is not None: + name = f"ROMDownloader: {self.platform} ({add_dir})" + self.post_to_discord(start_files, + end_files, + name=name + ) + + self.logger.info(create_bar(f"FINISH ROMDownloader")) + + def rclone_sync(self, + ftp_dir, + out_dir=None, + transfers=5, + ): + + if out_dir is None: + out_dir = os.getcwd() + + if not os.path.exists(out_dir): + os.makedirs(out_dir) + + cmd = f'rclone sync -P --transfers {transfers} "{self.remote_name}:{ftp_dir}" "{out_dir}"' + + # We mostly do full syncs here, but we can specify specific game names + if not self.sync_all: + + # Start with any negative filters + searches = [] + + if self.exclude_games is not None: + searches.extend(self.exclude_games) + + if len(searches) > 0: + pattern = get_file_pattern(searches) + else: + pattern = None + + if pattern: + cmd += add_rclone_filter(pattern=pattern, + filter_type="exclude", + ) + + # Now onto positive filters + searches = [] + + # Specific games + if self.include_games is not None: + searches.extend(self.include_games) + + if len(searches) > 0: + pattern = get_file_pattern(searches) + else: + pattern = None + + if pattern: + cmd += add_rclone_filter(pattern=pattern, + filter_type="include", + ) + + cmd += ' --filter "- *"' + + if self.dry_run: + self.logger.info(f"Dry run, would rclone_sync with:") + self.logger.info(cmd) + else: + # os.system(cmd) + # Execute the command and capture the output + with subprocess.Popen(cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as process: + for line in process.stdout: + # Log each line of the output using the provided logger + self.logger.info(line[:-1]) # Exclude the newline character + + return True + + def post_to_discord(self, + start_files, + end_files, + name, + max_per_message=10, + ): + """Create a discord post summarising files added and removed + + Args: + start_files (list): list of files at the start of the rclone + end_files (list): list of files at the end of the rclone + name (string): Name of the post title + max_per_message (int, optional): Maximum number of items per post. Defaults to 10. + """ + + items_added = list(set(end_files).difference(start_files)) + items_deleted = list(set(start_files).difference(end_files)) + + if len(items_added) > 0: + + for items_split in split(items_added, chunk_size=max_per_message): + + fields = [] + + field_dict = {"name": "Added", + "value": "\n".join(items_split) + } + fields.append(field_dict) + + if len(fields) > 0: + discord_push(url=self.discord_url, + name=name, + fields=fields, + ) + + if len(items_deleted) > 0: + + for items_split in split(items_deleted, chunk_size=max_per_message): + + fields = [] + + field_dict = {"name": "Deleted", + "value": "\n".join(items_split) + } + fields.append(field_dict) + + if len(fields) > 0: + discord_push(url=self.discord_url, + name=name, + fields=fields, + ) + + return True diff --git a/romsearch/modules/rommover.py b/romsearch/modules/rommover.py new file mode 100644 index 0000000..d6255bf --- /dev/null +++ b/romsearch/modules/rommover.py @@ -0,0 +1,167 @@ +import copy +import os +import shutil + +import romsearch +from ..util import load_yml, setup_logger, create_bar, unzip_file, load_json, save_json + + +class ROMMover: + + def __init__(self, + config_file, + platform, + game + ): + """ROM Moving and cache updating tool + + Because we do this per-platform, per-game, they need to be specified here + """ + + logger_add_dir = str(os.path.join(platform, game)) + + self.logger = setup_logger(log_level="info", + script_name=f"ROMMover", + additional_dir=logger_add_dir, + ) + + config = load_yml(config_file) + + self.raw_dir = config.get("raw_dir", None) + if self.raw_dir is None: + raise ValueError("raw_dir needs to be defined in config") + + self.rom_dir = config.get("rom_dir", None) + if self.rom_dir is None: + raise ValueError("rom_dir needs to be defined in config") + + cache_file = config.get("rommover", {}).get("cache_file", None) + if cache_file is None: + cache_file = os.path.join(os.getcwd(), f"cache ({platform}).json") + + if os.path.exists(cache_file): + cache = load_json(cache_file) + else: + cache = {} + + self.platform = platform + self.game = game + self.cache_file = cache_file + self.cache = cache + + # Pull in platform config that we need + mod_dir = os.path.dirname(romsearch.__file__) + platform_config_file = os.path.join(mod_dir, "configs", "platforms", f"{platform}.yml") + platform_config = load_yml(platform_config_file) + + self.platform_config = platform_config + self.unzip = self.platform_config.get("unzip", False) + + def run(self, + rom_dict, + ): + + self.logger.info(create_bar(f"START ROMMover")) + + roms_moved = self.move_roms(rom_dict) + self.save_cache() + + self.logger.info(create_bar(f"FINISH ROMMover")) + + return roms_moved + + def move_roms(self, rom_dict): + """Actually move the roms""" + + roms_moved = [] + + for rom_no, rom in enumerate(rom_dict): + + cache_mod_time = (self.cache.get(self.platform, {}). + get(self.game, {}). + get(rom, {}). + get("file_mod_time", 0) + ) + + if rom_dict[rom]["file_mod_time"] == cache_mod_time: + self.logger.info(f"No updates for {rom}, skipping") + continue + + if rom_no == 0: + delete_folder = True + else: + delete_folder = False + + # Move the main file + full_dir = os.path.join(self.raw_dir, self.platform) + full_rom = os.path.join(str(full_dir), rom) + self.move_file(full_rom, unzip=self.unzip, delete_folder=delete_folder) + self.logger.info(f"Moved {rom}") + + # If there are additional file to move/unzip, do that now + if "additional_dirs" in self.platform_config: + for add_dir in self.platform_config["additional_dirs"]: + + add_full_dir = f"{self.raw_dir} {add_dir}" + add_file = os.path.join(add_full_dir, rom) + if os.path.exists(add_file): + self.move_file(add_file, unzip=self.unzip) + self.logger.info(f"Moved {rom} {add_dir}") + + # Update the cache + self.cache_update(file=rom, rom_dict=rom_dict) + + roms_moved.append(rom) + + return roms_moved + + def move_file(self, + zip_file_name, + unzip=False, + delete_folder=False, + ): + """Move file to directory structure, optionally unzipping""" + + out_dir = os.path.join(self.rom_dir, self.platform, self.game) + + if delete_folder and os.path.exists(out_dir): + shutil.rmtree(out_dir) + + if not os.path.exists(out_dir): + os.makedirs(out_dir) + out_dir = str(out_dir) + + if unzip: + unzip_file(zip_file_name, out_dir) + else: + short_zip_file = os.path.split(zip_file_name)[-1] + out_file = os.path.join(out_dir, short_zip_file) + + # Remove this file if it already exists + if os.path.exists(out_file): + os.remove(out_file) + + os.link(zip_file_name, out_file) + + return True + + def cache_update(self, file, rom_dict): + """Update the cache with new file data""" + + if self.platform not in self.cache: + self.cache[self.platform] = {} + + if self.game not in self.cache[self.platform]: + self.cache[self.platform][self.game] = {} + + # If there's already something in there, clear it out + if not self.cache[self.platform][self.game]: + self.cache[self.platform][self.game] = {} + + self.cache[self.platform][self.game][file] = {"file_mod_time": rom_dict[file]["file_mod_time"]} + + def save_cache(self): + """Save out the cache file""" + + cache = copy.deepcopy(self.cache) + save_json(cache, self.cache_file) diff --git a/romsearch/modules/romparser.py b/romsearch/modules/romparser.py new file mode 100644 index 0000000..1d464fb --- /dev/null +++ b/romsearch/modules/romparser.py @@ -0,0 +1,393 @@ +import os +import re +import time +from datetime import datetime + +import romsearch +from ..util import (setup_logger, + create_bar, + load_yml, + load_json, + get_bracketed_file_pattern, + get_game_name, + ) + +DICT_DEFAULT_VALS = { + "bool": False, + "str": "", + "list": [] +} + + +def find_pattern(regex, search_str, group_number=0): + """ + Take a regex pattern and find potential matches within a search string + """ + regex_search_str = None + + regex_search = re.search(regex, search_str) + if regex_search: + regex_search_str = regex_search.group(group_number) + + return regex_search_str + + +def get_pattern_val(regex, tag, regex_type): + pattern_string = find_pattern(regex, tag) + + if pattern_string is not None: + pattern_string = pattern_string.strip("()") + + if regex_type == "bool": + pattern_val = True + elif regex_type == "str": + pattern_val = pattern_string + elif regex_type == "list": + # Split, and remove and trailing whitespace + pattern_string_split = pattern_string.split(",") + pattern_string_split = [s.strip() for s in pattern_string_split] + pattern_val = pattern_string_split + else: + raise ValueError("regex_type should be one of 'bool', 'str', or 'list'") + + else: + pattern_val = None + + return pattern_val + + +def get_regions(f, all_regions): + """Get regions from regex'ing the filename""" + + pattern = get_bracketed_file_pattern(all_regions) + + regions = re.findall(pattern, f)[0] + regions = regions.split(", ") + + return regions + + +def get_rev(f): + """Get revision number from regex'ing the filename""" + + rev = re.findall("(?<=Rev\\s)[0-9]+", f) + + if len(rev) > 0: + rev = int(rev[0]) + else: + rev = 0 + return rev + + +def get_lang(f, all_langs): + """Get languages from regex'ing the filename""" + + pattern = get_bracketed_file_pattern(all_langs) + lang = re.findall(pattern, f) + if len(lang) > 0: + lang = lang[0].split(",") + return lang + + +def get_ver(f): + """Get version from regex'ing the filename""" + + ver = re.findall("(?<=v)[0-9]+\\.?[0-9]*", f) + if len(ver) > 0: + ver = ver[0] + else: + ver = "0" + return ver + + +def match_exists(f, + match, + ): + """Get a True/False for whether some kind of match exists""" + + pattern = get_bracketed_file_pattern(match) + pattern_matched = re.findall(pattern, f) + + if len(pattern_matched) > 0: + return True + else: + return False + + +def get_file_time(f, + datetime_format, + ): + """Get created file time from the file itself""" + + if os.path.exists(f): + ti_m = os.path.getmtime(f) + date_ti_m = datetime.strptime(time.ctime(ti_m), "%a %b %d %H:%M:%S %Y") + else: + date_ti_m = datetime(year=1900, month=1, day=1, hour=0, minute=0, second=0) + date_ti_m_str = date_ti_m.strftime(format=datetime_format) + + return date_ti_m_str + + +class ROMParser: + + def __init__(self, + config_file, + platform, + game, + ): + """ROM parser tool + + This works per-game, per-platform, so must be specified here + + TODO: + - Default implied languages from regions + """ + + if platform is None: + raise ValueError("platform must be specified") + self.platform = platform + + logger_add_dir = str(os.path.join(platform, game)) + + self.logger = setup_logger(log_level="info", + script_name=f"ROMParser", + additional_dir=logger_add_dir, + ) + + config = load_yml(config_file) + + mod_dir = os.path.dirname(romsearch.__file__) + + default_config_file = os.path.join(mod_dir, "configs", "defaults.yml") + self.default_config = load_yml(default_config_file) + + regex_file = os.path.join(mod_dir, "configs", "regex.yml") + self.regex = load_yml(regex_file) + + platform_config_file = os.path.join(mod_dir, "configs", "platforms", f"{platform}.yml") + self.platform_config = load_yml(platform_config_file) + + self.raw_dir = config.get("raw_dir", None) + if not self.raw_dir: + raise ValueError("raw_dir must be specified in config.yml") + + self.use_dat = config.get("romparser", {}).get("use_dat", True) + self.use_retool = config.get("romparser", {}).get("use_retool", True) + self.use_filename = config.get("romparser", {}).get("use_filename", True) + self.dry_run = config.get("romparser", {}).get("dry_run", False) + + # If we're using the dat file, pull it out here + self.dat = None + if self.use_dat: + dat_dir = config.get("parsed_dat_dir", None) + if dat_dir is None: + raise ValueError("parsed_dat_dir must be specified in config.yml") + dat_file = os.path.join(dat_dir, f"{platform} (dat parsed).json") + if os.path.exists(dat_file): + self.dat = load_json(dat_file) + + # If we're using the retool file, pull it out here + self.retool = None + if self.use_retool: + dat_dir = config.get("parsed_dat_dir", None) + if dat_dir is None: + raise ValueError("parsed_dat_dir must be specified in config.yml") + retool_file = os.path.join(dat_dir, f"{platform} (retool).json") + if os.path.exists(retool_file): + self.retool = load_json(retool_file) + + def run(self, + files, + ): + """Run the ROM parser""" + + self.logger.info(create_bar(f"START ROMParser")) + + game_dict = {} + + for f in files: + game_dict[f] = self.parse_file(f) + + self.logger.info(create_bar(f"FINISH ROMParser")) + + return game_dict + + def parse_file(self, + f, + ): + """Parse useful info out of a specific file""" + + file_dict = {} + + if self.use_filename: + file_dict = self.parse_filename(f, file_dict) + if self.use_retool: + file_dict = self.parse_retool(f, file_dict) + if self.use_dat: + file_dict = self.parse_dat(f, file_dict) + + # TODO: Move the last dat bit out where it sets game if nothing else is set + + # File modification time + full_file_path = os.path.join(self.raw_dir, self.platform, f) + file_time = get_file_time(full_file_path, + datetime_format=self.default_config["datetime_format"], + ) + file_dict["file_mod_time"] = file_time + + self.logger.info(f"{f}: {file_dict}") + + return file_dict + + def parse_retool(self, f, file_dict=None): + """Parse info out of the retool file""" + + if file_dict is None: + file_dict = {} + + if self.retool is None: + self.logger.warning(f"No retool file found for {self.platform}. Skipping") + return file_dict + + # Pull out the game name + game_name = get_game_name(f) + + # Loop over the variants, see if we get a match + found_cat = False + for retool_dict in self.retool["variants"]: + + if found_cat: + continue + + retool_variants = [f["searchTerm"].lower() for f in retool_dict["titles"]] + + if game_name.lower() in retool_variants: + + found_cat = True + + # If we have categories, set these to True + retool_cats = retool_dict.get("categories", []) + for retool_cat in retool_cats: + file_cat = retool_cat.lower().replace(" ", "_") + file_dict[file_cat] = True + + return file_dict + + def parse_dat(self, f, file_dict=None): + """Parse info out of the dat file""" + + if file_dict is None: + file_dict = {} + + if self.dat is None: + self.logger.warning(f"No dat file found for {self.platform}. Skipping") + return file_dict + + # Remember there aren't zips in the dat entries + dat_entry = self.dat.get(f.strip(".zip"), None) + if not dat_entry: + self.logger.warning(f"No dat entry found for {f}. Skipping") + return file_dict + + dat_categories = self.default_config.get("dat_categories", []) + for dat_cat in dat_categories: + + dat_val = dat_entry.get("category", "") + cat_val = dat_val == dat_cat + + dat_cat_dict = dat_cat.lower().replace(" ", "_") + if dat_cat_dict in file_dict: + file_dict[dat_cat_dict] = file_dict[dat_cat_dict] | cat_val + else: + file_dict[dat_cat_dict] = cat_val + + if all([file_dict[d.lower().replace(" ", "_")] is False for d in dat_categories]): + file_dict["games"] = True + + return file_dict + + def parse_filename(self, f, file_dict=None): + """Parse info out of filename""" + + if file_dict is None: + file_dict = {} + + # Split file into tags + tags = [f'({x}' for x in f.strip(".zip").split(' (')][1:] + + for regex_key in self.regex: + + regex_type = self.regex[regex_key].get("type", "bool") + search_tags = self.regex[regex_key].get("search_tags", True) + group = self.regex[regex_key].get("group", None) + regex_flags = self.regex[regex_key].get("flags", "I") + transform_pattern = self.regex[regex_key].get("transform_pattern", None) + transform_repl = self.regex[regex_key].get("transform_repl", None) + + dict_default_val = DICT_DEFAULT_VALS.get(regex_type, None) + if dict_default_val is None: + raise ValueError(f"regex_type should be one of {list(DICT_DEFAULT_VALS.keys())}") + + if regex_key not in file_dict: + file_dict[regex_key] = dict_default_val + + if regex_flags == "NOFLAG": + regex_flags = re.NOFLAG + elif regex_flags == "I": + regex_flags = re.I + else: + raise ValueError("regex_flags should be one of 'NOFLAG', 'I'") + + pattern = self.regex[regex_key]["pattern"] + + if regex_type == "list": + pattern = pattern.replace(f"[{regex_key}]", "|".join(self.default_config[regex_key])) + + regex = re.compile(pattern, flags=regex_flags) + if search_tags: + + found_tag = False + + for tag in tags: + + if found_tag: + continue + + pattern_string = get_pattern_val(regex, + tag, + regex_type, + ) + if pattern_string is not None: + + if transform_pattern is not None: + pattern_string = re.sub(transform_pattern, transform_repl, pattern_string) + + file_dict[regex_key] = pattern_string + found_tag = True + else: + pattern_string = get_pattern_val(regex, + f, + regex_type, + ) + if pattern_string is not None: + file_dict[regex_key] = pattern_string + + # Update groups, if needed + if group is not None: + if group not in file_dict: + file_dict[group] = dict_default_val + + if regex_type == "bool": + file_dict[group] = file_dict[group] | file_dict[regex_key] + elif regex_type == "str": + if file_dict[group] and file_dict[regex_key]: + raise ValueError("Can't combine multiple groups with type str") + else: + file_dict[group] += file_dict[regex_key] + elif regex_type == "list": + file_dict[group].extend(file_dict[regex_key]) + else: + raise ValueError(f"regex_type should be one of {list(DICT_DEFAULT_VALS.keys())}") + + return file_dict diff --git a/romsearch/modules/romsearch.py b/romsearch/modules/romsearch.py new file mode 100644 index 0000000..3aa63c4 --- /dev/null +++ b/romsearch/modules/romsearch.py @@ -0,0 +1,207 @@ +import glob +import os +import re + +import numpy as np + +import romsearch +from .datparser import DATParser +from .dupeparser import DupeParser +from .gamefinder import GameFinder +from .romchooser import ROMChooser +from .romdownloader import ROMDownloader +from .rommover import ROMMover +from .romparser import ROMParser +from ..util import (load_yml, + setup_logger, + create_bar, + discord_push, + split, + load_json, + ) + + +class ROMSearch: + + def __init__(self, + config_file, + ): + """General search tool to get ROMs downloaded and organized into files""" + + self.config_file = config_file + config = load_yml(self.config_file) + + self.logger = setup_logger("info", "ROMSearch") + + # Read in the various pre-set configs we've got + mod_dir = os.path.dirname(romsearch.__file__) + + default_config_file = os.path.join(mod_dir, "configs", "defaults.yml") + default_config = load_yml(default_config_file) + + self.default_config = default_config + + # Pull in variables from the yaml file + self.raw_dir = config.get("raw_dir", None) + if self.raw_dir is None: + raise ValueError("raw_dir needs to be defined in config") + + self.rom_dir = config.get("rom_dir", None) + if self.rom_dir is None: + raise ValueError("rom_dir needs to be defined in config") + + # Pull out platforms, make sure they're all valid + platforms = config.get("platforms", None) + if platforms is None: + platforms = [] + if isinstance(platforms, str): + platforms = [platforms] + for platform in platforms: + if platform not in self.default_config["platforms"]: + raise ValueError(f"Platforms should be any of {self.default_config['platforms']}, not {platform}") + self.platforms = platforms + + # Which modules to run + self.run_romdownloader = config.get("run_romdownloader", True) + self.run_datparser = config.get("run_datparser", True) + self.run_dupeparser = config.get("run_dupeparser", True) + self.run_romchooser = config.get("run_romchooser", True) + self.run_rommover = config.get("run_rommover", True) + + # Finally, the discord URL if we're sending messages + self.discord_url = config.get("discord", {}).get("webhook_url", None) + + self.dry_run = config.get("romsearch", {}).get("dry_run", False) + + def run(self): + """Run ROMSearch""" + + self.logger.info(create_bar(f"START ROMSearch")) + + self.logger.info(f"Looping over platforms: {self.platforms}") + + all_roms_per_platform = {} + + for platform in self.platforms: + + self.logger.info(f"Running ROMSearch for {platform}") + + raw_dir = os.path.join(self.raw_dir, platform) + + # Run the rclone sync + if self.run_romdownloader: + downloader = ROMDownloader(config_file=self.config_file, + platform=platform, + ) + downloader.run() + + # Get the original directory, so we can safely move back after + orig_dir = os.getcwd() + os.chdir(raw_dir) + + all_files = glob.glob("*.zip") + all_files.sort() + + os.chdir(orig_dir) + + # Parse DAT files here, if we're doing that + if self.run_datparser: + dat_parser = DATParser(config_file=self.config_file, + platform=platform, + ) + dat_parser.run() + + # Get dupes here, if we're doing that + if self.run_dupeparser: + dupe_parser = DupeParser(config_file=self.config_file, + platform=platform, + ) + dupe_parser.run() + + # Find files + finder = GameFinder(config_file=self.config_file, + platform=platform, + ) + + all_games = finder.run(files=all_files) + + self.logger.info(f"Searching through {len(all_games)} games") + + all_roms_moved = [] + + for i, game in enumerate(all_games): + + self.logger.info(f"{i + 1}/{len(all_games)}: {game} (aliases {', '.join(all_games[game])}):") + + # regexing to match up to a ( + rom_files = [] + + for g in all_games[game]: + for f in all_files: + match = re.findall(f"{g}(?=\\s\\()", f) + if len(match) > 0: + rom_files.append(f) + rom_files = list(np.unique(rom_files)) + rom_files.sort() + + parse = ROMParser(self.config_file, + platform=platform, + game=game, + ) + rom_dict = parse.run(rom_files) + + if self.run_romchooser: + # Here, we'll parse down the number of files to one game, one ROM + chooser = ROMChooser(self.config_file, + platform=platform, + game=game, + ) + rom_dict = chooser.run(rom_dict) + + if len(rom_dict) == 0: + self.logger.info(f"All files filtered. Skipping") + continue + + # Print out all the ROMs we've now matched + rom_files = [f for f in rom_dict] + self.logger.info(f"Found ROM file(s): {rom_files}") + + if self.dry_run: + self.logger.info("Dry run, will not move any files") + continue + + if not self.run_rommover: + self.logger.debug("ROMMover is not running, will not move anything") + continue + + mover = ROMMover(self.config_file, + platform=platform, + game=game, + ) + roms_moved = mover.run(rom_dict) + all_roms_moved.extend(roms_moved) + + if len(all_roms_moved) > 0: + all_roms_per_platform[platform] = all_roms_moved + + # Post these to Discord in chunks of 10 + if self.discord_url is not None and len(all_roms_moved) > 0: + + for items_split in split(all_roms_moved): + + fields = [] + + field_dict = {"name": platform, + "value": "\n".join(items_split) + } + fields.append(field_dict) + + if len(fields) > 0: + discord_push(url=self.discord_url, + name="ROMSearch", + fields=fields, + ) + + self.logger.info(create_bar(f"FINISH ROMSearch")) + + return True diff --git a/romsearch/util/__init__.py b/romsearch/util/__init__.py new file mode 100644 index 0000000..b65a429 --- /dev/null +++ b/romsearch/util/__init__.py @@ -0,0 +1,19 @@ +from .discord import discord_push +from .general import split +from .io import load_yml, unzip_file, load_json, save_json +from .logger import setup_logger, create_bar +from .regex_matching import get_file_pattern, get_bracketed_file_pattern, get_game_name + +__all__ = [ + "create_bar", + "setup_logger", + "load_yml", + "get_bracketed_file_pattern", + "get_file_pattern", + "get_game_name", + "load_json", + "save_json", + "unzip_file", + "discord_push", + "split" +] diff --git a/romsearch/util/discord.py b/romsearch/util/discord.py new file mode 100644 index 0000000..1b450a7 --- /dev/null +++ b/romsearch/util/discord.py @@ -0,0 +1,22 @@ +from discordwebhook import Discord + +def discord_push(url, + name, + fields, + ): + """Post a message to Discord""" + + discord = Discord(url=url) + discord.post( + embeds=[ + { + "author": { + "name": name, + "url": "https://github.com/bbtufty/romsearch", + }, + "fields": fields + } + ], + ) + + return True diff --git a/romsearch/util/general.py b/romsearch/util/general.py new file mode 100644 index 0000000..6d97bab --- /dev/null +++ b/romsearch/util/general.py @@ -0,0 +1,10 @@ +def split(full_list, chunk_size=10): + """Split a list in chunks of size chunk_size + + Args: + full_list (list): list to split + chunk_size (int, optional): size of each chunk. Defaults to 10 + """ + + for i in range(0, len(full_list), chunk_size): + yield full_list[i:i + chunk_size] diff --git a/romsearch/util/io.py b/romsearch/util/io.py new file mode 100644 index 0000000..767c967 --- /dev/null +++ b/romsearch/util/io.py @@ -0,0 +1,49 @@ +import json +import os +import zipfile + +import yaml + + +def load_yml(f): + """Load YAML file""" + + with open(f, "r") as file: + config = yaml.safe_load(file) + + return config + + +def unzip_file(zip_file_name, + out_dir, + ): + """Unzip a file""" + + if not os.path.exists(out_dir): + os.makedirs(out_dir) + out_dir = str(out_dir) + + with zipfile.ZipFile(zip_file_name, 'r') as zip_file: + zip_file.extractall(out_dir) + + return True + + +def load_json(file): + """Load json file""" + + with open(file, "r", encoding="utf-8") as f: + j = json.load(f) + + return j + + +def save_json(data, out_file): + """Save json in a pretty way""" + + with open(out_file, "w", encoding="utf-8") as f: + json.dump(data, + f, + ensure_ascii=False, + indent=4, + ) diff --git a/romsearch/util/logger.py b/romsearch/util/logger.py new file mode 100644 index 0000000..210bdf9 --- /dev/null +++ b/romsearch/util/logger.py @@ -0,0 +1,129 @@ +import logging +import math +import os +from logging.handlers import RotatingFileHandler +from pathvalidate import sanitize_filename + +from .. import __version__ + + +def setup_logger(log_level, + script_name, + additional_dir="", + max_logs=9, + ): + """ + Set up the logger. + + Parameters: + log_level (str): The log level to use + script_name (str): The name of the script + additional_dir (str): Any additional directories to keep log files tidy + max_logs (int): Maximum number of log files to keep + + Returns: + A logger object for logging messages. + """ + + # Sanitize the directories if we need to + additional_dir = [sanitize_filename(f) for f in additional_dir.split(os.path.sep)] + + if os.environ.get('DOCKER_ENV'): + config_dir = os.getenv('CONFIG_DIR', '/config') + log_dir = os.path.join(config_dir, "logs", script_name, *additional_dir) + else: + log_dir = os.path.join(os.getcwd(), "logs", script_name, *additional_dir) + + if log_level not in ['debug', 'info', 'critical']: + log_level = 'info' + print(f"Invalid log level '{log_level}', defaulting to 'info'") + + # Create the log directory if it doesn't exist + if not os.path.exists(log_dir): + os.makedirs(log_dir) + + # Define the log file path, and sanitize if needs be + log_file = f"{log_dir}/{script_name}.log" + + # Check if log file already exists + if os.path.isfile(log_file): + for i in range(max_logs - 1, 0, -1): + old_log = f"{log_dir}/{script_name}.log.{i}" + new_log = f"{log_dir}/{script_name}.log.{i + 1}" + if os.path.exists(old_log): + if os.path.exists(new_log): + os.remove(new_log) + os.rename(old_log, new_log) + os.rename(log_file, f"{log_dir}/{script_name}.log.1") + + # Create a logger object with the script name + logger = logging.getLogger(script_name) + logger.propagate = False + + # Set the log level based on the provided parameter + log_level = log_level.upper() + if log_level == 'DEBUG': + logger.setLevel(logging.DEBUG) + elif log_level == 'INFO': + logger.setLevel(logging.INFO) + elif log_level == 'CRITICAL': + logger.setLevel(logging.CRITICAL) + else: + logger.critical(f"Invalid log level '{log_level}', defaulting to 'INFO'") + logger.setLevel(logging.INFO) + + # Define the log message format + formatter = logging.Formatter(fmt='%(asctime)s %(levelname)s: %(message)s', datefmt='%m/%d/%y %I:%M %p') + + # Create a RotatingFileHandler for log files + handler = RotatingFileHandler(log_file, delay=True, mode="w", backupCount=max_logs) + handler.setFormatter(formatter) + + # Add the file handler to the logger + logger.addHandler(handler) + + # Configure console logging with the specified log level + console_handler = logging.StreamHandler() + if log_level == 'DEBUG': + console_handler.setLevel(logging.DEBUG) + elif log_level == 'INFO': + console_handler.setLevel(logging.INFO) + elif log_level == 'CRITICAL': + console_handler.setLevel(logging.CRITICAL) + + # Add the console handler to the logger + logger.addHandler(console_handler) + + # Overwrite previous logger if exists + logging.getLogger(script_name).handlers.clear() + logging.getLogger(script_name).addHandler(handler) + logging.getLogger(script_name).addHandler(console_handler) + + # Insert version number at the head of every log file + name = script_name.replace("_", " ").upper() + logger.info(create_bar(f"{name} Version: {__version__}")) + + return logger + + +def create_bar(middle_text): + """ + Creates a separation bar with provided text in the center + + Args: + middle_text (str): The text to place in the center of the separation bar + + Returns: + str: The formatted separation bar + """ + total_length = 80 + if len(middle_text) == 1: + remaining_length = total_length - len(middle_text) - 2 + left_side_length = 0 + right_side_length = remaining_length + return f"\n{middle_text * left_side_length}{middle_text}{middle_text * right_side_length}\n" + else: + remaining_length = total_length - len(middle_text) - 4 + left_side_length = math.floor(remaining_length / 2) + right_side_length = remaining_length - left_side_length + return f"\n{'*' * left_side_length} {middle_text} {'*' * right_side_length}\n" diff --git a/romsearch/util/regex_matching.py b/romsearch/util/regex_matching.py new file mode 100644 index 0000000..3d1c391 --- /dev/null +++ b/romsearch/util/regex_matching.py @@ -0,0 +1,43 @@ +import re + + +def get_file_pattern(str_to_match): + if isinstance(str_to_match, list): + pattern = "|".join([f"{f}" for f in str_to_match]) + else: + pattern = f"{str_to_match}.*" + + pattern = f"({pattern})" + + return pattern + + +def get_bracketed_file_pattern(str_to_match, + ): + """Get file pattern to match bracketed things + + This is a little tricky since sometimes things can be matched a little unusually. + What we do here is allow for matches of letters, commas or spaces before and after, + but ensure there's no text immediately after the match. So "De" will match for a language + but not "Demo" + """ + + short_pattern = "[\\s\\-a-zA-Z,0-9]*?,?\\s?{},?\\s?(?![a-zA-Z])[\\s\\-a-zA-Z,0-9]*?" + + if isinstance(str_to_match, list): + # pattern = "|".join([f".*?{f}.*?" for f in str_to_match]) + pattern = "|".join([short_pattern.replace("{}", f) for f in str_to_match]) + else: + # pattern = f".*?{str_to_match}.*?" + pattern = short_pattern.replace("{}", str_to_match) + + pattern = f"\\(({pattern})\\)" + + return pattern + + +def get_game_name(f): + """Get game name from the ROM file naming convention""" + game_name = re.findall("^.*?(?=\\s\\()", f)[0] + + return game_name