From 20ead0fea654fe79148d816675fe1d5daa73e72f Mon Sep 17 00:00:00 2001 From: getzze Date: Tue, 19 Nov 2024 22:16:07 +0000 Subject: [PATCH] use a download_script --- appveyor.yml | 124 ++++++------- pyproject.toml | 84 +++++++++ scripts/demo.py | 4 +- scripts/download_library.py | 341 ++++++++++++++++++++++++++++++++++++ tox.ini | 8 +- 5 files changed, 483 insertions(+), 78 deletions(-) create mode 100644 scripts/download_library.py diff --git a/appveyor.yml b/appveyor.yml index 415f7b9..4ea15cc 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -70,30 +70,22 @@ for: - matrix: only: - - APPVEYOR_BUILD_WORKER_IMAGE: *windows_image + - APPVEYOR_BUILD_WORKER_IMAGE: *windows_image install: - "SET PATH=%PYTHON%;%PYTHON%/Scripts;%PATH%" - "python --version" - - "IF %PYTHON:~-4% == -x64 (SET ARCH=x64) ELSE (SET ARCH=i386)" - - "IF %PYTHON:~-4% == -x64 (SET PLATFORM_WHEEL=win_amd64) ELSE (SET PLATFORM_WHEEL=win32)" - - ps: "Start-FileDownload https://mediaarea.net/download/binary/mediainfo/${Env:MEDIAINFO_VERSION}/MediaInfo_CLI_${Env:MEDIAINFO_VERSION}_Windows_${Env:ARCH}.zip" - - ps: "unzip -o MediaInfo_CLI_${Env:MEDIAINFO_VERSION}_Windows_${Env:ARCH}.zip LIBCURL.DLL" - - ps: "Start-FileDownload https://mediaarea.net/download/binary/libmediainfo0/${Env:MEDIAINFO_VERSION}/MediaInfo_DLL_${Env:MEDIAINFO_VERSION}_Windows_${Env:ARCH}_WithoutInstaller.7z" - - ps: "7z -y x MediaInfo_DLL_${Env:MEDIAINFO_VERSION}_Windows_${Env:ARCH}_WithoutInstaller.7z MediaInfo.dll Developers/License.html" - # Required for tests to pass with tox, Windows looks for DLLs in PATH - - ps: "Copy-Item -Path MediaInfo.dll -Destination ${Env:PYTHON}" - - "move MediaInfo.dll pymediainfo" - - "move Developers\\License.html docs" - - "pip install tox" - build: off + - "pip install pdm tox" + - "pdm install --no-self" + build_script: + - ps: | + IF %PYTHON:~-4% == -x64 (SET ARCH=x86_64) ELSE (SET ARCH=i386) + pdm run build_win32_${Env:ARCH} test_script: - "tox" deploy_script: - ps: | If (($env:APPVEYOR_REPO_TAG -eq "true") -and ($env:TOXENV -eq $env:DEPLOY_TOXENV)) { - pip install twine wheel pdm - pdm build --no-sdist - wheel tags --remove --platform-tag=${Env:PLATFORM_WHEEL} dist/*-py3-none-any.whl + pip install twine wheel Invoke-Expression "twine upload --skip-existing dist/*.whl" } - @@ -104,80 +96,68 @@ for: set -eo pipefail PYTHON_VERSION="$(sed -E 's/^py(3)(.*)$/\1.\2/' <<< "$TOXENV")" source "${HOME}/venv${PYTHON_VERSION}/bin/activate" - pip install tox - curl https://mediaarea.net/download/binary/libmediainfo0/${MEDIAINFO_VERSION}/MediaInfo_DLL_${MEDIAINFO_VERSION}_Mac_x86_64+arm64.tar.bz2 \ - | tar xj MediaInfoLib/libmediainfo.0.dylib MediaInfoLib/License.html + pip install pdm tox + pdm install --no-self + build_script: | + pdm run build_darwin # Required for tests to pass with tox - cp MediaInfoLib/libmediainfo.0.dylib /usr/local/lib/ - build: off + cp src/pymediainfo/libmediainfo.0.dylib /usr/local/lib/ test_script: - "tox" deploy_script: | set -eo pipefail if [[ $APPVEYOR_REPO_TAG == "true" && $TOXENV == $DEPLOY_TOXENV ]]; then - mv MediaInfoLib/libmediainfo.0.dylib pymediainfo - mv MediaInfoLib/License.html docs - pip install twine wheel pdm - pdm build --no-sdist - wheel tags --remove --platform-tag=macosx-10.10-x86_64-macosx-11-universal2 dist/*-py3-none-any.whl + pip install twine wheel twine upload --skip-existing dist/*.whl fi - matrix: only: - APPVEYOR_BUILD_WORKER_IMAGE: *linux_image + - TOXENV: /^py.*/ install: | - set -eo pipefail - if [[ $TOXENV =~ doc.* ]]; then - source "${HOME}/venv${QA_PYTHON_VERSION}/bin/activate" - pip install tox + if [[ $TOXENV == pypy3 ]]; then + pushd /tmp + curl -sS "$PYPY_URL" | tar xj + PATH="$(pwd)/$(basename "$PYPY_URL" | sed -E 's/\.tar\.[^.]+$//')/bin/:$PATH" + python -m ensurepip + popd else - if [[ $TOXENV == pypy3 ]]; then - pushd /tmp - curl -sS "$PYPY_URL" | tar xj - PATH="$(pwd)/$(basename "$PYPY_URL" | sed -E 's/\.tar\.[^.]+$//')/bin/:$PATH" - python -m ensurepip - popd - else - PYTHON_VERSION="$(sed -E 's/^py(3)(.*)$/\1.\2/' <<< "$TOXENV")" - source "${HOME}/venv${PYTHON_VERSION}/bin/activate" - fi - # "python -m pip" will work with the unpacked PyPy too, "pip" won't - python -m pip install tox - curl -O https://mediaarea.net/download/binary/libmediainfo0/${MEDIAINFO_VERSION}/MediaInfo_DLL_${MEDIAINFO_VERSION}_Lambda_x86_64.zip - unzip MediaInfo_DLL_${MEDIAINFO_VERSION}_Lambda_x86_64.zip -d x86_64 - curl -O https://mediaarea.net/download/binary/libmediainfo0/${MEDIAINFO_VERSION}/MediaInfo_DLL_${MEDIAINFO_VERSION}_Lambda_arm64.zip - unzip MediaInfo_DLL_${MEDIAINFO_VERSION}_Lambda_arm64.zip -d arm64 + PYTHON_VERSION="$(sed -E 's/^py(3)(.*)$/\1.\2/' <<< "$TOXENV")" + source "${HOME}/venv${PYTHON_VERSION}/bin/activate" fi - build: off + # "python -m pip" will work with the unpacked PyPy too, "pip" won't + python -m pip install pdm tox + pdm install --no-self + build_script: | + # wheel for arm64 + pdm run build_linux_arm64 + # wheel for x86_64 + pdm run build_linux_x86_64 + test_script: | - if [[ $TOXENV =~ doc.* ]]; then - TOX_PARALLEL_NO_SPINNER=1 tox -p - else - # Use the previously downloaded library - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:x86_64/lib/" - # We want to see the progression of the tests so we can't run - # tox environments in parallel - tox - fi + # Use the previously downloaded library + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:x86_64/lib/" + # We want to see the progression of the tests so we can't run + # tox environments in parallel + tox deploy_script: | set -eo pipefail if [[ $APPVEYOR_REPO_TAG == "true" && $TOXENV == $DEPLOY_TOXENV ]]; then - pip install twine wheel pdm - # source distribution - pdm build --no-wheel - - # wheels - mv x86_64/LICENSE docs - # wheel x86_64 - cp x86_64/lib/libmediainfo.so.0 pymediainfo/libmediainfo.so.0 - pdm build --no-sdist - wheel tags --remove --platform-tag=manylinux_2_34-x86_64 dist/*-py3-none-any.whl - # wheel arm64 - mv -f arm64/lib/libmediainfo.so.0 pymediainfo/libmediainfo.so.0 - pdm build --no-sdist - wheel tags --remove --platform-tag=manylinux_2_34-arm64 dist/*-py3-none-any.whl - - # upload + pip install twine + pdm run clean_library + pdm build twine upload --skip-existing dist/*.gz dist/*.whl fi +- + matrix: + only: + - APPVEYOR_BUILD_WORKER_IMAGE: *linux_image + - TOXENV: /docs.*/ + install: | + set -eo pipefail + source "${HOME}/venv${QA_PYTHON_VERSION}/bin/activate" + pip install tox + build: off + test_script: | + TOX_PARALLEL_NO_SPINNER=1 tox -p diff --git a/pyproject.toml b/pyproject.toml index e9e1c50..b747d0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,9 @@ Bugs = "https://github.com/sbraz/pymediainfo/issues" [tool.pdm.version] source = "scm" +[tool.pdm.dev-dependencies] +download_library = ["wheel>=0.44", "requests", "tqdm"] + [tool.pdm.build] source-includes = ["scripts/", "tests/"] @@ -68,6 +71,87 @@ source-includes = ["scripts/", "tests/"] help = "Check type hints" cmd = "mypy --install-types --non-interactive --config-file=pyproject.toml {args:src tests}" +[tool.pdm.scripts.docs] +help = "Build and test documentation" +composite = [ + "sphinx-build -W --keep-going --color -b html docs docs/_build", + "sphinx-build -W --keep-going --color -b linkcheck docs docs/_build", +# "sphinx-build -W --keep-going --color -b doctest docs docs/_build", +] + +[tool.pdm.scripts.test-nocov] +help = "Run tests without coverage" +cmd = "pytest {args:-n auto}" + +[tool.pdm.scripts.test] +help = "Run tests with coverage" +cmd = "pytest --cov --cov-report=term-missing --cov-config=pyproject.toml {args:-n auto}" + +[tool.pdm.scripts.download_library] +help = "Download mediainfo library" +cmd = "python scripts/download_library.py {args:-c --auto}" + +[tool.pdm.scripts.clean_library] +help = "Clean mediainfo library" +cmd = "python scripts/download_library.py -c" + +[tool.pdm.scripts.tag_wheel] +help = "Tag the wheel for a specific platform" +shell = "python -m wheel tags --remove --platform-tag={args} dist/*-py3-none-any.whl 2> /dev/null" + +[tool.pdm.scripts.build_linux_x86_64] +help = "Build wheel with bundled library for Linux x86_64" +composite = [ + "download_library -c -p linux -a x86_64", + "pdm build --no-sdist", + "tag_wheel manylinux_2_34-x86_64", +] + +[tool.pdm.scripts.build_linux_arm64] +help = "Build wheel with bundled library for Linux arm64" +composite = [ + "download_library -c -p linux -a arm64", + "pdm build --no-sdist", + "tag_wheel manylinux_2_34-arm64", +] + +[tool.pdm.scripts.build_win32_x86_64] +help = "Build wheel with bundled library for Windows x64" +composite = [ + "download_library -c -p win32 -a x86_64", + "pdm build --no-sdist", + "tag_wheel win_amd64", +] + +[tool.pdm.scripts.build_win32_i386] +help = "Build wheel with bundled library for Windows x32" +composite = [ + "download_library -c -p win32 -a i386", + "pdm build --no-sdist", + "tag_wheel win32", +] + +[tool.pdm.scripts.build_darwin] +help = "Build wheel with bundled library for MacOS x86_64 and arm64" +composite = [ + "download_library -c -p darwin -a x86_64", + "pdm build --no-sdist", + "tag_wheel macosx-10.10-x86_64-macosx-11-universal2", +] + +[tool.pdm.scripts.build_all] +help = "Build all the wheels with bundled library and the sdist and wheel without library" +composite = [ + "build_linux_arm64", + "build_linux_x86_64", + "build_linux_win32_x86_64", + "build_linux_win32_i386", + "build_linux_darwin", + # remove any library before building sdist and barebone wheel + "clean_library", + "pdm build", +] + # https://mypy.readthedocs.io/en/stable/config_file.html [tool.mypy] diff --git a/scripts/demo.py b/scripts/demo.py index 0251145..b8d1697 100755 --- a/scripts/demo.py +++ b/scripts/demo.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 # ruff: noqa: T201 -""" -a demo that shows how to call pymediainfo -""" +"""A demo that shows how to call pymediainfo.""" import sys from pprint import pprint diff --git a/scripts/download_library.py b/scripts/download_library.py new file mode 100644 index 0000000..155c60e --- /dev/null +++ b/scripts/download_library.py @@ -0,0 +1,341 @@ +# ruff: noqa: T201 +"""Download binary library files from .""" + +from __future__ import annotations + +import hashlib +import os +import shutil +import sys +import tarfile +from dataclasses import dataclass +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import TYPE_CHECKING, Any +from zipfile import ZipFile + +import requests +import tqdm + +if TYPE_CHECKING: + from typing import Literal + + +#: Base URL for downloading MediaInfo library +BASE_URL: str = "https://mediaarea.net/download/binary/libmediainfo0" + +#: Version of the bundled MediaInfo library +MEDIAINFO_VERSION: str = "24.06" + +#: Hashes for the specific MediaInfo version, given the (platform, arch) +MEDIAINFO_HASHES: dict[tuple[str, str], str] = { + ("linux", "x86_64"): "a2935d6cd937709c4c3b616a71e947f2d99ffd23297c53f3c6a4c985a1563e8e", + ("linux", "arm64"): "46cdc6a6f366cad60c8dea9b6c592ace8e24f27e7d64c1214ee3ee02365d7f9d", + ("darwin", "x86_64"): "5298157cf67b52cb65b460242d5477200e54182d5dd31196636b7cf595f6f80f", + ("darwin", "arm64"): "5298157cf67b52cb65b460242d5477200e54182d5dd31196636b7cf595f6f80f", + ("win32", "x86_64"): "c14b9d67b3855229cfc556cff3c03e1d27640903602bec6f2153f3269760f0c1", + ("win32", "i386"): "93922404efe80f8f6e1dd402488445105aa547324baabcf8b3cb38f4996ee0be", +} + + +def get_file_sha256(file_path: os.PathLike | str, blocksize: int = 1 << 20) -> str: + """Get the SHA256 hash of a file.""" + sha256 = hashlib.sha256() + with open(file_path, "rb") as f: + while True: + data = f.read(blocksize) + if not data: + break + sha256.update(data) + return sha256.hexdigest() + + +@dataclass +class Downloader: + """Downloader for the mediainfo library files.""" + + platform: Literal["linux", "darwin", "win32"] + arch: Literal["x86_64", "arm64", "i386"] + + def __post_init__(self) -> None: + """Check that the combination of platform and arch is allowed.""" + allowed_arch = None + if self.platform in ("linux", "darwin"): + allowed_arch = ["x86_64", "arm64"] + elif self.platform == "win32": + allowed_arch = ["x86_64", "i386"] + else: + msg = f"platform not recognized: {self.platform}" + raise ValueError(msg) + + # Check the platform and arch is a valid combination + if allowed_arch is not None and self.arch not in allowed_arch: + msg = ( + f"for platform {self.platform}, arch {self.arch} is not allowed, " + f"must be one of {allowed_arch}" + ) + raise ValueError(msg) + + def get_compressed_file_name(self) -> str: + """Get the compressed file name.""" + if self.platform == "linux": + suffix = f"Lambda_{self.arch}.zip" + elif self.platform == "darwin": + suffix = "Mac_x86_64+arm64.tar.bz2" + elif self.platform == "win32": + win_arch = "x64" if self.arch == "x86_64" else self.arch + suffix = f"Windows_{win_arch}_WithoutInstaller.zip" + else: + msg = f"platform not recognized: {self.platform}" + raise ValueError(msg) + + return f"MediaInfo_DLL_{MEDIAINFO_VERSION}_{suffix}" + + def get_url(self) -> str: + """Get the url to download the mediainfo library.""" + compressed_file = self.get_compressed_file_name() + return f"{BASE_URL}/{MEDIAINFO_VERSION}/{compressed_file}" + + def compare_hash(self, h: str) -> bool: + """Compare downloaded hash with expected.""" + key = (self.platform, self.arch) + expected = MEDIAINFO_HASHES.get(key) + # Check expected hash exists + if expected is None: + msg = f"{key}, expected hash not found." + raise ValueError(msg) + + # Check hashes match + if expected != h: + msg = f"{key}, hash is different from expected: {h}" + raise ValueError(msg) + + return True + + def download_upstream( + self, + url: str, + outpath: os.PathLike, + *, + timeout: int = 20, + verbose: bool = True, + ) -> None: + """Download the compressed file from upstream URL.""" + response = requests.get(url, stream=True, timeout=timeout) + response.raise_for_status() + with open(outpath, "wb") as f: + for chunk in tqdm.tqdm(response.iter_content(chunk_size=8192), disable=not verbose): + f.write(chunk) + + downloaded_hash = get_file_sha256(outpath) + self.compare_hash(downloaded_hash) + + def unzip( + self, + file: os.PathLike | str, + folder: os.PathLike | str, + ) -> dict[str, str]: + """Extract compressed files.""" + file = Path(file) + folder = Path(folder) + compressed_file = self.get_compressed_file_name() + + if not file.is_file(): + msg = f"compressed file not found: {file.name!r}" + raise ValueError(msg) + tmp_dir = file.parent + + license_file: Path | None = None + lib_file: Path | None = None + # Linux + if compressed_file.endswith(".zip") and self.platform == "linux": + with ZipFile(file) as fd: + license_file = folder / "LICENSE" + fd.extract("LICENSE", tmp_dir) + shutil.move(os.fspath(tmp_dir / "LICENSE"), os.fspath(license_file)) + + lib_file = folder / "libmediainfo.so.0" + fd.extract("lib/libmediainfo.so.0.0.0", tmp_dir) + shutil.move(os.fspath(tmp_dir / "lib/libmediainfo.so.0.0.0"), os.fspath(lib_file)) + + # MacOS (darwin) + elif compressed_file.endswith(".tar.bz2") and self.platform == "darwin": + with tarfile.open(file) as fd: + kwargs: dict[str, Any] = {} + if sys.version_info >= (3, 12): + kwargs = {"filter": "data"} + + license_file = folder / "License.html" + fd.extract("MediaInfoLib/License.html", tmp_dir, **kwargs) + shutil.move( + os.fspath(tmp_dir / "MediaInfoLib/License.html"), + os.fspath(license_file), + ) + + lib_file = folder / "libmediainfo.0.dylib" + fd.extract("MediaInfoLib/libmediainfo.0.dylib", tmp_dir, **kwargs) + shutil.move( + os.fspath(tmp_dir / "MediaInfoLib/libmediainfo.0.dylib"), + os.fspath(lib_file), + ) + + # Windows (win32) + elif compressed_file.endswith(".zip") and self.platform == "win32": + with ZipFile(file) as fd: + license_file = folder / "License.html" + fd.extract("Developers/License.html", tmp_dir) + shutil.move(os.fspath(tmp_dir / "Developers/License.html"), os.fspath(license_file)) + + lib_file = folder / "MediaInfo.dll" + fd.extract("MediaInfo.dll", tmp_dir) + shutil.move(os.fspath(tmp_dir / "MediaInfo.dll"), os.fspath(lib_file)) + + files = {} + if license_file is not None and license_file.is_file(): + files["license"] = os.fspath(license_file.relative_to(folder)) + if lib_file is not None and lib_file.is_file(): + files["lib"] = os.fspath(lib_file.relative_to(folder)) + + return files + + def download( + self, + folder: os.PathLike | str, + *, + timeout: int = 20, + verbose: bool = True, + ) -> dict[str, str]: + """Download the library and license files.""" + folder = Path(folder) + + url = self.get_url() + compressed_file = self.get_compressed_file_name() + + extracted_files = {} + with TemporaryDirectory() as tmp_dir: + outpath = Path(tmp_dir) / compressed_file + if verbose: + print(f"Downloading mediainfo library: {compressed_file}") + self.download_upstream(url, outpath, timeout=timeout, verbose=verbose) + + if verbose: + print(f"Extract {compressed_file}") + extracted_files = self.unzip(outpath, folder) + + if verbose: + print(f"Extracted files: {extracted_files}") + return extracted_files + + +def download_files( + folder: os.PathLike | str, + platform: Literal["linux", "darwin", "win32"], + arch: Literal["x86_64", "arm64", "i386"], + *, + timeout: int = 20, + verbose: bool = True, +) -> dict[str, str]: + """Download the library and license files to the output folder.""" + downloader = Downloader(platform=platform, arch=arch) + return downloader.download(folder, timeout=timeout, verbose=verbose) + + +def clean_files( + folder: os.PathLike | str, + *, + verbose: bool = True, +) -> bool: + """Remove downloaded files in the output folder.""" + folder = Path(folder) + if not folder.is_dir(): + if verbose: + print(f"folder does not exist: {os.fspath(folder)!r}") + return False + + glob_patterns = ["License.html", "LICENSE", "MediaInfo.dll", "libmediainfo.*"] + + # list files to delete + to_delete = [] + for pattern in glob_patterns: + to_delete.extend(folder.glob(pattern)) + + # delete files + if verbose: + print(f"will delete files: {to_delete}") + for relative_path in to_delete: + (folder / relative_path).unlink() + + return True + + +if __name__ == "__main__": + import argparse + import platform + + default_folder = Path(__file__).parent.parent.resolve() / "src" / "pymediainfo" + + parser = argparse.ArgumentParser(description="download MediaInfo files from upstream.") + parser.add_argument( + "-p", + "--platform", + choices=["linux", "darwin", "win32"], + help="platform of the library", + ) + parser.add_argument( + "-a", + "--arch", + choices=["x86_64", "arm64", "i386"], + help="architecture of the library", + ) + parser.add_argument( + "--auto", + action="store_true", + help="use the current platform and arch", + ) + parser.add_argument( + "-v", + "--verbose", + help="print information.", + action="store_true", + ) + parser.add_argument( + "--timeout", + type=int, + help="url request timeout", + default=20, + ) + parser.add_argument( + "-o", + "--folder", + type=Path, + help="output folder", + default=default_folder, + ) + parser.add_argument( + "-c", + "--clean", + action="store_true", + help="clean the output folder of downloaded files.", + ) + + args = parser.parse_args() + + if args.auto: + args.platform = platform.system().lower() + args.arch = platform.machine().lower() + + # Clean folder + if args.clean: + clean_files(args.folder, verbose=args.verbose) + + # Download files + if args.platform is not None and args.arch is not None: + extracted_files = download_files( + args.folder, + args.platform, + args.arch, + verbose=args.verbose, + timeout=args.timeout, + ) + + sys.exit(0) diff --git a/tox.ini b/tox.ini index e2d89e3..2120216 100644 --- a/tox.ini +++ b/tox.ini @@ -18,14 +18,16 @@ deps = pytest-xdist setuptools_scm commands = - pytest -n auto {posargs} + pytest {posargs:-n auto} [testenv:docs] deps = + alabaster setuptools_scm sphinx - alabaster -commands = sphinx-build -d "{toxworkdir}/docs_doctree" docs "{toxworkdir}/docs_out" --color -W {posargs} +commands = + sphinx-build -W --keep-going --color -b html docs docs/_build + sphinx-build -W --keep-going --color -b linkcheck docs docs/_build [testenv:black] deps =