diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 42926e9..e2ef8f5 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -15,14 +15,13 @@ jobs: name: Code QA runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - run: pip install black flake8 isort - - run: black --version - - run: isort --version - - run: flake8 --version - - run: isort --check . - - run: black --check . - - run: flake8 . + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - uses: pre-commit/action@v3.0.1 + env: + SKIP: no-commit-to-branch checks: if: ${{ github.event_name == 'release' }} diff --git a/.gitignore b/.gitignore index 68bc17f..e6fdb41 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,9 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + + +*.onnx +*.grib +*.out +_version.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..1abaa90 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,69 @@ +repos: + +# Empty notebookds +- repo: local + hooks: + - id: clear-notebooks-output + name: clear-notebooks-output + files: tools/.*\.ipynb$ + stages: [commit] + language: python + entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace + additional_dependencies: [jupyter] + + +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-yaml # Check YAML files for syntax errors only + args: [--unsafe, --allow-multiple-documents] + - id: debug-statements # Check for debugger imports and py37+ breakpoint() + - id: end-of-file-fixer # Ensure files end in a newline + - id: trailing-whitespace # Trailing whitespace checker + - id: no-commit-to-branch # Prevent committing to main / master + - id: check-added-large-files # Check for large files added to git + - id: check-merge-conflict # Check for files that contain merge conflict + +- repo: https://github.com/psf/black-pre-commit-mirror + rev: 24.1.1 + hooks: + - id: black + args: [--line-length=120] + +- repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + args: + - -l 120 + - --force-single-line-imports + - --profile black + + +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.3.0 + hooks: + - id: ruff + exclude: '(dev/.*|.*_)\.py$' + args: + - --line-length=120 + - --fix + - --exit-non-zero-on-fix + - --preview + +- repo: https://github.com/sphinx-contrib/sphinx-lint + rev: v0.9.1 + hooks: + - id: sphinx-lint + +# For now, we use it. But it does not support a lot of sphinx features +- repo: https://github.com/dzhu/rstfmt + rev: v0.0.14 + hooks: + - id: rstfmt + +- repo: https://github.com/b8raoult/pre-commit-docconvert + rev: "0.1.4" + hooks: + - id: docconvert + args: ["numpy"] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..91c63b5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# (C) Copyright 2024 ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +# https://packaging.python.org/en/latest/guides/writing-pyproject-toml/ + +[build-system] +requires = [ + "setuptools>=60", + "setuptools-scm>=8", +] + +[project] +name = "ai-models-panguweather" + +description = "An ai-models plugin to run PanguWeather" +keywords = [ + "ai", + "tools", +] + +license = { file = "LICENSE" } +authors = [ + { name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "software.support@ecmwf.int" }, +] + +requires-python = ">=3.10" + +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] + +dynamic = [ + "version", +] + +dependencies = [ + "ai-models>=0.7.0", + "onnx", + "onnxruntime-silicon; sys_platform == 'darwin'", + "onnxruntime-gpu; sys_platform == 'linux'" +] + +optional-dependencies.dev = [ + "pre-commit", +] +urls.Repository = "https://github.com/ecmwf-lab/ai-models-panguweather" +entry-points."ai_models.model".panguweather = "ai_models_panguweather.model:PanguWeather" + +[tool.setuptools_scm] +version_file = "src/ai_models_panguweather/_version.py" diff --git a/setup.py b/setup.py deleted file mode 100644 index 7be6a76..0000000 --- a/setup.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python -# (C) Copyright 2023 ECMWF. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# In applying this licence, ECMWF does not waive the privileges and immunities -# granted to it by virtue of its status as an intergovernmental organisation -# nor does it submit to any jurisdiction. -# - - -import io -import os -import platform -import subprocess -import sys - -import setuptools - - -def read(fname): - file_path = os.path.join(os.path.dirname(__file__), fname) - return io.open(file_path, encoding="utf-8").read() - - -version = None -for line in read("ai_models_panguweather/__init__.py").split("\n"): - if line.startswith("__version__"): - version = line.split("=")[-1].strip()[1:-1] - - -assert version - - -def check_gpus(): - try: - n = 0 - for line in subprocess.check_output( - ["nvidia-smi", "-L"], - text=True, - ).split("\n"): - if line.startswith("GPU"): - n += 1 - return n - except (subprocess.CalledProcessError, FileNotFoundError): - return 0 - - -def has_gpu(): - return check_gpus() > 0 - - -onnxruntime = "onnxruntime" -if sys.platform == "darwin": - if platform.machine() == "arm64": - onnxruntime = "onnxruntime-silicon" - -if has_gpu(): - onnxruntime = "onnxruntime-gpu" - - -setuptools.setup( - name="ai-models-panguweather", - # python_requires="<3.11", # For now, does not support Python 3.11 - version=version, - description="An ai-models plugin to run PanguWeather", - long_description=read("README.md"), - long_description_content_type="text/markdown", - author="European Centre for Medium-Range Weather Forecasts (ECMWF)", - author_email="software.support@ecmwf.int", - license="Apache License Version 2.0", - url="https://github.com/ecmwf-lab/ai-models-panguweather", - packages=setuptools.find_packages(), - include_package_data=True, - setup_requires=["GPUtil"], - install_requires=[ - "ai-models>=0.4.0", - "onnx", - os.environ.get("ONNXRUNTIME", onnxruntime), - ], - zip_safe=True, - keywords="tool", - entry_points={ - "ai_models.model": [ - "panguweather = ai_models_panguweather.model:PanguWeather", - ] - }, - classifiers=[ - "Development Status :: 3 - Alpha", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Operating System :: OS Independent", - ], -) diff --git a/ai_models_panguweather/__init__.py b/src/ai_models_panguweather/__init__.py similarity index 100% rename from ai_models_panguweather/__init__.py rename to src/ai_models_panguweather/__init__.py diff --git a/ai_models_panguweather/model.py b/src/ai_models_panguweather/model.py similarity index 97% rename from ai_models_panguweather/model.py rename to src/ai_models_panguweather/model.py index 56c28c4..3a2f8b9 100644 --- a/ai_models_panguweather/model.py +++ b/src/ai_models_panguweather/model.py @@ -18,9 +18,7 @@ class PanguWeather(Model): # Download - download_url = ( - "https://get.ecmwf.int/repository/test-data/ai-models/pangu-weather/{file}" - ) + download_url = "https://get.ecmwf.int/repository/test-data/ai-models/pangu-weather/{file}" download_files = ["pangu_weather_24.onnx", "pangu_weather_6.onnx"] # Input diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 02cc0f9..0000000 --- a/tox.ini +++ /dev/null @@ -1,13 +0,0 @@ -[isort] -profile = black - -[flake8] -; ignore = E226,E302,E41 -max-line-length = 120 -; exclude = tests/* -; See https://black.readthedocs.io/en/stable/the_black_code_style.html -exclude = - dev/* - experiments - ?.py -extend-ignore = E203 diff --git a/utils/pangu-gfs-input.py b/utils/pangu-gfs-input.py index 652a3a6..64fe3ea 100755 --- a/utils/pangu-gfs-input.py +++ b/utils/pangu-gfs-input.py @@ -11,7 +11,7 @@ import sys -import climetlab as cml +import earthkit.data as ekd import tqdm date = sys.argv[1] @@ -24,10 +24,9 @@ time = int(time) time = f"{time:02d}" -gfs = cml.load_source( +gfs = ekd.from_source( "url-pattern", - "https://data.rda.ucar.edu/ds084.1/{year}/{date}/" - "gfs.0p25.{date}{time}.f{step}.grib2", + "https://data.rda.ucar.edu/ds084.1/{year}/{date}/" "gfs.0p25.{date}{time}.f{step}.grib2", date=date, time=time, step="000", @@ -62,7 +61,7 @@ ).order_by(param=param_sfc) print("Write", output) -out = cml.new_grib_output(output) +out = ekd.new_grib_output(output) G = {"gh": 9.80665} PARAM = {"gh": "z", "prmsl": "msl"}