From fdd42d69e89f104ebfd842de4a1060a5e643f097 Mon Sep 17 00:00:00 2001 From: Baudouin Raoult Date: Sun, 22 Sep 2024 07:35:45 +0000 Subject: [PATCH 1/4] Change climetlab to earthkit --- .gitignore | 5 +++++ utils/pangu-gfs-input.py | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 68bc17f..7d373a9 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,8 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + + +*.onnx +*.grib +*.out diff --git a/utils/pangu-gfs-input.py b/utils/pangu-gfs-input.py index 652a3a6..7db5345 100755 --- a/utils/pangu-gfs-input.py +++ b/utils/pangu-gfs-input.py @@ -11,7 +11,7 @@ import sys -import climetlab as cml +import earthkit.data as ekd import tqdm date = sys.argv[1] @@ -24,7 +24,7 @@ time = int(time) time = f"{time:02d}" -gfs = cml.load_source( +gfs = ekd.from_source( "url-pattern", "https://data.rda.ucar.edu/ds084.1/{year}/{date}/" "gfs.0p25.{date}{time}.f{step}.grib2", @@ -62,7 +62,7 @@ ).order_by(param=param_sfc) print("Write", output) -out = cml.new_grib_output(output) +out = ekd.new_grib_output(output) G = {"gh": 9.80665} PARAM = {"gh": "z", "prmsl": "msl"} From 6c93bc958d126f285549beeab64c636d439e0e99 Mon Sep 17 00:00:00 2001 From: Baudouin Raoult Date: Tue, 24 Sep 2024 07:22:43 +0000 Subject: [PATCH 2/4] QA --- .gitignore | 1 + .pre-commit-config.yaml | 69 ++++++++++++ pyproject.toml | 66 ++++++++++++ setup.py | 100 ------------------ .../ai_models_panguweather}/__init__.py | 0 .../ai_models_panguweather}/model.py | 4 +- tox.ini | 2 +- utils/pangu-gfs-input.py | 3 +- 8 files changed, 139 insertions(+), 106 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 pyproject.toml delete mode 100644 setup.py rename {ai_models_panguweather => src/ai_models_panguweather}/__init__.py (100%) rename {ai_models_panguweather => src/ai_models_panguweather}/model.py (97%) diff --git a/.gitignore b/.gitignore index 7d373a9..e6fdb41 100644 --- a/.gitignore +++ b/.gitignore @@ -163,3 +163,4 @@ cython_debug/ *.onnx *.grib *.out +_version.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..1abaa90 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,69 @@ +repos: + +# Empty notebookds +- repo: local + hooks: + - id: clear-notebooks-output + name: clear-notebooks-output + files: tools/.*\.ipynb$ + stages: [commit] + language: python + entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace + additional_dependencies: [jupyter] + + +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-yaml # Check YAML files for syntax errors only + args: [--unsafe, --allow-multiple-documents] + - id: debug-statements # Check for debugger imports and py37+ breakpoint() + - id: end-of-file-fixer # Ensure files end in a newline + - id: trailing-whitespace # Trailing whitespace checker + - id: no-commit-to-branch # Prevent committing to main / master + - id: check-added-large-files # Check for large files added to git + - id: check-merge-conflict # Check for files that contain merge conflict + +- repo: https://github.com/psf/black-pre-commit-mirror + rev: 24.1.1 + hooks: + - id: black + args: [--line-length=120] + +- repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + args: + - -l 120 + - --force-single-line-imports + - --profile black + + +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.3.0 + hooks: + - id: ruff + exclude: '(dev/.*|.*_)\.py$' + args: + - --line-length=120 + - --fix + - --exit-non-zero-on-fix + - --preview + +- repo: https://github.com/sphinx-contrib/sphinx-lint + rev: v0.9.1 + hooks: + - id: sphinx-lint + +# For now, we use it. But it does not support a lot of sphinx features +- repo: https://github.com/dzhu/rstfmt + rev: v0.0.14 + hooks: + - id: rstfmt + +- repo: https://github.com/b8raoult/pre-commit-docconvert + rev: "0.1.4" + hooks: + - id: docconvert + args: ["numpy"] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..91c63b5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# (C) Copyright 2024 ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +# https://packaging.python.org/en/latest/guides/writing-pyproject-toml/ + +[build-system] +requires = [ + "setuptools>=60", + "setuptools-scm>=8", +] + +[project] +name = "ai-models-panguweather" + +description = "An ai-models plugin to run PanguWeather" +keywords = [ + "ai", + "tools", +] + +license = { file = "LICENSE" } +authors = [ + { name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "software.support@ecmwf.int" }, +] + +requires-python = ">=3.10" + +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] + +dynamic = [ + "version", +] + +dependencies = [ + "ai-models>=0.7.0", + "onnx", + "onnxruntime-silicon; sys_platform == 'darwin'", + "onnxruntime-gpu; sys_platform == 'linux'" +] + +optional-dependencies.dev = [ + "pre-commit", +] +urls.Repository = "https://github.com/ecmwf-lab/ai-models-panguweather" +entry-points."ai_models.model".panguweather = "ai_models_panguweather.model:PanguWeather" + +[tool.setuptools_scm] +version_file = "src/ai_models_panguweather/_version.py" diff --git a/setup.py b/setup.py deleted file mode 100644 index 7be6a76..0000000 --- a/setup.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python -# (C) Copyright 2023 ECMWF. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# In applying this licence, ECMWF does not waive the privileges and immunities -# granted to it by virtue of its status as an intergovernmental organisation -# nor does it submit to any jurisdiction. -# - - -import io -import os -import platform -import subprocess -import sys - -import setuptools - - -def read(fname): - file_path = os.path.join(os.path.dirname(__file__), fname) - return io.open(file_path, encoding="utf-8").read() - - -version = None -for line in read("ai_models_panguweather/__init__.py").split("\n"): - if line.startswith("__version__"): - version = line.split("=")[-1].strip()[1:-1] - - -assert version - - -def check_gpus(): - try: - n = 0 - for line in subprocess.check_output( - ["nvidia-smi", "-L"], - text=True, - ).split("\n"): - if line.startswith("GPU"): - n += 1 - return n - except (subprocess.CalledProcessError, FileNotFoundError): - return 0 - - -def has_gpu(): - return check_gpus() > 0 - - -onnxruntime = "onnxruntime" -if sys.platform == "darwin": - if platform.machine() == "arm64": - onnxruntime = "onnxruntime-silicon" - -if has_gpu(): - onnxruntime = "onnxruntime-gpu" - - -setuptools.setup( - name="ai-models-panguweather", - # python_requires="<3.11", # For now, does not support Python 3.11 - version=version, - description="An ai-models plugin to run PanguWeather", - long_description=read("README.md"), - long_description_content_type="text/markdown", - author="European Centre for Medium-Range Weather Forecasts (ECMWF)", - author_email="software.support@ecmwf.int", - license="Apache License Version 2.0", - url="https://github.com/ecmwf-lab/ai-models-panguweather", - packages=setuptools.find_packages(), - include_package_data=True, - setup_requires=["GPUtil"], - install_requires=[ - "ai-models>=0.4.0", - "onnx", - os.environ.get("ONNXRUNTIME", onnxruntime), - ], - zip_safe=True, - keywords="tool", - entry_points={ - "ai_models.model": [ - "panguweather = ai_models_panguweather.model:PanguWeather", - ] - }, - classifiers=[ - "Development Status :: 3 - Alpha", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Operating System :: OS Independent", - ], -) diff --git a/ai_models_panguweather/__init__.py b/src/ai_models_panguweather/__init__.py similarity index 100% rename from ai_models_panguweather/__init__.py rename to src/ai_models_panguweather/__init__.py diff --git a/ai_models_panguweather/model.py b/src/ai_models_panguweather/model.py similarity index 97% rename from ai_models_panguweather/model.py rename to src/ai_models_panguweather/model.py index 56c28c4..3a2f8b9 100644 --- a/ai_models_panguweather/model.py +++ b/src/ai_models_panguweather/model.py @@ -18,9 +18,7 @@ class PanguWeather(Model): # Download - download_url = ( - "https://get.ecmwf.int/repository/test-data/ai-models/pangu-weather/{file}" - ) + download_url = "https://get.ecmwf.int/repository/test-data/ai-models/pangu-weather/{file}" download_files = ["pangu_weather_24.onnx", "pangu_weather_6.onnx"] # Input diff --git a/tox.ini b/tox.ini index 02cc0f9..b42be33 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ profile = black max-line-length = 120 ; exclude = tests/* ; See https://black.readthedocs.io/en/stable/the_black_code_style.html -exclude = +exclude = dev/* experiments ?.py diff --git a/utils/pangu-gfs-input.py b/utils/pangu-gfs-input.py index 7db5345..64fe3ea 100755 --- a/utils/pangu-gfs-input.py +++ b/utils/pangu-gfs-input.py @@ -26,8 +26,7 @@ gfs = ekd.from_source( "url-pattern", - "https://data.rda.ucar.edu/ds084.1/{year}/{date}/" - "gfs.0p25.{date}{time}.f{step}.grib2", + "https://data.rda.ucar.edu/ds084.1/{year}/{date}/" "gfs.0p25.{date}{time}.f{step}.grib2", date=date, time=time, step="000", From e0dd987d60f63d57b655ee09b2103982c60a9aff Mon Sep 17 00:00:00 2001 From: Baudouin Raoult Date: Tue, 24 Sep 2024 07:24:29 +0000 Subject: [PATCH 3/4] remove tox --- tox.ini | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 tox.ini diff --git a/tox.ini b/tox.ini deleted file mode 100644 index b42be33..0000000 --- a/tox.ini +++ /dev/null @@ -1,13 +0,0 @@ -[isort] -profile = black - -[flake8] -; ignore = E226,E302,E41 -max-line-length = 120 -; exclude = tests/* -; See https://black.readthedocs.io/en/stable/the_black_code_style.html -exclude = - dev/* - experiments - ?.py -extend-ignore = E203 From 4a8f340b79db64e626f92a95ce081d772e94eba3 Mon Sep 17 00:00:00 2001 From: Baudouin Raoult Date: Tue, 24 Sep 2024 07:26:13 +0000 Subject: [PATCH 4/4] update workflow --- .github/workflows/python-publish.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 42926e9..e2ef8f5 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -15,14 +15,13 @@ jobs: name: Code QA runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - run: pip install black flake8 isort - - run: black --version - - run: isort --version - - run: flake8 --version - - run: isort --check . - - run: black --check . - - run: flake8 . + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - uses: pre-commit/action@v3.0.1 + env: + SKIP: no-commit-to-branch checks: if: ${{ github.event_name == 'release' }}