From d8c97fcde7f675ab88894a24eee7916d18d3558d Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Thu, 28 Nov 2024 11:54:24 +0100 Subject: [PATCH 1/4] prototype of so-only wheel and cffi interface --- .github/workflows/build-wheel-linux.yml | 91 +++++++++++++++++++++++ .gitignore | 4 + python_interface/README.md | 7 ++ python_interface/pyproject.toml | 8 ++ python_interface/src/pymetkit/__init__.py | 18 +++++ python_wrapper/buildconfig | 14 ++++ python_wrapper/setup.cfg | 5 ++ python_wrapper/setup.py | 2 + 8 files changed, 149 insertions(+) create mode 100644 .github/workflows/build-wheel-linux.yml create mode 100644 python_interface/README.md create mode 100644 python_interface/pyproject.toml create mode 100644 python_interface/src/pymetkit/__init__.py create mode 100644 python_wrapper/buildconfig create mode 100644 python_wrapper/setup.cfg create mode 100644 python_wrapper/setup.py diff --git a/.github/workflows/build-wheel-linux.yml b/.github/workflows/build-wheel-linux.yml new file mode 100644 index 00000000..0dea81b0 --- /dev/null +++ b/.github/workflows/build-wheel-linux.yml @@ -0,0 +1,91 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + + +name: Build Python Wheel for Linux + +on: + # Trigger the workflow manually + workflow_dispatch: ~ + + # Allow to be called from another workflow + workflow_call: ~ + + # TODO automation trigger + +jobs: + + build: + + runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] + # TODO which manylinux do we want to build for? 2014? 2_28? 2_34? Matrix? + container: wheelmaker_2_28:0.1 + + name: Build manylinux_2_28 + + steps: + # TODO which project do we build odc wheel from -- this, or some bundle? Or can we obtain the compiled eckit artifact somehow? + - run: git clone --branch develop --depth=1 https://github.com/ecmwf/eckit.git /src/eckit + - run: /buildscripts/compile.sh ./eckit/python_wrapper/buildconfig + - uses: actions/checkout@v2 + - run: /buildscripts/compile.sh ./metkit/python_wrapper/buildconfig + + ################################################################ + - run: /buildscripts/wheel-linux.sh ./metkit/python_wrapper/buildconfig 3.11 + - uses: actions/upload-artifact@v4 + name: Upload wheel 3.11 + with: + name: wheel-manylinux2_28-3.11 + path: /build/wheel/*.whl + + # TODO other python versions, once the above is correct. + # NOTE if Matrix, then break into (compile & upload) ; (wheel & upload)[matix] steps + + test: + + needs: build + strategy: + fail-fast: false + matrix: + python-version: ["3.11"] # ["3.8", "3.9", "3.10", "3.11", "3.12"] # TODO enable + + name: Test with ${{ matrix.python-version }} + runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] + container: wheelmaker_2_28:0.1 + steps: + - uses: actions/checkout@v2 + - uses: actions/download-artifact@v4 + with: + name: wheel-manylinux2_28-${{ matrix.python-version }} + - run: /buildscripts/test-wheel.sh ${{ matrix.python-version }} + +# TODO enable and test +# deploy: +# +# if: ${{ github.ref_type == 'tag' || github.event_name == 'release' }} +# needs: [test, build] +# strategy: +# fail-fast: false +# matrix: +# python-version: ["3.11"] # ["3.8", "3.9", "3.10", "3.11", "3.12"] # TODO enable +# +# name: Deploy wheel ${{ matrix.python-version }} +# runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] +# container: wheelmaker_2_28:0.1 +# steps: +# - run: mkdir artifact-${{ matrix.python-version }} +# - uses: actions/checkout@v2 +# - uses: actions/download-artifact@v4 +# with: +# name: wheel-manylinux2_28-${{ matrix.python-version }} +# path: artifact-${{ matrix.python-version }} +# - run: | +# /buildsripts/upload-twine.sh ${{ matrix.python-version }} +# env: +# TWINE_USERNAME: __token__ +# TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.gitignore b/.gitignore index e4cea70b..350c3457 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,7 @@ doc/latex .*un~ *.ccls-cache share +__pycache__/ +*.pyc +*.egg-info/ + diff --git a/python_interface/README.md b/python_interface/README.md new file mode 100644 index 00000000..e349a5da --- /dev/null +++ b/python_interface/README.md @@ -0,0 +1,7 @@ +This is just a placeholder for proper python interface of metkit -- for now we just showcase `findlibs`-based finding of `.so`, and loading via cffi. + +To demonstrate functionality, install `eckit` and `metkit` binary wheels in your venv, and then you can `pip install -e . && python -c 'import metkit; metkit.version()`. + +To be done: +1. proper cffi setup +2. wheel building (note this is **not** a binary wheel! That happens in `../python_wrapper`, and contains no python code) diff --git a/python_interface/pyproject.toml b/python_interface/pyproject.toml new file mode 100644 index 00000000..c1aa4aff --- /dev/null +++ b/python_interface/pyproject.toml @@ -0,0 +1,8 @@ +[build-system] +requires = ["setuptools >= 61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "pymetkit" +version = "0.0.0" +dependencies = ["cffi", "findlibs"] diff --git a/python_interface/src/pymetkit/__init__.py b/python_interface/src/pymetkit/__init__.py new file mode 100644 index 00000000..47439340 --- /dev/null +++ b/python_interface/src/pymetkit/__init__.py @@ -0,0 +1,18 @@ +import cffi +import findlibs + +_lib = None + +# TODO expose the full functionality + +def _get_lib(): + global _lib + if _lib is None: + ffi = cffi.FFI() + ffi.cdef("unsigned int metkit_version_int();") + loc = findlibs.find("metkit", "metkitlibs") + _lib = ffi.dlopen(loc) + return _lib + +def metkit_version_int() -> int: + return _get_lib().metkit_version_int() diff --git a/python_wrapper/buildconfig b/python_wrapper/buildconfig new file mode 100644 index 00000000..e4fe1183 --- /dev/null +++ b/python_wrapper/buildconfig @@ -0,0 +1,14 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +# to be source'd by wheelmaker's compile.sh *and* wheel-linux.sh +# NOTE replace the whole thing with pyproject.toml? Less powerful, and quaint to use for sourcing ecbuild invocation + +NAME="metkit" +CMAKE_PARAMS="-DECKIT_PATH=/target/eckit" +PYPROJECT_DIR="python_wrapper" diff --git a/python_wrapper/setup.cfg b/python_wrapper/setup.cfg new file mode 100644 index 00000000..7d273367 --- /dev/null +++ b/python_wrapper/setup.cfg @@ -0,0 +1,5 @@ +[metadata] +description = "metkit" +long_description = file: README.md +long_description_content_type = text/markdown +author = file: AUTHORS diff --git a/python_wrapper/setup.py b/python_wrapper/setup.py new file mode 100644 index 00000000..c64ad6d7 --- /dev/null +++ b/python_wrapper/setup.py @@ -0,0 +1,2 @@ +from setup_utils import plain_setup +plain_setup() From 684a9ac090f84ff2cc6993c7f5ecc4d68df81fc2 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Thu, 12 Dec 2024 13:37:47 +0100 Subject: [PATCH 2/4] Update wheelmaker to 0.10 --- .github/workflows/build-wheel-linux.yml | 87 ++++++------------------- python_wrapper/buildconfig | 4 +- 2 files changed, 24 insertions(+), 67 deletions(-) diff --git a/.github/workflows/build-wheel-linux.yml b/.github/workflows/build-wheel-linux.yml index 0dea81b0..c146e1f7 100644 --- a/.github/workflows/build-wheel-linux.yml +++ b/.github/workflows/build-wheel-linux.yml @@ -19,73 +19,28 @@ on: # TODO automation trigger jobs: - build: - - runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] - # TODO which manylinux do we want to build for? 2014? 2_28? 2_34? Matrix? - container: wheelmaker_2_28:0.1 - name: Build manylinux_2_28 - - steps: - # TODO which project do we build odc wheel from -- this, or some bundle? Or can we obtain the compiled eckit artifact somehow? - - run: git clone --branch develop --depth=1 https://github.com/ecmwf/eckit.git /src/eckit - - run: /buildscripts/compile.sh ./eckit/python_wrapper/buildconfig - - uses: actions/checkout@v2 - - run: /buildscripts/compile.sh ./metkit/python_wrapper/buildconfig - - ################################################################ - - run: /buildscripts/wheel-linux.sh ./metkit/python_wrapper/buildconfig 3.11 - - uses: actions/upload-artifact@v4 - name: Upload wheel 3.11 - with: - name: wheel-manylinux2_28-3.11 - path: /build/wheel/*.whl - - # TODO other python versions, once the above is correct. - # NOTE if Matrix, then break into (compile & upload) ; (wheel & upload)[matix] steps - - test: - - needs: build - strategy: - fail-fast: false - matrix: - python-version: ["3.11"] # ["3.8", "3.9", "3.10", "3.11", "3.12"] # TODO enable - - name: Test with ${{ matrix.python-version }} runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] - container: wheelmaker_2_28:0.1 + # TODO which manylinux do we want to build for? 2014? 2_28? 2_34? Matrix? + container: + image: eccr.ecmwf.int/wheelmaker/2_28:0.10 + credentials: + username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} + password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v4 - with: - name: wheel-manylinux2_28-${{ matrix.python-version }} - - run: /buildscripts/test-wheel.sh ${{ matrix.python-version }} - -# TODO enable and test -# deploy: -# -# if: ${{ github.ref_type == 'tag' || github.event_name == 'release' }} -# needs: [test, build] -# strategy: -# fail-fast: false -# matrix: -# python-version: ["3.11"] # ["3.8", "3.9", "3.10", "3.11", "3.12"] # TODO enable -# -# name: Deploy wheel ${{ matrix.python-version }} -# runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] -# container: wheelmaker_2_28:0.1 -# steps: -# - run: mkdir artifact-${{ matrix.python-version }} -# - uses: actions/checkout@v2 -# - uses: actions/download-artifact@v4 -# with: -# name: wheel-manylinux2_28-${{ matrix.python-version }} -# path: artifact-${{ matrix.python-version }} -# - run: | -# /buildsripts/upload-twine.sh ${{ matrix.python-version }} -# env: -# TWINE_USERNAME: __token__ -# TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + # TODO convert this to be matrix-friendly. Note it's a bit tricky since + # we'd ideally not reexecute the compile step multiple times, but it + # (non-essentially) depends on a matrix-based step + # NOTE we dont use action checkout because it doesnt cleanup after itself correctly + - run: git clone --depth=1 --branch="${GITHUB_REF#refs/heads/}" https://github.com/$GITHUB_REPOSITORY /proj + - run: cd /proj && /buildscripts/prepare_deps.sh ./python_wrapper/buildconfig 3.11 + - run: cd /proj && /buildscripts/compile.sh ./python_wrapper/buildconfig + - run: cd /proj && PYTHONPATH=/buildscripts /buildscripts/wheel-linux.sh ./python_wrapper/buildconfig 3.11 + - run: cd /proj && /buildscripts/test-wheel.sh ./python_wrapper/buildconfig 3.11 /tmp/build/wheel/*whl + - run: cd /proj && /buildscripts/upload-pypi.sh /tmp/build/wheel/*whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + # NOTE temporary thing until all the mess gets cleared + - run: rm -rf ./* ./.git ./.github diff --git a/python_wrapper/buildconfig b/python_wrapper/buildconfig index e4fe1183..e8470d87 100644 --- a/python_wrapper/buildconfig +++ b/python_wrapper/buildconfig @@ -8,7 +8,9 @@ # to be source'd by wheelmaker's compile.sh *and* wheel-linux.sh # NOTE replace the whole thing with pyproject.toml? Less powerful, and quaint to use for sourcing ecbuild invocation +# TODO we duplicate information -- pyproject.toml's `name` and `packages` are derivable from $NAME and must stay consistent NAME="metkit" -CMAKE_PARAMS="-DECKIT_PATH=/target/eckit" +CMAKE_PARAMS="-DECKIT_PATH=/tmp/target/eckit" PYPROJECT_DIR="python_wrapper" +DEPENDENCIES='["eckit"]' From 6a5385abf96a94dee99bcc0dc8f4c09d96cf3c89 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Mon, 16 Dec 2024 12:58:29 +0100 Subject: [PATCH 3/4] Python package name consolidation --- .github/workflows/build-wheel-linux.yml | 2 +- python_wrapper/buildconfig | 4 ++-- python_wrapper/setup.cfg | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-wheel-linux.yml b/.github/workflows/build-wheel-linux.yml index c146e1f7..92fce300 100644 --- a/.github/workflows/build-wheel-linux.yml +++ b/.github/workflows/build-wheel-linux.yml @@ -24,7 +24,7 @@ jobs: runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] # TODO which manylinux do we want to build for? 2014? 2_28? 2_34? Matrix? container: - image: eccr.ecmwf.int/wheelmaker/2_28:0.10 + image: eccr.ecmwf.int/wheelmaker/2_28:latest credentials: username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} diff --git a/python_wrapper/buildconfig b/python_wrapper/buildconfig index e8470d87..30eb14e3 100644 --- a/python_wrapper/buildconfig +++ b/python_wrapper/buildconfig @@ -11,6 +11,6 @@ # TODO we duplicate information -- pyproject.toml's `name` and `packages` are derivable from $NAME and must stay consistent NAME="metkit" -CMAKE_PARAMS="-DECKIT_PATH=/tmp/target/eckit" +CMAKE_PARAMS="-DECKIT_PATH=/tmp/prereqs/eckitlib" PYPROJECT_DIR="python_wrapper" -DEPENDENCIES='["eckit"]' +DEPENDENCIES='["eckitlib"]' diff --git a/python_wrapper/setup.cfg b/python_wrapper/setup.cfg index 7d273367..3826781a 100644 --- a/python_wrapper/setup.cfg +++ b/python_wrapper/setup.cfg @@ -1,5 +1,5 @@ [metadata] -description = "metkit" +description = "metkitlib" long_description = file: README.md long_description_content_type = text/markdown author = file: AUTHORS From 5aa776235e3b5e3dbd68fe4ab27fd5c1b5126ded Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Fri, 10 Jan 2025 15:35:04 +0100 Subject: [PATCH 4/4] Renaming and dependency fix --- python_interface/pyproject.toml | 4 ++-- python_interface/src/{pymetkit => metkit}/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename python_interface/src/{pymetkit => metkit}/__init__.py (86%) diff --git a/python_interface/pyproject.toml b/python_interface/pyproject.toml index c1aa4aff..d109e433 100644 --- a/python_interface/pyproject.toml +++ b/python_interface/pyproject.toml @@ -3,6 +3,6 @@ requires = ["setuptools >= 61.0"] build-backend = "setuptools.build_meta" [project] -name = "pymetkit" +name = "metkit" version = "0.0.0" -dependencies = ["cffi", "findlibs"] +dependencies = ["cffi", "findlibs", "metkitlib"] diff --git a/python_interface/src/pymetkit/__init__.py b/python_interface/src/metkit/__init__.py similarity index 86% rename from python_interface/src/pymetkit/__init__.py rename to python_interface/src/metkit/__init__.py index 47439340..dceae240 100644 --- a/python_interface/src/pymetkit/__init__.py +++ b/python_interface/src/metkit/__init__.py @@ -10,7 +10,7 @@ def _get_lib(): if _lib is None: ffi = cffi.FFI() ffi.cdef("unsigned int metkit_version_int();") - loc = findlibs.find("metkit", "metkitlibs") + loc = findlibs.find("metkit", "metkitlib") _lib = ffi.dlopen(loc) return _lib