From bbf73661239ffbf34791db3b89858aa067ec00b7 Mon Sep 17 00:00:00 2001 From: Saransh Chopra Date: Tue, 26 Nov 2024 14:46:49 +0000 Subject: [PATCH 1/9] gh-439: a framework for testing array API compatibility --- .github/workflows/test.yml | 15 ++++++- CONTRIBUTING.md | 53 +++++++++++++++++++++++ noxfile.py | 10 +++++ tests/conftest.py | 89 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 166 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f6aef75d..867409a1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,10 +56,23 @@ jobs: env: FORCE_COLOR: 1 - - name: Run tests and generate coverage report + - name: Run NumPy tests and generate coverage report run: nox -s coverage-${{ matrix.python-version }} --verbose env: FORCE_COLOR: 1 + GLASS_ARRAY_BACKEND: numpy + + - name: Run array API strict tests + run: nox -s doctests-${{ matrix.python-version }} --verbose + env: + FORCE_COLOR: 1 + GLASS_ARRAY_BACKEND: array_api_strict + + - name: Run JAX tests + run: nox -s doctests-${{ matrix.python-version }} --verbose + env: + FORCE_COLOR: 1 + GLASS_ARRAY_BACKEND: jax - name: Coveralls requires XML report run: coverage xml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index efd3b302..2243c0f7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,6 +64,42 @@ following way - python -m pytest --cov --doctest-plus ``` +### Array API tests + +One can specify a particular array backend for testing by setting the +`GLASS_ARRAY_BACKEND` environment variable. The default array backend is NumPy. +_GLASS_ can be tested with every supported array library available in the +environment by setting `GLASS_ARRAY_BACKEND` to `all`. The testing framework +only installs NumPy automatically; hence, remaining array libraries should +either be installed manually or developers should use `Nox`. + +```bash +# run tests using numpy +python -m pytest +GLASS_ARRAY_BACKEND=numpy python -m pytest +# run tests using array_api_strict (should be installed manually) +GLASS_ARRAY_BACKEND=array_api_strict python -m pytest +# run tests using jax (should be installed manually) +GLASS_ARRAY_BACKEND=jax python -m pytest +# run tests using every supported array library available in the environment +GLASS_ARRAY_BACKEND=all python -m pytest +``` + +Moreover, one can mark a test to be compatible with the array API standard by +decorating it with `@array_api_compatible`. This will `parameterize` the test to +run on every array library specified through `GLASS_ARRAY_BACKEND` - + +```py +import types +from tests.conftest import array_api_compatible + + +@array_api_compatible +def test_something(xp: types.ModuleType): + # use `xp.` to access the array library functionality + ... +``` + ## Documenting _GLASS_'s documentation is mainly written in the form of @@ -173,6 +209,23 @@ syntax - nox -s tests-3.11 ``` +One can specify a particular array backend for testing by setting the +`GLASS_ARRAY_BACKEND` environment variable. The default array backend is NumPy. +_GLASS_ can be tested with every supported array library by setting +`GLASS_ARRAY_BACKEND` to `all`. + +```bash +# run tests using numpy +nox -s tests-3.11 +GLASS_ARRAY_BACKEND=numpy nox -s tests-3.11 +# run tests using array_api_strict +GLASS_ARRAY_BACKEND=array_api_strict nox -s tests-3.11 +# run tests using jax +GLASS_ARRAY_BACKEND=jax nox -s tests-3.11 +# run tests using every supported array library +GLASS_ARRAY_BACKEND=all nox -s tests-3.11 +``` + The following command can be used to deploy the docs on `localhost` - ```bash diff --git a/noxfile.py b/noxfile.py index 6a45d5c7..0be65a22 100644 --- a/noxfile.py +++ b/noxfile.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os from pathlib import Path import nox @@ -29,6 +30,15 @@ def lint(session: nox.Session) -> None: def tests(session: nox.Session) -> None: """Run the unit tests.""" session.install("-c", ".github/test-constraints.txt", "-e", ".[test]") + + array_backend = os.environ.get("GLASS_ARRAY_BACKEND") + if array_backend == "array_api_strict": + session.install("array_api_strict>=2") + elif array_backend == "jax": + session.install("jax>=0.4.32") + elif array_backend == "all": + session.install("array_api_strict>=2", "jax>=0.4.32") + session.run( "pytest", *session.posargs, diff --git a/tests/conftest.py b/tests/conftest.py index ab4ae201..35e35d49 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,100 @@ +import contextlib +import importlib.metadata +import os +import types + import numpy as np import numpy.typing as npt +import packaging.version import pytest from cosmology import Cosmology from glass import RadialWindow +# environment variable to specify array backends for testing +# can be: +# a particular array library (numpy, jax, array_api_strict, ...) +# all (try finding every supported array library available in the environment) +GLASS_ARRAY_BACKEND: str | bool = os.environ.get("GLASS_ARRAY_BACKEND", False) + + +def _check_version(lib: str, array_api_compliant_version: str) -> None: + """ + Check if installed library's version is compliant with the array API standard. + + Parameters + ---------- + lib + name of the library. + array_api_compliant_version + version of the library compliant with the array API standard. + + Raises + ------ + ImportError + If the installed version is not compliant with the array API standard. + """ + lib_version = packaging.version.Version(importlib.metadata.version(lib)) + if lib_version < packaging.version.Version(array_api_compliant_version): + msg = f"{lib} must be >= {array_api_compliant_version}; found {lib_version}" + raise ImportError(msg) + + +def _import_and_add_numpy(xp_available_backends: dict[str, types.ModuleType]) -> None: + """Add numpy to the backends dictionary.""" + _check_version("numpy", "2.1.0") + xp_available_backends.update({"numpy": np}) + + +def _import_and_add_array_api_strict( + xp_available_backends: dict[str, types.ModuleType], +) -> None: + """Add array_api_strict to the backends dictionary.""" + import array_api_strict + + _check_version("array_api_strict", "2.0.0") + xp_available_backends.update({"array_api_strict": array_api_strict}) + array_api_strict.set_array_api_strict_flags(api_version="2023.12") + + +def _import_and_add_jax(xp_available_backends: dict[str, types.ModuleType]) -> None: + """Add jax to the backends dictionary.""" + import jax + + _check_version("jax", "0.4.32") + xp_available_backends.update({"jax.numpy": jax.numpy}) + # enable 64 bit numbers + jax.config.update("jax_enable_x64", val=True) + + +# a dictionary with all array backends to test +xp_available_backends: dict[str, types.ModuleType] = {} + +# if no backend passed, use numpy by default +if not GLASS_ARRAY_BACKEND or GLASS_ARRAY_BACKEND == "numpy": + _import_and_add_numpy(xp_available_backends) +elif GLASS_ARRAY_BACKEND == "array_api_strict": + _import_and_add_array_api_strict(xp_available_backends) +elif GLASS_ARRAY_BACKEND == "jax": + _import_and_add_jax(xp_available_backends) +# if all, try importing every backend +elif GLASS_ARRAY_BACKEND == "all": + with contextlib.suppress(ImportError): + _import_and_add_numpy(xp_available_backends) + + with contextlib.suppress(ImportError): + _import_and_add_array_api_strict(xp_available_backends) + + with contextlib.suppress(ImportError): + _import_and_add_jax(xp_available_backends) +else: + msg = f"unsupported array backend: {GLASS_ARRAY_BACKEND}" + raise ValueError(msg) + +# use this as a decorator for tests involving array API compatible functions +array_api_compatible = pytest.mark.parametrize("xp", xp_available_backends.values()) + @pytest.fixture(scope="session") def cosmo() -> Cosmology: From e90885ccfa6bb1d8255bf3991bdaeb784b33d4c9 Mon Sep 17 00:00:00 2001 From: Saransh Chopra Date: Tue, 26 Nov 2024 14:51:50 +0000 Subject: [PATCH 2/9] fix tests --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 867409a1..ae3925b8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,13 +63,13 @@ jobs: GLASS_ARRAY_BACKEND: numpy - name: Run array API strict tests - run: nox -s doctests-${{ matrix.python-version }} --verbose + run: nox -s tests-${{ matrix.python-version }} --verbose env: FORCE_COLOR: 1 GLASS_ARRAY_BACKEND: array_api_strict - name: Run JAX tests - run: nox -s doctests-${{ matrix.python-version }} --verbose + run: nox -s tests-${{ matrix.python-version }} --verbose env: FORCE_COLOR: 1 GLASS_ARRAY_BACKEND: jax From 60c377f05ae9eee64f633d2f7795818b2afc7745 Mon Sep 17 00:00:00 2001 From: Saransh Chopra Date: Tue, 26 Nov 2024 15:18:44 +0000 Subject: [PATCH 3/9] run test once --- .github/workflows/test.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ae3925b8..edf2b2b6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,23 +56,11 @@ jobs: env: FORCE_COLOR: 1 - - name: Run NumPy tests and generate coverage report + - name: Run tests wih every array backend and generate coverage report run: nox -s coverage-${{ matrix.python-version }} --verbose env: FORCE_COLOR: 1 - GLASS_ARRAY_BACKEND: numpy - - - name: Run array API strict tests - run: nox -s tests-${{ matrix.python-version }} --verbose - env: - FORCE_COLOR: 1 - GLASS_ARRAY_BACKEND: array_api_strict - - - name: Run JAX tests - run: nox -s tests-${{ matrix.python-version }} --verbose - env: - FORCE_COLOR: 1 - GLASS_ARRAY_BACKEND: jax + GLASS_ARRAY_BACKEND: all - name: Coveralls requires XML report run: coverage xml From fe4dd97e2031520d981bc0b1c9becf8af9bf4217 Mon Sep 17 00:00:00 2001 From: Saransh Chopra Date: Tue, 26 Nov 2024 15:33:28 +0000 Subject: [PATCH 4/9] Update tests/conftest.py --- tests/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 35e35d49..1a024f11 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -43,6 +43,8 @@ def _check_version(lib: str, array_api_compliant_version: str) -> None: def _import_and_add_numpy(xp_available_backends: dict[str, types.ModuleType]) -> None: """Add numpy to the backends dictionary.""" + import numpy + _check_version("numpy", "2.1.0") xp_available_backends.update({"numpy": np}) From e01e388668efff47336a4571af8302c481d0d96a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 26 Nov 2024 15:36:13 +0000 Subject: [PATCH 5/9] pre-commit.ci: style fixes --- tests/conftest.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 1a024f11..35e35d49 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -43,8 +43,6 @@ def _check_version(lib: str, array_api_compliant_version: str) -> None: def _import_and_add_numpy(xp_available_backends: dict[str, types.ModuleType]) -> None: """Add numpy to the backends dictionary.""" - import numpy - _check_version("numpy", "2.1.0") xp_available_backends.update({"numpy": np}) From b4e5b4bdfec738e643f16242da93eb6a3fb9e5d8 Mon Sep 17 00:00:00 2001 From: Saransh Chopra Date: Wed, 27 Nov 2024 13:04:39 +0000 Subject: [PATCH 6/9] Apply suggestions from code review Co-authored-by: Patrick J. Roddy --- CONTRIBUTING.md | 2 +- noxfile.py | 6 +++--- tests/conftest.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2243c0f7..2dabf3c3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -89,7 +89,7 @@ Moreover, one can mark a test to be compatible with the array API standard by decorating it with `@array_api_compatible`. This will `parameterize` the test to run on every array library specified through `GLASS_ARRAY_BACKEND` - -```py +```python import types from tests.conftest import array_api_compatible diff --git a/noxfile.py b/noxfile.py index 0be65a22..61d0d680 100644 --- a/noxfile.py +++ b/noxfile.py @@ -33,11 +33,11 @@ def tests(session: nox.Session) -> None: array_backend = os.environ.get("GLASS_ARRAY_BACKEND") if array_backend == "array_api_strict": - session.install("array_api_strict>=2") + session.install(ARRAY_BACKENDS["array_api_strict"]) elif array_backend == "jax": - session.install("jax>=0.4.32") + session.install(ARRAY_BACKENDS["jax"]) elif array_backend == "all": - session.install("array_api_strict>=2", "jax>=0.4.32") + session.install(ARRAY_BACKENDS.values()) session.run( "pytest", diff --git a/tests/conftest.py b/tests/conftest.py index 35e35d49..5eead72c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,7 +16,7 @@ # can be: # a particular array library (numpy, jax, array_api_strict, ...) # all (try finding every supported array library available in the environment) -GLASS_ARRAY_BACKEND: str | bool = os.environ.get("GLASS_ARRAY_BACKEND", False) +GLASS_ARRAY_BACKEND: str = os.environ.get("GLASS_ARRAY_BACKEND", "") def _check_version(lib: str, array_api_compliant_version: str) -> None: From d19359d3311aeef2b68143397881d2465d67454d Mon Sep 17 00:00:00 2001 From: Saransh Chopra Date: Wed, 27 Nov 2024 13:07:13 +0000 Subject: [PATCH 7/9] fix noxfile --- CONTRIBUTING.md | 12 ++++++------ noxfile.py | 6 +++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2dabf3c3..00f55e8b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -202,11 +202,11 @@ nox -s tests Only `tests`, `coverage`, and the `doctests` session run on all supported Python versions by default. -To specify a particular Python version (for example `3.11`), use the following +To specify a particular Python version (for example `3.13`), use the following syntax - ```bash -nox -s tests-3.11 +nox -s tests-3.13 ``` One can specify a particular array backend for testing by setting the @@ -217,13 +217,13 @@ _GLASS_ can be tested with every supported array library by setting ```bash # run tests using numpy nox -s tests-3.11 -GLASS_ARRAY_BACKEND=numpy nox -s tests-3.11 +GLASS_ARRAY_BACKEND=numpy nox -s tests-3.13 # run tests using array_api_strict -GLASS_ARRAY_BACKEND=array_api_strict nox -s tests-3.11 +GLASS_ARRAY_BACKEND=array_api_strict nox -s tests-3.13 # run tests using jax -GLASS_ARRAY_BACKEND=jax nox -s tests-3.11 +GLASS_ARRAY_BACKEND=jax nox -s tests-3.13 # run tests using every supported array library -GLASS_ARRAY_BACKEND=all nox -s tests-3.11 +GLASS_ARRAY_BACKEND=all nox -s tests-3.13 ``` The following command can be used to deploy the docs on `localhost` - diff --git a/noxfile.py b/noxfile.py index 61d0d680..bdfd528a 100644 --- a/noxfile.py +++ b/noxfile.py @@ -17,6 +17,10 @@ "3.12", "3.13", ] +ARRAY_BACKENDS = { + "array_api_strict": "array_api_strict>=2", + "jax": "jax>=0.4.32", +} @nox.session @@ -37,7 +41,7 @@ def tests(session: nox.Session) -> None: elif array_backend == "jax": session.install(ARRAY_BACKENDS["jax"]) elif array_backend == "all": - session.install(ARRAY_BACKENDS.values()) + session.install(*ARRAY_BACKENDS.values()) session.run( "pytest", From 7414f6e1469b3f9bcd516566a478b31f5620cf5d Mon Sep 17 00:00:00 2001 From: Saransh Chopra Date: Wed, 27 Nov 2024 13:13:02 +0000 Subject: [PATCH 8/9] better comments --- tests/conftest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 5eead72c..cb2181a0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,6 +12,9 @@ from glass import RadialWindow +# Handling of array backends, inspired by- +# https://github.com/scipy/scipy/blob/36e349b6afbea057cb713fc314296f10d55194cc/scipy/conftest.py#L139 + # environment variable to specify array backends for testing # can be: # a particular array library (numpy, jax, array_api_strict, ...) @@ -96,6 +99,7 @@ def _import_and_add_jax(xp_available_backends: dict[str, types.ModuleType]) -> N array_api_compatible = pytest.mark.parametrize("xp", xp_available_backends.values()) +# Pytest fixtures @pytest.fixture(scope="session") def cosmo() -> Cosmology: class MockCosmology: From 1b0d92bcd396c9d320f54f973c9b7b1a3509e4d8 Mon Sep 17 00:00:00 2001 From: Saransh Chopra Date: Thu, 28 Nov 2024 11:26:42 +0000 Subject: [PATCH 9/9] 3.13 --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 00f55e8b..f761dbaa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -216,7 +216,7 @@ _GLASS_ can be tested with every supported array library by setting ```bash # run tests using numpy -nox -s tests-3.11 +nox -s tests-3.13 GLASS_ARRAY_BACKEND=numpy nox -s tests-3.13 # run tests using array_api_strict GLASS_ARRAY_BACKEND=array_api_strict nox -s tests-3.13