Refactor tests into scripts (#832)

* Complete testing script.

* Test scripts in CI and update `contributing.md` to use script.

.github/workflows/ci.yml

@@ -30,7 +30,6 @@ jobs:
     env:
       PYTHON_VERSION: ${{ matrix.python }}
       NUMBA_BOUNDSCHECK: ${{ matrix.numba_boundscheck }}
-      PYTHONFAULTHANDLER: '${{ github.workspace }}/faulthandler.log'
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v4
@@ -49,15 +48,7 @@ jobs:
         run: |
           pip install -e '.[tests]'
       - name: Run tests
-        run: |
-          if [ $(python -c 'import numpy as np; print(np.lib.NumpyVersion(np.__version__) >= "2.0.0a1")') = 'True' ]; then
-            pytest --pyargs sparse --doctest-modules --cov-report=xml:coverage_Numba.xml -n auto -vvv
-          else
-            pytest --pyargs sparse --cov-report=xml:coverage_Numba.xml -n auto -vvv
-          fi
-          python -c 'import finch'
-          SPARSE_BACKEND=Finch pytest --pyargs sparse/tests --cov-report=xml:coverage_Finch.xml -n auto -vvv
-          SPARSE_BACKEND=MLIR pytest --pyargs sparse/mlir_backend --cov-report=xml:coverage_MLIR.xml -n auto -vvv
+        run: ci/test_backends.sh
       - uses: codecov/codecov-action@v5
         if: always()
         with:
@@ -79,8 +70,7 @@ jobs:
         pip install -U setuptools wheel
         pip install '.[finch]' scipy
     - name: Run examples
-      run: |
-        source ci/test_examples.sh
+      run: ci/test_examples.sh
 
   notebooks:
     runs-on: ubuntu-latest
@@ -96,46 +86,36 @@ jobs:
       run: |
         pip install -e '.[notebooks]'
     - name: Run notebooks
-      run: |
-        source ci/test_notebooks.sh
+      run: ci/test_notebooks.sh
 
   array_api_tests:
     strategy:
       matrix:
         backend: ['Numba', 'Finch']
       fail-fast: false
+    env:
+      ARRAY_API_TESTS_DIR: ${{ github.workspace }}/array-api-tests
     runs-on: ubuntu-latest
     steps:
     - name: Checkout Repo
       uses: actions/checkout@v4
     - name: Checkout array-api-tests
-      uses: actions/checkout@v4
-      with:
-        repository: data-apis/array-api-tests
-        ref: '33f2d2ea2f3dd2b3ceeeb4519d55e08096184149'  # Latest commit as of 2024-05-29
-        submodules: 'true'
-        path: 'array-api-tests'
+      run: ci/clone_array_api_tests.sh
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
         python-version: '3.11'
         cache: 'pip'
     - name: Install build and test dependencies from PyPI
       run: |
-        pip install pytest-xdist -r array-api-tests/requirements.txt
+        pip install pytest-xdist -r "$ARRAY_API_TESTS_DIR/requirements.txt"
     - name: Build and install Sparse
       run: |
         pip install '.[finch]'
     - name: Run the test suite
       env:
-        ARRAY_API_TESTS_MODULE: sparse
         SPARSE_BACKEND: ${{ matrix.backend }}
-      run: |
-        cd ${GITHUB_WORKSPACE}/array-api-tests
-        if [ "${SPARSE_BACKEND}" = "Finch" ]; then
-            python -c 'import finch'
-        fi
-        pytest array_api_tests -v -c pytest.ini -n auto --max-examples=2 --derandomize --disable-deadline -o xfail_strict=True --xfails-file ${GITHUB_WORKSPACE}/ci/${{ matrix.backend }}-array-api-xfails.txt --skips-file ${GITHUB_WORKSPACE}/ci/${{ matrix.backend }}-array-api-skips.txt
+      run: ci/test_array_api.sh
 
 on:
   # Trigger the workflow on push or pull request,

.gitignore

@@ -84,6 +84,7 @@ results/
 # Notebooks converted to scripts.
 docs/examples_ipynb/
 
-# Pixi envs
+# Envs
 .pixi/
 pixi.lock
+.venv/

ci/array-api-tests-rev.txt

@@ -0,0 +1 @@
+33f2d2ea2f3dd2b3ceeeb4519d55e08096184149

ci/clone_array_api_tests.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+ARRAY_API_TESTS_DIR="${ARRAY_API_TESTS_DIR:-"../array-api-tests"}"
+if [ ! -d "$ARRAY_API_TESTS_DIR" ]; then
+  git clone --recursive https://github.com/data-apis/array-api-tests.git "$ARRAY_API_TESTS_DIR"
+fi
+
+git --git-dir="$ARRAY_API_TESTS_DIR/.git" --work-tree "$ARRAY_API_TESTS_DIR" clean -xddf
+git --git-dir="$ARRAY_API_TESTS_DIR/.git" --work-tree "$ARRAY_API_TESTS_DIR" fetch
+git --git-dir="$ARRAY_API_TESTS_DIR/.git" --work-tree "$ARRAY_API_TESTS_DIR" reset --hard $(cat "ci/array-api-tests-rev.txt")

ci/setup_env.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+if [ ! -d ".venv" ]; then
+  python -m venv .venv
+  source .venv/bin/activate
+  pip install -e .[all]
+  source ci/clone_array_api_tests.sh
+  pip install -r ../array-api-tests/requirements.txt
+  pip uninstall -y matrepr
+fi

ci/test_Finch.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+python -c 'import finch'
+PYTHONFAULTHANDLER="${HOME}/faulthandler.log" SPARSE_BACKEND=Finch pytest --pyargs sparse/tests --cov-report=xml:coverage_Finch.xml -n auto -vvv

ci/test_MLIR.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+SPARSE_BACKEND=MLIR pytest --pyargs sparse/mlir_backend --cov-report=xml:coverage_MLIR.xml -n auto -vvv

ci/test_Numba.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+if [ $(python -c 'import numpy as np; print(np.lib.NumpyVersion(np.__version__) >= "2.0.0a1")') = 'True' ]; then
+    pytest --pyargs sparse --doctest-modules --cov-report=xml:coverage_Numba.xml -n auto -vvv
+else
+    pytest --pyargs sparse --cov-report=xml:coverage_Numba.xml -n auto -vvv
+fi

ci/test_all.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+ACTIVATE_VENV="${ACTIVATE_VENV:-0}"
+
+if [ $ACTIVATE_VENV = "1" ]; then
+    source .venv/bin/activate
+fi
+
+source ci/test_backends.sh
+source ci/test_examples.sh
+source ci/test_notebooks.sh
+SPARSE_BACKEND="Numba" source ci/test_array_api.sh
+SPARSE_BACKEND="Finch" PYTHONFAULTHANDLER="${HOME}/faulthandler.log" source ci/test_array_api.sh

ci/test_array_api.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+source ci/clone_array_api_tests.sh
+
+if [ "${SPARSE_BACKEND}" = "Finch" ]; then
+    python -c 'import finch'
+fi
+ARRAY_API_TESTS_MODULE="sparse" pytest "$ARRAY_API_TESTS_DIR/array_api_tests/" -v -c "$ARRAY_API_TESTS_DIR/pytest.ini" --ci --max-examples=2 --derandomize --disable-deadline -o xfail_strict=True -n auto --xfails-file ../sparse/ci/${SPARSE_BACKEND}-array-api-xfails.txt --skips-file ../sparse/ci/${SPARSE_BACKEND}-array-api-skips.txt

ci/test_backends.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+source ci/test_Numba.sh
+source ci/test_Finch.sh
+source ci/test_MLIR.sh

ci/test_examples.sh

@@ -1,3 +1,6 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
 for example in $(find ./docs/examples/ -iname *.py); do
   CI_MODE=1 python $example
 done

ci/test_notebooks.sh

@@ -1 +1,4 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
 CI_MODE=1 pytest -n 4 --nbmake --nbmake-timeout=600 ./examples/*.ipynb

docs/contributing.md

@@ -69,7 +69,6 @@ check code style. You don't need to configure these extensions yourself. Once yo
 configured your environment, you can just `cd` to the root of your repository and run
 
 ```bash
-
 pytest --pyargs sparse
 ```
 
@@ -78,6 +77,16 @@ even though it doesn't fail on low coverage.
 
 Unit tests are automatically run on Travis CI for pull requests.
 
+### Advanced
+
+To run the complete set of unit tests run in CI for your platform, run the following
+in the repository root:
+
+```bash
+ci/setup_env.sh
+ACTIVATE_VENV=1 ci/test_all.sh
+```
+
 ## Coverage
 
 The `pytest` script automatically reports coverage, both on the terminal for

pixi.toml

@@ -26,9 +26,9 @@ mkdocs-section-index = "*"
 mkdocs-jupyter = "*"
 
 [feature.test.tasks]
-test = "pytest -n auto --doctest-modules"
-test-mlir = "pytest --pyargs sparse.mlir_backend -v"
-test-finch = { cmd = "pytest --pyargs sparse/tests -n auto -v", depends-on = ["precompile"] }
+test = "ci/test_Numba.sh"
+test-mlir = "ci/test_MLIR.sh"
+test-finch = "ci/test_Finch.sh"
 
 [feature.test.pypi-dependencies]
 pytest = ">=3.5"
@@ -71,6 +71,15 @@ scipy = ">=0.19"
 finch-mlir = ">=0.0.2"
 "PyYAML" = "*"
 
+[feature.barebones.dependencies]
+python = ">=3.10,<3.13"
+pip = ">=24"
+
+[feature.barebones.tasks]
+setup-env = {cmd = "ci/setup_env.sh" }
+test-all = { cmd = "ci/test_all.sh", env = { ACTIVATE_VENV = "1" }, depends-on = ["setup-env"] }
+test-finch = "ci/test_Finch.sh"
+
 [feature.mlir.activation.env]
 SPARSE_BACKEND = "MLIR"
 
@@ -80,3 +89,4 @@ doc = ["doc", "extra"]
 mlir-dev = {features = ["test", "mlir"], no-default-feature = true}
 finch-dev = {features = ["test", "finch"], no-default-feature = true}
 notebooks = ["extra", "mlir", "finch", "notebooks"]
+barebones = {features = ["barebones"], no-default-feature = true}

pyproject.toml

@@ -53,8 +53,9 @@ tests = [
 ]
 tox = ["sparse[tests]", "tox"]
 notebooks = ["sparse[tests]", "nbmake", "matplotlib"]
-all = ["sparse[docs,tox,notebooks]", "matrepr"]
+all = ["sparse[docs,tox,notebooks,mlir]", "matrepr"]
 finch = ["finch-tensor>=0.2.3"]
+mlir = ["finch-mlir>=0.0.2"]
 
 [project.urls]
 Documentation = "https://sparse.pydata.org/"