Improve documentation

.github/workflows/docs.yml

@@ -14,74 +14,26 @@ defaults:
     shell: bash
 
 jobs:
-  convert_scripts:
-    name: Translate scripts to notebooks
-    runs-on: ubuntu-latest
-    permissions:
-      pull-requests: write
-      contents: write
-    outputs:
-      commit_hash: ${{ steps.add-commit-push.outputs.commit_hash }}
-    container:
-      image: ghcr.io/ptb-mr/mrpro_py311:latest
-      options: --user runner
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.commit }}
-          ref: ${{ github.event.pull_request.head.ref }}
-          fetch-depth: 0
-
-      - name: Install mrpro and dependencies
-        run: pip install --upgrade --upgrade-strategy "eager" .[notebook]
-
-      - name: Translate scripts to notebooks
-        run: |
-          scripts=$(ls ./examples/*.py)
-          for script in $scripts
-            do jupytext --set-kernel "python3" --update --output ${script//.py/.ipynb} $script
-          done
-
-      - name: Check if any notebooks have been changed
-        uses: tj-actions/verify-changed-files@v20
-        id: verify-changed-notebooks
-        with:
-          files: ./examples/*.ipynb
-
-      - name: Commit notebooks
-        if: steps.verify-changed-notebooks.outputs.files_changed == 'true'
-        uses: actions4git/add-commit-push@v1
-        with:
-          commit-message: Notebooks updated
-
-      - name: Get hash of last commit
-        id: add-commit-push
-        run: echo "commit_hash=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
-
   get_notebooks:
     name: Get list of notebooks
-    needs: convert_scripts
     runs-on: ubuntu-latest
     steps:
       - name: Checkout mrpro repo
         uses: actions/checkout@v4
-        with:
-          ref: ${{ needs.convert_scripts.outputs.commit_hash }}
 
       - id: set-matrix
         run: |
-          echo "notebooks=$(find examples -type f -name '*.ipynb' | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
+          echo "notebook_paths=$(find examples/notebooks -type f -name '*.ipynb' | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
 
       - name: Notebook overview
         run: |
-          echo "jupyter-notebooks: ${{ steps.set-matrix.outputs.notebooks }}"
+          echo "jupyter-notebooks: ${{ steps.set-matrix.outputs.notebook_paths }}"
     outputs:
-      notebooks: ${{ steps.set-matrix.outputs.notebooks }}
+      notebook_paths: ${{ steps.set-matrix.outputs.notebook_paths }}
 
   run_notebook:
     name: Run notebook
-    needs: [convert_scripts, get_notebooks]
+    needs: get_notebooks
     runs-on: ubuntu-latest
     permissions:
       pull-requests: write
@@ -92,23 +44,21 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        notebook: ${{ fromJson(needs.get_notebooks.outputs.notebooks) }}
+        notebook_path: ${{ fromJson(needs.get_notebooks.outputs.notebook_paths) }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
-        with:
-          ref: ${{ needs.convert_scripts.outputs.commit_hash }}
 
       - name: Install mrpro and dependencies
         run: pip install --upgrade --upgrade-strategy "eager" .[notebook]
 
       - name: Notebook name
         run: |
-          echo "current jupyter-notebook: ${{ matrix.notebook }}"
+          echo "current jupyter-notebook: ${{ matrix.notebook_path }}"
 
       - name: Add nb-myst download badge
         run: |
-          notebook=${{ matrix.notebook }}
+          notebook=${{ matrix.notebook_path }}
           notebook_name=$(basename $notebook)
           download_badge_md="[![Download notebook](https://img.shields.io/badge/Download-notebook-blue?logo=jupyter)](path:$notebook_name)"
           python_command="import nbformat as nbf\n\
@@ -129,12 +79,12 @@ jobs:
         env:
           RUNNER: ${{ toJson(runner) }}
         with:
-          notebook: ./${{ matrix.notebook }}
+          notebook: ${{ matrix.notebook_path }}
 
       - name: Get artifact names
         id: artifact_names
         run: |
-          notebook=${{ matrix.notebook }}
+          notebook=${{ matrix.notebook_path }}
           echo "ARTIFACT_NAME=$(basename ${notebook/.ipynb})" >> $GITHUB_OUTPUT
           echo "IPYNB_EXECUTED=$(basename $notebook)" >> $GITHUB_OUTPUT
 
@@ -149,7 +99,7 @@ jobs:
 
   create_documentation:
     name: Build and deploy documentation
-    needs: [convert_scripts, run_notebook]
+    needs: run_notebook
     runs-on: ubuntu-latest
     container:
       image: ghcr.io/ptb-mr/mrpro_py311:latest
@@ -160,8 +110,6 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-        with:
-          ref: ${{ needs.convert_scripts.outputs.commit_hash }}
 
       - name: Install mrpro and dependencies
         run: pip install --upgrade --upgrade-strategy "eager" .[docs]

.gitignore

@@ -183,6 +183,7 @@ cython_debug/
 
 # Documentation
 **/_autosummary
+**/_notebooks
 
 # Ignore Conda environment directories:
 .conda
@@ -192,3 +193,4 @@ cython_debug/
 *~
 *.swp
 *.swo
+

.pre-commit-config.yaml

@@ -32,6 +32,42 @@ repos:
         args: [--double-quotes, --fix]
         exclude: ^tests/
 
+  - repo: https://github.com/kynan/nbstripout
+    rev: 0.8.0
+    hooks:
+    # cleans the .ipynbs (removes outputs, resets all cell-ids to 0..N, cleans steps)
+    # also clean any kernel information left after execution
+    - id: nbstripout
+      name: clean .ipynb output
+      args: [--extra-keys, "metadata.language_info"]
+      files: examples/notebooks
+
+  - repo: local
+    hooks:
+     - id: jupytext
+       name: convert .py to .ipynb
+       language: python
+
+       additional_dependencies:
+        - jupytext
+       entry: >
+        jupytext
+        --update
+        --pipe
+        "python .precommit/add_notebook_preamble.py {}"
+        --to
+        "../notebooks//ipynb"
+        --update-metadata
+        '{
+          "accelerator": "GPU",
+          "colab": {"gpuType": "T4","provenance": []},
+          "kernelspec": {"display_name": "Python 3 (ipykernel)","language": "python","name": "python3"}
+        }'
+       always_run: true
+       pass_filenames: true
+       files: ^examples/scripts/.*py
+       types_or: [python]
+
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.14.1
     hooks:
@@ -52,6 +88,33 @@ repos:
           - "--index-url=https://download.pytorch.org/whl/cpu"
           - "--extra-index-url=https://pypi.python.org/simple"
 
+  - repo: https://github.com/kynan/nbstripout
+    rev: 0.8.0
+    hooks:
+    # cleans the .ipynbs (removes outputs, resets all cell-ids to 0..N, cleans steps)
+    # also clean any kernel information left after execution
+    - id: nbstripout
+      name: clean .ipynb output
+      args: [--extra-keys, "metadata.language_info"]
+      files: examples/notebooks
+
+  - repo: https://github.com/mwouts/jupytext
+    rev: v1.16.6
+    hooks:
+     - id: jupytext
+       name: convert .py to .ipynb
+       args:
+         - --update
+         - --pipe
+         - "python .precommit/add_notebook_preemble.py {}"
+         - --to
+         - "../notebooks//ipynb"
+         - --update-metadata
+         - '{"accelerator": "GPU","colab": {"gpuType": "T4","provenance": []},"kernelspec": {"display_name": "Python 3 (ipykernel)","language": "python","name": "python3"}}'
+         - examples/scripts/*.py
+       always_run: true
+       pass_filenames: false
+
 ci:
   autofix_commit_msg: |
     [pre-commit] auto fixes from pre-commit hooks

.precommit/add_notebook_preamble.py

@@ -0,0 +1,36 @@
+"""Add a colab badge and pip install to notebooks."""
+
+import sys
+from pathlib import Path
+
+# the filename is the name of temp file created by jupytext, not an original notebook
+file = Path(sys.argv[1])
+# the temp filename for "iteratitive_sense_reconstruction.py" is like "iterative_sense_reconstruction-42_5f4kv.py"
+basename = file.stem.rpartition('-')[0]
+
+badge_svg = 'https://colab.research.google.com/assets/colab-badge.svg'
+ipynb_link = f'https://colab.research.google.com/github/PTB-MR/mrpro/blob/main/examples/notebooks/{basename}.ipynb'
+badge_markdown = f'[![Open In Colab]({badge_svg})]({ipynb_link})'
+badge_pyprocent = f'# %% [markdown]\n# {badge_markdown}\n'
+import_python = """# %% tags=["remove-cell"]
+import importlib
+
+if not importlib.util.find_spec('mrpro'):
+    %pip install mrpro[notebook]
+"""
+
+# the temp files of jupytext have the header which looks like:
+# ---
+# jupyter:
+#   jupytext:
+# multiple lines...
+# ---
+# we need to insert the #markdown cell after the header
+# insert the badge_pyprocent string after the second occurrence of '# ---'
+split_sequence = '# ---\n'
+old = file.read_text()
+split_text = old.split(split_sequence)
+new = ''.join(
+    [split_text[0], split_sequence, split_text[1], split_sequence, badge_pyprocent, import_python, *split_text[2:]]
+)
+file.write_text(new)

docs/Makefile

@@ -18,6 +18,7 @@ help:
 # be removed when https://github.com/sphinx-doc/sphinx/issues/1999 is fixed.
 clean:
 	rm -rfv "$(SOURCEDIR)/_autosummary"
+	rm -rfv "$(SOURCEDIR)/_notebooks"
 	$(SPHINXBUILD)  -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
 # Catch-all target: route all unknown targets to Sphinx using the new

docs/source/_static/custom.css

@@ -7,3 +7,14 @@ html {
 .wy-nav-content {
     max-width: 75% !important;
 }
+
+/* new line after each property */
+dl.py.property {
+    display: block !important;
+}
+
+/* example notebooks outputs */
+.output>.highlight {
+    background: #efefef !important;
+    color: inherit !important;
+}

docs/source/_templates/class_template.rst

@@ -4,9 +4,8 @@
 
 .. autoclass:: {{ objname }}
    :members:
+   :special-members: '__init__, __matmul__, __add__, __mul__, __or__, __and__, __radd__, __rmul__, __rmatmul__, __ror__, __rand__, __truediv__, __eq__, __pow__'
    :inherited-members: Module
    :show-inheritance:
 
-   {% block methods %}
-   .. automethod:: __init__
-   {% endblock %}
+