Remove conda (#294)

* Update README.md * Remove setup.py and setup.cfg * Add configuration files for pdm * Update .gitignore * Add requirements files * Update installation instructions to remove conda * Update README.md * Rename cleanup workflow for clarity * Update installation instructions in CONTRIBUTING.md * Refactor tox.ini: Update env_list and groups * Add PDM and Dependencies actions Update build-docs workflow Add cache-dependencies workflow Update cleanup-firebase workflow Update ci workflow Update analyze workflow Update analyze workflow * Update pyproject.toml dependencies and add pdm lock file * Update requirements files * Update dependency installation script * Activate environment after installing dependencies * Run workflows through pdm * Remove venv activation specific to linux * Update cml setup version * Update black version * Formatting changes * * Add command "clean" to pyproject.toml * Update makefile to move cleanup * Specify base python in tox.ini * Refactor tox.ini to add 'test' and 'lint' groups * Add Flake8 configuration to pyproject.toml for linting * Update dependencies and migrate from tox.ini to pyproject.toml * Update version bump command in CONTRIBUTING.md to use bumpver * bump version 1.0.8 -> 1.1.0 * Update flake8 ignore rules in CI workflow * bump version 1.1.0 -> 1.1.1 * Update docutils version and adjust Python requirements in lock files * Update version format to include 'v' prefix in __init__.py and pyproject.toml * Bump version to v1.1.1 This version includes performance improvements, bug fixes, and installation updates
mesoscope · Jan 17, 2025 · 22053fb · 22053fb
1 parent ecb33a2
commit 22053fb
Show file tree

Hide file tree

Showing 28 changed files with 4,059 additions and 294 deletions.
diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml
@@ -0,0 +1,12 @@
+name: PDM & Dependencies
+description: Install PDM and dependencies
+runs:
+  using: composite
+  steps:
+    - uses: ./.github/actions/pdm
+
+    - name: Install dependencies
+      shell: bash
+      # --only-keep flag in case the environment is restored from a stale cache.
+      run: |
+          pdm sync -d --only-keep
diff --git a/.github/actions/pdm/action.yml b/.github/actions/pdm/action.yml
@@ -0,0 +1,11 @@
+name: PDM
+description: Install PDM
+runs:
+  using: composite
+  steps:
+    - name: Set up PDM
+      uses: pdm-project/setup-pdm@v4
+      with:
+        python-version: "3.9"
+        # Cache all dependencies installed from pdm.lock
+        cache: true
diff --git a/.github/workflows/analyze.yml b/.github/workflows/analyze.yml
@@ -16,27 +16,18 @@ jobs:
                 os: [ubuntu-latest]
         steps:
             - uses: actions/[email protected]
-            - name: Set up Python ${{ matrix.python-version }}
-              uses: actions/setup-python@v5
-              with:
-                  python-version: ${{ matrix.python-version }}
-                  cache: "pip"
-                  cache-dependency-path: setup.py
-            - name: Install Dependencies
-              run: |
-                  python -m pip install --upgrade pip
-                  pip install .[all]
+            - uses: ./.github/actions/dependencies
             - name: Pack test recipe
               run: |
-                  pack -r cellpack/tests/recipes/v2/test_spheres.json -c cellpack/tests/packing-configs/test_config.json
+                  pdm run pack -r cellpack/tests/recipes/v2/test_spheres.json -c cellpack/tests/packing-configs/test_config.json
             - name: Modify JSON with PR sub_directory path
               run: |
                   jq --arg branch_name "${{  github.ref_name  }}" '.create_report.output_image_location |= "https://cellpack-results.s3.us-west-2.amazonaws.com/\($branch_name)/spheresSST/figures"' cellpack/tests/analysis-configs/PR_analysis_config.json > cellpack/tests/analysis-configs/PR_analysis_config_temp.json
                   mv cellpack/tests/analysis-configs/PR_analysis_config_temp.json cellpack/tests/analysis-configs/PR_analysis_config.json
             - name: Run analysis code
-              run: analyze -r cellpack/tests/recipes/v2/test_spheres.json -a cellpack/tests/analysis-configs/PR_analysis_config.json -p cellpack/tests/outputs/test_spheres/spheresSST
+              run: pdm run analyze -r cellpack/tests/recipes/v2/test_spheres.json -a cellpack/tests/analysis-configs/PR_analysis_config.json -p cellpack/tests/outputs/test_spheres/spheresSST
             - name: Upload results
-              uses: actions/upload-artifact@v3
+              uses: actions/upload-artifact@v4
               with:
                   name: results
                   path: cellpack/tests/outputs/test_spheres/
@@ -59,10 +50,10 @@ jobs:
                   aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
                   aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
                   aws-region: us-west-2
-            - uses: actions/download-artifact@v3
+            - uses: actions/download-artifact@v4
             - name: Copy files to results bucket
               run: aws s3 cp ./results s3://cellpack-results/${{  github.ref_name  }}/ --recursive --acl public-read
-            - uses: iterative/setup-cml@v1
+            - uses: iterative/setup-cml@v2
             - name: Update comment for dependabot
               if: ${{ github.actor == 'dependabot[bot]' }}
               env:

diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml
@@ -13,16 +13,7 @@ jobs:
     - uses: actions/[email protected]
       with:
         persist-credentials: false
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: 3.9
-        cache: "pip"
-        cache-dependency-path: setup.py
-    - name: Install Dependencies
-      run: |
-        pip install --upgrade pip
-        pip install .[dev]
+    - uses: ./.github/actions/dependencies
     - name: Generate Docs
       run: |
         make gen-docs

diff --git a/.github/workflows/cache-dependencies.yml b/.github/workflows/cache-dependencies.yml
@@ -0,0 +1,18 @@
+# The purpose of this workflow is to get more cache hits when other workflows build dependencies.
+# Github actions do not share caches between branches, except that all branches can pull results
+# from the default (main) branch. Therefore, this workflow runs on the main branch to keep the
+# latest dependencies cached for other branches to use.
+# Branches that change the dependencies will still get cache misses.
+name: Build and cache dependencies on main
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  dependencies:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/dependencies
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -31,19 +31,10 @@ jobs:
 
     steps:
     - uses: actions/[email protected]
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-        cache: "pip"
-        cache-dependency-path: setup.py
-    - name: Install Dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install .[all]
+    - uses: ./.github/actions/dependencies
     - name: Test with pytest
       run: |
-        pytest --cov cellpack/tests/
+        pdm run pytest --cov cellpack/tests/
     - name: Upload codecov
       uses: codecov/codecov-action@v4
 
@@ -52,22 +43,13 @@ jobs:
     if: ${{ !contains(github.event.head_commit.message, 'Bump version') }}
     steps:
     - uses: actions/[email protected]
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: 3.9
-        cache: "pip"
-        cache-dependency-path: setup.py
-    - name: Install Dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install .[test]
+    - uses: ./.github/actions/dependencies
     - name: Lint with flake8
       run: |
-        flake8 cellpack --count --verbose --show-source --statistics --ignore=E501,E277,W503,E203
+        pdm run flake8 cellpack --count --verbose --show-source --statistics --ignore=E203,E277,E402,E501,E721,W291,W503
     - name: Check with black
       run: |
-        black --check cellpack
+        pdm run black --check cellpack
 
   publish:
     if: success() && startsWith(github.ref, 'refs/tags/')
@@ -81,19 +63,7 @@ jobs:
 
     steps:
     - uses: actions/[email protected]
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: 3.9
-        cache: "pip"
-        cache-dependency-path: setup.py
-    - name: Install Dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel
-    - name: Build Package
-      run: |
-        python setup.py sdist bdist_wheel
+    - uses: ./.github/actions/dependencies
     - name: Publish to PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
+      run: pdm publish
 
diff --git a/.github/workflows/cleanup.yml → .github/workflows/cleanup-aws.yml b/.github/workflows/cleanup.yml → .github/workflows/cleanup-aws.yml
@@ -1,4 +1,4 @@
-name: cleanup
+name: Cleanup AWS S3 Bucket
 
 on: 
   pull_request:

diff --git a/.github/workflows/cleanup-firebase.yml b/.github/workflows/cleanup-firebase.yml
@@ -13,17 +13,10 @@ jobs:
                 os: [ubuntu-latest, windows-latest, macOS-latest]
         steps:
             -   uses: actions/[email protected]
-            -   name: Set up Python ${{ matrix.python-version }}
-                uses: actions/setup-python@v5
-                with:
-                    python-version: ${{ matrix.python-version }}
-            -   name: Install dependencies
-                run: |
-                    python -m pip install --upgrade pip
-                    pip install .[all]
+            -   uses: ./.github/actions/dependencies
             -   name: Cleanup Firebase Metadata
                 env:
                     FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }}
                     FIREBASE_EMAIL: ${{ secrets.FIREBASE_EMAIL }}
                 run: |
-                    python cellpack/bin/cleanup_tasks.py
+                    pdm run python cellpack/bin/cleanup_tasks.py
diff --git a/.github/workflows/make-requirements.yml b/.github/workflows/make-requirements.yml
@@ -0,0 +1,105 @@
+name: Make requirements files
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - "pdm.lock"
+
+jobs:
+  make-requirements:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macOS-latest, windows-latest]
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/pdm
+
+      - name: Generate requirements.txt
+        shell: bash
+        run: |
+          case "${{ matrix.os }}" in
+              "ubuntu-latest")
+                  export PLATFORM="linux"
+                  ;;
+              "macOS-latest")
+                  export PLATFORM="macos"
+                  ;;
+              "windows-latest")
+                  export PLATFORM="windows"
+                  ;;
+          esac
+
+          rm -rf requirements/*
+          mkdir -p requirements/$PLATFORM
+          pdm requirements requirements/$PLATFORM/requirements.txt
+
+      - name: Get platform variable
+        id: platform
+        shell: bash
+        run: |
+          case "${{ matrix.os }}" in
+              "ubuntu-latest")
+                  echo "::set-output name=platform::linux"
+                  ;;
+              "macOS-latest")
+                  echo "::set-output name=platform::macos"
+                  ;;
+              "windows-latest")
+                  echo "::set-output name=platform::windows"
+                  ;;
+          esac
+
+      - name: Upload requirements files
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.platform.outputs.platform }}-requirements
+          path: requirements/${{ steps.platform.outputs.platform }}
+
+  open-PR:
+    needs: [make-requirements]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Download linux requirements files
+        uses: actions/download-artifact@v4
+        with:
+          name: linux-requirements
+          path: requirements/linux
+
+      - name: Download windows requirements files
+        uses: actions/download-artifact@v4
+        with:
+          name: windows-requirements
+          path: requirements/windows
+
+      - name: Download macOS requirements files
+        uses: actions/download-artifact@v4
+        with:
+          name: macos-requirements
+          path: requirements/macos
+
+      - name: Clean-up CRLF
+        shell: bash
+        run: find requirements -type f -exec sed -i 's/\r//g' {} \;
+
+      - name: Get timestamp
+        id: timestamp
+        run: echo "::set-output name=timestamp::$(date +'%Y-%m-%d_%H-%M')"
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v5
+        with:
+          base: main
+          title: admin/requirements-update_${{ steps.timestamp.outputs.timestamp }}
+          body: Updating requirements.txt.
+
+            Due to some [challenges](https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#triggering-further-workflow-runs),
+            with getting this PR to trigger the tests, please manually close and re-open this PR.
+          branch: admin/requirements-update_${{ steps.timestamp.outputs.timestamp }}
+          commit-message: Updating requirements.txt after change to `pdm.lock` was pushed to `main`
+          delete-branch: true
diff --git a/.gitignore b/.gitignore
@@ -74,9 +74,6 @@ target/
 .ipynb_checkpoints
 *.ipynb
 
-# pyenv
-.python-version
-
 # celery beat schedule file
 celerybeat-schedule
 
@@ -122,4 +119,7 @@ data/
 **/converted/*
 
 # credentials
-.creds
+.creds
+
+# pdm files
+.pdm-python
diff --git a/.python-version b/.python-version
@@ -0,0 +1 @@
+3.9
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -15,10 +15,11 @@ Ready to contribute? Here's how to set up `cellpack` for local development.
     git clone [email protected]:{your_name_here}/cellpack.git
     ```
 
-3. Install the project in editable mode. (It is also recommended to work in a virtualenv or anaconda environment):
+3. Install the project in editable mode by following installation instructions [here](docs/INSTALL.md). (It is also recommended to work in a virtual environment):
 
     ```bash
     cd cellpack/
+    pip install -r requirements/linux/requirements.txt
     pip install -e .[dev]
     ```
 
@@ -55,8 +56,7 @@ Make sure all your changes are committed.
 Then run:
 
 ```bash
-bump2version patch # possible: major / minor / patch
-git push
+bumpver update --patch # possible: major / minor / patch
 git push --tags
 ```
 

diff --git a/Makefile b/Makefile
@@ -33,18 +33,18 @@ clean:  ## clean all build, python, and testing files
 	rm -fr build/
 	rm -fr dist/
 	rm -fr .eggs/
+	rm -fr .tox/
+	rm -fr .coverage
+	rm -fr coverage.xml
+	rm -fr htmlcov/
+	rm -fr .pytest_cache
 	find . -name '*.egg-info' -exec rm -fr {} +
 	find . -name '*.egg' -exec rm -f {} +
 	find . -name '*.pyc' -exec rm -f {} +
 	find . -name '*.pyo' -exec rm -f {} +
 	find . -name '*~' -exec rm -f {} +
 	find . -name '__pycache__' -exec rm -fr {} +
 	find . -name '*.py.bak' -exec rm -f {} +
-	rm -fr .tox/
-	rm -fr .coverage
-	rm -fr coverage.xml
-	rm -fr htmlcov/
-	rm -fr .pytest_cache
 
 build: ## run tox / run tests and lint
 	tox