Delete conda builds (#1367)

* Update README.md (#1366) * Update README.md * Update README.md * revert changes * initial commit * run pre-commit * delete existing builds * run precommit * delete previously available nightly builds and do not upload new ones * delete everything conda * run precommit * run precommit * try new permissions * update linux build * revert to old linux build * clean test_release * random test * revert change * update manylinux * update manylinux * update validate binaries.sh * revert manylinux update * update validate binaries --------- Co-authored-by: Andrew Ho <[email protected]>
pytorch · Nov 25, 2024 · 31291ef · 31291ef
1 parent aebad0c
commit 31291ef
Show file tree

Hide file tree

Showing 8 changed files with 9 additions and 272 deletions.
diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh
@@ -6,21 +6,10 @@
 # LICENSE file in the root directory of this source tree.
 
 AWS_ENABLED=1
-if [[ ${MATRIX_PACKAGE_TYPE} = "conda" ]]; then
 
-    if [[ "${MATRIX_PYTHON_VERSION}" = "3.11" ]]; then
-        PYTORCH_CONDA_CHANNEL="malfet -c ${PYTORCH_CONDA_CHANNEL}"
-    fi
+# shellcheck disable=SC2086
+pip install ${PYTORCH_PIP_PREFIX:-} torchdata --extra-index-url "${PYTORCH_PIP_DOWNLOAD_URL}"
 
-    # shellcheck disable=SC2086
-    conda install -y torchdata -c ${PYTORCH_CONDA_CHANNEL}
-    if [[ ${TARGET_OS:-} = "windows" ]]; then
-        AWS_ENABLED=0
-    fi
-else
-    # shellcheck disable=SC2086
-    pip install ${PYTORCH_PIP_PREFIX:-} torchdata --extra-index-url "${PYTORCH_PIP_DOWNLOAD_URL}"
-fi
 
 case "${AWS_ENABLED}" in
     "0")

diff --git a/.github/workflows/_build_test_upload.yml b/.github/workflows/_build_test_upload.yml
@@ -19,12 +19,6 @@ on:
     secrets:
       PYPI_TOKEN:
         required: false
-      CONDA_PYTORCHBOT_TOKEN:
-        required: false
-      CONDA_TEST_PYTORCHBOT_TOKEN:
-        required: false
-      CONDA_NIGHTLY_PYTORCHBOT_TOKEN:
-        required: false
 
 permissions:
   id-token: write
@@ -214,156 +208,11 @@ jobs:
             --password "$PYPI_TOKEN" \
             torchdata*.whl
 
-  conda_build_test:
-    needs: get_release_type
-    runs-on: ${{ matrix.os }}
-    container: ${{ startsWith( matrix.os, 'ubuntu' ) && 'pytorch/manylinux-cpu' || null }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os:
-          - windows-latest
-        python-version:
-          - 3.9
-          - "3.10"
-          - "3.11"
-          - "3.12"
-          - "3.13"
-    steps:
-      - name: Checkout Source Repository
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ inputs.branch }}
-          submodules: recursive
-      - name: Determine if build AWSSDK
-        shell: bash -l {0}
-        run: |
-          BUILD_S3=0
-          echo "value=$BUILD_S3" >> $GITHUB_OUTPUT
-        id: build_s3
-      - name: Add temp runner environment variables
-        shell: bash -l {0}
-        run: |
-          echo "MINICONDA_INSTALL_PATH=${RUNNER_TEMP}/miniconda" >> "${GITHUB_ENV}"
-          echo "CONDA_ENV_PATH=${RUNNER_TEMP}/conda_build_env" >> "${GITHUB_ENV}"
-      - name: Create Conda Env on Windows
-        if: ${{ startsWith( matrix.os, 'windows' ) }}
-        uses: conda-incubator/setup-miniconda@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          activate-environment: ${{ env.CONDA_ENV_PATH }}
-      - name: Build TorchData for Conda
-        shell: bash -l {0}
-        env:
-          PYTHON_VERSION: ${{ matrix.python-version }}
-          PYTORCH_VERSION: ${{ inputs.pytorch_version }}
-          BUILD_S3: ${{ steps.build_s3.outputs.value }}
-        run: |
-          set -ex
-          if ${{ ! startsWith( matrix.os, 'windows' ) }}; then
-            source "${MINICONDA_INSTALL_PATH}/etc/profile.d/conda.sh"
-          fi
-          conda activate "${CONDA_ENV_PATH}"
-          if [[ "${PYTHON_VERSION}" = "3.11" ]]; then
-            conda install -yq conda-build -c malfet -c conda-forge
-            export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c malfet"
-          else
-            conda install -yq conda-build
-          fi
-          packaging/build_conda.sh
-          # conda index ./conda-bld
-      - name: Upload Conda Package to Github
-        if: always()
-        uses: actions/upload-artifact@v3
-        with:
-          name: torchdata-artifact
-          path: conda-bld/*/torchdata-*.tar.bz2
-
-  conda_upload:
-    if: always() && inputs.branch != '' && inputs.do-upload == true
-    needs: [get_release_type, conda_build_test]
-    runs-on: ubuntu-latest
-    container: continuumio/miniconda3
-    environment: pytorchbot-env
-    outputs:
-      upload: ${{ steps.trigger_upload.outputs.value }}
-    steps:
-      - name: Download Artifacts from Github
-        continue-on-error: true
-        uses: actions/download-artifact@v3
-        with:
-          name: torchdata-artifact
-      - name: Determine if Conda Uploading is needed
-        run: |
-          upload=false
-          for pkg in ./*/torchdata-*.tar.bz2; do
-            upload=true
-            break
-          done
-          echo "value=$upload" >> $GITHUB_OUTPUT
-        id: trigger_upload
-      - name: Display All TorchData Conda Package
-        if: steps.trigger_upload.outputs.value == 'true'
-        run: ls -lh ./*/torchdata-*.tar.bz2
-      - name: Upload Packages to Conda
-        if: steps.trigger_upload.outputs.value == 'true'
-        shell: bash -l {0}
-        env:
-          CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }}
-          CONDA_NIGHTLY_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_NIGHTLY_PYTORCHBOT_TOKEN }}
-          CONDA_TEST_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_TEST_PYTORCHBOT_TOKEN }}
-        run: |
-          conda create -y --name conda_upload_env
-          conda activate conda_upload_env
-
-          conda install -yq anaconda-client
-          conda install -c conda-forge -yq jq
-
-          if [[ ${{ needs.get_release_type.outputs.type }} == 'official' ]]; then
-            CONDA_CHANNEL=pytorch
-            CONDA_TOKEN=${CONDA_PYTORCHBOT_TOKEN}
-          elif [[ ${{ needs.get_release_type.outputs.type }} == 'test' ]]; then
-            CONDA_CHANNEL=pytorch-${{ needs.get_release_type.outputs.type }}
-            CONDA_TOKEN=${CONDA_TEST_PYTORCHBOT_TOKEN}
-          else
-            CONDA_CHANNEL=pytorch-${{ needs.get_release_type.outputs.type }}
-            CONDA_TOKEN=${CONDA_NIGHTLY_PYTORCHBOT_TOKEN}
-          fi
-
-          if [[ ${{ needs.get_release_type.outputs.type }} == 'nightly' ]]; then
-            # Loop over all platforms [win-64, osx-64, linux-64]
-            for subdir in $(find . -type f -name '*torchdata*.tar.bz2' | sed -r 's|/[^/]+$||' | uniq | cut -f2 -d/) ; do
-              version=""
-              # Find existing conda packages on pytorch nightly
-              for val in $(conda search --json torchdata --channel=$CONDA_CHANNEL --subdir=$subdir | jq -r '.[][] | .version, .build'); do
-                if [[ -z $version ]]; then
-                  version=$val
-                else
-                  build=$val
-                  # Check if a new conda package built based on the same Python version and platform exists
-                  for new_package in ./$subdir/torchdata-*-$build.tar.bz2; do
-                    if [[ -f "$new_package" ]]; then
-                      echo "Removing $CONDA_CHANNEL/torchdata/$version/$subdir/torchdata-$version-$build.tar.bz2"
-                      anaconda -t "${CONDA_TOKEN}" remove -f "$CONDA_CHANNEL/torchdata/$version/$subdir/torchdata-$version-$build.tar.bz2"
-                      break
-                    fi
-                  done
-                  version=""
-                  build=""
-                fi
-              done
-              # Upload new conda packages per subdir
-              anaconda -t "${CONDA_TOKEN}" upload ./$subdir/torchdata-*.tar.bz2 -u "$CONDA_CHANNEL" --label main --no-progress --force
-            done
-          else
-            anaconda -t "${CONDA_TOKEN}" upload ./*/torchdata-*.tar.bz2 -u "$CONDA_CHANNEL" --label main --no-progress --force
-          fi
-
   build_docs:
     if: |
       always() && inputs.branch != '' &&
-      ( needs.wheel_upload.outputs.upload == 'true' || needs.conda_upload.outputs.upload == 'true' )
-    needs: [get_release_type, wheel_upload, conda_upload]
+      ( needs.wheel_upload.outputs.upload == 'true')
+    needs: [get_release_type, wheel_upload]
     runs-on: ubuntu-latest
     steps:
       - name: Setup Python 3.9

diff --git a/.github/workflows/build-conda-m1.yml b/.github/workflows/build-conda-m1.yml
diff --git a/.github/workflows/build_conda_linux.yml b/.github/workflows/build_conda_linux.yml
diff --git a/.github/workflows/nightly_release.yml b/.github/workflows/nightly_release.yml
@@ -18,5 +18,3 @@ jobs:
       branch: "main"
       pre_dev_release: true
       pytorch_version: ""
-    secrets:
-      CONDA_NIGHTLY_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_NIGHTLY_PYTORCHBOT_TOKEN }}
diff --git a/.github/workflows/pull_release.yml b/.github/workflows/pull_release.yml
@@ -7,6 +7,10 @@ on:
       - .github/workflows/pull_release.yml
       - .github/workflows/_build_test_upload.yml
 
+permissions:
+  contents: write
+  id-token: write
+
 jobs:
   build_test_upload:
     if: github.repository == 'pytorch/data'
@@ -16,5 +20,3 @@ jobs:
       pre_dev_release: true
       pytorch_version: ""
       do-upload: false
-    secrets:
-      CONDA_TEST_PYTORCHBOT_TOKEN: ""
diff --git a/.github/workflows/test_release.yml b/.github/workflows/test_release.yml
@@ -34,5 +34,3 @@ jobs:
       branch: "release/0.9"
       pre_dev_release: true
       pytorch_version: "2.5.0"
-    secrets:
-      CONDA_TEST_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_TEST_PYTORCHBOT_TOKEN }}
diff --git a/.github/workflows/validate-binaries.yml b/.github/workflows/validate-binaries.yml
@@ -47,7 +47,7 @@ jobs:
   validate-binaries:
     uses: pytorch/test-infra/.github/workflows/validate-domain-library.yml@main
     with:
-      package_type: "conda,wheel"
+      package_type: "wheel"
       os: ${{ inputs.os }}
       channel: ${{ inputs.channel }}
       repository: "pytorch/data"