diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh index 061e80282..66b4f6692 100755 --- a/.github/scripts/validate_binaries.sh +++ b/.github/scripts/validate_binaries.sh @@ -6,21 +6,10 @@ # LICENSE file in the root directory of this source tree. AWS_ENABLED=1 -if [[ ${MATRIX_PACKAGE_TYPE} = "conda" ]]; then - if [[ "${MATRIX_PYTHON_VERSION}" = "3.11" ]]; then - PYTORCH_CONDA_CHANNEL="malfet -c ${PYTORCH_CONDA_CHANNEL}" - fi +# shellcheck disable=SC2086 +pip install ${PYTORCH_PIP_PREFIX:-} torchdata --extra-index-url "${PYTORCH_PIP_DOWNLOAD_URL}" - # shellcheck disable=SC2086 - conda install -y torchdata -c ${PYTORCH_CONDA_CHANNEL} - if [[ ${TARGET_OS:-} = "windows" ]]; then - AWS_ENABLED=0 - fi -else - # shellcheck disable=SC2086 - pip install ${PYTORCH_PIP_PREFIX:-} torchdata --extra-index-url "${PYTORCH_PIP_DOWNLOAD_URL}" -fi case "${AWS_ENABLED}" in "0") diff --git a/.github/workflows/_build_test_upload.yml b/.github/workflows/_build_test_upload.yml index 8753d18ea..b5df9142b 100644 --- a/.github/workflows/_build_test_upload.yml +++ b/.github/workflows/_build_test_upload.yml @@ -19,12 +19,6 @@ on: secrets: PYPI_TOKEN: required: false - CONDA_PYTORCHBOT_TOKEN: - required: false - CONDA_TEST_PYTORCHBOT_TOKEN: - required: false - CONDA_NIGHTLY_PYTORCHBOT_TOKEN: - required: false permissions: id-token: write @@ -214,156 +208,11 @@ jobs: --password "$PYPI_TOKEN" \ torchdata*.whl - conda_build_test: - needs: get_release_type - runs-on: ${{ matrix.os }} - container: ${{ startsWith( matrix.os, 'ubuntu' ) && 'pytorch/manylinux-cpu' || null }} - strategy: - fail-fast: false - matrix: - os: - - windows-latest - python-version: - - 3.9 - - "3.10" - - "3.11" - - "3.12" - - "3.13" - steps: - - name: Checkout Source Repository - uses: actions/checkout@v3 - with: - ref: ${{ inputs.branch }} - submodules: recursive - - name: Determine if build AWSSDK - shell: bash -l {0} - run: | - BUILD_S3=0 - echo "value=$BUILD_S3" >> $GITHUB_OUTPUT - id: build_s3 - - name: Add temp runner environment variables - shell: bash -l {0} - run: | - echo "MINICONDA_INSTALL_PATH=${RUNNER_TEMP}/miniconda" >> "${GITHUB_ENV}" - echo "CONDA_ENV_PATH=${RUNNER_TEMP}/conda_build_env" >> "${GITHUB_ENV}" - - name: Create Conda Env on Windows - if: ${{ startsWith( matrix.os, 'windows' ) }} - uses: conda-incubator/setup-miniconda@v2 - with: - python-version: ${{ matrix.python-version }} - activate-environment: ${{ env.CONDA_ENV_PATH }} - - name: Build TorchData for Conda - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - PYTORCH_VERSION: ${{ inputs.pytorch_version }} - BUILD_S3: ${{ steps.build_s3.outputs.value }} - run: | - set -ex - if ${{ ! startsWith( matrix.os, 'windows' ) }}; then - source "${MINICONDA_INSTALL_PATH}/etc/profile.d/conda.sh" - fi - conda activate "${CONDA_ENV_PATH}" - if [[ "${PYTHON_VERSION}" = "3.11" ]]; then - conda install -yq conda-build -c malfet -c conda-forge - export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c malfet" - else - conda install -yq conda-build - fi - packaging/build_conda.sh - # conda index ./conda-bld - - name: Upload Conda Package to Github - if: always() - uses: actions/upload-artifact@v3 - with: - name: torchdata-artifact - path: conda-bld/*/torchdata-*.tar.bz2 - - conda_upload: - if: always() && inputs.branch != '' && inputs.do-upload == true - needs: [get_release_type, conda_build_test] - runs-on: ubuntu-latest - container: continuumio/miniconda3 - environment: pytorchbot-env - outputs: - upload: ${{ steps.trigger_upload.outputs.value }} - steps: - - name: Download Artifacts from Github - continue-on-error: true - uses: actions/download-artifact@v3 - with: - name: torchdata-artifact - - name: Determine if Conda Uploading is needed - run: | - upload=false - for pkg in ./*/torchdata-*.tar.bz2; do - upload=true - break - done - echo "value=$upload" >> $GITHUB_OUTPUT - id: trigger_upload - - name: Display All TorchData Conda Package - if: steps.trigger_upload.outputs.value == 'true' - run: ls -lh ./*/torchdata-*.tar.bz2 - - name: Upload Packages to Conda - if: steps.trigger_upload.outputs.value == 'true' - shell: bash -l {0} - env: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} - CONDA_NIGHTLY_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_NIGHTLY_PYTORCHBOT_TOKEN }} - CONDA_TEST_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_TEST_PYTORCHBOT_TOKEN }} - run: | - conda create -y --name conda_upload_env - conda activate conda_upload_env - - conda install -yq anaconda-client - conda install -c conda-forge -yq jq - - if [[ ${{ needs.get_release_type.outputs.type }} == 'official' ]]; then - CONDA_CHANNEL=pytorch - CONDA_TOKEN=${CONDA_PYTORCHBOT_TOKEN} - elif [[ ${{ needs.get_release_type.outputs.type }} == 'test' ]]; then - CONDA_CHANNEL=pytorch-${{ needs.get_release_type.outputs.type }} - CONDA_TOKEN=${CONDA_TEST_PYTORCHBOT_TOKEN} - else - CONDA_CHANNEL=pytorch-${{ needs.get_release_type.outputs.type }} - CONDA_TOKEN=${CONDA_NIGHTLY_PYTORCHBOT_TOKEN} - fi - - if [[ ${{ needs.get_release_type.outputs.type }} == 'nightly' ]]; then - # Loop over all platforms [win-64, osx-64, linux-64] - for subdir in $(find . -type f -name '*torchdata*.tar.bz2' | sed -r 's|/[^/]+$||' | uniq | cut -f2 -d/) ; do - version="" - # Find existing conda packages on pytorch nightly - for val in $(conda search --json torchdata --channel=$CONDA_CHANNEL --subdir=$subdir | jq -r '.[][] | .version, .build'); do - if [[ -z $version ]]; then - version=$val - else - build=$val - # Check if a new conda package built based on the same Python version and platform exists - for new_package in ./$subdir/torchdata-*-$build.tar.bz2; do - if [[ -f "$new_package" ]]; then - echo "Removing $CONDA_CHANNEL/torchdata/$version/$subdir/torchdata-$version-$build.tar.bz2" - anaconda -t "${CONDA_TOKEN}" remove -f "$CONDA_CHANNEL/torchdata/$version/$subdir/torchdata-$version-$build.tar.bz2" - break - fi - done - version="" - build="" - fi - done - # Upload new conda packages per subdir - anaconda -t "${CONDA_TOKEN}" upload ./$subdir/torchdata-*.tar.bz2 -u "$CONDA_CHANNEL" --label main --no-progress --force - done - else - anaconda -t "${CONDA_TOKEN}" upload ./*/torchdata-*.tar.bz2 -u "$CONDA_CHANNEL" --label main --no-progress --force - fi - build_docs: if: | always() && inputs.branch != '' && - ( needs.wheel_upload.outputs.upload == 'true' || needs.conda_upload.outputs.upload == 'true' ) - needs: [get_release_type, wheel_upload, conda_upload] + ( needs.wheel_upload.outputs.upload == 'true') + needs: [get_release_type, wheel_upload] runs-on: ubuntu-latest steps: - name: Setup Python 3.9 diff --git a/.github/workflows/build-conda-m1.yml b/.github/workflows/build-conda-m1.yml deleted file mode 100644 index 1f014acb9..000000000 --- a/.github/workflows/build-conda-m1.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Build M1 Conda - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: conda - os: macos-arm64 - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - name: pytorch/data - uses: pytorch/test-infra/.github/workflows/build_conda_macos.yml@main - with: - conda-package-directory: packaging/torchdata - repository: pytorch/data - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: "" - post-script: "" - package-name: torchdata - env-var-script: packaging/env-var-script.txt - smoke-test-script: test/smoke_test/smoke_test.py - runner-type: macos-m1-stable - trigger-event: ${{ github.event_name }} - secrets: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} - CONDA_PYTORCHBOT_TOKEN_TEST: ${{ secrets.CONDA_TEST_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build_conda_linux.yml b/.github/workflows/build_conda_linux.yml deleted file mode 100644 index ea791b412..000000000 --- a/.github/workflows/build_conda_linux.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Build Linux Conda - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: conda - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-cuda: disable - with-rocm: disable - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/data - pre-script: "" - post-script: "" - conda-package-directory: packaging/torchdata - smoke-test-script: test/smoke_test/smoke_test.py - package-name: torchdata - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_conda_linux.yml@main - with: - conda-package-directory: ${{ matrix.conda-package-directory }} - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - smoke-test-script: ${{ matrix.smoke-test-script }} - package-name: ${{ matrix.package-name }} - env-var-script: packaging/env-var-script.txt - trigger-event: ${{ github.event_name }} - secrets: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} - CONDA_PYTORCHBOT_TOKEN_TEST: ${{ secrets.CONDA_TEST_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/nightly_release.yml b/.github/workflows/nightly_release.yml index 52031ce85..b9179b9e4 100644 --- a/.github/workflows/nightly_release.yml +++ b/.github/workflows/nightly_release.yml @@ -18,5 +18,3 @@ jobs: branch: "main" pre_dev_release: true pytorch_version: "" - secrets: - CONDA_NIGHTLY_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_NIGHTLY_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/pull_release.yml b/.github/workflows/pull_release.yml index 4e4757539..ed9f421bf 100644 --- a/.github/workflows/pull_release.yml +++ b/.github/workflows/pull_release.yml @@ -7,6 +7,10 @@ on: - .github/workflows/pull_release.yml - .github/workflows/_build_test_upload.yml +permissions: + contents: write + id-token: write + jobs: build_test_upload: if: github.repository == 'pytorch/data' @@ -16,5 +20,3 @@ jobs: pre_dev_release: true pytorch_version: "" do-upload: false - secrets: - CONDA_TEST_PYTORCHBOT_TOKEN: "" diff --git a/.github/workflows/test_release.yml b/.github/workflows/test_release.yml index 51bf2bd5b..bd3c1b9c5 100644 --- a/.github/workflows/test_release.yml +++ b/.github/workflows/test_release.yml @@ -34,5 +34,3 @@ jobs: branch: "release/0.9" pre_dev_release: true pytorch_version: "2.5.0" - secrets: - CONDA_TEST_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_TEST_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/validate-binaries.yml b/.github/workflows/validate-binaries.yml index 97aed0d04..aca0f54a0 100644 --- a/.github/workflows/validate-binaries.yml +++ b/.github/workflows/validate-binaries.yml @@ -47,7 +47,7 @@ jobs: validate-binaries: uses: pytorch/test-infra/.github/workflows/validate-domain-library.yml@main with: - package_type: "conda,wheel" + package_type: "wheel" os: ${{ inputs.os }} channel: ${{ inputs.channel }} repository: "pytorch/data" diff --git a/torchdata/stateful_dataloader/README.md b/torchdata/stateful_dataloader/README.md index f42c8e3bd..e78bfa2c5 100644 --- a/torchdata/stateful_dataloader/README.md +++ b/torchdata/stateful_dataloader/README.md @@ -16,12 +16,12 @@ handles aggregation and distribution of state across multiprocess workers (but n ## Installation -`torchdata.stateful_dataloader` is currently available on the `torchdata` Nightly build. +`torchdata.stateful_dataloader` is currently available in `torchdata>=0.8.0`. Using pip: ```bash -pip install --pre torchdata --index-url https://download.pytorch.org/whl/nightly/cpu +pip install torchdata>=0.8.0 ``` Using conda: