Merge pull request #134 from duckdb/feature

Merge feature into main

.github/regression/micro.csv

@@ -0,0 +1,5 @@
+benchmark/micro/snapshot_performance/delta_scan.benchmark
+benchmark/micro/snapshot_performance/snapshot_no_pin.benchmark
+benchmark/micro/snapshot_performance/snapshot_no_pin_filter.benchmark
+benchmark/micro/snapshot_performance/snapshot_pin.benchmark
+benchmark/micro/snapshot_performance/snapshot_pin_filter.benchmark

.github/workflows/CloudTesting.yml

@@ -26,7 +26,7 @@ jobs:
           sudo apt-get update -y -qq
           sudo apt-get install -y -qq ninja-build make gcc-multilib g++-multilib zip unzip build-essential checkinstall curl libz-dev openssh-client
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           submodules: 'true'

.github/workflows/LocalTesting.yml

@@ -14,7 +14,6 @@ jobs:
   azurite-tests-linux:
     name: Azurite (local azure test server) tests (Linux)
     runs-on: ubuntu-latest
-    container: 'quay.io/pypa/manylinux2014_x86_64'
     env:
       VCPKG_TARGET_TRIPLET: 'x64-linux'
       GEN: Ninja
@@ -24,32 +23,37 @@ jobs:
       ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
         submodules: 'true'
 
+    - name: Install Ninja
+      shell: bash
+      run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
+
+    - name: Setup Ccache
+      uses: hendrikmuhs/ccache-action@main
+      with:
+        key: ${{ github.job }}
+
+    - name: Setup vcpkg
+      uses: lukka/[email protected]
+      with:
+        vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
+
+    - uses: actions/setup-node@v4
+
     - name: install Azure test service
       run: |
-        yum install -y nodejs npm
         npm install -g azurite
-        echo -e "[azure-cli]\nname=Azure CLI\nbaseurl=https://packages.microsoft.com/yumrepos/azure-cli\nenabled=1\ngpgcheck=1\ngpgkey=https://packages.microsoft.com/keys/microsoft.asc" | tee /etc/yum.repos.d/azure-cli.repo
-        yum install -y azure-cli
-
-    - name: Setup ManyLinux2014
-      run: |
-        ./duckdb/scripts/setup_manylinux2014.sh general aws-cli ccache ssh python_alias openssl
+        sudo apt-get install -y azure-cli 
 
     - name: Setup Rust for manylinux (dtolnay/rust-toolchain doesn't work due to curl being old here)
       run: |
         curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
         echo "$HOME/.cargo/bin" >> $GITHUB_PATH
 
-    - name: Setup vcpkg
-      uses: lukka/[email protected]
-      with:
-        vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
-
     - name: Handle OpenSSL dependency for rust build
       run: |
         echo "OPENSSL_ROOT_DIR=`pwd`/build/release/vcpkg_installed/x64-linux" >> $GITHUB_ENV
@@ -90,7 +94,7 @@ jobs:
       VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           submodules: 'true'
@@ -179,7 +183,7 @@ jobs:
       VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           submodules: 'true'
@@ -210,12 +214,14 @@ jobs:
 
       - name: Build
         shell: bash
-        run: make generate-data
+        run: |
+          make generate-data
+          make release
 
       - name: Test
         shell: bash
         run: |
-          GENERATED_DATA_AVAILABLE=1 make test
+          GENERATED_DATA_AVAILABLE=1 make test_release
 
   regression-test-benchmark-runner:
     name: Performance Regression Tests
@@ -280,13 +286,20 @@ jobs:
         if: always()
         shell: bash
         run: |
-          python3 ./duckdb/scripts/regression_test_runner.py --old=duckdb_delta/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/tpch_sf1_local.csv --verbose --threads=2 --root-dir=.
+          python3 ./duckdb/scripts/regression/test_runner.py --old=duckdb_delta/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/tpch_sf1_local.csv --verbose --threads=2 --root-dir=.
 
       - name: Regression Test TPC-DS
         if: always()
         shell: bash
         run: |
-          python ./duckdb/scripts/regression_test_runner.py --old=duckdb_delta/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/tpcds_sf1_local.csv --verbose --threads=2 --root-dir=.
+          python ./duckdb/scripts/regression/test_runner.py --old=duckdb_delta/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/tpcds_sf1_local.csv --verbose --threads=2 --root-dir=.
+
+      # FIXME: re-enable
+#      - name: Regression Test Micro
+#        if: always()
+#        shell: bash
+#        run: |
+#          python ./duckdb/scripts/regression/test_runner.py --old=duckdb_delta/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/micro.csv --verbose --threads=2 --root-dir=.
 
       - name: Test benchmark makefile
         shell: bash

.github/workflows/MainDistributionPipeline.yml

@@ -14,21 +14,22 @@ concurrency:
 jobs:
   duckdb-stable-build:
     name: Build extension binaries
-    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.1.3
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main
     with:
-      duckdb_version: v1.1.3
-      ci_tools_version: v1.1.3
+      duckdb_version: main
+      ci_tools_version: main
       extension_name: delta
       enable_rust: true
-      exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools'
+      exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw;linux_amd64_musl'
+      extra_toolchains: 'python3'
 
   duckdb-stable-deploy:
     name: Deploy extension binaries
     needs: duckdb-stable-build
-    uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.1.3
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@main
     secrets: inherit
     with:
       extension_name: delta
-      duckdb_version: v1.1.3
-      exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools'
-      deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
+      duckdb_version: main
+      ci_tools_version: main
+      exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw;linux_amd64_musl'

.gitignore

@@ -11,5 +11,5 @@ test/python/__pycache__/
 data/generated
 __azurite*__.json
 __blobstorage__
-.venv
+venv
 .vscode