staged upload scripts

.github/workflows/StagedUpload.yml

@@ -0,0 +1,45 @@
+name: Staged Upload
+on:
+  workflow_call:
+    inputs:
+      target_git_describe:
+        type: string
+  workflow_dispatch:
+    inputs:
+      target_git_describe:
+        type: string
+
+env:
+  GH_TOKEN: ${{ secrets.GH_TOKEN }}
+
+jobs:
+  staged-upload:
+    runs-on: ubuntu-latest
+    if: ${{ inputs.target_git_describe != '' }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+
+      - name: Install
+        shell: bash
+        run: pip install awscli
+
+      - name: Download from staging bucket
+        shell: bash
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }}
+        run: |
+          TARGET=$(git log -1 --format=%h)
+          mkdir to_be_uploaded
+          aws s3 cp --recursive "s3://duckdb-staging/$TARGET/${{ inputs.target_git_describe }}/$GITHUB_REPOSITORY/github_release" to_be_uploaded --region us-east-2
+
+      - name: Deploy
+        shell: bash
+        run: |
+          python3 scripts/asset-upload-gha.py to_be_uploaded/*

scripts/asset-upload-gha.py

@@ -0,0 +1,126 @@
+import json
+import os
+import sys
+import glob
+import time
+import urllib.request
+
+api_url = 'https://api.github.com/repos/duckdb/duckdb-odbc/'
+
+if len(sys.argv) < 2:
+    print("Usage: [filename1] [filename2] ... ")
+    exit(1)
+
+# this essentially should run on release tag builds to fill up release assets and master
+
+repo = os.getenv("GITHUB_REPOSITORY", "")
+if repo != "duckdb/duckdb-odbc":
+    print("Not running on forks. Exiting.")
+    exit(0)
+
+ref = os.getenv("GITHUB_REF", '')  # this env var is always present just not always used
+if ref == 'refs/heads/main':
+    print("Not running on main branch. Exiting.")
+    exit(0)
+elif ref.startswith('refs/tags/'):
+    tag = ref.replace('refs/tags/', '')
+else:
+    print("Not running on branches. Exiting.")
+    exit(0)
+
+
+print("Running on tag %s" % tag)
+
+
+token = os.getenv("GH_TOKEN", "")
+if token == "":
+    raise ValueError('need a GitHub token in GH_TOKEN')
+
+
+def internal_gh_api(suburl, filename='', method='GET'):
+    url = api_url + suburl
+    headers = {"Content-Type": "application/json", 'Authorization': 'token ' + token}
+
+    body_data = b''
+    raw_resp = None
+    if len(filename) > 0:
+        method = 'POST'
+        body_data = open(filename, 'rb')
+        headers["Content-Type"] = "binary/octet-stream"
+        headers["Content-Length"] = os.path.getsize(local_filename)
+        url = suburl  # cough
+
+    req = urllib.request.Request(url, body_data, headers)
+    req.get_method = lambda: method
+    print(f'GH API URL: "{url}" Filename: "{filename}" Method: "{method}"')
+    raw_resp = urllib.request.urlopen(req).read().decode()
+
+    if method != 'DELETE':
+        return json.loads(raw_resp)
+    else:
+        return {}
+
+
+def gh_api(suburl, filename='', method='GET'):
+    timeout = 1
+    nretries = 10
+    success = False
+    for i in range(nretries + 1):
+        try:
+            response = internal_gh_api(suburl, filename, method)
+            success = True
+        except urllib.error.HTTPError as e:
+            print(e.read().decode())  # gah
+        except Exception as e:
+            print(e)
+        if success:
+            break
+        print(f"Failed upload, retrying in {timeout} seconds... ({i}/{nretries})")
+        time.sleep(timeout)
+        timeout = timeout * 2
+    if not success:
+        raise Exception("Failed to open URL " + suburl)
+    return response
+
+
+# check if tag exists
+resp = gh_api('git/ref/tags/%s' % tag)
+if 'object' not in resp or 'sha' not in resp['object']:  # or resp['object']['sha'] != sha
+    raise ValueError('tag %s not found' % tag)
+
+resp = gh_api('releases/tags/%s' % tag)
+if 'id' not in resp or 'upload_url' not in resp:
+    raise ValueError('release does not exist for tag ' % tag)
+
+
+# double-check that release exists and has correct sha
+# disabled to not spam people watching releases
+# if 'id' not in resp or 'upload_url' not in resp or 'target_commitish' not in resp or resp['target_commitish'] != sha:
+# 	raise ValueError('release does not point to requested commit %s' % sha)
+
+# TODO this could be a paged response!
+assets = gh_api('releases/%s/assets' % resp['id'])
+
+upload_url = resp['upload_url'].split('{')[0]  # gah
+files = sys.argv[1:]
+for filename in files:
+    if '=' in filename:
+        parts = filename.split("=")
+        asset_filename = parts[0]
+        paths = glob.glob(parts[1])
+        if len(paths) != 1:
+            raise ValueError("Could not find file for pattern %s" % parts[1])
+        local_filename = paths[0]
+    else:
+        asset_filename = os.path.basename(filename)
+        local_filename = filename
+
+    # delete if present
+    for asset in assets:
+        if asset['name'] == asset_filename:
+            gh_api('releases/assets/%s' % asset['id'], method='DELETE')
+
+    resp = gh_api(f'{upload_url}?name={asset_filename}', filename=local_filename)
+    if 'id' not in resp:
+        raise ValueError('upload failed :/ ' + str(resp))
+    print("%s -> %s" % (local_filename, resp['browser_download_url']))