Keep all fleetd-base and fleetd-chrome artifacts.

.github/actions/r2-upload/action.yml

@@ -1,5 +1,5 @@
 name: R2 upload
-description: Upload a file to R2
+description: Upload files to R2, keeping the directory structure intact
 # Schema: https://json.schemastore.org/github-action.json
 
 # This action expects the following env vars to be set:
@@ -9,6 +9,10 @@ description: Upload a file to R2
 # - R2_BUCKET: The bucket to upload to
 
 inputs:
+  working-directory:
+    description: 'The working directory, relative to which the files will be uploaded.'
+    default: './'
+    required: false
   filenames:
     description: 'Comma-delimited names of the file(s) to upload. For example: file1,manifest.json,file with spaces.txt'
     required: true
@@ -18,8 +22,9 @@ runs:
   steps:
     - name: Upload file to R2
       shell: bash
+      working-directory: ${{ inputs.working-directory }}
       run: |
-        sudo ./.github/scripts/rclone-install.sh
+        sudo ${{ github.workspace }}/.github/scripts/rclone-install.sh
         mkdir -p ~/.config/rclone
         echo "[r2]
         type = s3
@@ -32,6 +37,11 @@ runs:
         " > ~/.config/rclone/rclone.conf
         : # Loop over each filename in the array of filenames and upload each one.
         IFS=$'\n'
-        for row in $(echo "${{ inputs.filenames }}" | tr "," "\n"); do
-          rclone copy --verbose "$row" r2:${{ env.R2_BUCKET }}/
+        for filename in $(echo "${{ inputs.filenames }}" | tr "," "\n"); do
+          upload_dir=${{ env.R2_BUCKET }}
+          dirname=$(dirname "$filename")
+          if [ "$dirname" != "" ] && [ "$dirname" != "." ]; then
+            upload_dir="$upload_dir"/"$dirname"
+          fi
+          rclone copy --verbose "$filename" r2:"$upload_dir"
         done

.github/workflows/release-fleetd-base.yml

@@ -1,4 +1,18 @@
-name: Upload fleetd base to https://download.fleetdm.com
+name: Release and upload fleetd base to https://download.fleetdm.com
+
+# This workflow checks TUF if there are updates to orbit, desktop, and osqueryd components of fleetd.
+# If there are updates, it builds and uploads the following files:
+# - stable/meta.json
+# - stable/tuf-meta.json
+# - stable/fleetd-base.pkg
+# - stable/fleetd-base-manifest.plist
+# - stable/fleetd-base.msi
+# - archive/stable/YYYY-MM-DD_HH-MM-SS/meta.json
+# - archive/stable/YYYY-MM-DD_HH-MM-SS/tuf-meta.json
+# - archive/stable/YYYY-MM-DD_HH-MM-SS/fleetd-base.pkg
+# - archive/stable/YYYY-MM-DD_HH-MM-SS/fleetd-base-manifest.plist
+# - archive/stable/YYYY-MM-DD_HH-MM-SS/fleetd-base.msi
+# Finally, it verifies the uploaded installers and their checksums.
 
 on:
   workflow_dispatch: # Manual
@@ -29,6 +43,7 @@ jobs:
   check-for-fleetd-component-updates:
     runs-on: ubuntu-latest
     outputs:
+      date_dir: ${{ steps.check-for-fleetd-component-updates.outputs.date_dir }}
       update_needed: ${{ steps.check-for-fleetd-component-updates.outputs.update_needed }}
     steps:
       - name: Harden Runner
@@ -49,25 +64,33 @@ jobs:
       - name: Check for fleetd component updates
         id: check-for-fleetd-component-updates
         run: |
-          go run tools/tuf/status/tuf-status.go channel-version -channel stable --components orbit,desktop,osqueryd --format json > latest-meta.json
-          curl -O $BASE_URL/meta.json
-          if diff latest-meta.json meta.json >/dev/null 2>&1
+          go run tools/tuf/status/tuf-status.go channel-version -channel stable --components orbit,desktop,osqueryd --format json > latest-tuf-meta.json
+          : # Check that latest-tuf-meta.json is valid
+          jq -e . >/dev/null 2>&1 <<< $(cat latest-tuf-meta.json)
+          : # Download the current TUF meta file in order to compare it with the latest
+          curl -O $BASE_URL/stable/tuf-meta.json
+          if diff latest-tuf-meta.json tuf-meta.json >/dev/null 2>&1
           then
             echo "update_needed=false" >> $GITHUB_OUTPUT
           else
             echo "update_needed=true" >> $GITHUB_OUTPUT
           fi
+          echo "date_dir=$(date -u +%Y-%m-%d_%H-%M-%S)" >> $GITHUB_OUTPUT
 
-      - name: Upload latest meta.json artifact
+      - name: Upload latest TUF meta artifact
         uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         with:
-          name: latest-meta.json
-          path: latest-meta.json
+          name: latest-tuf-meta.json
+          path: latest-tuf-meta.json
 
   update-fleetd-base-pkg:
     needs: [check-for-fleetd-component-updates]
     if: needs.check-for-fleetd-component-updates.outputs.update_needed == 'true'
     runs-on: macos-latest
+    outputs:
+      fleetd_base_pkg_sha256: ${{ steps.build-sign-notarize.outputs.fleetd_base_pkg_sha256 }}
+    env:
+      FULL_DATE_DIR: archive/stable/${{ needs.check-for-fleetd-component-updates.outputs.date_dir }}
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
@@ -101,6 +124,7 @@ jobs:
           rm certificate.p12
 
       - name: Build PKG, sign, and notarize
+        id: build-sign-notarize
         env:
           AC_USERNAME: ${{ secrets.APPLE_USERNAME }}
           AC_PASSWORD: ${{ secrets.APPLE_PASSWORD }}
@@ -109,8 +133,8 @@ jobs:
         run: |
           fleetctl package --type pkg --fleet-desktop --use-system-configuration --sign-identity $PACKAGE_SIGNING_IDENTITY_SHA1 --notarize
           mv fleet-osquery*.pkg fleetd-base.pkg
-          : # Calculate the SHA256 checksum of the package for the next step
-          echo "FLEETD_BASE_PKG_CHECKSUM=$(shasum -a 256 fleetd-base.pkg | cut -d ' ' -f 1)" >> $GITHUB_ENV
+          : # Calculate the SHA256 checksum of the package
+          echo "fleetd_base_pkg_sha256=$(shasum -a 256 fleetd-base.pkg | cut -d ' ' -f 1)" >> $GITHUB_OUTPUT
 
       - name: Create plist
         run: |
@@ -128,26 +152,78 @@ jobs:
                       <integer>32</integer>
                       <key>sha256s</key>
                       <array>
-                        <string>${{ env.FLEETD_BASE_PKG_CHECKSUM }}</string>
+                        <string>${{ steps.build-sign-notarize.outputs.fleetd_base_pkg_sha256 }}</string>
                       </array>
                       <key>url</key>
-                      <string>${{ env.BASE_URL }}/fleetd-base.pkg</string>
+                      <string>${{ env.BASE_URL }}/${{ env.FULL_DATE_DIR }}/fleetd-base.pkg</string>
                     </dict>
                   </array>
                 </dict>
               </array>
             </dict>
           </plist>' > fleetd-base-manifest.plist
 
+      - name: Set up files and directories for R2 upload
+        run: |
+          mkdir -p stable
+          mkdir -p ${{ env.FULL_DATE_DIR }}
+          cp fleetd-base.pkg stable/
+          cp fleetd-base-manifest.plist stable/
+          cp fleetd-base.pkg ${{ env.FULL_DATE_DIR }}/
+          cp fleetd-base-manifest.plist ${{ env.FULL_DATE_DIR }}/
+
       - name: Upload package
         uses: ./.github/actions/r2-upload
         with:
-          filenames: fleetd-base.pkg,fleetd-base-manifest.plist
+          filenames: stable/fleetd-base.pkg,stable/fleetd-base-manifest.plist,${{ env.FULL_DATE_DIR }}/fleetd-base.pkg,${{ env.FULL_DATE_DIR }}/fleetd-base-manifest.plist
 
-  update-fleetd-base-msi:
+  build-fleetd-base-msi:
     needs: [check-for-fleetd-component-updates]
     if: needs.check-for-fleetd-component-updates.outputs.update_needed == 'true'
     runs-on: ubuntu-latest
+    env:
+      FULL_DATE_DIR: archive/stable/${{ needs.check-for-fleetd-component-updates.outputs.date_dir }}
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
+        with:
+          egress-policy: audit
+
+      - name: Install fleetctl
+        run: npm install -g fleetctl
+
+      - name: Build MSI
+        id: build
+        run: |
+          fleetctl package --type msi --fleet-desktop --fleet-url dummy --enroll-secret dummy
+          mv fleet-osquery*.msi fleetd-base.msi
+
+      - name: Upload fleetd-base.msi for code signing
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # 4.3.3
+        with:
+          name: unsigned-windows
+          path: fleetd-base.msi
+
+  code-sign-windows:
+    needs: build-fleetd-base-msi
+    uses: ./.github/workflows/code-sign-windows.yml
+    with:
+      filename: fleetd-base.msi
+      upload_name: fleetd-base-msi
+    secrets:
+      DIGICERT_KEYLOCKER_CERTIFICATE: ${{ secrets.DIGICERT_KEYLOCKER_CERTIFICATE }}
+      DIGICERT_KEYLOCKER_PASSWORD: ${{ secrets.DIGICERT_KEYLOCKER_PASSWORD }}
+      DIGICERT_KEYLOCKER_HOST_URL: ${{ secrets.DIGICERT_KEYLOCKER_HOST_URL }}
+      DIGICERT_API_KEY: ${{ secrets.DIGICERT_API_KEY }}
+      DIGICERT_KEYLOCKER_CERTIFICATE_FINGERPRINT: ${{ secrets.DIGICERT_KEYLOCKER_CERTIFICATE_FINGERPRINT }}
+
+  update-fleetd-base-msi:
+    needs: [code-sign-windows, check-for-fleetd-component-updates]
+    runs-on: ubuntu-latest
+    outputs:
+      fleetd_base_msi_sha256: ${{ steps.prepare-files.outputs.fleetd_base_msi_sha256 }}
+    env:
+      FULL_DATE_DIR: archive/stable/${{ needs.check-for-fleetd-component-updates.outputs.date_dir }}
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
@@ -162,22 +238,31 @@ jobs:
             .github/scripts/rclone-install.sh
           sparse-checkout-cone-mode: false
 
-      - name: Install fleetctl
-        run: npm install -g fleetctl
+      - name: Download signed artifact
+        uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4.1.6
+        with:
+          name: fleetd-base-msi
 
-      - name: Build MSI
+      - name: Prepare files for R2 upload
+        id: prepare-files
         run: |
-          fleetctl package --type msi --fleet-desktop --fleet-url dummy --enroll-secret dummy
-          mv fleet-osquery*.msi fleetd-base.msi
+          mkdir -p stable
+          mkdir -p ${{ env.FULL_DATE_DIR }}
+          cp fleetd-base.msi stable/
+          cp fleetd-base.msi ${{ env.FULL_DATE_DIR }}/
+          : # Calculate the SHA256 checksum of the package
+          echo "fleetd_base_msi_sha256=$(shasum -a 256 fleetd-base.msi | cut -d ' ' -f 1)" >> $GITHUB_OUTPUT
 
       - name: Upload package
         uses: ./.github/actions/r2-upload
         with:
-          filenames: fleetd-base.msi
+          filenames: stable/fleetd-base.msi,${{ env.FULL_DATE_DIR }}/fleetd-base.msi
 
-  update-meta-json:
-    needs: [update-fleetd-base-pkg, update-fleetd-base-msi]
+  update-meta-files:
+    needs: [check-for-fleetd-component-updates, update-fleetd-base-pkg, update-fleetd-base-msi]
     runs-on: ubuntu-latest
+    env:
+      FULL_DATE_DIR: archive/stable/${{ needs.check-for-fleetd-component-updates.outputs.date_dir }}
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
@@ -192,15 +277,37 @@ jobs:
             .github/scripts/rclone-install.sh
           sparse-checkout-cone-mode: false
 
-      - name: Download latest-meta.json artifact
+      - name: Download latest-tuf-meta.json artifact
         uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
         with:
-          name: latest-meta.json
+          name: latest-tuf-meta.json
 
-      - name: Rename latest-meta.json to meta.json
-        run: mv latest-meta.json meta.json
+      - name: Set up files and directories for R2 upload
+        run: |
+          mkdir -p stable
+          mkdir -p ${{ env.FULL_DATE_DIR }}
+          echo '{
+            "fleetd_base_msi_url": "${{ env.BASE_URL }}/${{ env.FULL_DATE_DIR }}/fleetd-base.msi",
+            "fleetd_base_msi_sha256": "${{ needs.update-fleetd-base-msi.outputs.fleetd_base_msi_sha256 }}",
+            "fleetd_base_pkg_url": "${{ env.BASE_URL }}/${{ env.FULL_DATE_DIR }}/fleetd-base.pkg",
+            "fleetd_base_pkg_sha256": "${{ needs.update-fleetd-base-pkg.outputs.fleetd_base_pkg_sha256 }}",
+            "fleetd_base_manifest_plist_url": "${{ env.BASE_URL }}/${{ env.FULL_DATE_DIR }}/fleetd-base-manifest.plist",
+            "version": "${{ needs.check-for-fleetd-component-updates.outputs.date_dir }}"
+          }' > meta.json
+          : # Check that meta.json is valid
+          jq -e . >/dev/null 2>&1 <<< $(cat meta.json)
+          cp latest-tuf-meta.json stable/tuf-meta.json
+          cp latest-tuf-meta.json ${{ env.FULL_DATE_DIR }}/tuf-meta.json
+          cp meta.json stable/meta.json
+          cp meta.json ${{ env.FULL_DATE_DIR }}/meta.json
 
-      - name: Upload meta.json
+      - name: Upload meta files
         uses: ./.github/actions/r2-upload
         with:
-          filenames: meta.json
+          filenames: stable/meta.json,stable/tuf-meta.json,${{ env.FULL_DATE_DIR }}/meta.json,${{ env.FULL_DATE_DIR }}/tuf-meta.json
+
+  verify-fleetd-base:
+    needs: update-meta-files
+    uses: ./.github/workflows/verify-fleetd-base.yml
+    with:
+      base-url: "https://download-testing.fleetdm.com" # Production: "https://download.fleetdm.com" | Testing: "https://download-testing.fleetdm.com"

.github/workflows/release-fleetd-chrome-beta.yml

@@ -1,5 +1,13 @@
 name: Release fleetd-chrome beta
 
+# Build and upload the following files:
+# - fleetd.crx
+# - updates.xml
+# - meta.json
+# - archive/YYYY-MM-DD_HH-MM-SS/fleetd.crx
+# - archive/YYYY-MM-DD_HH-MM-SS/updates.xml
+# - archive/YYYY-MM-DD_HH-MM-SS/meta.json
+
 on:
   push:
    tags:
@@ -23,6 +31,8 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: read
+    env:
+      BASE_URL: https://chrome-beta.fleetdm.com
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
@@ -35,7 +45,7 @@ jobs:
       - name: Run test
         working-directory: ./ee/fleetd-chrome
         run: |
-          npm install && npm run test
+          npm install --no-save && npm run test
 
       - name: Set the version
         working-directory: ./ee/fleetd-chrome
@@ -54,10 +64,24 @@ jobs:
           /usr/bin/google-chrome --pack-extension=./dist --pack-extension-key=chrome.pem
 
       - name: Prepare files for upload
+        id: prepare-files
         working-directory: ./ee/fleetd-chrome
         run: |
           mv dist.crx fleetd.crx
           mv updates-beta.xml updates.xml
+          datedir=$(date -u +%Y-%m-%d_%H-%M-%S)
+          mkdir -p archive/$datedir
+          cp fleetd.crx archive/$datedir
+          cp updates.xml archive/$datedir
+          echo "{
+            \"fleetd_crx_url\": \"${{ env.BASE_URL }}/archive/$datedir/fleetd.crx\",
+            \"updates_xml\": \"${{ env.BASE_URL }}/archive/$datedir/updates.xml\",
+            \"version\": \"$datedir\"
+          }" > meta.json
+          : # Check that meta.json is valid
+          jq -e . >/dev/null 2>&1 <<< $(cat meta.json)
+          cp meta.json archive/$datedir
+          echo "datedir=$datedir" >> $GITHUB_OUTPUT
 
       - name: Upload extension
         uses: ./.github/actions/r2-upload
@@ -67,4 +91,5 @@ jobs:
           R2_ACCESS_KEY_SECRET: ${{ secrets.R2_CHROME_BETA_ACCESS_KEY_SECRET }}
           R2_BUCKET: chrome-beta
         with:
-          filenames: ./ee/fleetd-chrome/fleetd.crx,./ee/fleetd-chrome/updates.xml
+          working-directory: ./ee/fleetd-chrome
+          filenames: fleetd.crx,updates.xml,meta.json,archive/${{ steps.prepare-files.outputs.datedir }}/fleetd.crx,archive/${{ steps.prepare-files.outputs.datedir }}/updates.xml,archive/${{ steps.prepare-files.outputs.datedir }}/meta.json