[serverless] Upload serverless assets to Azure so we can release for a layer

.azure-pipelines/ultimate-pipeline.yml

@@ -4257,9 +4257,49 @@ stages:
           displayName: Publish ssi artifacts
           artifact: ssi-artifacts
 
+- stage: store_serverless_artifacts
+  condition: and(succeeded(), eq(variables['isBenchmarksOnlyBuild'], 'False'), eq(variables.isMainRepository, true))
+  dependsOn: [build_linux_tracer_r2r, build_arm64_tracer_r2r, build_linux_universal, build_arm64_universal]
+  jobs:
+    - job: combine
+      timeoutInMinutes: 60 #default value
+      pool:
+        vmImage: ubuntu-latest
+      steps:
+        - checkout: none
+        # Download the files to the expected locations
+        - task: DownloadPipelineArtifact@2
+          displayName: Download serverless tracer home x64
+          inputs:
+            artifact: linux-tracer-home-linux-x64-r2r
+            path: $(Build.ArtifactStagingDirectory)/x64
+
+        - task: DownloadPipelineArtifact@2
+          displayName: Download serverless universal home x64
+          inputs:
+            artifact: linux-universal-home-linux-x64
+            path: $(Build.ArtifactStagingDirectory)/x64
+
+        - task: DownloadPipelineArtifact@2
+          displayName: Download serverless tracer home arm64
+          inputs:
+            artifact: linux-tracer-home-linux-arm64-r2r
+            path: $(Build.ArtifactStagingDirectory)/arm64
+
+        - task: DownloadPipelineArtifact@2
+          displayName: Download serverless universal home arm64
+          inputs:
+            artifact: linux-universal-home-linux-arm64
+            path: $(Build.ArtifactStagingDirectory)/arm64
+
+        # publish all tar files as single artifact
+        - publish: "$(Build.ArtifactStagingDirectory)"
+          displayName: Publish serverless artifacts
+          artifact: serverless-artifacts
+
 - stage: upload_to_azure
   condition: and(succeeded(), eq(variables['isBenchmarksOnlyBuild'], 'False'), eq(variables.isMainRepository, true))
-  dependsOn: [package_windows, package_linux, package_arm64, dotnet_tool, merge_commit_id]
+  dependsOn: [package_windows, package_linux, package_arm64, dotnet_tool, merge_commit_id, store_serverless_artifacts]
   variables:
     targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
     targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
@@ -4485,6 +4525,19 @@ stages:
             artifact: windows-msi-x64
             path: $(Build.ArtifactStagingDirectory)
 
+        # Download the artifacts required by the serverless layer and create the zip file
+        # This exapands the directory - but we want a zip file, so re-zip it!
+        - task: DownloadPipelineArtifact@2
+          displayName: Download serverless artifacts
+          inputs:
+            artifact: serverless-artifacts
+            path: $(Agent.TempDirectory)/serverless-artifacts
+
+        - bash: |
+            cd $(Agent.TempDirectory)/serverless-artifacts
+            zip -r $(Build.ArtifactStagingDirectory)/serverless-artifacts.zip .
+          displayName: Zip serverless artifacts and store in staging directory
+
         - bash: |
             # Write tracer version number to version.txt
             echo "$(tracer_version)" > $(Build.ArtifactStagingDirectory)/version.txt

.gitlab/download-serverless-artifacts.sh

@@ -10,7 +10,22 @@ set -eo pipefail
 target_dir=artifacts
 mkdir -p $target_dir
 
+if [ -n "$CI_COMMIT_TAG" ] && [ -n "$CI_COMMIT_SHA" ]; then
+  echo "Downloading artifacts from Azure"
+  curl --location --fail \
+    --output $target_dir/serverless-artifacts.zip \
+    "https://apmdotnetci.blob.core.windows.net/apm-dotnet-ci-artifacts-master/${CI_COMMIT_SHA}/serverless-artifacts.zip"
+
+  # Extract top level artifact
+  unzip $target_dir/serverless-artifacts.zip -d $target_dir/
+  rm -f $target_dir/serverless-artifacts.zip
+
+  ls -l $target_dir
+  exit 0
+fi
+
 branchName="refs/heads/$CI_COMMIT_BRANCH"
+artifactName="serverless-artifacts"
 
 echo "Looking for azure devops PR builds for branch '$branchName' for commit '$CI_COMMIT_SHA' to start"
 
@@ -32,41 +47,31 @@ fi
 
 echo "Found build with id '$buildId' for commit '$CI_COMMIT_SHA' on branch '$branchName'"
 
-architectures=("x64" "arm64")
-for architecture in "${architectures[@]}"; do
-  echo "Looking for artifacts for architecture '$architecture'"
-
-  artifacts=("linux-tracer-home-linux-$architecture-r2r" "linux-universal-home-linux-$architecture")
-
-  # Now try to download the artifacts from the build
-  for artifactName in "${artifacts[@]}"; do
-    artifactsUrl="https://dev.azure.com/datadoghq/dd-trace-dotnet/_apis/build/builds/$buildId/artifacts?api-version=7.1&artifactName=$artifactName"
-
-    # Keep trying to get the artifact for 30 minutes
-    downloadUrl=""
-    TIMEOUT=1800
-    STARTED=0
-    until (( STARTED == TIMEOUT )) || [ ! -z "${downloadUrl}" ] ; do
-        echo "Checking for '$artifactName' at '$artifactsUrl'..."
-        # If the artifact doesn't exist, .resource.downloadUrl will be null, so we filter that out
-        downloadUrl=$(curl -s $artifactsUrl | jq -r '.resource.downloadUrl | select( . != null )')
-        sleep 100
-        (( STARTED += 100 ))
-    done
-    (( STARTED < TIMEOUT ))
-
-    if [ -z "${downloadUrl}" ]; then
-      echo "No downloadUrl found after 30 minutes for commit '$CI_COMMIT_SHA' on branch '$branchName'"
-      exit 1
-    fi
-
-    echo "Downloading '$artifactName' from '$downloadUrl'..."
-    curl -o $target_dir/artifacts.zip "$downloadUrl"
-    unzip $target_dir/artifacts.zip -d $target_dir/$architecture
-    mv $target_dir/$architecture/$artifactName/* $target_dir/$architecture
-    rm -rf $target_dir/artifacts.zip
-    rmdir $target_dir/$architecture/$artifactName
-  done
+# Now try to download the artifacts from the build
+artifactsUrl="https://dev.azure.com/datadoghq/dd-trace-dotnet/_apis/build/builds/$buildId/artifacts?api-version=7.1&artifactName=$artifactName"
+
+# Keep trying to get the artifact for 40 minutes
+TIMEOUT=2400
+STARTED=0
+until (( STARTED == TIMEOUT )) || [ ! -z "${downloadUrl}" ] ; do
+    echo "Checking for artifacts at '$artifactsUrl'..."
+    # If the artifact doesn't exist, .resource.downloadUrl will be null, so we filter that out
+    downloadUrl=$(curl -s $artifactsUrl | jq -r '.resource.downloadUrl | select( . != null )')
+    sleep 100
+    (( STARTED += 100 ))
 done
+(( STARTED < TIMEOUT ))
+
+if [ -z "${downloadUrl}" ]; then
+  echo "No downloadUrl found after 30 minutes for commit '$CI_COMMIT_SHA' on branch '$branchName'"
+  exit 1
+fi
+
+echo "Downloading artifacts from '$downloadUrl'..."
+curl -o $target_dir/artifacts.zip "$downloadUrl"
+unzip $target_dir/artifacts.zip -d $target_dir
+mv $target_dir/$artifactName/* $target_dir
+rm -rf $target_dir/artifacts.zip
+rmdir $target_dir/$artifactName
 
 ls -l $target_dir