diff --git a/.azure-pipelines/ultimate-pipeline.yml b/.azure-pipelines/ultimate-pipeline.yml index d73224c9eb38..1b4be12a27d5 100644 --- a/.azure-pipelines/ultimate-pipeline.yml +++ b/.azure-pipelines/ultimate-pipeline.yml @@ -4257,9 +4257,49 @@ stages: displayName: Publish ssi artifacts artifact: ssi-artifacts +- stage: store_serverless_artifacts + condition: and(succeeded(), eq(variables['isBenchmarksOnlyBuild'], 'False'), eq(variables.isMainRepository, true)) + dependsOn: [build_linux_tracer_r2r, build_arm64_tracer_r2r, build_linux_universal, build_arm64_universal] + jobs: + - job: combine + timeoutInMinutes: 60 #default value + pool: + vmImage: ubuntu-latest + steps: + - checkout: none + # Download the files to the expected locations + - task: DownloadPipelineArtifact@2 + displayName: Download serverless tracer home x64 + inputs: + artifact: linux-tracer-home-linux-x64-r2r + path: $(Build.ArtifactStagingDirectory)/x64 + + - task: DownloadPipelineArtifact@2 + displayName: Download serverless universal home x64 + inputs: + artifact: linux-universal-home-linux-x64 + path: $(Build.ArtifactStagingDirectory)/x64 + + - task: DownloadPipelineArtifact@2 + displayName: Download serverless tracer home arm64 + inputs: + artifact: linux-tracer-home-linux-arm64-r2r + path: $(Build.ArtifactStagingDirectory)/arm64 + + - task: DownloadPipelineArtifact@2 + displayName: Download serverless universal home arm64 + inputs: + artifact: linux-universal-home-linux-arm64 + path: $(Build.ArtifactStagingDirectory)/arm64 + + # publish all tar files as single artifact + - publish: "$(Build.ArtifactStagingDirectory)" + displayName: Publish serverless artifacts + artifact: serverless-artifacts + - stage: upload_to_azure condition: and(succeeded(), eq(variables['isBenchmarksOnlyBuild'], 'False'), eq(variables.isMainRepository, true)) - dependsOn: [package_windows, package_linux, package_arm64, dotnet_tool, merge_commit_id] + dependsOn: [package_windows, package_linux, package_arm64, dotnet_tool, merge_commit_id, store_serverless_artifacts] variables: targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']] targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']] @@ -4485,6 +4525,19 @@ stages: artifact: windows-msi-x64 path: $(Build.ArtifactStagingDirectory) + # Download the artifacts required by the serverless layer and create the zip file + # This exapands the directory - but we want a zip file, so re-zip it! + - task: DownloadPipelineArtifact@2 + displayName: Download serverless artifacts + inputs: + artifact: serverless-artifacts + path: $(Agent.TempDirectory)/serverless-artifacts + + - bash: | + cd $(Agent.TempDirectory)/serverless-artifacts + zip -r $(Build.ArtifactStagingDirectory)/serverless-artifacts.zip . + displayName: Zip serverless artifacts and store in staging directory + - bash: | # Write tracer version number to version.txt echo "$(tracer_version)" > $(Build.ArtifactStagingDirectory)/version.txt diff --git a/.gitlab/download-serverless-artifacts.sh b/.gitlab/download-serverless-artifacts.sh index 03bacd9d463b..0ca189c5c0b9 100755 --- a/.gitlab/download-serverless-artifacts.sh +++ b/.gitlab/download-serverless-artifacts.sh @@ -10,7 +10,22 @@ set -eo pipefail target_dir=artifacts mkdir -p $target_dir +if [ -n "$CI_COMMIT_TAG" ] && [ -n "$CI_COMMIT_SHA" ]; then + echo "Downloading artifacts from Azure" + curl --location --fail \ + --output $target_dir/serverless-artifacts.zip \ + "https://apmdotnetci.blob.core.windows.net/apm-dotnet-ci-artifacts-master/${CI_COMMIT_SHA}/serverless-artifacts.zip" + + # Extract top level artifact + unzip $target_dir/serverless-artifacts.zip -d $target_dir/ + rm -f $target_dir/serverless-artifacts.zip + + ls -l $target_dir + exit 0 +fi + branchName="refs/heads/$CI_COMMIT_BRANCH" +artifactName="serverless-artifacts" echo "Looking for azure devops PR builds for branch '$branchName' for commit '$CI_COMMIT_SHA' to start" @@ -32,41 +47,31 @@ fi echo "Found build with id '$buildId' for commit '$CI_COMMIT_SHA' on branch '$branchName'" -architectures=("x64" "arm64") -for architecture in "${architectures[@]}"; do - echo "Looking for artifacts for architecture '$architecture'" - - artifacts=("linux-tracer-home-linux-$architecture-r2r" "linux-universal-home-linux-$architecture") - - # Now try to download the artifacts from the build - for artifactName in "${artifacts[@]}"; do - artifactsUrl="https://dev.azure.com/datadoghq/dd-trace-dotnet/_apis/build/builds/$buildId/artifacts?api-version=7.1&artifactName=$artifactName" - - # Keep trying to get the artifact for 30 minutes - downloadUrl="" - TIMEOUT=1800 - STARTED=0 - until (( STARTED == TIMEOUT )) || [ ! -z "${downloadUrl}" ] ; do - echo "Checking for '$artifactName' at '$artifactsUrl'..." - # If the artifact doesn't exist, .resource.downloadUrl will be null, so we filter that out - downloadUrl=$(curl -s $artifactsUrl | jq -r '.resource.downloadUrl | select( . != null )') - sleep 100 - (( STARTED += 100 )) - done - (( STARTED < TIMEOUT )) - - if [ -z "${downloadUrl}" ]; then - echo "No downloadUrl found after 30 minutes for commit '$CI_COMMIT_SHA' on branch '$branchName'" - exit 1 - fi - - echo "Downloading '$artifactName' from '$downloadUrl'..." - curl -o $target_dir/artifacts.zip "$downloadUrl" - unzip $target_dir/artifacts.zip -d $target_dir/$architecture - mv $target_dir/$architecture/$artifactName/* $target_dir/$architecture - rm -rf $target_dir/artifacts.zip - rmdir $target_dir/$architecture/$artifactName - done +# Now try to download the artifacts from the build +artifactsUrl="https://dev.azure.com/datadoghq/dd-trace-dotnet/_apis/build/builds/$buildId/artifacts?api-version=7.1&artifactName=$artifactName" + +# Keep trying to get the artifact for 40 minutes +TIMEOUT=2400 +STARTED=0 +until (( STARTED == TIMEOUT )) || [ ! -z "${downloadUrl}" ] ; do + echo "Checking for artifacts at '$artifactsUrl'..." + # If the artifact doesn't exist, .resource.downloadUrl will be null, so we filter that out + downloadUrl=$(curl -s $artifactsUrl | jq -r '.resource.downloadUrl | select( . != null )') + sleep 100 + (( STARTED += 100 )) done +(( STARTED < TIMEOUT )) + +if [ -z "${downloadUrl}" ]; then + echo "No downloadUrl found after 30 minutes for commit '$CI_COMMIT_SHA' on branch '$branchName'" + exit 1 +fi + +echo "Downloading artifacts from '$downloadUrl'..." +curl -o $target_dir/artifacts.zip "$downloadUrl" +unzip $target_dir/artifacts.zip -d $target_dir +mv $target_dir/$artifactName/* $target_dir +rm -rf $target_dir/artifacts.zip +rmdir $target_dir/$artifactName ls -l $target_dir \ No newline at end of file