From ba7c4070d5d196d5dbbca4a7735e553cdfc3a509 Mon Sep 17 00:00:00 2001 From: Brandon Morelli Date: Fri, 9 Feb 2024 12:26:53 -0800 Subject: [PATCH 01/11] Update build_pipeline.yml --- .buildkite/build_pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/build_pipeline.yml b/.buildkite/build_pipeline.yml index 43ff5240baea..f2322aaaddeb 100644 --- a/.buildkite/build_pipeline.yml +++ b/.buildkite/build_pipeline.yml @@ -1,4 +1,6 @@ steps: + concurrency_group: docs-build-${BUILDKITE_BRANCH} + concurrency: 1 - input: "Build parameters" if: build.source == "ui" fields: @@ -37,8 +39,6 @@ steps: provider: "gcp" image: family/docs-ubuntu-2204 machineType: ${BUILD_MACHINE_TYPE} - concurrency_group: build-docs-${BUILDKITE_BRANCH} - concurrency: 1 notify: - email: "docs-status@elastic.co" if: build.state == "failed" From 22f63a9540149f6c9d640039bf570139921842e4 Mon Sep 17 00:00:00 2001 From: bmorelli25 Date: Fri, 9 Feb 2024 12:36:52 -0800 Subject: [PATCH 02/11] add group to each step --- .buildkite/build_pipeline.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.buildkite/build_pipeline.yml b/.buildkite/build_pipeline.yml index f2322aaaddeb..2f14e4f1f7aa 100644 --- a/.buildkite/build_pipeline.yml +++ b/.buildkite/build_pipeline.yml @@ -1,6 +1,4 @@ steps: - concurrency_group: docs-build-${BUILDKITE_BRANCH} - concurrency: 1 - input: "Build parameters" if: build.source == "ui" fields: @@ -26,10 +24,13 @@ steps: - label: "Fail the build" value: "" hint: "Should we ignore checking broken links? Should we allow to run the build without failing if there's a broken link? Ignoring broken links is dangerous not just because bad links will leak into the public site but because subsequent builds and pull requests that do not fix the links fail." + concurrency: 1 + concurrency_group: docs-build-${BUILDKITE_BRANCH} - wait - label: "Full rebuild or incremental build?" if: build.source == "schedule" command: ".buildkite/scripts/compare_commits.sh" + concurrency_group: docs-build-${BUILDKITE_BRANCH} - label: ":white_check_mark: Build docs" command: | export REBUILD="$(buildkite-agent meta-data get REBUILD --default '' --log-level fatal)" @@ -39,6 +40,7 @@ steps: provider: "gcp" image: family/docs-ubuntu-2204 machineType: ${BUILD_MACHINE_TYPE} + concurrency_group: docs-build-${BUILDKITE_BRANCH} notify: - email: "docs-status@elastic.co" if: build.state == "failed" From de67bd469e576e8c171d7dbc1ca57fe0b39884da Mon Sep 17 00:00:00 2001 From: bmorelli25 Date: Fri, 9 Feb 2024 13:06:18 -0800 Subject: [PATCH 03/11] =?UTF-8?q?cancel=20build=20if=20there=E2=80=99s=20a?= =?UTF-8?q?=20queue?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .buildkite/build_pipeline.yml | 4 ++++ .buildkite/scripts/check_queue.sh | 14 ++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 .buildkite/scripts/check_queue.sh diff --git a/.buildkite/build_pipeline.yml b/.buildkite/build_pipeline.yml index 2f14e4f1f7aa..2b60ba41584b 100644 --- a/.buildkite/build_pipeline.yml +++ b/.buildkite/build_pipeline.yml @@ -24,6 +24,10 @@ steps: - label: "Fail the build" value: "" hint: "Should we ignore checking broken links? Should we allow to run the build without failing if there's a broken link? Ignoring broken links is dangerous not just because bad links will leak into the public site but because subsequent builds and pull requests that do not fix the links fail." + - wait + - label: "Check for queue" + if: build.source == "schedule" + command: ".buildkite/scripts/check_queue.sh" concurrency: 1 concurrency_group: docs-build-${BUILDKITE_BRANCH} - wait diff --git a/.buildkite/scripts/check_queue.sh b/.buildkite/scripts/check_queue.sh new file mode 100644 index 000000000000..298690c5fed8 --- /dev/null +++ b/.buildkite/scripts/check_queue.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +last_build_url="https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds?branch=${BUILDKITE_BRANCH}" +cancel_build_url="https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_JOB_ID}/cancel" + +LAST_BUILD_STATE=$(curl -s -H "Authorization: Bearer ${BUILDKITE_API_TOKEN}" $last_successful_build_url | jq -r '.[1].status') + +echo "Determining if the last build is currently blocked." +if [[ "$LAST_BUILD_STATE" == "blocked" ]]; then + echo "The pipeline is congested. Canceling this build." + curl -sX PUT -H "Authorization: Bearer ${BUILDKITE_API_TOKEN}" $cancel_build_url +else + echo "The pipeline is ready for a new build." +fi From 12172c7c705e2d6c155b890686564b86e56f29fc Mon Sep 17 00:00:00 2001 From: bmorelli25 Date: Fri, 9 Feb 2024 13:09:26 -0800 Subject: [PATCH 04/11] remove concurrency gate from queue step --- .buildkite/build_pipeline.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.buildkite/build_pipeline.yml b/.buildkite/build_pipeline.yml index 2b60ba41584b..081d085fb0df 100644 --- a/.buildkite/build_pipeline.yml +++ b/.buildkite/build_pipeline.yml @@ -28,12 +28,11 @@ steps: - label: "Check for queue" if: build.source == "schedule" command: ".buildkite/scripts/check_queue.sh" - concurrency: 1 - concurrency_group: docs-build-${BUILDKITE_BRANCH} - wait - label: "Full rebuild or incremental build?" if: build.source == "schedule" command: ".buildkite/scripts/compare_commits.sh" + concurrency: 1 concurrency_group: docs-build-${BUILDKITE_BRANCH} - label: ":white_check_mark: Build docs" command: | From 0219016825b0c2cf52cd7ad802123cacf7e30184 Mon Sep 17 00:00:00 2001 From: bmorelli25 Date: Fri, 9 Feb 2024 13:12:46 -0800 Subject: [PATCH 05/11] comment out for testing purposes --- .buildkite/build_pipeline.yml | 56 +++++++++++++++++------------------ 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.buildkite/build_pipeline.yml b/.buildkite/build_pipeline.yml index 081d085fb0df..d3b4de012cac 100644 --- a/.buildkite/build_pipeline.yml +++ b/.buildkite/build_pipeline.yml @@ -1,36 +1,36 @@ steps: - - input: "Build parameters" - if: build.source == "ui" - fields: - - select: "Rebuild?" - key: "REBUILD" - default: "" - required: false - options: - - label: "no" - value: "" - - label: "yes" - value: "rebuild" - hint: "Should all books be rebuilt, regardless of what has changed? Build once with this set to true after every release." - - select: "How should broken links be handled?" - key: "BROKEN_LINKS" - default: "" - required: false - options: - - label: "Continue without warning" - value: "skiplinkcheck" - - label: "Continue, but log a warning" - value: "warnlinkcheck" - - label: "Fail the build" - value: "" - hint: "Should we ignore checking broken links? Should we allow to run the build without failing if there's a broken link? Ignoring broken links is dangerous not just because bad links will leak into the public site but because subsequent builds and pull requests that do not fix the links fail." - - wait + # - input: "Build parameters" + # if: build.source == "ui" + # fields: + # - select: "Rebuild?" + # key: "REBUILD" + # default: "" + # required: false + # options: + # - label: "no" + # value: "" + # - label: "yes" + # value: "rebuild" + # hint: "Should all books be rebuilt, regardless of what has changed? Build once with this set to true after every release." + # - select: "How should broken links be handled?" + # key: "BROKEN_LINKS" + # default: "" + # required: false + # options: + # - label: "Continue without warning" + # value: "skiplinkcheck" + # - label: "Continue, but log a warning" + # value: "warnlinkcheck" + # - label: "Fail the build" + # value: "" + # hint: "Should we ignore checking broken links? Should we allow to run the build without failing if there's a broken link? Ignoring broken links is dangerous not just because bad links will leak into the public site but because subsequent builds and pull requests that do not fix the links fail." + # - wait - label: "Check for queue" - if: build.source == "schedule" + # if: build.source == "schedule" command: ".buildkite/scripts/check_queue.sh" - wait - label: "Full rebuild or incremental build?" - if: build.source == "schedule" + # if: build.source == "schedule" command: ".buildkite/scripts/compare_commits.sh" concurrency: 1 concurrency_group: docs-build-${BUILDKITE_BRANCH} From 4d62453f8c1d54c4727f6c62d9d1c2f9620889ee Mon Sep 17 00:00:00 2001 From: bmorelli25 Date: Fri, 9 Feb 2024 13:30:34 -0800 Subject: [PATCH 06/11] fix concurrency gate --- .buildkite/build_pipeline.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.buildkite/build_pipeline.yml b/.buildkite/build_pipeline.yml index d3b4de012cac..23bdb2a31f96 100644 --- a/.buildkite/build_pipeline.yml +++ b/.buildkite/build_pipeline.yml @@ -25,16 +25,28 @@ steps: # value: "" # hint: "Should we ignore checking broken links? Should we allow to run the build without failing if there's a broken link? Ignoring broken links is dangerous not just because bad links will leak into the public site but because subsequent builds and pull requests that do not fix the links fail." # - wait + # This step is purposefully outside of the concurrency gate - label: "Check for queue" + key: check-queue # if: build.source == "schedule" command: ".buildkite/scripts/check_queue.sh" + - label: "Start of concurrency gate" + key: start-gate + depends_on: check-queue + command: echo "Start of concurrency gate" + concurrency_group: gate/docs-build-${BUILDKITE_BRANCH} + concurrency: 1 - wait - label: "Full rebuild or incremental build?" + key: build-type + depends_on: start-gate # if: build.source == "schedule" command: ".buildkite/scripts/compare_commits.sh" concurrency: 1 concurrency_group: docs-build-${BUILDKITE_BRANCH} - label: ":white_check_mark: Build docs" + key: build-docs + depends_on: build-type command: | export REBUILD="$(buildkite-agent meta-data get REBUILD --default '' --log-level fatal)" export BROKEN_LINKS="$(buildkite-agent meta-data get BROKEN_LINKS --default '' --log-level fatal)" @@ -44,6 +56,11 @@ steps: image: family/docs-ubuntu-2204 machineType: ${BUILD_MACHINE_TYPE} concurrency_group: docs-build-${BUILDKITE_BRANCH} + - label: "End of concurrency gate" + key: end-gate + depends_on: build-docs + command: echo "End of concurrency gate" + concurrency_group: gate/docs-build-${BUILDKITE_BRANCH} notify: - email: "docs-status@elastic.co" if: build.state == "failed" From 5a059e2bc7deac8ee444dfe4a4fdee84744e6d93 Mon Sep 17 00:00:00 2001 From: bmorelli25 Date: Fri, 9 Feb 2024 13:47:07 -0800 Subject: [PATCH 07/11] the fix for the fix --- .buildkite/build_pipeline.yml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/.buildkite/build_pipeline.yml b/.buildkite/build_pipeline.yml index 23bdb2a31f96..578376c94da7 100644 --- a/.buildkite/build_pipeline.yml +++ b/.buildkite/build_pipeline.yml @@ -30,23 +30,17 @@ steps: key: check-queue # if: build.source == "schedule" command: ".buildkite/scripts/check_queue.sh" + - wait - label: "Start of concurrency gate" - key: start-gate - depends_on: check-queue command: echo "Start of concurrency gate" concurrency_group: gate/docs-build-${BUILDKITE_BRANCH} concurrency: 1 - wait - label: "Full rebuild or incremental build?" - key: build-type - depends_on: start-gate # if: build.source == "schedule" command: ".buildkite/scripts/compare_commits.sh" - concurrency: 1 - concurrency_group: docs-build-${BUILDKITE_BRANCH} + - wait - label: ":white_check_mark: Build docs" - key: build-docs - depends_on: build-type command: | export REBUILD="$(buildkite-agent meta-data get REBUILD --default '' --log-level fatal)" export BROKEN_LINKS="$(buildkite-agent meta-data get BROKEN_LINKS --default '' --log-level fatal)" @@ -55,12 +49,11 @@ steps: provider: "gcp" image: family/docs-ubuntu-2204 machineType: ${BUILD_MACHINE_TYPE} - concurrency_group: docs-build-${BUILDKITE_BRANCH} + - wait - label: "End of concurrency gate" - key: end-gate - depends_on: build-docs command: echo "End of concurrency gate" concurrency_group: gate/docs-build-${BUILDKITE_BRANCH} + concurrency: 1 notify: - email: "docs-status@elastic.co" if: build.state == "failed" From e01aafdaae1d9cbfd3938edf901ce1024a918712 Mon Sep 17 00:00:00 2001 From: bmorelli25 Date: Fri, 9 Feb 2024 13:56:52 -0800 Subject: [PATCH 08/11] look three builds back --- .buildkite/scripts/check_queue.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/check_queue.sh b/.buildkite/scripts/check_queue.sh index 298690c5fed8..8ef42d2f7dfc 100644 --- a/.buildkite/scripts/check_queue.sh +++ b/.buildkite/scripts/check_queue.sh @@ -3,10 +3,14 @@ last_build_url="https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds?branch=${BUILDKITE_BRANCH}" cancel_build_url="https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_JOB_ID}/cancel" -LAST_BUILD_STATE=$(curl -s -H "Authorization: Bearer ${BUILDKITE_API_TOKEN}" $last_successful_build_url | jq -r '.[1].status') +# Don't look at this build (it's running now!) +# Don't look at the last build (it's okay if it's still running!) +# Look three builds back instead (if this build is still running, +# it means there's already one in the queue and we can safely cancel this one) +THIRD_TO_LAST_BUILD_STATE=$(curl -s -H "Authorization: Bearer ${BUILDKITE_API_TOKEN}" $last_successful_build_url | jq -r '.[2].status') -echo "Determining if the last build is currently blocked." -if [[ "$LAST_BUILD_STATE" == "blocked" ]]; then +echo "Determining if there are multiple builds waiting." +if [[ "$THIRD_TO_LAST_BUILD_STATE" == "running" ]]; then echo "The pipeline is congested. Canceling this build." curl -sX PUT -H "Authorization: Bearer ${BUILDKITE_API_TOKEN}" $cancel_build_url else From 31a5e1c303a3ffb8344c0e39eccd286bc584146a Mon Sep 17 00:00:00 2001 From: bmorelli25 Date: Fri, 9 Feb 2024 14:07:53 -0800 Subject: [PATCH 09/11] fix rookie mistake --- .buildkite/scripts/check_queue.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/check_queue.sh b/.buildkite/scripts/check_queue.sh index 8ef42d2f7dfc..b7556a33d0c3 100644 --- a/.buildkite/scripts/check_queue.sh +++ b/.buildkite/scripts/check_queue.sh @@ -1,13 +1,15 @@ #!/bin/bash -last_build_url="https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds?branch=${BUILDKITE_BRANCH}" +build_data_url="https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds?branch=${BUILDKITE_BRANCH}" cancel_build_url="https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_JOB_ID}/cancel" # Don't look at this build (it's running now!) # Don't look at the last build (it's okay if it's still running!) # Look three builds back instead (if this build is still running, # it means there's already one in the queue and we can safely cancel this one) -THIRD_TO_LAST_BUILD_STATE=$(curl -s -H "Authorization: Bearer ${BUILDKITE_API_TOKEN}" $last_successful_build_url | jq -r '.[2].status') +THIRD_TO_LAST_BUILD_STATE=$(curl -s -H "Authorization: Bearer ${BUILDKITE_API_TOKEN}" $build_data_url | jq -r '.[2].status') + +echo $(curl -s -H "Authorization: Bearer ${BUILDKITE_API_TOKEN}" $build_data_url) echo "Determining if there are multiple builds waiting." if [[ "$THIRD_TO_LAST_BUILD_STATE" == "running" ]]; then From a4b8790fb40edfd397ea6cdc2e3763627bd494ab Mon Sep 17 00:00:00 2001 From: bmorelli25 Date: Fri, 9 Feb 2024 14:13:21 -0800 Subject: [PATCH 10/11] rookie mistake here, there, everywhere --- .buildkite/scripts/check_queue.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.buildkite/scripts/check_queue.sh b/.buildkite/scripts/check_queue.sh index b7556a33d0c3..7260e089a863 100644 --- a/.buildkite/scripts/check_queue.sh +++ b/.buildkite/scripts/check_queue.sh @@ -7,9 +7,7 @@ cancel_build_url="https://api.buildkite.com/v2/organizations/elastic/pipelines/$ # Don't look at the last build (it's okay if it's still running!) # Look three builds back instead (if this build is still running, # it means there's already one in the queue and we can safely cancel this one) -THIRD_TO_LAST_BUILD_STATE=$(curl -s -H "Authorization: Bearer ${BUILDKITE_API_TOKEN}" $build_data_url | jq -r '.[2].status') - -echo $(curl -s -H "Authorization: Bearer ${BUILDKITE_API_TOKEN}" $build_data_url) +THIRD_TO_LAST_BUILD_STATE=$(curl -s -H "Authorization: Bearer ${BUILDKITE_API_TOKEN}" $build_data_url | jq -r '.[2].state') echo "Determining if there are multiple builds waiting." if [[ "$THIRD_TO_LAST_BUILD_STATE" == "running" ]]; then From ab1b944e824fe02010a06995a85a4977ddf0749c Mon Sep 17 00:00:00 2001 From: bmorelli25 Date: Fri, 9 Feb 2024 14:16:19 -0800 Subject: [PATCH 11/11] Use BUILDKITE_BUILD_NUMBER instead of BUILDKITE_BUILD_ID --- .buildkite/scripts/check_queue.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/check_queue.sh b/.buildkite/scripts/check_queue.sh index 7260e089a863..1a96d41566a3 100644 --- a/.buildkite/scripts/check_queue.sh +++ b/.buildkite/scripts/check_queue.sh @@ -1,7 +1,7 @@ #!/bin/bash build_data_url="https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds?branch=${BUILDKITE_BRANCH}" -cancel_build_url="https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_JOB_ID}/cancel" +cancel_build_url="https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}/cancel" # Don't look at this build (it's running now!) # Don't look at the last build (it's okay if it's still running!)