chore: upload test results to buildkite and datadog (solana-labs#27139)

* export test-stable result * export test-stable-perf result * export test-local-cluster result * export test-local-cluster-flakey result * export test-local-cluster-slow-1 result * export test-local-cluster-slow-2 result * export test-docs result * export test-stable-bpf result * upload test result to buildkite and datadog
CriesofCarrots · Aug 16, 2022 · 3a9bc4b · 3a9bc4b
1 parent 5f195ba
commit 3a9bc4b
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 9 deletions.
diff --git a/.buildkite/hooks/post-command b/.buildkite/hooks/post-command
@@ -20,6 +20,18 @@ else
     SUCCESS=false
   fi
 
+  if [[ "$BUILDKITE_BRANCH" == 'master' && -f "results.json" ]]; then
+    # prepare result file
+    awk '/{ "type": .* }/' results.json > sanitized-results.json
+
+    # upload to buildkite
+    buildkite-test-collector < sanitized-results.json
+
+    # upload to datadog
+    cargo2junit > results.xml < sanitized-results.json
+    datadog-ci junit upload --service solana results.xml
+  fi
+
   point_tags="pipeline=$BUILDKITE_PIPELINE_SLUG,job=$CI_LABEL,pr=$PR,success=$SUCCESS"
   point_tags="${point_tags// /\\ }"  # Escape spaces
 

diff --git a/ci/test-stable.sh b/ci/test-stable.sh
@@ -12,6 +12,12 @@ annotate() {
   }
 }
 
+exit_if_error() {
+   if [[ "$1" -ne 0 ]]; then
+    exit "$1"
+  fi
+}
+
 # Run the appropriate test based on entrypoint
 testName=$(basename "$0" .sh)
 
@@ -35,7 +41,8 @@ JOBS=$((JOBS>NPROC ? NPROC : JOBS))
 echo "Executing $testName"
 case $testName in
 test-stable)
-  _ "$cargo" stable test --jobs "$JOBS" --all --tests --exclude solana-local-cluster ${V:+--verbose} -- --nocapture
+  _ "$cargo" stable test --jobs "$JOBS" --all --tests --exclude solana-local-cluster ${V:+--verbose} -- -Z unstable-options --format json --report-time | tee results.json
+  exit_if_error "${PIPESTATUS[0]}"
   ;;
 test-stable-bpf)
   # Clear the C dependency files, if dependency moves these files are not regenerated
@@ -59,7 +66,8 @@ test-stable-bpf)
   _ make -C programs/bpf/c tests
   _ "$cargo" stable test \
     --manifest-path programs/bpf/Cargo.toml \
-    --no-default-features --features=bpf_c,bpf_rust -- --nocapture
+    --no-default-features --features=bpf_c,bpf_rust -- -Z unstable-options --format json --report-time | tee results.json
+  exit_if_error "${PIPESTATUS[0]}"
 
   # BPF Rust program unit tests
   for bpf_test in programs/bpf/rust/*; do
@@ -95,7 +103,8 @@ test-stable-bpf)
   _ "$cargo" stable test \
     --manifest-path programs/bpf/Cargo.toml \
     --no-default-features --features=bpf_c,bpf_rust assert_instruction_count \
-    -- --nocapture &> "${bpf_target_path}"/deploy/instuction_counts.txt
+    -- -Z unstable-options --format json --report-time |& tee results.json
+  awk '!/{ "type": .* }/' results.json > "${bpf_target_path}"/deploy/instuction_counts.txt
 
   bpf_dump_archive="bpf-dumps.tar.bz2"
   rm -f "$bpf_dump_archive"
@@ -120,27 +129,32 @@ test-stable-perf)
   fi
 
   _ "$cargo" stable build --bins ${V:+--verbose}
-  _ "$cargo" stable test --package solana-perf --package solana-ledger --package solana-core --lib ${V:+--verbose} -- --nocapture
+  _ "$cargo" stable test --package solana-perf --package solana-ledger --package solana-core --lib ${V:+--verbose} -- -Z unstable-options --format json --report-time | tee results.json
+  exit_if_error "${PIPESTATUS[0]}"
   _ "$cargo" stable run --manifest-path poh-bench/Cargo.toml ${V:+--verbose} -- --hashes-per-tick 10
   ;;
 test-local-cluster)
   _ "$cargo" stable build --release --bins ${V:+--verbose}
-  _ "$cargo" stable test --release --package solana-local-cluster --test local_cluster ${V:+--verbose} -- --nocapture --test-threads=1
+  _ "$cargo" stable test --release --package solana-local-cluster --test local_cluster ${V:+--verbose} -- --test-threads=1 -Z unstable-options --format json --report-time | tee results.json
+  exit_if_error "${PIPESTATUS[0]}"
   exit 0
   ;;
 test-local-cluster-flakey)
   _ "$cargo" stable build --release --bins ${V:+--verbose}
-  _ "$cargo" stable test --release --package solana-local-cluster --test local_cluster_flakey ${V:+--verbose} -- --nocapture --test-threads=1
+  _ "$cargo" stable test --release --package solana-local-cluster --test local_cluster_flakey ${V:+--verbose} -- --test-threads=1 -Z unstable-options --format json --report-time | tee results.json
+  exit_if_error "${PIPESTATUS[0]}"
   exit 0
   ;;
 test-local-cluster-slow-1)
   _ "$cargo" stable build --release --bins ${V:+--verbose}
-  _ "$cargo" stable test --release --package solana-local-cluster --test local_cluster_slow_1 ${V:+--verbose} -- --nocapture --test-threads=1
+  _ "$cargo" stable test --release --package solana-local-cluster --test local_cluster_slow_1 ${V:+--verbose} -- --test-threads=1 -Z unstable-options --format json --report-time | tee results.json
+  exit_if_error "${PIPESTATUS[0]}"
   exit 0
   ;;
 test-local-cluster-slow-2)
   _ "$cargo" stable build --release --bins ${V:+--verbose}
-  _ "$cargo" stable test --release --package solana-local-cluster --test local_cluster_slow_2 ${V:+--verbose} -- --nocapture --test-threads=1
+  _ "$cargo" stable test --release --package solana-local-cluster --test local_cluster_slow_2 ${V:+--verbose} -- --test-threads=1 -Z unstable-options --format json --report-time | tee results.json
+  exit_if_error "${PIPESTATUS[0]}"
   exit 0
   ;;
 test-wasm)
@@ -157,7 +171,8 @@ test-wasm)
   exit 0
   ;;
 test-docs)
-  _ "$cargo" stable test --jobs "$JOBS" --all --doc --exclude solana-local-cluster ${V:+--verbose} -- --nocapture
+  _ "$cargo" stable test --jobs "$JOBS" --all --doc --exclude solana-local-cluster ${V:+--verbose} -- -Z unstable-options --format json --report-time | tee results.json
+  exit_if_error "${PIPESTATUS[0]}"
   exit 0
   ;;
 *)