Skip to content

Commit

Permalink
ci: retry testmo requests (ydb-platform#9727)
Browse files Browse the repository at this point in the history
  • Loading branch information
nikitka authored Sep 27, 2024
1 parent 31a439a commit cfbb87c
Show file tree
Hide file tree
Showing 3 changed files with 76 additions and 7 deletions.
12 changes: 7 additions & 5 deletions .github/actions/test_ya/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,9 @@ runs:
shell: bash
run: |
set -x
echo "$(pwd)/ydb/ci/scripts" >> $GITHUB_PATH
export TMP_DIR=$(pwd)/tmp
rm -rf $TMP_DIR
mkdir -p $TMP_DIR
Expand Down Expand Up @@ -356,7 +358,7 @@ runs:
TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:resources:add-link --name build --url "$TESTMO_RUN_URL" --resources $CURRENT_PUBLIC_DIR/testmo.json
TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:resources:add-field --name git-sha --type string --value "${GITHUB_SHA:0:7}" --resources $CURRENT_PUBLIC_DIR/testmo.json
TESTMO_RUN_ID=$(
TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:run:create --instance "https://$TESTMO_PROXY_ADDR" --project-id ${{ inputs.testman_project_id }} \
TESTMO_TOKEN=${{ inputs.testman_token }} retry.sh -- testmo automation:run:create --instance "https://$TESTMO_PROXY_ADDR" --project-id ${{ inputs.testman_project_id }} \
--name "$TESTMO_RUN_NAME" --source "$TESTMO_SOURCE" --resources $CURRENT_PUBLIC_DIR/testmo.json \
--tags "$TESTMO_BRANCH_TAG" --tags "$TESTMO_EXTRA_TAG"
)
Expand Down Expand Up @@ -471,11 +473,11 @@ runs:
# archive unitest reports (transformed)
tar -C $TESTMO_JUNIT_REPORT_PARTS/.. -czf $PUBLIC_DIR/junit_parts.xml.tar.gz $(basename $TESTMO_JUNIT_REPORT_PARTS)

TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:run:submit-thread \
TESTMO_TOKEN=${{ inputs.testman_token }} retry.sh -- testmo automation:run:submit-thread \
--instance "https://$TESTMO_PROXY_ADDR" --run-id "$TESTMO_RUN_ID" \
--results "$TESTMO_JUNIT_REPORT_PARTS/*.xml"

TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:run:complete --instance "https://$TESTMO_PROXY_ADDR" --run-id $TESTMO_RUN_ID || true
TESTMO_TOKEN=${{ inputs.testman_token }} retry.sh -- testmo automation:run:complete --instance "https://$TESTMO_PROXY_ADDR" --run-id $TESTMO_RUN_ID || true
echo "runid=" >> $GITHUB_OUTPUT
fi

Expand Down Expand Up @@ -512,7 +514,7 @@ runs:
shell: bash
run: |
if [ ${{ steps.build.outputs.runid }} ]; then
TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:run:complete --instance "https://$TESTMO_PROXY_ADDR" --run-id ${{ steps.build.outputs.runid }} || true
TESTMO_TOKEN=${{ inputs.testman_token }} retry.sh -- testmo automation:run:complete --instance "https://$TESTMO_PROXY_ADDR" --run-id ${{ steps.build.outputs.runid }} || true
fi
if [ ${{ inputs.testman_token }} ]; then
kill $TESTMO_PROXY_PID
Expand Down
67 changes: 67 additions & 0 deletions ydb/ci/scripts/retry.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
#!/usr/bin/env bash

usage() {
echo "$0 -r [max_attempts] -t [timeout] -s [sleep] -- cmd"
}
if [ $# -lt 2 ]; then
usage
exit 1
fi

max_attempts=0
timeout=1800
sleep_time=30

while getopts 'r:t:s:' opt; do
case "$opt" in
r)
max_attempts=$OPTARG
;;
t)
timeout=$OPTARG
;;
s)
sleep_time=$OPTARG
;;
*)
usage
exit 1
;;
esac
done


shift $((OPTIND - 2))

if [[ "$1" == "--" ]]; then
shift # Shift past the double dash
else
echo "Error: Missing -- before command"
exit 1
fi

attempt_num=1
start_time=$(date +%s)

while true; do
elapsed=$(($(date +%s) - start_time))

if [ "$max_attempts" -ne 0 ] && [ "$attempt_num" -ge "$max_attempts" ]; then
echo "maximum attempts reached, exit" >&2
exit 10
fi

if [ "$timeout" -ne 0 ] && [ $elapsed -ge "$timeout" ]; then
echo "timeout reached, exit" >&2
exit 11
fi

if "$@"; then
exit
else
attempt_num=$(( attempt_num + 1 ))
if [ "$sleep_time" != "0" ]; then
sleep "$sleep_time"
fi
fi
done
4 changes: 2 additions & 2 deletions ydb/ci/testmo-proxy/testmo-proxy.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@ class Handler(http.server.BaseHTTPRequestHandler):
error_message_format = '{"status":"error", "code":"%(code)d", "message": "%(message)s"}\n'

# noinspection PyMissingConstructor
def __init__(self, target_url: str, timeout: Tuple[int, int], max_reqest_time: int):
def __init__(self, target_url: str, timeout: Tuple[int, int], max_request_time: int):
self._target_url = target_url
self._timeout = timeout
self._max_request_time = max_reqest_time
self._max_request_time = max_request_time

def __call__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
Expand Down

0 comments on commit cfbb87c

Please sign in to comment.