From 10e19f27ab891fbec96e6f654d1566cb7f3aef99 Mon Sep 17 00:00:00 2001 From: Dennis Trautwein Date: Tue, 3 Dec 2024 15:46:55 +0100 Subject: [PATCH] add: GitHub actions workflows --- .github/workflows/build_push.yml | 65 +++++++++++++++++++++ .github/workflows/ci.yml | 51 ---------------- .github/workflows/deploy.yml | 93 ++++++++++++++++++++++++++++++ .github/workflows/pull_request.yml | 12 ++++ .github/workflows/push_dev.yml | 27 +++++++++ .github/workflows/push_main.yml | 31 ++++++++++ .github/workflows/test.yml | 29 ++++++++++ cmd/ants/main.go | 16 ++--- 8 files changed, 265 insertions(+), 59 deletions(-) create mode 100644 .github/workflows/build_push.yml delete mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/deploy.yml create mode 100644 .github/workflows/pull_request.yml create mode 100644 .github/workflows/push_dev.yml create mode 100644 .github/workflows/push_main.yml create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/build_push.yml b/.github/workflows/build_push.yml new file mode 100644 index 0000000..982477a --- /dev/null +++ b/.github/workflows/build_push.yml @@ -0,0 +1,65 @@ +name: Test, Build & Push (dev) + +on: + workflow_call: + outputs: + image_tag: + description: "The image tag that was pushed to ECR" + value: ${{ jobs.push_to_ecr.outputs.image_tag }} + +# TODO: move to workflow inputs: +env: + AWS_REGION: us-east-1 + AWS_ROLE: arn:aws:iam::019120760881:role/prod-use1-github-oidc-role + AWS_ECR_REPOSITORY: probelab + +jobs: + push_to_ecr: + name: Build & Push + runs-on: ubuntu-latest + outputs: + image_tag: ${{ steps.meta.outputs.image }} + steps: + - name: Checking out the Repository + uses: actions/checkout@v4 + + - name: Configuring AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.AWS_REGION }} + role-to-assume: ${{ env.AWS_ROLE }} + role-session-name: PushToECR + + - name: Logging in to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Building ants Image Metadata + id: meta + run: | + SHA_SHORT=${{ github.sha }} + SHA_SHORT=${SHA_SHORT::7} + + NAMESPACE=${{ steps.login-ecr.outputs.registry }} + TAG="ants-sha${SHA_SHORT}" + + IMAGE="$NAMESPACE/$AWS_ECR_REPOSITORY:$TAG" + + echo "tag=$TAG" >> $GITHUB_OUTPUT + echo "image=$IMAGE" >> $GITHUB_OUTPUT + + - name: Checking if Image exists in ECR + id: check-ecr + run: | + aws ecr describe-images --repository-name $AWS_ECR_REPOSITORY --image-ids imageTag=${{ steps.meta.outputs.tag }} || exit_code=$? + echo "exit_code=$exit_code" >> $GITHUB_OUTPUT + + - name: Building Docker Image ${{ steps.meta.outputs.tag }} + id: build + if: steps.check-ecr.outputs.exit_code != 0 + run: docker build -t ${{ steps.meta.outputs.image }} . + + - name: Pushing Docker Image ${{ steps.meta.outputs.tag }} to Amazon ECR + id: push + if: steps.check-ecr.outputs.exit_code != 0 + run: docker push ${{ steps.meta.outputs.image }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index adc3e5e..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: continuous integration - -on: - push: - branches: [main] - pull_request: - -jobs: - build: - runs-on: ubuntu-latest - - strategy: - matrix: - go-version: [1.21, 1.22] - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - with: - submodules: true - - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: ${{ matrix.go-version }} - - - name: Cache Go modules - uses: actions/cache@v3 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ matrix.go-version }}- - - - name: Install dependencies - run: go mod download - - - name: Check Go fmt - run: | - if ! go fmt ./...; then - echo "Go fmt check failed" - exit 1 - fi - - - name: Run Go vet - run: go vet ./... - - - name: Run tests - run: go test ./... -v -cover diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..02d5ad1 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,93 @@ +name: Deployment (prod) + +on: + workflow_call: + secrets: + slack_webhook_url: + required: true + +# TODO: move to workflow inputs: +env: + AWS_REGION: us-east-1 + AWS_ROLE: arn:aws:iam::019120760881:role/prod-use1-github-oidc-role + AWS_ECS_TASK_DEFINITION: prod-use1-cmi-ants-celestia-watch-task + AWS_ECS_SERVICE: prod-use1-cmi-ants-celestia-watch + AWS_ECS_CLUSTER_NAME: default + AWS_ECR_REPOSITORY: probelab + +jobs: + deploy: + name: Deploy + runs-on: ubuntu-latest + needs: build_push + steps: + - name: Configuring AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.AWS_REGION }} + role-to-assume: ${{ env.AWS_ROLE }} + role-session-name: deploy-ants + + - name: Downloading latest Amazon ECS task definition + id: download + run: | + aws ecs describe-task-definition \ + --task-definition ${{ env.AWS_ECS_TASK_DEFINITION }} \ + --query taskDefinition > task-definition.json + + # Extract downloaded task definition revision + REVISION=$(cat task-definition.json | jq -r '.revision') + + # Store task definition revision + echo "task_definition=${{ env.AWS_ECS_TASK_DEFINITION }}:$REVISION" >> $GITHUB_OUTPUT + + # https://github.com/aws-actions/amazon-ecs-deploy-task-definition/issues/176 + # This isn't critical but just avoids some warning messages in the next step + - name: Removing invalid task definition fields + run: | + cat task-definition.json | jq -r 'del( + .taskDefinitionArn, + .requiresAttributes, + .compatibilities, + .revision, + .status, + .registeredAt, + .registeredBy + )' > task-definition-cleaned.json + + - name: Updating image tag of task definition ${{ steps.download.outputs.task_definition }} + id: task-def + uses: aws-actions/amazon-ecs-render-task-definition@v1 + with: + task-definition: task-definition-cleaned.json + container-name: prod-use1-cmi-ants-celestia-watch + image: ${{ needs.build_push.outputs.image_tag }} + + - name: Deploying to Amazon ECS + uses: aws-actions/amazon-ecs-deploy-task-definition@v2 + with: + task-definition: ${{ steps.task-def.outputs.task-definition }} + service: ${{ env.AWS_ECS_SERVICE }} + cluster: ${{ env.AWS_ECS_CLUSTER_NAME }} + wait-for-service-stability: true + wait-for-minutes: 15 # default is 30 + propagate-tags: SERVICE + enable-ecs-managed-tags: true + + - name: Publishing Success Notification to Slack + if: success() + uses: slackapi/slack-github-action@v2.0.0 + with: + webhook: ${{ secrets.slack_webhook_url }} + webhook-type: incoming-webhook + payload: | + text: "✅ Successfully deployed task definition ${{ steps.download.outputs.task_definition }}. " + + - name: Publishing Error Notification to Slack + if: failure() + uses: slackapi/slack-github-action@v2.0.0 + with: + webhook: ${{ secrets.slack_webhook_url }} + webhook-type: incoming-webhook + payload: | + text: "🚨 Deployment of task definition ${{ steps.download.outputs.task_definition }} failed. <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>" diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml new file mode 100644 index 0000000..ca061c7 --- /dev/null +++ b/.github/workflows/pull_request.yml @@ -0,0 +1,12 @@ +name: Test, Build & Push (dev) + +on: + pull_request: + +jobs: + test: + name: Ants + uses: ./.github/workflows/test.yml + permissions: + id-token: write + contents: read \ No newline at end of file diff --git a/.github/workflows/push_dev.yml b/.github/workflows/push_dev.yml new file mode 100644 index 0000000..a99bc88 --- /dev/null +++ b/.github/workflows/push_dev.yml @@ -0,0 +1,27 @@ +name: Test, Build & Push (dev) + +on: + workflow_dispatch: + push: + branches: + - dev + +env: + AWS_REGION: us-east-1 + AWS_ROLE: arn:aws:iam::019120760881:role/prod-use1-github-oidc-role + AWS_ECR_REPOSITORY: probelab + +jobs: + test: + name: Ants + uses: ./.github/workflows/test.yml + permissions: + id-token: write + contents: read + build_push: + name: Ants + uses: ./.github/workflows/build_push.yml + needs: test + permissions: + id-token: write + contents: read \ No newline at end of file diff --git a/.github/workflows/push_main.yml b/.github/workflows/push_main.yml new file mode 100644 index 0000000..0bbd5cd --- /dev/null +++ b/.github/workflows/push_main.yml @@ -0,0 +1,31 @@ +name: Deployment (prod) + +on: + workflow_dispatch: + +jobs: + test: + name: Ants + uses: ./.github/workflows/test.yml + permissions: + id-token: write + contents: read + + build_push: + name: Ants + uses: ./.github/workflows/build_push.yml + needs: test + permissions: + id-token: write + contents: read + + + deploy: + name: Ants + uses: ./.github/workflows/deploy.yml + needs: build_push + permissions: + id-token: write + contents: read + secrets: + slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..1b11c63 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,29 @@ +name: Test + +on: + workflow_call: +jobs: + test: + name: Test + runs-on: ubuntu-latest + steps: + - name: Checking out repository code + uses: actions/checkout@v4 + + - name: Setting up Golang + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Checking Go fmt + run: | + if ! go fmt ./...; then + echo "Go fmt check failed" + exit 1 + fi + + - name: Running vet + run: go vet ./... + + - name: Running Tests + run: go test ./... diff --git a/cmd/ants/main.go b/cmd/ants/main.go index 665824c..779ae61 100644 --- a/cmd/ants/main.go +++ b/cmd/ants/main.go @@ -54,7 +54,7 @@ var rootConfig = struct { FirstPort: 6000, UPnp: false, BatchSize: 1000, - BatchTime: time.Second, + BatchTime: 20 * time.Second, CrawlInterval: 120 * time.Minute, CacheSize: 10_000, BucketSize: 20, @@ -166,13 +166,6 @@ func main() { Destination: &rootConfig.KeyDBPath, Value: rootConfig.KeyDBPath, }, - &cli.IntFlag{ - Name: "num_ports", - Usage: "Number of ports ants can listen on", - EnvVars: []string{"ANTS_NUM_PORTS"}, - Destination: &rootConfig.NumPorts, - Value: rootConfig.NumPorts, - }, &cli.IntFlag{ Name: "first_port", Usage: "First port ants can listen on", @@ -180,6 +173,13 @@ func main() { Destination: &rootConfig.FirstPort, Value: rootConfig.FirstPort, }, + &cli.IntFlag{ + Name: "num_ports", + Usage: "Number of ports ants can listen on", + EnvVars: []string{"ANTS_NUM_PORTS"}, + Destination: &rootConfig.NumPorts, + Value: rootConfig.NumPorts, + }, &cli.BoolFlag{ Name: "upnp", Usage: "Enable UPnP",