Clickhouse support

.github/workflows/build_push.yml

@@ -0,0 +1,65 @@
+name: Test, Build & Push (dev)
+
+on:
+  workflow_call:
+    outputs:
+      image_tag:
+        description: "The image tag that was pushed to ECR"
+        value: ${{ jobs.push_to_ecr.outputs.image_tag }}
+
+# TODO: move to workflow inputs:
+env:
+  AWS_REGION: us-east-1
+  AWS_ROLE: arn:aws:iam::019120760881:role/prod-use1-github-oidc-role
+  AWS_ECR_REPOSITORY: probelab
+
+jobs:
+  push_to_ecr:
+    name: Build & Push
+    runs-on: ubuntu-latest
+    outputs:
+      image_tag: ${{ steps.meta.outputs.image }}
+    steps:
+      - name: Checking out the Repository
+        uses: actions/checkout@v4
+
+      - name: Configuring AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ env.AWS_REGION }}
+          role-to-assume: ${{ env.AWS_ROLE }}
+          role-session-name: PushToECR
+
+      - name: Logging in to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Building ants Image Metadata
+        id: meta
+        run: |
+          SHA_SHORT=${{ github.sha }}
+          SHA_SHORT=${SHA_SHORT::7}
+
+          NAMESPACE=${{ steps.login-ecr.outputs.registry }}
+          TAG="ants-sha${SHA_SHORT}"
+
+          IMAGE="$NAMESPACE/$AWS_ECR_REPOSITORY:$TAG"
+
+          echo "tag=$TAG" >> $GITHUB_OUTPUT
+          echo "image=$IMAGE" >> $GITHUB_OUTPUT
+
+      - name: Checking if Image exists in ECR
+        id: check-ecr
+        run: |
+          aws ecr describe-images --repository-name $AWS_ECR_REPOSITORY --image-ids imageTag=${{ steps.meta.outputs.tag }} || exit_code=$?
+          echo "exit_code=$exit_code" >> $GITHUB_OUTPUT
+
+      - name: Building Docker Image ${{ steps.meta.outputs.tag }}
+        id: build
+        if: steps.check-ecr.outputs.exit_code != 0
+        run: docker build -t ${{ steps.meta.outputs.image }} .
+
+      - name: Pushing Docker Image ${{ steps.meta.outputs.tag }} to Amazon ECR
+        id: push
+        if: steps.check-ecr.outputs.exit_code != 0
+        run: docker push ${{ steps.meta.outputs.image }}

.github/workflows/deploy.yml

@@ -0,0 +1,93 @@
+name: Deployment (prod)
+
+on:
+  workflow_call:
+    secrets:
+      slack_webhook_url:
+        required: true
+
+# TODO: move to workflow inputs:
+env:
+  AWS_REGION: us-east-1
+  AWS_ROLE: arn:aws:iam::019120760881:role/prod-use1-github-oidc-role
+  AWS_ECS_TASK_DEFINITION: prod-use1-cmi-ants-celestia-watch-task
+  AWS_ECS_SERVICE: prod-use1-cmi-ants-celestia-watch
+  AWS_ECS_CLUSTER_NAME: default
+  AWS_ECR_REPOSITORY: probelab
+
+jobs:
+  deploy:
+    name: Deploy
+    runs-on: ubuntu-latest
+    needs: build_push
+    steps:
+      - name: Configuring AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ env.AWS_REGION }}
+          role-to-assume: ${{ env.AWS_ROLE }}
+          role-session-name: deploy-ants
+
+      - name: Downloading latest Amazon ECS task definition
+        id: download
+        run: |
+          aws ecs describe-task-definition \
+            --task-definition ${{ env.AWS_ECS_TASK_DEFINITION }} \
+            --query taskDefinition > task-definition.json
+
+          # Extract downloaded task definition revision
+          REVISION=$(cat task-definition.json | jq -r '.revision')
+
+          # Store task definition revision
+          echo "task_definition=${{ env.AWS_ECS_TASK_DEFINITION }}:$REVISION" >> $GITHUB_OUTPUT
+
+      # https://github.com/aws-actions/amazon-ecs-deploy-task-definition/issues/176
+      # This isn't critical but just avoids some warning messages in the next step
+      - name: Removing invalid task definition fields
+        run: |
+          cat task-definition.json | jq -r 'del(
+            .taskDefinitionArn,
+            .requiresAttributes,
+            .compatibilities,
+            .revision,
+            .status,
+            .registeredAt,
+            .registeredBy
+          )' > task-definition-cleaned.json
+
+      - name: Updating image tag of task definition ${{ steps.download.outputs.task_definition }}
+        id: task-def
+        uses: aws-actions/amazon-ecs-render-task-definition@v1
+        with:
+          task-definition: task-definition-cleaned.json
+          container-name: prod-use1-cmi-ants-celestia-watch
+          image: ${{ needs.build_push.outputs.image_tag }}
+
+      - name: Deploying to Amazon ECS
+        uses: aws-actions/amazon-ecs-deploy-task-definition@v2
+        with:
+          task-definition: ${{ steps.task-def.outputs.task-definition }}
+          service: ${{ env.AWS_ECS_SERVICE }}
+          cluster: ${{ env.AWS_ECS_CLUSTER_NAME }}
+          wait-for-service-stability: true
+          wait-for-minutes: 15 # default is 30
+          propagate-tags: SERVICE
+          enable-ecs-managed-tags: true
+
+      - name: Publishing Success Notification to Slack
+        if: success()
+        uses: slackapi/[email protected]
+        with:
+          webhook: ${{ secrets.slack_webhook_url }}
+          webhook-type: incoming-webhook
+          payload: |
+            text: "✅ Successfully deployed task definition ${{ steps.download.outputs.task_definition }}. <https://${{ env.AWS_REGION }}.console.aws.amazon.com/ecs/v2/clusters/${{ env.AWS_ECS_CLUSTER_NAME }}/services/${{ env.AWS_ECS_SERVICE }}/health|View Service>"
+
+      - name: Publishing Error Notification to Slack
+        if: failure()
+        uses: slackapi/[email protected]
+        with:
+          webhook: ${{ secrets.slack_webhook_url }}
+          webhook-type: incoming-webhook
+          payload: |
+            text: "🚨 Deployment of task definition ${{ steps.download.outputs.task_definition }} failed. <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>"

.github/workflows/pull_request.yml

@@ -0,0 +1,12 @@
+name: Test, Build & Push (dev)
+
+on:
+  pull_request:
+
+jobs:
+  test:
+    name: Ants
+    uses: ./.github/workflows/test.yml
+    permissions:
+      id-token: write
+      contents: read

.github/workflows/push_dev.yml

@@ -0,0 +1,27 @@
+name: Test, Build & Push (dev)
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - dev
+
+env:
+  AWS_REGION: us-east-1
+  AWS_ROLE: arn:aws:iam::019120760881:role/prod-use1-github-oidc-role
+  AWS_ECR_REPOSITORY: probelab
+
+jobs:
+  test:
+    name: Ants
+    uses: ./.github/workflows/test.yml
+    permissions:
+      id-token: write
+      contents: read
+  build_push:
+    name: Ants
+    uses: ./.github/workflows/build_push.yml
+    needs: test
+    permissions:
+      id-token: write
+      contents: read

.github/workflows/push_main.yml

@@ -0,0 +1,31 @@
+name: Deployment (prod)
+
+on:
+  workflow_dispatch:
+
+jobs:
+  test:
+    name: Ants
+    uses: ./.github/workflows/test.yml
+    permissions:
+      id-token: write
+      contents: read
+
+  build_push:
+    name: Ants
+    uses: ./.github/workflows/build_push.yml
+    needs: test
+    permissions:
+      id-token: write
+      contents: read
+
+
+  deploy:
+    name: Ants
+    uses: ./.github/workflows/deploy.yml
+    needs: build_push
+    permissions:
+      id-token: write
+      contents: read
+    secrets:
+      slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/test.yml

@@ -0,0 +1,31 @@
+name: Test
+
+on:
+  workflow_call:
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checking out repository code
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: Setting up Golang
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Checking Go fmt
+        run: |
+          if ! go fmt ./...; then
+            echo "Go fmt check failed"
+            exit 1
+          fi
+
+      - name: Running vet
+        run: go vet ./...
+
+      - name: Running Tests
+        run: go test ./ # run tests only in root directory

Dockerfile

@@ -15,7 +15,7 @@ RUN go mod download
 COPY . ./
 
 RUN --mount=type=cache,target=/root/.cache/go-build \
-    CGO_ENABLED=1 GOOS=linux go build -ldflags "-X main.RawVersion='${RAW_VERSION}'" -o ants github.com/probe-lab/ants-watch/cmd/honeypot
+    CGO_ENABLED=1 GOOS=linux go build -ldflags "-X main.RawVersion='${RAW_VERSION}'" -o ants github.com/probe-lab/ants-watch/cmd/ants
 
 FROM alpine:3.18
 

Makefile

@@ -10,20 +10,17 @@ REPO_USER?=AWS
 REPO_REGION?=us-east-1
 
 
-
 tools:
 	go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/[email protected]
-	go install github.com/volatiletech/sqlboiler/[email protected]
-	go install github.com/volatiletech/sqlboiler/v4/drivers/[email protected]
-
-models:
-	sqlboiler --no-tests psql
 
 migrate-up:
-	migrate -database 'postgres://ants_watch:password@localhost:5432/ants_watch?sslmode=disable' -path db/migrations up
+	migrate -database 'clickhouse://localhost:9000?username=default&secure=false' -path db/migrations up
 
 migrate-down:
-	migrate -database 'postgres://ants_watch:password@localhost:5432/ants_watch?sslmode=disable' -path db/migrations down
+	migrate -database 'clickhouse://localhost:9000?username=default&secure=false' -path db/migrations down
+
+local-clickhouse:
+	docker run --name ants-clickhouse --rm -p 9000:9000 clickhouse/clickhouse-server
 
 .PHONY: build
 build:
@@ -32,6 +29,6 @@ build:
 
 .PHONY: push
 push:
-	aws ecr get-login-password --profile probelab --region ${REPO_REGION} | docker login --username ${REPO_USER} --password-stdin ${REPO}
+	aws ecr get-login-password --region ${REPO_REGION} | docker login --username ${REPO_USER} --password-stdin ${REPO}
 	docker tag ${IMAGE_NAME} ${REPO}/${IMAGE_NAME}
 	docker push ${REPO}/${IMAGE_NAME}