From f10198dc03170c1afd6c64d6f07239e20127b5c5 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 20 Nov 2024 12:40:58 +0100 Subject: [PATCH 01/12] Update deploy to onto-ns CD workflow --- .github/workflows/cd_onto-ns.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cd_onto-ns.yml b/.github/workflows/cd_onto-ns.yml index e189a76..e8bada1 100644 --- a/.github/workflows/cd_onto-ns.yml +++ b/.github/workflows/cd_onto-ns.yml @@ -10,11 +10,16 @@ on: jobs: check-service-changes: runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} + if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} outputs: - service-changes: ${{ steps.check-changes.outputs.service-changes }} + service-changes: ${{ steps.check-changes.outputs.service-changes || steps.passthrough.outputs.service-changes }} steps: + - name: Passthrough if manually deploying + if: github.event_name == 'workflow_dispatch' + id: passthrough + run: echo "service-changes=true" >> $GITHUB_OUTPUT + - name: Checkout repository uses: actions/checkout@v4 @@ -23,10 +28,7 @@ jobs: run: | git fetch --all --quiet - git diff --name-only origin/main -- | \ - grep -qE "(pyproject.toml|entities_service/((models/|service/)(.+/)*)?[^/]*\.py)" \ - && SERVICE_CHANGES=true \ - || SERVICE_CHANGES=false + git diff --name-only origin/main -- | grep -qE "(pyproject.toml|entities_service/((models|service)/(.+/)*)?[^/]*\.py)" && SERVICE_CHANGES=true || SERVICE_CHANGES=false echo "Changed files:" git diff --name-only origin/main -- From c0aa5fb1176ec83b022f14cb65b6f66c32196526 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 20 Nov 2024 12:47:00 +0100 Subject: [PATCH 02/12] Avoid running other steps in service change check job --- .github/workflows/cd_onto-ns.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cd_onto-ns.yml b/.github/workflows/cd_onto-ns.yml index e8bada1..bb98d62 100644 --- a/.github/workflows/cd_onto-ns.yml +++ b/.github/workflows/cd_onto-ns.yml @@ -18,12 +18,15 @@ jobs: - name: Passthrough if manually deploying if: github.event_name == 'workflow_dispatch' id: passthrough - run: echo "service-changes=true" >> $GITHUB_OUTPUT + run: | + echo "service-changes=true" >> $GITHUB_OUTPUT - name: Checkout repository + if: github.event_name != 'workflow_dispatch' uses: actions/checkout@v4 - name: Check changes + if: github.event_name != 'workflow_dispatch' id: check-changes run: | git fetch --all --quiet From ab0bffd81a5d65cc67fae08e9ec068db52d3ca44 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 20 Nov 2024 13:15:17 +0100 Subject: [PATCH 03/12] Retrieve SHA for proper file comparison --- .github/workflows/cd_onto-ns.yml | 17 +++++++++++++++-- .github/workflows/ci_tests.yml | 16 ++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd_onto-ns.yml b/.github/workflows/cd_onto-ns.yml index bb98d62..859c2bc 100644 --- a/.github/workflows/cd_onto-ns.yml +++ b/.github/workflows/cd_onto-ns.yml @@ -25,16 +25,29 @@ jobs: if: github.event_name != 'workflow_dispatch' uses: actions/checkout@v4 + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Download deployment artifact + if: github.event_name != 'workflow_dispatch' + uses: actions/download-artifact@v4 + with: + name: onto-ns-deployment + - name: Check changes if: github.event_name != 'workflow_dispatch' id: check-changes run: | git fetch --all --quiet - git diff --name-only origin/main -- | grep -qE "(pyproject.toml|entities_service/((models|service)/(.+/)*)?[^/]*\.py)" && SERVICE_CHANGES=true || SERVICE_CHANGES=false + before_sha="$(python -c "import json, pathlib; print(json.load(pathlib.Path('onto-ns-deployment.json').read_text())['before'])")" + + git diff --name-only ${before_sha} -- | grep -qE "(pyproject.toml|entities_service/((models|service)/(.+/)*)?[^/]*\.py)" && SERVICE_CHANGES=true || SERVICE_CHANGES=false echo "Changed files:" - git diff --name-only origin/main -- + git diff --name-only ${before_sha} -- echo "SERVICE_CHANGES=${SERVICE_CHANGES}" echo "service-changes=${SERVICE_CHANGES}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 890fc92..7385f7f 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -389,3 +389,19 @@ jobs: echo "Unknown error." exit 3 fi + + upload-artifact-for-onto-ns-deployment: + name: Upload artifact for onto-ns deployment + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + + steps: + - name: Generate artifact + run: | + echo "{\"before\":\"${{ github.event.before }}\"}" > onto-ns-deployment.json + + - name: Archive artifacts + uses: actions/upload-artifact@v4 + with: + name: onto-ns-deployment + path: onto-ns-deployment.json From 58aa163405e607fcf49feab7cea8f1c2ec27d929 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 20 Nov 2024 14:46:22 +0100 Subject: [PATCH 04/12] Update CLI docs generation CI job Add bash script to auto-update the CLI docs. --- .github/workflows/ci_tests.yml | 18 +++++++++----- build_cli_docs.sh | 44 ++++++++++++++++++++++++++++++++++ pyproject.toml | 5 +++- 3 files changed, 60 insertions(+), 7 deletions(-) create mode 100755 build_cli_docs.sh diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 7385f7f..4c3502d 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -359,7 +359,7 @@ jobs: run: | python -m pip install -U pip pip install -U setuptools wheel - pip install -U -e .[cli] + pip install -U -e .[cli,ci] - name: Typer version run: typer --version @@ -367,11 +367,16 @@ jobs: - name: Build CLI docs run: | # Build the CLI documentation - # Run it twice, since the first run may create a doc only for the `config` sub-typer app. - typer entities_service.cli.main utils docs --output /tmp/CLI.md > /dev/null + # We need to run this multiple times until the documentation is generated + # properly. Sometimes it only finds the sub-app `config` and not the main + # app `entities-service`. + typer entities_service.cli.main utils docs --output /tmp/CLI.md > /dev/null + until grep -q "# \`entities-service\`" /tmp/CLI.md; do + typer entities_service.cli.main utils docs --output /tmp/CLI.md > /dev/null + done - diff --suppress-common-lines /tmp/CLI.md docs/CLI.md 2>&1 > /tmp/CLI_diff.md && ERROR_CODE=0 || ERROR_CODE=$? + diff -u --suppress-common-lines /tmp/CLI.md docs/CLI.md > /tmp/CLI_diff.md 2>&1 ERROR_CODE=$? @@ -379,9 +384,10 @@ jobs: echo "diff encountered an error." exit 2 elif [ ${ERROR_CODE} -eq 1 ]; then - echo "CLI documentation has changed. Please update the documentation." + echo "CLI documentation has changed." + echo "Please update the documentation by running './build_cli_docs.sh' in a bash-compatible terminal." echo "Diff:" - cat /tmp/CLI_diff.md + cat /tmp/CLI_diff.md | ydiff -p cat exit 1 elif [ ${ERROR_CODE} -eq 0 ]; then echo "CLI documentation is up-to-date." diff --git a/build_cli_docs.sh b/build_cli_docs.sh new file mode 100755 index 0000000..8435d4a --- /dev/null +++ b/build_cli_docs.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +{ + typer --version > /dev/null 2>&1 +} || { + echo -e "\nāŒ 'typer' is not installed." && + echo -e "Please install it and the 'entities-service' CLI using:\n\n pip install .[cli]\n" && + exit 1 +} + +DOCS_PATH="docs/CLI.md" + +if [ ! -f ${DOCS_PATH} ]; then + echo "CLI documentation file not found at ${DOCS_PATH}." + exit 1 +fi + +rm /tmp/CLI.md 2> /dev/null + +# Generate CLI documentation +typer entities_service.cli.main utils docs --output /tmp/CLI.md > /dev/null +until grep -q "# \`entities-service\`" /tmp/CLI.md; do + typer entities_service.cli.main utils docs --output /tmp/CLI.md > /dev/null +done + +diff --suppress-common-lines /tmp/CLI.md ${DOCS_PATH} > /dev/null 2>&1 + +ERROR_CODE=$? + +if [ ${ERROR_CODE} -eq 2 ]; then + echo "āŒ diff encountered an error." + exit 2 +elif [ ${ERROR_CODE} -eq 1 ]; then + echo "āœØ CLI documentation will be updated." + echo "Diff:" + diff --suppress-common-lines --color /tmp/CLI.md ${DOCS_PATH} + mv -f /tmp/CLI.md ${DOCS_PATH} + echo -e "\nNow commit the changes to the documentation file:\n\n git add ${DOCS_PATH}\n git commit -m 'Update CLI documentation'\n" +elif [ ${ERROR_CODE} -eq 0 ]; then + echo "šŸ‘Œ CLI documentation is up-to-date." +else + echo "āŒ Unknown error." + exit 3 +fi diff --git a/pyproject.toml b/pyproject.toml index 16fa746..70df59e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,10 +51,13 @@ dependencies = [ ] [project.optional-dependencies] +ci = [ + "ydiff ~=1.4", +] cli = [ "httpx-auth ~=0.22.0", "pyyaml ~=6.0", - "typer[all] ~=0.12.5", + "typer ~=0.12.5", ] testing = [ "cryptography ~=43.0", From 0733b4435405363bbd7bb3abcf5d16b9c01f51d6 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 20 Nov 2024 14:59:54 +0100 Subject: [PATCH 05/12] Debug build-cli-docs CI job --- .github/workflows/ci_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 4c3502d..73b79aa 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -370,6 +370,7 @@ jobs: # We need to run this multiple times until the documentation is generated # properly. Sometimes it only finds the sub-app `config` and not the main # app `entities-service`. + set -x typer entities_service.cli.main utils docs --output /tmp/CLI.md > /dev/null until grep -q "# \`entities-service\`" /tmp/CLI.md; do From 8ac75d8a80dcfe2ebf76406ea72a366143468256 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 20 Nov 2024 15:01:43 +0100 Subject: [PATCH 06/12] Try avoid fail-fast --- .github/workflows/ci_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 73b79aa..2ed2650 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -370,7 +370,7 @@ jobs: # We need to run this multiple times until the documentation is generated # properly. Sometimes it only finds the sub-app `config` and not the main # app `entities-service`. - set -x + set -x +e typer entities_service.cli.main utils docs --output /tmp/CLI.md > /dev/null until grep -q "# \`entities-service\`" /tmp/CLI.md; do From 5a889a1c2f1c3cf6de3088b295d84dd09166ea01 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 20 Nov 2024 15:02:40 +0100 Subject: [PATCH 07/12] Don't print all executed statements --- .github/workflows/ci_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 2ed2650..ffbdacd 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -370,7 +370,7 @@ jobs: # We need to run this multiple times until the documentation is generated # properly. Sometimes it only finds the sub-app `config` and not the main # app `entities-service`. - set -x +e + set +e typer entities_service.cli.main utils docs --output /tmp/CLI.md > /dev/null until grep -q "# \`entities-service\`" /tmp/CLI.md; do From 0e662f001f89819ae30fdb18b2bfb5a5f72da260 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 20 Nov 2024 15:04:46 +0100 Subject: [PATCH 08/12] Remove usage of ydiff --- .github/workflows/ci_tests.yml | 6 +++--- pyproject.toml | 3 --- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index ffbdacd..eb5b093 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -359,7 +359,7 @@ jobs: run: | python -m pip install -U pip pip install -U setuptools wheel - pip install -U -e .[cli,ci] + pip install -U -e .[cli] - name: Typer version run: typer --version @@ -377,7 +377,7 @@ jobs: typer entities_service.cli.main utils docs --output /tmp/CLI.md > /dev/null done - diff -u --suppress-common-lines /tmp/CLI.md docs/CLI.md > /tmp/CLI_diff.md 2>&1 + diff --suppress-common-lines /tmp/CLI.md docs/CLI.md > /tmp/CLI_diff.md 2>&1 ERROR_CODE=$? @@ -388,7 +388,7 @@ jobs: echo "CLI documentation has changed." echo "Please update the documentation by running './build_cli_docs.sh' in a bash-compatible terminal." echo "Diff:" - cat /tmp/CLI_diff.md | ydiff -p cat + cat /tmp/CLI_diff.md exit 1 elif [ ${ERROR_CODE} -eq 0 ]; then echo "CLI documentation is up-to-date." diff --git a/pyproject.toml b/pyproject.toml index 70df59e..d53d035 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,9 +51,6 @@ dependencies = [ ] [project.optional-dependencies] -ci = [ - "ydiff ~=1.4", -] cli = [ "httpx-auth ~=0.22.0", "pyyaml ~=6.0", From e384020a8470194729e0dff00910d8b07e28ea34 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 20 Nov 2024 15:05:32 +0100 Subject: [PATCH 09/12] Use diff again instead of storing it --- .github/workflows/ci_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index eb5b093..9533287 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -377,7 +377,7 @@ jobs: typer entities_service.cli.main utils docs --output /tmp/CLI.md > /dev/null done - diff --suppress-common-lines /tmp/CLI.md docs/CLI.md > /tmp/CLI_diff.md 2>&1 + diff --suppress-common-lines /tmp/CLI.md docs/CLI.md > /dev/null 2>&1 ERROR_CODE=$? @@ -388,7 +388,7 @@ jobs: echo "CLI documentation has changed." echo "Please update the documentation by running './build_cli_docs.sh' in a bash-compatible terminal." echo "Diff:" - cat /tmp/CLI_diff.md + diff --suppress-common-lines --color /tmp/CLI.md docs/CLI.md exit 1 elif [ ${ERROR_CODE} -eq 0 ]; then echo "CLI documentation is up-to-date." From 0412a378fa490085d6b0779eb4f0154133acd7fa Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 20 Nov 2024 15:08:13 +0100 Subject: [PATCH 10/12] Update CLI documentation --- docs/CLI.md | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/docs/CLI.md b/docs/CLI.md index dea8bc1..93e737d 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -19,6 +19,7 @@ $ entities-service [OPTIONS] COMMAND [ARGS]... **Commands**: * `config`: Manage configuration options. +* `list`: List resources. * `login`: Login to the entities service. * `upload`: Upload (local) entities to a remote location. * `validate`: Validate (local) entities. @@ -112,6 +113,58 @@ $ entities-service config unset-all [OPTIONS] * `--help`: Show this message and exit. +## `entities-service list` + +List resources. + +**Usage**: + +```console +$ entities-service list [OPTIONS] COMMAND [ARGS]... +``` + +**Options**: + +* `--help`: Show this message and exit. + +**Commands**: + +* `entities`: List entities from the entities service. +* `namespaces`: List namespaces from the entities service. + +### `entities-service list entities` + +List entities from the entities service. + +**Usage**: + +```console +$ entities-service list entities [OPTIONS] [NAMESPACE]... +``` + +**Arguments**: + +* `[NAMESPACE]...`: Namespace(s) to list entities from. Defaults to the core namespace. If the namespace is a URL, the specific namespace will be extracted. + +**Options**: + +* `-a, --all`: List entities from all namespaces. +* `--help`: Show this message and exit. + +### `entities-service list namespaces` + +List namespaces from the entities service. + +**Usage**: + +```console +$ entities-service list namespaces [OPTIONS] +``` + +**Options**: + +* `--help`: Show this message and exit. + ## `entities-service login` Login to the entities service. From ebfb02621dd6eb251e9bcda7c95c6051793b9980 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 20 Nov 2024 15:10:08 +0100 Subject: [PATCH 11/12] =?UTF-8?q?Add=20emojis=20=F0=9F=98=8E=20to=20the=20?= =?UTF-8?q?print=20statements=20in=20CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci_tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 9533287..973f094 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -382,18 +382,18 @@ jobs: ERROR_CODE=$? if [ ${ERROR_CODE} -eq 2 ]; then - echo "diff encountered an error." + echo "āŒ diff encountered an error." exit 2 elif [ ${ERROR_CODE} -eq 1 ]; then - echo "CLI documentation has changed." + echo "āœØ CLI documentation has changed." echo "Please update the documentation by running './build_cli_docs.sh' in a bash-compatible terminal." echo "Diff:" diff --suppress-common-lines --color /tmp/CLI.md docs/CLI.md exit 1 elif [ ${ERROR_CODE} -eq 0 ]; then - echo "CLI documentation is up-to-date." + echo "šŸ‘Œ CLI documentation is up-to-date." else - echo "Unknown error." + echo "āŒ Unknown error." exit 3 fi From f3073281b86f8dc852beaff75ced484a2d61c9a7 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 20 Nov 2024 15:11:03 +0100 Subject: [PATCH 12/12] Add mongo-express to compose --- compose.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/compose.yml b/compose.yml index f95873f..8ad60f4 100644 --- a/compose.yml +++ b/compose.yml @@ -26,7 +26,6 @@ services: mongodb: image: mongo:8 - restart: always ports: - "27017:27017" networks: @@ -42,5 +41,22 @@ services: retries: 5 stop_grace_period: 1s + mongodb_gui: + image: mongo-express:1 + ports: + - "8081:8081" + environment: + ME_CONFIG_MONGODB_URL: mongodb://root:root@mongodb:27017/?authSource=admin + ME_CONFIG_MONGODB_PORT: 27017 + ME_CONFIG_MONGODB_ENABLE_ADMIN: "true" + ME_CONFIG_BASICAUTH_USERNAME: admin + ME_CONFIG_BASICAUTH_PASSWORD: admin + networks: + - entities_service_net + depends_on: + mongodb: + condition: service_healthy + restart: true + networks: entities_service_net: