Merge pull request #11 from ubuntu-robotics/feat/grafana_dashboards_w…

…ith_charm

Feat/grafana dashboards with charm

.github/workflows/rock-release-pr.yaml

@@ -0,0 +1,58 @@
+# The workflow canonical/observability/.github/workflows/rock-release-dev.yaml@main
+# https://github.com/canonical/observability/blob/34b7edb56094e8e3f01200ce49e01bf9dd6688ac/.github/workflows/rock-release-dev.yaml
+# does not allow setting a GHCR repo where to publish at the moment.
+# We thus copy it below and adapt it.
+
+name: Build ROCK and release pr tag to GHCR
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  main:
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    - name: Find the *latest* rockcraft.yaml
+      id: find-latest
+      run: |
+        latest_rockcraft_file=$(find $GITHUB_WORKSPACE -name "rockcraft.yaml" | sort -V | tail -n1)
+        rockcraft_dir=$(dirname ${latest_rockcraft_file#\./})
+        echo "latest-dir=$rockcraft_dir" >> $GITHUB_OUTPUT
+    - name: Build ROCK
+      uses: canonical/craft-actions/rockcraft-pack@main
+      with:
+        path: "${{ steps.find-latest.outputs.latest-dir }}"
+        verbosity: verbose
+      id: rockcraft
+
+    - name: Upload locally built ROCK artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: cos-registration-agent-rock
+        path: ${{ steps.rockcraft.outputs.rock }}
+
+    - name: Upload ROCK to ghcr.io
+      run: |
+        sudo skopeo --insecure-policy copy oci-archive:$(realpath ${{ steps.rockcraft.outputs.rock }}) docker://ghcr.io/ubuntu-robotics/cos-registration-server:pr --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}"
+    - name: Generate digest
+      run: |
+        digest=$(skopeo inspect oci-archive:$(realpath ${{ steps.rockcraft.outputs.rock }}) --format '{{.Digest}}')
+        echo "digest=${digest#*:}" >> "$GITHUB_OUTPUT"
+    - name: Install Syft
+      run: |
+        curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin
+    - name: Create SBOM
+      run: syft $(realpath ${{ steps.rockcraft.outputs.rock }}) -o spdx-json=cos-registration-server.sbom.json
+
+    - name: Upload SBOM
+      uses: actions/upload-artifact@v3
+      with:
+        name: cos-registration-server-sbom
+        path: "cos-registration-server.sbom.json"

README.md

@@ -73,7 +73,7 @@ requiring to access the device database.
 
 > | name      |  type     | data type               | description                                                           |
 > |-----------|-----------|-------------------------|-----------------------------------------------------------------------|
-> | None      |  required | {"uid": "string", "address": "string"}   | Unique ID and IP address of the device  |
+> | None      |  required | {"uid": "string", "address": "string", "grafana_dashboards"(optional): "list(grafana_json_dashboards)"}   | Unique ID and IP address of the device. Grafana dashboards is an optional list of Grafana JSON dashboards |
 
 
 ##### Responses
@@ -98,7 +98,7 @@ requiring to access the device database.
 
 > | http code     | content-type                      | response                                                            |
 > |---------------|-----------------------------------|---------------------------------------------------------------------|
-> | `200`         | `application/json`        | {"uid": "string", "creation_date": "string", "address": "string"}                                                         |
+> | `200`         | `application/json`        | {"uid": "string", "creation_date": "string", "address": "string", "grafana_dashboards": "list(grafana_json_dashboards)"}                                                         |
 > | `404`         | `text/html;charset=utf-8`        | None                                                         |
 </details>
 
@@ -109,7 +109,7 @@ requiring to access the device database.
 
 > | name      |  type     | data type               | description                                                           |
 > |-----------|-----------|-------------------------|-----------------------------------------------------------------------|
-> | None      |  required | {"address": "string"}   | Address to modify.  |
+> | None      |  required | {"field: "value"}   | Field to modify. Can be: address and grafana_dashboards |
 
 
 ##### Responses

cos_registration_server/api/grafana_dashboards.py

@@ -0,0 +1,37 @@
+import json
+from os import mkdir, remove
+from pathlib import Path
+from shutil import rmtree
+
+from devices.models import Device
+from django.conf import settings
+
+
+def add_grafana_dashboards(device):
+    dashboard_path = Path(settings.GRAFANA_DASHBOARD_PATH)
+    for dashboard in device.grafana_dashboards:
+        dashboard_title = dashboard["title"].replace(" ", "_")
+        dashboard["title"] = f'{device.uid}-{dashboard["title"]}'
+        dashboard_file = f"{device.uid}-{dashboard_title}.json"
+        with open(dashboard_path / dashboard_file, "w") as file:
+            json.dump(dashboard, file)
+
+
+def delete_grafana_dashboards(device):
+    dashboard_path = Path(settings.GRAFANA_DASHBOARD_PATH)
+
+    def _is_device_dashboard(p: Path) -> bool:
+        return p.is_file() and p.name.startswith(device.uid)
+
+    for dashboard in filter(
+        _is_device_dashboard, Path(dashboard_path).glob("*")
+    ):
+        remove(dashboard)
+
+
+def update_all_grafana_dashboards():
+    dashboard_path = Path(settings.GRAFANA_DASHBOARD_PATH)
+    rmtree(dashboard_path, ignore_errors=True)
+    mkdir(dashboard_path)
+    for device in Device.objects.all():
+        add_grafana_dashboards(device)

cos_registration_server/api/management/commands/update_all_grafana_dashboards.py

@@ -0,0 +1,9 @@
+from api.grafana_dashboards import update_all_grafana_dashboards
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+    help = "Update all the grafana dashboards in the folder"
+
+    def handle(self, *args, **options):
+        update_all_grafana_dashboards()

cos_registration_server/api/serializer.py

@@ -41,4 +41,4 @@ def validate_grafana_dashboards(self, value):
             raise serializers.ValidationError(
                 "gafana_dashboards is not a supported format (list)."
             )
-        return value
+        return dashboards

cos_registration_server/api/tests.py

@@ -1,28 +1,32 @@
 import json
+from os import mkdir, path
+from pathlib import Path
+from shutil import rmtree
 
 from devices.models import Device, default_dashboards_json_field
+from django.core.management import call_command
+from django.test import TestCase
 from django.urls import reverse
 from django.utils import timezone
 from rest_framework.test import APITestCase
 
-SIMPLE_GRAFANA_DASHBOARD = """{
-  "dashboard": {
-    "id": null,
-    "uid": null,
-    "title": "Production Overview",
-    "tags": [ "templated" ],
-    "timezone": "browser",
-    "schemaVersion": 16,
-    "refresh": "25s"
-  },
-  "message": "Made changes to xyz",
-  "overwrite": false
-}"""
-
 
 class DevicesViewTests(APITestCase):
     def setUp(self):
         self.url = reverse("api:devices")
+        self.grafana_dashboards_path = Path("grafana_dashboards")
+        rmtree(self.grafana_dashboards_path, ignore_errors=True)
+        mkdir(self.grafana_dashboards_path)
+
+        self.simple_grafana_dashboard = {
+            "id": None,
+            "uid": None,
+            "title": "Production Overview",
+            "tags": ["templated"],
+            "timezone": "browser",
+            "schemaVersion": 16,
+            "refresh": "25s",
+        }
 
     def create_device(self, **fields):
         data = {}
@@ -38,8 +42,8 @@ def test_get_nothing(self):
     def test_create_device(self):
         uid = "robot-1"
         address = "192.168.0.1"
-        custom_grafana_dashboards = eval(default_dashboards_json_field())
-        custom_grafana_dashboards.append(SIMPLE_GRAFANA_DASHBOARD)
+        custom_grafana_dashboards = default_dashboards_json_field()
+        custom_grafana_dashboards.append(self.simple_grafana_dashboard)
         response = self.create_device(
             uid=uid,
             address=address,
@@ -57,6 +61,15 @@ def test_create_device(self):
         self.assertEqual(
             Device.objects.get().grafana_dashboards, custom_grafana_dashboards
         )
+        with open(
+            self.grafana_dashboards_path / "robot-1-Production_Overview.json",
+            "r",
+        ) as file:
+            dashboard_data = json.load(file)
+            self.simple_grafana_dashboard[
+                "title"
+            ] = f'{uid}-{self.simple_grafana_dashboard["title"]}'
+            self.assertEqual(dashboard_data, self.simple_grafana_dashboard)
 
     def test_create_multiple_devices(self):
         devices = [
@@ -98,13 +111,13 @@ def test_grafana_dashboard_not_in_a_list(self):
         response = self.create_device(
             uid=uid,
             address=address,
-            grafana_dashboards=SIMPLE_GRAFANA_DASHBOARD,
+            grafana_dashboards=self.simple_grafana_dashboard,
         )
         self.assertEqual(Device.objects.count(), 0)
         self.assertContains(
             response,
             '{"grafana_dashboards": ["gafana_dashboards is not a supported '
-            'format (list).',
+            "format (list).",
             status_code=400,
         )
 
@@ -127,11 +140,27 @@ def test_grafana_dashboard_illformed_json(self):
 
 
 class DeviceViewTests(APITestCase):
+    def setUp(self):
+        self.grafana_dashboards_path = Path("grafana_dashboards")
+        rmtree(self.grafana_dashboards_path, ignore_errors=True)
+        mkdir(self.grafana_dashboards_path)
+        self.simple_grafana_dashboard = {
+            "id": None,
+            "uid": None,
+            "title": "Production Overview",
+            "tags": ["templated"],
+            "timezone": "browser",
+            "schemaVersion": 16,
+            "refresh": "25s",
+        }
+
     def url(self, uid):
         return reverse("api:device", args=(uid,))
 
-    def create_device(self, uid, address):
-        data = {"uid": uid, "address": address}
+    def create_device(self, **fields):
+        data = {}
+        for field, value in fields.items():
+            data[field] = value
         url = reverse("api:devices")
         return self.client.post(url, data, format="json")
 
@@ -142,7 +171,7 @@ def test_get_nonexistent_device(self):
     def test_get_device(self):
         uid = "robot-1"
         address = "192.168.1.2"
-        self.create_device(uid, address)
+        self.create_device(uid=uid, address=address)
         response = self.client.get(self.url(uid))
         self.assertEqual(response.status_code, 200)
         content_json = json.loads(response.content)
@@ -159,7 +188,7 @@ def test_get_device(self):
     def test_patch_device(self):
         uid = "robot-1"
         address = "192.168.1.2"
-        self.create_device(uid, address)
+        self.create_device(uid=uid, address=address)
         address = "192.168.1.200"
         data = {"address": address}
         response = self.client.patch(self.url(uid), data, format="json")
@@ -171,21 +200,31 @@ def test_patch_device(self):
     def test_patch_dashboards(self):
         uid = "robot-1"
         address = "192.168.1.2"
-        self.create_device(uid, address)
-        data = {"grafana_dashboards": [SIMPLE_GRAFANA_DASHBOARD]}
+        self.create_device(uid=uid, address=address)
+        data = {"grafana_dashboards": [self.simple_grafana_dashboard]}
         response = self.client.patch(self.url(uid), data, format="json")
         self.assertEqual(response.status_code, 200)
         content_json = json.loads(response.content)
+        # necessary since patching returns the modified title
+        self.simple_grafana_dashboard[
+            "title"
+        ] = f'{uid}-{self.simple_grafana_dashboard["title"]}'
         self.assertEqual(
             content_json["grafana_dashboards"][0],
-            SIMPLE_GRAFANA_DASHBOARD,
+            self.simple_grafana_dashboard,
         )
         self.assertEqual(content_json["address"], address)
+        with open(
+            self.grafana_dashboards_path / "robot-1-Production_Overview.json",
+            "r",
+        ) as file:
+            dashboard_data = json.load(file)
+            self.assertEqual(dashboard_data, self.simple_grafana_dashboard)
 
     def test_invalid_patch_device(self):
         uid = "robot-1"
         address = "192.168.1.2"
-        self.create_device(uid, address)
+        self.create_device(uid=uid, address=address)
         address = "192.168.1"  # invalid IP
         data = {"address": address}
         response = self.client.patch(self.url(uid), data, format="json")
@@ -194,10 +233,69 @@ def test_invalid_patch_device(self):
     def test_delete_device(self):
         uid = "robot-1"
         address = "192.168.1.2"
-        self.create_device(uid, address)
+        self.create_device(
+            uid=uid,
+            address=address,
+            grafana_dashboards=[self.simple_grafana_dashboard],
+        )
         response = self.client.get(self.url(uid))
         self.assertEqual(response.status_code, 200)
+        self.assertTrue(
+            path.isfile(
+                self.grafana_dashboards_path
+                / "robot-1-Production_Overview.json"
+            )
+        )
         response = self.client.delete(self.url(uid))
         self.assertEqual(response.status_code, 204)
         response = self.client.get(self.url(uid))
         self.assertEqual(response.status_code, 404)
+        self.assertFalse(
+            path.isfile(
+                self.grafana_dashboards_path
+                / "robot-1-Production_Overview.json"
+            )
+        )
+
+
+class CommandsTestCase(TestCase):
+    def setUp(self):
+        self.grafana_dashboards_path = Path("grafana_dashboards")
+        rmtree(self.grafana_dashboards_path, ignore_errors=True)
+        mkdir(self.grafana_dashboards_path)
+        self.simple_grafana_dashboard = {
+            "id": None,
+            "uid": None,
+            "title": "Production Overview",
+            "tags": ["templated"],
+            "timezone": "browser",
+            "schemaVersion": 16,
+            "refresh": "25s",
+        }
+
+    def test_update_all_dashboards(self):
+        robot_1 = Device(
+            uid="robot-1",
+            address="127.0.0.1",
+            grafana_dashboards=[self.simple_grafana_dashboard],
+        )
+        robot_1.save()
+        robot_2 = Device(
+            uid="robot-2",
+            address="127.0.0.1",
+            grafana_dashboards=[self.simple_grafana_dashboard],
+        )
+        robot_2.save()
+        call_command("update_all_grafana_dashboards")
+        self.assertTrue(
+            path.isfile(
+                self.grafana_dashboards_path
+                / "robot-1-Production_Overview.json"
+            )
+        )
+        self.assertTrue(
+            path.isfile(
+                self.grafana_dashboards_path
+                / "robot-2-Production_Overview.json"
+            )
+        )