From f62a86983e79fcd2b69c3bb5fb78ffeefdad2b08 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Fri, 15 Mar 2024 17:41:39 +0100 Subject: [PATCH 1/2] refactor(applications): applications specific data in a seperate model Now applications specific data are stored in a different model. There is now a relation ManyToMany between a device and applications data --- Makefile | 2 +- cos_registration_server/api/apps.py | 1 + cos_registration_server/api/serializer.py | 143 +++++-- cos_registration_server/api/tests.py | 353 +++++++++++++----- cos_registration_server/api/urls.py | 11 + cos_registration_server/api/views.py | 61 ++- .../applications/__init__.py | 1 + cos_registration_server/applications/admin.py | 7 + cos_registration_server/applications/apps.py | 10 + .../applications/migrations/0001_initial.py | 35 ++ .../applications/migrations/__init__.py | 0 .../applications/models.py | 35 ++ cos_registration_server/applications/tests.py | 57 +++ cos_registration_server/applications/urls.py | 3 + cos_registration_server/applications/views.py | 1 + .../cos_registration_server/settings.py | 1 + .../cos_registration_server/urls.py | 1 + cos_registration_server/devices/admin.py | 1 + cos_registration_server/devices/apps.py | 1 + .../devices/migrations/0001_initial.py | 16 +- ...a_dashboards_device_unique_uid_blocking.py | 27 -- cos_registration_server/devices/models.py | 61 +-- cos_registration_server/devices/tests.py | 72 +++- cos_registration_server/devices/urls.py | 1 + cos_registration_server/devices/views.py | 1 + 25 files changed, 666 insertions(+), 236 deletions(-) create mode 100644 cos_registration_server/applications/__init__.py create mode 100644 cos_registration_server/applications/admin.py create mode 100644 cos_registration_server/applications/apps.py create mode 100644 cos_registration_server/applications/migrations/0001_initial.py create mode 100644 cos_registration_server/applications/migrations/__init__.py create mode 100644 cos_registration_server/applications/models.py create mode 100644 cos_registration_server/applications/tests.py create mode 100644 cos_registration_server/applications/urls.py create mode 100644 cos_registration_server/applications/views.py delete mode 100644 cos_registration_server/devices/migrations/0002_device_grafana_dashboards_device_unique_uid_blocking.py diff --git a/Makefile b/Makefile index 6277acd..493b90e 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ lint: ## Run pep8, black, mypy linters. .PHONY: test test: lint ## Run tests and generate coverage report. - $(ENV_PREFIX)coverage run --source='.' cos_registration_server/manage.py test api devices + $(ENV_PREFIX)coverage run --source='.' cos_registration_server/manage.py test api devices applications $(ENV_PREFIX)coverage xml $(ENV_PREFIX)coverage html diff --git a/cos_registration_server/api/apps.py b/cos_registration_server/api/apps.py index e7d9396..4daf150 100644 --- a/cos_registration_server/api/apps.py +++ b/cos_registration_server/api/apps.py @@ -1,4 +1,5 @@ """API apps.""" + from django.apps import AppConfig diff --git a/cos_registration_server/api/serializer.py b/cos_registration_server/api/serializer.py index 9e3f798..2590cff 100644 --- a/cos_registration_server/api/serializer.py +++ b/cos_registration_server/api/serializer.py @@ -1,64 +1,141 @@ """API app serializer.""" + import json +from applications.models import Dashboard, GrafanaDashboard from devices.models import Device from rest_framework import serializers -from rest_framework.validators import UniqueValidator - -class DeviceSerializer(serializers.Serializer): - """Device Serializer class.""" - uid = serializers.CharField( - required=True, - validators=[UniqueValidator(queryset=Device.objects.all())], - ) - creation_date = serializers.DateTimeField(read_only=True) - address = serializers.IPAddressField(required=True) - grafana_dashboards = serializers.JSONField(required=False) +class DashboardSerializer: + """Dashboard Serializer class.""" - def create(self, validated_data): - """Create Device object from data. + class Meta: + """DashboardSerializer Meta class.""" - validated_data: Dict of complete and validated data. - """ - return Device.objects.create(**validated_data) + model = Dashboard + fields = ["uid", "dashboard"] def update(self, instance, validated_data): - """Update a Device from data. + """Update a Dashboard from data. instance: Device instance. validated_data: Dict of partial and validated data. """ - address = validated_data.get("address", instance.address) - instance.address = address - grafana_dashboards = validated_data.get( - "grafana_dashboards", instance.grafana_dashboards - ) - instance.grafana_dashboards = grafana_dashboards + dashboard = validated_data.get("dashboard", instance.dashboard) + instance.dashboard = dashboard instance.save() return instance - def validate_grafana_dashboards(self, value): - """Validate grafana dashboards data. + def validate_dashboard(self, value): + """Validate dashboards data. - value: Grafana dashboards provided data. - return: Grafana dashboards as list. + value: dashboard provided data. + return: dashboard json. raise: json.JSONDecodeError serializers.ValidationError """ if isinstance(value, str): try: - dashboards = json.loads(value) + dashboard = json.loads(value) except json.JSONDecodeError: raise serializers.ValidationError( - "Failed to load grafana_dashboards as json." + "Failed to load dashboard as json." ) else: - dashboards = value - if not isinstance(dashboards, list): + dashboard = value + if not isinstance(dashboard, dict): raise serializers.ValidationError( - "gafana_dashboards is not a supported format (list)." + "Dashboard is not a supported format (dict)." ) - return dashboards + return dashboard + + +class GrafanaDashboardSerializer( + DashboardSerializer, serializers.ModelSerializer +): + """Grafana Dashboard Serializer class.""" + + class Meta(DashboardSerializer.Meta): + """DashboardSerializer Meta class.""" + + model = GrafanaDashboard + + def create(self, validated_data): + """Create Grafana Dashboard object from data. + + validated_data: Dict of complete and validated data. + """ + return GrafanaDashboard.objects.create(**validated_data) + + +class DeviceSerializer(serializers.ModelSerializer): + """Device Serializer class.""" + + grafana_dashboards = serializers.SlugRelatedField( + many=True, + queryset=GrafanaDashboard.objects.all(), + slug_field="uid", + required=False, + ) + + class Meta: + """DeviceSerializer Meta class.""" + + model = Device + fields = ("uid", "creation_date", "address", "grafana_dashboards") + + def create(self, validated_data): + """Create Device object from data. + + validated_data: Dict of complete and validated data. + """ + grafana_dashboards_data = validated_data.pop("grafana_dashboards", {}) + device = Device.objects.create(**validated_data) + + for dashboard_uid in grafana_dashboards_data: + try: + dashboard = GrafanaDashboard.objects.get(uid=dashboard_uid) + device.grafana_dashboards.add(dashboard) + except GrafanaDashboard.DoesNotExist: + raise serializers.ValidationError( + f"GrafanaDashboard with UID {dashboard_uid}" + " does not exist." + ) + return device + + def update(self, instance, validated_data): + """Update a Device from data. + + instance: Device instance. + validated_data: Dict of partial and validated data. + """ + # Update device fields (if any) + for field, value in validated_data.items(): + if field != "grafana_dashboards": + setattr(instance, field, value) + instance.save() + + # Update Grafana dashboards + try: + grafana_dashboards_data = validated_data.pop("grafana_dashboards") + current_grafana_dashboards = instance.grafana_dashboards.all() + for dashboard in current_grafana_dashboards: + instance.grafana_dashboards.remove(dashboard) + + for dashboard_uid in grafana_dashboards_data: + try: + dashboard = GrafanaDashboard.objects.get(uid=dashboard_uid) + instance.grafana_dashboards.add(dashboard) + except GrafanaDashboard.DoesNotExist: + raise serializers.ValidationError( + f"Grafana Dashboard with UID {dashboard_uid}" + " does not exist." + ) + except KeyError: + # Handle partial updates without grafana_dashboards vs + # empty grafana_dashboards + pass + + return instance diff --git a/cos_registration_server/api/tests.py b/cos_registration_server/api/tests.py index e07f197..b9c5b8a 100644 --- a/cos_registration_server/api/tests.py +++ b/cos_registration_server/api/tests.py @@ -1,11 +1,7 @@ import json -from os import mkdir, path -from pathlib import Path -from shutil import rmtree -from devices.models import Device, default_dashboards_json_field -from django.core.management import call_command -from django.test import TestCase +from applications.models import GrafanaDashboard +from devices.models import Device from django.urls import reverse from django.utils import timezone from rest_framework.test import APITestCase @@ -14,9 +10,6 @@ class DevicesViewTests(APITestCase): def setUp(self): self.url = reverse("api:devices") - self.grafana_dashboards_path = Path("grafana_dashboards") - rmtree(self.grafana_dashboards_path, ignore_errors=True) - mkdir(self.grafana_dashboards_path) self.simple_grafana_dashboard = { "id": None, @@ -34,6 +27,11 @@ def create_device(self, **fields): data[field] = value return self.client.post(self.url, data, format="json") + def add_grafana_dashboard(self, uid, dashboard): + dashboard = GrafanaDashboard(uid=uid, dashboard=dashboard) + dashboard.save() + return dashboard + def test_get_nothing(self): response = self.client.get(self.url) self.assertEqual(response.status_code, 200) @@ -42,12 +40,14 @@ def test_get_nothing(self): def test_create_device(self): uid = "robot-1" address = "192.168.0.1" - custom_grafana_dashboards = default_dashboards_json_field() - custom_grafana_dashboards.append(self.simple_grafana_dashboard) + grafana_dashboard_uid = "dashboard-1" + self.add_grafana_dashboard( + uid=grafana_dashboard_uid, dashboard=self.simple_grafana_dashboard + ) response = self.create_device( uid=uid, address=address, - grafana_dashboards=custom_grafana_dashboards, + grafana_dashboards={grafana_dashboard_uid}, ) self.assertEqual(response.status_code, 201) self.assertEqual(Device.objects.count(), 1) @@ -59,17 +59,13 @@ def test_create_device(self): delta=timezone.timedelta(seconds=10), ) self.assertEqual( - Device.objects.get().grafana_dashboards, custom_grafana_dashboards - ) - with open( - self.grafana_dashboards_path / "robot-1-Production_Overview.json", - "r", - ) as file: - dashboard_data = json.load(file) - self.simple_grafana_dashboard[ - "title" - ] = f'{uid}-{self.simple_grafana_dashboard["title"]}' - self.assertEqual(dashboard_data, self.simple_grafana_dashboard) + Device.objects.get().grafana_dashboards.get().uid, + grafana_dashboard_uid, + ) + self.assertEqual( + Device.objects.get().grafana_dashboards.get().dashboard, + self.simple_grafana_dashboard, + ) def test_create_multiple_devices(self): devices = [ @@ -100,51 +96,43 @@ def test_create_already_present_uid(self): self.assertEqual(Device.objects.count(), 1) self.assertContains( response, - '{"uid": ["This field must be unique."]}', + '{"uid": ["device with this uid already exists."]}', status_code=400, ) - def test_grafana_dashboard_not_in_a_list(self): + def test_assiciate_device_with_non_existant_grafana_dashboard(self): uid = "robot-1" address = "192.168.0.1" - # we try to create the same one response = self.create_device( - uid=uid, - address=address, - grafana_dashboards=self.simple_grafana_dashboard, + uid=uid, address=address, grafana_dashboards={"dashboard-1"} ) self.assertEqual(Device.objects.count(), 0) self.assertContains( response, - '{"grafana_dashboards": ["gafana_dashboards is not a supported ' - "format (list).", + '{"grafana_dashboards": ["Object with uid=dashboard-1 does not exist."]}', status_code=400, ) - def test_grafana_dashboard_illformed_json(self): + def test_grafana_dashboards_not_in_a_list(self): uid = "robot-1" address = "192.168.0.1" # we try to create the same one response = self.create_device( uid=uid, address=address, - grafana_dashboards='[{"hello"=321}', + grafana_dashboards="dashboard-1", ) self.assertEqual(Device.objects.count(), 0) self.assertContains( response, - '{"grafana_dashboards": ["Failed to load grafana_dashboards ' - "as json.", + '{"grafana_dashboards": ["Expected a list of items but got type \\"str\\".', status_code=400, ) class DeviceViewTests(APITestCase): def setUp(self): - self.grafana_dashboards_path = Path("grafana_dashboards") - rmtree(self.grafana_dashboards_path, ignore_errors=True) - mkdir(self.grafana_dashboards_path) - self.simple_grafana_dashboard = { + self.simple_grafana_dashboard_json = { "id": None, "uid": None, "title": "Production Overview", @@ -153,6 +141,10 @@ def setUp(self): "schemaVersion": 16, "refresh": "25s", } + self.grafana_dashboard = GrafanaDashboard( + uid="dashboard-1", dashboard=self.simple_grafana_dashboard_json + ) + self.grafana_dashboard.save() def url(self, uid): return reverse("api:device", args=(uid,)) @@ -171,7 +163,11 @@ def test_get_nonexistent_device(self): def test_get_device(self): uid = "robot-1" address = "192.168.1.2" - self.create_device(uid=uid, address=address) + self.create_device( + uid=uid, + address=address, + grafana_dashboards={self.grafana_dashboard.uid}, + ) response = self.client.get(self.url(uid)) self.assertEqual(response.status_code, 200) content_json = json.loads(response.content) @@ -184,6 +180,9 @@ def test_get_device(self): timezone.now(), delta=timezone.timedelta(seconds=10), ) + self.assertEqual( + content_json["grafana_dashboards"], [self.grafana_dashboard.uid] + ) def test_patch_device(self): uid = "robot-1" @@ -197,29 +196,42 @@ def test_patch_device(self): self.assertEqual(content_json["uid"], uid) self.assertEqual(content_json["address"], address) - def test_patch_dashboards(self): + def test_patch_grafana_dashboards(self): uid = "robot-1" address = "192.168.1.2" self.create_device(uid=uid, address=address) - data = {"grafana_dashboards": [self.simple_grafana_dashboard]} + self.assertEqual(Device.objects.get().grafana_dashboards.count(), 0) + data = {"grafana_dashboards": [self.grafana_dashboard.uid]} response = self.client.patch(self.url(uid), data, format="json") self.assertEqual(response.status_code, 200) content_json = json.loads(response.content) - # necessary since patching returns the modified title - self.simple_grafana_dashboard[ - "title" - ] = f'{uid}-{self.simple_grafana_dashboard["title"]}' self.assertEqual( content_json["grafana_dashboards"][0], - self.simple_grafana_dashboard, + self.grafana_dashboard.uid, ) - self.assertEqual(content_json["address"], address) - with open( - self.grafana_dashboards_path / "robot-1-Production_Overview.json", - "r", - ) as file: - dashboard_data = json.load(file) - self.assertEqual(dashboard_data, self.simple_grafana_dashboard) + self.assertEqual( + Device.objects.get().grafana_dashboards.get().uid, + self.grafana_dashboard.uid, + ) + + def test_partial_patch_with_existing_grafana_dashboards(self): + uid = "robot-1" + address = "192.168.1.2" + self.create_device( + uid=uid, + address=address, + grafana_dashboards={self.grafana_dashboard.uid}, + ) + self.assertEqual(Device.objects.get().grafana_dashboards.count(), 1) + data = {"address": "192.168.1.3"} + response = self.client.patch(self.url(uid), data, format="json") + self.assertEqual(response.status_code, 200) + content_json = json.loads(response.content) + self.assertEqual( + content_json["address"], + data["address"], + ) + self.assertEqual(Device.objects.get().grafana_dashboards.count(), 1) def test_invalid_patch_device(self): uid = "robot-1" @@ -236,33 +248,140 @@ def test_delete_device(self): self.create_device( uid=uid, address=address, - grafana_dashboards=[self.simple_grafana_dashboard], + grafana_dashboards=[self.grafana_dashboard.uid], ) response = self.client.get(self.url(uid)) self.assertEqual(response.status_code, 200) - self.assertTrue( - path.isfile( - self.grafana_dashboards_path - / "robot-1-Production_Overview.json" - ) - ) response = self.client.delete(self.url(uid)) self.assertEqual(response.status_code, 204) response = self.client.get(self.url(uid)) self.assertEqual(response.status_code, 404) - self.assertFalse( - path.isfile( - self.grafana_dashboards_path - / "robot-1-Production_Overview.json" + + +class GrafanaDashboardsViewTests(APITestCase): + def setUp(self): + self.url = reverse("api:grafana_dashboards") + + self.simple_grafana_dashboard = { + "id": None, + "uid": None, + "title": "Production Overview", + "tags": ["templated"], + "timezone": "browser", + "schemaVersion": 16, + "refresh": "25s", + } + + def create_dashboard(self, **fields): + data = {} + for field, value in fields.items(): + data[field] = value + return self.client.post(self.url, data, format="json") + + def add_device(self, uid): + device = Device(uid=uid, address="127.0.0.1") + device.save() + return device + + def test_get_nothing(self): + response = self.client.get(self.url) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(json.loads(response.content)), 0) + + def test_create_dashboard(self): + grafana_dashboard_uid = "dashboard-1" + response = self.create_dashboard( + uid=grafana_dashboard_uid, dashboard=self.simple_grafana_dashboard + ) + self.assertEqual(response.status_code, 201) + self.assertEqual(GrafanaDashboard.objects.count(), 1) + self.assertEqual( + GrafanaDashboard.objects.get().uid, grafana_dashboard_uid + ) + self.assertEqual( + GrafanaDashboard.objects.get().dashboard, + self.simple_grafana_dashboard, + ) + + def test_create_multiple_dashboards(self): + dashboards = [ + {"uid": "d-1", "dashboard": '{"test1": "value"}'}, + {"uid": "d-2", "dashboard": '{"test2": "value"}'}, + {"uid": "d-3", "dashboard": '{"test3": "value"}'}, + ] + for dashboard in dashboards: + self.create_dashboard( + uid=dashboard["uid"], dashboard=dashboard["dashboard"] ) + + self.assertEqual(GrafanaDashboard.objects.count(), 3) + + response = self.client.get(self.url) + self.assertEqual(response.status_code, 200) + content_json = json.loads(response.content) + self.assertEqual(len(content_json), 3) + for i, dashboard in enumerate(content_json): + self.assertEqual(dashboards[i]["uid"], dashboard["uid"]) + self.assertEqual( + json.loads(dashboards[i]["dashboard"]), dashboard["dashboard"] + ) + + def test_create_already_present_uid(self): + grafana_dashboard_uid = "dashboard-1" + response = self.create_dashboard( + uid=grafana_dashboard_uid, dashboard=self.simple_grafana_dashboard + ) + self.assertEqual(response.status_code, 201) + self.assertEqual(GrafanaDashboard.objects.count(), 1) + # we try to create the same one + response = self.create_dashboard( + uid=grafana_dashboard_uid, dashboard=self.simple_grafana_dashboard + ) + self.assertEqual(GrafanaDashboard.objects.count(), 1) + self.assertContains( + response, + '{"uid": ["grafana dashboard with this uid already exists."]}', + status_code=400, + ) + + def test_create_dashboard_with_illformed_json(self): + grafana_dashboard_uid = "dashboard-1" + response = self.create_dashboard( + uid=grafana_dashboard_uid, dashboard='{"hello": {"test"sdsa"}}' + ) + self.assertEqual(response.status_code, 400) + self.assertEqual(GrafanaDashboard.objects.count(), 0) + self.assertContains( + response, + '{"dashboard": ["Failed to load dashboard as json."]}', + status_code=400, + ) + + def test_get_dashboards_associated_with_device(self): + grafana_dashboard_uid = "dashboard-1" + self.create_dashboard( + uid=grafana_dashboard_uid, dashboard=self.simple_grafana_dashboard + ) + self.add_device(uid="robot-1").grafana_dashboards.add( + GrafanaDashboard.objects.get() + ) + + response = self.client.get(self.url) + self.assertEqual(response.status_code, 200) + content_json = json.loads(response.content) + self.assertEqual( + content_json, + [ + { + "uid": grafana_dashboard_uid, + "dashboard": self.simple_grafana_dashboard, + } + ], ) -class CommandsTestCase(TestCase): +class GrafanaDashboardViewTests(APITestCase): def setUp(self): - self.grafana_dashboards_path = Path("grafana_dashboards") - rmtree(self.grafana_dashboards_path, ignore_errors=True) - mkdir(self.grafana_dashboards_path) self.simple_grafana_dashboard = { "id": None, "uid": None, @@ -273,29 +392,77 @@ def setUp(self): "refresh": "25s", } - def test_update_all_dashboards(self): - robot_1 = Device( - uid="robot-1", - address="127.0.0.1", - grafana_dashboards=[self.simple_grafana_dashboard], - ) - robot_1.save() - robot_2 = Device( - uid="robot-2", - address="127.0.0.1", - grafana_dashboards=[self.simple_grafana_dashboard], - ) - robot_2.save() - call_command("update_all_grafana_dashboards") - self.assertTrue( - path.isfile( - self.grafana_dashboards_path - / "robot-1-Production_Overview.json" - ) + def url(self, uid): + return reverse("api:grafana_dashboard", args=(uid,)) + + def create_dashboard(self, **fields): + data = {} + for field, value in fields.items(): + data[field] = value + url = reverse("api:grafana_dashboards") + return self.client.post(url, data, format="json") + + def add_device(self, uid): + device = Device(uid=uid, address="127.0.0.1") + device.save() + return device + + def test_get_nonexistent_dashboard(self): + response = self.client.get(self.url("future-dashboard")) + self.assertEqual(response.status_code, 404) + + def test_get_dashboard(self): + dashboard_uid = "dashboard-1" + self.create_dashboard( + uid=dashboard_uid, + dashboard=self.simple_grafana_dashboard, ) - self.assertTrue( - path.isfile( - self.grafana_dashboards_path - / "robot-2-Production_Overview.json" - ) + response = self.client.get(self.url(dashboard_uid)) + self.assertEqual(response.status_code, 200) + content_json = json.loads(response.content) + self.assertEqual(content_json["uid"], dashboard_uid) + self.assertEqual( + content_json["dashboard"], self.simple_grafana_dashboard ) + + def test_patch_dashboard(self): + dashboard_uid = "dashboard-1" + self.create_dashboard( + uid=dashboard_uid, + dashboard=self.simple_grafana_dashboard, + ) + data = {"dashboard": '{"test": "dash"}'} + response = self.client.patch( + self.url(dashboard_uid), data, format="json" + ) + self.assertEqual(response.status_code, 200) + content_json = json.loads(response.content) + self.assertEqual(content_json["uid"], dashboard_uid) + self.assertEqual( + content_json["dashboard"], json.loads(data["dashboard"]) + ) + + def test_invalid_patch_dashboard(self): + dashboard_uid = "dashboard-1" + self.create_dashboard( + uid=dashboard_uid, + dashboard=self.simple_grafana_dashboard, + ) + data = {"dashboard": '{"test": "das}'} + response = self.client.patch( + self.url(dashboard_uid), data, format="json" + ) + self.assertEqual(response.status_code, 400) + + def test_delete_dashboard(self): + dashboard_uid = "dashboard-1" + self.create_dashboard( + uid=dashboard_uid, + dashboard=self.simple_grafana_dashboard, + ) + response = self.client.get(self.url(dashboard_uid)) + self.assertEqual(response.status_code, 200) + response = self.client.delete(self.url(dashboard_uid)) + self.assertEqual(response.status_code, 204) + response = self.client.get(self.url(dashboard_uid)) + self.assertEqual(response.status_code, 404) diff --git a/cos_registration_server/api/urls.py b/cos_registration_server/api/urls.py index afc7f2b..315357b 100644 --- a/cos_registration_server/api/urls.py +++ b/cos_registration_server/api/urls.py @@ -1,4 +1,5 @@ """API urls.""" + from django.urls import path from . import views @@ -9,4 +10,14 @@ # make a version API url path("v1/devices", views.devices, name="devices"), path("v1/devices/", views.device, name="device"), + path( + "v1/applications/grafana/dashboards", + views.grafana_dashboards, + name="grafana_dashboards", + ), + path( + "v1/applications/grafana/dashboards/", + views.grafana_dashboard, + name="grafana_dashboard", + ), ] diff --git a/cos_registration_server/api/views.py b/cos_registration_server/api/views.py index 28b3caf..4b18b1a 100644 --- a/cos_registration_server/api/views.py +++ b/cos_registration_server/api/views.py @@ -1,14 +1,11 @@ """API views.""" -from api.serializer import DeviceSerializer + +from api.serializer import DeviceSerializer, GrafanaDashboardSerializer +from applications.models import GrafanaDashboard from devices.models import Device from django.http import HttpResponse, JsonResponse from rest_framework.parsers import JSONParser -from .grafana_dashboards import ( - add_grafana_dashboards, - delete_grafana_dashboards, -) - def devices(request): """Devices API view. @@ -27,7 +24,6 @@ def devices(request): serialized = DeviceSerializer(data=data) if serialized.is_valid(): serialized.save() - add_grafana_dashboards(serialized.instance) return JsonResponse(serialized.data, status=201) return JsonResponse(serialized.errors, status=400) @@ -52,11 +48,56 @@ def device(request, uid): serialized = DeviceSerializer(device, data=data, partial=True) if serialized.is_valid(): serialized.save() - delete_grafana_dashboards(serialized.instance) - add_grafana_dashboards(serialized.instance) return JsonResponse(serialized.data) return JsonResponse(serialized.errors, status=400) elif request.method == "DELETE": device.delete() - delete_grafana_dashboards(device) + return HttpResponse(status=204) + + +def grafana_dashboards(request): + """Grafana dashboards API view. + + request: Http request (GET,POST). + return: Http JSON response. + """ + if request.method == "GET": + dashboards = GrafanaDashboard.objects.all() + serialized = GrafanaDashboardSerializer(dashboards, many=True) + return JsonResponse(serialized.data, safe=False) + elif request.method == "POST": + data = JSONParser().parse(request) + serialized = GrafanaDashboardSerializer(data=data) + if serialized.is_valid(): + serialized.save() + return JsonResponse(serialized.data, status=201) + return JsonResponse(serialized.errors, status=400) + + +def grafana_dashboard(request, uid): + """Grafana dashboard API view. + + request: Http request (GET,PACH, DELETE). + uid: GrafanaDashboard UID passed in the URL. + return: Http JSON response. + """ + try: + dashboard = GrafanaDashboard.objects.get(uid=uid) + except GrafanaDashboard.DoesNotExist: + return HttpResponse(status=404) + + if request.method == "GET": + serialized = GrafanaDashboardSerializer(dashboard) + return JsonResponse(serialized.data) + if request.method == "PATCH": + data = JSONParser().parse(request) + serialized = GrafanaDashboardSerializer( + dashboard, data=data, partial=True + ) + if serialized.is_valid(): + serialized.save() + return JsonResponse(serialized.data) + return JsonResponse(serialized.errors, status=400) + elif request.method == "DELETE": + dashboard.delete() return HttpResponse(status=204) diff --git a/cos_registration_server/applications/__init__.py b/cos_registration_server/applications/__init__.py new file mode 100644 index 0000000..18d1e6c --- /dev/null +++ b/cos_registration_server/applications/__init__.py @@ -0,0 +1 @@ +"""Applications django application.""" diff --git a/cos_registration_server/applications/admin.py b/cos_registration_server/applications/admin.py new file mode 100644 index 0000000..78dd972 --- /dev/null +++ b/cos_registration_server/applications/admin.py @@ -0,0 +1,7 @@ +"""Device administration registration.""" + +from django.contrib import admin + +from .models import GrafanaDashboard + +admin.site.register(GrafanaDashboard) diff --git a/cos_registration_server/applications/apps.py b/cos_registration_server/applications/apps.py new file mode 100644 index 0000000..442cd27 --- /dev/null +++ b/cos_registration_server/applications/apps.py @@ -0,0 +1,10 @@ +"""Devices apps.""" + +from django.apps import AppConfig + + +class ApplicationsConfig(AppConfig): + """ApplicationsConfig class.""" + + default_auto_field = "django.db.models.BigAutoField" + name = "applications" diff --git a/cos_registration_server/applications/migrations/0001_initial.py b/cos_registration_server/applications/migrations/0001_initial.py new file mode 100644 index 0000000..e59e82e --- /dev/null +++ b/cos_registration_server/applications/migrations/0001_initial.py @@ -0,0 +1,35 @@ +# Generated by Django 4.2.11 on 2024-03-15 14:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="GrafanaDashboard", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("uid", models.CharField(max_length=200, unique=True)), + ( + "dashboard", + models.JSONField(verbose_name="Dashboard json field"), + ), + ], + options={ + "abstract": False, + }, + ), + ] diff --git a/cos_registration_server/applications/migrations/__init__.py b/cos_registration_server/applications/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cos_registration_server/applications/models.py b/cos_registration_server/applications/models.py new file mode 100644 index 0000000..5268b31 --- /dev/null +++ b/cos_registration_server/applications/models.py @@ -0,0 +1,35 @@ +"""Applications DB model.""" + +from django.db import models + + +class Dashboard(models.Model): + """Application dashboard. + + This class represent an application dashboard in the DB. + + uid: Unique ID of the dashboard. + dashboard: Dashboard JSON. + """ + + uid = models.CharField(max_length=200, unique=True) + dashboard = models.JSONField("Dashboard json field") + + class Meta: + """Model Meta class overwritting.""" + + abstract = True + + def __str__(self): + """Str representation of a dashboard.""" + return self.uid + + +class GrafanaDashboard(Dashboard): # noqa: DJ08 + """Grafana dashboard. + + This class represent a Grafana dashboard in the DB. + + uid: Unique ID of the dashboard. + dashboard: Dashboard JSON. + """ diff --git a/cos_registration_server/applications/tests.py b/cos_registration_server/applications/tests.py new file mode 100644 index 0000000..740cc07 --- /dev/null +++ b/cos_registration_server/applications/tests.py @@ -0,0 +1,57 @@ +from devices.models import Device +from django.db.utils import IntegrityError +from django.test import TestCase + +from .models import GrafanaDashboard + +SIMPLE_GRAFANA_DASHBOARD = { + "id": None, + "uid": None, + "title": "Production Overview", + "tags": ["templated"], + "timezone": "browser", + "schemaVersion": 16, + "refresh": "25s", +} + + +class GrafanaDashboardModelTests(TestCase): + def test_creation_of_a_dashboard(self): + dashboard_name = "first_dashboard" + grafana_dashboard = GrafanaDashboard( + uid=dashboard_name, dashboard=SIMPLE_GRAFANA_DASHBOARD + ) + self.assertEqual(grafana_dashboard.uid, dashboard_name) + self.assertEqual(grafana_dashboard.dashboard, SIMPLE_GRAFANA_DASHBOARD) + + def test_dashboard_str(self): + dashboard_name = "first_dashboard" + grafana_dashboard = GrafanaDashboard( + uid=dashboard_name, dashboard=SIMPLE_GRAFANA_DASHBOARD + ) + self.assertEqual(str(grafana_dashboard), dashboard_name) + + def test_dashboard_uid_uniqueness(self): + dashboard_name = "first_dashboard" + GrafanaDashboard( + uid=dashboard_name, dashboard=SIMPLE_GRAFANA_DASHBOARD + ).save() + + self.assertRaises( + IntegrityError, + GrafanaDashboard( + uid=dashboard_name, dashboard=SIMPLE_GRAFANA_DASHBOARD + ).save, + ) + + def test_device_from_a_dashboard(self): + dashboard_name = "first_dashboard" + grafana_dashboard = GrafanaDashboard( + uid=dashboard_name, dashboard=SIMPLE_GRAFANA_DASHBOARD + ) + grafana_dashboard.save() + device = Device(uid="robot", address="127.0.0.1") + device.save() + device.grafana_dashboards.add(grafana_dashboard) + + self.assertEqual(grafana_dashboard.devices.all()[0].uid, "robot") diff --git a/cos_registration_server/applications/urls.py b/cos_registration_server/applications/urls.py new file mode 100644 index 0000000..4da9a3b --- /dev/null +++ b/cos_registration_server/applications/urls.py @@ -0,0 +1,3 @@ +"""Devices urls.""" + +app_name = "applications" diff --git a/cos_registration_server/applications/views.py b/cos_registration_server/applications/views.py new file mode 100644 index 0000000..4f9001c --- /dev/null +++ b/cos_registration_server/applications/views.py @@ -0,0 +1 @@ +"""Devices views.""" diff --git a/cos_registration_server/cos_registration_server/settings.py b/cos_registration_server/cos_registration_server/settings.py index 3dfee55..e0b877a 100644 --- a/cos_registration_server/cos_registration_server/settings.py +++ b/cos_registration_server/cos_registration_server/settings.py @@ -38,6 +38,7 @@ INSTALLED_APPS = [ "api.apps.ApiConfig", "devices.apps.DevicesConfig", + "applications.apps.ApplicationsConfig", "django.contrib.admin", "django.contrib.auth", "django.contrib.contenttypes", diff --git a/cos_registration_server/cos_registration_server/urls.py b/cos_registration_server/cos_registration_server/urls.py index f0bb4fc..43aa4a1 100644 --- a/cos_registration_server/cos_registration_server/urls.py +++ b/cos_registration_server/cos_registration_server/urls.py @@ -14,6 +14,7 @@ 1. Import the include() function: from django.urls import include, path 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ + from django.contrib import admin from django.urls import include, path diff --git a/cos_registration_server/devices/admin.py b/cos_registration_server/devices/admin.py index 8403e75..70df80b 100644 --- a/cos_registration_server/devices/admin.py +++ b/cos_registration_server/devices/admin.py @@ -1,4 +1,5 @@ """Device administration registration.""" + from django.contrib import admin from .models import Device diff --git a/cos_registration_server/devices/apps.py b/cos_registration_server/devices/apps.py index f52b8f7..b7ffd24 100644 --- a/cos_registration_server/devices/apps.py +++ b/cos_registration_server/devices/apps.py @@ -1,4 +1,5 @@ """Devices apps.""" + from django.apps import AppConfig diff --git a/cos_registration_server/devices/migrations/0001_initial.py b/cos_registration_server/devices/migrations/0001_initial.py index b388c51..5246745 100644 --- a/cos_registration_server/devices/migrations/0001_initial.py +++ b/cos_registration_server/devices/migrations/0001_initial.py @@ -1,12 +1,15 @@ -# Generated by Django 4.2.9 on 2024-01-17 15:58 +# Generated by Django 4.2.11 on 2024-03-15 14:33 from django.db import migrations, models class Migration(migrations.Migration): + initial = True - dependencies = [] + dependencies = [ + ("applications", "0001_initial"), + ] operations = [ migrations.CreateModel( @@ -21,7 +24,7 @@ class Migration(migrations.Migration): verbose_name="ID", ), ), - ("uid", models.CharField(max_length=200)), + ("uid", models.CharField(max_length=200, unique=True)), ( "creation_date", models.DateTimeField( @@ -32,6 +35,13 @@ class Migration(migrations.Migration): "address", models.GenericIPAddressField(verbose_name="device IP"), ), + ( + "grafana_dashboards", + models.ManyToManyField( + related_name="devices", + to="applications.grafanadashboard", + ), + ), ], ), ] diff --git a/cos_registration_server/devices/migrations/0002_device_grafana_dashboards_device_unique_uid_blocking.py b/cos_registration_server/devices/migrations/0002_device_grafana_dashboards_device_unique_uid_blocking.py deleted file mode 100644 index 9620118..0000000 --- a/cos_registration_server/devices/migrations/0002_device_grafana_dashboards_device_unique_uid_blocking.py +++ /dev/null @@ -1,27 +0,0 @@ -# Generated by Django 4.2.9 on 2024-02-01 17:44 - -import devices.models -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("devices", "0001_initial"), - ] - - operations = [ - migrations.AddField( - model_name="device", - name="grafana_dashboards", - field=models.JSONField( - default=devices.models.default_dashboards_json_field, - verbose_name="Dashboards json field", - ), - ), - migrations.AddConstraint( - model_name="device", - constraint=models.UniqueConstraint( - fields=("uid",), name="unique_uid_blocking" - ), - ), - ] diff --git a/cos_registration_server/devices/models.py b/cos_registration_server/devices/models.py index 89d57d3..8597d9d 100644 --- a/cos_registration_server/devices/models.py +++ b/cos_registration_server/devices/models.py @@ -1,18 +1,7 @@ """Device DB model.""" -import json -from django.core.exceptions import ValidationError +from applications.models import GrafanaDashboard from django.db import models -from django.db.models.constraints import UniqueConstraint - - -def default_dashboards_json_field(): - """Return default value for dashboards. - - Default json values are usually dict but - here we need a list - """ - return [] class Device(models.Model): @@ -23,56 +12,16 @@ class Device(models.Model): uid: Unique ID of the device. creation_date: Creation date of the device. address: IP address of the device. - grafana_dashboards: list of Grafana dashboards. + grafana_dashboards: Grafana dashboards relations. """ - uid = models.CharField(max_length=200) + uid = models.CharField(max_length=200, unique=True) creation_date = models.DateTimeField("creation date", auto_now_add=True) address = models.GenericIPAddressField("device IP") - grafana_dashboards = models.JSONField( - "Dashboards json field", default=default_dashboards_json_field + grafana_dashboards = models.ManyToManyField( + GrafanaDashboard, related_name="devices" ) - class Meta: - """Model Meta class overwritting.""" - - constraints = [ - UniqueConstraint(fields=["uid"], name="unique_uid_blocking") - ] - def __str__(self): """Str representation of a device.""" return self.uid - - def clean(self): - """Model clean overwritting. - - Default Django model method to validate the model. - Not automatically called. - - raise: - json.JSONDecodeError - ValidationError - """ - # make sure the grafana_dashboards is containing an array of dashboards - dashboards = [] - if isinstance(self.grafana_dashboards, str): - try: - dashboards = json.loads(self.grafana_dashboards) - except json.JSONDecodeError: - raise ValidationError( - "Failed to load grafana_dashboards as json" - ) - elif isinstance(self.grafana_dashboards, list): - dashboards = self.grafana_dashboards - else: - raise ValidationError( - f"Unknow type for grafana_dashboards: \ - {type(self.grafana_dashboards)}" - ) - - if dashboards is None or not isinstance(dashboards, list): - raise ValidationError( - 'gafana_dashboards is not well formated. \ - Make sure all the dashboards are within "dashbords": [] ' - ) diff --git a/cos_registration_server/devices/tests.py b/cos_registration_server/devices/tests.py index f842ea5..95d03e6 100644 --- a/cos_registration_server/devices/tests.py +++ b/cos_registration_server/devices/tests.py @@ -1,10 +1,12 @@ from html import escape +from applications.models import GrafanaDashboard +from django.db.utils import IntegrityError from django.test import Client, TestCase from django.urls import reverse from django.utils import timezone -from .models import Device, default_dashboards_json_field +from .models import Device SIMPLE_GRAFANA_DASHBOARD = { "id": None, @@ -22,15 +24,14 @@ def test_creation_of_a_device(self): device = Device( uid="hello-123", creation_date=timezone.now(), address="127.0.0.1" ) + device.save() self.assertEqual(device.uid, "hello-123") self.assertEqual(str(device.address), "127.0.0.1") self.assertLessEqual(device.creation_date, timezone.now()) self.assertGreater( device.creation_date, timezone.now() - timezone.timedelta(hours=1) ) - self.assertEquals( - device.grafana_dashboards, default_dashboards_json_field() - ) + self.assertEqual(len(device.grafana_dashboards.all()), 0) def test_device_str(self): device = Device( @@ -38,20 +39,65 @@ def test_device_str(self): ) self.assertEqual(str(device), "hello-123") - def test_device_grafana_dashboards(self): - custom_grafana_dashboards = default_dashboards_json_field() - custom_grafana_dashboards.append(SIMPLE_GRAFANA_DASHBOARD) + def test_device_create_grafana_dashboards(self): device = Device( - uid="hello-123", - creation_date=timezone.now(), - address="127.0.0.1", - grafana_dashboards=custom_grafana_dashboards, + uid="hello-123", creation_date=timezone.now(), address="127.0.0.1" + ) + device.save() + device.grafana_dashboards.create( + uid="first_dashboard", dashboard=SIMPLE_GRAFANA_DASHBOARD ) self.assertEqual( - device.grafana_dashboards[0], + device.grafana_dashboards.all()[0].uid, + "first_dashboard", + ) + self.assertEqual( + device.grafana_dashboards.all()[0].dashboard, SIMPLE_GRAFANA_DASHBOARD, ) + def test_device_create_grafana_dashboards_then_delete_device(self): + device = Device( + uid="hello-123", creation_date=timezone.now(), address="127.0.0.1" + ) + device.save() + device.grafana_dashboards.create( + uid="first_dashboard", dashboard=SIMPLE_GRAFANA_DASHBOARD + ) + device.delete() + self.assertEqual( + GrafanaDashboard.objects.all()[0].uid, + "first_dashboard", + ) + + def test_device_relate_grafana_dashboards(self): + grafana_dashboard = GrafanaDashboard( + uid="first_dashboard", dashboard=SIMPLE_GRAFANA_DASHBOARD + ) + grafana_dashboard.save() + device = Device( + uid="hello-123", creation_date=timezone.now(), address="127.0.0.1" + ) + device.save() + device.grafana_dashboards.add(grafana_dashboard) + self.assertEqual( + device.grafana_dashboards.all()[0].uid, + "first_dashboard", + ) + self.assertEqual( + device.grafana_dashboards.all()[0].dashboard, + SIMPLE_GRAFANA_DASHBOARD, + ) + + def test_device_uid_uniqueness(self): + uid = "123" + Device(uid=uid, address="127.0.0.1").save() + + self.assertRaises( + IntegrityError, + Device(uid=uid, address="192.168.0.1").save, + ) + def create_device(uid, address): return Device.objects.create(uid=uid, address=address) @@ -115,7 +161,7 @@ def test_listed_device(self): self.assertContains( response, f"Device {device.uid} with ip {device.address}, was created on the" - f" {device.creation_date.strftime('%b. %d, %Y, %-I')}", + f" {device.creation_date.strftime('%B %d, %Y, %-I')}", ) self.assertContains( response, self.base_url + "/cos-grafana/f/" + device.uid + "/" diff --git a/cos_registration_server/devices/urls.py b/cos_registration_server/devices/urls.py index beaab06..a01502f 100644 --- a/cos_registration_server/devices/urls.py +++ b/cos_registration_server/devices/urls.py @@ -1,4 +1,5 @@ """Devices urls.""" + from django.urls import path from . import views diff --git a/cos_registration_server/devices/views.py b/cos_registration_server/devices/views.py index b5e0352..156a979 100644 --- a/cos_registration_server/devices/views.py +++ b/cos_registration_server/devices/views.py @@ -1,4 +1,5 @@ """Devices views.""" + from django.http import HttpResponse from django.shortcuts import render from django.views import generic From 312fe8825e80365fb9257efaf93eaa4c884ec20d Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Fri, 15 Mar 2024 17:42:17 +0100 Subject: [PATCH 2/2] fix(grafana_dashboards): remove grafana dashboards file generation The grafana dashboard file generation doesn't belong here. It belongs in the charm. --- .../api/grafana_dashboards.py | 51 ------------------- .../api/management/__init__.py | 1 - .../api/management/commands/__init__.py | 1 - .../commands/update_all_grafana_dashboards.py | 13 ----- 4 files changed, 66 deletions(-) delete mode 100644 cos_registration_server/api/grafana_dashboards.py delete mode 100644 cos_registration_server/api/management/__init__.py delete mode 100644 cos_registration_server/api/management/commands/__init__.py delete mode 100644 cos_registration_server/api/management/commands/update_all_grafana_dashboards.py diff --git a/cos_registration_server/api/grafana_dashboards.py b/cos_registration_server/api/grafana_dashboards.py deleted file mode 100644 index 353476f..0000000 --- a/cos_registration_server/api/grafana_dashboards.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Dashboards files management utilities.""" -import json -from os import mkdir, remove -from pathlib import Path -from shutil import rmtree - -from devices.models import Device -from django.conf import settings - - -def add_grafana_dashboards(device): - """Add Grafana dashboards of a device on disk. - - device: A device saved in the DB. - """ - dashboard_path = Path(settings.GRAFANA_DASHBOARD_PATH) - for dashboard in device.grafana_dashboards: - dashboard_title = dashboard["title"].replace(" ", "_") - dashboard["title"] = f'{device.uid}-{dashboard["title"]}' - dashboard_file = f"{device.uid}-{dashboard_title}.json" - with open(dashboard_path / dashboard_file, "w") as file: - json.dump(dashboard, file) - - -def delete_grafana_dashboards(device): - """Delete Grafana dashboards of a device on disk. - - device: A device saved in the DB. - """ - dashboard_path = Path(settings.GRAFANA_DASHBOARD_PATH) - - def _is_device_dashboard(p: Path) -> bool: - return p.is_file() and p.name.startswith(device.uid) - - for dashboard in filter( - _is_device_dashboard, Path(dashboard_path).glob("*") - ): - remove(dashboard) - - -def update_all_grafana_dashboards(): - """Update the Grafana dashboards of all devices. - - This makes sure all the dashboards stored in the DB - and only them are written on the disk. - """ - dashboard_path = Path(settings.GRAFANA_DASHBOARD_PATH) - rmtree(dashboard_path, ignore_errors=True) - mkdir(dashboard_path) - for device in Device.objects.all(): - add_grafana_dashboards(device) diff --git a/cos_registration_server/api/management/__init__.py b/cos_registration_server/api/management/__init__.py deleted file mode 100644 index 271f13e..0000000 --- a/cos_registration_server/api/management/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""API managment.""" diff --git a/cos_registration_server/api/management/commands/__init__.py b/cos_registration_server/api/management/commands/__init__.py deleted file mode 100644 index 948e4db..0000000 --- a/cos_registration_server/api/management/commands/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Additional API app django admin commands.""" diff --git a/cos_registration_server/api/management/commands/update_all_grafana_dashboards.py b/cos_registration_server/api/management/commands/update_all_grafana_dashboards.py deleted file mode 100644 index 5af92da..0000000 --- a/cos_registration_server/api/management/commands/update_all_grafana_dashboards.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Update all dashboards django admin command.""" -from api.grafana_dashboards import update_all_grafana_dashboards -from django.core.management.base import BaseCommand - - -class Command(BaseCommand): - """Update all Grafana dashboards command class.""" - - help = "Update all the grafana dashboards in the folder" - - def handle(self, *args, **options): - """Handle the call to update_all_grafana_dashboards command.""" - update_all_grafana_dashboards()