From 463b9859e8205babbe04a0a09c43051cca142e8c Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Thu, 11 Jan 2024 17:35:32 +0100 Subject: [PATCH 01/49] feat(requirement): add django --- requirements.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index b05f2a6..d3e4ba5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1 @@ -# This template is a low-dependency template. -# By default there is no requirements added here. -# Add the requirements you need to this file. -# or run `make init` to create this file automatically based on the template. -# You can also run `make switch-to-poetry` to use the poetry package manager. +django From 8e280ad7d531149fdab1f0d12f4b17cb0bac2f18 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Thu, 11 Jan 2024 17:40:06 +0100 Subject: [PATCH 02/49] feat(django): project creation --- cos_registration_server/__main__.py | 6 - cos_registration_server/base.py | 17 --- cos_registration_server/cli.py | 28 ---- .../{ => cos_registration_server}/VERSION | 0 .../{ => cos_registration_server}/__init__.py | 0 .../cos_registration_server/asgi.py | 16 +++ .../cos_registration_server/settings.py | 123 ++++++++++++++++++ .../cos_registration_server/urls.py | 22 ++++ .../cos_registration_server/wsgi.py | 16 +++ cos_registration_server/manage.py | 22 ++++ setup.py | 5 +- 11 files changed, 200 insertions(+), 55 deletions(-) delete mode 100644 cos_registration_server/__main__.py delete mode 100644 cos_registration_server/base.py delete mode 100644 cos_registration_server/cli.py rename cos_registration_server/{ => cos_registration_server}/VERSION (100%) rename cos_registration_server/{ => cos_registration_server}/__init__.py (100%) create mode 100644 cos_registration_server/cos_registration_server/asgi.py create mode 100644 cos_registration_server/cos_registration_server/settings.py create mode 100644 cos_registration_server/cos_registration_server/urls.py create mode 100644 cos_registration_server/cos_registration_server/wsgi.py create mode 100755 cos_registration_server/manage.py diff --git a/cos_registration_server/__main__.py b/cos_registration_server/__main__.py deleted file mode 100644 index 870a349..0000000 --- a/cos_registration_server/__main__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Entry point for cos_registration_server.""" - -from cos_registration_server.cli import main # pragma: no cover - -if __name__ == "__main__": # pragma: no cover - main() diff --git a/cos_registration_server/base.py b/cos_registration_server/base.py deleted file mode 100644 index cdeb245..0000000 --- a/cos_registration_server/base.py +++ /dev/null @@ -1,17 +0,0 @@ -""" -cos_registration_server base module. - -This is the principal module of the cos_registration_server project. -here you put your main classes and objects. - -Be creative! do whatever you want! - -If you want to replace this with a Flask application run: - - $ make init - -and then choose `flask` as template. -""" - -# example constant variable -NAME = "cos_registration_server" diff --git a/cos_registration_server/cli.py b/cos_registration_server/cli.py deleted file mode 100644 index 5156550..0000000 --- a/cos_registration_server/cli.py +++ /dev/null @@ -1,28 +0,0 @@ -"""CLI interface for cos_registration_server project. - -Be creative! do whatever you want! - -- Install click or typer and create a CLI app -- Use builtin argparse -- Start a web application -- Import things from your .base module -""" - - -def main(): # pragma: no cover - """ - The main function executes on commands: - `python -m cos_registration_server` and `$ cos_registration_server `. - - This is your program's entry point. - - You can change this function to do whatever you want. - Examples: - * Run a test suite - * Run a server - * Do some other stuff - * Run a command line application (Click, Typer, ArgParse) - * List all available tasks - * Run an application (Flask, FastAPI, Django, etc.) - """ - print("This will do something") diff --git a/cos_registration_server/VERSION b/cos_registration_server/cos_registration_server/VERSION similarity index 100% rename from cos_registration_server/VERSION rename to cos_registration_server/cos_registration_server/VERSION diff --git a/cos_registration_server/__init__.py b/cos_registration_server/cos_registration_server/__init__.py similarity index 100% rename from cos_registration_server/__init__.py rename to cos_registration_server/cos_registration_server/__init__.py diff --git a/cos_registration_server/cos_registration_server/asgi.py b/cos_registration_server/cos_registration_server/asgi.py new file mode 100644 index 0000000..68eef8b --- /dev/null +++ b/cos_registration_server/cos_registration_server/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for cos_registration_server project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cos_registration_server.settings") + +application = get_asgi_application() diff --git a/cos_registration_server/cos_registration_server/settings.py b/cos_registration_server/cos_registration_server/settings.py new file mode 100644 index 0000000..5faa0f0 --- /dev/null +++ b/cos_registration_server/cos_registration_server/settings.py @@ -0,0 +1,123 @@ +""" +Django settings for cos_registration_server project. + +Generated by 'django-admin startproject' using Django 4.2.9. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/4.2/ref/settings/ +""" + +from pathlib import Path + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = "django-insecure-$b#5b(2h9_%p439=#ev0!dkde9wqt=rgoc!jvi-y93^@+wcvw8" + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", +] + +MIDDLEWARE = [ + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", +] + +ROOT_URLCONF = "cos_registration_server.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + +WSGI_APPLICATION = "cos_registration_server.wsgi.application" + + +# Database +# https://docs.djangoproject.com/en/4.2/ref/settings/#databases + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": BASE_DIR / "db.sqlite3", + } +} + + +# Password validation +# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/4.2/topics/i18n/ + +LANGUAGE_CODE = "en-us" + +TIME_ZONE = "UTC" + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/4.2/howto/static-files/ + +STATIC_URL = "static/" + +# Default primary key field type +# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" diff --git a/cos_registration_server/cos_registration_server/urls.py b/cos_registration_server/cos_registration_server/urls.py new file mode 100644 index 0000000..be0be84 --- /dev/null +++ b/cos_registration_server/cos_registration_server/urls.py @@ -0,0 +1,22 @@ +""" +URL configuration for cos_registration_server project. + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/4.2/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path + +urlpatterns = [ + path("admin/", admin.site.urls), +] diff --git a/cos_registration_server/cos_registration_server/wsgi.py b/cos_registration_server/cos_registration_server/wsgi.py new file mode 100644 index 0000000..536a730 --- /dev/null +++ b/cos_registration_server/cos_registration_server/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for cos_registration_server project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cos_registration_server.settings") + +application = get_wsgi_application() diff --git a/cos_registration_server/manage.py b/cos_registration_server/manage.py new file mode 100755 index 0000000..b3451be --- /dev/null +++ b/cos_registration_server/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cos_registration_server.settings") + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index ec2fdfa..656e8f7 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ def read_requirements(path): setup( name="cos_registration_server", - version=read("cos_registration_server", "VERSION"), + version=read("cos_registration_server/cos_registration_server", "VERSION"), description="Awesome cos_registration_server created by ubuntu-robotics", url="https://github.com/ubuntu-robotics/cos-registration-server/", long_description=read("README.md"), @@ -39,8 +39,5 @@ def read_requirements(path): author="ubuntu-robotics", packages=find_packages(exclude=["tests", ".github"]), install_requires=read_requirements("requirements.txt"), - entry_points={ - "console_scripts": ["cos-registration-server = cos_registration_server.__main__:main"] - }, extras_require={"test": read_requirements("requirements-test.txt")}, ) From 73336ec5122fc4d86facdd765a80a18b29fb22c7 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Thu, 11 Jan 2024 17:46:31 +0100 Subject: [PATCH 03/49] feat(registration): registration app --- cos_registration_server/registration/__init__.py | 0 cos_registration_server/registration/admin.py | 3 +++ cos_registration_server/registration/apps.py | 6 ++++++ cos_registration_server/registration/migrations/__init__.py | 0 cos_registration_server/registration/models.py | 3 +++ cos_registration_server/registration/tests.py | 3 +++ cos_registration_server/registration/views.py | 3 +++ 7 files changed, 18 insertions(+) create mode 100644 cos_registration_server/registration/__init__.py create mode 100644 cos_registration_server/registration/admin.py create mode 100644 cos_registration_server/registration/apps.py create mode 100644 cos_registration_server/registration/migrations/__init__.py create mode 100644 cos_registration_server/registration/models.py create mode 100644 cos_registration_server/registration/tests.py create mode 100644 cos_registration_server/registration/views.py diff --git a/cos_registration_server/registration/__init__.py b/cos_registration_server/registration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cos_registration_server/registration/admin.py b/cos_registration_server/registration/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/cos_registration_server/registration/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/cos_registration_server/registration/apps.py b/cos_registration_server/registration/apps.py new file mode 100644 index 0000000..7c2d6f4 --- /dev/null +++ b/cos_registration_server/registration/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class RegistrationConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "registration" diff --git a/cos_registration_server/registration/migrations/__init__.py b/cos_registration_server/registration/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cos_registration_server/registration/models.py b/cos_registration_server/registration/models.py new file mode 100644 index 0000000..71a8362 --- /dev/null +++ b/cos_registration_server/registration/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/cos_registration_server/registration/tests.py b/cos_registration_server/registration/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/cos_registration_server/registration/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/cos_registration_server/registration/views.py b/cos_registration_server/registration/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/cos_registration_server/registration/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. From c486c7b8655e1dbeee11e0ce245a404cd96ff3fc Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Mon, 15 Jan 2024 14:49:06 +0100 Subject: [PATCH 04/49] feat(rename registration into device) --- .../cos_registration_server/settings.py | 1 + .../cos_registration_server/urls.py | 3 ++- .../{registration => devices}/__init__.py | 0 cos_registration_server/devices/admin.py | 5 +++++ .../{registration => devices}/apps.py | 4 ++-- .../{registration => devices}/migrations/__init__.py | 0 cos_registration_server/devices/models.py | 10 ++++++++++ .../{registration => devices}/tests.py | 0 cos_registration_server/devices/urls.py | 11 +++++++++++ cos_registration_server/registration/admin.py | 3 --- cos_registration_server/registration/models.py | 3 --- cos_registration_server/registration/views.py | 3 --- 12 files changed, 31 insertions(+), 12 deletions(-) rename cos_registration_server/{registration => devices}/__init__.py (100%) create mode 100644 cos_registration_server/devices/admin.py rename cos_registration_server/{registration => devices}/apps.py (59%) rename cos_registration_server/{registration => devices}/migrations/__init__.py (100%) create mode 100644 cos_registration_server/devices/models.py rename cos_registration_server/{registration => devices}/tests.py (100%) create mode 100644 cos_registration_server/devices/urls.py delete mode 100644 cos_registration_server/registration/admin.py delete mode 100644 cos_registration_server/registration/models.py delete mode 100644 cos_registration_server/registration/views.py diff --git a/cos_registration_server/cos_registration_server/settings.py b/cos_registration_server/cos_registration_server/settings.py index 5faa0f0..09e6515 100644 --- a/cos_registration_server/cos_registration_server/settings.py +++ b/cos_registration_server/cos_registration_server/settings.py @@ -31,6 +31,7 @@ # Application definition INSTALLED_APPS = [ + "devices.apps.DevicesConfig", "django.contrib.admin", "django.contrib.auth", "django.contrib.contenttypes", diff --git a/cos_registration_server/cos_registration_server/urls.py b/cos_registration_server/cos_registration_server/urls.py index be0be84..ca43891 100644 --- a/cos_registration_server/cos_registration_server/urls.py +++ b/cos_registration_server/cos_registration_server/urls.py @@ -15,8 +15,9 @@ 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ from django.contrib import admin -from django.urls import path +from django.urls import include, path urlpatterns = [ + path("devices/", include("devices.urls")), path("admin/", admin.site.urls), ] diff --git a/cos_registration_server/registration/__init__.py b/cos_registration_server/devices/__init__.py similarity index 100% rename from cos_registration_server/registration/__init__.py rename to cos_registration_server/devices/__init__.py diff --git a/cos_registration_server/devices/admin.py b/cos_registration_server/devices/admin.py new file mode 100644 index 0000000..cedeffc --- /dev/null +++ b/cos_registration_server/devices/admin.py @@ -0,0 +1,5 @@ +from django.contrib import admin + +from .models import Device + +admin.site.register(Device) diff --git a/cos_registration_server/registration/apps.py b/cos_registration_server/devices/apps.py similarity index 59% rename from cos_registration_server/registration/apps.py rename to cos_registration_server/devices/apps.py index 7c2d6f4..b06641c 100644 --- a/cos_registration_server/registration/apps.py +++ b/cos_registration_server/devices/apps.py @@ -1,6 +1,6 @@ from django.apps import AppConfig -class RegistrationConfig(AppConfig): +class DevicesConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" - name = "registration" + name = "devices" diff --git a/cos_registration_server/registration/migrations/__init__.py b/cos_registration_server/devices/migrations/__init__.py similarity index 100% rename from cos_registration_server/registration/migrations/__init__.py rename to cos_registration_server/devices/migrations/__init__.py diff --git a/cos_registration_server/devices/models.py b/cos_registration_server/devices/models.py new file mode 100644 index 0000000..8de0a93 --- /dev/null +++ b/cos_registration_server/devices/models.py @@ -0,0 +1,10 @@ +from django.db import models + +class Device(models.Model): + + uid = models.CharField(max_length=200) + creation_date = models.DateTimeField("creation date", auto_now_add=True) + address = models.GenericIPAddressField("device IP") + + def __str__(self): + return self.uid diff --git a/cos_registration_server/registration/tests.py b/cos_registration_server/devices/tests.py similarity index 100% rename from cos_registration_server/registration/tests.py rename to cos_registration_server/devices/tests.py diff --git a/cos_registration_server/devices/urls.py b/cos_registration_server/devices/urls.py new file mode 100644 index 0000000..1589bd3 --- /dev/null +++ b/cos_registration_server/devices/urls.py @@ -0,0 +1,11 @@ +from django.urls import path + +from . import views + + +app_name = "devices" + +urlpatterns = [ + path("", views.devices.as_view(), name="devices"), + path("", views.device, name="device"), +] diff --git a/cos_registration_server/registration/admin.py b/cos_registration_server/registration/admin.py deleted file mode 100644 index 8c38f3f..0000000 --- a/cos_registration_server/registration/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/cos_registration_server/registration/models.py b/cos_registration_server/registration/models.py deleted file mode 100644 index 71a8362..0000000 --- a/cos_registration_server/registration/models.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.db import models - -# Create your models here. diff --git a/cos_registration_server/registration/views.py b/cos_registration_server/registration/views.py deleted file mode 100644 index 91ea44a..0000000 --- a/cos_registration_server/registration/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. From 6b84ba162aafd599ea44fd49797a1b1f93603ef8 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Mon, 15 Jan 2024 14:52:28 +0100 Subject: [PATCH 05/49] feat(devices): devices views --- .../devices/templates/devices/device.html | 6 ++++ .../devices/templates/devices/devices.html | 10 +++++++ cos_registration_server/devices/views.py | 29 +++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 cos_registration_server/devices/templates/devices/device.html create mode 100644 cos_registration_server/devices/templates/devices/devices.html create mode 100644 cos_registration_server/devices/views.py diff --git a/cos_registration_server/devices/templates/devices/device.html b/cos_registration_server/devices/templates/devices/device.html new file mode 100644 index 0000000..c8f3bd1 --- /dev/null +++ b/cos_registration_server/devices/templates/devices/device.html @@ -0,0 +1,6 @@ +

Device {{device.uid}} with ip {{device.address}}, was created on the {{device.creation_date}}

+
    +{% for name, link in links_dict.items %} +
  • {{ name }}
    • +{% endfor %} +
diff --git a/cos_registration_server/devices/templates/devices/devices.html b/cos_registration_server/devices/templates/devices/devices.html new file mode 100644 index 0000000..cb37768 --- /dev/null +++ b/cos_registration_server/devices/templates/devices/devices.html @@ -0,0 +1,10 @@ +{% if devices_list %} +

Devices list:

+ +{% else %} +

No devices are available.

+{% endif %} diff --git a/cos_registration_server/devices/views.py b/cos_registration_server/devices/views.py new file mode 100644 index 0000000..4e23b1e --- /dev/null +++ b/cos_registration_server/devices/views.py @@ -0,0 +1,29 @@ +from django.shortcuts import render +from django.http import HttpResponse +from django.template import loader +from django.views import generic + +from .models import Device + +class devices(generic.ListView): + template_name = "devices/devices.html" + context_object_name = "devices_list" + + def get_queryset(self): + return Device.objects.all() + +def device(request, uid): + try: + device = Device.objects.get(uid=uid) + except Device.DoesNotExist: + return HttpResponse(f"devices not found") + base_url = request.META['HTTP_HOST'] + grafana_folder = base_url + '/cos-grafana/f/' + uid + '/' + foxglove = base_url + '/cos-foxglove-studio/?ds=foxglove-websocket&ds.url=ws%3A%2F%2F' + device.address + '%3A8765/' + bag_files = base_url + '/cos-ros2bag-fileserver/' + uid + '/' + links = {'grafana folder': grafana_folder, 'foxglove': foxglove, 'bag_files': bag_files} + context = { + "device": device, + "links_dict": links, + } + return render(request, 'devices/device.html', context) From f3b9a70e4f9632d89b9af3cf4888f782f7281ceb Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Mon, 15 Jan 2024 14:54:02 +0100 Subject: [PATCH 06/49] feat(API): add API --- cos_registration_server/api/__init__.py | 0 cos_registration_server/api/admin.py | 3 ++ cos_registration_server/api/apps.py | 6 +++ .../api/migrations/__init__.py | 0 cos_registration_server/api/models.py | 0 cos_registration_server/api/serializer.py | 16 +++++++ cos_registration_server/api/tests.py | 3 ++ cos_registration_server/api/urls.py | 9 ++++ cos_registration_server/api/views.py | 44 +++++++++++++++++++ .../cos_registration_server/settings.py | 2 + .../cos_registration_server/urls.py | 1 + requirements.txt | 1 + 12 files changed, 85 insertions(+) create mode 100644 cos_registration_server/api/__init__.py create mode 100644 cos_registration_server/api/admin.py create mode 100644 cos_registration_server/api/apps.py create mode 100644 cos_registration_server/api/migrations/__init__.py create mode 100644 cos_registration_server/api/models.py create mode 100644 cos_registration_server/api/serializer.py create mode 100644 cos_registration_server/api/tests.py create mode 100644 cos_registration_server/api/urls.py create mode 100644 cos_registration_server/api/views.py diff --git a/cos_registration_server/api/__init__.py b/cos_registration_server/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cos_registration_server/api/admin.py b/cos_registration_server/api/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/cos_registration_server/api/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/cos_registration_server/api/apps.py b/cos_registration_server/api/apps.py new file mode 100644 index 0000000..878e7d5 --- /dev/null +++ b/cos_registration_server/api/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ApiConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "api" diff --git a/cos_registration_server/api/migrations/__init__.py b/cos_registration_server/api/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cos_registration_server/api/models.py b/cos_registration_server/api/models.py new file mode 100644 index 0000000..e69de29 diff --git a/cos_registration_server/api/serializer.py b/cos_registration_server/api/serializer.py new file mode 100644 index 0000000..909c2b2 --- /dev/null +++ b/cos_registration_server/api/serializer.py @@ -0,0 +1,16 @@ +from rest_framework import serializers +from devices.models import Device + + +class DeviceSerializer(serializers.Serializer): + uid = serializers.CharField(required=True) + creation_date = serializers.DateTimeField(read_only=True) + address = serializers.IPAddressField(required=True) + + def create(self, validated_data): + return Device.objects.create(**validated_data) + + def update(self, instance, validated_data): + instance.address = validated_data.get('address', instance.address) + instance.save() + return instance diff --git a/cos_registration_server/api/tests.py b/cos_registration_server/api/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/cos_registration_server/api/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/cos_registration_server/api/urls.py b/cos_registration_server/api/urls.py new file mode 100644 index 0000000..c69fd9b --- /dev/null +++ b/cos_registration_server/api/urls.py @@ -0,0 +1,9 @@ +from django.urls import path + +from . import views + +urlpatterns = [ + # make a version API url + path("v1/devices", views.devices, name="devices"), + path("v1/devices/", views.device, name="device"), +] diff --git a/cos_registration_server/api/views.py b/cos_registration_server/api/views.py new file mode 100644 index 0000000..cff4783 --- /dev/null +++ b/cos_registration_server/api/views.py @@ -0,0 +1,44 @@ +from django.http import HttpResponse, JsonResponse +from django.views.decorators.csrf import csrf_exempt +from rest_framework.parsers import JSONParser + +from devices.models import Device +from api.serializer import DeviceSerializer + +@csrf_exempt +def devices(request): + if request.method == 'GET': + devices = Device.objects.all() + serialized = DeviceSerializer(devices, many=True) + return JsonResponse(serialized.data, safe=False) + elif request.method == 'POST': + data = JSONParser().parse(request) + serialized = DeviceSerializer(data=data) + if serialized.is_valid(): + print(f"HEEEEERRRRRR: {serialized.validated_data['uid']}") + if Device.objects.filter(uid=serialized.validated_data['uid']).exists(): + return JsonResponse({"error": "Device uid already exists"}, status=409) + serialized.save() + return JsonResponse(serialized.data, status=201) + return JsonResponse(serialized.errors, status=400) + +@csrf_exempt +def device(request, uid): + try: + device = Device.objects.get(uid=uid) + except Device.DoesNotExist: + return HttpResponse(status=404) + + if request.method == 'GET': + serialized = DeviceSerializer(device) + return JsonResponse(serialized.data) + if request.method == 'PATCH': + data = JSONParser().parse(request) + serialized = DeviceSerializer(device, data=data, partial=True) + if serialized.is_valid(): + serialized.save() + return JsonResponse(serialized.data) + return JsonResponse(serialized.errors, status=400) + elif request.method == 'DELETE': + device.delete() + return HttpResponse(status=204) diff --git a/cos_registration_server/cos_registration_server/settings.py b/cos_registration_server/cos_registration_server/settings.py index 09e6515..71178f5 100644 --- a/cos_registration_server/cos_registration_server/settings.py +++ b/cos_registration_server/cos_registration_server/settings.py @@ -31,6 +31,7 @@ # Application definition INSTALLED_APPS = [ + "api.apps.ApiConfig", "devices.apps.DevicesConfig", "django.contrib.admin", "django.contrib.auth", @@ -38,6 +39,7 @@ "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", + "rest_framework", ] MIDDLEWARE = [ diff --git a/cos_registration_server/cos_registration_server/urls.py b/cos_registration_server/cos_registration_server/urls.py index ca43891..f0bb4fc 100644 --- a/cos_registration_server/cos_registration_server/urls.py +++ b/cos_registration_server/cos_registration_server/urls.py @@ -18,6 +18,7 @@ from django.urls import include, path urlpatterns = [ + path("api/", include("api.urls")), path("devices/", include("devices.urls")), path("admin/", admin.site.urls), ] diff --git a/requirements.txt b/requirements.txt index d3e4ba5..6e5370e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ django +djangorestframework From da559a0127974db3dfe2a66ebfb22366302b304a Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Mon, 15 Jan 2024 14:59:08 +0100 Subject: [PATCH 07/49] fix(formating): apply black --- .../cos_registration_server/asgi.py | 4 ++- .../cos_registration_server/settings.py | 4 ++- .../cos_registration_server/wsgi.py | 4 ++- cos_registration_server/devices/models.py | 2 +- cos_registration_server/devices/urls.py | 1 - cos_registration_server/devices/views.py | 25 +++++++++++++------ cos_registration_server/manage.py | 4 ++- 7 files changed, 31 insertions(+), 13 deletions(-) diff --git a/cos_registration_server/cos_registration_server/asgi.py b/cos_registration_server/cos_registration_server/asgi.py index 68eef8b..a96de7c 100644 --- a/cos_registration_server/cos_registration_server/asgi.py +++ b/cos_registration_server/cos_registration_server/asgi.py @@ -11,6 +11,8 @@ from django.core.asgi import get_asgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cos_registration_server.settings") +os.environ.setdefault( + "DJANGO_SETTINGS_MODULE", "cos_registration_server.settings" +) application = get_asgi_application() diff --git a/cos_registration_server/cos_registration_server/settings.py b/cos_registration_server/cos_registration_server/settings.py index 71178f5..5fb74ba 100644 --- a/cos_registration_server/cos_registration_server/settings.py +++ b/cos_registration_server/cos_registration_server/settings.py @@ -20,7 +20,9 @@ # See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = "django-insecure-$b#5b(2h9_%p439=#ev0!dkde9wqt=rgoc!jvi-y93^@+wcvw8" +SECRET_KEY = ( + "django-insecure-$b#5b(2h9_%p439=#ev0!dkde9wqt=rgoc!jvi-y93^@+wcvw8" +) # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True diff --git a/cos_registration_server/cos_registration_server/wsgi.py b/cos_registration_server/cos_registration_server/wsgi.py index 536a730..794e519 100644 --- a/cos_registration_server/cos_registration_server/wsgi.py +++ b/cos_registration_server/cos_registration_server/wsgi.py @@ -11,6 +11,8 @@ from django.core.wsgi import get_wsgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cos_registration_server.settings") +os.environ.setdefault( + "DJANGO_SETTINGS_MODULE", "cos_registration_server.settings" +) application = get_wsgi_application() diff --git a/cos_registration_server/devices/models.py b/cos_registration_server/devices/models.py index 8de0a93..ba016e7 100644 --- a/cos_registration_server/devices/models.py +++ b/cos_registration_server/devices/models.py @@ -1,7 +1,7 @@ from django.db import models -class Device(models.Model): +class Device(models.Model): uid = models.CharField(max_length=200) creation_date = models.DateTimeField("creation date", auto_now_add=True) address = models.GenericIPAddressField("device IP") diff --git a/cos_registration_server/devices/urls.py b/cos_registration_server/devices/urls.py index 1589bd3..b6efc6a 100644 --- a/cos_registration_server/devices/urls.py +++ b/cos_registration_server/devices/urls.py @@ -2,7 +2,6 @@ from . import views - app_name = "devices" urlpatterns = [ diff --git a/cos_registration_server/devices/views.py b/cos_registration_server/devices/views.py index 4e23b1e..f873a10 100644 --- a/cos_registration_server/devices/views.py +++ b/cos_registration_server/devices/views.py @@ -1,10 +1,11 @@ -from django.shortcuts import render from django.http import HttpResponse +from django.shortcuts import render from django.template import loader from django.views import generic from .models import Device + class devices(generic.ListView): template_name = "devices/devices.html" context_object_name = "devices_list" @@ -12,18 +13,28 @@ class devices(generic.ListView): def get_queryset(self): return Device.objects.all() + def device(request, uid): try: device = Device.objects.get(uid=uid) except Device.DoesNotExist: return HttpResponse(f"devices not found") - base_url = request.META['HTTP_HOST'] - grafana_folder = base_url + '/cos-grafana/f/' + uid + '/' - foxglove = base_url + '/cos-foxglove-studio/?ds=foxglove-websocket&ds.url=ws%3A%2F%2F' + device.address + '%3A8765/' - bag_files = base_url + '/cos-ros2bag-fileserver/' + uid + '/' - links = {'grafana folder': grafana_folder, 'foxglove': foxglove, 'bag_files': bag_files} + base_url = request.META["HTTP_HOST"] + grafana_folder = base_url + "/cos-grafana/f/" + uid + "/" + foxglove = ( + base_url + + "/cos-foxglove-studio/?ds=foxglove-websocket&ds.url=ws%3A%2F%2F" + + device.address + + "%3A8765/" + ) + bag_files = base_url + "/cos-ros2bag-fileserver/" + uid + "/" + links = { + "grafana folder": grafana_folder, + "foxglove": foxglove, + "bag_files": bag_files, + } context = { "device": device, "links_dict": links, } - return render(request, 'devices/device.html', context) + return render(request, "devices/device.html", context) diff --git a/cos_registration_server/manage.py b/cos_registration_server/manage.py index b3451be..ee25aa0 100755 --- a/cos_registration_server/manage.py +++ b/cos_registration_server/manage.py @@ -6,7 +6,9 @@ def main(): """Run administrative tasks.""" - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cos_registration_server.settings") + os.environ.setdefault( + "DJANGO_SETTINGS_MODULE", "cos_registration_server.settings" + ) try: from django.core.management import execute_from_command_line except ImportError as exc: From a5228934f43bc38ad852beebd5985a708691743e Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Mon, 15 Jan 2024 14:59:27 +0100 Subject: [PATCH 08/49] fix(api formating): apply black --- cos_registration_server/api/serializer.py | 4 ++-- cos_registration_server/api/views.py | 23 ++++++++++++++--------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/cos_registration_server/api/serializer.py b/cos_registration_server/api/serializer.py index 909c2b2..3dec7f8 100644 --- a/cos_registration_server/api/serializer.py +++ b/cos_registration_server/api/serializer.py @@ -1,5 +1,5 @@ -from rest_framework import serializers from devices.models import Device +from rest_framework import serializers class DeviceSerializer(serializers.Serializer): @@ -11,6 +11,6 @@ def create(self, validated_data): return Device.objects.create(**validated_data) def update(self, instance, validated_data): - instance.address = validated_data.get('address', instance.address) + instance.address = validated_data.get("address", instance.address) instance.save() return instance diff --git a/cos_registration_server/api/views.py b/cos_registration_server/api/views.py index cff4783..57abe15 100644 --- a/cos_registration_server/api/views.py +++ b/cos_registration_server/api/views.py @@ -1,27 +1,32 @@ +from api.serializer import DeviceSerializer +from devices.models import Device from django.http import HttpResponse, JsonResponse from django.views.decorators.csrf import csrf_exempt from rest_framework.parsers import JSONParser -from devices.models import Device -from api.serializer import DeviceSerializer @csrf_exempt def devices(request): - if request.method == 'GET': + if request.method == "GET": devices = Device.objects.all() serialized = DeviceSerializer(devices, many=True) return JsonResponse(serialized.data, safe=False) - elif request.method == 'POST': + elif request.method == "POST": data = JSONParser().parse(request) serialized = DeviceSerializer(data=data) if serialized.is_valid(): print(f"HEEEEERRRRRR: {serialized.validated_data['uid']}") - if Device.objects.filter(uid=serialized.validated_data['uid']).exists(): - return JsonResponse({"error": "Device uid already exists"}, status=409) + if Device.objects.filter( + uid=serialized.validated_data["uid"] + ).exists(): + return JsonResponse( + {"error": "Device uid already exists"}, status=409 + ) serialized.save() return JsonResponse(serialized.data, status=201) return JsonResponse(serialized.errors, status=400) + @csrf_exempt def device(request, uid): try: @@ -29,16 +34,16 @@ def device(request, uid): except Device.DoesNotExist: return HttpResponse(status=404) - if request.method == 'GET': + if request.method == "GET": serialized = DeviceSerializer(device) return JsonResponse(serialized.data) - if request.method == 'PATCH': + if request.method == "PATCH": data = JSONParser().parse(request) serialized = DeviceSerializer(device, data=data, partial=True) if serialized.is_valid(): serialized.save() return JsonResponse(serialized.data) return JsonResponse(serialized.errors, status=400) - elif request.method == 'DELETE': + elif request.method == "DELETE": device.delete() return HttpResponse(status=204) From 5a40d516df528b8738708c578e167cc40661e599 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Mon, 15 Jan 2024 15:05:07 +0100 Subject: [PATCH 09/49] fix(device): apply linter --- .../cos_registration_server/settings.py | 15 ++++++++++----- cos_registration_server/devices/views.py | 3 +-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/cos_registration_server/cos_registration_server/settings.py b/cos_registration_server/cos_registration_server/settings.py index 5fb74ba..6f2e66b 100644 --- a/cos_registration_server/cos_registration_server/settings.py +++ b/cos_registration_server/cos_registration_server/settings.py @@ -11,6 +11,7 @@ """ from pathlib import Path +from typing import List # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -27,7 +28,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [] +ALLOWED_HOSTS = List[str] # Application definition @@ -91,16 +92,20 @@ AUTH_PASSWORD_VALIDATORS = [ { - "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", + "NAME": "django.contrib.auth.password_validation.\ + UserAttributeSimilarityValidator", }, { - "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + "NAME": "django.contrib.auth.password_validation.\ + MinimumLengthValidator", }, { - "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + "NAME": "django.contrib.auth.password_validation.\ + CommonPasswordValidator", }, { - "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + "NAME": "django.contrib.auth.password_validation.\ + NumericPasswordValidator", }, ] diff --git a/cos_registration_server/devices/views.py b/cos_registration_server/devices/views.py index f873a10..3cc5ce2 100644 --- a/cos_registration_server/devices/views.py +++ b/cos_registration_server/devices/views.py @@ -1,6 +1,5 @@ from django.http import HttpResponse from django.shortcuts import render -from django.template import loader from django.views import generic from .models import Device @@ -18,7 +17,7 @@ def device(request, uid): try: device = Device.objects.get(uid=uid) except Device.DoesNotExist: - return HttpResponse(f"devices not found") + return HttpResponse(f"device {uid} not found") base_url = request.META["HTTP_HOST"] grafana_folder = base_url + "/cos-grafana/f/" + uid + "/" foxglove = ( From a62ea12d773101cd7355d81a6ca63a2c794d485a Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Mon, 15 Jan 2024 15:05:17 +0100 Subject: [PATCH 10/49] fix(API): apply linter --- cos_registration_server/api/admin.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cos_registration_server/api/admin.py b/cos_registration_server/api/admin.py index 8c38f3f..846f6b4 100644 --- a/cos_registration_server/api/admin.py +++ b/cos_registration_server/api/admin.py @@ -1,3 +1 @@ -from django.contrib import admin - # Register your models here. From b880be5272cfee7bac4d849c73775b5b7712720f Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Wed, 17 Jan 2024 12:27:25 +0100 Subject: [PATCH 11/49] test(devices): add model and view test --- .../cos_registration_server/settings.py | 2 +- cos_registration_server/devices/tests.py | 104 +++++++++++++++++- 2 files changed, 103 insertions(+), 3 deletions(-) diff --git a/cos_registration_server/cos_registration_server/settings.py b/cos_registration_server/cos_registration_server/settings.py index 6f2e66b..0a762ad 100644 --- a/cos_registration_server/cos_registration_server/settings.py +++ b/cos_registration_server/cos_registration_server/settings.py @@ -28,7 +28,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = List[str] +ALLOWED_HOSTS = ["*"] # Application definition diff --git a/cos_registration_server/devices/tests.py b/cos_registration_server/devices/tests.py index 7ce503c..042f74c 100644 --- a/cos_registration_server/devices/tests.py +++ b/cos_registration_server/devices/tests.py @@ -1,3 +1,103 @@ -from django.test import TestCase +from html import escape -# Create your tests here. +from django.test import Client, TestCase +from django.urls import reverse +from django.utils import timezone + +from .models import Device + + +class DeviceModelTests(TestCase): + def test_creation_of_a_device(self): + device = Device( + uid="hello-123", creation_date=timezone.now(), address="127.0.0.1" + ) + self.assertEqual(device.uid, "hello-123") + self.assertEqual(str(device.address), "127.0.0.1") + self.assertLessEqual(device.creation_date, timezone.now()) + self.assertGreater( + device.creation_date, timezone.now() - timezone.timedelta(hours=1) + ) + + def test_device_str(self): + device = Device( + uid="hello-123", creation_date=timezone.now(), address="127.0.0.1" + ) + self.assertEqual(str(device), "hello-123") + + +def create_device(uid, address): + return Device.objects.create(uid=uid, address=address) + + +class DevicesViewTests(TestCase): + def test_no_devices(self): + response = self.client.get(reverse("devices:devices")) + self.assertEqual(response.status_code, 200) + self.assertContains(response, "No devices are available.") + self.assertQuerySetEqual(response.context["devices_list"], []) + + def test_two_devices(self): + device_1 = create_device("robot-1", "192.168.0.1") + device_2 = create_device("robot-2", "192.168.0.2") + + response = self.client.get(reverse("devices:devices")) + self.assertEqual(response.status_code, 200) + self.assertContains(response, "Devices list:") + self.assertContains(response, "robot-1") + self.assertContains(response, "robot-2") + self.assertQuerySetEqual( + list(response.context["devices_list"]), [device_1, device_2] + ) + + def test_one_device_then_two(self): + device_1 = create_device("robot-1", "192.168.0.1") + + response = self.client.get(reverse("devices:devices")) + self.assertEqual(response.status_code, 200) + self.assertContains(response, "Devices list:") + self.assertQuerySetEqual(response.context["devices_list"], [device_1]) + + device_2 = create_device("robot-2", "192.168.0.2") + + response = self.client.get(reverse("devices:devices")) + self.assertEqual(response.status_code, 200) + self.assertContains(response, "Devices list:") + self.assertQuerySetEqual( + list(response.context["devices_list"]), [device_1, device_2] + ) + + +class DeviceViewTests(TestCase): + def setUp(self): + # custom client with META HTTP_HOST specified + self.base_url = "192.168.1.2:8080" + self.client = Client(HTTP_HOST=self.base_url) + + def test_unlisted_device(self): + url = reverse("devices:device", args=("future-robot",)) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertContains(response, "device future-robot not found") + + def test_listed_device(self): + device = create_device("robot-1", "192.168.0.23") + url = reverse("devices:device", args=(device.uid,)) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertContains( + response, + f"Device {device.uid} with ip {device.address}, was created on the" + f" {device.creation_date.strftime('%b. %d, %Y, %I')}", + ) + self.assertContains( + response, self.base_url + "/cos-grafana/f/" + device.uid + "/" + ) + self.assertContains( + response, + self.base_url + + "/cos-foxglove-studio/" + + escape("?ds=foxglove-websocket&ds.url=ws%3A%2F%2F") + + device.address + + "%3A8765/", + ) From 1912a091fd01890aac1059cd0d118367835598de Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Wed, 17 Jan 2024 15:39:41 +0100 Subject: [PATCH 12/49] test(api): add unit test for the API --- cos_registration_server/api/admin.py | 1 - cos_registration_server/api/models.py | 0 cos_registration_server/api/tests.py | 129 +++++++++++++++++++++++++- cos_registration_server/api/urls.py | 2 + cos_registration_server/api/views.py | 1 - 5 files changed, 129 insertions(+), 4 deletions(-) delete mode 100644 cos_registration_server/api/admin.py delete mode 100644 cos_registration_server/api/models.py diff --git a/cos_registration_server/api/admin.py b/cos_registration_server/api/admin.py deleted file mode 100644 index 846f6b4..0000000 --- a/cos_registration_server/api/admin.py +++ /dev/null @@ -1 +0,0 @@ -# Register your models here. diff --git a/cos_registration_server/api/models.py b/cos_registration_server/api/models.py deleted file mode 100644 index e69de29..0000000 diff --git a/cos_registration_server/api/tests.py b/cos_registration_server/api/tests.py index 7ce503c..743af40 100644 --- a/cos_registration_server/api/tests.py +++ b/cos_registration_server/api/tests.py @@ -1,3 +1,128 @@ -from django.test import TestCase +import json -# Create your tests here. +from devices.models import Device +from django.urls import reverse +from django.utils import timezone +from rest_framework.test import APITestCase + + +class DevicesViewTests(APITestCase): + def setUp(self): + self.url = reverse("api:devices") + + def create_device(self, uid, address): + data = {"uid": uid, "address": address} + return self.client.post(self.url, data, format="json") + + def test_get_nothing(self): + response = self.client.get(self.url) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(json.loads(response.content)), 0) + + def test_create_device(self): + uid = "robot-1" + address = "192.168.0.1" + response = self.create_device(uid, address) + self.assertEqual(response.status_code, 201) + self.assertEqual(Device.objects.count(), 1) + self.assertEqual(Device.objects.get().uid, uid) + self.assertEqual(Device.objects.get().address, address) + self.assertAlmostEqual( + Device.objects.get().creation_date, + timezone.now(), + delta=timezone.timedelta(seconds=10), + ) + + def test_create_multiple_devices(self): + devices = [ + {"uid": "robot-1", "address": "192.168.0.1"}, + {"uid": "robot-2", "address": "192.168.0.2"}, + {"uid": "robot-3", "address": "192.168.0.3"}, + ] + for device in devices: + self.create_device(device["uid"], device["address"]) + self.assertEqual(Device.objects.count(), 3) + + response = self.client.get(self.url) + self.assertEqual(response.status_code, 200) + content_json = json.loads(response.content) + self.assertEqual(len(content_json), 3) + for i, device in enumerate(content_json): + self.assertEqual(devices[i]["uid"], device["uid"]) + self.assertEqual(devices[i]["address"], device["address"]) + + def test_create_already_present_uid(self): + uid = "robot-1" + address = "192.168.0.1" + response = self.create_device(uid, address) + self.assertEqual(response.status_code, 201) + self.assertEqual(Device.objects.count(), 1) + # we try to create the same one + response = self.create_device(uid, address) + self.assertEqual(Device.objects.count(), 1) + self.assertContains( + response, "Device uid already exists", status_code=409 + ) + + +class DeviceViewTests(APITestCase): + def url(self, uid): + return reverse("api:device", args=(uid,)) + + def create_device(self, uid, address): + data = {"uid": uid, "address": address} + url = reverse("api:devices") + return self.client.post(url, data, format="json") + + def test_get_nonexistent_device(self): + response = self.client.get(self.url("future-robot")) + self.assertEqual(response.status_code, 404) + + def test_get_device(self): + uid = "robot-1" + address = "192.168.1.2" + self.create_device(uid, address) + response = self.client.get(self.url(uid)) + self.assertEqual(response.status_code, 200) + content_json = json.loads(response.content) + self.assertEqual(content_json["uid"], uid) + self.assertEqual(content_json["address"], address) + self.assertAlmostEqual( + timezone.datetime.fromisoformat( + content_json["creation_date"].replace("Z", "+00:00") + ), + timezone.now(), + delta=timezone.timedelta(seconds=10), + ) + + def test_patch_device(self): + uid = "robot-1" + address = "192.168.1.2" + self.create_device(uid, address) + address = "192.168.1.200" + data = {"address": address} + response = self.client.patch(self.url(uid), data, format="json") + self.assertEqual(response.status_code, 200) + content_json = json.loads(response.content) + self.assertEqual(content_json["uid"], uid) + self.assertEqual(content_json["address"], address) + + def test_invalid_patch_device(self): + uid = "robot-1" + address = "192.168.1.2" + self.create_device(uid, address) + address = "192.168.1" # invalid IP + data = {"address": address} + response = self.client.patch(self.url(uid), data, format="json") + self.assertEqual(response.status_code, 400) + + def test_delete_device(self): + uid = "robot-1" + address = "192.168.1.2" + self.create_device(uid, address) + response = self.client.get(self.url(uid)) + self.assertEqual(response.status_code, 200) + response = self.client.delete(self.url(uid)) + self.assertEqual(response.status_code, 204) + response = self.client.get(self.url(uid)) + self.assertEqual(response.status_code, 404) diff --git a/cos_registration_server/api/urls.py b/cos_registration_server/api/urls.py index c69fd9b..c7e0065 100644 --- a/cos_registration_server/api/urls.py +++ b/cos_registration_server/api/urls.py @@ -2,6 +2,8 @@ from . import views +app_name = "api" + urlpatterns = [ # make a version API url path("v1/devices", views.devices, name="devices"), diff --git a/cos_registration_server/api/views.py b/cos_registration_server/api/views.py index 57abe15..fc457b6 100644 --- a/cos_registration_server/api/views.py +++ b/cos_registration_server/api/views.py @@ -15,7 +15,6 @@ def devices(request): data = JSONParser().parse(request) serialized = DeviceSerializer(data=data) if serialized.is_valid(): - print(f"HEEEEERRRRRR: {serialized.validated_data['uid']}") if Device.objects.filter( uid=serialized.validated_data["uid"] ).exists(): From aabe6de0f558bcfc7ae3184ca3a325cb15493eff Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Wed, 17 Jan 2024 15:44:32 +0100 Subject: [PATCH 13/49] fix(devices-test): use 12h as decimal number --- cos_registration_server/devices/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cos_registration_server/devices/tests.py b/cos_registration_server/devices/tests.py index 042f74c..51b88ad 100644 --- a/cos_registration_server/devices/tests.py +++ b/cos_registration_server/devices/tests.py @@ -88,7 +88,7 @@ def test_listed_device(self): self.assertContains( response, f"Device {device.uid} with ip {device.address}, was created on the" - f" {device.creation_date.strftime('%b. %d, %Y, %I')}", + f" {device.creation_date.strftime('%b. %d, %Y, %-I')}", ) self.assertContains( response, self.base_url + "/cos-grafana/f/" + device.uid + "/" From 6515394abc99fcf1acc9fd5691be16468e99ba22 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Wed, 17 Jan 2024 16:47:47 +0100 Subject: [PATCH 14/49] test(makefile): adapt makefile test for django --- Makefile | 8 +------- .../cos_registration_server/settings.py | 1 - mypy.ini | 2 ++ setup.py | 2 +- tests/__init__.py | 0 tests/conftest.py | 14 -------------- tests/test_base.py | 5 ----- 7 files changed, 4 insertions(+), 28 deletions(-) create mode 100644 mypy.ini delete mode 100644 tests/__init__.py delete mode 100644 tests/conftest.py delete mode 100644 tests/test_base.py diff --git a/Makefile b/Makefile index 3c478d8..86dcd9a 100644 --- a/Makefile +++ b/Makefile @@ -27,25 +27,19 @@ install: ## Install the project in dev mode. fmt: ## Format code using black & isort. $(ENV_PREFIX)isort cos_registration_server/ $(ENV_PREFIX)black -l 79 cos_registration_server/ - $(ENV_PREFIX)black -l 79 tests/ .PHONY: lint lint: ## Run pep8, black, mypy linters. $(ENV_PREFIX)flake8 cos_registration_server/ $(ENV_PREFIX)black -l 79 --check cos_registration_server/ - $(ENV_PREFIX)black -l 79 --check tests/ $(ENV_PREFIX)mypy --ignore-missing-imports cos_registration_server/ .PHONY: test test: lint ## Run tests and generate coverage report. - $(ENV_PREFIX)pytest -v --cov-config .coveragerc --cov=cos_registration_server -l --tb=short --maxfail=1 tests/ + $(ENV_PREFIX)coverage run --source='.' cos_registration_server/manage.py test api devices $(ENV_PREFIX)coverage xml $(ENV_PREFIX)coverage html -.PHONY: watch -watch: ## Run tests on every change. - ls **/**.py | entr $(ENV_PREFIX)pytest -s -vvv -l --tb=long --maxfail=1 tests/ - .PHONY: clean clean: ## Clean unused files. @find ./ -name '*.pyc' -exec rm -f {} \; diff --git a/cos_registration_server/cos_registration_server/settings.py b/cos_registration_server/cos_registration_server/settings.py index 0a762ad..49c0263 100644 --- a/cos_registration_server/cos_registration_server/settings.py +++ b/cos_registration_server/cos_registration_server/settings.py @@ -11,7 +11,6 @@ """ from pathlib import Path -from typing import List # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..a4d9e5f --- /dev/null +++ b/mypy.ini @@ -0,0 +1,2 @@ +[mypy] +exclude = migrations/ diff --git a/setup.py b/setup.py index 656e8f7..1ca3171 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ def read_requirements(path): long_description=read("README.md"), long_description_content_type="text/markdown", author="ubuntu-robotics", - packages=find_packages(exclude=["tests", ".github"]), + packages=find_packages(exclude=[".github"]), install_requires=read_requirements("requirements.txt"), extras_require={"test": read_requirements("requirements-test.txt")}, ) diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 1cbb7b1..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,14 +0,0 @@ -import sys -import pytest - - -# each test runs on cwd to its temp dir -@pytest.fixture(autouse=True) -def go_to_tmpdir(request): - # Get the fixture dynamically by its name. - tmpdir = request.getfixturevalue("tmpdir") - # ensure local test created packages can be imported - sys.path.insert(0, str(tmpdir)) - # Chdir only for the duration of the test. - with tmpdir.as_cwd(): - yield diff --git a/tests/test_base.py b/tests/test_base.py deleted file mode 100644 index e8d003c..0000000 --- a/tests/test_base.py +++ /dev/null @@ -1,5 +0,0 @@ -from cos_registration_server.base import NAME - - -def test_base(): - assert NAME == "cos_registration_server" From 161947f95dad0d65d217804afc3df6903da6cda0 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Wed, 17 Jan 2024 16:51:01 +0100 Subject: [PATCH 15/49] ci(remove windows and mac) --- .github/workflows/main.yml | 40 -------------------------------------- 1 file changed, 40 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6cbcfd9..4910438 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -50,43 +50,3 @@ jobs: run: make test - name: "Upload coverage to Codecov" uses: codecov/codecov-action@v3 - # with: - # fail_ci_if_error: true - - tests_mac: - needs: linter - strategy: - fail-fast: false - matrix: - python-version: [3.9] - os: [macos-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install project - run: make install - - name: Run tests - run: make test - - tests_win: - needs: linter - strategy: - fail-fast: false - matrix: - python-version: [3.9] - os: [windows-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install Pip - run: pip install --user --upgrade pip - - name: Install project - run: pip install -e .[test] - - name: run tests - run: pytest -s -vvvv -l --tb=long tests From fd0dcb27dac431a8adebf13a704c47e983fadec1 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Wed, 17 Jan 2024 17:10:17 +0100 Subject: [PATCH 16/49] feat(migration): initial --- .../cos_registration_server/settings.py | 16 ++++---- .../devices/migrations/0001_initial.py | 37 +++++++++++++++++++ 2 files changed, 45 insertions(+), 8 deletions(-) create mode 100644 cos_registration_server/devices/migrations/0001_initial.py diff --git a/cos_registration_server/cos_registration_server/settings.py b/cos_registration_server/cos_registration_server/settings.py index 49c0263..cf2f93e 100644 --- a/cos_registration_server/cos_registration_server/settings.py +++ b/cos_registration_server/cos_registration_server/settings.py @@ -91,20 +91,20 @@ AUTH_PASSWORD_VALIDATORS = [ { - "NAME": "django.contrib.auth.password_validation.\ - UserAttributeSimilarityValidator", + "NAME": "django.contrib.auth.password_validation." + "UserAttributeSimilarityValidator", }, { - "NAME": "django.contrib.auth.password_validation.\ - MinimumLengthValidator", + "NAME": "django.contrib.auth.password_validation." + "MinimumLengthValidator", }, { - "NAME": "django.contrib.auth.password_validation.\ - CommonPasswordValidator", + "NAME": "django.contrib.auth.password_validation." + "CommonPasswordValidator", }, { - "NAME": "django.contrib.auth.password_validation.\ - NumericPasswordValidator", + "NAME": "django.contrib.auth.password_validation." + "NumericPasswordValidator", }, ] diff --git a/cos_registration_server/devices/migrations/0001_initial.py b/cos_registration_server/devices/migrations/0001_initial.py new file mode 100644 index 0000000..b388c51 --- /dev/null +++ b/cos_registration_server/devices/migrations/0001_initial.py @@ -0,0 +1,37 @@ +# Generated by Django 4.2.9 on 2024-01-17 15:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="Device", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("uid", models.CharField(max_length=200)), + ( + "creation_date", + models.DateTimeField( + auto_now_add=True, verbose_name="creation date" + ), + ), + ( + "address", + models.GenericIPAddressField(verbose_name="device IP"), + ), + ], + ), + ] From e53a68bcc4971ad06756c409d239dd057b91e11f Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Wed, 17 Jan 2024 18:36:49 +0100 Subject: [PATCH 17/49] doc(readme): add project info to the readme --- Makefile | 4 ++ README.md | 148 ++++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 136 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 86dcd9a..dc52ffa 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,10 @@ install: ## Install the project in dev mode. @echo "Don't forget to run 'make virtualenv' if you got errors." $(ENV_PREFIX)pip install -e .[test] +.PHONY: runserver +runserver: ## Django run server. + $(ENV_PREFIX)python3 cos_registration_server/manage.py runserver + .PHONY: fmt fmt: ## Format code using black & isort. $(ENV_PREFIX)isort cos_registration_server/ diff --git a/README.md b/README.md index b6cfd19..ffd177b 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,146 @@ # COS registration server +COS registration server is responsible for storing devices registered by the +COS registration agent. +The COS registration server is a django server consisting of a database, +an API as well as front end views. +With COS registration server a device can register itself and update +its informations. +A user can also visualize the registered devices as well as discovering +customized device applications. + [![codecov](https://codecov.io/gh/ubuntu-robotics/cos-registration-server/branch/main/graph/badge.svg?token=cos-registration-server_token_here)](https://codecov.io/gh/ubuntu-robotics/cos-registration-server) [![CI](https://github.com/ubuntu-robotics/cos-registration-server/actions/workflows/main.yml/badge.svg)](https://github.com/ubuntu-robotics/cos-registration-server/actions/workflows/main.yml) -## Install it from PyPI +## Features + +The COS registration server consist of two applications: Devices and API. + +### Devices + +This application is describing the database as well as offering views for +the user to visualize the list of devices and the specificities of one device. + +#### Device model +The Device model represent a device stored in the database. +It consists of: +- UID: Unique ID per device. Typically, the name of the instance of the serial number. +- Creation date: DateTime of the device creation in the server. +- Address: IP address or hostname of the device. + +#### View: `devices/` + +It provides a simple list of all the registered devices. +Every device listed provides a link to explore the specificities of a specific +device. + +#### View: `devices/` + +It provides a view of all the information registered in the database: +UID, creation date and address. +Additionally, this view provides a list of links of COS applications +personalised for the given device. +The COS applications personalised links are: +- Grafana: Pointing to the Grafana dashboard folder of the device. +- Foxglove: Pointing to the Foxglove app with the proper websocket connected. +- Bag files: Pointing to the file server directory storing the bag files of the +device. + +### API +The API can be used by the COS registration agent but also by any service +requiring to access the device database. + +#### Devices +
+ GET api/v1/devices (Get the list of all the devices) + +#### Parameters + +> None + +#### Responses + +> | http code | content-type | response | +> |---------------|-----------------------------------|---------------------------------------------------------------------| +> | `200` | `application/json` | List of devices. | +
+ +
+ POST api/v1/devices (Add a device to the database) + +##### Parameters + +> | name | type | data type | description | +> |-----------|-----------|-------------------------|-----------------------------------------------------------------------| +> | None | required | {"uid": "string", "address": "string"} | Unique ID and IP address of the device | + + +##### Responses + +> | http code | content-type | response | +> |---------------|-----------------------------------|---------------------------------------------------------------------| +> | `201` | `application/json` | {"uid": "string", "creation_date": "string", "address": "string"} | +> | `400` | `application/json` | {"field": "error details"} | +> | `409` | `application/json` | {"error": "Device uid already exists"} | +
+ +#### Device + +
+ GET api/v1/device/<str:uid> (Get the details of a device) + +#### Parameters + +> None + +#### Responses + +> | http code | content-type | response | +> |---------------|-----------------------------------|---------------------------------------------------------------------| +> | `200` | `application/json` | {"uid": "string", "creation_date": "string", "address": "string"} | +> | `404` | `text/html;charset=utf-8` | None | +
+ +
+ PATCH api/v1/device/<str:uid> (Modify the attribute of a device) + +##### Parameters + +> | name | type | data type | description | +> |-----------|-----------|-------------------------|-----------------------------------------------------------------------| +> | None | required | {"address": "string"} | Address to modify. | + + +##### Responses + +> | http code | content-type | response | +> |---------------|-----------------------------------|---------------------------------------------------------------------| +> | `201` | `application/json` | {"uid": "string", "creation_date": "string", "address": "string"} | +> | `400` | `application/json` | {"field": "error details"} | +> | `404` | `text/html;charset=utf-8` | None | +
+ +
+ DELETE api/v1/device/<str:uid> (Delete a device from the database) + +#### Parameters + +> None -```bash -pip install cos_registration_server -``` +#### Responses -## Usage +> | http code | content-type | response | +> |---------------|-----------------------------------|---------------------------------------------------------------------| +> | `204` | `text/html;charset=utf-8` | None | +> | `404` | `text/html;charset=utf-8` | None | +
-```py -from cos_registration_server import BaseClass -from cos_registration_server import base_function +## Installation -BaseClass().base_method() -base_function() -``` +`make install` -```bash -$ python -m cos_registration_server -#or -$ cos-registration-server -``` +`make runserver` ## Development From 196ad060102220772efb0990b7f76799f0be8f16 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Thu, 18 Jan 2024 17:21:03 +0100 Subject: [PATCH 18/49] feat(secret_key): don't use hardcoded secret key --- .github/workflows/main.yml | 2 ++ Makefile | 5 +++++ README.md | 7 +++++++ .../cos_registration_server/settings.py | 13 +++++-------- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4910438..f1af244 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,6 +28,8 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install project run: make install + - name: Generate django secret key + run: export SECRET_KEY_DJANGO=$(make secretkey) - name: Run linter run: make lint diff --git a/Makefile b/Makefile index dc52ffa..25ff105 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,11 @@ install: ## Install the project in dev mode. runserver: ## Django run server. $(ENV_PREFIX)python3 cos_registration_server/manage.py runserver +.PHONY: secretkey +runserver: ## Generate the dajngo secret key + $(ENV_PREFIX)python3 -c 'from django.core.management.utils import get_random_secret_key; print(get_random_secret_key())' + + .PHONY: fmt fmt: ## Format code using black & isort. $(ENV_PREFIX)isort cos_registration_server/ diff --git a/README.md b/README.md index ffd177b..4976f68 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,13 @@ requiring to access the device database. ## Installation +First we must generate a secret key for our django to sign data. +The secret key must be a large random value and it must be kept secret. + +A secret key can be generated with the following command: +``` +export SECRET_KEY_DJANGO=$(make secretkey) +``` `make install` diff --git a/cos_registration_server/cos_registration_server/settings.py b/cos_registration_server/cos_registration_server/settings.py index cf2f93e..541f2ae 100644 --- a/cos_registration_server/cos_registration_server/settings.py +++ b/cos_registration_server/cos_registration_server/settings.py @@ -10,19 +10,16 @@ https://docs.djangoproject.com/en/4.2/ref/settings/ """ +import os from pathlib import Path # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent - -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/ - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = ( - "django-insecure-$b#5b(2h9_%p439=#ev0!dkde9wqt=rgoc!jvi-y93^@+wcvw8" -) +try: + SECRET_KEY = os.environ["SECRET_KEY_DJANGO"] +except KeyError: + pass # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True From 1df549a70f34985b31aeb06630a110be3e113a69 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Thu, 18 Jan 2024 17:25:03 +0100 Subject: [PATCH 19/49] feat(allow hosts): allow extension of the list --- README.md | 9 ++++++--- .../cos_registration_server/settings.py | 8 +++++++- cos_registration_server/devices/tests.py | 2 +- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4976f68..543f2e4 100644 --- a/README.md +++ b/README.md @@ -141,9 +141,12 @@ First we must generate a secret key for our django to sign data. The secret key must be a large random value and it must be kept secret. A secret key can be generated with the following command: -``` -export SECRET_KEY_DJANGO=$(make secretkey) -``` + +`export SECRET_KEY_DJANGO=$(make secretkey)` + +Additionally, we can expand the allowed hosts list by defining `ALLOWED_HOST_DJANGO`: + +`export ALLOWED_HOST_DJANGO="cos-registration-server.com"` `make install` diff --git a/cos_registration_server/cos_registration_server/settings.py b/cos_registration_server/cos_registration_server/settings.py index 541f2ae..124c9d9 100644 --- a/cos_registration_server/cos_registration_server/settings.py +++ b/cos_registration_server/cos_registration_server/settings.py @@ -24,8 +24,14 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = ["*"] +ALLOWED_HOSTS = ["localhost", "127.0.0.1"] +# to be able to add the webserver address +try: + additional_host = os.environ["ALLOWED_HOST_DJANGO"] + ALLOWED_HOSTS.append(additional_host) +except KeyError: + pass # Application definition diff --git a/cos_registration_server/devices/tests.py b/cos_registration_server/devices/tests.py index 51b88ad..1c9998b 100644 --- a/cos_registration_server/devices/tests.py +++ b/cos_registration_server/devices/tests.py @@ -71,7 +71,7 @@ def test_one_device_then_two(self): class DeviceViewTests(TestCase): def setUp(self): # custom client with META HTTP_HOST specified - self.base_url = "192.168.1.2:8080" + self.base_url = "127.0.0.1:8080" self.client = Client(HTTP_HOST=self.base_url) def test_unlisted_device(self): From 0873ae4ea9d990c80c7b5f297d770783083f8ee8 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Thu, 18 Jan 2024 17:25:13 +0100 Subject: [PATCH 20/49] feat(debug): set it to false --- cos_registration_server/cos_registration_server/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cos_registration_server/cos_registration_server/settings.py b/cos_registration_server/cos_registration_server/settings.py index 124c9d9..cd48621 100644 --- a/cos_registration_server/cos_registration_server/settings.py +++ b/cos_registration_server/cos_registration_server/settings.py @@ -22,7 +22,7 @@ pass # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = False ALLOWED_HOSTS = ["localhost", "127.0.0.1"] From 3896895551780249f92dcf03257dc87cc5b55e12 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Thu, 18 Jan 2024 17:25:41 +0100 Subject: [PATCH 21/49] feat(CSRF): enable CSRF --- cos_registration_server/api/views.py | 3 --- cos_registration_server/cos_registration_server/settings.py | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cos_registration_server/api/views.py b/cos_registration_server/api/views.py index fc457b6..7c6f2d2 100644 --- a/cos_registration_server/api/views.py +++ b/cos_registration_server/api/views.py @@ -1,11 +1,9 @@ from api.serializer import DeviceSerializer from devices.models import Device from django.http import HttpResponse, JsonResponse -from django.views.decorators.csrf import csrf_exempt from rest_framework.parsers import JSONParser -@csrf_exempt def devices(request): if request.method == "GET": devices = Device.objects.all() @@ -26,7 +24,6 @@ def devices(request): return JsonResponse(serialized.errors, status=400) -@csrf_exempt def device(request, uid): try: device = Device.objects.get(uid=uid) diff --git a/cos_registration_server/cos_registration_server/settings.py b/cos_registration_server/cos_registration_server/settings.py index cd48621..f3cc495 100644 --- a/cos_registration_server/cos_registration_server/settings.py +++ b/cos_registration_server/cos_registration_server/settings.py @@ -133,3 +133,7 @@ # https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" + +SESSION_COOKIE_SECURE = True + +CSRF_COOKIE_SECURE = True From 241c385b82c7bf3bc4822e6527945e32a8a53205 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Thu, 18 Jan 2024 17:27:16 +0100 Subject: [PATCH 22/49] feat(database): provide database location with env var --- README.md | 6 +++++- .../cos_registration_server/settings.py | 8 +++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 543f2e4..35e0b88 100644 --- a/README.md +++ b/README.md @@ -144,10 +144,14 @@ A secret key can be generated with the following command: `export SECRET_KEY_DJANGO=$(make secretkey)` -Additionally, we can expand the allowed hosts list by defining `ALLOWED_HOST_DJANGO`: +We can expand the allowed hosts list: `export ALLOWED_HOST_DJANGO="cos-registration-server.com"` +Additionally, we can provide a directory for the database: + +`export DATABASE_BASE_DIR_DJANGO="/var/lib"` + `make install` `make runserver` diff --git a/cos_registration_server/cos_registration_server/settings.py b/cos_registration_server/cos_registration_server/settings.py index f3cc495..05ab39e 100644 --- a/cos_registration_server/cos_registration_server/settings.py +++ b/cos_registration_server/cos_registration_server/settings.py @@ -81,10 +81,16 @@ # Database # https://docs.djangoproject.com/en/4.2/ref/settings/#databases +# to be able to store the database in a juju storage +try: + database_base_dir = Path(os.environ["DATABASE_BASE_DIR_DJANGO"]) +except KeyError: + database_base_dir = BASE_DIR + DATABASES = { "default": { "ENGINE": "django.db.backends.sqlite3", - "NAME": BASE_DIR / "db.sqlite3", + "NAME": database_base_dir / "db.sqlite3", } } From 34b8c9cc4a2f5281387fd2fae5c293e7bc5de9e4 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Fri, 19 Jan 2024 11:23:37 +0100 Subject: [PATCH 23/49] fix(ci): use python3.10 --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f1af244..b104a7e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.9] + python-version: [3.10] os: [ubuntu-latest] runs-on: ${{ matrix.os }} steps: @@ -38,7 +38,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.9] + python-version: [3.10] os: [ubuntu-latest] runs-on: ${{ matrix.os }} steps: From 78ed7b2a0adfe8ece67244eb6c7b2d55bc7e9480 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Mon, 22 Jan 2024 11:22:57 +0100 Subject: [PATCH 24/49] fix(makefile): start dev server from makefile --- CONTRIBUTING.md | 4 ++++ Makefile | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bda67a0..01b44c1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,6 +21,10 @@ then activate it with `source .venv/bin/activate`. Run `make install` to install the project in develop mode. +## Start the developer server + +Run `make runserver` to start the django developer server. + ## Run the tests to ensure everything is working Run `make test` to run the tests. diff --git a/Makefile b/Makefile index 25ff105..c03f563 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ runserver: ## Django run server. $(ENV_PREFIX)python3 cos_registration_server/manage.py runserver .PHONY: secretkey -runserver: ## Generate the dajngo secret key +secretkey: ## Generate the django secret key. $(ENV_PREFIX)python3 -c 'from django.core.management.utils import get_random_secret_key; print(get_random_secret_key())' @@ -80,9 +80,9 @@ virtualenv: ## Create a virtual environment. release: ## Create a new tag for release. @echo "WARNING: This operation will create s version tag and push to github" @read -p "Version? (provide the next x.y.z semver) : " TAG - @echo "$${TAG}" > cos_registration_server/VERSION + @echo "$${TAG}" > cos_registration_server/cos_registration_server/VERSION @$(ENV_PREFIX)gitchangelog > HISTORY.md - @git add cos_registration_server/VERSION HISTORY.md + @git add cos_registration_server/cos_registration_server/VERSION HISTORY.md @git commit -m "release: version $${TAG} 🚀" @echo "creating git tag : $${TAG}" @git tag $${TAG} From bdded1719f8c1e06a8cef77d68f1f61b3dc6b30b Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Mon, 22 Jan 2024 11:24:29 +0100 Subject: [PATCH 25/49] feat(setting): define standard static_root --- cos_registration_server/cos_registration_server/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cos_registration_server/cos_registration_server/settings.py b/cos_registration_server/cos_registration_server/settings.py index 05ab39e..c739538 100644 --- a/cos_registration_server/cos_registration_server/settings.py +++ b/cos_registration_server/cos_registration_server/settings.py @@ -134,6 +134,7 @@ # https://docs.djangoproject.com/en/4.2/howto/static-files/ STATIC_URL = "static/" +STATIC_ROOT = "static/" # Default primary key field type # https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field From 7e4504c94e49af3cd1faab78590f737f4c05a732 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Mon, 22 Jan 2024 11:25:03 +0100 Subject: [PATCH 26/49] feat(setup): include template files --- MANIFEST.in | 3 +-- cos_registration_server/__init__.py | 0 setup.py | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 cos_registration_server/__init__.py diff --git a/MANIFEST.in b/MANIFEST.in index 144a8df..12c3bcd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,4 @@ include LICENSE include HISTORY.md -include Containerfile -graft tests graft cos_registration_server +recursive-include cos_registration_server/devices/templates * diff --git a/cos_registration_server/__init__.py b/cos_registration_server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/setup.py b/setup.py index 1ca3171..3ec2045 100644 --- a/setup.py +++ b/setup.py @@ -38,6 +38,7 @@ def read_requirements(path): long_description_content_type="text/markdown", author="ubuntu-robotics", packages=find_packages(exclude=[".github"]), + include_package_data=True, install_requires=read_requirements("requirements.txt"), extras_require={"test": read_requirements("requirements-test.txt")}, ) From a4fe0441e16698fd473f62dacec540e127dfc354 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Mon, 22 Jan 2024 11:28:22 +0100 Subject: [PATCH 27/49] feat(snap): add snap to distribute the server --- .gitignore | 3 ++ .../cos_registration_server/settings.py | 1 + requirements.txt | 2 ++ snap/hooks/configure | 8 +++++ snap/hooks/install | 6 ++++ snap/local/launcher.bash | 9 ++++++ snap/snapcraft.yaml | 32 +++++++++++++++++++ 7 files changed, 61 insertions(+) create mode 100755 snap/hooks/configure create mode 100755 snap/hooks/install create mode 100755 snap/local/launcher.bash create mode 100644 snap/snapcraft.yaml diff --git a/.gitignore b/.gitignore index 2d0fadb..c01f0f4 100644 --- a/.gitignore +++ b/.gitignore @@ -130,3 +130,6 @@ dmypy.json # templates .github/templates/* + +# snap +*.snap diff --git a/cos_registration_server/cos_registration_server/settings.py b/cos_registration_server/cos_registration_server/settings.py index c739538..72d5eb4 100644 --- a/cos_registration_server/cos_registration_server/settings.py +++ b/cos_registration_server/cos_registration_server/settings.py @@ -55,6 +55,7 @@ "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", + "whitenoise.middleware.WhiteNoiseMiddleware", ] ROOT_URLCONF = "cos_registration_server.urls" diff --git a/requirements.txt b/requirements.txt index 6e5370e..8f9d992 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ django djangorestframework +daphne +whitenoise diff --git a/snap/hooks/configure b/snap/hooks/configure new file mode 100755 index 0000000..37c53f9 --- /dev/null +++ b/snap/hooks/configure @@ -0,0 +1,8 @@ +#!/usr/bin/bash + +cd $SNAP/lib/python3.10/site-packages/cos_registration_server + +export DATABASE_BASE_DIR_DJANGO=$SNAP_COMMON + +# migrate the database +$SNAP/bin/python3 manage.py migrate diff --git a/snap/hooks/install b/snap/hooks/install new file mode 100755 index 0000000..19a9698 --- /dev/null +++ b/snap/hooks/install @@ -0,0 +1,6 @@ +#!/usr/bin/bash + +SECRET_KEY=$($SNAP/bin/python3 -c \ +'from django.core.management.utils import get_random_secret_key; +print(get_random_secret_key())') +snapctl set secret-key=$SECRET_KEY diff --git a/snap/local/launcher.bash b/snap/local/launcher.bash new file mode 100755 index 0000000..4ae0eb1 --- /dev/null +++ b/snap/local/launcher.bash @@ -0,0 +1,9 @@ +#!/usr/bin/bash + +# necessary for daphne as the option --root-path doesn't work +cd $SNAP/lib/python3.10/site-packages/cos_registration_server + +export DATABASE_BASE_DIR_DJANGO=$SNAP_COMMON +export SECRET_KEY_DJANGO=$(snapctl get secret-key) + +$SNAP/bin/daphne cos_registration_server.asgi:application diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml new file mode 100644 index 0000000..125cbe5 --- /dev/null +++ b/snap/snapcraft.yaml @@ -0,0 +1,32 @@ +name: cos-registration-server +summary: The COS registration server +description: | + COS registation server with production server + and static fileserver +version: '0.1' +base: core22 +confinement: strict + +parts: + cos-registration-server: + plugin: python + source: . + python-packages: ["django"] + override-build: | + craftctl default + cd $CRAFT_PART_INSTALL/lib/python3.10/site-packages/cos_registration_server + PYTHONPATH=../ python3 manage.py collectstatic --no-input + + local: + plugin: dump + source: snap/local + organize: + '*.bash' : usr/bin/ + +apps: + cos-registration-server: + command: usr/bin/launcher.bash + daemon: simple + plugs: + - network + - network-bind From bd34979b26b6817d920f3c44242cddb009df0fec Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Mon, 22 Jan 2024 12:03:13 +0100 Subject: [PATCH 28/49] feat(snap): create-super-user --- snap/local/create_super_user.bash | 9 +++++++++ snap/snapcraft.yaml | 3 +++ 2 files changed, 12 insertions(+) create mode 100755 snap/local/create_super_user.bash diff --git a/snap/local/create_super_user.bash b/snap/local/create_super_user.bash new file mode 100755 index 0000000..28fdd5e --- /dev/null +++ b/snap/local/create_super_user.bash @@ -0,0 +1,9 @@ +#!/usr/bin/bash + +# necessary for daphne as the option --root-path doesn't work +cd $SNAP/lib/python3.10/site-packages/cos_registration_server + +export DATABASE_BASE_DIR_DJANGO=$SNAP_COMMON +export SECRET_KEY_DJANGO=$(snapctl get secret-key) + +python manage.py createsuperuser diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 125cbe5..4c76349 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -30,3 +30,6 @@ apps: plugs: - network - network-bind + + create-super-user: + command: usr/bin/create_super_user.bash From ad22dc2e8f32287d3a84c37af7663fe9ce952f72 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Wed, 24 Jan 2024 15:48:36 +0100 Subject: [PATCH 29/49] refactor(test) --- .github/workflows/{main.yml => tests.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{main.yml => tests.yml} (100%) diff --git a/.github/workflows/main.yml b/.github/workflows/tests.yml similarity index 100% rename from .github/workflows/main.yml rename to .github/workflows/tests.yml From 4fe65762801f83c7642af5ed064b16513910fe6d Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Wed, 24 Jan 2024 15:48:54 +0100 Subject: [PATCH 30/49] feat(rock): create a rock --- .gitignore | 2 ++ requirements.txt | 1 + rock/local/configure.bash | 8 +++++++ rock/local/create_super_user.bash | 9 ++++++++ rock/local/install.bash | 8 +++++++ rock/local/launcher.bash | 9 ++++++++ rockcraft.yaml | 37 +++++++++++++++++++++++++++++++ 7 files changed, 74 insertions(+) create mode 100755 rock/local/configure.bash create mode 100755 rock/local/create_super_user.bash create mode 100755 rock/local/install.bash create mode 100755 rock/local/launcher.bash create mode 100644 rockcraft.yaml diff --git a/.gitignore b/.gitignore index c01f0f4..561d037 100644 --- a/.gitignore +++ b/.gitignore @@ -133,3 +133,5 @@ dmypy.json # snap *.snap +# rock +*.rock diff --git a/requirements.txt b/requirements.txt index 8f9d992..fae131d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ django djangorestframework daphne whitenoise +tzdata diff --git a/rock/local/configure.bash b/rock/local/configure.bash new file mode 100755 index 0000000..5db650c --- /dev/null +++ b/rock/local/configure.bash @@ -0,0 +1,8 @@ +#!/usr/bin/bash + +cd /lib/python3.10/site-packages/cos_registration_server + +export DATABASE_BASE_DIR_DJANGO="~/server_data" + +# migrate the database +/bin/python3 manage.py migrate diff --git a/rock/local/create_super_user.bash b/rock/local/create_super_user.bash new file mode 100755 index 0000000..39642a0 --- /dev/null +++ b/rock/local/create_super_user.bash @@ -0,0 +1,9 @@ +#!/usr/bin/bash + +# necessary for daphne as the option --root-path doesn't work +cd /lib/python3.10/site-packages/cos_registration_server + +export DATABASE_BASE_DIR_DJANGO="~/server_data" +export SECRET_KEY_DJANGO=$(cat ~/server_data/secret_key) + +python manage.py createsuperuser diff --git a/rock/local/install.bash b/rock/local/install.bash new file mode 100755 index 0000000..cea8041 --- /dev/null +++ b/rock/local/install.bash @@ -0,0 +1,8 @@ +#!/usr/bin/bash + +mkdir ~/server_data + +SECRET_KEY=$(/bin/python3 -c \ +'from django.core.management.utils import get_random_secret_key; +print(get_random_secret_key())') +echo $SECRET_KEY > ~/server_data/secret_key diff --git a/rock/local/launcher.bash b/rock/local/launcher.bash new file mode 100755 index 0000000..b7cc132 --- /dev/null +++ b/rock/local/launcher.bash @@ -0,0 +1,9 @@ +#!/usr/bin/bash + +# necessary for daphne as the option --root-path doesn't work +cd /lib/python3.10/site-packages/cos_registration_server + +export DATABASE_BASE_DIR_DJANGO="~/server_data" +export SECRET_KEY_DJANGO=$(cat ~/server_data/secret_key) + +/bin/daphne cos_registration_server.asgi:application diff --git a/rockcraft.yaml b/rockcraft.yaml new file mode 100644 index 0000000..1b3ea31 --- /dev/null +++ b/rockcraft.yaml @@ -0,0 +1,37 @@ +name: cos-registration-server +summary: The COS registration server +description: | + COS registation server with production server + and static fileserver +version: '0.1' +license: GPL-3.0 + +base: ubuntu@22.04 +build-base: ubuntu@22.04 +platforms: + amd64: + + +services: + cos-registration-server: + override: replace + startup: enabled + command: /usr/bin/launcher.bash + +# Parts section +parts: + cos-registration-server: + plugin: python + source: . + python-packages: ["django"] + stage-packages: ["python3", "python3-venv"] + override-build: | + craftctl default + cd $CRAFT_PART_INSTALL/lib/python3.10/site-packages/cos_registration_server + PYTHONPATH=../ python3 manage.py collectstatic --no-input + + local: + plugin: dump + source: rock/local + organize: + '*.bash' : usr/bin/ From eb01c8466fb3c874d6f9439689a5da7b98b15919 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Wed, 24 Jan 2024 15:49:02 +0100 Subject: [PATCH 31/49] CI(rock) --- .github/workflows/pull-request.yaml | 35 +++++++++++++++ .github/workflows/rock-release-dev.yaml | 58 +++++++++++++++++++++++++ .github/workflows/tests.yml | 6 +-- rock/local/configure.bash | 2 +- rock/local/create_super_user.bash | 4 +- rock/local/install.bash | 4 +- rock/local/launcher.bash | 4 +- 7 files changed, 103 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/pull-request.yaml create mode 100644 .github/workflows/rock-release-dev.yaml diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml new file mode 100644 index 0000000..dec12e7 --- /dev/null +++ b/.github/workflows/pull-request.yaml @@ -0,0 +1,35 @@ +name: Pull Requests + +on: + workflow_dispatch: {} + pull_request: + branches: + - main + +jobs: + main: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Determine any rockcraft.yaml changed in the PR + id: changed-files + uses: tj-actions/changed-files@v35 + with: + files: "./rockcraft.yaml" + + - name: Setup LXD + if: steps.changed-files.outputs.any_changed + uses: canonical/setup-lxd@v0.1.0 + with: + channel: latest/stable + + - name: Install dependencies + if: steps.changed-files.outputs.any_changed + run: | + sudo snap install --classic rockcraft + - name: Build ROCK + if: steps.changed-files.outputs.any_changed + run: | + rockcraft pack --verbose diff --git a/.github/workflows/rock-release-dev.yaml b/.github/workflows/rock-release-dev.yaml new file mode 100644 index 0000000..19e3274 --- /dev/null +++ b/.github/workflows/rock-release-dev.yaml @@ -0,0 +1,58 @@ +# The workflow canonical/observability/.github/workflows/rock-release-dev.yaml@main +# https://github.com/canonical/observability/blob/34b7edb56094e8e3f01200ce49e01bf9dd6688ac/.github/workflows/rock-release-dev.yaml +# does not allow setting a GHCR repo where to publish at the moment. +# We thus copy it below and adapt it. + +name: Build ROCK and release dev tag to GHCR + +on: + workflow_dispatch: + push: + branches: + - main + +jobs: + main: + runs-on: ubuntu-latest + steps: + + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Find the *latest* rockcraft.yaml + id: find-latest + run: | + latest_rockcraft_file=$(find $GITHUB_WORKSPACE -name "rockcraft.yaml" | sort -V | tail -n1) + rockcraft_dir=$(dirname ${latest_rockcraft_file#\./}) + echo "latest-dir=$rockcraft_dir" >> $GITHUB_OUTPUT + - name: Build ROCK + uses: canonical/craft-actions/rockcraft-pack@main + with: + path: "${{ steps.find-latest.outputs.latest-dir }}" + verbosity: verbose + id: rockcraft + + - name: Upload locally built ROCK artifact + uses: actions/upload-artifact@v3 + with: + name: cos-registration-agent-rock + path: ${{ steps.rockcraft.outputs.rock }} + + - name: Upload ROCK to ghcr.io + run: | + sudo skopeo --insecure-policy copy oci-archive:$(realpath ${{ steps.rockcraft.outputs.rock }}) docker://ghcr.io/ubuntu-robotics/cos-registration-server:dev --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" + - name: Generate digest + run: | + digest=$(skopeo inspect oci-archive:$(realpath ${{ steps.rockcraft.outputs.rock }}) --format '{{.Digest}}') + echo "digest=${digest#*:}" >> "$GITHUB_OUTPUT" + - name: Install Syft + run: | + curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin + - name: Create SBOM + run: syft $(realpath ${{ steps.rockcraft.outputs.rock }}) -o spdx-json=cos-registration-server.sbom.json + + - name: Upload SBOM + uses: actions/upload-artifact@v3 + with: + name: cos-registration-server-sbom + path: "cos-registration-server.sbom.json" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b104a7e..8d5ab1c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,6 +1,6 @@ # This is a basic workflow to help you get started with Actions -name: CI +name: tests # Controls when the workflow will run on: @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.10] + python-version: ["3.10"] os: [ubuntu-latest] runs-on: ${{ matrix.os }} steps: @@ -38,7 +38,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.10] + python-version: ["3.10"] os: [ubuntu-latest] runs-on: ${{ matrix.os }} steps: diff --git a/rock/local/configure.bash b/rock/local/configure.bash index 5db650c..a94987a 100755 --- a/rock/local/configure.bash +++ b/rock/local/configure.bash @@ -2,7 +2,7 @@ cd /lib/python3.10/site-packages/cos_registration_server -export DATABASE_BASE_DIR_DJANGO="~/server_data" +export DATABASE_BASE_DIR_DJANGO="/server_data" # migrate the database /bin/python3 manage.py migrate diff --git a/rock/local/create_super_user.bash b/rock/local/create_super_user.bash index 39642a0..db06a17 100755 --- a/rock/local/create_super_user.bash +++ b/rock/local/create_super_user.bash @@ -3,7 +3,7 @@ # necessary for daphne as the option --root-path doesn't work cd /lib/python3.10/site-packages/cos_registration_server -export DATABASE_BASE_DIR_DJANGO="~/server_data" -export SECRET_KEY_DJANGO=$(cat ~/server_data/secret_key) +export DATABASE_BASE_DIR_DJANGO="/server_data" +export SECRET_KEY_DJANGO=$(cat /server_data/secret_key) python manage.py createsuperuser diff --git a/rock/local/install.bash b/rock/local/install.bash index cea8041..3b653bc 100755 --- a/rock/local/install.bash +++ b/rock/local/install.bash @@ -1,8 +1,8 @@ #!/usr/bin/bash -mkdir ~/server_data +mkdir /server_data SECRET_KEY=$(/bin/python3 -c \ 'from django.core.management.utils import get_random_secret_key; print(get_random_secret_key())') -echo $SECRET_KEY > ~/server_data/secret_key +echo $SECRET_KEY > /server_data/secret_key diff --git a/rock/local/launcher.bash b/rock/local/launcher.bash index b7cc132..a80e033 100755 --- a/rock/local/launcher.bash +++ b/rock/local/launcher.bash @@ -3,7 +3,7 @@ # necessary for daphne as the option --root-path doesn't work cd /lib/python3.10/site-packages/cos_registration_server -export DATABASE_BASE_DIR_DJANGO="~/server_data" -export SECRET_KEY_DJANGO=$(cat ~/server_data/secret_key) +export DATABASE_BASE_DIR_DJANGO="/server_data" +export SECRET_KEY_DJANGO=$(cat /server_data/secret_key) /bin/daphne cos_registration_server.asgi:application From 6ae4dba749e7604a3c875451cc8760ef6b2cf268 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Thu, 25 Jan 2024 10:06:43 +0100 Subject: [PATCH 32/49] fix(daphne): change bind address --- rock/local/launcher.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rock/local/launcher.bash b/rock/local/launcher.bash index a80e033..af66d97 100755 --- a/rock/local/launcher.bash +++ b/rock/local/launcher.bash @@ -6,4 +6,4 @@ cd /lib/python3.10/site-packages/cos_registration_server export DATABASE_BASE_DIR_DJANGO="/server_data" export SECRET_KEY_DJANGO=$(cat /server_data/secret_key) -/bin/daphne cos_registration_server.asgi:application +/bin/daphne -b 0.0.0.0 cos_registration_server.asgi:application From c77567eca0f72c7b038528a52b4f6eacbf746edc Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Thu, 25 Jan 2024 11:46:40 +0100 Subject: [PATCH 33/49] feat(django): USE_X_FORWARDED_HOST --- cos_registration_server/cos_registration_server/settings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cos_registration_server/cos_registration_server/settings.py b/cos_registration_server/cos_registration_server/settings.py index 72d5eb4..16b70a6 100644 --- a/cos_registration_server/cos_registration_server/settings.py +++ b/cos_registration_server/cos_registration_server/settings.py @@ -145,3 +145,6 @@ SESSION_COOKIE_SECURE = True CSRF_COOKIE_SECURE = True + +# To keep the reverse proxy prefix when forwarding. +USE_X_FORWARDED_HOST = True From 7b9a1f9bdcc1b58684559778b0636666bf4399d8 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Wed, 31 Jan 2024 17:01:21 +0100 Subject: [PATCH 34/49] fix(gitignore): ignore generated static files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 561d037..496bf0d 100644 --- a/.gitignore +++ b/.gitignore @@ -135,3 +135,5 @@ dmypy.json *.snap # rock *.rock +# static files +cos_registration_server/static/ From d700d71cd98fe38ee1ce92b442ad9a585d074be0 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Wed, 31 Jan 2024 17:02:13 +0100 Subject: [PATCH 35/49] fix(gunicorn): use guniron instead of daphne --- requirements.txt | 2 +- rock/local/launcher.bash | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index fae131d..d58dcfa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ django djangorestframework -daphne whitenoise tzdata +gunicorn diff --git a/rock/local/launcher.bash b/rock/local/launcher.bash index af66d97..42adbaf 100755 --- a/rock/local/launcher.bash +++ b/rock/local/launcher.bash @@ -1,9 +1,8 @@ #!/usr/bin/bash -# necessary for daphne as the option --root-path doesn't work cd /lib/python3.10/site-packages/cos_registration_server export DATABASE_BASE_DIR_DJANGO="/server_data" export SECRET_KEY_DJANGO=$(cat /server_data/secret_key) -/bin/daphne -b 0.0.0.0 cos_registration_server.asgi:application +gunicorn --bind 0.0.0.0:8000 cos_registration_server.wsgi From ae527a4b49e6d6fd4413030e118bfdf2acdcb2ea Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Wed, 31 Jan 2024 17:02:48 +0100 Subject: [PATCH 36/49] fix(settings): manage static files with script_name --- .../cos_registration_server/settings.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cos_registration_server/cos_registration_server/settings.py b/cos_registration_server/cos_registration_server/settings.py index 16b70a6..8611117 100644 --- a/cos_registration_server/cos_registration_server/settings.py +++ b/cos_registration_server/cos_registration_server/settings.py @@ -134,8 +134,11 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/4.2/howto/static-files/ -STATIC_URL = "static/" -STATIC_ROOT = "static/" +STATIC_URL = os.getenv("SCRIPT_NAME", "") + "/static/" + +STATIC_ROOT = BASE_DIR / "static/" + +WHITENOISE_STATIC_PREFIX = "static/" # Default primary key field type # https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field From 641b9e24a19775b919550d5f8003866cb2ae4afb Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Wed, 31 Jan 2024 17:03:10 +0100 Subject: [PATCH 37/49] fix(create_super_user): accept parameters --- rock/local/create_super_user.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rock/local/create_super_user.bash b/rock/local/create_super_user.bash index db06a17..f17a69e 100755 --- a/rock/local/create_super_user.bash +++ b/rock/local/create_super_user.bash @@ -6,4 +6,4 @@ cd /lib/python3.10/site-packages/cos_registration_server export DATABASE_BASE_DIR_DJANGO="/server_data" export SECRET_KEY_DJANGO=$(cat /server_data/secret_key) -python manage.py createsuperuser +python3 manage.py createsuperuser $@ From 380b7135e8b6c35eacc0516a6829fcd8dad5d9ef Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Wed, 31 Jan 2024 17:03:37 +0100 Subject: [PATCH 38/49] fix(security): disable csrf for now --- cos_registration_server/cos_registration_server/settings.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cos_registration_server/cos_registration_server/settings.py b/cos_registration_server/cos_registration_server/settings.py index 8611117..02fcc34 100644 --- a/cos_registration_server/cos_registration_server/settings.py +++ b/cos_registration_server/cos_registration_server/settings.py @@ -51,7 +51,6 @@ "django.middleware.security.SecurityMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.common.CommonMiddleware", - "django.middleware.csrf.CsrfViewMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", @@ -145,9 +144,6 @@ DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" -SESSION_COOKIE_SECURE = True - -CSRF_COOKIE_SECURE = True - # To keep the reverse proxy prefix when forwarding. USE_X_FORWARDED_HOST = True + From 1036eef446ccc67c3b55fce0fb9188ee1d0f45b4 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Thu, 1 Feb 2024 18:03:21 +0100 Subject: [PATCH 39/49] feat(models): add grafana dashboard to Device model --- ...a_dashboards_device_unique_uid_blocking.py | 27 +++++++++++++ cos_registration_server/devices/models.py | 40 +++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 cos_registration_server/devices/migrations/0002_device_grafana_dashboards_device_unique_uid_blocking.py diff --git a/cos_registration_server/devices/migrations/0002_device_grafana_dashboards_device_unique_uid_blocking.py b/cos_registration_server/devices/migrations/0002_device_grafana_dashboards_device_unique_uid_blocking.py new file mode 100644 index 0000000..9620118 --- /dev/null +++ b/cos_registration_server/devices/migrations/0002_device_grafana_dashboards_device_unique_uid_blocking.py @@ -0,0 +1,27 @@ +# Generated by Django 4.2.9 on 2024-02-01 17:44 + +import devices.models +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("devices", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="device", + name="grafana_dashboards", + field=models.JSONField( + default=devices.models.default_dashboards_json_field, + verbose_name="Dashboards json field", + ), + ), + migrations.AddConstraint( + model_name="device", + constraint=models.UniqueConstraint( + fields=("uid",), name="unique_uid_blocking" + ), + ), + ] diff --git a/cos_registration_server/devices/models.py b/cos_registration_server/devices/models.py index ba016e7..e311e76 100644 --- a/cos_registration_server/devices/models.py +++ b/cos_registration_server/devices/models.py @@ -1,10 +1,50 @@ +import json + +from django.core.exceptions import ValidationError from django.db import models +from django.db.models.constraints import UniqueConstraint + + +def default_dashboards_json_field(): + return "[]" class Device(models.Model): uid = models.CharField(max_length=200) creation_date = models.DateTimeField("creation date", auto_now_add=True) address = models.GenericIPAddressField("device IP") + grafana_dashboards = models.JSONField( + "Dashboards json field", default=default_dashboards_json_field + ) def __str__(self): return self.uid + + class Meta: + constraints = [ + UniqueConstraint(fields=["uid"], name="unique_uid_blocking") + ] + + def clean(self): + # make sure the grafana_dashboards is containing an array of dashboards + dashboards = [] + if isinstance(self.grafana_dashboards, str): + try: + dashboards = json.loads(self.grafana_dashboards) + except json.JSONDecodeError: + raise ValidationError( + "Failed to load grafana_dashboards as json" + ) + elif isinstance(self.grafana_dashboards, list): + dashboards = self.grafana_dashboards + else: + raise ValidationError( + f"Unknow type for grafana_dashboards: \ + {type(self.grafana_dashboards)}" + ) + + if dashboards is None or not isinstance(dashboards, list): + raise ValidationError( + 'gafana_dashboards is not well formated. \ + Make sure all the dashboards are within "dashbords": [] ' + ) From edf9db9bcdef4fb81ab4e64435fef60e6ab79c89 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Thu, 1 Feb 2024 18:03:42 +0100 Subject: [PATCH 40/49] test(device): grafana dashboards --- cos_registration_server/devices/tests.py | 33 +++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/cos_registration_server/devices/tests.py b/cos_registration_server/devices/tests.py index 1c9998b..231ebe7 100644 --- a/cos_registration_server/devices/tests.py +++ b/cos_registration_server/devices/tests.py @@ -4,7 +4,21 @@ from django.urls import reverse from django.utils import timezone -from .models import Device +from .models import Device, default_dashboards_json_field + +SIMPLE_GRAFANA_DASHBOARD = """{ + "dashboard": { + "id": null, + "uid": null, + "title": "Production Overview", + "tags": [ "templated" ], + "timezone": "browser", + "schemaVersion": 16, + "refresh": "25s" + }, + "message": "Made changes to xyz", + "overwrite": false +}""" class DeviceModelTests(TestCase): @@ -18,6 +32,9 @@ def test_creation_of_a_device(self): self.assertGreater( device.creation_date, timezone.now() - timezone.timedelta(hours=1) ) + self.assertEquals( + device.grafana_dashboards, default_dashboards_json_field() + ) def test_device_str(self): device = Device( @@ -25,6 +42,20 @@ def test_device_str(self): ) self.assertEqual(str(device), "hello-123") + def test_device_grafana_dashboards(self): + custom_grafana_dashboards = eval(default_dashboards_json_field()) + custom_grafana_dashboards.append(SIMPLE_GRAFANA_DASHBOARD) + device = Device( + uid="hello-123", + creation_date=timezone.now(), + address="127.0.0.1", + grafana_dashboards=custom_grafana_dashboards, + ) + self.assertEqual( + str(device.grafana_dashboards[0]), + SIMPLE_GRAFANA_DASHBOARD, + ) + def create_device(uid, address): return Device.objects.create(uid=uid, address=address) From e5eb868a7a26a62c71a9a18e151d966eba063204 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Thu, 1 Feb 2024 18:04:53 +0100 Subject: [PATCH 41/49] fix(serializer): use validator for unique uid --- cos_registration_server/api/serializer.py | 6 +++++- cos_registration_server/api/views.py | 6 ------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/cos_registration_server/api/serializer.py b/cos_registration_server/api/serializer.py index 3dec7f8..ef68acb 100644 --- a/cos_registration_server/api/serializer.py +++ b/cos_registration_server/api/serializer.py @@ -1,9 +1,13 @@ from devices.models import Device from rest_framework import serializers +from rest_framework.validators import UniqueValidator class DeviceSerializer(serializers.Serializer): - uid = serializers.CharField(required=True) + uid = serializers.CharField( + required=True, + validators=[UniqueValidator(queryset=Device.objects.all())], + ) creation_date = serializers.DateTimeField(read_only=True) address = serializers.IPAddressField(required=True) diff --git a/cos_registration_server/api/views.py b/cos_registration_server/api/views.py index 7c6f2d2..6a96f2c 100644 --- a/cos_registration_server/api/views.py +++ b/cos_registration_server/api/views.py @@ -13,12 +13,6 @@ def devices(request): data = JSONParser().parse(request) serialized = DeviceSerializer(data=data) if serialized.is_valid(): - if Device.objects.filter( - uid=serialized.validated_data["uid"] - ).exists(): - return JsonResponse( - {"error": "Device uid already exists"}, status=409 - ) serialized.save() return JsonResponse(serialized.data, status=201) return JsonResponse(serialized.errors, status=400) From fdd67ad193957ec7969250d4ba493e0e7037fc4b Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Thu, 1 Feb 2024 18:24:29 +0100 Subject: [PATCH 42/49] feat(serializer): add grafana_dashboard as well as validation --- cos_registration_server/api/serializer.py | 26 ++++++++++++++++++- .../cos_registration_server/settings.py | 1 - 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/cos_registration_server/api/serializer.py b/cos_registration_server/api/serializer.py index ef68acb..00c6849 100644 --- a/cos_registration_server/api/serializer.py +++ b/cos_registration_server/api/serializer.py @@ -1,3 +1,5 @@ +import json + from devices.models import Device from rest_framework import serializers from rest_framework.validators import UniqueValidator @@ -10,11 +12,33 @@ class DeviceSerializer(serializers.Serializer): ) creation_date = serializers.DateTimeField(read_only=True) address = serializers.IPAddressField(required=True) + grafana_dashboards = serializers.JSONField(required=False) def create(self, validated_data): return Device.objects.create(**validated_data) def update(self, instance, validated_data): - instance.address = validated_data.get("address", instance.address) + address = validated_data.get("address", instance.address) + instance.address = address + grafana_dashboards = validated_data.get( + "grafana_dashboards", instance.grafana_dashboards + ) + instance.grafana_dashboards = grafana_dashboards instance.save() return instance + + def validate_grafana_dashboards(self, value): + if isinstance(value, str): + try: + dashboards = json.loads(value) + except json.JSONDecodeError: + raise serializers.ValidationError( + "Failed to load grafana_dashboards as json." + ) + else: + dashboards = value + if not isinstance(dashboards, list): + raise serializers.ValidationError( + "gafana_dashboards is not a supported format (list)." + ) + return value diff --git a/cos_registration_server/cos_registration_server/settings.py b/cos_registration_server/cos_registration_server/settings.py index 02fcc34..e6a5453 100644 --- a/cos_registration_server/cos_registration_server/settings.py +++ b/cos_registration_server/cos_registration_server/settings.py @@ -146,4 +146,3 @@ # To keep the reverse proxy prefix when forwarding. USE_X_FORWARDED_HOST = True - From 45166c7e55ac8c27a23c32e98bb533fb1feea5e5 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Thu, 1 Feb 2024 18:25:05 +0100 Subject: [PATCH 43/49] feat(devices_view): don't return dashboards when we get all the devices --- cos_registration_server/api/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cos_registration_server/api/views.py b/cos_registration_server/api/views.py index 6a96f2c..8d60289 100644 --- a/cos_registration_server/api/views.py +++ b/cos_registration_server/api/views.py @@ -6,7 +6,9 @@ def devices(request): if request.method == "GET": - devices = Device.objects.all() + devices = Device.objects.all().values( + "uid", "creation_date", "address" + ) serialized = DeviceSerializer(devices, many=True) return JsonResponse(serialized.data, safe=False) elif request.method == "POST": From e390d49857d483b8705e51e93afc8abb82ee1f32 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Thu, 1 Feb 2024 18:25:43 +0100 Subject: [PATCH 44/49] test(API): api with grafana dashboards --- cos_registration_server/api/tests.py | 91 +++++++++++++++++++++++++--- 1 file changed, 83 insertions(+), 8 deletions(-) diff --git a/cos_registration_server/api/tests.py b/cos_registration_server/api/tests.py index 743af40..81d4588 100644 --- a/cos_registration_server/api/tests.py +++ b/cos_registration_server/api/tests.py @@ -1,17 +1,33 @@ import json -from devices.models import Device +from devices.models import Device, default_dashboards_json_field from django.urls import reverse from django.utils import timezone from rest_framework.test import APITestCase +SIMPLE_GRAFANA_DASHBOARD = """{ + "dashboard": { + "id": null, + "uid": null, + "title": "Production Overview", + "tags": [ "templated" ], + "timezone": "browser", + "schemaVersion": 16, + "refresh": "25s" + }, + "message": "Made changes to xyz", + "overwrite": false +}""" + class DevicesViewTests(APITestCase): def setUp(self): self.url = reverse("api:devices") - def create_device(self, uid, address): - data = {"uid": uid, "address": address} + def create_device(self, **fields): + data = {} + for field, value in fields.items(): + data[field] = value return self.client.post(self.url, data, format="json") def test_get_nothing(self): @@ -22,7 +38,13 @@ def test_get_nothing(self): def test_create_device(self): uid = "robot-1" address = "192.168.0.1" - response = self.create_device(uid, address) + custom_grafana_dashboards = eval(default_dashboards_json_field()) + custom_grafana_dashboards.append(SIMPLE_GRAFANA_DASHBOARD) + response = self.create_device( + uid=uid, + address=address, + grafana_dashboards=custom_grafana_dashboards, + ) self.assertEqual(response.status_code, 201) self.assertEqual(Device.objects.count(), 1) self.assertEqual(Device.objects.get().uid, uid) @@ -32,6 +54,9 @@ def test_create_device(self): timezone.now(), delta=timezone.timedelta(seconds=10), ) + self.assertEqual( + Device.objects.get().grafana_dashboards, custom_grafana_dashboards + ) def test_create_multiple_devices(self): devices = [ @@ -40,7 +65,7 @@ def test_create_multiple_devices(self): {"uid": "robot-3", "address": "192.168.0.3"}, ] for device in devices: - self.create_device(device["uid"], device["address"]) + self.create_device(uid=device["uid"], address=device["address"]) self.assertEqual(Device.objects.count(), 3) response = self.client.get(self.url) @@ -54,14 +79,50 @@ def test_create_multiple_devices(self): def test_create_already_present_uid(self): uid = "robot-1" address = "192.168.0.1" - response = self.create_device(uid, address) + response = self.create_device(uid=uid, address=address) self.assertEqual(response.status_code, 201) self.assertEqual(Device.objects.count(), 1) # we try to create the same one - response = self.create_device(uid, address) + response = self.create_device(uid=uid, address=address) self.assertEqual(Device.objects.count(), 1) self.assertContains( - response, "Device uid already exists", status_code=409 + response, + '{"uid": ["This field must be unique."]}', + status_code=400, + ) + + def test_grafana_dashboard_not_in_a_list(self): + uid = "robot-1" + address = "192.168.0.1" + # we try to create the same one + response = self.create_device( + uid=uid, + address=address, + grafana_dashboards=SIMPLE_GRAFANA_DASHBOARD, + ) + self.assertEqual(Device.objects.count(), 0) + self.assertContains( + response, + '{"grafana_dashboards": ["gafana_dashboards is not a supported ' + 'format (list).', + status_code=400, + ) + + def test_grafana_dashboard_illformed_json(self): + uid = "robot-1" + address = "192.168.0.1" + # we try to create the same one + response = self.create_device( + uid=uid, + address=address, + grafana_dashboards='[{"hello"=321}', + ) + self.assertEqual(Device.objects.count(), 0) + self.assertContains( + response, + '{"grafana_dashboards": ["Failed to load grafana_dashboards ' + "as json.", + status_code=400, ) @@ -107,6 +168,20 @@ def test_patch_device(self): self.assertEqual(content_json["uid"], uid) self.assertEqual(content_json["address"], address) + def test_patch_dashboards(self): + uid = "robot-1" + address = "192.168.1.2" + self.create_device(uid, address) + data = {"grafana_dashboards": [SIMPLE_GRAFANA_DASHBOARD]} + response = self.client.patch(self.url(uid), data, format="json") + self.assertEqual(response.status_code, 200) + content_json = json.loads(response.content) + self.assertEqual( + content_json["grafana_dashboards"][0], + SIMPLE_GRAFANA_DASHBOARD, + ) + self.assertEqual(content_json["address"], address) + def test_invalid_patch_device(self): uid = "robot-1" address = "192.168.1.2" From 47e51c2f390e4d7a6545244e48fab4eabbe7e766 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Fri, 2 Feb 2024 14:13:23 +0100 Subject: [PATCH 45/49] CI(rock): create and publish rock when a PR is open This way we can test rocks integrated to a charm --- .github/workflows/rock-release-pr.yaml | 58 ++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 .github/workflows/rock-release-pr.yaml diff --git a/.github/workflows/rock-release-pr.yaml b/.github/workflows/rock-release-pr.yaml new file mode 100644 index 0000000..c0ec20c --- /dev/null +++ b/.github/workflows/rock-release-pr.yaml @@ -0,0 +1,58 @@ +# The workflow canonical/observability/.github/workflows/rock-release-dev.yaml@main +# https://github.com/canonical/observability/blob/34b7edb56094e8e3f01200ce49e01bf9dd6688ac/.github/workflows/rock-release-dev.yaml +# does not allow setting a GHCR repo where to publish at the moment. +# We thus copy it below and adapt it. + +name: Build ROCK and release pr tag to GHCR + +on: + workflow_dispatch: + pull_request: + branches: + - main + +jobs: + main: + runs-on: ubuntu-latest + steps: + + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Find the *latest* rockcraft.yaml + id: find-latest + run: | + latest_rockcraft_file=$(find $GITHUB_WORKSPACE -name "rockcraft.yaml" | sort -V | tail -n1) + rockcraft_dir=$(dirname ${latest_rockcraft_file#\./}) + echo "latest-dir=$rockcraft_dir" >> $GITHUB_OUTPUT + - name: Build ROCK + uses: canonical/craft-actions/rockcraft-pack@main + with: + path: "${{ steps.find-latest.outputs.latest-dir }}" + verbosity: verbose + id: rockcraft + + - name: Upload locally built ROCK artifact + uses: actions/upload-artifact@v3 + with: + name: cos-registration-agent-rock + path: ${{ steps.rockcraft.outputs.rock }} + + - name: Upload ROCK to ghcr.io + run: | + sudo skopeo --insecure-policy copy oci-archive:$(realpath ${{ steps.rockcraft.outputs.rock }}) docker://ghcr.io/ubuntu-robotics/cos-registration-server:pr --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" + - name: Generate digest + run: | + digest=$(skopeo inspect oci-archive:$(realpath ${{ steps.rockcraft.outputs.rock }}) --format '{{.Digest}}') + echo "digest=${digest#*:}" >> "$GITHUB_OUTPUT" + - name: Install Syft + run: | + curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin + - name: Create SBOM + run: syft $(realpath ${{ steps.rockcraft.outputs.rock }}) -o spdx-json=cos-registration-server.sbom.json + + - name: Upload SBOM + uses: actions/upload-artifact@v3 + with: + name: cos-registration-server-sbom + path: "cos-registration-server.sbom.json" From 53513d0215b6f3addea2a9908aaead9c1f2e591e Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Fri, 2 Feb 2024 14:07:59 +0100 Subject: [PATCH 46/49] feat(dashboard): write dashboards in a directory --- cos_registration_server/api/dashboards.py | 37 +++++++++++++++++++ .../api/management/__init__.py | 0 .../api/management/commands/__init__.py | 0 .../commands/update_all_dashboards.py | 9 +++++ cos_registration_server/api/serializer.py | 2 +- cos_registration_server/api/tests.py | 2 +- cos_registration_server/api/views.py | 6 +++ .../cos_registration_server/settings.py | 5 +++ cos_registration_server/devices/models.py | 2 +- rock/local/configure.bash | 4 ++ 10 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 cos_registration_server/api/dashboards.py create mode 100644 cos_registration_server/api/management/__init__.py create mode 100644 cos_registration_server/api/management/commands/__init__.py create mode 100644 cos_registration_server/api/management/commands/update_all_dashboards.py diff --git a/cos_registration_server/api/dashboards.py b/cos_registration_server/api/dashboards.py new file mode 100644 index 0000000..60d9e15 --- /dev/null +++ b/cos_registration_server/api/dashboards.py @@ -0,0 +1,37 @@ +import json +from os import mkdir, remove +from pathlib import Path +from shutil import rmtree + +from devices.models import Device +from django.conf import settings + + +def add_dashboards(device): + dashboard_path = Path(settings.GRAFANA_DASHBOARD_PATH) + for dashboard in device.grafana_dashboards: + dashboard_title = dashboard["title"].replace(" ", "_") + dashboard["title"] = f'{device.uid}-{dashboard["title"]}' + dashboard_file = f"{device.uid}-{dashboard_title}.json" + with open(dashboard_path / dashboard_file, "w") as file: + json.dump(dashboard, file) + + +def delete_dashboards(device): + dashboard_path = Path(settings.GRAFANA_DASHBOARD_PATH) + + def _is_device_dashboard(p: Path) -> bool: + return p.is_file() and p.name.startswith(device.uid) + + for dashboard in filter( + _is_device_dashboard, Path(dashboard_path).glob("*") + ): + remove(dashboard) + + +def update_all_dashboards(): + dashboard_path = Path(settings.GRAFANA_DASHBOARD_PATH) + rmtree(dashboard_path, ignore_errors=True) + mkdir(dashboard_path) + for device in Device.objects.all(): + add_dashboards(device) diff --git a/cos_registration_server/api/management/__init__.py b/cos_registration_server/api/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cos_registration_server/api/management/commands/__init__.py b/cos_registration_server/api/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cos_registration_server/api/management/commands/update_all_dashboards.py b/cos_registration_server/api/management/commands/update_all_dashboards.py new file mode 100644 index 0000000..abb8886 --- /dev/null +++ b/cos_registration_server/api/management/commands/update_all_dashboards.py @@ -0,0 +1,9 @@ +from api.dashboards import update_all_dashboards +from django.core.management.base import BaseCommand + + +class Command(BaseCommand): + help = "Update all the grafana dashboards in the folder" + + def handle(self, *args, **options): + update_all_dashboards() diff --git a/cos_registration_server/api/serializer.py b/cos_registration_server/api/serializer.py index 00c6849..a356543 100644 --- a/cos_registration_server/api/serializer.py +++ b/cos_registration_server/api/serializer.py @@ -41,4 +41,4 @@ def validate_grafana_dashboards(self, value): raise serializers.ValidationError( "gafana_dashboards is not a supported format (list)." ) - return value + return dashboards diff --git a/cos_registration_server/api/tests.py b/cos_registration_server/api/tests.py index 81d4588..2fc9749 100644 --- a/cos_registration_server/api/tests.py +++ b/cos_registration_server/api/tests.py @@ -104,7 +104,7 @@ def test_grafana_dashboard_not_in_a_list(self): self.assertContains( response, '{"grafana_dashboards": ["gafana_dashboards is not a supported ' - 'format (list).', + "format (list).", status_code=400, ) diff --git a/cos_registration_server/api/views.py b/cos_registration_server/api/views.py index 8d60289..9efee76 100644 --- a/cos_registration_server/api/views.py +++ b/cos_registration_server/api/views.py @@ -3,6 +3,8 @@ from django.http import HttpResponse, JsonResponse from rest_framework.parsers import JSONParser +from .dashboards import add_dashboards, delete_dashboards + def devices(request): if request.method == "GET": @@ -16,6 +18,7 @@ def devices(request): serialized = DeviceSerializer(data=data) if serialized.is_valid(): serialized.save() + add_dashboards(serialized.instance) return JsonResponse(serialized.data, status=201) return JsonResponse(serialized.errors, status=400) @@ -34,8 +37,11 @@ def device(request, uid): serialized = DeviceSerializer(device, data=data, partial=True) if serialized.is_valid(): serialized.save() + delete_dashboards(serialized.instance) + add_dashboards(serialized.instance) return JsonResponse(serialized.data) return JsonResponse(serialized.errors, status=400) elif request.method == "DELETE": device.delete() + delete_dashboards(device) return HttpResponse(status=204) diff --git a/cos_registration_server/cos_registration_server/settings.py b/cos_registration_server/cos_registration_server/settings.py index e6a5453..3dfee55 100644 --- a/cos_registration_server/cos_registration_server/settings.py +++ b/cos_registration_server/cos_registration_server/settings.py @@ -146,3 +146,8 @@ # To keep the reverse proxy prefix when forwarding. USE_X_FORWARDED_HOST = True + +# Where robot dashboards are written on disk +GRAFANA_DASHBOARD_PATH = os.getenv( + "GRAFANA_DASHBOARD_PATH", "grafana_dashboards/" +) diff --git a/cos_registration_server/devices/models.py b/cos_registration_server/devices/models.py index e311e76..a5409a7 100644 --- a/cos_registration_server/devices/models.py +++ b/cos_registration_server/devices/models.py @@ -6,7 +6,7 @@ def default_dashboards_json_field(): - return "[]" + return [] class Device(models.Model): diff --git a/rock/local/configure.bash b/rock/local/configure.bash index a94987a..1a9521c 100755 --- a/rock/local/configure.bash +++ b/rock/local/configure.bash @@ -3,6 +3,10 @@ cd /lib/python3.10/site-packages/cos_registration_server export DATABASE_BASE_DIR_DJANGO="/server_data" +export SECRET_KEY_DJANGO=$(cat /server_data/secret_key) # migrate the database /bin/python3 manage.py migrate + +# update all the dashboards +python3 manage.py update_all_dashboards From 96e288367a69d7ce1547c1517a319ad3d66388f9 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Wed, 7 Feb 2024 17:57:21 +0100 Subject: [PATCH 47/49] fix(tests): clean tests for grafana_dashboards changes --- cos_registration_server/api/tests.py | 27 ++++++++++++++---------- cos_registration_server/devices/tests.py | 20 +++++++----------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/cos_registration_server/api/tests.py b/cos_registration_server/api/tests.py index 2fc9749..22660e5 100644 --- a/cos_registration_server/api/tests.py +++ b/cos_registration_server/api/tests.py @@ -1,28 +1,29 @@ import json +from os import mkdir +from shutil import rmtree from devices.models import Device, default_dashboards_json_field from django.urls import reverse from django.utils import timezone from rest_framework.test import APITestCase -SIMPLE_GRAFANA_DASHBOARD = """{ - "dashboard": { - "id": null, - "uid": null, +SIMPLE_GRAFANA_DASHBOARD = { + "id": None, + "uid": None, "title": "Production Overview", - "tags": [ "templated" ], + "tags": ["templated"], "timezone": "browser", "schemaVersion": 16, - "refresh": "25s" - }, - "message": "Made changes to xyz", - "overwrite": false -}""" + "refresh": "25s", +} class DevicesViewTests(APITestCase): def setUp(self): self.url = reverse("api:devices") + self.grafana_dashboards_path = "grafana_dashboards" + rmtree(self.grafana_dashboards_path, ignore_errors=True) + mkdir(self.grafana_dashboards_path) def create_device(self, **fields): data = {} @@ -38,7 +39,7 @@ def test_get_nothing(self): def test_create_device(self): uid = "robot-1" address = "192.168.0.1" - custom_grafana_dashboards = eval(default_dashboards_json_field()) + custom_grafana_dashboards = default_dashboards_json_field() custom_grafana_dashboards.append(SIMPLE_GRAFANA_DASHBOARD) response = self.create_device( uid=uid, @@ -176,6 +177,10 @@ def test_patch_dashboards(self): response = self.client.patch(self.url(uid), data, format="json") self.assertEqual(response.status_code, 200) content_json = json.loads(response.content) + # necessary since patching returns the modified title + SIMPLE_GRAFANA_DASHBOARD[ + "title" + ] = f'{uid}-{SIMPLE_GRAFANA_DASHBOARD["title"]}' self.assertEqual( content_json["grafana_dashboards"][0], SIMPLE_GRAFANA_DASHBOARD, diff --git a/cos_registration_server/devices/tests.py b/cos_registration_server/devices/tests.py index 231ebe7..f842ea5 100644 --- a/cos_registration_server/devices/tests.py +++ b/cos_registration_server/devices/tests.py @@ -6,19 +6,15 @@ from .models import Device, default_dashboards_json_field -SIMPLE_GRAFANA_DASHBOARD = """{ - "dashboard": { - "id": null, - "uid": null, +SIMPLE_GRAFANA_DASHBOARD = { + "id": None, + "uid": None, "title": "Production Overview", - "tags": [ "templated" ], + "tags": ["templated"], "timezone": "browser", "schemaVersion": 16, - "refresh": "25s" - }, - "message": "Made changes to xyz", - "overwrite": false -}""" + "refresh": "25s", +} class DeviceModelTests(TestCase): @@ -43,7 +39,7 @@ def test_device_str(self): self.assertEqual(str(device), "hello-123") def test_device_grafana_dashboards(self): - custom_grafana_dashboards = eval(default_dashboards_json_field()) + custom_grafana_dashboards = default_dashboards_json_field() custom_grafana_dashboards.append(SIMPLE_GRAFANA_DASHBOARD) device = Device( uid="hello-123", @@ -52,7 +48,7 @@ def test_device_grafana_dashboards(self): grafana_dashboards=custom_grafana_dashboards, ) self.assertEqual( - str(device.grafana_dashboards[0]), + device.grafana_dashboards[0], SIMPLE_GRAFANA_DASHBOARD, ) From 356e9b3f2a9cb10a648b5383bce70a338a785161 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Wed, 7 Feb 2024 18:27:40 +0100 Subject: [PATCH 48/49] test(dashboards): dashboards files are properly created --- cos_registration_server/api/tests.py | 99 +++++++++++++++++++++------- 1 file changed, 74 insertions(+), 25 deletions(-) diff --git a/cos_registration_server/api/tests.py b/cos_registration_server/api/tests.py index 22660e5..e62b328 100644 --- a/cos_registration_server/api/tests.py +++ b/cos_registration_server/api/tests.py @@ -1,5 +1,6 @@ import json -from os import mkdir +from os import mkdir, path +from pathlib import Path from shutil import rmtree from devices.models import Device, default_dashboards_json_field @@ -7,24 +8,24 @@ from django.utils import timezone from rest_framework.test import APITestCase -SIMPLE_GRAFANA_DASHBOARD = { - "id": None, - "uid": None, - "title": "Production Overview", - "tags": ["templated"], - "timezone": "browser", - "schemaVersion": 16, - "refresh": "25s", -} - class DevicesViewTests(APITestCase): def setUp(self): self.url = reverse("api:devices") - self.grafana_dashboards_path = "grafana_dashboards" + self.grafana_dashboards_path = Path("grafana_dashboards") rmtree(self.grafana_dashboards_path, ignore_errors=True) mkdir(self.grafana_dashboards_path) + self.simple_grafana_dashboard = { + "id": None, + "uid": None, + "title": "Production Overview", + "tags": ["templated"], + "timezone": "browser", + "schemaVersion": 16, + "refresh": "25s", + } + def create_device(self, **fields): data = {} for field, value in fields.items(): @@ -40,7 +41,7 @@ def test_create_device(self): uid = "robot-1" address = "192.168.0.1" custom_grafana_dashboards = default_dashboards_json_field() - custom_grafana_dashboards.append(SIMPLE_GRAFANA_DASHBOARD) + custom_grafana_dashboards.append(self.simple_grafana_dashboard) response = self.create_device( uid=uid, address=address, @@ -58,6 +59,15 @@ def test_create_device(self): self.assertEqual( Device.objects.get().grafana_dashboards, custom_grafana_dashboards ) + with open( + self.grafana_dashboards_path / "robot-1-Production_Overview.json", + "r", + ) as file: + dashboard_data = json.load(file) + self.simple_grafana_dashboard[ + "title" + ] = f'{uid}-{self.simple_grafana_dashboard["title"]}' + self.assertEqual(dashboard_data, self.simple_grafana_dashboard) def test_create_multiple_devices(self): devices = [ @@ -99,7 +109,7 @@ def test_grafana_dashboard_not_in_a_list(self): response = self.create_device( uid=uid, address=address, - grafana_dashboards=SIMPLE_GRAFANA_DASHBOARD, + grafana_dashboards=self.simple_grafana_dashboard, ) self.assertEqual(Device.objects.count(), 0) self.assertContains( @@ -128,11 +138,27 @@ def test_grafana_dashboard_illformed_json(self): class DeviceViewTests(APITestCase): + def setUp(self): + self.grafana_dashboards_path = Path("grafana_dashboards") + rmtree(self.grafana_dashboards_path, ignore_errors=True) + mkdir(self.grafana_dashboards_path) + self.simple_grafana_dashboard = { + "id": None, + "uid": None, + "title": "Production Overview", + "tags": ["templated"], + "timezone": "browser", + "schemaVersion": 16, + "refresh": "25s", + } + def url(self, uid): return reverse("api:device", args=(uid,)) - def create_device(self, uid, address): - data = {"uid": uid, "address": address} + def create_device(self, **fields): + data = {} + for field, value in fields.items(): + data[field] = value url = reverse("api:devices") return self.client.post(url, data, format="json") @@ -143,7 +169,7 @@ def test_get_nonexistent_device(self): def test_get_device(self): uid = "robot-1" address = "192.168.1.2" - self.create_device(uid, address) + self.create_device(uid=uid, address=address) response = self.client.get(self.url(uid)) self.assertEqual(response.status_code, 200) content_json = json.loads(response.content) @@ -160,7 +186,7 @@ def test_get_device(self): def test_patch_device(self): uid = "robot-1" address = "192.168.1.2" - self.create_device(uid, address) + self.create_device(uid=uid, address=address) address = "192.168.1.200" data = {"address": address} response = self.client.patch(self.url(uid), data, format="json") @@ -172,25 +198,31 @@ def test_patch_device(self): def test_patch_dashboards(self): uid = "robot-1" address = "192.168.1.2" - self.create_device(uid, address) - data = {"grafana_dashboards": [SIMPLE_GRAFANA_DASHBOARD]} + self.create_device(uid=uid, address=address) + data = {"grafana_dashboards": [self.simple_grafana_dashboard]} response = self.client.patch(self.url(uid), data, format="json") self.assertEqual(response.status_code, 200) content_json = json.loads(response.content) # necessary since patching returns the modified title - SIMPLE_GRAFANA_DASHBOARD[ + self.simple_grafana_dashboard[ "title" - ] = f'{uid}-{SIMPLE_GRAFANA_DASHBOARD["title"]}' + ] = f'{uid}-{self.simple_grafana_dashboard["title"]}' self.assertEqual( content_json["grafana_dashboards"][0], - SIMPLE_GRAFANA_DASHBOARD, + self.simple_grafana_dashboard, ) self.assertEqual(content_json["address"], address) + with open( + self.grafana_dashboards_path / "robot-1-Production_Overview.json", + "r", + ) as file: + dashboard_data = json.load(file) + self.assertEqual(dashboard_data, self.simple_grafana_dashboard) def test_invalid_patch_device(self): uid = "robot-1" address = "192.168.1.2" - self.create_device(uid, address) + self.create_device(uid=uid, address=address) address = "192.168.1" # invalid IP data = {"address": address} response = self.client.patch(self.url(uid), data, format="json") @@ -199,10 +231,27 @@ def test_invalid_patch_device(self): def test_delete_device(self): uid = "robot-1" address = "192.168.1.2" - self.create_device(uid, address) + self.create_device( + uid=uid, + address=address, + grafana_dashboards=[self.simple_grafana_dashboard], + ) response = self.client.get(self.url(uid)) self.assertEqual(response.status_code, 200) + self.assertTrue( + path.isfile( + self.grafana_dashboards_path + / "robot-1-Production_Overview.json" + ) + ) response = self.client.delete(self.url(uid)) self.assertEqual(response.status_code, 204) response = self.client.get(self.url(uid)) self.assertEqual(response.status_code, 404) + self.assertFalse( + path.isfile( + self.grafana_dashboards_path + / "robot-1-Production_Overview.json" + ) + ) + From 89ccdd9bcf27abf797ce87dd76ab19b2aafd95c8 Mon Sep 17 00:00:00 2001 From: Guillaumebeuzeboc Date: Wed, 7 Feb 2024 18:27:57 +0100 Subject: [PATCH 49/49] test(update_all_dashboard): test django command --- cos_registration_server/api/tests.py | 44 ++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/cos_registration_server/api/tests.py b/cos_registration_server/api/tests.py index e62b328..8039700 100644 --- a/cos_registration_server/api/tests.py +++ b/cos_registration_server/api/tests.py @@ -4,6 +4,8 @@ from shutil import rmtree from devices.models import Device, default_dashboards_json_field +from django.core.management import call_command +from django.test import TestCase from django.urls import reverse from django.utils import timezone from rest_framework.test import APITestCase @@ -255,3 +257,45 @@ def test_delete_device(self): ) ) + +class CommandsTestCase(TestCase): + def setUp(self): + self.grafana_dashboards_path = Path("grafana_dashboards") + rmtree(self.grafana_dashboards_path, ignore_errors=True) + mkdir(self.grafana_dashboards_path) + self.simple_grafana_dashboard = { + "id": None, + "uid": None, + "title": "Production Overview", + "tags": ["templated"], + "timezone": "browser", + "schemaVersion": 16, + "refresh": "25s", + } + + def test_update_all_dashboards(self): + robot_1 = Device( + uid="robot-1", + address="127.0.0.1", + grafana_dashboards=[self.simple_grafana_dashboard], + ) + robot_1.save() + robot_2 = Device( + uid="robot-2", + address="127.0.0.1", + grafana_dashboards=[self.simple_grafana_dashboard], + ) + robot_2.save() + call_command("update_all_dashboards") + self.assertTrue( + path.isfile( + self.grafana_dashboards_path + / "robot-1-Production_Overview.json" + ) + ) + self.assertTrue( + path.isfile( + self.grafana_dashboards_path + / "robot-2-Production_Overview.json" + ) + )