Validation logging refactor 🧝🏼

CHANGELOG.md

@@ -7,7 +7,9 @@
 - **Added**: Advanced admin filters
 - **Added**: [Basic job history](https://github.com/FIIT-Databases/tester/issues/3)
 - **Added**: Creator in Evaluation
+- **Added**: [Show request method #4](https://github.com/FIIT-Databases/tester/issues/4)
 - **Changed**: Evaluations are managed from Django admin now
+- **Changed**: Major task record logging refactor (messages introduced)
 - **Changed**: Django 5.0 upgrade
 - **Changed**: Use Debian-based containers
 

apps/core/jobs.py

@@ -4,7 +4,6 @@
 import random
 import string
 from difflib import HtmlDiff
-from http import HTTPStatus
 from json import JSONDecodeError
 from time import sleep
 from typing import Optional
@@ -111,7 +110,7 @@ def run(self):
             else:
                 url = f"http://127.0.0.1:9050{scenario.url}"
 
-            record = TaskRecord(task=self._task, scenario=scenario, url=url)
+            record = TaskRecord(task=self._task, scenario=scenario, url=url, status=TaskRecord.Status.OK)
 
             s = Session()
             # retry = Retry(connect=6, backoff_factor=2)
@@ -130,78 +129,55 @@ def run(self):
                 r = s.send(req.prepare(), timeout=settings.DBS_TESTER_TIMEOUT)
             except Timeout as e:
                 record.status = TaskRecord.Status.TIMEOUT
-                record.message = str(e)
+                record.messages.append(str(e))
                 record.save()
                 continue
             except BaseException as e:
                 record.status = TaskRecord.Status.ERROR
-                record.message = str(e)
+                record.messages.append(str(e))
                 record.save()
                 continue
 
             record.duration = r.elapsed
-
-            try:
-                r.raise_for_status()
-            except HTTPError as e:
-                record.response = r.content
-                if r.status_code == scenario.status_code:
-                    record.status = TaskRecord.Status.OK
-                else:
-                    record.status = TaskRecord.Status.INVALID_HTTP_STATUS
-                record.message = str(e)
-                record.additional_data = {"status_code": r.status_code}
-                record.save()
-                s.close()
-                continue
+            record.response = r.content
 
             if r.status_code != scenario.status_code:
-                record.response = r.content
-                record.status = TaskRecord.Status.INVALID_HTTP_STATUS
-                record.message = str(TaskRecord.Status.INVALID_HTTP_STATUS)
-                record.save()
-                s.close()
-                continue
+                record.status = TaskRecord.Status.INVALID
+                record.messages.append(
+                    f"Invalid HTTP Status code (received={r.status_code}, expected={scenario.status_code})"
+                )
 
-            if r.status_code != HTTPStatus.NO_CONTENT:
+            if r.content:
                 try:
                     response = r.json()
+                    try:
+                        record.response = json.dumps(
+                            {key: response[key] for key in response if key not in (scenario.ignored_properties or [])},
+                            sort_keys=True,
+                            indent=4,
+                        )
+                    except TypeError:
+                        record.response = json.dumps(response, sort_keys=True, indent=4)
+
+                    valid_response = json.dumps(scenario.response, sort_keys=True, indent=4)
+
+                    if record.response != valid_response:
+                        valid_lines = valid_response.splitlines(keepends=True)
+                        response_lines = record.response.splitlines(keepends=True)
+                        d = HtmlDiff()
+                        record.diff_type = TaskRecord.DiffType.HTML
+                        record.diff = d.make_table(
+                            valid_lines,
+                            response_lines,
+                            fromdesc=_("Valid response"),
+                            todesc=_("Your response"),
+                        )
+                        record.status = TaskRecord.Status.INVALID
+                        record.messages.append(f"JSON Mismatch")
                 except (InvalidJSONError, JSONDecodeError) as e:
-                    record.response = r.content
-                    record.status = TaskRecord.Status.INVALID_JSON
-                    record.message = str(e)
-                    record.save()
-                    s.close()
-                    continue
-
-                s.close()
-
-                try:
-                    record.response = json.dumps(
-                        {key: response[key] for key in response if key not in (scenario.ignored_properties or [])},
-                        sort_keys=True,
-                        indent=4,
-                    )
-                except TypeError:
-                    record.response = json.dumps(response, sort_keys=True, indent=4)
-
-                valid_response = json.dumps(scenario.response, sort_keys=True, indent=4)
-
-                if record.response == valid_response:
-                    record.status = TaskRecord.Status.OK
-                else:
-                    valid_lines = valid_response.splitlines(keepends=True)
-                    response_lines = record.response.splitlines(keepends=True)
-                    d = HtmlDiff()
-                    record.diff_type = TaskRecord.DiffType.HTML
-                    record.diff = d.make_table(
-                        valid_lines,
-                        response_lines,
-                        fromdesc=_("Valid response"),
-                        todesc=_("Your response"),
-                    )
-                    record.status = TaskRecord.Status.MISMATCH
-
+                    record.status = TaskRecord.Status.INVALID
+                    record.messages.append("Invalid JSON")
+                    record.additional_data["exception"] = str(e)
                 record.save()
 
         self._task.status = Task.Status.DONE

apps/core/migrations/0012_task_record_messages.py

@@ -0,0 +1,42 @@
+# Generated by Django 5.0.2 on 2024-02-10 02:18
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0011_evaluation_creator"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="taskrecord",
+            name="message",
+        ),
+        migrations.AddField(
+            model_name="taskrecord",
+            name="messages",
+            field=models.JSONField(default=list),
+        ),
+        migrations.AlterField(
+            model_name="scenario",
+            name="response",
+            field=models.JSONField(null=True),
+        ),
+        migrations.AlterField(
+            model_name="taskrecord",
+            name="status",
+            field=models.CharField(
+                choices=[
+                    ("ok", "OK"),
+                    ("invalid", "Invalid"),
+                    ("invalid_json", "parse error or invalid content-type"),
+                    ("invalid_http_status", "invalid http status code"),
+                    ("timeout", "timeout"),
+                    ("mismatch", "response do not match"),
+                    ("error", "4xx or 500x"),
+                ],
+                max_length=20,
+            ),
+        ),
+    ]

apps/core/models/scenario.py

@@ -25,7 +25,7 @@ class Method(models.TextChoices):
     assigment = models.ForeignKey(Assignment, on_delete=models.CASCADE, related_name="scenarios")
     url = models.CharField(max_length=200, help_text=_("Endpoint handle (/v1/hello)"))
     method = models.CharField(choices=Method.choices, help_text=_("HTTP method"), max_length=10, default=Method.GET)
-    response = models.JSONField()
+    response = models.JSONField(null=True)
     body = models.JSONField(null=True, help_text=_("HTTP body"), blank=True)
     ignored_properties = ArrayField(models.CharField(max_length=50), null=True)
     status_code = models.SmallIntegerField(default=HTTPStatus.OK)

apps/core/models/task_record.py

@@ -15,10 +15,8 @@ class Meta:
 
     class Status(models.TextChoices):
         OK = "ok", _("OK")
-        INVALID_JSON = "invalid_json", _("parse error or invalid content-type")
-        INVALID_HTTP_STATUS = "invalid_http_status", _("invalid http status code")
-        TIMEOUT = "timeout", _("timeout")
-        MISMATCH = "mismatch", _("response do not match")
+        INVALID = "invalid", _("Invalid")
+        TIMEOUT = "timeout", _("Timeout")
         ERROR = "error", _("4xx or 500x")
 
     class DiffType(models.TextChoices):
@@ -32,7 +30,7 @@ class DiffType(models.TextChoices):
     response = models.TextField(null=True)
     diff = models.TextField(null=True)
     diff_type = models.TextField(choices=DiffType.choices, max_length=15, null=True)
-    message = models.TextField()
+    messages = models.JSONField(default=list)
     duration = models.DurationField(null=True)
     additional_data = models.JSONField(default=dict)