Hr update events bugfixes

CHANGES.md

@@ -1,3 +1,9 @@
+# 2023-11-24 (5.17.18)
+
+* Bugfix: Update nested table when nested field name has underscore.
+* Bugfix: Update parent table when parent table has shortname for update events.
+* Bugfix: Only check for row existence when table exists.
+
 # 2023-10-18 (5.17.17)
 
 * Bugfix: Ignore id when copying data from temp table to main table for nested tables.

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = amsterdam-schema-tools
-version = 5.17.17
+version = 5.17.18
 url = https://github.com/amsterdam/schema-tools
 license = Mozilla Public 2.0
 author = Team Data Diensten, van het Dataplatform onder de Directie Digitale Voorzieningen (Gemeente Amsterdam)

src/schematools/events/full.py

@@ -55,7 +55,7 @@ def parent_id_field(self):
     def parent_id_value(self, prepared_row: dict) -> str:
         return ".".join(
             [
-                str(prepared_row[to_snake_case(f"{self.parent_schema_table.id}_{fn}")])
+                str(prepared_row[to_snake_case(f"{self.parent_schema_table.shortname}_{fn}")])
                 for fn in self.parent_schema_table.identifier
             ]
         )
@@ -421,16 +421,16 @@ def _process_row(
 
         event_type = event_meta["event_type"]
 
-        if (
-            run_configuration.check_existence_on_add
-            and event_type == "ADD"
-            and self._row_exists_in_database(run_configuration, id_value)
-        ):
-            logger.info("Row with id %s already exists in database. Skipping.", row["id"])
-            return
-
         db_operation = None
         if run_configuration.update_table:
+            if (
+                run_configuration.check_existence_on_add
+                and event_type == "ADD"
+                and self._row_exists_in_database(run_configuration, id_value)
+            ):
+                logger.info("Row with id %s already exists in database. Skipping.", row["id"])
+                return
+
             db_operation_name, needs_select = EVENT_TYPE_MAPPINGS[event_type]
             db_operation = getattr(table, db_operation_name)()
 
@@ -472,7 +472,7 @@ def _update_nested_tables(
             if is_delete:
                 continue
 
-            if value := prepared_row.get(field.id, []):
+            if value := prepared_row.get(to_snake_case(field.id), []):
                 if rows := self._prepare_nested_rows(field, value, id_value):
                     self.conn.execute(table.insert(), rows)
 

tests/test_events_full.py

@@ -1248,6 +1248,78 @@ def test_full_load_shortnames(here, db_schema, tconn, local_metadata, hr_simple_
     assert records[0]["heeft_hoofdvestiging_id"] == "24902480"
 
 
+def test_full_load_shortnames_update(here, db_schema, tconn, local_metadata, hr_simple_schema):
+    importer = EventsProcessor([hr_simple_schema], tconn, local_metadata=local_metadata)
+
+    # First import an object with nested objects
+    events = [
+        (
+            {
+                "event_type": "ADD",
+                "event_id": 1,
+                "dataset_id": "hr",
+                "table_id": "maatschappelijkeactiviteiten",
+            },
+            {
+                "kvknummer": 42,
+                "email_adressen": [
+                    {
+                        "email_adres": "[email protected]",
+                    },
+                    {
+                        "email_adres": "[email protected]",
+                    },
+                ],
+            },
+        )
+    ]
+    importer.process_events(events)
+
+    # Not testing contents here, but merely the fact that the right tables are used without errors
+    records = [dict(r) for r in tconn.execute("SELECT * FROM hr_mac")]
+    assert len(records) == 1
+    assert records[0]["heeft_hoofdvestiging_id"] is None
+
+    nested_records = [dict(r) for r in tconn.execute("SELECT * FROM hr_mac_email_adressen")]
+    assert len(nested_records) == 2
+    assert nested_records[0]["parent_id"] == "42"
+    assert nested_records[0]["email_adres"] == "[email protected]"
+
+    # Now test adding a relation object that references a parent table with short name
+    events = [
+        (
+            {
+                "dataset_id": "hr",
+                "table_id": "maatschappelijkeactiviteiten_heeftHoofdvestiging",
+                "event_type": "ADD",
+                "event_id": 1658565091,
+                "tid": "42.AMSBI.24902480",
+                "generated_timestamp": "2023-10-05T09:59:05.314873",
+            },
+            {
+                "mac_kvknummer": "42",
+                "mac_id": "42",
+                "heeft_hoofdvestiging_vestigingsnummer": "24902480",
+                "heeft_hoofdvestiging_id": "24902480",
+                "begin_geldigheid": None,
+                "eind_geldigheid": None,
+                "id": 457172,
+            },
+        )
+    ]
+
+    importer.process_events(events)
+    rel_records = [dict(r) for r in tconn.execute("SELECT * FROM hr_mac_heeft_hoofdvestiging")]
+    assert len(rel_records) == 1
+    assert rel_records[0]["id"] == 457172
+    assert rel_records[0]["mac_id"] == "42"
+    assert rel_records[0]["heeft_hoofdvestiging_id"] == "24902480"
+
+    records = [dict(r) for r in tconn.execute("SELECT * FROM hr_mac")]
+    assert len(records) == 1
+    assert records[0]["heeft_hoofdvestiging_id"] == "24902480"
+
+
 def test_reset_lasteventid_after_incomplete_full_load(
     here, db_schema, tconn, local_metadata, nap_schema, gebieden_schema
 ):