[#IOPID-2346] Add alert rule for profile deletion poison queue + move alerts to error action group

src/domains/elt/_modules/function_apps/data.tf

@@ -8,6 +8,11 @@ data "azurerm_monitor_action_group" "error_action_group" {
   resource_group_name = local.resource_group_name_common
 }
 
+data "azurerm_monitor_action_group" "quarantine_error_action_group" {
+  name                = "${replace(var.project, "-", "")}quarantineerror"
+  resource_group_name = local.resource_group_name_common
+}
+
 data "azurerm_monitor_action_group" "io_com_action_group" {
   name                = "io-p-com-error-ag-01"
   resource_group_name = "io-p-itn-msgs-rg-01"

src/domains/elt/_modules/function_apps/monitor.tf

@@ -1,8 +1,3 @@
-data "azurerm_monitor_action_group" "quarantine_error_action_group" {
-  resource_group_name = local.resource_group_name_common
-  name                = "${replace(var.project, "-", "")}quarantineerror"
-}
-
 resource "azurerm_monitor_diagnostic_setting" "queue_diagnostic_setting" {
   name                       = "${var.project}-fnelt-internal-st-queue-ds-01"
   target_resource_id         = "${data.azurerm_storage_account.function_elt_internal_storage.id}/queueServices/default"
@@ -50,7 +45,7 @@ resource "azurerm_monitor_scheduled_query_rules_alert_v2" "service_preferences_f
 
   action {
     action_groups = [
-      data.azurerm_monitor_action_group.quarantine_error_action_group.id,
+      data.azurerm_monitor_action_group.error_action_group.id,
     ]
   }
 
@@ -84,7 +79,41 @@ resource "azurerm_monitor_scheduled_query_rules_alert_v2" "profiles_failure_aler
 
   action {
     action_groups = [
-      data.azurerm_monitor_action_group.quarantine_error_action_group.id,
+      data.azurerm_monitor_action_group.error_action_group.id,
+    ]
+  }
+
+  tags = var.tags
+}
+
+resource "azurerm_monitor_scheduled_query_rules_alert_v2" "profile_deletion_failure_alert_rule" {
+  enabled             = true
+  name                = "[CITIZEN-AUTH | iopfneltsdt] Failures on ${local.profile_deletion_failure_queue_name}-poison"
+  resource_group_name = var.resource_group_name
+  location            = var.location
+
+  scopes                  = [data.azurerm_storage_account.function_elt_internal_storage.id]
+  description             = "Permanent failures processing Profiles export to PDND. REQUIRED MANUAL ACTION"
+  severity                = 1
+  auto_mitigation_enabled = false
+
+  window_duration      = "PT15M" # Select the interval that's used to group the data points by using the aggregation type function. Choose an Aggregation granularity (period) that's greater than the Frequency of evaluation to reduce the likelihood of missing the first evaluation period of an added time series.
+  evaluation_frequency = "PT15M" # Select how often the alert rule is to be run. Select a frequency that's smaller than the aggregation granularity to generate a sliding window for the evaluation.
+
+  criteria {
+    query                   = <<-QUERY
+      StorageQueueLogs
+        | where OperationName contains "PutMessage"
+        | where Uri contains "${local.profile_deletion_failure_queue_name}-poison"
+      QUERY
+    operator                = "GreaterThan"
+    threshold               = 0
+    time_aggregation_method = "Count"
+  }
+
+  action {
+    action_groups = [
+      data.azurerm_monitor_action_group.error_action_group.id,
     ]
   }