wip

src/elastic-cloud/02_elastic_organization/01_organization.tf

@@ -5,6 +5,15 @@ resource "ec_organization" "pagopa_ec_org" {
     },
     "[email protected]" = {
       organization_role = "organization-admin"
+    },
+    "[email protected]" = {
+      organization_role = "organization-admin"
+    },
+    "[email protected]" = {
+      organization_role = "organization-admin"
+    },
+    "[email protected]" = {
+      organization_role = "organization-admin"
     }
   }
 

src/elastic-cloud/05_elstic_resources_app/02_application.tf

@@ -1,7 +1,22 @@
+resource "elasticstack_kibana_space" "kibana_space" {
+  for_each = local.spaces
+  space_id          = "${each.value}-${var.env}"
+  name              = "${each.value}-${var.env}"
+  description       = "Space for ${each.value}-${var.env}"
+  disabled_features = []
+}
+
+
 module "app_resources" {
   source = "./tf_module"
   for_each = local.configurations
 
-  configuration = each.value
+  configuration = each.value.conf
   default_snapshot_policy_name = var.default_snapshot_policy_name
+  env = var.env
+  space_id = elasticstack_kibana_space.kibana_space[each.value.space_name].id
+  dashboard_folder = each.value.dashboard_folder
 }
+
+
+

src/elastic-cloud/05_elstic_resources_app/99_locals.tf

@@ -1,9 +1,17 @@
 locals {
-  prefix = "pagopa"
-  config_files = fileset(path.module, "config/*.json" )
-  configurations ={ for f in local.config_files : basename(f) =>  jsondecode(templatefile(f, {
-    env = var.env
-    env_separator = "${var.env}-"
-    wait_for_snapshot = var.lifecycle_policy_wait_for_snapshot
-  }))}
+  prefix             = "pagopa"
+  config_folder_name = "config"
+  config_files = fileset(path.module, "${local.config_folder_name}/*/*.json")
+  configurations     = {
+    for f in local.config_files : basename(f) => {
+      conf = jsondecode(templatefile(f, {
+            env               = var.env
+            env_separator     = "${var.env}-"
+            wait_for_snapshot = var.lifecycle_policy_wait_for_snapshot
+          }))
+      space_name = substr(dirname(f), length("${local.config_folder_name}/"), -1)
+      dashboard_folder = "${dirname(f)}/dashboards"
+    }
+  }
+  spaces = { for f in local.config_files : substr(dirname(f), length("${local.config_folder_name}/"), -1) => substr(dirname(f), length("${local.config_folder_name}/"), -1)}
 }

src/elastic-cloud/05_elstic_resources_app/config/README.md

@@ -0,0 +1,21 @@
+# kibana space configuration
+
+
+
+this folder contains the kibana spaces, indexes and data stream configuration for every application.
+
+
+folder structure:
+
+```commandline
+├── ndp //determines the name of the kibana space
+│ └── nodo.json // resources and visualizations associated with the above space
+└── printit // another kibana space
+    ├── dashboards // dashboard for this application
+    │ └── dashboard_massive.ndjson
+    └── printit.json
+```
+
+
+Define a new json file containing all the required information to allow the creation of a space and all required resource to manage the logs of an application.
+The file name must match

src/elastic-cloud/05_elstic_resources_app/config/ndp/nodo.json

@@ -0,0 +1,169 @@
+{
+  "id": "nodo",
+  "displayName": "Nodo dei Pagamenti ${env}",
+  "ingestPipeline": {
+  "processors": [
+    {
+      "json": {
+        "field": "message",
+        "target_field": "json_message",
+        "on_failure" : [
+          {
+            "set" : {
+              "field" : "error-info.message",
+              "value" : "Field 'message' is not a json format and not create 'json_message"
+            }
+          },
+          {
+            "set": {
+              "description": "Record error detail",
+              "field": "error-info.message-detail",
+              "value": "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message {{ _ingest.on_failure_message }}"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "set": {
+        "field": "app",
+        "copy_from": "json_message.app",
+        "ignore_failure": true,
+        "on_failure" : [
+          {
+            "set" : {
+              "field" : "error-info.app",
+              "value" : "Field 'json_message.app' not found"
+            }
+          },
+          {
+            "set": {
+              "description": "Record error detail",
+              "field": "error-info.app-detail",
+              "value": "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message {{ _ingest.on_failure_message }}"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "set": {
+        "if": "ctx.app?.reJsonLog != 'true'",
+        "field": "message",
+        "copy_from": "json_message.message",
+        "ignore_failure": true,
+        "on_failure" : [
+          {
+            "set" : {
+              "field" : "error-info.message",
+              "value" : "Field 'json_message.message' not found"
+            }
+          },
+          {
+            "set": {
+              "description": "Record error detail",
+              "field": "error-info.message-detail",
+              "value": "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message {{ _ingest.on_failure_message }}"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "json": {
+        "if": "ctx.app?.reJsonLog == 'true'",
+        "field": "json_message.message",
+        "target_field": "re",
+        "on_failure" : [
+          {
+            "set" : {
+              "field" : "error-re-info.message",
+              "value" : "Field 'json_message.message' is not a json format and not create 're"
+            }
+          },
+          {
+            "set": {
+              "description": "Record error detail",
+              "field": "error-re-info.message-detail",
+              "value": "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message {{ _ingest.on_failure_message }}"
+            }
+          },
+          {
+            "remove": {
+              "field": "json_message",
+              "ignore_missing": true
+            }
+          }
+        ]
+      }
+    },
+    {
+      "set": {
+        "if": "ctx.app?.reJsonLog == 'true'",
+        "field": "message",
+        "copy_from": "re.internalMessage",
+        "ignore_failure": true,
+        "on_failure" : [
+          {
+            "set" : {
+              "field" : "error-re-internalMessage-info.message",
+              "value" : "Field 're.internalMessage' not found"
+            }
+          },
+          {
+            "set": {
+              "description": "Record error detail",
+              "field": "error-re-internalMessage-info.message-detail",
+              "value": "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message {{ _ingest.on_failure_message }}"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "drop": {
+        "if": "ctx.app?.reXmlLog == 'true'"
+      }
+    },
+    {
+      "remove": {
+        "field": "json_message",
+        "ignore_missing": true
+      }
+    }
+  ]
+},
+  "ilm": {
+    "hot": {
+      "minAge": "0m",
+      "rollover": {
+        "maxPrimarySize": "50gb",
+        "maxAge": "2d"
+      }
+    },
+    "warm": {
+      "minAge": "2d",
+      "setPriority": 50
+    },
+    "cold": {
+      "minAge": "4d",
+      "setPriority": 0
+    },
+    "delete": {
+      "minAge": "7d",
+      "deleteSearchableSnapshot": true,
+      "waitForSnapshot": ${wait_for_snapshot}
+    }
+  },
+  "indexTemplate": {
+    "indexPattern" : "logs-${env_separator}print-payment-*"
+  },
+  "dataStream": [
+    "logs-${env_separator}print-payment-notice-service-default",
+    "logs-${env_separator}print-payment-notice-generator-default",
+    "logs-${env_separator}print-payment-notice-functions-default"
+  ],
+  "dataView": {
+    "indexIdentifier": "${env_separator}print-payment"
+  }
+}

src/elastic-cloud/05_elstic_resources_app/config/printit.json → src/elastic-cloud/05_elstic_resources_app/config/printit/printit.json

@@ -1,9 +1,6 @@
 {
-  "id": "printit-${env}",
+  "id": "printit",
   "displayName": "Print It ${env}",
-  "space": {
-    "disabledFeatures": []
-  },
   "ingestPipeline": {
     "processors": [
       {

src/elastic-cloud/05_elstic_resources_app/tf_module/local_defaults.tf

@@ -0,0 +1,5 @@
+locals {
+  default_ilm = {
+
+  }
+}

src/elastic-cloud/05_elstic_resources_app/tf_module/resources.tf

@@ -1,27 +1,19 @@
 locals {
   data_streams = { for d in var.configuration.dataStream : d =>  d}
+  application_id = "${var.configuration.id}-${var.env}"
+  dashboards = { for df in fileset("${var.dashboard_folder}", "/*.ndjson") : basename(df) => "${var.dashboard_folder}/${df}" }
 }
 
-output "conf" {
-  value = var.configuration
-}
-
-resource "elasticstack_kibana_space" "kibana_space" {
-  space_id          = var.configuration.id
-  name              = "${var.configuration.displayName}"
-  description       = "Space for ${var.configuration.displayName}"
-  disabled_features = var.configuration.space.disabledFeatures
-}
 
 resource "elasticstack_elasticsearch_ingest_pipeline" "ingest_pipeline" {
-  name        = "${var.configuration.id}-pipeline"
+  name        = "${local.application_id}-pipeline"
   description = "Ingest pipeline for ${var.configuration.displayName}"
 
   processors = [ for p in var.configuration.ingestPipeline.processors : jsonencode(p)]
 }
 
 resource "elasticstack_elasticsearch_index_lifecycle" "index_lifecycle" {
-  name = "${var.configuration.id}-ilm"
+  name = "${local.application_id}-ilm"
 
   hot {
     min_age = var.configuration.ilm.hot.minAge
@@ -64,31 +56,31 @@ resource "elasticstack_elasticsearch_index_lifecycle" "index_lifecycle" {
 }
 
 resource "elasticstack_elasticsearch_component_template" "component_template" {
-  name = "${var.configuration.id}@custom"
+  name = "${local.application_id}@custom"
 
   template {
 
     settings = jsonencode({
        "index": {
-        "default_pipeline": elasticstack_elasticsearch_ingest_pipeline.ingest_pipeline.name, #fixme count index
+        "default_pipeline": elasticstack_elasticsearch_ingest_pipeline.ingest_pipeline.name,
         "lifecycle": {
-          "name": elasticstack_elasticsearch_index_lifecycle.index_lifecycle.name #fixme count index
+          "name": elasticstack_elasticsearch_index_lifecycle.index_lifecycle.name
         }
       }
     })
   }
 
   metadata = jsonencode({
-    description = "Settings for ${var.configuration.id}"
+    description = "Settings for ${local.application_id}"
   })
 }
 
 resource "elasticstack_elasticsearch_index_template" "index_template" {
-  name = "${var.configuration.id}-idxtpl"
+  name = "${local.application_id}-idxtpl"
 
   priority       = 500
   index_patterns = [ var.configuration.indexTemplate.indexPattern ]
-  composed_of = [elasticstack_elasticsearch_component_template.component_template.name] #fixme count index
+  composed_of = [elasticstack_elasticsearch_component_template.component_template.name]
 
   data_stream {
     allow_custom_routing = false
@@ -106,7 +98,7 @@ resource "elasticstack_elasticsearch_index_template" "index_template" {
   }
 
   metadata = jsonencode({
-    "description" = "Index template for ${var.configuration.id}"
+    "description" = "Index template for ${local.application_id}"
   })
 }
 
@@ -123,11 +115,19 @@ resource "elasticstack_elasticsearch_data_stream" "data_stream" {
 
 
 resource "elasticstack_kibana_data_view" "kibana_data_view" {
-  space_id = elasticstack_kibana_space.kibana_space.id #fixme count index
+  space_id = var.space_id
   data_view = {
     name            = "Log ${var.configuration.dataView.indexIdentifier}"
     title           = "logs-${var.configuration.dataView.indexIdentifier}-*"
     time_field_name = "@timestamp"
   }
 
 }
+
+resource "elasticstack_kibana_import_saved_objects" "dashboard" {
+  for_each = local.dashboards
+
+  overwrite     = true
+
+  file_contents = file(each.value)
+}

src/elastic-cloud/05_elstic_resources_app/tf_module/variables.tf

@@ -5,3 +5,15 @@ variable "configuration" {
 variable "default_snapshot_policy_name" {
   type = string
 }
+
+variable "space_id" {
+  type = string
+}
+
+variable "env" {
+  type = string
+}
+
+variable "dashboard_folder" {
+  type = string
+}