From e5493c296d592c48dffe1d1f94fd227e90f315f1 Mon Sep 17 00:00:00 2001 From: Andrea Grillo Date: Mon, 8 Apr 2024 12:23:01 +0200 Subject: [PATCH] [EC-246] Move ELT resources to their own configuration (#940) --- .github/workflows/elt_cd.yaml | 18 ++ .github/workflows/elt_ci.yaml | 23 ++ src/core/README.md | 14 +- src/core/data.tf | 16 +- src/core/events.tf | 2 +- src/core/function_elt.tf | 264 ------------------ .../elt/_modules/function_apps/data.tf | 115 ++++++++ .../function_apps/function_app_elt.tf | 177 ++++++++++++ .../elt/_modules/function_apps/locals.tf | 18 ++ .../elt/_modules/function_apps/main.tf | 8 + .../elt/_modules/function_apps/outputs.tf | 8 + .../elt/_modules/function_apps/tests.tf | 3 + .../elt/_modules/function_apps/variables.tf | 69 +++++ src/domains/elt/_modules/networking/data.tf | 4 + src/domains/elt/_modules/networking/locals.tf | 3 + src/domains/elt/_modules/networking/main.tf | 8 + .../elt/_modules/networking/outputs.tf | 14 + .../elt/_modules/networking/subnet_elt.tf | 24 ++ .../elt/_modules/networking/variables.tf | 14 + .../elt/_modules/storage_accounts/main.tf | 8 + .../elt/_modules/storage_accounts/outputs.tf | 34 +++ .../storage_accounts/storage_account_elt.tf | 18 ++ .../storage_accounts/storage_tables.tf | 41 +++ .../_modules/storage_accounts/variables.tf | 19 ++ .../elt/prod/westeurope/.terraform.lock.hcl | 47 ++++ src/domains/elt/prod/westeurope/README.md | 34 +++ src/domains/elt/prod/westeurope/locals.tf | 17 ++ src/domains/elt/prod/westeurope/main.tf | 81 ++++++ src/domains/elt/prod/westeurope/outputs.tf | 21 ++ 29 files changed, 838 insertions(+), 284 deletions(-) create mode 100644 .github/workflows/elt_cd.yaml create mode 100644 .github/workflows/elt_ci.yaml create mode 100644 src/domains/elt/_modules/function_apps/data.tf create mode 100644 src/domains/elt/_modules/function_apps/function_app_elt.tf create mode 100644 src/domains/elt/_modules/function_apps/locals.tf create mode 100644 src/domains/elt/_modules/function_apps/main.tf create mode 100644 src/domains/elt/_modules/function_apps/outputs.tf create mode 100644 src/domains/elt/_modules/function_apps/tests.tf create mode 100644 src/domains/elt/_modules/function_apps/variables.tf create mode 100644 src/domains/elt/_modules/networking/data.tf create mode 100644 src/domains/elt/_modules/networking/locals.tf create mode 100644 src/domains/elt/_modules/networking/main.tf create mode 100644 src/domains/elt/_modules/networking/outputs.tf create mode 100644 src/domains/elt/_modules/networking/subnet_elt.tf create mode 100644 src/domains/elt/_modules/networking/variables.tf create mode 100644 src/domains/elt/_modules/storage_accounts/main.tf create mode 100644 src/domains/elt/_modules/storage_accounts/outputs.tf create mode 100644 src/domains/elt/_modules/storage_accounts/storage_account_elt.tf create mode 100644 src/domains/elt/_modules/storage_accounts/storage_tables.tf create mode 100644 src/domains/elt/_modules/storage_accounts/variables.tf create mode 100644 src/domains/elt/prod/westeurope/.terraform.lock.hcl create mode 100644 src/domains/elt/prod/westeurope/README.md create mode 100644 src/domains/elt/prod/westeurope/locals.tf create mode 100644 src/domains/elt/prod/westeurope/main.tf create mode 100644 src/domains/elt/prod/westeurope/outputs.tf diff --git a/.github/workflows/elt_cd.yaml b/.github/workflows/elt_cd.yaml new file mode 100644 index 000000000..e06f4bde5 --- /dev/null +++ b/.github/workflows/elt_cd.yaml @@ -0,0 +1,18 @@ +name: Continuous Delivery on elt + +on: + workflow_dispatch: + push: + branches: + - main + paths: + - "src/domains/elt**" + +jobs: + release_prod: + uses: ./.github/workflows/call_release.yaml + name: Terraform Apply + secrets: inherit + with: + environment: prod + dir: "src/domains/elt/prod/westeurope" diff --git a/.github/workflows/elt_ci.yaml b/.github/workflows/elt_ci.yaml new file mode 100644 index 000000000..2bc3c2f73 --- /dev/null +++ b/.github/workflows/elt_ci.yaml @@ -0,0 +1,23 @@ +name: Continuous Integration on elt + +on: + workflow_dispatch: + pull_request: + types: + - opened + - edited + - synchronize + - reopened + - ready_for_review + paths: + - "src/domains/elt**" + - ".github/workflows/elt_**" + +jobs: + code_review_prod: + uses: ./.github/workflows/call_code_review.yaml + name: Terraform Plan + secrets: inherit + with: + environment: prod + dir: "src/domains/elt/prod/westeurope" diff --git a/src/core/README.md b/src/core/README.md index 887a0a4f6..ddce252e2 100644 --- a/src/core/README.md +++ b/src/core/README.md @@ -72,8 +72,6 @@ | [function\_assets\_cdn](#module\_function\_assets\_cdn) | git::https://github.com/pagopa/terraform-azurerm-v3.git//function_app | v7.61.0 | | [function\_assets\_cdn\_snet](#module\_function\_assets\_cdn\_snet) | git::https://github.com/pagopa/terraform-azurerm-v3.git//subnet | v7.61.0 | | [function\_assets\_cdn\_staging\_slot](#module\_function\_assets\_cdn\_staging\_slot) | git::https://github.com/pagopa/terraform-azurerm-v3.git//function_app_slot | v7.61.0 | -| [function\_elt](#module\_function\_elt) | git::https://github.com/pagopa/terraform-azurerm-v3.git//function_app | v7.62.0 | -| [function\_elt\_snetout](#module\_function\_elt\_snetout) | git::https://github.com/pagopa/terraform-azurerm-v3.git//subnet | v7.61.0 | | [function\_public](#module\_function\_public) | git::https://github.com/pagopa/terraform-azurerm-v3.git//function_app | v7.61.0 | | [function\_public\_staging\_slot](#module\_function\_public\_staging\_slot) | git::https://github.com/pagopa/terraform-azurerm-v3.git//function_app_slot | v7.61.0 | | [function\_services](#module\_function\_services) | git::https://github.com/pagopa/terraform-azurerm-v3.git//function_app | v7.61.0 | @@ -88,7 +86,6 @@ | [redis\_common\_snet](#module\_redis\_common\_snet) | git::https://github.com/pagopa/terraform-azurerm-v3.git//subnet | v7.61.0 | | [services\_snet](#module\_services\_snet) | git::https://github.com/pagopa/terraform-azurerm-v3.git//subnet | v7.61.0 | | [shared\_1\_snet](#module\_shared\_1\_snet) | git::https://github.com/pagopa/terraform-azurerm-v3.git//subnet | v7.61.0 | -| [storage\_account\_elt](#module\_storage\_account\_elt) | git::https://github.com/pagopa/terraform-azurerm-v3.git//storage_account | v7.61.0 | | [storage\_api](#module\_storage\_api) | git::https://github.com/pagopa/terraform-azurerm-v3.git//storage_account | v7.61.0 | | [storage\_api\_object\_replication\_to\_replica](#module\_storage\_api\_object\_replication\_to\_replica) | git::https://github.com/pagopa/terraform-azurerm-v3.git//storage_object_replication | v7.61.0 | | [storage\_api\_replica](#module\_storage\_api\_replica) | git::https://github.com/pagopa/terraform-azurerm-v3.git//storage_account | v7.61.0 | @@ -294,7 +291,6 @@ | [azurerm_resource_group.container_registry_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | | [azurerm_resource_group.data](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | | [azurerm_resource_group.default_roleassignment_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | -| [azurerm_resource_group.elt_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | | [azurerm_resource_group.event_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | | [azurerm_resource_group.grafana_dashboard_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | | [azurerm_resource_group.rg_common](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | @@ -308,17 +304,9 @@ | [azurerm_resource_group.shared_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | | [azurerm_resource_group.weu_beta_vnet_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | | [azurerm_resource_group.weu_prod01_vnet_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | -| [azurerm_storage_container.container_messages_report_step1](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_container) | resource | -| [azurerm_storage_container.container_messages_report_step_final](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_container) | resource | | [azurerm_storage_container.storage_api_cached](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_container) | resource | | [azurerm_storage_container.storage_api_message_content](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_container) | resource | | [azurerm_storage_queue.storage_account_apievents_events_queue](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_queue) | resource | -| [azurerm_storage_table.fneltcommands](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_table) | resource | -| [azurerm_storage_table.fnelterrors](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_table) | resource | -| [azurerm_storage_table.fnelterrors_message_status](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_table) | resource | -| [azurerm_storage_table.fnelterrors_messages](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_table) | resource | -| [azurerm_storage_table.fnelterrors_notification_status](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_table) | resource | -| [azurerm_storage_table.fneltexports](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_table) | resource | | [azurerm_storage_table.locked_profiles](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_table) | resource | | [azurerm_storage_table.storage_api_faileduserdataprocessing](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_table) | resource | | [azurerm_storage_table.storage_api_subscriptionsfeedbyday](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_table) | resource | @@ -442,7 +430,6 @@ | [azurerm_key_vault_secret.monitor_notification_slack_email](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault_secret) | data source | | [azurerm_key_vault_secret.sec_storage_id](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault_secret) | data source | | [azurerm_key_vault_secret.sec_workspace_id](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault_secret) | data source | -| [azurerm_key_vault_secret.services_exclusion_list](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault_secret) | data source | | [azurerm_linux_function_app.app_messages_1](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/linux_function_app) | data source | | [azurerm_linux_function_app.app_messages_2](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/linux_function_app) | data source | | [azurerm_linux_function_app.eucovidcert](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/linux_function_app) | data source | @@ -465,6 +452,7 @@ | [azurerm_storage_account.userbackups](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/storage_account) | data source | | [azurerm_storage_account.userdatadownload](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/storage_account) | data source | | [azurerm_subnet.function_eucovidcert_snet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subnet) | data source | +| [azurerm_subnet.function_let_snet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subnet) | data source | | [azurerm_subnet.functions_fast_login_snet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subnet) | data source | | [azurerm_subnet.ioweb_profile_snet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subnet) | data source | | [azurerm_subnet.services_cms_backoffice_snet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subnet) | data source | diff --git a/src/core/data.tf b/src/core/data.tf index 5c043b923..27b6a1fa2 100644 --- a/src/core/data.tf +++ b/src/core/data.tf @@ -62,12 +62,6 @@ locals { storage_account_notifications_queue_push_notifications = "push-notifications" } -# KeyVault values - start -data "azurerm_key_vault_secret" "services_exclusion_list" { - name = "io-fn-services-SERVICEID-EXCLUSION-LIST" - key_vault_id = module.key_vault_common.id -} - # Event hubs data "azurerm_eventhub_authorization_rule" "io-p-payments-weu-prod01-evh-ns_payment-updates_io-fn-messages-cqrs" { @@ -312,3 +306,13 @@ data "azurerm_linux_function_app" "app_messages_2" { resource_group_name = "${local.project}-app-messages-rg-2" name = "${local.project}-app-messages-fn-2" } + +# +# ELT +# + +data "azurerm_subnet" "function_let_snet" { + name = "fn3eltout" + resource_group_name = azurerm_resource_group.rg_common.name + virtual_network_name = module.vnet_common.name +} diff --git a/src/core/events.tf b/src/core/events.tf index a4dc3d0e7..04200a584 100644 --- a/src/core/events.tf +++ b/src/core/events.tf @@ -41,7 +41,7 @@ module "event_hub" { default_action = "Deny", virtual_network_rule = [ { - subnet_id = module.function_elt_snetout.id, + subnet_id = data.azurerm_subnet.function_let_snet.id ignore_missing_virtual_network_service_endpoint = false } ], diff --git a/src/core/function_elt.tf b/src/core/function_elt.tf index 1d20102ad..b28b04f64 100644 --- a/src/core/function_elt.tf +++ b/src/core/function_elt.tf @@ -1,267 +1,3 @@ -locals { - function_elt = { - app_settings = { - FUNCTIONS_WORKER_RUNTIME = "node" - FUNCTIONS_WORKER_PROCESS_COUNT = 4 - NODE_ENV = "production" - // Keepalive fields are all optionals - FETCH_KEEPALIVE_ENABLED = "true" - FETCH_KEEPALIVE_SOCKET_ACTIVE_TTL = "110000" - FETCH_KEEPALIVE_MAX_SOCKETS = "40" - FETCH_KEEPALIVE_MAX_FREE_SOCKETS = "10" - FETCH_KEEPALIVE_FREE_SOCKET_TIMEOUT = "30000" - FETCH_KEEPALIVE_TIMEOUT = "60000" - COSMOSDB_NAME = "db" - COSMOSDB_URI = data.azurerm_cosmosdb_account.cosmos_api.endpoint - COSMOSDB_KEY = data.azurerm_cosmosdb_account.cosmos_api.primary_key - COSMOS_API_CONNECTION_STRING = format("AccountEndpoint=%s;AccountKey=%s;", data.azurerm_cosmosdb_account.cosmos_api.endpoint, data.azurerm_cosmosdb_account.cosmos_api.primary_key) - TARGETKAFKA_clientId = "IO_FUNCTIONS_ELT" - TARGETKAFKA_brokers = local.event_hub.connection - TARGETKAFKA_ssl = "true" - TARGETKAFKA_sasl_mechanism = "plain" - TARGETKAFKA_sasl_username = "$ConnectionString" - TARGETKAFKA_sasl_password = module.event_hub.keys["io-cosmosdb-services.io-fn-elt"].primary_connection_string - TARGETKAFKA_maxInFlightRequests = "1" - TARGETKAFKA_idempotent = "true" - TARGETKAFKA_transactionalId = "IO_ELT" - TARGETKAFKA_topic = "io-cosmosdb-services" - - SERVICES_TOPIC_NAME = "io-cosmosdb-services" - SERVICES_TOPIC_CONNECTION_STRING = module.event_hub.keys["io-cosmosdb-services.io-fn-elt"].primary_connection_string - SERVICES_LEASES_PREFIX = "services-001" - - MESSAGES_TOPIC_NAME = "pdnd-io-cosmosdb-messages" - MESSAGES_TOPIC_CONNECTION_STRING = module.event_hub.keys["pdnd-io-cosmosdb-messages.io-fn-elt"].primary_connection_string - MESSAGES_LEASES_PREFIX = "messages-001" - - MESSAGE_STATUS_TOPIC_NAME = "pdnd-io-cosmosdb-message-status" - MESSAGE_STATUS_TOPIC_CONNECTION_STRING = module.event_hub.keys["pdnd-io-cosmosdb-message-status.io-fn-elt"].primary_connection_string - MESSAGE_STATUS_LEASES_PREFIX = "message-status-001" - - NOTIFICATION_STATUS_TOPIC_NAME = "pdnd-io-cosmosdb-notification-status" - NOTIFICATION_STATUS_TOPIC_CONNECTION_STRING = module.event_hub.keys["pdnd-io-cosmosdb-notification-status.io-fn-elt"].primary_connection_string - NOTIFICATION_STATUS_LEASES_PREFIX = "notification-status-001" - - - ERROR_STORAGE_ACCOUNT = module.storage_account_elt.name - ERROR_STORAGE_KEY = module.storage_account_elt.primary_access_key - ERROR_STORAGE_TABLE = azurerm_storage_table.fnelterrors.name - ERROR_STORAGE_TABLE_MESSAGES = azurerm_storage_table.fnelterrors_messages.name - ERROR_STORAGE_TABLE_MESSAGE_STATUS = azurerm_storage_table.fnelterrors_message_status.name - ERROR_STORAGE_TABLE_NOTIFICATION_STATUS = azurerm_storage_table.fnelterrors_notification_status.name - - COMMAND_STORAGE = module.storage_account_elt.primary_connection_string - COMMAND_STORAGE_TABLE = azurerm_storage_table.fneltcommands.name - IMPORT_TOPIC_NAME = "import-command" - IMPORT_TOPIC_CONNECTION_STRING = module.event_hub.keys["import-command.io-fn-elt"].primary_connection_string - - PROFILE_TOPIC_NAME = "io-cosmosdb-profiles" - PROFILE_TOPIC_CONNECTION_STRING = module.event_hub.keys["io-cosmosdb-profiles.io-fn-elt"].primary_connection_string - - COSMOSDB_REPLICA_NAME = "db" - COSMOSDB_REPLICA_URI = replace(data.azurerm_cosmosdb_account.cosmos_api.endpoint, "io-p-cosmos-api", "io-p-cosmos-api-northeurope") - COSMOSDB_REPLICA_KEY = data.azurerm_cosmosdb_account.cosmos_api.primary_key - COSMOSDB_REPLICA_LOCATION = "North Europe" - - MESSAGE_EXPORTS_COMMAND_TABLE = azurerm_storage_table.fneltexports.name - MESSAGE_EXPORT_STEP_1_CONTAINER = azurerm_storage_container.container_messages_report_step1.name - MESSAGE_EXPORT_STEP_FINAL_CONTAINER = azurerm_storage_container.container_messages_report_step_final.name - - COSMOS_CHUNK_SIZE = "1000" - COSMOS_DEGREE_OF_PARALLELISM = "2" - MESSAGE_CONTENT_CHUNK_SIZE = "200" - - SERVICEID_EXCLUSION_LIST = data.azurerm_key_vault_secret.services_exclusion_list.value - - PN_SERVICE_ID = var.pn_service_id - - #iopstapi connection string - MessageContentPrimaryStorageConnection = module.storage_api.primary_connection_string - #iopstapireplica connection string - MessageContentStorageConnection = module.storage_api_replica.primary_connection_string - ServiceInfoBlobStorageConnection = module.assets_cdn.primary_connection_string - - MESSAGES_FAILURE_QUEUE_NAME = "pdnd-io-cosmosdb-messages-failure" - MESSAGE_STATUS_FAILURE_QUEUE_NAME = "pdnd-io-cosmosdb-message-status-failure" - SERVICES_FAILURE_QUEUE_NAME = "pdnd-io-cosmosdb-services-failure" - - INTERNAL_TEST_FISCAL_CODES = local.test_users - } - } -} - -resource "azurerm_resource_group" "elt_rg" { - name = format("%s-elt-rg", local.project) - location = var.location - - tags = var.tags -} - -module "function_elt_snetout" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//subnet?ref=v7.61.0" - name = "fn3eltout" - address_prefixes = var.cidr_subnet_fnelt - resource_group_name = azurerm_resource_group.rg_common.name - virtual_network_name = module.vnet_common.name - private_endpoint_network_policies_enabled = true - - service_endpoints = [ - "Microsoft.EventHub", - "Microsoft.Storage", - "Microsoft.AzureCosmosDB", - ] - - delegation = { - name = "default" - service_delegation = { - name = "Microsoft.Web/serverFarms" - actions = ["Microsoft.Network/virtualNetworks/subnets/action"] - } - } -} - -#tfsec:ignore:azure-storage-queue-services-logging-enabled:exp:2022-05-01 # already ignored, maybe a bug in tfsec -module "function_elt" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//function_app?ref=v7.62.0" - - resource_group_name = azurerm_resource_group.elt_rg.name - name = "${local.project}-fn-elt" - domain = "IO-COMMONS" - storage_account_name = "${replace(local.project, "-", "")}stfnelt" - app_service_plan_name = "${local.project}-plan-fnelt" - location = var.location - health_check_path = "/api/v1/info" - subnet_id = module.function_elt_snetout.id - runtime_version = "~4" - node_version = "18" - application_insights_instrumentation_key = azurerm_application_insights.application_insights.instrumentation_key - - app_service_plan_info = { - kind = "elastic" - sku_tier = "ElasticPremium" - sku_size = "EP1" - maximum_elastic_worker_count = 1 - worker_count = null - zone_balancing_enabled = null - } - - app_settings = merge( - local.function_elt.app_settings, { - "AzureWebJobs.CosmosApiServicesChangeFeed.Disabled" = "1" - "AzureWebJobs.CosmosApiMessageStatusChangeFeed.Disabled" = "1" - "AzureWebJobs.CosmosApiMessagesChangeFeed.Disabled" = "1" - "AzureWebJobs.AnalyticsMessagesChangeFeedInboundProcessorAdapter.Disabled" = "0" - "AzureWebJobs.AnalyticsMessagesStorageQueueInboundProcessorAdapter.Disabled" = "0" - "AzureWebJobs.AnalyticsMessageStatusChangeFeedInboundProcessorAdapter.Disabled" = "0" - "AzureWebJobs.AnalyticsMessageStatusStorageQueueInbloundAdapter.Disabled" = "0" - "AzureWebJobs.AnalyticsServiceChangeFeedInboundProcessorAdapter.Disabled" = "0" - "AzureWebJobs.AnalyticsServiceStorageQueueInboundProcessorAdapter.Disabled" = "0" - } - ) - - storage_account_info = { - account_kind = "StorageV2" - account_tier = "Standard" - account_replication_type = "GZRS" - access_tier = "Hot" - advanced_threat_protection_enable = true - use_legacy_defender_version = false - } - - internal_storage = { - "enable" = true, - "private_endpoint_subnet_id" = module.private_endpoints_subnet.id, - "private_dns_zone_blob_ids" = [azurerm_private_dns_zone.privatelink_blob_core.id], - "private_dns_zone_queue_ids" = [azurerm_private_dns_zone.privatelink_queue_core.id], - "private_dns_zone_table_ids" = [azurerm_private_dns_zone.privatelink_table_core.id], - "queues" = [ - local.function_elt.app_settings.MESSAGES_FAILURE_QUEUE_NAME, - "${local.function_elt.app_settings.MESSAGES_FAILURE_QUEUE_NAME}-poison", - local.function_elt.app_settings.MESSAGE_STATUS_FAILURE_QUEUE_NAME, - "${local.function_elt.app_settings.MESSAGE_STATUS_FAILURE_QUEUE_NAME}-poison", - local.function_elt.app_settings.SERVICES_FAILURE_QUEUE_NAME, - "${local.function_elt.app_settings.SERVICES_FAILURE_QUEUE_NAME}-poison" - ], - "containers" = [], - "blobs_retention_days" = 1, - } - - allowed_subnets = [ - module.azdoa_snet[0].id, - ] - - allowed_ips = local.app_insights_ips_west_europe - - # Action groups for alerts - action = [ - { - action_group_id = azurerm_monitor_action_group.error_action_group.id - webhook_properties = {} - } - ] - - tags = var.tags -} - -#tfsec:ignore:azure-storage-default-action-deny -#tfsec:ignore:azure-storage-queue-services-logging-enabled:exp:2022-05-01 # already ignored, maybe a bug in tfsec -module "storage_account_elt" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//storage_account?ref=v7.61.0" - - name = replace(format("%s-stelt", local.project), "-", "") - account_kind = "StorageV2" - account_tier = "Standard" - account_replication_type = "GZRS" - access_tier = "Hot" - resource_group_name = azurerm_resource_group.elt_rg.name - location = var.location - advanced_threat_protection = true - public_network_access_enabled = true - - tags = var.tags -} - -resource "azurerm_storage_table" "fnelterrors" { - name = "fnelterrors" - storage_account_name = module.storage_account_elt.name -} - -resource "azurerm_storage_table" "fnelterrors_messages" { - name = "fnelterrorsMessages" - storage_account_name = module.storage_account_elt.name -} - -resource "azurerm_storage_table" "fnelterrors_message_status" { - name = "fnelterrorsMessageStatus" - storage_account_name = module.storage_account_elt.name -} - -resource "azurerm_storage_table" "fnelterrors_notification_status" { - name = "fnelterrorsNotificationStatus" - storage_account_name = module.storage_account_elt.name -} - -resource "azurerm_storage_table" "fneltcommands" { - name = "fneltcommands" - storage_account_name = module.storage_account_elt.name -} - -resource "azurerm_storage_table" "fneltexports" { - name = "fneltexports" - storage_account_name = module.storage_account_elt.name -} - -resource "azurerm_storage_container" "container_messages_report_step1" { - name = "messages-report-step1" - storage_account_name = module.storage_account_elt.name - container_access_type = "private" -} - -resource "azurerm_storage_container" "container_messages_report_step_final" { - name = "messages-report-step-final" - storage_account_name = module.storage_account_elt.name - container_access_type = "private" -} diff --git a/src/domains/elt/_modules/function_apps/data.tf b/src/domains/elt/_modules/function_apps/data.tf new file mode 100644 index 000000000..2b016f1df --- /dev/null +++ b/src/domains/elt/_modules/function_apps/data.tf @@ -0,0 +1,115 @@ +data "azurerm_application_insights" "application_insights" { + name = format("%s-ai-common", var.project) + resource_group_name = local.resource_group_name_common +} + +data "azurerm_monitor_action_group" "error_action_group" { + name = "${replace("${var.project}", "-", "")}error" + resource_group_name = local.resource_group_name_common +} + +data "azurerm_subnet" "snet_azdoa" { + name = "azure-devops" + virtual_network_name = var.vnet_name + resource_group_name = local.resource_group_name_common +} + +data "azurerm_subnet" "private_endpoints_subnet" { + name = "pendpoints" + virtual_network_name = var.vnet_name + resource_group_name = local.resource_group_name_common +} + +data "azurerm_private_dns_zone" "privatelink_blob_core" { + name = "privatelink.blob.core.windows.net" + resource_group_name = local.resource_group_name_common +} + +data "azurerm_private_dns_zone" "privatelink_queue_core" { + name = "privatelink.queue.core.windows.net" + resource_group_name = local.resource_group_name_common +} + +data "azurerm_private_dns_zone" "privatelink_table_core" { + name = "privatelink.table.core.windows.net" + resource_group_name = local.resource_group_name_common +} + +data "azurerm_cosmosdb_account" "cosmos_api" { + name = format("%s-cosmos-api", var.project) + resource_group_name = format("%s-rg-internal", var.project) +} + +data "azurerm_eventhub_authorization_rule" "evh_ns_io_cosmos_fn" { + name = "io-fn-elt" + namespace_name = "${var.project}-evh-ns" + eventhub_name = "io-cosmosdb-services" + resource_group_name = "${var.project}-evt-rg" +} + +data "azurerm_eventhub_authorization_rule" "evh_ns_pnpg_cosmos_fn" { + name = "io-fn-elt" + namespace_name = "${var.project}-evh-ns" + eventhub_name = "pdnd-io-cosmosdb-messages" + resource_group_name = "${var.project}-evt-rg" +} + +data "azurerm_eventhub_authorization_rule" "evh_ns_pdnd_io_cosmos_notification_status_fn" { + name = "io-fn-elt" + namespace_name = "${var.project}-evh-ns" + eventhub_name = "pdnd-io-cosmosdb-notification-status" + resource_group_name = "${var.project}-evt-rg" +} + +data "azurerm_eventhub_authorization_rule" "evh_ns_pdnd_io_cosmos_message_status_fn" { + name = "io-fn-elt" + namespace_name = "${var.project}-evh-ns" + eventhub_name = "pdnd-io-cosmosdb-message-status" + resource_group_name = "${var.project}-evt-rg" +} + +data "azurerm_eventhub_authorization_rule" "evh_ns_pdnd_io_cosmos_fn" { + name = "io-fn-elt" + namespace_name = "${var.project}-evh-ns" + eventhub_name = "pdnd-io-cosmosdb-messages" + resource_group_name = "${var.project}-evt-rg" +} + +data "azurerm_eventhub_authorization_rule" "evh_ns_import_command_fn" { + name = "io-fn-elt" + namespace_name = "${var.project}-evh-ns" + eventhub_name = "import-command" + resource_group_name = "${var.project}-evt-rg" +} + +data "azurerm_eventhub_authorization_rule" "evh_ns_io_cosmos_profiles_fn" { + name = "io-fn-elt" + namespace_name = "${var.project}-evh-ns" + eventhub_name = "io-cosmosdb-profiles" + resource_group_name = "${var.project}-evt-rg" +} + +data "azurerm_key_vault" "kv_common" { + name = "${var.project}-kv-common" + resource_group_name = local.resource_group_name_common +} + +data "azurerm_key_vault_secret" "services_exclusion_list" { + name = "io-fn-services-SERVICEID-EXCLUSION-LIST" + key_vault_id = data.azurerm_key_vault.kv_common.id +} + +data "azurerm_storage_account" "storage_api" { + name = replace("${var.project}stapi", "-", "") + resource_group_name = local.resource_group_name_internal +} + +data "azurerm_storage_account" "storage_api_replica" { + name = replace("${var.project}stapireplica", "-", "") + resource_group_name = local.resource_group_name_internal +} + +data "azurerm_storage_account" "storage_assets_cdn" { + name = replace(format("%s-stcdnassets", var.project), "-", "") + resource_group_name = local.resource_group_name_common +} diff --git a/src/domains/elt/_modules/function_apps/function_app_elt.tf b/src/domains/elt/_modules/function_apps/function_app_elt.tf new file mode 100644 index 000000000..386f99cd7 --- /dev/null +++ b/src/domains/elt/_modules/function_apps/function_app_elt.tf @@ -0,0 +1,177 @@ +locals { + function_elt = { + app_settings = { + FUNCTIONS_WORKER_RUNTIME = "node" + FUNCTIONS_WORKER_PROCESS_COUNT = 4 + NODE_ENV = "production" + + // Keepalive fields are all optionals + FETCH_KEEPALIVE_ENABLED = "true" + FETCH_KEEPALIVE_SOCKET_ACTIVE_TTL = "110000" + FETCH_KEEPALIVE_MAX_SOCKETS = "40" + FETCH_KEEPALIVE_MAX_FREE_SOCKETS = "10" + FETCH_KEEPALIVE_FREE_SOCKET_TIMEOUT = "30000" + FETCH_KEEPALIVE_TIMEOUT = "60000" + + COSMOSDB_NAME = "db" + COSMOSDB_URI = data.azurerm_cosmosdb_account.cosmos_api.endpoint + COSMOSDB_KEY = data.azurerm_cosmosdb_account.cosmos_api.primary_key + COSMOS_API_CONNECTION_STRING = format("AccountEndpoint=%s;AccountKey=%s;", data.azurerm_cosmosdb_account.cosmos_api.endpoint, data.azurerm_cosmosdb_account.cosmos_api.primary_key) + + TARGETKAFKA_clientId = "IO_FUNCTIONS_ELT" + TARGETKAFKA_brokers = local.event_hub_connection + TARGETKAFKA_ssl = "true" + TARGETKAFKA_sasl_mechanism = "plain" + TARGETKAFKA_sasl_username = "$ConnectionString" + TARGETKAFKA_sasl_password = data.azurerm_eventhub_authorization_rule.evh_ns_io_cosmos_fn.primary_connection_string + TARGETKAFKA_maxInFlightRequests = "1" + TARGETKAFKA_idempotent = "true" + TARGETKAFKA_transactionalId = "IO_ELT" + TARGETKAFKA_topic = "io-cosmosdb-services" + + SERVICES_TOPIC_NAME = "io-cosmosdb-services" + SERVICES_TOPIC_CONNECTION_STRING = data.azurerm_eventhub_authorization_rule.evh_ns_io_cosmos_fn.primary_connection_string + SERVICES_LEASES_PREFIX = "services-001" + + MESSAGES_TOPIC_NAME = "pdnd-io-cosmosdb-messages" + MESSAGES_TOPIC_CONNECTION_STRING = data.azurerm_eventhub_authorization_rule.evh_ns_pdnd_io_cosmos_fn.primary_connection_string + MESSAGES_LEASES_PREFIX = "messages-001" + + MESSAGE_STATUS_TOPIC_NAME = "pdnd-io-cosmosdb-message-status" + MESSAGE_STATUS_TOPIC_CONNECTION_STRING = data.azurerm_eventhub_authorization_rule.evh_ns_pdnd_io_cosmos_message_status_fn.primary_connection_string + MESSAGE_STATUS_LEASES_PREFIX = "message-status-001" + + NOTIFICATION_STATUS_TOPIC_NAME = "pdnd-io-cosmosdb-notification-status" + NOTIFICATION_STATUS_TOPIC_CONNECTION_STRING = data.azurerm_eventhub_authorization_rule.evh_ns_pdnd_io_cosmos_notification_status_fn.primary_connection_string + NOTIFICATION_STATUS_LEASES_PREFIX = "notification-status-001" + + ERROR_STORAGE_ACCOUNT = var.storage_account_name + ERROR_STORAGE_KEY = var.storage_account_primary_access_key + ERROR_STORAGE_TABLE = var.storage_account_tables.fnelterrors + ERROR_STORAGE_TABLE_MESSAGES = var.storage_account_tables.fnelterrors_messages + ERROR_STORAGE_TABLE_MESSAGE_STATUS = var.storage_account_tables.fnelterrors_message_status + ERROR_STORAGE_TABLE_NOTIFICATION_STATUS = var.storage_account_tables.fnelterrors_notification_status + + COMMAND_STORAGE = var.storage_account_primary_connection_string + COMMAND_STORAGE_TABLE = var.storage_account_tables.fneltcommands + IMPORT_TOPIC_NAME = "import-command" + IMPORT_TOPIC_CONNECTION_STRING = data.azurerm_eventhub_authorization_rule.evh_ns_import_command_fn.primary_connection_string + + PROFILE_TOPIC_NAME = "io-cosmosdb-profiles" + PROFILE_TOPIC_CONNECTION_STRING = data.azurerm_eventhub_authorization_rule.evh_ns_io_cosmos_profiles_fn.primary_connection_string + + COSMOSDB_REPLICA_NAME = "db" + COSMOSDB_REPLICA_URI = replace(data.azurerm_cosmosdb_account.cosmos_api.endpoint, "io-p-cosmos-api", "io-p-cosmos-api-northeurope") + COSMOSDB_REPLICA_KEY = data.azurerm_cosmosdb_account.cosmos_api.primary_key + COSMOSDB_REPLICA_LOCATION = var.secondary_location_display_name + + MESSAGE_EXPORTS_COMMAND_TABLE = var.storage_account_tables.fneltexports + MESSAGE_EXPORT_STEP_1_CONTAINER = var.storage_account_containers.container_messages_report_step1 + MESSAGE_EXPORT_STEP_FINAL_CONTAINER = var.storage_account_containers.container_messages_report_step_final + + COSMOS_CHUNK_SIZE = "1000" + COSMOS_DEGREE_OF_PARALLELISM = "2" + MESSAGE_CONTENT_CHUNK_SIZE = "200" + + SERVICEID_EXCLUSION_LIST = data.azurerm_key_vault_secret.services_exclusion_list.value + + PN_SERVICE_ID = local.pn_service_id + + #iopstapi connection string + MessageContentPrimaryStorageConnection = data.azurerm_storage_account.storage_api.primary_connection_string + #iopstapireplica connection string + MessageContentStorageConnection = data.azurerm_storage_account.storage_api_replica.primary_connection_string + ServiceInfoBlobStorageConnection = data.azurerm_storage_account.storage_assets_cdn.primary_connection_string + + MESSAGES_FAILURE_QUEUE_NAME = "pdnd-io-cosmosdb-messages-failure" + MESSAGE_STATUS_FAILURE_QUEUE_NAME = "pdnd-io-cosmosdb-message-status-failure" + SERVICES_FAILURE_QUEUE_NAME = "pdnd-io-cosmosdb-services-failure" + + INTERNAL_TEST_FISCAL_CODES = module.tests.test_users.all + } + } +} + +#tfsec:ignore:azure-storage-queue-services-logging-enabled:exp:2022-05-01 # already ignored, maybe a bug in tfsec +module "function_elt" { + source = "github.com/pagopa/terraform-azurerm-v3//function_app?ref=v7.67.1" + + resource_group_name = var.resource_group_name + name = "${var.project}-fn-elt" + location = var.location + domain = "IO-COMMONS" + + storage_account_name = "${replace(var.project, "-", "")}stfnelt" + app_service_plan_name = "${var.project}-plan-fnelt" + health_check_path = "/api/v1/info" + subnet_id = var.subnet_id + runtime_version = "~4" + node_version = "18" + application_insights_instrumentation_key = data.azurerm_application_insights.application_insights.instrumentation_key + + app_service_plan_info = { + kind = "elastic" + sku_tier = "ElasticPremium" + sku_size = "EP1" + maximum_elastic_worker_count = 1 + worker_count = null + zone_balancing_enabled = null + } + + app_settings = merge( + local.function_elt.app_settings, { + "AzureWebJobs.CosmosApiServicesChangeFeed.Disabled" = "1" + "AzureWebJobs.CosmosApiMessageStatusChangeFeed.Disabled" = "1" + "AzureWebJobs.CosmosApiMessagesChangeFeed.Disabled" = "1" + "AzureWebJobs.AnalyticsMessagesChangeFeedInboundProcessorAdapter.Disabled" = "0" + "AzureWebJobs.AnalyticsMessagesStorageQueueInboundProcessorAdapter.Disabled" = "0" + "AzureWebJobs.AnalyticsMessageStatusChangeFeedInboundProcessorAdapter.Disabled" = "0" + "AzureWebJobs.AnalyticsMessageStatusStorageQueueInbloundAdapter.Disabled" = "0" + "AzureWebJobs.AnalyticsServiceChangeFeedInboundProcessorAdapter.Disabled" = "0" + "AzureWebJobs.AnalyticsServiceStorageQueueInboundProcessorAdapter.Disabled" = "0" + } + ) + + storage_account_info = { + account_kind = "StorageV2" + account_tier = "Standard" + account_replication_type = "GZRS" + access_tier = "Hot" + advanced_threat_protection_enable = true + use_legacy_defender_version = false + } + + internal_storage = { + "enable" = true, + "private_endpoint_subnet_id" = data.azurerm_subnet.private_endpoints_subnet.id + "private_dns_zone_blob_ids" = [data.azurerm_private_dns_zone.privatelink_blob_core.id], + "private_dns_zone_queue_ids" = [data.azurerm_private_dns_zone.privatelink_queue_core.id], + "private_dns_zone_table_ids" = [data.azurerm_private_dns_zone.privatelink_table_core.id], + "queues" = [ + local.function_elt.app_settings.MESSAGES_FAILURE_QUEUE_NAME, + "${local.function_elt.app_settings.MESSAGES_FAILURE_QUEUE_NAME}-poison", + local.function_elt.app_settings.MESSAGE_STATUS_FAILURE_QUEUE_NAME, + "${local.function_elt.app_settings.MESSAGE_STATUS_FAILURE_QUEUE_NAME}-poison", + local.function_elt.app_settings.SERVICES_FAILURE_QUEUE_NAME, + "${local.function_elt.app_settings.SERVICES_FAILURE_QUEUE_NAME}-poison" + ], + "containers" = [], + "blobs_retention_days" = 1, + } + + allowed_subnets = [ + data.azurerm_subnet.snet_azdoa.id, + ] + + allowed_ips = local.app_insights_ips_west_europe + + # Action groups for alerts + action = [ + { + action_group_id = data.azurerm_monitor_action_group.error_action_group.id + webhook_properties = {} + } + ] + + tags = var.tags +} diff --git a/src/domains/elt/_modules/function_apps/locals.tf b/src/domains/elt/_modules/function_apps/locals.tf new file mode 100644 index 000000000..6e23393a3 --- /dev/null +++ b/src/domains/elt/_modules/function_apps/locals.tf @@ -0,0 +1,18 @@ +locals { + + resource_group_name_common = "${var.project}-rg-common" + resource_group_name_internal = "${var.project}-rg-internal" + + app_insights_ips_west_europe = [ + "51.144.56.96/28", + "51.144.56.112/28", + "51.144.56.128/28", + "51.144.56.144/28", + "51.144.56.160/28", + "51.144.56.176/28", + ] + + event_hub_connection = "${format("%s-evh-ns", var.project)}.servicebus.windows.net:9093" + + pn_service_id = "01G40DWQGKY5GRWSNM4303VNRP" +} diff --git a/src/domains/elt/_modules/function_apps/main.tf b/src/domains/elt/_modules/function_apps/main.tf new file mode 100644 index 000000000..fe3796d05 --- /dev/null +++ b/src/domains/elt/_modules/function_apps/main.tf @@ -0,0 +1,8 @@ +terraform { + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + } + } +} diff --git a/src/domains/elt/_modules/function_apps/outputs.tf b/src/domains/elt/_modules/function_apps/outputs.tf new file mode 100644 index 000000000..0b9de8c3e --- /dev/null +++ b/src/domains/elt/_modules/function_apps/outputs.tf @@ -0,0 +1,8 @@ +output "function_app_elt" { + value = { + id = module.function_elt.id + name = module.function_elt.name + app_service_plan_name = module.function_elt.app_service_plan_name + resource_group_name = module.function_elt.resource_group_name + } +} diff --git a/src/domains/elt/_modules/function_apps/tests.tf b/src/domains/elt/_modules/function_apps/tests.tf new file mode 100644 index 000000000..28d58ff4d --- /dev/null +++ b/src/domains/elt/_modules/function_apps/tests.tf @@ -0,0 +1,3 @@ +module "tests" { + source = "../../../tests" +} diff --git a/src/domains/elt/_modules/function_apps/variables.tf b/src/domains/elt/_modules/function_apps/variables.tf new file mode 100644 index 000000000..7e486c692 --- /dev/null +++ b/src/domains/elt/_modules/function_apps/variables.tf @@ -0,0 +1,69 @@ +variable "project" { + type = string + description = "IO prefix and short environment" +} + +variable "location" { + type = string + description = "Azure region" +} + +variable "secondary_location_display_name" { + type = string + description = "Azure redundancy region display name" +} + +variable "tags" { + type = map(any) + description = "Resource tags" +} + +variable "resource_group_name" { + type = string + description = "Name of the resource group where resources will be created" +} + +variable "vnet_name" { + type = string + description = "VNet name" +} + +variable "subnet_id" { + type = string + description = "Id of the subnet to use for Function App" +} + +variable "storage_account_name" { + type = string + description = "Storage account name used to handle errors" +} + +variable "storage_account_primary_access_key" { + type = string + sensitive = true + description = "Storage account primary access key used to handle errors" +} + +variable "storage_account_primary_connection_string" { + type = string + sensitive = true + description = "Storage account primary connection string used to handle errors" +} + +variable "storage_account_tables" { + type = object({ + fnelterrors = string + fnelterrors_messages = string + fnelterrors_message_status = string + fnelterrors_notification_status = string + fneltcommands = string + fneltexports = string + }) +} + +variable "storage_account_containers" { + type = object({ + container_messages_report_step1 = string + container_messages_report_step_final = string + }) +} diff --git a/src/domains/elt/_modules/networking/data.tf b/src/domains/elt/_modules/networking/data.tf new file mode 100644 index 000000000..e219f9575 --- /dev/null +++ b/src/domains/elt/_modules/networking/data.tf @@ -0,0 +1,4 @@ +data "azurerm_virtual_network" "vnet_common" { + name = "${var.project}-vnet-common" + resource_group_name = local.resource_group_common +} diff --git a/src/domains/elt/_modules/networking/locals.tf b/src/domains/elt/_modules/networking/locals.tf new file mode 100644 index 000000000..3cedaf4f1 --- /dev/null +++ b/src/domains/elt/_modules/networking/locals.tf @@ -0,0 +1,3 @@ +locals { + resource_group_common = "${var.project}-rg-common" +} diff --git a/src/domains/elt/_modules/networking/main.tf b/src/domains/elt/_modules/networking/main.tf new file mode 100644 index 000000000..fe3796d05 --- /dev/null +++ b/src/domains/elt/_modules/networking/main.tf @@ -0,0 +1,8 @@ +terraform { + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + } + } +} diff --git a/src/domains/elt/_modules/networking/outputs.tf b/src/domains/elt/_modules/networking/outputs.tf new file mode 100644 index 000000000..443162fc5 --- /dev/null +++ b/src/domains/elt/_modules/networking/outputs.tf @@ -0,0 +1,14 @@ +output "vnet_common" { + value = { + id = data.azurerm_virtual_network.vnet_common.id + name = data.azurerm_virtual_network.vnet_common.name + resource_group_name = data.azurerm_virtual_network.vnet_common.resource_group_name + } +} + +output "subnet_elt" { + value = { + id = module.function_elt_snet.id + name = module.function_elt_snet.name + } +} diff --git a/src/domains/elt/_modules/networking/subnet_elt.tf b/src/domains/elt/_modules/networking/subnet_elt.tf new file mode 100644 index 000000000..ae609ef4a --- /dev/null +++ b/src/domains/elt/_modules/networking/subnet_elt.tf @@ -0,0 +1,24 @@ +module "function_elt_snet" { + source = "github.com/pagopa/terraform-azurerm-v3//subnet?ref=v7.67.1" + + name = "fn3eltout" + address_prefixes = var.cidr_subnet_elt + resource_group_name = data.azurerm_virtual_network.vnet_common.resource_group_name + virtual_network_name = data.azurerm_virtual_network.vnet_common.name + + private_endpoint_network_policies_enabled = true + + service_endpoints = [ + "Microsoft.EventHub", + "Microsoft.Storage", + "Microsoft.AzureCosmosDB", + ] + + delegation = { + name = "default" + service_delegation = { + name = "Microsoft.Web/serverFarms" + actions = ["Microsoft.Network/virtualNetworks/subnets/action"] + } + } +} diff --git a/src/domains/elt/_modules/networking/variables.tf b/src/domains/elt/_modules/networking/variables.tf new file mode 100644 index 000000000..4e0c80447 --- /dev/null +++ b/src/domains/elt/_modules/networking/variables.tf @@ -0,0 +1,14 @@ +variable "project" { + type = string + description = "IO prefix and short environment" +} + +variable "tags" { + type = map(any) + description = "Resource tags" +} + +variable "cidr_subnet_elt" { + type = list(string) + description = "CIDR block for Elt functions subnet" +} diff --git a/src/domains/elt/_modules/storage_accounts/main.tf b/src/domains/elt/_modules/storage_accounts/main.tf new file mode 100644 index 000000000..fe3796d05 --- /dev/null +++ b/src/domains/elt/_modules/storage_accounts/main.tf @@ -0,0 +1,8 @@ +terraform { + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + } + } +} diff --git a/src/domains/elt/_modules/storage_accounts/outputs.tf b/src/domains/elt/_modules/storage_accounts/outputs.tf new file mode 100644 index 000000000..792a2005b --- /dev/null +++ b/src/domains/elt/_modules/storage_accounts/outputs.tf @@ -0,0 +1,34 @@ +output "storage_account_elt" { + value = { + id = module.storage_account_elt.id + name = module.storage_account_elt.name + } +} + +output "storage_account_elt_primary_access_key" { + value = module.storage_account_elt.primary_access_key + sensitive = true +} + +output "storage_account_elt_primary_connection_string" { + value = module.storage_account_elt.primary_connection_string + sensitive = true +} + +output "storage_account_tables" { + value = { + fnelterrors = azurerm_storage_table.fnelterrors.name + fnelterrors_messages = azurerm_storage_table.fnelterrors_messages.name + fnelterrors_message_status = azurerm_storage_table.fnelterrors_message_status.name + fnelterrors_notification_status = azurerm_storage_table.fnelterrors_notification_status.name + fneltcommands = azurerm_storage_table.fneltcommands.name + fneltexports = azurerm_storage_table.fneltexports.name + } +} + +output "storage_account_containers" { + value = { + container_messages_report_step1 = azurerm_storage_container.container_messages_report_step1.name + container_messages_report_step_final = azurerm_storage_container.container_messages_report_step_final.name + } +} diff --git a/src/domains/elt/_modules/storage_accounts/storage_account_elt.tf b/src/domains/elt/_modules/storage_accounts/storage_account_elt.tf new file mode 100644 index 000000000..fe15d02f7 --- /dev/null +++ b/src/domains/elt/_modules/storage_accounts/storage_account_elt.tf @@ -0,0 +1,18 @@ +#tfsec:ignore:azure-storage-default-action-deny +#tfsec:ignore:azure-storage-queue-services-logging-enabled:exp:2022-05-01 # already ignored, maybe a bug in tfsec +module "storage_account_elt" { + source = "github.com/pagopa/terraform-azurerm-v3//storage_account?ref=v7.67.1" + + name = replace(format("%s-stelt", var.project), "-", "") + resource_group_name = var.resource_group_name + location = var.location + + account_kind = "StorageV2" + account_tier = "Standard" + account_replication_type = "GZRS" + access_tier = "Hot" + advanced_threat_protection = true + public_network_access_enabled = true + + tags = var.tags +} diff --git a/src/domains/elt/_modules/storage_accounts/storage_tables.tf b/src/domains/elt/_modules/storage_accounts/storage_tables.tf new file mode 100644 index 000000000..4152ba6e5 --- /dev/null +++ b/src/domains/elt/_modules/storage_accounts/storage_tables.tf @@ -0,0 +1,41 @@ +resource "azurerm_storage_table" "fnelterrors" { + name = "fnelterrors" + storage_account_name = module.storage_account_elt.name +} + +resource "azurerm_storage_table" "fnelterrors_messages" { + name = "fnelterrorsMessages" + storage_account_name = module.storage_account_elt.name +} + +resource "azurerm_storage_table" "fnelterrors_message_status" { + name = "fnelterrorsMessageStatus" + storage_account_name = module.storage_account_elt.name +} + +resource "azurerm_storage_table" "fnelterrors_notification_status" { + name = "fnelterrorsNotificationStatus" + storage_account_name = module.storage_account_elt.name +} + +resource "azurerm_storage_table" "fneltcommands" { + name = "fneltcommands" + storage_account_name = module.storage_account_elt.name +} + +resource "azurerm_storage_table" "fneltexports" { + name = "fneltexports" + storage_account_name = module.storage_account_elt.name +} + +resource "azurerm_storage_container" "container_messages_report_step1" { + name = "messages-report-step1" + storage_account_name = module.storage_account_elt.name + container_access_type = "private" +} + +resource "azurerm_storage_container" "container_messages_report_step_final" { + name = "messages-report-step-final" + storage_account_name = module.storage_account_elt.name + container_access_type = "private" +} diff --git a/src/domains/elt/_modules/storage_accounts/variables.tf b/src/domains/elt/_modules/storage_accounts/variables.tf new file mode 100644 index 000000000..043c8b564 --- /dev/null +++ b/src/domains/elt/_modules/storage_accounts/variables.tf @@ -0,0 +1,19 @@ +variable "project" { + type = string + description = "IO prefix and short environment" +} + +variable "location" { + type = string + description = "Azure region" +} + +variable "tags" { + type = map(any) + description = "Resource tags" +} + +variable "resource_group_name" { + type = string + description = "Name of the resource group where resources will be created" +} diff --git a/src/domains/elt/prod/westeurope/.terraform.lock.hcl b/src/domains/elt/prod/westeurope/.terraform.lock.hcl new file mode 100644 index 000000000..ad7145221 --- /dev/null +++ b/src/domains/elt/prod/westeurope/.terraform.lock.hcl @@ -0,0 +1,47 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "3.94.0" + constraints = ">= 3.30.0, >= 3.76.0, <= 3.94.0, <= 3.97.1" + hashes = [ + "h1:Kd1Vhk4bPbiP0ZWo1pDEW1De3oNbODgh2bhX9Y6AJ6I=", + "h1:a51ZYUp5uuboql399mflWZDrErlhhYz0ujJFsc9gjhg=", + "h1:a8L0H+sq8UBeArGs/jzQYEnJ2rNmR8Um3BOGBA1m1t8=", + "h1:t3fM/PO8PLAA5mK3esAypp01V6Vh75kjPnNqxQeVrV0=", + "zh:20d102bc63096ade82f8da81c91afaffa858aa56fe9a7ad02f24f5ae5618bc53", + "zh:3ddb9d6173a4fdb9b2352a76324ee321976915544ae66cbb863c7a60f0593f05", + "zh:4bc6c62142f67192d2def11f4fd419c54dddd89a5448af036bfc60b15eb0509a", + "zh:4c5120c2101a51524af32c4220c5e376f97a227730dd92ec0b06ac677e4b39f2", + "zh:585fa7ab876d09899cd2d842f12bc28c34556b4d47919eceadefab6fa47f909f", + "zh:59de7ea462470dee7088fc4deeff48e1ffd286eaca1185c219be68dadde745b8", + "zh:8421a46dd3bc4bc2eb56f7eb9b91cc84a66070b72195a805862c6022adee2da0", + "zh:a2fcb5a091d5944dc50f1e51f53fa4d370810a507fbf4122920d756083d8df19", + "zh:beb6b93a2a16942625bb6ac1e52bf26878e35f5562f3173279423ca66553b6d7", + "zh:c6846892ea68f49c838d90b75793d1f3a866871dd701ccb575b1eecccd4e7051", + "zh:ddd59492b6d5ce4c83f06a5b16c520048f3e9bb898bab4f3910042f5c01ffeda", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.2" + hashes = [ + "h1:IMVAUHKoydFrlPrl9OzasDnw/8ntZFerCC9iXw1rXQY=", + "h1:m467k2tZ9cdFFgHW7LPBK2GLPH43LC6wc3ppxr8yvoE=", + "h1:vWAsYRd7MjYr3adj8BVKRohVfHpWQdvkIwUQ2Jf5FVM=", + "h1:zT1ZbegaAYHwQa+QwIFugArWikRJI9dqohj8xb0GY88=", + "zh:3248aae6a2198f3ec8394218d05bd5e42be59f43a3a7c0b71c66ec0df08b69e7", + "zh:32b1aaa1c3013d33c245493f4a65465eab9436b454d250102729321a44c8ab9a", + "zh:38eff7e470acb48f66380a73a5c7cdd76cc9b9c9ba9a7249c7991488abe22fe3", + "zh:4c2f1faee67af104f5f9e711c4574ff4d298afaa8a420680b0cb55d7bbc65606", + "zh:544b33b757c0b954dbb87db83a5ad921edd61f02f1dc86c6186a5ea86465b546", + "zh:696cf785090e1e8cf1587499516b0494f47413b43cb99877ad97f5d0de3dc539", + "zh:6e301f34757b5d265ae44467d95306d61bef5e41930be1365f5a8dcf80f59452", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:913a929070c819e59e94bb37a2a253c228f83921136ff4a7aa1a178c7cce5422", + "zh:aa9015926cd152425dbf86d1abdbc74bfe0e1ba3d26b3db35051d7b9ca9f72ae", + "zh:bb04798b016e1e1d49bcc76d62c53b56c88c63d6f2dfe38821afef17c416a0e1", + "zh:c23084e1b23577de22603cff752e59128d83cfecc2e6819edadd8cf7a10af11e", + ] +} diff --git a/src/domains/elt/prod/westeurope/README.md b/src/domains/elt/prod/westeurope/README.md new file mode 100644 index 000000000..6c60ad4c1 --- /dev/null +++ b/src/domains/elt/prod/westeurope/README.md @@ -0,0 +1,34 @@ + + +## Requirements + +| Name | Version | +|------|---------| +| [azurerm](#requirement\_azurerm) | <= 3.97.1 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [function\_apps](#module\_function\_apps) | ../../_modules/function_apps | n/a | +| [networking](#module\_networking) | ../../_modules/networking | n/a | +| [storage\_accounts](#module\_storage\_accounts) | ../../_modules/storage_accounts | n/a | + +## Resources + +| Name | Type | +|------|------| +| [azurerm_resource_group.elt_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | + +## Inputs + +No inputs. + +## Outputs + +| Name | Description | +|------|-------------| +| [function\_app\_elt](#output\_function\_app\_elt) | n/a | +| [resource\_group\_elt](#output\_resource\_group\_elt) | n/a | +| [storage\_account\_elt](#output\_storage\_account\_elt) | n/a | + diff --git a/src/domains/elt/prod/westeurope/locals.tf b/src/domains/elt/prod/westeurope/locals.tf new file mode 100644 index 000000000..09ff321da --- /dev/null +++ b/src/domains/elt/prod/westeurope/locals.tf @@ -0,0 +1,17 @@ +locals { + prefix = "io" + env_short = "p" + project = "${local.prefix}-${local.env_short}" + + location = "westeurope" + secondary_location_display_name = "North Europe" + + tags = { + CostCenter = "TS310 - PAGAMENTI & SERVIZI" + CreatedBy = "Terraform" + Environment = "Prod" + Owner = "IO" + Source = "https://github.com/pagopa/io-infra/blob/main/src/domains/elt/prod/westeurope" + ManagementTeam = "IO Platform" + } +} diff --git a/src/domains/elt/prod/westeurope/main.tf b/src/domains/elt/prod/westeurope/main.tf new file mode 100644 index 000000000..e904e92f0 --- /dev/null +++ b/src/domains/elt/prod/westeurope/main.tf @@ -0,0 +1,81 @@ +terraform { + + backend "azurerm" { + resource_group_name = "terraform-state-rg" + storage_account_name = "tfinfprodio" + container_name = "terraform-state" + key = "io-infra.elt.tfstate" + } + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "<= 3.97.1" + } + } +} + +provider "azurerm" { + features {} +} + +resource "azurerm_resource_group" "elt_rg" { + name = format("%s-elt-rg", local.project) + location = local.location + + tags = local.tags +} + +module "networking" { + source = "../../_modules/networking" + + project = local.project + + # inferred from vnet-common with cidr 10.0.0.0/16 + # https://github.com/pagopa/io-infra/blob/d5101ef7b24bc262b8a7773a9690a00afe9ec92e/src/core/network.tf#L8 + cidr_subnet_elt = ["10.0.11.0/24"] + + tags = local.tags +} + +module "storage_accounts" { + source = "../../_modules/storage_accounts" + + project = local.project + location = local.location + resource_group_name = azurerm_resource_group.elt_rg.name + + tags = local.tags +} + +module "function_apps" { + source = "../../_modules/function_apps" + + project = local.project + location = local.location + secondary_location_display_name = local.secondary_location_display_name + resource_group_name = azurerm_resource_group.elt_rg.name + + vnet_name = module.networking.vnet_common.name + subnet_id = module.networking.subnet_elt.id + + storage_account_name = module.storage_accounts.storage_account_elt.name + storage_account_primary_access_key = module.storage_accounts.storage_account_elt_primary_access_key + storage_account_primary_connection_string = module.storage_accounts.storage_account_elt_primary_connection_string + + storage_account_tables = { + fnelterrors = module.storage_accounts.storage_account_tables.fnelterrors + fnelterrors_messages = module.storage_accounts.storage_account_tables.fnelterrors_messages + fnelterrors_message_status = module.storage_accounts.storage_account_tables.fnelterrors_message_status + fnelterrors_notification_status = module.storage_accounts.storage_account_tables.fnelterrors_notification_status + fneltcommands = module.storage_accounts.storage_account_tables.fneltcommands + fneltexports = module.storage_accounts.storage_account_tables.fneltexports + } + + storage_account_containers = { + container_messages_report_step1 = module.storage_accounts.storage_account_containers.container_messages_report_step1 + container_messages_report_step_final = module.storage_accounts.storage_account_containers.container_messages_report_step_final + } + + tags = local.tags +} diff --git a/src/domains/elt/prod/westeurope/outputs.tf b/src/domains/elt/prod/westeurope/outputs.tf new file mode 100644 index 000000000..eade4d339 --- /dev/null +++ b/src/domains/elt/prod/westeurope/outputs.tf @@ -0,0 +1,21 @@ +output "resource_group_elt" { + value = { + id = azurerm_resource_group.elt_rg.id + name = azurerm_resource_group.elt_rg.name + } +} + +output "function_app_elt" { + value = { + id = module.function_apps.function_app_elt.id + name = module.function_apps.function_app_elt.name + app_service_plan_name = module.function_apps.function_app_elt.app_service_plan_name + } +} + +output "storage_account_elt" { + value = { + id = module.storage_accounts.storage_account_elt.id + name = module.storage_accounts.storage_account_elt.name + } +}