Merge branch 'main' into core-network-subnets-change

.github/workflows/build_validation_develop.yml

@@ -1,11 +1,11 @@
 ---
 name: Build Validation
 
-on:  # yamllint disable-line rule:truthy
+on: # yamllint disable-line rule:truthy
   pull_request:
     branches:
       - main
-      - 'feature/**'
+      - "feature/**"
 
 # for each ref (branch/pr) run just the most recent,
 # cancel other pending/running ones
@@ -52,6 +52,16 @@ jobs:
             terraform_workspace_services:
               - templates/workspace_services/**/terraform/**/*.tf
 
+      - uses: hashicorp/setup-terraform@v3
+        if: ${{ steps.filter.outputs.terraform == 'true' }}
+        with:
+          terraform_version: "1.9.8"
+
+      - uses: hashicorp/setup-terraform@v3
+        if: ${{ steps.filter.outputs.terraform == 'true' }}
+        with:
+          terraform_version: "1.9.8"
+
       - name: Terraform format check
         if: ${{ steps.filter.outputs.terraform == 'true' }}
         run: terraform fmt -check -recursive
@@ -107,7 +117,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           VALIDATE_TERRAFORM_TFLINT: true
           TERRAFORM_TFLINT_CONFIG_FILE: .tflint_core.hcl
-          FILTER_REGEX_INCLUDE: './core/.*'
+          FILTER_REGEX_INCLUDE: "./core/.*"
 
       - name: Workspace Tags
         if: ${{ steps.filter.outputs.terraform_workspaces == 'true' }}
@@ -118,7 +128,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           VALIDATE_TERRAFORM_TFLINT: true
           TERRAFORM_TFLINT_CONFIG_FILE: .tflint_workspaces.hcl
-          FILTER_REGEX_INCLUDE: './templates/workspaces/.*'
+          FILTER_REGEX_INCLUDE: "./templates/workspaces/.*"
 
       - name: Workspace Services Tags
         if: ${{ steps.filter.outputs.terraform_workspace_services == 'true' }}
@@ -129,8 +139,8 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           VALIDATE_TERRAFORM_TFLINT: true
           TERRAFORM_TFLINT_CONFIG_FILE: .tflint_workspace_services.hcl
-          FILTER_REGEX_INCLUDE: './templates/workspaces/.*'
-          FILTER_REGEX_EXCLUDE: '.*user_resource.*'
+          FILTER_REGEX_INCLUDE: "./templates/workspaces/.*"
+          FILTER_REGEX_EXCLUDE: ".*user_resource.*"
 
       - name: User Resources Tags
         if: ${{ steps.filter.outputs.terraform_workspace_services == 'true' }}
@@ -141,7 +151,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           VALIDATE_TERRAFORM_TFLINT: true
           TERRAFORM_TFLINT_CONFIG_FILE: .tflint_user_resources.hcl
-          FILTER_REGEX_INCLUDE: './templates/workspace_services/.*/user_resources/.*'
+          FILTER_REGEX_INCLUDE: "./templates/workspace_services/.*/user_resources/.*"
 
       - name: Shared Services Tags
         if: ${{ steps.filter.outputs.terraform_shared_services == 'true' }}
@@ -152,4 +162,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           VALIDATE_TERRAFORM_TFLINT: true
           TERRAFORM_TFLINT_CONFIG_FILE: .tflint_shared_services.hcl
-          FILTER_REGEX_INCLUDE: './templates/shared_services/.*'
+          FILTER_REGEX_INCLUDE: "./templates/shared_services/.*"

.github/workflows/codeql-analysis.yml

@@ -12,13 +12,13 @@
 #
 name: "CodeQL"
 
-on:  # yamllint disable-line rule:truthy
+on: # yamllint disable-line rule:truthy
   push:
     branches: [main]
   pull_request:
     branches: [main]
   schedule:
-    - cron: '41 3 * * 5'
+    - cron: "41 3 * * 5"
 
 # for each ref (branch/pr) run just the most recent,
 # cancel other pending/running ones
@@ -29,7 +29,7 @@ concurrency:
 jobs:
   analyze:
     name: Analyze
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     permissions:
       actions: read
       contents: read
@@ -38,7 +38,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        language: ['python', 'java', 'javascript', 'typescript']
+        language: ["python", "java", "javascript", "typescript"]
 
     steps:
       - name: Checkout repository

CHANGELOG.md

@@ -14,6 +14,7 @@ ENHANCEMENTS:
 * Split log entries with [Log chunk X of Y] for better readability. ([#3992](https://github.com/microsoft/AzureTRE/issues/3992))
 * Expose APP_SERVICE_SKU build variable to allow enablement of App Gateway WAF ([#4111](https://github.com/microsoft/AzureTRE/pull/4111))
 * Update Terraform to use Azure AD authentication rather than storage account keys ([#4103](https://github.com/microsoft/AzureTRE/issues/4103))
+* Consolidate Terraform upgrade scripts ([#4099](https://github.com/microsoft/AzureTRE/issues/4099))
 * Storage accounts should use infrastructure encryption ([#4001](https://github.com/microsoft/AzureTRE/issues/4001))
 * Update obsolete Terraform properties ([#4136](https://github.com/microsoft/AzureTRE/issues/4136))
 * Update Guacamole version and dependencies ([#4140](https://github.com/microsoft/AzureTRE/issues/4140))
@@ -48,6 +49,7 @@ BUG FIXES:
 * Fix VM actions where Workspace shared storage doesn't allow shared key access ([#4222](https://github.com/microsoft/AzureTRE/issues/4222))
 * Fix public exposure in Guacamole service ([[#4199](https://github.com/microsoft/AzureTRE/issues/4199)])
 * Fix Azure ML network tags to use name rather than ID ([[#4151](https://github.com/microsoft/AzureTRE/issues/4151)])
+* Add option to force tunnel TRE's Firewall ([#4237](https://github.com/microsoft/AzureTRE/issues/4237))
 
 COMPONENTS:
 

Makefile

@@ -149,7 +149,7 @@ terraform-upgrade:
 	&& . ${MAKEFILE_DIR}/devops/scripts/check_dependencies.sh env \
 	&& . ${MAKEFILE_DIR}/devops/scripts/load_and_validate_env.sh \
 	&& . ${MAKEFILE_DIR}/devops/scripts/load_env.sh ${DIR}/.env \
-	&& cd ${DIR}/terraform/ && ./upgrade.sh
+	&& ./devops/scripts/upgrade.sh ${DIR}
 
 terraform-import:
 	$(call target_title, "Importing ${DIR} with Terraform") \
@@ -309,8 +309,10 @@ deploy-shared-service:
 	&& ${MAKEFILE_DIR}/devops/scripts/deploy_shared_service.sh $${PROPS}
 
 firewall-install:
-	$(MAKE) bundle-build bundle-publish bundle-register deploy-shared-service \
-	DIR=${MAKEFILE_DIR}/templates/shared_services/firewall/ BUNDLE_TYPE=shared_service
+	. ${MAKEFILE_DIR}/devops/scripts/check_dependencies.sh env \
+	&& $(MAKE) bundle-build bundle-publish bundle-register deploy-shared-service \
+	DIR=${MAKEFILE_DIR}/templates/shared_services/firewall/ BUNDLE_TYPE=shared_service \
+	PROPS="$${FIREWALL_SKU+--firewall_sku $${FIREWALL_SKU} }$${FIREWALL_FORCE_TUNNEL_IP+--firewall_force_tunnel_ip $${FIREWALL_FORCE_TUNNEL_IP} }"
 
 static-web-upload:
 	$(call target_title, "Uploading to static website") \

config.sample.yaml

@@ -10,7 +10,7 @@ management:
   acr_name: __CHANGE_ME__
   # ID of external Key Vault to store CMKs in (only required if enable_cmk_encryption is true)
   # external_key_store_id: __CHANGE_ME__
-  # Name of Key Vault for encryption keys, required only if enable_cmk_encryption is true and not using external_key_store_id
+  # Name of Key Vault for encryption, required if enable_cmk_encryption is true and external_key_store_id is not set
   # encryption_kv_name: __CHANGE_ME__
   # Azure Resource Manager credentials used for CI/CD pipelines
   arm_subscription_id: __CHANGE_ME__
@@ -46,6 +46,7 @@ tre:
   # The TRE Web UI is deployed by default.
   # Uncomment the following to disable deployment of the Web UI.
   # deploy_ui: false
+  # firewall_force_tunnel_ip: __CHANGE_ME__
   firewall_sku: Standard
   app_gateway_sku: Standard_v2
 

core/version.txt

@@ -1 +1 @@
-__version__ = "0.11.15"
+__version__ = "0.11.16"

devops/scripts/upgrade.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+set -e
+
+# This script is used to upgrade terraform providers in a specified directory
+
+# Usage: ./upgrade.sh <directory>
+
+DIR=$1
+
+# Load environment variables from .env file
+if [ -f "$DIR/.env" ]; then
+  set -a
+  # shellcheck source=/dev/null
+  . "$DIR/.env"
+  set +a
+fi
+
+# Ensure TF_VAR_mgmt_resource_group_name is set
+if [ -z "${TF_VAR_mgmt_resource_group_name}" ]; then
+  echo "Error: TF_VAR_mgmt_resource_group_name is not set."
+  exit 1
+fi
+
+# Ensure TF_VAR_mgmt_storage_account_name is set
+if [ -z "${TF_VAR_mgmt_storage_account_name}" ]; then
+  echo "Error: TF_VAR_mgmt_storage_account_name is not set."
+  exit 1
+fi
+
+# Ensure TF_VAR_terraform_state_container_name is set
+if [ -z "${TF_VAR_terraform_state_container_name}" ]; then
+  echo "Error: TF_VAR_terraform_state_container_name is not set."
+  exit 1
+fi
+
+# Ensure TRE_ID is set
+if [ -z "${TRE_ID}" ]; then
+  echo "Error: TRE_ID is not set."
+  exit 1
+fi
+
+# Infer the key from the directory names
+PARENT_DIR=$(basename "$(dirname "$DIR")")
+GRANDPARENT_DIR=$(basename "$(dirname "$(dirname "$DIR")")")
+
+if [[ "$GRANDPARENT_DIR" == "workspaces" || "$GRANDPARENT_DIR" == "shared_services" ]]; then
+  KEY="${TRE_ID?}_${TF_VAR_id?}_${PARENT_DIR}"
+elif [[ "$GRANDPARENT_DIR" == "workspace_services" ]]; then
+  KEY="${TRE_ID?}_${TF_VAR_workspace_id?}_${TF_VAR_id?}_${PARENT_DIR}"
+elif [[ "$GRANDPARENT_DIR" == "user_resources" ]]; then
+  KEY="${TRE_ID?}_${TF_VAR_workspace_id?}_${TF_VAR_workspace_service_id?}_${TF_VAR_id?}_${PARENT_DIR}"
+else
+  KEY="${TRE_ID?}_${PARENT_DIR}"
+fi
+
+# Run terraform init with upgrade and reconfigure options
+terraform -chdir="$DIR/terraform" init -upgrade -reconfigure -input=false -backend=true \
+    -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \
+    -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name}" \
+    -backend-config="container_name=${TF_VAR_terraform_state_container_name}" \
+    -backend-config="key=${KEY}"

docs/tre-admins/configure-firewall-force-tunneling.md

@@ -0,0 +1,21 @@
+# Forced Tunneling to External Firewall in TRE
+
+Azure TRE deploys and manages an Azure firewall to ensure creation of workspace level rules can be automated when TRE workspaces and other services are created without manual intervention.
+It is highly recommended leaving the Azure TRE firewall in place. If there is still the requirement to send all traffic through a centralized enterprise firewall, such as that deployed as part of an Azure landing zone, then forced tunnelling should be used. The centralized firewall will need a superset of rules used by the TRE.
+
+To setup forced tunneling to an external firewall, follow these steps:
+
+## 1. Set the firewall_force_tunnel_ip parameter in the config.yaml file
+Provide the external firewall's IP address:
+
+```json
+firewall_force_tunnel_ip: 192.168.0.4
+```
+This automatically creates a route table to direct TRE’s traffic to the specified IP.
+
+## 2. Manually Connect TRE to Your Firewall
+Configure connectivity between TRE’s VNet and your external firewall using one of the following methods:
+
+1. **VNet Peering**: Peer the TRE VNet with your firewall’s VNet.
+1. **ExpressRoute**: Use a private connection for firewalls located on-premises.
+1. **Site-to-Site VPN**: Establish a VPN connection as an alternative.

mkdocs.yml

@@ -140,6 +140,7 @@ nav:
           - Supported Clouds: tre-admins/supported-clouds.md
           - Customer Managed Keys: tre-admins/customer-managed-keys.md
           - Custom Domain Name: tre-admins/custom-domain.md
+          - Firewall Force Tunneling: tre-admins/configure-firewall-force-tunneling.md
 
       - Development: # Docs related to the developing code for the AzureTRE
           - Local Development: using-tre/local-development/local-development.md

templates/shared_services/firewall/parameters.json

@@ -63,6 +63,12 @@
       "source": {
         "env": "ARM_ENVIRONMENT"
       }
+    },
+    {
+      "name": "firewall_force_tunnel_ip",
+      "source": {
+        "env": "FIREWALL_FORCE_TUNNEL_IP"
+      }
     }
   ]
 }