diff --git a/CHANGELOG.md b/CHANGELOG.md
index e99edc513f..e2ea6f1cb7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -32,6 +32,7 @@ ENHANCEMENTS:
 * Upgrade Python version from 3.8 to 3.12 ([#3949](https://github.com/microsoft/AzureTRE/issues/3949))Upgrade Python version from 3.8 to 3.12 (#3949)
 * Disable storage account key usage ([[#4227](https://github.com/microsoft/AzureTRE/issues/4227)])
 * Update Guacamole dependencies ([[#4232](https://github.com/microsoft/AzureTRE/issues/4232)])
+* Core key vault firewall should not be set to "Allow public access from all networks" ([#4250](https://github.com/microsoft/AzureTRE/issues/4250))
 
 BUG FIXES:
 * Update KeyVault references in API to use the version so Terraform cascades the update ([#4112](https://github.com/microsoft/AzureTRE/pull/4112))
diff --git a/core/terraform/.terraform.lock.hcl b/core/terraform/.terraform.lock.hcl
index 1c20359910..7c6b650d30 100644
--- a/core/terraform/.terraform.lock.hcl
+++ b/core/terraform/.terraform.lock.hcl
@@ -82,6 +82,25 @@ provider "registry.terraform.io/hashicorp/local" {
   ]
 }
 
+provider "registry.terraform.io/hashicorp/null" {
+  version = "3.2.3"
+  hashes = [
+    "h1:+AnORRgFbRO6qqcfaQyeX80W0eX3VmjadjnUFUJTiXo=",
+    "zh:22d062e5278d872fe7aed834f5577ba0a5afe34a3bdac2b81f828d8d3e6706d2",
+    "zh:23dead00493ad863729495dc212fd6c29b8293e707b055ce5ba21ee453ce552d",
+    "zh:28299accf21763ca1ca144d8f660688d7c2ad0b105b7202554ca60b02a3856d3",
+    "zh:55c9e8a9ac25a7652df8c51a8a9a422bd67d784061b1de2dc9fe6c3cb4e77f2f",
+    "zh:756586535d11698a216291c06b9ed8a5cc6a4ec43eee1ee09ecd5c6a9e297ac1",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:9d5eea62fdb587eeb96a8c4d782459f4e6b73baeece4d04b4a40e44faaee9301",
+    "zh:a6355f596a3fb8fc85c2fb054ab14e722991533f87f928e7169a486462c74670",
+    "zh:b5a65a789cff4ada58a5baffc76cb9767dc26ec6b45c00d2ec8b1b027f6db4ed",
+    "zh:db5ab669cf11d0e9f81dc380a6fdfcac437aea3d69109c7aef1a5426639d2d65",
+    "zh:de655d251c470197bcbb5ac45d289595295acb8f829f6c781d4a75c8c8b7c7dd",
+    "zh:f5c68199f2e6076bce92a12230434782bf768103a427e9bb9abee99b116af7b5",
+  ]
+}
+
 provider "registry.terraform.io/hashicorp/random" {
   version     = "3.6.3"
   constraints = ">= 3.0.0, ~> 3.6"
diff --git a/core/terraform/deploy.sh b/core/terraform/deploy.sh
index e71fb14ae1..e60128e2cb 100755
--- a/core/terraform/deploy.sh
+++ b/core/terraform/deploy.sh
@@ -5,6 +5,12 @@ set -o pipefail
 set -o nounset
 # set -o xtrace
 
+# add trap to remove deployment network exceptions
+trap 'source "../../devops/scripts/remove_deployment_network_exceptions.sh"' EXIT
+
+# now add deployment network exceptions
+source "../../devops/scripts/add_deployment_network_exceptions.sh"
+
 # This is where we can migrate any Terraform before we plan and apply
 # For instance deprecated Terraform resources
 # shellcheck disable=SC1091
diff --git a/core/terraform/destroy.sh b/core/terraform/destroy.sh
index 92b6b75c4c..804a2088d1 100755
--- a/core/terraform/destroy.sh
+++ b/core/terraform/destroy.sh
@@ -5,6 +5,12 @@ set -o pipefail
 set -o nounset
 # set -o xtrace
 
+# add trap to remove deployment network exceptions on script exit
+trap 'source "$script_dir/remove_deployment_network_exceptions.sh"' EXIT
+
+# now add deployment network exceptions
+source "$script_dir/add_deployment_network_exceptions.sh"
+
 # These variables are loaded in for us
 # shellcheck disable=SC2154
 ../../devops/scripts/terraform_wrapper.sh -g "${TF_VAR_mgmt_resource_group_name}" \
diff --git a/core/terraform/keyvault.tf b/core/terraform/keyvault.tf
index 5d75ae9176..33107c5e57 100644
--- a/core/terraform/keyvault.tf
+++ b/core/terraform/keyvault.tf
@@ -1,14 +1,55 @@
 resource "azurerm_key_vault" "kv" {
-  name                      = "kv-${var.tre_id}"
+  name                      = local.kv_name
   tenant_id                 = data.azurerm_client_config.current.tenant_id
   location                  = azurerm_resource_group.core.location
   resource_group_name       = azurerm_resource_group.core.name
   sku_name                  = "standard"
   enable_rbac_authorization = true
   purge_protection_enabled  = var.kv_purge_protection_enabled
-  tags                      = local.tre_core_tags
+  tags                      = merge(local.tre_core_tags, { "${local.tre_deployment_network_exception_tag}" = "true" })
 
-  lifecycle { ignore_changes = [access_policy, tags] }
+  public_network_access_enabled = local.kv_public_network_access_enabled
+
+  network_acls {
+    default_action = local.kv_network_default_action
+    bypass         = local.kv_network_bypass
+    ip_rules       = [ local.myip ]  # exception for deployment IP, this is removed in remove_deployment_network_exceptions.sh
+  }
+
+  lifecycle {
+    ignore_changes = [access_policy, tags]
+  }
+
+  # create provisioner required due to https://github.com/hashicorp/terraform-provider-azurerm/issues/18970
+  #
+  provisioner "local-exec" {
+    when = create
+    command = <<EOT
+az keyvault update --name ${local.kv_name} --public-network-access ${local.kv_public_network_access_enabled ? "Enabled" : "Disabled"} --default-action ${local.kv_network_default_action} --bypass ${local.kv_network_bypass} --output none
+az keyvault network-rule add --name ${local.kv_name} --ip-address ${local.myip} --output none
+EOT
+  }
+}
+
+locals {
+  kv_name = "kv-${var.tre_id}"
+  kv_public_network_access_enabled = true
+  kv_network_default_action = var.enable_local_debugging ? "Allow" : "Deny"
+  kv_network_bypass = "AzureServices"
+}
+
+# provisioner required due to ignore_changes = [tags] in azurerm_key_vault.kv
+#
+resource "null_resource" "add_deployment_tag" {
+  triggers = {
+    always_run = timestamp()
+  }
+
+  provisioner "local-exec" {
+    command = "az resource update --ids ${azurerm_key_vault.kv.id} --set 'tags.${local.tre_deployment_network_exception_tag}=\"true\"' --output none"
+  }
+
+  depends_on = [azurerm_key_vault.kv]
 }
 
 resource "azurerm_role_assignment" "keyvault_deployer_role" {
diff --git a/core/terraform/locals.tf b/core/terraform/locals.tf
index 22d327f96f..376dff495c 100644
--- a/core/terraform/locals.tf
+++ b/core/terraform/locals.tf
@@ -5,6 +5,10 @@ locals {
     tre_core_service_id = var.tre_id
   }
 
+  # used to tag resources (e.g. kv, sa) that require a IP exception for the deployment runner
+  # adding to their network firewall during TRE deployment (and removing at the end of deployment)
+  tre_deployment_network_exception_tag = "tre_deployment_network_exception"
+
   api_diagnostic_categories_enabled = [
     "AppServiceHTTPLogs", "AppServiceConsoleLogs", "AppServiceAppLogs",
     "AppServiceAuditLogs", "AppServiceIPSecAuditLogs", "AppServicePlatformLogs", "AppServiceAntivirusScanAuditLogs"
diff --git a/core/terraform/scripts/letsencrypt.sh b/core/terraform/scripts/letsencrypt.sh
index cb1e68f4a1..a45a48f00b 100755
--- a/core/terraform/scripts/letsencrypt.sh
+++ b/core/terraform/scripts/letsencrypt.sh
@@ -3,6 +3,13 @@ set -e
 
 script_dir=$(realpath "$(dirname "${BASH_SOURCE[0]}")")
 
+# add trap to remove deployment network exceptions on script exit
+trap 'source "$script_dir/../../../devops/scripts/remove_deployment_network_exceptions.sh"' EXIT
+
+# now add deployment network exceptions
+source "$script_dir/../../../devops/scripts/add_deployment_network_exceptions.sh"
+
+
 if [[ -z ${STORAGE_ACCOUNT} ]]; then
   echo "STORAGE_ACCOUNT not set"
   exit 1
diff --git a/devops/scripts/add_deployment_network_exceptions.sh b/devops/scripts/add_deployment_network_exceptions.sh
new file mode 100644
index 0000000000..172fd914bb
--- /dev/null
+++ b/devops/scripts/add_deployment_network_exceptions.sh
@@ -0,0 +1,100 @@
+#!/bin/bash
+
+TRE_DEPLOYMENT_NETWORK_EXCEPTION_TAG="tre_deployment_network_exception"
+
+function main() {
+
+    set -o errexit
+    set -o pipefail
+
+
+    # parse params/set up inputs
+    #
+    if [[ -z "$TRE_ID" ]]; then
+      echo -e "Could not open deployment network exceptions: TRE_ID is not set\nExiting...\n"
+      exit 1
+    fi
+
+    local MY_IP="${PUBLIC_DEPLOYMENT_IP_ADDRESS:-}"
+
+    if [[ -z "$MY_IP" ]]; then
+      MY_IP=$(curl -s "ipecho.net/plain"; echo)
+    fi
+
+    local TRE_CORE_RG="rg-${TRE_ID}"
+
+
+    # find resources that require network exceptions
+    #
+    echo -e "\nQuerying resources that require network exceptions adding for deployment..."
+
+    if [[ -z "$(az group list --query "[?name=='$TRE_CORE_RG']" --output tsv)" ]]; then
+        echo -e " Core resource group $TRE_CORE_RG not found\n"
+        return 0
+    fi
+
+    local AZ_IDS
+    AZ_IDS=$(az resource list --resource-group "$TRE_CORE_RG" --query "[?tags.${TRE_DEPLOYMENT_NETWORK_EXCEPTION_TAG}=='true'].id" --output tsv)
+
+    if [ -z "$AZ_IDS" ]; then
+      echo -e " No resources found\n"
+      return 0
+    fi
+
+
+    # add network exceptions
+    #
+    local AZ_ID
+    for AZ_ID in $AZ_IDS; do
+
+      local RESOURCE_TYPE
+      RESOURCE_TYPE=$(az resource show --ids "${AZ_ID}" --query 'type' --output tsv)
+
+      if [ "$RESOURCE_TYPE" == "Microsoft.KeyVault/vaults" ]; then
+        add_keyvault_network_exception "$AZ_ID" "$MY_IP"
+      fi
+
+    done
+
+    echo ""
+
+}
+
+function add_keyvault_network_exception() {
+  local AZ_ID="$1"
+  local MY_IP="$2"
+
+  local KV_NAME
+  KV_NAME=$(basename "$AZ_ID")
+
+  echo " Adding keyvault deployment network exception for $KV_NAME"
+
+  az keyvault network-rule add --name "$KV_NAME" --ip-address "$MY_IP" --output none
+
+  local ATTEMPT=1
+  local MAX_ATTEMPTS=10
+
+  while true; do
+
+    if KV_OUTPUT=$(az keyvault secret list --vault-name "$KV_NAME" --query '[].name' --output tsv 2>&1); then
+      echo "  Keyvault $KV_NAME is now accessible"
+      break
+    fi
+
+    if [ $ATTEMPT -eq $MAX_ATTEMPTS ]; then
+      echo -e "Could not add deployment network exception for $KV_NAME"
+      echo -e "Unable to access keyvault $KV_NAME after $ATTEMPT/$MAX_ATTEMPTS.\n"
+      echo -e "$KV_OUTPUT\n"
+
+      exit 1
+    fi
+
+    echo "  Unable to access keyvault $KV_NAME after $ATTEMPT/$MAX_ATTEMPTS. Waiting for network rules to take effect."
+    sleep 5
+    ((ATTEMPT++))
+
+  done
+
+}
+
+main "$@"
diff --git a/devops/scripts/destroy_env_no_terraform.sh b/devops/scripts/destroy_env_no_terraform.sh
index e327f313c6..a8a975aacf 100755
--- a/devops/scripts/destroy_env_no_terraform.sh
+++ b/devops/scripts/destroy_env_no_terraform.sh
@@ -66,6 +66,14 @@ then
   no_wait_option="--no-wait"
 fi
 
+script_dir=$(realpath "$(dirname "${BASH_SOURCE[0]}")")
+
+# add trap to remove deployment network exceptions on script exit
+trap 'source "$script_dir/remove_deployment_network_exceptions.sh"' EXIT
+
+# now add deployment network exceptions
+source "$script_dir/add_deployment_network_exceptions.sh"
+
 group_show_result=$(az group show --name "${core_tre_rg}" > /dev/null 2>&1; echo $?)
 if [[ "$group_show_result" !=  "0" ]]; then
   echo "Resource group ${core_tre_rg} not found - skipping destroy"
diff --git a/devops/scripts/key_vault_list.sh b/devops/scripts/key_vault_list.sh
index faa1aa9384..4d21867561 100755
--- a/devops/scripts/key_vault_list.sh
+++ b/devops/scripts/key_vault_list.sh
@@ -7,6 +7,14 @@ fi
 
 echo "DEBUG: Check keyvault and secrets exist"
 
+script_dir=$(realpath "$(dirname "${BASH_SOURCE[0]}")")
+
+# add trap to remove deployment network exceptions on script exit
+trap 'source "$script_dir/remove_deployment_network_exceptions.sh"' EXIT
+
+# now add deployment network exceptions
+source "$script_dir/add_deployment_network_exceptions.sh"
+
 echo "az keyvault show"
 az keyvault show --name kv-${TRE_ID}
 
diff --git a/devops/scripts/remove_deployment_network_exceptions.sh b/devops/scripts/remove_deployment_network_exceptions.sh
new file mode 100644
index 0000000000..91fd8d60c9
--- /dev/null
+++ b/devops/scripts/remove_deployment_network_exceptions.sh
@@ -0,0 +1,74 @@
+#!/bin/bash
+
+TRE_DEPLOYER_FW_EXCEPTION_TAG="tre_deployment_network_exception"
+
+function main() {
+
+    set -o errexit
+    set -o pipefail
+
+    # parse params/set up inputs
+    #
+    if [[ -z "$TRE_ID" ]]; then
+      echo -e "Could not close deployment network exceptions: TRE_ID is not set\n"
+      exit 1
+    fi
+
+    local MY_IP="${PUBLIC_DEPLOYMENT_IP_ADDRESS:-}"
+
+    if [[ -z "$MY_IP" ]]; then
+      MY_IP=$(curl -s "ipecho.net/plain"; echo)
+    fi
+
+    local TRE_CORE_RG="rg-${TRE_ID}"
+
+
+    # find resources that require network exceptions
+    #
+    echo -e "\nQuerying resources that require network exceptions removing for deployment..."
+
+    if [[ -z "$(az group list --query "[?name=='$TRE_CORE_RG']" --output tsv)" ]]; then
+        echo -e " Core resource group $TRE_CORE_RG not found\n"
+        return 0
+    fi
+
+    local AZ_IDS
+    AZ_IDS=$(az resource list --resource-group "$TRE_CORE_RG" --query "[?tags.${TRE_DEPLOYMENT_NETWORK_EXCEPTION_TAG}=='true'].id" --output tsv)
+
+    if [ -z "$AZ_IDS" ]; then
+      echo -e " No resources found\n"
+      return 0
+    fi
+
+
+    # remove network exceptions
+    #
+    local AZ_ID
+    for AZ_ID in $AZ_IDS; do
+
+      local RESOURCE_TYPE
+      RESOURCE_TYPE=$(az resource show --ids "${AZ_ID}" --query 'type' --output tsv)
+
+      if [ "$RESOURCE_TYPE" == "Microsoft.KeyVault/vaults" ]; then
+        remove_keyvault_network_exception "$AZ_ID" "$MY_IP"
+      fi
+
+    done
+
+    echo ""
+
+}
+
+function remove_keyvault_network_exception() {
+  local AZ_ID="$1"
+  local MY_IP="$2"
+
+  local KV_NAME
+  KV_NAME=$(basename "$AZ_ID")
+
+  echo " Removing keyvault deployment network exception for $KV_NAME"
+
+  az keyvault network-rule remove --name "$KV_NAME" --ip-address "$MY_IP" --output none
+}
+
+main "$@"
diff --git a/devops/scripts/set_contributor_sp_secrets.sh b/devops/scripts/set_contributor_sp_secrets.sh
index 95a07da877..ecc09d2a38 100755
--- a/devops/scripts/set_contributor_sp_secrets.sh
+++ b/devops/scripts/set_contributor_sp_secrets.sh
@@ -16,6 +16,16 @@ set -e
 #
 
 echo -e "\n\e[34m»»» 🤖 \e[96mCreating (or updating) service principal ID and secret to Key Vault\e[0m..."
+
+script_dir=$(realpath "$(dirname "${BASH_SOURCE[0]}")")
+
+# add trap to remove deployment network exceptions on script exit
+trap 'source "$script_dir/remove_deployment_network_exceptions.sh"' EXIT
+
+# now add deployment network exceptions
+source "$script_dir/add_deployment_network_exceptions.sh"
+
+
 key_vault_name="kv-$TRE_ID"
 az account set --subscription $ARM_SUBSCRIPTION_ID
 az keyvault secret set --name deployment-processor-azure-client-id --vault-name $key_vault_name --value $RESOURCE_PROCESSOR_CLIENT_ID