From 415b35bae8e1e92e6949d472b987105f369be650 Mon Sep 17 00:00:00 2001 From: Shuya Ma <87669292+shuyama1@users.noreply.github.com> Date: Wed, 8 Jan 2025 11:45:36 -0800 Subject: [PATCH] Add GOOGLE_VMWAREENGINE_PROJECT env var for vmwareengine tests (#12707) --- .ci/gcb-pr-downstream-generation-and-test.yml | 4 +- .ci/gcb-push-downstream.yml | 3 + .ci/gcb-vcr-nightly.yml | 4 +- .ci/magician/cmd/check_cassettes.go | 1 + .ci/magician/cmd/test_eap_vcr.go | 1 + .ci/magician/cmd/test_terraform_vcr.go | 1 + .ci/magician/cmd/vcr_cassette_update.go | 1 + .ci/magician/vcr/tester.go | 1 + mmv1/api/resource/examples.go | 28 ++++---- mmv1/provider/template_data.go | 66 ++++++++++--------- .../terraform/env_var_context.go.tmpl | 2 + .../components/builds/build_parameters.kt | 10 +++ .../terraform/.teamcity/settings.kts | 7 ++ .../terraform/.teamcity/tests/test_utils.kt | 3 + .../terraform/envvar/envvar_utils.go | 11 ++++ 15 files changed, 96 insertions(+), 47 deletions(-) diff --git a/.ci/gcb-pr-downstream-generation-and-test.yml b/.ci/gcb-pr-downstream-generation-and-test.yml index 675217116245..a1dc4555ad6b 100644 --- a/.ci/gcb-pr-downstream-generation-and-test.yml +++ b/.ci/gcb-pr-downstream-generation-and-test.yml @@ -251,7 +251,7 @@ steps: - name: 'gcr.io/graphite-docker-images/go-plus' id: gcb-tpg-vcr-test entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' - secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS", "GITHUB_TOKEN_MAGIC_MODULES", "GOOGLE_BILLING_ACCOUNT", "GOOGLE_CHRONICLE_INSTANCE_ID", "GOOGLE_CUST_ID", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION"] + secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS", "GITHUB_TOKEN_MAGIC_MODULES", "GOOGLE_BILLING_ACCOUNT", "GOOGLE_CHRONICLE_INSTANCE_ID", "GOOGLE_CUST_ID", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION", "GOOGLE_VMWAREENGINE_PROJECT"] waitFor: ["diff"] env: - BASE_BRANCH=$_BASE_BRANCH @@ -314,3 +314,5 @@ availableSecrets: env: SA_KEY - versionName: projects/673497134629/secrets/ci-test-public-advertised-prefix-description/versions/latest env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION + - versionName: projects/673497134629/secrets/ci-test-vmwareengine-project/versions/latest + env: GOOGLE_VMWAREENGINE_PROJECT diff --git a/.ci/gcb-push-downstream.yml b/.ci/gcb-push-downstream.yml index 2249dc6d39b2..9d3066a8bc36 100644 --- a/.ci/gcb-push-downstream.yml +++ b/.ci/gcb-push-downstream.yml @@ -203,6 +203,7 @@ steps: - "GOOGLE_SERVICE_ACCOUNT" - "SA_KEY" - "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION" + - "GOOGLE_VMWAREENGINE_PROJECT" env: - "COMMIT_SHA=$COMMIT_SHA" - "GOOGLE_REGION=us-central1" @@ -249,3 +250,5 @@ availableSecrets: env: SA_KEY - versionName: projects/673497134629/secrets/ci-test-public-advertised-prefix-description/versions/latest env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION + - versionName: projects/673497134629/secrets/ci-test-vmwareengine-project/versions/latest + env: GOOGLE_VMWAREENGINE_PROJECT diff --git a/.ci/gcb-vcr-nightly.yml b/.ci/gcb-vcr-nightly.yml index ee0fc58c518f..e81b744fe4fd 100644 --- a/.ci/gcb-vcr-nightly.yml +++ b/.ci/gcb-vcr-nightly.yml @@ -3,7 +3,7 @@ steps: - name: 'gcr.io/graphite-docker-images/go-plus' id: gcb-vcr-nightly entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' - secretEnv: ["GOOGLE_BILLING_ACCOUNT", "GOOGLE_CHRONICLE_INSTANCE_ID", "GOOGLE_CUST_ID", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION", "GITHUB_TOKEN_CLASSIC"] + secretEnv: ["GOOGLE_BILLING_ACCOUNT", "GOOGLE_CHRONICLE_INSTANCE_ID", "GOOGLE_CUST_ID", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION", "GITHUB_TOKEN_CLASSIC", "GOOGLE_VMWAREENGINE_PROJECT"] env: - "GOOGLE_REGION=us-central1" - "GOOGLE_ZONE=us-central1-a" @@ -48,3 +48,5 @@ availableSecrets: env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION - versionName: projects/673497134629/secrets/github-classic--repo-workflow/versions/latest env: GITHUB_TOKEN_CLASSIC + - versionName: projects/673497134629/secrets/ci-test-vmwareengine-project/versions/latest + env: GOOGLE_VMWAREENGINE_PROJECT diff --git a/.ci/magician/cmd/check_cassettes.go b/.ci/magician/cmd/check_cassettes.go index 497fcd0c09e9..74b5ec3ce3d6 100644 --- a/.ci/magician/cmd/check_cassettes.go +++ b/.ci/magician/cmd/check_cassettes.go @@ -34,6 +34,7 @@ var ccRequiredEnvironmentVariables = [...]string{ var ccOptionalEnvironmentVariables = [...]string{ "GOOGLE_CHRONICLE_INSTANCE_ID", + "GOOGLE_VMWAREENGINE_PROJECT", } var checkCassettesCmd = &cobra.Command{ diff --git a/.ci/magician/cmd/test_eap_vcr.go b/.ci/magician/cmd/test_eap_vcr.go index 4bf1f272169b..867c1a7ac800 100644 --- a/.ci/magician/cmd/test_eap_vcr.go +++ b/.ci/magician/cmd/test_eap_vcr.go @@ -43,6 +43,7 @@ var tevOptionalEnvironmentVariables = [...]string{ "GOOGLE_ORG_2", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION", "GOOGLE_SERVICE_ACCOUNT", + "GOOGLE_VMWAREENGINE_PROJECT", } var testEAPVCRCmd = &cobra.Command{ diff --git a/.ci/magician/cmd/test_terraform_vcr.go b/.ci/magician/cmd/test_terraform_vcr.go index ce92d213b54f..3dd30d9a3d1d 100644 --- a/.ci/magician/cmd/test_terraform_vcr.go +++ b/.ci/magician/cmd/test_terraform_vcr.go @@ -57,6 +57,7 @@ var ttvRequiredEnvironmentVariables = [...]string{ var ttvOptionalEnvironmentVariables = [...]string{ "GOOGLE_CHRONICLE_INSTANCE_ID", + "GOOGLE_VMWAREENGINE_PROJECT", } type analytics struct { diff --git a/.ci/magician/cmd/vcr_cassette_update.go b/.ci/magician/cmd/vcr_cassette_update.go index f90a0c0a6541..c03ef43ce470 100644 --- a/.ci/magician/cmd/vcr_cassette_update.go +++ b/.ci/magician/cmd/vcr_cassette_update.go @@ -40,6 +40,7 @@ var vcuRequiredEnvironmentVariables = [...]string{ var vcuOptionalEnvironmentVariables = [...]string{ "GOOGLE_CHRONICLE_INSTANCE_ID", + "GOOGLE_VMWAREENGINE_PROJECT", } var ( diff --git a/.ci/magician/vcr/tester.go b/.ci/magician/vcr/tester.go index e898740080ad..2bd55a12169e 100644 --- a/.ci/magician/vcr/tester.go +++ b/.ci/magician/vcr/tester.go @@ -92,6 +92,7 @@ var safeToLog = map[string]bool{ "GOOGLE_REGION": true, "GOOGLE_SERVICE_ACCOUNT": true, "GOOGLE_TEST_DIRECTORY": true, + "GOOGLE_VMWAREENGINE_PROJECT": true, "GOOGLE_ZONE": true, "GOPATH": true, "HOME": true, diff --git a/mmv1/api/resource/examples.go b/mmv1/api/resource/examples.go index 895b63585c38..3304589a5c86 100644 --- a/mmv1/api/resource/examples.go +++ b/mmv1/api/resource/examples.go @@ -73,6 +73,7 @@ type Examples struct { // - :CUST_ID // - :IDENTITY_USER // - :CHRONICLE_ID + // - :VMWAREENGINE_PROJECT // This list corresponds to the `get*FromEnv` methods in provider_test.go. TestEnvVars map[string]string `yaml:"test_env_vars,omitempty"` @@ -215,19 +216,20 @@ func (e *Examples) SetHCLText() { originalTestEnvVars := e.TestEnvVars docTestEnvVars := make(map[string]string) docs_defaults := map[string]string{ - "PROJECT_NAME": "my-project-name", - "CREDENTIALS": "my/credentials/filename.json", - "REGION": "us-west1", - "ORG_ID": "123456789", - "ORG_DOMAIN": "example.com", - "ORG_TARGET": "123456789", - "BILLING_ACCT": "000000-0000000-0000000-000000", - "MASTER_BILLING_ACCT": "000000-0000000-0000000-000000", - "SERVICE_ACCT": "my@service-account.com", - "CUST_ID": "A01b123xz", - "IDENTITY_USER": "cloud_identity_user", - "PAP_DESCRIPTION": "description", - "CHRONICLE_ID": "00000000-0000-0000-0000-000000000000", + "PROJECT_NAME": "my-project-name", + "CREDENTIALS": "my/credentials/filename.json", + "REGION": "us-west1", + "ORG_ID": "123456789", + "ORG_DOMAIN": "example.com", + "ORG_TARGET": "123456789", + "BILLING_ACCT": "000000-0000000-0000000-000000", + "MASTER_BILLING_ACCT": "000000-0000000-0000000-000000", + "SERVICE_ACCT": "my@service-account.com", + "CUST_ID": "A01b123xz", + "IDENTITY_USER": "cloud_identity_user", + "PAP_DESCRIPTION": "description", + "CHRONICLE_ID": "00000000-0000-0000-0000-000000000000", + "VMWAREENGINE_PROJECT": "my-vmwareengine-project", } // Apply doc defaults to test_env_vars from YAML diff --git a/mmv1/provider/template_data.go b/mmv1/provider/template_data.go index 4a90da836d86..50cc1bbb0e79 100644 --- a/mmv1/provider/template_data.go +++ b/mmv1/provider/template_data.go @@ -119,22 +119,23 @@ func (td *TemplateData) GenerateTestFile(filePath string, resource api.Resource) templatePath, } tmplInput := TestInput{ - Res: resource, - ImportPath: td.ImportPath(), - PROJECT_NAME: "my-project-name", - CREDENTIALS: "my/credentials/filename.json", - REGION: "us-west1", - ORG_ID: "123456789", - ORG_DOMAIN: "example.com", - ORG_TARGET: "123456789", - PROJECT_NUMBER: "1111111111111", - BILLING_ACCT: "000000-0000000-0000000-000000", - MASTER_BILLING_ACCT: "000000-0000000-0000000-000000", - SERVICE_ACCT: "my@service-account.com", - CUST_ID: "A01b123xz", - IDENTITY_USER: "cloud_identity_user", - PAP_DESCRIPTION: "description", - CHRONICLE_ID: "00000000-0000-0000-0000-000000000000", + Res: resource, + ImportPath: td.ImportPath(), + PROJECT_NAME: "my-project-name", + CREDENTIALS: "my/credentials/filename.json", + REGION: "us-west1", + ORG_ID: "123456789", + ORG_DOMAIN: "example.com", + ORG_TARGET: "123456789", + PROJECT_NUMBER: "1111111111111", + BILLING_ACCT: "000000-0000000-0000000-000000", + MASTER_BILLING_ACCT: "000000-0000000-0000000-000000", + SERVICE_ACCT: "my@service-account.com", + CUST_ID: "A01b123xz", + IDENTITY_USER: "cloud_identity_user", + PAP_DESCRIPTION: "description", + CHRONICLE_ID: "00000000-0000-0000-0000-000000000000", + VMWAREENGINE_PROJECT: "my-vmwareengine-project", } td.GenerateFile(filePath, templatePath, tmplInput, true, templates...) @@ -283,20 +284,21 @@ func FixImports(outputPath string, dumpDiffs bool) { } type TestInput struct { - Res api.Resource - ImportPath string - PROJECT_NAME string - CREDENTIALS string - REGION string - ORG_ID string - ORG_DOMAIN string - ORG_TARGET string - PROJECT_NUMBER string - BILLING_ACCT string - MASTER_BILLING_ACCT string - SERVICE_ACCT string - CUST_ID string - IDENTITY_USER string - PAP_DESCRIPTION string - CHRONICLE_ID string + Res api.Resource + ImportPath string + PROJECT_NAME string + CREDENTIALS string + REGION string + ORG_ID string + ORG_DOMAIN string + ORG_TARGET string + PROJECT_NUMBER string + BILLING_ACCT string + MASTER_BILLING_ACCT string + SERVICE_ACCT string + CUST_ID string + IDENTITY_USER string + PAP_DESCRIPTION string + CHRONICLE_ID string + VMWAREENGINE_PROJECT string } diff --git a/mmv1/templates/terraform/env_var_context.go.tmpl b/mmv1/templates/terraform/env_var_context.go.tmpl index b21a761b48cd..faa834e9f018 100644 --- a/mmv1/templates/terraform/env_var_context.go.tmpl +++ b/mmv1/templates/terraform/env_var_context.go.tmpl @@ -28,6 +28,8 @@ "{{$varKey}}": envvar.GetTestPublicAdvertisedPrefixDescriptionFromEnv(t), {{- else if eq $varVal "CHRONICLE_ID" }} "{{$varKey}}": envvar.GetTestChronicleInstanceIdFromEnv(t), + {{- else if eq $varVal "VMWAREENGINE_PROJECT" }} + "{{$varKey}}": envvar.GetTestVmwareengineProjectFromEnv(t), {{- else if eq $varVal "ZONE" }} "{{$varKey}}": envvar.GetTestZoneFromEnv(), {{- end }} diff --git a/mmv1/third_party/terraform/.teamcity/components/builds/build_parameters.kt b/mmv1/third_party/terraform/.teamcity/components/builds/build_parameters.kt index 8b1f9ac42d0f..4acd6411e05d 100644 --- a/mmv1/third_party/terraform/.teamcity/components/builds/build_parameters.kt +++ b/mmv1/third_party/terraform/.teamcity/components/builds/build_parameters.kt @@ -61,6 +61,11 @@ class AllContextParameters( val chronicleInstanceIdBeta: String, val chronicleInstanceIdVcr: String, + // GOOGLE_VMWAREENGINE_PROJECT + val vmwareengineProjectGa: String, + val vmwareengineProjectBeta: String, + val vmwareengineProjectVcr: String, + // Values that are the same across GA, Beta, and VCR testing environments val billingAccount: String, // GOOGLE_BILLING_ACCOUNT val billingAccount2: String, // GOOGLE_BILLING_ACCOUNT_2 @@ -94,6 +99,7 @@ class AccTestConfiguration( val projectNumber: String, val region: String, val serviceAccount: String, + val vmwareengineProject: String, val zone: String, // VCR specific @@ -120,6 +126,7 @@ fun getGaAcceptanceTestConfig(allConfig: AllContextParameters): AccTestConfigura allConfig.projectNumberGa, allConfig.region, allConfig.serviceAccountGa, + allConfig.vmwareengineProjectGa, allConfig.zone, allConfig.infraProject, allConfig.vcrBucketName, @@ -143,6 +150,7 @@ fun getBetaAcceptanceTestConfig(allConfig: AllContextParameters): AccTestConfigu allConfig.projectNumberBeta, allConfig.region, allConfig.serviceAccountBeta, + allConfig.vmwareengineProjectBeta, allConfig.zone, allConfig.infraProject, allConfig.vcrBucketName, @@ -166,6 +174,7 @@ fun getVcrAcceptanceTestConfig(allConfig: AllContextParameters): AccTestConfigur allConfig.projectNumberVcr, allConfig.region, allConfig.serviceAccountVcr, + allConfig.vmwareengineProjectVcr, allConfig.zone, allConfig.infraProject, allConfig.vcrBucketName, @@ -190,6 +199,7 @@ fun ParametrizedWithType.configureGoogleSpecificTestParameters(config: AccTestCo hiddenVariable("env.GOOGLE_ZONE", config.zone, "The google zone to use") hiddenVariable("env.GOOGLE_IDENTITY_USER", config.identityUser, "The user for the identity platform") hiddenVariable("env.GOOGLE_CHRONICLE_INSTANCE_ID", config.chronicleInstanceId, "The id of the Chronicle instance") + hiddenVariable("env.GOOGLE_VMWAREENGINE_PROJECT", config.vmwareengineProject, "The project used for vmwareengine tests") hiddenPasswordVariable("env.GOOGLE_CREDENTIALS", config.credentials, "The Google credentials for this test runner") } diff --git a/mmv1/third_party/terraform/.teamcity/settings.kts b/mmv1/third_party/terraform/.teamcity/settings.kts index f9bab43f5ee5..46faab219f69 100644 --- a/mmv1/third_party/terraform/.teamcity/settings.kts +++ b/mmv1/third_party/terraform/.teamcity/settings.kts @@ -49,6 +49,10 @@ val org2Vcr = DslContext.getParameter("org2Vcr", "") val chronicleInstanceIdGa = DslContext.getParameter("chronicleInstanceIdGa", "") val chronicleInstanceIdBeta = DslContext.getParameter("chronicleInstanceIdBeta", "") val chronicleInstanceIdVcr = DslContext.getParameter("chronicleInstanceIdVcr", "") +// GOOGLE_VMWAREENGINE_PROJECT +val vmwareengineProjectGa = DslContext.getParameter("vmwareengineProjectGa", "") +val vmwareengineProjectBeta = DslContext.getParameter("vmwareengineProjectBeta", "") +val vmwareengineProjectVcr = DslContext.getParameter("vmwareengineProjectVcr", "") // Values that are the same across GA, Beta, and VCR testing environments val billingAccount = DslContext.getParameter("billingAccount", "") // GOOGLE_BILLING_ACCOUNT @@ -91,6 +95,9 @@ var allContextParams = AllContextParameters( chronicleInstanceIdGa, chronicleInstanceIdBeta, chronicleInstanceIdVcr, + vmwareengineProjectGa, + vmwareengineProjectBeta, + vmwareengineProjectVcr, billingAccount, billingAccount2, custId, diff --git a/mmv1/third_party/terraform/.teamcity/tests/test_utils.kt b/mmv1/third_party/terraform/.teamcity/tests/test_utils.kt index 71ad0bcd4a26..e2e481679b89 100644 --- a/mmv1/third_party/terraform/.teamcity/tests/test_utils.kt +++ b/mmv1/third_party/terraform/.teamcity/tests/test_utils.kt @@ -44,6 +44,9 @@ fun testContextParameters(): AllContextParameters { "chronicleInstanceIdGa", "chronicleInstanceIdBeta", "chronicleInstanceIdVcr", + "vmwareengineProjectGa", + "vmwareengineProjectBeta", + "vmwareengineProjectVcr", "billingAccount", "billingAccount2", "custId", diff --git a/mmv1/third_party/terraform/envvar/envvar_utils.go b/mmv1/third_party/terraform/envvar/envvar_utils.go index 32945dd99143..537242cef5e9 100644 --- a/mmv1/third_party/terraform/envvar/envvar_utils.go +++ b/mmv1/third_party/terraform/envvar/envvar_utils.go @@ -115,6 +115,12 @@ var ImpersonateServiceAccountEnvVars = []string{ "GOOGLE_IMPERSONATE_SERVICE_ACCOUNT", } +// This value is the project used for vmwareengine tests. A separate project is needed +// due to the limited quota allocated to each testing project +var vmwareengineProjectEnvVars = []string{ + "GOOGLE_VMWAREENGINE_PROJECT", +} + // AccTestPreCheck ensures at least one of the project env variables is set. func GetTestProjectNumberFromEnv() string { return transport_tpg.MultiEnvSearch(ProjectNumberEnvVars) @@ -215,6 +221,11 @@ func GetTestChronicleInstanceIdFromEnv(t *testing.T) string { return transport_tpg.MultiEnvSearch(ChronicleInstanceIdEnvVars) } +func GetTestVmwareengineProjectFromEnv(t *testing.T) string { + SkipIfEnvNotSet(t, vmwareengineProjectEnvVars...) + return transport_tpg.MultiEnvSearch(vmwareengineProjectEnvVars) +} + func SkipIfEnvNotSet(t *testing.T, envs ...string) { if t == nil { log.Printf("[DEBUG] Not running inside of test - skip skipping")