Support environment variables in config profiles

changes/17309-support-env-vars-profiles

@@ -0,0 +1 @@
+* Support environment variables in configuration profiles for GitOps.

cmd/fleetctl/apply.go

@@ -69,9 +69,11 @@ func applyCommand() *cli.Command {
 				fmt.Fprintf(c.App.Writer, format, a...)
 			}
 
-			opts := fleet.ApplySpecOptions{
-				Force:  flForce,
-				DryRun: flDryRun,
+			opts := fleet.ApplyClientSpecOptions{
+				ApplySpecOptions: fleet.ApplySpecOptions{
+					Force:  flForce,
+					DryRun: flDryRun,
+				},
 			}
 			if policiesTeamName := c.String("policies-team"); policiesTeamName != "" {
 				opts.TeamForPolicies = policiesTeamName

cmd/fleetctl/preview.go

@@ -342,7 +342,7 @@ Use the stop and reset subcommands to manage the server and dependencies once st
 			}
 			// this only applies standard queries, the base directory is not used,
 			// so pass in the current working directory.
-			_, err = client.ApplyGroup(c.Context, specs, ".", logf, fleet.ApplySpecOptions{})
+			_, err = client.ApplyGroup(c.Context, specs, ".", logf, fleet.ApplyClientSpecOptions{})
 			if err != nil {
 				return err
 			}

pkg/spec/gitops.go

@@ -58,7 +58,7 @@ type GitOps struct {
 	Software []*fleet.TeamSpecSoftware
 }
 
-// GitOpsFromBytes parses a GitOps yaml file.
+// GitOpsFromFile parses a GitOps yaml file.
 func GitOpsFromFile(filePath, baseDir string) (*GitOps, error) {
 	b, err := os.ReadFile(filePath)
 	if err != nil {

pkg/spec/gitops_test.go

@@ -59,20 +59,32 @@ func gitOpsFromString(t *testing.T, s string) (*GitOps, error) {
 func TestValidGitOpsYaml(t *testing.T) {
 	t.Parallel()
 	tests := map[string]struct {
-		filePath string
-		isTeam   bool
+		environment map[string]string
+		filePath    string
+		isTeam      bool
 	}{
 		"global_config_no_paths": {
 			filePath: "testdata/global_config_no_paths.yml",
 		},
 		"global_config_with_paths": {
+			environment: map[string]string{
+				"LINUX_OS":                      "linux",
+				"DISTRIBUTED_DENYLIST_DURATION": "0",
+				"ORG_NAME":                      "Fleet Device Management",
+			},
 			filePath: "testdata/global_config.yml",
 		},
 		"team_config_no_paths": {
 			filePath: "testdata/team_config_no_paths.yml",
 			isTeam:   true,
 		},
 		"team_config_with_paths": {
+			environment: map[string]string{
+				"POLICY":                          "policy",
+				"LINUX_OS":                        "linux",
+				"DISTRIBUTED_DENYLIST_DURATION":   "0",
+				"ENABLE_FAILING_POLICIES_WEBHOOK": "true",
+			},
 			filePath: "testdata/team_config.yml",
 			isTeam:   true,
 		},
@@ -83,14 +95,35 @@ func TestValidGitOpsYaml(t *testing.T) {
 		name := name
 		t.Run(
 			name, func(t *testing.T) {
-				t.Parallel()
+				if len(test.environment) > 0 {
+					for k, v := range test.environment {
+						os.Setenv(k, v)
+					}
+					t.Cleanup(func() {
+						for k := range test.environment {
+							os.Unsetenv(k)
+						}
+					})
+				} else {
+					t.Parallel()
+				}
+
 				gitops, err := GitOpsFromFile(test.filePath, "./testdata")
 				require.NoError(t, err)
 
 				if test.isTeam {
 					// Check team settings
 					assert.Equal(t, "Team1", *gitops.TeamName)
 					assert.Contains(t, gitops.TeamSettings, "webhook_settings")
+					webhookSettings, ok := gitops.TeamSettings["webhook_settings"].(map[string]interface{})
+					assert.True(t, ok, "webhook_settings not found")
+					assert.Contains(t, webhookSettings, "failing_policies_webhook")
+					failingPoliciesWebhook, ok := webhookSettings["failing_policies_webhook"].(map[string]interface{})
+					assert.True(t, ok, "webhook_settings not found")
+					assert.Contains(t, failingPoliciesWebhook, "enable_failing_policies_webhook")
+					enableFailingPoliciesWebhook, ok := failingPoliciesWebhook["enable_failing_policies_webhook"].(bool)
+					assert.True(t, ok)
+					assert.True(t, enableFailingPoliciesWebhook)
 					assert.Contains(t, gitops.TeamSettings, "host_expiry_settings")
 					assert.Contains(t, gitops.TeamSettings, "features")
 					assert.Contains(t, gitops.TeamSettings, "secrets")
@@ -105,6 +138,9 @@ func TestValidGitOpsYaml(t *testing.T) {
 					assert.True(t, ok, "server_settings not found")
 					assert.Equal(t, "https://fleet.example.com", serverSettings.(map[string]interface{})["server_url"])
 					assert.Contains(t, gitops.OrgSettings, "org_info")
+					orgInfo, ok := gitops.OrgSettings["org_info"].(map[string]interface{})
+					assert.True(t, ok)
+					assert.Equal(t, "Fleet Device Management", orgInfo["org_name"])
 					assert.Contains(t, gitops.OrgSettings, "smtp_settings")
 					assert.Contains(t, gitops.OrgSettings, "sso_settings")
 					assert.Contains(t, gitops.OrgSettings, "integrations")
@@ -151,12 +187,13 @@ func TestValidGitOpsYaml(t *testing.T) {
 
 				// Check agent options
 				assert.NotNil(t, gitops.AgentOptions)
-				assert.Contains(t, string(*gitops.AgentOptions), "distributed_denylist_duration")
+				assert.Contains(t, string(*gitops.AgentOptions), "\"distributed_denylist_duration\":0")
 
 				// Check queries
 				require.Len(t, gitops.Queries, 3)
 				assert.Equal(t, "Scheduled query stats", gitops.Queries[0].Name)
 				assert.Equal(t, "orbit_info", gitops.Queries[1].Name)
+				assert.Equal(t, "darwin,linux,windows", gitops.Queries[1].Platform)
 				assert.Equal(t, "osquery_info", gitops.Queries[2].Name)
 
 				// Check policies
@@ -165,6 +202,7 @@ func TestValidGitOpsYaml(t *testing.T) {
 				assert.Equal(t, "Passing policy", gitops.Policies[1].Name)
 				assert.Equal(t, "No root logins (macOS, Linux)", gitops.Policies[2].Name)
 				assert.Equal(t, "🔥 Failing policy", gitops.Policies[3].Name)
+				assert.Equal(t, "linux", gitops.Policies[3].Platform)
 				assert.Equal(t, "😊😊 Failing policy", gitops.Policies[4].Name)
 			},
 		)
@@ -283,7 +321,7 @@ queries:
 `
 	_, err = gitOpsFromString(t, config)
 	require.Error(t, err)
-	require.Contains(t, err.Error(), "variable \"NOT_DEFINED\" not set")
+	require.Contains(t, err.Error(), "environment variable \"NOT_DEFINED\" not set")
 }
 
 func TestMixingGlobalAndTeamConfig(t *testing.T) {

pkg/spec/testdata/agent-options.yml

@@ -1,10 +1,10 @@
 command_line_flags:
-  distributed_denylist_duration: 0
+  distributed_denylist_duration: $DISTRIBUTED_DENYLIST_DURATION
 config:
   decorators:
     load:
-    - SELECT uuid AS host_uuid FROM system_info;
-    - SELECT hostname AS hostname FROM system_info;
+      - SELECT uuid AS host_uuid FROM system_info;
+      - SELECT hostname AS hostname FROM system_info;
   options:
     disable_distributed: false
     distributed_interval: 10

pkg/spec/testdata/org-settings.yml

@@ -11,7 +11,7 @@ org_info:
   contact_url: https://fleetdm.com/company/contact
   org_logo_url: ""
   org_logo_url_light_background: ""
-  org_name: Fleet Device Management
+  org_name: $ORG_NAME
 smtp_settings:
   authentication_method: authmethod_plain
   authentication_type: authtype_username_password

pkg/spec/testdata/team-settings.yml

@@ -3,7 +3,7 @@ secrets:
   - secret: "ABC"
 webhook_settings:
   failing_policies_webhook:
-    enable_failing_policies_webhook: true
+    enable_failing_policies_webhook: $ENABLE_FAILING_POLICIES_WEBHOOK
     destination_url: https://example.tines.com/webhook
     policy_ids: [1, 2, 3, 4, 5, 6 ,7, 8, 9]
 features:

pkg/spec/testdata/team_config.yml

@@ -20,7 +20,7 @@ policies:
   - path: ./top.policies.yml
   - path: ./top.policies2.yml
   - path: ./empty.yml
-  - name: 😊😊 Failing policy
+  - name: 😊😊 Failing $POLICY
     platform: linux
     description: This policy should always fail.
     resolution: There is no resolution for this policy.

pkg/spec/testdata/top.policies2.yml

@@ -1,5 +1,5 @@
 - name: 🔥 Failing policy
-  platform: linux
+  platform: $LINUX_OS
   description: This policy should always fail.
   resolution: There is no resolution for this policy.
   query: SELECT 1 FROM osquery_info WHERE start_time < 0;

pkg/spec/testdata/top.queries.yml

@@ -12,7 +12,7 @@
 - name: orbit_info
   query: SELECT * from orbit_info;
   interval: 0
-  platform: darwin,linux,windows
+  platform: darwin,$LINUX_OS,windows
   min_osquery_version: all
   observer_can_run: false
   automations_enabled: true

server/fleet/app.go

@@ -1055,6 +1055,16 @@ type ApplyTeamSpecOptions struct {
 	DryRunAssumptions *TeamSpecsDryRunAssumptions
 }
 
+// ApplyClientSpecOptions embeds a ApplySpecOptions and adds additional client
+// side configuration.
+type ApplyClientSpecOptions struct {
+	ApplySpecOptions
+
+	// ExpandEnvConfigProfiles enables expansion of environment variables in
+	// configuration profiles.
+	ExpandEnvConfigProfiles bool
+}
+
 // RawQuery returns the ApplySpecOptions url-encoded for use in an URL's
 // query string parameters. It only sets the parameters that are not the
 // default values.

server/service/client.go

@@ -303,7 +303,7 @@
 
 // getProfilesContents takes file paths and creates a slice of profile payloads
 // ready to batch-apply.
-func getProfilesContents(baseDir string, profiles []fleet.MDMProfileSpec) ([]fleet.MDMProfileBatchPayload, error) {
+func getProfilesContents(baseDir string, profiles []fleet.MDMProfileSpec, expandEnv bool) ([]fleet.MDMProfileBatchPayload, error) {
 	// map to check for duplicate names
 	extByName := make(map[string]string, len(profiles))
 	result := make([]fleet.MDMProfileBatchPayload, 0, len(profiles))
@@ -315,6 +315,13 @@
 			return nil, fmt.Errorf("applying fleet config: %w", err)
 		}
 
+		if expandEnv {
+			fileContents, err = spec.ExpandEnvBytes(fileContents)
+			if err != nil {
+				return nil, fmt.Errorf("expanding environment on file %q: %w", profile.Path, err)
+			}
+		}
+
 		// by default, use the file name. macOS profiles use their PayloadDisplayName
 		ext := filepath.Ext(filePath)
 		name := strings.TrimSuffix(filepath.Base(filePath), ext)
@@ -345,7 +352,7 @@
 	specs *spec.Group,
 	baseDir string,
 	logf func(format string, args ...interface{}),
-	opts fleet.ApplySpecOptions,
+	opts fleet.ApplyClientSpecOptions,
 ) (map[string]uint, error) {
 	logfn := func(format string, args ...interface{}) {
 		if logf != nil {
@@ -424,7 +431,7 @@
 		// custom settings but windows is present and empty (but mac is absent),
 		// shouldn't that clear the windows ones?
 		if (windowsCustomSettings != nil && macosCustomSettings != nil) || len(allCustomSettings) > 0 {
-			fileContents, err := getProfilesContents(baseDir, allCustomSettings)
+			fileContents, err := getProfilesContents(baseDir, allCustomSettings, opts.ExpandEnvConfigProfiles)
 			if err != nil {
 				return nil, err
 			}
@@ -439,7 +446,7 @@
 					assumeEnabled = ok && assumeEnabled
 				}
 			}
-			if err := c.ApplyNoTeamProfiles(fileContents, opts, assumeEnabled); err != nil {
+			if err := c.ApplyNoTeamProfiles(fileContents, opts.ApplySpecOptions, assumeEnabled); err != nil {
 				return nil, fmt.Errorf("applying custom settings: %w", err)
 			}
 		}
@@ -481,11 +488,11 @@
 					Name:           filepath.Base(f),
 				}
 			}
-			if err := c.ApplyNoTeamScripts(scriptPayloads, opts); err != nil {
+			if err := c.ApplyNoTeamScripts(scriptPayloads, opts.ApplySpecOptions); err != nil {
 				return nil, fmt.Errorf("applying custom settings: %w", err)
 			}
 		}
-		if err := c.ApplyAppConfig(specs.AppConfig, opts); err != nil {
+		if err := c.ApplyAppConfig(specs.AppConfig, opts.ApplySpecOptions); err != nil {
 			return nil, fmt.Errorf("applying fleet config: %w", err)
 		}
 		if opts.DryRun {
@@ -514,7 +521,7 @@
 
 		tmFileContents := make(map[string][]fleet.MDMProfileBatchPayload, len(tmMDMSettings))
 		for k, paths := range tmMDMSettings {
-			fileContents, err := getProfilesContents(baseDir, paths)
+			fileContents, err := getProfilesContents(baseDir, paths, opts.ExpandEnvConfigProfiles)
 			if err != nil {
 				return nil, err
 			}
@@ -622,7 +629,7 @@
 		// non-existing team gets created.
 		var err error
 		teamOpts := fleet.ApplyTeamSpecOptions{
-			ApplySpecOptions:  opts,
+			ApplySpecOptions:  opts.ApplySpecOptions,
 			DryRunAssumptions: specs.TeamsDryRunAssumptions,
 		}
 		teamIDsByName, err = c.ApplyTeams(specs.Teams, teamOpts)
@@ -659,14 +666,14 @@
 		}
 		if len(tmScriptsPayloads) > 0 {
 			for tmName, scripts := range tmScriptsPayloads {
-				if err := c.ApplyTeamScripts(tmName, scripts, opts); err != nil {
+				if err := c.ApplyTeamScripts(tmName, scripts, opts.ApplySpecOptions); err != nil {
 					return nil, fmt.Errorf("applying scripts for team %q: %w", tmName, err)
 				}
 			}
 		}
 		if len(tmSoftwarePayloads) > 0 {
 			for tmName, software := range tmSoftwarePayloads {
 				if err := c.ApplyTeamSoftwareInstallers(tmName, software, opts); err != nil {
 					return nil, fmt.Errorf("applying software installers for team %q: %w", tmName, err)
 				}
 			}
@@ -1193,7 +1200,12 @@
 	}
 
 	// Apply org settings, scripts, enroll secrets, and controls
-	teamIDsByName, err := c.ApplyGroup(ctx, &group, baseDir, logf, fleet.ApplySpecOptions{DryRun: dryRun})
+	teamIDsByName, err := c.ApplyGroup(ctx, &group, baseDir, logf, fleet.ApplyClientSpecOptions{
+		ApplySpecOptions: fleet.ApplySpecOptions{
+			DryRun: dryRun,
+		},
+		ExpandEnvConfigProfiles: true,
+	})
 	if err != nil {
 		return nil, err
 	}