27 configuration support several splunk servers and define their roles

.gitignore

@@ -21,5 +21,6 @@
 go.work
 
 # runtime files
-splunk_exporter.yml
-/splunk_exporter
+/splunk_exporter.yml
+/splunk_exporter
+deploy/default.yml

README.md

@@ -17,7 +17,7 @@ You will need a configuration file, follow [`splunk_exporter_example.yml`](./spl
 You need docker compose installed, a bash helper is provided to start the exporter and the whole test bench as a [docker compose environment](./deploy/README.md).
 
 ```shell
-cd /deploy
+cd deploy/
 bash run.sh
 ```
 
@@ -27,27 +27,44 @@ To stop it:
 docker compose down
 ```
 
-## 🙋 Contribute
+## 👷 Contribute
 
 After doing some changes, possible to re-deploy splunk_exporter with the following command
 ```shell
 docker compose up -d --build splunk_exporter
 ```
 
+## 🛠️ Configuration
+
+Splunk exporter needs to access management APIs
+See an example configuration file in [`splunk_exporter_example.yml`](./splunk_exporter_example.yml).
+
 ## 📏 metrics
 
 All metrics are **Gauge**.
 
 ### from API
 
-| Prefix                                                 | Description                                       |
-| ------------------------------------------------------ | ------------------------------------------------- |
-| `splunk_exporter_index_`                               | Numerical data coming from data/indexes endpoint. |
-| `splunk_exporter_indexer_throughput_bytes_per_seconds` | average data throughput in indexer                |
-| `splunk_exporter_metric_`                              | Export from metric indexes                        |
-| `splunk_exporter_health_splunkd`                       | Health status from local splunkd                  |
-| `splunk_exporter_health_deployment`                    | Health status from deployment                     |
+| Prefix                                                 | Labels                        | Description                                       |
+| ------------------------------------------------------ | ----------------------------- | ------------------------------------------------- |
+| `splunk_exporter_index_`                               | `index_name`                  | Numerical data coming from data/indexes endpoint. |
+| `splunk_exporter_indexer_throughput_bytes_per_seconds` | _None_                        | Average data throughput in indexer                |
+| `splunk_exporter_metric_`                              | Dimensions returned by Splunk | Export from metric indexes                        |
+| `splunk_exporter_health_splunkd`                       | `name`                        | Health status from local splunkd                  |
+| `splunk_exporter_health_deployment`                    | `instance_id`, `name`         | Health status from deployment                     |
+
+## 🧑‍🔬 Testing
+
+```shell
+go test -v ./...
+```
 
-## ⛔ Limitations
+## ✨ Roadmap
 
-Currently, only one splunk instance is supported
+| Item                  | Status            |
+| --------------------- | ----------------- |
+| Metrics indexes       | ✅ Done            |
+| Indexes metrics       | 🕰️ Ongoing         |
+| Savedsearches metrics | 🔜 Next            |
+| System metrics        | ❓ Not planned yet |
+| Ingestion pipeline    | ❓ Not planned yet |

config/config.go

@@ -19,6 +19,8 @@ type Metric struct {
 type Config struct {
 	URL      string   `yaml:"url"`
 	Token    string   `yaml:"token"`
+	Username string   `yaml:"username"`
+	Password string   `yaml:"password"`
 	Insecure bool     `yaml:"insecure"` // defaults to false
 	Metrics  []Metric `yaml:"metrics"`
 }
@@ -61,6 +63,7 @@ func (sc *SafeConfig) ReloadConfig(confFile string, logger log.Logger) (err erro
 		return fmt.Errorf("error reading config file: %s", err)
 	}
 	defer yamlReader.Close()
+
 	decoder := yaml.NewDecoder(yamlReader)
 	decoder.KnownFields(true)
 

config/config_test.go

@@ -6,10 +6,43 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-func TestLoadConfig(t *testing.T) {
+// TestLoadConfigToken
+// Given
+//
+//	A valid config file using a token
+//
+// When
+//
+//	reloading the config
+//
+// Then
+//
+//	Config reload happens without error
+func TestLoadConfigToken(t *testing.T) {
 	sc := NewSafeConfig(prometheus.NewRegistry())
 
-	err := sc.ReloadConfig("testdata/splunk_exporter-good.yml", nil)
+	err := sc.ReloadConfig("testdata/splunk_exporter-token-good.yml", nil)
+	if err != nil {
+		t.Errorf("Error loading config %v: %v", "splunk_exporter-good.yml", err)
+	}
+}
+
+// TestLoadConfigUser
+// Given
+//
+//	A valid config file using a username and password
+//
+// When
+//
+//	reloading the config
+//
+// Then
+//
+//	Config reload happens without error
+func TestLoadConfigUser(t *testing.T) {
+	sc := NewSafeConfig(prometheus.NewRegistry())
+
+	err := sc.ReloadConfig("testdata/splunk_exporter-user-good.yml", nil)
 	if err != nil {
 		t.Errorf("Error loading config %v: %v", "splunk_exporter-good.yml", err)
 	}

deploy/splunk_exporter.yml.src → config/testdata/splunk_exporter-user-good.yml

@@ -1,5 +1,6 @@
 url: https://splunk:8089
-token: '${SPLUNK_TOKEN}'
+username: toto
+password: tutu
 insecure: true
 metrics:
   - index: _metrics

deploy/docker-compose.yml

@@ -55,18 +55,55 @@ services:
     command: /splunk_exporter --config.file /splunk_exporter.yml --log.level=debug
     # add /etc/ssl/certs/ca-certificates.crt if needed
 
+  # splunk:
+  #   image: splunk/splunk:9.2
+  #   restart: unless-stopped
+  #   container_name: splunk
+  #   environment:
+  #     - SPLUNK_START_ARGS=--accept-license
+  #     - SPLUNK_PASSWORD=splunkadmin
+  #   expose:
+  #     - 8000
+  #     - 8089
+  #   ports:
+  #     - 8000:8000
+  #     - 8089:8089
+  #   networks:
+  #     - monitoring
+
   splunk:
-    image: splunk/splunk:9.2
-    restart: unless-stopped
     container_name: splunk
+    networks:
+      monitoring:
+        aliases:
+          - splunk
+    image: ${SPLUNK_IMAGE:-splunk/splunk:latest}
+    hostname: splunk
     environment:
       - SPLUNK_START_ARGS=--accept-license
+      - SPLUNK_STANDALONE_URL=splunk
+      - DEBUG=true
       - SPLUNK_PASSWORD=splunkadmin
-    expose:
+    ports:
       - 8000
       - 8089
-    ports:
-      - 8000:8000
-      - 8089:8089
+
+  dmc:
+    container_name: dmc
     networks:
-      - monitoring
+      monitoring:
+        aliases:
+          - dmc
+    image: ${SPLUNK_IMAGE:-splunk/splunk:latest}
+    command: start
+    hostname: dmc
+    environment:
+      - SPLUNK_START_ARGS=--accept-license
+      - SPLUNK_STANDALONE_URL=splunk
+      - SPLUNK_ROLE=splunk_monitor
+      - SPLUNK_LICENSE_URI
+      - SPLUNK_PASSWORD=splunkadmin
+      - DEBUG=true
+    ports:
+      - 8000
+      - 8089

deploy/run.sh

@@ -5,19 +5,12 @@ set -e
 # print commands
 set -v
 
-# initiate conf file
-touch ./splunk_exporter.yml
-
 # Start the stack
-docker compose up -d prometheus grafana splunk
-
-# Wait for splunk to be initialized
-until docker logs -n1 splunk 2>/dev/null | grep -q -m 1 '^Ansible playbook complete'; do sleep 0.2; done
-
-# Generate api key
-export SPLUNK_TOKEN=$(curl -k -u admin:splunkadmin -X POST https://localhost:8089/services/authorization/tokens?output_mode=json --data name=admin --data audience=splunk_exporter | jq -r '.entry[0].content.token')
-cat splunk_exporter.yml.src | envsubst > splunk_exporter.yml
+export SPLUNK_IMAGE="splunk/splunk:9.3"
+docker run --rm -it ${SPLUNK_IMAGE:-splunk/splunk:latest} create-defaults > default.yml
+docker compose up -d --remove-orphans 
 
-# start splunk_exporter
-docker compose up -d
+# Please wait for Splunk to be initialized, check this with the command:
+# docker compose logs dmc -f
+# If you need to reload config, you may use the following command:
 # curl -X POST http://localhost:9115/-/reload

deploy/splunk_exporter.yml

@@ -0,0 +1,9 @@
+url: https://dmc:8089
+username: admin
+password: splunkadmin
+insecure: true
+metrics:
+  - index: _metrics
+    name: spl.intr.disk_objects.Indexes.data.total_event_count
+  - index: _metrics
+    name: spl.intr.disk_objects.Indexes.data.total_bucket_count

exporter/exporter.go

@@ -42,46 +42,81 @@ type Exporter struct {
 }
 
 func (e *Exporter) UpdateConf(conf *config.Config) {
-	// FIXME need to re-validate params
-	e.splunk.Client.TLSInsecureSkipVerify = conf.Insecure
-	e.splunk.Client.URL = conf.URL
-	e.splunk.Client.Authenticator = authenticators.Token{
-		Token: conf.Token,
+
+	opts := SplunkOpts{
+		URI:      conf.URL,
+		Token:    conf.Token,
+		Username: conf.Username,
+		Password: conf.Password,
+		Insecure: conf.Insecure,
+	}
+
+	client, err := getSplunkClient(opts, e.logger)
+
+	if err != nil {
+		level.Error(e.logger).Log("msg", "Could not get Splunk client", "err", err)
 	}
+	e.splunk.Client = client
 }
 
 type SplunkOpts struct {
 	URI      string
 	Token    string
+	Username string
+	Password string
 	Insecure bool
 }
 
-// New creates a new exporter for Splunk metrics
-func New(opts SplunkOpts, logger log.Logger, metricsConf []config.Metric) (*Exporter, error) {
+// getSplunkClient generates a Splunk client from parameters
+// this function validates parameters and returns an error if they are not valid.
+func getSplunkClient(opts SplunkOpts, logger log.Logger) (*splunkclient.Client, error) {
 
-	uri := opts.URI
-	if !strings.Contains(uri, "://") {
-		uri = "https://" + uri
+	if !strings.Contains(opts.URI, "://") {
+		opts.URI = "https://" + opts.URI
 	}
-	u, err := url.Parse(uri)
+	u, err := url.Parse(opts.URI)
 	if err != nil {
 		return nil, fmt.Errorf("invalid splunk URL: %s", err)
 	}
 	if u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
-		return nil, fmt.Errorf("invalid splunk URL: %s", uri)
+		return nil, fmt.Errorf("invalid splunk URL: %s", opts.URI)
 	}
 
-	authenticator := authenticators.Token{
-		Token: opts.Token,
+	var authenticator splunkclient.Authenticator
+	if len(opts.Token) > 0 {
+		level.Info(logger).Log("msg", "Token is defined, we will use it for authentication.")
+		authenticator = authenticators.Token{
+			Token: opts.Token,
+		}
+	} else {
+		level.Info(logger).Log("msg", "Token is not defined, we will use password authentication.", "username", opts.Username)
+		if len(opts.Password) == 0 {
+			level.Warn(logger).Log("msg", "Password seems to be undefined.")
+		}
+		authenticator = &authenticators.Password{
+			Username: opts.Username,
+			Password: opts.Password,
+		}
 	}
 	client := splunkclient.Client{
 		URL:                   opts.URI,
 		Authenticator:         authenticator,
 		TLSInsecureSkipVerify: opts.Insecure,
 	}
+	return &client, nil
+}
+
+// New creates a new exporter for Splunk metrics
+func New(opts SplunkOpts, logger log.Logger, metricsConf []config.Metric) (*Exporter, error) {
+
+	client, err := getSplunkClient(opts, logger)
+
+	if err != nil {
+		level.Error(logger).Log("msg", "Could not get Splunk client", "err", err)
+	}
 
 	spk := splunklib.Splunk{
-		Client: &client,
+		Client: client,
 		Logger: logger,
 	}
 
@@ -126,13 +161,13 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
 // collectConfiguredMetrics gets metric measures from splunk indexes as specified by configuration
 func (e *Exporter) collectConfiguredMetrics(ch chan<- prometheus.Metric) bool {
 
-	return e.indexedMetrics.ProcessMeasures(ch)
+	return e.indexedMetrics.CollectMeasures(ch)
 
 }
 
 // collectHealthMetrics grabs metrics from Splunk Health endpoints
 func (e *Exporter) collectHealthMetrics(ch chan<- prometheus.Metric) bool {
-	return e.healthMetrics.ProcessMeasures(ch)
+	return e.healthMetrics.CollectMeasures(ch)
 }
 
 func (e *Exporter) collectIndexerMetrics(ch chan<- prometheus.Metric) bool {

exporter/exporter_test.go

@@ -0,0 +1,4 @@
+package exporter
+
+// We shall test properly metrics being collected (from testdata/deploymenthealth.json for example).
+// We’re waiting for https://github.com/prometheus/client_golang/issues/1639 to be resolved for this.