From 956843a5ca87b51147f9150f669c313b1f9e43e6 Mon Sep 17 00:00:00 2001 From: Pete Wall Date: Fri, 17 Jan 2025 17:33:19 -0600 Subject: [PATCH] Add example and test for using terraform Signed-off-by: Pete Wall --- charts/k8s-monitoring/Makefile | 3 +- .../terraform/README.md | 225 +++++++++++++++++ .../terraform/grafana-k8s-monitoring.tf | 74 ++++++ .../terraform/provider.tf | 15 ++ .../deployment-alternatives/terraform/vars.tf | 44 ++++ .../terraform-deployment/.gitignore | 4 + .../terraform-deployment/.norender | 0 .../terraform-deployment/README.md | 232 ++++++++++++++++++ .../terraform-deployment/deploy.sh | 7 + .../deployments/grafana.yaml | 56 +++++ .../deployments/loki.yaml | 71 ++++++ .../deployments/prometheus.yaml | 69 ++++++ .../deployments/query-test.yaml | 62 +++++ .../grafana-k8s-monitoring.tf | 49 ++++ .../terraform-deployment/provider.tf | 15 ++ .../terraform-deployment/values.yaml | 26 ++ .../integration/terraform-deployment/vars.tf | 44 ++++ scripts/run-cluster-test.sh | 9 +- 18 files changed, 1002 insertions(+), 3 deletions(-) create mode 100644 charts/k8s-monitoring/docs/examples/deployment-alternatives/terraform/README.md create mode 100644 charts/k8s-monitoring/docs/examples/deployment-alternatives/terraform/grafana-k8s-monitoring.tf create mode 100644 charts/k8s-monitoring/docs/examples/deployment-alternatives/terraform/provider.tf create mode 100644 charts/k8s-monitoring/docs/examples/deployment-alternatives/terraform/vars.tf create mode 100644 charts/k8s-monitoring/tests/integration/terraform-deployment/.gitignore create mode 100644 charts/k8s-monitoring/tests/integration/terraform-deployment/.norender create mode 100644 charts/k8s-monitoring/tests/integration/terraform-deployment/README.md create mode 100755 charts/k8s-monitoring/tests/integration/terraform-deployment/deploy.sh create mode 100644 charts/k8s-monitoring/tests/integration/terraform-deployment/deployments/grafana.yaml create mode 100644 charts/k8s-monitoring/tests/integration/terraform-deployment/deployments/loki.yaml create mode 100644 charts/k8s-monitoring/tests/integration/terraform-deployment/deployments/prometheus.yaml create mode 100644 charts/k8s-monitoring/tests/integration/terraform-deployment/deployments/query-test.yaml create mode 100644 charts/k8s-monitoring/tests/integration/terraform-deployment/grafana-k8s-monitoring.tf create mode 100644 charts/k8s-monitoring/tests/integration/terraform-deployment/provider.tf create mode 100644 charts/k8s-monitoring/tests/integration/terraform-deployment/values.yaml create mode 100644 charts/k8s-monitoring/tests/integration/terraform-deployment/vars.tf diff --git a/charts/k8s-monitoring/Makefile b/charts/k8s-monitoring/Makefile index c27a154ed..7f57ffcf1 100644 --- a/charts/k8s-monitoring/Makefile +++ b/charts/k8s-monitoring/Makefile @@ -90,7 +90,8 @@ EXAMPLE_OUTPUT_FILES = $(EXAMPLE_VALUES_FILES:values.yaml=output.yaml) EXAMPLE_ALLOY_FILES = $(foreach file,$(EXAMPLE_VALUES_FILES),$(call alloy_configs, $(file))) EXAMPLE_README_FILES = $(EXAMPLE_VALUES_FILES:values.yaml=README.md) -INTEGRATION_TEST_VALUES_FILES = $(shell find tests/integration -name values.yaml) +NON_RENDERED_INTEGRATION_TEST_VALUES_FILES = $(shell find tests/integration -name .norender | sed 's/.norender/values.yaml/') +INTEGRATION_TEST_VALUES_FILES = $(filter-out $(NON_RENDERED_INTEGRATION_TEST_VALUES_FILES),$(shell find tests/integration -name values.yaml)) INTEGRATION_TEST_OUTPUT_FILES = $(INTEGRATION_TEST_VALUES_FILES:values.yaml=.rendered/output.yaml) PLATFORM_TEST_VALUES_FILES = $(shell find tests/platform -name values.yaml) diff --git a/charts/k8s-monitoring/docs/examples/deployment-alternatives/terraform/README.md b/charts/k8s-monitoring/docs/examples/deployment-alternatives/terraform/README.md new file mode 100644 index 000000000..d2e8b5e83 --- /dev/null +++ b/charts/k8s-monitoring/docs/examples/deployment-alternatives/terraform/README.md @@ -0,0 +1,225 @@ +# Terraform Deployment + +Some may want to use [Terraform](https://www.terraform.io/) to deploy the Kubernetes Monitoring Helm chart. This is +accomplished with the use of +the [Helm provider](https://registry.terraform.io/providers/hashicorp/helm/latest/docs), and +its [helm_release](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release#create_namespace) +resource. To use this, adapt the provider to connect to your own Kubernetes cluster and modify the `vars.tf` file to the +specific values for your deployment. If you want to provide additional values, follow the same pattern or look at +the [helm_release documentation](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) +for more options. + +## Files + +This example shows the various files used by Terraform to define and deploy the Kubernetes Monitoring Helm chart. + +### `provider.tf` + +This file shows the inclusion and instantiation of the Helm provider. + +```terraform +terraform { + required_providers { + helm = { + source = "hashicorp/helm" + version = "2.17.0" + } + } +} +provider "helm" { + kubernetes { + # Replace this with values that provide connection to your cluster + config_path = "~/.kube/config" + config_context = "my-cluster-context" + } +} +``` + +### `grafana-k8s-monitoring.tf` + +This file defines how to deploy the Helm chart as well as how to translate the Terraform vars into Helm chart values. +It also embeds a limited version of the Helm chart's values file as a string for chart configuration that does not +contain credentials. + +```terraform +resource "helm_release" "grafana-k8s-monitoring" { + name = "grafana-k8s-monitoring" + repository = "https://grafana.github.io/helm-charts" + chart = "k8s-monitoring" + namespace = var.namespace + create_namespace = true + atomic = true + values = [<<-EOT + destinations: + - name: metrics-destination + type: prometheus + auth: + type: basic + - name: logs-destination + type: loki + auth: + type: basic + + clusterMetrics: + enabled: true + clusterEvents: + enabled: true + podLogs: + enabled: true + + alloy-metrics: + enabled: true + alloy-singleton: + enabled: true + alloy-logs: + enabled: true + EOT + ] + + set { + name = "cluster.name" + value = var.cluster-name + } + + set { + name = "destinations[0].url" + value = var.prometheus-url + } + + set { + name = "destinations[0].auth.username" + value = var.prometheus-username + } + + set { + name = "destinations[0].auth.password" + value = var.prometheus-password + } + + set { + name = "destinations[1].url" + value = var.loki-url + } + + set { + name = "destinations[1].auth.username" + value = var.loki-username + } + + set { + name = "destinations[1].auth.password" + value = var.loki-password + } + + set { + name = "destinations[1].tenantId" + value = var.loki-tenantid + } +} +``` + +### `vars.tf` + +This file provides the variables and their values that'll be used during deployment. + +```terraform +variable "namespace" { + type = string + default = "monitoring" +} + +variable "cluster-name" { + type = string + default = "terraform-test" +} + +variable "prometheus-url" { + type = string + default = "https://prometheus.example.com/api/v1/write" +} + +variable "prometheus-username" { + type = string + default = "12345" +} + +variable "prometheus-password" { + type = string + default = "It's a secret to everyone" +} + +variable "loki-url" { + type = string + default = "https://loki.example.com/loki/api/v1/push" +} + +variable "loki-username" { + type = string + default = "12345" +} + +variable "loki-password" { + type = string + default = "It's a secret to everyone" +} + +variable "loki-tenantid" { + type = string + default = "1" +} +``` + +## Deploying + +Run `terraform init` and `terraform apply` to deploy this Helm chart to your cluster. + +```shell +$ terraform init +Initializing the backend... + +Initializing provider plugins... +- Finding hashicorp/helm versions matching "2.17.0"... +- Installing hashicorp/helm v2.17.0... +- Installed hashicorp/helm v2.17.0 (signed by HashiCorp) + +Terraform has created a lock file .terraform.lock.hcl to record the provider +selections it made above. Include this file in your version control repository +so that Terraform can guarantee to make the same selections by default when +you run "terraform init" in the future. + +Terraform has been successfully initialized! + +You may now begin working with Terraform. Try running "terraform plan" to see +any changes that are required for your infrastructure. All Terraform commands +should now work. + +If you ever set or change modules or backend configuration for Terraform, +rerun this command to reinitialize your working directory. If you forget, other +commands will detect it and remind you to do so if necessary. +$ terraform apply + +Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the following symbols: + + create + +Terraform will perform the following actions: + + # helm_release.grafana-k8s-monitoring will be created + + resource "helm_release" "grafana-k8s-monitoring" { + ... + } +Plan: 1 to add, 0 to change, 0 to destroy. + +Do you want to perform these actions? + Terraform will perform the actions described above. + Only 'yes' will be accepted to approve. + + Enter a value: yes + +helm_release.grafana-k8s-monitoring: Creating... +helm_release.grafana-k8s-monitoring: Still creating... [10s elapsed] +helm_release.grafana-k8s-monitoring: Still creating... [20s elapsed] +vhelm_release.grafana-k8s-monitoring: Creation complete after 27s [id=grafana-k8s-monitoring] + +Apply complete! Resources: 1 added, 0 changed, 0 destroyed. +$ +``` diff --git a/charts/k8s-monitoring/docs/examples/deployment-alternatives/terraform/grafana-k8s-monitoring.tf b/charts/k8s-monitoring/docs/examples/deployment-alternatives/terraform/grafana-k8s-monitoring.tf new file mode 100644 index 000000000..28cffa478 --- /dev/null +++ b/charts/k8s-monitoring/docs/examples/deployment-alternatives/terraform/grafana-k8s-monitoring.tf @@ -0,0 +1,74 @@ +resource "helm_release" "grafana-k8s-monitoring" { + name = "grafana-k8s-monitoring" + repository = "https://grafana.github.io/helm-charts" + chart = "k8s-monitoring" + namespace = var.namespace + create_namespace = true + atomic = true + values = [<<-EOT + destinations: + - name: metrics-destination + type: prometheus + auth: + type: basic + - name: logs-destination + type: loki + auth: + type: basic + + clusterMetrics: + enabled: true + clusterEvents: + enabled: true + podLogs: + enabled: true + + alloy-metrics: + enabled: true + alloy-singleton: + enabled: true + alloy-logs: + enabled: true + EOT + ] + + set { + name = "cluster.name" + value = var.cluster-name + } + + set { + name = "destinations[0].url" + value = var.prometheus-url + } + + set { + name = "destinations[0].auth.username" + value = var.prometheus-username + } + + set { + name = "destinations[0].auth.password" + value = var.prometheus-password + } + + set { + name = "destinations[1].url" + value = var.loki-url + } + + set { + name = "destinations[1].auth.username" + value = var.loki-username + } + + set { + name = "destinations[1].auth.password" + value = var.loki-password + } + + set { + name = "destinations[1].tenantId" + value = var.loki-tenantid + } + } \ No newline at end of file diff --git a/charts/k8s-monitoring/docs/examples/deployment-alternatives/terraform/provider.tf b/charts/k8s-monitoring/docs/examples/deployment-alternatives/terraform/provider.tf new file mode 100644 index 000000000..818ace9b4 --- /dev/null +++ b/charts/k8s-monitoring/docs/examples/deployment-alternatives/terraform/provider.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + helm = { + source = "hashicorp/helm" + version = "2.17.0" + } + } +} +provider "helm" { + kubernetes { + # Replace this with values that provide connection to your cluster + config_path = "~/.kube/config" + config_context = "my-cluster-context" + } +} diff --git a/charts/k8s-monitoring/docs/examples/deployment-alternatives/terraform/vars.tf b/charts/k8s-monitoring/docs/examples/deployment-alternatives/terraform/vars.tf new file mode 100644 index 000000000..21e56f8de --- /dev/null +++ b/charts/k8s-monitoring/docs/examples/deployment-alternatives/terraform/vars.tf @@ -0,0 +1,44 @@ +variable "namespace" { + type = string + default = "monitoring" +} + +variable "cluster-name" { + type = string + default = "terraform-test" +} + +variable "prometheus-url" { + type = string + default = "https://prometheus.example.com/api/v1/write" +} + +variable "prometheus-username" { + type = string + default = "12345" +} + +variable "prometheus-password" { + type = string + default = "It's a secret to everyone" +} + +variable "loki-url" { + type = string + default = "https://loki.example.com/loki/api/v1/push" +} + +variable "loki-username" { + type = string + default = "12345" +} + +variable "loki-password" { + type = string + default = "It's a secret to everyone" +} + +variable "loki-tenantid" { + type = string + default = "1" +} diff --git a/charts/k8s-monitoring/tests/integration/terraform-deployment/.gitignore b/charts/k8s-monitoring/tests/integration/terraform-deployment/.gitignore new file mode 100644 index 000000000..c05f3837d --- /dev/null +++ b/charts/k8s-monitoring/tests/integration/terraform-deployment/.gitignore @@ -0,0 +1,4 @@ +.terraform +.terraform.lock.hcl +kubeconfig.yaml +terraform.tfstate* diff --git a/charts/k8s-monitoring/tests/integration/terraform-deployment/.norender b/charts/k8s-monitoring/tests/integration/terraform-deployment/.norender new file mode 100644 index 000000000..e69de29bb diff --git a/charts/k8s-monitoring/tests/integration/terraform-deployment/README.md b/charts/k8s-monitoring/tests/integration/terraform-deployment/README.md new file mode 100644 index 000000000..ac679b8b2 --- /dev/null +++ b/charts/k8s-monitoring/tests/integration/terraform-deployment/README.md @@ -0,0 +1,232 @@ +# Terraform Deployment + +Some may want to use [Terraform](https://www.terraform.io/) to deploy the Kubernetes Monitoring Helm chart. This is +accomplished with the use of +the [Helm provider](https://registry.terraform.io/providers/hashicorp/helm/latest/docs), and +its [helm_release](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release#create_namespace) +resource. To use this, adapt the provider to connect to your own Kubernetes cluster and modify the `vars.tf` file to the +specific values for your deployment. If you want to provide additional values, follow the same pattern or look at +the [helm_release documentation](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) +for more options. + +## Files + +This example shows the various files used by Terraform to define and deploy the Kubernetes Monitoring Helm chart. + +### `provider.tf` + +This file shows the inclusion and instantiation of the Helm provider. + +```terraform +terraform { + required_providers { + helm = { + source = "hashicorp/helm" + version = "2.12.1" + } + } +} +provider "helm" { + kubernetes { + # Replace this with values that provide connection to your cluster + config_path = "~/.kube/config" + config_context = "my-cluster-context" + } +} +``` + +### `grafana-k8s-monitoring.tf` + +This file defines how to deploy the Helm chart as well as how to translate the Terraform vars into Helm chart values. + +```terraform +resource "helm_release" "grafana-k8s-monitoring" { + name = "grafana-k8s-monitoring" + repository = "https://grafana.github.io/helm-charts" + chart = "k8s-monitoring" + namespace = var.namespace + create_namespace = true + atomic = true + + set { + name = "cluster.name" + value = var.cluster-name + } + + set { + name = "externalServices.prometheus.host" + value = var.prometheus-url + } + + set { + name = "externalServices.prometheus.basicAuth.username" + value = var.prometheus-username + } + + set { + name = "externalServices.prometheus.basicAuth.password" + value = var.prometheus-password + } + + set { + name = "externalServices.loki.host" + value = var.loki-url + } + + set { + name = "externalServices.loki.basicAuth.username" + value = var.loki-username + } + + set { + name = "externalServices.loki.basicAuth.password" + value = var.loki-password + } + + set { + name = "externalServices.tempo.host" + value = var.tempo-url + } + + set { + name = "externalServices.tempo.basicAuth.username" + value = var.tempo-username + } + + set { + name = "externalServices.tempo.basicAuth.password" + value = var.tempo-password + } + + set { + name = "traces.enabled" + value = true + } + + set { + name = "opencost.opencost.exporter.defaultClusterId" + value = var.cluster-name + } + + set { + name = "opencost.opencost.prometheus.external.url" + value = "${var.prometheus-url}/api/prom" + } +} +``` + +### `vars.tf` + +This file provides the variables and their values that'll be send to the Helm chart during deployment. + +```terraform +variable "namespace" { + type = string + default = "monitoring" +} + +variable "cluster-name" { + type = string + default = "terraform-test" +} + +variable "prometheus-url" { + type = string + default = "https://prometheus.example.com" +} + +variable "prometheus-username" { + type = number + default = 12345 +} + +variable "prometheus-password" { + type = string + default = "It's a secret to everyone" +} + +variable "loki-url" { + type = string + default = "https://loki.example.com" +} + +variable "loki-username" { + type = number + default = 12345 +} + +variable "loki-password" { + type = string + default = "It's a secret to everyone" +} + +variable "tempo-url" { + type = string + default = "https://tempo.example.com" +} + +variable "tempo-username" { + type = number + default = 12345 +} + +variable "tempo-password" { + type = string + default = "It's a secret to everyone" +} +``` + +## Deploying + +Run `terraform init` and `terraform apply` to deploy this Helm chart to your cluster. + +```shell +$ terraform init +Initializing the backend... + +Initializing provider plugins... +- Finding hashicorp/helm versions matching "2.12.1"... +- Installing hashicorp/helm v2.12.1... +- Installed hashicorp/helm v2.12.1 (signed by HashiCorp) + +Terraform has created a lock file .terraform.lock.hcl to record the provider +selections it made above. Include this file in your version control repository +so that Terraform can guarantee to make the same selections by default when +you run "terraform init" in the future. + +Terraform has been successfully initialized! + +You may now begin working with Terraform. Try running "terraform plan" to see +any changes that are required for your infrastructure. All Terraform commands +should now work. + +If you ever set or change modules or backend configuration for Terraform, +rerun this command to reinitialize your working directory. If you forget, other +commands will detect it and remind you to do so if necessary. +$ terraform apply + +Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the following symbols: + + create + +Terraform will perform the following actions: + + # helm_release.grafana-k8s-monitoring will be created + + resource "helm_release" "grafana-k8s-monitoring" { + ... + } +Plan: 1 to add, 0 to change, 0 to destroy. + +Do you want to perform these actions? + Terraform will perform the actions described above. + Only 'yes' will be accepted to approve. + + Enter a value: yes + +helm_release.grafana-k8s-monitoring: Creating... +helm_release.grafana-k8s-monitoring: Still creating... [10s elapsed] +helm_release.grafana-k8s-monitoring: Still creating... [20s elapsed] +vhelm_release.grafana-k8s-monitoring: Creation complete after 27s [id=grafana-k8s-monitoring] + +Apply complete! Resources: 1 added, 0 changed, 0 destroyed. +$ +``` diff --git a/charts/k8s-monitoring/tests/integration/terraform-deployment/deploy.sh b/charts/k8s-monitoring/tests/integration/terraform-deployment/deploy.sh new file mode 100755 index 000000000..666659aba --- /dev/null +++ b/charts/k8s-monitoring/tests/integration/terraform-deployment/deploy.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +TEST_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +pushd "${TEST_DIR}" || exit 1 +kind get kubeconfig --name="$(yq '.cluster.name' values.yaml)" > kubeconfig.yaml +terraform init +terraform apply -auto-approve diff --git a/charts/k8s-monitoring/tests/integration/terraform-deployment/deployments/grafana.yaml b/charts/k8s-monitoring/tests/integration/terraform-deployment/deployments/grafana.yaml new file mode 100644 index 000000000..b7476aef8 --- /dev/null +++ b/charts/k8s-monitoring/tests/integration/terraform-deployment/deployments/grafana.yaml @@ -0,0 +1,56 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: grafana +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: grafana + namespace: grafana +spec: + interval: 1m + url: https://grafana.github.io/helm-charts +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: grafana + namespace: grafana +spec: + interval: 1m + chart: + spec: + chart: grafana + sourceRef: + kind: HelmRepository + name: grafana + namespace: grafana + interval: 1m + values: + datasources: + datasources.yaml: + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + url: http://prometheus-server.prometheus.svc:9090 + isDefault: true + basicAuth: true + basicAuthUser: promuser + jsonData: + tlsSkipVerify: true + secureJsonData: + basicAuthPassword: prometheuspassword + + - name: Loki + type: loki + url: http://loki-gateway.loki.svc:8080 + basicAuth: true + basicAuthUser: loki + jsonData: + httpHeaderName1: X-Scope-OrgID + secureJsonData: + basicAuthPassword: lokipassword + httpHeaderValue1: "1" diff --git a/charts/k8s-monitoring/tests/integration/terraform-deployment/deployments/loki.yaml b/charts/k8s-monitoring/tests/integration/terraform-deployment/deployments/loki.yaml new file mode 100644 index 000000000..d189f8f5c --- /dev/null +++ b/charts/k8s-monitoring/tests/integration/terraform-deployment/deployments/loki.yaml @@ -0,0 +1,71 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: loki +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: grafana + namespace: loki +spec: + interval: 1m + url: https://grafana.github.io/helm-charts +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: loki + namespace: loki +spec: + interval: 1m + chart: + spec: + chart: loki + sourceRef: + kind: HelmRepository + name: grafana + namespace: loki + interval: + values: + deploymentMode: SingleBinary + loki: + commonConfig: + replication_factor: 1 + storage: + type: 'filesystem' + schemaConfig: + configs: + - from: "2024-01-01" + store: tsdb + index: + prefix: loki_index_ + period: 24h + object_store: filesystem # we're storing on filesystem so there's no real persistence here. + schema: v13 + singleBinary: + replicas: 1 + read: + replicas: 0 + backend: + replicas: 0 + write: + replicas: 0 + + chunksCache: + enabled: false + resultsCache: + enabled: false + lokiCanary: + enabled: false + test: + enabled: false + + gateway: + basicAuth: + enabled: true + username: loki + password: lokipassword + service: + port: 8080 diff --git a/charts/k8s-monitoring/tests/integration/terraform-deployment/deployments/prometheus.yaml b/charts/k8s-monitoring/tests/integration/terraform-deployment/deployments/prometheus.yaml new file mode 100644 index 000000000..4b3ca4444 --- /dev/null +++ b/charts/k8s-monitoring/tests/integration/terraform-deployment/deployments/prometheus.yaml @@ -0,0 +1,69 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: prometheus +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: prometheus-community + namespace: prometheus +spec: + interval: 1m + url: https://prometheus-community.github.io/helm-charts +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: prometheus + namespace: prometheus +spec: + interval: 1m + chart: + spec: + chart: prometheus + version: "^25" + sourceRef: + kind: HelmRepository + name: prometheus-community + namespace: prometheus + interval: 1m + values: + server: + extraFlags: + - enable-feature=otlp-write-receiver + - enable-feature=remote-write-receiver + + persistentVolume: + enabled: false + + probeHeaders: + - name: "Authorization" + value: "Basic cHJvbXVzZXI6cHJvbWV0aGV1c3Bhc3N3b3Jk" + + service: + servicePort: 9090 + + serverFiles: + prometheus.yml: + scrape_configs: [] + web.yml: + basic_auth_users: + promuser: $2a$12$1UJsAG4QnhjjDzqcSVkZmeDxxjgIFOAmzfuVTybTuhhDnYgfuAbAq # "prometheuspassword" + + configmapReload: + prometheus: + enabled: false + + alertmanager: + enabled: false + + kube-state-metrics: + enabled: false + + prometheus-node-exporter: + enabled: false + + prometheus-pushgateway: + enabled: false diff --git a/charts/k8s-monitoring/tests/integration/terraform-deployment/deployments/query-test.yaml b/charts/k8s-monitoring/tests/integration/terraform-deployment/deployments/query-test.yaml new file mode 100644 index 000000000..d6eef0958 --- /dev/null +++ b/charts/k8s-monitoring/tests/integration/terraform-deployment/deployments/query-test.yaml @@ -0,0 +1,62 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: k8s-monitoring-test +spec: + interval: 1m + url: https://github.com/grafana/k8s-monitoring-helm + ref: + branch: main + ignore: | + /* + !/charts/k8s-monitoring-test +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: k8s-monitoring-test +spec: + interval: 1m + chart: + spec: + chart: charts/k8s-monitoring-test + sourceRef: + kind: GitRepository + name: k8s-monitoring-test + interval: 1m + dependsOn: + - name: loki + namespace: loki + - name: prometheus + namespace: prometheus + values: + tests: + - env: + CLUSTER: terraform-test + PROMETHEUS_URL: http://prometheus-server.prometheus.svc:9090/api/v1/query + PROMETHEUS_USER: promuser + PROMETHEUS_PASS: prometheuspassword + LOKI_URL: http://loki.loki.svc:3100/loki/api/v1/query + LOKI_TENANTID: 1 + LOKI_USER: loki + LOKI_PASS: lokipassword + queries: + # Cluster metrics + - query: kube_node_info{cluster="$CLUSTER"} + type: promql + + # Cluster events + - query: count_over_time({cluster="$CLUSTER", job="integrations/kubernetes/eventhandler"}[1h]) + type: logql + + # Pod logs + - query: count_over_time({cluster="$CLUSTER", job!="integrations/kubernetes/eventhandler"}[1h]) + type: logql + + # DPM check + - query: avg(count_over_time(scrape_samples_scraped{cluster="$CLUSTER"}[1m])) + type: promql + expect: + value: 1 + operator: == diff --git a/charts/k8s-monitoring/tests/integration/terraform-deployment/grafana-k8s-monitoring.tf b/charts/k8s-monitoring/tests/integration/terraform-deployment/grafana-k8s-monitoring.tf new file mode 100644 index 000000000..a3b2b8750 --- /dev/null +++ b/charts/k8s-monitoring/tests/integration/terraform-deployment/grafana-k8s-monitoring.tf @@ -0,0 +1,49 @@ +resource "helm_release" "grafana-k8s-monitoring" { + name = "grafana-k8s-monitoring" + chart = "../../../../k8s-monitoring" + namespace = var.namespace + create_namespace = true + atomic = true + + values = [file("values.yaml")] + + set { + name = "cluster.name" + value = var.cluster-name + } + + set { + name = "destinations[0].url" + value = var.prometheus-url + } + + set { + name = "destinations[0].auth.username" + value = var.prometheus-username + } + + set { + name = "destinations[0].auth.password" + value = var.prometheus-password + } + + set { + name = "destinations[1].url" + value = var.loki-url + } + + set { + name = "destinations[1].auth.username" + value = var.loki-username + } + + set { + name = "destinations[1].auth.password" + value = var.loki-password + } + + set { + name = "destinations[1].tenantId" + value = var.loki-tenantid + } +} \ No newline at end of file diff --git a/charts/k8s-monitoring/tests/integration/terraform-deployment/provider.tf b/charts/k8s-monitoring/tests/integration/terraform-deployment/provider.tf new file mode 100644 index 000000000..44a80c06a --- /dev/null +++ b/charts/k8s-monitoring/tests/integration/terraform-deployment/provider.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + helm = { + source = "hashicorp/helm" + version = "2.17.0" + } + } +} +provider "helm" { + kubernetes { + # Replace this with values that provide connection to your cluster + config_path = "kubeconfig.yaml" + config_context = "kind-terraform-test" + } +} diff --git a/charts/k8s-monitoring/tests/integration/terraform-deployment/values.yaml b/charts/k8s-monitoring/tests/integration/terraform-deployment/values.yaml new file mode 100644 index 000000000..97026cb5f --- /dev/null +++ b/charts/k8s-monitoring/tests/integration/terraform-deployment/values.yaml @@ -0,0 +1,26 @@ +cluster: + name: terraform-test + +destinations: + - name: localPrometheus + type: prometheus + auth: + type: basic + - name: localLoki + type: loki + auth: + type: basic + +clusterMetrics: + enabled: true +clusterEvents: + enabled: true +podLogs: + enabled: true + +alloy-metrics: + enabled: true +alloy-singleton: + enabled: true +alloy-logs: + enabled: true diff --git a/charts/k8s-monitoring/tests/integration/terraform-deployment/vars.tf b/charts/k8s-monitoring/tests/integration/terraform-deployment/vars.tf new file mode 100644 index 000000000..88e10f9ec --- /dev/null +++ b/charts/k8s-monitoring/tests/integration/terraform-deployment/vars.tf @@ -0,0 +1,44 @@ +variable "namespace" { + type = string + default = "monitoring" +} + +variable "cluster-name" { + type = string + default = "terraform-test" +} + +variable "prometheus-url" { + type = string + default = "http://prometheus-server.prometheus.svc:9090/api/v1/write" +} + +variable "prometheus-username" { + type = string + default = "promuser" +} + +variable "prometheus-password" { + type = string + default = "prometheuspassword" +} + +variable "loki-url" { + type = string + default = "http://loki.loki.svc:3100/loki/api/v1/push" +} + +variable "loki-username" { + type = string + default = "loki" +} + +variable "loki-password" { + type = string + default = "lokipassword" +} + +variable "loki-tenantid" { + type = string + default = "1" +} diff --git a/scripts/run-cluster-test.sh b/scripts/run-cluster-test.sh index e1089da60..479aca6e6 100755 --- a/scripts/run-cluster-test.sh +++ b/scripts/run-cluster-test.sh @@ -109,8 +109,13 @@ done # Deploy k8s-monitoring # OpenCost's defaultClusterId is set to the cluster name always, even if OpenCost is not enabled -echo helm upgrade --install k8smon ${PARENT_DIR}/charts/k8s-monitoring -f ${TEST_DIRECTORY}/values.yaml --set "cluster.name=${clusterName}" --set "clusterMetrics.opencost.opencost.exporter.defaultClusterId=${clusterName}" --wait -helm upgrade --install k8smon ${PARENT_DIR}/charts/k8s-monitoring -f ${TEST_DIRECTORY}/values.yaml --set "cluster.name=${clusterName}" --set "clusterMetrics.opencost.opencost.exporter.defaultClusterId=${clusterName}" --wait +if [ -f "${TEST_DIRECTORY}/deploy.sh" ]; then + echo "Running ${TEST_DIRECTORY}/deploy.sh" + ${TEST_DIRECTORY}/deploy.sh +else + echo helm upgrade --install k8smon ${PARENT_DIR}/charts/k8s-monitoring -f ${TEST_DIRECTORY}/values.yaml --set "cluster.name=${clusterName}" --set "clusterMetrics.opencost.opencost.exporter.defaultClusterId=${clusterName}" --wait + helm upgrade --install k8smon ${PARENT_DIR}/charts/k8s-monitoring -f ${TEST_DIRECTORY}/values.yaml --set "cluster.name=${clusterName}" --set "clusterMetrics.opencost.opencost.exporter.defaultClusterId=${clusterName}" --wait +fi # Run tests echo helm test k8s-monitoring-test --logs