Skip to content

Commit

Permalink
Add exclusion rules for prom operator objects and static label helpers (
Browse files Browse the repository at this point in the history 
#1127)

* Add exclusion rules for prom operator objects and static label helpers

Signed-off-by: Pete Wall <[email protected]>

* Fix bad section name

Signed-off-by: Pete Wall <[email protected]>

* Add the integration label to the labels to keep list.

It'll be dropped later, and it's required for the integration matcher later

Signed-off-by: Pete Wall <[email protected]>

---------

Signed-off-by: Pete Wall <[email protected]>
  • Loading branch information
@petewall
petewall authored Jan 16, 2025
1 parent 430bcfa commit 68bc50a
Show file tree
Hide file tree
Showing 61 changed files with 381 additions and 140 deletions.
2 changes: 2 additions & 0 deletions charts/k8s-monitoring/charts/feature-pod-logs/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ Be sure perform actual integration testing in a live environment in the main [k8
| extraLogProcessingStages | string | `""` | Stage blocks to be added to the loki.process component for pod logs. ([docs](https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#blocks)) This value is templated so that you can refer to other values from this file. |
| labels | object | `{"app_kubernetes_io_name":"app.kubernetes.io/name"}` | Log labels to set with values copied from the Kubernetes Pod labels. Format: `<log_label>: <kubernetes_label>`. |
| labelsToKeep | list | `["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name"]` | The list of labels to keep on the logs, all other pipeline labels will be dropped. |
| staticLabels | object | `{}` | Log labels to set with static values. |
| staticLabelsFrom | object | `{}` | Log labels to set with static values, not quoted so it can reference config components. |

### Pod Discovery

Expand Down
39 changes: 27 additions & 12 deletions charts/k8s-monitoring/charts/feature-pod-logs/templates/_common_log_processing.alloy.tpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,28 +37,43 @@ loki.process "pod_logs" {
]
}

{{- if .Values.extraLogProcessingStages }}
{{ tpl .Values.extraLogProcessingStages $ | indent 2 }}
{{- end -}}

{{- /* the stage.structured_metadata block needs to be conditionalized because the support for enabling structured metadata can be disabled */ -}}
{{- /* through the loki limits_conifg on a per-tenant basis, even if there are no values defined or there are values defined but it is disabled */ -}}
{{- /* in Loki, the write will fail. */ -}}
{{- if gt (len (keys .Values.structuredMetadata)) 0 }}
{{- /* the stage.structured_metadata block needs to be conditionalized because the support for enabling structured metadata can be disabled */ -}}
{{- /* through the loki limits_conifg on a per-tenant basis, even if there are no values defined or there are values defined but it is disabled */ -}}
{{- /* in Loki, the write will fail. */ -}}
{{- if .Values.structuredMetadata }}
// set the structured metadata values
stage.structured_metadata {
values = {
{{- range $key, $value := .Values.structuredMetadata }}
{{- range $key, $value := .Values.structuredMetadata }}
{{ $key | quote }} = {{ if $value }}{{ $value | quote }}{{ else }}{{ $key | quote }}{{ end }},
{{- end }}
{{- end }}
}
}
{{- end }}
{{- end }}
{{- with .Values.labelsToKeep }}

// Only keep the labels that are defined in the `keepLabels` list.
stage.label_keep {
values = {{ .Values.labelsToKeep | toJson }}
values = {{ append . "integration" | toJson }}
}
{{- end }}

{{- if or .Values.staticLabels .Values.staticLabelsFrom }}

stage.static_labels {
values = {
{{- range $key, $value := .Values.staticLabels }}
{{ $key }} = {{ $value | quote }},
{{- end }}
{{- range $key, $value := .Values.staticLabelsFrom }}
{{ $key }} = {{ $value }},
{{- end }}
}
}
{{- end }}
{{- if .Values.extraLogProcessingStages }}
{{ tpl .Values.extraLogProcessingStages $ | indent 2 }}
{{- end }}

forward_to = argument.logs_destinations.value
}
Expand Down
164 changes: 160 additions & 4 deletions charts/k8s-monitoring/charts/feature-pod-logs/tests/default_test.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ tests:
// Only keep the labels that are defined in the `keepLabels` list.
stage.label_keep {
values = ["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name"]
values = ["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name","integration"]
}
forward_to = argument.logs_destinations.value
Expand Down Expand Up @@ -293,14 +293,13 @@ tests:
// Only keep the labels that are defined in the `keepLabels` list.
stage.label_keep {
values = ["k8s_container_name","k8s_namespace","k8s_pod_name","k8s_pod_label_app_kubernetes_io_name"]
values = ["k8s_container_name","k8s_namespace","k8s_pod_name","k8s_pod_label_app_kubernetes_io_name","integration"]
}
forward_to = argument.logs_destinations.value
}
}
- it: should allow setting structured metadata
set:
deployAsConfigMap: true
Expand Down Expand Up @@ -450,9 +449,166 @@ tests:
// Only keep the labels that are defined in the `keepLabels` list.
stage.label_keep {
values = ["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name"]
values = ["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name","integration"]
}
forward_to = argument.logs_destinations.value
}
}
- it: should allow setting static labels
set:
deployAsConfigMap: true
staticLabels:
region: central
staticLabelsFrom:
color: sys.env("COLOR")
asserts:
- isKind:
of: ConfigMap
- equal:
path: data["module.alloy"]
value: |-
declare "pod_logs" {
argument "logs_destinations" {
comment = "Must be a list of log destinations where collected logs should be forwarded to"
}
discovery.relabel "filtered_pods" {
targets = discovery.kubernetes.pods.targets
rule {
source_labels = ["__meta_kubernetes_namespace"]
action = "replace"
target_label = "namespace"
}
rule {
source_labels = ["__meta_kubernetes_pod_name"]
action = "replace"
target_label = "pod"
}
rule {
source_labels = ["__meta_kubernetes_pod_container_name"]
action = "replace"
target_label = "container"
}
rule {
source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"]
separator = "/"
action = "replace"
replacement = "$1"
target_label = "job"
}
// set the container runtime as a label
rule {
action = "replace"
source_labels = ["__meta_kubernetes_pod_container_id"]
regex = "^(\\S+):\\/\\/.+$"
replacement = "$1"
target_label = "tmp_container_runtime"
}
// set the job label from the k8s.grafana.com/logs.job annotation if it exists
rule {
source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_logs_job"]
regex = "(.+)"
target_label = "job"
}
// make all labels on the pod available to the pipeline as labels,
// they are omitted before write to loki via stage.label_keep unless explicitly set
rule {
action = "labelmap"
regex = "__meta_kubernetes_pod_label_(.+)"
}
// make all annotations on the pod available to the pipeline as labels,
// they are omitted before write to loki via stage.label_keep unless explicitly set
rule {
action = "labelmap"
regex = "__meta_kubernetes_pod_annotation_(.+)"
}
}
discovery.kubernetes "pods" {
role = "pod"
selectors {
role = "pod"
field = "spec.nodeName=" + sys.env("HOSTNAME")
}
}
discovery.relabel "filtered_pods_with_paths" {
targets = discovery.relabel.filtered_pods.output
rule {
source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"]
separator = "/"
action = "replace"
replacement = "/var/log/pods/*$1/*.log"
target_label = "__path__"
}
}
local.file_match "pod_logs" {
path_targets = discovery.relabel.filtered_pods_with_paths.output
}
loki.source.file "pod_logs" {
targets = local.file_match.pod_logs.targets
forward_to = [loki.process.pod_logs.receiver]
}
loki.process "pod_logs" {
stage.match {
selector = "{tmp_container_runtime=~\"containerd|cri-o\"}"
// the cri processing stage extracts the following k/v pairs: log, stream, time, flags
stage.cri {}
// Set the extract flags and stream values as labels
stage.labels {
values = {
flags = "",
stream = "",
}
}
}
stage.match {
selector = "{tmp_container_runtime=\"docker\"}"
// the docker processing stage extracts the following k/v pairs: log, stream, time
stage.docker {}
// Set the extract stream value as a label
stage.labels {
values = {
stream = "",
}
}
}
// Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster,
// namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary
// container runtime label as it is no longer needed.
stage.label_drop {
values = [
"filename",
"tmp_container_runtime",
]
}
// Only keep the labels that are defined in the `keepLabels` list.
stage.label_keep {
values = ["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name","integration"]
}
stage.static_labels {
values = {
region = "central",
color = sys.env("COLOR"),
}
}
forward_to = argument.logs_destinations.value
}
}
6 changes: 6 additions & 0 deletions charts/k8s-monitoring/charts/feature-pod-logs/values.schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,12 @@
"namespaces": {
"type": "array"
},
"staticLabels": {
"type": "object"
},
"staticLabelsFrom": {
"type": "object"
},
"structuredMetadata": {
"type": "object"
},
Expand Down
8 changes: 8 additions & 0 deletions charts/k8s-monitoring/charts/feature-pod-logs/values.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,14 @@ extraDiscoveryRules: ""
labels:
app_kubernetes_io_name: app.kubernetes.io/name

# -- Log labels to set with static values.
# @section -- Log Processing
staticLabels: {}

# -- Log labels to set with static values, not quoted so it can reference config components.
# @section -- Log Processing
staticLabelsFrom: {}

# -- Log labels to set with values copied from the Kubernetes Pod annotations.
# Format: `<log_label>: <kubernetes_annotation>`.
# @section -- Log Processing
Expand Down
3 changes: 3 additions & 0 deletions charts/k8s-monitoring/charts/feature-prometheus-operator-objects/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ Be sure perform actual integration testing in a live environment in the main [k8
| Key | Type | Default | Description |
|-----|------|---------|-------------|
| podMonitors.enabled | bool | `true` | Enable discovery of Prometheus Operator PodMonitor objects. |
| podMonitors.excludeNamespaces | list | `[]` | Which namespaces to not look for PodMonitor objects. |
| podMonitors.extraDiscoveryRules | string | `""` | Rule blocks to be added to the prometheus.operator.podmonitors component for PodMonitors. These relabeling rules are applied pre-scrape against the targets from service discovery. The relabelings defined in the PodMonitor object are applied first, then these relabelings are applied. Before the scrape, any remaining target labels that start with `__` (i.e. `__meta_kubernetes*`) are dropped. ([docs](https://grafana.com/docs/alloy/latest/reference/components/discovery/discovery.relabel/#rule-block)) |
| podMonitors.extraMetricProcessingRules | string | `""` | Rule blocks to be added to the prometheus.relabel component for PodMonitor objects. These relabeling rules are applied post-scrape against the metrics returned from the scraped target, no `__meta*` labels are present. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus/prometheus.relabel/#rule-block)) |
| podMonitors.maxCacheSize | string | `nil` | Sets the max_cache_size for cadvisor prometheus.relabel component. This should be at least 2x-5x your largest scrape target or samples appended rate. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus/prometheus.relabel/#arguments)) Overrides global.maxCacheSize |
Expand All @@ -87,6 +88,7 @@ Be sure perform actual integration testing in a live environment in the main [k8
| Key | Type | Default | Description |
|-----|------|---------|-------------|
| probes.enabled | bool | `true` | Enable discovery of Prometheus Operator Probe objects. |
| probes.excludeNamespaces | list | `[]` | Which namespaces to not look for Probe objects. |
| probes.extraDiscoveryRules | string | `""` | Rule blocks to be added to the prometheus.operator.probes component for Probes. These relabeling rules are applied pre-scrape against the targets from service discovery. The relabelings defined in the PodMonitor object are applied first, then these relabelings are applied. Before the scrape, any remaining target labels that start with `__` (i.e. `__meta_kubernetes*`) are dropped. ([docs](https://grafana.com/docs/alloy/latest/reference/components/discovery/discovery.relabel/#rule-block)) |
| probes.extraMetricProcessingRules | string | `""` | Rule blocks to be added to the prometheus.relabel component for Probe objects. These relabeling rules are applied post-scrape against the metrics returned from the scraped target, no `__meta*` labels are present. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus/prometheus.relabel/#rule-block)) |
| probes.maxCacheSize | string | `nil` | Sets the max_cache_size for cadvisor prometheus.relabel component. This should be at least 2x-5x your largest scrape target or samples appended rate. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus/prometheus.relabel/#arguments)) Overrides global.maxCacheSize |
Expand All @@ -101,6 +103,7 @@ Be sure perform actual integration testing in a live environment in the main [k8
| Key | Type | Default | Description |
|-----|------|---------|-------------|
| serviceMonitors.enabled | bool | `true` | Enable discovery of Prometheus Operator ServiceMonitor objects. |
| serviceMonitors.excludeNamespaces | list | `[]` | Which namespaces to not look for ServiceMonitor objects. |
| serviceMonitors.extraDiscoveryRules | string | `""` | Rule blocks to be added to the prometheus.operator.probes component for Probes. These relabeling rules are applied pre-scrape against the targets from service discovery. The relabelings defined in the PodMonitor object are applied first, then these relabelings are applied. Before the scrape, any remaining target labels that start with `__` (i.e. `__meta_kubernetes*`) are dropped. ([docs](https://grafana.com/docs/alloy/latest/reference/components/discovery/discovery.relabel/#rule-block)) |
| serviceMonitors.extraMetricProcessingRules | string | `""` | Rule blocks to be added to the prometheus.relabel component for ServiceMonitor objects. These relabeling rules are applied post-scrape against the metrics returned from the scraped target, no `__meta*` labels are present. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus/prometheus.relabel/#rule-block)) |
| serviceMonitors.maxCacheSize | string | `nil` | Sets the max_cache_size for cadvisor prometheus.relabel component. This should be at least 2x-5x your largest scrape target or samples appended rate. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus/prometheus.relabel/#arguments)) Overrides global.maxCacheSize |
Expand Down
7 changes: 7 additions & 0 deletions ...s-monitoring/charts/feature-prometheus-operator-objects/templates/_pod_monitors.alloy.tpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,13 @@ prometheus.operator.podmonitors "pod_monitors" {
default_scrape_interval = {{ .Values.podMonitors.scrapeInterval | default .Values.global.scrapeInterval | quote }}
}

{{- with .Values.podMonitors.excludeNamespaces }}
rule {
source_labels = ["__meta_kubernetes_namespace"]
regex = {{ . | join "|" | quote }}
action = "drop"
}
{{- end }}
{{- if .Values.podMonitors.extraDiscoveryRules }}
{{ .Values.podMonitors.extraDiscoveryRules | indent 2 }}
{{- end }}
Expand Down
7 changes: 7 additions & 0 deletions charts/k8s-monitoring/charts/feature-prometheus-operator-objects/templates/_probes.alloy.tpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,13 @@ prometheus.operator.probes "pod_monitors" {
default_scrape_interval = {{ .Values.probes.scrapeInterval | default .Values.global.scrapeInterval | quote }}
}

{{- with .Values.probes.excludeNamespaces }}
rule {
source_labels = ["__meta_kubernetes_namespace"]
regex = {{ . | join "|" | quote }}
action = "drop"
}
{{- end }}
{{- if .Values.probes.extraDiscoveryRules }}
{{ .Values.probes.extraDiscoveryRules | indent 2 }}
{{- end }}
Expand Down
7 changes: 7 additions & 0 deletions ...nitoring/charts/feature-prometheus-operator-objects/templates/_service_monitors.alloy.tpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,13 @@ prometheus.operator.servicemonitors "service_monitors" {
default_scrape_interval = {{ .Values.serviceMonitors.scrapeInterval | default .Values.global.scrapeInterval | quote }}
}

{{- with .Values.serviceMonitors.excludeNamespaces }}
rule {
source_labels = ["__meta_kubernetes_namespace"]
regex = {{ . | join "|" | quote }}
action = "drop"
}
{{- end }}
{{- if .Values.serviceMonitors.extraDiscoveryRules }}
{{ .Values.serviceMonitors.extraDiscoveryRules | indent 2 }}
{{- end }}
Expand Down
9 changes: 9 additions & 0 deletions charts/k8s-monitoring/charts/feature-prometheus-operator-objects/values.schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@
"enabled": {
"type": "boolean"
},
"excludeNamespaces": {
"type": "array"
},
"extraDiscoveryRules": {
"type": "string"
},
Expand Down Expand Up @@ -73,6 +76,9 @@
"enabled": {
"type": "boolean"
},
"excludeNamespaces": {
"type": "array"
},
"extraDiscoveryRules": {
"type": "string"
},
Expand Down Expand Up @@ -110,6 +116,9 @@
"enabled": {
"type": "boolean"
},
"excludeNamespaces": {
"type": "array"
},
"extraDiscoveryRules": {
"type": "string"
},
Expand Down
Loading

0 comments on commit 68bc50a

Please sign in to comment.