diff --git a/charts/kafka-connect/Chart.yaml b/charts/kafka-connect/Chart.yaml index 2e5c7cd..785af42 100644 --- a/charts/kafka-connect/Chart.yaml +++ b/charts/kafka-connect/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.5 +version: 0.1.6 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/kafka-connect/templates/connect-cluster.yaml b/charts/kafka-connect/templates/connect-cluster.yaml index a1fdd24..4973a30 100644 --- a/charts/kafka-connect/templates/connect-cluster.yaml +++ b/charts/kafka-connect/templates/connect-cluster.yaml @@ -28,6 +28,14 @@ spec: operator: In values: - amd64 +{{- if $.Values.monitoring.enabled }} + metricsConfig: + type: jmxPrometheusExporter + valueFrom: + configMapKeyRef: + name: {{ printf "%s-metrics" (include "kafka-connect.name" $) }} + key: metrics-config.yml +{{ end }} config: group.id: {{ $config.groupId | default $connect_name }} config.providers: {{ $config.providersClass | keys | join ", " }} diff --git a/charts/kafka-connect/templates/connect-metrics.yaml b/charts/kafka-connect/templates/connect-metrics.yaml new file mode 100644 index 0000000..be834ce --- /dev/null +++ b/charts/kafka-connect/templates/connect-metrics.yaml @@ -0,0 +1,185 @@ +{{- if .Values.monitoring.enabled }} +kind: ConfigMap +apiVersion: v1 +metadata: + name: {{ printf "%s-metrics" (include "kafka-connect.name" .) }} + labels: + app: strimzi +data: + metrics-config.yml: | + # Inspired by kafka-connect rules + # https://github.com/prometheus/jmx_exporter/blob/master/example_configs/kafka-connect.yml + # See https://github.com/prometheus/jmx_exporter for more info about JMX Prometheus Exporter metrics + lowercaseOutputName: true + lowercaseOutputLabelNames: true + rules: + #kafka.connect:type=app-info,client-id="{clientid}" + #kafka.consumer:type=app-info,client-id="{clientid}" + #kafka.producer:type=app-info,client-id="{clientid}" + - pattern: 'kafka.(.+)<>start-time-ms' + name: kafka_$1_start_time_seconds + labels: + clientId: "$2" + help: "Kafka $1 JMX metric start time seconds" + type: GAUGE + valueFactor: 0.001 + - pattern: 'kafka.(.+)<>(commit-id|version): (.+)' + name: kafka_$1_$3_info + value: 1 + labels: + clientId: "$2" + $3: "$4" + help: "Kafka $1 JMX metric info version and commit-id" + type: UNTYPED + + #kafka.consumer:type=consumer-fetch-manager-metrics,client-id="{clientid}",topic="{topic}"", partition="{partition}" + - pattern: kafka.consumer<>(.+-total) + name: kafka_consumer_fetch_manager_$4 + labels: + clientId: "$1" + topic: "$2" + partition: "$3" + help: "Kafka Consumer JMX metric type consumer-fetch-manager-metrics" + type: COUNTER + - pattern: kafka.consumer<>(compression-rate|.+-avg|.+-replica|.+-lag|.+-lead) + name: kafka_consumer_fetch_manager_$4 + labels: + clientId: "$1" + topic: "$2" + partition: "$3" + help: "Kafka Consumer JMX metric type consumer-fetch-manager-metrics" + type: GAUGE + + #kafka.producer:type=producer-topic-metrics,client-id="{clientid}",topic="{topic}" + - pattern: kafka.producer<>(.+-total) + name: kafka_producer_topic_$3 + labels: + clientId: "$1" + topic: "$2" + help: "Kafka Producer JMX metric type producer-topic-metrics" + type: COUNTER + - pattern: kafka.producer<>(compression-rate|.+-avg|.+rate) + name: kafka_producer_topic_$3 + labels: + clientId: "$1" + topic: "$2" + help: "Kafka Producer JMX metric type producer-topic-metrics" + type: GAUGE + + #kafka.connect:type=connect-node-metrics,client-id="{clientid}",node-id="{nodeid}" + #kafka.consumer:type=consumer-node-metrics,client-id=consumer-1,node-id="{nodeid}" + - pattern: kafka.(.+)<>(.+-total) + name: kafka_$2_$5 + labels: + clientId: "$3" + nodeId: "$4" + help: "Kafka $1 JMX metric type $2" + type: COUNTER + - pattern: kafka.(.+)<>(.+-avg|.+-rate) + name: kafka_$2_$5 + labels: + clientId: "$3" + nodeId: "$4" + help: "Kafka $1 JMX metric type $2" + type: GAUGE + + #kafka.connect:type=kafka-metrics-count,client-id="{clientid}" + #kafka.consumer:type=consumer-fetch-manager-metrics,client-id="{clientid}" + #kafka.consumer:type=consumer-coordinator-metrics,client-id="{clientid}" + #kafka.consumer:type=consumer-metrics,client-id="{clientid}" + - pattern: kafka.(.+)<>(.+-total) + name: kafka_$2_$4 + labels: + clientId: "$3" + help: "Kafka $1 JMX metric type $2" + type: COUNTER + - pattern: kafka.(.+)<>(.+-avg|.+-bytes|.+-count|.+-ratio|.+-age|.+-flight|.+-threads|.+-connectors|.+-tasks|.+-ago) + name: kafka_$2_$4 + labels: + clientId: "$3" + help: "Kafka $1 JMX metric type $2" + type: GAUGE + + #kafka.connect:type=connector-metrics,connector="{connector}" + - pattern: 'kafka.connect<>(connector-class|connector-type|connector-version|status): (.+)' + name: kafka_connect_connector_$2 + value: 1 + labels: + connector: "$1" + $2: "$3" + help: "Kafka Connect $2 JMX metric type connector" + type: GAUGE + + #kafka.connect:type=connector-task-metrics,connector="{connector}",task="{task}<> status" + - pattern: 'kafka.connect<>status: ([a-z-]+)' + name: kafka_connect_connector_task_status + value: 1 + labels: + connector: "$1" + task: "$2" + status: "$3" + help: "Kafka Connect JMX Connector task status" + type: GAUGE + + #kafka.connect:type=task-error-metrics,connector="{connector}",task="{task}" + #kafka.connect:type=source-task-metrics,connector="{connector}",task="{task}" + #kafka.connect:type=sink-task-metrics,connector="{connector}",task="{task}" + #kafka.connect:type=connector-task-metrics,connector="{connector}",task="{task}" + - pattern: kafka.connect<>(.+-total) + name: kafka_connect_$1_$4 + labels: + connector: "$2" + task: "$3" + help: "Kafka Connect JMX metric type $1" + type: COUNTER + - pattern: kafka.connect<>(.+-count|.+-ms|.+-ratio|.+-seq-no|.+-rate|.+-max|.+-avg|.+-failures|.+-requests|.+-timestamp|.+-logged|.+-errors|.+-retries|.+-skipped) + name: kafka_connect_$1_$4 + labels: + connector: "$2" + task: "$3" + help: "Kafka Connect JMX metric type $1" + type: GAUGE + + #kafka.connect:type=connect-worker-metrics,connector="{connector}" + - pattern: kafka.connect<>([a-z-]+) + name: kafka_connect_worker_$2 + labels: + connector: "$1" + help: "Kafka Connect JMX metric $1" + type: GAUGE + + #kafka.connect:type=connect-worker-metrics + - pattern: kafka.connect<>([a-z-]+-total) + name: kafka_connect_worker_$1 + help: "Kafka Connect JMX metric worker" + type: COUNTER + - pattern: kafka.connect<>([a-z-]+) + name: kafka_connect_worker_$1 + help: "Kafka Connect JMX metric worker" + type: GAUGE + + #kafka.connect:type=connect-worker-rebalance-metrics,leader-name|connect-protocol + - pattern: 'kafka.connect<>(leader-name|connect-protocol): (.+)' + name: kafka_connect_worker_rebalance_$1 + value: 1 + labels: + $1: "$2" + help: "Kafka Connect $2 JMX metric type worker rebalance" + type: UNTYPED + + #kafka.connect:type=connect-worker-rebalance-metrics + - pattern: kafka.connect<>([a-z-]+-total) + name: kafka_connect_worker_rebalance_$1 + help: "Kafka Connect JMX metric rebalance information" + type: COUNTER + - pattern: kafka.connect<>([a-z-]+) + name: kafka_connect_worker_rebalance_$1 + help: "Kafka Connect JMX metric rebalance information" + type: GAUGE + + #kafka.connect:type=connect-coordinator-metrics + - pattern: kafka.connect<>(assigned-connectors|assigned-tasks) + name: kafka_connect_coordinator_$1 + help: "Kafka Connect JMX metric assignment information" + type: GAUGE +{{- end }} \ No newline at end of file diff --git a/charts/kafka-connect/templates/podMonitor.yaml b/charts/kafka-connect/templates/podMonitor.yaml new file mode 100644 index 0000000..7551c2b --- /dev/null +++ b/charts/kafka-connect/templates/podMonitor.yaml @@ -0,0 +1,51 @@ +{{- if .Values.monitoring.enabled }} +{{- if .Values.monitoring.podMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: {{ printf "%s-metrics" (include "kafka-connect.name" .) }} + labels: + app: strimzi +spec: + selector: + matchExpressions: + - key: "strimzi.io/kind" + operator: In + values: ["Kafka", "KafkaConnect", "KafkaMirrorMaker", "KafkaMirrorMaker2"] + namespaceSelector: + matchNames: + - {{ $.Release.Namespace }} + podMetricsEndpoints: + - path: /metrics + port: tcp-prometheus + relabelings: + - separator: ; + regex: __meta_kubernetes_pod_label_(strimzi_io_.+) + replacement: $1 + action: labelmap + - sourceLabels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + targetLabel: namespace + replacement: $1 + action: replace + - sourceLabels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + targetLabel: kubernetes_pod_name + replacement: $1 + action: replace + - sourceLabels: [__meta_kubernetes_pod_node_name] + separator: ; + regex: (.*) + targetLabel: node_name + replacement: $1 + action: replace + - sourceLabels: [__meta_kubernetes_pod_host_ip] + separator: ; + regex: (.*) + targetLabel: node_ip + replacement: $1 + action: replace +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kafka-connect/values.yaml b/charts/kafka-connect/values.yaml index d80c64e..ddaa9c3 100644 --- a/charts/kafka-connect/values.yaml +++ b/charts/kafka-connect/values.yaml @@ -30,6 +30,11 @@ serviceAccount: # If not set and create is true, a name is generated using the fullname template name: "" +monitoring: + enabled: false + podMonitor: + enabled: false + kafka: bootstrapServers: "broker1:9098,broker2:9098" saslMechanism: "AWS_MSK_IAM"