From 828ce31742b62be6ace46aa33b77cffe3a85cc1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geyslan=20Greg=C3=B3rio?= Date: Mon, 6 Jan 2025 19:16:54 -0300 Subject: [PATCH] chore(performance): update tracee dashboard Add panels measuring Tracee's CPU time and updates Host ones. --- .../provisioning/dashboards/tracee.json | 864 +++++++++++++----- 1 file changed, 621 insertions(+), 243 deletions(-) diff --git a/performance/dashboard/provisioning/dashboards/tracee.json b/performance/dashboard/provisioning/dashboards/tracee.json index 4a9344af0665..1a33c2840cd7 100644 --- a/performance/dashboard/provisioning/dashboards/tracee.json +++ b/performance/dashboard/provisioning/dashboards/tracee.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 2, + "id": 1, "links": [], "panels": [ { @@ -773,7 +773,7 @@ "type": "stat" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, @@ -781,229 +781,602 @@ "y": 19 }, "id": 40, - "panels": [], - "title": "Perf Event Buffer (METRICS=1)", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PDC1078F23EBDF0E5" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PDC1078F23EBDF0E5" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 38, - "w": 8, - "x": 0, - "y": 20 - }, - "id": 39, - "interval": "10s", - "options": { - "displayMode": "gradient", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + }, + "overrides": [] + }, + "gridPos": { + "h": 38, + "w": 8, + "x": 0, + "y": 1236 + }, + "id": 39, + "interval": "10s", + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.4.0-202090", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "tracee_ebpf_bpf_perf_event_submit_attempts", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{event_name}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Submit Attempts", + "type": "bargauge" }, - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" + { + "datasource": { + "type": "prometheus", + "uid": "PDC1078F23EBDF0E5" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 38, + "w": 8, + "x": 8, + "y": 1236 + }, + "id": 41, + "interval": "10s", + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.4.0-202090", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "tracee_ebpf_bpf_perf_event_submit_failures", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{event_name}}", + "range": true, + "refId": "A", + "useBackend": false + } ], - "fields": "", - "values": false + "title": "Submit Failures", + "type": "bargauge" }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" - }, - "pluginVersion": "11.4.0-202090", - "targets": [ { - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "tracee_ebpf_bpf_perf_event_submit_attempts", - "format": "time_series", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{event_name}}", - "range": true, - "refId": "A", - "useBackend": false + "datasource": { + "type": "prometheus", + "uid": "PDC1078F23EBDF0E5" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 38, + "w": 8, + "x": 16, + "y": 1236 + }, + "id": 42, + "interval": "10s", + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.4.0-202090", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "expr": "tracee_ebpf_bpf_perf_event_submit_failures/tracee_ebpf_bpf_perf_event_submit_attempts", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{event_name}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Failures Ratio", + "type": "bargauge" } ], - "title": "Submit Attempts", - "type": "bargauge" + "title": "Perf Event Buffer (METRICS=1)", + "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "PDC1078F23EBDF0E5" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - } - ] - } - }, - "overrides": [] - }, + "collapsed": true, "gridPos": { - "h": 38, - "w": 8, - "x": 8, + "h": 1, + "w": 24, + "x": 0, "y": 20 }, - "id": 41, - "interval": "10s", - "options": { - "displayMode": "gradient", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "id": 11, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PROMETHEUS" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": true, + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 0, + "y": 21 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-202090", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PROMETHEUS" + }, + "editorMode": "code", + "expr": "(rate(process_cpu_seconds_total{job=\"tracee\"}[$__rate_interval]) * 100)\n/\non()\ncount(node_cpu_seconds_total{mode=\"idle\"})", + "legendFormat": "Tracee", + "range": true, + "refId": "USER" + } + ], + "title": "CPU Time (Tracee)", + "transparent": true, + "type": "timeseries" }, - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" + { + "datasource": { + "type": "prometheus", + "uid": "PDC1078F23EBDF0E5" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 2 + }, + { + "color": "red", + "value": 2.5 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 21 + }, + "id": 48, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0-202090", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PDC1078F23EBDF0E5" + }, + "editorMode": "code", + "expr": "avg(rate(process_cpu_seconds_total{job=\"tracee\"}[$__rate_interval]) * 100)\n/\ncount(node_cpu_seconds_total{mode=\"idle\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } ], - "fields": "", - "values": false + "title": "CPU Time (Tracee) - avg", + "type": "stat" }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" - }, - "pluginVersion": "11.4.0-202090", - "targets": [ { - "disableTextWrap": false, - "editorMode": "builder", - "expr": "tracee_ebpf_bpf_perf_event_submit_failures", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "{{event_name}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Submit Failures", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PDC1078F23EBDF0E5" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" + "datasource": { + "type": "prometheus", + "uid": "PROMETHEUS" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - } - ] + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": true, + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 38, - "w": 8, - "x": 16, - "y": 20 - }, - "id": 42, - "interval": "10s", - "options": { - "displayMode": "gradient", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" + "gridPos": { + "h": 6, + "w": 18, + "x": 0, + "y": 27 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-202090", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PROMETHEUS" + }, + "editorMode": "code", + "expr": "(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval]))) * 100", + "legendFormat": "All but idle", + "range": true, + "refId": "USER" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PROMETHEUS" + }, + "editorMode": "code", + "expr": "(rate(process_cpu_seconds_total{job=\"tracee\"}[$__rate_interval]) * 100)\n/\non()\ncount(node_cpu_seconds_total{mode=\"idle\"})", + "hide": false, + "legendFormat": "Tracee", + "range": true, + "refId": "A" + } ], - "fields": "", - "values": false + "title": "CPU Time (All but idle/Tracee)", + "transparent": true, + "type": "timeseries" }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" - }, - "pluginVersion": "11.4.0-202090", - "targets": [ { - "disableTextWrap": false, - "editorMode": "code", - "expr": "tracee_ebpf_bpf_perf_event_submit_failures/tracee_ebpf_bpf_perf_event_submit_attempts", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "{{event_name}}", - "range": true, - "refId": "A", - "useBackend": false + "datasource": { + "type": "prometheus", + "uid": "PDC1078F23EBDF0E5" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "red", + "value": 2 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 27 + }, + "id": 49, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0-202090", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PDC1078F23EBDF0E5" + }, + "editorMode": "code", + "expr": "(\n (rate(process_cpu_seconds_total{job=\"tracee\"}[$__rate_interval]) * 100)\n /\n on()\n count(node_cpu_seconds_total{mode=\"idle\"})\n)\n/\n(\n (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval]))) * 100\n)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU Time (All but idle/Tracee) - ratio", + "type": "stat" } ], - "title": "Failures Ratio", - "type": "bargauge" + "title": "CPU", + "type": "row" }, { "collapsed": true, @@ -1011,7 +1384,7 @@ "h": 1, "w": 24, "x": 0, - "y": 58 + "y": 21 }, "id": 12, "panels": [ @@ -1078,7 +1451,7 @@ "h": 6, "w": 15, "x": 0, - "y": 153 + "y": 58 }, "id": 32, "options": { @@ -1143,7 +1516,7 @@ "h": 6, "w": 3, "x": 15, - "y": 153 + "y": 58 }, "id": 14, "options": { @@ -1206,7 +1579,7 @@ "h": 6, "w": 3, "x": 18, - "y": 153 + "y": 58 }, "id": 15, "options": { @@ -1271,7 +1644,7 @@ "h": 12, "w": 3, "x": 21, - "y": 153 + "y": 58 }, "id": 19, "options": { @@ -1399,7 +1772,7 @@ "h": 6, "w": 15, "x": 0, - "y": 159 + "y": 150 }, "id": 16, "options": { @@ -1464,7 +1837,7 @@ "h": 6, "w": 3, "x": 15, - "y": 159 + "y": 150 }, "id": 17, "options": { @@ -1527,7 +1900,7 @@ "h": 6, "w": 3, "x": 18, - "y": 159 + "y": 150 }, "id": 18, "options": { @@ -1629,7 +2002,7 @@ "h": 7, "w": 15, "x": 0, - "y": 165 + "y": 156 }, "id": 20, "options": { @@ -1720,7 +2093,7 @@ "h": 7, "w": 3, "x": 15, - "y": 165 + "y": 156 }, "id": 22, "options": { @@ -1820,7 +2193,7 @@ "h": 7, "w": 15, "x": 0, - "y": 172 + "y": 163 }, "id": 21, "options": { @@ -1885,7 +2258,7 @@ "h": 7, "w": 3, "x": 15, - "y": 172 + "y": 163 }, "id": 23, "options": { @@ -1931,7 +2304,7 @@ "h": 1, "w": 24, "x": 0, - "y": 59 + "y": 22 }, "id": 24, "panels": [ @@ -1999,7 +2372,7 @@ "h": 7, "w": 15, "x": 0, - "y": 453 + "y": 59 }, "id": 13, "options": { @@ -2064,7 +2437,7 @@ "h": 7, "w": 3, "x": 15, - "y": 453 + "y": 59 }, "id": 26, "options": { @@ -2164,7 +2537,7 @@ "h": 7, "w": 15, "x": 0, - "y": 460 + "y": 84 }, "id": 27, "options": { @@ -2229,7 +2602,7 @@ "h": 7, "w": 3, "x": 15, - "y": 460 + "y": 84 }, "id": 28, "options": { @@ -2275,16 +2648,16 @@ "h": 1, "w": 24, "x": 0, - "y": 60 + "y": 23 }, - "id": 11, + "id": 47, "panels": [ { "datasource": { "type": "prometheus", - "uid": "PROMETHEUS" + "uid": "PDC1078F23EBDF0E5" }, - "description": "Each line represents ONE SPECIFIC CPU percentage, and all of them are summing ALL USER CPU TIME in a stack.", + "description": "Each line represents ONE SPECIFIC CPU percentage.", "fieldConfig": { "defaults": { "color": { @@ -2300,7 +2673,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 8, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -2309,6 +2682,9 @@ }, "insertNulls": false, "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -2318,7 +2694,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "normal" + "mode": "none" }, "thresholdsStyle": { "mode": "area" @@ -2332,7 +2708,8 @@ "color": "transparent" } ] - } + }, + "unit": "percent" }, "overrides": [] }, @@ -2340,9 +2717,9 @@ "h": 6, "w": 24, "x": 0, - "y": 454 + "y": 48 }, - "id": 5, + "id": 8, "options": { "legend": { "calcs": [], @@ -2363,22 +2740,22 @@ "uid": "PROMETHEUS" }, "editorMode": "code", - "expr": "rate(node_cpu_seconds_total{mode=\"user\"}[$__rate_interval])*100", - "legendFormat": "USER", + "expr": "(1 - rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])) * 100", + "legendFormat": "Core {{cpu}}", "range": true, - "refId": "USER" + "refId": "Cores" } ], - "title": "USER PER CPU TIME STACKED", + "title": "CPU Time (All but idle - per core)", "transparent": true, "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PDC1078F23EBDF0E5" + "uid": "PROMETHEUS" }, - "description": "Each line represents ONE SPECIFIC CPU percentage, and all of them are summing ALL SYSTEM CPU TIME in a stack.", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -2426,7 +2803,8 @@ "color": "transparent" } ] - } + }, + "unit": "percent" }, "overrides": [] }, @@ -2434,9 +2812,9 @@ "h": 6, "w": 24, "x": 0, - "y": 460 + "y": 54 }, - "id": 8, + "id": 44, "options": { "legend": { "calcs": [], @@ -2457,13 +2835,13 @@ "uid": "PROMETHEUS" }, "editorMode": "code", - "expr": "rate(node_cpu_seconds_total{mode=\"system\"}[$__rate_interval])*100", - "legendFormat": "SYSTEM", + "expr": "(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval]))) * 100", + "legendFormat": "All but idle", "range": true, - "refId": "SYSTEM" + "refId": "USER" } ], - "title": "SYSTEM PERCPU TIME STACKED", + "title": "CPU Time (All but idle - avg)", "transparent": true, "type": "timeseries" }, @@ -2528,7 +2906,7 @@ "h": 6, "w": 24, "x": 0, - "y": 466 + "y": 60 }, "id": 9, "options": { @@ -2618,12 +2996,12 @@ "refId": "FS CACHE + BUFFERS" } ], - "title": "HOST MEMORY IN GB", + "title": "Memory (GB)", "transparent": true, "type": "timeseries" } ], - "title": "System", + "title": "Host", "type": "row" } ],