From 83d4dcadc2a8401b5277a7d369c7098d592539c7 Mon Sep 17 00:00:00 2001 From: PierreDemailly <39910767+PierreDemailly@users.noreply.github.com> Date: Sun, 10 Dec 2023 11:25:41 +0100 Subject: [PATCH] fix(agent): self monitoring duplicated errors (#171) --- .../src/notifiers/agentFailure.notifier.ts | 3 +- src/agent/src/tasks/asyncTask.ts | 3 +- .../sigyn.config.json | 32 +++++++++++++++++++ .../test/FT/mocks/sigyn-test-notifier.js | 8 ++++- src/agent/test/FT/selfMonitoring.spec.ts | 30 ++++++++++------- 5 files changed, 61 insertions(+), 15 deletions(-) diff --git a/src/agent/src/notifiers/agentFailure.notifier.ts b/src/agent/src/notifiers/agentFailure.notifier.ts index a11689b..d764fb3 100644 --- a/src/agent/src/notifiers/agentFailure.notifier.ts +++ b/src/agent/src/notifiers/agentFailure.notifier.ts @@ -57,10 +57,11 @@ export class AgentFailureNotifier extends Notifier { async #agentFailureAlertData(alert: AgentFailureAlert) { const { failures } = alert; + const errors = new Set(failures.map(({ message }) => message)); return { agentFailure: { - errors: failures.reduce((pre, { message }) => (pre ? `${pre}, ${message}` : message), ""), + errors: [...errors].join(", "), rules: getAgentFailureRules(alert) }, severity: kAgentFailureSeverity diff --git a/src/agent/src/tasks/asyncTask.ts b/src/agent/src/tasks/asyncTask.ts index 6d2248d..88055f3 100644 --- a/src/agent/src/tasks/asyncTask.ts +++ b/src/agent/src/tasks/asyncTask.ts @@ -24,11 +24,12 @@ export function asyncTask(ruleConfig: SigynInitializedRule, options: AsyncTaskOp return; } + logger.info(`[${ruleConfig.name}](state: polling|start: ${start}|int: ${Date.now() - start}|query: ${ruleConfig.logql})`); + try { const { logs } = await lokiApi.queryRangeStream(ruleConfig.logql, { start }); - logger.info(`[${ruleConfig.name}](state: polling|start: ${start}|end: ${Date.now()}|query: ${ruleConfig.logql})`); const createAlert = await rule.walkOnLogs(logs); if (createAlert) { diff --git a/src/agent/test/FT/fixtures/no-self-monitoring-filters/sigyn.config.json b/src/agent/test/FT/fixtures/no-self-monitoring-filters/sigyn.config.json index 8819afc..a46e429 100644 --- a/src/agent/test/FT/fixtures/no-self-monitoring-filters/sigyn.config.json +++ b/src/agent/test/FT/fixtures/no-self-monitoring-filters/sigyn.config.json @@ -24,6 +24,38 @@ "title": "Alert" } } + }, + { + "name": "State KO >= 80% no filters 2", + "logql": "{app=\"sigyn\"} |~ `state: (ok|ko)` | regexp `state: (?Pok|ko)`", + "polling": "200ms", + "alert": { + "on": { + "label": "state", + "value": "ko", + "percentThreshold": 80, + "minimumLabelCount": 10 + }, + "template": { + "title": "Alert" + } + } + }, + { + "name": "State KO >= 80% no filters 3", + "logql": "{app=\"sigyn\"} |~ `state: (ok|ko)` | regexp `state: (?Pok|ko)`", + "polling": "200ms", + "alert": { + "on": { + "label": "state", + "value": "ko", + "percentThreshold": 80, + "minimumLabelCount": 10 + }, + "template": { + "title": "Alert" + } + } } ], "selfMonitoring": { diff --git a/src/agent/test/FT/mocks/sigyn-test-notifier.js b/src/agent/test/FT/mocks/sigyn-test-notifier.js index d12d981..4d63b9a 100644 --- a/src/agent/test/FT/mocks/sigyn-test-notifier.js +++ b/src/agent/test/FT/mocks/sigyn-test-notifier.js @@ -1,8 +1,10 @@ let calls = 0; +let args = null; // This is the notifier execute function that will be called by the agent -export function execute() { +export function execute(...fnArgs) { calls++; + args = fnArgs; } export function resetCalls() { @@ -12,3 +14,7 @@ export function resetCalls() { export function getCalls() { return calls; } + +export function getArgs() { + return args; +} diff --git a/src/agent/test/FT/selfMonitoring.spec.ts b/src/agent/test/FT/selfMonitoring.spec.ts index 1a8ff54..0ea985e 100644 --- a/src/agent/test/FT/selfMonitoring.spec.ts +++ b/src/agent/test/FT/selfMonitoring.spec.ts @@ -13,7 +13,7 @@ import { asyncTask } from "../../src/tasks/asyncTask"; import { MockLogger } from "./helpers"; import { Rule } from "../../src/rules"; import { getDB, initDB } from "../../src/database"; -import { resetCalls, getCalls } from "./mocks/sigyn-test-notifier"; +import { resetCalls, getCalls, getArgs } from "./mocks/sigyn-test-notifier"; // CONSTANTS const kFixturePath = path.join(__dirname, "/fixtures"); @@ -92,22 +92,28 @@ describe("Self-monitoring", () => { it("should send alert as rule matches ruleFilters", async() => { initDB(kLogger, { databaseFilename: ".temp/test-agent.sqlite3" }); const config = await initConfig(kRuleMatchRuleFiltersConfigLocation); - const rule = new Rule(config.rules[0], { logger: kLogger }); - rule.init(); - const task = asyncTask( - config.rules[0], { - logger: kLogger, - lokiApi: kMockLokiApi as any, - rule - } - ); + for (const ruleConfig of config.rules) { + const rule = new Rule(ruleConfig, { logger: kLogger }); + rule.init(); + const task = asyncTask( + config.rules[0], { + logger: kLogger, + lokiApi: kMockLokiApi as any, + rule + } + ); + + task.execute(); + } - task.execute(); await setTimeout(kTimeout); - assert.equal(getCalls(), 1); + assert.equal(getCalls(), 3); + + const errors = getArgs()[0].data.agentFailure.errors; + assert.equal(errors, "Failed", "should not have duplicated errors"); }); it("should send alert as rule matches errorFilters", async() => {