fix(agent): self monitoring duplicated errors (#171)

MyUnisoft · Dec 10, 2023 · 83d4dca · 83d4dca
1 parent abe7861
commit 83d4dca
Show file tree

Hide file tree

Showing 5 changed files with 61 additions and 15 deletions.
diff --git a/src/agent/src/notifiers/agentFailure.notifier.ts b/src/agent/src/notifiers/agentFailure.notifier.ts
@@ -57,10 +57,11 @@ export class AgentFailureNotifier extends Notifier<AgentFailureAlert> {
 
   async #agentFailureAlertData(alert: AgentFailureAlert) {
     const { failures } = alert;
+    const errors = new Set(failures.map(({ message }) => message));
 
     return {
       agentFailure: {
-        errors: failures.reduce((pre, { message }) => (pre ? `${pre}, ${message}` : message), ""),
+        errors: [...errors].join(", "),
         rules: getAgentFailureRules(alert)
       },
       severity: kAgentFailureSeverity

diff --git a/src/agent/src/tasks/asyncTask.ts b/src/agent/src/tasks/asyncTask.ts
@@ -24,11 +24,12 @@ export function asyncTask(ruleConfig: SigynInitializedRule, options: AsyncTaskOp
       return;
     }
 
+    logger.info(`[${ruleConfig.name}](state: polling|start: ${start}|int: ${Date.now() - start}|query: ${ruleConfig.logql})`);
+
     try {
       const { logs } = await lokiApi.queryRangeStream<string>(ruleConfig.logql, {
         start
       });
-      logger.info(`[${ruleConfig.name}](state: polling|start: ${start}|end: ${Date.now()}|query: ${ruleConfig.logql})`);
 
       const createAlert = await rule.walkOnLogs(logs);
       if (createAlert) {

diff --git a/src/agent/test/FT/fixtures/no-self-monitoring-filters/sigyn.config.json b/src/agent/test/FT/fixtures/no-self-monitoring-filters/sigyn.config.json
@@ -24,6 +24,38 @@
           "title": "Alert"
         }
       }
+    },
+    {
+      "name": "State KO >= 80% no filters 2",
+      "logql": "{app=\"sigyn\"} |~ `state: (ok|ko)` | regexp `state: (?P<state>ok|ko)`",
+      "polling": "200ms",
+      "alert": {
+        "on": {
+          "label": "state",
+          "value": "ko",
+          "percentThreshold": 80,
+          "minimumLabelCount": 10
+        },
+        "template": {
+          "title": "Alert"
+        }
+      }
+    },
+    {
+      "name": "State KO >= 80% no filters 3",
+      "logql": "{app=\"sigyn\"} |~ `state: (ok|ko)` | regexp `state: (?P<state>ok|ko)`",
+      "polling": "200ms",
+      "alert": {
+        "on": {
+          "label": "state",
+          "value": "ko",
+          "percentThreshold": 80,
+          "minimumLabelCount": 10
+        },
+        "template": {
+          "title": "Alert"
+        }
+      }
     }
   ],
   "selfMonitoring": {

diff --git a/src/agent/test/FT/mocks/sigyn-test-notifier.js b/src/agent/test/FT/mocks/sigyn-test-notifier.js
@@ -1,8 +1,10 @@
 let calls = 0;
+let args = null;
 
 // This is the notifier execute function that will be called by the agent
-export function execute() {
+export function execute(...fnArgs) {
   calls++;
+  args = fnArgs;
 }
 
 export function resetCalls() {
@@ -12,3 +14,7 @@ export function resetCalls() {
 export function getCalls() {
   return calls;
 }
+
+export function getArgs() {
+  return args;
+}
diff --git a/src/agent/test/FT/selfMonitoring.spec.ts b/src/agent/test/FT/selfMonitoring.spec.ts
@@ -13,7 +13,7 @@ import { asyncTask } from "../../src/tasks/asyncTask";
 import { MockLogger } from "./helpers";
 import { Rule } from "../../src/rules";
 import { getDB, initDB } from "../../src/database";
-import { resetCalls, getCalls } from "./mocks/sigyn-test-notifier";
+import { resetCalls, getCalls, getArgs } from "./mocks/sigyn-test-notifier";
 
 // CONSTANTS
 const kFixturePath = path.join(__dirname, "/fixtures");
@@ -92,22 +92,28 @@ describe("Self-monitoring", () => {
   it("should send alert as rule matches ruleFilters", async() => {
     initDB(kLogger, { databaseFilename: ".temp/test-agent.sqlite3" });
     const config = await initConfig(kRuleMatchRuleFiltersConfigLocation);
-    const rule = new Rule(config.rules[0], { logger: kLogger });
-    rule.init();
 
-    const task = asyncTask(
-      config.rules[0], {
-        logger: kLogger,
-        lokiApi: kMockLokiApi as any,
-        rule
-      }
-    );
+    for (const ruleConfig of config.rules) {
+      const rule = new Rule(ruleConfig, { logger: kLogger });
+      rule.init();
+      const task = asyncTask(
+        config.rules[0], {
+          logger: kLogger,
+          lokiApi: kMockLokiApi as any,
+          rule
+        }
+      );
+
+      task.execute();
+    }
 
-    task.execute();
 
     await setTimeout(kTimeout);
 
-    assert.equal(getCalls(), 1);
+    assert.equal(getCalls(), 3);
+
+    const errors = getArgs()[0].data.agentFailure.errors;
+    assert.equal(errors, "Failed", "should not have duplicated errors");
   });
 
   it("should send alert as rule matches errorFilters", async() => {