fix(agent): self monitoring inaccurate throttle (#160)

src/agent/src/notifiers/agentFailure.notifier.ts

@@ -37,20 +37,20 @@ export class AgentFailureNotifier extends Notifier<AgentFailureAlert> {
   }
 
   async sendNotification(alert: AgentFailureAlert) {
-    const { notifierConfig } = alert;
-    const notifierOptions = {
-      ...notifierConfig,
-      data: await this.#agentFailureAlertData(alert),
-      template: this.config.selfMonitoring!.template
-    };
-
     try {
+      const { notifierConfig } = alert;
+      const notifierOptions = {
+        ...notifierConfig,
+        data: await this.#agentFailureAlertData(alert),
+        template: this.config.selfMonitoring!.template
+      };
+
       await this.execute(notifierOptions);
 
       this.logger.info(`[SELF-MONITORING](notify: success|notifier: ${notifierConfig.notifier})`);
     }
     catch (error) {
-      this.logger.error(`[SELF-MONITORING](notify: error|notifier: ${notifierConfig.notifier}|message: ${error.message})`);
+      this.logger.error(`[SELF-MONITORING](notify: error|notifier: ${alert.notifierConfig.notifier}|message: ${error.message})`);
     }
   }
 

src/agent/src/rules.ts

@@ -249,7 +249,7 @@ export class Rule {
       }
 
       const alerts = getDB().prepare(
-        "SELECT alertId, key, value FROM alertLabels WHERE key IN (?)"
+        `SELECT alertId, key, value FROM alertLabels WHERE key IN (${labelScope.map(() => "?").join(",")})`
       ).all(labelScope) as { alertId: number; key: string; value: string }[];
 
       const alertIds = alerts.flatMap((alert) => {
@@ -264,7 +264,7 @@ export class Rule {
         return 0;
       }
 
-      return (getDB().prepare("SELECT COUNT(id) as count FROM alerts WHERE id IN (?) AND createdAt >= ?").get(
+      return (getDB().prepare(`SELECT COUNT(id) as count FROM alerts WHERE id IN (${alertIds.map(() => "?").join(",")}) AND createdAt >= ?`).get(
         alertIds,
         interval
       ) as { count: number }).count;

src/agent/src/utils/selfMonitoring.ts

@@ -12,7 +12,7 @@ import { AgentFailureAlert } from "../notifiers/agentFailure.notifier";
 export function getAgentFailureRules(alert: AgentFailureAlert): string {
   const ruleIds = new Set(alert.failures.map(({ ruleId }) => ruleId));
   const failures = getDB()
-    .prepare("SELECT name FROM rules WHERE id IN (?)")
+    .prepare(`SELECT name FROM rules WHERE id IN (${[...ruleIds].map(() => "?").join(",")})`)
     .all([...ruleIds]) as { name: string }[];
 
   return failures.map(({ name }) => name).join(", ");
@@ -31,10 +31,18 @@ function hasAgentFailureThrottle(throttle: SigynInitializedSelfMonitoring["throt
     .get(intervalDate) as { count: number });
   const agentFailuresAlertCount = agentFailuresAlert?.count ?? 0;
 
-  if (agentFailuresAlertCount <= activationThreshold) {
+  if (activationThreshold > 0 && agentFailuresAlertCount <= activationThreshold) {
     return false;
   }
 
+  if (count > 0 && agentFailuresAlertCount - activationThreshold > count) {
+    return false;
+  }
+
+  if (activationThreshold > 0 && agentFailuresAlertCount > activationThreshold) {
+    return true;
+  }
+
   return agentFailuresAlertCount === 1 ? false : agentFailuresAlertCount - activationThreshold <= count;
 }