From 9aa9a4dea39569f00b2da599cb88b330894030fa Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Fri, 29 Nov 2024 15:14:38 +0000 Subject: [PATCH 1/3] Remove deprecated metrics. --- src/Metrics.ts | 6 ------ src/feeds/FeedReader.ts | 6 ------ 2 files changed, 12 deletions(-) diff --git a/src/Metrics.ts b/src/Metrics.ts index c639e6b32..e96557bd0 100644 --- a/src/Metrics.ts +++ b/src/Metrics.ts @@ -27,9 +27,6 @@ export class Metrics { public readonly feedsCount; public readonly feedFetchMs; public readonly feedsFailing; - public readonly feedsCountDeprecated; - public readonly feedsFetchMsDeprecated; - public readonly feedsFailingDeprecated; constructor(private registry: Registry = register) { @@ -55,9 +52,6 @@ export class Metrics { this.feedsCount = new Gauge({ name: "hookshot_feeds_count", help: "Number of RSS feeds that hookshot is subscribed to", labelNames: [], registers: [this.registry]}); this.feedFetchMs = new Gauge({ name: "hookshot_feeds_fetch_ms", help: "Time taken for hookshot to fetch all feeds", labelNames: [], registers: [this.registry]}); this.feedsFailing = new Gauge({ name: "hookshot_feeds_failing", help: "Number of RSS feeds that hookshot is failing to read", labelNames: ["reason"], registers: [this.registry]}); - this.feedsCountDeprecated = new Gauge({ name: "feed_count", help: "(Deprecated) Number of RSS feeds that hookshot is subscribed to", labelNames: [], registers: [this.registry]}); - this.feedsFetchMsDeprecated = new Gauge({ name: "feed_fetch_ms", help: "(Deprecated) Time taken for hookshot to fetch all feeds", labelNames: [], registers: [this.registry]}); - this.feedsFailingDeprecated = new Gauge({ name: "feed_failing", help: "(Deprecated) Number of RSS feeds that hookshot is failing to read", labelNames: ["reason"], registers: [this.registry]}); collectDefaultMetrics({ register: this.registry, diff --git a/src/feeds/FeedReader.ts b/src/feeds/FeedReader.ts index 699598ad6..2da8697cd 100644 --- a/src/feeds/FeedReader.ts +++ b/src/feeds/FeedReader.ts @@ -128,7 +128,6 @@ export class FeedReader { this.feedQueue.push(normalisedUrl); feeds.add(normalisedUrl); Metrics.feedsCount.inc(); - Metrics.feedsCountDeprecated.inc(); } }); connectionManager.on('connection-removed', removed => { @@ -156,7 +155,6 @@ export class FeedReader { this.feedsFailingHttp.delete(normalisedUrl); this.feedsFailingParsing.delete(normalisedUrl); Metrics.feedsCount.dec(); - Metrics.feedsCountDeprecated.dec(); }); log.debug('Loaded feed URLs:', [...feeds].join(', ')); @@ -187,7 +185,6 @@ export class FeedReader { } this.feedQueue.populate([...observedFeedUrls]); Metrics.feedsCount.set(observedFeedUrls.size); - Metrics.feedsCountDeprecated.set(observedFeedUrls.size); return observedFeedUrls; } @@ -306,8 +303,6 @@ export class FeedReader { // Update on each iteration Metrics.feedsFailing.set({ reason: "http" }, this.feedsFailingHttp.size ); Metrics.feedsFailing.set({ reason: "parsing" }, this.feedsFailingParsing.size); - Metrics.feedsFailingDeprecated.set({ reason: "http" }, this.feedsFailingHttp.size ); - Metrics.feedsFailingDeprecated.set({ reason: "parsing" }, this.feedsFailingParsing.size); log.debug(`Checking for updates in ${this.feedQueue.length()} RSS/Atom feeds (worker: ${workerId})`); @@ -322,7 +317,6 @@ export class FeedReader { } const elapsed = Date.now() - fetchingStarted; Metrics.feedFetchMs.set(elapsed); - Metrics.feedsFetchMsDeprecated.set(elapsed); sleepFor = Math.max(this.sleepingInterval - elapsed, 0); log.debug(`Feed fetching took ${elapsed / 1000}s, sleeping for ${sleepFor / 1000}s`); From c6d15adf7f2a9106f1dfd6ca963a4f3fcc644eae Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Fri, 29 Nov 2024 15:17:05 +0000 Subject: [PATCH 2/3] Add a new metric for recently failing fields. --- docs/metrics.md | 7 +------ src/Metrics.ts | 2 ++ src/feeds/FeedReader.ts | 20 ++++++++++++++------ 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/docs/metrics.md b/docs/metrics.md index 94cdfa679..69ef4fc98 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -36,6 +36,7 @@ Below is the generated list of Prometheus metrics for Hookshot. | hookshot_feeds_count | Number of RSS feeds that hookshot is subscribed to | | | hookshot_feeds_fetch_ms | Time taken for hookshot to fetch all feeds | | | hookshot_feeds_failing | Number of RSS feeds that hookshot is failing to read | reason | +| hookshot_feeds_failing_recent | Number of RSS feeds that hookshot is failing to read that have begun to fail recently | reason | ## matrix | Metric | Help | Labels | |--------|------|--------| @@ -43,12 +44,6 @@ Below is the generated list of Prometheus metrics for Hookshot. | matrix_api_calls_failed | Number of Matrix client API calls which failed | method | | matrix_appservice_events | Number of events sent over the AS API | | | matrix_appservice_decryption_failed | Number of events sent over the AS API that failed to decrypt | | -## feed -| Metric | Help | Labels | -|--------|------|--------| -| feed_count | (Deprecated) Number of RSS feeds that hookshot is subscribed to | | -| feed_fetch_ms | (Deprecated) Time taken for hookshot to fetch all feeds | | -| feed_failing | (Deprecated) Number of RSS feeds that hookshot is failing to read | reason | ## process | Metric | Help | Labels | |--------|------|--------| diff --git a/src/Metrics.ts b/src/Metrics.ts index e96557bd0..b44fa0c8f 100644 --- a/src/Metrics.ts +++ b/src/Metrics.ts @@ -27,6 +27,7 @@ export class Metrics { public readonly feedsCount; public readonly feedFetchMs; public readonly feedsFailing; + public readonly feedsFailingRecent; constructor(private registry: Registry = register) { @@ -52,6 +53,7 @@ export class Metrics { this.feedsCount = new Gauge({ name: "hookshot_feeds_count", help: "Number of RSS feeds that hookshot is subscribed to", labelNames: [], registers: [this.registry]}); this.feedFetchMs = new Gauge({ name: "hookshot_feeds_fetch_ms", help: "Time taken for hookshot to fetch all feeds", labelNames: [], registers: [this.registry]}); this.feedsFailing = new Gauge({ name: "hookshot_feeds_failing", help: "Number of RSS feeds that hookshot is failing to read", labelNames: ["reason"], registers: [this.registry]}); + this.feedsFailingRecent = new Gauge({ name: "hookshot_feeds_failing_recent", help: "Number of RSS feeds that hookshot is failing to read that have begun to fail recently", labelNames: ["reason"], registers: [this.registry]}); collectDefaultMetrics({ register: this.registry, diff --git a/src/feeds/FeedReader.ts b/src/feeds/FeedReader.ts index 2da8697cd..7ea97d68d 100644 --- a/src/feeds/FeedReader.ts +++ b/src/feeds/FeedReader.ts @@ -17,6 +17,12 @@ const BACKOFF_TIME_MAX_MS = 24 * 60 * 60 * 1000; const BACKOFF_POW = 1.05; const BACKOFF_TIME_MS = 5 * 1000; +/** + * If a feed fails this many times or more, we consider it effectively dead + * and while we might retry it, it won't be counted on the metrics. + */ +const FEEDS_FAILING_METRIC_MAX_DORMANT = 25; + export class FeedError extends Error { constructor( public url: string, @@ -88,9 +94,9 @@ export class FeedReader { // A set of last modified times for each url. private cacheTimes: Map = new Map(); - // Reason failures to url map. - private feedsFailingHttp = new Set(); - private feedsFailingParsing = new Set(); + // Reason failures to url map. // url -> fail count. + private feedsFailingHttp = new Map(); + private feedsFailingParsing = new Map(); static readonly seenEntriesEventType = "uk.half-shot.matrix-hookshot.feed.reader.seenEntries"; @@ -282,9 +288,9 @@ export class FeedReader { } catch (err: unknown) { // TODO: Proper Rust Type error. if ((err as Error).message.includes('Failed to fetch feed due to HTTP')) { - this.feedsFailingHttp.add(url); + this.feedsFailingHttp.set(url, (this.feedsFailingHttp.get(url) ?? 0) + 1); } else { - this.feedsFailingParsing.add(url); + this.feedsFailingParsing.set(url, (this.feedsFailingParsing.get(url) ?? 0) + 1); } const backoffDuration = this.feedQueue.backoff(url); const error = err instanceof Error ? err : new Error(`Unknown error ${err}`); @@ -301,8 +307,10 @@ export class FeedReader { public async pollFeeds(workerId: number): Promise { // Update on each iteration - Metrics.feedsFailing.set({ reason: "http" }, this.feedsFailingHttp.size ); + Metrics.feedsFailing.set({ reason: "http" }, this.feedsFailingHttp.size); Metrics.feedsFailing.set({ reason: "parsing" }, this.feedsFailingParsing.size); + Metrics.feedsFailingRecent.set({ reason: "http" }, [...this.feedsFailingHttp.values()].filter(v => v < FEEDS_FAILING_METRIC_MAX_DORMANT).length); + Metrics.feedsFailingRecent.set({ reason: "parsing" }, [...this.feedsFailingParsing.values()].filter(v => v < FEEDS_FAILING_METRIC_MAX_DORMANT).length); log.debug(`Checking for updates in ${this.feedQueue.length()} RSS/Atom feeds (worker: ${workerId})`); From 236da5d4a6e23984af2829e826d7f0b080ab57d6 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Fri, 29 Nov 2024 15:18:00 +0000 Subject: [PATCH 3/3] changelog --- changelog.d/997.feature | 1 + changelog.d/997.removal | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/997.feature create mode 100644 changelog.d/997.removal diff --git a/changelog.d/997.feature b/changelog.d/997.feature new file mode 100644 index 000000000..b09999fdc --- /dev/null +++ b/changelog.d/997.feature @@ -0,0 +1 @@ +Add a new metric `hookshot_feeds_failing_recent` which only includes feeds that have recently started failing. diff --git a/changelog.d/997.removal b/changelog.d/997.removal new file mode 100644 index 000000000..bef9355d8 --- /dev/null +++ b/changelog.d/997.removal @@ -0,0 +1 @@ +Remove legacy & deprecated field metrics.