From 498c569b7258784f8393edf845801fbede862234 Mon Sep 17 00:00:00 2001 From: Mickael Bourgois Date: Wed, 6 Nov 2024 12:42:55 +0100 Subject: [PATCH 1/2] CLDSRV-573: Catch prom cluster timeout Fix crashes of primary because of prom-client 5s timeout. Mostly to happen at startup when workers are not ready. Should also fix error write EPIPE in workers by preventing primary to crash. --- lib/utilities/monitoringHandler.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/utilities/monitoringHandler.js b/lib/utilities/monitoringHandler.js index 6a3cab87f4..d744acf01a 100644 --- a/lib/utilities/monitoringHandler.js +++ b/lib/utilities/monitoringHandler.js @@ -24,7 +24,15 @@ async function routeHandler(req, res, cb) { if (req.method !== 'GET') { return cb(errors.BadRequest, []); } - const metrics = await aggregatorRegistry.clusterMetrics(); + let metrics; + try { + // Catch timeout on IPC between worker and primary + // prom-client has a 5s hardcoded timeout + metrics = await aggregatorRegistry.clusterMetrics(); + } catch (err) { + return cb(err, { message: err.toString() }); + } + const contentLen = Buffer.byteLength(metrics, 'utf8'); res.writeHead(200, { 'Content-Length': contentLen, @@ -48,7 +56,7 @@ function monitoringHandler(clientIP, req, res, log) { function monitoringEndHandler(err, results) { writeResponse(res, err, results, error => { if (error) { - return log.end().warn('monitoring error', { err: error }); + return log.end().warn('monitoring error', { err: error, error: results }); } return log.end(); }); From cbaa5d66b08c80e51ec60a7441a7e552a19dc43f Mon Sep 17 00:00:00 2001 From: Mickael Bourgois Date: Wed, 6 Nov 2024 12:49:01 +0100 Subject: [PATCH 2/2] CLDSRV-573: Bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 555add73cd..440d1209f0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "s3", - "version": "7.70.55", + "version": "7.70.56", "description": "S3 connector", "main": "index.js", "engines": {