perf(core): Introduce backend performance benchmarking

.github/workflows/benchmark.yml

@@ -0,0 +1,56 @@
+name: Benchmark
+
+on:
+  pull_request:
+    paths:
+      - 'packages/cli/**'
+      - 'packages/core/**'
+      - 'packages/workflow/**'
+  workflow_dispatch:
+
+jobs:
+  benchmark:
+    name: Benchmark
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    env:
+      DB_POSTGRESDB_PASSWORD: password
+    steps:
+      - uses: actions/[email protected]
+
+      - name: Start Postgres
+        uses: isbang/[email protected]
+        with:
+          compose-file: ./.github/docker-compose.yml
+          services: postgres
+
+      - run: corepack enable
+
+      - uses: actions/[email protected]
+        with:
+          node-version: 18.x
+          cache: pnpm
+
+      - run: pnpm install --frozen-lockfile
+
+      - name: Build
+        if: ${{ inputs.cacheKey == '' }}
+        run: pnpm build:backend
+
+      - name: Restore cached build artifacts
+        if: ${{ inputs.cacheKey != '' }}
+        uses: actions/cache/[email protected]
+        with:
+          path: ./packages/**/dist
+          key: ${{ inputs.cacheKey }}
+
+      - run: pnpm build:benchmark
+        working-directory: packages/cli
+
+      - name: Benchmark
+        uses: CodSpeedHQ/action@v2
+        with:
+          working-directory: packages/cli
+          run: |
+            pnpm benchmark:sqlite
+            pnpm benchmark:postgres

package.json

@@ -10,6 +10,7 @@
   "packageManager": "[email protected]",
   "scripts": {
     "preinstall": "node scripts/block-npm-install.js",
+    "benchmark": "pnpm --filter=n8n benchmark",
     "build": "turbo run build",
     "build:backend": "pnpm --filter=!@n8n/chat --filter=!n8n-design-system --filter=!n8n-editor-ui build",
     "build:frontend": "pnpm --filter=@n8n/chat --filter=n8n-design-system --filter=n8n-editor-ui build",
@@ -95,7 +96,8 @@
       "[email protected]": "patches/[email protected]",
       "@types/[email protected]": "patches/@[email protected]",
       "@types/[email protected]": "patches/@[email protected]",
-      "[email protected]": "patches/[email protected]"
+      "[email protected]": "patches/[email protected]",
+      "@codspeed/[email protected]": "patches/@[email protected]"
     }
   }
 }

packages/cli/package.json

@@ -20,9 +20,12 @@
     "bin": "n8n"
   },
   "scripts": {
+    "benchmark:sqlite": "NODE_ENV=benchmark N8N_LOG_LEVEL=silent DB_TYPE=sqlite node dist/benchmark/main.js",
+    "benchmark:postgres": "NODE_ENV=benchmark N8N_LOG_LEVEL=silent DB_TYPE=postgresdb node dist/benchmark/main.js",
     "clean": "rimraf dist .turbo",
     "typecheck": "tsc",
     "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && node scripts/build.mjs",
+    "build:benchmark": "tsc -p tsconfig.benchmark.json && tsc-alias -p tsconfig.benchmark.json && node scripts/build.mjs && node dist/benchmark/scripts/list-suites.js",
     "buildAndDev": "pnpm run build && pnpm run dev",
     "dev": "concurrently -k -n \"TypeScript,Node\" -c \"yellow.bold,cyan.bold\" \"npm run watch\" \"nodemon\"",
     "dev:worker": "concurrently -k -n \"TypeScript,Node\" -c \"yellow.bold,cyan.bold\" \"npm run watch\" \"nodemon worker\"",
@@ -60,6 +63,7 @@
     "!dist/**/e2e.*"
   ],
   "devDependencies": {
+    "@codspeed/tinybench-plugin": "^3.1.0",
     "@redocly/cli": "^1.6.0",
     "@types/aws4": "^1.5.1",
     "@types/basic-auth": "^1.1.3",
@@ -87,6 +91,7 @@
     "chokidar": "^3.5.2",
     "concurrently": "^8.2.0",
     "ioredis-mock": "^8.8.1",
+    "tinybench": "^2.6.0",
     "ts-essentials": "^7.0.3"
   },
   "dependencies": {

packages/cli/src/AbstractServer.ts

@@ -154,7 +154,7 @@ export abstract class AbstractServer {
 
 		this.server.on('error', (error: Error & { code: string }) => {
 			if (error.code === 'EADDRINUSE') {
-				console.log(
+				this.logger.info(
 					`n8n's port ${PORT} is already in use. Do you have another instance of n8n running already?`,
 				);
 				process.exit(1);
@@ -167,7 +167,7 @@ export abstract class AbstractServer {
 
 		await this.setupHealthCheck();
 
-		console.log(`n8n ready on ${ADDRESS}, port ${PORT}`);
+		this.logger.info(`n8n ready on ${ADDRESS}, port ${PORT}`);
 	}
 
 	async start(): Promise<void> {
@@ -236,11 +236,11 @@ export abstract class AbstractServer {
 		await this.configure();
 
 		if (!inTest) {
-			console.log(`Version: ${N8N_VERSION}`);
+			this.logger.info(`Version: ${N8N_VERSION}`);
 
 			const defaultLocale = config.getEnv('defaultLocale');
 			if (defaultLocale !== 'en') {
-				console.log(`Locale: ${defaultLocale}`);
+				this.logger.info(`Locale: ${defaultLocale}`);
 			}
 
 			await this.externalHooks.run('n8n.ready', [this, config]);

packages/cli/src/benchmark/benchmark.md

@@ -0,0 +1,65 @@
+# Benchmark
+
+This package contains benchmarks to measure the execution time of n8n backend operations in sqlite and Postgres.
+
+Benchmarks are organized into **suites** for the scenario to benchmark, **tasks** for operations in that scenario, and **hooks** for setup and teardown, implemented on top of [`tinybench`](https://github.com/tinylibs/tinybench). Execution in CI is delegated to [Codspeed](https://codspeed.io/) to keep measurements consistent and to monitor improvements and regressions.
+
+## Running benchmarks
+
+To run benchmarks:
+
+```sh
+pnpm build:benchmark
+pnpm benchmark:sqlite # or
+pnpm benchmark:postgres
+```
+
+Locally, the benchmarking run can be configured via [environment variables](https://docs.n8n.io/hosting/configuration/environment-variables/benchmarking). In CI, the configuration is set by Codspeed.
+
+## Creating benchmarks
+
+To create benchmarks:
+
+1. Create a file at `suites/**/{suiteId}-{suiteTitle}.ts`.
+2. Include a `suite()` call for the scenario to benchmark.
+3. Inside the suite, include one or more `task()` calls for operations in that scenario. `task()` must contain only the specific operation whose execution time to measure. Move any per-task setup and teardown to `beforeEachTask()` and `afterEachTask()` in the suite.
+4. Include workflows at `suites/workflows/{suiteId}-{ordinalNumber}`. During setup, workflows at this dir are saved in the temp DB and activated in memory.
+5. Run `pnpm build:benchmark` to add the suite and its tasks to the index below.
+
+## Index of benchmarking suites
+
+> **Note**: All workflows with default settings unless otherwise specified, e.g. `EXECUTIONS_MODE` is `regular` unless `queue` is specified.
+
+<!-- BENCHMARK_SUITES_LIST -->
+
+### 001 - Production workflow with authless webhook node
+
+- [using "Respond immediately" mode](./suites/workflows/001-1.json)
+- [using "When last node finishes" mode](./suites/workflows/001-2.json)
+- [using "Respond to Webhook node" mode](./suites/workflows/001-3.json)
+
+<!-- /BENCHMARK_SUITES_LIST -->
+
+## Reading benchmarks
+
+In a benchmarking run, a task is repeatedly executed for a duration and for a number of iterations - the run will continue until the number of iterations is reached, even if this exceeds the duration.
+
+```
+BENCHMARK suites/001-production-webhook-with-authless-webhook-node.suite.ts [sqlite]
+
+  • using "Respond immediately" mode
+    · Ran 27 iterations in 509.992 ms at a rate of 52.941 op/s
+    · p75 20.251 ms ··· p99 64.570 ms ··· p999 64.570 ms
+    · min 8.363 ms ···· max 64.570 ms ··· mean 18.888 ms
+    · MoE ±4.1% ··· std err 02.037 ms ··· std dev 10.586 ms
+```
+
+`p{n}` is the percentile, i.e. the percentage of data points in a distribution that are less than or equal to a value. For example, `p75` being 20.251 ms means that 75% of the 27 iterations for the task `using "Respond immediately" mode` took 20.251 ms or less. `p75` is the execution time that the majority of users experience, `p99` captures worst-case scenarios for all but 1% of users, and `p999` includes performance at extreme cases for the slowest 0.1% of users.
+
+`min` is the shortest execution time recorded across all iterations of the task, `max` is the longest, and `mean` is the average.
+
+`MoE` (margin of error) reflects how much the sample mean is expected to differ from the true population mean. For example, a margin of error of ±4.1% in the task `using "Respond immediately" mode` suggests that, if the benchmarking run were repeated multiple times, the sample mean would fall within 4.1% of the true population mean in 95% of those runs, assuming a standard confidence level. This range indicates the variability we might see due to the randomness of selecting a sample.
+
+`std err` (standard error) reflects how closely a sample mean is expected to approximate the true population mean. A smaller standard error indicates that the sample mean is likely to be a more accurate estimate of the population mean because the variation among sample means is less. For example, in the task `using "Respond immediately" mode`, the standard error is 2.037 ms, which suggests that the sample mean is expected to differ from the true population mean by 2.037 ms on average.
+
+`std dev` (standard deviation) is the amount of dispersion across samples. When low, it indicates that the samples tend to be close to the mean; when high, it indicates that the samples are spread out over a wider range. For example, in the task `using "Respond immediately" mode`, the standard deviation is 10.586 ms, which suggests that the execution times varied significantly across iterations.

packages/cli/src/benchmark/lib/agent.ts

@@ -0,0 +1,23 @@
+import axios from 'axios';
+import { BACKEND_BASE_URL, INSTANCE_ONWER } from './constants';
+import { ApplicationError } from 'n8n-workflow';
+
+export const agent = axios.create({ baseURL: BACKEND_BASE_URL });
+
+export async function authenticateAgent() {
+	const response = await agent.post('/rest/login', {
+		email: INSTANCE_ONWER.EMAIL,
+		password: INSTANCE_ONWER.PASSWORD,
+	});
+
+	const cookies = response.headers['set-cookie'];
+
+	if (!cookies || cookies.length !== 1) {
+		throw new ApplicationError('Expected cookie', { level: 'warning' });
+	}
+
+	const [cookie] = cookies;
+
+	agent.defaults.headers.Cookie = cookie;
+	agent.defaults.headers['x-n8n-api-key'] = INSTANCE_ONWER.API_KEY;
+}

packages/cli/src/benchmark/lib/api.ts

@@ -0,0 +1,98 @@
+import 'reflect-metadata';
+import path from 'node:path';
+import type Bench from 'tinybench';
+import { assert } from 'n8n-workflow';
+import glob from 'fast-glob';
+import callsites from 'callsites';
+import type { Suites, Task, Callback } from './types';
+import { DuplicateHookError } from './errors/duplicate-hook.error';
+import { DuplicateSuiteError } from './errors/duplicate-suite.error';
+
+const suites: Suites = {};
+
+export async function collectSuites() {
+	const files = await glob('**/*.suite.js', {
+		cwd: path.join('dist', 'benchmark'),
+		absolute: true,
+	});
+
+	for (const f of files) {
+		await import(f);
+	}
+
+	return suites;
+}
+
+export function registerSuites(bench: Bench) {
+	for (const { name: suiteName, hooks, tasks } of Object.values(suites)) {
+		/**
+		 * In tinybench, `beforeAll` and `afterAll` refer to all _iterations_ of
+		 * a single task, while `beforeEach` and `afterEach` refer to each _iteration_.
+		 *
+		 * In jest and vitest, `beforeAll` and `afterAll` refer to all _tests_,
+		 * while `beforeEach` and `afterEach` refer to each _test_.
+		 *
+		 * This API renames tinybench's hooks to prevent confusion from familiarity with jest.
+		 */
+		const options: Record<string, Callback> = {};
+
+		if (hooks.beforeEachTask) options.beforeAll = hooks.beforeEachTask;
+		if (hooks.afterEachTask) options.afterAll = hooks.afterEachTask;
+
+		for (const t of tasks) {
+			const taskName = process.env.CI === 'true' ? [suiteName, t.name].join('::') : t.name;
+
+			bench.add(taskName, t.operation, options);
+		}
+	}
+}
+
+function suiteKey() {
+	const key = callsites()
+		.map((site) => site.getFileName())
+		.filter((site): site is string => site !== null)
+		.find((site) => site.endsWith('.suite.js'));
+
+	assert(key !== undefined);
+
+	return key.replace(/^.*benchmark\//, '').replace(/\.js$/, '.ts');
+}
+
+export function suite(suiteName: string, suiteFn: () => void) {
+	const key = suiteKey();
+
+	if (suites[key]) throw new DuplicateSuiteError(key);
+
+	suites[key] = { name: suiteName, hooks: {}, tasks: [] };
+
+	suiteFn();
+}
+
+export function task(taskName: string, operation: Task['operation']) {
+	const key = suiteKey();
+
+	suites[key].tasks.push({
+		name: taskName,
+		operation,
+	});
+}
+
+export function beforeEachTask(fn: Callback) {
+	const key = suiteKey();
+
+	if (suites[key]?.hooks.beforeEachTask) {
+		throw new DuplicateHookError('beforeEachTask', key);
+	}
+
+	suites[key].hooks.beforeEachTask = fn;
+}
+
+export function afterEachTask(fn: Callback) {
+	const key = suiteKey();
+
+	if (suites[key]?.hooks.afterEachTask) {
+		throw new DuplicateHookError('afterEachTask', key);
+	}
+
+	suites[key].hooks.afterEachTask = fn;
+}

packages/cli/src/benchmark/lib/constants.ts

@@ -0,0 +1,9 @@
+export const BACKEND_BASE_URL = 'http://127.0.0.1:5678'; // localhost on GitHub Actions runners refuses connections
+
+export const INSTANCE_ONWER = {
+	EMAIL: '[email protected]',
+	PASSWORD: 'password',
+	FIRST_NAME: 'Instance',
+	LAST_NAME: 'Owner',
+	API_KEY: 'n8n_api_123',
+};

packages/cli/src/benchmark/lib/errors/duplicate-hook.error.ts

@@ -0,0 +1,10 @@
+import { ApplicationError } from 'n8n-workflow';
+
+export class DuplicateHookError extends ApplicationError {
+	constructor(hookName: 'beforeEachTask' | 'afterEachTask', key: string) {
+		super(
+			`Duplicate \`${hookName}\` hook found at \`${key}\`. Please define a single \`${hookName}\` hook for this file.`,
+			{ level: 'warning' },
+		);
+	}
+}

packages/cli/src/benchmark/lib/errors/duplicate-suite.error.ts

@@ -0,0 +1,9 @@
+import { ApplicationError } from 'n8n-workflow';
+
+export class DuplicateSuiteError extends ApplicationError {
+	constructor(key: string) {
+		super(`Duplicate suite found at \`${key}\`. Please define a single suite for this file.`, {
+			level: 'warning',
+		});
+	}
+}

packages/cli/src/benchmark/lib/errors/postgres-connection.error.ts

@@ -0,0 +1,12 @@
+import { ApplicationError } from 'n8n-workflow';
+import type { DataSourceOptions } from '@n8n/typeorm';
+
+export class PostgresConnectionError extends ApplicationError {
+	constructor(error: unknown, pgOptions: DataSourceOptions) {
+		super('Failed to connect to Postgres - check your Postgres configuration', {
+			level: 'warning',
+			cause: error,
+			extra: { postgresConfig: { pgOptions } },
+		});
+	}
+}