Skip to content

Commit d6d7efb

Browse files
committed
Cleanup
1 parent 57724d5 commit d6d7efb

15 files changed

+169
-184
lines changed

packages/cli/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
"clean": "rimraf dist .turbo",
2525
"typecheck": "tsc",
2626
"build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && node scripts/build.mjs",
27-
"build:benchmark": "tsc -p tsconfig.benchmark.json && tsc-alias -p tsconfig.benchmark.json && node scripts/build.mjs && node dist/benchmark/scripts/document-suites.js",
27+
"build:benchmark": "tsc -p tsconfig.benchmark.json && tsc-alias -p tsconfig.benchmark.json && node scripts/build.mjs && node dist/benchmark/scripts/list-suites.js",
2828
"buildAndDev": "pnpm run build && pnpm run dev",
2929
"dev": "concurrently -k -n \"TypeScript,Node\" -c \"yellow.bold,cyan.bold\" \"npm run watch\" \"nodemon\"",
3030
"dev:worker": "concurrently -k -n \"TypeScript,Node\" -c \"yellow.bold,cyan.bold\" \"npm run watch\" \"nodemon worker\"",

packages/cli/src/benchmark/benchmark.md

+4-6
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,10 @@ To create a benchmark, @TODO
1616
1717
<!-- BENCHMARK_SUITES_LIST -->
1818

19-
### 1. Production workflow with authless webhook node
19+
### 001 - Production workflow with authless webhook node
2020

21-
Suite file: `webhook.tasks.js`
22-
23-
(1.1) using "Respond immediately" mode
24-
(1.2) using "When last node finishes" mode
25-
(1.3) using "Respond to Webhook" node mode
21+
- [using "Respond immediately" mode](./suites/workflows/001-1.json)
22+
- [using "When last node finishes" mode](./suites/workflows/001-2.json)
23+
- [using "Respond to Webhook" node mode](./suites/workflows/001-3.json)
2624

2725
<!-- /BENCHMARK_SUITES_LIST -->

packages/cli/src/benchmark/lib/hooks.ts

+12-32
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,9 @@ import type { WorkflowRequest } from '@/workflows/workflow.request';
1515
import { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
1616
import { Logger } from '@/Logger';
1717

18-
/**
19-
* Create a temp `.n8n` dir for encryption key, sqlite DB, etc.
20-
*/
21-
function n8nDir() {
18+
const logger = Container.get(Logger);
19+
20+
function tempN8nDir() {
2221
const baseDirPath = path.join(tmpdir(), 'n8n-benchmarks/');
2322

2423
mkdirSync(baseDirPath, { recursive: true });
@@ -41,14 +40,11 @@ function n8nDir() {
4140
instanceSettings.n8nFolder = _n8nDir;
4241
Container.set(InstanceSettings, instanceSettings);
4342

44-
Container.get(Logger).info(`Temp .n8n dir location: ${instanceSettings.n8nFolder}`);
43+
logger.info(`[Benchmarking] Temp .n8n dir location: ${instanceSettings.n8nFolder}`);
4544
}
4645

47-
/**
48-
* Load into DB and activate in memory all workflows to use in benchmarks.
49-
*/
50-
async function prepareWorkflows(owner: User) {
51-
const files = await glob('workflows/*.json', {
46+
async function loadWorkflows(owner: User) {
47+
const files = await glob('suites/workflows/*.json', {
5248
cwd: path.join('dist', 'benchmark'),
5349
absolute: true,
5450
});
@@ -64,41 +60,25 @@ async function prepareWorkflows(owner: User) {
6460
// @ts-ignore @TODO Fix typing
6561
await Container.get(WorkflowsController).create({ body: workflow, user: owner });
6662
}
67-
68-
await Container.get(ActiveWorkflowRunner).init();
6963
}
7064

7165
let main: Start;
7266

73-
/**
74-
* Start the main n8n process to use in benchmarks.
75-
*/
76-
async function mainProcess() {
77-
const args: string[] = [];
78-
const _config = new Config({ root: __dirname });
67+
export async function globalSetup() {
68+
tempN8nDir();
7969

80-
main = new Start(args, _config);
70+
main = new Start([], new Config({ root: __dirname }));
8171

8272
await main.init();
8373
await main.run();
84-
}
85-
86-
/**
87-
* Setup to run before once all benchmarks.
88-
*/
89-
export async function globalSetup() {
90-
n8nDir();
91-
92-
await mainProcess();
9374

9475
const owner = await Container.get(UserRepository).createTestOwner();
9576

96-
await prepareWorkflows(owner); // @TODO: Load all here or as part of each benchmark's `beforeEach`?
77+
await loadWorkflows(owner);
78+
79+
await Container.get(ActiveWorkflowRunner).init();
9780
}
9881

99-
/**
100-
* Teardown to run after all benchmarks.
101-
*/
10282
export async function globalTeardown() {
10383
await main.stopProcess();
10484
}
+108-1
Original file line numberDiff line numberDiff line change
@@ -1 +1,108 @@
1-
export { suite, collectSuites, task, beforeEach, afterEach } from './suites';
1+
import 'reflect-metadata';
2+
import path from 'node:path';
3+
import type Bench from 'tinybench';
4+
import { assert } from 'n8n-workflow';
5+
import glob from 'fast-glob';
6+
import callsites from 'callsites';
7+
import type { Suites, Task, Callback } from './types';
8+
import { DuplicateHookError } from './errors/duplicate-hook.error';
9+
import { DuplicateSuiteError } from './errors/duplicate-suite.error';
10+
11+
const suites: Suites = {};
12+
13+
export async function collectSuites() {
14+
const files = await glob('**/*.suite.js', {
15+
cwd: path.join('dist', 'benchmark'),
16+
absolute: true,
17+
});
18+
19+
for (const f of files) {
20+
await import(f);
21+
}
22+
23+
return suites;
24+
}
25+
26+
export function registerSuites(bench: Bench) {
27+
for (const { hooks, tasks } of Object.values(suites)) {
28+
/**
29+
* In tinybench, `beforeAll` and `afterAll` refer to all iterations of
30+
* a single task, while `beforeEach` and `afterEach` refer to each iteration.
31+
*
32+
* In jest and vitest, `beforeAll` and `afterAll` refer to all tests in a suite,
33+
* while `beforeEach` and `afterEach` refer to each individual test.
34+
*
35+
* We rename tinybench's hooks to prevent confusion from this difference.
36+
*/
37+
const options: Record<string, Callback> = {};
38+
39+
if (hooks.beforeEach) options.beforeAll = hooks.beforeEach;
40+
if (hooks.afterEach) options.afterAll = hooks.afterEach;
41+
42+
for (const t of tasks) {
43+
bench.add(t.name, t.operation, options);
44+
}
45+
}
46+
}
47+
48+
function suiteFilePath() {
49+
const filePath = callsites()
50+
.map((site) => site.getFileName())
51+
.filter((site): site is string => site !== null)
52+
.find((site) => site.endsWith('.suite.js'));
53+
54+
assert(filePath !== undefined);
55+
56+
return filePath;
57+
}
58+
59+
export function suite(suiteName: string, suiteFn: () => void) {
60+
const filePath = suiteFilePath();
61+
62+
if (suites[filePath]) throw new DuplicateSuiteError(filePath);
63+
64+
suites[filePath] = { name: suiteName, hooks: {}, tasks: [] };
65+
66+
suiteFn();
67+
}
68+
69+
export function task(taskName: string, operation: Task['operation']) {
70+
const filePath = suiteFilePath();
71+
72+
suites[filePath].tasks.push({
73+
name: suites[filePath].name + ' ' + taskName,
74+
operation,
75+
});
76+
}
77+
78+
// @TODO: Rename next two utils to dismbiguate?
79+
80+
/**
81+
* Setup step to run once before all iterations of each benchmarking task in a suite.
82+
*/
83+
export function beforeEach(fn: Callback) {
84+
const filePath = suiteFilePath();
85+
86+
if (suites[filePath]?.hooks.beforeEach) {
87+
throw new DuplicateHookError('beforeEach', filePath);
88+
}
89+
90+
suites[filePath].hooks.beforeEach = fn;
91+
}
92+
93+
/**
94+
* Teardown step to run once after all iterations of each benchmarking task in a suite.
95+
*/
96+
export function afterEach(fn: Callback) {
97+
const filePath = suiteFilePath();
98+
99+
if (suites[filePath]?.hooks.afterEach) {
100+
throw new DuplicateHookError('afterEach', filePath);
101+
}
102+
103+
suites[filePath].hooks.afterEach = fn;
104+
}
105+
106+
export const BACKEND_BASE_URL = 'http://localhost:5678';
107+
108+
export type { Suites } from './types';

packages/cli/src/benchmark/lib/suites.ts

-104
This file was deleted.

packages/cli/src/benchmark/lib/types.ts

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
/**
2-
* Benchmarking suites, i.e. `*.tasks.ts` files containing benchmarking tasks.
2+
* Benchmarking suites, i.e. `*.suite.ts` files containing benchmarking tasks.
33
*/
44
export type Suites = {
5-
[suiteFilepath: string]: {
5+
[suiteFilePath: string]: {
66
name: string;
77
hooks: {
88
beforeEach?: Callback;
@@ -16,7 +16,7 @@ export type Suites = {
1616
* A benchmarking task, i.e. a single operation whose performance to measure.
1717
*/
1818
export type Task = {
19-
description: string;
19+
name: string;
2020
operation: Callback;
2121
};
2222

packages/cli/src/benchmark/main.ts

+6-6
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
import 'reflect-metadata';
2-
import * as hooks from './lib/hooks';
3-
import { collectSuites, registerSuites } from './lib/suites';
2+
import Container from 'typedi';
43
import config from '@/config';
5-
import { UnsupportedDatabaseError } from './lib/errors/unsupported-database.error';
64
import { Logger } from '@/Logger';
7-
import Container from 'typedi';
5+
import * as hooks from './lib/hooks';
6+
import { collectSuites, registerSuites } from './lib';
7+
import { UnsupportedDatabaseError } from './lib/errors/unsupported-db.error';
88

99
/* eslint-disable import/no-extraneous-dependencies */
1010
import Bench from 'tinybench';
@@ -23,11 +23,11 @@ async function main() {
2323
const logger = Container.get(Logger);
2424

2525
if (count === 0) {
26-
logger.info('No benchmarking suites found. Exiting...');
26+
logger.info('[Benchmarking] Found no suites. Exiting...');
2727
return;
2828
}
2929

30-
logger.info(`Running ${count} benchmarking ${count === 1 ? 'suite' : 'suites'}...`);
30+
logger.info(`[Benchmarking] Running ${count} ${count === 1 ? 'suite' : 'suites'}...`);
3131

3232
await hooks.globalSetup();
3333

0 commit comments

Comments
 (0)