Skip to content

Commit 60d6c52

Browse files
CMCDragonkaiCMCDragonkai
committed
Initial port of DB from EFS
1 parent 89c5f3d commit 60d6c52

15 files changed

+1493
-4
lines changed

‎benches/DB1KiB.ts

+101
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
import type { EFSWorkerModule } from '@/workers';
2+
3+
import os from 'os';
4+
import path from 'path';
5+
import fs from 'fs';
6+
import b from 'benny';
7+
import { spawn, Worker } from 'threads';
8+
import Logger, { LogLevel, StreamHandler } from '@matrixai/logger';
9+
import { WorkerManager } from '@matrixai/workers';
10+
import { DB } from '@/db';
11+
import * as utils from '@/utils';
12+
import packageJson from '../package.json';
13+
14+
const logger = new Logger('DB1KiB Bench', LogLevel.WARN, [new StreamHandler()]);
15+
16+
async function main() {
17+
const cores = os.cpus().length;
18+
logger.warn(`Cores: ${cores}`);
19+
const workerManager = new WorkerManager<EFSWorkerModule>({ logger });
20+
await workerManager.start({
21+
workerFactory: () => spawn(new Worker('../src/workers/efsWorker')),
22+
cores,
23+
});
24+
const dataDir = await fs.promises.mkdtemp(
25+
path.join(os.tmpdir(), 'encryptedfs-benches-'),
26+
);
27+
const dbKey = await utils.generateKey(256);
28+
// Db1 doesn't use workers
29+
const dbPath1 = `${dataDir}/db1`;
30+
const db1 = await DB.createDB({ dbKey, dbPath: dbPath1, logger });
31+
await db1.start();
32+
// Db2 uses workers
33+
const dbPath2 = `${dataDir}/db2`;
34+
const db2 = await DB.createDB({ dbKey, dbPath: dbPath2, logger });
35+
await db2.start();
36+
db2.setWorkerManager(workerManager);
37+
const data0 = utils.getRandomBytesSync(0);
38+
const data1KiB = utils.getRandomBytesSync(1024);
39+
const summary = await b.suite(
40+
'DB1KiB',
41+
b.add('get 1 KiB of data', async () => {
42+
await db1.put([], '1kib', data1KiB, true);
43+
return async () => {
44+
await db1.get([], '1kib', true);
45+
};
46+
}),
47+
b.add('put 1 KiB of data', async () => {
48+
await db1.put([], '1kib', data1KiB, true);
49+
}),
50+
b.add('put zero data', async () => {
51+
await db1.put([], '0', data0, true);
52+
}),
53+
b.add('put zero data then del', async () => {
54+
await db1.put([], '0', data0, true);
55+
await db1.del([], '0');
56+
}),
57+
b.add('get 1 KiB of data with workers', async () => {
58+
await db2.put([], '1kib', data1KiB, true);
59+
return async () => {
60+
await db2.get([], '1kib', true);
61+
};
62+
}),
63+
b.add('put 1 KiB of data with workers', async () => {
64+
await db2.put([], '1kib', data1KiB, true);
65+
}),
66+
b.add('put zero data with workers', async () => {
67+
await db2.put([], '0', data0, true);
68+
}),
69+
b.add('put zero data then del with workers', async () => {
70+
await db2.put([], '0', data0, true);
71+
await db2.del([], '0');
72+
}),
73+
b.cycle(),
74+
b.complete(),
75+
b.save({
76+
file: 'DB1KiB',
77+
folder: 'benches/results',
78+
version: packageJson.version,
79+
details: true,
80+
}),
81+
b.save({
82+
file: 'DB1KiB',
83+
folder: 'benches/results',
84+
format: 'chart.html',
85+
}),
86+
);
87+
await db1.stop();
88+
await db2.stop();
89+
await fs.promises.rm(dataDir, {
90+
force: true,
91+
recursive: true,
92+
});
93+
await workerManager.stop();
94+
return summary;
95+
}
96+
97+
if (require.main === module) {
98+
main();
99+
}
100+
101+
export default main;

‎benches/DB1MiB.ts

+85
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
import type { EFSWorkerModule } from '@/workers';
2+
3+
import os from 'os';
4+
import path from 'path';
5+
import fs from 'fs';
6+
import b from 'benny';
7+
import { spawn, Worker } from 'threads';
8+
import Logger, { LogLevel, StreamHandler } from '@matrixai/logger';
9+
import { WorkerManager } from '@matrixai/workers';
10+
import { DB } from '@/db';
11+
import * as utils from '@/utils';
12+
import packageJson from '../package.json';
13+
14+
const logger = new Logger('DB1MiB Bench', LogLevel.WARN, [new StreamHandler()]);
15+
16+
async function main() {
17+
const cores = os.cpus().length;
18+
logger.warn(`Cores: ${cores}`);
19+
const workerManager = new WorkerManager<EFSWorkerModule>({ logger });
20+
await workerManager.start({
21+
workerFactory: () => spawn(new Worker('../src/workers/efsWorker')),
22+
cores,
23+
});
24+
const dataDir = await fs.promises.mkdtemp(
25+
path.join(os.tmpdir(), 'encryptedfs-benches-'),
26+
);
27+
const dbKey = await utils.generateKey(256);
28+
// Db1 doesn't use workers
29+
const dbPath1 = `${dataDir}/db1`;
30+
const db1 = await DB.createDB({ dbKey, dbPath: dbPath1, logger });
31+
await db1.start();
32+
// Db2 uses workers
33+
const dbPath2 = `${dataDir}/db2`;
34+
const db2 = await DB.createDB({ dbKey, dbPath: dbPath2, logger });
35+
await db2.start();
36+
const data1MiB = utils.getRandomBytesSync(1024 * 1024);
37+
const summary = await b.suite(
38+
'DB1MiB',
39+
b.add('get 1 MiB of data', async () => {
40+
await db1.put([], '1mib', data1MiB, true);
41+
return async () => {
42+
await db1.get([], '1mib', true);
43+
};
44+
}),
45+
b.add('put 1 MiB of data', async () => {
46+
await db1.put([], '1mib', data1MiB, true);
47+
}),
48+
b.add('get 1 MiB of data with workers', async () => {
49+
await db2.put([], '1mib', data1MiB, true);
50+
return async () => {
51+
await db2.get([], '1mib', true);
52+
};
53+
}),
54+
b.add('put 1 MiB of data with workers', async () => {
55+
await db2.put([], '1mib', data1MiB, true);
56+
}),
57+
b.cycle(),
58+
b.complete(),
59+
b.save({
60+
file: 'DB1MiB',
61+
folder: 'benches/results',
62+
version: packageJson.version,
63+
details: true,
64+
}),
65+
b.save({
66+
file: 'DB1MiB',
67+
folder: 'benches/results',
68+
format: 'chart.html',
69+
}),
70+
);
71+
await db1.stop();
72+
await db2.stop();
73+
await fs.promises.rm(dataDir, {
74+
force: true,
75+
recursive: true,
76+
});
77+
await workerManager.stop();
78+
return summary;
79+
}
80+
81+
if (require.main === module) {
82+
main();
83+
}
84+
85+
export default main;

‎benches/index.ts

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/usr/bin/env node
2+
3+
import fs from 'fs';
4+
import si from 'systeminformation';
5+
import DB1KiBBench from './DB1KiB';
6+
import DB1MiBBench from './DB1MiB';
7+
8+
async function main(): Promise<void> {
9+
await DB1KiBBench();
10+
await DB1MiBBench();
11+
const systemData = await si.get({
12+
cpu: '*',
13+
osInfo: 'platform, distro, release, kernel, arch',
14+
system: 'model, manufacturer',
15+
});
16+
await fs.promises.writeFile(
17+
'benches/results/system.json',
18+
JSON.stringify(systemData, null, 2),
19+
);
20+
}
21+
22+
main();

‎package-lock.json

+24-3
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

+4-1
Original file line numberDiff line numberDiff line change
@@ -33,14 +33,17 @@
3333
},
3434
"devDependencies": {
3535
"@types/jest": "^26.0.20",
36-
"@types/node": "^13.13.1",
36+
"@types/node": "^14.14.35",
37+
"@types/node-forge": "^0.10.4",
3738
"@typescript-eslint/eslint-plugin": "^4.12.0",
3839
"@typescript-eslint/parser": "^4.12.0",
3940
"benny": "^3.6.15",
4041
"eslint": "^7.17.0",
4142
"eslint-config-prettier": "^7.1.0",
4243
"eslint-plugin-prettier": "^3.3.1",
4344
"jest": "^26.6.3",
45+
"lexicographic-integer": "^1.1.0",
46+
"node-forge": "^0.10.0",
4447
"prettier": "^2.2.1",
4548
"systeminformation": "^5.8.9",
4649
"ts-jest": "^26.4.4",

0 commit comments

Comments
 (0)