Skip to content

Commit 79686b6

Browse files
MarshallOfSoundclaude
MarshallOfSound
and
claude
authored
perf: 5-7x faster packing, 15-20% faster extraction (#414)
* fix: resolve pickle resize overflow, stream flush race, and integrity block boundary bugs Added 81 robustness tests that uncovered three bugs: - Pickle resize allocated wrong buffer size, causing RangeError on large headers - writeFileListToStream/streamFilesystem didn't await stream flush before returning - Integrity hash emitted spurious empty block when file size hit exact 4MB boundary Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix(test): use path.join for deeply nested path test on Windows Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix(test): use os.tmpdir() instead of shared dir for Windows compatibility Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix(test): use wrappedFs to bypass Electron ASAR interception Under Electron, node:fs intercepts .asar paths as virtual archives. Tests that read/write raw .asar files need original-fs via wrappedFs. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * perf: optimize pack and extract performance (5-7x faster packing) Key optimizations: - Buffer-based integrity hashing for files ≤2MB (avoids stream setup overhead) - Batched writes in writeFileListToStream (single write instead of per-file) - Bulk archive read in extractAll (one readSync for entire data section) - Synchronous fast path in insertFile (eliminates Promise allocation per file) - Set-based lookups for ordering and unpackDirs (O(1) vs O(n)) - Parallel metadata resolution in createPackageFromFiles - Optimized crawl symlink filter with early exit Adds benchmark suite (yarn benchmark / yarn benchmark:quick). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * ci: run tests on all PRs, not just those targeting main Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * test: remove robustness-spec.ts (superseded by split spec files from #417) --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 24191b4 commit 79686b6

14 files changed

Lines changed: 1343 additions & 110 deletions

‎.github/workflows/test.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@ name: Test
22

33
on:
44
pull_request:
5-
branches:
6-
- main
75
schedule:
86
- cron: '0 22 * * 3'
97
workflow_call:

‎benchmark/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
fixtures/
2+
results.json

‎benchmark/generate-fixtures.ts‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
import fs from 'node:fs';
2+
import path from 'node:path';
3+
import crypto from 'node:crypto';
4+
5+
const BENCHMARK_DIR = path.join(import.meta.dirname, 'fixtures');
6+
7+
export type FixtureConfig = {
8+
name: string;
9+
fileCount: number;
10+
/** Average file size in bytes */
11+
avgFileSize: number;
12+
/** Directory depth */
13+
depth: number;
14+
/** Number of subdirectories per level */
15+
breadth: number;
16+
};
17+
18+
export const FIXTURES: FixtureConfig[] = [
19+
{ name: 'small', fileCount: 10, avgFileSize: 1024, depth: 2, breadth: 2 },
20+
{ name: 'medium', fileCount: 500, avgFileSize: 4096, depth: 4, breadth: 4 },
21+
{ name: 'large', fileCount: 5000, avgFileSize: 8192, depth: 5, breadth: 5 },
22+
{ name: 'few-large-files', fileCount: 20, avgFileSize: 1024 * 1024, depth: 2, breadth: 2 },
23+
{ name: 'many-small-files', fileCount: 10000, avgFileSize: 256, depth: 3, breadth: 10 },
24+
{ name: 'deep-tree', fileCount: 1000, avgFileSize: 2048, depth: 10, breadth: 2 },
25+
];
26+
27+
function generateRandomContent(size: number): Buffer {
28+
return crypto.randomBytes(size);
29+
}
30+
31+
function generateDirectoryPaths(depth: number, breadth: number): string[] {
32+
const dirs: string[] = [''];
33+
for (let d = 0; d < depth; d++) {
34+
const currentLevel = dirs.filter((dir) => dir.split('/').length - 1 === d);
35+
for (const parent of currentLevel) {
36+
for (let b = 0; b < breadth; b++) {
37+
dirs.push(parent ? `${parent}/dir_${d}_${b}` : `dir_${d}_${b}`);
38+
}
39+
}
40+
}
41+
return dirs;
42+
}
43+
44+
export function generateFixture(config: FixtureConfig): string {
45+
const fixtureDir = path.join(BENCHMARK_DIR, config.name);
46+
47+
if (fs.existsSync(fixtureDir)) {
48+
return fixtureDir;
49+
}
50+
51+
fs.mkdirSync(fixtureDir, { recursive: true });
52+
53+
const dirs = generateDirectoryPaths(config.depth, config.breadth);
54+
55+
// Create all directories
56+
for (const dir of dirs) {
57+
if (dir) {
58+
fs.mkdirSync(path.join(fixtureDir, dir), { recursive: true });
59+
}
60+
}
61+
62+
// Distribute files across directories
63+
for (let i = 0; i < config.fileCount; i++) {
64+
const dir = dirs[i % dirs.length];
65+
// Vary file sizes: 50% to 150% of average
66+
const sizeVariation = 0.5 + Math.random();
67+
const size = Math.max(1, Math.floor(config.avgFileSize * sizeVariation));
68+
const content = generateRandomContent(size);
69+
const ext = ['.txt', '.js', '.json', '.bin', '.dat'][i % 5];
70+
const filePath = path.join(fixtureDir, dir, `file_${i}${ext}`);
71+
fs.writeFileSync(filePath, content);
72+
}
73+
74+
return fixtureDir;
75+
}
76+
77+
export function cleanFixtures() {
78+
if (fs.existsSync(BENCHMARK_DIR)) {
79+
fs.rmSync(BENCHMARK_DIR, { recursive: true, force: true });
80+
}
81+
}
82+
83+
// When run directly, generate all fixtures
84+
if (process.argv[1] === import.meta.filename) {
85+
console.log('Generating benchmark fixtures...');
86+
cleanFixtures();
87+
for (const config of FIXTURES) {
88+
const start = performance.now();
89+
const dir = generateFixture(config);
90+
const elapsed = (performance.now() - start).toFixed(1);
91+
const totalSize = getTotalSize(dir);
92+
console.log(
93+
` ${config.name}: ${config.fileCount} files, ${formatBytes(totalSize)} in ${elapsed}ms`,
94+
);
95+
}
96+
console.log('Done.');
97+
}
98+
99+
function getTotalSize(dir: string): number {
100+
let total = 0;
101+
for (const entry of fs.readdirSync(dir, { withFileTypes: true, recursive: true })) {
102+
if (entry.isFile()) {
103+
total += fs.statSync(path.join(entry.parentPath, entry.name)).size;
104+
}
105+
}
106+
return total;
107+
}
108+
109+
export function formatBytes(bytes: number): string {
110+
if (bytes < 1024) return `${bytes} B`;
111+
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
112+
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
113+
}

‎benchmark/profile-extract.ts‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
import fs from 'node:fs';
2+
import path from 'node:path';
3+
import os from 'node:os';
4+
import { generateFixture, FIXTURES } from './generate-fixtures.js';
5+
import { createPackage, extractAll, listPackage, getRawHeader, uncache } from '../lib/asar.js';
6+
import { readArchiveHeaderSync, readFilesystemSync } from '../lib/disk.js';
7+
8+
const config = FIXTURES.find((f) => f.name === 'many-small-files')!;
9+
const fixtureDir = generateFixture(config);
10+
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'asar-prof-extract-'));
11+
const archiveFile = path.join(tmpDir, 'test.asar');
12+
await createPackage(fixtureDir, archiveFile);
13+
const archiveSize = fs.statSync(archiveFile).size;
14+
15+
console.log(`=== Extract breakdown: ${config.fileCount} files, archive ${(archiveSize / 1024).toFixed(0)} KB ===\n`);
16+
17+
// 1. Header parsing
18+
{
19+
const t = performance.now();
20+
for (let i = 0; i < 10; i++) readArchiveHeaderSync(archiveFile);
21+
console.log(`header parse (avg): ${((performance.now() - t) / 10).toFixed(2)}ms`);
22+
}
23+
24+
// 2. listFiles
25+
{
26+
uncache(archiveFile);
27+
const filesystem = readFilesystemSync(archiveFile);
28+
const t = performance.now();
29+
const files = filesystem.listFiles();
30+
console.log(`listFiles: ${(performance.now() - t).toFixed(2)}ms (${files.length} entries)`);
31+
32+
// 3. getFile lookups
33+
const followLinks = process.platform === 'win32';
34+
const t2 = performance.now();
35+
for (const fullPath of files) {
36+
const filename = fullPath.substr(1);
37+
filesystem.getFile(filename, followLinks);
38+
}
39+
console.log(`getFile (all): ${(performance.now() - t2).toFixed(2)}ms`);
40+
}
41+
42+
// 4. Read all file data from archive with individual readSync
43+
{
44+
uncache(archiveFile);
45+
const filesystem = readFilesystemSync(archiveFile);
46+
const files = filesystem.listFiles();
47+
const followLinks = process.platform === 'win32';
48+
const fd = fs.openSync(archiveFile, 'r');
49+
const headerSize = filesystem.getHeaderSize();
50+
51+
let totalRead = 0;
52+
const t = performance.now();
53+
for (const fullPath of files) {
54+
const filename = fullPath.substr(1);
55+
const file = filesystem.getFile(filename, followLinks);
56+
if ('size' in file && !('files' in file) && !('link' in file) && !file.unpacked) {
57+
const buffer = Buffer.alloc(file.size);
58+
if (file.size > 0) {
59+
const offset = 8 + headerSize + parseInt(file.offset);
60+
fs.readSync(fd, buffer, 0, file.size, offset);
61+
totalRead += file.size;
62+
}
63+
}
64+
}
65+
fs.closeSync(fd);
66+
console.log(`readSync (individual): ${(performance.now() - t).toFixed(2)}ms (${totalRead} bytes)`);
67+
68+
// 5. Read all data in one shot
69+
const t2 = performance.now();
70+
const fd2 = fs.openSync(archiveFile, 'r');
71+
const dataStart = 8 + headerSize;
72+
const dataSize = archiveSize - dataStart;
73+
const allData = Buffer.alloc(dataSize);
74+
fs.readSync(fd2, allData, 0, dataSize, dataStart);
75+
fs.closeSync(fd2);
76+
console.log(`readSync (one shot): ${(performance.now() - t2).toFixed(2)}ms (${dataSize} bytes)`);
77+
}
78+
79+
// 6. writeFileSync overhead
80+
{
81+
const extractDir = path.join(tmpDir, 'write-test');
82+
fs.mkdirSync(extractDir, { recursive: true });
83+
84+
// Pre-generate content
85+
const content = Buffer.alloc(256, 0x41);
86+
const filePaths: string[] = [];
87+
for (let i = 0; i < 10000; i++) {
88+
const dir = path.join(extractDir, `d${i % 10}`);
89+
fs.mkdirSync(dir, { recursive: true });
90+
filePaths.push(path.join(dir, `f${i}.txt`));
91+
}
92+
93+
const t = performance.now();
94+
for (const fp of filePaths) {
95+
fs.writeFileSync(fp, content);
96+
}
97+
console.log(`writeFileSync (10k): ${(performance.now() - t).toFixed(1)}ms`);
98+
99+
fs.rmSync(extractDir, { recursive: true });
100+
}
101+
102+
// 7. mkdirSync overhead (with recursive)
103+
{
104+
const extractDir = path.join(tmpDir, 'mkdir-test');
105+
const dirs: string[] = [];
106+
for (let i = 0; i < 1000; i++) {
107+
dirs.push(path.join(extractDir, `a${i % 10}`, `b${i % 100}`, `c${i}`));
108+
}
109+
const t = performance.now();
110+
for (const d of dirs) {
111+
fs.mkdirSync(d, { recursive: true });
112+
}
113+
console.log(`mkdirSync (1k): ${(performance.now() - t).toFixed(1)}ms`);
114+
fs.rmSync(extractDir, { recursive: true });
115+
}
116+
117+
// 8. Full extractAll for reference
118+
{
119+
const extractDir = path.join(tmpDir, 'full-extract');
120+
uncache(archiveFile);
121+
const t = performance.now();
122+
extractAll(archiveFile, extractDir);
123+
console.log(`\nextractAll (total): ${(performance.now() - t).toFixed(1)}ms`);
124+
fs.rmSync(extractDir, { recursive: true });
125+
}
126+
127+
// 9. Pack insertFile loop overhead (without pre-read)
128+
{
129+
console.log('\n--- Pack insert loop breakdown ---');
130+
const { crawl } = await import('../lib/crawlfs.js');
131+
const [filenames, metadata] = await crawl(fixtureDir + '/**/*', { dot: true });
132+
133+
// Just the insert loop with readFileSync + hash
134+
const { Filesystem } = await import('../lib/filesystem.js');
135+
const filesystem = new Filesystem(fixtureDir);
136+
137+
const fileEntries = filenames.filter((f) => metadata[f]?.type === 'file');
138+
const dirEntries = filenames.filter((f) => metadata[f]?.type === 'directory');
139+
140+
let t = performance.now();
141+
for (const d of dirEntries) filesystem.insertDirectory(d, false);
142+
console.log(`insertDirectory: ${(performance.now() - t).toFixed(1)}ms (${dirEntries.length})`);
143+
144+
t = performance.now();
145+
for (const f of fileEntries) {
146+
await filesystem.insertFile(f, () => fs.createReadStream(f), false, metadata[f]);
147+
}
148+
console.log(`insertFile (all): ${(performance.now() - t).toFixed(1)}ms (${fileEntries.length})`);
149+
}
150+
151+
fs.rmSync(tmpDir, { recursive: true, force: true });

‎benchmark/profile-memory.ts‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
import fs from 'node:fs';
2+
import path from 'node:path';
3+
import os from 'node:os';
4+
import { generateFixture, formatBytes, FIXTURES } from './generate-fixtures.js';
5+
import { createPackage, createPackageFromFiles, uncache } from '../lib/asar.js';
6+
import { crawl } from '../lib/crawlfs.js';
7+
8+
function getMemoryUsage() {
9+
if (global.gc) global.gc();
10+
return process.memoryUsage();
11+
}
12+
13+
function printMem(label: string, before: NodeJS.MemoryUsage, after: NodeJS.MemoryUsage) {
14+
const heapDelta = after.heapUsed - before.heapUsed;
15+
const rssDelta = after.rss - before.rss;
16+
console.log(
17+
` ${label.padEnd(30)} ` +
18+
`heap: ${formatBytes(after.heapUsed).padStart(10)} (Δ ${(heapDelta >= 0 ? '+' : '') + formatBytes(heapDelta)}) ` +
19+
`rss: ${formatBytes(after.rss).padStart(10)} (Δ ${(rssDelta >= 0 ? '+' : '') + formatBytes(rssDelta)})`
20+
);
21+
}
22+
23+
async function profileFixture(name: string) {
24+
const config = FIXTURES.find((f) => f.name === name)!;
25+
const fixtureDir = generateFixture(config);
26+
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), `asar-mem-${name}-`));
27+
const archiveFile = path.join(tmpDir, `${name}.asar`);
28+
29+
// Get data size
30+
let dataSize = 0;
31+
for (const entry of fs.readdirSync(fixtureDir, { withFileTypes: true, recursive: true })) {
32+
if (entry.isFile()) dataSize += fs.statSync(path.join(entry.parentPath, entry.name)).size;
33+
}
34+
35+
console.log(`\n=== ${name} (${config.fileCount} files, ${formatBytes(dataSize)}) ===`);
36+
37+
// Measure: createPackage (full pipeline)
38+
{
39+
const before = getMemoryUsage();
40+
41+
// Track peak heap during packing
42+
let peakHeap = before.heapUsed;
43+
let peakRss = before.rss;
44+
const interval = setInterval(() => {
45+
const mem = process.memoryUsage();
46+
if (mem.heapUsed > peakHeap) peakHeap = mem.heapUsed;
47+
if (mem.rss > peakRss) peakRss = mem.rss;
48+
}, 1);
49+
50+
if (fs.existsSync(archiveFile)) fs.unlinkSync(archiveFile);
51+
await createPackage(fixtureDir, archiveFile);
52+
53+
clearInterval(interval);
54+
const after = getMemoryUsage();
55+
// One final peak check
56+
if (after.heapUsed > peakHeap) peakHeap = after.heapUsed;
57+
if (after.rss > peakRss) peakRss = after.rss;
58+
59+
printMem('createPackage (after)', before, after);
60+
console.log(
61+
` ${'peak heap'.padEnd(30)} ${formatBytes(peakHeap).padStart(10)} (Δ +${formatBytes(peakHeap - before.heapUsed)}) ` +
62+
`rss: ${formatBytes(peakRss).padStart(10)} (Δ +${formatBytes(peakRss - before.rss)})`
63+
);
64+
console.log(` ${'data / peak-heap-delta'.padEnd(30)} ${((peakHeap - before.heapUsed) / dataSize).toFixed(2)}x data size`);
65+
}
66+
67+
// Measure: createPackageFromFiles (pre-crawled)
68+
{
69+
const [filenames, metadata] = await crawl(fixtureDir + '/**/*', { dot: true });
70+
71+
if (global.gc) global.gc();
72+
const before = getMemoryUsage();
73+
74+
let peakHeap = before.heapUsed;
75+
let peakRss = before.rss;
76+
const interval = setInterval(() => {
77+
const mem = process.memoryUsage();
78+
if (mem.heapUsed > peakHeap) peakHeap = mem.heapUsed;
79+
if (mem.rss > peakRss) peakRss = mem.rss;
80+
}, 1);
81+
82+
if (fs.existsSync(archiveFile)) fs.unlinkSync(archiveFile);
83+
await createPackageFromFiles(fixtureDir, archiveFile, [...filenames], { ...metadata });
84+
85+
clearInterval(interval);
86+
const after = getMemoryUsage();
87+
if (after.heapUsed > peakHeap) peakHeap = after.heapUsed;
88+
if (after.rss > peakRss) peakRss = after.rss;
89+
90+
printMem('createPackageFromFiles (after)', before, after);
91+
console.log(
92+
` ${'peak heap'.padEnd(30)} ${formatBytes(peakHeap).padStart(10)} (Δ +${formatBytes(peakHeap - before.heapUsed)}) ` +
93+
`rss: ${formatBytes(peakRss).padStart(10)} (Δ +${formatBytes(peakRss - before.rss)})`
94+
);
95+
console.log(` ${'data / peak-heap-delta'.padEnd(30)} ${((peakHeap - before.heapUsed) / dataSize).toFixed(2)}x data size`);
96+
}
97+
98+
fs.rmSync(tmpDir, { recursive: true, force: true });
99+
}
100+
101+
console.log('Memory profiling with --expose-gc');
102+
console.log(`Node.js: ${process.version}, Platform: ${os.platform()} ${os.arch()}`);
103+
104+
await profileFixture('small');
105+
await profileFixture('medium');
106+
await profileFixture('large');
107+
await profileFixture('few-large-files');
108+
await profileFixture('many-small-files');
109+
await profileFixture('deep-tree');

0 commit comments

Comments
 (0)