made benchmarks more resilient to OS filesystem caching issues

Author: JoshuaWise

benchmark/benchmark.js

@@ -19,7 +19,7 @@ const display = (result) => {
 (async () => {
 	process.on('unhandledRejection', (err) => { throw err; });
 	const ctx = JSON.parse(process.argv[2]);
-	const db = await require('./drivers').get(ctx.driver)(ctx.filename, ctx.pragma);
+	const db = await require('./drivers').get(ctx.driver)('../temp/benchmark.db', ctx.pragma);
 	const fn = require(`./types/${ctx.type}`)[ctx.driver](db, ctx);
 	if (typeof fn === 'function') setImmediate(sync, fn);
 	else setImmediate(async, await fn);

benchmark/index.js

@@ -24,6 +24,14 @@ const filterBySearchTerms = (searchTerms) => (trial) => {
 	return searchTerms.every(arg => terms.includes(arg));
 };
 
+const sortTrials = (a, b) => {
+	const aRo = require(`./types/${a.type}`).readonly;
+	const bRo = require(`./types/${b.type}`).readonly;
+	if (typeof aRo !== 'boolean') throw new TypeError(`Missing readonly export in benchmark type ${a.type}`);
+	if (typeof bRo !== 'boolean') throw new TypeError(`Missing readonly export in benchmark type ${b.type}`);
+	return bRo - aRo;
+};
+
 const displayTrialName = (trial) => {
 	if (trial.description) return console.log(clc.magenta(`--- ${trial.description} ---`));
 	const name = `${trial.type} ${trial.table} (${trial.columns.join(', ')})`;
@@ -33,8 +41,7 @@ const displayTrialName = (trial) => {
 
 const createContext = (trial, driver) => {
 	const { data: _unused, ...tableInfo } = tables.get(trial.table);
-	const ctx = { ...trial, ...tableInfo, driver, filename: `../temp/${iteration++}.db` };
-	return JSON.stringify(ctx);
+	return JSON.stringify({ ...trial, ...tableInfo, driver });
 };
 
 const erase = () => {
@@ -43,20 +50,19 @@ const erase = () => {
 
 // Determine which trials should be executed.
 process.chdir(__dirname);
-const trials = getTrials(process.argv.slice(2));
+const trials = getTrials(process.argv.slice(2)).sort(sortTrials);
 if (!trials.length) {
 	console.log(clc.yellow('No matching benchmarks found!'));
 	process.exit();
 }
 
-// Create the temporary databases needed to run the benchmark trials.
+// Create the temporary database needed to run the benchmark trials.
 console.log('Generating tables...');
-const drivers = require('./drivers');
-const tables = require('./seed')(drivers.size * trials.length);
+const tables = require('./seed')();
 process.stdout.write(erase());
 
 // Execute each trial for each available driver.
-let iteration = 0;
+const drivers = require('./drivers');
 const nameLength = [...drivers.keys()].reduce((m, d) => Math.max(m, d.length), 0);
 for (const trial of trials) {
 	displayTrialName(trial);

benchmark/seed.js

@@ -26,17 +26,17 @@ const tables = new Map([
 ]);
 
 /*
-	This function creates a number of pre-populated databases that are deleted
-	when the process exits.
+	This function creates a pre-populated database that is deleted when the
+	process exits.
  */
 
-module.exports = (numberOfDatabases = 2) => {
+module.exports = () => {
 	const tempDir = path.join(__dirname, '..', 'temp');
 	process.on('exit', () => fs.removeSync(tempDir));
 	fs.removeSync(tempDir);
 	fs.ensureDirSync(tempDir);
 
-	const db = require('../.')(path.join(tempDir, '0.db'));
+	const db = require('../.')(path.join(tempDir, 'benchmark.db'));
 	db.pragma('journal_mode = OFF');
 	db.pragma('synchronous = OFF');
 
@@ -48,6 +48,5 @@ module.exports = (numberOfDatabases = 2) => {
 	}
 
 	db.close();
-	for (let i = 1; i < numberOfDatabases; ++i) fs.copySync(db.name, path.join(tempDir, `${i}.db`));
 	return tables;
 };

benchmark/types/insert.js

@@ -1,5 +1,5 @@
 'use strict';
-// Inserting rows individually (`.run()`)
+exports.readonly = false; // Inserting rows individually (`.run()`)
 
 exports['better-sqlite3'] = (db, { table, columns }) => {
 	const stmt = db.prepare(`INSERT INTO ${table} (${columns.join(', ')}) VALUES (${columns.map(x => '@' + x).join(', ')})`);

benchmark/types/select-all.js

@@ -1,5 +1,5 @@
 'use strict';
-// Reading 100 rows into an array (`.all()`)
+exports.readonly = true; // Reading 100 rows into an array (`.all()`)
 
 exports['better-sqlite3'] = (db, { table, columns, count }) => {
 	const stmt = db.prepare(`SELECT ${columns.join(', ')} FROM ${table} WHERE rowid >= ? LIMIT 100`);

benchmark/types/select-iterate.js

@@ -1,5 +1,5 @@
 'use strict';
-// Iterating over 100 rows (`.iterate()`)
+exports.readonly = true; // Iterating over 100 rows (`.iterate()`)
 
 exports['better-sqlite3'] = (db, { table, columns, count }) => {
 	const stmt = db.prepare(`SELECT ${columns.join(', ')} FROM ${table} WHERE rowid >= ? LIMIT 100`);

benchmark/types/select.js

@@ -1,5 +1,5 @@
 'use strict';
-// Reading rows individually (`.get()`)
+exports.readonly = true; // Reading rows individually (`.get()`)
 
 exports['better-sqlite3'] = (db, { table, columns, count }) => {
 	const stmt = db.prepare(`SELECT ${columns.join(', ')} FROM ${table} WHERE rowid = ?`);

benchmark/types/transaction.js

@@ -1,5 +1,5 @@
 'use strict';
-// Inserting 100 rows in a single transaction
+exports.readonly = false; // Inserting 100 rows in a single transaction
 
 exports['better-sqlite3'] = (db, { table, columns }) => {
 	const stmt = db.prepare(`INSERT INTO ${table} (${columns.join(', ')}) VALUES (${columns.map(x => '@' + x).join(', ')})`);
@@ -10,7 +10,7 @@ exports['better-sqlite3'] = (db, { table, columns }) => {
 	return () => trx(row);
 };
 
-exports['node-sqlite3'] = async (db, { table, columns, driver, filename, pragma }) => {
+exports['node-sqlite3'] = async (db, { table, columns, driver, pragma }) => {
 	const sql = `INSERT INTO ${table} (${columns.join(', ')}) VALUES (${columns.map(x => '@' + x).join(', ')})`;
 	const row = Object.assign({}, ...Object.entries(await db.get(`SELECT * FROM ${table} LIMIT 1`))
 		.filter(([k]) => columns.includes(k))
@@ -22,7 +22,7 @@ exports['node-sqlite3'] = async (db, { table, columns, driver, filename, pragma
 		connection for each transaction.
 		(http://github.com/mapbox/node-sqlite3/issues/304#issuecomment-45242331)
 	 */
-	return () => open(filename, pragma).then(async (db) => {
+	return () => open('../temp/benchmark.db', pragma).then(async (db) => {
 		try {
 			await db.run('BEGIN');
 			try {