feat(bruno-sqlite): move cache collection fetch to bruno-sqlite

packages/bruno-app/src/components/Preferences/Cache/index.js

@@ -1,9 +1,10 @@
-import React, { useEffect, useState, useCallback } from 'react';
+import React from 'react';
 import { useSelector, useDispatch } from 'react-redux';
 import { savePreferences, clearHttpHttpsAgentCache } from 'providers/ReduxStore/slices/app';
 import toast from 'react-hot-toast';
 import get from 'lodash/get';
 import { IconEraser } from '@tabler/icons';
+import { useSqliteQuery, useSqliteMutation } from '@usebruno/sqlite/web';
 import { useTheme } from 'providers/Theme';
 import ToggleSwitch from 'components/ToggleSwitch';
 import ActionIcon from 'ui/ActionIcon';
@@ -14,24 +15,15 @@ const Cache = () => {
   const preferences = useSelector((state) => state.app.preferences);
   const dispatch = useDispatch();
   const { theme } = useTheme();
-  const { ipcRenderer } = window;
 
   const fileCacheEnabled = get(preferences, 'cache.file.enabled', false);
   const sslSessionEnabled = get(preferences, 'cache.sslSession.enabled', false);
 
-  const [fileCacheSize, setFileCacheSize] = useState(null);
+  const { data: fileCacheSizeRow } = useSqliteQuery('file_index_size');
+  const fileCacheSize = fileCacheSizeRow?.bytes ?? null;
 
-  const refreshFileCacheSize = useCallback(() => {
-    if (!ipcRenderer) return;
-    ipcRenderer
-      .invoke('renderer:get-file-cache-size')
-      .then((size) => setFileCacheSize(size))
-      .catch(() => setFileCacheSize(null));
-  }, [ipcRenderer]);
-
-  useEffect(() => {
-    refreshFileCacheSize();
-  }, [refreshFileCacheSize, fileCacheEnabled]);
+  const clearFileCache = useSqliteMutation('file_index_clear');
+  const vacuumFileCache = useSqliteMutation('file_index_vacuum');
 
   const persist = (next) => {
     dispatch(savePreferences({ ...preferences, cache: next })).catch(() => {
@@ -57,15 +49,14 @@ const Cache = () => {
     }
   };
 
-  const handleClearFileCache = () => {
-    if (!ipcRenderer) return;
-    ipcRenderer
-      .invoke('renderer:clear-file-cache')
-      .then((size) => {
-        setFileCacheSize(size);
-        toast.success('File cache cleared');
-      })
-      .catch(() => toast.error('Failed to clear file cache'));
+  const handleClearFileCache = async () => {
+    try {
+      await clearFileCache.mutateAsync({});
+      await vacuumFileCache.mutateAsync({});
+      toast.success('File cache cleared');
+    } catch (error) {
+      toast.error('Failed to clear file cache');
+    }
   };
 
   const handleClearSslSession = () => {

packages/bruno-electron/src/ipc/mount.js

@@ -4,13 +4,6 @@ const { MountManager } = require('../services/mount');
 const manager = new MountManager();
 
 const registerMountIpc = () => {
-  ipcMain.handle('renderer:get-file-cache-size', () => manager.getCacheSize());
-
-  ipcMain.handle('renderer:clear-file-cache', () => {
-    manager.clearCache();
-    return manager.getCacheSize();
-  });
-
   ipcMain.handle(
     'renderer:mount-collection-v2',
     async (event, { collectionUid, collectionPathname, brunoConfig }) => {

packages/bruno-electron/src/ipc/sqlite.js

@@ -1,29 +1,52 @@
+const fs = require('node:fs');
 const path = require('path');
 const { app, ipcMain } = require('electron');
 const { createDatabase, registerSQLiteIpc, SQLITE_MUTATION_CHANNEL } = require('@usebruno/sqlite');
 
 let ipc = null;
 
+const LEGACY_FILE_INDEX_DB = 'mount-snapshots.db';
+const LEGACY_FILE_INDEX_SUFFIXES = ['', '-journal', '-wal', '-shm'];
+
+const removeLegacyFileIndex = () => {
+  const legacyPath = path.join(app.getPath('userData'), LEGACY_FILE_INDEX_DB);
+  if (!fs.existsSync(legacyPath)) return;
+
+  for (const suffix of LEGACY_FILE_INDEX_SUFFIXES) {
+    try {
+      fs.rmSync(legacyPath + suffix, { force: true, maxRetries: 3 });
+    } catch (err) {
+      console.warn(`failed to remove ${LEGACY_FILE_INDEX_DB}${suffix}: `, err);
+    }
+  }
+};
+
 class SqliteEventModel {
   _db = null;
   _statements = null;
   _window = null;
   constructor(window) {
     this._window = window;
     const { db, statements } = createDatabase(path.join(app.getPath('userData'), 'bruno.db'), {
+      pragmas: { journal_mode: 'WAL' },
       onMutation: (event) => {
         this._window?.webContents?.send(SQLITE_MUTATION_CHANNEL, event);
       }
     });
     this._db = db;
     this._statements = statements;
+    removeLegacyFileIndex();
     registerSQLiteIpc(ipcMain, statements);
   }
 
   get statements() {
     return this._statements;
   }
 
+  get db() {
+    return this._db;
+  }
+
   shutdown() {
     if (this._db) {
       this._db.close();
@@ -48,4 +71,6 @@ const shutdown = () => {
 
 const getStatements = () => (ipc ? ipc.statements : null);
 
-module.exports = { registerSqliteIpc, shutdown, getStatements };
+const getDatabase = () => (ipc ? ipc.db : null);
+
+module.exports = { registerSqliteIpc, shutdown, getStatements, getDatabase };

packages/bruno-electron/src/services/mount/file-index.js

@@ -1,6 +1,6 @@
 const fs = require('node:fs');
 const path = require('node:path');
-const { Database } = require('../storage');
+const { getStatements, getDatabase } = require('../../ipc/sqlite');
 const {
   hashFile,
   hashFileAsync,
@@ -12,47 +12,20 @@ const {
   walk
 } = require('../../utils/mount');
 
-const MIGRATIONS = [
-  {
-    version: 1,
-    up: `
-      CREATE TABLE IF NOT EXISTS file_index_entries (
-        collection_path TEXT NOT NULL,
-        relative_path TEXT NOT NULL,
-        id TEXT NOT NULL,
-        mtime INTEGER NOT NULL,
-        hash TEXT NOT NULL,
-        data TEXT NOT NULL,
-        PRIMARY KEY (collection_path, relative_path)
-      ) WITHOUT ROWID;
-      CREATE INDEX IF NOT EXISTS idx_collection_path ON file_index_entries(collection_path);
-    `
-  },
-  {
-    version: 2,
-    up: `
-      ALTER TABLE file_index_entries ADD COLUMN raw TEXT;
-      UPDATE file_index_entries SET mtime = 0, hash = '';
-      ALTER TABLE file_index_entries ADD COLUMN created_at INTEGER;
-      ALTER TABLE file_index_entries ADD COLUMN updated_at INTEGER;
-      UPDATE file_index_entries SET created_at = unixepoch(), updated_at = unixepoch();
-    `
-  }
-];
-
 // TODO: Check for trigger (ON UPDATE) and then see if we can use that to update updated_at
 
 class FileIndex {
+  #statements;
   #db;
-  #dbPath;
+  #applicationVersion;
 
-  constructor({ dbPath } = {}) {
-    this.#dbPath = dbPath || path.join(require('electron').app.getPath('userData'), 'mount-snapshots.db');
-    this.#db = new Database({ path: this.#dbPath, migrations: MIGRATIONS, readBigInts: true });
-  }
-
-  close() {
-    this.#db.close();
+  constructor() {
+    this.#statements = getStatements();
+    this.#db = getDatabase();
+    if (!this.#statements || !this.#db) {
+      throw new Error('the file cache is unavailable: the sqlite database is not open');
+    }
+    this.#applicationVersion = require('electron').app.getVersion();
   }
 
   async status(collectionPath, options = {}) {
@@ -74,6 +47,10 @@ class FileIndex {
         const hash = await hashFileAsync(absolutePath);
         return { kind: 'added', entry: { relativePath, absolutePath, mtime, hash } };
       }
+      if (prior.applicationVersion !== this.#applicationVersion) {
+        const hash = await hashFileAsync(absolutePath);
+        return { kind: 'updated', entry: { relativePath, absolutePath, mtime, hash, prevHash: prior.hash } };
+      }
       if (prior.mtime === mtime) return { kind: 'unchanged', relativePath };
       const hash = await hashFileAsync(absolutePath);
       if (hash === prior.hash) return { kind: 'unchanged', relativePath };
@@ -101,27 +78,14 @@ class FileIndex {
     return { added, updated, removed };
   }
 
-  clear() {
-    this.#db.exec('DELETE FROM file_index_entries');
-    // VACUUM so the file actually shrinks after the DELETE
-    this.#db.exec('VACUUM');
-  }
-
   clearCollection(collectionPath) {
-    const root = normalize(collectionPath);
-    this.#db.run('DELETE FROM file_index_entries WHERE collection_path = ?', root);
-  }
-
-  get dbPath() {
-    return this.#dbPath;
+    this.#statements.execute('file_index_clear_collection', { collection_path: normalize(collectionPath) });
   }
 
   entries(collectionPath) {
-    const root = normalize(collectionPath);
-    const rows = this.#db.all(
-      'SELECT relative_path AS relativePath, data, raw FROM file_index_entries WHERE collection_path = ?',
-      root
-    );
+    const rows = this.#statements.execute('file_index_entries_for_collection', {
+      collection_path: normalize(collectionPath)
+    });
     const map = new Map();
     for (const row of rows) {
       map.set(row.relativePath, { data: JSON.parse(row.data), raw: row.raw });
@@ -131,48 +95,34 @@ class FileIndex {
 
   stage(collectionPath, entry) {
     const root = normalize(collectionPath);
-    const { op, relativePath } = entry;
+    const { op } = entry;
+    const relativePath = path.normalize(entry.relativePath);
 
     if (op === 'remove') {
-      this.#db.run(
-        'DELETE FROM file_index_entries WHERE collection_path = ? AND relative_path = ?',
-        root,
-        relativePath
-      );
+      this.#statements.execute('file_index_delete_entry', { collection_path: root, relative_path: relativePath });
       return;
     }
 
     const { mtime, hash, data, raw } = entry;
-    const id = idForAbsolutePath(path.join(root, relativePath));
-    this.#db.run(
-      `
-      INSERT INTO file_index_entries (collection_path, relative_path, id, mtime, hash, data, raw, created_at, updated_at)
-      VALUES (?, ?, ?, ?, ?, ?, ?, unixepoch(), unixepoch())
-      ON CONFLICT(collection_path, relative_path) DO UPDATE SET
-        mtime = excluded.mtime,
-        hash = excluded.hash,
-        data = excluded.data,
-        raw = excluded.raw,
-        updated_at = unixepoch()
-    `,
-      root,
-      relativePath,
-      id,
+    this.#statements.execute('file_index_upsert', {
+      collection_path: root,
+      relative_path: relativePath,
+      id: idForAbsolutePath(path.join(root, relativePath)),
       mtime,
       hash,
-      JSON.stringify(data),
-      raw ?? null
-    );
+      data: JSON.stringify(data),
+      raw: raw ?? null,
+      application_version: this.#applicationVersion
+    });
   }
 
   stageParsed(collectionPath, absolutePath, data) {
-    const root = normalize(collectionPath);
-    const relativePath = path.relative(root, normalize(absolutePath));
-    if (relativePath.startsWith('..') || path.isAbsolute(relativePath)) return;
+    const target = this.#resolveTarget(collectionPath, absolutePath);
+    if (!target) return;
     const stat = fs.statSync(absolutePath, { bigint: true });
-    this.stage(root, {
+    this.stage(target.root, {
       op: 'add',
-      relativePath,
+      relativePath: target.relativePath,
       mtime: stat.mtimeNs,
       hash: hashFile(absolutePath),
       raw: fs.readFileSync(absolutePath, 'utf8'),
@@ -181,21 +131,24 @@ class FileIndex {
   }
 
   unstagePath(collectionPath, absolutePath) {
+    const target = this.#resolveTarget(collectionPath, absolutePath);
+    if (!target) return;
+    this.stage(target.root, { op: 'remove', relativePath: target.relativePath });
+  }
+
+  #resolveTarget(collectionPath, absolutePath) {
     const root = normalize(collectionPath);
-    const relativePath = path.relative(root, normalize(absolutePath));
-    if (relativePath.startsWith('..') || path.isAbsolute(relativePath)) return;
-    this.stage(root, { op: 'remove', relativePath });
+    const relativePath = path.normalize(path.relative(root, normalize(absolutePath)));
+    if (relativePath.startsWith('..') || path.isAbsolute(relativePath)) return null;
+    return { root, relativePath };
   }
 
   transaction(callback) {
-    return this.#db.transaction(callback);
+    return this.#db._transaction(callback);
   }
 
   #loadStored(collectionPath) {
-    const rows = this.#db.all(
-      'SELECT relative_path AS relativePath, id, mtime, hash FROM file_index_entries WHERE collection_path = ?',
-      collectionPath
-    );
+    const rows = this.#statements.execute('file_index_stored', { collection_path: collectionPath });
     const map = new Map();
     for (const row of rows) {
       map.set(row.relativePath, row);

packages/bruno-electron/src/services/mount/manager.js

@@ -138,23 +138,7 @@ class MountManager {
       Array.from(this.#mounts.keys()).map((uid) => this.unmount(uid).catch(() => {}))
     );
     await destroyPool().catch(() => {});
-    if (this.#index) {
-      this.#index.close();
-      this.#index = null;
-    }
-  }
-
-  getCacheSize() {
-    try {
-      return fs.statSync(this.#getIndex().dbPath).size;
-    } catch (err) {
-      if (err && err.code === 'ENOENT') return 0;
-      throw err;
-    }
-  }
-
-  clearCache() {
-    this.#getIndex().clear();
+    this.#index = null;
   }
 
   clearCollectionIndex(collectionPath) {
@@ -227,7 +211,7 @@ class MountManager {
   }
 
   #getIndex() {
-    if (!this.#index) this.#index = new FileIndex({});
+    if (!this.#index) this.#index = new FileIndex();
     return this.#index;
   }
 }