chore: add new benchmarks

Julien-R44 · Julien-R44 · commit efd760b4f37f · 2025-02-02T02:49:09.000+01:00
diff --git a/docs/content/docs/options.md b/docs/content/docs/options.md
@@ -75,15 +75,15 @@ Note that event if errors are suppressed and not thrown, they will still be logg
 
 ### `grace`
 
-Default `undefined`
+Default `false`
 
 Levels: `global`, `store`, `operation`
 
-A duration to define the [grace period](./grace_periods.md).
+A duration to define the [grace period](./grace_periods.md). Also can be `false` to disable grace periods.
 
 ### `graceBackoff`
 
-Default: `undefined`
+Default: `10s`
 
 Levels: `global`, `store`, `operation`
 
diff --git a/examples/hono/package.json b/examples/hono/package.json
@@ -10,7 +10,7 @@
     "hono": "^4.6.20"
   },
   "devDependencies": {
-    "@types/node": "^22.12.0",
+    "@types/node": "^22.13.0",
     "tsx": "^4.19.2"
   }
 }
diff --git a/package.json b/package.json
@@ -24,7 +24,7 @@
     "@japa/runner": "^4.2.0",
     "@julr/tooling-configs": "^3.1.0",
     "@swc/core": "^1.10.12",
-    "@types/node": "^22.12.0",
+    "@types/node": "^22.13.0",
     "c8": "^10.1.3",
     "copyfiles": "^2.4.1",
     "cross-env": "^7.0.3",
diff --git a/packages/bentocache/benchmarks/README.md b/packages/bentocache/benchmarks/README.md
@@ -0,0 +1,34 @@
+# Benchmarks
+
+> [!IMPORTANT]  
+> The benchmarks are not meant to be a definitive proof of which library is the best. They are mainly here to see if we make any performance regressions. And also for fun. Do not take them too seriously.
+
+At the time of writing, every librairies seems on pair with each other when using a single tier cache. The real differences come when using a two-tier cache, only CacheManager and Bentocache support this feature. 
+
+- `mtier_get_key` : Just get a key from the cache stack.
+
+```
+┌─────────┬────────────────┬──────────────────┬──────────────────┬────────────────────────┬────────────────────────┬─────────┐
+│ (index) │ Task name      │ Latency avg (ns) │ Latency med (ns) │ Throughput avg (ops/s) │ Throughput med (ops/s) │ Samples │
+├─────────┼────────────────┼──────────────────┼──────────────────┼────────────────────────┼────────────────────────┼─────────┤
+│ 0       │ 'BentoCache'   │ '854.21 ± 0.75%' │ '776.00 ± 10.00' │ '1252083 ± 0.02%'      │ '1288660 ± 16823'      │ 1170677 │
+│ 1       │ 'CacheManager' │ '1917.8 ± 2.90%' │ '1770.0 ± 27.00' │ '555669 ± 0.02%'       │ '564972 ± 8752'        │ 521437  │
+└─────────┴────────────────┴──────────────────┴──────────────────┴────────────────────────┴────────────────────────┴─────────┘
+```
+
+- `mtier_get_or_set` : Get a key from the cache stack, if it does not exist, set it.
+
+```
+┌─────────┬────────────────┬───────────────────┬──────────────────┬────────────────────────┬────────────────────────┬─────────┐
+│ (index) │ Task name      │ Latency avg (ns)  │ Latency med (ns) │ Throughput avg (ops/s) │ Throughput med (ops/s) │ Samples │
+├─────────┼────────────────┼───────────────────┼──────────────────┼────────────────────────┼────────────────────────┼─────────┤
+│ 0       │ 'BentoCache'   │ '8766.7 ± 98.44%' │ '967.00 ± 36.00' │ '987455 ± 0.07%'       │ '1034126 ± 39988'      │ 157942  │
+│ 1       │ 'CacheManager' │ '14364 ± 97.77%'  │ '1743.0 ± 36.00' │ '549195 ± 0.08%'       │ '573723 ± 11610'       │ 97211   │
+└─────────┴────────────────┴───────────────────┴──────────────────┴────────────────────────┴────────────────────────┴─────────┘
+```
+
+Please if you see any mistake in the benchmarks, open an issue and I will correct it.
+
+- Run: 02/02/2025
+- Node 22.11.0
+- WSL2, Ubuntu 24.04, 32G RAM, 8 cores
diff --git a/packages/bentocache/benchmarks/mtier_get_key.ts b/packages/bentocache/benchmarks/mtier_get_key.ts
@@ -4,11 +4,10 @@
 import 'dotenv/config'
 
 import Keyv from 'keyv'
-import { Redis } from 'ioredis'
 import { Bench } from 'tinybench'
-import KeyvTiered from '@keyv/tiered'
-import { multiCaching, caching } from 'cache-manager'
-import { redisStore } from 'cache-manager-ioredis-yet'
+import KeyvRedis from '@keyv/redis'
+import { createCache } from 'cache-manager'
+import { CacheableMemory } from 'cacheable'
 
 import { BentoCache } from '../index.js'
 import { bentostore } from '../src/bento_store.js'
@@ -21,47 +20,33 @@ const bench = new Bench()
 const bentocache = new BentoCache({
   default: 'tiered',
   stores: {
-    redis: bentostore().useL2Layer(redisDriver({ connection: REDIS_CREDENTIALS })),
     tiered: bentostore()
       .useL1Layer(memoryDriver({}))
       .useL2Layer(redisDriver({ connection: REDIS_CREDENTIALS })),
   },
 })
 
-const bentocacheTiered = bentocache.use('tiered')
-const keyvRedis = new Keyv('redis://localhost:6379')
-const keyv = new KeyvTiered({ remote: keyvRedis as any, local: new Keyv() })
+const bento = bentocache.use('tiered')
 
-const cacheManagerMemory = await caching('memory')
-const cacheManagerRedis = await caching(await redisStore({ host: 'localhost', port: 6379 }))
-const multiCache = multiCaching([cacheManagerMemory, cacheManagerRedis])
-
-await keyv.set('key', 'value')
-await bentocacheTiered.set({ key: 'key', value: 'value' })
-await multiCache.set('key', 'value')
+const cacheManager = createCache({
+  stores: [
+    new Keyv({ store: new CacheableMemory() }),
+    new Keyv({ store: new KeyvRedis('redis://localhost:6379') }),
+  ],
+})
 
-const ioredis = new Redis()
+await bento.set({ key: 'bento:key', value: 'value', ttl: '10s' })
+await cacheManager.set('cm:key', 'value', 10_000)
 
-/**
- * Simple get benchmark
- */
 bench
   .add('BentoCache', async () => {
-    await bentocacheTiered.get({ key: 'key' })
-  })
-  .add('Keyv', async () => {
-    await keyv.get('key')
+    await bento.get({ key: 'bento:key' })
   })
   .add('CacheManager', async () => {
-    await multiCache.get('key')
+    await cacheManager.get('cm:key')
   })
 
 await bench.run()
 console.table(bench.table())
 
-await Promise.all([
-  bentocache.disconnectAll(),
-  ioredis.quit(),
-  cacheManagerRedis.store.client.disconnect(),
-  keyvRedis.disconnect(),
-])
+await Promise.all([bentocache.disconnectAll(), cacheManager.disconnect()])
diff --git a/packages/bentocache/benchmarks/mtier_get_or_set.ts b/packages/bentocache/benchmarks/mtier_get_or_set.ts
@@ -0,0 +1,64 @@
+/**
+ * Benchmark a single get operation on a tiered store ( memory + redis )
+ */
+import 'dotenv/config'
+
+import Keyv from 'keyv'
+import { Bench } from 'tinybench'
+import KeyvRedis from '@keyv/redis'
+import { createCache } from 'cache-manager'
+import { CacheableMemory } from 'cacheable'
+import { setTimeout } from 'node:timers/promises'
+
+import { BentoCache } from '../index.js'
+import { bentostore } from '../src/bento_store.js'
+import { redisDriver } from '../src/drivers/redis.js'
+import { memoryDriver } from '../src/drivers/memory.js'
+import { REDIS_CREDENTIALS } from '../tests/helpers/index.js'
+
+const bench = new Bench({})
+
+const bentocache = new BentoCache({
+  default: 'tiered',
+  stores: {
+    tiered: bentostore()
+      .useL1Layer(memoryDriver({}))
+      .useL2Layer(redisDriver({ connection: REDIS_CREDENTIALS })),
+  },
+})
+
+const bento = bentocache.use('tiered')
+
+const cacheManager = createCache({
+  stores: [
+    new Keyv({ store: new CacheableMemory() }),
+    new Keyv({ store: new KeyvRedis('redis://localhost:6379') }),
+  ],
+})
+
+const getFromDb = async () => {
+  await setTimeout(400)
+  return 'value'
+}
+
+bench
+  .add('BentoCache', async () => {
+    return await bento.getOrSet({
+      key: 'bento:key',
+      factory: getFromDb,
+      ttl: 100,
+    })
+  })
+  .add('CacheManager', async () => {
+    const result = await cacheManager.get('cm:key')
+    if (result === null) {
+      await cacheManager.set('cm:key', await getFromDb(), 100)
+    }
+
+    return result ?? 'value'
+  })
+
+await bench.run()
+console.table(bench.table())
+
+await Promise.all([bentocache.disconnectAll(), cacheManager.disconnect()])
diff --git a/packages/bentocache/benchmarks/mtier_set_key.ts b/packages/bentocache/benchmarks/mtier_set_key.ts
@@ -1,14 +1,13 @@
 /**
- * Benchmarking a set key operation with a tiered cache ( memory + redis )
+ * Benchmark a single set operation on a tiered store ( memory + redis )
  */
 import 'dotenv/config'
 
 import Keyv from 'keyv'
-import { Redis } from 'ioredis'
 import { Bench } from 'tinybench'
-import KeyvTiered from '@keyv/tiered'
-import { multiCaching, caching } from 'cache-manager'
-import { redisStore } from 'cache-manager-ioredis-yet'
+import KeyvRedis from '@keyv/redis'
+import { createCache } from 'cache-manager'
+import { CacheableMemory } from 'cacheable'
 
 import { BentoCache } from '../index.js'
 import { bentostore } from '../src/bento_store.js'
@@ -21,44 +20,38 @@ const bench = new Bench()
 const bentocache = new BentoCache({
   default: 'tiered',
   stores: {
-    redis: bentostore().useL2Layer(redisDriver({ connection: REDIS_CREDENTIALS })),
     tiered: bentostore()
       .useL1Layer(memoryDriver({}))
       .useL2Layer(redisDriver({ connection: REDIS_CREDENTIALS })),
   },
 })
 
-const bentocacheTiered = bentocache.use('tiered')
-const keyvRedis = new Keyv('redis://localhost:6379')
-const keyv = new KeyvTiered({ remote: keyvRedis as any, local: new Keyv() })
+const bento = bentocache.use('tiered')
 
-const cacheManagerMemory = await caching('memory')
-const cacheManagerRedis = await caching(await redisStore({ host: 'localhost', port: 6379 }))
-const multiCache = multiCaching([cacheManagerMemory, cacheManagerRedis])
+const cacheManager = createCache({
+  stores: [
+    new Keyv({
+      store: new CacheableMemory({ ttl: 60_000, lruSize: 5000 }),
+    }),
 
-await keyv.set('key', 'value')
-await bentocacheTiered.set({ key: 'key', value: 'value' })
-await multiCache.set('key', 'value')
+    new Keyv({
+      store: new KeyvRedis('redis://localhost:6379'),
+    }),
+  ],
+})
 
-const ioredis = new Redis()
+await bento.set({ key: 'key', value: 'value' })
+await cacheManager.set('key', 'value')
 
 bench
   .add('BentoCache', async () => {
-    await bentocacheTiered.set({ key: 'key', value: 'value' })
-  })
-  .add('Keyv', async () => {
-    await keyv.set('key', 'value')
+    await bento.set({ key: 'key', value: 10 })
   })
   .add('CacheManager', async () => {
-    await multiCache.set('key', 'value')
+    await cacheManager.set('key', 10)
   })
 
 await bench.run()
 console.table(bench.table())
 
-await Promise.all([
-  bentocache.disconnectAll(),
-  ioredis.quit(),
-  cacheManagerRedis.store.client.disconnect(),
-  keyvRedis.disconnect(),
-])
+await Promise.all([bentocache.disconnectAll(), cacheManager.disconnect()])
diff --git a/packages/bentocache/benchmarks/onetier_get_key.ts b/packages/bentocache/benchmarks/onetier_get_key.ts
@@ -6,8 +6,8 @@ import 'dotenv/config'
 import Keyv from 'keyv'
 import { Redis } from 'ioredis'
 import { Bench } from 'tinybench'
-import { caching } from 'cache-manager'
-import { redisStore } from 'cache-manager-ioredis-yet'
+import KeyvRedis from '@keyv/redis'
+import { createCache } from 'cache-manager'
 
 import { BentoCache } from '../index.js'
 import { bentostore } from '../src/bento_store.js'
@@ -23,9 +23,10 @@ const bentocache = new BentoCache({
   },
 })
 
-const keyv = new Keyv('redis://localhost:6379')
-
-const cacheManager = await caching(await redisStore({ host: 'localhost', port: 6379 }))
+const keyv = new Keyv(new KeyvRedis('redis://localhost:6379'))
+const cacheManager = await createCache({
+  stores: [new Keyv(new KeyvRedis('redis://localhost:6379'))],
+})
 
 await keyv.set('key', 'value')
 await bentocache.set({ key: 'key', value: 'value' })
@@ -53,6 +54,6 @@ console.table(bench.table())
 await Promise.all([
   bentocache.disconnectAll(),
   ioredis.quit(),
-  cacheManager.store.client.disconnect(),
+  cacheManager.disconnect(),
   keyv.disconnect(),
 ])
diff --git a/packages/bentocache/benchmarks/onetier_set_key.ts b/packages/bentocache/benchmarks/onetier_set_key.ts
@@ -1,40 +1,48 @@
 /**
- * Benchmark a single set operation on a memory store
+ * Benchmark a single set operation on a redis store
  */
 import 'dotenv/config'
 
 import Keyv from 'keyv'
 import { Bench } from 'tinybench'
-import { caching } from 'cache-manager'
+import KeyvRedis from '@keyv/redis'
+import { createCache } from 'cache-manager'
 
 import { BentoCache } from '../index.js'
 import { bentostore } from '../src/bento_store.js'
-import { memoryDriver } from '../src/drivers/memory.js'
+import { redisDriver } from '../src/drivers/redis.js'
+import { REDIS_CREDENTIALS } from '../tests/helpers/index.js'
 
 const bench = new Bench()
 
 const bentocache = new BentoCache({
-  default: 'memory',
-  stores: { memory: bentostore().useL1Layer(memoryDriver()) },
-}).use('memory')
+  default: 'redis',
+  stores: {
+    redis: bentostore().useL2Layer(redisDriver({ connection: REDIS_CREDENTIALS })),
+  },
+})
 
-const keyv = new Keyv()
-const cacheManager = await caching('memory')
+const keyv = new Keyv(new KeyvRedis('redis://localhost:6379'))
+const cacheManager = await createCache({
+  stores: [new Keyv(new KeyvRedis('redis://localhost:6379'))],
+})
 
 await keyv.set('key', 'value')
 await bentocache.set({ key: 'key', value: 'value' })
 await cacheManager.set('key', 'value')
 
 bench
   .add('BentoCache', async () => {
-    await bentocache.get({ key: 'key' })
+    await bentocache.set({ key: 'key', value: 'foo' })
   })
   .add('Keyv', async () => {
-    await keyv.get('key')
+    await keyv.set('key', 'foo')
   })
   .add('CacheManager', async () => {
-    await cacheManager.get('key')
+    await cacheManager.set('key', 'foo')
   })
 
 await bench.run()
 console.table(bench.table())
+
+await Promise.all([bentocache.disconnect(), cacheManager.disconnect(), keyv.disconnect()])
diff --git a/packages/bentocache/package.json b/packages/bentocache/package.json
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml

Original file line number	Diff line number	Diff line change
`@@ -10,7 +10,7 @@`
`10`	`10`	`"hono": "^4.6.20"`
`11`	`11`	`},`
`12`	`12`	`"devDependencies": {`
`13`		`- "@types/node": "^22.12.0",`
	`13`	`+ "@types/node": "^22.13.0",`
`14`	`14`	`"tsx": "^4.19.2"`
`15`	`15`	`}`
`16`	`16`	`}`