feat: fine grained storage persister (#5125)

* feat: fine grained storage persister

* feat: persister option

* fix: invalidate query just after restoring

* fix: types, prettier, added comments

* fix: move pieces

* fix: prettier

* fix: add error handling

* fix: drop client dependency, fix type inference

* fix: formatting

* fix: pass query as param

* chore: some cleanup

* test: add tests

* test: fix import order

* test: add react test for fine grained persister

* feat: queryFilter predicate

* fix: test import

* docs: add docs and migration guide entry

* docs and filters

---------

Co-authored-by: Dominik Dorfmeister <[email protected]>

Author: DamianOsipiuk

docs/config.json

@@ -331,6 +331,10 @@
             {
               "label": "broadcastQueryClient (Experimental)",
               "to": "react/plugins/broadcastQueryClient"
+            },
+            {
+              "label": "createPersister (Experimental)",
+              "to": "react/plugins/createPersister"
             }
           ]
         },

docs/react/guides/migrating-to-v5.md

@@ -455,6 +455,10 @@ Infinite Queries can be prefetched like regular Queries. Per default, only the f
 
 See the [useQueries docs](../reference/useQueries#combine) for more details.
 
+### Experimental `fine grained storage persister`
+
+See the [experimental_createPersister docs](../plugins/createPersister) for more details.
+
 [//]: # 'FrameworkSpecificNewFeatures'
 
 ### Typesafe way to create Query Options

docs/react/plugins/createPersister.md

@@ -0,0 +1,120 @@
+---
+id: createPersister
+title: experimental_createPersister
+---
+
+## Installation
+
+This utility comes as a separate package and is available under the `'@tanstack/query-persist-client-core'` import.
+
+```bash
+npm install @tanstack/query-persist-client-core
+```
+
+or
+
+```bash
+pnpm add @tanstack/query-persist-client-core
+```
+
+or
+
+```bash
+yarn add @tanstack/query-persist-client-core
+```
+
+> Note: This util is also included in the `@tanstack/react-query-persist-client` package, so you do not need to install it separately if you are using that package.
+
+## Usage
+
+- Import the `experimental_createPersister` function
+- Create a new `experimental_createPersister`
+  - you can pass any `storage` to it that adheres to the `AsyncStorage` or `Storage` interface - the example below uses the async-storage from React Native.
+- Pass that `persister` as an option to your Query. This can be done either by passing it to the `defaultOptions` of the `QueryClient` or to any `useQuery` hook instance.
+  - If you pass this `persister` as `defaultOptions`, all queries will be persisted to the provided `storage`. You can additionally narrow this down by passing `filters`. In contrast to the `persistClient` plugin, this will not persist the whole query client as a single item, but each query separately. As a key, the query hash is used.
+  - If you provide this `persister` to a single `useQuery` hook, only this Query will be persisted.
+
+This way, you do not need to store whole `QueryClient`, but choose what is worth to be persisted in your application. Each query is lazily restored (when the Query is first used) and persisted (after each run of the `queryFn`), so it does not need to be throttled. `staleTime` is also respected after restoring the Query, so if data is considered `stale`, it will be refetched immediately after restoring. If data is `fresh`, the `queryFn` will not run.
+
+```tsx
+import AsyncStorage from '@react-native-async-storage/async-storage'
+import { QueryClient } from '@tanstack/react-query'
+import { experimental_createPersister } from '@tanstack/query-persist-client-core'
+
+const queryClient = new QueryClient({
+  defaultOptions: {
+    queries: {
+      gcTime: 1000 * 60 * 60 * 24, // 24 hours
+      persister: experimental_createPersister({
+        storage: AsyncStorage,
+      }),
+    },
+  },
+})
+```
+
+## API
+
+### `experimental_createPersister`
+
+```tsx
+experimental_createPersister(options: StoragePersisterOptions)
+```
+
+#### `Options`
+
+```tsx
+export interface StoragePersisterOptions {
+  /** The storage client used for setting and retrieving items from cache.
+   * For SSR pass in `undefined`.
+   */
+  storage: AsyncStorage | Storage | undefined | null
+  /**
+   * How to serialize the data to storage.
+   * @default `JSON.stringify`
+   */
+  serialize?: (persistedQuery: PersistedQuery) => string
+  /**
+   * How to deserialize the data from storage.
+   * @default `JSON.parse`
+   */
+  deserialize?: (cachedString: string) => PersistedQuery
+  /**
+   * A unique string that can be used to forcefully invalidate existing caches,
+   * if they do not share the same buster string
+   */
+  buster?: string
+  /**
+   * The max-allowed age of the cache in milliseconds.
+   * If a persisted cache is found that is older than this
+   * time, it will be discarded
+   */
+  maxAge?: number
+  /**
+   * Prefix to be used for storage key.
+   * Storage key is a combination of prefix and query hash in a form of `prefix-queryHash`.
+   */
+  prefix?: string
+  /**
+   * Filters to narrow down which Queries should be persisted.
+   */
+  filters?: QueryFilters
+}
+
+interface AsyncStorage {
+  getItem: (key: string) => Promise<string | undefined | null>
+  setItem: (key: string, value: string) => Promise<unknown>
+  removeItem: (key: string) => Promise<void>
+}
+```
+
+The default options are:
+
+```tsx
+{
+  prefix = 'tanstack-query',
+  maxAge = 1000 * 60 * 60 * 24,
+  serialize = JSON.stringify,
+  deserialize = JSON.parse,
+}
+```

examples/vue/persister/package.json

@@ -8,10 +8,12 @@
     "serve": "vite preview"
   },
   "dependencies": {
+    "@tanstack/query-core": "^5.0.0-beta.0",
     "@tanstack/query-persist-client-core": "^5.0.0-beta.0",
     "@tanstack/query-sync-storage-persister": "^5.0.0-beta.0",
     "@tanstack/vue-query": "^5.0.0-beta.0",
-    "vue": "^3.3.0"
+    "vue": "^3.3.0",
+    "idb-keyval": "^6.2.0"
   },
   "devDependencies": {
     "@vitejs/plugin-vue": "^4.2.3",

examples/vue/persister/src/Post.vue

@@ -1,8 +1,10 @@
 <script lang="ts">
+import { get, set, del } from 'idb-keyval'
 import { defineComponent } from 'vue'
 import { useQuery } from '@tanstack/vue-query'
 
 import { Post } from './types'
+import { experimental_createPersister } from '@tanstack/query-persist-client-core'
 
 const fetcher = async (id: number): Promise<Post> =>
   await fetch(`https://jsonplaceholder.typicode.com/posts/${id}`).then(
@@ -20,8 +22,15 @@ export default defineComponent({
   emits: ['setPostId'],
   setup(props) {
     const { isPending, isError, isFetching, data, error } = useQuery({
-      queryKey: ['post', props.postId],
+      queryKey: ['post', props.postId] as const,
       queryFn: () => fetcher(props.postId),
+      persister: experimental_createPersister({
+        storage: {
+          getItem: (key: string) => get(key),
+          setItem: (key: string, value: string) => set(key, value),
+          removeItem: (key: string) => del(key),
+        },
+      }),
     })
 
     return { isPending, isError, isFetching, data, error }

examples/vue/persister/src/Posts.vue

@@ -1,12 +1,16 @@
 <script lang="ts">
 import { defineComponent } from 'vue'
 import { useQuery } from '@tanstack/vue-query'
+import { experimental_createPersister } from '@tanstack/query-persist-client-core'
 
 import { Post } from './types'
 
 const fetcher = async (): Promise<Post[]> =>
-  await fetch('https://jsonplaceholder.typicode.com/posts').then((response) =>
-    response.json(),
+  await fetch('https://jsonplaceholder.typicode.com/posts').then(
+    (response) =>
+      new Promise((resolve) =>
+        setTimeout(() => resolve(response.json()), 1000),
+      ),
   )
 
 export default defineComponent({
@@ -19,18 +23,39 @@ export default defineComponent({
   },
   emits: ['setPostId'],
   setup() {
-    const { isPending, isError, isFetching, data, error, refetch } = useQuery({
-      queryKey: ['posts'],
-      queryFn: fetcher,
+    const {
+      isPending,
+      isError,
+      isFetching,
+      isRefetching,
+      data,
+      error,
+      refetch,
+    } = useQuery({
+      queryKey: ['posts'] as const,
+      queryFn: () => fetcher(),
+      persister: experimental_createPersister({
+        storage: localStorage,
+      }),
+      staleTime: 5000,
     })
 
-    return { isPending, isError, isFetching, data, error, refetch }
+    return {
+      isPending,
+      isRefetching,
+      isError,
+      isFetching,
+      data,
+      error,
+      refetch,
+    }
   },
 })
 </script>
 
 <template>
   <h1>Posts</h1>
+  <div v-if="isRefetching">Refetching...</div>
   <div v-if="isPending">Loading...</div>
   <div v-else-if="isError">An error has occurred: {{ error }}</div>
   <div v-else-if="data">

examples/vue/persister/src/main.ts

@@ -1,7 +1,5 @@
 import { createApp } from 'vue'
 import { VueQueryPlugin } from '@tanstack/vue-query'
-import { persistQueryClient } from '@tanstack/query-persist-client-core'
-import { createSyncStoragePersister } from '@tanstack/query-sync-storage-persister'
 import type { VueQueryPluginOptions } from '@tanstack/vue-query'
 
 import App from './App.vue'
@@ -11,16 +9,10 @@ const vueQueryOptions: VueQueryPluginOptions = {
     defaultOptions: {
       queries: {
         gcTime: 1000 * 60 * 60 * 24,
-        staleTime: 1000 * 60 * 60 * 24,
+        // staleTime: 1000 * 10,
       },
     },
   },
-  clientPersister: (queryClient) => {
-    return persistQueryClient({
-      queryClient,
-      persister: createSyncStoragePersister({ storage: localStorage }),
-    })
-  },
 }
 
 createApp(App).use(VueQueryPlugin, vueQueryOptions).mount('#app')

packages/query-async-storage-persister/src/index.ts

@@ -1,16 +1,11 @@
 import { asyncThrottle } from './asyncThrottle'
 import type {
+  AsyncStorage,
   PersistedClient,
   Persister,
   Promisable,
 } from '@tanstack/query-persist-client-core'
 
-interface AsyncStorage {
-  getItem: (key: string) => Promise<string | null>
-  setItem: (key: string, value: string) => Promise<unknown>
-  removeItem: (key: string) => Promise<void>
-}
-
 export type AsyncPersistRetryer = (props: {
   persistedClient: PersistedClient
   error: Error

packages/query-core/src/infiniteQueryBehavior.ts

@@ -11,8 +11,8 @@ export function infiniteQueryBehavior<TQueryFnData, TError, TData, TPageParam>(
   pages?: number,
 ): QueryBehavior<TQueryFnData, TError, InfiniteData<TData, TPageParam>> {
   return {
-    onFetch: (context) => {
-      context.fetchFn = async () => {
+    onFetch: (context, query) => {
+      const fetchFn = async () => {
         const options = context.options as InfiniteQueryPageParamsOptions<TData>
         const direction = context.fetchOptions?.meta?.fetchMore?.direction
         const oldPages = context.state.data?.pages || []
@@ -114,6 +114,21 @@ export function infiniteQueryBehavior<TQueryFnData, TError, TData, TPageParam>(
 
         return result
       }
+      if (context.options.persister) {
+        context.fetchFn = () => {
+          return context.options.persister?.(
+            fetchFn as any,
+            {
+              queryKey: context.queryKey,
+              meta: context.options.meta,
+              signal: context.signal,
+            },
+            query,
+          )
+        }
+      } else {
+        context.fetchFn = fetchFn
+      }
     },
   }
 }

packages/query-core/src/query.ts

@@ -71,6 +71,7 @@ export interface QueryBehavior<
 > {
   onFetch: (
     context: FetchContext<TQueryFnData, TError, TData, TQueryKey>,
+    query: Query,
   ) => void
 }
 
@@ -392,6 +393,14 @@ export class Query<
         )
       }
       this.#abortSignalConsumed = false
+      if (this.options.persister) {
+        return this.options.persister(
+          this.options.queryFn,
+          queryFnContext as QueryFunctionContext<TQueryKey>,
+          this as unknown as Query,
+        )
+      }
+
       return this.options.queryFn(
         queryFnContext as QueryFunctionContext<TQueryKey>,
       )
@@ -413,6 +422,7 @@ export class Query<
 
     this.options.behavior?.onFetch(
       context as FetchContext<TQueryFnData, TError, TData, TQueryKey>,
+      this as unknown as Query,
     )
 
     // Store state in case the current fetch needs to be reverted