feat!: on demand rsc (#8364)

Currently, Payload renders all custom components on initial compile of
the admin panel. This is problematic for two key reasons:
1. Custom components do not receive contextual data, i.e. fields do not
receive their field data, edit views do not receive their document data,
etc.
2. Components are unnecessarily rendered before they are used

This was initially required to support React Server Components within
the Payload Admin Panel for two key reasons:
1. Fields can be dynamically rendered within arrays, blocks, etc.
2. Documents can be recursively rendered within a "drawer" UI, i.e.
relationship fields
3. Payload supports server/client component composition 

In order to achieve this, components need to be rendered on the server
and passed as "slots" to the client. Currently, the pattern for this is
to render custom server components in the "client config". Then when a
view or field is needed to be rendered, we first check the client config
for a "pre-rendered" component, otherwise render our client-side
fallback component.

But for the reasons listed above, this pattern doesn't exactly make
custom server components very useful within the Payload Admin Panel,
which is where this PR comes in. Now, instead of pre-rendering all
components on initial compile, we're able to render custom components
_on demand_, only as they are needed.

To achieve this, we've established [this
pattern](#8481) of React
Server Functions in the Payload Admin Panel. With Server Functions, we
can iterate the Payload Config and return JSX through React's
`text/x-component` content-type. This means we're able to pass
contextual props to custom components, such as data for fields and
views.

## Breaking Changes

1. Add the following to your root layout file, typically located at
`(app)/(payload)/layout.tsx`:

    ```diff
    /* THIS FILE WAS GENERATED AUTOMATICALLY BY PAYLOAD. */
    /* DO NOT MODIFY IT BECAUSE IT COULD BE REWRITTEN AT ANY TIME. */
    + import type { ServerFunctionClient } from 'payload'

    import config from '@payload-config'
    import { RootLayout } from '@payloadcms/next/layouts'
    import { handleServerFunctions } from '@payloadcms/next/utilities'
    import React from 'react'

    import { importMap } from './admin/importMap.js'
    import './custom.scss'

    type Args = {
      children: React.ReactNode
    }

+ const serverFunctions: ServerFunctionClient = async function (args) {
    +  'use server'
    +  return handleServerFunctions({
    +    ...args,
    +    config,
    +    importMap,
    +  })
    + }

    const Layout = ({ children }: Args) => (
      <RootLayout
        config={config}
        importMap={importMap}
    +  serverFunctions={serverFunctions}
      >
        {children}
      </RootLayout>
    )

    export default Layout
    ```

2. If you were previously posting to the `/api/form-state` endpoint, it
no longer exists. Instead, you'll need to invoke the `form-state` Server
Function, which can be done through the _new_ `getFormState` utility:

    ```diff
    - import { getFormState } from '@payloadcms/ui'
    - const { state } = await getFormState({
    -   apiRoute: '',
    -   body: {
    -     // ...
    -   },
    -   serverURL: ''
    - })

    + const { getFormState } = useServerFunctions()
    +
    + const { state } = await getFormState({
    +   // ...
    + })
    ```

## Breaking Changes

```diff
- useFieldProps()
- useCellProps()
```

More details coming soon.

---------

Co-authored-by: Alessio Gravili <[email protected]>
Co-authored-by: Jarrod Flesch <[email protected]>
Co-authored-by: James <[email protected]>

Author: 4 people

app/(payload)/layout.tsx

@@ -1,8 +1,10 @@
 /* THIS FILE WAS GENERATED AUTOMATICALLY BY PAYLOAD. */
 /* DO NOT MODIFY IT BECAUSE IT COULD BE REWRITTEN AT ANY TIME. */
-import configPromise from '@payload-config'
-import { RootLayout } from '@payloadcms/next/layouts'
 // import '@payloadcms/ui/styles.css' // Uncomment this line if `@payloadcms/ui` in `tsconfig.json` points to `/ui/dist` instead of `/ui/src`
+import type { ServerFunctionClient } from 'payload'
+
+import config from '@payload-config'
+import { handleServerFunctions, RootLayout } from '@payloadcms/next/layouts'
 import React from 'react'
 
 import { importMap } from './admin/importMap.js'
@@ -12,8 +14,17 @@ type Args = {
   children: React.ReactNode
 }
 
+const serverFunction: ServerFunctionClient = async function (args) {
+  'use server'
+  return handleServerFunctions({
+    ...args,
+    config,
+    importMap,
+  })
+}
+
 const Layout = ({ children }: Args) => (
-  <RootLayout config={configPromise} importMap={importMap}>
+  <RootLayout config={config} importMap={importMap} serverFunction={serverFunction}>
     {children}
   </RootLayout>
 )

docs/admin/fields.mdx

@@ -228,7 +228,6 @@ The following additional properties are also provided to the `field` prop:
 
 | Property         | Description                                                                                                                                                                                                                      |
 | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **`_isPresentational`** | A boolean indicating that the field is purely visual and does not directly affect data or change data shape, i.e. the [UI Field](../fields/ui). |
 | **`_path`** | A string representing the direct, dynamic path to the field at runtime, i.e. `myGroup.myArray[0].myField`. |
 | **`_schemaPath`** | A string representing the direct, static path to the [Field Config](../fields/overview), i.e. `myGroup.myArray.myField` |
 

docs/lexical/converters.mdx

@@ -370,7 +370,12 @@ const yourEditorState: SerializedEditorState // <= your current editor state her
 
 // Import editor state into your headless editor
 try {
-  headlessEditor.setEditorState(headlessEditor.parseEditorState(yourEditorState)) // This should commit the editor state immediately
+  headlessEditor.update(
+    () => {
+      headlessEditor.setEditorState(headlessEditor.parseEditorState(yourEditorState))
+    },
+    { discrete: true },  // This should commit the editor state immediately
+  )
 } catch (e) {
   logger.error({ err: e }, 'ERROR parsing editor state')
 }
@@ -382,8 +387,6 @@ headlessEditor.getEditorState().read(() => {
 })
 ```
 
-The `.setEditorState()` function immediately updates your editor state. Thus, there's no need for the `discrete: true` flag when reading the state afterward.
-
 ## Lexical => Plain Text
 
 Export content from the Lexical editor into plain text using these steps:
@@ -401,8 +404,13 @@ const yourEditorState: SerializedEditorState // <= your current editor state her
 
 // Import editor state into your headless editor
 try {
-  headlessEditor.setEditorState(headlessEditor.parseEditorState(yourEditorState)) // This should commit the editor state immediately
-} catch (e) {
+    headlessEditor.update(
+    () => {
+      headlessEditor.setEditorState(headlessEditor.parseEditorState(yourEditorState))
+    },
+    { discrete: true },  // This should commit the editor state immediately
+  )
+  } catch (e) {
   logger.error({ err: e }, 'ERROR parsing editor state')
 }
 

examples/custom-components/src/app/(payload)/layout.tsx

@@ -1,6 +1,6 @@
 /* THIS FILE WAS GENERATED AUTOMATICALLY BY PAYLOAD. */
 /* DO NOT MODIFY IT BECAUSE IT COULD BE REWRITTEN AT ANY TIME. */
-import configPromise from '@payload-config'
+import config from '@payload-config'
 import '@payloadcms/next/css'
 import { RootLayout } from '@payloadcms/next/layouts'
 import React from 'react'
@@ -13,7 +13,7 @@ type Args = {
 }
 
 const Layout = ({ children }: Args) => (
-  <RootLayout config={configPromise} importMap={importMap}>
+  <RootLayout config={config} importMap={importMap}>
     {children}
   </RootLayout>
 )

examples/hierarchy/src/app/(payload)/layout.tsx

@@ -1,8 +1,8 @@
 /* THIS FILE WAS GENERATED AUTOMATICALLY BY PAYLOAD. */
+/* DO NOT MODIFY IT BECAUSE IT COULD BE REWRITTEN AT ANY TIME. */
 import configPromise from '@payload-config'
 import '@payloadcms/next/css'
 import { RootLayout } from '@payloadcms/next/layouts'
-/* DO NOT MODIFY IT BECAUSE IT COULD BE REWRITTEN AT ANY TIME. */
 import React from 'react'
 
 import './custom.scss'

examples/multi-tenant/src/app/(payload)/layout.tsx

@@ -1,6 +1,6 @@
 /* THIS FILE WAS GENERATED AUTOMATICALLY BY PAYLOAD. */
 /* DO NOT MODIFY IT BECAUSE IT COULD BE REWRITTEN AT ANY TIME. */
-import configPromise from '@payload-config'
+import config from '@payload-config'
 import '@payloadcms/next/css'
 import { RootLayout } from '@payloadcms/next/layouts'
 import React from 'react'
@@ -13,7 +13,7 @@ type Args = {
 }
 
 const Layout = ({ children }: Args) => (
-  <RootLayout config={configPromise} importMap={importMap}>
+  <RootLayout config={config} importMap={importMap}>
     {children}
   </RootLayout>
 )

next.config.mjs

@@ -19,6 +19,11 @@ const config = withBundleAnalyzer(
     typescript: {
       ignoreBuildErrors: true,
     },
+    experimental: {
+      serverActions: {
+        bodySizeLimit: '5mb',
+      },
+    },
     env: {
       PAYLOAD_CORE_DEV: 'true',
       ROOT_DIR: path.resolve(dirname),

package.json

@@ -58,19 +58,20 @@
     "dev:generate-importmap": "pnpm runts ./test/generateImportMap.ts",
     "dev:generate-types": "pnpm runts ./test/generateTypes.ts",
     "dev:postgres": "cross-env PAYLOAD_DATABASE=postgres pnpm runts ./test/dev.ts",
+    "dev:prod": "cross-env NODE_OPTIONS=--no-deprecation tsx ./test/dev.ts --prod",
     "dev:vercel-postgres": "cross-env PAYLOAD_DATABASE=vercel-postgres pnpm runts ./test/dev.ts",
     "devsafe": "node ./scripts/delete-recursively.js '**/.next' && pnpm dev",
     "docker:restart": "pnpm docker:stop --remove-orphans && pnpm docker:start",
     "docker:start": "docker compose -f packages/plugin-cloud-storage/docker-compose.yml up -d",
     "docker:stop": "docker compose -f packages/plugin-cloud-storage/docker-compose.yml down",
     "force:build": "pnpm run build:core:force",
     "lint": "turbo run lint --concurrency 1 --continue",
-    "lint-staged": "lint-staged",
+    "lint-staged": "node ./scripts/run-lint-staged.js",
     "lint:fix": "turbo run lint:fix --concurrency 1 --continue",
     "obliterate-playwright-cache-macos": "rm -rf ~/Library/Caches/ms-playwright && find /System/Volumes/Data/private/var/folders -type d -name 'playwright*' -exec rm -rf {} +",
     "prepare": "husky",
-    "prepare-run-test-against-prod": "pnpm bf && rm -rf test/packed && rm -rf test/node_modules && rm -f test/pnpm-lock.yaml && pnpm run script:pack --all --no-build --dest test/packed && pnpm runts test/setupProd.ts && cd test && pnpm i --ignore-workspace && cd ..",
-    "prepare-run-test-against-prod:ci": "rm -rf test/node_modules && rm -f test/pnpm-lock.yaml && pnpm run script:pack --all --no-build --dest test/packed && pnpm runts test/setupProd.ts && cd test && pnpm i --ignore-workspace && cd ..",
+    "prepare-run-test-against-prod": "pnpm bf && rm -rf test/packed && rm -rf test/node_modules && rm -rf app && rm -f test/pnpm-lock.yaml && pnpm run script:pack --all --no-build --dest test/packed && pnpm runts test/setupProd.ts && cd test && pnpm i --ignore-workspace && cd ..",
+    "prepare-run-test-against-prod:ci": "rm -rf test/node_modules && rm -rf app && rm -f test/pnpm-lock.yaml && pnpm run script:pack --all --no-build --dest test/packed && pnpm runts test/setupProd.ts && cd test && pnpm i --ignore-workspace && cd ..",
     "reinstall": "pnpm clean:all && pnpm install",
     "release:alpha": "pnpm runts ./scripts/release.ts --bump prerelease --tag alpha",
     "release:beta": "pnpm runts ./scripts/release.ts --bump prerelease --tag beta",

packages/db-mongodb/src/countGlobalVersions.ts

@@ -0,0 +1,48 @@
+import type { QueryOptions } from 'mongoose'
+import type { CountGlobalVersions, PayloadRequest } from 'payload'
+
+import { flattenWhereToOperators } from 'payload'
+
+import type { MongooseAdapter } from './index.js'
+
+import { withSession } from './withSession.js'
+
+export const countGlobalVersions: CountGlobalVersions = async function countGlobalVersions(
+  this: MongooseAdapter,
+  { global, locale, req = {} as PayloadRequest, where },
+) {
+  const Model = this.versions[global]
+  const options: QueryOptions = await withSession(this, req)
+
+  let hasNearConstraint = false
+
+  if (where) {
+    const constraints = flattenWhereToOperators(where)
+    hasNearConstraint = constraints.some((prop) => Object.keys(prop).some((key) => key === 'near'))
+  }
+
+  const query = await Model.buildQuery({
+    locale,
+    payload: this.payload,
+    where,
+  })
+
+  // useEstimatedCount is faster, but not accurate, as it ignores any filters. It is thus set to true if there are no filters.
+  const useEstimatedCount = hasNearConstraint || !query || Object.keys(query).length === 0
+
+  if (!useEstimatedCount && Object.keys(query).length === 0 && this.disableIndexHints !== true) {
+    // Improve the performance of the countDocuments query which is used if useEstimatedCount is set to false by adding
+    // a hint. By default, if no hint is provided, MongoDB does not use an indexed field to count the returned documents,
+    // which makes queries very slow. This only happens when no query (filter) is provided. If one is provided, it uses
+    // the correct indexed field
+    options.hint = {
+      _id: 1,
+    }
+  }
+
+  const result = await Model.countDocuments(query, options)
+
+  return {
+    totalDocs: result,
+  }
+}

packages/db-mongodb/src/countVersions.ts

@@ -0,0 +1,48 @@
+import type { QueryOptions } from 'mongoose'
+import type { CountVersions, PayloadRequest } from 'payload'
+
+import { flattenWhereToOperators } from 'payload'
+
+import type { MongooseAdapter } from './index.js'
+
+import { withSession } from './withSession.js'
+
+export const countVersions: CountVersions = async function countVersions(
+  this: MongooseAdapter,
+  { collection, locale, req = {} as PayloadRequest, where },
+) {
+  const Model = this.versions[collection]
+  const options: QueryOptions = await withSession(this, req)
+
+  let hasNearConstraint = false
+
+  if (where) {
+    const constraints = flattenWhereToOperators(where)
+    hasNearConstraint = constraints.some((prop) => Object.keys(prop).some((key) => key === 'near'))
+  }
+
+  const query = await Model.buildQuery({
+    locale,
+    payload: this.payload,
+    where,
+  })
+
+  // useEstimatedCount is faster, but not accurate, as it ignores any filters. It is thus set to true if there are no filters.
+  const useEstimatedCount = hasNearConstraint || !query || Object.keys(query).length === 0
+
+  if (!useEstimatedCount && Object.keys(query).length === 0 && this.disableIndexHints !== true) {
+    // Improve the performance of the countDocuments query which is used if useEstimatedCount is set to false by adding
+    // a hint. By default, if no hint is provided, MongoDB does not use an indexed field to count the returned documents,
+    // which makes queries very slow. This only happens when no query (filter) is provided. If one is provided, it uses
+    // the correct indexed field
+    options.hint = {
+      _id: 1,
+    }
+  }
+
+  const result = await Model.countDocuments(query, options)
+
+  return {
+    totalDocs: result,
+  }
+}

packages/db-mongodb/src/index.ts

@@ -12,6 +12,8 @@ import type { CollectionModel, GlobalModel, MigrateDownArgs, MigrateUpArgs } fro
 
 import { connect } from './connect.js'
 import { count } from './count.js'
+import { countGlobalVersions } from './countGlobalVersions.js'
+import { countVersions } from './countVersions.js'
 import { create } from './create.js'
 import { createGlobal } from './createGlobal.js'
 import { createGlobalVersion } from './createGlobalVersion.js'
@@ -154,7 +156,6 @@ export function mongooseAdapter({
       collections: {},
       connection: undefined,
       connectOptions: connectOptions || {},
-      count,
       disableIndexHints,
       globals: undefined,
       mongoMemoryServer,
@@ -166,6 +167,9 @@ export function mongooseAdapter({
       beginTransaction: transactionOptions === false ? defaultBeginTransaction() : beginTransaction,
       commitTransaction,
       connect,
+      count,
+      countGlobalVersions,
+      countVersions,
       create,
       createGlobal,
       createGlobalVersion,

packages/db-mongodb/src/utilities/buildJoinAggregation.ts

@@ -1,8 +1,6 @@
 import type { PipelineStage } from 'mongoose'
 import type { CollectionSlug, JoinQuery, SanitizedCollectionConfig, Where } from 'payload'
 
-import { combineQueries } from 'payload'
-
 import type { MongooseAdapter } from '../index.js'
 
 import { buildSortParam } from '../queries/buildSortParam.js'
@@ -60,19 +58,19 @@ export const buildJoinAggregation = async ({
     for (const join of joinConfig[slug]) {
       const joinModel = adapter.collections[join.field.collection]
 
-      if (projection && !projection[join.schemaPath]) {
+      if (projection && !projection[join.joinPath]) {
         continue
       }
 
-      if (joins?.[join.schemaPath] === false) {
+      if (joins?.[join.joinPath] === false) {
         continue
       }
 
       const {
         limit: limitJoin = join.field.defaultLimit ?? 10,
         sort: sortJoin = join.field.defaultSort || collectionConfig.defaultSort,
         where: whereJoin,
-      } = joins?.[join.schemaPath] || {}
+      } = joins?.[join.joinPath] || {}
 
       const sort = buildSortParam({
         config: adapter.payload.config,
@@ -105,7 +103,7 @@ export const buildJoinAggregation = async ({
 
       if (adapter.payload.config.localization && locale === 'all') {
         adapter.payload.config.localization.localeCodes.forEach((code) => {
-          const as = `${versions ? `version.${join.schemaPath}` : join.schemaPath}${code}`
+          const as = `${versions ? `version.${join.joinPath}` : join.joinPath}${code}`
 
           aggregate.push(
             {
@@ -146,7 +144,7 @@ export const buildJoinAggregation = async ({
       } else {
         const localeSuffix =
           join.field.localized && adapter.payload.config.localization && locale ? `.${locale}` : ''
-        const as = `${versions ? `version.${join.schemaPath}` : join.schemaPath}${localeSuffix}`
+        const as = `${versions ? `version.${join.joinPath}` : join.joinPath}${localeSuffix}`
 
         aggregate.push(
           {

packages/db-mongodb/src/utilities/handleError.ts

@@ -19,8 +19,8 @@ export const handleError = ({
         collection,
         errors: [
           {
-            field: Object.keys(error.keyValue)[0],
             message: req.t('error:valueMustBeUnique'),
+            path: Object.keys(error.keyValue)[0],
           },
         ],
         global,

packages/db-postgres/src/index.ts

@@ -4,6 +4,8 @@ import {
   beginTransaction,
   commitTransaction,
   count,
+  countGlobalVersions,
+  countVersions,
   create,
   createGlobal,
   createGlobalVersion,
@@ -126,6 +128,8 @@ export function postgresAdapter(args: Args): DatabaseAdapterObj<PostgresAdapter>
       convertPathToJSONTraversal,
       count,
       countDistinct,
+      countGlobalVersions,
+      countVersions,
       create,
       createGlobal,
       createGlobalVersion,

packages/db-sqlite/src/index.ts

@@ -5,6 +5,8 @@ import {
   beginTransaction,
   commitTransaction,
   count,
+  countGlobalVersions,
+  countVersions,
   create,
   createGlobal,
   createGlobalVersion,
@@ -114,6 +116,8 @@ export function sqliteAdapter(args: Args): DatabaseAdapterObj<SQLiteAdapter> {
       convertPathToJSONTraversal,
       count,
       countDistinct,
+      countGlobalVersions,
+      countVersions,
       create,
       createGlobal,
       createGlobalVersion,

packages/db-vercel-postgres/src/index.ts

@@ -4,6 +4,8 @@ import {
   beginTransaction,
   commitTransaction,
   count,
+  countGlobalVersions,
+  countVersions,
   create,
   createGlobal,
   createGlobalVersion,
@@ -127,6 +129,8 @@ export function vercelPostgresAdapter(args: Args = {}): DatabaseAdapterObj<Verce
       convertPathToJSONTraversal,
       count,
       countDistinct,
+      countGlobalVersions,
+      countVersions,
       create,
       createGlobal,
       createGlobalVersion,