Backport #2538: fix(next): prewarm SWC plugin cache before parallel loaders

.changeset/prewarm-next-swc-cache.md

@@ -0,0 +1,5 @@
+---
+'@workflow/next': patch
+---
+
+Prewarm the Workflow SWC plugin cache before Next.js starts parallel loader workers.

packages/next/package.json

@@ -28,6 +28,7 @@
   "scripts": {
     "build": "tsc",
     "dev": "tsc --watch",
+    "test": "vitest run src",
     "clean": "tsc --build --clean && rm -rf dist docs",
     "prepack": "mkdir -p docs && cp ../../docs/content/docs/getting-started/next.mdx ./docs/ && cp -r ../../docs/content/docs/api-reference/workflow-next ./docs/api-reference",
     "postpack": "rm -rf docs"
@@ -45,7 +46,8 @@
     "@types/semver": "7.7.1",
     "@types/watchpack": "2.4.4",
     "@workflow/tsconfig": "workspace:*",
-    "next": "16.2.1"
+    "next": "16.2.1",
+    "vitest": "catalog:"
   },
   "peerDependencies": {
     "next": ">13"

packages/next/src/index.test.ts

@@ -10,7 +10,12 @@
 import { dirname, join } from 'node:path';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
-const { buildMock, builderConfigs, getNextBuilderMock } = vi.hoisted(() => {
+const {
+  buildMock,
+  builderConfigs,
+  getNextBuilderMock,
+  prewarmWorkflowSwcPluginCacheMock,
+} = vi.hoisted(() => {
   const buildMock = vi.fn(async () => {});
   const builderConfigs: Record<string, unknown>[] = [];
   const getNextBuilderMock = vi.fn(async () => {
@@ -22,27 +27,27 @@
       }
     };
   });
+  const prewarmWorkflowSwcPluginCacheMock = vi.fn();
 
   return {
     buildMock,
     builderConfigs,
     getNextBuilderMock,
+    prewarmWorkflowSwcPluginCacheMock,
   };
 });
 
 vi.mock('./builder.js', () => ({
   getNextBuilder: getNextBuilderMock,
 }));
 
+vi.mock('./swc-plugin-cache.js', () => ({
+  prewarmWorkflowSwcPluginCache: prewarmWorkflowSwcPluginCacheMock,
+}));
+
 import { withWorkflow } from './index.js';
 
-const loaderStubPath = join(
-  process.cwd(),
-  'packages',
-  'next',
-  'src',
-  'loader.js'
-);
+const loaderStubPath = join(__dirname, 'loader.js');
 const hadLoaderStub = existsSync(loaderStubPath);
 const realTmpDir = realpathSync(tmpdir());
 
@@ -65,6 +70,7 @@
     buildMock.mockClear();
     builderConfigs.length = 0;
     getNextBuilderMock.mockClear();
+    prewarmWorkflowSwcPluginCacheMock.mockClear();
 
     if (!hadLoaderStub) {
       writeFileSync(loaderStubPath, 'module.exports = {};\n', 'utf-8');
@@ -114,6 +120,102 @@
     });
   });
 
+  it.each([
+    'phase-production-build',
+    'phase-development-server',
+  ])('prewarms the SWC plugin cache during %s', async (phase) => {
+    const config = withWorkflow({});
+
+    await config(phase, { defaultConfig: {} });
+
+    expect(prewarmWorkflowSwcPluginCacheMock).toHaveBeenCalledOnce();
+    expect(prewarmWorkflowSwcPluginCacheMock).toHaveBeenCalledWith(
+      process.cwd()
+    );
+  });
+
+  it('does not prewarm the SWC plugin cache for the production server', async () => {
+    const config = withWorkflow({});
+
+    await config('phase-production-server', { defaultConfig: {} });
+
+    expect(prewarmWorkflowSwcPluginCacheMock).not.toHaveBeenCalled();
+  });
+
+  it('configures diagnostics inside the default Next.js dist dir', async () => {
+    const config = withWorkflow({});
+
+    await config('phase-production-build', {
+      defaultConfig: {},
+    });
+
+    expect(builderConfigs[0]).toMatchObject({
+      distDir: '.next',
+      diagnosticsDir: '.next/diagnostics',
+    });
+  });
+
+  it('configures diagnostics inside a custom Next.js dist dir', async () => {
+    const config = withWorkflow({
+      distDir: 'build-output',
+    });
+
+    await config('phase-production-build', {
+      defaultConfig: {},
+    });
+
+    expect(builderConfigs[0]).toMatchObject({
+      distDir: 'build-output',
+      diagnosticsDir: 'build-output/diagnostics',
+    });
+  });
+
+  it('externalizes the built-in Vercel world while preserving user externals', async () => {
+    const config = withWorkflow({
+      serverExternalPackages: ['@node-rs/xxhash'],
+    });
+
+    const nextConfig = await config('phase-production-build', {
+      defaultConfig: {},
+    });
+
+    expect(nextConfig.serverExternalPackages).toEqual([
+      '@node-rs/xxhash',
+      '@workflow/world-vercel',
+      '@vercel/queue',
+      '@vercel/oidc',
+      '@vercel/cli-auth',
+      '@napi-rs/keyring',
+    ]);
+    expect(nextConfig.outputFileTracingIncludes).toBeUndefined();
+  });
+
+  it('preserves user webpack externals without adding Vercel world dependency externals', async () => {
+    const userWebpack = vi.fn((webpackConfig: any) => {
+      webpackConfig.externals = [{ react: 'commonjs react' }];
+      return webpackConfig;
+    });
+    const config = withWorkflow({
+      webpack: userWebpack,
+    });
+
+    const nextConfig = await config('phase-production-build', {
+      defaultConfig: {},
+    });
+    const webpackConfig = nextConfig.webpack?.(
+      {
+        externals: [],
+        module: {
+          rules: [],
+        },
+      },
+      {} as any
+    );
+
+    expect(userWebpack).toHaveBeenCalledOnce();
+    expect(webpackConfig?.externals).toEqual([{ react: 'commonjs react' }]);
+  });
+
   it('removes workflow packages from serverExternalPackages for this build', async () => {
     const projectDir = mkdtempSync(
       join(realTmpDir, 'workflow-next-server-external-')

packages/next/src/index.ts

@@ -229,6 +229,17 @@ export function withWorkflow(
     phase: string,
     ctx: { defaultConfig: NextConfig }
   ) {
+    if (
+      phase === 'phase-development-server' ||
+      phase === 'phase-production-build'
+    ) {
+      const { prewarmWorkflowSwcPluginCache } = await import(
+        './swc-plugin-cache.js'
+      );
+      // Loader workers inherit this cwd and read from the same SWC cache.
+      prewarmWorkflowSwcPluginCache(process.cwd());
+    }
+
     const loaderPath = require.resolve('./loader');
     let nextConfig: NextConfig;
 

packages/next/src/loader.ts

@@ -293,6 +293,7 @@ export default function workflowLoader(
         },
         target: 'es2022',
         experimental: {
+          cacheRoot: join(workingDir, '.swc'),
           plugins: [[swcPluginPath, { mode, moduleSpecifier }]],
         },
         transform: {

packages/next/src/swc-plugin-cache.integration.test.ts

@@ -0,0 +1,62 @@
+import { execFile } from 'node:child_process';
+import { mkdtemp, readdir, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { promisify } from 'node:util';
+import { describe, expect, it, onTestFinished } from 'vitest';
+import { prewarmWorkflowSwcPluginCache } from './swc-plugin-cache.js';
+
+const execFileAsync = promisify(execFile);
+
+const WORKER_SOURCE = `
+const [swcCorePath, pluginPath, cacheRoot, workerId] = process.argv.slice(1);
+const { transformSync } = require(swcCorePath);
+
+transformSync(
+  \`export async function worker\${workerId}() { 'use step'; }\`,
+  {
+    filename: \`worker-\${workerId}.js\`,
+    swcrc: false,
+    jsc: {
+      experimental: {
+        cacheRoot,
+        plugins: [[pluginPath, { mode: 'step' }]],
+      },
+    },
+  }
+);
+`;
+
+describe('Workflow SWC plugin cache prewarming integration', () => {
+  it('makes the compiled plugin safe for parallel worker reads', async () => {
+    const projectRoot = await mkdtemp(
+      join(tmpdir(), 'workflow-swc-plugin-cache-')
+    );
+    onTestFinished(() => rm(projectRoot, { recursive: true }));
+
+    prewarmWorkflowSwcPluginCache(projectRoot);
+
+    const cacheRoot = join(projectRoot, '.swc');
+    const cacheFiles = await readdir(cacheRoot, { recursive: true });
+    expect(cacheFiles.some((file) => file.endsWith('.wasmer-v7'))).toBe(true);
+
+    const swcCorePath = require.resolve('@swc/core');
+    const swcPluginPath = require.resolve('@workflow/swc-plugin');
+    await Promise.all(
+      Array.from({ length: 16 }, (_, workerId) =>
+        execFileAsync(
+          process.execPath,
+          [
+            '-e',
+            WORKER_SOURCE,
+            swcCorePath,
+            swcPluginPath,
+            cacheRoot,
+            String(workerId),
+          ],
+          { cwd: projectRoot }
+        )
+      )
+    );
+  }, 30_000);
+});

packages/next/src/swc-plugin-cache.ts

@@ -0,0 +1,20 @@
+import { join } from 'node:path';
+import { transformSync } from '@swc/core';
+
+/**
+ * SWC does not atomically write its Wasmer cache. Compile the plugin before
+ * Next.js starts parallel loader workers.
+ * @see https://github.com/swc-project/swc/issues/10065
+ */
+export function prewarmWorkflowSwcPluginCache(projectRoot: string): void {
+  transformSync(`async function step() { 'use step'; }`, {
+    filename: '__workflow_swc_cache_warmup__.js',
+    swcrc: false,
+    jsc: {
+      experimental: {
+        cacheRoot: join(projectRoot, '.swc'),
+        plugins: [[require.resolve('@workflow/swc-plugin'), { mode: 'step' }]],
+      },
+    },
+  });
+}