diff --git a/.changeset/remove-next-lazy-discovery.md b/.changeset/remove-next-lazy-discovery.md new file mode 100644 index 0000000000..eafdcd498b --- /dev/null +++ b/.changeset/remove-next-lazy-discovery.md @@ -0,0 +1,8 @@ +--- +'@workflow/next': major +'@workflow/builders': patch +--- + +Remove the Next.js lazy discovery/deferred builder path and the `workflows.lazyDiscovery` option. + +Fall back to direct generated-file overwrites on Windows when atomic rename is blocked by Next.js dev server file handles. diff --git a/.github/workflows/event-log-race-repro.yml b/.github/workflows/event-log-race-repro.yml index 3e3bec4da3..fc4341cdca 100644 --- a/.github/workflows/event-log-race-repro.yml +++ b/.github/workflows/event-log-race-repro.yml @@ -68,7 +68,6 @@ jobs: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} WORKFLOW_PUBLIC_MANIFEST: '1' - WORKFLOW_NEXT_LAZY_DISCOVERY: '0' steps: - name: Checkout Repo diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1665b31aaa..f62fc66b87 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -353,7 +353,6 @@ jobs: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} WORKFLOW_PUBLIC_MANIFEST: '1' - WORKFLOW_NEXT_LAZY_DISCOVERY: ${{ matrix.app.canary != true && (matrix.app.name == 'nextjs-turbopack' || matrix.app.name == 'nextjs-webpack') && '0' || '' }} steps: - name: Checkout Repo uses: actions/checkout@v4 @@ -477,7 +476,6 @@ jobs: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} WORKFLOW_PUBLIC_MANIFEST: '1' - WORKFLOW_NEXT_LAZY_DISCOVERY: ${{ matrix.app.lazyDiscovery == false && '0' || matrix.app.lazyDiscovery == true && '1' || '' }} steps: - name: Checkout Repo @@ -561,7 +559,6 @@ jobs: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} WORKFLOW_PUBLIC_MANIFEST: '1' - WORKFLOW_NEXT_LAZY_DISCOVERY: ${{ matrix.app.lazyDiscovery == false && '0' || matrix.app.lazyDiscovery == true && '1' || '' }} steps: - name: Checkout Repo @@ -666,7 +663,6 @@ jobs: WORKFLOW_PUBLIC_MANIFEST: '1' WORKFLOW_TARGET_WORLD: "@workflow/world-postgres" WORKFLOW_POSTGRES_URL: "postgres://world:world@localhost:5432/world" - WORKFLOW_NEXT_LAZY_DISCOVERY: ${{ matrix.app.lazyDiscovery == false && '0' || matrix.app.lazyDiscovery == true && '1' || '' }} steps: - name: Checkout Repo @@ -848,7 +844,11 @@ jobs: NODE_OPTIONS: "--enable-source-maps" APP_NAME: "nextjs-turbopack" DEPLOYMENT_URL: "http://localhost:3000" - DEV_TEST_CONFIG: '{"generatedStepPath":"app/.well-known/workflow/v1/flow/__step_registrations.js","generatedWorkflowPath":"app/.well-known/workflow/v1/flow/route.js","apiFilePath":"app/api/chat/route.ts","apiFileImportPath":"../../..","port":3000}' + # Use a real workflow file for the Windows HMR test. Most of this + # workbench's workflows are symlinks into workbench/example, and + # Windows/Turbopack can rebuild from the symlink path while still + # serving the previous module contents from the realpath cache. + DEV_TEST_CONFIG: '{"generatedStepPath":"app/.well-known/workflow/v1/flow/__step_registrations.js","generatedWorkflowPath":"app/.well-known/workflow/v1/flow/route.js","apiFilePath":"app/api/chat/route.ts","apiFileImportPath":"../../..","port":3000,"testWorkflowFile":"96_many_steps.ts"}' - name: Print Next.js server logs if: always() diff --git a/docs/content/docs/v4/api-reference/workflow-next/with-workflow.mdx b/docs/content/docs/v4/api-reference/workflow-next/with-workflow.mdx index 9f8c0bd4da..c314a01077 100644 --- a/docs/content/docs/v4/api-reference/workflow-next/with-workflow.mdx +++ b/docs/content/docs/v4/api-reference/workflow-next/with-workflow.mdx @@ -68,7 +68,6 @@ const nextConfig: NextConfig = {}; export default withWorkflow(nextConfig, { workflows: { - lazyDiscovery: true, local: { port: 4000, }, @@ -79,7 +78,6 @@ export default withWorkflow(nextConfig, { | Option | Type | Default | Description | | --- | --- | --- | --- | -| `workflows.lazyDiscovery` | `boolean` | `false` | When `true`, defers workflow discovery until files are requested instead of scanning eagerly at startup. Useful for large projects where startup time matters. | | `workflows.local.port` | `number` | — | Overrides the `PORT` environment variable for local development. Has no effect when deployed to Vercel. | | `workflows.sourcemap` | `boolean \| 'inline' \| 'linked' \| 'external' \| 'both'` | `'inline'` (dev) / `false` (prod) | Controls source maps on generated workflow bundles. See [Source maps](#source-maps) below. | diff --git a/docs/content/docs/v5/api-reference/workflow-next/with-workflow.mdx b/docs/content/docs/v5/api-reference/workflow-next/with-workflow.mdx index 038c1a4d5e..acfc585691 100644 --- a/docs/content/docs/v5/api-reference/workflow-next/with-workflow.mdx +++ b/docs/content/docs/v5/api-reference/workflow-next/with-workflow.mdx @@ -92,7 +92,6 @@ const nextConfig: NextConfig = {}; export default withWorkflow(nextConfig, { workflows: { - lazyDiscovery: false, local: { port: 4000, }, @@ -103,7 +102,6 @@ export default withWorkflow(nextConfig, { | Option | Type | Default | Description | | --- | --- | --- | --- | -| `workflows.lazyDiscovery` | `boolean` | `true` | Defers workflow discovery until files are requested instead of scanning eagerly at startup. Set to `false` to force eager discovery (scanning the project up front). Requires a Next.js version that supports deferred entries; older versions fall back to eager discovery automatically. | | `workflows.local.port` | `number` | — | Overrides the `PORT` environment variable for local development. Has no effect when deployed to Vercel. | | `workflows.sourcemap` | `boolean \| 'inline' \| 'linked' \| 'external' \| 'both'` | `'inline'` (dev) / `false` (prod) | Controls source maps on generated workflow bundles. See [Source maps](#source-maps) below. | diff --git a/packages/builders/src/base-builder.ts b/packages/builders/src/base-builder.ts index 044142a23f..2e17355966 100644 --- a/packages/builders/src/base-builder.ts +++ b/packages/builders/src/base-builder.ts @@ -1,5 +1,12 @@ import { randomUUID } from 'node:crypto'; -import { mkdir, readFile, realpath, rename, writeFile } from 'node:fs/promises'; +import { + mkdir, + readFile, + realpath, + rename, + rm, + writeFile, +} from 'node:fs/promises'; import { createRequire } from 'node:module'; import { basename, dirname, join, relative, resolve } from 'node:path'; import { promisify } from 'node:util'; @@ -467,6 +474,37 @@ export abstract class BaseBuilder { return state; } + /** + * Writes generated files atomically where possible. On Windows, Next.js can + * briefly hold generated route files open while compiling them, which makes + * rename-over-existing fail with EPERM/EACCES. In that case, fall back to a + * direct overwrite so watch rebuilds can still make progress. + */ + private async writeGeneratedFile( + targetPath: string, + content: string + ): Promise { + const tempPath = `${targetPath}.${randomUUID()}.tmp`; + await writeFile(tempPath, content); + try { + await rename(tempPath, targetPath); + } catch (error) { + const errorCode = + error && typeof error === 'object' && 'code' in error + ? (error as NodeJS.ErrnoException).code + : undefined; + if ( + process.platform === 'win32' && + (errorCode === 'EPERM' || errorCode === 'EACCES') + ) { + await writeFile(targetPath, content); + await rm(tempPath, { force: true }); + return; + } + throw error; + } + } + /** * Writes debug information to a JSON file for troubleshooting build issues. * Uses atomic write (temp file + rename) to prevent race conditions when @@ -504,12 +542,7 @@ export abstract class BaseBuilder { 2 ); - // Write atomically: write to temp file, then rename. - // rename() is atomic on POSIX systems and provides best-effort atomicity on Windows. - // Prevents race conditions where concurrent builds read partially-written files. - const tempPath = `${targetPath}.${randomUUID()}.tmp`; - await writeFile(tempPath, mergedData); - await rename(tempPath, targetPath); + await this.writeGeneratedFile(targetPath, mergedData); } catch (error: unknown) { console.warn('Failed to write debug file:', error); } @@ -1213,11 +1246,7 @@ export const POST = workflowEntrypoint(workflowCode${workflowEntrypointOptionsCo const outputDir = dirname(outfile); await mkdir(outputDir, { recursive: true }); - // Atomic write: write to temp file then rename to prevent - // file watchers from reading partial file during write - const tempPath = `${outfile}.${randomUUID()}.tmp`; - await writeFile(tempPath, workflowFunctionCode); - await rename(tempPath, outfile); + await this.writeGeneratedFile(outfile, workflowFunctionCode); return; } @@ -1396,10 +1425,7 @@ const workflowCode = \`${escapedVMCode}\`; export const POST = workflowEntrypoint(workflowCode${workflowEntrypointOptionsCode});`; if (!bundleFinalOutput) { - // Write directly (Next.js will bundle) - const tempPath = `${flowOutfile}.${randomUUID()}.tmp`; - await writeFile(tempPath, combinedFunctionCode); - await rename(tempPath, flowOutfile); + await this.writeGeneratedFile(flowOutfile, combinedFunctionCode); } else { // Bundle the combined code for standalone use const bundleStartTime = Date.now(); @@ -1469,9 +1495,7 @@ export const POST = workflowEntrypoint(workflowCode${workflowEntrypointOptionsCo const outputDir = dirname(flowOutfile); await mkdir(outputDir, { recursive: true }); - const tempPath = `${flowOutfile}.${randomUUID()}.tmp`; - await writeFile(tempPath, code); - await rename(tempPath, flowOutfile); + await this.writeGeneratedFile(flowOutfile, code); }; if (this.config.watch) { diff --git a/packages/core/e2e/dev.test.ts b/packages/core/e2e/dev.test.ts index b283a08196..f4750d67cf 100644 --- a/packages/core/e2e/dev.test.ts +++ b/packages/core/e2e/dev.test.ts @@ -1,10 +1,7 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { afterEach, beforeAll, describe, expect, test } from 'vitest'; -import { - getWorkbenchAppPath, - isNextLazyDiscoveryEnabledForTest, -} from './utils'; +import { getWorkbenchAppPath } from './utils'; export interface DevTestConfig { generatedStepPath: string; @@ -41,10 +38,6 @@ export function createDevTests(config?: DevTestConfig) { // Each prewarm/trigger fetch is hard-bounded by this so cleanup never hangs // on a wedged dev server. const PREWARM_FETCH_TIMEOUT_MS = 5_000; - // Workflow requests can include Next's first compile of the API route and - // the deferred flow route. CI routinely exceeds 5s there even when the - // workflow itself is healthy, so keep those requests bounded separately. - const WORKFLOW_FETCH_TIMEOUT_MS = 30_000; // The afterEach cleanup can issue two *sequential* prewarms (before and // after deleting an added file) while the dev server is mid-rebuild — the // teardown of a test that added a workflow file and edited an import is @@ -65,14 +58,6 @@ export function createDevTests(config?: DevTestConfig) { const usesNextFlowRoute = generatedWorkflow.includes( path.join('app', '.well-known', 'workflow', 'v1', 'flow', 'route.js') ); - const usesDeferredBuilder = - isNextLazyDiscoveryEnabledForTest() && usesNextFlowRoute; - const usesNextEagerBuilder = - !isNextLazyDiscoveryEnabledForTest() && usesNextFlowRoute; - const deferredWorkflowCodePath = path.join( - path.dirname(generatedWorkflow), - '__workflow_code.txt' - ); const workflowManifestPath = path.join( appPath, 'app/.well-known/workflow/v1/manifest.json' @@ -86,6 +71,15 @@ export function createDevTests(config?: DevTestConfig) { Object.keys(entry) ); }; + const readManifestWorkflowFunctionNames = async (): Promise => { + const manifestJson = await fs.readFile(workflowManifestPath, 'utf8'); + const manifest = JSON.parse(manifestJson) as { + workflows?: Record>; + }; + return Object.values(manifest.workflows || {}).flatMap((entry) => + Object.keys(entry) + ); + }; const readFileIfExists = async ( filePath: string ): Promise => { @@ -104,12 +98,9 @@ export function createDevTests(config?: DevTestConfig) { } }; const readGeneratedWorkflowOutput = async (): Promise => { - const outputs = [ - await readFileIfExists(generatedWorkflow), - usesDeferredBuilder - ? await readFileIfExists(deferredWorkflowCodePath) - : null, - ].filter((output): output is string => output !== null); + const outputs = [await readFileIfExists(generatedWorkflow)].filter( + (output): output is string => output !== null + ); if (outputs.length === 0) { throw new Error('Generated workflow outputs were not found'); @@ -129,101 +120,6 @@ export function createDevTests(config?: DevTestConfig) { }); }; - const triggerWorkflowRun = async ( - workflowName: string, - args: unknown[] = [] - ) => { - if (!deploymentUrl) { - return; - } - - const response = await fetch( - new URL('/api/workflows/start', deploymentUrl), - { - method: 'POST', - headers: { - 'content-type': 'application/json', - }, - body: JSON.stringify({ - workflowName, - args, - }), - signal: AbortSignal.timeout(WORKFLOW_FETCH_TIMEOUT_MS), - } - ); - - if (!response.ok) { - throw new Error( - `Failed to trigger workflow "${workflowName}": ${response.status}` - ); - } - }; - - const triggerPagesWorkflowRun = async ({ - workflowFile, - workflowFn, - args = [], - }: { - workflowFile: string; - workflowFn: string; - args?: unknown[]; - }): Promise => { - if (!deploymentUrl) { - throw new Error('DEPLOYMENT_URL is required to start a workflow'); - } - - const url = new URL('/api/trigger-pages', deploymentUrl); - url.searchParams.set('workflowFile', workflowFile); - url.searchParams.set('workflowFn', workflowFn); - if (args.length > 0) { - url.searchParams.set('args', args.map(String).join(',')); - } - - const response = await fetch(url, { - method: 'POST', - signal: AbortSignal.timeout(WORKFLOW_FETCH_TIMEOUT_MS), - }); - if (!response.ok) { - throw new Error( - `Failed to trigger workflow "${workflowFn}" from "${workflowFile}": ${ - response.status - } ${await response.text()}` - ); - } - - const result = (await response.json()) as { runId?: unknown }; - if (typeof result.runId !== 'string') { - throw new Error( - `Workflow trigger response did not include a runId: ${JSON.stringify( - result - )}` - ); - } - return result.runId; - }; - - const awaitPagesWorkflowRun = async (runId: string): Promise => { - if (!deploymentUrl) { - throw new Error('DEPLOYMENT_URL is required to await a workflow'); - } - - const url = new URL('/api/trigger-pages', deploymentUrl); - url.searchParams.set('runId', runId); - - const response = await fetch(url, { - signal: AbortSignal.timeout(WORKFLOW_FETCH_TIMEOUT_MS), - }); - if (!response.ok) { - throw new Error( - `Failed to await workflow run "${runId}": ${ - response.status - } ${await response.text()}` - ); - } - - return await response.json(); - }; - const prewarm = async () => { // Pre-warm the app with bounded requests so cleanup hooks cannot hang. await Promise.all([ @@ -290,7 +186,7 @@ export function createDevTests(config?: DevTestConfig) { restoreFiles.length = 0; }, CLEANUP_HOOK_TIMEOUT_MS); - test('should rebuild on workflow change', { timeout: 30_000 }, async () => { + test('should rebuild on workflow change', { timeout: 70_000 }, async () => { const workflowFile = path.join(appPath, workflowsDir, testWorkflowFile); const content = await fs.readFile(workflowFile, 'utf8'); @@ -309,14 +205,22 @@ export async function myNewWorkflow() { await pollUntil({ description: 'generated workflow to include myNewWorkflow', + timeoutMs: usesNextFlowRoute ? 50_000 : 25_000, check: async () => { + if (usesNextFlowRoute) { + const manifestFunctionNames = + await readManifestWorkflowFunctionNames(); + expect(manifestFunctionNames).toContain('myNewWorkflow'); + return; + } + const workflowContent = await readGeneratedWorkflowOutput(); expect(workflowContent).toContain('myNewWorkflow'); }, }); }); - test('should rebuild on step change', { timeout: 30_000 }, async () => { + test('should rebuild on step change', { timeout: 70_000 }, async () => { const stepFile = path.join(appPath, workflowsDir, testWorkflowFile); const content = await fs.readFile(stepFile, 'utf8'); @@ -334,6 +238,7 @@ export async function myNewStep() { restoreFiles.push({ path: stepFile, content }); await pollUntil({ description: 'generated step outputs to include myNewStep', + timeoutMs: usesNextFlowRoute ? 50_000 : 25_000, check: async () => { const stepRouteContent = await readFileIfExists(generatedStep); if (stepRouteContent?.includes('myNewStep')) { @@ -341,9 +246,8 @@ export async function myNewStep() { } // Next flow-route builders regenerate manifest.json on every - // rebuild. In lazy mode there is no standalone step registration - // file; in eager mode the bundled file may not preserve function - // names as plain text. + // rebuild. The bundled file may not preserve function names as + // plain text. if (usesNextFlowRoute) { const manifestFunctionNames = await readManifestStepFunctionNames(); expect(manifestFunctionNames).toContain('myNewStep'); @@ -355,107 +259,6 @@ export async function myNewStep() { }); }); - test.skipIf(!usesDeferredBuilder)( - 'should execute updated workflow logic after HMR without restart', - { timeout: 120_000 }, - async () => { - const workflowFileName = '98_duplicate_case.ts'; - const workflowFile = path.join(appPath, workflowsDir, workflowFileName); - const workflowFileKey = `workflows/${workflowFileName}`; - const workflowFn = 'addTenWorkflow'; - const content = await fs.readFile(workflowFile, 'utf8'); - const originalLine = ' const b = await add(a, 3);'; - const updatedLine = ' const b = await add(a, 30);'; - - if (!content.includes(originalLine)) { - throw new Error( - `Expected ${workflowFile} to contain ${JSON.stringify( - originalLine - )}` - ); - } - - const baselineRunId = await triggerPagesWorkflowRun({ - workflowFile: workflowFileKey, - workflowFn, - args: [100], - }); - await expect(awaitPagesWorkflowRun(baselineRunId)).resolves.toBe(110); - - await fs.writeFile( - workflowFile, - content.replace(originalLine, updatedLine) - ); - restoreFiles.push({ path: workflowFile, content }); - - await pollUntil({ - description: - 'updated workflow logic to be used by the deferred flow route', - timeoutMs: 75_000, - check: async () => { - const runId = await triggerPagesWorkflowRun({ - workflowFile: workflowFileKey, - workflowFn, - args: [100], - }); - await expect(awaitPagesWorkflowRun(runId)).resolves.toBe(137); - }, - }); - } - ); - - test.skipIf(!usesDeferredBuilder)( - 'should rebuild on imported step dependency change', - { timeout: 60_000 }, - async () => { - const importedStepFile = path.join( - appPath, - workflowsDir, - '_imported_step_only.ts' - ); - const content = await fs.readFile(importedStepFile, 'utf8'); - const marker = 'importedStepOnlyHotReloadMarker'; - - await fs.writeFile( - importedStepFile, - `${content} - -export async function ${marker}() { - 'use step' - return 'updated' -} -` - ); - restoreFiles.push({ path: importedStepFile, content }); - - const apiFile = path.join(appPath, finalConfig.apiFilePath); - const apiFileContent = await fs.readFile(apiFile, 'utf8'); - - await pollUntil({ - description: - 'manifest.json to include imported step hot-reload marker', - timeoutMs: 50_000, - check: async () => { - try { - await triggerWorkflowRun('importedStepOnlyWorkflow'); - } catch (error) { - // Turbopack on Windows occasionally caches a stale resolver - // failure (e.g. `Could not parse module - // '@workflow/core/dist/runtime/start.js'`) after an HMR - // cascade and returns 500 to every request until something - // invalidates its cache. Rewriting the api file is enough to - // force a fresh resolve on the next request, so we treat the - // 500 as transient and keep polling instead of bailing out. - await fs.writeFile(apiFile, apiFileContent); - throw error; - } - const manifestFunctionNames = await readManifestStepFunctionNames(); - expect(manifestFunctionNames).toContain(marker); - }, - }); - } - ); - test( 'should rebuild on adding workflow file', { timeout: 60_000 }, @@ -490,7 +293,7 @@ ${apiFileContent}` description: 'generated workflow to include newWorkflowFile', timeoutMs: 50_000, check: async () => { - if (usesNextEagerBuilder) { + if (usesNextFlowRoute) { const manifestJson = await fs.readFile( workflowManifestPath, 'utf8' @@ -513,143 +316,6 @@ ${apiFileContent}` }); } ); - - test.skipIf(!usesDeferredBuilder)( - 'should include steps discovered from workflow imports', - { timeout: 60_000 }, - async () => { - const workflowFile = path.join( - appPath, - workflowsDir, - 'discovered-via-workflow.ts' - ); - const stepFile = path.join( - appPath, - workflowsDir, - 'discovered-via-workflow-step.ts' - ); - - await fs.writeFile( - workflowFile, - `'use workflow'; -import { discoveredViaWorkflowStep } from './discovered-via-workflow-step'; - -export async function discoveredViaWorkflow() { - await discoveredViaWorkflowStep(); - return 'ok'; -} -` - ); - await fs.writeFile( - stepFile, - `'use step'; - -export async function discoveredViaWorkflowStep() { - return 'ok'; -} -` - ); - restoreFiles.push({ path: workflowFile, content: '' }); - restoreFiles.push({ path: stepFile, content: '' }); - - const apiFile = path.join(appPath, finalConfig.apiFilePath); - const apiFileContent = await fs.readFile(apiFile, 'utf8'); - restoreFiles.push({ path: apiFile, content: apiFileContent }); - - await fs.writeFile( - apiFile, - `import '${finalConfig.apiFileImportPath}/${workflowsDir}/discovered-via-workflow'; -${apiFileContent}` - ); - - await pollUntil({ - description: - 'manifest.json to include discoveredViaWorkflowStep after discovery', - timeoutMs: 25_000, - check: async () => { - await fetchWithTimeout('/api/chat'); - const manifestFunctionNames = await readManifestStepFunctionNames(); - expect(manifestFunctionNames).toContain( - 'discoveredViaWorkflowStep' - ); - }, - }); - - // Tear down in-test (rather than relying on afterEach) so we can wait - // for the deferred builder to drop the discovered step from the - // manifest before the next test file runs. Rewrite the temporary - // sources to inert modules before deleting them; otherwise the rebuild - // can race with deletion and get stuck on an ENOENT from the SWC - // transform while the generated route still imports the old files. - await fs.writeFile(apiFile, apiFileContent); - await fs.writeFile(workflowFile, 'export const removed = true;\n'); - await fs.writeFile(stepFile, 'export const removed = true;\n'); - await pollUntil({ - description: - 'manifest.json to drop discoveredViaWorkflowStep after cleanup', - timeoutMs: 25_000, - check: async () => { - await fetchWithTimeout('/api/chat'); - const manifestFunctionNames = await readManifestStepFunctionNames(); - expect(manifestFunctionNames).not.toContain( - 'discoveredViaWorkflowStep' - ); - }, - }); - await fs.unlink(workflowFile); - await fs.unlink(stepFile); - for (const trackedPath of [apiFile, workflowFile, stepFile]) { - const idx = restoreFiles.findIndex( - (item) => item.path === trackedPath - ); - if (idx !== -1) { - restoreFiles.splice(idx, 1); - } - } - } - ); - - test.skipIf(!usesDeferredBuilder)( - 'should reference package step sources discovered via manifest entries', - { timeout: 30_000 }, - async () => { - await pollUntil({ - description: - 'generated workflow outputs to reference @workflow/ai package steps', - timeoutMs: 25_000, - check: async () => { - await fetchWithTimeout('/api/chat'); - const manifestJson = await fs.readFile( - workflowManifestPath, - 'utf8' - ); - const manifest = JSON.parse(manifestJson) as { - steps?: Record; - }; - const manifestStepFiles = Object.keys(manifest.steps || {}); - expect( - manifestStepFiles.some((filePath) => - /ai\/(src|dist)\/agent\/durable-agent\.(ts|js)$/.test(filePath) - ) - ).toBe(true); - - // Package step sources are imported directly (not copied). Verify - // the generated route imports the @workflow/ai package or - // otherwise references `durable-agent` via its resolved path. - const generatedRouteContent = - (await readFileIfExists(generatedStep)) ?? - (await readFileIfExists(generatedWorkflow)); - if (!generatedRouteContent) { - throw new Error('generated workflow outputs were not found'); - } - expect( - generatedRouteContent.includes('@workflow/ai') || - generatedRouteContent.includes('durable-agent') - ).toBe(true); - }, - }); - } - ); }); } diff --git a/packages/core/e2e/local-build.test.ts b/packages/core/e2e/local-build.test.ts index a5198c1f8a..9cdbc0e526 100644 --- a/packages/core/e2e/local-build.test.ts +++ b/packages/core/e2e/local-build.test.ts @@ -3,10 +3,7 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { describe, expect, test } from 'vitest'; import { usesVercelWorld } from '../../utils/src/world-target'; -import { - getWorkbenchAppPath, - isNextLazyDiscoveryEnabledForTest, -} from './utils'; +import { getWorkbenchAppPath } from './utils'; interface CommandResult { stdout: string; @@ -92,18 +89,6 @@ const DIAGNOSTICS_MANIFEST_PATHS: Record = { 'nextjs-turbopack': '.next/diagnostics/workflows-manifest.json', }; -const DEFERRED_BUILD_MODE_PROJECTS = new Set([ - 'nextjs-webpack', - 'nextjs-turbopack', -]); -const DEFERRED_BUILD_UNSUPPORTED_WARNING = 'lazyDiscovery requires Next.js >='; -const EAGER_DISCOVERY_LOG = 'Discovering workflow directives'; -const WORKFLOW_BUNDLE_LOGS = [ - 'Created intermediate workflow bundle', - 'Created final workflow bundle', -]; -const STEP_BUNDLE_LOG = 'Created steps bundle'; - describe.each([ 'example', 'nextjs-webpack', @@ -133,22 +118,6 @@ describe.each([ expect(result.output).not.toContain('Error:'); - if ( - DEFERRED_BUILD_MODE_PROJECTS.has(project) && - isNextLazyDiscoveryEnabledForTest() - ) { - const deferredBuildSupported = !result.output.includes( - DEFERRED_BUILD_UNSUPPORTED_WARNING - ); - if (deferredBuildSupported) { - expect(result.output).not.toContain(EAGER_DISCOVERY_LOG); - for (const workflowBundleLog of WORKFLOW_BUNDLE_LOGS) { - expect(result.output).not.toContain(workflowBundleLog); - } - expect(result.output).not.toContain(STEP_BUNDLE_LOG); - } - } - const diagnosticsManifestPath = usesVercelWorld() ? '.vercel/output/diagnostics/workflows-manifest.json' : DIAGNOSTICS_MANIFEST_PATHS[project]; diff --git a/packages/core/e2e/utils.test.ts b/packages/core/e2e/utils.test.ts index 4952ceabfc..c75e3a38ed 100644 --- a/packages/core/e2e/utils.test.ts +++ b/packages/core/e2e/utils.test.ts @@ -6,11 +6,9 @@ const ORIGINAL_ENV = { ...process.env }; function setStepSourceMapEnv({ appName, dev, - lazyDiscovery, }: { appName: string; dev: boolean; - lazyDiscovery?: boolean; }) { process.env.APP_NAME = appName; process.env.DEPLOYMENT_URL = 'http://localhost:3000'; @@ -20,12 +18,6 @@ function setStepSourceMapEnv({ } else { delete process.env.DEV_TEST_CONFIG; } - - if (lazyDiscovery === undefined) { - delete process.env.WORKFLOW_NEXT_LAZY_DISCOVERY; - } else { - process.env.WORKFLOW_NEXT_LAZY_DISCOVERY = lazyDiscovery ? '1' : '0'; - } } afterEach(() => { @@ -33,41 +25,19 @@ afterEach(() => { }); describe('hasStepSourceMaps', () => { - test('expects source filenames for webpack local dev with lazy discovery enabled', () => { + test('does not expect source filenames for webpack local dev', () => { setStepSourceMapEnv({ appName: 'nextjs-webpack', dev: true, - lazyDiscovery: true, - }); - - expect(hasStepSourceMaps()).toBe(true); - }); - - test('does not expect source filenames for webpack local dev with lazy discovery disabled', () => { - setStepSourceMapEnv({ - appName: 'nextjs-webpack', - dev: true, - lazyDiscovery: false, - }); - - expect(hasStepSourceMaps()).toBe(false); - }); - - test('does not expect source filenames for turbopack local dev with lazy discovery disabled', () => { - setStepSourceMapEnv({ - appName: 'nextjs-turbopack', - dev: true, - lazyDiscovery: false, }); expect(hasStepSourceMaps()).toBe(false); }); - test('does not expect source filenames for turbopack local dev with lazy discovery enabled', () => { + test('does not expect source filenames for turbopack local dev', () => { setStepSourceMapEnv({ appName: 'nextjs-turbopack', dev: true, - lazyDiscovery: true, }); expect(hasStepSourceMaps()).toBe(false); @@ -77,7 +47,6 @@ describe('hasStepSourceMaps', () => { setStepSourceMapEnv({ appName: 'nextjs-webpack', dev: false, - lazyDiscovery: false, }); expect(hasStepSourceMaps()).toBe(false); diff --git a/packages/core/e2e/utils.ts b/packages/core/e2e/utils.ts index b669393089..b211e6008b 100644 --- a/packages/core/e2e/utils.ts +++ b/packages/core/e2e/utils.ts @@ -6,7 +6,6 @@ import { fileURLToPath } from 'node:url'; import { createVercelWorld } from '@workflow/world-vercel'; import { onTestFailed } from 'vitest'; import { getTrustedSourcesHeaders } from '../../../scripts/trusted-sources-headers.mjs'; -import { parseEnvironmentFlag } from '../../next/src/environment-flag.js'; import type { Run } from '../src/runtime'; import { getWorld, setWorld } from '../src/runtime'; @@ -95,10 +94,6 @@ function splitArgs(raw: string): string[] { return value.split(/\s+/); } -export function isNextLazyDiscoveryEnabledForTest(): boolean { - return parseEnvironmentFlag(process.env.WORKFLOW_NEXT_LAZY_DISCOVERY) ?? true; -} - export function getWorkbenchAppPath(overrideAppName?: string): string { const explicitWorkbenchPath = process.env.WORKBENCH_APP_PATH; const appName = process.env.APP_NAME ?? overrideAppName; @@ -135,10 +130,10 @@ export function hasStepSourceMaps(): boolean { if (appName === 'nextjs-turbopack') { return false; } - // Webpack's eager Next flow route executes steps from the generated - // __step_registrations.js bundle. Lazy discovery imports step sources through - // the flow route and preserves source filenames in local dev stacks. - if (appName === 'nextjs-webpack' && !isNextLazyDiscoveryEnabledForTest()) { + // Webpack's Next flow route executes steps from the generated + // __step_registrations.js bundle, which does not preserve source filenames + // in local dev stacks. + if (appName === 'nextjs-webpack') { return false; } // V2 carve-out: the V2 combined flow handler does not yet wire up inline @@ -436,8 +431,8 @@ export function getFallbackWorkflowId( ): string { const fileWithoutExt = workflowFile.replace(/\.tsx?$/, ''); // Keep this in sync with the SWC transform ID format. This fallback is - // intentionally coupled so tests can continue running when deferred manifest - // publication lags behind discovery in staged/out-of-monorepo scenarios. + // intentionally coupled so tests can continue running when manifest + // publication lags in staged/out-of-monorepo scenarios. return `workflow//./${fileWithoutExt}//${workflowFn}`; } @@ -463,8 +458,8 @@ export async function getWorkflowMetadata( return metadata; } - // Deferred discovery can grow the manifest during test execution, so poll - // briefly before failing to avoid races in staged/out-of-monorepo mode. + // Manifest publication can lag in staged/out-of-monorepo tests, so poll + // briefly before failing to avoid races. const deadline = Date.now() + manifestRetryTimeoutMs; while (Date.now() < deadline) { manifest = await fetchManifest(deploymentUrl, { forceRefresh: true }); @@ -479,9 +474,9 @@ export async function getWorkflowMetadata( await sleep(manifestRetryIntervalMs); } - // Deferred discovery can lag behind manifest publication in staged/out-of- - // monorepo tests. Fall back to the deterministic workflow ID format used by - // the transform so tests can continue exercising runtime behavior. + // Manifest publication can lag in staged/out-of-monorepo tests. Fall back to + // the deterministic workflow ID format used by the transform so tests can + // continue exercising runtime behavior. const fallbackWorkflowId = getFallbackWorkflowId(workflowFile, workflowFn); console.warn( `Workflow "${workflowFn}" not found in manifest for "${workflowFile}" after ${manifestRetryTimeoutMs}ms; ` + diff --git a/packages/next/src/builder-deferred.test.ts b/packages/next/src/builder-deferred.test.ts deleted file mode 100644 index 22fcedbd56..0000000000 --- a/packages/next/src/builder-deferred.test.ts +++ /dev/null @@ -1,191 +0,0 @@ -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join, relative } from 'node:path'; -import { afterEach, describe, expect, it, vi } from 'vitest'; -import { getNextBuilderDeferred } from './builder-deferred.js'; - -const tempDirs: string[] = []; -// biome-ignore lint/security/noGlobalEval: The test preserves the builder's dynamic import shim while stubbing one import. -const originalEval = globalThis.eval; - -afterEach(async () => { - await Promise.all( - tempDirs.map((dir) => rm(dir, { recursive: true, force: true })) - ); - tempDirs.length = 0; - vi.unstubAllGlobals(); -}); - -describe('NextDeferredBuilder', () => { - it('lets Next bundle step registrations from source imports', async () => { - const workingDir = await mkdtemp(join(tmpdir(), 'workflow-next-deferred-')); - tempDirs.push(workingDir); - vi.stubGlobal('eval', (source: string) => { - if (source === 'import("@workflow/builders")') { - return import('@workflow/builders'); - } - return originalEval(source); - }); - - const NextDeferredBuilder = await getNextBuilderDeferred(); - const builder = new NextDeferredBuilder({ - dirs: [], - workingDir, - buildTarget: 'next', - workflowsBundlePath: '', - stepsBundlePath: '', - webhookBundlePath: '', - }) as any; - - const workflowFile = join(workingDir, 'workflows/example.ts'); - const routeDir = join(workingDir, 'app/.well-known/workflow/v1/flow'); - const flowOutfile = join(routeDir, 'route.js.temp'); - const stepManifest = { - steps: { - 'workflows/example.ts': { - step: { stepId: 'step//./workflows/example//step' }, - }, - }, - }; - const workflowManifest = { - workflows: { - 'workflows/example.ts': { - run: { workflowId: 'workflow//./workflows/example//run' }, - }, - }, - }; - - builder.createStepsBundle = vi.fn(); - builder.createWorkflowsBundle = vi.fn(async () => ({ - interimBundleText: 'globalThis.__private_workflows = new Map();', - manifest: workflowManifest, - })); - builder.createDeferredStepManifest = vi.fn(async () => stepManifest); - - const result = await builder.createDeferredFlowRoute({ - inputFiles: [workflowFile], - flowOutfile, - discoveredEntries: { - discoveredSteps: new Set([workflowFile]), - discoveredWorkflows: new Set([workflowFile]), - discoveredSerdeFiles: new Set(), - }, - }); - - expect(builder.createStepsBundle).not.toHaveBeenCalled(); - expect(result.manifest).toEqual({ - ...stepManifest, - workflows: workflowManifest.workflows, - classes: {}, - }); - - const routeCode = await readFile(flowOutfile, 'utf8'); - const expectedStepImport = relative(routeDir, workflowFile).replace( - /\\/g, - '/' - ); - expect(routeCode).toContain("import 'workflow/internal/builtins';"); - expect(routeCode).toContain(`import "${expectedStepImport}";`); - expect(routeCode).toContain( - "import { workflowEntrypoint } from 'workflow/runtime';" - ); - expect(routeCode).not.toContain('__step_registrations'); - }); - - it('loads workflow code from disk for dev deferred flow routes', async () => { - const workingDir = await mkdtemp(join(tmpdir(), 'workflow-next-deferred-')); - tempDirs.push(workingDir); - vi.stubGlobal('eval', (source: string) => { - if (source === 'import("@workflow/builders")') { - return import('@workflow/builders'); - } - return originalEval(source); - }); - - const NextDeferredBuilder = await getNextBuilderDeferred(); - const builder = new NextDeferredBuilder({ - dirs: [], - workingDir, - buildTarget: 'next', - workflowsBundlePath: '', - stepsBundlePath: '', - webhookBundlePath: '', - watch: true, - }) as any; - - const workflowCode = 'globalThis.__private_workflows = new Map();'; - const workflowFile = join(workingDir, 'workflows/example.ts'); - const routeDir = join(workingDir, 'app/.well-known/workflow/v1/flow'); - const flowOutfile = join(routeDir, 'route.js.temp'); - - builder.createWorkflowsBundle = vi.fn(async () => ({ - interimBundleText: workflowCode, - manifest: { workflows: {}, classes: {} }, - })); - builder.createDeferredStepManifest = vi.fn(async () => ({})); - - await builder.createDeferredFlowRoute({ - inputFiles: [workflowFile], - flowOutfile, - discoveredEntries: { - discoveredSteps: new Set(), - discoveredWorkflows: new Set([workflowFile]), - discoveredSerdeFiles: new Set(), - }, - }); - - const routeCode = await readFile(flowOutfile, 'utf8'); - expect(routeCode).toContain( - "import { readFile, stat } from 'node:fs/promises';" - ); - expect(routeCode).toContain('async function getWorkflowHandler()'); - expect(routeCode).toContain('export async function POST(req)'); - expect(routeCode).not.toContain('const workflowCode = `'); - await expect( - readFile(join(routeDir, '__workflow_code.txt'), 'utf8') - ).resolves.toBe(workflowCode); - }); - - it('imports workspace package step sources from dist output outside packages directories', async () => { - const workingDir = await mkdtemp(join(tmpdir(), 'workflow-next-deferred-')); - tempDirs.push(workingDir); - vi.stubGlobal('eval', (source: string) => { - if (source === 'import("@workflow/builders")') { - return import('@workflow/builders'); - } - return originalEval(source); - }); - - const NextDeferredBuilder = await getNextBuilderDeferred(); - const builder = new NextDeferredBuilder({ - dirs: [], - workingDir, - buildTarget: 'next', - workflowsBundlePath: '', - stepsBundlePath: '', - webhookBundlePath: '', - }) as any; - - const packageDir = join(workingDir, '../libs/demo'); - const packageSourceFile = join(packageDir, 'src/runtime/run.ts'); - const packageDistFile = join(packageDir, 'dist/runtime/run.js'); - const routeDir = join(workingDir, 'app/.well-known/workflow/v1/flow'); - await mkdir(join(packageDir, 'src/runtime'), { recursive: true }); - await mkdir(join(packageDir, 'dist/runtime'), { recursive: true }); - await writeFile( - join(packageDir, 'package.json'), - JSON.stringify({ name: '@demo/workflow-steps', type: 'module' }) - ); - await writeFile(packageSourceFile, 'export async function step() {}'); - await writeFile(packageDistFile, 'export async function step() {}'); - - const importSpecifier = await builder.getDeferredRouteImportSpecifier( - packageSourceFile, - routeDir - ); - - expect(importSpecifier).toBe( - relative(routeDir, packageDistFile).replace(/\\/g, '/') - ); - }); -}); diff --git a/packages/next/src/builder-deferred.ts b/packages/next/src/builder-deferred.ts deleted file mode 100644 index 0b62c6cd8d..0000000000 --- a/packages/next/src/builder-deferred.ts +++ /dev/null @@ -1,2123 +0,0 @@ -import { createHash } from 'node:crypto'; -import { constants, existsSync, readFileSync } from 'node:fs'; -import { - access, - mkdir, - open, - readdir, - readFile, - rm, - stat, - writeFile, -} from 'node:fs/promises'; -import { createRequire } from 'node:module'; -import os from 'node:os'; -import { - basename, - dirname, - extname, - isAbsolute, - join, - relative, - resolve, -} from 'node:path'; -import type { WorkflowManifest } from '@workflow/builders'; -import { - cleanupStaleSocketInfoFiles, - createSocketServer, - SOCKET_INFO_FILENAME, - type SocketIO, - type SocketServerConfig, -} from './socket-server.js'; - -const ROUTE_STUB_FILE_MARKER = 'WORKFLOW_ROUTE_STUB_FILE'; -const ROUTE_STUB_MARKER_SCAN_BYTES = 4 * 1024; -const DEV_WORKFLOW_CODE_FILENAME = '__workflow_code.txt'; - -let CachedNextBuilderDeferred: any; - -// Create the deferred Next builder dynamically by extending the ESM BaseBuilder. -// Exported as getNextBuilderDeferred() to allow CommonJS modules to import from -// the ESM @workflow/builders package via dynamic import at runtime. -export async function getNextBuilderDeferred() { - if (CachedNextBuilderDeferred) { - return CachedNextBuilderDeferred; - } - - // V2: STEP_QUEUE_TRIGGER and enhanced-resolve infrastructure - // were removed because the V2 combined handler eliminates the separate step - // route/topic. The step copy import rewriting (getRelativeImportSpecifier, - // getStepCopyFileName, rewriteRelativeImportsForCopiedStep) from main was also - // removed — V2 doesn't use step copies. If step copy support is needed, it - // should land as a complete feature set. - const { - BaseBuilder: BaseBuilderClass, - WORKFLOW_QUEUE_TRIGGER, - createWorkflowEntrypointOptionsCode, - detectWorkflowPatterns, - applySwcTransform, - getImportPath, - resolveWorkflowAliasRelativePath, - // biome-ignore lint/security/noGlobalEval: Need to use eval here to avoid TypeScript from transpiling the import statement into `require()` - } = (await eval( - 'import("@workflow/builders")' - )) as typeof import('@workflow/builders'); - - class NextDeferredBuilder extends BaseBuilderClass { - private socketIO?: SocketIO; - private readonly discoveredWorkflowFiles = new Set(); - private readonly discoveredStepFiles = new Set(); - private readonly discoveredSerdeFiles = new Set(); - private trackedDependencyFiles = new Set(); - private deferredBuildQueue = Promise.resolve(); - private cacheInitialized = false; - private cacheWriteTimer: NodeJS.Timeout | null = null; - private deferredRebuildTimer: NodeJS.Timeout | null = null; - private lastDeferredBuildSignature: string | null = null; - - async build() { - const outputDir = await this.findAppDirectory(); - - await this.initializeDiscoveryState(); - await this.cleanupGeneratedArtifactsOnBoot(outputDir); - - await this.writeStubFiles(outputDir); - await this.createDiscoverySocketServer(); - } - - async onBeforeDeferredEntries(): Promise { - await this.initializeDiscoveryState(); - await this.validateDiscoveredEntryFiles(); - const implicitStepFiles = await this.resolveImplicitStepFiles(); - - const pendingBuild = this.deferredBuildQueue.then(() => - this.buildDeferredEntriesUntilStable(implicitStepFiles) - ); - - // Keep the queue chain alive even when the current build fails so future - // callbacks can enqueue another attempt without triggering unhandled - // rejection warnings. - this.deferredBuildQueue = pendingBuild.catch(() => { - // Error is surfaced through `pendingBuild` below. - }); - - await pendingBuild; - } - - private getCurrentInputFiles(implicitStepFiles: string[]): string[] { - return Array.from( - new Set([ - ...this.discoveredWorkflowFiles, - ...this.discoveredStepFiles, - ...this.discoveredSerdeFiles, - ...implicitStepFiles, - ]) - ).sort(); - } - - private async buildDeferredEntriesUntilStable( - implicitStepFiles: string[] - ): Promise { - // A successful build can discover additional transitive dependency files - // (via source maps), which changes the signature and may require one more - // build pass to include newly discovered serde files. - const maxBuildPasses = 3; - - for (let buildPass = 0; buildPass < maxBuildPasses; buildPass++) { - const inputFiles = this.getCurrentInputFiles(implicitStepFiles); - const buildSignature = - await this.createDeferredBuildSignature(inputFiles); - const shouldForceBuildForGeneratedRoutes = - await this.shouldForceBuildForGeneratedRoutes(); - if ( - buildSignature === this.lastDeferredBuildSignature && - !shouldForceBuildForGeneratedRoutes - ) { - return; - } - - try { - await this.buildDiscoveredFiles(inputFiles, implicitStepFiles); - } catch (error) { - if (this.config.watch) { - await this.validateDiscoveredEntryFiles(); - const recoveredInputFiles = - this.getCurrentInputFiles(implicitStepFiles); - const recoveredSignature = - await this.createDeferredBuildSignature(recoveredInputFiles); - if (recoveredSignature !== buildSignature) { - // A file was added/removed while this build was running; retry - // immediately with the refreshed discovered-entry state. - continue; - } - console.warn( - '[workflow] Deferred entries build failed. Will retry only after inputs change.', - error - ); - this.lastDeferredBuildSignature = buildSignature; - return; - } else { - throw error; - } - } - this.lastDeferredBuildSignature = buildSignature; - - if (!this.config.watch) { - // Production builds can persist newly discovered deferred-entry files to - // the cache after the first pass completes. Reload that cache before we - // decide whether the input signature stabilized so staged tarball builds - // can immediately replay with the expanded step set. - await new Promise((resolve) => setTimeout(resolve, 250)); - await this.loadWorkflowsCache(); - } - - const postBuildInputFiles = - this.getCurrentInputFiles(implicitStepFiles); - const postBuildSignature = - await this.createDeferredBuildSignature(postBuildInputFiles); - if (postBuildSignature === buildSignature) { - return; - } - } - - console.warn( - '[workflow] Deferred entries build signature did not stabilize after 3 passes.' - ); - } - - private async resolveImplicitStepFiles(): Promise { - const workflowStdlibPath = this.resolveWorkflowStdlibStepFilePath(); - return workflowStdlibPath ? [workflowStdlibPath] : []; - } - - private async shouldForceBuildForGeneratedRoutes(): Promise { - const outputDir = await this.findAppDirectory(); - const generatedRouteFiles = [ - join(outputDir, '.well-known/workflow/v1/flow/route.js'), - join(outputDir, '.well-known/workflow/v1/webhook/[token]/route.js'), - ]; - - for (const routeFilePath of generatedRouteFiles) { - const routeState = await this.getGeneratedRouteState(routeFilePath); - if (routeState === 'missing' || routeState === 'stub') { - return true; - } - } - - return false; - } - - private async getGeneratedRouteState( - routeFilePath: string - ): Promise<'missing' | 'stub' | 'generated'> { - let routeStats: Awaited>; - try { - routeStats = await stat(routeFilePath); - } catch { - return 'missing'; - } - if (!routeStats.isFile()) { - return 'missing'; - } - - try { - const routeFileHandle = await open(routeFilePath, 'r'); - try { - const markerScanBuffer = Buffer.alloc(ROUTE_STUB_MARKER_SCAN_BYTES); - const { bytesRead } = await routeFileHandle.read( - markerScanBuffer, - 0, - ROUTE_STUB_MARKER_SCAN_BYTES, - 0 - ); - const markerScanSource = markerScanBuffer.toString( - 'utf8', - 0, - bytesRead - ); - return markerScanSource.includes(ROUTE_STUB_FILE_MARKER) - ? 'stub' - : 'generated'; - } finally { - await routeFileHandle.close(); - } - } catch { - return 'missing'; - } - } - - private resolveWorkflowStdlibStepFilePath(): string | null { - let workflowCjsEntry: string; - try { - workflowCjsEntry = require.resolve('workflow', { - paths: [this.config.workingDir], - }); - } catch { - return null; - } - - const workflowDistDir = dirname(workflowCjsEntry); - const workflowStdlibPath = this.normalizeDiscoveredFilePath( - join(workflowDistDir, 'stdlib.js') - ); - return existsSync(workflowStdlibPath) ? workflowStdlibPath : null; - } - - private async reconcileDiscoveredEntries({ - workflowCandidates, - stepCandidates, - serdeCandidates, - validatePatterns, - }: { - workflowCandidates: Iterable; - stepCandidates: Iterable; - serdeCandidates?: Iterable; - validatePatterns: boolean; - }): Promise<{ - workflowFiles: Set; - stepFiles: Set; - serdeFiles: Set; - }> { - const candidatesByFile = new Map< - string, - { - hasWorkflowCandidate: boolean; - hasStepCandidate: boolean; - hasSerdeCandidate: boolean; - } - >(); - - for (const filePath of workflowCandidates) { - const normalizedPath = this.normalizeDiscoveredFilePath(filePath); - const existing = candidatesByFile.get(normalizedPath); - if (existing) { - existing.hasWorkflowCandidate = true; - } else { - candidatesByFile.set(normalizedPath, { - hasWorkflowCandidate: true, - hasStepCandidate: false, - hasSerdeCandidate: false, - }); - } - } - - for (const filePath of stepCandidates) { - const normalizedPath = this.normalizeDiscoveredFilePath(filePath); - const existing = candidatesByFile.get(normalizedPath); - if (existing) { - existing.hasStepCandidate = true; - } else { - candidatesByFile.set(normalizedPath, { - hasWorkflowCandidate: false, - hasStepCandidate: true, - hasSerdeCandidate: false, - }); - } - } - - if (serdeCandidates) { - for (const filePath of serdeCandidates) { - const normalizedPath = this.normalizeDiscoveredFilePath(filePath); - const existing = candidatesByFile.get(normalizedPath); - if (existing) { - existing.hasSerdeCandidate = true; - } else { - candidatesByFile.set(normalizedPath, { - hasWorkflowCandidate: false, - hasStepCandidate: false, - hasSerdeCandidate: true, - }); - } - } - } - - const fileEntries = Array.from(candidatesByFile.entries()).sort( - ([a], [b]) => a.localeCompare(b) - ); - const validatedEntries = await Promise.all( - fileEntries.map(async ([filePath, candidates]) => { - try { - const fileStats = await stat(filePath); - if (!fileStats.isFile()) { - return null; - } - - if (!validatePatterns) { - return { - filePath, - hasUseWorkflow: candidates.hasWorkflowCandidate, - hasUseStep: candidates.hasStepCandidate, - hasSerde: candidates.hasSerdeCandidate, - }; - } - - const source = await readFile(filePath, 'utf-8'); - const patterns = detectWorkflowPatterns(source); - return { - filePath, - hasUseWorkflow: patterns.hasUseWorkflow, - hasUseStep: patterns.hasUseStep, - hasSerde: patterns.hasSerde, - }; - } catch { - return null; - } - }) - ); - - const workflowFiles = new Set(); - const stepFiles = new Set(); - const serdeFiles = new Set(); - for (const entry of validatedEntries) { - if (!entry) { - continue; - } - if (entry.hasUseWorkflow) { - workflowFiles.add(entry.filePath); - } - if (entry.hasUseStep) { - stepFiles.add(entry.filePath); - } - if (entry.hasSerde) { - serdeFiles.add(entry.filePath); - } - } - - return { workflowFiles, stepFiles, serdeFiles }; - } - - private async validateDiscoveredEntryFiles(): Promise { - const workflowCandidates = new Set(this.discoveredWorkflowFiles); - const stepCandidates = new Set(this.discoveredStepFiles); - const serdeCandidates = new Set(this.discoveredSerdeFiles); - const { workflowFiles, stepFiles, serdeFiles } = - await this.reconcileDiscoveredEntries({ - workflowCandidates, - stepCandidates, - serdeCandidates, - validatePatterns: true, - }); - - // Reconcile validated entries against the snapshot we started with so - // file discoveries that arrive during validation are preserved. - let workflowsChanged = false; - let stepsChanged = false; - let serdeChanged = false; - - for (const filePath of workflowCandidates) { - if (!workflowFiles.has(filePath)) { - workflowsChanged = - this.discoveredWorkflowFiles.delete(filePath) || workflowsChanged; - } - } - for (const filePath of workflowFiles) { - if (!this.discoveredWorkflowFiles.has(filePath)) { - this.discoveredWorkflowFiles.add(filePath); - workflowsChanged = true; - } - } - - for (const filePath of stepCandidates) { - if (!stepFiles.has(filePath)) { - stepsChanged = - this.discoveredStepFiles.delete(filePath) || stepsChanged; - } - } - for (const filePath of stepFiles) { - if (!this.discoveredStepFiles.has(filePath)) { - this.discoveredStepFiles.add(filePath); - stepsChanged = true; - } - } - - for (const filePath of serdeCandidates) { - if (!serdeFiles.has(filePath)) { - serdeChanged = - this.discoveredSerdeFiles.delete(filePath) || serdeChanged; - } - } - for (const filePath of serdeFiles) { - if (!this.discoveredSerdeFiles.has(filePath)) { - this.discoveredSerdeFiles.add(filePath); - serdeChanged = true; - } - } - - if (workflowsChanged || stepsChanged) { - this.scheduleWorkflowsCacheWrite(); - } - } - - private async buildDiscoveredFiles( - inputFiles: string[], - implicitStepFiles: string[] - ) { - const outputDir = await this.findAppDirectory(); - const workflowGeneratedDir = join(outputDir, '.well-known/workflow/v1'); - const cacheDir = join(this.config.workingDir, this.getDistDir(), 'cache'); - await mkdir(cacheDir, { recursive: true }); - const manifestBuildDir = join(cacheDir, 'workflow-generated-manifest'); - const tempRouteFileName = 'route.js.temp'; - const trackedDiscoveredEntries = - await this.collectTrackedDiscoveredEntries(); - const discoveredStepFileCandidates = Array.from( - new Set([ - ...this.discoveredStepFiles, - ...trackedDiscoveredEntries.discoveredSteps, - ...implicitStepFiles, - ]) - ).sort(); - const discoveredWorkflowFileCandidates = Array.from( - new Set([ - ...this.discoveredWorkflowFiles, - ...trackedDiscoveredEntries.discoveredWorkflows, - ]) - ).sort(); - const discoveredSerdeFileCandidates = Array.from( - new Set([ - ...this.discoveredSerdeFiles, - ...trackedDiscoveredEntries.discoveredSerdeFiles, - ]) - ).sort(); - const discoveredWorkflowFiles = await this.filterExistingFiles( - discoveredWorkflowFileCandidates - ); - const existingStepFileCandidates = await this.filterExistingFiles( - discoveredStepFileCandidates - ); - const discoveredStepFiles = await this.collectTransitiveStepFiles({ - entryFiles: [...existingStepFileCandidates, ...discoveredWorkflowFiles], - stepFiles: existingStepFileCandidates, - }); - const existingSerdeFileCandidates = await this.filterExistingFiles( - discoveredSerdeFileCandidates - ); - const discoveredSerdeFiles = await this.collectTransitiveSerdeFiles({ - entryFiles: [...discoveredStepFiles, ...discoveredWorkflowFiles], - serdeFiles: existingSerdeFileCandidates, - }); - const existingInputFiles = await this.filterExistingFiles(inputFiles); - const buildInputFiles = Array.from( - new Set([ - ...existingInputFiles, - ...discoveredStepFiles, - ...discoveredWorkflowFiles, - ...discoveredSerdeFiles, - ]) - ).sort(); - const discoveredEntries = { - discoveredSteps: new Set(discoveredStepFiles), - discoveredWorkflows: new Set(discoveredWorkflowFiles), - discoveredSerdeFiles: new Set(discoveredSerdeFiles), - }; - - // Ensure output directories exist - await mkdir(workflowGeneratedDir, { recursive: true }); - - await this.writeFileIfChanged( - join(workflowGeneratedDir, '.gitignore'), - '*' - ); - - const tsconfigPath = await this.findTsConfigPath(); - - const flowRouteDir = join(workflowGeneratedDir, 'flow'); - await mkdir(flowRouteDir, { recursive: true }); - - const combinedResult = await this.createDeferredFlowRoute({ - inputFiles: buildInputFiles, - flowOutfile: join(flowRouteDir, tempRouteFileName), - tsconfigPath, - discoveredEntries, - }); - await this.buildWebhookRoute({ - workflowGeneratedDir, - routeFileName: tempRouteFileName, - }); - await this.refreshTrackedDependencyFiles( - workflowGeneratedDir, - tempRouteFileName - ); - - const manifest = { - steps: { ...combinedResult?.manifest?.steps }, - workflows: { ...combinedResult?.manifest?.workflows }, - classes: { ...combinedResult?.manifest?.classes }, - }; - - const manifestFilePath = join(workflowGeneratedDir, 'manifest.json'); - const manifestBuildPath = join(manifestBuildDir, 'manifest.json'); - const workflowBundlePath = join( - workflowGeneratedDir, - `flow/${tempRouteFileName}` - ); - const manifestJson = await this.createManifest({ - workflowBundlePath, - manifestDir: manifestBuildDir, - manifest, - }); - if (manifestJson) { - await this.rewriteJsonFileWithStableKeyOrder(manifestBuildPath); - await this.copyFileIfChanged(manifestBuildPath, manifestFilePath); - } else { - await rm(manifestBuildPath, { force: true }); - await rm(manifestFilePath, { force: true }); - } - - await this.writeFunctionsConfig(outputDir); - - // V2: Combined route (flow) — step registrations already at final path - await this.copyFileIfChanged( - join(workflowGeneratedDir, `flow/${tempRouteFileName}`), - join(workflowGeneratedDir, 'flow/route.js') - ); - await this.copyFileIfChanged( - join(workflowGeneratedDir, `webhook/[token]/${tempRouteFileName}`), - join(workflowGeneratedDir, 'webhook/[token]/route.js') - ); - - // Expose manifest as a static file when WORKFLOW_PUBLIC_MANIFEST=1. - // Next.js serves files from public/ at the root URL. - if (this.shouldExposePublicManifest && manifestJson) { - const publicManifestDir = join( - this.config.workingDir, - 'public/.well-known/workflow/v1' - ); - await mkdir(publicManifestDir, { recursive: true }); - if (process.env.VERCEL_DEPLOYMENT_ID === undefined) { - await this.writeFileIfChanged( - join(publicManifestDir, '.gitignore'), - '*' - ); - } - await this.copyFileIfChanged( - manifestFilePath, - join(publicManifestDir, 'manifest.json') - ); - } - - // Notify deferred entry loaders waiting on route.js stubs. - this.socketIO?.emit('build-complete'); - } - - private async createDeferredFlowRoute({ - inputFiles, - flowOutfile, - tsconfigPath, - discoveredEntries, - }: { - inputFiles: string[]; - flowOutfile: string; - tsconfigPath?: string; - discoveredEntries: { - discoveredSteps: Set; - discoveredWorkflows: Set; - discoveredSerdeFiles: Set; - }; - }): Promise<{ manifest: WorkflowManifest }> { - const workflowResult = await this.createWorkflowsBundle({ - inputFiles, - outfile: `${flowOutfile}.__wf_tmp.js`, - format: 'esm', - bundleFinalOutput: false, - tsconfigPath, - discoveredEntries, - }); - - const workflowVMCode = workflowResult.interimBundleText; - if (!workflowVMCode) { - throw new Error( - 'createWorkflowsBundle did not return interimBundleText' - ); - } - - try { - await rm(`${flowOutfile}.__wf_tmp.js`, { force: true }); - } catch { - // Ignore cleanup errors. - } - - const stepAndSerdeFiles = Array.from( - new Set([ - ...discoveredEntries.discoveredSteps, - ...discoveredEntries.discoveredSerdeFiles, - ]) - ).sort(); - const stepImports = await this.createDeferredStepSideEffectImports( - stepAndSerdeFiles, - dirname(flowOutfile) - ); - const stepManifest = - await this.createDeferredStepManifest(stepAndSerdeFiles); - const escapedVMCode = workflowVMCode.replace(/[\\`$]/g, '\\$&'); - const workflowEntrypointOptionsCode = - createWorkflowEntrypointOptionsCode(); - let routeCode: string; - - if (this.config.watch) { - const workflowCodePath = join( - dirname(flowOutfile), - DEV_WORKFLOW_CODE_FILENAME - ); - await this.writeFileIfChanged(workflowCodePath, workflowVMCode); - routeCode = `// biome-ignore-all lint: generated file -/* eslint-disable */ -import 'workflow/internal/builtins'; -${stepImports} -import { readFile, stat } from 'node:fs/promises'; -import { workflowEntrypoint } from 'workflow/runtime'; - -const workflowCodePath = ${JSON.stringify(workflowCodePath)}; -let cachedWorkflowHandler; -let cachedWorkflowCodeSignature; - -async function getWorkflowHandler() { - const workflowCodeStats = await stat(workflowCodePath); - const workflowCodeSignature = \`\${workflowCodeStats.size}:\${workflowCodeStats.mtimeMs}\`; - if (!cachedWorkflowHandler || cachedWorkflowCodeSignature !== workflowCodeSignature) { - const workflowCode = await readFile(workflowCodePath, 'utf8'); - cachedWorkflowHandler = workflowEntrypoint(workflowCode${workflowEntrypointOptionsCode}); - cachedWorkflowCodeSignature = workflowCodeSignature; - } - return cachedWorkflowHandler; -} - -export async function POST(req) { - return (await getWorkflowHandler())(req); -}`; - } else { - routeCode = `// biome-ignore-all lint: generated file -/* eslint-disable */ -import 'workflow/internal/builtins'; -${stepImports} -import { workflowEntrypoint } from 'workflow/runtime'; - -const workflowCode = \`${escapedVMCode}\`; - -export const POST = workflowEntrypoint(workflowCode${workflowEntrypointOptionsCode});`; - } - - await this.writeFileIfChanged(flowOutfile, routeCode); - - return { - manifest: { - ...stepManifest, - workflows: { - ...stepManifest.workflows, - ...workflowResult.manifest.workflows, - }, - classes: { - ...stepManifest.classes, - ...workflowResult.manifest.classes, - }, - }, - }; - } - - private async createDeferredStepSideEffectImports( - files: string[], - routeDir: string - ): Promise { - const importSpecifiers = await Promise.all( - files.map((file) => - this.getDeferredRouteImportSpecifier(file, routeDir) - ) - ); - - return importSpecifiers - .map((specifier) => `import ${JSON.stringify(specifier)};`) - .join('\n'); - } - - private async getDeferredRouteImportSpecifier( - file: string, - routeDir: string - ): Promise { - const { importPath, isPackage } = getImportPath( - file, - this.config.workingDir - ); - - if (isPackage) { - return importPath; - } - - const absolutePath = await this.resolveDeferredRouteImportFilePath( - resolve(this.config.workingDir, importPath) - ); - let relativePath = relative(routeDir, absolutePath).replace(/\\/g, '/'); - if (!relativePath.startsWith('./') && !relativePath.startsWith('../')) { - relativePath = `./${relativePath}`; - } - return relativePath; - } - - private async resolveDeferredRouteImportFilePath( - filePath: string - ): Promise { - const packageDistPath = - this.resolvePackageSourceDistFilePathForRouteImport(filePath); - if (packageDistPath) { - return packageDistPath; - } - - return filePath; - } - - private resolvePackageSourceDistFilePathForRouteImport( - filePath: string - ): string | null { - const relativeToWorkingDir = relative(this.config.workingDir, filePath); - if ( - !relativeToWorkingDir.startsWith('..') && - !isAbsolute(relativeToWorkingDir) - ) { - return null; - } - - const packageRoot = this.findPackageRootForRouteImportSource(filePath); - if (!packageRoot) { - return null; - } - - const relativeToPackageRoot = relative(packageRoot, filePath).replace( - /\\/g, - '/' - ); - const srcPrefix = 'src/'; - if (!relativeToPackageRoot.startsWith(srcPrefix)) { - return null; - } - - const extension = extname(relativeToPackageRoot); - const outputExtension = - extension === '.mts' ? '.mjs' : extension === '.cts' ? '.cjs' : '.js'; - const withoutExtension = relativeToPackageRoot.slice( - srcPrefix.length, - relativeToPackageRoot.length - extension.length - ); - const distPath = join( - packageRoot, - 'dist', - `${withoutExtension}${outputExtension}` - ); - - return existsSync(distPath) ? distPath : null; - } - - private findPackageRootForRouteImportSource( - filePath: string - ): string | null { - let currentDir = dirname(filePath); - - while (true) { - if (existsSync(join(currentDir, 'package.json'))) { - return currentDir; - } - - const parentDir = dirname(currentDir); - if (parentDir === currentDir) { - return null; - } - currentDir = parentDir; - } - } - - private async createDeferredStepManifest( - files: string[] - ): Promise { - const manifest: WorkflowManifest = {}; - const manifestFiles = Array.from( - new Set([ - ...files, - ...(await this.resolveBuiltInStepFilesForManifest()), - ]) - ).sort(); - - for (const file of manifestFiles) { - const source = await readFile(file, 'utf8').catch(() => undefined); - if (!source) { - continue; - } - - const relativeFilepath = await this.getRelativeFilenameForSwc(file); - const { workflowManifest } = await applySwcTransform( - relativeFilepath, - source, - 'step', - file, - this.transformProjectRoot, - this.moduleSpecifierRoot - ); - this.mergeWorkflowManifest(manifest, workflowManifest); - } - - return manifest; - } - - private async resolveBuiltInStepFilesForManifest(): Promise { - try { - const workflowRequire = createRequire( - join(this.config.workingDir, 'package.json') - ); - return [ - this.normalizeDiscoveredFilePath( - workflowRequire.resolve('workflow/internal/builtins') - ), - ]; - } catch { - return []; - } - } - - private mergeWorkflowManifest( - target: WorkflowManifest, - source: WorkflowManifest - ): void { - target.workflows = Object.assign( - target.workflows || {}, - source.workflows - ); - target.steps = Object.assign(target.steps || {}, source.steps); - target.classes = Object.assign(target.classes || {}, source.classes); - } - - private async cleanupGeneratedArtifactsOnBoot( - outputDir: string - ): Promise { - const workflowGeneratedDir = join(outputDir, '.well-known/workflow/v1'); - const flowRouteDir = join(workflowGeneratedDir, 'flow'); - const stepRouteDir = join(workflowGeneratedDir, 'step'); - const webhookRouteDir = join(workflowGeneratedDir, 'webhook/[token]'); - - const staleArtifactPaths = [ - join(flowRouteDir, 'route.js.temp'), - join(flowRouteDir, 'route.js.temp.debug.json'), - join(flowRouteDir, 'route.js.debug.json'), - join(flowRouteDir, DEV_WORKFLOW_CODE_FILENAME), - join(flowRouteDir, '__step_registrations.route.js.temp'), - join(flowRouteDir, '__step_registrations.route.js.temp.debug.json'), - // V2: clean up stale V1 step route directory - stepRouteDir, - join(webhookRouteDir, 'route.js.temp'), - join(workflowGeneratedDir, 'manifest.json'), - ]; - - await Promise.all([ - ...staleArtifactPaths.map((stalePath) => - rm(stalePath, { recursive: true, force: true }) - ), - cleanupStaleSocketInfoFiles( - join(this.config.workingDir, this.getDistDir()) - ), - this.removeStaleDeferredTempFiles(flowRouteDir), - this.removeStaleDeferredTempFiles(webhookRouteDir), - ]); - } - - private async removeStaleDeferredTempFiles( - routeDir: string - ): Promise { - const routeEntries = await readdir(routeDir, { - withFileTypes: true, - }).catch(() => []); - await Promise.all( - routeEntries - .filter( - (entry) => - entry.isFile() && - entry.name.startsWith('route.js.') && - entry.name.endsWith('.tmp') - ) - .map((entry) => - rm(join(routeDir, entry.name), { - force: true, - }) - ) - ); - } - - private async createDiscoverySocketServer(): Promise { - if (this.socketIO || process.env.WORKFLOW_SOCKET_PORT) { - return; - } - - process.env.WORKFLOW_SOCKET_INFO_PATH = this.getSocketInfoFilePath(); - const config: SocketServerConfig = { - isDevServer: Boolean(this.config.watch), - socketInfoFilePath: this.getSocketInfoFilePath(), - onFileDiscovered: ( - filePath: string, - hasWorkflow: boolean, - hasStep: boolean, - hasSerde: boolean - ) => { - const normalizedFilePath = this.normalizeDiscoveredFilePath(filePath); - let hasCacheTrackingChange = false; - const wasTrackedDependency = - this.trackedDependencyFiles.has(normalizedFilePath); - - if (hasWorkflow) { - if (!this.discoveredWorkflowFiles.has(normalizedFilePath)) { - this.discoveredWorkflowFiles.add(normalizedFilePath); - hasCacheTrackingChange = true; - } - } else { - const wasDeleted = - this.discoveredWorkflowFiles.delete(normalizedFilePath); - hasCacheTrackingChange = wasDeleted || hasCacheTrackingChange; - } - - if (hasStep) { - if (!this.discoveredStepFiles.has(normalizedFilePath)) { - this.discoveredStepFiles.add(normalizedFilePath); - hasCacheTrackingChange = true; - } - } else { - const wasDeleted = - this.discoveredStepFiles.delete(normalizedFilePath); - hasCacheTrackingChange = wasDeleted || hasCacheTrackingChange; - } - - if (hasSerde) { - if (!this.discoveredSerdeFiles.has(normalizedFilePath)) { - hasCacheTrackingChange = true; - } - this.discoveredSerdeFiles.add(normalizedFilePath); - } else { - const wasDeleted = - this.discoveredSerdeFiles.delete(normalizedFilePath); - hasCacheTrackingChange = wasDeleted || hasCacheTrackingChange; - } - - if (hasCacheTrackingChange) { - this.scheduleWorkflowsCacheWrite(); - } - - if ( - hasWorkflow || - hasStep || - hasSerde || - hasCacheTrackingChange || - wasTrackedDependency - ) { - this.scheduleDeferredRebuild(); - } - }, - onTriggerBuild: () => { - this.scheduleDeferredRebuild(); - }, - }; - - this.socketIO = await createSocketServer(config); - } - - private async initializeDiscoveryState(): Promise { - if (this.cacheInitialized) { - return; - } - - await this.loadWorkflowsCache(); - // Deferred mode must not run eager input-graph discovery; entries are - // discovered via loader->socket notifications during Next's build. - this.cacheInitialized = true; - } - - private getDistDir(): string { - return (this.config as { distDir?: string }).distDir || '.next'; - } - - private getWorkflowsCacheFilePath(): string { - return join( - this.config.workingDir, - this.getDistDir(), - 'cache', - 'workflows.json' - ); - } - - private getSocketInfoFilePath(): string { - return join( - this.config.workingDir, - this.getDistDir(), - SOCKET_INFO_FILENAME - ); - } - - private findPackageJsonPath(filePath: string): string | null { - let currentDir = dirname(filePath); - let previousDir = ''; - - while (currentDir !== previousDir) { - const packageJsonPath = join(currentDir, 'package.json'); - if (existsSync(packageJsonPath)) { - return packageJsonPath; - } - previousDir = currentDir; - currentDir = dirname(currentDir); - } - - return null; - } - - private shouldPreferSourceBackedPackagePath(filePath: string): boolean { - const normalizedPath = filePath.replace(/\\/g, '/'); - // Only prefer source for workspace packages (not in node_modules). - // For tarball-installed packages, using source-backed paths causes - // esbuild to bundle the full source tree (including world.ts with - // process.cwd()) instead of externalizing properly. - if ( - normalizedPath.includes('/packages/') && - !normalizedPath.includes('/node_modules/') - ) { - return true; - } - - if (!normalizedPath.includes('/node_modules/')) { - return false; - } - - const packageJsonPath = this.findPackageJsonPath(filePath); - if (!packageJsonPath) { - return false; - } - - try { - const packageJson = JSON.parse( - readFileSync(packageJsonPath, 'utf-8') - ) as { name?: unknown }; - return ( - packageJson.name === 'workflow' || - (typeof packageJson.name === 'string' && - packageJson.name.startsWith('@workflow/')) - ); - } catch { - return false; - } - } - - private normalizeDiscoveredFilePath(filePath: string): string { - return isAbsolute(filePath) - ? filePath - : resolve(this.config.workingDir, filePath); - } - - private async filterExistingFiles(filePaths: string[]): Promise { - const normalizedFilePaths = Array.from( - new Set( - filePaths.map((filePath) => - this.normalizeDiscoveredFilePath(filePath) - ) - ) - ).sort(); - - const existingFiles = await Promise.all( - normalizedFilePaths.map(async (filePath) => { - try { - const fileStats = await stat(filePath); - return fileStats.isFile() ? filePath : null; - } catch { - return null; - } - }) - ); - - return existingFiles.filter((filePath): filePath is string => - Boolean(filePath) - ); - } - - private async createDeferredBuildSignature( - inputFiles: string[] - ): Promise { - const normalizedFiles = Array.from( - new Set([ - ...inputFiles.map((filePath) => - this.normalizeDiscoveredFilePath(filePath) - ), - ...this.trackedDependencyFiles, - ]) - ).sort(); - - const signatureParts = await Promise.all( - normalizedFiles.map(async (filePath) => { - try { - const fileStats = await stat(filePath); - return `${filePath}:${fileStats.size}:${Math.trunc(fileStats.mtimeMs)}`; - } catch { - return `${filePath}:missing`; - } - }) - ); - - const signatureHash = createHash('sha256'); - for (const signaturePart of signatureParts) { - signatureHash.update(signaturePart); - signatureHash.update('\n'); - } - - return signatureHash.digest('hex'); - } - - private async collectTrackedDiscoveredEntries(): Promise<{ - discoveredSteps: string[]; - discoveredWorkflows: string[]; - discoveredSerdeFiles: string[]; - }> { - if (this.trackedDependencyFiles.size === 0) { - return { - discoveredSteps: [], - discoveredWorkflows: [], - discoveredSerdeFiles: [], - }; - } - - const { workflowFiles, stepFiles, serdeFiles } = - await this.reconcileDiscoveredEntries({ - workflowCandidates: this.trackedDependencyFiles, - stepCandidates: this.trackedDependencyFiles, - serdeCandidates: this.trackedDependencyFiles, - validatePatterns: true, - }); - - return { - discoveredSteps: Array.from(stepFiles).sort(), - discoveredWorkflows: Array.from(workflowFiles).sort(), - discoveredSerdeFiles: Array.from(serdeFiles).sort(), - }; - } - - private async refreshTrackedDependencyFiles( - workflowGeneratedDir: string, - routeFileName: string - ): Promise { - const bundleFiles = [ - join(workflowGeneratedDir, `step/${routeFileName}`), - join(workflowGeneratedDir, `flow/${routeFileName}`), - ]; - const trackedFiles = new Set(); - - for (const bundleFile of bundleFiles) { - const bundleSources = await this.extractBundleSourceFiles(bundleFile); - for (const sourceFile of bundleSources) { - trackedFiles.add(sourceFile); - } - } - - if (trackedFiles.size > 0) { - this.trackedDependencyFiles = trackedFiles; - } - } - - private async extractBundleSourceFiles( - bundleFilePath: string - ): Promise { - let bundleContents: string; - try { - bundleContents = await readFile(bundleFilePath, 'utf-8'); - } catch { - return []; - } - - const baseDirectory = dirname(bundleFilePath); - const localSourceFiles = new Set(); - - // Extract inline base64 sourcemaps via string scanning. A previous - // implementation used `bundleContents.matchAll(/...[A-Za-z0-9+/=]+.../g)` - // which overflows V8's regex stack - // (`RangeError: Maximum call stack size exceeded at RegExpStringIterator.next`) - // on bundles with large inline sourcemaps — V8's irregexp uses - // recursion for greedy character-class quantifiers and certain long - // base64 inputs exhaust the call stack. - const sourceMapPrefix = '//# sourceMappingURL=data:application/json'; - const base64Marker = ';base64,'; - const sourceMapMatches: Array<{ base64Value: string }> = []; - let scanFrom = 0; - while (scanFrom < bundleContents.length) { - const prefixIdx = bundleContents.indexOf(sourceMapPrefix, scanFrom); - if (prefixIdx === -1) break; - const base64Start = bundleContents.indexOf( - base64Marker, - prefixIdx + sourceMapPrefix.length - ); - if (base64Start === -1) { - scanFrom = prefixIdx + sourceMapPrefix.length; - continue; - } - // Bail if a comma appears before `;base64,` — that means this - // wasn't a `data:application/json[;params];base64,...` URL after - // all (the comma would terminate the data URL parameters). - const commaBefore = bundleContents.indexOf(',', prefixIdx); - if (commaBefore !== -1 && commaBefore < base64Start) { - scanFrom = base64Start; - continue; - } - const valueStart = base64Start + base64Marker.length; - // Base64 payload terminates at first non-base64 char (newline, - // closing `*/`, etc.). Scanning a small alphabet by char is far - // cheaper than backtracking a regex over a multi-MB capture. - let valueEnd = valueStart; - while (valueEnd < bundleContents.length) { - const code = bundleContents.charCodeAt(valueEnd); - // A-Z 0x41-0x5A, a-z 0x61-0x7A, 0-9 0x30-0x39, '+' 0x2B, - // '/' 0x2F, '=' 0x3D - if ( - !( - (code >= 0x41 && code <= 0x5a) || - (code >= 0x61 && code <= 0x7a) || - (code >= 0x30 && code <= 0x39) || - code === 0x2b || - code === 0x2f || - code === 0x3d - ) - ) { - break; - } - valueEnd++; - } - if (valueEnd > valueStart) { - sourceMapMatches.push({ - base64Value: bundleContents.slice(valueStart, valueEnd), - }); - } - scanFrom = valueEnd; - } - - for (const match of sourceMapMatches) { - const base64Value = match.base64Value; - if (!base64Value) { - continue; - } - - let sourceMap: { sourceRoot?: unknown; sources?: unknown }; - try { - sourceMap = JSON.parse( - Buffer.from(base64Value, 'base64').toString('utf-8') - ) as { sourceRoot?: unknown; sources?: unknown }; - } catch { - continue; - } - - const sourceRoot = - typeof sourceMap.sourceRoot === 'string' ? sourceMap.sourceRoot : ''; - const sources = Array.isArray(sourceMap.sources) - ? sourceMap.sources.filter( - (source): source is string => typeof source === 'string' - ) - : []; - - for (const source of sources) { - if (source.startsWith('webpack://') || source.startsWith('<')) { - continue; - } - - let resolvedSourcePath: string; - if (source.startsWith('file://')) { - try { - resolvedSourcePath = decodeURIComponent(new URL(source).pathname); - } catch { - continue; - } - } else if (isAbsolute(source)) { - resolvedSourcePath = source; - } else { - resolvedSourcePath = resolve(baseDirectory, sourceRoot, source); - } - - const normalizedSourcePath = - this.normalizeDiscoveredFilePath(resolvedSourcePath); - const normalizedSourcePathForCheck = normalizedSourcePath.replace( - /\\/g, - '/' - ); - const isSourceBackedPackagePath = - this.shouldPreferSourceBackedPackagePath(normalizedSourcePath); - if ( - normalizedSourcePathForCheck.includes('/.well-known/workflow/') || - (!isSourceBackedPackagePath && - normalizedSourcePathForCheck.includes('/node_modules/')) || - (!isSourceBackedPackagePath && - normalizedSourcePathForCheck.includes('/.pnpm/')) || - normalizedSourcePathForCheck.includes('/.next/') || - normalizedSourcePathForCheck.endsWith('/virtual-entry.js') - ) { - continue; - } - - localSourceFiles.add(normalizedSourcePath); - } - } - - return Array.from(localSourceFiles); - } - - private scheduleWorkflowsCacheWrite(): void { - if (this.cacheWriteTimer) { - clearTimeout(this.cacheWriteTimer); - } - - this.cacheWriteTimer = setTimeout(() => { - this.cacheWriteTimer = null; - void this.writeWorkflowsCache().catch((error) => { - console.warn('Failed to write workflow discovery cache', error); - }); - }, 50); - } - - private scheduleDeferredRebuild(): void { - if (this.deferredRebuildTimer) { - clearTimeout(this.deferredRebuildTimer); - } - - this.deferredRebuildTimer = setTimeout(() => { - this.deferredRebuildTimer = null; - void this.onBeforeDeferredEntries().catch((error) => { - console.warn( - '[workflow] Deferred rebuild after source update failed.', - error - ); - }); - }, 75); - } - - private async readWorkflowsCache(): Promise<{ - workflowFiles: string[]; - stepFiles: string[]; - } | null> { - const cacheFilePath = this.getWorkflowsCacheFilePath(); - - try { - const cacheContents = await readFile(cacheFilePath, 'utf-8'); - const parsed = JSON.parse(cacheContents) as { - workflowFiles?: unknown; - stepFiles?: unknown; - }; - - const workflowFiles = Array.isArray(parsed.workflowFiles) - ? parsed.workflowFiles.filter( - (item): item is string => typeof item === 'string' - ) - : []; - const stepFiles = Array.isArray(parsed.stepFiles) - ? parsed.stepFiles.filter( - (item): item is string => typeof item === 'string' - ) - : []; - - return { workflowFiles, stepFiles }; - } catch { - return null; - } - } - - private async loadWorkflowsCache(): Promise { - const cachedData = await this.readWorkflowsCache(); - if (!cachedData) { - return; - } - const { workflowFiles, stepFiles, serdeFiles } = - await this.reconcileDiscoveredEntries({ - workflowCandidates: cachedData.workflowFiles, - stepCandidates: cachedData.stepFiles, - serdeCandidates: this.discoveredSerdeFiles, - validatePatterns: true, - }); - - this.discoveredWorkflowFiles.clear(); - this.discoveredStepFiles.clear(); - this.discoveredSerdeFiles.clear(); - for (const filePath of workflowFiles) { - this.discoveredWorkflowFiles.add(filePath); - } - for (const filePath of stepFiles) { - this.discoveredStepFiles.add(filePath); - } - for (const filePath of serdeFiles) { - this.discoveredSerdeFiles.add(filePath); - } - } - - private async writeWorkflowsCache(): Promise { - const cacheFilePath = this.getWorkflowsCacheFilePath(); - const cacheDir = join(this.config.workingDir, this.getDistDir(), 'cache'); - await mkdir(cacheDir, { recursive: true }); - - const cacheData = { - workflowFiles: Array.from(this.discoveredWorkflowFiles).sort(), - stepFiles: Array.from(this.discoveredStepFiles).sort(), - }; - - await writeFile(cacheFilePath, JSON.stringify(cacheData, null, 2)); - } - - private async writeStubFiles(outputDir: string): Promise { - // Turbopack currently has a worker-concurrency limitation for pending - // virtual entries. Warn if parallelism is too low to reliably discover. - const parallelismCount = os.availableParallelism(); - if (process.env.TURBOPACK && parallelismCount < 4) { - console.warn( - `Available parallelism of ${parallelismCount} is less than needed 4. This can cause workflows/steps to fail to discover properly in turbopack` - ); - } - - const routeStubContent = [ - `// ${ROUTE_STUB_FILE_MARKER}`, - 'export const __workflowRouteStub = true;', - ].join('\n'); - const workflowGeneratedDir = join(outputDir, '.well-known/workflow/v1'); - - await mkdir(join(workflowGeneratedDir, 'flow'), { recursive: true }); - await mkdir(join(workflowGeneratedDir, 'webhook/[token]'), { - recursive: true, - }); - - await this.writeFileIfChanged( - join(workflowGeneratedDir, '.gitignore'), - '*' - ); - - // V2: Only flow + webhook stubs needed (no separate step route). - // Stubs are replaced by generated output once discovery finishes. - await this.writeFileIfChanged( - join(workflowGeneratedDir, 'flow/route.js'), - routeStubContent - ); - await this.writeFileIfChanged( - join(workflowGeneratedDir, 'webhook/[token]/route.js'), - routeStubContent - ); - } - - protected async getInputFiles(): Promise { - // Read Next.js's app-paths-manifest.json from a previous build to - // determine which files are actual route entrypoints. This avoids - // predicting Next.js conventions with regexes and instead reads - // from Next.js's own output. - const nextDir = join(this.config.workingDir, '.next'); - const manifestPath = join(nextDir, 'app-paths-manifest.json'); - try { - const manifestContent = readFileSync(manifestPath, 'utf-8'); - const manifest = JSON.parse(manifestContent) as Record; - // The manifest maps route paths to their source files. - // Extract the source file paths and resolve them. - const manifestFiles = new Set(); - for (const sourcePath of Object.values(manifest)) { - const resolved = resolve(nextDir, 'server', sourcePath); - // The manifest points to built output; find the source file - // by matching against the base builder's full file list. - manifestFiles.add(resolved); - } - - // Use the manifest route paths to filter the input files. - // A file is included if it matches a known route segment from - // the manifest (e.g., app/api/route contains 'app/api/route'). - const inputFiles = await super.getInputFiles(); - const routeSegments = Object.keys(manifest).map((route) => - route.replace(/^\//, '').replace(/\/route$/, '') - ); - return inputFiles.filter((item) => - routeSegments.some((segment) => item.includes(segment)) - ); - } catch { - // No manifest from a previous build — fall back to the base - // builder's full file scan. This is safe but slower; subsequent - // builds will use the manifest. - return super.getInputFiles(); - } - } - - private async writeFunctionsConfig(outputDir: string) { - // we don't run this in development mode as it's not needed - if (process.env.NODE_ENV === 'development') { - return; - } - // V2: Single combined trigger handles both workflow and step execution - const generatedConfig = { - version: '0', - workflows: { - maxDuration: 'max', - experimentalTriggers: [WORKFLOW_QUEUE_TRIGGER], - }, - }; - - await this.writeFileIfChanged( - join(outputDir, '.well-known/workflow/v1/config.json'), - JSON.stringify(generatedConfig, null, 2) - ); - } - - private async writeFileIfChanged( - filePath: string, - contents: string | Buffer - ): Promise { - const nextBuffer = Buffer.isBuffer(contents) - ? contents - : Buffer.from(contents); - - try { - const currentBuffer = await readFile(filePath); - if (currentBuffer.equals(nextBuffer)) { - return false; - } - } catch { - // File does not exist yet or cannot be read; write a fresh copy. - } - - await mkdir(dirname(filePath), { recursive: true }); - await writeFile(filePath, nextBuffer); - return true; - } - - private async copyFileIfChanged( - sourcePath: string, - destinationPath: string - ): Promise { - const sourceContents = await readFile(sourcePath); - return this.writeFileIfChanged(destinationPath, sourceContents); - } - - private sortJsonValue(value: unknown): unknown { - if (Array.isArray(value)) { - return value.map((item) => this.sortJsonValue(item)); - } - if (value && typeof value === 'object') { - const sortedEntries = Object.entries(value as Record) - .sort(([a], [b]) => a.localeCompare(b)) - .map(([key, entryValue]) => [key, this.sortJsonValue(entryValue)]); - return Object.fromEntries(sortedEntries); - } - return value; - } - - private async rewriteJsonFileWithStableKeyOrder( - filePath: string - ): Promise { - try { - const contents = await readFile(filePath, 'utf-8'); - const parsed = JSON.parse(contents) as unknown; - const normalized = this.sortJsonValue(parsed); - await this.writeFileIfChanged( - filePath, - `${JSON.stringify(normalized, null, 2)}\n` - ); - } catch { - // Manifest may not exist (e.g. manifest generation failed); ignore. - } - } - - private extractRelativeImportSpecifiers(source: string): string[] { - return this.extractImportSpecifiers(source).filter((specifier) => - specifier.startsWith('.') - ); - } - - private extractImportSpecifiers(source: string): string[] { - const relativeSpecifiers = new Set(); - const importPatterns = [ - /from\s+['"]([^'"]+)['"]/g, - /import\s+['"]([^'"]+)['"]/g, - /import\(\s*['"]([^'"]+)['"]\s*\)/g, - /require\(\s*['"]([^'"]+)['"]\s*\)/g, - ]; - - for (const importPattern of importPatterns) { - for (const match of source.matchAll(importPattern)) { - const specifier = match[1]; - if (specifier) { - relativeSpecifiers.add(specifier); - } - } - } - - return Array.from(relativeSpecifiers); - } - - private async getRelativeFilenameForSwc(filePath: string): Promise { - const workingDir = this.config.workingDir; - const normalizedWorkingDir = workingDir - .replace(/\\/g, '/') - .replace(/\/$/, ''); - const normalizedFilepath = filePath.replace(/\\/g, '/'); - - // Windows fix: Use case-insensitive comparison to work around drive letter casing issues. - const lowerWd = normalizedWorkingDir.toLowerCase(); - const lowerPath = normalizedFilepath.toLowerCase(); - - let relativeFilename: string; - if (lowerPath.startsWith(`${lowerWd}/`)) { - relativeFilename = normalizedFilepath.substring( - normalizedWorkingDir.length + 1 - ); - } else if (lowerPath === lowerWd) { - relativeFilename = '.'; - } else { - relativeFilename = relative(workingDir, filePath).replace(/\\/g, '/'); - if (relativeFilename.startsWith('../')) { - const aliasedRelativePath = await resolveWorkflowAliasRelativePath( - filePath, - workingDir - ); - if (aliasedRelativePath) { - relativeFilename = aliasedRelativePath; - } else { - relativeFilename = relativeFilename - .split('/') - .filter((part) => part !== '..') - .join('/'); - } - } - } - - if (relativeFilename.includes(':') || relativeFilename.startsWith('/')) { - relativeFilename = basename(normalizedFilepath); - } - - return relativeFilename; - } - - private resolveImportTargetWithExtensionFallbacks( - targetPath: string - ): string { - if (existsSync(targetPath)) { - return targetPath; - } - - const extensionMatch = targetPath.match(/(\.[^./\\]+)$/); - const extension = extensionMatch?.[1]?.toLowerCase(); - if (!extension) { - return targetPath; - } - - const extensionFallbacks = - extension === '.js' - ? ['.ts', '.tsx', '.mts', '.cts'] - : extension === '.mjs' - ? ['.mts'] - : extension === '.cjs' - ? ['.cts'] - : extension === '.jsx' - ? ['.tsx'] - : []; - - if (extensionFallbacks.length === 0) { - return targetPath; - } - - const targetWithoutExtension = targetPath.slice(0, -extension.length); - for (const fallbackExtension of extensionFallbacks) { - const fallbackPath = `${targetWithoutExtension}${fallbackExtension}`; - if (existsSync(fallbackPath)) { - return fallbackPath; - } - } - - return targetPath; - } - - private shouldSkipTransitiveStepFile(filePath: string): boolean { - const normalizedPath = filePath.replace(/\\/g, '/'); - const isSourceBackedPackagePath = - this.shouldPreferSourceBackedPackagePath(filePath); - return ( - normalizedPath.includes('/.well-known/workflow/') || - normalizedPath.includes('/.next/') || - (!isSourceBackedPackagePath && - (normalizedPath.includes('/node_modules/') || - normalizedPath.includes('/.pnpm/'))) - ); - } - - private async resolveTransitiveStepImportTargetPath( - sourceFilePath: string, - specifier: string - ): Promise { - const specifierMatch = specifier.match(/^([^?#]+)(.*)$/); - const importPath = specifierMatch?.[1] ?? specifier; - - if (!importPath.startsWith('.')) { - if (importPath !== 'workflow' && !importPath.startsWith('@workflow/')) { - return null; - } - - try { - const resolvedPath = - createRequire(sourceFilePath).resolve(importPath); - const normalizedResolvedPath = - this.normalizeDiscoveredFilePath(resolvedPath); - if (this.shouldSkipTransitiveStepFile(normalizedResolvedPath)) { - return null; - } - - const fileStats = await stat(normalizedResolvedPath); - return fileStats.isFile() ? normalizedResolvedPath : null; - } catch { - return null; - } - } - - const absoluteTargetPath = resolve(dirname(sourceFilePath), importPath); - - const candidatePaths = new Set([ - this.resolveImportTargetWithExtensionFallbacks(absoluteTargetPath), - ]); - - if (!extname(absoluteTargetPath)) { - const extensionCandidates = [ - '.ts', - '.tsx', - '.mts', - '.cts', - '.js', - '.jsx', - '.mjs', - '.cjs', - ]; - for (const extensionCandidate of extensionCandidates) { - candidatePaths.add(`${absoluteTargetPath}${extensionCandidate}`); - candidatePaths.add( - join(absoluteTargetPath, `index${extensionCandidate}`) - ); - } - } - - for (const candidatePath of candidatePaths) { - const resolvedPath = - this.resolveImportTargetWithExtensionFallbacks(candidatePath); - const normalizedResolvedPath = - this.normalizeDiscoveredFilePath(resolvedPath); - - if (this.shouldSkipTransitiveStepFile(normalizedResolvedPath)) { - continue; - } - - try { - const fileStats = await stat(normalizedResolvedPath); - if (fileStats.isFile()) { - return normalizedResolvedPath; - } - } catch { - // Try the next candidate path. - } - } - - return null; - } - - private async collectTransitiveStepFiles({ - entryFiles, - stepFiles, - }: { - entryFiles: string[]; - stepFiles: string[]; - }): Promise { - const normalizedEntryFiles = Array.from( - new Set( - entryFiles.map((entryFile) => - this.normalizeDiscoveredFilePath(entryFile) - ) - ) - ).sort(); - const normalizedStepSeedFiles = Array.from( - new Set( - stepFiles.map((stepFile) => - this.normalizeDiscoveredFilePath(stepFile) - ) - ) - ).sort(); - const discoveredStepFiles = new Set(normalizedStepSeedFiles); - const queuedFiles = Array.from( - new Set([...normalizedEntryFiles, ...normalizedStepSeedFiles]) - ); - const visitedFiles = new Set(); - const sourceCache = new Map(); - const patternCache = new Map< - string, - ReturnType | null - >(); - - const getSource = async (filePath: string): Promise => { - if (sourceCache.has(filePath)) { - return sourceCache.get(filePath) ?? null; - } - try { - const source = await readFile(filePath, 'utf-8'); - sourceCache.set(filePath, source); - return source; - } catch { - sourceCache.set(filePath, null); - return null; - } - }; - - const getPatterns = async ( - filePath: string - ): Promise | null> => { - if (patternCache.has(filePath)) { - return patternCache.get(filePath) ?? null; - } - const source = await getSource(filePath); - if (source === null) { - patternCache.set(filePath, null); - return null; - } - const patterns = detectWorkflowPatterns(source); - patternCache.set(filePath, patterns); - return patterns; - }; - - while (queuedFiles.length > 0) { - const currentFile = queuedFiles.pop(); - if (!currentFile || visitedFiles.has(currentFile)) { - continue; - } - visitedFiles.add(currentFile); - - const currentSource = await getSource(currentFile); - if (currentSource === null) { - continue; - } - - const importSpecifiers = this.extractImportSpecifiers(currentSource); - for (const specifier of importSpecifiers) { - const resolvedImportPath = - await this.resolveTransitiveStepImportTargetPath( - currentFile, - specifier - ); - if (!resolvedImportPath) { - continue; - } - - if (!visitedFiles.has(resolvedImportPath)) { - queuedFiles.push(resolvedImportPath); - } - - const importPatterns = await getPatterns(resolvedImportPath); - if (importPatterns?.hasUseStep) { - discoveredStepFiles.add(resolvedImportPath); - } - } - } - - return Array.from(discoveredStepFiles).sort(); - } - - private async collectTransitiveSerdeFiles({ - entryFiles, - serdeFiles, - }: { - entryFiles: string[]; - serdeFiles: string[]; - }): Promise { - const normalizedEntryFiles = Array.from( - new Set( - entryFiles.map((entryFile) => - this.normalizeDiscoveredFilePath(entryFile) - ) - ) - ).sort(); - const normalizedSerdeSeedFiles = Array.from( - new Set( - serdeFiles.map((serdeFile) => - this.normalizeDiscoveredFilePath(serdeFile) - ) - ) - ).sort(); - // Intentionally re-validate serde seeds against source patterns. - // This keeps previously discovered/manual seed entries from sticking when - // files no longer match serde patterns. - const discoveredSerdeFiles = new Set(); - const queuedFiles = Array.from( - new Set([...normalizedEntryFiles, ...normalizedSerdeSeedFiles]) - ); - const visitedFiles = new Set(); - const sourceCache = new Map(); - const patternCache = new Map< - string, - ReturnType | null - >(); - - const getSource = async (filePath: string): Promise => { - if (sourceCache.has(filePath)) { - return sourceCache.get(filePath) ?? null; - } - try { - const source = await readFile(filePath, 'utf-8'); - sourceCache.set(filePath, source); - return source; - } catch { - sourceCache.set(filePath, null); - return null; - } - }; - - const getPatterns = async ( - filePath: string - ): Promise | null> => { - if (patternCache.has(filePath)) { - return patternCache.get(filePath) ?? null; - } - const source = await getSource(filePath); - if (source === null) { - patternCache.set(filePath, null); - return null; - } - const patterns = detectWorkflowPatterns(source); - patternCache.set(filePath, patterns); - return patterns; - }; - - for (const serdeSeedFile of normalizedSerdeSeedFiles) { - const seedPatterns = await getPatterns(serdeSeedFile); - if (seedPatterns?.hasSerde) { - discoveredSerdeFiles.add(serdeSeedFile); - } - } - - while (queuedFiles.length > 0) { - const currentFile = queuedFiles.pop(); - if (!currentFile || visitedFiles.has(currentFile)) { - continue; - } - visitedFiles.add(currentFile); - - const currentSource = await getSource(currentFile); - if (currentSource === null) { - continue; - } - - const relativeImportSpecifiers = - this.extractRelativeImportSpecifiers(currentSource); - for (const specifier of relativeImportSpecifiers) { - const resolvedImportPath = - await this.resolveTransitiveStepImportTargetPath( - currentFile, - specifier - ); - if (!resolvedImportPath) { - continue; - } - - if (!visitedFiles.has(resolvedImportPath)) { - queuedFiles.push(resolvedImportPath); - } - - const importPatterns = await getPatterns(resolvedImportPath); - if (importPatterns?.hasSerde) { - discoveredSerdeFiles.add(resolvedImportPath); - } - } - } - - // AST-level verification: run SWC detect mode on regex-matched candidates - // to confirm they actually define serde classes. This prevents SDK internal - // files (which match serde regex patterns but define no classes) from being - // bundled into the workflow sandbox. - const projectRoot = this.config.projectRoot || this.config.workingDir; - const verifiedSerdeFiles: string[] = []; - await Promise.all( - Array.from(discoveredSerdeFiles).map(async (filePath) => { - const source = await getSource(filePath); - if (!source) return; - try { - const relativeFilename = - await this.getRelativeFilenameForSwc(filePath); - const { workflowManifest } = await applySwcTransform( - relativeFilename, - source, - 'detect', - filePath, - projectRoot, - this.moduleSpecifierRoot - ); - // Only include files that actually define serde classes - const hasClasses = - workflowManifest.classes && - Object.values(workflowManifest.classes).some( - (entries) => Object.keys(entries).length > 0 - ); - if (hasClasses) { - verifiedSerdeFiles.push(filePath); - } - } catch { - // If detect fails, include the file to be safe - verifiedSerdeFiles.push(filePath); - } - }) - ); - - return verifiedSerdeFiles.sort(); - } - - private async buildWebhookRoute({ - workflowGeneratedDir, - routeFileName = 'route.js', - }: { - workflowGeneratedDir: string; - routeFileName?: string; - }): Promise { - const webhookRouteFile = join( - workflowGeneratedDir, - `webhook/[token]/${routeFileName}` - ); - await this.createWebhookBundle({ - outfile: webhookRouteFile, - bundle: false, // Next.js doesn't need bundling - }); - } - - private async findAppDirectory(): Promise { - const appDir = resolve(this.config.workingDir, 'app'); - const srcAppDir = resolve(this.config.workingDir, 'src/app'); - const pagesDir = resolve(this.config.workingDir, 'pages'); - const srcPagesDir = resolve(this.config.workingDir, 'src/pages'); - - // Helper to check if a path exists and is a directory - const isDirectory = async (path: string): Promise => { - try { - await access(path, constants.F_OK); - const stats = await stat(path); - if (!stats.isDirectory()) { - throw new Error(`Path exists but is not a directory: ${path}`); - } - return true; - } catch (e) { - if (e instanceof Error && e.message.includes('not a directory')) { - throw e; - } - return false; - } - }; - - // Check if app directory exists - if (await isDirectory(appDir)) { - return appDir; - } - - // Check if src/app directory exists - if (await isDirectory(srcAppDir)) { - return srcAppDir; - } - - // If no app directory exists, check for pages directory and create app next to it - if (await isDirectory(pagesDir)) { - // Create app directory next to pages directory - await mkdir(appDir, { recursive: true }); - return appDir; - } - - if (await isDirectory(srcPagesDir)) { - // Create src/app directory next to src/pages directory - await mkdir(srcAppDir, { recursive: true }); - return srcAppDir; - } - - throw new Error( - 'Could not find Next.js app or pages directory. Expected one of: "app", "src/app", "pages", or "src/pages" to exist.' - ); - } - } - - CachedNextBuilderDeferred = NextDeferredBuilder; - return NextDeferredBuilder; -} diff --git a/packages/next/src/builder-eager.ts b/packages/next/src/builder-eager.ts index 93d587d951..91dc88304b 100644 --- a/packages/next/src/builder-eager.ts +++ b/packages/next/src/builder-eager.ts @@ -3,7 +3,6 @@ import { access, copyFile, mkdir, stat, writeFile } from 'node:fs/promises'; import { extname, join, resolve } from 'node:path'; import type { WorkflowManifest } from '@workflow/builders'; import Watchpack from 'watchpack'; -import { cleanupStaleSocketInfoFiles } from './socket-server.js'; let CachedNextBuilderEager: any; @@ -25,16 +24,6 @@ export async function getNextBuilderEager() { class NextBuilder extends BaseBuilderClass { async build() { - // Eager mode never starts a discovery socket server, so any leftover - // workflow-socket.json is from a previous deferred-mode build and - // would make the webpack loader connect to a dead port. - await cleanupStaleSocketInfoFiles( - join( - this.config.workingDir, - (this.config as { distDir?: string }).distDir || '.next' - ) - ); - const outputDir = await this.findAppDirectory(); const workflowGeneratedDir = join(outputDir, '.well-known/workflow/v1'); @@ -102,20 +91,29 @@ export async function getNextBuilderEager() { 'Invariant: expected steps build context in watch mode' ); } + if (!combinedResult?.interimBundleCtx || !combinedResult?.bundleFinal) { + throw new Error( + 'Invariant: expected workflows bundle context in watch mode' + ); + } // Use stepsCtx for the watch rebuild (workflow interim ctx from combined) let workflowsCtx = { - interimBundleCtx: combinedResult?.interimBundleCtx!, - bundleFinal: combinedResult?.bundleFinal!, + interimBundleCtx: combinedResult.interimBundleCtx, + bundleFinal: combinedResult.bundleFinal, }; const normalizePath = (pathname: string) => pathname.replace(/\\/g, '/'); - const knownFiles = new Set(); type WatchpackTimeInfoEntry = { safeTime: number; timestamp?: number; }; + type FileChanges = { + addedFiles: string[]; + modifiedFiles: string[]; + removedFiles: string[]; + }; let previousTimeInfo = new Map(); const watchableExtensions = new Set([ @@ -222,62 +220,26 @@ export async function getNextBuilderEager() { await writeManifest(newCombined.manifest); }; - const logBuildMessages = ( - result: { - errors?: import('esbuild').Message[]; - warnings?: import('esbuild').Message[]; - }, - label: string - ) => { - const logByType = ( - messages: import('esbuild').Message[] | undefined, - method: 'error' | 'warn' - ) => { - if (!messages || messages.length === 0) { - return; - } - const descriptor = method === 'error' ? 'errors' : 'warnings'; - console[method](`${descriptor} while rebuilding ${label}`); - for (const message of messages) { - console[method](message); - } - }; - - logByType(result.errors, 'error'); - logByType(result.warnings, 'warn'); - }; - - const rebuildExistingFiles = async () => { - const rebuiltStepStart = Date.now(); - const stepsResult = await stepsCtx!.rebuild(); - logBuildMessages(stepsResult, 'steps bundle'); - console.log( - 'Rebuilt steps bundle', - `${Date.now() - rebuiltStepStart}ms` - ); + const isWatchableFile = (path: string) => + watchableExtensions.has(extname(path)); - const rebuiltWorkflowStart = Date.now(); - const workflowResult = await workflowsCtx.interimBundleCtx.rebuild(); - logBuildMessages(workflowResult, 'workflows bundle'); + const normalizeWatchpackPaths = (paths?: Iterable) => { + const normalizedPaths: string[] = []; + if (!paths) { + return normalizedPaths; + } - if ( - !workflowResult.outputFiles || - workflowResult.outputFiles.length === 0 - ) { - console.error( - 'No output generated while rebuilding workflows bundle' - ); - return; + for (const path of paths) { + const normalizedPath = normalizePath(path); + if (isWatchableFile(normalizedPath)) { + normalizedPaths.push(normalizedPath); + } } - await workflowsCtx.bundleFinal(workflowResult.outputFiles[0].text); - console.log( - 'Rebuilt workflow bundle', - `${Date.now() - rebuiltWorkflowStart}ms` - ); + + return normalizedPaths; }; - const isWatchableFile = (path: string) => - watchableExtensions.has(extname(path)); + const unique = (paths: string[]) => [...new Set(paths)]; const getComparableTimestamp = (entry: WatchpackTimeInfoEntry) => entry.timestamp ?? entry.safeTime; @@ -326,7 +288,7 @@ export async function getNextBuilderEager() { const determineFileChanges = ( currentEntries: Map, previousEntries: Map - ) => { + ): FileChanges => { const removedFiles = findRemovedFiles( currentEntries, previousEntries @@ -343,50 +305,90 @@ export async function getNextBuilderEager() { }; }; - let isInitial = true; - - watcher.on('aggregated', () => { - const currentEntries = readTimeInfoEntries(); - const { addedFiles, modifiedFiles, removedFiles } = - determineFileChanges(currentEntries, previousTimeInfo); + const mergeFileChanges = ({ + currentEntries, + previousEntries, + timestampChanges, + eventChangedFiles, + eventRemovedFiles, + }: { + currentEntries: Map; + previousEntries: Map; + timestampChanges: FileChanges; + eventChangedFiles: string[]; + eventRemovedFiles: string[]; + }): FileChanges => ({ + addedFiles: unique([ + ...timestampChanges.addedFiles, + ...eventChangedFiles.filter( + (path) => currentEntries.has(path) && !previousEntries.has(path) + ), + ]), + modifiedFiles: unique([ + ...timestampChanges.modifiedFiles, + ...eventChangedFiles, + ]), + removedFiles: unique([ + ...timestampChanges.removedFiles, + ...eventRemovedFiles, + ]), + }); - previousTimeInfo = currentEntries; + const hasFileChanges = ({ + addedFiles, + modifiedFiles, + removedFiles, + }: FileChanges) => + addedFiles.length > 0 || + modifiedFiles.length > 0 || + removedFiles.length > 0; - if (isInitial) { - isInitial = false; - return; - } + let isInitial = true; - if ( - addedFiles.length === 0 && - modifiedFiles.length === 0 && - removedFiles.length === 0 - ) { - return; - } + watcher.on( + 'aggregated', + (changes?: Set, removals?: Set) => { + const currentEntries = readTimeInfoEntries(); + const eventChangedFiles = normalizeWatchpackPaths(changes); + const eventRemovedFiles = normalizeWatchpackPaths(removals); + const timestampChanges = determineFileChanges( + currentEntries, + previousTimeInfo + ); - for (const removal of removedFiles) { - knownFiles.delete(removal); - } - for (const added of addedFiles) { - knownFiles.add(added); - } + const fileChanges = mergeFileChanges({ + currentEntries, + previousEntries: previousTimeInfo, + timestampChanges, + eventChangedFiles, + eventRemovedFiles, + }); + + previousTimeInfo = currentEntries; + + if (isInitial) { + isInitial = false; + if ( + eventChangedFiles.length === 0 && + eventRemovedFiles.length === 0 + ) { + return; + } + } - enqueue(async () => { - if (addedFiles.length > 0 || removedFiles.length > 0) { - await fullRebuild(); + if (!hasFileChanges(fileChanges)) { return; } - if (modifiedFiles.length > 0) { - await rebuildExistingFiles(); - } - }); - }); + enqueue(async () => { + await fullRebuild(); + }); + } + ); watcher.watch({ directories: [this.config.workingDir], - startTime: 0, + startTime: Date.now(), }); } } diff --git a/packages/next/src/builder.test.ts b/packages/next/src/builder.test.ts deleted file mode 100644 index 6e2d92035e..0000000000 --- a/packages/next/src/builder.test.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { afterEach, describe, expect, it } from 'vitest'; -import { shouldUseDeferredBuilder } from './builder.js'; -import { parseEnvironmentFlag } from './environment-flag.js'; - -const originalLazyDiscoveryEnv = process.env.WORKFLOW_NEXT_LAZY_DISCOVERY; - -afterEach(() => { - if (originalLazyDiscoveryEnv === undefined) { - delete process.env.WORKFLOW_NEXT_LAZY_DISCOVERY; - } else { - process.env.WORKFLOW_NEXT_LAZY_DISCOVERY = originalLazyDiscoveryEnv; - } -}); - -describe('shouldUseDeferredBuilder', () => { - it('treats WORKFLOW_NEXT_LAZY_DISCOVERY=0 as disabled', () => { - process.env.WORKFLOW_NEXT_LAZY_DISCOVERY = '0'; - - expect(shouldUseDeferredBuilder('16.2.1')).toBe(false); - }); - - it('treats WORKFLOW_NEXT_LAZY_DISCOVERY=false as disabled', () => { - process.env.WORKFLOW_NEXT_LAZY_DISCOVERY = 'false'; - - expect(shouldUseDeferredBuilder('16.2.1')).toBe(false); - }); - - it('treats WORKFLOW_NEXT_LAZY_DISCOVERY=off as disabled', () => { - process.env.WORKFLOW_NEXT_LAZY_DISCOVERY = 'off'; - - expect(parseEnvironmentFlag(process.env.WORKFLOW_NEXT_LAZY_DISCOVERY)).toBe( - false - ); - expect(shouldUseDeferredBuilder('16.2.1')).toBe(false); - }); - - it('enables deferred mode for compatible versions when the env is enabled', () => { - process.env.WORKFLOW_NEXT_LAZY_DISCOVERY = '1'; - - expect(shouldUseDeferredBuilder('16.2.1')).toBe(true); - }); -}); diff --git a/packages/next/src/builder.ts b/packages/next/src/builder.ts index e4a4e9ec56..b8574d714f 100644 --- a/packages/next/src/builder.ts +++ b/packages/next/src/builder.ts @@ -1,39 +1,5 @@ -import semver from 'semver'; -import { getNextBuilderDeferred } from './builder-deferred.js'; import { getNextBuilderEager } from './builder-eager.js'; -import { parseEnvironmentFlag } from './environment-flag.js'; - -export const DEFERRED_BUILDER_MIN_VERSION = '16.2.0-canary.48'; - -export const WORKFLOW_DEFERRED_ENTRIES = [ - '/.well-known/workflow/v1/flow', - '/.well-known/workflow/v1/webhook/[token]', -] as const; - -let warnedAboutFlagAndVersion = false; - -export function shouldUseDeferredBuilder(nextVersion: string): boolean { - const flagEnabled = - parseEnvironmentFlag(process.env.WORKFLOW_NEXT_LAZY_DISCOVERY) ?? false; - const versionCompatible = semver.gte( - nextVersion, - DEFERRED_BUILDER_MIN_VERSION - ); - - if (flagEnabled && !versionCompatible && !warnedAboutFlagAndVersion) { - warnedAboutFlagAndVersion = true; - console.warn( - `lazyDiscovery requires Next.js >= ${DEFERRED_BUILDER_MIN_VERSION} (found ${nextVersion}); falling back to eager workflow discovery.` - ); - } - - return flagEnabled && versionCompatible; -} - -export async function getNextBuilder(nextVersion: string) { - if (shouldUseDeferredBuilder(nextVersion)) { - return getNextBuilderDeferred(); - } +export async function getNextBuilder(_nextVersion: string) { return getNextBuilderEager(); } diff --git a/packages/next/src/environment-flag.ts b/packages/next/src/environment-flag.ts deleted file mode 100644 index 3436a7303c..0000000000 --- a/packages/next/src/environment-flag.ts +++ /dev/null @@ -1,16 +0,0 @@ -const FALSE_ENV_VALUES = new Set(['0', 'false', 'no', 'off']); - -export function parseEnvironmentFlag( - rawValue: string | undefined -): boolean | undefined { - const normalizedValue = rawValue?.trim().toLowerCase(); - if (!normalizedValue) { - return undefined; - } - - if (FALSE_ENV_VALUES.has(normalizedValue)) { - return false; - } - - return true; -} diff --git a/packages/next/src/index.test.ts b/packages/next/src/index.test.ts index b9419893f4..6fc41bfa0e 100644 --- a/packages/next/src/index.test.ts +++ b/packages/next/src/index.test.ts @@ -10,12 +10,7 @@ import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -const { - buildMock, - builderConfigs, - getNextBuilderMock, - shouldUseDeferredBuilderMock, -} = vi.hoisted(() => { +const { buildMock, builderConfigs, getNextBuilderMock } = vi.hoisted(() => { const buildMock = vi.fn(async () => {}); const builderConfigs: Record[] = []; const getNextBuilderMock = vi.fn(async () => { @@ -27,24 +22,16 @@ const { } }; }); - const shouldUseDeferredBuilderMock = vi.fn(() => false); return { buildMock, builderConfigs, getNextBuilderMock, - shouldUseDeferredBuilderMock, }; }); vi.mock('./builder.js', () => ({ getNextBuilder: getNextBuilderMock, - shouldUseDeferredBuilder: shouldUseDeferredBuilderMock, - WORKFLOW_DEFERRED_ENTRIES: [ - '/.well-known/workflow/v1/flow', - '/.well-known/workflow/v1/step', - '/.well-known/workflow/v1/webhook/[token]', - ], })); import { withWorkflow } from './index.js'; @@ -70,7 +57,6 @@ describe('withWorkflow builder config', () => { PORT: process.env.PORT, VERCEL_DEPLOYMENT_ID: process.env.VERCEL_DEPLOYMENT_ID, WORKFLOW_LOCAL_DATA_DIR: process.env.WORKFLOW_LOCAL_DATA_DIR, - WORKFLOW_NEXT_LAZY_DISCOVERY: process.env.WORKFLOW_NEXT_LAZY_DISCOVERY, WORKFLOW_NEXT_PRIVATE_BUILT: process.env.WORKFLOW_NEXT_PRIVATE_BUILT, WORKFLOW_TARGET_WORLD: process.env.WORKFLOW_TARGET_WORLD, }; @@ -79,7 +65,6 @@ describe('withWorkflow builder config', () => { buildMock.mockClear(); builderConfigs.length = 0; getNextBuilderMock.mockClear(); - shouldUseDeferredBuilderMock.mockClear(); if (!hadLoaderStub) { writeFileSync(loaderStubPath, 'module.exports = {};\n', 'utf-8'); @@ -88,7 +73,6 @@ describe('withWorkflow builder config', () => { delete process.env.PORT; delete process.env.VERCEL_DEPLOYMENT_ID; delete process.env.WORKFLOW_LOCAL_DATA_DIR; - delete process.env.WORKFLOW_NEXT_LAZY_DISCOVERY; delete process.env.WORKFLOW_NEXT_PRIVATE_BUILT; delete process.env.WORKFLOW_TARGET_WORLD; }); @@ -130,21 +114,6 @@ describe('withWorkflow builder config', () => { }); }); - it('enables lazyDiscovery by default', async () => { - withWorkflow({}); - expect(process.env.WORKFLOW_NEXT_LAZY_DISCOVERY).toBe('1'); - }); - - it('enables lazyDiscovery when explicitly set to true', async () => { - withWorkflow({}, { workflows: { lazyDiscovery: true } }); - expect(process.env.WORKFLOW_NEXT_LAZY_DISCOVERY).toBe('1'); - }); - - it('disables lazyDiscovery when explicitly set to false', async () => { - withWorkflow({}, { workflows: { lazyDiscovery: false } }); - expect(process.env.WORKFLOW_NEXT_LAZY_DISCOVERY).toBeUndefined(); - }); - it('configures diagnostics inside the default Next.js dist dir', async () => { const config = withWorkflow({}); @@ -219,36 +188,6 @@ describe('withWorkflow builder config', () => { expect(webpackConfig?.externals).toEqual([{ react: 'commonjs react' }]); }); - it('preserves an explicit lazyDiscovery disable override', () => { - process.env.WORKFLOW_NEXT_LAZY_DISCOVERY = '0'; - - withWorkflow( - {}, - { - workflows: { - lazyDiscovery: true, - }, - } - ); - - expect(process.env.WORKFLOW_NEXT_LAZY_DISCOVERY).toBe('0'); - }); - - it('treats an empty lazyDiscovery env override as unset', () => { - process.env.WORKFLOW_NEXT_LAZY_DISCOVERY = ''; - - withWorkflow( - {}, - { - workflows: { - lazyDiscovery: true, - }, - } - ); - - expect(process.env.WORKFLOW_NEXT_LAZY_DISCOVERY).toBe('1'); - }); - it('removes workflow packages from serverExternalPackages for this build', async () => { const projectDir = mkdtempSync( join(realTmpDir, 'workflow-next-server-external-') diff --git a/packages/next/src/index.ts b/packages/next/src/index.ts index dcaaf26cf0..c363b4cc5e 100644 --- a/packages/next/src/index.ts +++ b/packages/next/src/index.ts @@ -3,12 +3,7 @@ import { copyFile, mkdir, readFile } from 'node:fs/promises'; import { dirname, isAbsolute, join } from 'node:path'; import type { NextConfig } from 'next'; import semver from 'semver'; -import { - getNextBuilder, - shouldUseDeferredBuilder, - WORKFLOW_DEFERRED_ENTRIES, -} from './builder.js'; -import { parseEnvironmentFlag } from './environment-flag.js'; +import { getNextBuilder } from './builder.js'; const VERCEL_WORLD_PACKAGE = '@workflow/world-vercel'; const VERCEL_WORLD_DEPENDENCY_PACKAGES = [ @@ -326,7 +321,6 @@ export function withWorkflow( workflows, }: { workflows?: { - lazyDiscovery?: boolean; local?: { port?: number; }; @@ -341,25 +335,6 @@ export function withWorkflow( }; } = {} ) { - // lazyDiscovery defaults to true; pass `lazyDiscovery: false` to force eager - // discovery (scanning the project at startup) instead of deferring workflow - // discovery until files are requested. The `WORKFLOW_NEXT_LAZY_DISCOVERY` - // environment variable, if set, takes precedence over the option. - const lazyDiscoveryOverride = parseEnvironmentFlag( - process.env.WORKFLOW_NEXT_LAZY_DISCOVERY - ); - if (lazyDiscoveryOverride === undefined) { - if (workflows?.lazyDiscovery === false) { - delete process.env.WORKFLOW_NEXT_LAZY_DISCOVERY; - } else { - process.env.WORKFLOW_NEXT_LAZY_DISCOVERY = '1'; - } - } else { - process.env.WORKFLOW_NEXT_LAZY_DISCOVERY = lazyDiscoveryOverride - ? '1' - : '0'; - } - if (!process.env.VERCEL_DEPLOYMENT_ID) { if (!process.env.WORKFLOW_TARGET_WORLD) { process.env.WORKFLOW_TARGET_WORLD = 'local'; @@ -380,8 +355,6 @@ export function withWorkflow( ctx: { defaultConfig: NextConfig } ) { const loaderPath = require.resolve('./loader'); - let runDeferredBuildFromCallback: (() => Promise) | undefined; - let nextConfig: NextConfig; if (typeof nextConfigOrFn === 'function') { @@ -460,7 +433,6 @@ export function withWorkflow( const existingRules = nextConfig.turbopack.rules as any; const nextVersion = resolveNextVersion(process.cwd()); const supportsTurboCondition = semver.gte(nextVersion, 'v16.0.0'); - const useDeferredBuilder = shouldUseDeferredBuilder(nextVersion); const shouldWatch = process.env.NODE_ENV === 'development'; let workflowBuilderPromise: Promise | undefined; @@ -495,10 +467,6 @@ export function withWorkflow( stepsBundlePath: '', // not used in base webhookBundlePath: '', // node used in base sourcemap: workflows?.sourcemap, - suppressCreateWorkflowsBundleLogs: useDeferredBuilder, - suppressCreateWorkflowsBundleWarnings: useDeferredBuilder, - suppressCreateWebhookBundleLogs: useDeferredBuilder, - suppressCreateManifestLogs: useDeferredBuilder, externalPackages: [ // server-only and client-only are pseudo-packages handled by Next.js // during its build process. We mark them as external to prevent esbuild @@ -515,50 +483,6 @@ export function withWorkflow( return workflowBuilderPromise; }; - if (useDeferredBuilder) { - runDeferredBuildFromCallback = async () => { - const workflowBuilder = await getWorkflowBuilder(); - if (typeof workflowBuilder.onBeforeDeferredEntries === 'function') { - await workflowBuilder.onBeforeDeferredEntries(); - } - }; - - const existingExperimental = (nextConfig.experimental ?? {}) as Record< - string, - any - >; - const existingDeferredEntries = Array.isArray( - existingExperimental.deferredEntries - ) - ? existingExperimental.deferredEntries - : []; - const existingOnBeforeDeferredEntries = - typeof existingExperimental.onBeforeDeferredEntries === 'function' - ? existingExperimental.onBeforeDeferredEntries - : undefined; - - nextConfig.experimental = { - ...existingExperimental, - - // biome-ignore lint/suspicious/noTsIgnore: expect-error is wrong as it will work on valid version - // @ts-ignore this is only available in canary Next.js - deferredEntries: [ - ...new Set([ - ...existingDeferredEntries, - ...WORKFLOW_DEFERRED_ENTRIES, - ]), - ], - onBeforeDeferredEntries: async (...args: unknown[]) => { - if (existingOnBeforeDeferredEntries) { - await existingOnBeforeDeferredEntries(...args); - } - if (runDeferredBuildFromCallback) { - await runDeferredBuildFromCallback(); - } - }, - }; - } - for (const key of [ '*.tsx', '*.ts', diff --git a/packages/next/src/loader.test.ts b/packages/next/src/loader.test.ts deleted file mode 100644 index b13ee68561..0000000000 --- a/packages/next/src/loader.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { shouldNotifySocketForDiscoveredPattern } from './loader.js'; - -describe('workflow loader discovery notifications', () => { - it('notifies for unchanged files that still contain workflow patterns', () => { - expect( - shouldNotifySocketForDiscoveredPattern(false, { - hasWorkflow: true, - hasStep: false, - hasSerde: false, - }) - ).toBe(true); - expect( - shouldNotifySocketForDiscoveredPattern(false, { - hasWorkflow: false, - hasStep: true, - hasSerde: false, - }) - ).toBe(true); - expect( - shouldNotifySocketForDiscoveredPattern(false, { - hasWorkflow: false, - hasStep: false, - hasSerde: true, - }) - ).toBe(true); - }); - - it('only notifies for plain files when pattern state changed', () => { - const plainPatternState = { - hasWorkflow: false, - hasStep: false, - hasSerde: false, - }; - - expect( - shouldNotifySocketForDiscoveredPattern(false, plainPatternState) - ).toBe(false); - expect( - shouldNotifySocketForDiscoveredPattern(true, plainPatternState) - ).toBe(true); - }); -}); diff --git a/packages/next/src/loader.ts b/packages/next/src/loader.ts index 60d38c79a7..efa200a269 100644 --- a/packages/next/src/loader.ts +++ b/packages/next/src/loader.ts @@ -1,14 +1,6 @@ import { existsSync } from 'node:fs'; -import { readFile } from 'node:fs/promises'; -import { connect, type Socket } from 'node:net'; import { dirname, join, relative } from 'node:path'; import { transform } from '@swc/core'; -import { - parseMessage, - SOCKET_INFO_FILENAME, - type SocketMessage, - serializeMessage, -} from './socket-server.js'; type DecoratorOptionsWithConfigPath = import('@workflow/builders').DecoratorOptionsWithConfigPath; @@ -27,75 +19,6 @@ type LoaderStaticDependencies = { }; let cachedLoaderStaticDependencies: LoaderStaticDependencies | null = null; -type DiscoveredPatternState = { - hasWorkflow: boolean; - hasStep: boolean; - hasSerde: boolean; -}; -const discoveredPatternStateByFilePath = new Map< - string, - DiscoveredPatternState ->(); - -export function shouldNotifySocketForDiscoveredPattern( - patternStateChanged: boolean, - patternState: DiscoveredPatternState -): boolean { - return ( - patternStateChanged || - patternState.hasWorkflow || - patternState.hasStep || - patternState.hasSerde - ); -} - -// Cache socket connection to avoid reconnecting on every file. -let socketClientPromise: Promise | null = null; -let socketClient: Socket | null = null; -let socketClientKey: string | null = null; - -type SocketCredentials = { - port: number; - authToken: string; -}; - -/** - * Wrap a TCP connect failure with context about where the connection was - * attempted, where the credentials came from, and which source file was - * being processed when it failed. ECONNREFUSED is the common case here, and - * the message points the user at the most likely root cause (stale - * socket-info file). - */ -function annotateConnectionError( - originalError: unknown, - credentials: SocketCredentials -): Error { - const errorCode = - originalError instanceof Error && - 'code' in originalError && - typeof (originalError as { code?: unknown }).code === 'string' - ? ((originalError as { code: string }).code as string) - : undefined; - const errorMessage = - originalError instanceof Error - ? originalError.message - : String(originalError); - - const lines = [ - `Workflow discovery socket connect failed: ${errorCode ?? errorMessage} (127.0.0.1:${credentials.port})`, - ]; - - const annotated = new Error(lines.join('\n')); - if (originalError instanceof Error) { - (annotated as { cause?: unknown }).cause = originalError; - } - return annotated; -} - -const ROUTE_STUB_FILE_MARKER = 'WORKFLOW_ROUTE_STUB_FILE'; -const ROUTE_STUB_BUILD_WAIT_TIMEOUT_MS = 120_000; -let pendingDeferredRouteStubBuildPromise: Promise | null = null; - function registerFileDependency( loaderContext: WorkflowLoaderContext, dependencyPath: string @@ -104,35 +27,6 @@ function registerFileDependency( loaderContext.addBuildDependency?.(dependencyPath); } -function updateDiscoveredPatternState( - filePath: string, - nextState: DiscoveredPatternState -): { shouldNotify: boolean; previousState?: DiscoveredPatternState } { - const previousState = discoveredPatternStateByFilePath.get(filePath); - const hasAnyPattern = - nextState.hasWorkflow || nextState.hasStep || nextState.hasSerde; - - if (!hasAnyPattern) { - if (!previousState) { - return { shouldNotify: false }; - } - discoveredPatternStateByFilePath.delete(filePath); - return { shouldNotify: true, previousState }; - } - - if ( - previousState && - previousState.hasWorkflow === nextState.hasWorkflow && - previousState.hasStep === nextState.hasStep && - previousState.hasSerde === nextState.hasSerde - ) { - return { shouldNotify: false, previousState }; - } - - discoveredPatternStateByFilePath.set(filePath, nextState); - return { shouldNotify: true, previousState }; -} - function addIfExists(files: Set, dependencyPath: string): void { if (existsSync(dependencyPath)) { files.add(dependencyPath); @@ -160,7 +54,6 @@ function resolveLoaderStaticDependencies(): LoaderStaticDependencies { const files = new Set([ __filename, - require.resolve('./socket-server'), swcPluginPath, swcPluginBuildHashPath, workflowBuildersPath, @@ -186,354 +79,6 @@ function registerTransformDependencies( return staticDependencies.swcPluginPath; } -function resetSocketClient(cachedSocket?: Socket): void { - if (cachedSocket && socketClient && socketClient !== cachedSocket) { - return; - } - - socketClientPromise = null; - socketClient = null; - socketClientKey = null; -} - -async function writeSocketMessage( - socket: Socket, - message: string -): Promise { - await new Promise((resolve, reject) => { - socket.write(message, (error?: Error | null) => { - if (error) { - reject(error); - return; - } - resolve(); - }); - }); -} - -function getSocketInfoFilePath(): string | null { - const configuredPath = process.env.WORKFLOW_SOCKET_INFO_PATH; - if (configuredPath) { - return configuredPath; - } - - // Fallback for worker processes that don't inherit dynamic env updates - // from the process that created the socket server. - const distDir = process.env.WORKFLOW_NEXT_DIST_DIR || '.next'; - const cwdFallbackPath = join(process.cwd(), distDir, SOCKET_INFO_FILENAME); - const projectRoot = process.env.WORKFLOW_PROJECT_ROOT; - if (projectRoot) { - const projectRootFallbackPath = join( - projectRoot, - distDir, - SOCKET_INFO_FILENAME - ); - if (existsSync(projectRootFallbackPath)) { - return projectRootFallbackPath; - } - } - return cwdFallbackPath; -} - -function getSocketCredentialsFromEnv(): SocketCredentials | null { - const socketPort = process.env.WORKFLOW_SOCKET_PORT; - const authToken = process.env.WORKFLOW_SOCKET_AUTH; - if (!socketPort || !authToken) { - return null; - } - - const port = Number.parseInt(socketPort, 10); - if (Number.isNaN(port)) { - return null; - } - return { port, authToken }; -} - -async function getSocketCredentialsFromFile(): Promise { - const socketInfoFilePath = getSocketInfoFilePath(); - if (!socketInfoFilePath) { - return null; - } - if (!existsSync(socketInfoFilePath)) { - return null; - } - - try { - const raw = await readFile(socketInfoFilePath, 'utf8'); - const parsed = JSON.parse(raw) as { - port?: unknown; - authToken?: unknown; - }; - const authToken = - typeof parsed.authToken === 'string' ? parsed.authToken : null; - const numericPort = - typeof parsed.port === 'number' - ? parsed.port - : Number.parseInt(String(parsed.port), 10); - - if (!authToken || Number.isNaN(numericPort)) { - return null; - } - return { - port: numericPort, - authToken, - }; - } catch { - return null; - } -} - -async function getSocketCredentials(): Promise { - const envCredentials = getSocketCredentialsFromEnv(); - if (envCredentials) { - return envCredentials; - } - return await getSocketCredentialsFromFile(); -} - -async function getSocketClient(): Promise { - const socketCredentials = await getSocketCredentials(); - if (!socketCredentials) { - return null; - } - - if (socketClient?.destroyed) { - resetSocketClient(socketClient); - } - - const currentSocketKey = `${socketCredentials.port}:${socketCredentials.authToken}`; - if (socketClientKey && socketClientKey !== currentSocketKey) { - if (socketClient) { - resetSocketClient(socketClient); - } else { - resetSocketClient(); - } - } - - if (!socketClientPromise) { - socketClientPromise = (async () => { - try { - const socket = connect({ - port: socketCredentials.port, - host: '127.0.0.1', - }); - - // Wait for connection - await new Promise((resolve, reject) => { - const onConnect = () => { - socket.setNoDelay(true); - cleanup(); - resolve(); - }; - const onError = (error: Error) => { - cleanup(); - reject(annotateConnectionError(error, socketCredentials)); - }; - const timeout = setTimeout(() => { - cleanup(); - socket.destroy(); - reject( - annotateConnectionError( - new Error('Socket connection timeout'), - socketCredentials - ) - ); - }, 1000); - const cleanup = () => { - clearTimeout(timeout); - socket.off('connect', onConnect); - socket.off('error', onError); - }; - - socket.on('connect', onConnect); - socket.on('error', onError); - }); - - socket.on('close', () => { - resetSocketClient(socket); - }); - socket.on('error', () => { - resetSocketClient(socket); - }); - - socketClient = socket; - socketClientKey = currentSocketKey; - return socket; - } catch (error) { - resetSocketClient(); - throw error; - } - })(); - } - return socketClientPromise; -} - -async function notifySocketServer( - filename: string, - hasWorkflow: boolean, - hasStep: boolean, - hasSerde: boolean -): Promise { - const socketCredentials = await getSocketCredentials(); - if (!socketCredentials) { - return; - } - - const socket = await getSocketClient(); - if (!socket) { - throw new Error('Invariant: missing workflow socket connection'); - } - - const message: SocketMessage = { - type: 'file-discovered', - filePath: filename, - hasWorkflow, - hasStep, - hasSerde, - }; - const serializedMessage = serializeMessage( - message, - socketCredentials.authToken - ); - - try { - await writeSocketMessage(socket, serializedMessage); - } catch (error) { - resetSocketClient(socket); - const reconnectedSocket = await getSocketClient(); - if (!reconnectedSocket) { - throw error; - } - await writeSocketMessage(reconnectedSocket, serializedMessage); - } -} - -function isWorkflowRouteStubSource(source: string): boolean { - return source.includes(ROUTE_STUB_FILE_MARKER); -} - -async function createSocketConnection( - socketCredentials: SocketCredentials, - timeoutMs = 1_000 -): Promise { - return await new Promise((resolve, reject) => { - const socket = connect({ - port: socketCredentials.port, - host: '127.0.0.1', - }); - const timeout = setTimeout(() => { - cleanup(); - socket.destroy(); - reject( - annotateConnectionError( - new Error('Socket connection timeout'), - socketCredentials - ) - ); - }, timeoutMs); - const cleanup = () => { - clearTimeout(timeout); - socket.off('connect', onConnect); - socket.off('error', onError); - }; - const onConnect = () => { - socket.setNoDelay(true); - cleanup(); - resolve(socket); - }; - const onError = (error: Error) => { - cleanup(); - socket.destroy(); - reject(annotateConnectionError(error, socketCredentials)); - }; - - socket.on('connect', onConnect); - socket.on('error', onError); - }); -} - -async function waitForDeferredBuildComplete( - socket: Socket, - authToken: string, - timeoutMs = ROUTE_STUB_BUILD_WAIT_TIMEOUT_MS -): Promise { - await new Promise((resolve, reject) => { - let buffer = ''; - const timeout = setTimeout(() => { - cleanup(); - reject( - new Error('Timed out waiting for deferred route build completion') - ); - }, timeoutMs); - const cleanup = () => { - clearTimeout(timeout); - socket.off('data', onData); - socket.off('error', onError); - socket.off('close', onClose); - socket.off('end', onClose); - }; - const onError = (error: Error) => { - cleanup(); - reject(error); - }; - const onClose = () => { - cleanup(); - reject(new Error('Socket closed before deferred route build completed')); - }; - const onData = (chunk: Buffer) => { - buffer += chunk.toString(); - let newlineIndex = buffer.indexOf('\n'); - while (newlineIndex !== -1) { - const line = buffer.slice(0, newlineIndex); - buffer = buffer.slice(newlineIndex + 1); - newlineIndex = buffer.indexOf('\n'); - - const message = parseMessage(line, authToken); - if (message?.type === 'build-complete') { - cleanup(); - resolve(); - return; - } - } - }; - - socket.on('data', onData); - socket.on('error', onError); - socket.on('close', onClose); - socket.on('end', onClose); - }); -} - -async function triggerDeferredRouteStubBuildAndWait(): Promise { - const socketCredentials = await getSocketCredentials(); - if (!socketCredentials) { - return; - } - const socket = await createSocketConnection(socketCredentials); - try { - await writeSocketMessage( - socket, - serializeMessage({ type: 'trigger-build' }, socketCredentials.authToken) - ); - await waitForDeferredBuildComplete(socket, socketCredentials.authToken); - } finally { - socket.destroy(); - } -} - -async function ensureDeferredRouteStubBuildAndWait(): Promise { - if (pendingDeferredRouteStubBuildPromise) { - return pendingDeferredRouteStubBuildPromise; - } - const pendingPromise = triggerDeferredRouteStubBuildAndWait(); - pendingDeferredRouteStubBuildPromise = pendingPromise.finally(() => { - if (pendingDeferredRouteStubBuildPromise === pendingPromise) { - pendingDeferredRouteStubBuildPromise = null; - } - }); - return pendingDeferredRouteStubBuildPromise; -} - async function getBuildersModule(): Promise< typeof import('@workflow/builders') > { @@ -680,52 +225,12 @@ export default function workflowLoader( const isGeneratedWorkflowFile = await checkGeneratedFile(filename); // Skip generated workflow route files to avoid re-processing them. - // Deferred route stubs are a special case: wait for the generated route - // output to become available before returning. if (isGeneratedWorkflowFile) { - if ( - process.env.WORKFLOW_NEXT_LAZY_DISCOVERY === '1' && - isWorkflowRouteStubSource(normalizedSource) - ) { - try { - await ensureDeferredRouteStubBuildAndWait(); - const refreshedSource = await readFile(filename, 'utf8'); - if (!isWorkflowRouteStubSource(refreshedSource)) { - return { code: refreshedSource, map: sourceMap }; - } - } catch (error) { - console.warn( - `[workflow] Failed waiting for deferred route build for ${filename}, using stub output`, - error - ); - } - } return { code: normalizedSource, map: sourceMap }; } // Detect workflow patterns in the source code. const patterns = await detectPatterns(sourceForTransform); - // Always notify discovery tracking, even for `false/false`, so files that - // previously had workflow/step usage are removed from the tracked sets. - const nextPatternState: DiscoveredPatternState = { - hasWorkflow: patterns.hasUseWorkflow, - hasStep: patterns.hasUseStep, - hasSerde: patterns.hasSerde, - }; - const { shouldNotify } = updateDiscoveredPatternState( - filename, - nextPatternState - ); - if ( - shouldNotifySocketForDiscoveredPattern(shouldNotify, nextPatternState) - ) { - await notifySocketServer( - filename, - nextPatternState.hasWorkflow, - nextPatternState.hasStep, - nextPatternState.hasSerde - ); - } // Check if file needs transformation based on patterns and path if (!(await checkShouldTransform(filename, patterns))) { diff --git a/packages/next/src/socket-server.ts b/packages/next/src/socket-server.ts deleted file mode 100644 index 2f82c219c5..0000000000 --- a/packages/next/src/socket-server.ts +++ /dev/null @@ -1,258 +0,0 @@ -import { randomBytes } from 'node:crypto'; -import { mkdir, rm, writeFile } from 'node:fs/promises'; -import { createServer, type Server, type Socket } from 'node:net'; -import { dirname, join } from 'node:path'; - -/** - * Magic preamble that must prefix all messages to authenticate them as workflow messages. - * This prevents accidental processing of messages from port scanners or other local processes. - */ -const MESSAGE_PREAMBLE = 'WF:'; - -/** - * Generate a random authentication token for this server session. - * Clients must include this token in all messages. - */ -function generateAuthToken(): string { - return randomBytes(16).toString('hex'); -} - -/** - * Message types that can be sent between loader and builder - */ -export type SocketMessage = - | { - type: 'file-discovered'; - filePath: string; - hasWorkflow: boolean; - hasStep: boolean; - hasSerde: boolean; - } - | { type: 'trigger-build' } - | { type: 'build-complete' }; - -/** - * Configuration for the socket server - */ -export interface SocketServerConfig { - isDevServer: boolean; - onFileDiscovered: ( - filePath: string, - hasWorkflow: boolean, - hasStep: boolean, - hasSerde: boolean - ) => void; - onTriggerBuild: () => void; - socketInfoFilePath?: string; -} - -/** - * Interface for the socket IO instance returned by createSocketServer - */ -export interface SocketIO { - emit(event: 'build-complete'): void; - getAuthToken(): string; -} - -/** - * Filename for the socket-info file. - */ -export const SOCKET_INFO_FILENAME = 'workflow-socket.json'; - -/** - * Previous filesystem location for the socket-info file. This file lives - * inside `.next/cache/`, which Vercel and Turborepo preserve across builds — - * a stale file from a prior build would cause the loader to attempt to - * connect to a dead port (ECONNREFUSED). The current location is a sibling - * of `cache/` so it isn't preserved. - * - * Exported so the builders can unlink the legacy path at boot, cleaning up - * any leftover file written by older versions of the SDK. - */ -export const LEGACY_SOCKET_INFO_RELATIVE_PATH = join( - 'cache', - SOCKET_INFO_FILENAME -); - -function getDefaultSocketInfoFilePath(): string { - return join(process.cwd(), '.next', SOCKET_INFO_FILENAME); -} - -/** - * Remove any stale socket-info files at boot. - * @param distDir absolute path to the project's `.next` directory. - */ -export async function cleanupStaleSocketInfoFiles( - distDir: string -): Promise { - await Promise.all([ - rm(join(distDir, SOCKET_INFO_FILENAME), { force: true }), - rm(join(distDir, LEGACY_SOCKET_INFO_RELATIVE_PATH), { force: true }), - ]); -} - -/** - * Serialize a message with authentication preamble - */ -export function serializeMessage( - message: SocketMessage, - authToken: string -): string { - return `${MESSAGE_PREAMBLE}${authToken}:${JSON.stringify(message)}\n`; -} - -/** - * Parse and authenticate a message from the socket - * Returns the parsed message if valid, null otherwise - */ -export function parseMessage( - line: string, - authToken: string -): SocketMessage | null { - const trimmed = line.trim(); - if (!trimmed) { - return null; - } - - // Check for preamble - if (!trimmed.startsWith(MESSAGE_PREAMBLE)) { - console.warn('Received message without valid preamble, ignoring'); - return null; - } - - // Extract auth token and payload - const withoutPreamble = trimmed.slice(MESSAGE_PREAMBLE.length); - const colonIndex = withoutPreamble.indexOf(':'); - if (colonIndex === -1) { - console.warn('Received message without auth token separator, ignoring'); - return null; - } - - const messageToken = withoutPreamble.slice(0, colonIndex); - const payload = withoutPreamble.slice(colonIndex + 1); - - // Verify auth token - if (messageToken !== authToken) { - console.warn('Received message with invalid auth token, ignoring'); - return null; - } - - // Parse JSON payload - try { - return JSON.parse(payload) as SocketMessage; - } catch (error) { - console.error('Failed to parse socket message JSON:', error); - return null; - } -} - -/** - * Create a TCP socket server for loader<->builder communication. - * Returns a SocketIO interface for broadcasting messages and the auth token. - * - * SECURITY: Server listens on 127.0.0.1 (localhost only) and uses - * message authentication to prevent processing of unauthorized messages. - */ -export async function createSocketServer( - config: SocketServerConfig -): Promise { - const authToken = generateAuthToken(); - const clients = new Set(); - let buildTriggered = false; - - const server: Server = createServer((socket: Socket) => { - socket.setNoDelay(true); - clients.add(socket); - - // Send build-complete if build already finished (production mode) - if (buildTriggered && !config.isDevServer) { - socket.write(serializeMessage({ type: 'build-complete' }, authToken)); - } - - let buffer = ''; - - socket.on('data', (data: Buffer) => { - buffer += data.toString(); - - // Process complete messages (newline-delimited) - let newlineIndex = buffer.indexOf('\n'); - while (newlineIndex !== -1) { - const line = buffer.slice(0, newlineIndex); - buffer = buffer.slice(newlineIndex + 1); - newlineIndex = buffer.indexOf('\n'); - - const message = parseMessage(line, authToken); - if (!message) { - continue; - } - - if (message.type === 'file-discovered') { - config.onFileDiscovered( - message.filePath, - message.hasWorkflow, - message.hasStep, - message.hasSerde - ); - } else if (message.type === 'trigger-build') { - config.onTriggerBuild(); - } - } - }); - - socket.on('end', () => { - clients.delete(socket); - }); - - socket.on('error', (err: Error) => { - console.error('Socket error:', err); - clients.delete(socket); - }); - }); - - // Listen on random available port (localhost only) - await new Promise((resolve, reject) => { - server.once('error', reject); - server.listen(0, '127.0.0.1', () => { - const address = server.address(); - if (address && typeof address === 'object') { - const socketInfoFilePath = - config.socketInfoFilePath || getDefaultSocketInfoFilePath(); - void (async () => { - try { - await mkdir(dirname(socketInfoFilePath), { recursive: true }); - await writeFile( - socketInfoFilePath, - JSON.stringify( - { - port: address.port, - authToken, - }, - null, - 2 - ) - ); - process.env.WORKFLOW_SOCKET_INFO_PATH = socketInfoFilePath; - process.env.WORKFLOW_SOCKET_PORT = String(address.port); - process.env.WORKFLOW_SOCKET_AUTH = authToken; - resolve(); - } catch (error) { - reject(error); - } - })(); - return; - } - reject(new Error('Failed to obtain workflow socket server address')); - }); - }); - - return { - emit: (_event: 'build-complete') => { - buildTriggered = true; - const message = serializeMessage({ type: 'build-complete' }, authToken); - for (const client of clients) { - client.write(message); - } - }, - getAuthToken: () => authToken, - }; -} diff --git a/scripts/create-test-matrix.mjs b/scripts/create-test-matrix.mjs index be2573ea9f..fdcf726533 100644 --- a/scripts/create-test-matrix.mjs +++ b/scripts/create-test-matrix.mjs @@ -109,23 +109,11 @@ for (const app of [ }, ]) { matrix.app.push( - createMatrixEntry(app.name, app.project, DEV_TEST_CONFIGS[app.name], { - lazyDiscovery: true, - runLabel: 'stable lazyDiscovery enabled', - artifactSuffix: 'stable-lazy-discovery-enabled', - }) - ); - matrix.app.push( - createMatrixEntry(app.name, app.project, DEV_TEST_CONFIGS[app.name], { - lazyDiscovery: false, - runLabel: 'stable lazyDiscovery disabled', - artifactSuffix: 'stable-lazy-discovery-disabled', - }) + createMatrixEntry(app.name, app.project, DEV_TEST_CONFIGS[app.name]) ); matrix.app.push( createMatrixEntry(app.name, app.project, DEV_TEST_CONFIGS[app.name], { canary: true, - lazyDiscovery: true, }) ); }