Add lazy workflow/step discovery via deferredEntries in next

.changeset/slick-lamps-press.md

@@ -0,0 +1,7 @@
+---
+"@workflow/builders": patch
+"@workflow/core": patch
+"@workflow/next": patch
+---
+
+Add lazy workflow/step discovery via deferredEntries in next

packages/builders/src/base-builder.ts

@@ -1,5 +1,5 @@
 import { randomUUID } from 'node:crypto';
-import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
+import { mkdir, readFile, realpath, rename, writeFile } from 'node:fs/promises';
 import { basename, dirname, join, relative, resolve } from 'node:path';
 import { promisify } from 'node:util';
 import { pluralize } from '@workflow/utils';
@@ -25,6 +25,12 @@ const enhancedResolve = promisify(enhancedResolveOriginal);
 const EMIT_SOURCEMAPS_FOR_DEBUGGING =
   process.env.WORKFLOW_EMIT_SOURCEMAPS_FOR_DEBUGGING === '1';
 
+export interface DiscoveredEntries {
+  discoveredSteps: string[];
+  discoveredWorkflows: string[];
+  discoveredSerdeFiles: string[];
+}
+
 /**
  * Base class for workflow builders. Provides common build logic for transforming
  * workflow source files into deployable bundles using esbuild and SWC.
@@ -100,11 +106,7 @@ export abstract class BaseBuilder {
   protected async discoverEntries(
     inputs: string[],
     outdir: string
-  ): Promise<{
-    discoveredSteps: string[];
-    discoveredWorkflows: string[];
-    discoveredSerdeFiles: string[];
-  }> {
+  ): Promise<DiscoveredEntries> {
     const previousResult = this.discoveredEntries.get(inputs);
 
     if (previousResult) {
@@ -270,23 +272,26 @@ export abstract class BaseBuilder {
     outfile,
     externalizeNonSteps,
     tsconfigPath,
+    discoveredEntries,
   }: {
     tsconfigPath?: string;
     inputFiles: string[];
     outfile: string;
     format?: 'cjs' | 'esm';
     externalizeNonSteps?: boolean;
+    discoveredEntries?: DiscoveredEntries;
   }): Promise<{
     context: esbuild.BuildContext | undefined;
     manifest: WorkflowManifest;
   }> {
     // These need to handle watching for dev to scan for
     // new entries and changes to existing ones
-    const {
-      discoveredSteps: stepFiles,
-      discoveredWorkflows: workflowFiles,
-      discoveredSerdeFiles: serdeFiles,
-    } = await this.discoverEntries(inputFiles, dirname(outfile));
+    const discovered =
+      discoveredEntries ??
+      (await this.discoverEntries(inputFiles, dirname(outfile)));
+    const stepFiles = [...discovered.discoveredSteps].sort();
+    const workflowFiles = [...discovered.discoveredWorkflows].sort();
+    const serdeFiles = [...discovered.discoveredSerdeFiles].sort();
 
     // Include serde files that aren't already step files for cross-context class registration.
     // Classes need to be registered in the step bundle so they can be deserialized
@@ -368,6 +373,27 @@ export abstract class BaseBuilder {
     export { stepEntrypoint as POST } from 'workflow/runtime';`;
 
     // Bundle with esbuild and our custom SWC plugin
+    const entriesToBundle = externalizeNonSteps
+      ? [...stepFiles, ...(resolvedBuiltInSteps ? [resolvedBuiltInSteps] : [])]
+      : undefined;
+    const normalizedEntriesToBundle = entriesToBundle
+      ? Array.from(
+          new Set(
+            (
+              await Promise.all(
+                entriesToBundle.map(async (entryToBundle) => {
+                  const resolvedEntry = await realpath(entryToBundle).catch(
+                    () => undefined
+                  );
+                  return resolvedEntry
+                    ? [entryToBundle, resolvedEntry]
+                    : [entryToBundle];
+                })
+              )
+            ).flat()
+          )
+        )
+      : undefined;
     const esbuildCtx = await esbuild.context({
       banner: {
         js: '// biome-ignore-all lint: generated file\n/* eslint-disable */\n',
@@ -414,12 +440,7 @@ export abstract class BaseBuilder {
         createPseudoPackagePlugin(),
         createSwcPlugin({
           mode: 'step',
-          entriesToBundle: externalizeNonSteps
-            ? [
-                ...stepFiles,
-                ...(resolvedBuiltInSteps ? [resolvedBuiltInSteps] : []),
-              ]
-            : undefined,
+          entriesToBundle: normalizedEntriesToBundle,
           outdir: outfile ? dirname(outfile) : undefined,
           workflowManifest,
         }),
@@ -494,21 +515,24 @@ export abstract class BaseBuilder {
     outfile,
     bundleFinalOutput = true,
     tsconfigPath,
+    discoveredEntries,
   }: {
     tsconfigPath?: string;
     inputFiles: string[];
     outfile: string;
     format?: 'cjs' | 'esm';
     bundleFinalOutput?: boolean;
+    discoveredEntries?: DiscoveredEntries;
   }): Promise<{
     manifest: WorkflowManifest;
     interimBundleCtx?: esbuild.BuildContext;
     bundleFinal?: (interimBundleResult: string) => Promise<void>;
   }> {
-    const {
-      discoveredWorkflows: workflowFiles,
-      discoveredSerdeFiles: serdeFiles,
-    } = await this.discoverEntries(inputFiles, dirname(outfile));
+    const discovered =
+      discoveredEntries ??
+      (await this.discoverEntries(inputFiles, dirname(outfile)));
+    const workflowFiles = [...discovered.discoveredWorkflows].sort();
+    const serdeFiles = [...discovered.discoveredSerdeFiles].sort();
 
     // Include serde files that aren't already workflow files for cross-context class registration.
     // Classes need to be registered in the workflow bundle so they can be deserialized

packages/builders/src/swc-esbuild-plugin.ts

@@ -1,5 +1,5 @@
-import { readFile } from 'node:fs/promises';
-import { relative } from 'node:path';
+import { access, readFile, realpath } from 'node:fs/promises';
+import { basename, relative, resolve } from 'node:path';
 import { promisify } from 'node:util';
 import enhancedResolveOrig from 'enhanced-resolve';
 import type { Plugin } from 'esbuild';
@@ -55,6 +55,42 @@ const NODE_ESM_RESOLVE_OPTIONS = {
   conditionNames: ['node', 'import'],
 };
 
+async function resolveWorkflowAliasRelativePath(
+  absoluteFilePath: string,
+  workingDir: string
+): Promise<string | undefined> {
+  const fileName = basename(absoluteFilePath);
+  const aliasDirs = ['workflows', 'src/workflows'];
+  const resolvedFilePath = await realpath(absoluteFilePath).catch(
+    () => undefined
+  );
+  if (!resolvedFilePath) {
+    return undefined;
+  }
+
+  const aliases = await Promise.all(
+    aliasDirs.map(async (aliasDir) => {
+      const candidatePath = resolve(workingDir, aliasDir, fileName);
+      try {
+        await access(candidatePath);
+      } catch {
+        return undefined;
+      }
+      const resolvedCandidatePath = await realpath(candidatePath).catch(
+        () => undefined
+      );
+      if (!resolvedCandidatePath) {
+        return undefined;
+      }
+      return resolvedCandidatePath === resolvedFilePath
+        ? `${aliasDir}/${fileName}`
+        : undefined;
+    })
+  );
+
+  return aliases.find((aliasPath): aliasPath is string => Boolean(aliasPath));
+}
+
 export function createSwcPlugin(options: SwcPluginOptions): Plugin {
   return {
     name: 'swc-workflow-plugin',
@@ -187,10 +223,16 @@ export function createSwcPlugin(options: SwcPluginOptions): Plugin {
             // Handle files discovered outside the working directory
             // These come back as ../path/to/file, but we want just path/to/file
             if (relativeFilepath.startsWith('../')) {
-              relativeFilepath = relativeFilepath
-                .split('/')
-                .filter((part) => part !== '..')
-                .join('/');
+              const aliasedRelativePath =
+                await resolveWorkflowAliasRelativePath(args.path, workingDir);
+              if (aliasedRelativePath) {
+                relativeFilepath = aliasedRelativePath;
+              } else {
+                relativeFilepath = relativeFilepath
+                  .split('/')
+                  .filter((part) => part !== '..')
+                  .join('/');
+              }
             }
           }
 

packages/core/e2e/dev.test.ts

@@ -1,6 +1,6 @@
 import fs from 'fs/promises';
 import path from 'path';
-import { afterEach, describe, expect, test } from 'vitest';
+import { afterEach, beforeAll, describe, expect, test } from 'vitest';
 import { getWorkbenchAppPath } from './utils';
 
 export interface DevTestConfig {
@@ -44,7 +44,19 @@
     const workflowsDir = finalConfig.workflowsDir ?? 'workflows';
     const restoreFiles: Array<{ path: string; content: string }> = [];
 
+    const prewarm = async () => {
+      // pre-warm for dev watching
+      await fetch(new URL('/', process.env.DEPLOYMENT_URL)).catch(() => {});
+      await fetch(new URL('/api/chat', process.env.DEPLOYMENT_URL)).catch(
+        () => {}
+      );
+    };
+
+    beforeAll(async () => {
+      await prewarm();
+    });
+
     afterEach(async () => {
       await Promise.all(
         restoreFiles.map(async (item) => {
           if (item.content === '') {
@@ -54,6 +66,7 @@
           }
         })
       );
+      await prewarm();
       restoreFiles.length = 0;
     });
 
@@ -145,6 +158,7 @@
 
         while (true) {
           try {
+            await fetch(new URL('/api/chat', process.env.DEPLOYMENT_URL));
             const workflowContent = await fs.readFile(
               generatedWorkflow,
               'utf8'

packages/next/package.json

@@ -45,7 +45,7 @@
     "@types/node": "catalog:",
     "@types/semver": "7.7.1",
     "@types/watchpack": "2.4.4",
-    "next": "16.1.6"
+    "next": "https://files-r8fgftscl-vtest314-ijjk-testing.vercel.app"
   },
   "peerDependencies": {
     "next": ">13"