perf(bundle-source): cut multi-entry agoric bundling time and add detailed profiling

packages/bundle-source/README.md

@@ -119,6 +119,64 @@ map for every physical module.
 It is not yet quite clever enough to collect source maps for sources that do
 not exist.
 
+## Profiling
+
+`bundle-source` can emit Chrome trace files for performance analysis.
+This works for programmatic usage and CLI usage, including builds in larger
+repos like `agoric-sdk`.
+
+Enable with environment variables:
+
+```console
+ENDO_BUNDLE_SOURCE_PROFILE=1 \
+ENDO_BUNDLE_SOURCE_PROFILE_DIR=/tmp/bs-profiles \
+yarn bundle-source app.js > /tmp/app-bundle.json
+```
+
+Each bundle call writes one `*.trace.json` file. Open these in Chrome tracing
+tools or convert for Speedscope.
+
+You can also control profiling in code:
+
+```js
+await bundleSource('program.js', {
+  profile: {
+    enabled: true,
+    traceDir: '/tmp/bs-profiles',
+    // or traceFile: '/tmp/specific.trace.json'
+  },
+});
+```
+
+Environment variables:
+- `ENDO_BUNDLE_SOURCE_PROFILE`: enable profiling when truthy (`1`, `true`, `yes`, `on`)
+- `ENDO_BUNDLE_SOURCE_PROFILE_DIR`: output directory for generated trace files
+- `ENDO_BUNDLE_SOURCE_PROFILE_FILE`: explicit output file for a single run
+- `ENDO_BUNDLE_SOURCE_PROFILE_STDERR`: if truthy, prints each generated trace path to stderr
+
+Merge and summarize many profile traces:
+
+```console
+yarn workspace @endo/bundle-source trace:merge -- /tmp/bs-profiles
+```
+
+This generates:
+- `merged.trace.json` for trace viewers.
+- `summary.json` with aggregate span statistics.
+- `summary.md` with a top spans table by total duration.
+
+Profile bundling all `source-spec-registry.js` entries from an `agoric-sdk`
+checkout using the current checkout's `bundle-source`:
+
+```console
+yarn workspace @endo/bundle-source profile:agoric-bundling -- \
+  --agoric-sdk-root /opt/agoric/agoric-sdk \
+  --out-dir /tmp/profile-agoric-bundling
+```
+
+The tool writes bundles, raw traces, merged trace, and summary files to
+`--out-dir`, and prints a top-spans summary table at the end.
+
 ## `moduleFormat` explanations
 
 <a id="getexport-moduleformat"></a>

packages/bundle-source/package.json

@@ -16,6 +16,8 @@
     "prepack": "tsc --build tsconfig.build.json",
     "postpack": "git clean -fX \"*.d.ts*\" \"*.d.cts*\" \"*.d.mts*\" \"*.tsbuildinfo\"",
     "test": "ava",
+    "profile:agoric-bundling": "tools/profile-agoric-bundling.mts",
+    "trace:merge": "node tools/trace-merge.js",
     "test:xs": "exit 0",
     "lint-fix": "eslint --fix '**/*.js'",
     "lint": "yarn lint:types && yarn lint:eslint",

packages/bundle-source/src/endo.js

@@ -21,7 +21,7 @@ const textDecoder = new TextDecoder();
  */
 export const makeBundlingKit = (
   { pathResolve, userInfo, computeSha512, platform, env },
-  { cacheSourceMaps, elideComments, noTransforms, commonDependencies },
+  { cacheSourceMaps, elideComments, noTransforms, commonDependencies, profiler },
 ) => {
   if (noTransforms && elideComments) {
     throw new Error(
@@ -31,8 +31,8 @@ export const makeBundlingKit = (
 
   /** @type {Set<Promise<void>>} */
   const sourceMapJobs = new Set();
-  /** @type {(sourceMap: string, sourceDescriptor: SourceMapDescriptor) => Promise<void>} */
-  let writeSourceMap = async () => {};
+  /** @type {((sourceMap: string, sourceDescriptor: SourceMapDescriptor) => Promise<void>) | undefined} */
+  let writeSourceMap;
   if (cacheSourceMaps) {
     if (!computeSha512) {
       throw new Error('computeSha512 is required when cacheSourceMaps is true');
@@ -48,63 +48,71 @@ export const makeBundlingKit = (
       sourceMap,
       { sha512, compartment: packageLocation, module: moduleSpecifier },
     ) => {
-      const location = new URL(moduleSpecifier, packageLocation).href;
-      const locationSha512 = computeSha512(location);
-      const locationSha512Head = locationSha512.slice(0, 2);
-      const locationSha512Tail = locationSha512.slice(2);
-      const sha512Head = sha512.slice(0, 2);
-      const sha512Tail = sha512.slice(2);
-      const sourceMapTrackerDirectory = pathResolve(
-        sourceMapsTrackerDirectory,
-        locationSha512Head,
-      );
-      const sourceMapTrackerPath = pathResolve(
-        sourceMapTrackerDirectory,
-        locationSha512Tail,
-      );
-      const sourceMapDirectory = pathResolve(
-        sourceMapsCacheDirectory,
-        sha512Head,
-      );
-      const sourceMapPath = pathResolve(
-        sourceMapDirectory,
-        `${sha512Tail}.map.json`,
+      const endWriteSourceMap = profiler?.startSpan(
+        'bundleSource.writeSourceMap',
+        { moduleSpecifier, packageLocation },
       );
+      try {
+        const location = new URL(moduleSpecifier, packageLocation).href;
+        const locationSha512 = computeSha512(location);
+        const locationSha512Head = locationSha512.slice(0, 2);
+        const locationSha512Tail = locationSha512.slice(2);
+        const sha512Head = sha512.slice(0, 2);
+        const sha512Tail = sha512.slice(2);
+        const sourceMapTrackerDirectory = pathResolve(
+          sourceMapsTrackerDirectory,
+          locationSha512Head,
+        );
+        const sourceMapTrackerPath = pathResolve(
+          sourceMapTrackerDirectory,
+          locationSha512Tail,
+        );
+        const sourceMapDirectory = pathResolve(
+          sourceMapsCacheDirectory,
+          sha512Head,
+        );
+        const sourceMapPath = pathResolve(
+          sourceMapDirectory,
+          `${sha512Tail}.map.json`,
+        );
 
-      await fs.promises
-        .readFile(sourceMapTrackerPath, 'utf-8')
-        .then(async oldSha512 => {
-          oldSha512 = oldSha512.trim();
-          if (oldSha512 === sha512) {
-            return;
-          }
-          const oldSha512Head = oldSha512.slice(0, 2);
-          const oldSha512Tail = oldSha512.slice(2);
-          const oldSourceMapDirectory = pathResolve(
-            sourceMapsCacheDirectory,
-            oldSha512Head,
-          );
-          const oldSourceMapPath = pathResolve(
-            oldSourceMapDirectory,
-            `${oldSha512Tail}.map.json`,
-          );
-          await fs.promises.unlink(oldSourceMapPath);
-          const entries = await fs.promises.readdir(oldSourceMapDirectory);
-          if (entries.length === 0) {
-            await fs.promises.rmdir(oldSourceMapDirectory);
-          }
-        })
-        .catch(error => {
-          if (error.code !== 'ENOENT') {
-            throw error;
-          }
-        });
+        await fs.promises
+          .readFile(sourceMapTrackerPath, 'utf-8')
+          .then(async oldSha512 => {
+            oldSha512 = oldSha512.trim();
+            if (oldSha512 === sha512) {
+              return;
+            }
+            const oldSha512Head = oldSha512.slice(0, 2);
+            const oldSha512Tail = oldSha512.slice(2);
+            const oldSourceMapDirectory = pathResolve(
+              sourceMapsCacheDirectory,
+              oldSha512Head,
+            );
+            const oldSourceMapPath = pathResolve(
+              oldSourceMapDirectory,
+              `${oldSha512Tail}.map.json`,
+            );
+            await fs.promises.unlink(oldSourceMapPath);
+            const entries = await fs.promises.readdir(oldSourceMapDirectory);
+            if (entries.length === 0) {
+              await fs.promises.rmdir(oldSourceMapDirectory);
+            }
+          })
+          .catch(error => {
+            if (error.code !== 'ENOENT') {
+              throw error;
+            }
+          });
 
-      await fs.promises.mkdir(sourceMapDirectory, { recursive: true });
-      await fs.promises.writeFile(sourceMapPath, sourceMap);
+        await fs.promises.mkdir(sourceMapDirectory, { recursive: true });
+        await fs.promises.writeFile(sourceMapPath, sourceMap);
 
-      await fs.promises.mkdir(sourceMapTrackerDirectory, { recursive: true });
-      await fs.promises.writeFile(sourceMapTrackerPath, sha512);
+        await fs.promises.mkdir(sourceMapTrackerDirectory, { recursive: true });
+        await fs.promises.writeFile(sourceMapTrackerPath, sha512);
+      } finally {
+        endWriteSourceMap?.();
+      }
     };
   }
 
@@ -122,25 +130,41 @@ export const makeBundlingKit = (
     location,
     sourceMap,
   ) => {
+    const endTransformModule = profiler?.startSpan('bundleSource.transformModule', {
+      parser,
+      specifier,
+      location,
+    });
     if (!['mjs', 'cjs'].includes(parser)) {
       throw Error(`Parser ${parser} not supported in evadeEvalCensor`);
     }
     const babelSourceType = parser === 'mjs' ? 'module' : 'script';
     const source = textDecoder.decode(sourceBytes);
     const priorSourceMap =
       typeof sourceMap === 'string' ? sourceMap : undefined;
-    const { code: object, map } = await evadeCensor(source, {
-      sourceType: babelSourceType,
-      sourceMap: priorSourceMap,
-      sourceUrl: new URL(specifier, location).href,
-      elideComments,
-    });
-    const objectBytes = textEncoder.encode(object);
-    return {
-      bytes: objectBytes,
-      parser,
-      sourceMap: map ? JSON.stringify(map) : undefined,
-    };
+    /** @type {number | undefined} */
+    let outputBytes;
+    try {
+      const { code: object, map } = await evadeCensor(source, {
+        sourceType: babelSourceType,
+        sourceMap: priorSourceMap,
+        sourceUrl: new URL(specifier, location).href,
+        elideComments,
+        profileStartSpan: profiler?.startSpan,
+      });
+      const objectBytes = textEncoder.encode(object);
+      outputBytes = objectBytes.length;
+      return {
+        bytes: objectBytes,
+        parser,
+        sourceMap: map ? JSON.stringify(map) : undefined,
+      };
+    } finally {
+      endTransformModule?.({
+        inputBytes: sourceBytes.length,
+        outputBytes,
+      });
+    }
   };
 
   /** @type {ParserForLanguageLike} */
@@ -196,16 +220,24 @@ export const makeBundlingKit = (
       packageLocation,
       options = undefined,
     ) {
+      const endTypeErasure = profiler?.startSpan('bundleSource.typeErase', {
+        parser: 'mts',
+        specifier,
+      });
       const sourceText = textDecoder.decode(sourceBytes);
       const objectText = tsBlankSpace(sourceText);
       const objectBytes = textEncoder.encode(objectText);
-      return parserForLanguage.mjs.parse(
-        objectBytes,
-        specifier,
-        moduleLocation,
-        packageLocation,
-        options,
-      );
+      try {
+        return parserForLanguage.mjs.parse(
+          objectBytes,
+          specifier,
+          moduleLocation,
+          packageLocation,
+          options,
+        );
+      } finally {
+        endTypeErasure?.();
+      }
     },
     heuristicImports: false,
     synchronous: false,
@@ -219,16 +251,24 @@ export const makeBundlingKit = (
       packageLocation,
       options = undefined,
     ) {
+      const endTypeErasure = profiler?.startSpan('bundleSource.typeErase', {
+        parser: 'cts',
+        specifier,
+      });
       const sourceText = textDecoder.decode(sourceBytes);
       const objectText = tsBlankSpace(sourceText);
       const objectBytes = textEncoder.encode(objectText);
-      return parserForLanguage.cjs.parse(
-        objectBytes,
-        specifier,
-        moduleLocation,
-        packageLocation,
-        options,
-      );
+      try {
+        return parserForLanguage.cjs.parse(
+          objectBytes,
+          specifier,
+          moduleLocation,
+          packageLocation,
+          options,
+        );
+      } finally {
+        endTypeErasure?.();
+      }
     },
     heuristicImports: true,
     synchronous: false,
@@ -237,9 +277,11 @@ export const makeBundlingKit = (
   parserForLanguage = { ...parserForLanguage, mts: mtsParser, cts: ctsParser };
 
   /** @type {BundlingKit['sourceMapHook']} */
-  const sourceMapHook = (sourceMap, sourceDescriptor) => {
-    sourceMapJobs.add(writeSourceMap(sourceMap, sourceDescriptor));
-  };
+  const sourceMapHook =
+    writeSourceMap &&
+    ((sourceMap, sourceDescriptor) => {
+      sourceMapJobs.add(writeSourceMap(sourceMap, sourceDescriptor));
+    });
 
   const workspaceLanguageForExtension = { mts: 'mts', cts: 'cts' };
   const workspaceModuleLanguageForExtension = { ts: 'mts' };

packages/bundle-source/src/exports.d.ts

@@ -1,4 +1,5 @@
 export type {
+  BundleProfilingOptions,
   BundleOptions,
   BundleSource,
   BundleSourceResult,