fix: Add globalPreload to ts-node/esm for node 20

README.md

@@ -230,7 +230,7 @@ To test your version of `env` for compatibility with `-S`:
 
 ## node flags and other tools
 
-You can register ts-node without using our CLI: `node -r ts-node/register` and `node --loader ts-node/esm`
+You can register ts-node without using our CLI: `node -r ts-node/register`, `node --loader ts-node/esm`, or `node --import ts-node/import` in node 20.6 and above.
 
 In many cases, setting [`NODE_OPTIONS`](https://nodejs.org/api/cli.html#cli_node_options_options) will enable `ts-node` within other node tools, child processes, and worker threads.  This can be combined with other node flags.
 

ava.config.cjs

@@ -16,7 +16,7 @@ module.exports = {
     CONCURRENT_TESTS: '4'
   },
   require: ['./src/test/remove-env-var-force-color.js'],
-  nodeArguments: ['--loader', './src/test/test-loader.mjs', '--no-warnings'],
+  nodeArguments: ['--loader', './src/test/test-loader/loader.mjs', '--no-warnings'],
   timeout: '300s',
   concurrency: 4,
   // We do chdir -- maybe other things -- that you can't do in worker_threads.

child-loader.mjs

@@ -1,8 +1,11 @@
-import { fileURLToPath } from 'url';
 import { createRequire } from 'module';
+import { fileURLToPath } from 'url';
+
 const require = createRequire(fileURLToPath(import.meta.url));
 
 // TODO why use require() here?  I think we can just `import`
 /** @type {import('./dist/child-loader')} */
 const childLoader = require('./dist/child/child-loader');
-export const { resolve, load, getFormat, transformSource } = childLoader;
+export const { resolve, load, getFormat, transformSource, bindFromLoaderThread } = childLoader;
+
+bindFromLoaderThread(import.meta.url);

esm.mjs

@@ -4,4 +4,4 @@ const require = createRequire(fileURLToPath(import.meta.url));
 
 /** @type {import('./dist/esm')} */
 const esm = require('./dist/esm');
-export const { resolve, load, getFormat, transformSource } = esm.registerAndCreateEsmHooks();
+export const { initialize, resolve, load, getFormat, transformSource, globalPreload } = esm.registerAndCreateEsmHooks();

package.json

@@ -113,7 +113,7 @@
   "homepage": "https://typestrong.org/ts-node",
   "devDependencies": {
     "@TypeStrong/fs-fixture-builder": "https://github.com/Typestrong/fs-fixture-builder.git#3099e53621daf99db971af29c96145dc115693cd",
-    "@cspotcode/ava-lib": "https://github.com/cspotcode/ava-lib#bbbed83f393342b51dc6caf2ddf775a3e89371d8",
+    "@cspotcode/ava-lib": "https://github.com/cspotcode/ava-lib#805aab17b2b89c388596b6dc2b4eece403c5fb87",
     "@cspotcode/expect-stream": "https://github.com/cspotcode/node-expect-stream#4e425ff1eef240003af8716291e80fbaf3e3ae8f",
     "@microsoft/api-extractor": "^7.19.4",
     "@swc/core": "1.3.32",

src/bin.ts

@@ -71,6 +71,7 @@ export interface BootstrapState {
   parseArgvResult: ReturnType<typeof parseArgv>;
   phase2Result?: ReturnType<typeof phase2>;
   phase3Result?: ReturnType<typeof phase3>;
+  isLoaderThread?: boolean;
 }
 
 /** @internal */
@@ -441,7 +442,7 @@ function getEntryPointInfo(state: BootstrapState) {
 }
 
 function phase4(payload: BootstrapState) {
-  const { isInChildProcess, tsNodeScript } = payload;
+  const { isInChildProcess, tsNodeScript, isLoaderThread } = payload;
   const { version, showConfig, restArgs, code, print, argv } = payload.parseArgvResult;
   const { cwd } = payload.phase2Result!;
   const { preloadedConfig } = payload.phase3Result!;
@@ -522,8 +523,12 @@ function phase4(payload: BootstrapState) {
 
   if (replStuff) replStuff.state.path = join(cwd, REPL_FILENAME(service.ts.version));
 
-  if (isInChildProcess)
+  if (isInChildProcess) {
     (require('./child/child-loader') as typeof import('./child/child-loader')).lateBindHooks(createEsmHooks(service));
+    // we should not do anything else at this point in the loader thread,
+    // let the entrypoint run the actual program.
+    if (isLoaderThread) return;
+  }
 
   // Bind REPL service to ts-node compiler service (chicken-and-egg problem)
   replStuff?.repl.setService(service);

src/child/child-loader.ts

@@ -1,5 +1,24 @@
 import type { NodeLoaderHooksAPI1, NodeLoaderHooksAPI2 } from '..';
 import { filterHooksByAPIVersion } from '../esm';
+import { URL } from 'url';
+import { bootstrap } from '../bin';
+import { versionGteLt } from '../util';
+import { argPrefix, decompress } from './argv-payload';
+
+// On node v20, we cannot lateBind the hooks from outside the loader thread
+// so it has to be done in the loader thread.
+export function bindFromLoaderThread(loaderURL: string) {
+  // If we aren't in a loader thread, then skip this step.
+  if (!versionGteLt(process.versions.node, '20.0.0')) return;
+
+  const url = new URL(loaderURL);
+  const base64Payload = url.searchParams.get(argPrefix);
+  if (!base64Payload) throw new Error('unexpected loader url');
+  const state = decompress(base64Payload);
+  state.isInChildProcess = true;
+  state.isLoaderThread = true;
+  bootstrap(state);
+}
 
 let hooks: NodeLoaderHooksAPI1 & NodeLoaderHooksAPI2;
 

src/child/spawn-child.ts

@@ -10,16 +10,20 @@ import { argPrefix, compress } from './argv-payload';
  *   the child process.
  */
 export function callInChild(state: BootstrapState) {
+  const loaderURL = pathToFileURL(require.resolve('../../child-loader.mjs'));
+  const compressedState = compress(state);
+  loaderURL.searchParams.set(argPrefix, compressedState);
+
   const child = spawn(
     process.execPath,
     [
       '--require',
       require.resolve('./child-require.js'),
       '--loader',
       // Node on Windows doesn't like `c:\` absolute paths here; must be `file:///c:/`
-      pathToFileURL(require.resolve('../../child-loader.mjs')).toString(),
+      loaderURL.toString(),
       require.resolve('./child-entrypoint.js'),
-      `${argPrefix}${compress(state)}`,
+      `${argPrefix}${compressedState}`,
       ...state.parseArgvResult.restArgs,
     ],
     {

src/esm.ts

@@ -1,9 +1,10 @@
-import { register, RegisterOptions, Service } from './index';
+import { register, RegisterOptions, Service, type TSError } from './index';
 import { parse as parseUrl, format as formatUrl, UrlWithStringQuery, fileURLToPath, pathToFileURL } from 'url';
 import { extname, resolve as pathResolve } from 'path';
 import * as assert from 'assert';
 import { normalizeSlashes, versionGteLt } from './util';
 import { createRequire } from 'module';
+import type { MessagePort } from 'worker_threads';
 
 // Note: On Windows, URLs look like this: file:///D:/dev/@TypeStrong/ts-node-examples/foo.ts
 
@@ -43,6 +44,7 @@ export namespace NodeLoaderHooksAPI1 {
 export interface NodeLoaderHooksAPI2 {
   resolve: NodeLoaderHooksAPI2.ResolveHook;
   load: NodeLoaderHooksAPI2.LoadHook;
+  globalPreload?: NodeLoaderHooksAPI2.GlobalPreloadHook;
 }
 export namespace NodeLoaderHooksAPI2 {
   export type ResolveHook = (
@@ -74,6 +76,18 @@ export namespace NodeLoaderHooksAPI2 {
   export interface NodeImportAssertions {
     type?: 'json';
   }
+  export type GlobalPreloadHook = (context?: { port: MessagePort }) => string;
+}
+
+export interface NodeLoaderHooksAPI3 {
+  resolve: NodeLoaderHooksAPI2.ResolveHook;
+  load: NodeLoaderHooksAPI2.LoadHook;
+  initialize?: NodeLoaderHooksAPI3.InitializeHook;
+}
+export namespace NodeLoaderHooksAPI3 {
+  // technically this can be anything that can be passed through a postMessage channel,
+  // but defined here based on how ts-node uses it.
+  export type InitializeHook = (data: any) => void | Promise<void>;
 }
 
 export type NodeLoaderHooksFormat = 'builtin' | 'commonjs' | 'dynamic' | 'json' | 'module' | 'wasm';
@@ -84,17 +98,24 @@ export interface NodeImportAssertions {
 }
 
 // The hooks API changed in node version X so we need to check for backwards compatibility.
-const newHooksAPI = versionGteLt(process.versions.node, '16.12.0');
+const hooksAPIVersion = versionGteLt(process.versions.node, '21.0.0')
+  ? 3
+  : versionGteLt(process.versions.node, '16.12.0')
+  ? 2
+  : 1;
 
 /** @internal */
 export function filterHooksByAPIVersion(
-  hooks: NodeLoaderHooksAPI1 & NodeLoaderHooksAPI2
-): NodeLoaderHooksAPI1 | NodeLoaderHooksAPI2 {
-  const { getFormat, load, resolve, transformSource } = hooks;
+  hooks: NodeLoaderHooksAPI1 & NodeLoaderHooksAPI2 & NodeLoaderHooksAPI3
+): NodeLoaderHooksAPI1 | NodeLoaderHooksAPI2 | NodeLoaderHooksAPI3 {
+  const { getFormat, load, resolve, transformSource, globalPreload, initialize } = hooks;
   // Explicit return type to avoid TS's non-ideal inferred type
-  const hooksAPI: NodeLoaderHooksAPI1 | NodeLoaderHooksAPI2 = newHooksAPI
-    ? { resolve, load, getFormat: undefined, transformSource: undefined }
-    : { resolve, getFormat, transformSource, load: undefined };
+  const hooksAPI: NodeLoaderHooksAPI1 | NodeLoaderHooksAPI2 | NodeLoaderHooksAPI3 =
+    hooksAPIVersion === 3
+      ? { resolve, load, initialize, globalPreload: undefined, transformSource: undefined, getFormat: undefined }
+      : hooksAPIVersion === 2
+      ? { resolve, load, globalPreload, initialize: undefined, getFormat: undefined, transformSource: undefined }
+      : { resolve, getFormat, transformSource, initialize: undefined, globalPreload: undefined, load: undefined };
   return hooksAPI;
 }
 
@@ -111,14 +132,44 @@ export function createEsmHooks(tsNodeService: Service) {
   const nodeResolveImplementation = tsNodeService.getNodeEsmResolver();
   const nodeGetFormatImplementation = tsNodeService.getNodeEsmGetFormat();
   const extensions = tsNodeService.extensions;
+  const useLoaderThread = versionGteLt(process.versions.node, '20.0.0');
 
   const hooksAPI = filterHooksByAPIVersion({
     resolve,
     load,
     getFormat,
     transformSource,
+    globalPreload: useLoaderThread ? globalPreload : undefined,
+    initialize: undefined,
   });
 
+  function globalPreload({ port }: { port?: MessagePort } = {}) {
+    // The loader thread doesn't get process.stderr.isTTY properly,
+    // so this signal lets us infer it based on the state of the main
+    // thread, but only relevant if options.pretty is unset.
+    let stderrTTYSignal: string;
+    if (port && tsNodeService.options.pretty === undefined) {
+      port.on('message', (data: { stderrIsTTY?: boolean }) => {
+        if (data.stderrIsTTY) {
+          tsNodeService.setPrettyErrors(true);
+        }
+      });
+      stderrTTYSignal = `
+        port.postMessage({
+          stderrIsTTY: !!process.stderr.isTTY
+        });
+      `;
+    } else {
+      stderrTTYSignal = '';
+    }
+    return `
+      const { createRequire } = getBuiltin('module');
+      const require = createRequire(${JSON.stringify(__filename)});
+      ${stderrTTYSignal}
+      require('./index').register();
+    `;
+  }
+
   function isFileUrlOrNodeStyleSpecifier(parsed: UrlWithStringQuery) {
     // We only understand file:// URLs, but in node, the specifier can be a node-style `./foo` or `foo`
     const { protocol } = parsed;
@@ -211,7 +262,7 @@ export function createEsmHooks(tsNodeService: Service) {
     format: NodeLoaderHooksFormat;
     source: string | Buffer | undefined;
   }> {
-    return addShortCircuitFlag(async () => {
+    return await addShortCircuitFlag(async () => {
       // If we get a format hint from resolve() on the context then use it
       // otherwise call the old getFormat() hook using node's old built-in defaultGetFormat() that ships with ts-node
       const format =
@@ -239,8 +290,23 @@ export function createEsmHooks(tsNodeService: Service) {
         });
 
         // Call the old hook
-        const { source: transformedSource } = await transformSource(rawSource, { url, format }, defaultTransformSource);
-        source = transformedSource;
+        try {
+          const { source: transformedSource } = await transformSource(
+            rawSource,
+            { url, format },
+            defaultTransformSource
+          );
+          source = transformedSource;
+        } catch (er) {
+          // throw an error that can make it through the loader thread
+          // comms channel intact.
+          const tsErr = er as TSError;
+          const err = new Error(tsErr.message.trimEnd());
+          const { diagnosticCodes } = tsErr;
+          Object.assign(err, { diagnosticCodes });
+          Error.captureStackTrace(err, load);
+          throw err;
+        }
       }
 
       return { format, source };
@@ -348,11 +414,12 @@ export function createEsmHooks(tsNodeService: Service) {
 }
 
 async function addShortCircuitFlag<T>(fn: () => Promise<T>) {
-  const ret = await fn();
-  // Not sure if this is necessary; being lazy.  Can revisit in the future.
-  if (ret == null) return ret;
-  return {
-    ...ret,
-    shortCircuit: true,
-  };
+  return fn().then((ret) => {
+    // Not sure if this is necessary; being lazy.  Can revisit in the future.
+    if (ret == null) return ret;
+    return {
+      ...ret,
+      shortCircuit: true,
+    };
+  });
 }

src/index.ts

@@ -480,6 +480,7 @@ export interface Service {
   ignored(fileName: string): boolean;
   compile(code: string, fileName: string, lineOffset?: number): string;
   getTypeInfo(code: string, fileName: string, position: number): TypeInfo;
+  setPrettyErrors(pretty: boolean): void;
   /** @internal */
   configFilePath: string | undefined;
   /** @internal */
@@ -690,7 +691,13 @@ export function createFromPreloadedConfig(foundConfigResult: ReturnType<typeof f
   }
 
   // Install source map support and read from memory cache.
+  const useBuiltInSourceMaps = versionGteLt(process.versions.node, '20.0.0');
   function installSourceMapSupport() {
+    if (useBuiltInSourceMaps) {
+      //@ts-ignore added to node somewhat recently, not yet in DT.
+      process.setSourceMapsEnabled(true);
+      return;
+    }
     const sourceMapSupport = require('@cspotcode/source-map-support') as typeof _sourceMapSupport;
     sourceMapSupport.install({
       environment: 'node',
@@ -719,11 +726,16 @@ export function createFromPreloadedConfig(foundConfigResult: ReturnType<typeof f
     });
   }
 
-  const shouldHavePrettyErrors = options.pretty === undefined ? process.stdout.isTTY : options.pretty;
+  let shouldHavePrettyErrors!: boolean;
+  let formatDiagnostics: (diagnostics: readonly _ts.Diagnostic[], host: _ts.FormatDiagnosticsHost) => string;
 
-  const formatDiagnostics = shouldHavePrettyErrors
-    ? ts.formatDiagnosticsWithColorAndContext || ts.formatDiagnostics
-    : ts.formatDiagnostics;
+  function setPrettyErrors(pretty: boolean) {
+    shouldHavePrettyErrors = pretty;
+    formatDiagnostics = shouldHavePrettyErrors
+      ? ts.formatDiagnosticsWithColorAndContext || ts.formatDiagnostics
+      : ts.formatDiagnostics;
+  }
+  setPrettyErrors(options.pretty !== undefined ? options.pretty : !!process.stderr.isTTY);
 
   function createTSError(diagnostics: ReadonlyArray<_ts.Diagnostic>) {
     const diagnosticText = formatDiagnostics(diagnostics, diagnosticHost);
@@ -1282,6 +1294,7 @@ export function createFromPreloadedConfig(foundConfigResult: ReturnType<typeof f
     getNodeEsmGetFormat,
     getNodeCjsLoader,
     extensions,
+    setPrettyErrors,
   };
 }