diff --git a/.vscode/launch.json b/.vscode/launch.json index 6084882d779..d52370624c1 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -39,6 +39,24 @@ "console": "integratedTerminal", "internalConsoleOptions": "neverOpen" }, + { + "type": "node", + "request": "launch", + "name": "Debug Build in Selected Project (Heft)", + "cwd": "${fileDirname}", + "runtimeArgs": [ + "--nolazy", + "--inspect-brk", + "${workspaceFolder}/apps/heft/lib/start.js", + "--debug", + "build" + ], + "skipFiles": ["/**"], + "outFiles": [], + "sourceMaps": true, + "console": "integratedTerminal", + "internalConsoleOptions": "neverOpen" + }, { "name": "Attach", "type": "node", diff --git a/build-tests/heft-typescript-composite-test/src/indexA.ts b/build-tests/heft-typescript-composite-test/src/indexA.ts index 6b4db719843..4394a4bf61e 100644 --- a/build-tests/heft-typescript-composite-test/src/indexA.ts +++ b/build-tests/heft-typescript-composite-test/src/indexA.ts @@ -8,3 +8,5 @@ import(/* webpackChunkName: 'chunk' */ './chunks/ChunkClass') .catch((e) => { console.log('Error: ' + e.message); }); + +export {}; diff --git a/build-tests/heft-typescript-composite-test/src/indexB.ts b/build-tests/heft-typescript-composite-test/src/indexB.ts index 16401835981..e122ca67a73 100644 --- a/build-tests/heft-typescript-composite-test/src/indexB.ts +++ b/build-tests/heft-typescript-composite-test/src/indexB.ts @@ -1 +1,3 @@ console.log('dostuff'); + +export {}; diff --git a/build-tests/heft-typescript-composite-test/tsconfig-base.json b/build-tests/heft-typescript-composite-test/tsconfig-base.json index 2c750bee0ce..520a0cf3447 100644 --- a/build-tests/heft-typescript-composite-test/tsconfig-base.json +++ b/build-tests/heft-typescript-composite-test/tsconfig-base.json @@ -16,6 +16,9 @@ "noUnusedLocals": true, "types": ["heft-jest", "webpack-env"], + "isolatedModules": true, + "importsNotUsedAsValues": "error", + "module": "esnext", "moduleResolution": "node", "target": "es5", diff --git a/build-tests/heft-webpack4-everything-test/config/typescript.json b/build-tests/heft-webpack4-everything-test/config/typescript.json index 76f0a6e9340..e672741b44e 100644 --- a/build-tests/heft-webpack4-everything-test/config/typescript.json +++ b/build-tests/heft-webpack4-everything-test/config/typescript.json @@ -55,5 +55,7 @@ // "excludeGlobs": [ // "some/path/*.css" // ] - } + }, + + "useTranspilerWorker": true } diff --git a/build-tests/heft-webpack4-everything-test/package.json b/build-tests/heft-webpack4-everything-test/package.json index 4edb9b07111..bd054a1895f 100644 --- a/build-tests/heft-webpack4-everything-test/package.json +++ b/build-tests/heft-webpack4-everything-test/package.json @@ -5,7 +5,7 @@ "private": true, "scripts": { "build": "heft build --clean", - "start": "heft start", + "start": "heft build-watch --serve", "_phase:build": "heft run --only build -- --clean", "_phase:test": "heft run --only test -- --clean" }, diff --git a/build-tests/heft-webpack4-everything-test/src/indexA.ts b/build-tests/heft-webpack4-everything-test/src/indexA.ts index 6b4db719843..4394a4bf61e 100644 --- a/build-tests/heft-webpack4-everything-test/src/indexA.ts +++ b/build-tests/heft-webpack4-everything-test/src/indexA.ts @@ -8,3 +8,5 @@ import(/* webpackChunkName: 'chunk' */ './chunks/ChunkClass') .catch((e) => { console.log('Error: ' + e.message); }); + +export {}; diff --git a/build-tests/heft-webpack4-everything-test/src/indexB.ts b/build-tests/heft-webpack4-everything-test/src/indexB.ts index 16401835981..e122ca67a73 100644 --- a/build-tests/heft-webpack4-everything-test/src/indexB.ts +++ b/build-tests/heft-webpack4-everything-test/src/indexB.ts @@ -1 +1,3 @@ console.log('dostuff'); + +export {}; diff --git a/build-tests/heft-webpack4-everything-test/tsconfig.json b/build-tests/heft-webpack4-everything-test/tsconfig.json index 8a059e84bb0..eb547d1f50d 100644 --- a/build-tests/heft-webpack4-everything-test/tsconfig.json +++ b/build-tests/heft-webpack4-everything-test/tsconfig.json @@ -16,6 +16,7 @@ "noUnusedLocals": true, "types": ["heft-jest", "webpack-env"], "incremental": true, + "isolatedModules": true, "module": "esnext", "moduleResolution": "node", diff --git a/common/reviews/api/heft-typescript-plugin.api.md b/common/reviews/api/heft-typescript-plugin.api.md index b97ad70b765..e0671977644 100644 --- a/common/reviews/api/heft-typescript-plugin.api.md +++ b/common/reviews/api/heft-typescript-plugin.api.md @@ -59,6 +59,7 @@ export interface ITypeScriptConfigurationJson { // (undocumented) project?: string; staticAssetsToCopy?: IStaticAssetsCopyConfiguration; + useTranspilerWorker?: boolean; } // @beta (undocumented) diff --git a/heft-plugins/heft-typescript-plugin/src/TranspilerWorker.ts b/heft-plugins/heft-typescript-plugin/src/TranspilerWorker.ts new file mode 100644 index 00000000000..19984ec334a --- /dev/null +++ b/heft-plugins/heft-typescript-plugin/src/TranspilerWorker.ts @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. +import { parentPort, workerData } from 'node:worker_threads'; + +import type * as TTypescript from 'typescript'; +import type { + ITranspilationErrorMessage, + ITranspilationRequestMessage, + ITranspilationSuccessMessage, + ITypescriptWorkerData +} from './types'; +import type { ExtendedTypeScript } from './internalTypings/TypeScriptInternals'; +import { configureProgramForMultiEmit } from './configureProgramForMultiEmit'; + +const typedWorkerData: ITypescriptWorkerData = workerData; + +const ts: ExtendedTypeScript = require(typedWorkerData.typeScriptToolPath); + +function handleMessage(message: ITranspilationRequestMessage | false): void { + if (!message) { + process.exit(0); + } + + try { + const response: ITranspilationSuccessMessage = runTranspiler(message); + parentPort!.postMessage(response); + } catch (err) { + const errorResponse: ITranspilationErrorMessage = { + requestId: message.requestId, + type: 'error', + result: { + message: err.message, + ...Object.fromEntries(Object.entries(err)) + } + }; + parentPort!.postMessage(errorResponse); + } +} + +function runTranspiler(message: ITranspilationRequestMessage): ITranspilationSuccessMessage { + const { requestId, compilerOptions, moduleKindsToEmit, fileNames } = message; + + const fullySkipTypeCheck: boolean = + compilerOptions.importsNotUsedAsValues === ts.ImportsNotUsedAsValues.Error; + + for (const [option, value] of Object.entries(ts.getDefaultCompilerOptions())) { + if (compilerOptions[option] === undefined) { + compilerOptions[option] = value; + } + } + + const { target: rawTarget } = compilerOptions; + + for (const option of ts.transpileOptionValueCompilerOptions) { + compilerOptions[option.name] = option.transpileOptionValue; + } + + compilerOptions.suppressOutputPathCheck = true; + compilerOptions.skipDefaultLibCheck = true; + compilerOptions.preserveValueImports = true; + + const sourceFileByPath: Map = new Map(); + + for (const fileName of fileNames) { + const sourceText: string | undefined = ts.sys.readFile(fileName); + if (sourceText) { + const sourceFile: TTypescript.SourceFile = ts.createSourceFile(fileName, sourceText, rawTarget!); + sourceFile.hasNoDefaultLib = fullySkipTypeCheck; + sourceFileByPath.set(fileName, sourceFile); + } + } + + const newLine: string = ts.getNewLineCharacter(compilerOptions); + + const compilerHost: TTypescript.CompilerHost = { + getSourceFile: (fileName: string) => sourceFileByPath.get(fileName), + writeFile: ts.sys.writeFile, + getDefaultLibFileName: () => 'lib.d.ts', + useCaseSensitiveFileNames: () => true, + getCanonicalFileName: (fileName: string) => fileName, + getCurrentDirectory: () => '', + getNewLine: () => newLine, + fileExists: (fileName: string) => sourceFileByPath.has(fileName), + readFile: () => '', + directoryExists: () => true, + getDirectories: () => [] + }; + + const program: TTypescript.Program = ts.createProgram(fileNames, compilerOptions, compilerHost); + + configureProgramForMultiEmit(program, ts, moduleKindsToEmit, 'transpile'); + + const result: TTypescript.EmitResult = program.emit(undefined, undefined, undefined, undefined, undefined); + + const response: ITranspilationSuccessMessage = { + requestId, + type: 'success', + result + }; + + return response; +} + +parentPort!.on('message', handleMessage); diff --git a/heft-plugins/heft-typescript-plugin/src/TypeScriptBuilder.ts b/heft-plugins/heft-typescript-plugin/src/TypeScriptBuilder.ts index 3371d85c55d..a79298c8143 100644 --- a/heft-plugins/heft-typescript-plugin/src/TypeScriptBuilder.ts +++ b/heft-plugins/heft-typescript-plugin/src/TypeScriptBuilder.ts @@ -3,6 +3,8 @@ import * as crypto from 'crypto'; import * as path from 'path'; +import { Worker } from 'worker_threads'; + import * as semver from 'semver'; import type * as TTypescript from 'typescript'; import { @@ -11,8 +13,7 @@ import { type IPackageJson, Path, Async, - FileError, - InternalError + FileError } from '@rushstack/node-core-library'; import type { IScopedLogger } from '@rushstack/heft'; @@ -20,24 +21,13 @@ import type { ExtendedTypeScript, IExtendedSolutionBuilder } from './internalTyp import { TypeScriptCachedFileSystem } from './fileSystem/TypeScriptCachedFileSystem'; import type { ITypeScriptConfigurationJson } from './TypeScriptPlugin'; import type { PerformanceMeasurer, PerformanceMeasurerAsync } from './Performance'; - -interface ICachedEmitModuleKind { - moduleKind: TTypescript.ModuleKind; - - outFolderPath: string; - - /** - * File extension to use instead of '.js' for emitted ECMAScript files. - * For example, '.cjs' to indicate commonjs content, or '.mjs' to indicate ECMAScript modules. - */ - jsExtensionOverride: string | undefined; - - /** - * Set to true if this is the emit kind that is specified in the tsconfig.json. - * Declarations are only emitted for the primary module kind. - */ - isPrimary: boolean; -} +import type { + ICachedEmitModuleKind, + ITranspilationRequestMessage, + ITranspilationResponseMessage, + ITypescriptWorkerData +} from './types'; +import { configureProgramForMultiEmit } from './configureProgramForMultiEmit'; export interface ITypeScriptBuilderConfiguration extends ITypeScriptConfigurationJson { /** @@ -125,16 +115,17 @@ interface IPendingWork { (): void; } +interface ITranspileSignal { + resolve: (result: TTypescript.EmitResult) => void; + reject: (error: Error) => void; +} + const OLDEST_SUPPORTED_TS_MAJOR_VERSION: number = 2; const OLDEST_SUPPORTED_TS_MINOR_VERSION: number = 9; const NEWEST_SUPPORTED_TS_MAJOR_VERSION: number = 5; const NEWEST_SUPPORTED_TS_MINOR_VERSION: number = 0; -// symbols for attaching hidden metadata to ts.Program instances. -const INNER_GET_COMPILER_OPTIONS_SYMBOL: unique symbol = Symbol('getCompilerOptions'); -const INNER_EMIT_SYMBOL: unique symbol = Symbol('emit'); - interface ITypeScriptTool { ts: ExtendedTypeScript; measureSync: PerformanceMeasurer; @@ -151,6 +142,11 @@ interface ITypeScriptTool { executing: boolean; + worker: Worker | undefined; + workerExitPromise: Promise | undefined; + pendingTranspilePromises: Map>; + pendingTranspileSignals: Map; + reportDiagnostic: TTypescript.DiagnosticReporter; clearTimeout: (timeout: IPendingWork) => void; setTimeout: (timeout: (...args: T) => void, ms: number, ...args: T) => IPendingWork; @@ -175,6 +171,8 @@ export class TypeScriptBuilder { private _tool: ITypeScriptTool | undefined = undefined; + private _nextRequestId: number = 0; + private get _tsCacheFilePath(): string { if (!this.__tsCacheFilePath) { // TypeScript internally handles if the tsconfig options have changed from when the tsbuildinfo file was created. @@ -361,7 +359,13 @@ export class TypeScriptBuilder { onChangeDetected(); } return timeout; - } + }, + + worker: undefined, + workerExitPromise: undefined, + + pendingTranspilePromises: new Map(), + pendingTranspileSignals: new Map() }; } @@ -371,147 +375,22 @@ export class TypeScriptBuilder { performance.enable(); if (onChangeDetected !== undefined) { - this._runWatch(this._tool); + await this._runWatchAsync(this._tool); } else if (this._useSolutionBuilder) { - this._runSolutionBuild(this._tool); + await this._runSolutionBuildAsync(this._tool); } else { await this._runBuildAsync(this._tool); } } - private _configureProgramForMultiEmit( - innerProgram: TTypescript.Program, - ts: ExtendedTypeScript - ): { changedFiles: Set } { - interface IProgramWithMultiEmit extends TTypescript.Program { - // Attach the originals to the Program instance to avoid modifying the same Program twice. - // Don't use WeakMap because this Program could theoretically get a { ... } applied to it. - [INNER_GET_COMPILER_OPTIONS_SYMBOL]?: TTypescript.Program['getCompilerOptions']; - [INNER_EMIT_SYMBOL]?: TTypescript.Program['emit']; - } - - const program: IProgramWithMultiEmit = innerProgram; - - // Check to see if this Program has already been modified. - let { [INNER_EMIT_SYMBOL]: innerEmit, [INNER_GET_COMPILER_OPTIONS_SYMBOL]: innerGetCompilerOptions } = - program; - - if (!innerGetCompilerOptions) { - program[INNER_GET_COMPILER_OPTIONS_SYMBOL] = innerGetCompilerOptions = program.getCompilerOptions; - } - - if (!innerEmit) { - program[INNER_EMIT_SYMBOL] = innerEmit = program.emit; - } - - let foundPrimary: boolean = false; - let defaultModuleKind: TTypescript.ModuleKind; - - const multiEmitMap: Map = new Map(); - for (const moduleKindToEmit of this._moduleKindsToEmit) { - const kindCompilerOptions: TTypescript.CompilerOptions = moduleKindToEmit.isPrimary - ? { - ...innerGetCompilerOptions() - } - : { - ...innerGetCompilerOptions(), - module: moduleKindToEmit.moduleKind, - outDir: moduleKindToEmit.outFolderPath, - - // Don't emit declarations for secondary module kinds - declaration: false, - declarationMap: false - }; - if (!kindCompilerOptions.outDir) { - throw new InternalError('Expected compilerOptions.outDir to be assigned'); - } - multiEmitMap.set(moduleKindToEmit, kindCompilerOptions); - - if (moduleKindToEmit.isPrimary) { - if (foundPrimary) { - throw new Error('Multiple primary module emit kinds encountered.'); - } else { - foundPrimary = true; - } - - defaultModuleKind = moduleKindToEmit.moduleKind; - } - } - - const changedFiles: Set = new Set(); - - program.emit = ( - targetSourceFile?: TTypescript.SourceFile, - writeFile?: TTypescript.WriteFileCallback, - cancellationToken?: TTypescript.CancellationToken, - emitOnlyDtsFiles?: boolean, - customTransformers?: TTypescript.CustomTransformers - ) => { - if (emitOnlyDtsFiles) { - return program[INNER_EMIT_SYMBOL]!( - targetSourceFile, - writeFile, - cancellationToken, - emitOnlyDtsFiles, - customTransformers - ); - } - - if (targetSourceFile && changedFiles) { - changedFiles.add(targetSourceFile); - } - - const originalCompilerOptions: TTypescript.CompilerOptions = - program[INNER_GET_COMPILER_OPTIONS_SYMBOL]!(); - - let defaultModuleKindResult: TTypescript.EmitResult; - const diagnostics: TTypescript.Diagnostic[] = []; - let emitSkipped: boolean = false; - try { - for (const [moduleKindToEmit, kindCompilerOptions] of multiEmitMap) { - program.getCompilerOptions = () => kindCompilerOptions; - // Need to mutate the compiler options for the `module` field specifically, because emitWorker() captures - // options in the closure and passes it to `ts.getTransformers()` - originalCompilerOptions.module = moduleKindToEmit.moduleKind; - const flavorResult: TTypescript.EmitResult = program[INNER_EMIT_SYMBOL]!( - targetSourceFile, - writeFile && wrapWriteFile(writeFile, moduleKindToEmit.jsExtensionOverride), - cancellationToken, - emitOnlyDtsFiles, - customTransformers - ); - - emitSkipped = emitSkipped || flavorResult.emitSkipped; - // Need to aggregate diagnostics because some are impacted by the target module type - for (const diagnostic of flavorResult.diagnostics) { - diagnostics.push(diagnostic); - } - - if (moduleKindToEmit.moduleKind === defaultModuleKind) { - defaultModuleKindResult = flavorResult; - } - } - - const mergedDiagnostics: readonly TTypescript.Diagnostic[] = - ts.sortAndDeduplicateDiagnostics(diagnostics); - - return { - ...defaultModuleKindResult!, - changedSourceFiles: changedFiles, - diagnostics: mergedDiagnostics, - emitSkipped - }; - } finally { - // Restore the original compiler options and module kind for future calls - program.getCompilerOptions = program[INNER_GET_COMPILER_OPTIONS_SYMBOL]!; - originalCompilerOptions.module = defaultModuleKind; - } - }; - return { changedFiles }; - } - - public _runWatch(tool: ITypeScriptTool): void { - const { ts, measureSync: measureTsPerformance, pendingOperations, rawDiagnostics } = tool; + public async _runWatchAsync(tool: ITypeScriptTool): Promise { + const { + ts, + measureSync: measureTsPerformance, + pendingOperations, + rawDiagnostics, + pendingTranspilePromises + } = tool; if (!tool.solutionBuilder && !tool.watchProgram) { //#region CONFIGURE @@ -547,13 +426,27 @@ export class TypeScriptBuilder { pendingOperations.delete(operation); operation(); } + if (pendingTranspilePromises.size) { + const emitResults: TTypescript.EmitResult[] = await Promise.all(pendingTranspilePromises.values()); + for (const { diagnostics } of emitResults) { + for (const diagnostic of diagnostics) { + rawDiagnostics.push(diagnostic); + } + } + } + // eslint-disable-next-line require-atomic-updates tool.executing = false; } this._logDiagnostics(ts, rawDiagnostics); } public async _runBuildAsync(tool: ITypeScriptTool): Promise { - const { ts, measureSync: measureTsPerformance, measureAsync: measureTsPerformanceAsync } = tool; + const { + ts, + measureSync: measureTsPerformance, + measureAsync: measureTsPerformanceAsync, + pendingTranspilePromises + } = tool; //#region CONFIGURE const { @@ -579,17 +472,26 @@ export class TypeScriptBuilder { let builderProgram: TTypescript.BuilderProgram | undefined = undefined; let innerProgram: TTypescript.Program; + const isolatedModules: boolean = + !!this._configuration.useTranspilerWorker && !!tsconfig.options.isolatedModules; + const mode: 'both' | 'declaration' = isolatedModules ? 'declaration' : 'both'; + + let fileNames: string[] | undefined; + if (tsconfig.options.incremental) { // Use ts.createEmitAndSemanticDiagnositcsBuilderProgram directly because the customizations performed by // _getCreateBuilderProgram duplicate those performed in this function for non-incremental build. + const oldProgram: TTypescript.EmitAndSemanticDiagnosticsBuilderProgram | undefined = + ts.readBuilderProgram(tsconfig.options, compilerHost); builderProgram = ts.createEmitAndSemanticDiagnosticsBuilderProgram( tsconfig.fileNames, tsconfig.options, compilerHost, - ts.readBuilderProgram(tsconfig.options, compilerHost), + oldProgram, ts.getConfigFileParsingDiagnostics(tsconfig), tsconfig.projectReferences ); + fileNames = getFilesToTranspileFromBuilderProgram(builderProgram); innerProgram = builderProgram.getProgram(); } else { innerProgram = ts.createProgram({ @@ -600,6 +502,7 @@ export class TypeScriptBuilder { oldProgram: undefined, configFileParsingDiagnostics: ts.getConfigFileParsingDiagnostics(tsconfig) }); + fileNames = getFilesToTranspileFromProgram(innerProgram); } // Prefer the builder program, since it is what gives us incremental builds @@ -608,6 +511,11 @@ export class TypeScriptBuilder { this._logReadPerformance(ts); //#endregion + if (isolatedModules) { + // Kick the transpilation worker. + this._queueTranspileInWorker(tool, genericProgram.getCompilerOptions(), fileNames); + } + //#region ANALYSIS const { duration: diagnosticsDurationMs, diagnostics: preDiagnostics } = measureTsPerformance( 'Analyze', @@ -632,7 +540,7 @@ export class TypeScriptBuilder { filesToWrite.push({ filePath, data }); }; - const { changedFiles } = this._configureProgramForMultiEmit(innerProgram, ts); + const { changedFiles } = configureProgramForMultiEmit(innerProgram, ts, this._moduleKindsToEmit, mode); const emitResult: TTypescript.EmitResult = genericProgram.emit(undefined, writeFileCallback); //#endregion @@ -657,6 +565,17 @@ export class TypeScriptBuilder { ); //#endregion + this._configuration.emitChangedFilesCallback(innerProgram, changedFiles); + + if (pendingTranspilePromises.size) { + const emitResults: TTypescript.EmitResult[] = await Promise.all(pendingTranspilePromises.values()); + for (const { diagnostics } of emitResults) { + for (const diagnostic of diagnostics) { + rawDiagnostics.push(diagnostic); + } + } + } + const { duration: writeDuration } = await writePromise; this._typescriptTerminal.writeVerboseLine(`I/O Write: ${writeDuration}ms (${filesToWrite.length} files)`); @@ -664,13 +583,14 @@ export class TypeScriptBuilder { // Reset performance counters in case any are used in the callback ts.performance.disable(); ts.performance.enable(); - this._configuration.emitChangedFilesCallback(innerProgram, changedFiles); + + await this._cleanupWorkerAsync(); } - public _runSolutionBuild(tool: ITypeScriptTool): void { + public async _runSolutionBuildAsync(tool: ITypeScriptTool): Promise { this._typescriptTerminal.writeVerboseLine(`Using solution mode`); - const { ts, measureSync, rawDiagnostics } = tool; + const { ts, measureSync, rawDiagnostics, pendingTranspilePromises } = tool; rawDiagnostics.length = 0; if (!tool.solutionBuilder) { @@ -705,7 +625,18 @@ export class TypeScriptBuilder { tool.solutionBuilder.build(); //#endregion + if (pendingTranspilePromises.size) { + const emitResults: TTypescript.EmitResult[] = await Promise.all(pendingTranspilePromises.values()); + for (const { diagnostics } of emitResults) { + for (const diagnostic of diagnostics) { + rawDiagnostics.push(diagnostic); + } + } + } + this._logDiagnostics(ts, rawDiagnostics); + + await this._cleanupWorkerAsync(); } private _logDiagnostics(ts: ExtendedTypeScript, rawDiagnostics: TTypescript.Diagnostic[]): void { @@ -1084,6 +1015,16 @@ export class TypeScriptBuilder { this._logReadPerformance(ts); + const isolatedModules: boolean = + !!this._configuration.useTranspilerWorker && !!compilerOptions!.isolatedModules; + const mode: 'both' | 'declaration' = isolatedModules ? 'declaration' : 'both'; + + if (isolatedModules) { + // Kick the transpilation worker. + const fileNamesToTranspile: string[] = getFilesToTranspileFromBuilderProgram(newProgram); + this._queueTranspileInWorker(this._tool!, compilerOptions!, fileNamesToTranspile); + } + const { emit: originalEmit } = newProgram; const emit: TTypescript.Program['emit'] = ( @@ -1097,7 +1038,12 @@ export class TypeScriptBuilder { const innerCompilerOptions: TTypescript.CompilerOptions = innerProgram.getCompilerOptions(); - const { changedFiles } = this._configureProgramForMultiEmit(innerProgram, ts); + const { changedFiles } = configureProgramForMultiEmit( + innerProgram, + ts, + this._moduleKindsToEmit, + mode + ); const result: TTypescript.EmitResult = originalEmit.call( newProgram, @@ -1294,32 +1240,122 @@ export class TypeScriptBuilder { throw new Error(`"${moduleKindName}" is not a valid module kind name.`); } } -} -const JS_EXTENSION_REGEX: RegExp = /\.js(\.map)?$/; + private _queueTranspileInWorker( + tool: ITypeScriptTool, + compilerOptions: TTypescript.CompilerOptions, + fileNames: string[] + ): void { + const { pendingTranspilePromises, pendingTranspileSignals } = tool; + let maybeWorker: Worker | undefined = tool.worker; + if (!maybeWorker) { + const workerData: ITypescriptWorkerData = { + typeScriptToolPath: this._configuration.typeScriptToolPath + }; + tool.worker = maybeWorker = new Worker(require.resolve('./TranspilerWorker.js'), { + workerData: workerData + }); -function wrapWriteFile( - baseWriteFile: TTypescript.WriteFileCallback, - jsExtensionOverride: string | undefined -): TTypescript.WriteFileCallback { - if (!jsExtensionOverride) { - return baseWriteFile; - } + maybeWorker.on('message', (response: ITranspilationResponseMessage) => { + const { requestId: resolvingRequestId, type, result } = response; + const signal: ITranspileSignal | undefined = pendingTranspileSignals.get(resolvingRequestId); + + if (type === 'error') { + const error: Error = Object.assign(new Error(result.message), result); + if (signal) { + signal.reject(error); + } else { + this._typescriptTerminal.writeErrorLine( + `Unexpected worker rejection for request with id ${resolvingRequestId}: ${error}` + ); + } + } else if (signal) { + signal.resolve(result); + } else { + this._typescriptTerminal.writeErrorLine( + `Unexpected worker resolution for request with id ${resolvingRequestId}` + ); + } + + pendingTranspileSignals.delete(resolvingRequestId); + pendingTranspilePromises.delete(resolvingRequestId); + }); + + tool.workerExitPromise = new Promise( + (resolve: (exitCode: number) => void, reject: (err: Error) => void) => { + maybeWorker!.once('exit', resolve); + + maybeWorker!.once('error', (err: Error) => { + for (const { reject: rejectTranspile } of pendingTranspileSignals.values()) { + rejectTranspile(err); + } + pendingTranspileSignals.clear(); + reject(err); + }); + } + ); + } + + // make linter happy + const worker: Worker = maybeWorker; + + const requestId: number = ++this._nextRequestId; + const transpilePromise: Promise = new Promise( + (resolve: (result: TTypescript.EmitResult) => void, reject: (err: Error) => void) => { + pendingTranspileSignals.set(requestId, { resolve, reject }); + + this._typescriptTerminal.writeLine(`Asynchronously transpiling ${compilerOptions.configFilePath}`); + const request: ITranspilationRequestMessage = { + compilerOptions, + fileNames, + moduleKindsToEmit: this._moduleKindsToEmit, + requestId + }; - const replacementExtension: string = `${jsExtensionOverride}$1`; - return ( - fileName: string, - data: string, - writeBOM: boolean, - onError?: ((message: string) => void) | undefined, - sourceFiles?: readonly TTypescript.SourceFile[] | undefined - ) => { - return baseWriteFile( - fileName.replace(JS_EXTENSION_REGEX, replacementExtension), - data, - writeBOM, - onError, - sourceFiles + worker.postMessage(request); + } ); - }; + + pendingTranspilePromises.set(requestId, transpilePromise); + } + + private async _cleanupWorkerAsync(): Promise { + const tool: ITypeScriptTool | undefined = this._tool; + if (!tool) { + return; + } + + const { worker, workerExitPromise } = tool; + if (worker && workerExitPromise) { + worker.postMessage(false); + this._typescriptTerminal.writeLine(`Waiting for worker to exit`); + await workerExitPromise; + tool.worker = undefined; + tool.workerExitPromise = undefined; + } + } +} + +function getFilesToTranspileFromBuilderProgram(builderProgram: TTypescript.BuilderProgram): string[] { + const changedFilesSet: Set = ( + builderProgram as unknown as { getState(): { changedFilesSet: Set } } + ).getState().changedFilesSet; + const fileNames: string[] = []; + for (const fileName of changedFilesSet) { + const sourceFile: TTypescript.SourceFile | undefined = builderProgram.getSourceFile(fileName); + if (sourceFile && !sourceFile.isDeclarationFile) { + fileNames.push(sourceFile.fileName); + } + } + return fileNames; +} + +function getFilesToTranspileFromProgram(program: TTypescript.Program): string[] { + const fileNames: string[] = []; + for (const sourceFile of program.getSourceFiles()) { + if (!sourceFile.isDeclarationFile) { + fileNames.push(sourceFile.fileName); + } + } + return fileNames; } diff --git a/heft-plugins/heft-typescript-plugin/src/TypeScriptPlugin.ts b/heft-plugins/heft-typescript-plugin/src/TypeScriptPlugin.ts index f15a530123e..db0104f7f61 100644 --- a/heft-plugins/heft-typescript-plugin/src/TypeScriptPlugin.ts +++ b/heft-plugins/heft-typescript-plugin/src/TypeScriptPlugin.ts @@ -70,6 +70,11 @@ export interface ITypeScriptConfigurationJson { */ buildProjectReferences?: boolean; + /** + * If true, and the tsconfig has \"isolatedModules\": true, then transpilation will happen in parallel in a worker thread. + */ + useTranspilerWorker?: boolean; + /* * Specifies the tsconfig.json file that will be used for compilation. Equivalent to the "project" argument for the 'tsc' and 'tslint' command line tools. * @@ -363,6 +368,8 @@ export default class TypeScriptPlugin implements IHeftTaskPlugin { buildProjectReferences: typeScriptConfigurationJson?.buildProjectReferences, + useTranspilerWorker: typeScriptConfigurationJson?.useTranspilerWorker, + tsconfigPath: getTsconfigFilePath(heftConfiguration, typeScriptConfigurationJson), additionalModuleKindsToEmit: typeScriptConfigurationJson?.additionalModuleKindsToEmit, emitCjsExtensionForCommonJS: !!typeScriptConfigurationJson?.emitCjsExtensionForCommonJS, diff --git a/heft-plugins/heft-typescript-plugin/src/configureProgramForMultiEmit.ts b/heft-plugins/heft-typescript-plugin/src/configureProgramForMultiEmit.ts new file mode 100644 index 00000000000..6cfbd7cfe6b --- /dev/null +++ b/heft-plugins/heft-typescript-plugin/src/configureProgramForMultiEmit.ts @@ -0,0 +1,183 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. +import type * as TTypescript from 'typescript'; +import { InternalError } from '@rushstack/node-core-library'; + +import type { ExtendedTypeScript } from './internalTypings/TypeScriptInternals'; +import type { ICachedEmitModuleKind } from './types'; + +// symbols for attaching hidden metadata to ts.Program instances. +const INNER_GET_COMPILER_OPTIONS_SYMBOL: unique symbol = Symbol('getCompilerOptions'); +const INNER_EMIT_SYMBOL: unique symbol = Symbol('emit'); + +const JS_EXTENSION_REGEX: RegExp = /\.js(\.map)?$/; + +function wrapWriteFile( + this: void, + baseWriteFile: TTypescript.WriteFileCallback, + jsExtensionOverride: string | undefined +): TTypescript.WriteFileCallback { + if (!jsExtensionOverride) { + return baseWriteFile; + } + + const replacementExtension: string = `${jsExtensionOverride}$1`; + return ( + fileName: string, + data: string, + writeBOM: boolean, + onError?: ((message: string) => void) | undefined, + sourceFiles?: readonly TTypescript.SourceFile[] | undefined + ) => { + return baseWriteFile( + fileName.replace(JS_EXTENSION_REGEX, replacementExtension), + data, + writeBOM, + onError, + sourceFiles + ); + }; +} + +export function configureProgramForMultiEmit( + this: void, + innerProgram: TTypescript.Program, + ts: ExtendedTypeScript, + moduleKindsToEmit: ICachedEmitModuleKind[], + mode: 'transpile' | 'declaration' | 'both' +): { changedFiles: Set } { + interface IProgramWithMultiEmit extends TTypescript.Program { + // Attach the originals to the Program instance to avoid modifying the same Program twice. + // Don't use WeakMap because this Program could theoretically get a { ... } applied to it. + [INNER_GET_COMPILER_OPTIONS_SYMBOL]?: TTypescript.Program['getCompilerOptions']; + [INNER_EMIT_SYMBOL]?: TTypescript.Program['emit']; + } + + const program: IProgramWithMultiEmit = innerProgram; + + // Check to see if this Program has already been modified. + let { [INNER_EMIT_SYMBOL]: innerEmit, [INNER_GET_COMPILER_OPTIONS_SYMBOL]: innerGetCompilerOptions } = + program; + + if (!innerGetCompilerOptions) { + program[INNER_GET_COMPILER_OPTIONS_SYMBOL] = innerGetCompilerOptions = program.getCompilerOptions; + } + + if (!innerEmit) { + program[INNER_EMIT_SYMBOL] = innerEmit = program.emit; + } + + let foundPrimary: boolean = false; + let defaultModuleKind: TTypescript.ModuleKind; + + const multiEmitMap: Map = new Map(); + for (const moduleKindToEmit of moduleKindsToEmit) { + const kindCompilerOptions: TTypescript.CompilerOptions = moduleKindToEmit.isPrimary + ? { + ...innerGetCompilerOptions() + } + : { + ...innerGetCompilerOptions(), + module: moduleKindToEmit.moduleKind, + outDir: moduleKindToEmit.outFolderPath, + + // Don't emit declarations for secondary module kinds + declaration: false, + declarationMap: false + }; + if (!kindCompilerOptions.outDir) { + throw new InternalError('Expected compilerOptions.outDir to be assigned'); + } + if (mode === 'transpile') { + kindCompilerOptions.declaration = false; + kindCompilerOptions.declarationMap = false; + } else if (mode === 'declaration') { + kindCompilerOptions.emitDeclarationOnly = true; + } + + if (moduleKindToEmit.isPrimary || mode !== 'declaration') { + multiEmitMap.set(moduleKindToEmit, kindCompilerOptions); + } + + if (moduleKindToEmit.isPrimary) { + if (foundPrimary) { + throw new Error('Multiple primary module emit kinds encountered.'); + } else { + foundPrimary = true; + } + + defaultModuleKind = moduleKindToEmit.moduleKind; + } + } + + const changedFiles: Set = new Set(); + + program.emit = ( + targetSourceFile?: TTypescript.SourceFile, + writeFile?: TTypescript.WriteFileCallback, + cancellationToken?: TTypescript.CancellationToken, + emitOnlyDtsFiles?: boolean, + customTransformers?: TTypescript.CustomTransformers + ) => { + if (emitOnlyDtsFiles) { + return program[INNER_EMIT_SYMBOL]!( + targetSourceFile, + writeFile, + cancellationToken, + emitOnlyDtsFiles, + customTransformers + ); + } + + if (targetSourceFile && changedFiles) { + changedFiles.add(targetSourceFile); + } + + const originalCompilerOptions: TTypescript.CompilerOptions = + program[INNER_GET_COMPILER_OPTIONS_SYMBOL]!(); + + let defaultModuleKindResult: TTypescript.EmitResult; + const diagnostics: TTypescript.Diagnostic[] = []; + let emitSkipped: boolean = false; + try { + for (const [moduleKindToEmit, kindCompilerOptions] of multiEmitMap) { + program.getCompilerOptions = () => kindCompilerOptions; + // Need to mutate the compiler options for the `module` field specifically, because emitWorker() captures + // options in the closure and passes it to `ts.getTransformers()` + originalCompilerOptions.module = moduleKindToEmit.moduleKind; + const flavorResult: TTypescript.EmitResult = program[INNER_EMIT_SYMBOL]!( + targetSourceFile, + writeFile && wrapWriteFile(writeFile, moduleKindToEmit.jsExtensionOverride), + cancellationToken, + emitOnlyDtsFiles, + customTransformers + ); + + emitSkipped = emitSkipped || flavorResult.emitSkipped; + // Need to aggregate diagnostics because some are impacted by the target module type + for (const diagnostic of flavorResult.diagnostics) { + diagnostics.push(diagnostic); + } + + if (moduleKindToEmit.moduleKind === defaultModuleKind) { + defaultModuleKindResult = flavorResult; + } + } + + const mergedDiagnostics: readonly TTypescript.Diagnostic[] = + ts.sortAndDeduplicateDiagnostics(diagnostics); + + return { + ...defaultModuleKindResult!, + changedSourceFiles: changedFiles, + diagnostics: mergedDiagnostics, + emitSkipped + }; + } finally { + // Restore the original compiler options and module kind for future calls + program.getCompilerOptions = program[INNER_GET_COMPILER_OPTIONS_SYMBOL]!; + originalCompilerOptions.module = defaultModuleKind; + } + }; + return { changedFiles }; +} diff --git a/heft-plugins/heft-typescript-plugin/src/internalTypings/TypeScriptInternals.ts b/heft-plugins/heft-typescript-plugin/src/internalTypings/TypeScriptInternals.ts index a4d78dbfbdf..5701e0f9ddb 100644 --- a/heft-plugins/heft-typescript-plugin/src/internalTypings/TypeScriptInternals.ts +++ b/heft-plugins/heft-typescript-plugin/src/internalTypings/TypeScriptInternals.ts @@ -45,6 +45,14 @@ export interface IExtendedTypeScript { getCount(measureName: string): number; }; + transpileOptionValueCompilerOptions: { + name: keyof TTypescript.CompilerOptions; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + transpileOptionValue: any; + }[]; + + getNewLineCharacter(compilerOptions: TTypescript.CompilerOptions): string; + /** * https://github.com/microsoft/TypeScript/blob/782c09d783e006a697b4ba6d1e7ec2f718ce8393/src/compiler/utilities.ts#L6540 */ diff --git a/heft-plugins/heft-typescript-plugin/src/schemas/typescript.schema.json b/heft-plugins/heft-typescript-plugin/src/schemas/typescript.schema.json index af0f3ba1097..2406afaa8e3 100644 --- a/heft-plugins/heft-typescript-plugin/src/schemas/typescript.schema.json +++ b/heft-plugins/heft-typescript-plugin/src/schemas/typescript.schema.json @@ -52,6 +52,11 @@ "type": "boolean" }, + "useTranspilerWorker": { + "description": "If true, and the tsconfig has \"isolatedModules\": true, then transpilation will happen in parallel in a worker thread.", + "type": "boolean" + }, + "project": { "description": "Specifies the tsconfig.json file that will be used for compilation. Equivalent to the \"project\" argument for the 'tsc' and 'tslint' command line tools. The default value is \"./tsconfig.json\".", "type": "string" diff --git a/heft-plugins/heft-typescript-plugin/src/types.ts b/heft-plugins/heft-typescript-plugin/src/types.ts new file mode 100644 index 00000000000..aea0ee72955 --- /dev/null +++ b/heft-plugins/heft-typescript-plugin/src/types.ts @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. +import type * as TTypescript from 'typescript'; + +export interface ITypescriptWorkerData { + /** + * Path to the version of TypeScript to use. + */ + typeScriptToolPath: string; +} + +export interface ITranspilationRequestMessage { + /** + * Unique identifier for this request. + */ + requestId: number; + /** + * The tsconfig compiler options to use for the request. + */ + compilerOptions: TTypescript.CompilerOptions; + /** + * The variants to emit. + */ + moduleKindsToEmit: ICachedEmitModuleKind[]; + /** + * The set of files to build. + */ + fileNames: string[]; +} + +export interface ITranspilationSuccessMessage { + requestId: number; + type: 'success'; + result: TTypescript.EmitResult; +} + +export interface ITranspilationErrorMessage { + requestId: number; + type: 'error'; + result: { + message: string; + [key: string]: unknown; + }; +} + +export type ITranspilationResponseMessage = ITranspilationSuccessMessage | ITranspilationErrorMessage; + +export interface ICachedEmitModuleKind { + moduleKind: TTypescript.ModuleKind; + + outFolderPath: string; + + /** + * File extension to use instead of '.js' for emitted ECMAScript files. + * For example, '.cjs' to indicate commonjs content, or '.mjs' to indicate ECMAScript modules. + */ + jsExtensionOverride: string | undefined; + + /** + * Set to true if this is the emit kind that is specified in the tsconfig.json. + * Declarations are only emitted for the primary module kind. + */ + isPrimary: boolean; +} diff --git a/heft-plugins/heft-typescript-plugin/tsconfig.json b/heft-plugins/heft-typescript-plugin/tsconfig.json index 7512871fdbf..14f45ad644e 100644 --- a/heft-plugins/heft-typescript-plugin/tsconfig.json +++ b/heft-plugins/heft-typescript-plugin/tsconfig.json @@ -2,6 +2,7 @@ "extends": "./node_modules/@rushstack/heft-node-rig/profiles/default/tsconfig-base.json", "compilerOptions": { - "types": ["node"] + "types": ["node"], + "lib": ["ES2019"] } }