From 87cfece2a67ad3520e55654f1f403144aaee1c4e Mon Sep 17 00:00:00 2001 From: Elliot Nelson Date: Wed, 15 Feb 2023 08:38:27 -0500 Subject: [PATCH] Initial implementation of rush-http-build-cache-plugin --- apps/rush/package.json | 1 + apps/rush/src/start-dev.ts | 1 + .../enelson-http-cache_2023-02-15-13-36.json | 10 + .../rush/browser-approved-packages.json | 4 + common/config/rush/pnpm-lock.yaml | 29 ++ libraries/rush-lib/package.json | 3 +- .../src/pluginFramework/PluginManager.ts | 1 + .../rush-http-build-cache-plugin/README.md | 94 +++++ .../config/jest.config.json | 14 + .../config/rig.json | 18 + .../rush-http-build-cache-plugin/package.json | 36 ++ .../rush-plugin-manifest.json | 11 + .../src/HttpBuildCacheProvider.test.ts | 55 +++ .../src/HttpBuildCacheProvider.ts | 384 ++++++++++++++++++ .../src/RushHttpBuildCachePlugin.ts | 75 ++++ .../src/exec.test.ts | 13 + .../rush-http-build-cache-plugin/src/exec.ts | 35 ++ .../rush-http-build-cache-plugin/src/index.ts | 4 + .../src/schemas/plugin-config.schema.json | 39 ++ .../tsconfig.json | 6 + rush.json | 6 + 21 files changed, 838 insertions(+), 1 deletion(-) create mode 100644 common/changes/@microsoft/rush/enelson-http-cache_2023-02-15-13-36.json create mode 100644 rush-plugins/rush-http-build-cache-plugin/README.md create mode 100644 rush-plugins/rush-http-build-cache-plugin/config/jest.config.json create mode 100644 rush-plugins/rush-http-build-cache-plugin/config/rig.json create mode 100644 rush-plugins/rush-http-build-cache-plugin/package.json create mode 100644 rush-plugins/rush-http-build-cache-plugin/rush-plugin-manifest.json create mode 100644 rush-plugins/rush-http-build-cache-plugin/src/HttpBuildCacheProvider.test.ts create mode 100644 rush-plugins/rush-http-build-cache-plugin/src/HttpBuildCacheProvider.ts create mode 100644 rush-plugins/rush-http-build-cache-plugin/src/RushHttpBuildCachePlugin.ts create mode 100644 rush-plugins/rush-http-build-cache-plugin/src/exec.test.ts create mode 100644 rush-plugins/rush-http-build-cache-plugin/src/exec.ts create mode 100644 rush-plugins/rush-http-build-cache-plugin/src/index.ts create mode 100644 rush-plugins/rush-http-build-cache-plugin/src/schemas/plugin-config.schema.json create mode 100644 rush-plugins/rush-http-build-cache-plugin/tsconfig.json diff --git a/apps/rush/package.json b/apps/rush/package.json index 23a6012fff3..51d9bae0c82 100644 --- a/apps/rush/package.json +++ b/apps/rush/package.json @@ -47,6 +47,7 @@ "@rushstack/heft-node-rig": "workspace:*", "@rushstack/rush-amazon-s3-build-cache-plugin": "workspace:*", "@rushstack/rush-azure-storage-build-cache-plugin": "workspace:*", + "@rushstack/rush-http-build-cache-plugin": "workspace:*", "@types/heft-jest": "1.0.1", "@types/node": "14.18.36", "@types/semver": "7.3.5" diff --git a/apps/rush/src/start-dev.ts b/apps/rush/src/start-dev.ts index 064b1f4f384..46e79f2c0b5 100644 --- a/apps/rush/src/start-dev.ts +++ b/apps/rush/src/start-dev.ts @@ -27,6 +27,7 @@ function includePlugin(pluginName: string, pluginPackageName?: string): void { includePlugin('rush-amazon-s3-build-cache-plugin'); includePlugin('rush-azure-storage-build-cache-plugin'); +includePlugin('rush-http-build-cache-plugin'); // Including this here so that developers can reuse it without installing the plugin a second time includePlugin('rush-azure-interactive-auth-plugin', '@rushstack/rush-azure-storage-build-cache-plugin'); diff --git a/common/changes/@microsoft/rush/enelson-http-cache_2023-02-15-13-36.json b/common/changes/@microsoft/rush/enelson-http-cache_2023-02-15-13-36.json new file mode 100644 index 00000000000..94560cc615a --- /dev/null +++ b/common/changes/@microsoft/rush/enelson-http-cache_2023-02-15-13-36.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "Add built-in plugin rush-http-build-cache-plugin", + "type": "none" + } + ], + "packageName": "@microsoft/rush" +} \ No newline at end of file diff --git a/common/config/rush/browser-approved-packages.json b/common/config/rush/browser-approved-packages.json index 8a8d72ffe65..1ef3100cce9 100644 --- a/common/config/rush/browser-approved-packages.json +++ b/common/config/rush/browser-approved-packages.json @@ -34,6 +34,10 @@ "name": "@reduxjs/toolkit", "allowedCategories": [ "libraries" ] }, + { + "name": "@rushstack/rush-http-build-cache-plugin", + "allowedCategories": [ "libraries" ] + }, { "name": "@rushstack/rush-themed-ui", "allowedCategories": [ "libraries" ] diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 2a1a2e1f46e..38a8e3b685c 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -253,6 +253,7 @@ importers: '@rushstack/node-core-library': workspace:* '@rushstack/rush-amazon-s3-build-cache-plugin': workspace:* '@rushstack/rush-azure-storage-build-cache-plugin': workspace:* + '@rushstack/rush-http-build-cache-plugin': workspace:* '@types/heft-jest': 1.0.1 '@types/node': 14.18.36 '@types/semver': 7.3.5 @@ -269,6 +270,7 @@ importers: '@rushstack/heft-node-rig': link:../../rigs/heft-node-rig '@rushstack/rush-amazon-s3-build-cache-plugin': link:../../rush-plugins/rush-amazon-s3-build-cache-plugin '@rushstack/rush-azure-storage-build-cache-plugin': link:../../rush-plugins/rush-azure-storage-build-cache-plugin + '@rushstack/rush-http-build-cache-plugin': link:../../rush-plugins/rush-http-build-cache-plugin '@types/heft-jest': 1.0.1 '@types/node': 14.18.36 '@types/semver': 7.3.5 @@ -2397,6 +2399,33 @@ importers: '@types/heft-jest': 1.0.1 '@types/node': 14.18.36 + ../../rush-plugins/rush-http-build-cache-plugin: + specifiers: + '@microsoft/rush-lib': workspace:* + '@rushstack/eslint-config': workspace:* + '@rushstack/heft': workspace:* + '@rushstack/heft-node-rig': workspace:* + '@rushstack/node-core-library': workspace:* + '@rushstack/rush-sdk': workspace:* + '@types/heft-jest': 1.0.1 + '@types/node': 14.18.36 + '@types/node-fetch': 2.6.2 + https-proxy-agent: ~5.0.0 + node-fetch: 2.6.7 + dependencies: + '@rushstack/node-core-library': link:../../libraries/node-core-library + '@rushstack/rush-sdk': link:../../libraries/rush-sdk + https-proxy-agent: 5.0.1 + node-fetch: 2.6.7 + devDependencies: + '@microsoft/rush-lib': link:../../libraries/rush-lib + '@rushstack/eslint-config': link:../../eslint/eslint-config + '@rushstack/heft': link:../../apps/heft + '@rushstack/heft-node-rig': link:../../rigs/heft-node-rig + '@types/heft-jest': 1.0.1 + '@types/node': 14.18.36 + '@types/node-fetch': 2.6.2 + ../../rush-plugins/rush-litewatch-plugin: specifiers: '@rushstack/eslint-config': workspace:* diff --git a/libraries/rush-lib/package.json b/libraries/rush-lib/package.json index 2d9daf24009..db57be7ae11 100644 --- a/libraries/rush-lib/package.json +++ b/libraries/rush-lib/package.json @@ -88,6 +88,7 @@ }, "publishOnlyDependencies": { "@rushstack/rush-amazon-s3-build-cache-plugin": "workspace:*", - "@rushstack/rush-azure-storage-build-cache-plugin": "workspace:*" + "@rushstack/rush-azure-storage-build-cache-plugin": "workspace:*", + "@rushstack/rush-http-build-cache-plugin": "workspace:*" } } diff --git a/libraries/rush-lib/src/pluginFramework/PluginManager.ts b/libraries/rush-lib/src/pluginFramework/PluginManager.ts index ee5b0a37549..01c3cc9f304 100644 --- a/libraries/rush-lib/src/pluginFramework/PluginManager.ts +++ b/libraries/rush-lib/src/pluginFramework/PluginManager.ts @@ -75,6 +75,7 @@ export class PluginManager { tryAddBuiltInPlugin('rush-amazon-s3-build-cache-plugin'); tryAddBuiltInPlugin('rush-azure-storage-build-cache-plugin'); + tryAddBuiltInPlugin('rush-http-build-cache-plugin'); // This is a secondary plugin inside the `@rushstack/rush-azure-storage-build-cache-plugin` // package. Because that package comes with Rush (for now), it needs to get registered here. // If the necessary config file doesn't exist, this plugin doesn't do anything. diff --git a/rush-plugins/rush-http-build-cache-plugin/README.md b/rush-plugins/rush-http-build-cache-plugin/README.md new file mode 100644 index 00000000000..abae1b4b19f --- /dev/null +++ b/rush-plugins/rush-http-build-cache-plugin/README.md @@ -0,0 +1,94 @@ +# @rushstack/rush-http-build-cache-plugin + +A Rush plugin that uses HTTP/HTTPS to manage cache objects. + +Authentication is provided via standard `Authorization` HTTP headers, with the value (a Bearer or Basic token) configured using a custom node script. Your "tokenHandler" node script is automatically called when the user updates their cloud credentials. + +## Configuration + +To use the HTTP build cache plugin, enable it in `common/config/rush/build-cache.json`: + +```json +{ + "buildCacheEnabled": true, + + "cacheProvider": "http" +} +``` + +Once enabled, configure the HTTP build cache in config file `common/config/rush-plugins/rush-http-build-cache-plugin.json`: + +```json +{ + "url": "https://build-cache.example.com", + "tokenHandler": "node common/scripts/custom-script-that-returns-an-authentication-header.js", + "isCacheWriteAllowed": false +} +``` + +- url: The server to store cache objects. +- tokenHandler: A script that can print the Authorization header expected by the server. The value printed to `stdout` by this command should be an exact header, for example, `Bearer ab98d8c878d937290d979a9097c90dfffff` or `Basic 098abc7dff==`. +- isCacheWriteAllowed: A flag that determines if the plugin should write to the cache. + +## HTTP Cache Server Requirements + +The HTTP build cache plugin can use almost any HTTP/HTTPS backend for remote caching, as long as it honors the following rules: + + - Uses `Authorization: Bearer xxx` or `Authorization: Basic xxx` headers for authentication. + - Accepts GET requests for cache reads. + - Accepts PUT requests for cache writes (with a raw request body -- no `form/multipart`). + - Cache hits return HTTP 200 with the file in the response body. + - Successful cache writes return HTTP 2xx (200-299). + - Cache misses return HTTP 404 or HTTP 403. + - Invalid or missing authentication returns HTTP 401. + +## Examples + +### Gradle Build Cache Server + +The Gradle Build Cache Server (typically used to support Gradle Remote Build Cache) meets all of the requirements above, so if you don't have another server in mind, you can use it as your remote backend. + +First, start up and configure your build cache node locally: + + - Download latest JAR file from https://docs.gradle.com/build-cache-node/ + - Start the service: `java -jar build-cache-node-14.0.jar start` + - Copy the startup banner information, and navigate to the specified localhost port + - Enter the temporary username/password credentials printed in the startup banner + - Click Build Cache > Settings > Cache Access Control and grant "Read & write" access to Anonymous + +Second, create your `rush-http-build-cache-plugin.json` file as described in the Configuration section: + + - Note that your `url` should end with `/cache`, for example, `http://localhost:5071/cache` + - To test reading and writing, set `isCacheWriteAllowed: true`. + - Configure `tokenHandler` to point to a script that prints a Basic or Bearer Authorization value (this can be a dummy string if you granted Read and Write to Anonymous in your build cache node configuration). + +Note that the Gradle Build Cache Server has a stricter format for its cache keys (they should be a simple hexadecimal hash with no non-alphanumeric characters). Configure this setting in your `common/config/rush/build-cache.json` file: + +```json +{ + "cacheEntryNamePattern": "[hash]" +} +``` + +Last, initialize your cache credentials using Rush: + +```console +rush update-cloud-credentials --interactive +``` + +To test out your remote build cache with full debugging output (for spotting any errors reading or writing the cache), run with the `--debug` flag: + +```console +rush --debug build --verbose +``` + +> If you go on to deploy Rush remote build caching to your developers using the Gradle Build Cache, update your `tokenHandler` +> script to reflect your use case -- for example, you could require each developer to have a designated username/token configured +> via environment variables, and configure Cache Access Control with the corresponding entries. In this case the `tokenHandler` +> script should read the environment variables and print out an Authorization header, for example: +> +> ```javascript +> // common/scripts/build-cache-auth.js +> const credentials = `${process.env.CACHE_USER}:${process.env.CACHE_TOKEN}`; +> console.log('Basic ' + Buffer.from(credentials).toString('base64')); +> ``` diff --git a/rush-plugins/rush-http-build-cache-plugin/config/jest.config.json b/rush-plugins/rush-http-build-cache-plugin/config/jest.config.json new file mode 100644 index 00000000000..fa6760fad1e --- /dev/null +++ b/rush-plugins/rush-http-build-cache-plugin/config/jest.config.json @@ -0,0 +1,14 @@ +{ + "extends": "@rushstack/heft-node-rig/profiles/default/config/jest.config.json", + "clearMocks": true, + "restoreMocks": true, + "collectCoverage": true, + "coverageThreshold": { + "global": { + "branches": 4, + "functions": 15, + "lines": 4, + "statements": 4 + } + } +} diff --git a/rush-plugins/rush-http-build-cache-plugin/config/rig.json b/rush-plugins/rush-http-build-cache-plugin/config/rig.json new file mode 100644 index 00000000000..f497ba3bb0a --- /dev/null +++ b/rush-plugins/rush-http-build-cache-plugin/config/rig.json @@ -0,0 +1,18 @@ +{ + // The "rig.json" file directs tools to look for their config files in an external package. + // Documentation for this system: https://www.npmjs.com/package/@rushstack/rig-package + "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", + + /** + * (Required) The name of the rig package to inherit from. + * It should be an NPM package name with the "-rig" suffix. + */ + "rigPackageName": "@rushstack/heft-node-rig", + + /** + * (Optional) Selects a config profile from the rig package. The name must consist of + * lowercase alphanumeric words separated by hyphens, for example "sample-profile". + * If omitted, then the "default" profile will be used." + */ + "rigProfile": "default" +} diff --git a/rush-plugins/rush-http-build-cache-plugin/package.json b/rush-plugins/rush-http-build-cache-plugin/package.json new file mode 100644 index 00000000000..9e289c5a2b0 --- /dev/null +++ b/rush-plugins/rush-http-build-cache-plugin/package.json @@ -0,0 +1,36 @@ +{ + "name": "@rushstack/rush-http-build-cache-plugin", + "version": "5.92.0", + "description": "Rush plugin for generic HTTP cloud build cache", + "repository": { + "type": "git", + "url": "https://github.com/microsoft/rushstack", + "directory": "rush-plugins/rush-http-build-cache-plugin" + }, + "homepage": "https://rushjs.io", + "main": "lib/index.js", + "types": "lib/index.d.ts", + "license": "MIT", + "scripts": { + "build": "heft build --clean", + "start": "heft test --clean --watch", + "test": "heft test", + "_phase:build": "heft build --clean", + "_phase:test": "heft test --no-build" + }, + "dependencies": { + "@rushstack/node-core-library": "workspace:*", + "@rushstack/rush-sdk": "workspace:*", + "https-proxy-agent": "~5.0.0", + "node-fetch": "2.6.7" + }, + "devDependencies": { + "@microsoft/rush-lib": "workspace:*", + "@rushstack/eslint-config": "workspace:*", + "@rushstack/heft": "workspace:*", + "@rushstack/heft-node-rig": "workspace:*", + "@types/heft-jest": "1.0.1", + "@types/node": "14.18.36", + "@types/node-fetch": "2.6.2" + } +} diff --git a/rush-plugins/rush-http-build-cache-plugin/rush-plugin-manifest.json b/rush-plugins/rush-http-build-cache-plugin/rush-plugin-manifest.json new file mode 100644 index 00000000000..126dfa796b3 --- /dev/null +++ b/rush-plugins/rush-http-build-cache-plugin/rush-plugin-manifest.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/rush-plugin-manifest.schema.json", + "plugins": [ + { + "pluginName": "rush-http-build-cache-plugin", + "description": "Rush plugin for generic HTTP build cache", + "entryPoint": "lib/index.js", + "optionsSchema": "lib/schemas/plugin-config.schema.json" + } + ] +} diff --git a/rush-plugins/rush-http-build-cache-plugin/src/HttpBuildCacheProvider.test.ts b/rush-plugins/rush-http-build-cache-plugin/src/HttpBuildCacheProvider.test.ts new file mode 100644 index 00000000000..20da3767c5e --- /dev/null +++ b/rush-plugins/rush-http-build-cache-plugin/src/HttpBuildCacheProvider.test.ts @@ -0,0 +1,55 @@ +jest.mock('node-fetch', function () { + return Object.assign(jest.fn(), jest.requireActual('node-fetch')); +}); + +import fetch, { Response } from 'node-fetch'; +import { HttpBuildCacheProvider } from './HttpBuildCacheProvider'; +import { RushSession, EnvironmentConfiguration } from '@rushstack/rush-sdk'; +import { StringBufferTerminalProvider, Terminal } from '@rushstack/node-core-library'; + +const EXAMPLE_OPTIONS = { + url: 'https://buildcache.example.acme.com', + tokenHandler: 'node tokenHandler.js', + uploadMethod: 'POST', + isCacheWriteAllowed: false, + pluginName: 'example-plugin', + rushProjectRoot: '/repo' +}; + +describe('HttpBuildCacheProvider', () => { + let terminalBuffer: StringBufferTerminalProvider; + let terminal!: Terminal; + + beforeEach(() => { + terminalBuffer = new StringBufferTerminalProvider(); + terminal = new Terminal(terminalBuffer); + }); + + describe('tryGetCacheEntryBufferByIdAsync', () => { + it('prints warning if read credentials are not available', async () => { + jest.spyOn(EnvironmentConfiguration, 'buildCacheCredential', 'get').mockReturnValue(undefined); + + const session: RushSession = {} as RushSession; + const provider = new HttpBuildCacheProvider(EXAMPLE_OPTIONS, session); + + mocked(fetch).mockResolvedValue( + new Response('Unauthorized', { + status: 401, + statusText: 'Unauthorized' + }) + ); + + const result = await provider.tryGetCacheEntryBufferByIdAsync(terminal, 'some-key'); + expect(result).toBe(undefined); + expect(fetch).toHaveBeenCalledWith('https://buildcache.example.acme.com/some-key', { + body: undefined, + headers: {}, + method: 'GET', + redirect: 'follow' + }); + expect(terminalBuffer.getWarningOutput()).toEqual( + 'Error getting cache entry: Error: Credentials for https://buildcache.example.acme.com/ have not been provided.[n]In CI, verify that RUSH_BUILD_CACHE_CREDENTIAL contains a valid Authorization header value.[n][n]For local developers, run:[n][n] rush update-cloud-credentials --interactive[n][n]' + ); + }); + }); +}); diff --git a/rush-plugins/rush-http-build-cache-plugin/src/HttpBuildCacheProvider.ts b/rush-plugins/rush-http-build-cache-plugin/src/HttpBuildCacheProvider.ts new file mode 100644 index 00000000000..64abea7b8ad --- /dev/null +++ b/rush-plugins/rush-http-build-cache-plugin/src/HttpBuildCacheProvider.ts @@ -0,0 +1,384 @@ +import { ITerminal } from '@rushstack/node-core-library'; +import { + ICloudBuildCacheProvider, + ICredentialCacheEntry, + CredentialCache, + RushSession, + EnvironmentConfiguration +} from '@rushstack/rush-sdk'; +import fetch, { BodyInit, Response } from 'node-fetch'; +import { exec } from './exec'; + +enum CredentialsOptions { + Optional, + Required, + Omit +} + +enum FailureType { + None, + Informational, + Warning, + Error, + Authentication +} + +/** + * @public + */ +export interface IHttpBuildCacheProviderOptions { + url: string; + tokenHandler?: string; + uploadMethod?: string; + headers?: Record; + cacheKeyPrefix?: string; + isCacheWriteAllowed: boolean; + pluginName: string; + rushProjectRoot: string; +} + +export class HttpBuildCacheProvider implements ICloudBuildCacheProvider { + private readonly _pluginName: string; + private readonly _rushSession: RushSession; + private readonly _rushProjectRoot: string; + private readonly _environmentCredential: string | undefined; + private readonly _isCacheWriteAllowedByConfiguration: boolean; + private readonly _url: URL; + private readonly _uploadMethod: string; + private readonly _headers: Record; + private readonly _cacheKeyPrefix: string; + private readonly _tokenHandler: string | undefined; + private __credentialCacheId: string | undefined; + + public get isCacheWriteAllowed(): boolean { + return EnvironmentConfiguration.buildCacheWriteAllowed ?? this._isCacheWriteAllowedByConfiguration; + } + + public constructor(options: IHttpBuildCacheProviderOptions, rushSession: RushSession) { + this._pluginName = options.pluginName; + this._rushSession = rushSession; + this._rushProjectRoot = options.rushProjectRoot; + + this._environmentCredential = EnvironmentConfiguration.buildCacheCredential; + this._isCacheWriteAllowedByConfiguration = options.isCacheWriteAllowed; + this._url = new URL(options.url); + this._uploadMethod = options.uploadMethod ?? 'PUT'; + this._headers = options.headers ?? {}; + this._tokenHandler = options.tokenHandler; + this._cacheKeyPrefix = options.cacheKeyPrefix ?? ''; + } + + public async tryGetCacheEntryBufferByIdAsync( + terminal: ITerminal, + cacheId: string + ): Promise { + try { + const result = await this._http({ + terminal: terminal, + relUrl: `${this._cacheKeyPrefix}${cacheId}`, + method: 'GET', + body: undefined, + warningText: 'Could not get cache entry', + readBody: true + }); + + return Buffer.isBuffer(result) ? result : undefined; + } catch (e) { + terminal.writeWarningLine(`Error getting cache entry: ${e}`); + return undefined; + } + } + + public async trySetCacheEntryBufferAsync( + terminal: ITerminal, + cacheId: string, + objectBuffer: Buffer + ): Promise { + if (!this.isCacheWriteAllowed) { + terminal.writeErrorLine('Writing to cache is not allowed in the current configuration.'); + return false; + } + + terminal.writeDebugLine('Uploading object with cacheId: ', cacheId); + + try { + const result = await this._http({ + terminal: terminal, + relUrl: `${this._cacheKeyPrefix}${cacheId}`, + method: this._uploadMethod, + body: objectBuffer, + warningText: 'Could not write cache entry', + readBody: false + }); + + return result !== false; + } catch (e) { + terminal.writeWarningLine(`Error uploading cache entry: ${e}`); + return false; + } + } + + public async updateCachedCredentialAsync(terminal: ITerminal, credential: string): Promise { + await CredentialCache.usingAsync( + { + supportEditing: true + }, + async (credentialsCache: CredentialCache) => { + credentialsCache.setCacheEntry(this._credentialCacheId, { + credential: credential + }); + await credentialsCache.saveIfModifiedAsync(); + } + ); + } + + public async updateCachedCredentialInteractiveAsync(terminal: ITerminal): Promise { + if (typeof this._tokenHandler !== 'string') { + throw new Error( + `The interactive cloud credentials flow is not configured.\n` + + `Set the 'tokenHandler' setting in 'common/config/rush-plugins/${this._pluginName}.json' to a command that writes your credentials to standard output and exits with code 0 ` + + `or provide your credentials to rush using the --credential flag instead. Credentials must be the ` + + `'Authorization' header expected by ${this._url.href}` + ); + } + + const cmd: string = this._tokenHandler; + terminal.writeVerboseLine(`Running '${cmd}' to get credentials`); + const result = await exec(cmd, this._rushProjectRoot); + + terminal.writeErrorLine(result.stderr); + + if (result.error) { + throw new Error(`Could not obtain credentials. The command '${cmd}' failed.`); + } + + const credential = result.stdout.trim(); + terminal.writeVerboseLine('Got credentials'); + + await this.updateCachedCredentialAsync(terminal, credential); + + terminal.writeLine('Updated credentials cache'); + } + + public async deleteCachedCredentialsAsync(terminal: ITerminal): Promise { + await CredentialCache.usingAsync( + { + supportEditing: true + }, + async (credentialsCache: CredentialCache) => { + credentialsCache.deleteCacheEntry(this._credentialCacheId); + await credentialsCache.saveIfModifiedAsync(); + } + ); + } + + private get _credentialCacheId(): string { + if (!this.__credentialCacheId) { + const cacheIdParts: string[] = [this._url.href]; + + if (this._isCacheWriteAllowedByConfiguration) { + cacheIdParts.push('cacheWriteAllowed'); + } + + this.__credentialCacheId = cacheIdParts.join('|'); + } + + return this.__credentialCacheId; + } + + private async _http(options: { + terminal: ITerminal; + relUrl: string; + method: string; + body: BodyInit | undefined; + warningText: string; + readBody: boolean; + credentialOptions?: CredentialsOptions; + }): Promise { + const { terminal, relUrl, method, body, warningText, readBody, credentialOptions } = options; + const safeCredentialOptions = credentialOptions ?? CredentialsOptions.Optional; + const credentials = await this._tryGetCredentials(safeCredentialOptions); + const url = new URL(relUrl, this._url).href; + + const headers: Record = {}; + if (typeof credentials === 'string') { + headers.Authorization = credentials; + } + + for (const [key, value] of Object.entries(this._headers)) { + if (typeof value === 'string') { + headers[key] = value; + } + } + + const bodyLength = (body as { length: number })?.length || 'unknown'; + + terminal.writeDebugLine(`[http-build-cache] request: ${method} ${url} ${bodyLength} bytes`); + + const response = await fetch(url, { + method: method, + headers: headers, + body: body, + redirect: 'follow' + }); + + if (!response.ok) { + if (typeof credentials !== 'string' && safeCredentialOptions === CredentialsOptions.Optional) { + // We tried fetching the resource without credentials and that did not work out + // Try again but require credentials this time + // This will trigger the provider to request credentials + return await this._http({ ...options, credentialOptions: CredentialsOptions.Required }); + } + + this._reportFailure(terminal, method, response, false, warningText); + return false; + } + + const result: Buffer | boolean = readBody ? Buffer.from(await response.arrayBuffer()) : true; + + terminal.writeDebugLine( + `[http-build-cache] actual response: ${response.status} ${url} ${ + result === true ? 'true' : result.length + } bytes` + ); + + return result; + } + + private async _tryGetCredentials(options: CredentialsOptions.Required): Promise; + private async _tryGetCredentials(options: CredentialsOptions.Optional): Promise; + private async _tryGetCredentials(options: CredentialsOptions.Omit): Promise; + private async _tryGetCredentials(options: CredentialsOptions): Promise; + private async _tryGetCredentials(options: CredentialsOptions): Promise { + if (options === CredentialsOptions.Omit) { + return; + } + + let credentials: string | undefined = this._environmentCredential; + + if (typeof credentials !== 'string') { + credentials = await this._tryGetCredentialsFromCache(); + } + + if (typeof credentials !== 'string' && options === CredentialsOptions.Required) { + throw new Error( + [ + `Credentials for ${this._url.href} have not been provided.`, + `In CI, verify that RUSH_BUILD_CACHE_CREDENTIAL contains a valid Authorization header value.`, + ``, + `For local developers, run:`, + ``, + ` rush update-cloud-credentials --interactive`, + `` + ].join('\n') + ); + } + + return credentials; + } + + private async _tryGetCredentialsFromCache(): Promise { + let cacheEntry: ICredentialCacheEntry | undefined; + + await CredentialCache.usingAsync( + { + supportEditing: false + }, + (credentialsCache: CredentialCache) => { + cacheEntry = credentialsCache.tryGetCacheEntry(this._credentialCacheId); + } + ); + + if (cacheEntry) { + const expirationTime: number | undefined = cacheEntry.expires?.getTime(); + if (!expirationTime || expirationTime >= Date.now()) { + return cacheEntry.credential; + } + } + } + + private _getFailureType(requestMethod: string, response: Response, isRedirect: boolean): FailureType { + if (response.ok) { + return FailureType.None; + } + + switch (response.status) { + case 503: { + // Don't fail production builds with warnings for transient issues + return FailureType.Informational; + } + + case 401: + case 403: + case 407: { + if (requestMethod === 'GET' && (isRedirect || response.redirected)) { + // Cache misses for GET requests are not errors + // This is a workaround behavior where a server can issue a redirect and we fail to authenticate at the new location. + // We do not want to signal this as an authentication failure because the authorization header is not passed on to redirects. + // i.e The authentication header was accepted for the first request and therefore subsequent failures + // where it was not present should not be attributed to the header. + // This scenario usually comes up with services that redirect to pre-signed URLS that don't actually exist. + // Those services then usually treat the 404 as a 403 to prevent leaking information. + return FailureType.None; + } + + return FailureType.Authentication; + } + + case 404: { + if (requestMethod === 'GET') { + // Cache misses for GET requests are not errors + return FailureType.None; + } + } + } + + // Let dev builds succeed, let Prod builds fail + return FailureType.Warning; + } + + private _reportFailure( + terminal: ITerminal, + requestMethod: string, + response: Response, + isRedirect: boolean, + message: string + ): void { + switch (this._getFailureType(requestMethod, response, isRedirect)) { + default: { + terminal.writeErrorLine(`${message}: HTTP ${response.status}: ${response.statusText}`); + break; + } + + case FailureType.Warning: { + terminal.writeWarningLine(`${message}: HTTP ${response.status}: ${response.statusText}`); + break; + } + + case FailureType.Informational: { + terminal.writeLine(`${message}: HTTP ${response.status}: ${response.statusText}`); + break; + } + + case FailureType.None: { + terminal.writeDebugLine(`${message}: HTTP ${response.status}: ${response.statusText}`); + break; + } + + case FailureType.Authentication: { + throw new Error( + [ + `${this._url.href} responded with ${response.status}: ${response.statusText}.`, + `Credentials may be misconfigured of have expired.`, + `In CI, verify that RUSH_BUILD_CACHE_CREDENTIAL contains a valid Authorization header value.`, + ``, + `For local developers, run:`, + ``, + ` rush update-cloud-credentials --interactive`, + `` + ].join('\n') + ); + } + } + } +} diff --git a/rush-plugins/rush-http-build-cache-plugin/src/RushHttpBuildCachePlugin.ts b/rush-plugins/rush-http-build-cache-plugin/src/RushHttpBuildCachePlugin.ts new file mode 100644 index 00000000000..2a206290b52 --- /dev/null +++ b/rush-plugins/rush-http-build-cache-plugin/src/RushHttpBuildCachePlugin.ts @@ -0,0 +1,75 @@ +import { Import } from '@rushstack/node-core-library'; +import type { IRushPlugin, RushSession, RushConfiguration } from '@rushstack/rush-sdk'; +import type { HttpBuildCacheProvider, IHttpBuildCacheProviderOptions } from './HttpBuildCacheProvider'; + +const HttpBuildCacheProviderModule: typeof import('./HttpBuildCacheProvider') = Import.lazy( + './HttpBuildCacheProvider', + require +); + +const PLUGIN_NAME: string = 'HttpBuildCachePlugin'; + +/** + * @public + */ +export interface IRushHttpBuildCachePluginOptions { + /** + * The url to the service that caches builds. + */ + url: string; + + /** + * An optional set of HTTP headers to pass to the cache server. + */ + headers?: Record; + + /** + * An optional command that prints the endpoint's credentials to stdout. + */ + tokenHandler?: string; + + /** + * Prefix for cache keys. + */ + cacheKeyPrefix?: string; + + /** + * If set to true, allow writing to the cache. Defaults to false. + */ + isCacheWriteAllowed?: boolean; +} + +/** + * @public + */ +export class RushHttpBuildCachePlugin implements IRushPlugin { + public readonly pluginName: string = PLUGIN_NAME; + private readonly _options: IRushHttpBuildCachePluginOptions; + + public constructor(options: IRushHttpBuildCachePluginOptions) { + this._options = options; + } + + public apply(rushSession: RushSession, rushConfig: RushConfiguration): void { + rushSession.hooks.initialize.tap(this.pluginName, () => { + rushSession.registerCloudBuildCacheProviderFactory( + 'http', + (buildCacheConfig): HttpBuildCacheProvider => { + const { url, headers, tokenHandler, cacheKeyPrefix, isCacheWriteAllowed } = this._options; + + const options: IHttpBuildCacheProviderOptions = { + pluginName: this.pluginName, + rushProjectRoot: rushConfig.rushJsonFolder, + url: url, + headers: headers, + tokenHandler: tokenHandler, + cacheKeyPrefix: cacheKeyPrefix, + isCacheWriteAllowed: !!isCacheWriteAllowed + }; + + return new HttpBuildCacheProviderModule.HttpBuildCacheProvider(options, rushSession); + } + ); + }); + } +} diff --git a/rush-plugins/rush-http-build-cache-plugin/src/exec.test.ts b/rush-plugins/rush-http-build-cache-plugin/src/exec.test.ts new file mode 100644 index 00000000000..309806fcfbd --- /dev/null +++ b/rush-plugins/rush-http-build-cache-plugin/src/exec.test.ts @@ -0,0 +1,13 @@ +import { exec } from './exec'; + +describe('exec', function () { + it('can exec a process and capture output', async function () { + // Act + const cmd = process.argv0 + ` --eval "console.log(1); console.error(2); process.exit(3);"`; + const result = await exec(cmd, process.cwd()); + + expect(result.error?.message).toEqual(`Command failed: ${cmd}\n2\n`); + expect(result.stderr).toEqual('2\n'); + expect(result.stdout).toEqual('1\n'); + }); +}); diff --git a/rush-plugins/rush-http-build-cache-plugin/src/exec.ts b/rush-plugins/rush-http-build-cache-plugin/src/exec.ts new file mode 100644 index 00000000000..6952df1aee6 --- /dev/null +++ b/rush-plugins/rush-http-build-cache-plugin/src/exec.ts @@ -0,0 +1,35 @@ +import { exec as child_process_exec } from 'child_process'; + +/** + * The result of spawning a command + */ +export interface IExecResult { + /** + * The standard output of the spawned command + */ + stdout: string; + + /** + * The standard error of the spawned command + */ + stderr: string; + + /** + * The exit code of the spawned command + */ + error?: Error; +} + +/** + * Spawn a child process and obtain the contents of its output streams as text + * @param cmd - Command to execute + * @param args - Arguments to pass to command + * @returns The exit code and output of the executed command + */ +export function exec(cmd: string, cwd: string): Promise { + return new Promise(function (resolve: (result: IExecResult) => void, reject: (error: unknown) => void) { + child_process_exec(cmd, { cwd }, function (error, stdout, stderr) { + resolve({ stdout, stderr, error: error || undefined }); + }); + }); +} diff --git a/rush-plugins/rush-http-build-cache-plugin/src/index.ts b/rush-plugins/rush-http-build-cache-plugin/src/index.ts new file mode 100644 index 00000000000..17e67ac1859 --- /dev/null +++ b/rush-plugins/rush-http-build-cache-plugin/src/index.ts @@ -0,0 +1,4 @@ +import { RushHttpBuildCachePlugin } from './RushHttpBuildCachePlugin'; + +export default RushHttpBuildCachePlugin; +export { IHttpBuildCacheProviderOptions } from './HttpBuildCacheProvider'; diff --git a/rush-plugins/rush-http-build-cache-plugin/src/schemas/plugin-config.schema.json b/rush-plugins/rush-http-build-cache-plugin/src/schemas/plugin-config.schema.json new file mode 100644 index 00000000000..0d825245c0c --- /dev/null +++ b/rush-plugins/rush-http-build-cache-plugin/src/schemas/plugin-config.schema.json @@ -0,0 +1,39 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Configuration for build cache with HTTPS server", + "type": "object", + "required": ["url"], + "properties": { + "url": { + "type": "string", + "description": "(Required) The URL of the server that stores the caches (e.g. \"https://build-caches.example.com\").", + "format": "uri" + }, + "method": { + "type": "string", + "description": "(Required) The URL of the server that stores the caches (e.g. \"https://build-caches.example.com\").", + "enum": ["PUT", "POST", "GET", "PATCH"], + "default": "PUT" + }, + "headers": { + "type": "object", + "description": "(Optional) HTTP headers to pass to the cache server", + "properties": {}, + "additionalProperties": { + "type": "string" + } + }, + "tokenHandler": { + "type": "string", + "description": "(Optional) Shell command that prints the authorization token needed to communicate with the HTTPS server and exits with code 0." + }, + "cacheKeyPrefix": { + "type": "string", + "description": "(Optional) prefix for cache keys." + }, + "isCacheWriteAllowed": { + "type": "boolean", + "description": "(Optional) If set to true, allow writing to the cache. Defaults to false." + } + } +} diff --git a/rush-plugins/rush-http-build-cache-plugin/tsconfig.json b/rush-plugins/rush-http-build-cache-plugin/tsconfig.json new file mode 100644 index 00000000000..6798ece7344 --- /dev/null +++ b/rush-plugins/rush-http-build-cache-plugin/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "./node_modules/@rushstack/heft-node-rig/profiles/default/tsconfig-base.json", + "compilerOptions": { + "types": ["node", "heft-jest"] + } +} diff --git a/rush.json b/rush.json index 085a872ecf9..56a875ca1fd 100644 --- a/rush.json +++ b/rush.json @@ -1020,6 +1020,12 @@ "reviewCategory": "libraries", "versionPolicyName": "rush" }, + { + "packageName": "@rushstack/rush-http-build-cache-plugin", + "projectFolder": "rush-plugins/rush-http-build-cache-plugin", + "reviewCategory": "libraries", + "versionPolicyName": "rush" + }, { "packageName": "@rushstack/rush-litewatch-plugin", "projectFolder": "rush-plugins/rush-litewatch-plugin",