Initial implementation of rush-http-build-cache-plugin

apps/rush/package.json

@@ -47,6 +47,7 @@
     "@rushstack/heft-node-rig": "workspace:*",
     "@rushstack/rush-amazon-s3-build-cache-plugin": "workspace:*",
     "@rushstack/rush-azure-storage-build-cache-plugin": "workspace:*",
+    "@rushstack/rush-http-build-cache-plugin": "workspace:*",
     "@types/heft-jest": "1.0.1",
     "@types/node": "14.18.36",
     "@types/semver": "7.3.5"

apps/rush/src/start-dev.ts

@@ -27,6 +27,7 @@ function includePlugin(pluginName: string, pluginPackageName?: string): void {
 
 includePlugin('rush-amazon-s3-build-cache-plugin');
 includePlugin('rush-azure-storage-build-cache-plugin');
+includePlugin('rush-http-build-cache-plugin');
 // Including this here so that developers can reuse it without installing the plugin a second time
 includePlugin('rush-azure-interactive-auth-plugin', '@rushstack/rush-azure-storage-build-cache-plugin');
 

common/changes/@microsoft/rush/enelson-http-cache_2023-02-15-13-36.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "Add built-in plugin rush-http-build-cache-plugin",
+      "type": "none"
+    }
+  ],
+  "packageName": "@microsoft/rush"
+}

common/config/rush/browser-approved-packages.json

@@ -34,6 +34,10 @@
       "name": "@reduxjs/toolkit",
       "allowedCategories": [ "libraries" ]
     },
+    {
+      "name": "@rushstack/rush-http-build-cache-plugin",
+      "allowedCategories": [ "libraries" ]
+    },
     {
       "name": "@rushstack/rush-themed-ui",
       "allowedCategories": [ "libraries" ]

libraries/rush-lib/package.json

@@ -88,6 +88,7 @@
   },
   "publishOnlyDependencies": {
     "@rushstack/rush-amazon-s3-build-cache-plugin": "workspace:*",
-    "@rushstack/rush-azure-storage-build-cache-plugin": "workspace:*"
+    "@rushstack/rush-azure-storage-build-cache-plugin": "workspace:*",
+    "@rushstack/rush-http-build-cache-plugin": "workspace:*"
   }
 }

libraries/rush-lib/src/pluginFramework/PluginManager.ts

@@ -75,6 +75,7 @@ export class PluginManager {
 
     tryAddBuiltInPlugin('rush-amazon-s3-build-cache-plugin');
     tryAddBuiltInPlugin('rush-azure-storage-build-cache-plugin');
+    tryAddBuiltInPlugin('rush-http-build-cache-plugin');
     // This is a secondary plugin inside the `@rushstack/rush-azure-storage-build-cache-plugin`
     // package. Because that package comes with Rush (for now), it needs to get registered here.
     // If the necessary config file doesn't exist, this plugin doesn't do anything.

rush-plugins/rush-http-build-cache-plugin/README.md

@@ -0,0 +1,94 @@
+# @rushstack/rush-http-build-cache-plugin
+
+A Rush plugin that uses HTTP/HTTPS to manage cache objects.
+
+Authentication is provided via standard `Authorization` HTTP headers, with the value (a Bearer or Basic token) configured using a custom node script. Your "tokenHandler" node script is automatically called when the user updates their cloud credentials.
+
+## Configuration
+
+To use the HTTP build cache plugin, enable it in `common/config/rush/build-cache.json`:
+
+```json
+{
+    "buildCacheEnabled": true,
+
+    "cacheProvider": "http"
+}
+```
+
+Once enabled, configure the HTTP build cache in config file `common/config/rush-plugins/rush-http-build-cache-plugin.json`:
+
+```json
+{
+    "url": "https://build-cache.example.com",
+    "tokenHandler": "node common/scripts/custom-script-that-returns-an-authentication-header.js",
+    "isCacheWriteAllowed": false
+}
+```
+
+- url: The server to store cache objects.
+- tokenHandler: A script that can print the Authorization header expected by the server. The value printed to `stdout` by this command should be an exact header, for example, `Bearer ab98d8c878d937290d979a9097c90dfffff` or `Basic 098abc7dff==`.
+- isCacheWriteAllowed: A flag that determines if the plugin should write to the cache.
+
+## HTTP Cache Server Requirements
+
+The HTTP build cache plugin can use almost any HTTP/HTTPS backend for remote caching, as long as it honors the following rules:
+
+ - Uses `Authorization: Bearer xxx` or `Authorization: Basic xxx` headers for authentication.
+ - Accepts GET requests for cache reads.
+ - Accepts PUT requests for cache writes (with a raw request body -- no `form/multipart`).
+ - Cache hits return HTTP 200 with the file in the response body.
+ - Successful cache writes return HTTP 2xx (200-299).
+ - Cache misses return HTTP 404 or HTTP 403.
+ - Invalid or missing authentication returns HTTP 401.
+
+## Examples
+
+### Gradle Build Cache Server
+
+The Gradle Build Cache Server (typically used to support Gradle Remote Build Cache) meets all of the requirements above, so if you don't have another server in mind, you can use it as your remote backend.
+
+First, start up and configure your build cache node locally:
+
+ - Download latest JAR file from https://docs.gradle.com/build-cache-node/
+ - Start the service: `java -jar build-cache-node-14.0.jar start`
+ - Copy the startup banner information, and navigate to the specified localhost port
+ - Enter the temporary username/password credentials printed in the startup banner
+ - Click Build Cache > Settings > Cache Access Control and grant "Read & write" access to Anonymous
+
+Second, create your `rush-http-build-cache-plugin.json` file as described in the Configuration section:
+
+ - Note that your `url` should end with `/cache`, for example, `http://localhost:5071/cache`
+ - To test reading and writing, set `isCacheWriteAllowed: true`.
+ - Configure `tokenHandler` to point to a script that prints a Basic or Bearer Authorization value (this can be a dummy string if you granted Read and Write to Anonymous in your build cache node configuration).
+
+Note that the Gradle Build Cache Server has a stricter format for its cache keys (they should be a simple hexadecimal hash with no non-alphanumeric characters). Configure this setting in your `common/config/rush/build-cache.json` file:
+
+```json
+{
+    "cacheEntryNamePattern": "[hash]"
+}
+```
+
+Last, initialize your cache credentials using Rush:
+
+```console
+rush update-cloud-credentials --interactive
+```
+
+To test out your remote build cache with full debugging output (for spotting any errors reading or writing the cache), run with the `--debug` flag:
+
+```console
+rush --debug build --verbose
+```
+
+> If you go on to deploy Rush remote build caching to your developers using the Gradle Build Cache, update your `tokenHandler`
+> script to reflect your use case -- for example, you could require each developer to have a designated username/token configured
+> via environment variables, and configure Cache Access Control with the corresponding entries. In this case the `tokenHandler`
+> script should read the environment variables and print out an Authorization header, for example:
+>
+> ```javascript
+> // common/scripts/build-cache-auth.js
+> const credentials = `${process.env.CACHE_USER}:${process.env.CACHE_TOKEN}`;
+> console.log('Basic ' + Buffer.from(credentials).toString('base64'));
+> ```

rush-plugins/rush-http-build-cache-plugin/config/jest.config.json

@@ -0,0 +1,14 @@
+{
+  "extends": "@rushstack/heft-node-rig/profiles/default/config/jest.config.json",
+  "clearMocks": true,
+  "restoreMocks": true,
+  "collectCoverage": true,
+  "coverageThreshold": {
+    "global": {
+      "branches": 4,
+      "functions": 15,
+      "lines": 4,
+      "statements": 4
+    }
+  }
+}

rush-plugins/rush-http-build-cache-plugin/config/rig.json

@@ -0,0 +1,18 @@
+{
+  // The "rig.json" file directs tools to look for their config files in an external package.
+  // Documentation for this system: https://www.npmjs.com/package/@rushstack/rig-package
+  "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
+
+  /**
+   * (Required) The name of the rig package to inherit from.
+   * It should be an NPM package name with the "-rig" suffix.
+   */
+  "rigPackageName": "@rushstack/heft-node-rig",
+
+  /**
+   * (Optional) Selects a config profile from the rig package.  The name must consist of
+   * lowercase alphanumeric words separated by hyphens, for example "sample-profile".
+   * If omitted, then the "default" profile will be used."
+   */
+  "rigProfile": "default"
+}

rush-plugins/rush-http-build-cache-plugin/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "@rushstack/rush-http-build-cache-plugin",
+  "version": "5.92.0",
+  "description": "Rush plugin for generic HTTP cloud build cache",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/microsoft/rushstack",
+    "directory": "rush-plugins/rush-http-build-cache-plugin"
+  },
+  "homepage": "https://rushjs.io",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "license": "MIT",
+  "scripts": {
+    "build": "heft build --clean",
+    "start": "heft test --clean --watch",
+    "test": "heft test",
+    "_phase:build": "heft build --clean",
+    "_phase:test": "heft test --no-build"
+  },
+  "dependencies": {
+    "@rushstack/node-core-library": "workspace:*",
+    "@rushstack/rush-sdk": "workspace:*",
+    "https-proxy-agent": "~5.0.0",
+    "node-fetch": "2.6.7"
+  },
+  "devDependencies": {
+    "@microsoft/rush-lib": "workspace:*",
+    "@rushstack/eslint-config": "workspace:*",
+    "@rushstack/heft": "workspace:*",
+    "@rushstack/heft-node-rig": "workspace:*",
+    "@types/heft-jest": "1.0.1",
+    "@types/node": "14.18.36",
+    "@types/node-fetch": "2.6.2"
+  }
+}

rush-plugins/rush-http-build-cache-plugin/rush-plugin-manifest.json

@@ -0,0 +1,11 @@
+{
+  "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/rush-plugin-manifest.schema.json",
+  "plugins": [
+    {
+      "pluginName": "rush-http-build-cache-plugin",
+      "description": "Rush plugin for generic HTTP build cache",
+      "entryPoint": "lib/index.js",
+      "optionsSchema": "lib/schemas/plugin-config.schema.json"
+    }
+  ]
+}

rush-plugins/rush-http-build-cache-plugin/src/HttpBuildCacheProvider.test.ts

@@ -0,0 +1,55 @@
+jest.mock('node-fetch', function () {
+  return Object.assign(jest.fn(), jest.requireActual('node-fetch'));
+});
+
+import fetch, { Response } from 'node-fetch';
+import { HttpBuildCacheProvider } from './HttpBuildCacheProvider';
+import { RushSession, EnvironmentConfiguration } from '@rushstack/rush-sdk';
+import { StringBufferTerminalProvider, Terminal } from '@rushstack/node-core-library';
+
+const EXAMPLE_OPTIONS = {
+  url: 'https://buildcache.example.acme.com',
+  tokenHandler: 'node tokenHandler.js',
+  uploadMethod: 'POST',
+  isCacheWriteAllowed: false,
+  pluginName: 'example-plugin',
+  rushProjectRoot: '/repo'
+};
+
+describe('HttpBuildCacheProvider', () => {
+  let terminalBuffer: StringBufferTerminalProvider;
+  let terminal!: Terminal;
+
+  beforeEach(() => {
+    terminalBuffer = new StringBufferTerminalProvider();
+    terminal = new Terminal(terminalBuffer);
+  });
+
+  describe('tryGetCacheEntryBufferByIdAsync', () => {
+    it('prints warning if read credentials are not available', async () => {
+      jest.spyOn(EnvironmentConfiguration, 'buildCacheCredential', 'get').mockReturnValue(undefined);
+
+      const session: RushSession = {} as RushSession;
+      const provider = new HttpBuildCacheProvider(EXAMPLE_OPTIONS, session);
+
+      mocked(fetch).mockResolvedValue(
+        new Response('Unauthorized', {
+          status: 401,
+          statusText: 'Unauthorized'
+        })
+      );
+
+      const result = await provider.tryGetCacheEntryBufferByIdAsync(terminal, 'some-key');
+      expect(result).toBe(undefined);
+      expect(fetch).toHaveBeenCalledWith('https://buildcache.example.acme.com/some-key', {
+        body: undefined,
+        headers: {},
+        method: 'GET',
+        redirect: 'follow'
+      });
+      expect(terminalBuffer.getWarningOutput()).toEqual(
+        'Error getting cache entry: Error: Credentials for https://buildcache.example.acme.com/ have not been provided.[n]In CI, verify that RUSH_BUILD_CACHE_CREDENTIAL contains a valid Authorization header value.[n][n]For local developers, run:[n][n]    rush update-cloud-credentials --interactive[n][n]'
+      );
+    });
+  });
+});