Allow a PAT still to be configured to avoid rate limiting (#3785)

Unauthenticated GitHub API calls are more heavily rate limited. Passing
a token will allow a greater number of requests to be made.

This restores the PAT utility code, but tries to use the default
github.token now that ark and positron repositories are public, avoiding
the need for secrets in PR workflows.

Follow-up to #3739

.github/workflows/positron-ci.yml

@@ -36,6 +36,7 @@ jobs:
         env:
           PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
           ELECTRON_SKIP_BINARY_DOWNLOAD: 1
+          POSITRON_GITHUB_PAT: ${{ github.token }}
         run: |
           # Install Yarn
           npm install -g yarn

.github/workflows/positron-full-test.yml

@@ -41,6 +41,7 @@ jobs:
         env:
           PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
           ELECTRON_SKIP_BINARY_DOWNLOAD: 1
+          POSITRON_GITHUB_PAT: ${{ github.token }}
         run: |
           # Install Yarn
           npm install -g yarn

.github/workflows/positron-python-ci.yml

@@ -372,6 +372,7 @@ jobs:
         env:
           TEST_FILES_SUFFIX: testvirtualenvs
           CI_PYTHON_VERSION: ${{ matrix.python }}
+          POSITRON_GITHUB_PAT: ${{ github.token }}
         uses: GabrielBB/[email protected]
         with:
           run: yarn testSingleWorkspace
@@ -381,6 +382,7 @@ jobs:
       - name: Run single-workspace tests
         env:
           CI_PYTHON_VERSION: ${{ matrix.python }}
+          POSITRON_GITHUB_PAT: ${{ github.token }}
         uses: GabrielBB/[email protected]
         with:
           run: yarn testSingleWorkspace
@@ -390,6 +392,7 @@ jobs:
       - name: Run debugger tests
         env:
           CI_PYTHON_VERSION: ${{ matrix.python }}
+          POSITRON_GITHUB_PAT: ${{ github.token }}
         uses: GabrielBB/[email protected]
         with:
           run: yarn testDebugger
@@ -401,5 +404,7 @@ jobs:
         if: matrix.test-suite == 'functional'
 
       - name: Run smoke tests
+        env:
+          POSITRON_GITHUB_PAT: ${{ github.token }}
         run: yarn tsc && node ./out/test/smokeTest.js
         if: matrix.test-suite == 'smoke'

extensions/positron-python/src/test/positron/testElectron.ts

@@ -3,7 +3,7 @@
  *  Licensed under the Elastic License 2.0. See LICENSE.txt for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { spawnSync } from 'child_process';
+import { exec, spawnSync } from 'child_process';
 import * as fs from 'fs-extra';
 import { IncomingMessage } from 'http';
 import * as https from 'https';
@@ -26,6 +26,29 @@ const httpsGetAsync = (opts: string | https.RequestOptions) =>
         req.once('error', reject);
     });
 
+/**
+ * Helper to execute a command and return the stdout and stderr.
+ *
+ * @param command The command to execute.
+ * @param stdin Optional stdin to pass to the command.
+ * @returns A promise that resolves with the stdout and stderr of the command.
+ */
+async function executeCommand(command: string, stdin?: string): Promise<{ stdout: string; stderr: string }> {
+    return new Promise((resolve, reject) => {
+        const process = exec(command, (error, stdout, stderr) => {
+            if (error) {
+                reject(error);
+            } else {
+                resolve({ stdout, stderr });
+            }
+        });
+        if (stdin) {
+            process.stdin!.write(stdin);
+            process.stdin!.end();
+        }
+    });
+}
+
 /**
  * Helper to execute a command (quoting arguments) and return stdout.
  *
@@ -49,17 +72,145 @@ function spawnSyncCommand(command: string, args?: string[]): string {
  * Roughly equivalent to `downloadAndUnzipVSCode` from `@vscode/test-electron`.
  */
 export async function downloadAndUnzipPositron(): Promise<{ version: string; executablePath: string }> {
+    // Adapted from: https://github.com/posit-dev/positron/extensions/positron-r/scripts/install-kernel.ts.
+
+    // We need a Github Personal Access Token (PAT) to download Positron. Because this is sensitive
+    // information, there are a lot of ways to set it. We try the following in order:
+
+    // (1) The GITHUB_PAT environment variable.
+    // (2) The POSITRON_GITHUB_PAT environment variable.
+    // (3) The git config setting 'credential.https://api.github.com.token'.
+    // (4) The git credential store.
+
+    // (1) Get the GITHUB_PAT from the environment.
+    let githubPat = process.env.GITHUB_PAT;
+    let gitCredential = false;
+    if (githubPat) {
+        console.log('Using Github PAT from environment variable GITHUB_PAT.');
+    } else {
+        // (2) Try POSITRON_GITHUB_PAT (it's what the build script sets)
+        githubPat = process.env.POSITRON_GITHUB_PAT;
+        if (githubPat) {
+            console.log('Using Github PAT from environment variable POSITRON_GITHUB_PAT.');
+        }
+    }
+
+    // (3) If no GITHUB_PAT is set, try to get it from git config. This provides a
+    // convenient non-interactive way to set the PAT.
+    if (!githubPat) {
+        try {
+            const { stdout } = await executeCommand('git config --get credential.https://api.github.com.token');
+            githubPat = stdout.trim();
+            if (githubPat) {
+                console.log(`Using Github PAT from git config setting 'credential.https://api.github.com.token'.`);
+            }
+        } catch (error) {
+            // We don't care if this fails; we'll try `git credential` next.
+        }
+    }
+
+    // (4) If no GITHUB_PAT is set, try to get it from git credential.
+    if (!githubPat) {
+        // Explain to the user what's about to happen.
+        console.log(
+            `Attempting to retrieve a Github Personal Access Token from git in order\n` +
+                `to download Positron. If you are prompted for a username and\n` +
+                `password, enter your Github username and a Personal Access Token with the\n` +
+                `'repo' scope. You can read about how to create a Personal Access Token here: \n` +
+                `\n` +
+                `https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens\n` +
+                `\n` +
+                `You can set a PAT later by rerunning this command and supplying the PAT at this prompt,\n` +
+                `or by running 'git config credential.https://api.github.com.token YOUR_GITHUB_PAT'\n`,
+        );
+        const { stdout } = await executeCommand(
+            'git credential fill',
+            `protocol=https\nhost=github.com\npath=/repos/posit-dev/positron/releases\n`,
+        );
+
+        gitCredential = true;
+        // Extract the `password = ` line from the output.
+        const passwordLine = stdout.split('\n').find((line: string) => line.startsWith('password='));
+        if (passwordLine) {
+            [, githubPat] = passwordLine.split('=');
+            console.log(`Using Github PAT returned from 'git credential'.`);
+        }
+    }
+
+    if (!githubPat) {
+        throw new Error(`No Github PAT was found. Unable to download Positron.`);
+    }
+
+    const headers: Record<string, string> = {
+        Accept: 'application/vnd.github.v3.raw', // eslint-disable-line
+        'User-Agent': 'positron-python-tests', // eslint-disable-line
+    };
+    // If we have a githubPat, set it for better rate limiting.
+    if (githubPat) {
+        headers.Authorization = `token ${githubPat}`;
+    }
+
     const response = await httpsGetAsync({
-        headers: {
-            Accept: 'application/vnd.github.v3.raw', // eslint-disable-line
-            'User-Agent': 'positron-python-tests', // eslint-disable-line
-        },
+        headers,
         method: 'GET',
         protocol: 'https:',
         hostname: 'api.github.com',
         path: `/repos/posit-dev/positron/releases`,
     });
 
+    // Special handling for PATs originating from `git credential`.
+    if (gitCredential && response.statusCode === 200) {
+        // If the PAT hasn't been approved yet, do so now. This stores the credential in
+        // the system credential store (or whatever `git credential` uses on the system).
+        // Without this step, the user will be prompted for a username and password the
+        // next time they try to download Positron.
+        const { stdout, stderr } = await executeCommand(
+            'git credential approve',
+            `protocol=https\n` +
+                `host=github.com\n` +
+                `path=/repos/posit-dev/positron/releases\n` +
+                `username=\n` +
+                `password=${githubPat}\n`,
+        );
+        console.log(stdout);
+        if (stderr) {
+            console.warn(
+                `Unable to approve PAT. You may be prompted for a username and ` +
+                    `password the next time you download Positron.`,
+            );
+            console.error(stderr);
+        }
+    } else if (gitCredential && response.statusCode && response.statusCode > 400 && response.statusCode < 500) {
+        // This handles the case wherein we got an invalid PAT from `git credential`. In this
+        // case we need to clean up the PAT from the credential store, so that we don't
+        // continue to use it.
+        const { stdout, stderr } = await executeCommand(
+            'git credential reject',
+            `protocol=https\n` +
+                `host=github.com\n` +
+                `path=/repos/posit-dev/positron/releases\n` +
+                `username=\n` +
+                `password=${githubPat}\n`,
+        );
+        console.log(stdout);
+        if (stderr) {
+            console.error(stderr);
+            throw new Error(
+                `The stored PAT returned by 'git credential' is invalid, but\n` +
+                    `could not be removed. Please manually remove the PAT from 'git credential'\n` +
+                    `for the host 'github.com'`,
+            );
+        }
+        throw new Error(
+            `The PAT returned by 'git credential' is invalid. Positron cannot be\n` +
+                `downloaded.\n\n` +
+                `Check to be sure that your Personal Access Token:\n` +
+                '- Has the `repo` scope\n' +
+                '- Is not expired\n' +
+                '- Has been authorized for the "posit-dev" organization on Github (Configure SSO)\n',
+        );
+    }
+
     let responseBody = '';
     response.on('data', (chunk) => {
         responseBody += chunk;
@@ -131,11 +282,10 @@ export async function downloadAndUnzipPositron(): Promise<{ version: string; exe
 
     console.log(`Downloading Positron for ${platform} from ${asset.url}`);
     const url = new URL(asset.url);
+    // Reset the Accept header to download the asset.
+    headers.Accept = 'application/octet-stream';
     const dlRequestOptions: https.RequestOptions = {
-        headers: {
-            Accept: 'application/octet-stream', // eslint-disable-line
-            'User-Agent': 'positron-python-tests', // eslint-disable-line
-        },
+        headers,
         method: 'GET',
         protocol: url.protocol,
         hostname: url.hostname,