diff --git a/.github/workflows/positron-python-ci.yml b/.github/workflows/positron-python-ci.yml index f23422ecb10..d4425796329 100644 --- a/.github/workflows/positron-python-ci.yml +++ b/.github/workflows/positron-python-ci.yml @@ -17,7 +17,7 @@ on: defaults: run: - working-directory: 'extensions/positron-python' + working-directory: 'extensions/positron-python' env: NODE_VERSION: '20.12.1' @@ -216,7 +216,8 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] python: ['3.x'] - test-suite: [ts-unit, venv, single-workspace, debugger, functional, smoke] + test-suite: + [ts-unit, venv, single-workspace, debugger, functional, smoke] # TODO: Add integration tests on windows and ubuntu. This requires updating # src/test/positron/testElectron.ts to support installing Positron on these platforms. exclude: @@ -257,7 +258,9 @@ jobs: cache: 'pip' - name: Install Node dependencies - run: npm ci --fetch-timeout 120000 + run: | + git config credential.https://api.github.com.token ${{ secrets.POSITRON_GITHUB_PAT }} + npm ci --fetch-timeout 120000 - name: Run `gulp prePublishNonBundle` run: npm run prePublish @@ -405,6 +408,7 @@ jobs: - name: Run smoke tests env: - POSITRON_GITHUB_PAT: ${{ github.token }} - run: npx tsc && node ./out/test/smokeTest.js + POSITRON_GITHUB_PAT: ${{ secrets.POSITRON_GITHUB_PAT }} + run: | + npx tsc && node ./out/test/smokeTest.js if: matrix.test-suite == 'smoke' diff --git a/extensions/positron-python/.gitignore b/extensions/positron-python/.gitignore index 39571ab0b1b..fcaa70ab8fe 100644 --- a/extensions/positron-python/.gitignore +++ b/extensions/positron-python/.gitignore @@ -53,6 +53,8 @@ tags # --- Start Positron --- python_files/posit/positron/tests/images python_files/posit/positron/_vendor/** +scripts/*.js +resources/pet/** # --- End Positron --- python-env-tools/** # coverage files produced as test output diff --git a/extensions/positron-python/package.json b/extensions/positron-python/package.json index ab0c540fff2..d1595cf39ac 100644 --- a/extensions/positron-python/package.json +++ b/extensions/positron-python/package.json @@ -699,7 +699,7 @@ "onExP", "preview" ], - "scope": "machine", + "scope": "machine-overridable", "type": "string" }, "python.pipenvPath": { @@ -1800,14 +1800,15 @@ } }, "scripts": { - "package": "gulp clean && gulp prePublishBundle && vsce package -o ms-python-insiders.vsix", + "package": "gulp clean && gulp prePublishBundle && vsce package -o positron-python-insiders.vsix", "prePublish": "gulp clean && gulp prePublishNonBundle", "compile": "tsc -watch -p ./", "compileApi": "node ./node_modules/typescript/lib/tsc.js -b ./pythonExtensionApi/tsconfig.json", "compiled": "deemon npm run compile", "kill-compiled": "deemon --kill npm run compile", "checkDependencies": "gulp checkDependencies", - "postinstall": "gulp installPythonLibs", + "install-pet": "ts-node scripts/install-pet.ts", + "postinstall": "gulp installPythonLibs && ts-node scripts/post-install.ts", "test": "node ./out/test/standardTest.js && node ./out/test/multiRootTest.js", "test:unittests": "mocha --config ./build/.mocha.unittests.json", "test:unittests:cover": "nyc --no-clean --nycrc-path ./build/.nycrc mocha --config ./build/.mocha.unittests.json", @@ -1950,5 +1951,10 @@ "webpack-require-from": "^1.8.6", "worker-loader": "^3.0.8", "yargs": "^15.3.1" - } + }, + "positron": { + "externalDependencies": { + "pet": "v2024.16.0" + } + } } diff --git a/extensions/positron-python/scripts/install-pet.ts b/extensions/positron-python/scripts/install-pet.ts new file mode 100644 index 00000000000..6726b2f99ff --- /dev/null +++ b/extensions/positron-python/scripts/install-pet.ts @@ -0,0 +1,382 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable global-require */ +/* eslint-disable arrow-body-style */ +/*--------------------------------------------------------------------------------------------- + * Copyright (C) 2025 Posit Software, PBC. All rights reserved. + * Licensed under the Elastic License 2.0. See LICENSE.txt for license information. + *--------------------------------------------------------------------------------------------*/ + +import decompress from 'decompress'; +import fs from 'fs'; +import https from 'https'; +import path from 'path'; +import { IncomingMessage } from 'http'; +import { promisify } from 'util'; +import { platform, arch } from 'os'; + +/** + * This script is a forked copy of the `install-kernel` script from the + * positron-r and positron-supervisor extension; it is responsible for downloading + * a copy of the Python Environment Tools repo and/or using a local version. + * + * In the future, we could consider some way to share this script between the + * extensions (note that some URLs, paths, and messages are different) or + * provide a shared library for downloading and installing binaries from Github + * releases. + */ + +// Promisify some filesystem functions. +const readFileAsync = promisify(fs.readFile); +const writeFileAsync = async (filePath: string, data: any) => { + const dir = path.dirname(filePath); + await fs.promises.mkdir(dir, { recursive: true }); + return fs.promises.writeFile(filePath, data); +}; +const existsAsync = promisify(fs.exists); + +// Create a promisified version of https.get. We can't use the built-in promisify +// because the callback doesn't follow the promise convention of (error, result). +export const httpsGetAsync = (opts: https.RequestOptions): Promise => { + return new Promise((resolve, reject) => { + const req = https.get(opts, resolve); + req.once('error', reject); + }); +}; + +/** + * Gets the version of Python Environment Tool specified in package.json. + * + * @returns The version of Python Environment Tool specified in package.json, or null if it cannot be determined. + */ +async function getVersionFromPackageJson(): Promise { + try { + const packageJson = JSON.parse(await readFileAsync('package.json', 'utf-8')); + return packageJson.positron.externalDependencies?.pet || null; + } catch (error) { + throw new Error(`Error reading package.json: ${error}`); + } +} + +/** + * Gets the version of Python Environment Tools installed locally by reading a `VERSION` file that's written + * by this `install-kernel` script. + * + * @returns The version of Python Environment Tool installed locally, or null if PET is not installed. + */ +async function getLocalPetVersion(): Promise { + const versionFile = path.join('resources', 'pet', 'VERSION'); + try { + const petExists = await existsAsync(versionFile); + if (!petExists) { + return null; + } + return readFileAsync(versionFile, 'utf-8'); + } catch (error) { + throw new Error(`Error determining Python Environment Tools version: ${error}`); + } +} + +/** + * Helper to execute a command and return the stdout and stderr. + * + * @param command The command to execute. + * @param stdin Optional stdin to pass to the command. + * @returns A promise that resolves with the stdout and stderr of the command. + */ +async function executeCommand(command: string, stdin?: string): Promise<{ stdout: string; stderr: string }> { + const { exec } = require('child_process'); + return new Promise((resolve, reject) => { + const process = exec(command, (error: any, stdout: string, stderr: string) => { + if (error) { + reject(error); + } else { + resolve({ stdout, stderr }); + } + }); + if (stdin) { + process.stdin.write(stdin); + process.stdin.end(); + } + }); +} + +/** + * Downloads the specified version of Python Environment Tool and replaces the local directory. + * + * @param version The version of Python Environment Tool to download. + * @param githubPat A Github Personal Access Token with the appropriate rights + * to download the release. + * @param gitCredential Whether the PAT originated from the `git credential` command. + */ +async function downloadAndReplacePet(version: string, githubPat: string, gitCredential: boolean): Promise { + try { + const headers: Record = { + Accept: 'application/vnd.github.v3.raw', // eslint-disable-line + 'User-Agent': 'positron-pet-downloader', // eslint-disable-line + }; + // If we have a githubPat, set it for better rate limiting. + if (githubPat) { + headers.Authorization = `token ${githubPat}`; + } + const requestOptions: https.RequestOptions = { + headers, + method: 'GET', + protocol: 'https:', + hostname: 'api.github.com', + path: `/repos/posit-dev/positron-pet-builds/releases`, + }; + + const response = (await httpsGetAsync(requestOptions as any)) as any; + + // Special handling for PATs originating from `git credential`. + if (gitCredential && response.statusCode === 200) { + // If the PAT hasn't been approved yet, do so now. This stores the credential in + // the system credential store (or whatever `git credential` uses on the system). + // Without this step, the user will be prompted for a username and password the + // next time they try to download PET. + const { stdout, stderr } = await executeCommand( + 'git credential approve', + `protocol=https\n` + + `host=github.com\n` + + `path=/repos/posit-dev/positron-pet-builds/releases\n` + + `username=\n` + + `password=${githubPat}\n`, + ); + console.log(stdout); + if (stderr) { + console.warn( + `Unable to approve PAT. You may be prompted for a username and ` + + `password the next time you download Python Environment Tools.`, + ); + console.error(stderr); + } + } else if (gitCredential && response.statusCode > 400 && response.statusCode < 500) { + // This handles the case wherein we got an invalid PAT from `git credential`. In this + // case we need to clean up the PAT from the credential store, so that we don't + // continue to use it. + const { stdout, stderr } = await executeCommand( + 'git credential reject', + `protocol=https\n` + + `host=github.com\n` + + `path=/repos/posit-dev/positron-pet-builds/releases\n` + + `username=\n` + + `password=${githubPat}\n`, + ); + console.log(stdout); + if (stderr) { + console.error(stderr); + throw new Error( + `The stored PAT returned by 'git credential' is invalid, but\n` + + `could not be removed. Please manually remove the PAT from 'git credential'\n` + + `for the host 'github.com'`, + ); + } + throw new Error( + `The PAT returned by 'git credential' is invalid. Python Environment Tool cannot be\n` + + `downloaded.\n\n` + + `Check to be sure that your Personal Access Token:\n` + + '- Has the `repo` scope\n' + + '- Is not expired\n' + + '- Has been authorized for the "posit-dev" organization on Github (Configure SSO)\n', + ); + } + + let responseBody = ''; + + response.on('data', (chunk: any) => { + responseBody += chunk; + }); + + response.on('end', async () => { + if (response.statusCode !== 200) { + throw new Error( + `Failed to download Python Environment Tool: HTTP ${response.statusCode}\n\n ${responseBody}`, + ); + } + const releases = JSON.parse(responseBody); + if (!Array.isArray(releases)) { + throw new Error(`Unexpected response from Github:\n\n ${responseBody}`); + } + const release = releases.find((asset: any) => asset.tag_name === version); + if (!release) { + throw new Error(`Could not find Python Environment Tool ${version} in the releases.`); + } + let os: string; + switch (platform()) { + case 'win32': + os = 'windows-x64'; + break; + case 'darwin': + os = 'darwin-universal'; + break; + case 'linux': + os = arch() === 'arm64' ? 'linux-arm64' : 'linux-x64'; + break; + default: { + throw new Error(`Unsupported platform ${platform()}.`); + } + } + + const assetName = `pet-${version}-${os}.zip`; + const asset = release.assets.find((asset: any) => asset.name === assetName); + if (!asset) { + throw new Error(`Could not find Python Environment Tool with asset name ${assetName} in the release.`); + } + console.log(`Downloading Python Environment Tool ${version} from ${asset.url}...`); + const url = new URL(asset.url); + // Reset the Accept header to download the asset. + headers.Accept = 'application/octet-stream'; + const requestOptions: https.RequestOptions = { + headers, + method: 'GET', + protocol: url.protocol, + hostname: url.hostname, + path: url.pathname, + }; + + let dlResponse = (await httpsGetAsync(requestOptions)) as any; + while (dlResponse.statusCode === 302) { + // Follow redirects. + dlResponse = (await httpsGetAsync(dlResponse.headers.location)) as any; + } + let binaryData = Buffer.alloc(0); + + dlResponse.on('data', (chunk: any) => { + binaryData = Buffer.concat([binaryData, chunk]); + }); + dlResponse.on('end', async () => { + const extensionParent = path.dirname(__dirname); + const petDir = path.join(extensionParent, 'python-env-tools'); + // Create the resources/pet directory if it doesn't exist. + if (!(await existsAsync(petDir))) { + await fs.promises.mkdir(petDir); + } + + console.log(`Successfully downloaded PET ${version} (${binaryData.length} bytes).`); + const zipFileDest = path.join(petDir, 'pet.zip'); + await writeFileAsync(zipFileDest, binaryData); + + await decompress(zipFileDest, petDir, { strip: 1 }).then(() => { + console.log(`Successfully unzipped Python Environment Tool ${version}.`); + }); + + // Clean up the zipfile. + await fs.promises.unlink(zipFileDest); + + // Write a VERSION file with the version number. + await writeFileAsync(path.join('resources', 'pet', 'VERSION'), version); + }); + }); + } catch (error) { + throw new Error(`Error downloading Pet: ${error}`); + } +} + +async function main() { + // Before we do any work, check to see if there is a locally built copy of + // the Python Environment Tool in the `pet / target` directory. If so, we'll assume + // that the user is a kernel developer and skip the download; this version + // will take precedence over any downloaded version. + const extensionParent = path.dirname(__dirname); + const petFolder = path.join(extensionParent, 'python-env-tools'); + if (fs.existsSync(petFolder)) { + console.log(`Using locally built PET in ${petFolder}.`); + return; + } + console.log(`No locally built Python Environment Tool found in ${petFolder}; checking downloaded version.`); + + const packageJsonVersion = await getVersionFromPackageJson(); + const localPetVersion = await getLocalPetVersion(); + + if (!packageJsonVersion) { + throw new Error('Could not determine PET version from package.json.'); + } + + console.log(`package.json version: ${packageJsonVersion} `); + console.log(`Downloaded PET version: ${localPetVersion || 'Not found'} `); + + if (packageJsonVersion === localPetVersion) { + console.log('Versions match. No action required.'); + return; + } + + // We need a Github Personal Access Token (PAT) to download PET. Because this is sensitive + // information, there are a lot of ways to set it. We try the following in order: + + // (1) The GITHUB_PAT environment variable. + // (2) The POSITRON_GITHUB_PAT environment variable. + // (3) The git config setting 'credential.https://api.github.com.token'. + // (4) The git credential store. + + // (1) Get the GITHUB_PAT from the environment. + let githubPat = process.env.GITHUB_PAT; + let gitCredential = false; + if (githubPat) { + console.log('Using Github PAT from environment variable GITHUB_PAT.'); + } else { + // (2) Try POSITRON_GITHUB_PAT (it's what the build script sets) + githubPat = process.env.POSITRON_GITHUB_PAT; + if (githubPat) { + console.log('Using Github PAT from environment variable POSITRON_GITHUB_PAT.'); + } + } + + // (3) If no GITHUB_PAT is set, try to get it from git config. This provides a + // convenient non-interactive way to set the PAT. + if (!githubPat) { + try { + const { stdout } = await executeCommand('git config --get credential.https://api.github.com.token'); + githubPat = stdout.trim(); + if (githubPat) { + console.log(`Using Github PAT from git config setting 'credential.https://api.github.com.token'.`); + } + } catch (error) { + // We don't care if this fails; we'll try `git credential` next. + } + } + + // (4) If no GITHUB_PAT is set, try to get it from git credential. + if (!githubPat) { + // Explain to the user what's about to happen. + console.log( + `Attempting to retrieve a Github Personal Access Token from git in order\n` + + `to download Python Environment Tool ${packageJsonVersion}. If you are prompted for a username and\n` + + `password, enter your Github username and a Personal Access Token with the\n` + + `'repo' scope. You can read about how to create a Personal Access Token here: \n` + + `\n` + + `https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens\n` + + `\n` + + `If you don't want to set up a Personal Access Token now, just press Enter twice to set \n` + + `a blank value for the password. Python Environment Tool will not be downloaded, but you will still be\n` + + `able to run Positron with Python support.\n` + + `\n` + + `You can set a PAT later by running yarn again and supplying the PAT at this prompt,\n` + + `or by running 'git config credential.https://api.github.com.token YOUR_GITHUB_PAT'\n`, + ); + const { stdout } = await executeCommand( + 'git credential fill', + `protocol=https\n host=github.com\n path=/repos/posit-dev/pet/releases\n`, + ); + + gitCredential = true; + // Extract the `password = ` line from the output. + const passwordLine = stdout.split('\n').find((line: string) => line.startsWith('password=')); + if (passwordLine) { + [, githubPat] = passwordLine.split('='); + console.log(`Using Github PAT returned from 'git credential'.`); + } + } + + if (!githubPat) { + throw new Error( + `No Github PAT was found. Unable to download PET ${packageJsonVersion}.\n` + + `You can still run Positron with Python Support.`, + ); + } + + await downloadAndReplacePet(packageJsonVersion, githubPat, gitCredential); +} + +main().catch((error) => { + console.error('An error occurred:', error); +}); diff --git a/extensions/positron-python/scripts/post-install.ts b/extensions/positron-python/scripts/post-install.ts new file mode 100644 index 00000000000..28f7754a318 --- /dev/null +++ b/extensions/positron-python/scripts/post-install.ts @@ -0,0 +1,11 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (C) 2025 Posit Software, PBC. All rights reserved. + * Licensed under the Elastic License 2.0. See LICENSE.txt for license information. + *--------------------------------------------------------------------------------------------*/ + +const { execSync } = require('child_process'); + +// Install or update the PET server binary +execSync('npm run install-pet', { + stdio: 'inherit', +}); diff --git a/extensions/positron-python/src/client/positron/runtime.ts b/extensions/positron-python/src/client/positron/runtime.ts index 189608aa46a..af30ee47431 100644 --- a/extensions/positron-python/src/client/positron/runtime.ts +++ b/extensions/positron-python/src/client/positron/runtime.ts @@ -67,7 +67,7 @@ export async function createPythonRuntimeMetadata( traceInfo(`createPythonRuntime: startup behavior: ${startupBehavior}`); // Get the Python version from sysVersion since only that includes alpha/beta info (e.g '3.12.0b1') - const pythonVersion = interpreter.sysVersion?.split(' ')[0] ?? '0.0.1'; + const pythonVersion = interpreter.sysVersion?.split(' ')[0] || interpreter.version?.raw || '0.0.1'; const envName = interpreter.envName ?? ''; const runtimeSource = interpreter.envType; diff --git a/extensions/positron-python/src/client/pythonEnvironments/base/locators/common/nativePythonFinder.ts b/extensions/positron-python/src/client/pythonEnvironments/base/locators/common/nativePythonFinder.ts index 2829c137608..579fa59bacb 100644 --- a/extensions/positron-python/src/client/pythonEnvironments/base/locators/common/nativePythonFinder.ts +++ b/extensions/positron-python/src/client/pythonEnvironments/base/locators/common/nativePythonFinder.ts @@ -25,8 +25,11 @@ import { untildify } from '../../../../common/helpers'; import { traceError } from '../../../../logging'; const PYTHON_ENV_TOOLS_PATH = isWindows() - ? path.join(EXTENSION_ROOT_DIR, 'python-env-tools', 'bin', 'pet.exe') - : path.join(EXTENSION_ROOT_DIR, 'python-env-tools', 'bin', 'pet'); + ? // --- Start Positron --- + // update path to reflect the location of the PET binary + path.join(EXTENSION_ROOT_DIR, 'python-env-tools', 'pet.exe') + : path.join(EXTENSION_ROOT_DIR, 'python-env-tools', 'pet'); +// --- End Positron --- export interface NativeEnvInfo { displayName?: string; diff --git a/extensions/positron-python/tsconfig.extension.json b/extensions/positron-python/tsconfig.extension.json index 2c38a7d8d83..b538bd8039f 100644 --- a/extensions/positron-python/tsconfig.extension.json +++ b/extensions/positron-python/tsconfig.extension.json @@ -31,6 +31,7 @@ "typings/vscode-proposed/*.d.ts", "types/*.d.ts", "positron-dts/positron.d.ts", - "positron-dts/ui-comm.d.ts" + "positron-dts/ui-comm.d.ts", + "python-env-tools/*" ] } diff --git a/extensions/positron-python/tsconfig.json b/extensions/positron-python/tsconfig.json index 8bbeda230f5..f5bb89a6233 100644 --- a/extensions/positron-python/tsconfig.json +++ b/extensions/positron-python/tsconfig.json @@ -15,7 +15,7 @@ "ES2019", "ES2020" ], - "types": ["reflect-metadata"], + "types": ["reflect-metadata", "node"], "sourceMap": true, "rootDir": "src", "experimentalDecorators": true, diff --git a/test/e2e/tests/top-action-bar/interpreter-pet.test.ts b/test/e2e/tests/top-action-bar/interpreter-pet.test.ts new file mode 100644 index 00000000000..d734954b124 --- /dev/null +++ b/test/e2e/tests/top-action-bar/interpreter-pet.test.ts @@ -0,0 +1,31 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (C) 2024 Posit Software, PBC. All rights reserved. + * Licensed under the Elastic License 2.0. See LICENSE.txt for license information. + *--------------------------------------------------------------------------------------------*/ + +import { test, tags } from '../_test.setup'; + +test.use({ + suiteId: __filename +}); + +const desiredPython = process.env.POSITRON_PY_VER_SEL!; + +test.describe('Top Action Bar - Interpreter Dropdown', { + tag: [tags.WEB, tags.CRITICAL, tags.WIN, tags.TOP_ACTION_BAR, tags.INTERPRETER] +}, () => { + + test.afterEach(async function ({ app }) { + await app.workbench.console.barClearButton.click(); + }); + + test.beforeAll(async function ({ userSettings }) { + await userSettings.set([['python.locator', 'native']]); + }); + + test('Python - starts and shows running [C707212]', async function ({ app }) { + await app.workbench.interpreter.selectInterpreter('Python', desiredPython); + await app.workbench.interpreter.verifyInterpreterIsSelected(desiredPython); + await app.workbench.interpreter.verifyInterpreterIsRunning(desiredPython); + }); +});