Fix display and discovery for interpreters included via settings or found in /opt/python (#7015)

### Summary

- addresses #6270
- addresses #7058

#### Implementation

- set the environment `source` to `PythonEnvSource.UserSettings` for
interpreters specified via
`python.interpreters.include`/`python.interpreters.override` for both
the JS and Native locators
- ensure user settings-specified interpreters are tagged as Custom
instead of Unknown
- if an interpreter was added via the settings, but is a Venv, it will
show as a Venv instead of Custom; same goes for Uv/Conda/etc.
- ensure `/opt/python`-installed interpreters tagged as Global instead
of Unknown

#### Known Issues
See screenshots in the QA Notes section below for examples
- when using the Native locator, the Python: Select Interpreter
quickpick does not display as much information as when the JS locator is
used. In particular, text like "custom" is not included

### Release Notes

#### New Features

- Python interpreters installed at `/opt/python` or included via
interpreter settings are no longer tagged as "Unknown" (#6270)

#### Bug Fixes

- Fixed issue with missing or duplicated Python interpreters when
installed in `opt/python` or included via interpreter settings (#7058)


### QA Notes

@:interpreters

#### Setup
- ensure at least one python installation exists at `/opt/python`,
    - e.g. `/opt/python/3.10.4/bin/python`
- ensure at least one python installation is specified in
`python.interpreters.include`
- e.g. `~/scratch/3.10.4/bin/python` exists and the following is set:
    ```
    "python.interpreters.include": [
            "~/scratch"
    ],
    ```

#### Settings+UI Combinations

1. "python.locator": "js" and "console.multipleConsoleSessions": false

    ##### Interpreter Dropdown

<img width="386" alt="jslocator+nomulticonsole+interpreterdropdown"
src="https://github.com/user-attachments/assets/7b941ea0-1308-4012-8b81-64ac4003203f"
/>

##### Python: Select Interpreter Quickpick
(`console.multipleConsoleSessions` can be true or false)

<img width="607" alt="jslocator+pythonquickpick"
src="https://github.com/user-attachments/assets/27a74920-e6ff-4cc7-a54a-0d70815b9c4e"
/>

2. "python.locator": "js" and "console.multipleConsoleSessions": true

    ##### Multi Console Start Session Quickpick

<img width="612" alt="jslocator+multiconsole+startsessionquickpick"
src="https://github.com/user-attachments/assets/1bbc9f68-e2a5-4f21-a46b-6c796d5e6a6f"
/>

3. "python.locator": "native" and "console.multipleConsoleSessions":
false

    ##### Interpreter Dropdown

<img width="383"
alt="locatornative_muticonsolefalse_interpreterdropdown"
src="https://github.com/user-attachments/assets/c7626328-ce33-43d4-8129-63db24ceef46"
/>

##### Python: Select Interpreter Quickpick
(`console.multipleConsoleSessions` can be true or false)

<img width="609" alt="locatornative_quickpick"
src="https://github.com/user-attachments/assets/eb676ba6-9a15-4792-9382-e426c07207ab"
/>

4. "python.locator": "native" and "console.multipleConsoleSessions":
true

    ##### Multi Console Start Session Quickpick

<img width="610" alt="locatornative_muticonsoletrue"
src="https://github.com/user-attachments/assets/70aad74d-f4c9-47fe-a8ea-8a2ad437e8f9"
/>

This change is also reflected in the dropdown in the Project Wizard,
which retrieves the same information as the Interpreter Dropdown / Start
Session Quickpick:

<img width="707" alt="image"
src="https://github.com/user-attachments/assets/686ee754-0e53-4505-b586-fa6672a2e053"
/>

Author: sharon-wang

extensions/positron-python/src/client/common/stringUtils.ts

@@ -44,3 +44,14 @@ export function replaceAll(source: string, substr: string, newSubstr: string): s
 
     return source.replace(new RegExp(escapeRegExp(substr), 'g'), newSubstr);
 }
+
+// --- Start Positron ---
+/**
+ * Returns the shortest string from an array of strings.
+ * @param strings - The strings to compare.
+ * @returns The shortest string.
+ */
+export function getShortestString(strings: string[]): string {
+    return strings.reduce((a, b) => (a.length <= b.length ? a : b));
+}
+// --- End Positron ---

extensions/positron-python/src/client/interpreter/configuration/environmentTypeComparer.ts

@@ -331,6 +331,9 @@ function getPrioritizedEnvironmentType(): EnvironmentType[] {
         EnvironmentType.MicrosoftStore,
         EnvironmentType.Global,
         EnvironmentType.System,
+        // --- Start Positron ---
+        EnvironmentType.Custom,
+        // --- End Positron ---
         EnvironmentType.Unknown,
     ];
 }

extensions/positron-python/src/client/interpreter/configuration/interpreterSelector/commands/setInterpreter.ts

@@ -727,6 +727,10 @@ function getGroup(item: IInterpreterQuickPickItem, workspacePath?: string) {
         return EnvGroups.Workspace;
     }
     switch (item.interpreter.envType) {
+        // --- Start Positron ---
+        case EnvironmentType.Custom:
+            return EnvGroups.Global;
+        // --- End Positron ---
         case EnvironmentType.Global:
         case EnvironmentType.System:
         case EnvironmentType.Unknown:

extensions/positron-python/src/client/positron/interpreterSettings.ts

@@ -90,14 +90,14 @@ function getOverrideInterpreters(): string[] {
  * @returns List of custom environment directories to look for environments.
  */
 export function getCustomEnvDirs(): string[] {
-    const overrideDirs = getOverrideInterpreters();
-    if (overrideDirs.length > 0) {
-        return mapInterpretersToInstallDirs(overrideDirs);
+    const overrideInterpreters = getOverrideInterpreters();
+    if (overrideInterpreters.length > 0) {
+        return mapInterpretersToInstallDirs(overrideInterpreters);
     }
 
-    const includeDirs = getIncludedInterpreters();
-    if (includeDirs.length > 0) {
-        return mapInterpretersToInstallDirs(includeDirs);
+    const includedInterpreters = getIncludedInterpreters();
+    if (includedInterpreters.length > 0) {
+        return mapInterpretersToInstallDirs(includedInterpreters);
     }
 
     return [];
@@ -149,6 +149,19 @@ export function shouldIncludeInterpreter(interpreterPath: string): boolean {
     return true;
 }
 
+/**
+ * Check if an interpreter path is a custom environment.
+ * An interpreter is a custom environment if it exists in any of the custom search directories.
+ * @param interpreterPath The interpreter path to check
+ * @returns Whether the interpreter is a custom environment.
+ */
+export async function isCustomEnvironment(interpreterPath: string): Promise<boolean> {
+    const overrideInterpreters = getOverrideInterpreters();
+    const includeInterpreters = getIncludedInterpreters();
+    const customDirs = mapInterpretersToInstallDirs([...overrideInterpreters, ...includeInterpreters]);
+    return customDirs.some((dir) => isParentPath(interpreterPath, dir));
+}
+
 /**
  * Checks if an interpreter path is included in the settings.
  * @param interpreterPath The interpreter path to check
@@ -287,54 +300,52 @@ export function printInterpreterDebugInfo(interpreters: PythonEnvironment[]): vo
 /**
  * Maps a list of interpreter paths to their installation directories.
  * @param interpreterPaths List of interpreter paths to map to their installation directories.
- * @returns
+ * @returns List of unique installation directories.
  */
 function mapInterpretersToInstallDirs(interpreterPaths: string[]): string[] {
-    return interpreterPaths.map((interpreterPath) => {
-        // If it's already a directory, return it as-is.
-        if (isDirectorySync(interpreterPath)) {
-            return interpreterPath;
-        }
+    return Array.from(
+        new Set(
+            interpreterPaths.map((interpreterPath) => {
+                // If it's already a directory, return it as-is.
+                if (isDirectorySync(interpreterPath)) {
+                    return interpreterPath;
+                }
 
-        // If it's a file, we need to return the installation directory so that the Python locators can find it.
-        // e.g. ~/scratch/3.10.4/bin/python -> ~/scratch/3.10.4
-        // The locators expect a list of environment directories and don't seem to handle individual interpreter files.
-        // The installation directory is the grandparent directory, which upholds the JS locator's DEFAULT_SEARCH_DEPTH of 2
-        // see extensions/positron-python/src/client/pythonEnvironments/base/locators/lowLevel/userSpecifiedEnvLocator.ts
-        // The Native Python Locator seems to use the same search depth of 2, although not explicitly documented in the python extension.
-        let parentDir: string | undefined;
-        let installDir: string | undefined;
-        try {
-            // parentDir tends to be the bin directory, which is the parent of the interpreter file.
-            parentDir = path.dirname(interpreterPath);
-            // installDir tends to be the python version directory, AKA the installation directory, which is the parent of the bin directory.
-            installDir = path.dirname(parentDir);
-        } catch (error) {
-            traceError(
-                `[mapInterpretersToInterpreterDirs]: Failed to get install directory for Python interpreter ${interpreterPath}`,
-                error,
-            );
-        }
+                // If it's a file, we need to return the installation directory so that the Python locators can find it.
+                // e.g. ~/scratch/3.10.4/bin/python -> ~/scratch/3.10.4
+                let parentDir: string | undefined;
+                let installDir: string | undefined;
+                try {
+                    parentDir = path.dirname(interpreterPath);
+                    installDir = path.dirname(parentDir);
+                } catch (error) {
+                    traceError(
+                        `[mapInterpretersToInterpreterDirs]: Failed to get install directory for Python interpreter ${interpreterPath}`,
+                        error,
+                    );
+                }
 
-        if (installDir) {
-            traceVerbose(
-                `[mapInterpretersToInterpreterDirs]: Mapped ${interpreterPath} to installation directory ${installDir}`,
-            );
-            return installDir;
-        }
+                if (installDir) {
+                    traceVerbose(
+                        `[mapInterpretersToInterpreterDirs]: Mapped ${interpreterPath} to installation directory ${installDir}`,
+                    );
+                    return installDir;
+                }
 
-        if (parentDir) {
-            traceInfo(
-                `[mapInterpretersToInterpreterDirs]: Expected ${interpreterPath} to be located in a Python installation directory. It may not be discoverable.`,
-            );
-            return parentDir;
-        }
+                if (parentDir) {
+                    traceInfo(
+                        `[mapInterpretersToInterpreterDirs]: Expected ${interpreterPath} to be located in a Python installation directory. It may not be discoverable.`,
+                    );
+                    return parentDir;
+                }
 
-        traceInfo(
-            `[mapInterpretersToInterpreterDirs]: Unable to map ${interpreterPath} to an installation directory. It may not be discoverable.`,
-        );
-        return interpreterPath;
-    });
+                traceInfo(
+                    `[mapInterpretersToInterpreterDirs]: Unable to map ${interpreterPath} to an installation directory. It may not be discoverable.`,
+                );
+                return interpreterPath;
+            }),
+        ),
+    );
 }
 
 /**

extensions/positron-python/src/client/pythonEnvironments/base/info/index.ts

@@ -102,6 +102,12 @@ export enum PythonEnvSource {
      */
     WindowsRegistry = 'windows registry',
     // If source turns out to be useful we will expand this enum to contain more details sources.
+    // --- Start Positron ---
+    /**
+     * Environment was found via user settings
+     */
+    UserSettings = 'user settings',
+    // --- End Positron ---
 }
 
 /**

extensions/positron-python/src/client/pythonEnvironments/base/locators/common/nativePythonFinder.ts

@@ -30,6 +30,8 @@ import { traceError } from '../../../../logging';
 // --- Start Positron ---
 import { getCustomEnvDirs } from '../../../../positron/interpreterSettings';
 import { traceVerbose } from '../../../../logging';
+import { ADDITIONAL_POSIX_BIN_PATHS } from '../../../common/posixUtils';
+import { PythonEnvSource } from '../../info/index';
 // --- End Positron ---
 
 const PYTHON_ENV_TOOLS_PATH = isWindows()
@@ -53,6 +55,9 @@ export interface NativeEnvInfo {
     project?: string;
     arch?: 'x64' | 'x86';
     symlinks?: string[];
+    // --- Start Positron ---
+    source?: PythonEnvSource[];
+    // --- End Positron ---
 }
 
 export interface NativeEnvManagerInfo {
@@ -463,17 +468,17 @@ function getEnvironmentDirs(): string[] {
  * Gets the list of additional directories to add to environment directories.
  * @returns List of directories to add to environment directories.
  */
-function getAdditionalEnvDirs(): string[] {
+export function getAdditionalEnvDirs(): string[] {
     const additionalDirs: string[] = [];
 
     // Add additional dirs to search for Python environments on non-Windows platforms.
+    // See JS locator equivalent `getAdditionalPosixBinaries` in extensions/positron-python/src/client/pythonEnvironments/base/locators/lowLevel/posixKnownPathsLocator.ts
     if (!isWindows()) {
-        // /opt/python is a recommended Python installation location on Posit Workbench.
-        // see: https://docs.posit.co/ide/server-pro/python/installing_python.html
-        additionalDirs.push('/opt/python');
+        additionalDirs.push(...ADDITIONAL_POSIX_BIN_PATHS);
     }
 
     // Add user-specified Python search directories.
+    // See JS locator equivalent in extensions/positron-python/src/client/pythonEnvironments/base/locators/lowLevel/userSpecifiedEnvLocator.ts
     const customEnvDirs = getCustomEnvDirs();
     additionalDirs.push(...customEnvDirs);
 

extensions/positron-python/src/client/pythonEnvironments/base/locators/common/nativePythonUtils.ts

@@ -16,6 +16,7 @@ export enum NativePythonEnvironmentKind {
     Poetry = 'Poetry',
     // --- Start Positron ---
     Uv = 'Uv',
+    Custom = 'Custom',
     // --- End Positron ---
     MacPythonOrg = 'MacPythonOrg',
     MacCommandLineTools = 'MacCommandLineTools',
@@ -47,6 +48,9 @@ const mapping = new Map<NativePythonEnvironmentKind, PythonEnvKind>([
     [NativePythonEnvironmentKind.MacCommandLineTools, PythonEnvKind.System],
     [NativePythonEnvironmentKind.MacPythonOrg, PythonEnvKind.System],
     [NativePythonEnvironmentKind.MacXCode, PythonEnvKind.System],
+    // --- Start Positron ---
+    [NativePythonEnvironmentKind.Custom, PythonEnvKind.Custom],
+    // --- End Positron ---
 ]);
 
 export function categoryToKind(category?: NativePythonEnvironmentKind, logger?: LogOutputChannel): PythonEnvKind {

extensions/positron-python/src/client/pythonEnvironments/base/locators/lowLevel/posixKnownPathsLocator.ts

@@ -1,14 +1,11 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-// --- Start Positron ---
-import path from 'path';
-// --- End Positron ---
-
 import * as os from 'os';
 import { gte } from 'semver';
 import { PythonEnvKind, PythonEnvSource } from '../../info';
 import { BasicEnvInfo, IPythonEnvsIterator, Locator } from '../../locator';
+// eslint-disable-next-line import/no-duplicates
 import { commonPosixBinPaths, getPythonBinFromPosixPaths } from '../../../common/posixUtils';
 import { isPyenvShimDir } from '../../../common/environmentManagers/pyenv';
 import { getOSType, OSType } from '../../../../common/utils/platform';
@@ -17,7 +14,11 @@ import { traceError, traceInfo, traceVerbose } from '../../../../logging';
 import { StopWatch } from '../../../../common/utils/stopWatch';
 
 // --- Start Positron ---
-import { findInterpretersInDir, looksLikeBasicGlobalPython } from '../../../common/commonUtils';
+// eslint-disable-next-line import/order
+import path from 'path';
+// eslint-disable-next-line import/no-duplicates
+import { ADDITIONAL_POSIX_BIN_PATHS } from '../../../common/posixUtils';
+import { findInterpretersInDir } from '../../../common/commonUtils';
 // --- End Positron ---
 
 export class PosixKnownPathsLocator extends Locator<BasicEnvInfo> {
@@ -44,15 +45,19 @@ export class PosixKnownPathsLocator extends Locator<BasicEnvInfo> {
                 const knownDirs = (await commonPosixBinPaths()).filter((dirname) => !isPyenvShimDir(dirname));
 
                 // --- Start Positron ---
-                const additionalDirs = getAdditionalPosixDirs();
-                for await (const dir of additionalDirs) {
-                    knownDirs.push(dir);
-                }
+                const additionalDirs = await getAdditionalPosixBinDirs();
+                knownDirs.push(...additionalDirs);
                 // --- End Positron ---
 
                 let pythonBinaries = await getPythonBinFromPosixPaths(knownDirs);
                 traceVerbose(`Found ${pythonBinaries.length} python binaries in posix paths`);
 
+                // --- Start Positron ---
+                traceVerbose(
+                    `[PosixKnownPathsLocator] Python binaries found in posix paths: ${pythonBinaries.join(', ')}`,
+                );
+                // --- End Positron ---
+
                 // Filter out MacOS system installs of Python 2 if necessary.
                 if (isMacPython2Deprecated) {
                     pythonBinaries = pythonBinaries.filter((binary) => !isMacDefaultPythonPath(binary));
@@ -78,7 +83,7 @@ export class PosixKnownPathsLocator extends Locator<BasicEnvInfo> {
 
 // --- Start Positron ---
 /**
- * Gets additional directories to look for Python binaries on Posix systems.
+ * Gets additional Python bin dirs on Posix systems.
  *
  * For example, `/opt/python/3.10.4/bin` will be returned if the machine has Python 3.10.4 installed
  * in `/opt/python/3.10.4/bin/python`.
@@ -90,20 +95,15 @@ export class PosixKnownPathsLocator extends Locator<BasicEnvInfo> {
  *                    Default is 2 levels.
  * @returns Paths to Python binaries found in additional locations for Posix systems.
  */
-export async function* getAdditionalPosixDirs(searchDepth = 2): AsyncGenerator<string> {
-    const additionalLocations = [
-        // /opt/python is a recommended Python installation location on Posit Workbench.
-        // see: https://docs.posit.co/ide/server-pro/python/installing_python.html
-        '/opt/python',
-    ];
-    for (const location of additionalLocations) {
-        const additionalDirs = findInterpretersInDir(location, searchDepth);
-        for await (const dir of additionalDirs) {
-            const { filename } = dir;
-            if (await looksLikeBasicGlobalPython(filename)) {
-                yield path.dirname(filename);
-            }
+async function getAdditionalPosixBinDirs(searchDepth = 2): Promise<string[]> {
+    const additionalDirs = [];
+    for (const location of ADDITIONAL_POSIX_BIN_PATHS) {
+        const executables = findInterpretersInDir(location, searchDepth);
+        for await (const entry of executables) {
+            const { filename } = entry;
+            additionalDirs.push(path.dirname(filename));
         }
     }
+    return Array.from(new Set(additionalDirs));
 }
 // --- End Positron ---