From 11827148746c35111676705731053ca9e7db2689 Mon Sep 17 00:00:00 2001 From: Glen Chung Date: Wed, 16 Oct 2024 17:23:16 -0700 Subject: [PATCH] Support A/B Compiler Arguments Traits - Depends on cpptools' update to provide ProjectContextResult. - Send "standardVersion" trait in completion prompt by default. - Added the following new traits - intelliSenseDisclaimer: compiler information disclaimer. - intelliSenseDisclaimerBeginning: to note the beginning of IntelliSense information. - compilerArguments: a list of compiler command arguments that could affect Copilot generating completions. - directAsks: direct asking Copilot to do something instead of providing an argument. - intelliSenseDisclaimerEnd: to note the end of IntelliSense information. - A/B Experimental flags - copilotcppTraits: deprecated, no longer used. - copilotcppExcludeTraits:: deprecated, no longer used. - copilotcppIncludeTraits: string array to include individual trait, i.e., compilerArguments. - copilotcppMsvcCompilerArgumentFilter: map of regex string to absence prompt for MSVC. - copilotcppClangCompilerArgumentFilter: map of regex string to absence prompt for Clang. - copilotcppGccCompilerArgumentFilter: map of regex string to absence prompt for GCC. - copilotcppCompilerArgumentDirectAskMap: map of argument to prompt. --- Extension/src/LanguageServer/client.ts | 34 +- .../src/LanguageServer/copilotProviders.ts | 140 +++++-- Extension/src/LanguageServer/lmTool.ts | 159 ++++++- Extension/src/LanguageServer/utils.ts | 6 + Extension/src/telemetry.ts | 4 +- .../tests/copilotProviders.test.ts | 325 +++++++++++---- .../SingleRootProject/tests/lmTool.test.ts | 388 ++++++++++++++++++ 7 files changed, 916 insertions(+), 140 deletions(-) create mode 100644 Extension/test/scenarios/SingleRootProject/tests/lmTool.test.ts diff --git a/Extension/src/LanguageServer/client.ts b/Extension/src/LanguageServer/client.ts index 6dd00d741b..82f50a6904 100644 --- a/Extension/src/LanguageServer/client.ts +++ b/Extension/src/LanguageServer/client.ts @@ -541,6 +541,19 @@ export interface ChatContextResult { targetArchitecture: string; } +export interface FileContextResult { + compilerArguments: string[]; +} + +export interface ProjectContextResult { + language: string; + standardVersion: string; + compiler: string; + targetPlatform: string; + targetArchitecture: string; + fileContext: FileContextResult; +} + // Requests const PreInitializationRequest: RequestType = new RequestType('cpptools/preinitialize'); const InitializationRequest: RequestType = new RequestType('cpptools/initialize'); @@ -560,7 +573,8 @@ const GoToDirectiveInGroupRequest: RequestType = new RequestType('cpptools/generateDoxygenComment'); const ChangeCppPropertiesRequest: RequestType = new RequestType('cpptools/didChangeCppProperties'); const IncludesRequest: RequestType = new RequestType('cpptools/getIncludes'); -const CppContextRequest: RequestType = new RequestType('cpptools/getChatContext'); +const CppContextRequest: RequestType = new RequestType('cpptools/getChatContext'); +const ProjectContextRequest: RequestType = new RequestType('cpptools/getProjectContext'); // Notifications to the server const DidOpenNotification: NotificationType = new NotificationType('textDocument/didOpen'); @@ -791,7 +805,8 @@ export interface Client { setShowConfigureIntelliSenseButton(show: boolean): void; addTrustedCompiler(path: string): Promise; getIncludes(maxDepth: number, token: vscode.CancellationToken): Promise; - getChatContext(token: vscode.CancellationToken): Promise; + getChatContext(uri: vscode.Uri, token: vscode.CancellationToken): Promise; + getProjectContext(uri: vscode.Uri, token: vscode.CancellationToken): Promise; } export function createClient(workspaceFolder?: vscode.WorkspaceFolder): Client { @@ -2220,10 +2235,18 @@ export class DefaultClient implements Client { () => this.languageClient.sendRequest(IncludesRequest, params, token), token); } - public async getChatContext(token: vscode.CancellationToken): Promise { + public async getChatContext(uri: vscode.Uri, token: vscode.CancellationToken): Promise { + const params: TextDocumentIdentifier = { uri: uri.toString() }; + await withCancellation(this.ready, token); + return DefaultClient.withLspCancellationHandling( + () => this.languageClient.sendRequest(CppContextRequest, params, token), token); + } + + public async getProjectContext(uri: vscode.Uri, token: vscode.CancellationToken): Promise { + const params: TextDocumentIdentifier = { uri: uri.toString() }; await withCancellation(this.ready, token); return DefaultClient.withLspCancellationHandling( - () => this.languageClient.sendRequest(CppContextRequest, null, token), token); + () => this.languageClient.sendRequest(ProjectContextRequest, params, token), token); } /** @@ -4129,5 +4152,6 @@ class NullClient implements Client { setShowConfigureIntelliSenseButton(show: boolean): void { } addTrustedCompiler(path: string): Promise { return Promise.resolve(); } getIncludes(maxDepth: number, token: vscode.CancellationToken): Promise { return Promise.resolve({} as GetIncludesResult); } - getChatContext(token: vscode.CancellationToken): Promise { return Promise.resolve({} as ChatContextResult); } + getChatContext(uri: vscode.Uri, token: vscode.CancellationToken): Promise { return Promise.resolve({} as ChatContextResult); } + getProjectContext(uri: vscode.Uri, token: vscode.CancellationToken): Promise { return Promise.resolve({} as ProjectContextResult); } } diff --git a/Extension/src/LanguageServer/copilotProviders.ts b/Extension/src/LanguageServer/copilotProviders.ts index f23554f76d..2433e16953 100644 --- a/Extension/src/LanguageServer/copilotProviders.ts +++ b/Extension/src/LanguageServer/copilotProviders.ts @@ -5,9 +5,13 @@ 'use strict'; import * as vscode from 'vscode'; +import { localize } from 'vscode-nls'; import * as util from '../common'; -import { ChatContextResult, GetIncludesResult } from './client'; +import * as logger from '../logger'; +import * as telemetry from '../telemetry'; +import { GetIncludesResult } from './client'; import { getActiveClient } from './extension'; +import { getCompilerArgumentFilterMap, getProjectContext } from './lmTool'; export interface CopilotTrait { name: string; @@ -34,35 +38,113 @@ export async function registerRelatedFilesProvider(): Promise { for (const languageId of ['c', 'cpp', 'cuda-cpp']) { api.registerRelatedFilesProvider( { extensionId: util.extensionContext.extension.id, languageId }, - async (_uri: vscode.Uri, context: { flags: Record }, token: vscode.CancellationToken) => { - - const getIncludesHandler = async () => (await getIncludesWithCancellation(1, token))?.includedFiles.map(file => vscode.Uri.file(file)) ?? []; - const getTraitsHandler = async () => { - const chatContext: ChatContextResult | undefined = await (getActiveClient().getChatContext(token) ?? undefined); - - if (!chatContext) { - return undefined; + async (uri: vscode.Uri, context: { flags: Record }, token: vscode.CancellationToken) => { + const start = performance.now(); + const telemetryProperties: Record = {}; + const telemetryMetrics: Record = {}; + try { + const getIncludesHandler = async () => (await getIncludesWithCancellation(1, token))?.includedFiles.map(file => vscode.Uri.file(file)) ?? []; + const getTraitsHandler = async () => { + const projectContext = await getProjectContext(uri, context, token); + + if (!projectContext) { + return undefined; + } + + let traits: CopilotTrait[] = [ + { name: "intelliSenseDisclaimer", value: '', includeInPrompt: true, promptTextOverride: `IntelliSense is currently configured with the following compiler information. It reflects the active configuration, and the project may have more configurations targeting different platforms.` }, + { name: "intelliSenseDisclaimerBeginning", value: '', includeInPrompt: true, promptTextOverride: `Beginning of IntelliSense information.` } + ]; + if (projectContext.language) { + traits.push({ name: "language", value: projectContext.language, includeInPrompt: true, promptTextOverride: `The language is ${projectContext.language}.` }); + } + if (projectContext.compiler) { + traits.push({ name: "compiler", value: projectContext.compiler, includeInPrompt: true, promptTextOverride: `This project compiles using ${projectContext.compiler}.` }); + } + if (projectContext.standardVersion) { + traits.push({ name: "standardVersion", value: projectContext.standardVersion, includeInPrompt: true, promptTextOverride: `This project uses the ${projectContext.standardVersion} language standard.` }); + } + if (projectContext.targetPlatform) { + traits.push({ name: "targetPlatform", value: projectContext.targetPlatform, includeInPrompt: true, promptTextOverride: `This build targets ${projectContext.targetPlatform}.` }); + } + if (projectContext.targetArchitecture) { + traits.push({ name: "targetArchitecture", value: projectContext.targetArchitecture, includeInPrompt: true, promptTextOverride: `This build targets ${projectContext.targetArchitecture}.` }); + } + + if (projectContext.compiler) { + // We will process compiler arguments based on copilotcppXXXCompilerArgumentFilters and copilotcppCompilerArgumentDirectAskMap feature flags. + // The copilotcppXXXCompilerArgumentFilters are maps. The keys are regex strings for filtering and the values, if not empty, + // are the prompt text to use when no arguments are found. + // copilotcppCompilerArgumentDirectAskMap map individual matched argument to a prompt text. + // For duplicate matches, the last one will be used. + const filterMap = getCompilerArgumentFilterMap(projectContext.compiler, context); + if (filterMap !== undefined) { + const directAskMap: Record = context.flags.copilotcppCompilerArgumentDirectAskMap ? JSON.parse(context.flags.copilotcppCompilerArgumentDirectAskMap as string) : {}; + let directAsks: string = ''; + const remainingArguments: string[] = []; + + for (const key in filterMap) { + if (!key) { + continue; + } + + const matchedArgument = projectContext.compilerArguments[key] as string; + if (matchedArgument?.length > 0) { + if (directAskMap[matchedArgument]) { + directAsks += `${directAskMap[matchedArgument]} `; + } else { + remainingArguments.push(matchedArgument); + } + } else if (filterMap[key]) { + // Use the prompt text in the absence of argument. + directAsks += `${filterMap[key]} `; + } + } + + if (remainingArguments.length > 0) { + const compilerArgumentsValue = remainingArguments.join(", "); + traits.push({ name: "compilerArguments", value: compilerArgumentsValue, includeInPrompt: true, promptTextOverride: `The compiler arguments include: ${compilerArgumentsValue}.` }); + } + + if (directAsks) { + traits.push({ name: "directAsks", value: directAsks, includeInPrompt: true, promptTextOverride: directAsks }); + } + } + } + + traits.push({ name: "intelliSenseDisclaimerEnd", value: '', includeInPrompt: true, promptTextOverride: `End of IntelliSense information.` }); + + const includeTraitsArray = context.flags.copilotcppIncludeTraits ? context.flags.copilotcppIncludeTraits as string[] : []; + const includeTraits = new Set(includeTraitsArray); + telemetryProperties["includeTraits"] = includeTraitsArray.join(','); + + // standardVersion trait is enabled by default. + traits = traits.filter(trait => includeTraits.has(trait.name) || trait.name === 'standardVersion'); + + telemetryProperties["traits"] = traits.map(trait => trait.name).join(','); + return traits.length > 0 ? traits : undefined; + }; + + // Call both handlers in parallel + const traitsPromise = getTraitsHandler(); + const includesPromise = getIncludesHandler(); + + return { entries: await includesPromise, traits: await traitsPromise }; + } + catch (exception) { + try { + const err: Error = exception as Error; + logger.getOutputChannelLogger().appendLine(localize("copilot.relatedfilesprovider.error", "Error while retrieving result. Reason: {0}", err.message)); } - - let traits: CopilotTrait[] = [ - { name: "language", value: chatContext.language, includeInPrompt: true, promptTextOverride: `The language is ${chatContext.language}.` }, - { name: "compiler", value: chatContext.compiler, includeInPrompt: true, promptTextOverride: `This project compiles using ${chatContext.compiler}.` }, - { name: "standardVersion", value: chatContext.standardVersion, includeInPrompt: true, promptTextOverride: `This project uses the ${chatContext.standardVersion} language standard.` }, - { name: "targetPlatform", value: chatContext.targetPlatform, includeInPrompt: true, promptTextOverride: `This build targets ${chatContext.targetPlatform}.` }, - { name: "targetArchitecture", value: chatContext.targetArchitecture, includeInPrompt: true, promptTextOverride: `This build targets ${chatContext.targetArchitecture}.` } - ]; - - const excludeTraits = context.flags.copilotcppExcludeTraits as string[] ?? []; - traits = traits.filter(trait => !excludeTraits.includes(trait.name)); - - return traits.length > 0 ? traits : undefined; - }; - - // Call both handlers in parallel - const traitsPromise = ((context.flags.copilotcppTraits as boolean) ?? false) ? getTraitsHandler() : Promise.resolve(undefined); - const includesPromise = getIncludesHandler(); - - return { entries: await includesPromise, traits: await traitsPromise }; + catch { + // Intentionally swallow any exception. + } + telemetryProperties["error"] = "true"; + throw exception; // Throw the exception for auto-retry. + } finally { + telemetryMetrics['duration'] = performance.now() - start; + telemetry.logCopilotEvent('RelatedFilesProvider', telemetryProperties, telemetryMetrics); + } } ); } diff --git a/Extension/src/LanguageServer/lmTool.ts b/Extension/src/LanguageServer/lmTool.ts index c3fad8b6eb..746ff3829f 100644 --- a/Extension/src/LanguageServer/lmTool.ts +++ b/Extension/src/LanguageServer/lmTool.ts @@ -9,9 +9,13 @@ import { localize } from 'vscode-nls'; import * as util from '../common'; import * as logger from '../logger'; import * as telemetry from '../telemetry'; -import { ChatContextResult } from './client'; +import { ChatContextResult, ProjectContextResult } from './client'; import { getClients } from './extension'; +import { checkDuration } from './utils'; +const MSVC: string = 'MSVC'; +const Clang: string = 'Clang'; +const GCC: string = 'GCC'; const knownValues: { [Property in keyof ChatContextResult]?: { [id: string]: string } } = { language: { 'c': 'C', @@ -19,9 +23,9 @@ const knownValues: { [Property in keyof ChatContextResult]?: { [id: string]: str 'cuda-cpp': 'CUDA C++' }, compiler: { - 'msvc': 'MSVC', - 'clang': 'Clang', - 'gcc': 'GCC' + 'msvc': MSVC, + 'clang': Clang, + 'gcc': GCC }, standardVersion: { 'c++98': 'C++98', @@ -44,6 +48,141 @@ const knownValues: { [Property in keyof ChatContextResult]?: { [id: string]: str } }; +function formatChatContext(context: ChatContextResult | ProjectContextResult): void { + type KnownKeys = 'language' | 'standardVersion' | 'compiler' | 'targetPlatform'; + for (const key in knownValues) { + const knownKey = key as KnownKeys; + if (knownValues[knownKey] && context[knownKey]) { + // Clear the value if it's not in the known values. + context[knownKey] = knownValues[knownKey][context[knownKey]] || ""; + } + } +} + +export interface ProjectContext { + language: string; + standardVersion: string; + compiler: string; + targetPlatform: string; + targetArchitecture: string; + compilerArguments: Record; +} + +export function getCompilerArgumentFilterMap(compiler: string, context: { flags: Record }): Record | undefined { + // The copilotcppXXXCompilerArgumentFilters are maps. + // The keys are regex strings and the values, if not empty, are the prompt text to use when no arguments are found. + let filterMap: Record | undefined; + try { + switch (compiler) { + case MSVC: + if (context.flags.copilotcppMsvcCompilerArgumentFilter !== undefined) { + filterMap = JSON.parse(context.flags.copilotcppMsvcCompilerArgumentFilter as string); + } + break; + case Clang: + if (context.flags.copilotcppClangCompilerArgumentFilter !== undefined) { + filterMap = JSON.parse(context.flags.copilotcppClangCompilerArgumentFilter as string); + } + break; + case GCC: + if (context.flags.copilotcppGccCompilerArgumentFilter !== undefined) { + filterMap = JSON.parse(context.flags.copilotcppGccCompilerArgumentFilter as string); + } + break; + } + } + catch { + // Intentionally swallow any exception. + } + return filterMap; +} + +function filterCompilerArguments(compiler: string, compilerArguments: string[], context: { flags: Record }, telemetryProperties: Record): Record { + const filterMap = getCompilerArgumentFilterMap(compiler, context); + if (filterMap === undefined) { + return {}; + } + + const combinedArguments = compilerArguments.join(' '); + const result: Record = {}; + const filteredCompilerArguments: string[] = []; + for (const key in filterMap) { + if (!key) { + continue; + } + const filter = new RegExp(key, 'g'); + const filtered = combinedArguments.match(filter); + if (filtered) { + filteredCompilerArguments.push(...filtered); + result[key] = filtered[filtered.length - 1]; + } + } + + if (filteredCompilerArguments.length > 0) { + // Telemetry to learn about the argument distribution. The filtered arguments are expected to be non-PII. + telemetryProperties["filteredCompilerArguments"] = filteredCompilerArguments.join(','); + telemetryProperties["filters"] = Object.keys(filterMap).filter(filter => !!filter).join(','); + } + + return result; +} + +export async function getProjectContext(uri: vscode.Uri, context: { flags: Record }, token: vscode.CancellationToken): Promise { + const telemetryProperties: Record = {}; + const telemetryMetrics: Record = {}; + try { + const projectContext = await checkDuration(async () => await getClients()?.ActiveClient?.getProjectContext(uri, token) ?? undefined); + telemetryMetrics["duration"] = projectContext.duration; + if (!projectContext.result) { + return undefined; + } + + formatChatContext(projectContext.result); + + const result: ProjectContext = { + language: projectContext.result.language, + standardVersion: projectContext.result.standardVersion, + compiler: projectContext.result.compiler, + targetPlatform: projectContext.result.targetPlatform, + targetArchitecture: projectContext.result.targetArchitecture, + compilerArguments: {} + }; + + if (projectContext.result.language) { + telemetryProperties["language"] = projectContext.result.language; + } + if (projectContext.result.compiler) { + telemetryProperties["compiler"] = projectContext.result.compiler; + } + if (projectContext.result.standardVersion) { + telemetryProperties["standardVersion"] = projectContext.result.standardVersion; + } + if (projectContext.result.targetPlatform) { + telemetryProperties["targetPlatform"] = projectContext.result.targetPlatform; + } + if (projectContext.result.targetArchitecture) { + telemetryProperties["targetArchitecture"] = projectContext.result.targetArchitecture; + } + telemetryMetrics["compilerArgumentCount"] = projectContext.result.fileContext.compilerArguments.length; + result.compilerArguments = filterCompilerArguments(projectContext.result.compiler, projectContext.result.fileContext.compilerArguments, context, telemetryProperties); + + return result; + } + catch (exception) { + try { + const err: Error = exception as Error; + logger.getOutputChannelLogger().appendLine(localize("copilot.projectcontext.error", "Error while retrieving the project context. Reason: {0}", err.message)); + } + catch { + // Intentionally swallow any exception. + } + telemetryProperties["error"] = "true"; + return undefined; + } finally { + telemetry.logCopilotEvent('ProjectContext', telemetryProperties, telemetryMetrics); + } +} + export class CppConfigurationLanguageModelTool implements vscode.LanguageModelTool { public async invoke(options: vscode.LanguageModelToolInvocationOptions, token: vscode.CancellationToken): Promise { return new vscode.LanguageModelToolResult([ @@ -58,18 +197,12 @@ export class CppConfigurationLanguageModelTool implements vscode.LanguageModelTo return 'The active document is not a C, C++, or CUDA file.'; } - const chatContext: ChatContextResult | undefined = await (getClients()?.ActiveClient?.getChatContext(token) ?? undefined); + const chatContext: ChatContextResult | undefined = await (getClients()?.ActiveClient?.getChatContext(currentDoc.uri, token) ?? undefined); if (!chatContext) { return 'No configuration information is available for the active document.'; } - for (const key in knownValues) { - const knownKey = key as keyof ChatContextResult; - if (knownValues[knownKey] && chatContext[knownKey]) { - // Clear the value if it's not in the known values. - chatContext[knownKey] = knownValues[knownKey][chatContext[knownKey]] || ""; - } - } + formatChatContext(chatContext); let contextString = ""; if (chatContext.language) { @@ -100,7 +233,7 @@ export class CppConfigurationLanguageModelTool implements vscode.LanguageModelTo telemetryProperties["error"] = "true"; return ""; } finally { - telemetry.logLanguageModelToolEvent('cpp', telemetryProperties); + telemetry.logCopilotEvent('Chat/Tool/cpp', telemetryProperties); } } diff --git a/Extension/src/LanguageServer/utils.ts b/Extension/src/LanguageServer/utils.ts index e8c8073ee1..3a46a486a9 100644 --- a/Extension/src/LanguageServer/utils.ts +++ b/Extension/src/LanguageServer/utils.ts @@ -112,3 +112,9 @@ export async function withCancellation(promise: Promise, token: vscode.Can }); }); } + +export async function checkDuration(fn: () => Promise): Promise<{ result: T; duration: number }> { + const start = performance.now(); + const result = await fn(); + return { result, duration: performance.now() - start }; +} diff --git a/Extension/src/telemetry.ts b/Extension/src/telemetry.ts index 600ffa4c45..7465de7df5 100644 --- a/Extension/src/telemetry.ts +++ b/Extension/src/telemetry.ts @@ -123,10 +123,10 @@ export function logLanguageServerEvent(eventName: string, properties?: Record, metrics?: Record): void { +export function logCopilotEvent(eventName: string, properties?: Record, metrics?: Record): void { const sendTelemetry = () => { if (experimentationTelemetry) { - const eventNamePrefix: string = "C_Cpp/Copilot/Chat/Tool/"; + const eventNamePrefix: string = "C_Cpp/Copilot/"; experimentationTelemetry.sendTelemetryEvent(eventNamePrefix + eventName, properties, metrics); } }; diff --git a/Extension/test/scenarios/SingleRootProject/tests/copilotProviders.test.ts b/Extension/test/scenarios/SingleRootProject/tests/copilotProviders.test.ts index 54052e122d..626ed287c0 100644 --- a/Extension/test/scenarios/SingleRootProject/tests/copilotProviders.test.ts +++ b/Extension/test/scenarios/SingleRootProject/tests/copilotProviders.test.ts @@ -9,11 +9,14 @@ import * as proxyquire from 'proxyquire'; import * as sinon from 'sinon'; import * as vscode from 'vscode'; import * as util from '../../../../src/common'; -import { ChatContextResult, DefaultClient, GetIncludesResult } from '../../../../src/LanguageServer/client'; +import { DefaultClient, GetIncludesResult } from '../../../../src/LanguageServer/client'; import { CopilotApi, CopilotTrait } from '../../../../src/LanguageServer/copilotProviders'; import * as extension from '../../../../src/LanguageServer/extension'; +import * as lmTool from '../../../../src/LanguageServer/lmTool'; +import { ProjectContext } from '../../../../src/LanguageServer/lmTool'; +import * as telemetry from '../../../../src/telemetry'; -describe('registerRelatedFilesProvider', () => { +describe('copilotProviders Tests', () => { let moduleUnderTest: any; let mockCopilotApi: sinon.SinonStubbedInstance; let getActiveClientStub: sinon.SinonStub; @@ -21,6 +24,7 @@ describe('registerRelatedFilesProvider', () => { let vscodeGetExtensionsStub: sinon.SinonStub; let callbackPromise: Promise<{ entries: vscode.Uri[]; traits?: CopilotTrait[] }> | undefined; let vscodeExtension: vscode.Extension; + let telemetryStub: sinon.SinonStub; const includedFiles = process.platform === 'win32' ? ['c:\\system\\include\\vector', 'c:\\system\\include\\string', 'C:\\src\\my_project\\foo.h'] : @@ -68,25 +72,28 @@ describe('registerRelatedFilesProvider', () => { activeClientStub = sinon.createStubInstance(DefaultClient); getActiveClientStub = sinon.stub(extension, 'getActiveClient').returns(activeClientStub); activeClientStub.getIncludes.resolves({ includedFiles: [] }); + telemetryStub = sinon.stub(telemetry, 'logCopilotEvent').returns(); }); afterEach(() => { sinon.restore(); }); - const arrange = ({ vscodeExtension, getIncludeFiles, chatContext, rootUri, flags }: - { vscodeExtension?: vscode.Extension; getIncludeFiles?: GetIncludesResult; chatContext?: ChatContextResult; rootUri?: vscode.Uri; flags?: Record } = - { vscodeExtension: undefined, getIncludeFiles: undefined, chatContext: undefined, rootUri: undefined, flags: {} } + const arrange = ({ vscodeExtension, getIncludeFiles, projectContext, rootUri, flags }: + { vscodeExtension?: vscode.Extension; getIncludeFiles?: GetIncludesResult; projectContext?: ProjectContext; rootUri?: vscode.Uri; flags?: Record } = + { vscodeExtension: undefined, getIncludeFiles: undefined, projectContext: undefined, rootUri: undefined, flags: {} } ) => { activeClientStub.getIncludes.resolves(getIncludeFiles); - activeClientStub.getChatContext.resolves(chatContext); + sinon.stub(lmTool, 'getProjectContext').resolves(projectContext); sinon.stub(activeClientStub, 'RootUri').get(() => rootUri); mockCopilotApi.registerRelatedFilesProvider.callsFake((_providerId: { extensionId: string; languageId: string }, callback: (uri: vscode.Uri, context: { flags: Record }, cancellationToken: vscode.CancellationToken) => Promise<{ entries: vscode.Uri[]; traits?: CopilotTrait[] }>) => { - const tokenSource = new vscode.CancellationTokenSource(); - try { - callbackPromise = callback(vscode.Uri.parse('file:///test-extension-path'), { flags: flags ?? {} }, tokenSource.token); - } finally { - tokenSource.dispose(); + if (_providerId.languageId === 'cpp') { + const tokenSource = new vscode.CancellationTokenSource(); + try { + callbackPromise = callback(vscode.Uri.parse('file:///test-extension-path'), { flags: flags ?? {} }, tokenSource.token); + } finally { + tokenSource.dispose(); + } } return { @@ -97,7 +104,7 @@ describe('registerRelatedFilesProvider', () => { vscodeGetExtensionsStub = sinon.stub(vscode.extensions, 'getExtension').returns(vscodeExtension); }; - it('should register provider', async () => { + it('should register provider.', async () => { arrange( { vscodeExtension: vscodeExtension } ); @@ -108,13 +115,13 @@ describe('registerRelatedFilesProvider', () => { ok(mockCopilotApi.registerRelatedFilesProvider.calledWithMatch(sinon.match({ extensionId: 'test-extension-id', languageId: sinon.match.in(['c', 'cpp', 'cuda-cpp']) })), 'registerRelatedFilesProvider should be called with the correct providerId and languageId'); }); - it('should not add #cpp traits when ChatContext isn\'t available.', async () => { + it('should not provide project context traits when project context isn\'t available.', async () => { arrange({ vscodeExtension: vscodeExtension, getIncludeFiles: { includedFiles }, - chatContext: undefined, + projectContext: undefined, rootUri, - flags: { copilotcppTraits: true } + flags: {} }); await moduleUnderTest.registerRelatedFilesProvider(); @@ -130,122 +137,258 @@ describe('registerRelatedFilesProvider', () => { ok(result.traits === undefined, 'result.traits should be undefined'); }); - it('should not add #cpp traits when copilotcppTraits flag is false.', async () => { + const projectContextNoArgs: ProjectContext = { + language: 'C++', + standardVersion: 'C++20', + compiler: 'MSVC', + targetPlatform: 'Windows', + targetArchitecture: 'x64', + compilerArguments: {} + }; + + it('provides standardVersion trait by default.', async () => { arrange({ vscodeExtension: vscodeExtension, getIncludeFiles: { includedFiles }, - chatContext: { - language: 'c++', - standardVersion: 'c++20', - compiler: 'msvc', - targetPlatform: 'windows', - targetArchitecture: 'x64' - }, + projectContext: projectContextNoArgs, rootUri, - flags: { copilotcppTraits: false } + flags: {} }); await moduleUnderTest.registerRelatedFilesProvider(); const result = await callbackPromise; - ok(vscodeGetExtensionsStub.calledOnce, 'vscode.extensions.getExtension should be called once'); - ok(mockCopilotApi.registerRelatedFilesProvider.calledWithMatch(sinon.match({ extensionId: 'test-extension-id', languageId: sinon.match.in(['c', 'cpp', 'cuda-cpp']) })), 'registerRelatedFilesProvider should be called with the correct providerId and languageId'); - ok(getActiveClientStub.callCount !== 0, 'getActiveClient should be called'); - ok(callbackPromise, 'callbackPromise should be defined'); ok(result, 'result should be defined'); - ok(result.entries.length === 1, 'result.entries should have 1 included file'); - ok(result.entries[0].toString() === expectedInclude, `result.entries should have "${expectedInclude}"`); - ok(result.traits === undefined, 'result.traits should be undefined'); + ok(result.traits, 'result.traits should be defined'); + ok(result.traits.length === 1, 'result.traits should have 1 trait'); + ok(result.traits.find((trait) => trait.name === 'standardVersion'), 'result.traits should have a standardVersion trait'); + ok(result.traits.find((trait) => trait.name === 'standardVersion')?.value === 'C++20', 'result.traits should have a standardVersion trait with value "C++20"'); + ok(result.traits.find((trait) => trait.name === 'standardVersion')?.includeInPrompt, 'result.traits should have a standardVersion trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'standardVersion')?.promptTextOverride === 'This project uses the C++20 language standard.', 'result.traits should have a standardVersion trait with promptTextOverride'); }); - it('should add #cpp traits when copilotcppTraits flag is true.', async () => { + it('provides traits per copilotcppIncludeTraits.', async () => { arrange({ vscodeExtension: vscodeExtension, getIncludeFiles: { includedFiles }, - chatContext: { - language: 'c++', - standardVersion: 'c++20', - compiler: 'msvc', - targetPlatform: 'windows', - targetArchitecture: 'x64' - }, + projectContext: projectContextNoArgs, rootUri, - flags: { copilotcppTraits: true } + flags: { copilotcppIncludeTraits: ['intelliSenseDisclaimer', 'intelliSenseDisclaimerBeginning', 'language', 'compiler', 'targetPlatform', 'targetArchitecture', 'intelliSenseDisclaimerEnd'] } }); await moduleUnderTest.registerRelatedFilesProvider(); const result = await callbackPromise; + ok(result, 'result should be defined'); + ok(result.traits, 'result.traits should be defined'); + ok(result.traits.length === 8, 'result.traits should have 8 traits if none are excluded'); + ok(result.traits.find((trait) => trait.name === 'intelliSenseDisclaimer'), 'result.traits should have a intellisense trait'); + ok(result.traits.find((trait) => trait.name === 'intelliSenseDisclaimer')?.includeInPrompt, 'result.traits should have a intellisense trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'intelliSenseDisclaimer')?.promptTextOverride === 'IntelliSense is currently configured with the following compiler information. It reflects the active configuration, and the project may have more configurations targeting different platforms.', 'result.traits should have a intellisense trait with promptTextOverride'); + ok(result.traits.find((trait) => trait.name === 'intelliSenseDisclaimerBeginning'), 'result.traits should have a intellisenseBegin trait'); + ok(result.traits.find((trait) => trait.name === 'intelliSenseDisclaimerBeginning')?.includeInPrompt, 'result.traits should have a intellisenseBegin trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'intelliSenseDisclaimerBeginning')?.promptTextOverride === 'Beginning of IntelliSense information.', 'result.traits should have a intellisenseBegin trait with promptTextOverride'); + ok(result.traits.find((trait) => trait.name === 'language'), 'result.traits should have a language trait'); + ok(result.traits.find((trait) => trait.name === 'language')?.value === 'C++', 'result.traits should have a language trait with value "C++"'); + ok(result.traits.find((trait) => trait.name === 'language')?.includeInPrompt, 'result.traits should have a language trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'language')?.promptTextOverride === 'The language is C++.', 'result.traits should have a language trait with promptTextOverride'); + ok(result.traits.find((trait) => trait.name === 'compiler'), 'result.traits should have a compiler trait'); + ok(result.traits.find((trait) => trait.name === 'compiler')?.value === 'MSVC', 'result.traits should have a compiler trait with value "MSVC"'); + ok(result.traits.find((trait) => trait.name === 'compiler')?.includeInPrompt, 'result.traits should have a compiler trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'compiler')?.promptTextOverride === 'This project compiles using MSVC.', 'result.traits should have a compiler trait with promptTextOverride'); + ok(result.traits.find((trait) => trait.name === 'standardVersion'), 'result.traits should have a standardVersion trait'); + ok(result.traits.find((trait) => trait.name === 'standardVersion')?.value === 'C++20', 'result.traits should have a standardVersion trait with value "C++20"'); + ok(result.traits.find((trait) => trait.name === 'standardVersion')?.includeInPrompt, 'result.traits should have a standardVersion trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'standardVersion')?.promptTextOverride === 'This project uses the C++20 language standard.', 'result.traits should have a standardVersion trait with promptTextOverride'); + ok(result.traits.find((trait) => trait.name === 'targetPlatform'), 'result.traits should have a targetPlatform trait'); + ok(result.traits.find((trait) => trait.name === 'targetPlatform')?.value === 'Windows', 'result.traits should have a targetPlatform trait with value "Windows"'); + ok(result.traits.find((trait) => trait.name === 'targetPlatform')?.includeInPrompt, 'result.traits should have a targetPlatform trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'targetPlatform')?.promptTextOverride === 'This build targets Windows.', 'result.traits should have a targetPlatform trait with promptTextOverride'); + ok(result.traits.find((trait) => trait.name === 'targetArchitecture'), 'result.traits should have a targetArchitecture trait'); + ok(result.traits.find((trait) => trait.name === 'targetArchitecture')?.value === 'x64', 'result.traits should have a targetArchitecture trait with value "x64"'); + ok(result.traits.find((trait) => trait.name === 'targetArchitecture')?.includeInPrompt, 'result.traits should have a targetArchitecture trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'targetArchitecture')?.promptTextOverride === 'This build targets x64.', 'result.traits should have a targetArchitecture trait with promptTextOverride'); + ok(result.traits.find((trait) => trait.name === 'intelliSenseDisclaimerEnd'), 'result.traits should have a intellisenseEnd trait'); + ok(result.traits.find((trait) => trait.name === 'intelliSenseDisclaimerEnd')?.includeInPrompt, 'result.traits should have a intellisenseEnd trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'intelliSenseDisclaimerEnd')?.promptTextOverride === 'End of IntelliSense information.', 'result.traits should have a intellisenseEnd trait with promptTextOverride'); + }); + + it('handles errors during provider registration.', async () => { + arrange({}); + + await moduleUnderTest.registerRelatedFilesProvider(); + ok(vscodeGetExtensionsStub.calledOnce, 'vscode.extensions.getExtension should be called once'); - ok(mockCopilotApi.registerRelatedFilesProvider.calledThrice, 'registerRelatedFilesProvider should be called three times'); - ok(mockCopilotApi.registerRelatedFilesProvider.calledWithMatch(sinon.match({ extensionId: 'test-extension-id', languageId: sinon.match.in(['c', 'cpp', 'cuda-cpp']) })), 'registerRelatedFilesProvider should be called with the correct providerId and languageId'); - ok(getActiveClientStub.callCount !== 0, 'getActiveClient should be called'); - ok(callbackPromise, 'callbackPromise should be defined'); + ok(mockCopilotApi.registerRelatedFilesProvider.notCalled, 'registerRelatedFilesProvider should not be called'); + }); + + const projectContext: ProjectContext = { + language: 'C++', + standardVersion: 'C++17', + compiler: 'MSVC', + targetPlatform: 'Windows', + targetArchitecture: 'x64', + compilerArguments: { "/std:c++\d+": '/std:c++17', "/GR-?": '/GR-', "/EH[ascr-]+": '/EHs-c-', "/await": '/await' } + }; + + it('provides compiler argument traits.', async () => { + arrange({ + vscodeExtension: vscodeExtension, + getIncludeFiles: { includedFiles: ['c:\\system\\include\\vector', 'c:\\system\\include\\string', 'C:\\src\\my_project\\foo.h'] }, + projectContext: projectContext, + rootUri: vscode.Uri.file('C:\\src\\my_project'), + flags: { + copilotcppIncludeTraits: ['compilerArguments'], + copilotcppMsvcCompilerArgumentFilter: '{"/std:c++\d+": "", "/GR-?": "", "/EH[ascr-]+": "", "/await": ""}' + } + }); + await moduleUnderTest.registerRelatedFilesProvider(); + + const result = await callbackPromise; + ok(result, 'result should be defined'); - ok(result.entries.length === 1, 'result.entries should have 1 included file'); - ok(result.entries[0].toString() === expectedInclude, `result.entries should have "${expectedInclude}"`); ok(result.traits, 'result.traits should be defined'); - ok(result.traits.length === 5, 'result.traits should have 5 traits'); - ok(result.traits[0].name === 'language', 'result.traits[0].name should be "language"'); - ok(result.traits[0].value === 'c++', 'result.traits[0].value should be "c++"'); - ok(result.traits[0].includeInPrompt, 'result.traits[0].includeInPrompt should be true'); - ok(result.traits[0].promptTextOverride === 'The language is c++.', 'result.traits[0].promptTextOverride should be "The language is c++."'); - ok(result.traits[1].name === 'compiler', 'result.traits[1].name should be "compiler"'); - ok(result.traits[1].value === 'msvc', 'result.traits[1].value should be "msvc"'); - ok(result.traits[1].includeInPrompt, 'result.traits[1].includeInPrompt should be true'); - ok(result.traits[1].promptTextOverride === 'This project compiles using msvc.', 'result.traits[1].promptTextOverride should be "This project compiles using msvc."'); - ok(result.traits[2].name === 'standardVersion', 'result.traits[2].name should be "standardVersion"'); - ok(result.traits[2].value === 'c++20', 'result.traits[2].value should be "c++20"'); - ok(result.traits[2].includeInPrompt, 'result.traits[2].includeInPrompt should be true'); - ok(result.traits[2].promptTextOverride === 'This project uses the c++20 language standard.', 'result.traits[2].promptTextOverride should be "This project uses the c++20 language standard."'); - ok(result.traits[3].name === 'targetPlatform', 'result.traits[3].name should be "targetPlatform"'); - ok(result.traits[3].value === 'windows', 'result.traits[3].value should be "windows"'); - ok(result.traits[3].includeInPrompt, 'result.traits[3].includeInPrompt should be true'); - ok(result.traits[3].promptTextOverride === 'This build targets windows.', 'result.traits[3].promptTextOverride should be "This build targets windows."'); - ok(result.traits[4].name === 'targetArchitecture', 'result.traits[4].name should be "targetArchitecture"'); - ok(result.traits[4].value === 'x64', 'result.traits[4].value should be "x64"'); - ok(result.traits[4].includeInPrompt, 'result.traits[4].includeInPrompt should be true'); - ok(result.traits[4].promptTextOverride === 'This build targets x64.', 'result.traits[4].promptTextOverride should be "This build targets x64."'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments'), 'result.traits should have a compiler arguments trait'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments')?.value === '/std:c++17, /GR-, /EHs-c-, /await', 'result.traits should have a compiler arguments trait with value "/std:c++17, /GR-, /EHs-c-, /await"'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments')?.includeInPrompt, 'result.traits should have a compiler arguments trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments')?.promptTextOverride === 'The compiler arguments include: /std:c++17, /GR-, /EHs-c-, /await.', 'result.traits should have a compiler arguments trait with promptTextOverride'); + ok(!result.traits.find((trait) => trait.name === 'directAsks'), 'result.traits should not have a direct asks trait'); }); - it('should exclude #cpp traits per copilotcppExcludeTraits.', async () => { - const excludeTraits = ['compiler', 'targetPlatform']; + it('provide direct ask traits of compiler arguments.', async () => { arrange({ vscodeExtension: vscodeExtension, - getIncludeFiles: { includedFiles }, - chatContext: { - language: 'c++', - standardVersion: 'c++20', - compiler: 'msvc', - targetPlatform: 'windows', - targetArchitecture: 'x64' + getIncludeFiles: { includedFiles: ['c:\\system\\include\\vector', 'c:\\system\\include\\string', 'C:\\src\\my_project\\foo.h'] }, + projectContext: projectContext, + rootUri: vscode.Uri.file('C:\\src\\my_project'), + flags: { + copilotcppIncludeTraits: ['directAsks', 'compilerArguments'], + copilotcppMsvcCompilerArgumentFilter: '{"/std:c++\d+": "", "/await": "", "/GR-?": "", "/EH[ascr-]+": ""}', + copilotcppCompilerArgumentDirectAskMap: '{"/GR-": "Do not generate code using RTTI keywords.", "/EHs-c-": "Do not generate code using exception handling keywords."}' + } + }); + await moduleUnderTest.registerRelatedFilesProvider(); + + const result = await callbackPromise; + + ok(result, 'result should be defined'); + ok(result.traits, 'result.traits should be defined'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments'), 'result.traits should have a compiler arguments trait'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments')?.value === '/std:c++17, /await', 'result.traits should have a compiler arguments trait with value "/std:c++17, /await"'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments')?.includeInPrompt, 'result.traits should have a compiler arguments trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments')?.promptTextOverride === 'The compiler arguments include: /std:c++17, /await.', 'result.traits should have a compiler arguments trait with promptTextOverride'); + ok(result.traits.find((trait) => trait.name === 'directAsks'), 'result.traits should have a direct asks trait'); + ok(result.traits.find((trait) => trait.name === 'directAsks')?.value === 'Do not generate code using RTTI keywords. Do not generate code using exception handling keywords. ', 'result.traits should have a direct asks value'); + ok(result.traits.find((trait) => trait.name === 'directAsks')?.includeInPrompt, 'result.traits should have a direct asks trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'directAsks')?.promptTextOverride === 'Do not generate code using RTTI keywords. Do not generate code using exception handling keywords. ', 'result.traits should have a direct ask trait with promptTextOverride'); + ok(telemetryStub.calledOnce, 'Telemetry should be called once'); + ok(telemetryStub.calledWithMatch('RelatedFilesProvider', sinon.match({ + "includeTraits": 'directAsks,compilerArguments', + 'traits': 'standardVersion,compilerArguments,directAsks' + }), sinon.match({ + 'duration': sinon.match.number + }))); + }); + + it('ignore compilerArguments trait if empty.', async () => { + arrange({ + vscodeExtension: vscodeExtension, + getIncludeFiles: { includedFiles: ['c:\\system\\include\\vector', 'c:\\system\\include\\string', 'C:\\src\\my_project\\foo.h'] }, + projectContext: projectContext, + rootUri: vscode.Uri.file('C:\\src\\my_project'), + flags: { + copilotcppIncludeTraits: ['directAsks', 'compilerArguments'], + copilotcppMsvcCompilerArgumentFilter: '{"/std:c++\d+": "", "/await": "", "/GR-?": "", "/EH[ascr-]+": ""}', + copilotcppCompilerArgumentDirectAskMap: '{"/GR-": "abc.", "/EHs-c-": "def.", "/std:c++17": "ghi.", "/await": "jkl."}' + } + }); + await moduleUnderTest.registerRelatedFilesProvider(); + + const result = await callbackPromise; + + ok(result, 'result should be defined'); + ok(result.traits, 'result.traits should be defined'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments') === undefined, 'result.traits should not have a compiler arguments trait'); + ok(result.traits.find((trait) => trait.name === 'directAsks'), 'result.traits should have a direct asks trait'); + ok(telemetryStub.calledOnce, 'Telemetry should be called once'); + ok(telemetryStub.calledWithMatch('RelatedFilesProvider', sinon.match({ + "includeTraits": 'directAsks,compilerArguments', + 'traits': 'standardVersion,directAsks' + }))); + }); + + it('uses only last argument from the duplicates.', async () => { + arrange({ + vscodeExtension: vscodeExtension, + getIncludeFiles: { includedFiles: ['c:\\system\\include\\vector', 'c:\\system\\include\\string', 'C:\\src\\my_project\\foo.h'] }, + projectContext: { + language: 'C++', + standardVersion: 'C++20', + compiler: 'MSVC', + targetPlatform: 'Windows', + targetArchitecture: 'x64', + compilerArguments: { "/std:c++\d+": '/std:c++20', "/await": '/await' } }, - rootUri, - flags: { copilotcppTraits: true, copilotcppExcludeTraits: excludeTraits } + rootUri: vscode.Uri.file('C:\\src\\my_project'), + flags: { + copilotcppIncludeTraits: ['compilerArguments'], + copilotcppMsvcCompilerArgumentFilter: '{"/std:c++\d+": "", "/await": ""}' + } }); await moduleUnderTest.registerRelatedFilesProvider(); const result = await callbackPromise; - ok(vscodeGetExtensionsStub.calledOnce, 'vscode.extensions.getExtension should be called once'); - ok(mockCopilotApi.registerRelatedFilesProvider.calledThrice, 'registerRelatedFilesProvider should be called three times'); - ok(mockCopilotApi.registerRelatedFilesProvider.calledWithMatch(sinon.match({ extensionId: 'test-extension-id', languageId: sinon.match.in(['c', 'cpp', 'cuda-cpp']) })), 'registerRelatedFilesProvider should be called with the correct providerId and languageId'); - ok(getActiveClientStub.callCount !== 0, 'getActiveClient should be called'); - ok(callbackPromise, 'callbackPromise should be defined'); ok(result, 'result should be defined'); - ok(result.entries.length === 1, 'result.entries should have 1 included file'); - ok(result.entries[0].toString() === expectedInclude, `result.entries should have "${expectedInclude}"`); ok(result.traits, 'result.traits should be defined'); - ok(result.traits.length === 3, 'result.traits should have 3 traits'); - ok(result.traits.filter(trait => excludeTraits.includes(trait.name)).length === 0, 'result.traits should not include excluded traits'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments'), 'result.traits should have a compiler arguments trait'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments')?.value === '/std:c++20, /await', 'result.traits should have a compiler arguments trait with value "/std:c++20, /await"'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments')?.includeInPrompt, 'result.traits should have a compiler arguments trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments')?.promptTextOverride === 'The compiler arguments include: /std:c++20, /await.', 'result.traits should have a compiler arguments trait with promptTextOverride'); }); - it('should handle errors during provider registration', async () => { - arrange({}); + it('provides direct asks trait for absence of arguments.', async () => { + arrange({ + vscodeExtension: vscodeExtension, + getIncludeFiles: { includedFiles: ['c:\\system\\include\\vector', 'c:\\system\\include\\string', 'C:\\src\\my_project\\foo.h'] }, + projectContext: projectContextNoArgs, + rootUri: vscode.Uri.file('C:\\src\\my_project'), + flags: { + copilotcppIncludeTraits: ['directAsks'], + copilotcppMsvcCompilerArgumentFilter: + '{"/FOO": "/FOO is not set.", "/BAR": "/BAR is not set."}' + } + }); + await moduleUnderTest.registerRelatedFilesProvider(); + + const result = await callbackPromise; + ok(result, 'result should be defined'); + ok(result.traits, 'result.traits should be defined'); + ok(result.traits.find((trait) => trait.name === 'directAsks'), 'result.traits should have a direct asks trait'); + ok(result.traits.find((trait) => trait.name === 'directAsks')?.value === '/FOO is not set. /BAR is not set. ', 'result.traits should have a direct asks value'); + ok(result.traits.find((trait) => trait.name === 'directAsks')?.includeInPrompt, 'result.traits should have a direct asks trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'directAsks')?.promptTextOverride === "/FOO is not set. /BAR is not set. ", 'result.traits should have a direct ask trait with promptTextOverride'); + }); + + it('does not accept empty regex.', async () => { + arrange({ + vscodeExtension: vscodeExtension, + getIncludeFiles: { includedFiles: ['c:\\system\\include\\vector', 'c:\\system\\include\\string', 'C:\\src\\my_project\\foo.h'] }, + projectContext: projectContextNoArgs, + rootUri: vscode.Uri.file('C:\\src\\my_project'), + flags: { + copilotcppIncludeTraits: ['directAsks'], + copilotcppMsvcCompilerArgumentFilter: + '{"": "Empty regex not allowed."}' + } + }); await moduleUnderTest.registerRelatedFilesProvider(); - ok(vscodeGetExtensionsStub.calledOnce, 'vscode.extensions.getExtension should be called once'); - ok(mockCopilotApi.registerRelatedFilesProvider.notCalled, 'registerRelatedFilesProvider should not be called'); + const result = await callbackPromise; + + ok(result, 'result should be defined'); + ok(result.traits, 'result.traits should be defined'); + ok(result.traits.find((trait) => trait.name === 'directAsks') === undefined, 'result.traits should not have a direct asks trait'); }); }); diff --git a/Extension/test/scenarios/SingleRootProject/tests/lmTool.test.ts b/Extension/test/scenarios/SingleRootProject/tests/lmTool.test.ts new file mode 100644 index 0000000000..f11c048e1f --- /dev/null +++ b/Extension/test/scenarios/SingleRootProject/tests/lmTool.test.ts @@ -0,0 +1,388 @@ +/* -------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All Rights Reserved. + * See 'LICENSE' in the project root for license information. + * ------------------------------------------------------------------------------------------ */ + +import { ok } from 'assert'; +import { afterEach, beforeEach, describe, it } from 'mocha'; +import * as sinon from 'sinon'; +import * as vscode from 'vscode'; +import * as util from '../../../../src/common'; +import { ChatContextResult, DefaultClient, ProjectContextResult } from '../../../../src/LanguageServer/client'; +import { ClientCollection } from '../../../../src/LanguageServer/clientCollection'; +import * as extension from '../../../../src/LanguageServer/extension'; +import { CppConfigurationLanguageModelTool, getProjectContext } from '../../../../src/LanguageServer/lmTool'; +import * as telemetry from '../../../../src/telemetry'; + +describe('CppConfigurationLanguageModelTool Tests', () => { + let mockLanguageModelToolInvocationOptions: sinon.SinonStubbedInstance>; + let activeClientStub: sinon.SinonStubbedInstance; + let mockTextEditorStub: MockTextEditor; + let mockTextDocumentStub: sinon.SinonStubbedInstance; + let telemetryStub: sinon.SinonStub; + + class MockLanguageModelToolInvocationOptions implements vscode.LanguageModelToolInvocationOptions { + tokenizationOptions?: vscode.LanguageModelToolTokenizationOptions | undefined; + toolInvocationToken: undefined; + input: undefined; + } + class MockTextEditor implements vscode.TextEditor { + constructor(selection: vscode.Selection, selections: readonly vscode.Selection[], visibleRanges: readonly vscode.Range[], options: vscode.TextEditorOptions, document: vscode.TextDocument, viewColumn?: vscode.ViewColumn) { + this.selection = selection; + this.selections = selections; + this.visibleRanges = visibleRanges; + this.options = options; + this.viewColumn = viewColumn; + this.document = document; + } + selection: vscode.Selection; + selections: readonly vscode.Selection[]; + visibleRanges: readonly vscode.Range[]; + options: vscode.TextEditorOptions; + viewColumn: vscode.ViewColumn | undefined; + edit(_callback: (editBuilder: vscode.TextEditorEdit) => void, _options?: { readonly undoStopBefore: boolean; readonly undoStopAfter: boolean }): Thenable { + throw new Error('Method not implemented.'); + } + insertSnippet(_snippet: vscode.SnippetString, _location?: vscode.Position | vscode.Range | readonly vscode.Position[] | readonly vscode.Range[], _options?: { readonly undoStopBefore: boolean; readonly undoStopAfter: boolean }): Thenable { + throw new Error('Method not implemented.'); + } + setDecorations(_decorationType: vscode.TextEditorDecorationType, _rangesOrOptions: readonly vscode.Range[] | readonly vscode.DecorationOptions[]): void { + throw new Error('Method not implemented.'); + } + revealRange(_range: vscode.Range, _revealType?: vscode.TextEditorRevealType): void { + throw new Error('Method not implemented.'); + } + show(_column?: vscode.ViewColumn): void { + throw new Error('Method not implemented.'); + } + hide(): void { + throw new Error('Method not implemented.'); + } + document: vscode.TextDocument; + } + class MockTextDocument implements vscode.TextDocument { + uri: vscode.Uri; + constructor(uri: vscode.Uri, fileName: string, isUntitled: boolean, languageId: string, version: number, isDirty: boolean, isClosed: boolean, eol: vscode.EndOfLine, lineCount: number) { + this.uri = uri; + this.fileName = fileName; + this.isUntitled = isUntitled; + this.languageId = languageId; + this.version = version; + this.isDirty = isDirty; + this.isClosed = isClosed; + this.eol = eol; + this.lineCount = lineCount; + } + fileName: string; + isUntitled: boolean; + languageId: string; + version: number; + isDirty: boolean; + isClosed: boolean; + save(): Thenable { + throw new Error('Method not implemented.'); + } + eol: vscode.EndOfLine; + lineCount: number; + + lineAt(line: number): vscode.TextLine; + // eslint-disable-next-line @typescript-eslint/unified-signatures + lineAt(position: vscode.Position): vscode.TextLine; + lineAt(_arg: number | vscode.Position): vscode.TextLine { + throw new Error('Method not implemented.'); + } + offsetAt(_position: vscode.Position): number { + throw new Error('Method not implemented.'); + } + positionAt(_offset: number): vscode.Position { + throw new Error('Method not implemented.'); + } + getText(_range?: vscode.Range): string { + throw new Error('Method not implemented.'); + } + getWordRangeAtPosition(_position: vscode.Position, _regex?: RegExp): vscode.Range | undefined { + throw new Error('Method not implemented.'); + } + validateRange(_range: vscode.Range): vscode.Range { + throw new Error('Method not implemented.'); + } + validatePosition(_position: vscode.Position): vscode.Position { + throw new Error('Method not implemented.'); + } + } + beforeEach(() => { + sinon.stub(util, 'extensionContext').value({ extension: { id: 'test-extension-id' } }); + + mockTextDocumentStub = sinon.createStubInstance(MockTextDocument); + mockTextEditorStub = new MockTextEditor(new vscode.Selection(0, 0, 0, 0), [], [], { tabSize: 4 }, mockTextDocumentStub); + mockLanguageModelToolInvocationOptions = new MockLanguageModelToolInvocationOptions(); + activeClientStub = sinon.createStubInstance(DefaultClient); + const clientsStub = sinon.createStubInstance(ClientCollection); + sinon.stub(extension, 'getClients').returns(clientsStub); + sinon.stub(clientsStub, 'ActiveClient').get(() => activeClientStub); + activeClientStub.getIncludes.resolves({ includedFiles: [] }); + sinon.stub(vscode.window, 'activeTextEditor').get(() => mockTextEditorStub); + telemetryStub = sinon.stub(telemetry, 'logCopilotEvent').returns(); + }); + + afterEach(() => { + sinon.restore(); + }); + + const arrangeChatContextFromCppTools = ({ chatContextFromCppTools, isCpp, isHeaderFile }: + { chatContextFromCppTools?: ChatContextResult; isCpp?: boolean; isHeaderFile?: boolean } = + { chatContextFromCppTools: undefined, isCpp: undefined, isHeaderFile: false } + ) => { + activeClientStub.getChatContext.resolves(chatContextFromCppTools); + sinon.stub(util, 'isCpp').returns(isCpp ?? true); + sinon.stub(util, 'isHeaderFile').returns(isHeaderFile ?? false); + }; + + const arrangeProjectContextFromCppTools = ({ projectContextFromCppTools, isCpp, isHeaderFile }: + { projectContextFromCppTools?: ProjectContextResult; isCpp?: boolean; isHeaderFile?: boolean } = + { projectContextFromCppTools: undefined, isCpp: undefined, isHeaderFile: false } + ) => { + activeClientStub.getProjectContext.resolves(projectContextFromCppTools); + sinon.stub(util, 'isCpp').returns(isCpp ?? true); + sinon.stub(util, 'isHeaderFile').returns(isHeaderFile ?? false); + }; + + it('should log telemetry and provide #cpp chat context.', async () => { + arrangeChatContextFromCppTools({ + chatContextFromCppTools: { + language: 'cpp', + standardVersion: 'c++20', + compiler: 'msvc', + targetPlatform: 'windows', + targetArchitecture: 'x64' + } + }); + + const result = await new CppConfigurationLanguageModelTool().invoke(mockLanguageModelToolInvocationOptions, new vscode.CancellationTokenSource().token); + + ok(telemetryStub.calledOnce, 'Telemetry should be called once'); + ok(telemetryStub.calledWithMatch('Chat/Tool/cpp', sinon.match({ + "language": 'C++', + "compiler": 'MSVC', + "standardVersion": 'C++20', + "targetPlatform": 'Windows', + "targetArchitecture": 'x64' + }))); + ok(result, 'result should not be undefined'); + const text = result.content[0] as vscode.LanguageModelTextPart; + ok(text, 'result should contain a text part'); + ok(text.value === 'The user is working on a C++ project. The project uses language version C++20. The project compiles using the MSVC compiler. The project targets the Windows platform. The project targets the x64 architecture. '); + }); + + const testGetProjectContext = async ({ + compiler, + expectedCompiler, + context, + compilerArguments: compilerArguments, + expectedCompilerArguments + }: { + compiler: string; + expectedCompiler: string; + context: { flags: Record }; + compilerArguments: string[]; + expectedCompilerArguments: Record; + }) => { + arrangeProjectContextFromCppTools({ + projectContextFromCppTools: { + language: 'cpp', + standardVersion: 'c++20', + compiler: compiler, + targetPlatform: 'windows', + targetArchitecture: 'x64', + fileContext: { + compilerArguments: compilerArguments + } + } + }); + + const result = await getProjectContext(mockTextDocumentStub.uri, context, new vscode.CancellationTokenSource().token); + + ok(result, 'result should not be undefined'); + ok(result.language === 'C++'); + ok(result.compiler === expectedCompiler); + ok(result.standardVersion === 'C++20'); + ok(result.targetPlatform === 'Windows'); + ok(result.targetArchitecture === 'x64'); + ok(JSON.stringify(result.compilerArguments) === JSON.stringify(expectedCompilerArguments)); + }; + + it('should provide compilerArguments based on copilotcppMsvcCompilerArgumentFilter.', async () => { + await testGetProjectContext({ + compiler: 'msvc', + expectedCompiler: 'MSVC', + context: { flags: { copilotcppMsvcCompilerArgumentFilter: '{"foo-?": ""}' } }, + compilerArguments: ['foo', 'bar', 'abc', 'foo-'], + expectedCompilerArguments: { 'foo-?': 'foo-' } + }); + }); + + it('should provide compilerArguments based on copilotcppClangCompilerArgumentFilter.', async () => { + await testGetProjectContext({ + compiler: 'clang', + expectedCompiler: 'Clang', + context: { flags: { copilotcppClangCompilerArgumentFilter: '{"foo": "", "bar": ""}' } }, + compilerArguments: ['foo', 'bar', 'abc'], + expectedCompilerArguments: { 'foo': 'foo', 'bar': 'bar' } + }); + }); + + it('should support spaces between argument and value.', async () => { + await testGetProjectContext({ + compiler: 'clang', + expectedCompiler: 'Clang', + context: { flags: { copilotcppClangCompilerArgumentFilter: '{"-std\\\\sc\\\\+\\\\+\\\\d+": ""}' } }, + compilerArguments: ['-std', 'c++17', '-std', 'foo', '-std', 'c++11', '-std', 'bar'], + expectedCompilerArguments: { '-std\\sc\\+\\+\\d+': '-std c++11' } + }); + }); + + it('should provide compilerArguments based on copilotcppGccCompilerArgumentFilter.', async () => { + await testGetProjectContext({ + compiler: 'gcc', + expectedCompiler: 'GCC', + context: { flags: { copilotcppGccCompilerArgumentFilter: '{"foo": "", "bar": ""}' } }, + compilerArguments: ['foo', 'bar', 'abc', 'bar', 'foo', 'bar'], + expectedCompilerArguments: { 'foo': 'foo', 'bar': 'bar' } + }); + }); + + it('should provide empty array for each regex if nothing matches.', async () => { + await testGetProjectContext({ + compiler: 'msvc', + expectedCompiler: 'MSVC', + context: { flags: { copilotcppMsvcCompilerArgumentFilter: '{"foo": "", "bar": ""}' } }, + compilerArguments: [], + expectedCompilerArguments: {} + }); + }); + + it('should filter out all compilerArguments by default.', async () => { + await testGetProjectContext({ + compiler: 'gcc', + expectedCompiler: 'GCC', + context: { flags: {} }, + compilerArguments: ['foo', 'bar'], + expectedCompilerArguments: {} + }); + }); + + it('should filter out all compilerArguments for empty regex.', async () => { + await testGetProjectContext({ + compiler: 'msvc', + expectedCompiler: 'MSVC', + context: { flags: { copilotcppMsvcCompilerArgumentFilter: '{"": ""}' } }, + compilerArguments: ['foo', 'bar'], + expectedCompilerArguments: {} + }); + }); + + it('should filter out all compilerArguments for empty copilotcppMsvcCompilerArgumentFilter.', async () => { + await testGetProjectContext({ + compiler: 'msvc', + expectedCompiler: 'MSVC', + context: { + flags: { + copilotcppMsvcCompilerArgumentFilter: '' + } + }, + compilerArguments: ['foo', 'bar'], + expectedCompilerArguments: {} + }); + }); + + it('should filter out all compilerArguments for invalid copilotcppMsvcCompilerArgumentFilter.', async () => { + await testGetProjectContext({ + compiler: 'msvc', + expectedCompiler: 'MSVC', + context: { + flags: { + copilotcppMsvcCompilerArgumentFilter: 'Not a map' + } + }, + compilerArguments: ['foo', 'bar'], + expectedCompilerArguments: {} + }); + }); + + it('should filter out all compilerArguments for unknown compilers.', async () => { + await testGetProjectContext({ + compiler: 'unknown', + expectedCompiler: '', + context: { + flags: { + copilotcppMsvcCompilerArgumentFilter: '{"(foo|bar)": ""}', + copilotcppClangCompilerArgumentFilter: '{"(foo|bar)": ""}', + copilotcppGccCompilerArgumentFilter: '{"(foo|bar)": ""}' + } + }, + compilerArguments: ['foo', 'bar'], + expectedCompilerArguments: {} + }); + }); + + it('should send telemetry.', async () => { + const input = { + compiler: 'msvc', + expectedCompiler: 'MSVC', + context: { flags: { copilotcppMsvcCompilerArgumentFilter: '{"foo-?": "", "": "", "bar": "", "xyz": ""}' } }, + compilerArguments: ['foo', 'bar', 'foo-', 'abc'], + expectedCompilerArguments: { 'foo-?': 'foo-', 'bar': 'bar' } + }; + await testGetProjectContext(input); + + ok(telemetryStub.calledOnce, 'Telemetry should be called once'); + ok(telemetryStub.calledWithMatch('ProjectContext', sinon.match({ + "language": 'C++', + "compiler": input.expectedCompiler, + "standardVersion": 'C++20', + "targetPlatform": 'Windows', + "targetArchitecture": 'x64', + "filteredCompilerArguments": "foo,foo-,bar", + "filters": "foo-?,bar,xyz" + }), sinon.match({ + "compilerArgumentCount": input.compilerArguments.length, + 'duration': sinon.match.number + }))); + }); + + it('should not send telemetry for unknown values', async () => { + arrangeProjectContextFromCppTools({ + projectContextFromCppTools: { + language: 'java', + standardVersion: 'gnu++17', + compiler: 'javac', + targetPlatform: 'arduino', + targetArchitecture: 'bar', + fileContext: { + compilerArguments: [] + } + } + }); + + const result = await getProjectContext(mockTextDocumentStub.uri, { flags: {} }, new vscode.CancellationTokenSource().token); + + ok(telemetryStub.calledOnce, 'Telemetry should be called once'); + ok(telemetryStub.calledWithMatch('ProjectContext', sinon.match({ + "targetArchitecture": 'bar' + }), sinon.match({ + "compilerArgumentCount": 0 + }))); + ok(telemetryStub.calledWithMatch('ProjectContext', sinon.match(property => + property['language'] === undefined && + property['compiler'] === undefined && + property['standardVersion'] === undefined && + property['targetPlatform'] === undefined))); + ok(result, 'result should not be undefined'); + ok(result.language === ''); + ok(result.compiler === ''); + ok(result.standardVersion === ''); + ok(result.targetPlatform === ''); + ok(result.targetArchitecture === 'bar'); + ok(Object.keys(result.compilerArguments).length === 0); + }); +});