Merge multiple compile_commands.json files

Extension/package.json

@@ -956,6 +956,38 @@
                             ]
                         },
                         "scope": "resource"
+                    },
+                    "C_Cpp.mergeCompileCommands": {
+                        "type": [
+                            "object",
+                            "null"
+                        ],
+                        "markdownDescription": "%c_cpp.configuration.mergeCompileCommands.markdownDescription%",
+                        "scope": "machine-overridable",
+                        "default": {
+                            "sources": [],
+                            "destination": ""
+                        },
+                        "required": [
+                            "sources",
+                            "destination"
+                        ],
+                        "properties": {
+                            "sources": {
+                                "type": "array",
+                                "items": {
+                                    "type": "string"
+                                },
+                                "uniqueItems": true,
+                                "markdownDescription": "%c_cpp.configuration.mergeCompileCommands.sources.markdownDescription%",
+                                "default": []
+                            },
+                            "destination": {
+                                "type": "string",
+                                "markdownDescription": "%c_cpp.configuration.mergeCompileCommands.destination.markdownDescription%",
+                                "default": ""
+                            }
+                        }
                     }
                 }
             },

Extension/package.nls.json

@@ -724,6 +724,24 @@
             "Markdown text between `` should not be translated or localized (they represent literal text) and the capitalization, spacing, and punctuation (including the ``) should not be altered."
         ]
     },
+    "c_cpp.configuration.mergeCompileCommands.markdownDescription": {
+        "message": "Collect and merge all `compile_commands.json` listed in `sources` and save to `destination`",
+        "comment": [
+            "Markdown text between `` should not be translated or localized (they represent literal text) and the capitalization, spacing, and punctuation (including the ``) should not be altered."
+        ]
+    },
+    "c_cpp.configuration.mergeCompileCommands.sources.markdownDescription": {
+        "message": "List of `compile_commands.json` files to merge. (glob patterns are not supported)",
+        "comment": [
+            "Markdown text between `` should not be translated or localized (they represent literal text) and the capitalization, spacing, and punctuation (including the ``) should not be altered."
+        ]
+    },
+    "c_cpp.configuration.mergeCompileCommands.destination.markdownDescription": {
+        "message": "The destination file to save the merged `compile_commands.json` to.",
+        "comment": [
+            "Markdown text between `` should not be translated or localized (they represent literal text) and the capitalization, spacing, and punctuation (including the ``) should not be altered."
+        ]
+    },
     "c_cpp.configuration.updateChannel.deprecationMessage": "This setting is deprecated. Pre-release extensions are now available via the Marketplace.",
     "c_cpp.configuration.default.dotConfig.markdownDescription": {
         "message": "The value to use in a configuration if `dotConfig` is not specified, or the value to insert if `${default}` is present in `dotConfig`.",

Extension/src/LanguageServer/client.ts

@@ -3933,6 +3933,7 @@ export class DefaultClient implements Client {
         if (this.innerLanguageClient !== undefined && this.configuration !== undefined) {
             void this.languageClient.sendNotification(IntervalTimerNotification).catch(logAndReturn.undefined);
             this.configuration.checkCppProperties();
+            //this.configuration.checkMergeCompileCommands();
             this.configuration.checkCompileCommands();
         }
     }

Extension/src/LanguageServer/configurations.ts

@@ -21,7 +21,7 @@ import * as telemetry from '../telemetry';
 import { DefaultClient } from './client';
 import { CustomConfigurationProviderCollection, getCustomConfigProviders } from './customProviders';
 import { PersistentFolderState } from './persistentState';
-import { CppSettings, OtherSettings } from './settings';
+import { CppSettings, MergeCompileCommands, OtherSettings } from './settings';
 import { SettingsPanel } from './settingsPanel';
 import { ConfigurationType, getUI } from './ui';
 import escapeStringRegExp = require('escape-string-regexp');
@@ -33,6 +33,14 @@ const configVersion: number = 4;
 
 type Environment = { [key: string]: string | string[] };
 
+interface CompileCommand {
+    directory: string;
+    file: string;
+    output?: string;
+    command: string; // The command string includes both commands and arguments (if any).
+    arguments?: string[];
+}
+
 // No properties are set in the config since we want to apply vscode settings first (if applicable).
 // That code won't trigger if another value is already set.
 // The property defaults are moved down to applyDefaultIncludePathsAndFrameworks.
@@ -160,6 +168,10 @@ export class CppProperties {
     private diagnosticCollection: vscode.DiagnosticCollection;
     private prevSquiggleMetrics: Map<string, { [key: string]: number }> = new Map<string, { [key: string]: number }>();
     private settingsPanel?: SettingsPanel;
+    private mergeCompileCommands?: MergeCompileCommands;
+    private mergeCompileCommandsFileWatchers: fs.FSWatcher[] = [];
+    private mergeCompileCommandsFileWatcherFallbackTime: Date = new Date(); // Used when file watching fails.
+    private mergeCompileCommandsFileWatcherTimer?: NodeJS.Timeout;
 
     // Any time the default settings are parsed and assigned to `this.configurationJson`,
     // we want to track when the default includes have been added to it.
@@ -1104,12 +1116,175 @@ export class CppProperties {
             }
         }
 
+        this.mergeCompileCommands = settings.mergeCompileCommands;
+        this.checkMergeCompileCommands();
+
         this.updateCompileCommandsFileWatchers();
         if (!this.configurationIncomplete) {
             this.onConfigurationsChanged();
         }
     }
 
+    private resolveMergeCompileCommandsPaths(): MergeCompileCommands | undefined {
+        if (!this.mergeCompileCommands) {
+            return undefined;
+        }
+        var result: MergeCompileCommands = { sources: [], destination: "" };
+        this.mergeCompileCommands.sources.forEach(path => { result.sources.push(this.resolvePath(path)); });
+        result.destination = this.resolvePath(this.mergeCompileCommands.destination);
+        return result;
+    }
+
+    public checkMergeCompileCommands(): void {
+        // Check for changes on settings changed / in case of file watcher failure.
+        // clear all file watchers
+        console.log("manually checking merge compile commands");
+        this.mergeCompileCommandsFileWatchers.forEach((watcher: fs.FSWatcher) => watcher.close());
+        this.mergeCompileCommandsFileWatchers = []; // reset it
+
+        const mergeCompileCommands: MergeCompileCommands | undefined = this.resolveMergeCompileCommandsPaths();
+        if (mergeCompileCommands == undefined) {
+            console.log("merge compile commands not found, returning");
+            return;
+        }
+        // first, check if the destination file doesn't exist,
+        // if so, try to create it
+        if (!fs.existsSync(mergeCompileCommands.destination)) {
+            console.log("destination file not found, trying to create it");
+            this.onMergeCompileCommandsFiles();
+            return;
+        }
+
+        // check if any of the sources changed since last time we manually checked
+        var shouldMerge: boolean = false;
+        mergeCompileCommands.sources.forEach((source) => {
+            try {
+                const stats = fs.statSync(source);
+                if (stats.mtime > this.mergeCompileCommandsFileWatcherFallbackTime) {
+                    // source file changed since last time we manually checked
+                    console.log(source, " is newer than last time we manually checked");
+                    this.mergeCompileCommandsFileWatcherFallbackTime = new Date();
+                    shouldMerge = true;
+                }
+                else {
+                    console.log(source, " is older than last time we manually checked");
+                }
+            }
+            catch (err: any) {
+                if (err.code === "ENOENT") {
+                    // source file doesn't exist 
+                    console.log(source, " doesn't exist");
+                }
+            }
+        });
+        if (shouldMerge) {
+            this.onMergeCompileCommandsFiles();
+            return;
+        }
+        this.updateMergeCompileCommandsFileWatchers();
+    }
+
+    public updateMergeCompileCommandsFileWatchers(): void {
+        this.mergeCompileCommandsFileWatchers.forEach((watcher: fs.FSWatcher) => watcher.close());
+        this.mergeCompileCommandsFileWatchers = []; // reset it
+        const mergeCompileCommands = this.resolveMergeCompileCommandsPaths();
+        if (mergeCompileCommands &&
+            mergeCompileCommands.sources.length > 0 &&
+            mergeCompileCommands.destination.length > 0) {
+            const filePaths: Set<string> = new Set<string>();
+            mergeCompileCommands.sources.forEach((source: string) => {
+                filePaths.add(source);
+            });
+            try {
+                filePaths.forEach((path: string) => {
+                    this.mergeCompileCommandsFileWatchers.push(fs.watch(path, () => {
+                        console.log(path, " file watcher triggered");
+                        // on file changed:
+                        // - clear the old timer if it exists
+                        if (this.mergeCompileCommandsFileWatcherTimer) {
+                            clearInterval(this.mergeCompileCommandsFileWatcherTimer);
+                        }
+                        // - set a new timer to wait 1 second before processing the changes
+                        this.mergeCompileCommandsFileWatcherTimer = setTimeout(() => {
+                            // - merge all the compile_commands.json files even if only one changed
+                            this.onMergeCompileCommandsFiles();
+                            // - clear the timer
+                            if (this.mergeCompileCommandsFileWatcherTimer) {
+                                clearInterval(this.mergeCompileCommandsFileWatcherTimer);
+                            }
+                            this.mergeCompileCommandsFileWatcherTimer = undefined;
+                        }, 1000);
+                    }));
+                });
+            } catch (e: any) {
+                if (e.code == "ENOENT") {
+                    console.log("file doesn't exist: ", path);
+                    // TODO: add to a low cycle periodic check list until it exists
+                } else {
+                    console.log("file watcher error: ", e.code)
+                    console.log("file watcher limit reached, trying to manually check for changes");
+                    this.checkMergeCompileCommands();
+                }
+            }
+        }
+    }
+
+    private onMergeCompileCommandsFiles(): void {
+        console.log("trying to merge compile commands");
+        const mergeCompileCommands: MergeCompileCommands | undefined = this.resolveMergeCompileCommandsPaths();
+        if (mergeCompileCommands === undefined ||
+            mergeCompileCommands.destination.length === 0 ||
+            mergeCompileCommands.sources.length === 0) {
+            console.log("merge compile commands settings are null, returning");
+            return;
+        }
+
+        var dst = mergeCompileCommands.destination;
+        const dst_dir = path.dirname(dst);
+        try {
+            fs.mkdirSync(dst_dir, { recursive: true });
+        }
+        catch (err: any) {
+            const failedToCreate: string = localize("failed.to.create.config.folder", 'Failed to create "{0}"', dst_dir);
+            void vscode.window.showErrorMessage(`${failedToCreate}: ${err.message}`);
+            return;
+        }
+        if (fs.existsSync(dst) && fs.statSync(dst).isDirectory()) {
+            dst = path.join(dst, "merged_compile_commands.json");
+        }
+
+        // merge all the json files
+        const mergedCompiledCommands: CompileCommand[] = [];
+        mergeCompileCommands.sources.forEach(src => {
+            try {
+                const fileData = fs.readFileSync(src);
+                const fileCommands = JSON.parse(fileData.toString()) as CompileCommand[];
+                mergedCompiledCommands.push(...fileCommands);
+            }
+            catch (err: any) {
+                const failedToRead: string = localize("failed.to.read.compile.commands", 'Failed to read "{0}"', src);
+                void vscode.window.showErrorMessage(`${failedToRead}: ${err.message}`);
+                // NOTE: we don't return here but try to merge the rest of the files
+            }
+        });
+
+        // try to save to the dst file
+        try {
+            const output = JSON.stringify(mergedCompiledCommands, null, 4);
+            fs.writeFileSync(dst, output);
+        }
+        catch (e: any) {
+            const failedToWrite: string = localize("failed.to.write.compile.commands", 'Failed to write "{0}"', dst);
+            void vscode.window.showErrorMessage(`${failedToWrite}: ${e.message}`);
+            return;
+        }
+
+        // if we got here, the merge was successful
+        // set up file watchers again
+        console.log("merge successful");
+        this.updateMergeCompileCommandsFileWatchers();
+    }
+
     private compileCommandsFileWatcherTimer?: NodeJS.Timeout;
     private compileCommandsFileWatcherFiles: Set<string> = new Set<string>();
 
@@ -2329,6 +2504,9 @@ export class CppProperties {
         this.compileCommandsFileWatchers.forEach((watcher: fs.FSWatcher) => watcher.close());
         this.compileCommandsFileWatchers = []; // reset it
 
+        this.mergeCompileCommandsFileWatchers.forEach((watcher: fs.FSWatcher) => watcher.close());
+        this.mergeCompileCommandsFileWatchers = []; // reset it
+
         this.diagnosticCollection.dispose();
     }
 }

Extension/src/LanguageServer/settings.ts

@@ -33,6 +33,11 @@ export interface Associations {
     [key: string]: string;
 }
 
+export interface MergeCompileCommands {
+    sources: string[];
+    destination: string;
+}
+
 // Settings that can be undefined have default values assigned in the native code or are meant to return undefined.
 export interface WorkspaceFolderSettingsParams {
     uri: string | undefined;
@@ -403,6 +408,7 @@ export class CppSettings extends Settings {
     public get defaultForcedInclude(): string[] | undefined { return this.getArrayOfStringsWithUndefinedDefault("default.forcedInclude"); }
     public get defaultIntelliSenseMode(): string | undefined { return this.getAsStringOrUndefined("default.intelliSenseMode"); }
     public get defaultCompilerPath(): string | null { return this.getAsString("default.compilerPath", true); }
+    public get mergeCompileCommands(): MergeCompileCommands | undefined { return this.getMergeCompileCommands(); }
 
     public set defaultCompilerPath(value: string) {
         const defaultCompilerPathStr: string = "default.compilerPath";
@@ -703,6 +709,14 @@ export class CppSettings extends Settings {
         return setting.default as Associations;
     }
 
+    private getMergeCompileCommands(): MergeCompileCommands | undefined {
+        const value: any = super.Section.get<any>("mergeCompileCommands");
+        //const setting = getRawSetting("C_Cpp.mergeCompileCommands", true);
+        // todo: add some validation here
+
+        return value as MergeCompileCommands;
+    }
+
     // Checks a given enum value against a list of valid enum values from package.json.
     private isValidEnum(enumDescription: any, value: any): value is string {
         if (isString(value) && isArray(enumDescription) && enumDescription.length > 0) {