[http-client-csharp] the post processor should always keep customized code as root documents (#5481)

Fixes #5441 
Previously in our generator, we have two instances of
`GeneratedCodeWorkspace`: one for the project that is being generated
right now (the generated code project), one for the existing part of the
generated library (the customized code project).
This leads to an issue that in the post processor, only the generated
documents are passed into the post processor, therefore the post
processor actually does not know about the existence of the customized
files.

This is not very correct because the generated files must need those
customized files to work properly therefore they should be in the same
project.
This PR refactors this part to change the structure of
`GeneratedCodeWorkspace`: now we only create one instance of
`GeneratedCodeWorkspace`, and the project inside it will be initialized
with shared files and all the customized files in it.

In this way, when we get to the post processor, it should be able to
access all the necessary documents.

---------

Co-authored-by: Wei Hu <[email protected]>

Author: ArcturusZhang

packages/http-client-csharp/generator/Microsoft.Generator.CSharp/src/CSharpGen.cs

@@ -29,7 +29,7 @@ public async Task ExecuteAsync()
             var generatedTestOutputPath = CodeModelPlugin.Instance.Configuration.TestGeneratedDirectory;
 
             GeneratedCodeWorkspace workspace = await GeneratedCodeWorkspace.Create();
-            await CodeModelPlugin.Instance.InitializeSourceInputModelAsync();
+            CodeModelPlugin.Instance.SourceInputModel = new SourceInputModel(await workspace.GetCompilationAsync());
 
             var output = CodeModelPlugin.Instance.OutputLibrary;
             Directory.CreateDirectory(Path.Combine(generatedSourceOutputPath, "Models"));

packages/http-client-csharp/generator/Microsoft.Generator.CSharp/src/CodeModelPlugin.cs

@@ -4,7 +4,6 @@
 using System;
 using System.Collections.Generic;
 using System.ComponentModel.Composition;
-using System.Threading.Tasks;
 using Microsoft.CodeAnalysis;
 using Microsoft.Generator.CSharp.Input;
 using Microsoft.Generator.CSharp.Primitives;
@@ -61,7 +60,17 @@ protected CodeModelPlugin()
 
         // Extensibility points to be implemented by a plugin
         public virtual TypeFactory TypeFactory { get; }
-        public virtual SourceInputModel SourceInputModel => _sourceInputModel ?? throw new InvalidOperationException($"SourceInputModel has not been initialized yet");
+
+        private SourceInputModel? _sourceInputModel;
+        public virtual SourceInputModel SourceInputModel
+        {
+            get => _sourceInputModel ?? throw new InvalidOperationException($"SourceInputModel has not been initialized yet");
+            internal set
+            {
+                _sourceInputModel = value;
+            }
+        }
+
         public virtual string LicenseString => string.Empty;
         public virtual OutputLibrary OutputLibrary { get; } = new();
         public virtual InputLibrary InputLibrary => _inputLibrary;
@@ -89,14 +98,6 @@ public void AddSharedSourceDirectory(string sharedSourceDirectory)
             _sharedSourceDirectories.Add(sharedSourceDirectory);
         }
 
-        private SourceInputModel? _sourceInputModel;
-
-        internal async Task InitializeSourceInputModelAsync()
-        {
-            GeneratedCodeWorkspace existingCode = GeneratedCodeWorkspace.CreateExistingCodeProject([Instance.Configuration.ProjectDirectory], Instance.Configuration.ProjectGeneratedDirectory);
-            _sourceInputModel =  new SourceInputModel(await existingCode.GetCompilationAsync());
-        }
-
         internal HashSet<string> TypesToKeep { get; } = new();
         //TODO consider using TypeProvider so we can have a fully qualified name to filter on
         //https://github.com/microsoft/typespec/issues/4418

packages/http-client-csharp/generator/Microsoft.Generator.CSharp/src/PostProcessing/GeneratedCodeWorkspace.cs

@@ -34,7 +34,6 @@ internal class GeneratedCodeWorkspace
         private static readonly string[] _sharedFolders = [SharedFolder];
 
         private Project _project;
-        private Compilation? _compilation;
         private Dictionary<string, string> PlainFiles { get; }
 
         private GeneratedCodeWorkspace(Project generatedCodeProject)
@@ -107,7 +106,7 @@ public async Task AddGeneratedFile(CodeFile codefile)
         private async Task<Document> ProcessDocument(Document document)
         {
             var syntaxTree = await document.GetSyntaxTreeAsync();
-            var compilation = await GetProjectCompilationAsync();
+            var compilation = await GetCompilationAsync();
             if (syntaxTree != null)
             {
                 var semanticModel = compilation.GetSemanticModel(syntaxTree);
@@ -143,12 +142,15 @@ private static Project CreateGeneratedCodeProject()
 
         internal static async Task<GeneratedCodeWorkspace> Create()
         {
+            // prepare the generated code project
             var projectTask = Interlocked.Exchange(ref _cachedProject, null);
-            var generatedCodeProject = projectTask != null ? await projectTask : CreateGeneratedCodeProject();
+            var project = projectTask != null ? await projectTask : CreateGeneratedCodeProject();
 
             var outputDirectory = CodeModelPlugin.Instance.Configuration.OutputDirectory;
             var projectDirectory = CodeModelPlugin.Instance.Configuration.ProjectDirectory;
+            var generatedDirectory = CodeModelPlugin.Instance.Configuration.ProjectGeneratedDirectory;
 
+            // add all documents except the documents from the generated directory
             if (Path.IsPathRooted(projectDirectory) && Path.IsPathRooted(outputDirectory))
             {
                 projectDirectory = Path.GetFullPath(projectDirectory);
@@ -157,37 +159,15 @@ internal static async Task<GeneratedCodeWorkspace> Create()
                 Directory.CreateDirectory(projectDirectory);
                 Directory.CreateDirectory(outputDirectory);
 
-                generatedCodeProject = AddDirectory(generatedCodeProject, projectDirectory, skipPredicate: sourceFile => sourceFile.StartsWith(outputDirectory));
+                project = AddDirectory(project, projectDirectory, skipPredicate: sourceFile => sourceFile.StartsWith(generatedDirectory));
             }
 
             foreach (var sharedSourceFolder in CodeModelPlugin.Instance.SharedSourceDirectories)
             {
-                generatedCodeProject = AddDirectory(generatedCodeProject, sharedSourceFolder, folders: _sharedFolders);
+                project = AddDirectory(project, sharedSourceFolder, folders: _sharedFolders);
             }
 
-            generatedCodeProject = generatedCodeProject.WithParseOptions(new CSharpParseOptions(preprocessorSymbols: new[] { "EXPERIMENTAL" }));
-            return new GeneratedCodeWorkspace(generatedCodeProject);
-        }
-
-        internal static GeneratedCodeWorkspace CreateExistingCodeProject(IEnumerable<string> projectDirectories, string generatedDirectory)
-        {
-            var workspace = new AdhocWorkspace();
-            var newOptionSet = workspace.Options.WithChangedOption(FormattingOptions.NewLine, LanguageNames.CSharp, NewLine);
-            workspace.TryApplyChanges(workspace.CurrentSolution.WithOptions(newOptionSet));
-            Project project = workspace.AddProject("ExistingCode", LanguageNames.CSharp);
-
-            foreach (var projectDirectory in projectDirectories)
-            {
-                if (Path.IsPathRooted(projectDirectory))
-                {
-                    project = AddDirectory(project, Path.GetFullPath(projectDirectory), skipPredicate: sourceFile => sourceFile.StartsWith(generatedDirectory));
-                }
-            }
-
-            project = project
-                .AddMetadataReferences(_assemblyMetadataReferences.Value.Concat(CodeModelPlugin.Instance.AdditionalMetadataReferences))
-                .WithCompilationOptions(new CSharpCompilationOptions(
-                    OutputKind.DynamicallyLinkedLibrary, metadataReferenceResolver: _metadataReferenceResolver.Value, nullableContextOptions: NullableContextOptions.Disable));
+            project = project.WithParseOptions(new CSharpParseOptions(preprocessorSymbols: new[] { "EXPERIMENTAL" }));
 
             return new GeneratedCodeWorkspace(project);
         }
@@ -248,11 +228,5 @@ public async Task PostProcessAsync()
                     break;
             }
         }
-
-        private async Task<Compilation> GetProjectCompilationAsync()
-        {
-            _compilation ??= await _project.GetCompilationAsync();
-            return _compilation!;
-        }
     }
 }

packages/http-client-csharp/generator/Microsoft.Generator.CSharp/test/common/Helpers.cs

@@ -1,11 +1,15 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Runtime.CompilerServices;
 using System.Threading.Tasks;
 using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.CodeAnalysis.Formatting;
+using NUnit.Framework;
 
 namespace Microsoft.Generator.CSharp.Tests.Common
 {
@@ -43,8 +47,36 @@ public static async Task<Compilation> GetCompilationFromDirectoryAsync(
         {
             var directory = GetAssetFileOrDirectoryPath(false, parameters, method, filePath);
             var codeGenAttributeFiles = Path.Combine(_assemblyLocation, "..", "..", "..", "..", "..", "Microsoft.Generator.CSharp.Customization", "src");
-            var workspace = GeneratedCodeWorkspace.CreateExistingCodeProject([directory, codeGenAttributeFiles], Path.Combine(directory, "Generated"));
-            return await workspace.GetCompilationAsync();
+            var project = CreateExistingCodeProject([directory, codeGenAttributeFiles], Path.Combine(directory, "Generated"));
+            var compilation = await project.GetCompilationAsync();
+            Assert.IsNotNull(compilation);
+            return compilation!;
+        }
+
+        private static Project CreateExistingCodeProject(IEnumerable<string> projectDirectories, string generatedDirectory)
+        {
+            var workspace = new AdhocWorkspace();
+            var newOptionSet = workspace.Options.WithChangedOption(FormattingOptions.NewLine, LanguageNames.CSharp, "\n");
+            workspace.TryApplyChanges(workspace.CurrentSolution.WithOptions(newOptionSet));
+            Project project = workspace.AddProject("ExistingCode", LanguageNames.CSharp);
+
+            foreach (var projectDirectory in projectDirectories)
+            {
+                if (Path.IsPathRooted(projectDirectory))
+                {
+                    project = GeneratedCodeWorkspace.AddDirectory(project, Path.GetFullPath(projectDirectory), skipPredicate: sourceFile => sourceFile.StartsWith(generatedDirectory));
+                }
+            }
+
+            project = project
+                .AddMetadataReferences([
+                    MetadataReference.CreateFromFile(typeof(object).Assembly.Location),
+                    ..CodeModelPlugin.Instance.AdditionalMetadataReferences
+                    ])
+                .WithCompilationOptions(new CSharpCompilationOptions(
+                    OutputKind.DynamicallyLinkedLibrary, metadataReferenceResolver: new WorkspaceMetadataReferenceResolver(), nullableContextOptions: NullableContextOptions.Disable));
+
+            return project;
         }
     }
 }