ajusted xmldoc pipeline (runs to completion)

rubberduck-vba · Nov 10, 2024 · b8231f2 · b8231f2
1 parent 2432f5d
commit b8231f2
Show file tree

Hide file tree

Showing 24 changed files with 297 additions and 307 deletions.
diff --git a/rubberduckvba.Server/Api/Features/FeaturesController.cs b/rubberduckvba.Server/Api/Features/FeaturesController.cs
@@ -66,17 +66,17 @@ public async Task<ActionResult<FeatureViewModel>> Info([FromRoute] string name)
 
         var model = new FeatureViewModel(feature with
         {
-            Description = md.FormatMarkdownDocument(feature.Description, true),
-            ShortDescription = md.FormatMarkdownDocument(feature.ShortDescription),
+            Description = feature.Description,
+            ShortDescription = feature.ShortDescription,
 
             ParentId = feature.Id,
             ParentName = feature.Name,
             ParentTitle = feature.Title,
 
             Features = feature.Features.Select(subFeature => subFeature with
             {
-                Description = md.FormatMarkdownDocument(subFeature.Description),
-                ShortDescription = md.FormatMarkdownDocument(subFeature.ShortDescription)
+                Description = subFeature.Description,
+                ShortDescription = subFeature.ShortDescription
             }).ToArray(),
 
             Inspections = feature.Inspections,

diff --git a/rubberduckvba.Server/ContentSynchronization/Pipeline/Abstract/DataflowJoinBlock.cs b/rubberduckvba.Server/ContentSynchronization/Pipeline/Abstract/DataflowJoinBlock.cs
@@ -43,6 +43,7 @@ public JoinBlock<T1, T2> CreateBlock(GroupingDataflowBlockOptions options, ISour
         Block = new JoinBlock<T1, T2>(options);
         LinkSources(source1, source2);
 
+        Parent.TraceBlockCompletion(Block, Name);
         return Block;
     }
 
@@ -51,12 +52,11 @@ public JoinBlock<T1, T2> CreateBlock<TContext>(GroupingDataflowBlockOptions opti
     {
         Block = new JoinBlock<T1, T2>(options);
 
-        var sourceBlock1 = source1.TryGetBlock() as ISourceBlock<T1>;
-        var sourceBlock2 = source2.TryGetBlock() as ISourceBlock<T2>;
-        if (sourceBlock1 != null && sourceBlock2 != null)
-        {
-            LinkSources(sourceBlock1, sourceBlock2);
-        }
+        var sourceBlock1 = source1.TryGetBlock() as ISourceBlock<T1> ?? throw new ArgumentNullException(nameof(source1));
+        var sourceBlock2 = source2.TryGetBlock() as ISourceBlock<T2> ?? throw new ArgumentNullException(nameof(source2));
+
+        LinkSources(sourceBlock1, sourceBlock2);
+        Parent.TraceBlockCompletion(Block, Name);
 
         return Block;
     }
@@ -121,6 +121,7 @@ public JoinBlock<T1, T2, T3> CreateBlock(GroupingDataflowBlockOptions options, I
         Block = new JoinBlock<T1, T2, T3>(options);
         LinkSources(source1, source2, source3);
 
+        Parent.TraceBlockCompletion(Block, Name);
         return Block;
     }
 
@@ -129,13 +130,12 @@ public JoinBlock<T1, T2, T3> CreateBlock<TContext>(GroupingDataflowBlockOptions
     {
         Block = new JoinBlock<T1, T2, T3>(options);
 
-        var sourceBlock1 = source1.TryGetBlock() as ISourceBlock<T1>;
-        var sourceBlock2 = source2.TryGetBlock() as ISourceBlock<T2>;
-        var sourceBlock3 = source3.TryGetBlock() as ISourceBlock<T3>;
-        if (sourceBlock1 != null && sourceBlock2 != null && sourceBlock3 != null)
-        {
-            LinkSources(sourceBlock1, sourceBlock2, sourceBlock3);
-        }
+        var sourceBlock1 = source1.TryGetBlock() as ISourceBlock<T1> ?? throw new ArgumentNullException(nameof(source1));
+        var sourceBlock2 = source2.TryGetBlock() as ISourceBlock<T2> ?? throw new ArgumentNullException(nameof(source2));
+        var sourceBlock3 = source3.TryGetBlock() as ISourceBlock<T3> ?? throw new ArgumentNullException(nameof(source3));
+
+        LinkSources(sourceBlock1, sourceBlock2, sourceBlock3);
+        Parent.TraceBlockCompletion(Block, Name);
 
         return Block;
     }

diff --git a/rubberduckvba.Server/ContentSynchronization/Pipeline/Abstract/ExecutionDataflowBlockBase.cs b/rubberduckvba.Server/ContentSynchronization/Pipeline/Abstract/ExecutionDataflowBlockBase.cs
@@ -165,9 +165,9 @@ protected void LinkToSources(params ISectionBlock<TContext>[] sources)
                     }
                     else if (srcBlock != null)
                     {
-                        // could be e.g. ActionBlock<TInput>, which can't directly source another block.
-                        completionTasks.Add(srcBlock.Completion);
-                        Logger.LogDebug(Context.Parameters, $"{Name} | ⚠️ Source block ({srcBlock.GetType().Name}) is not ISourceBlock<{typeof(TInput).Name}>. If this is accidental, pipeline may not complete.");
+                        //completionTasks.Add(srcBlock.Completion);
+                        Logger.LogWarning(Context.Parameters, $"{Name} | ⚠️ Source block ({srcBlock.GetType().Name}) is not ISourceBlock<{typeof(TInput).Name}>. Pipeline may not complete.");
+                        throw new InvalidOperationException($"Source block ({srcBlock.GetType().Name}) is not ISourceBlock<{typeof(TInput).Name}>");
                     }
                     else
                     {
@@ -194,6 +194,7 @@ protected void WaitAllTasks(params Task[] completionTasks)
                 sw.Stop();
                 Block.Complete();
                 Logger.LogInformation(Context.Parameters, $"{Name} | ☑️ Block completed | ⏱️ {sw.Elapsed}");
+                Parent.LogBlockCompletionDetails();
             }));
         }
     }
@@ -203,6 +204,7 @@ protected void LinkFromSource(Func<TInput> blockOutput, params Task[] waitForTas
         if (waitForTasks?.Any() ?? false)
         {
             Logger.LogTrace(Context.Parameters, $"{Name} | 🕑 Awaiting the completion of {waitForTasks.Length} source block task{(waitForTasks.Length > 1 ? "s" : string.Empty)}");
+
             _whenAllTasks.Add(Task.WhenAll(waitForTasks).ContinueWith(t =>
             {
                 try
@@ -216,18 +218,14 @@ protected void LinkFromSource(Func<TInput> blockOutput, params Task[] waitForTas
                     }
 
                     Logger.LogTrace(Context.Parameters, $"{Name} | 🚀 Block accepted input ({typeof(TInput).Name})");
+                    Block.Complete();
                 }
                 catch (Exception exception)
                 {
                     Logger.LogException(Context.Parameters, exception);
                     throw;
                 }
-                finally
-                {
-                    Block.Complete();
-                    Logger.LogTrace(Context.Parameters, $"{Name} | ☑️ Block completed");
-                }
-            }, Token, TaskContinuationOptions.OnlyOnRanToCompletion, TaskScheduler.Current));
+            }, Token, TaskContinuationOptions.NotOnCanceled, TaskScheduler.Current));
         }
     }
 

diff --git a/rubberduckvba.Server/ContentSynchronization/Pipeline/Abstract/PipelineSection.cs b/rubberduckvba.Server/ContentSynchronization/Pipeline/Abstract/PipelineSection.cs
@@ -27,6 +27,7 @@ protected PipelineSection(IPipeline<TContext, bool> parent, CancellationTokenSou
     private readonly List<Task> _tracedCompletionTasks = [];
     public void TraceBlockCompletion(IDataflowBlock block, string name)
     {
+        Logger.LogTrace(Context.Parameters, $"{GetType().Name} | ☑️ Tracing block completion for {name}.");
         _tracedCompletionTasks.Add(block.Completion.ContinueWith(t =>
         {
             string status;
@@ -42,22 +43,26 @@ public void TraceBlockCompletion(IDataflowBlock block, string name)
                     status = "task ran to completion.";
                     break;
                 default:
-                    status = $"task is in unexpected state {t.Status}.";
+                    status = $"task is in unexpected state '{t.Status}'.";
                     break;
             }
             Logger.LogTrace(Context.Parameters, $"{GetType().Name} | ✅ Dataflow block completion task completed | {name} ({block.GetType().Name}) | {status}");
-        }).ContinueWith(t =>
-        {
-            var details = Blocks.Select(
-                block => $"{(
-                      block.Value.Completion.IsCompletedSuccessfully ? "✔️"
-                    : block.Value.Completion.IsFaulted ? "✖️"
-                    : block.Value.Completion.IsCanceled ? "⛔"
-                    : "🕑")} {block.Key} : {block.Value.Completion.Status}");
-            Logger.LogTrace(Context.Parameters, "Pipeline block completion status details" + Environment.NewLine + string.Join(Environment.NewLine, details));
+            LogBlockCompletionDetails();
         }));
     }
 
+    public void LogBlockCompletionDetails()
+    {
+        var details = Blocks.Select(
+            block => $"{(
+                  block.Value.Completion.IsCompletedSuccessfully ? "✔️"
+                : block.Value.Completion.IsFaulted ? "✖️"
+                : block.Value.Completion.IsCanceled ? "⛔"
+                : "🕑")} {block.Key} : {block.Value.Completion.Status}");
+
+        Logger.LogTrace(Context.Parameters, "Pipeline block completion status details" + Environment.NewLine + string.Join(Environment.NewLine, details));
+    }
+
     public void FaultPipelineBlock(IDataflowBlock block, Exception exception)
     {
         block.Fault(exception);

diff --git a/rubberduckvba.Server/ContentSynchronization/Pipeline/Sections/Context/SyncContext.cs b/rubberduckvba.Server/ContentSynchronization/Pipeline/Sections/Context/SyncContext.cs
@@ -29,6 +29,7 @@ public void LoadParameters(SyncRequestParameters parameters)
     public StagingContext StagingContext => ContextNotInitializedException.ThrowIfNull(_staging);
 
 
+    private Dictionary<int, TagGraph> _latestTagsByAssetId = [];
     private TagGraph? _main;
     public TagGraph RubberduckDbMain
         => ContextNotInitializedException.ThrowIfNull(_main);
@@ -38,6 +39,10 @@ public void LoadRubberduckDbMain(TagGraph main)
         InvalidContextParameterException.ThrowIfNull(nameof(main), main);
         ContextAlreadyInitializedException.ThrowIfNotNull(_main);
         _main ??= main;
+        foreach (var asset in _main.Assets)
+        {
+            _latestTagsByAssetId[asset.Id] = _main;
+        }
     }
 
     private TagGraph? _next;
@@ -49,6 +54,17 @@ public void LoadRubberduckDbNext(TagGraph next)
         InvalidContextParameterException.ThrowIfNull(nameof(next), next);
         ContextAlreadyInitializedException.ThrowIfNotNull(_next);
         _next ??= next;
+        foreach (var asset in _next.Assets)
+        {
+            _latestTagsByAssetId[asset.Id] = _next;
+        }
+    }
+
+
+    public bool TryGetTagByAssetId(int assetId, out TagGraph tag)
+    {
+        var tags = ContextNotInitializedException.ThrowIfNull(_latestTagsByAssetId);
+        return tags.TryGetValue(assetId, out tag!);
     }
 
     private ImmutableDictionary<int, Tag>? _tagsById;
@@ -65,6 +81,7 @@ public void LoadDbTags(IEnumerable<Tag> tags)
         _tagsById = tags.ToImmutableDictionary(tag => tag.Id);
     }
 
+
     private ImmutableDictionary<string, InspectionDefaultConfig>? _inspectionConfig;
     public ImmutableDictionary<string, InspectionDefaultConfig> InspectionDefaultConfig
         => ContextNotInitializedException.ThrowIfNull(_inspectionConfig);
@@ -80,7 +97,7 @@ public ImmutableHashSet<Inspection> Inspections
         => ContextNotInitializedException.ThrowIfNull(_inspections);
 
     private ImmutableHashSet<QuickFix>? _quickfixes;
-    public ImmutableHashSet<QuickFix> Quickfixes
+    public ImmutableHashSet<QuickFix> QuickFixes
         => ContextNotInitializedException.ThrowIfNull(_quickfixes);
 
     private ImmutableHashSet<Annotation>? _annotations;

diff --git a/...ba.Server/ContentSynchronization/Pipeline/Sections/SyncXmldoc/BroadcastQuickFixesBlock.cs b/...ba.Server/ContentSynchronization/Pipeline/Sections/SyncXmldoc/BroadcastQuickFixesBlock.cs
@@ -1,13 +1,28 @@
 using rubberduckvba.Server.ContentSynchronization.Pipeline.Abstract;
 using rubberduckvba.Server.ContentSynchronization.Pipeline.Sections.Context;
 using rubberduckvba.Server.Model;
-using System.Xml.Linq;
 
 namespace rubberduckvba.Server.ContentSynchronization.Pipeline.Sections.SyncXmldoc;
 
-public class BroadcastQuickFixesBlock : BroadcastBlockBase<(TagAsset, XDocument, IEnumerable<QuickFix>), SyncContext>
+public class BroadcastQuickFixesBlock : BroadcastBlockBase<IEnumerable<QuickFix>, SyncContext>
 {
-    public BroadcastQuickFixesBlock(PipelineSection<SyncContext> parent, CancellationTokenSource tokenSource, ILogger logger) 
+    public BroadcastQuickFixesBlock(PipelineSection<SyncContext> parent, CancellationTokenSource tokenSource, ILogger logger)
+        : base(parent, tokenSource, logger)
+    {
+    }
+}
+
+public class BroadcastAnnotationsBlock : BroadcastBlockBase<IEnumerable<Annotation>, SyncContext>
+{
+    public BroadcastAnnotationsBlock(PipelineSection<SyncContext> parent, CancellationTokenSource tokenSource, ILogger logger)
+        : base(parent, tokenSource, logger)
+    {
+    }
+}
+
+public class BroadcastInspectionsBlock : BroadcastBlockBase<IEnumerable<Inspection>, SyncContext>
+{
+    public BroadcastInspectionsBlock(PipelineSection<SyncContext> parent, CancellationTokenSource tokenSource, ILogger logger)
         : base(parent, tokenSource, logger)
     {
     }

diff --git a/...kvba.Server/ContentSynchronization/Pipeline/Sections/SyncXmldoc/FeatureItemBufferBlock.cs b/...kvba.Server/ContentSynchronization/Pipeline/Sections/SyncXmldoc/FeatureItemBufferBlock.cs
diff --git a/...ckvba.Server/ContentSynchronization/Pipeline/Sections/SyncXmldoc/MergeAnnotationsBlock.cs b/...ckvba.Server/ContentSynchronization/Pipeline/Sections/SyncXmldoc/MergeAnnotationsBlock.cs
@@ -19,23 +19,23 @@ public override IEnumerable<Annotation> Transform(IEnumerable<Annotation> input)
     {
         try
         {
-
-            var main = input
-                .Where(e => Context.RubberduckDbMain.Assets.Any(asset => asset.Id == e.TagAssetId))
+            var main = Context.StagingContext.Annotations
+                .Where(annotation => Context.RubberduckDbMain.Assets.Any(asset => asset.Id == annotation.TagAssetId))
                 .ToList();
 
-            var next = input
-                .Where(e => Context.RubberduckDbNext.Assets.Any(asset => asset.Id == e.TagAssetId))
+            var next = Context.StagingContext.Annotations
+                .Where(annotation => Context.RubberduckDbNext.Assets.Any(asset => asset.Id == annotation.TagAssetId))
                 .ToList();
 
             Logger.LogDebug(Context.Parameters, $"Merging annotations. Main x{main.Count} | Next x{next.Count}");
             var dbItems = Context.Annotations.ToDictionary(e => e.Name);
+
             var merged = _service.Merge(dbItems, main, next);
             Logger.LogDebug(Context.Parameters, $"Merged x{merged.Count()}");
 
             Context.StagingContext.Annotations = new(merged);
 
-            Logger.LogDebug(Context.Parameters, $"Updated: {Context.StagingContext.Annotations.Count()} | New: {Context.StagingContext.Annotations.Count()}");
+            Logger.LogDebug(Context.Parameters, $"Updated: {Context.StagingContext.Annotations.Count(e => e.Id != default)} | New: {Context.StagingContext.Annotations.Count(e => e.Id == default)}");
             return merged;
         }
         finally

diff --git a/...ckvba.Server/ContentSynchronization/Pipeline/Sections/SyncXmldoc/MergeInspectionsBlock.cs b/...ckvba.Server/ContentSynchronization/Pipeline/Sections/SyncXmldoc/MergeInspectionsBlock.cs
@@ -19,12 +19,12 @@ public override IEnumerable<Inspection> Transform(IEnumerable<Inspection> input)
     {
         try
         {
-            var main = input
-                .Where(e => Context.RubberduckDbMain.Assets.Any(asset => asset.Id == e.TagAssetId))
+            var main = Context.StagingContext.Inspections
+                .Where(inspection => Context.RubberduckDbMain.Assets.Any(asset => asset.Id == inspection.TagAssetId))
                 .ToList();
 
-            var next = input
-                .Where(e => Context.RubberduckDbNext.Assets.Any(asset => asset.Id == e.TagAssetId))
+            var next = Context.StagingContext.Inspections
+                .Where(inspection => Context.RubberduckDbNext.Assets.Any(asset => asset.Id == inspection.TagAssetId))
                 .ToList();
 
             Logger.LogDebug(Context.Parameters, $"Merging inspections. Main x{main.Count} | Next x{next.Count}");

diff --git a/...uckvba.Server/ContentSynchronization/Pipeline/Sections/SyncXmldoc/MergeQuickFixesBlock.cs b/...uckvba.Server/ContentSynchronization/Pipeline/Sections/SyncXmldoc/MergeQuickFixesBlock.cs
@@ -19,17 +19,16 @@ public override IEnumerable<QuickFix> Transform(IEnumerable<QuickFix> input)
     {
         try
         {
-
-            var main = input
-                .Where(e => Context.RubberduckDbMain.Assets.Any(asset => asset.Id == e.TagAssetId))
+            var main = Context.StagingContext.QuickFixes
+                .Where(quickfix => Context.RubberduckDbMain.Assets.Any(asset => asset.Id == quickfix.TagAssetId))
                 .ToList();
 
-            var next = input
+            var next = Context.StagingContext.QuickFixes
                 .Where(e => Context.RubberduckDbNext.Assets.Any(asset => asset.Id == e.TagAssetId))
                 .ToList();
 
             Logger.LogDebug(Context.Parameters, $"Merging quickfixes. Main x{main.Count} | Next x{next.Count}");
-            var dbItems = Context.Quickfixes.ToDictionary(e => e.Name);
+            var dbItems = Context.QuickFixes.ToDictionary(e => e.Name);
             var merged = _service.Merge(dbItems, main, next);
             Logger.LogDebug(Context.Parameters, $"Merged x{merged.Count()}");
 

diff --git a/...r/ContentSynchronization/Pipeline/Sections/SyncXmldoc/ParseInspectionXElementInfoBlock.cs b/...r/ContentSynchronization/Pipeline/Sections/SyncXmldoc/ParseInspectionXElementInfoBlock.cs
@@ -5,7 +5,7 @@
 
 namespace rubberduckvba.Server.ContentSynchronization.Pipeline.Sections.SyncXmldoc;
 
-public class ParseInspectionXElementInfoBlock : TransformBlockBase<(TagAsset asset, XElementInfo info), Inspection, SyncContext>
+public class ParseInspectionXElementInfoBlock : TransformBlockBase<(TagAsset asset, XElementInfo info, IEnumerable<QuickFix> quickfixes), Inspection, SyncContext>
 {
     private readonly XmlDocInspectionParser _parser;
 
@@ -16,21 +16,20 @@ public ParseInspectionXElementInfoBlock(PipelineSection<SyncContext> parent, Can
         _parser = parser;
     }
 
-    public override async Task<Inspection> TransformAsync((TagAsset asset, XElementInfo info) input)
+    public override async Task<Inspection> TransformAsync((TagAsset asset, XElementInfo info, IEnumerable<QuickFix> quickfixes) input)
     {
         if (input.asset is null)
         {
             return default!;
         }
 
         var feature = Context.Features["Inspections"];
-        var quickfixes = Context.Features["QuickFixes"].QuickFixes; // NOTE: this makes parsing quickfixes a requirement for inspections
 
         var config = Context.InspectionDefaultConfig.TryGetValue(input.info.Name, out var value) ? value : null;
         var isPreRelease = Context.RubberduckDbNext.Assets.Any(asset => asset.Id == input.asset.Id);
         var name = input.info.Element.Attribute("name")!.Value;
 
-        var result = await _parser.ParseAsync(input.asset.Id, feature.Id, quickfixes, name, input.info.Element, config, isPreRelease);
+        var result = await _parser.ParseAsync(input.asset.Id, feature.Id, input.quickfixes, name, input.info.Element, config, isPreRelease);
 
         return result with { TagAssetId = input.asset.Id };
     }

diff --git a/...duckvba.Server/ContentSynchronization/Pipeline/Sections/SyncXmldoc/PrepareStagingBlock.cs b/...duckvba.Server/ContentSynchronization/Pipeline/Sections/SyncXmldoc/PrepareStagingBlock.cs
@@ -0,0 +1,26 @@
+using rubberduckvba.Server.ContentSynchronization.Pipeline.Abstract;
+using rubberduckvba.Server.ContentSynchronization.Pipeline.Sections.Context;
+using rubberduckvba.Server.Model;
+
+namespace rubberduckvba.Server.ContentSynchronization.Pipeline.Sections.SyncXmldoc;
+
+public class PrepareStagingBlock : TransformBlockBase<Tuple<IEnumerable<Annotation>, IEnumerable<QuickFix>, IEnumerable<Inspection>>, StagingContext, SyncContext>
+{
+    public PrepareStagingBlock(PipelineSection<SyncContext> parent, CancellationTokenSource tokenSource, ILogger logger)
+        : base(parent, tokenSource, logger)
+    {
+    }
+
+    public override StagingContext Transform(Tuple<IEnumerable<Annotation>, IEnumerable<QuickFix>, IEnumerable<Inspection>> input)
+    {
+        var parameters = Context.StagingContext.Parameters;
+        var (annotations, quickfixes, inspections) = (input.Item1, input.Item2, input.Item3);
+
+        return new StagingContext(parameters)
+        {
+            Inspections = new(inspections),
+            QuickFixes = new(quickfixes),
+            Annotations = new(annotations)
+        };
+    }
+}