Replace health check publisher and scheduler with ResourceHealthCheck…

…Service and introduce ResourceReadyEvent. (#5870) * Raise ResourceHealthyEvent. * Control dispatch of events. * Use HashSet * Unit tests for blocking/nonblock/sequential/concurrent tests. * Make sure health checks don't run after entering a health state. * Disable failing mongo test. * WIP - reworking things a bit. * Working (not optimized) * Add tracing. * Cleaning up tests. * PR feedback. * Tests! * Slow down health checks on resources in a stable state. * Move resourcesStartedMonitoring into a local variable. * Update src/Aspire.Hosting/Health/ResourceHealthCheckService.cs Co-authored-by: David Fowler <[email protected]> * Update src/Aspire.Hosting/Health/ResourceHealthCheckService.cs Co-authored-by: David Fowler <[email protected]> * do-while * Don't crash out when a poor health check is registered. --------- Co-authored-by: David Fowler <[email protected]>
dotnet · Sep 30, 2024 · 94008ce · 94008ce
1 parent f7f20e4
commit 94008ce
Show file tree

Hide file tree

Showing 24 changed files with 825 additions and 159 deletions.
diff --git a/playground/waitfor/WaitForSandbox.AppHost/Program.cs b/playground/waitfor/WaitForSandbox.AppHost/Program.cs
@@ -3,21 +3,18 @@
 
 var builder = DistributedApplication.CreateBuilder(args);
 
-var pg = builder.AddPostgres("pg")
+var db = builder.AddPostgres("pg")
                 .PublishAsAzurePostgresFlexibleServer()
-                .WithPgAdmin();
-
-var db = pg.AddDatabase("db");
+                .WithPgAdmin()
+                .AddDatabase("db");
 
 var dbsetup = builder.AddProject<Projects.WaitForSandbox_DbSetup>("dbsetup")
-                     .WithReference(db)
-                     .WaitFor(pg);
+                     .WithReference(db).WaitFor(db);
 
 builder.AddProject<Projects.WaitForSandbox_ApiService>("api")
        .WithExternalHttpEndpoints()
-       .WaitForCompletion(dbsetup)
-       .WaitFor(db)
-       .WithReference(db);
+       .WithReference(db).WaitFor(db)
+       .WaitForCompletion(dbsetup);
 
 #if !SKIP_DASHBOARD_REFERENCE
 // This project is only added in playground projects to support development/debugging

diff --git a/src/Aspire.Hosting/ApplicationModel/AfterEndpointsAllocatedEvent.cs b/src/Aspire.Hosting/ApplicationModel/AfterEndpointsAllocatedEvent.cs
@@ -1,7 +1,6 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using System.Diagnostics.CodeAnalysis;
 using Aspire.Hosting.Eventing;
 
 namespace Aspire.Hosting.ApplicationModel;
@@ -27,7 +26,6 @@ namespace Aspire.Hosting.ApplicationModel;
 /// });
 /// </code>
 /// </example>
-[Experimental("ASPIREEVENTING001", UrlFormat = "https://aka.ms/dotnet/aspire/diagnostics#{0}")]
 public class AfterEndpointsAllocatedEvent(IServiceProvider services, DistributedApplicationModel model) : IDistributedApplicationEvent
 {
     /// <summary>

diff --git a/src/Aspire.Hosting/ApplicationModel/AfterResourcesCreatedEvent.cs b/src/Aspire.Hosting/ApplicationModel/AfterResourcesCreatedEvent.cs
@@ -1,7 +1,6 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using System.Diagnostics.CodeAnalysis;
 using Aspire.Hosting.Eventing;
 
 namespace Aspire.Hosting.ApplicationModel;
@@ -27,7 +26,6 @@ namespace Aspire.Hosting.ApplicationModel;
 /// });
 /// </code>
 /// </example>
-[Experimental("ASPIREEVENTING001", UrlFormat = "https://aka.ms/dotnet/aspire/diagnostics#{0}")]
 public class AfterResourcesCreatedEvent(IServiceProvider services, DistributedApplicationModel model) : IDistributedApplicationEvent
 {
     /// <summary>

diff --git a/src/Aspire.Hosting/ApplicationModel/BeforeResourceStartedEvent.cs b/src/Aspire.Hosting/ApplicationModel/BeforeResourceStartedEvent.cs
@@ -1,7 +1,6 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using System.Diagnostics.CodeAnalysis;
 using Aspire.Hosting.Eventing;
 
 namespace Aspire.Hosting.ApplicationModel;
@@ -14,7 +13,6 @@ namespace Aspire.Hosting.ApplicationModel;
 /// <remarks>
 /// Resources that are created by orchestrators may not yet be ready to handle requests.
 /// </remarks>
-[Experimental("ASPIREEVENTING001", UrlFormat = "https://aka.ms/dotnet/aspire/diagnostics#{0}")]
 public class BeforeResourceStartedEvent(IResource resource, IServiceProvider services) : IDistributedApplicationResourceEvent
 {
     /// <inheritdoc />

diff --git a/src/Aspire.Hosting/ApplicationModel/BeforeStartEvent.cs b/src/Aspire.Hosting/ApplicationModel/BeforeStartEvent.cs
@@ -1,7 +1,6 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using System.Diagnostics.CodeAnalysis;
 using Aspire.Hosting.Eventing;
 
 namespace Aspire.Hosting.ApplicationModel;
@@ -27,7 +26,6 @@ namespace Aspire.Hosting.ApplicationModel;
 /// });
 /// </code>
 /// </example>
-[Experimental("ASPIREEVENTING001", UrlFormat = "https://aka.ms/dotnet/aspire/diagnostics#{0}")]
 public class BeforeStartEvent(IServiceProvider services, DistributedApplicationModel model) : IDistributedApplicationEvent
 {
     /// <summary>

diff --git a/src/Aspire.Hosting/ApplicationModel/ResourceExtensions.cs b/src/Aspire.Hosting/ApplicationModel/ResourceExtensions.cs
@@ -54,6 +54,47 @@ public static bool TryGetAnnotationsOfType<T>(this IResource resource, [NotNullW
         }
     }
 
+    /// <summary>
+    /// Attempts to retrieve all annotations of the specified type from the given resource including from parents.
+    /// </summary>
+    /// <typeparam name="T">The type of annotation to retrieve.</typeparam>
+    /// <param name="resource">The resource to retrieve annotations from.</param>
+    /// <param name="result">When this method returns, contains the annotations of the specified type, if found; otherwise, null.</param>
+    /// <returns>true if annotations of the specified type were found; otherwise, false.</returns>
+    public static bool TryGetAnnotationsIncludingAncestorsOfType<T>(this IResource resource, [NotNullWhen(true)] out IEnumerable<T>? result) where T : IResourceAnnotation
+    {
+        var matchingTypeAnnotations = resource.Annotations.OfType<T>();
+
+        if (resource is IResourceWithParent resourceWithParent)
+        {
+            if (resourceWithParent.Parent.TryGetAnnotationsIncludingAncestorsOfType<T>(out var ancestorMatchingTypeAnnotations))
+            {
+                result = matchingTypeAnnotations.Concat(ancestorMatchingTypeAnnotations);
+                return true;
+            }
+            else if (matchingTypeAnnotations.Any())
+            {
+                result = matchingTypeAnnotations;
+                return true;
+            }
+            else
+            {
+                result = null;
+                return false;
+            }
+        }
+        else if (matchingTypeAnnotations.Any())
+        {
+            result = matchingTypeAnnotations.ToArray();
+            return true;
+        }
+        else
+        {
+            result = null;
+            return false;
+        }
+    }
+
     /// <summary>
     /// Attempts to get the environment variables from the given resource.
     /// </summary>

diff --git a/src/Aspire.Hosting/ApplicationModel/ResourceHealthyEvent.cs b/src/Aspire.Hosting/ApplicationModel/ResourceHealthyEvent.cs
@@ -0,0 +1,27 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Aspire.Hosting.Eventing;
+
+namespace Aspire.Hosting.ApplicationModel;
+
+/// <summary>
+/// Event that is raised when a resource initially transitions to a ready state.
+/// </summary>
+/// <param name="resource">The resource that is in a ready state.</param>
+/// <param name="services">The service provider for the app host.</param>
+/// <remarks>
+/// This event is only fired once per resource the first time it transitions to a ready state.
+/// </remarks>
+public class ResourceReadyEvent(IResource resource, IServiceProvider services) : IDistributedApplicationResourceEvent
+{
+    /// <summary>
+    /// The resource that is in a healthy state.
+    /// </summary>
+    public IResource Resource => resource;
+
+    /// <summary>
+    /// The service provider for the app host.
+    /// </summary>
+    public IServiceProvider Services => services;
+}
diff --git a/src/Aspire.Hosting/ApplicationModel/ResourceNotificationService.cs b/src/Aspire.Hosting/ApplicationModel/ResourceNotificationService.cs
@@ -178,7 +178,7 @@ static bool IsContinuableState(CustomResourceSnapshot snapshot) =>
     /// This method returns a task that will complete with the resource is healthy. A resource
     /// without <see cref="HealthCheckAnnotation"/> annotations will be considered healthy.
     /// </remarks>
-    public Task<ResourceEvent> WaitForResourceHealthyAsync(string resourceName, CancellationToken cancellationToken)
+    public Task<ResourceEvent> WaitForResourceHealthyAsync(string resourceName, CancellationToken cancellationToken = default)
     {
         return WaitForResourceAsync(resourceName, re => re.Snapshot.HealthStatus == HealthStatus.Healthy, cancellationToken: cancellationToken);
     }

diff --git a/src/Aspire.Hosting/Dashboard/DashboardServiceData.cs b/src/Aspire.Hosting/Dashboard/DashboardServiceData.cs
@@ -50,7 +50,7 @@ static GenericResourceSnapshot CreateResourceSnapshot(IResource resource, string
                     ExitCode = snapshot.ExitCode,
                     State = snapshot.State?.Text,
                     StateStyle = snapshot.State?.Style,
-                    HealthState = resource.TryGetLastAnnotation<HealthCheckAnnotation>(out _) ? snapshot.HealthStatus switch
+                    HealthState = resource.TryGetAnnotationsIncludingAncestorsOfType<HealthCheckAnnotation>(out _) ? snapshot.HealthStatus switch
                     {
                         HealthStatus.Healthy => HealthStateKind.Healthy,
                         HealthStatus.Unhealthy => HealthStateKind.Unhealthy,

diff --git a/src/Aspire.Hosting/DistributedApplicationBuilder.cs b/src/Aspire.Hosting/DistributedApplicationBuilder.cs
@@ -311,14 +311,7 @@ private void ConfigureHealthChecks()
         {
             return new ConfigureOptions<HealthCheckPublisherOptions>(options =>
             {
-                if (ExecutionContext.IsRunMode)
-                {
-                    // In run mode we route requests to the health check scheduler.
-                    var hcs = sp.GetRequiredService<ResourceHealthCheckScheduler>();
-                    options.Predicate = hcs.Predicate;
-                    options.Period = TimeSpan.FromSeconds(5);
-                }
-                else
+                if (ExecutionContext.IsPublishMode)
                 {
                     // In publish mode we don't run any checks.
                     options.Predicate = (check) => false;
@@ -328,9 +321,8 @@ private void ConfigureHealthChecks()
 
         if (ExecutionContext.IsRunMode)
         {
-            _innerBuilder.Services.AddSingleton<IHealthCheckPublisher, ResourceNotificationHealthCheckPublisher>();
-            _innerBuilder.Services.AddSingleton<ResourceHealthCheckScheduler>();
-            _innerBuilder.Services.AddHostedService<ResourceHealthCheckScheduler>(sp => sp.GetRequiredService<ResourceHealthCheckScheduler>());
+            _innerBuilder.Services.AddSingleton<ResourceHealthCheckService>();
+            _innerBuilder.Services.AddHostedService<ResourceHealthCheckService>(sp => sp.GetRequiredService<ResourceHealthCheckService>());
         }
     }
 

diff --git a/src/Aspire.Hosting/Eventing/DistributedApplicationEventSubscription.cs b/src/Aspire.Hosting/Eventing/DistributedApplicationEventSubscription.cs
@@ -1,7 +1,6 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using System.Diagnostics.CodeAnalysis;
 using Aspire.Hosting.ApplicationModel;
 
 namespace Aspire.Hosting.Eventing;
@@ -10,7 +9,6 @@ namespace Aspire.Hosting.Eventing;
 /// Represents a subscription to an event that is published during the lifecycle of the AppHost.
 /// </summary>
 /// <param name="callback">Callback to invoke when the event is published.</param>
-[Experimental("ASPIREEVENTING001", UrlFormat = "https://aka.ms/dotnet/aspire/diagnostics#{0}")]
 public class DistributedApplicationEventSubscription(Func<IDistributedApplicationEvent, CancellationToken, Task> callback)
 {
     /// <summary>
@@ -22,7 +20,6 @@ public class DistributedApplicationEventSubscription(Func<IDistributedApplicatio
 /// <summary>
 /// Represents a subscription to an event that is published during the lifecycle of the AppHost for a specific resource.
 /// </summary>
-[Experimental("ASPIREEVENTING001", UrlFormat = "https://aka.ms/dotnet/aspire/diagnostics#{0}")]
 public class DistributedApplicationResourceEventSubscription(IResource? resource, Func<IDistributedApplicationResourceEvent, CancellationToken, Task> callback)
     : DistributedApplicationEventSubscription((@event, cancellationToken) => callback((IDistributedApplicationResourceEvent)@event, cancellationToken))
 {

diff --git a/src/Aspire.Hosting/Eventing/DistributedApplicationEventing.cs b/src/Aspire.Hosting/Eventing/DistributedApplicationEventing.cs
@@ -2,36 +2,78 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System.Collections.Concurrent;
-using System.Diagnostics.CodeAnalysis;
 using Aspire.Hosting.ApplicationModel;
 
 namespace Aspire.Hosting.Eventing;
 
 /// <inheritdoc cref="IDistributedApplicationEventing" />
-[Experimental("ASPIREEVENTING001", UrlFormat = "https://aka.ms/dotnet/aspire/diagnostics#{0}")]
 public class DistributedApplicationEventing : IDistributedApplicationEventing
 {
     private readonly ConcurrentDictionary<Type, List<DistributedApplicationEventSubscription>> _eventSubscriptionListLookup = new();
     private readonly ConcurrentDictionary<DistributedApplicationEventSubscription, Type> _subscriptionEventTypeLookup = new();
 
     /// <inheritdoc cref="IDistributedApplicationEventing.PublishAsync{T}(T, CancellationToken)" />
-    [Experimental("ASPIREEVENTING001", UrlFormat = "https://aka.ms/dotnet/aspire/diagnostics#{0}")]
-    public async Task PublishAsync<T>(T @event, CancellationToken cancellationToken = default) where T : IDistributedApplicationEvent
+    [System.Diagnostics.CodeAnalysis.SuppressMessage("ApiDesign", "RS0026:Do not add multiple public overloads with optional parameters", Justification = "Cancellation token")]
+    public Task PublishAsync<T>(T @event, CancellationToken cancellationToken = default) where T : IDistributedApplicationEvent
+    {
+        return PublishAsync(@event, EventDispatchBehavior.BlockingSequential, cancellationToken);
+    }
+
+    /// <inheritdoc cref="IDistributedApplicationEventing.PublishAsync{T}(T, CancellationToken)" />
+    [System.Diagnostics.CodeAnalysis.SuppressMessage("ApiDesign", "RS0026:Do not add multiple public overloads with optional parameters", Justification = "Cancellation token")]
+    public async Task PublishAsync<T>(T @event, EventDispatchBehavior dispatchBehavior, CancellationToken cancellationToken = default) where T : IDistributedApplicationEvent
     {
         if (_eventSubscriptionListLookup.TryGetValue(typeof(T), out var subscriptions))
         {
-            // Taking a snapshot of the subscription list to avoid any concurrency issues
-            // whilst we iterate over the subscriptions. Subscribers could result in the
-            // subscriptions being removed from the list.
-            foreach (var subscription in subscriptions.ToArray())
+            if (dispatchBehavior == EventDispatchBehavior.BlockingConcurrent || dispatchBehavior == EventDispatchBehavior.NonBlockingConcurrent)
+            {
+                var pendingSubscriptionCallbacks = new List<Task>(subscriptions.Count);
+                foreach (var subscription in subscriptions.ToArray())
+                {
+                    var pendingSubscriptionCallback = subscription.Callback(@event, cancellationToken);
+                    pendingSubscriptionCallbacks.Add(pendingSubscriptionCallback);
+                }
+
+                if (dispatchBehavior == EventDispatchBehavior.NonBlockingConcurrent)
+                {
+                    // Non-blocking concurrent.
+                    _ = Task.Run(async () =>
+                    {
+                        await Task.WhenAll(pendingSubscriptionCallbacks).ConfigureAwait(false);
+                    }, default);
+                }
+                else
+                {
+                    // Blocking concurrent.
+                    await Task.WhenAll(pendingSubscriptionCallbacks).ConfigureAwait(false);
+                }
+            }
+            else
             {
-                await subscription.Callback(@event, cancellationToken).ConfigureAwait(false);
+                if (dispatchBehavior == EventDispatchBehavior.NonBlockingSequential)
+                {
+                    // Non-blocking sequential.
+                    _ = Task.Run(async () =>
+                    {
+                        foreach (var subscription in subscriptions.ToArray())
+                        {
+                            await subscription.Callback(@event, cancellationToken).ConfigureAwait(false);
+                        }
+                    }, default);
+                }
+                else
+                {
+                    // Blocking sequential.
+                    foreach (var subscription in subscriptions.ToArray())
+                    {
+                        await subscription.Callback(@event, cancellationToken).ConfigureAwait(false);
+                    }
+                }
             }
         }
     }
 
     /// <inheritdoc cref="IDistributedApplicationEventing.Subscribe{T}(Func{T, CancellationToken, Task})" />
-    [Experimental("ASPIREEVENTING001", UrlFormat = "https://aka.ms/dotnet/aspire/diagnostics#{0}")]
     public DistributedApplicationEventSubscription Subscribe<T>(Func<T, CancellationToken, Task> callback) where T : IDistributedApplicationEvent
     {
         var subscription = new DistributedApplicationEventSubscription(async (@event, ct) =>
@@ -61,7 +103,6 @@ public DistributedApplicationEventSubscription Subscribe<T>(Func<T, Cancellation
     }
 
     /// <inheritdoc cref="IDistributedApplicationEventing.Subscribe{T}(Func{T, CancellationToken, Task})" />
-    [Experimental("ASPIREEVENTING001", UrlFormat = "https://aka.ms/dotnet/aspire/diagnostics#{0}")]
     public DistributedApplicationEventSubscription Subscribe<T>(IResource resource, Func<T, CancellationToken, Task> callback) where T : IDistributedApplicationResourceEvent
     {
         var resourceFilteredCallback = async (T @event, CancellationToken cancellationToken) =>
@@ -76,7 +117,6 @@ public DistributedApplicationEventSubscription Subscribe<T>(IResource resource,
     }
 
     /// <inheritdoc cref="IDistributedApplicationEventing.Unsubscribe(DistributedApplicationEventSubscription)" />
-    [Experimental("ASPIREEVENTING001", UrlFormat = "https://aka.ms/dotnet/aspire/diagnostics#{0}")]
     public void Unsubscribe(DistributedApplicationEventSubscription subscription)
     {
         if (_subscriptionEventTypeLookup.TryGetValue(subscription, out var eventType))

diff --git a/src/Aspire.Hosting/Eventing/EventDispatchBehavior.cs b/src/Aspire.Hosting/Eventing/EventDispatchBehavior.cs
@@ -0,0 +1,30 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace Aspire.Hosting.Eventing;
+
+/// <summary>
+/// Controls how events are dispatched to subscribers.
+/// </summary>
+public enum EventDispatchBehavior
+{
+    /// <summary>
+    /// Fires events sequentially and blocks until they are all processed.
+    /// </summary>
+    BlockingSequential,
+
+    /// <summary>
+    /// Fires events concurrently and blocks until they are all processed.
+    /// </summary>
+    BlockingConcurrent,
+
+    /// <summary>
+    /// Fires events sequentially but does not block.
+    /// </summary>
+    NonBlockingSequential,
+
+    /// <summary>
+    /// Fires events concurrently but does not block.
+    /// </summary>
+    NonBlockingConcurrent
+}
diff --git a/src/Aspire.Hosting/Eventing/IDistributedApplicationEvent.cs b/src/Aspire.Hosting/Eventing/IDistributedApplicationEvent.cs
@@ -1,23 +1,20 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using System.Diagnostics.CodeAnalysis;
 using Aspire.Hosting.ApplicationModel;
 
 namespace Aspire.Hosting.Eventing;
 
 /// <summary>
 /// Represents an event that is published during the lifecycle of the AppHost.
 /// </summary>
-[Experimental("ASPIREEVENTING001", UrlFormat = "https://aka.ms/dotnet/aspire/diagnostics#{0}")]
 public interface IDistributedApplicationEvent
 {
 }
 
 /// <summary>
 /// Represents an event that is published during the lifecycle of the AppHost for a specific resource.
 /// </summary>
-[Experimental("ASPIREEVENTING001", UrlFormat = "https://aka.ms/dotnet/aspire/diagnostics#{0}")]
 public interface IDistributedApplicationResourceEvent : IDistributedApplicationEvent
 {
     /// <summary>