From 3bd8a75880a883815c2ad13903f358bee5fdbef0 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Fri, 22 Nov 2024 13:51:49 +1100 Subject: [PATCH] Fix flaky ResourceHealthCheckService test. (#6730) Introduce time provider to the resource health check service and use channels for sampling loop time. --- .../DistributedApplicationBuilder.cs | 2 ++ .../Health/ResourceHealthCheckService.cs | 8 ++--- .../Aspire.Hosting.Tests.csproj | 3 +- .../Health/ResourceHealthCheckServiceTests.cs | 31 ++++++++++--------- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/src/Aspire.Hosting/DistributedApplicationBuilder.cs b/src/Aspire.Hosting/DistributedApplicationBuilder.cs index def5cd589b..01d174b4fe 100644 --- a/src/Aspire.Hosting/DistributedApplicationBuilder.cs +++ b/src/Aspire.Hosting/DistributedApplicationBuilder.cs @@ -132,6 +132,8 @@ public DistributedApplicationBuilder(DistributedApplicationOptions options) LogBuilderConstructing(options, innerBuilderOptions); _innerBuilder = new HostApplicationBuilder(innerBuilderOptions); + _innerBuilder.Services.AddSingleton(TimeProvider.System); + _innerBuilder.Logging.AddFilter("Microsoft.Hosting.Lifetime", LogLevel.Warning); _innerBuilder.Logging.AddFilter("Microsoft.AspNetCore.Server.Kestrel", LogLevel.Error); _innerBuilder.Logging.AddFilter("Aspire.Hosting.Dashboard", LogLevel.Error); diff --git a/src/Aspire.Hosting/Health/ResourceHealthCheckService.cs b/src/Aspire.Hosting/Health/ResourceHealthCheckService.cs index 810994ce02..0f795d556e 100644 --- a/src/Aspire.Hosting/Health/ResourceHealthCheckService.cs +++ b/src/Aspire.Hosting/Health/ResourceHealthCheckService.cs @@ -11,7 +11,7 @@ namespace Aspire.Hosting.Health; -internal class ResourceHealthCheckService(ILogger logger, ResourceNotificationService resourceNotificationService, HealthCheckService healthCheckService, IServiceProvider services, IDistributedApplicationEventing eventing) : BackgroundService +internal class ResourceHealthCheckService(ILogger logger, ResourceNotificationService resourceNotificationService, HealthCheckService healthCheckService, IServiceProvider services, IDistributedApplicationEventing eventing, TimeProvider timeProvider) : BackgroundService { private readonly Dictionary _latestEvents = new(); @@ -62,7 +62,7 @@ await eventing.PublishAsync( var registrationKeysToCheck = annotations.DistinctBy(a => a.Key).Select(a => a.Key).ToFrozenSet(); - using var timer = new PeriodicTimer(TimeSpan.FromSeconds(5)); + using var timer = new PeriodicTimer(TimeSpan.FromSeconds(5), timeProvider); do { @@ -144,11 +144,11 @@ await resourceNotificationService.PublishUpdateAsync(resource, s => async Task SlowDownMonitoringAsync(ResourceEvent lastEvent, CancellationToken cancellationToken) { - var releaseAfter = DateTime.Now.AddSeconds(30); + var releaseAfter = timeProvider.GetUtcNow().AddSeconds(30); // If we've waited for 30 seconds, or we received an updated event, or the health status is no longer // healthy then we stop slowing down the monitoring loop. - while (DateTime.Now < releaseAfter && _latestEvents[lastEvent.Resource.Name] == lastEvent && lastEvent.Snapshot.HealthStatus == HealthStatus.Healthy) + while (timeProvider.GetUtcNow() < releaseAfter && _latestEvents[lastEvent.Resource.Name] == lastEvent && lastEvent.Snapshot.HealthStatus == HealthStatus.Healthy) { await Task.Delay(1000, cancellationToken).ConfigureAwait(false); } diff --git a/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj b/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj index 484d9f3940..3b0beec413 100644 --- a/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj +++ b/tests/Aspire.Hosting.Tests/Aspire.Hosting.Tests.csproj @@ -1,4 +1,4 @@ - + $(DefaultTargetFramework) @@ -19,6 +19,7 @@ + diff --git a/tests/Aspire.Hosting.Tests/Health/ResourceHealthCheckServiceTests.cs b/tests/Aspire.Hosting.Tests/Health/ResourceHealthCheckServiceTests.cs index 70bff83252..37317f6b8f 100644 --- a/tests/Aspire.Hosting.Tests/Health/ResourceHealthCheckServiceTests.cs +++ b/tests/Aspire.Hosting.Tests/Health/ResourceHealthCheckServiceTests.cs @@ -2,10 +2,12 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Diagnostics; +using System.Threading.Channels; using Aspire.Hosting.Utils; using Microsoft.AspNetCore.InternalTesting; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Time.Testing; using Xunit; using Xunit.Abstractions; @@ -129,12 +131,14 @@ public async Task HealthCheckIntervalDoesNotSlowBeforeSteadyHealthyState() { using var builder = TestDistributedApplicationBuilder.Create(testOutputHelper); - AutoResetEvent? are = null; + var channel = Channel.CreateUnbounded(); - builder.Services.AddHealthChecks().AddCheck("resource_check", () => - { - are?.Set(); + var timeProvider = new FakeTimeProvider(DateTimeOffset.Now); + builder.Services.AddSingleton(timeProvider); + builder.Services.AddHealthChecks().AddAsyncCheck("resource_check", async () => + { + await channel.Writer.WriteAsync(timeProvider.GetUtcNow()); return HealthCheckResult.Unhealthy(); }); @@ -154,21 +158,18 @@ await rns.PublishUpdateAsync(resource.Resource, s => s with }).DefaultTimeout(); await rns.WaitForResourceAsync(resource.Resource.Name, KnownResourceStates.Running, abortTokenSource.Token).DefaultTimeout(); - are = new AutoResetEvent(false); + var firstCheck = await channel.Reader.ReadAsync(abortTokenSource.Token).DefaultTimeout(); + timeProvider.Advance(TimeSpan.FromSeconds(5)); - // Allow one event to through since it could be half way through. - are.WaitOne(); - - var stopwatch = Stopwatch.StartNew(); - are.WaitOne(); - stopwatch.Stop(); + var secondCheck = await channel.Reader.ReadAsync(abortTokenSource.Token).DefaultTimeout(); + timeProvider.Advance(TimeSpan.FromSeconds(5)); - // When not in a healthy state the delay should be ~3 seconds but - // we'll check for 10 seconds to make sure we haven't got down - // the 30 second slow path. - Assert.True(stopwatch.ElapsedMilliseconds < 10000); + var thirdCheck = await channel.Reader.ReadAsync(abortTokenSource.Token).DefaultTimeout(); await app.StopAsync(abortTokenSource.Token).DefaultTimeout(); + + var duration = thirdCheck - firstCheck; + Assert.Equal(TimeSpan.FromSeconds(10), duration); } [Fact]