Skip to content

Commit

Permalink
Move SlowDownMonitoringAsync. (#6309)
Browse files Browse the repository at this point in the history
* Move SlowDownMonitoringAsync.

* Add tests.
  • Loading branch information
mitchdenny authored Oct 16, 2024
1 parent 6f261e9 commit b87cab2
Show file tree
Hide file tree
Showing 2 changed files with 95 additions and 7 deletions.
9 changes: 2 additions & 7 deletions src/Aspire.Hosting/Health/ResourceHealthCheckService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -83,10 +83,9 @@ await eventing.PublishAsync(
cancellationToken).ConfigureAwait(false);
}

if (_latestEvents[resource.Name].Snapshot.HealthStatus == report.Status)
if (_latestEvents[resource.Name] is { } latestEvent && latestEvent.Snapshot.HealthStatus == report.Status)
{
// If the last health status is the same as this health status then we don't need
// to publish anything as it just creates noise.
await SlowDownMonitoringAsync(latestEvent, cancellationToken).ConfigureAwait(false);
continue;
}

Expand All @@ -103,10 +102,6 @@ await resourceNotificationService.PublishUpdateAsync(resource, s =>
HealthReports = healthReports
};
}).ConfigureAwait(false);

var lastEvent = _latestEvents[resource.Name];
await SlowDownMonitoringAsync(lastEvent, cancellationToken).ConfigureAwait(false);

}
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
{
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.Diagnostics;
using Aspire.Hosting.Utils;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Diagnostics.HealthChecks;
Expand All @@ -11,6 +12,98 @@ namespace Aspire.Hosting.Tests.Health;

public class ResourceHealthCheckServiceTests(ITestOutputHelper testOutputHelper)
{
[Fact]
public async Task HealthCheckIntervalSlowsAfterSteadyHealthyState()
{
using var builder = TestDistributedApplicationBuilder.Create(testOutputHelper);

AutoResetEvent? are = null;

builder.Services.AddHealthChecks().AddCheck("resource_check", () =>
{
are?.Set();

return HealthCheckResult.Healthy();
});

var resource = builder.AddResource(new ParentResource("resource"))
.WithHealthCheck("resource_check");

using var app = builder.Build();
var rns = app.Services.GetRequiredService<ResourceNotificationService>();

var abortTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(120));

await app.StartAsync(abortTokenSource.Token);

await rns.PublishUpdateAsync(resource.Resource, s => s with
{
State = KnownResourceStates.Running
});
await rns.WaitForResourceHealthyAsync(resource.Resource.Name, abortTokenSource.Token);

are = new AutoResetEvent(false);

// Allow one event to through since it could be half way through.
are.WaitOne();

var stopwatch = Stopwatch.StartNew();
are.WaitOne();
stopwatch.Stop();

// Delay is 30 seconds but we allow for a (ridiculous) 10 second margin of error.
Assert.True(stopwatch.ElapsedMilliseconds > 20000);

await app.StopAsync(abortTokenSource.Token);
}

[Fact]
public async Task HealthCheckIntervalDoesNotSlowBeforeSteadyHealthyState()
{
using var builder = TestDistributedApplicationBuilder.Create(testOutputHelper);

AutoResetEvent? are = null;

builder.Services.AddHealthChecks().AddCheck("resource_check", () =>
{
are?.Set();

return HealthCheckResult.Unhealthy();
});

var resource = builder.AddResource(new ParentResource("resource"))
.WithHealthCheck("resource_check");

using var app = builder.Build();
var rns = app.Services.GetRequiredService<ResourceNotificationService>();

var abortTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(120));

await app.StartAsync(abortTokenSource.Token);

await rns.PublishUpdateAsync(resource.Resource, s => s with
{
State = KnownResourceStates.Running
});
await rns.WaitForResourceAsync(resource.Resource.Name, KnownResourceStates.Running, abortTokenSource.Token);

are = new AutoResetEvent(false);

// Allow one event to through since it could be half way through.
are.WaitOne();

var stopwatch = Stopwatch.StartNew();
are.WaitOne();
stopwatch.Stop();

// When not in a healthy state the delay should be ~3 seconds but
// we'll check for 10 seconds to make sure we haven't got down
// the 30 second slow path.
Assert.True(stopwatch.ElapsedMilliseconds < 10000);

await app.StopAsync(abortTokenSource.Token);
}

[Fact]
public async Task ResourcesWithoutHealthCheckAnnotationsGetReadyEventFired()
{
Expand Down

0 comments on commit b87cab2

Please sign in to comment.