diff --git a/.gitignore b/.gitignore index 736d9bfd9..1b5e17f30 100644 --- a/.gitignore +++ b/.gitignore @@ -300,4 +300,4 @@ docs/_site/* #MAC .DS_Store -*.orig \ No newline at end of file +*.orig diff --git a/adding-a-new-scraper.md b/adding-a-new-scraper.md index 6fc139092..45d02c019 100644 --- a/adding-a-new-scraper.md +++ b/adding-a-new-scraper.md @@ -11,7 +11,7 @@ This guide walks you through the process of adding a new scraper type. 1. Add your new scraping type to the `Promitor.Core.Scraping.Configuration.Model.ResourceType` 2. Describe the resource for which you're creating scraping metrics by creating `MetricDefinition`and inherit from `Promitor.Core.Scraping.Configuration.Model.Metrics.MetricDefinition` - this class should go in `.\src\Promitor.Core.Scraping\Configuration\Model\Metrics\ResourceTypes` 2. Create an example entry for your scraper in `.\samples\promitor-sample.yaml` -3. Create a new Deserializer in `.\src\Promitor.Core.Scraping\Configuration\Serialization\Deserializers`. This must inherit from `AzureMetricDeserializer`. +3. Create a new Deserializer in `.\src\Promitor.Core.Scraping\Configuration\Serialization\Deserializers`. This must inherit from `GenericAzureMetricDeserializer`. 3. Update `Promitor.Core.Scraping.Factories.MetricDeserializerFactory` to handle your new resource type by returning a new instance of the Deserializer you created in the previous step. 4. Provide a unit test in `.\src\Promitor.Scraper.Tests.Unit\Serialization\MetricsDeclaration\` that tests the deserialization based on our sample diff --git a/docs/configuration/metrics/cosmosdb.md b/docs/configuration/metrics/cosmosdb.md new file mode 100644 index 000000000..48870e3f8 --- /dev/null +++ b/docs/configuration/metrics/cosmosdb.md @@ -0,0 +1,27 @@ +--- +layout: default +title: Cosmos Db Declaration +--- + +## Cosmos Db +You can declare to scrape Cosmos Db via the `CosmosDb` resource type. + +The following fields need to be provided: +- `dbName`- The name of the Cosmos Db to be scraped + +All supported metrics are documented in the official [Azure Monitor documentation](https://docs.microsoft.com/en-us/azure/azure-monitor/platform/metrics-supported#microsoftcontainerinstancecontainergroups). + +Example: +```yaml + name: demo_cosmos_totalrequests + description: "Demo cosmos query" + resourceType: CosmosDb + dbName: cognitiveanalytics + azureMetricConfiguration: + metricName: TotalRequests + aggregation: + type: Count +``` + +[← back to metrics declarations](/configuration/metrics) +[← back to introduction](/) \ No newline at end of file diff --git a/docs/configuration/metrics/index.md b/docs/configuration/metrics/index.md index ed5249991..aa2c0cda3 100644 --- a/docs/configuration/metrics/index.md +++ b/docs/configuration/metrics/index.md @@ -87,6 +87,7 @@ We also provide a simplified way to configure the following Azure resources: - [Azure Virtual Machine](virtual-machine) - [Azure Network Interface](network-interface) - [Azure Storage Queue](storage-queue) +- [Cosmos DB](cosmosdb) Want to help out? Create an issue and [contribute a new scraper](https://github.com/tomkerkhove/promitor/blob/master/adding-a-new-scraper.md). diff --git a/src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/CosmosDbMetricDefinition.cs b/src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/CosmosDbMetricDefinition.cs new file mode 100644 index 000000000..2f026326a --- /dev/null +++ b/src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/CosmosDbMetricDefinition.cs @@ -0,0 +1,9 @@ +namespace Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes +{ + public class CosmosDbMetricDefinition : MetricDefinition + { + public string DbName { get; set; } + + public override ResourceType ResourceType { get; } = ResourceType.CosmosDb; + } +} diff --git a/src/Promitor.Core.Scraping/Configuration/Model/ResourceType.cs b/src/Promitor.Core.Scraping/Configuration/Model/ResourceType.cs index d4d0a8010..55543b29f 100644 --- a/src/Promitor.Core.Scraping/Configuration/Model/ResourceType.cs +++ b/src/Promitor.Core.Scraping/Configuration/Model/ResourceType.cs @@ -10,5 +10,6 @@ public enum ResourceType VirtualMachine = 5, ContainerRegistry = 6, NetworkInterface = 7, + CosmosDb = 8 } } \ No newline at end of file diff --git a/src/Promitor.Core.Scraping/Configuration/Serialization/Core/MetricDeserializer.cs b/src/Promitor.Core.Scraping/Configuration/Serialization/Core/MetricDeserializer.cs index dd518bdc3..e7fa2eee3 100644 --- a/src/Promitor.Core.Scraping/Configuration/Serialization/Core/MetricDeserializer.cs +++ b/src/Promitor.Core.Scraping/Configuration/Serialization/Core/MetricDeserializer.cs @@ -1,4 +1,4 @@ -using System.Collections.Generic; +using System.Collections.Generic; using GuardNet; using Microsoft.Extensions.Logging; using Promitor.Core.Scraping.Configuration.Model.Metrics; diff --git a/src/Promitor.Core.Scraping/Configuration/Serialization/Deserializers/CosmosDbMetricDeserializer.cs b/src/Promitor.Core.Scraping/Configuration/Serialization/Deserializers/CosmosDbMetricDeserializer.cs new file mode 100644 index 000000000..05490e8e1 --- /dev/null +++ b/src/Promitor.Core.Scraping/Configuration/Serialization/Deserializers/CosmosDbMetricDeserializer.cs @@ -0,0 +1,23 @@ +using Promitor.Core.Scraping.Configuration.Model.Metrics; +using Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes; +using YamlDotNet.RepresentationModel; + +namespace Promitor.Core.Scraping.Configuration.Serialization.Deserializers +{ + internal class CosmosDbMetricDeserializer : GenericAzureMetricDeserializer + { + /// Deserializes the specified Cosmos DB metric node from the YAML configuration file. + /// The metric node to deserialize to Cosmos DB configuration + /// A new object (strongly typed as a ) + internal override MetricDefinition Deserialize(YamlMappingNode metricNode) + { + var metricDefinition = base.DeserializeMetricDefinition(metricNode); + + var dbName = metricNode.Children[new YamlScalarNode("dbName")]; + + metricDefinition.DbName = dbName?.ToString(); + + return metricDefinition; + } + } +} diff --git a/src/Promitor.Core.Scraping/Factories/MetricDeserializerFactory.cs b/src/Promitor.Core.Scraping/Factories/MetricDeserializerFactory.cs index 301c83735..e36f3d6cb 100644 --- a/src/Promitor.Core.Scraping/Factories/MetricDeserializerFactory.cs +++ b/src/Promitor.Core.Scraping/Factories/MetricDeserializerFactory.cs @@ -23,6 +23,8 @@ internal static GenericAzureMetricDeserializer GetDeserializerFor(Configuration. return new ContainerRegistryMetricDeserializer(); case Configuration.Model.ResourceType.NetworkInterface: return new NetworkInterfaceMetricDeserializer(); + case Configuration.Model.ResourceType.CosmosDb: + return new CosmosDbMetricDeserializer(); } throw new ArgumentOutOfRangeException($@"Resource Type {resource} not supported."); diff --git a/src/Promitor.Core.Scraping/Factories/MetricScraperFactory.cs b/src/Promitor.Core.Scraping/Factories/MetricScraperFactory.cs index 7d5c67499..edf7ecaeb 100644 --- a/src/Promitor.Core.Scraping/Factories/MetricScraperFactory.cs +++ b/src/Promitor.Core.Scraping/Factories/MetricScraperFactory.cs @@ -40,6 +40,8 @@ public static IScraper CreateScraper(ResourceType metricDefini return new NetworkInterfaceScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker); case ResourceType.ContainerRegistry: return new ContainerRegistryScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker); + case ResourceType.CosmosDb: + return new CosmosDbScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker); default: throw new ArgumentOutOfRangeException(); } diff --git a/src/Promitor.Core.Scraping/ResourceTypes/CosmosDbScraper.cs b/src/Promitor.Core.Scraping/ResourceTypes/CosmosDbScraper.cs new file mode 100644 index 000000000..247031db1 --- /dev/null +++ b/src/Promitor.Core.Scraping/ResourceTypes/CosmosDbScraper.cs @@ -0,0 +1,31 @@ +using System; +using System.Threading.Tasks; +using Microsoft.Azure.Management.Monitor.Fluent.Models; +using Microsoft.Extensions.Logging; +using Promitor.Core.Scraping.Configuration.Model; +using Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes; +using Promitor.Core.Telemetry.Interfaces; +using Promitor.Integrations.AzureMonitor; + +namespace Promitor.Core.Scraping.ResourceTypes +{ + public class CosmosDbScraper : Scraper + { + private const string ResourceUriTemplate = "subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.DocumentDB/databaseAccounts/{2}"; + + public CosmosDbScraper(AzureMetadata azureMetadata, AzureMonitorClient azureMonitorClient, ILogger logger, IExceptionTracker exceptionTracker) + : base(azureMetadata, azureMonitorClient, logger, exceptionTracker) + { + } + + protected override async Task ScrapeResourceAsync(string subscriptionId, string resourceGroupName, CosmosDbMetricDefinition metricDefinition, AggregationType aggregationType, TimeSpan aggregationInterval) + { + var resourceUri = string.Format(ResourceUriTemplate, subscriptionId, resourceGroupName, metricDefinition.DbName); + + var metricName = metricDefinition.AzureMetricConfiguration.MetricName; + var foundMetricValue = await AzureMonitorClient.QueryMetricAsync(metricName, aggregationType, aggregationInterval, resourceUri); + + return foundMetricValue; + } + } +} diff --git a/src/Promitor.Core.Scraping/Scraper.cs b/src/Promitor.Core.Scraping/Scraper.cs index 36a61adb1..e1495aefb 100644 --- a/src/Promitor.Core.Scraping/Scraper.cs +++ b/src/Promitor.Core.Scraping/Scraper.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Threading.Tasks; using GuardNet; using Microsoft.Azure.Management.Monitor.Fluent.Models; diff --git a/src/Promitor.Scraper.Host/Validation/Factories/MetricValidatorFactory.cs b/src/Promitor.Scraper.Host/Validation/Factories/MetricValidatorFactory.cs index 195d9e8f1..afad7621b 100644 --- a/src/Promitor.Scraper.Host/Validation/Factories/MetricValidatorFactory.cs +++ b/src/Promitor.Scraper.Host/Validation/Factories/MetricValidatorFactory.cs @@ -25,6 +25,8 @@ internal static IMetricValidator GetValidatorFor(ResourceType resourceType) return new NetworkInterfaceMetricValidator(); case ResourceType.ContainerRegistry: return new ContainerRegistryMetricValidator(); + case ResourceType.CosmosDb: + return new CosmosDbMetricValidator(); } throw new ArgumentOutOfRangeException(nameof(resourceType), $"No validation rules are defined for metric type '{resourceType}'"); diff --git a/src/Promitor.Scraper.Host/Validation/MetricDefinitions/ResourceTypes/CosmosDbMetricValidator.cs b/src/Promitor.Scraper.Host/Validation/MetricDefinitions/ResourceTypes/CosmosDbMetricValidator.cs new file mode 100644 index 000000000..53e41ac79 --- /dev/null +++ b/src/Promitor.Scraper.Host/Validation/MetricDefinitions/ResourceTypes/CosmosDbMetricValidator.cs @@ -0,0 +1,19 @@ +using System.Collections.Generic; +using GuardNet; +using Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes; + +namespace Promitor.Scraper.Host.Validation.MetricDefinitions.ResourceTypes +{ + internal class CosmosDbMetricValidator : MetricValidator + { + protected override IEnumerable Validate(CosmosDbMetricDefinition cosmosDbMetricDefinition) + { + Guard.NotNull(cosmosDbMetricDefinition, nameof(cosmosDbMetricDefinition)); + + if (string.IsNullOrWhiteSpace(cosmosDbMetricDefinition.DbName)) + { + yield return "No Cosmos db Name is configured"; + } + } + } +} \ No newline at end of file diff --git a/src/Promitor.Scraper.Tests.Unit/Builders/MetricsDeclarationBuilder.cs b/src/Promitor.Scraper.Tests.Unit/Builders/MetricsDeclarationBuilder.cs index 14a3c2151..116d1db57 100644 --- a/src/Promitor.Scraper.Tests.Unit/Builders/MetricsDeclarationBuilder.cs +++ b/src/Promitor.Scraper.Tests.Unit/Builders/MetricsDeclarationBuilder.cs @@ -106,6 +106,21 @@ public string Build() return this; } + public MetricsDeclarationBuilder WithCosmosDbMetric(string metricName = "promitor-cosmosdb", string metricDescription = "Description for a metric", string dbName = "promitor-cosmosdb", string azureMetricName = "TotalRequests") + { + var azureMetricConfiguration = CreateAzureMetricConfiguration(azureMetricName); + var metric = new CosmosDbMetricDefinition + { + Name = metricName, + Description = metricDescription, + DbName = dbName, + AzureMetricConfiguration = azureMetricConfiguration + }; + _metrics.Add(metric); + + return this; + } + public MetricsDeclarationBuilder WithAzureStorageQueueMetric(string metricName = "promitor", string metricDescription = "Description for a metric", string queueName = "promitor-queue", string accountName = "promitor-account", string sasToken = "?sig=promitor", string azureMetricName = AzureStorageConstants.Queues.Metrics.MessageCount) { var azureMetricConfiguration = CreateAzureMetricConfiguration(azureMetricName); diff --git a/src/Promitor.Scraper.Tests.Unit/Serialization/MetricsDeclaration/MetricsDeclarationWithCosmosDbYamlSerializationTests.cs b/src/Promitor.Scraper.Tests.Unit/Serialization/MetricsDeclaration/MetricsDeclarationWithCosmosDbYamlSerializationTests.cs new file mode 100644 index 000000000..d0625ab8a --- /dev/null +++ b/src/Promitor.Scraper.Tests.Unit/Serialization/MetricsDeclaration/MetricsDeclarationWithCosmosDbYamlSerializationTests.cs @@ -0,0 +1,78 @@ +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using Bogus; +using Microsoft.Extensions.Logging.Abstractions; +using Promitor.Core.Scraping.Configuration.Model; +using Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes; +using Promitor.Core.Scraping.Configuration.Serialization.Core; +using Xunit; +using MetricDefinition = Promitor.Core.Scraping.Configuration.Model.Metrics.MetricDefinition; + +namespace Promitor.Scraper.Tests.Unit.Serialization.MetricsDeclaration +{ + [Category("Unit")] + public class MetricsDeclarationWithCosmosDbYamlSerializationTests : YamlSerializationTests + { + [Theory] + [InlineData("promitor1", @"* */1 * * * *", @"* */2 * * * *")] + [InlineData(null, null, null)] + public void YamlSerialization_SerializeAndDeserializeValidConfigForCosmosDb_SucceedsWithIdenticalOutput(string resourceGroupName, string defaultScrapingInterval, string metricScrapingInterval) + { + // Arrange + var azureMetadata = GenerateBogusAzureMetadata(); + var cosmosDbMetricDefinition = GenerateBogusCosmosDbMetricDefinition(resourceGroupName, metricScrapingInterval); + var metricDefaults = GenerateBogusMetricDefaults(defaultScrapingInterval); + var scrapingConfiguration = new Core.Scraping.Configuration.Model.MetricsDeclaration + { + AzureMetadata = azureMetadata, + MetricDefaults = metricDefaults, + Metrics = new List + { + cosmosDbMetricDefinition + } + }; + var configurationSerializer = new ConfigurationSerializer(NullLogger.Instance); + + // Act + var serializedConfiguration = configurationSerializer.Serialize(scrapingConfiguration); + var deserializedConfiguration = configurationSerializer.Deserialize(serializedConfiguration); + + // Assert + Assert.NotNull(deserializedConfiguration); + AssertAzureMetadata(deserializedConfiguration, azureMetadata); + AssertMetricDefaults(deserializedConfiguration, metricDefaults); + Assert.NotNull(deserializedConfiguration.Metrics); + Assert.Single(deserializedConfiguration.Metrics); + var deserializedMetricDefinition = deserializedConfiguration.Metrics.FirstOrDefault(); + AssertMetricDefinition(deserializedMetricDefinition, cosmosDbMetricDefinition); + var deserializedCosmosDbMetricDefinition = deserializedMetricDefinition as CosmosDbMetricDefinition; + AssertCosmosDbMetricDefinition(deserializedCosmosDbMetricDefinition, cosmosDbMetricDefinition); + } + + private static void AssertCosmosDbMetricDefinition(CosmosDbMetricDefinition deserializedCosmosDbMetricDefinition, CosmosDbMetricDefinition cosmosDbMetricDefinition) + { + Assert.NotNull(deserializedCosmosDbMetricDefinition); + Assert.Equal(cosmosDbMetricDefinition.DbName, deserializedCosmosDbMetricDefinition.DbName); + } + + private CosmosDbMetricDefinition GenerateBogusCosmosDbMetricDefinition(string resourceGroupName, string metricScrapingInterval) + { + var bogusScrapingInterval = GenerateBogusScrapingInterval(metricScrapingInterval); + var bogusAzureMetricConfiguration = GenerateBogusAzureMetricConfiguration(); + + var bogusGenerator = new Faker() + .StrictMode(ensureRulesForAllProperties: true) + .RuleFor(metricDefinition => metricDefinition.Name, faker => faker.Name.FirstName()) + .RuleFor(metricDefinition => metricDefinition.Description, faker => faker.Lorem.Sentence(wordCount: 6)) + .RuleFor(metricDefinition => metricDefinition.ResourceType, faker => ResourceType.CosmosDb) + .RuleFor(metricDefinition => metricDefinition.DbName, faker => faker.Name.LastName()) + .RuleFor(metricDefinition => metricDefinition.AzureMetricConfiguration, faker => bogusAzureMetricConfiguration) + .RuleFor(metricDefinition => metricDefinition.ResourceGroupName, faker => resourceGroupName) + .RuleFor(metricDefinition => metricDefinition.Scraping, faker => bogusScrapingInterval) + .Ignore(metricDefinition => metricDefinition.ResourceGroupName); + + return bogusGenerator.Generate(); + } + } +} \ No newline at end of file diff --git a/src/Promitor.Scraper.Tests.Unit/Validation/Metrics/ResourceTypes/CosmosDbMetricsDeclarationValidationStepTests.cs b/src/Promitor.Scraper.Tests.Unit/Validation/Metrics/ResourceTypes/CosmosDbMetricsDeclarationValidationStepTests.cs new file mode 100644 index 000000000..a24e56906 --- /dev/null +++ b/src/Promitor.Scraper.Tests.Unit/Validation/Metrics/ResourceTypes/CosmosDbMetricsDeclarationValidationStepTests.cs @@ -0,0 +1,97 @@ +using System.ComponentModel; +using Promitor.Scraper.Host.Validation.Steps; +using Promitor.Scraper.Tests.Unit.Builders; +using Promitor.Scraper.Tests.Unit.Stubs; +using Xunit; + +namespace Promitor.Scraper.Tests.Unit.Validation.Metrics.ResourceTypes +{ + [Category("Unit")] + public class CosmosDbMetricsDeclarationValidationStepTests + { + [Fact] + public void CosmosDbMetricsDeclaration_DeclarationWithoutAzureMetricName_Succeeds() + { + // Arrange + var rawDeclaration = MetricsDeclarationBuilder.WithMetadata() + .WithCosmosDbMetric(azureMetricName: string.Empty) + .Build(); + var metricsDeclarationProvider = new MetricsDeclarationProviderStub(rawDeclaration); + + // Act + var scrapingScheduleValidationStep = new MetricsDeclarationValidationStep(metricsDeclarationProvider); + var validationResult = scrapingScheduleValidationStep.Run(); + + // Assert + Assert.False(validationResult.IsSuccessful, "Validation is successful"); + } + + [Fact] + public void CosmosDbMetricsDeclaration_DeclarationWithoutMetricDescription_Succeeded() + { + // Arrange + var rawDeclaration = MetricsDeclarationBuilder.WithMetadata() + .WithCosmosDbMetric(metricDescription: string.Empty) + .Build(); + var metricsDeclarationProvider = new MetricsDeclarationProviderStub(rawDeclaration); + + // Act + var scrapingScheduleValidationStep = new MetricsDeclarationValidationStep(metricsDeclarationProvider); + var validationResult = scrapingScheduleValidationStep.Run(); + + // Assert + Assert.True(validationResult.IsSuccessful, "Validation was not successful"); + } + + [Fact] + public void CosmosDbMetricsDeclaration_DeclarationWithoutMetricName_Fails() + { + // Arrange + var rawDeclaration = MetricsDeclarationBuilder.WithMetadata() + .WithCosmosDbMetric(string.Empty) + .Build(); + var metricsDeclarationProvider = new MetricsDeclarationProviderStub(rawDeclaration); + + // Act + var scrapingScheduleValidationStep = new MetricsDeclarationValidationStep(metricsDeclarationProvider); + var validationResult = scrapingScheduleValidationStep.Run(); + + // Assert + Assert.False(validationResult.IsSuccessful, "Validation is successful"); + } + + [Fact] + public void CosmosDbMetricsDeclaration_DeclarationWithoutDbName_Fails() + { + // Arrange + var rawDeclaration = MetricsDeclarationBuilder.WithMetadata() + .WithCosmosDbMetric(dbName: string.Empty) + .Build(); + var metricsDeclarationProvider = new MetricsDeclarationProviderStub(rawDeclaration); + + // Act + var scrapingScheduleValidationStep = new MetricsDeclarationValidationStep(metricsDeclarationProvider); + var validationResult = scrapingScheduleValidationStep.Run(); + + // Assert + Assert.False(validationResult.IsSuccessful, "Validation is successful"); + } + + [Fact] + public void CosmosDbMetricsDeclaration_ValidDeclaration_Succeeds() + { + // Arrange + var rawMetricsDeclaration = MetricsDeclarationBuilder.WithMetadata() + .WithCosmosDbMetric() + .Build(); + var metricsDeclarationProvider = new MetricsDeclarationProviderStub(rawMetricsDeclaration); + + // Act + var scrapingScheduleValidationStep = new MetricsDeclarationValidationStep(metricsDeclarationProvider); + var validationResult = scrapingScheduleValidationStep.Run(); + + // Assert + Assert.True(validationResult.IsSuccessful, "Validation was not successful"); + } + } +} \ No newline at end of file