Implementation of Cosmos DB scraper (#472)

* Implementation of Cosmos DB scraper with TotalRequests metric

* Implementation of Cosmos DB Scraper

* fixed minor type in CosmosDB serialization unit test

.gitignore

@@ -300,4 +300,4 @@ docs/_site/*
 #MAC
 .DS_Store
 
-*.orig
+*.orig

adding-a-new-scraper.md

@@ -11,7 +11,7 @@ This guide walks you through the process of adding a new scraper type.
 1. Add your new scraping type to the  `Promitor.Core.Scraping.Configuration.Model.ResourceType`
 2. Describe the resource for which you're creating scraping metrics by creating `<New-Type>MetricDefinition`and inherit from `Promitor.Core.Scraping.Configuration.Model.Metrics.MetricDefinition` - this class should go in `.\src\Promitor.Core.Scraping\Configuration\Model\Metrics\ResourceTypes`
 2. Create an example entry for your scraper in `.\samples\promitor-sample.yaml`
-3. Create a new Deserializer in `.\src\Promitor.Core.Scraping\Configuration\Serialization\Deserializers`. This must inherit from `AzureMetricDeserializer`.
+3. Create a new Deserializer in `.\src\Promitor.Core.Scraping\Configuration\Serialization\Deserializers`. This must inherit from `GenericAzureMetricDeserializer`.
 3. Update `Promitor.Core.Scraping.Factories.MetricDeserializerFactory` to handle your new resource type by returning a new instance of the Deserializer you created in the previous step.
 4. Provide a unit test in `.\src\Promitor.Scraper.Tests.Unit\Serialization\MetricsDeclaration\` that tests the deserialization based on our sample
 

docs/configuration/metrics/cosmosdb.md

@@ -0,0 +1,27 @@
+---
+layout: default
+title: Cosmos Db Declaration
+---
+
+## Cosmos Db
+You can declare to scrape Cosmos Db via the `CosmosDb` resource type.
+
+The following fields need to be provided:
+- `dbName`- The name of the Cosmos Db to be scraped
+
+All supported metrics are documented in the official [Azure Monitor documentation](https://docs.microsoft.com/en-us/azure/azure-monitor/platform/metrics-supported#microsoftcontainerinstancecontainergroups).
+
+Example:
+```yaml
+    name: demo_cosmos_totalrequests
+    description: "Demo cosmos query"
+    resourceType: CosmosDb
+    dbName: cognitiveanalytics
+    azureMetricConfiguration:
+      metricName: TotalRequests
+      aggregation:
+        type: Count
+```
+
+[← back to metrics declarations](/configuration/metrics)
+[← back to introduction](/)

docs/configuration/metrics/index.md

@@ -87,6 +87,7 @@ We also provide a simplified way to configure the following Azure resources:
 - [Azure Virtual Machine](virtual-machine)
 - [Azure Network Interface](network-interface)
 - [Azure Storage Queue](storage-queue)
+- [Cosmos DB](cosmosdb)
 
 Want to help out? Create an issue and [contribute a new scraper](https://github.com/tomkerkhove/promitor/blob/master/adding-a-new-scraper.md).
 

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/CosmosDbMetricDefinition.cs

@@ -0,0 +1,9 @@
+namespace Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes
+{
+    public class CosmosDbMetricDefinition : MetricDefinition
+    {
+        public string DbName { get; set; }
+
+        public override ResourceType ResourceType { get; } = ResourceType.CosmosDb;
+    }
+}

src/Promitor.Core.Scraping/Configuration/Model/ResourceType.cs

@@ -10,5 +10,6 @@ public enum ResourceType
         VirtualMachine = 5,
         ContainerRegistry = 6,
         NetworkInterface = 7,
+		CosmosDb = 8
     }
 }

src/Promitor.Core.Scraping/Configuration/Serialization/Core/MetricDeserializer.cs

@@ -1,4 +1,4 @@
-using System.Collections.Generic;
+using System.Collections.Generic;
 using GuardNet;
 using Microsoft.Extensions.Logging;
 using Promitor.Core.Scraping.Configuration.Model.Metrics;

src/Promitor.Core.Scraping/Configuration/Serialization/Deserializers/CosmosDbMetricDeserializer.cs

@@ -0,0 +1,23 @@
+using Promitor.Core.Scraping.Configuration.Model.Metrics;
+using Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes;
+using YamlDotNet.RepresentationModel;
+
+namespace Promitor.Core.Scraping.Configuration.Serialization.Deserializers
+{
+    internal class CosmosDbMetricDeserializer : GenericAzureMetricDeserializer
+    {
+        /// <summary>Deserializes the specified Cosmos DB metric node from the YAML configuration file.</summary>
+        /// <param name="metricNode">The metric node to deserialize to Cosmos DB configuration</param>
+        /// <returns>A new <see cref="MetricDefinition"/> object (strongly typed as a <see cref="CosmosDbMetricDefinition"/>) </returns>
+        internal override MetricDefinition Deserialize(YamlMappingNode metricNode)
+        {
+            var metricDefinition = base.DeserializeMetricDefinition<CosmosDbMetricDefinition>(metricNode);
+
+            var dbName = metricNode.Children[new YamlScalarNode("dbName")];
+
+            metricDefinition.DbName = dbName?.ToString();
+
+            return metricDefinition;
+        }
+    }
+}

src/Promitor.Core.Scraping/Factories/MetricDeserializerFactory.cs

@@ -23,6 +23,8 @@ internal static GenericAzureMetricDeserializer GetDeserializerFor(Configuration.
                     return new ContainerRegistryMetricDeserializer();
                 case Configuration.Model.ResourceType.NetworkInterface:
                     return new NetworkInterfaceMetricDeserializer();
+				case Configuration.Model.ResourceType.CosmosDb:
+                    return new CosmosDbMetricDeserializer();
             }
 
             throw new ArgumentOutOfRangeException($@"Resource Type {resource} not supported.");

src/Promitor.Core.Scraping/Factories/MetricScraperFactory.cs

@@ -40,6 +40,8 @@ public static IScraper<MetricDefinition> CreateScraper(ResourceType metricDefini
                     return new NetworkInterfaceScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker);
                 case ResourceType.ContainerRegistry:
                     return new ContainerRegistryScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker);
+				case ResourceType.CosmosDb:
+                    return new CosmosDbScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker);
                 default:
                     throw new ArgumentOutOfRangeException();
             }

src/Promitor.Core.Scraping/ResourceTypes/CosmosDbScraper.cs

@@ -0,0 +1,31 @@
+using System;
+using System.Threading.Tasks;
+using Microsoft.Azure.Management.Monitor.Fluent.Models;
+using Microsoft.Extensions.Logging;
+using Promitor.Core.Scraping.Configuration.Model;
+using Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes;
+using Promitor.Core.Telemetry.Interfaces;
+using Promitor.Integrations.AzureMonitor;
+
+namespace Promitor.Core.Scraping.ResourceTypes
+{
+    public class CosmosDbScraper : Scraper<CosmosDbMetricDefinition>
+    {
+        private const string ResourceUriTemplate = "subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.DocumentDB/databaseAccounts/{2}";
+
+        public CosmosDbScraper(AzureMetadata azureMetadata, AzureMonitorClient azureMonitorClient, ILogger logger, IExceptionTracker exceptionTracker)
+            : base(azureMetadata, azureMonitorClient, logger, exceptionTracker)
+        {
+        }
+
+        protected override async Task<double> ScrapeResourceAsync(string subscriptionId, string resourceGroupName, CosmosDbMetricDefinition metricDefinition, AggregationType aggregationType, TimeSpan aggregationInterval)
+        {
+            var resourceUri = string.Format(ResourceUriTemplate, subscriptionId, resourceGroupName, metricDefinition.DbName);
+
+            var metricName = metricDefinition.AzureMetricConfiguration.MetricName;
+            var foundMetricValue = await AzureMonitorClient.QueryMetricAsync(metricName, aggregationType, aggregationInterval, resourceUri);
+
+            return foundMetricValue;
+        }
+    }
+}

src/Promitor.Core.Scraping/Scraper.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Threading.Tasks;
 using GuardNet;
 using Microsoft.Azure.Management.Monitor.Fluent.Models;

src/Promitor.Scraper.Host/Validation/Factories/MetricValidatorFactory.cs

@@ -25,6 +25,8 @@ internal static IMetricValidator GetValidatorFor(ResourceType resourceType)
                     return new NetworkInterfaceMetricValidator();
                 case ResourceType.ContainerRegistry:
                     return new ContainerRegistryMetricValidator();
+                case ResourceType.CosmosDb:
+                    return new CosmosDbMetricValidator();
             }
 
             throw new ArgumentOutOfRangeException(nameof(resourceType), $"No validation rules are defined for metric type '{resourceType}'");

src/Promitor.Scraper.Host/Validation/MetricDefinitions/ResourceTypes/CosmosDbMetricValidator.cs

@@ -0,0 +1,19 @@
+using System.Collections.Generic;
+using GuardNet;
+using Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes;
+
+namespace Promitor.Scraper.Host.Validation.MetricDefinitions.ResourceTypes
+{
+    internal class CosmosDbMetricValidator : MetricValidator<CosmosDbMetricDefinition>
+    {
+        protected override IEnumerable<string> Validate(CosmosDbMetricDefinition cosmosDbMetricDefinition)
+        {
+            Guard.NotNull(cosmosDbMetricDefinition, nameof(cosmosDbMetricDefinition));
+
+            if (string.IsNullOrWhiteSpace(cosmosDbMetricDefinition.DbName))
+            {
+                yield return "No Cosmos db Name is configured";
+            }
+        }
+    }
+}

src/Promitor.Scraper.Tests.Unit/Builders/MetricsDeclarationBuilder.cs

@@ -106,6 +106,21 @@ public string Build()
             return this;
         }
 
+        public MetricsDeclarationBuilder WithCosmosDbMetric(string metricName = "promitor-cosmosdb", string metricDescription = "Description for a metric", string dbName = "promitor-cosmosdb", string azureMetricName = "TotalRequests")
+        {
+            var azureMetricConfiguration = CreateAzureMetricConfiguration(azureMetricName);
+            var metric = new CosmosDbMetricDefinition
+            {
+                Name = metricName,
+                Description = metricDescription,
+                DbName = dbName,
+                AzureMetricConfiguration = azureMetricConfiguration
+            };
+            _metrics.Add(metric);
+
+            return this;
+        }
+
         public MetricsDeclarationBuilder WithAzureStorageQueueMetric(string metricName = "promitor", string metricDescription = "Description for a metric", string queueName = "promitor-queue", string accountName = "promitor-account", string sasToken = "?sig=promitor", string azureMetricName = AzureStorageConstants.Queues.Metrics.MessageCount)
         {
             var azureMetricConfiguration = CreateAzureMetricConfiguration(azureMetricName);

src/Promitor.Scraper.Tests.Unit/Serialization/MetricsDeclaration/MetricsDeclarationWithCosmosDbYamlSerializationTests.cs

@@ -0,0 +1,78 @@
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Linq;
+using Bogus;
+using Microsoft.Extensions.Logging.Abstractions;
+using Promitor.Core.Scraping.Configuration.Model;
+using Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes;
+using Promitor.Core.Scraping.Configuration.Serialization.Core;
+using Xunit;
+using MetricDefinition = Promitor.Core.Scraping.Configuration.Model.Metrics.MetricDefinition;
+
+namespace Promitor.Scraper.Tests.Unit.Serialization.MetricsDeclaration
+{
+    [Category("Unit")]
+    public class MetricsDeclarationWithCosmosDbYamlSerializationTests : YamlSerializationTests<CosmosDbMetricDefinition>
+    {
+        [Theory]
+        [InlineData("promitor1", @"* */1 * * * *", @"* */2 * * * *")]
+        [InlineData(null, null, null)]
+        public void YamlSerialization_SerializeAndDeserializeValidConfigForCosmosDb_SucceedsWithIdenticalOutput(string resourceGroupName, string defaultScrapingInterval, string metricScrapingInterval)
+        {
+            // Arrange
+            var azureMetadata = GenerateBogusAzureMetadata();
+            var cosmosDbMetricDefinition = GenerateBogusCosmosDbMetricDefinition(resourceGroupName, metricScrapingInterval);
+            var metricDefaults = GenerateBogusMetricDefaults(defaultScrapingInterval);
+            var scrapingConfiguration = new Core.Scraping.Configuration.Model.MetricsDeclaration
+            {
+                AzureMetadata = azureMetadata,
+                MetricDefaults = metricDefaults,
+                Metrics = new List<MetricDefinition>
+                {
+                    cosmosDbMetricDefinition
+                }
+            };
+            var configurationSerializer = new ConfigurationSerializer(NullLogger.Instance);
+
+            // Act
+            var serializedConfiguration = configurationSerializer.Serialize(scrapingConfiguration);
+            var deserializedConfiguration = configurationSerializer.Deserialize(serializedConfiguration);
+
+            // Assert
+            Assert.NotNull(deserializedConfiguration);
+            AssertAzureMetadata(deserializedConfiguration, azureMetadata);
+            AssertMetricDefaults(deserializedConfiguration, metricDefaults);
+            Assert.NotNull(deserializedConfiguration.Metrics);
+            Assert.Single(deserializedConfiguration.Metrics);
+            var deserializedMetricDefinition = deserializedConfiguration.Metrics.FirstOrDefault();
+            AssertMetricDefinition(deserializedMetricDefinition, cosmosDbMetricDefinition);
+            var deserializedCosmosDbMetricDefinition = deserializedMetricDefinition as CosmosDbMetricDefinition;
+            AssertCosmosDbMetricDefinition(deserializedCosmosDbMetricDefinition, cosmosDbMetricDefinition);
+        }
+
+        private static void AssertCosmosDbMetricDefinition(CosmosDbMetricDefinition deserializedCosmosDbMetricDefinition, CosmosDbMetricDefinition cosmosDbMetricDefinition)
+        {
+            Assert.NotNull(deserializedCosmosDbMetricDefinition);
+            Assert.Equal(cosmosDbMetricDefinition.DbName, deserializedCosmosDbMetricDefinition.DbName);
+        }
+
+        private CosmosDbMetricDefinition GenerateBogusCosmosDbMetricDefinition(string resourceGroupName, string metricScrapingInterval)
+        {
+            var bogusScrapingInterval = GenerateBogusScrapingInterval(metricScrapingInterval);
+            var bogusAzureMetricConfiguration = GenerateBogusAzureMetricConfiguration();
+
+            var bogusGenerator = new Faker<CosmosDbMetricDefinition>()
+                .StrictMode(ensureRulesForAllProperties: true)
+                .RuleFor(metricDefinition => metricDefinition.Name, faker => faker.Name.FirstName())
+                .RuleFor(metricDefinition => metricDefinition.Description, faker => faker.Lorem.Sentence(wordCount: 6))
+                .RuleFor(metricDefinition => metricDefinition.ResourceType, faker => ResourceType.CosmosDb)
+                .RuleFor(metricDefinition => metricDefinition.DbName, faker => faker.Name.LastName())
+                .RuleFor(metricDefinition => metricDefinition.AzureMetricConfiguration, faker => bogusAzureMetricConfiguration)
+                .RuleFor(metricDefinition => metricDefinition.ResourceGroupName, faker => resourceGroupName)
+                .RuleFor(metricDefinition => metricDefinition.Scraping, faker => bogusScrapingInterval)
+                .Ignore(metricDefinition => metricDefinition.ResourceGroupName);
+
+            return bogusGenerator.Generate();
+        }
+    }
+}

src/Promitor.Scraper.Tests.Unit/Validation/Metrics/ResourceTypes/CosmosDbMetricsDeclarationValidationStepTests.cs

@@ -0,0 +1,97 @@
+using System.ComponentModel;
+using Promitor.Scraper.Host.Validation.Steps;
+using Promitor.Scraper.Tests.Unit.Builders;
+using Promitor.Scraper.Tests.Unit.Stubs;
+using Xunit;
+
+namespace Promitor.Scraper.Tests.Unit.Validation.Metrics.ResourceTypes
+{
+    [Category("Unit")]
+    public class CosmosDbMetricsDeclarationValidationStepTests
+    {
+        [Fact]
+        public void CosmosDbMetricsDeclaration_DeclarationWithoutAzureMetricName_Succeeds()
+        {
+            // Arrange
+            var rawDeclaration = MetricsDeclarationBuilder.WithMetadata()
+                .WithCosmosDbMetric(azureMetricName: string.Empty)
+                .Build();
+            var metricsDeclarationProvider = new MetricsDeclarationProviderStub(rawDeclaration);
+
+            // Act
+            var scrapingScheduleValidationStep = new MetricsDeclarationValidationStep(metricsDeclarationProvider);
+            var validationResult = scrapingScheduleValidationStep.Run();
+
+            // Assert
+            Assert.False(validationResult.IsSuccessful, "Validation is successful");
+        }
+
+        [Fact]
+        public void CosmosDbMetricsDeclaration_DeclarationWithoutMetricDescription_Succeeded()
+        {
+            // Arrange
+            var rawDeclaration = MetricsDeclarationBuilder.WithMetadata()
+                .WithCosmosDbMetric(metricDescription: string.Empty)
+                .Build();
+            var metricsDeclarationProvider = new MetricsDeclarationProviderStub(rawDeclaration);
+
+            // Act
+            var scrapingScheduleValidationStep = new MetricsDeclarationValidationStep(metricsDeclarationProvider);
+            var validationResult = scrapingScheduleValidationStep.Run();
+
+            // Assert
+            Assert.True(validationResult.IsSuccessful, "Validation was not successful");
+        }
+
+        [Fact]
+        public void CosmosDbMetricsDeclaration_DeclarationWithoutMetricName_Fails()
+        {
+            // Arrange
+            var rawDeclaration = MetricsDeclarationBuilder.WithMetadata()
+                .WithCosmosDbMetric(string.Empty)
+                .Build();
+            var metricsDeclarationProvider = new MetricsDeclarationProviderStub(rawDeclaration);
+
+            // Act
+            var scrapingScheduleValidationStep = new MetricsDeclarationValidationStep(metricsDeclarationProvider);
+            var validationResult = scrapingScheduleValidationStep.Run();
+
+            // Assert
+            Assert.False(validationResult.IsSuccessful, "Validation is successful");
+        }
+
+        [Fact]
+        public void CosmosDbMetricsDeclaration_DeclarationWithoutDbName_Fails()
+        {
+            // Arrange
+            var rawDeclaration = MetricsDeclarationBuilder.WithMetadata()
+                .WithCosmosDbMetric(dbName: string.Empty)
+                .Build();
+            var metricsDeclarationProvider = new MetricsDeclarationProviderStub(rawDeclaration);
+
+            // Act
+            var scrapingScheduleValidationStep = new MetricsDeclarationValidationStep(metricsDeclarationProvider);
+            var validationResult = scrapingScheduleValidationStep.Run();
+
+            // Assert
+            Assert.False(validationResult.IsSuccessful, "Validation is successful");
+        }
+
+        [Fact]
+        public void CosmosDbMetricsDeclaration_ValidDeclaration_Succeeds()
+        {
+            // Arrange
+            var rawMetricsDeclaration = MetricsDeclarationBuilder.WithMetadata()
+                .WithCosmosDbMetric()
+                .Build();
+            var metricsDeclarationProvider = new MetricsDeclarationProviderStub(rawMetricsDeclaration);
+
+            // Act
+            var scrapingScheduleValidationStep = new MetricsDeclarationValidationStep(metricsDeclarationProvider);
+            var validationResult = scrapingScheduleValidationStep.Run();
+
+            // Assert
+            Assert.True(validationResult.IsSuccessful, "Validation was not successful");
+        }
+    }
+}