Merge pull request #24 from brminnick/Optimize-GetMicrosoftLearnContr…

…ibutors-API

Optimize GetMicrosoftLearnContributors API

AzureAdvocates.Functions/AzureAdvocates.Functions.csproj

@@ -3,14 +3,17 @@
     <TargetFramework>netcoreapp3.1</TargetFramework>
     <AzureFunctionsVersion>v3</AzureFunctionsVersion>
     <LangVersion>latest</LangVersion>
+    <Nullable>enable</Nullable>
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="GitHubApiStatus.Extensions" Version="2.0.0-pre9" />
     <PackageReference Include="Microsoft.Azure.Functions.Extensions" Version="1.1.0" />
     <PackageReference Include="Microsoft.NET.Sdk.Functions" Version="3.0.11" />
   </ItemGroup>
   <ItemGroup>
-    <ProjectReference Include="..\GitHubReadmeWebTrends.Common\GitHubReadmeWebTrends.Common.csproj" />
+    <ProjectReference Include="..\GitHubReadmeWebTrends.Common\GitHubReadmeWebTrends.Common.csproj">
+      <GlobalPropertiesToRemove></GlobalPropertiesToRemove>
+    </ProjectReference>
   </ItemGroup>
   <ItemGroup>
     <None Update="host.json">

AzureAdvocates.Functions/Functions/GetMicrosoftLearnContributors.cs

@@ -1,80 +1,43 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Net;
 using System.Net.Http;
-using System.Net.Http.Headers;
-using System.Threading;
 using System.Threading.Tasks;
-using GitHubApiStatus;
-using GitHubReadmeWebTrends.Common;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Azure.WebJobs;
 using Microsoft.Azure.WebJobs.Extensions.Http;
 using Microsoft.Extensions.Logging;
 
-namespace GitHubReadmeWebTrends.Functions
+namespace AzureAdvocates.Functions
 {
     class GetMicrosoftLearnContributors
     {
-        const string _defaultTeam = "";
+        readonly BlobStorageService _blobStorageService;
 
-        readonly CloudAdvocateService _cloudAdvocateService;
-        readonly IGitHubApiStatusService _gitHubApiStatusService;
-        readonly GitHubGraphQLApiService _gitHubGraphQLApiService;
-
-        public GetMicrosoftLearnContributors(CloudAdvocateService cloudAdvocateService,
-                                                IGitHubApiStatusService gitHubApiStatusService,
-                                                GitHubGraphQLApiService gitHubGraphQLApiService)
-        {
-            _cloudAdvocateService = cloudAdvocateService;
-            _gitHubApiStatusService = gitHubApiStatusService;
-            _gitHubGraphQLApiService = gitHubGraphQLApiService;
-        }
+        public GetMicrosoftLearnContributors(BlobStorageService blobStorageService) => _blobStorageService = blobStorageService;
 
         [FunctionName(nameof(GetMicrosoftLearnContributors))]
         public async Task<IActionResult> Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = nameof(GetMicrosoftLearnContributors) + "/{from:datetime}/{to:datetime}/{team?}")] HttpRequestMessage req, DateTime from, DateTime to, string? team, ILogger log)
         {
             log.LogInformation($"{nameof(GetMicrosoftLearnContributors)} Started");
 
-            var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(2));
-            var gitHubApiStatus = await _gitHubApiStatusService.GetApiRateLimits(cancellationTokenSource.Token).ConfigureAwait(false);
-            if (gitHubApiStatus.GraphQLApi.RemainingRequestCount < 4000)
-            {
-                return new ObjectResult($"Maximum GitHub API Limit Reached. GitHub API Limit will reset in {gitHubApiStatus.GraphQLApi.RateLimitReset_TimeRemaining.Minutes + 1} minute(s). Try again at {gitHubApiStatus.GraphQLApi.RateLimitReset_DateTime.UtcDateTime} UTC")
-                {
-                    StatusCode = (int)HttpStatusCode.Forbidden
-                };
-            }
+            var microsoftLearnContributionsList = await _blobStorageService.GetCloudAdvocateMicrosoftLearnContributors().ConfigureAwait(false);
 
-            var cloudAdvocateList = new List<CloudAdvocateGitHubUserModel>();
-            await foreach (var advocate in _cloudAdvocateService.GetAzureAdvocates().ConfigureAwait(false))
+            var filteredCloudAdvocateContributions = new List<CloudAdvocateGitHubContributorModel>();
+            foreach (var advocateContribution in microsoftLearnContributionsList)
             {
-                if (team is null || advocate.MicrosoftTeam.Equals(team, StringComparison.OrdinalIgnoreCase))
+                if (team is null || advocateContribution.MicrosoftTeam.Equals(team, StringComparison.OrdinalIgnoreCase))
                 {
-                    log.LogInformation($"Found Advocate: {advocate.FullName}");
-                    cloudAdvocateList.Add(advocate);
-                }
-            }
+                    log.LogInformation($"Adding Advocate: {advocateContribution.FullName}");
 
-            var microsoftLearnPullRequests = new List<RepositoryPullRequest>();
-            await foreach (var pullRequestList in _gitHubGraphQLApiService.GetMicrosoftLearnPullRequests().ConfigureAwait(false))
-            {
-                microsoftLearnPullRequests.AddRange(pullRequestList);
-                log.LogInformation($"Added {pullRequestList.Count} Pull Requests from {pullRequestList.FirstOrDefault()?.RepositoryName}");
-            }
+                    var filteredPullRequests = advocateContribution.PullRequests.Where(x => x.CreatedAt.IsWithinRange(from, to)).ToList();
+                    var filteredCloudAdvocateContribution = new CloudAdvocateGitHubContributorModel(filteredPullRequests, advocateContribution.FullName, advocateContribution.GitHubUserName, advocateContribution.MicrosoftAlias, advocateContribution.MicrosoftTeam);
 
-            var cloudAdvocateContributions = new List<GitHubContributorModel>();
-            foreach (var cloudAdvocate in cloudAdvocateList)
-            {
-                var cloudAdvocateContributorModel = new GitHubContributorModel(microsoftLearnPullRequests.Where(x => x.Author.Equals(cloudAdvocate.GitHubUserName, StringComparison.OrdinalIgnoreCase) && x.CreatedAt.IsWithinRange(from, to)), cloudAdvocate);
-
-                cloudAdvocateContributions.Add(cloudAdvocateContributorModel);
-
-                log.LogInformation($"Added {cloudAdvocateContributorModel.PullRequests.Count} Pull Requests for {cloudAdvocate.FullName}");
+                    filteredCloudAdvocateContributions.Add(filteredCloudAdvocateContribution);
+                }
             }
 
-            return new OkObjectResult(cloudAdvocateContributions);
+            return new OkObjectResult(filteredCloudAdvocateContributions);
         }
     }
 

AzureAdvocates.Functions/Functions/UpdateMicrosoftLearnContributors.cs

@@ -0,0 +1,71 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using GitHubApiStatus;
+using GitHubReadmeWebTrends.Common;
+using Microsoft.Azure.WebJobs;
+using Microsoft.Extensions.Logging;
+
+namespace AzureAdvocates.Functions
+{
+    class UpdateMicrosoftLearnContributors
+    {
+        readonly BlobStorageService _blobStorageService;
+        readonly CloudAdvocateService _cloudAdvocateService;
+        readonly IGitHubApiStatusService _gitHubApiStatusService;
+        readonly GitHubGraphQLApiService _gitHubGraphQLApiService;
+
+        public UpdateMicrosoftLearnContributors(BlobStorageService blobStorageService,
+                                                CloudAdvocateService cloudAdvocateService,
+                                                IGitHubApiStatusService gitHubApiStatusService,
+                                                GitHubGraphQLApiService gitHubGraphQLApiService)
+        {
+            _blobStorageService = blobStorageService;
+            _cloudAdvocateService = cloudAdvocateService;
+            _gitHubApiStatusService = gitHubApiStatusService;
+            _gitHubGraphQLApiService = gitHubGraphQLApiService;
+        }
+
+        [FunctionName(nameof(UpdateMicrosoftLearnContributors))]
+        public async Task Run([TimerTrigger("0 0 */6 * * *")] TimerInfo timer, ILogger log)
+        {
+            log.LogInformation($"{nameof(UpdateMicrosoftLearnContributors)} Started");
+
+            var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(2));
+            var gitHubApiStatus = await _gitHubApiStatusService.GetApiRateLimits(cancellationTokenSource.Token).ConfigureAwait(false);
+            if (gitHubApiStatus.GraphQLApi.RemainingRequestCount < 4000)
+            {
+                log.LogError($"Maximum GitHub API Limit Reached. GitHub API Limit will reset in {gitHubApiStatus.GraphQLApi.RateLimitReset_TimeRemaining.Minutes + 1} minute(s). Try again at {gitHubApiStatus.GraphQLApi.RateLimitReset_DateTime.UtcDateTime} UTC");
+                return;
+            }
+
+            var cloudAdvocateList = new List<CloudAdvocateGitHubUserModel>();
+            await foreach (var advocate in _cloudAdvocateService.GetAzureAdvocates().ConfigureAwait(false))
+            {
+                cloudAdvocateList.Add(advocate);
+            }
+
+            var microsoftLearnPullRequests = new List<RepositoryPullRequest>();
+            await foreach (var pullRequestList in _gitHubGraphQLApiService.GetMicrosoftLearnPullRequests().ConfigureAwait(false))
+            {
+                microsoftLearnPullRequests.AddRange(pullRequestList);
+                log.LogInformation($"Added {pullRequestList.Count} Pull Requests from {pullRequestList.FirstOrDefault()?.RepositoryName}");
+            }
+
+            var cloudAdvocateContributions = new List<CloudAdvocateGitHubContributorModel>();
+            foreach (var cloudAdvocate in cloudAdvocateList)
+            {
+                var cloudAdvocateContributorModel = new CloudAdvocateGitHubContributorModel(microsoftLearnPullRequests.Where(x => x.Author.Equals(cloudAdvocate.GitHubUserName, StringComparison.OrdinalIgnoreCase)), cloudAdvocate);
+
+                cloudAdvocateContributions.Add(cloudAdvocateContributorModel);
+
+                log.LogInformation($"Added {cloudAdvocateContributorModel.PullRequests.Count} Pull Requests for {cloudAdvocate.FullName}");
+            }
+
+            var blobName = $"Contributions_{DateTime.UtcNow:o}.json";
+            await _blobStorageService.UploadCloudAdvocateMicrosoftLearnContributions(cloudAdvocateContributions, blobName).ConfigureAwait(false);
+        }
+    }
+}

AzureAdvocates.Functions/Models/GitHubContributorModel.cs

@@ -1,23 +1,26 @@
 using System.Collections.Generic;
 using System.Linq;
 using GitHubReadmeWebTrends.Common;
+using Newtonsoft.Json;
 
-namespace GitHubReadmeWebTrends.Functions
+namespace AzureAdvocates.Functions
 {
-    class GitHubContributorModel : CloudAdvocateGitHubUserModel
+    class CloudAdvocateGitHubContributorModel : CloudAdvocateGitHubUserModel
     {
-        public GitHubContributorModel(in IEnumerable<RepositoryPullRequest> pullReuests, CloudAdvocateGitHubUserModel cloudAdvocateGitHubUserModel)
-            : this(pullReuests, cloudAdvocateGitHubUserModel.FullName, cloudAdvocateGitHubUserModel.GitHubUserName, cloudAdvocateGitHubUserModel.MicrosoftAlias, cloudAdvocateGitHubUserModel.MicrosoftTeam)
+        public CloudAdvocateGitHubContributorModel(IEnumerable<RepositoryPullRequest> pullRequests, CloudAdvocateGitHubUserModel cloudAdvocateGitHubUserModel)
+            : this(pullRequests, cloudAdvocateGitHubUserModel.FullName, cloudAdvocateGitHubUserModel.GitHubUserName, cloudAdvocateGitHubUserModel.MicrosoftAlias, cloudAdvocateGitHubUserModel.MicrosoftTeam)
         {
 
         }
 
-        public GitHubContributorModel(in IEnumerable<RepositoryPullRequest> pullReuests, in string fullName, in string gitHubUserName, in string microsoftAlias, in string microsoftTeam)
+        [JsonConstructor]
+        public CloudAdvocateGitHubContributorModel(IEnumerable<RepositoryPullRequest> pullRequests, string fullName, string gitHubUserName, string microsoftAlias, string microsoftTeam)
             : base(fullName, gitHubUserName, microsoftAlias, microsoftTeam)
         {
-            PullRequests = pullReuests.ToList();
+            PullRequests = pullRequests.ToList(); ;
         }
 
+        [JsonProperty("pullRequests")]
         public IReadOnlyList<RepositoryPullRequest> PullRequests { get; }
     }
 }

AzureAdvocates.Functions/Services/BlobStorageService.cs

@@ -0,0 +1,63 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using System.Collections.Generic;
+using Microsoft.WindowsAzure.Storage.Blob;
+using Newtonsoft.Json;
+
+namespace AzureAdvocates.Functions
+{
+    class BlobStorageService
+    {
+        const string _microsoftLearnContributionsContainerName = "cloudadvocatemicrosoftlearncontributions";
+        readonly CloudBlobClient _blobClient;
+
+        public BlobStorageService(CloudBlobClient cloudBlobClient) => _blobClient = cloudBlobClient;
+
+        public Task UploadCloudAdvocateMicrosoftLearnContributions(IEnumerable<CloudAdvocateGitHubContributorModel> azureDataCenterIpRangeModel, string blobName)
+        {
+            var container = GetBlobContainer(_microsoftLearnContributionsContainerName);
+            var blob = container.GetBlockBlobReference(blobName);
+
+            return blob.UploadTextAsync(JsonConvert.SerializeObject(azureDataCenterIpRangeModel));
+        }
+
+        public async Task<IReadOnlyList<CloudAdvocateGitHubContributorModel>> GetCloudAdvocateMicrosoftLearnContributors()
+        {
+            var blobList = new List<CloudBlockBlob>();
+            await foreach (var blob in GetBlobs<CloudBlockBlob>(_microsoftLearnContributionsContainerName).ConfigureAwait(false))
+            {
+                blobList.Add(blob);
+            }
+
+            var gitHubContributorListBlob = blobList.OrderByDescending(x => x.Properties.Created).First();
+            var serializedGitHubContributorList = await gitHubContributorListBlob.DownloadTextAsync().ConfigureAwait(false);
+
+            return JsonConvert.DeserializeObject<IReadOnlyList<CloudAdvocateGitHubContributorModel>>(serializedGitHubContributorList) ?? throw new NullReferenceException();
+        }
+
+        async IAsyncEnumerable<T> GetBlobs<T>(string containerName, string prefix = "", int? maxresultsPerQuery = null, BlobListingDetails blobListingDetails = BlobListingDetails.None) where T : ICloudBlob
+        {
+            var blobContainer = GetBlobContainer(containerName);
+
+            BlobContinuationToken? continuationToken = null;
+
+            do
+            {
+                var response = await blobContainer.ListBlobsSegmentedAsync(prefix, true, blobListingDetails, maxresultsPerQuery, continuationToken, null, null).ConfigureAwait(false);
+                continuationToken = response?.ContinuationToken;
+
+                var blobListFromResponse = response?.Results?.OfType<T>() ?? Enumerable.Empty<T>();
+
+                foreach (var blob in blobListFromResponse)
+                {
+                    yield return blob;
+                }
+
+            } while (continuationToken != null);
+
+        }
+
+        CloudBlobContainer GetBlobContainer(string containerName) => _blobClient.GetContainerReference(containerName);
+    }
+}

AzureAdvocates.Functions/Startup.cs

@@ -2,14 +2,24 @@
 using AzureAdvocates.Functions;
 using GitHubReadmeWebTrends.Common;
 using Microsoft.Azure.Functions.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.WindowsAzure.Storage;
+using Microsoft.WindowsAzure.Storage.Blob;
 
 [assembly: FunctionsStartup(typeof(Startup))]
 namespace AzureAdvocates.Functions
 {
     class Startup : FunctionsStartup
     {
         readonly static string _token = Environment.GetEnvironmentVariable("Token") ?? string.Empty;
+        static readonly string _storageConnectionString = Environment.GetEnvironmentVariable("AzureWebJobsStorage") ?? string.Empty;
 
-        public override void Configure(IFunctionsHostBuilder builder) => StartupService.ConfigureServices(builder.Services, _token);
+        public override void Configure(IFunctionsHostBuilder builder)
+        {
+            builder.Services.AddSingleton<CloudBlobClient>(CloudStorageAccount.Parse(_storageConnectionString).CreateCloudBlobClient());
+            builder.Services.AddSingleton<BlobStorageService>();
+
+            StartupService.ConfigureServices(builder.Services, _token);
+        }
     }
 }

GitHubReadmeWebTrends.Common/Models/CloudAdvocateGitHubUserModel.cs

@@ -1,4 +1,6 @@
-namespace GitHubReadmeWebTrends.Common
+using Newtonsoft.Json;
+
+namespace GitHubReadmeWebTrends.Common
 {
     public class CloudAdvocateGitHubUserModel
     {
@@ -10,9 +12,16 @@ public CloudAdvocateGitHubUserModel(in string fullName, in string gitHubUserName
             MicrosoftTeam = microsoftTeam;
         }
 
+        [JsonProperty("fullName")]
         public string FullName { get; }
+
+        [JsonProperty("gitHubUserName")]
         public string GitHubUserName { get; }
+
+        [JsonProperty("microsoftAlias")]
         public string MicrosoftAlias { get; }
+
+        [JsonProperty("microsoftTeam")]
         public string MicrosoftTeam { get; }
     }
 }

GitHubReadmeWebTrends.Common/Models/PullRequest.cs

@@ -1,9 +1,11 @@
 using System;
+using Newtonsoft.Json;
+
 namespace GitHubReadmeWebTrends.Common
 {
     public class PullRequest
     {
-        [Newtonsoft.Json.JsonConstructor]
+        [JsonConstructor]
         public PullRequest(string id, Uri url, DateTimeOffset createdAt, bool merged, DateTimeOffset? mergedAt, string baseRefName, Author author)
             : this(id, url, createdAt, merged, mergedAt, baseRefName, author?.Login ?? string.Empty)
         {
@@ -21,25 +23,48 @@ public PullRequest(string id, Uri uri, DateTimeOffset createdAt, bool merged, Da
             Author = author;
         }
 
+        [JsonProperty("id")]
         public string Id { get; }
+
+        [JsonProperty("url")]
         public Uri Uri { get; }
+
+        [JsonProperty("createdAt")]
         public DateTimeOffset CreatedAt { get; }
+
+        [JsonProperty("merged")]
         public bool IsMerged { get; }
+
+        [JsonProperty("mergedAt")]
         public DateTimeOffset? MergedAt { get; }
+
+        [JsonProperty("baseRefName")]
         public string BaseRefName { get; }
+
+        [JsonProperty("author")]
         public string Author { get; }
     }
 
     public class RepositoryPullRequest : PullRequest
     {
         public RepositoryPullRequest(string repositoryName, string repositoryOwner, PullRequest pullRequest)
-            : base(pullRequest.Id, pullRequest.Uri, pullRequest.CreatedAt, pullRequest.IsMerged, pullRequest.MergedAt, pullRequest.BaseRefName, pullRequest.Author)
+            : this(repositoryName, repositoryOwner, pullRequest.Id, pullRequest.Uri, pullRequest.CreatedAt, pullRequest.IsMerged, pullRequest.MergedAt, pullRequest.BaseRefName, pullRequest.Author)
+        {
+
+        }
+
+        [JsonConstructor]
+        public RepositoryPullRequest(string repositoryName, string repositoryOwner, string id, Uri url, DateTimeOffset createdAt, bool merged, DateTimeOffset? mergedAt, string baseRefName, string author)
+            : base(id, url, createdAt, merged, mergedAt, baseRefName, author)
         {
             RepositoryName = repositoryName;
             RepositoryOwner = repositoryOwner;
         }
 
+        [JsonProperty("repositoryName")]
         public string RepositoryName { get; }
+
+        [JsonProperty("repositoryOwner")]
         public string RepositoryOwner { get; }
     }
 }