diff --git a/AzureAdvocates.Functions/AzureAdvocates.Functions.csproj b/AzureAdvocates.Functions/AzureAdvocates.Functions.csproj index 3ebf653..fb43669 100644 --- a/AzureAdvocates.Functions/AzureAdvocates.Functions.csproj +++ b/AzureAdvocates.Functions/AzureAdvocates.Functions.csproj @@ -3,6 +3,7 @@ netcoreapp3.1 v3 latest + enable @@ -10,7 +11,9 @@ - + + + diff --git a/AzureAdvocates.Functions/Functions/GetMicrosoftLearnContributors.cs b/AzureAdvocates.Functions/Functions/GetMicrosoftLearnContributors.cs index 9497b12..3d5ad74 100644 --- a/AzureAdvocates.Functions/Functions/GetMicrosoftLearnContributors.cs +++ b/AzureAdvocates.Functions/Functions/GetMicrosoftLearnContributors.cs @@ -1,80 +1,43 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Net; using System.Net.Http; -using System.Net.Http.Headers; -using System.Threading; using System.Threading.Tasks; -using GitHubApiStatus; -using GitHubReadmeWebTrends.Common; using Microsoft.AspNetCore.Mvc; using Microsoft.Azure.WebJobs; using Microsoft.Azure.WebJobs.Extensions.Http; using Microsoft.Extensions.Logging; -namespace GitHubReadmeWebTrends.Functions +namespace AzureAdvocates.Functions { class GetMicrosoftLearnContributors { - const string _defaultTeam = ""; + readonly BlobStorageService _blobStorageService; - readonly CloudAdvocateService _cloudAdvocateService; - readonly IGitHubApiStatusService _gitHubApiStatusService; - readonly GitHubGraphQLApiService _gitHubGraphQLApiService; - - public GetMicrosoftLearnContributors(CloudAdvocateService cloudAdvocateService, - IGitHubApiStatusService gitHubApiStatusService, - GitHubGraphQLApiService gitHubGraphQLApiService) - { - _cloudAdvocateService = cloudAdvocateService; - _gitHubApiStatusService = gitHubApiStatusService; - _gitHubGraphQLApiService = gitHubGraphQLApiService; - } + public GetMicrosoftLearnContributors(BlobStorageService blobStorageService) => _blobStorageService = blobStorageService; [FunctionName(nameof(GetMicrosoftLearnContributors))] public async Task Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = nameof(GetMicrosoftLearnContributors) + "/{from:datetime}/{to:datetime}/{team?}")] HttpRequestMessage req, DateTime from, DateTime to, string? team, ILogger log) { log.LogInformation($"{nameof(GetMicrosoftLearnContributors)} Started"); - var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(2)); - var gitHubApiStatus = await _gitHubApiStatusService.GetApiRateLimits(cancellationTokenSource.Token).ConfigureAwait(false); - if (gitHubApiStatus.GraphQLApi.RemainingRequestCount < 4000) - { - return new ObjectResult($"Maximum GitHub API Limit Reached. GitHub API Limit will reset in {gitHubApiStatus.GraphQLApi.RateLimitReset_TimeRemaining.Minutes + 1} minute(s). Try again at {gitHubApiStatus.GraphQLApi.RateLimitReset_DateTime.UtcDateTime} UTC") - { - StatusCode = (int)HttpStatusCode.Forbidden - }; - } + var microsoftLearnContributionsList = await _blobStorageService.GetCloudAdvocateMicrosoftLearnContributors().ConfigureAwait(false); - var cloudAdvocateList = new List(); - await foreach (var advocate in _cloudAdvocateService.GetAzureAdvocates().ConfigureAwait(false)) + var filteredCloudAdvocateContributions = new List(); + foreach (var advocateContribution in microsoftLearnContributionsList) { - if (team is null || advocate.MicrosoftTeam.Equals(team, StringComparison.OrdinalIgnoreCase)) + if (team is null || advocateContribution.MicrosoftTeam.Equals(team, StringComparison.OrdinalIgnoreCase)) { - log.LogInformation($"Found Advocate: {advocate.FullName}"); - cloudAdvocateList.Add(advocate); - } - } + log.LogInformation($"Adding Advocate: {advocateContribution.FullName}"); - var microsoftLearnPullRequests = new List(); - await foreach (var pullRequestList in _gitHubGraphQLApiService.GetMicrosoftLearnPullRequests().ConfigureAwait(false)) - { - microsoftLearnPullRequests.AddRange(pullRequestList); - log.LogInformation($"Added {pullRequestList.Count} Pull Requests from {pullRequestList.FirstOrDefault()?.RepositoryName}"); - } + var filteredPullRequests = advocateContribution.PullRequests.Where(x => x.CreatedAt.IsWithinRange(from, to)).ToList(); + var filteredCloudAdvocateContribution = new CloudAdvocateGitHubContributorModel(filteredPullRequests, advocateContribution.FullName, advocateContribution.GitHubUserName, advocateContribution.MicrosoftAlias, advocateContribution.MicrosoftTeam); - var cloudAdvocateContributions = new List(); - foreach (var cloudAdvocate in cloudAdvocateList) - { - var cloudAdvocateContributorModel = new GitHubContributorModel(microsoftLearnPullRequests.Where(x => x.Author.Equals(cloudAdvocate.GitHubUserName, StringComparison.OrdinalIgnoreCase) && x.CreatedAt.IsWithinRange(from, to)), cloudAdvocate); - - cloudAdvocateContributions.Add(cloudAdvocateContributorModel); - - log.LogInformation($"Added {cloudAdvocateContributorModel.PullRequests.Count} Pull Requests for {cloudAdvocate.FullName}"); + filteredCloudAdvocateContributions.Add(filteredCloudAdvocateContribution); + } } - return new OkObjectResult(cloudAdvocateContributions); + return new OkObjectResult(filteredCloudAdvocateContributions); } } diff --git a/AzureAdvocates.Functions/Functions/UpdateMicrosoftLearnContributors.cs b/AzureAdvocates.Functions/Functions/UpdateMicrosoftLearnContributors.cs new file mode 100644 index 0000000..8390e64 --- /dev/null +++ b/AzureAdvocates.Functions/Functions/UpdateMicrosoftLearnContributors.cs @@ -0,0 +1,71 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using GitHubApiStatus; +using GitHubReadmeWebTrends.Common; +using Microsoft.Azure.WebJobs; +using Microsoft.Extensions.Logging; + +namespace AzureAdvocates.Functions +{ + class UpdateMicrosoftLearnContributors + { + readonly BlobStorageService _blobStorageService; + readonly CloudAdvocateService _cloudAdvocateService; + readonly IGitHubApiStatusService _gitHubApiStatusService; + readonly GitHubGraphQLApiService _gitHubGraphQLApiService; + + public UpdateMicrosoftLearnContributors(BlobStorageService blobStorageService, + CloudAdvocateService cloudAdvocateService, + IGitHubApiStatusService gitHubApiStatusService, + GitHubGraphQLApiService gitHubGraphQLApiService) + { + _blobStorageService = blobStorageService; + _cloudAdvocateService = cloudAdvocateService; + _gitHubApiStatusService = gitHubApiStatusService; + _gitHubGraphQLApiService = gitHubGraphQLApiService; + } + + [FunctionName(nameof(UpdateMicrosoftLearnContributors))] + public async Task Run([TimerTrigger("0 0 */6 * * *")] TimerInfo timer, ILogger log) + { + log.LogInformation($"{nameof(UpdateMicrosoftLearnContributors)} Started"); + + var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + var gitHubApiStatus = await _gitHubApiStatusService.GetApiRateLimits(cancellationTokenSource.Token).ConfigureAwait(false); + if (gitHubApiStatus.GraphQLApi.RemainingRequestCount < 4000) + { + log.LogError($"Maximum GitHub API Limit Reached. GitHub API Limit will reset in {gitHubApiStatus.GraphQLApi.RateLimitReset_TimeRemaining.Minutes + 1} minute(s). Try again at {gitHubApiStatus.GraphQLApi.RateLimitReset_DateTime.UtcDateTime} UTC"); + return; + } + + var cloudAdvocateList = new List(); + await foreach (var advocate in _cloudAdvocateService.GetAzureAdvocates().ConfigureAwait(false)) + { + cloudAdvocateList.Add(advocate); + } + + var microsoftLearnPullRequests = new List(); + await foreach (var pullRequestList in _gitHubGraphQLApiService.GetMicrosoftLearnPullRequests().ConfigureAwait(false)) + { + microsoftLearnPullRequests.AddRange(pullRequestList); + log.LogInformation($"Added {pullRequestList.Count} Pull Requests from {pullRequestList.FirstOrDefault()?.RepositoryName}"); + } + + var cloudAdvocateContributions = new List(); + foreach (var cloudAdvocate in cloudAdvocateList) + { + var cloudAdvocateContributorModel = new CloudAdvocateGitHubContributorModel(microsoftLearnPullRequests.Where(x => x.Author.Equals(cloudAdvocate.GitHubUserName, StringComparison.OrdinalIgnoreCase)), cloudAdvocate); + + cloudAdvocateContributions.Add(cloudAdvocateContributorModel); + + log.LogInformation($"Added {cloudAdvocateContributorModel.PullRequests.Count} Pull Requests for {cloudAdvocate.FullName}"); + } + + var blobName = $"Contributions_{DateTime.UtcNow:o}.json"; + await _blobStorageService.UploadCloudAdvocateMicrosoftLearnContributions(cloudAdvocateContributions, blobName).ConfigureAwait(false); + } + } +} diff --git a/AzureAdvocates.Functions/Models/GitHubContributorModel.cs b/AzureAdvocates.Functions/Models/GitHubContributorModel.cs index d73c68a..6c4680d 100644 --- a/AzureAdvocates.Functions/Models/GitHubContributorModel.cs +++ b/AzureAdvocates.Functions/Models/GitHubContributorModel.cs @@ -1,23 +1,26 @@ using System.Collections.Generic; using System.Linq; using GitHubReadmeWebTrends.Common; +using Newtonsoft.Json; -namespace GitHubReadmeWebTrends.Functions +namespace AzureAdvocates.Functions { - class GitHubContributorModel : CloudAdvocateGitHubUserModel + class CloudAdvocateGitHubContributorModel : CloudAdvocateGitHubUserModel { - public GitHubContributorModel(in IEnumerable pullReuests, CloudAdvocateGitHubUserModel cloudAdvocateGitHubUserModel) - : this(pullReuests, cloudAdvocateGitHubUserModel.FullName, cloudAdvocateGitHubUserModel.GitHubUserName, cloudAdvocateGitHubUserModel.MicrosoftAlias, cloudAdvocateGitHubUserModel.MicrosoftTeam) + public CloudAdvocateGitHubContributorModel(IEnumerable pullRequests, CloudAdvocateGitHubUserModel cloudAdvocateGitHubUserModel) + : this(pullRequests, cloudAdvocateGitHubUserModel.FullName, cloudAdvocateGitHubUserModel.GitHubUserName, cloudAdvocateGitHubUserModel.MicrosoftAlias, cloudAdvocateGitHubUserModel.MicrosoftTeam) { } - public GitHubContributorModel(in IEnumerable pullReuests, in string fullName, in string gitHubUserName, in string microsoftAlias, in string microsoftTeam) + [JsonConstructor] + public CloudAdvocateGitHubContributorModel(IEnumerable pullRequests, string fullName, string gitHubUserName, string microsoftAlias, string microsoftTeam) : base(fullName, gitHubUserName, microsoftAlias, microsoftTeam) { - PullRequests = pullReuests.ToList(); + PullRequests = pullRequests.ToList(); ; } + [JsonProperty("pullRequests")] public IReadOnlyList PullRequests { get; } } } diff --git a/AzureAdvocates.Functions/Services/BlobStorageService.cs b/AzureAdvocates.Functions/Services/BlobStorageService.cs new file mode 100644 index 0000000..1e15b75 --- /dev/null +++ b/AzureAdvocates.Functions/Services/BlobStorageService.cs @@ -0,0 +1,63 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using System.Collections.Generic; +using Microsoft.WindowsAzure.Storage.Blob; +using Newtonsoft.Json; + +namespace AzureAdvocates.Functions +{ + class BlobStorageService + { + const string _microsoftLearnContributionsContainerName = "cloudadvocatemicrosoftlearncontributions"; + readonly CloudBlobClient _blobClient; + + public BlobStorageService(CloudBlobClient cloudBlobClient) => _blobClient = cloudBlobClient; + + public Task UploadCloudAdvocateMicrosoftLearnContributions(IEnumerable azureDataCenterIpRangeModel, string blobName) + { + var container = GetBlobContainer(_microsoftLearnContributionsContainerName); + var blob = container.GetBlockBlobReference(blobName); + + return blob.UploadTextAsync(JsonConvert.SerializeObject(azureDataCenterIpRangeModel)); + } + + public async Task> GetCloudAdvocateMicrosoftLearnContributors() + { + var blobList = new List(); + await foreach (var blob in GetBlobs(_microsoftLearnContributionsContainerName).ConfigureAwait(false)) + { + blobList.Add(blob); + } + + var gitHubContributorListBlob = blobList.OrderByDescending(x => x.Properties.Created).First(); + var serializedGitHubContributorList = await gitHubContributorListBlob.DownloadTextAsync().ConfigureAwait(false); + + return JsonConvert.DeserializeObject>(serializedGitHubContributorList) ?? throw new NullReferenceException(); + } + + async IAsyncEnumerable GetBlobs(string containerName, string prefix = "", int? maxresultsPerQuery = null, BlobListingDetails blobListingDetails = BlobListingDetails.None) where T : ICloudBlob + { + var blobContainer = GetBlobContainer(containerName); + + BlobContinuationToken? continuationToken = null; + + do + { + var response = await blobContainer.ListBlobsSegmentedAsync(prefix, true, blobListingDetails, maxresultsPerQuery, continuationToken, null, null).ConfigureAwait(false); + continuationToken = response?.ContinuationToken; + + var blobListFromResponse = response?.Results?.OfType() ?? Enumerable.Empty(); + + foreach (var blob in blobListFromResponse) + { + yield return blob; + } + + } while (continuationToken != null); + + } + + CloudBlobContainer GetBlobContainer(string containerName) => _blobClient.GetContainerReference(containerName); + } +} diff --git a/AzureAdvocates.Functions/Startup.cs b/AzureAdvocates.Functions/Startup.cs index 3022a93..d8d2ba1 100644 --- a/AzureAdvocates.Functions/Startup.cs +++ b/AzureAdvocates.Functions/Startup.cs @@ -2,6 +2,9 @@ using AzureAdvocates.Functions; using GitHubReadmeWebTrends.Common; using Microsoft.Azure.Functions.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; [assembly: FunctionsStartup(typeof(Startup))] namespace AzureAdvocates.Functions @@ -9,7 +12,14 @@ namespace AzureAdvocates.Functions class Startup : FunctionsStartup { readonly static string _token = Environment.GetEnvironmentVariable("Token") ?? string.Empty; + static readonly string _storageConnectionString = Environment.GetEnvironmentVariable("AzureWebJobsStorage") ?? string.Empty; - public override void Configure(IFunctionsHostBuilder builder) => StartupService.ConfigureServices(builder.Services, _token); + public override void Configure(IFunctionsHostBuilder builder) + { + builder.Services.AddSingleton(CloudStorageAccount.Parse(_storageConnectionString).CreateCloudBlobClient()); + builder.Services.AddSingleton(); + + StartupService.ConfigureServices(builder.Services, _token); + } } } diff --git a/GitHubReadmeWebTrends.Common/Models/CloudAdvocateGitHubUserModel.cs b/GitHubReadmeWebTrends.Common/Models/CloudAdvocateGitHubUserModel.cs index 4024bdc..035c220 100644 --- a/GitHubReadmeWebTrends.Common/Models/CloudAdvocateGitHubUserModel.cs +++ b/GitHubReadmeWebTrends.Common/Models/CloudAdvocateGitHubUserModel.cs @@ -1,4 +1,6 @@ -namespace GitHubReadmeWebTrends.Common +using Newtonsoft.Json; + +namespace GitHubReadmeWebTrends.Common { public class CloudAdvocateGitHubUserModel { @@ -10,9 +12,16 @@ public CloudAdvocateGitHubUserModel(in string fullName, in string gitHubUserName MicrosoftTeam = microsoftTeam; } + [JsonProperty("fullName")] public string FullName { get; } + + [JsonProperty("gitHubUserName")] public string GitHubUserName { get; } + + [JsonProperty("microsoftAlias")] public string MicrosoftAlias { get; } + + [JsonProperty("microsoftTeam")] public string MicrosoftTeam { get; } } } diff --git a/GitHubReadmeWebTrends.Common/Models/PullRequest.cs b/GitHubReadmeWebTrends.Common/Models/PullRequest.cs index c11bb2a..65df03d 100644 --- a/GitHubReadmeWebTrends.Common/Models/PullRequest.cs +++ b/GitHubReadmeWebTrends.Common/Models/PullRequest.cs @@ -1,9 +1,11 @@ using System; +using Newtonsoft.Json; + namespace GitHubReadmeWebTrends.Common { public class PullRequest { - [Newtonsoft.Json.JsonConstructor] + [JsonConstructor] public PullRequest(string id, Uri url, DateTimeOffset createdAt, bool merged, DateTimeOffset? mergedAt, string baseRefName, Author author) : this(id, url, createdAt, merged, mergedAt, baseRefName, author?.Login ?? string.Empty) { @@ -21,25 +23,48 @@ public PullRequest(string id, Uri uri, DateTimeOffset createdAt, bool merged, Da Author = author; } + [JsonProperty("id")] public string Id { get; } + + [JsonProperty("url")] public Uri Uri { get; } + + [JsonProperty("createdAt")] public DateTimeOffset CreatedAt { get; } + + [JsonProperty("merged")] public bool IsMerged { get; } + + [JsonProperty("mergedAt")] public DateTimeOffset? MergedAt { get; } + + [JsonProperty("baseRefName")] public string BaseRefName { get; } + + [JsonProperty("author")] public string Author { get; } } public class RepositoryPullRequest : PullRequest { public RepositoryPullRequest(string repositoryName, string repositoryOwner, PullRequest pullRequest) - : base(pullRequest.Id, pullRequest.Uri, pullRequest.CreatedAt, pullRequest.IsMerged, pullRequest.MergedAt, pullRequest.BaseRefName, pullRequest.Author) + : this(repositoryName, repositoryOwner, pullRequest.Id, pullRequest.Uri, pullRequest.CreatedAt, pullRequest.IsMerged, pullRequest.MergedAt, pullRequest.BaseRefName, pullRequest.Author) + { + + } + + [JsonConstructor] + public RepositoryPullRequest(string repositoryName, string repositoryOwner, string id, Uri url, DateTimeOffset createdAt, bool merged, DateTimeOffset? mergedAt, string baseRefName, string author) + : base(id, url, createdAt, merged, mergedAt, baseRefName, author) { RepositoryName = repositoryName; RepositoryOwner = repositoryOwner; } + [JsonProperty("repositoryName")] public string RepositoryName { get; } + + [JsonProperty("repositoryOwner")] public string RepositoryOwner { get; } } }