From b6c4df5ecd726d67050f82ae1afd0494f40926f6 Mon Sep 17 00:00:00 2001 From: Brandon Minnick <13558917+brminnick@users.noreply.github.com> Date: Sun, 25 Oct 2020 21:58:43 -0700 Subject: [PATCH] Check RemainingRequests for both GraphQL API & REST API --- .../Services/GitHubApiService.cs | 17 ----- .../Services/GitHubApiStatusService.cs | 42 +++++++++++ .../Services/GitHubGraphQLApiService.cs | 32 ++++++-- .../Services/StartupService.cs | 2 +- .../Functions/GetReadmeFunction.cs | 74 ++++++++----------- 5 files changed, 100 insertions(+), 67 deletions(-) delete mode 100644 GitHubReadmeWebTrends.Common/Services/GitHubApiService.cs create mode 100644 GitHubReadmeWebTrends.Common/Services/GitHubApiStatusService.cs diff --git a/GitHubReadmeWebTrends.Common/Services/GitHubApiService.cs b/GitHubReadmeWebTrends.Common/Services/GitHubApiService.cs deleted file mode 100644 index 9d67cbb..0000000 --- a/GitHubReadmeWebTrends.Common/Services/GitHubApiService.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System; -using System.Net; -using GitHubApiStatus; -using Refit; - -namespace GitHubReadmeWebTrends.Common -{ - public class GitHubApiExceptionService : GitHubApiStatusService - { - public bool HasReachedMaximimApiCallLimit(in Exception exception) => exception switch - { - ApiException apiException when apiException.StatusCode is HttpStatusCode.Forbidden => HasReachedMaximimApiCallLimit(apiException.Headers), - GraphQLException graphQLException => HasReachedMaximimApiCallLimit(graphQLException.ResponseHeaders), - _ => false - }; - } -} diff --git a/GitHubReadmeWebTrends.Common/Services/GitHubApiStatusService.cs b/GitHubReadmeWebTrends.Common/Services/GitHubApiStatusService.cs new file mode 100644 index 0000000..9330921 --- /dev/null +++ b/GitHubReadmeWebTrends.Common/Services/GitHubApiStatusService.cs @@ -0,0 +1,42 @@ +using System; +using System.Net; +using System.Threading.Tasks; +using Refit; + +namespace GitHubReadmeWebTrends.Common +{ + public class GitHubApiStatusService : GitHubApiStatus.GitHubApiStatusService + { + readonly GitHubRestApiService _gitHubRestApiService; + readonly GitHubGraphQLApiService _gitHubGraphQLApiService; + + public GitHubApiStatusService(GitHubRestApiService gitHubRestApiService, GitHubGraphQLApiService gitHubGraphQLApiService) => + (_gitHubRestApiService, _gitHubGraphQLApiService) = (gitHubRestApiService, gitHubGraphQLApiService); + + public bool HasReachedMaximimApiCallLimit(in Exception exception) => exception switch + { + ApiException apiException when apiException.StatusCode is HttpStatusCode.Forbidden => HasReachedMaximimApiCallLimit(apiException.Headers), + GraphQLException graphQLException => HasReachedMaximimApiCallLimit(graphQLException.ResponseHeaders), + _ => false + }; + + public async Task<(int gitHubRestApiRequestsRemaining, int gitHubGraphQLApiRequestsRemaining)> GetRemaininRequestCount() + { + int gitHubRestApiRequestsRemaining, gitHubGraphQLApiRequestsRemaining; + try + { + var restApiResponse = await _gitHubRestApiService.GetResponseMessage().ConfigureAwait(false); + var graphQLApiResponse = await _gitHubGraphQLApiService.GetViewerInformationResponse().ConfigureAwait(false); + + gitHubRestApiRequestsRemaining = GetRemainingRequestCount(restApiResponse.Headers); + gitHubGraphQLApiRequestsRemaining = GetRemainingRequestCount(graphQLApiResponse.Headers); + } + catch (Exception e) when (HasReachedMaximimApiCallLimit(e)) + { + gitHubRestApiRequestsRemaining = gitHubGraphQLApiRequestsRemaining = 0; + } + + return (gitHubRestApiRequestsRemaining, gitHubGraphQLApiRequestsRemaining); + } + } +} diff --git a/GitHubReadmeWebTrends.Common/Services/GitHubGraphQLApiService.cs b/GitHubReadmeWebTrends.Common/Services/GitHubGraphQLApiService.cs index b762148..8460bbb 100644 --- a/GitHubReadmeWebTrends.Common/Services/GitHubGraphQLApiService.cs +++ b/GitHubReadmeWebTrends.Common/Services/GitHubGraphQLApiService.cs @@ -12,16 +12,30 @@ public class GitHubGraphQLApiService public GitHubGraphQLApiService(IGitHubGraphQLApiClient gitHubGraphQLApiClient) => _gitHubGraphQLApiClient = gitHubGraphQLApiClient; - public Task>> CreateBranch(string repositoryId, string repositoryName, string branchOid, Guid guid) => - ExecuteGraphQLRequest(_gitHubGraphQLApiClient.CreateBranchMutation(new CreateBranchMutationContent(repositoryId, repositoryName, branchOid, guid))); - public Task>> GetRepository(string repositoryOwner, string repositoryName) => - ExecuteGraphQLRequest(_gitHubGraphQLApiClient.RepositoryConnectionQuery(new RepositoryConnectionQueryContent(repositoryOwner, repositoryName))); + public Task CreateBranch(string repositoryId, string repositoryName, string branchOid, Guid guid) => + GetGraphQLResponseData(CreateBranchResponse(repositoryId, repositoryName, branchOid, guid)); - public Task>> CreatePullRequest(in string repositoryId, in string baseRefName, in string headRefName, in string title, in string body, in Guid clientMutationId, in bool maintainerCanModify = true, in bool draft = false) => + public Task GetRepository(string repositoryOwner, string repositoryName) => + GetGraphQLResponseData(GetRepositoryResponse(repositoryOwner, repositoryName)); + + public Task CreatePullRequest(in string repositoryId, in string baseRefName, in string headRefName, in string title, in string body, in Guid clientMutationId, in bool maintainerCanModify = true, in bool draft = false) => + GetGraphQLResponseData(CreatePullRequestResponse(repositoryId, baseRefName, headRefName, title, body, clientMutationId, maintainerCanModify, draft)); + + public Task GetViewerInformation() => + GetGraphQLResponseData(GetViewerInformationResponse()); + + public Task>> CreateBranchResponse(string repositoryId, string repositoryName, string branchOid, Guid guid) => + ExecuteGraphQLRequest(_gitHubGraphQLApiClient.CreateBranchMutation(new CreateBranchMutationContent(repositoryId, repositoryName, branchOid, guid))); + + public Task>> GetRepositoryResponse(string repositoryOwner, string repositoryName) => + ExecuteGraphQLRequest(_gitHubGraphQLApiClient.RepositoryConnectionQuery(new RepositoryConnectionQueryContent(repositoryOwner, repositoryName))); + + public Task>> CreatePullRequestResponse(in string repositoryId, in string baseRefName, in string headRefName, in string title, in string body, in Guid clientMutationId, in bool maintainerCanModify = true, in bool draft = false) => ExecuteGraphQLRequest(_gitHubGraphQLApiClient.CreatePullRequestMutation(new CreatePullRequestMutationContent(repositoryId, baseRefName, headRefName, title, body, clientMutationId, maintainerCanModify, draft))); - public Task>> GetViewerInformation() => ExecuteGraphQLRequest(_gitHubGraphQLApiClient.ViewerLoginQuery(new ViewerLoginQueryContent())); + public Task>> GetViewerInformationResponse() => + ExecuteGraphQLRequest(_gitHubGraphQLApiClient.ViewerLoginQuery(new ViewerLoginQueryContent())); public async IAsyncEnumerable> GetRepositories(string repositoryOwner, int numberOfRepositoriesPerRequest = 100) { @@ -36,6 +50,12 @@ public async IAsyncEnumerable> GetRepositories(string re while (repositoryConnection?.PageInfo?.HasNextPage is true); } + async Task GetGraphQLResponseData(Task>> graphQLRequestTask) + { + var response = await ExecuteGraphQLRequest(graphQLRequestTask).ConfigureAwait(false); + return response.Content.Data; + } + async Task>> ExecuteGraphQLRequest(Task>> graphQLRequestTask) { var response = await graphQLRequestTask.ConfigureAwait(false); diff --git a/GitHubReadmeWebTrends.Common/Services/StartupService.cs b/GitHubReadmeWebTrends.Common/Services/StartupService.cs index 0f36f7d..9999929 100644 --- a/GitHubReadmeWebTrends.Common/Services/StartupService.cs +++ b/GitHubReadmeWebTrends.Common/Services/StartupService.cs @@ -39,7 +39,7 @@ public static void ConfigureServices(in IServiceCollection services) services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); static TimeSpan sleepDurationProvider(int attemptNumber) => TimeSpan.FromSeconds(Math.Pow(2, attemptNumber)); diff --git a/GitHubReadmeWebTrends.Functions/Functions/GetReadmeFunction.cs b/GitHubReadmeWebTrends.Functions/Functions/GetReadmeFunction.cs index b0bd5b9..4bd55dc 100644 --- a/GitHubReadmeWebTrends.Functions/Functions/GetReadmeFunction.cs +++ b/GitHubReadmeWebTrends.Functions/Functions/GetReadmeFunction.cs @@ -18,20 +18,20 @@ public class GetReadmeFunction readonly CloudQueueClient _cloudQueueClient; readonly GitHubRestApiService _gitHubRestApiService; readonly GitHubGraphQLApiService _gitHubGraphQLApiService; - readonly GitHubApiExceptionService _gitHubApiExceptionService; + readonly GitHubApiStatusService _gitHubApiStatusService; public GetReadmeFunction(CloudQueueClient cloudQueueClient, IHttpClientFactory httpClientFactory, GitHubRestApiService gitHubApiService, - GitHubGraphQLApiService gitHubGraphQLApiService, - GitHubApiExceptionService gitHubApiExceptionService) + GitHubApiStatusService gitHubApiStatusService, + GitHubGraphQLApiService gitHubGraphQLApiService) { _httpClient = httpClientFactory.CreateClient(); _cloudQueueClient = cloudQueueClient; _gitHubRestApiService = gitHubApiService; + _gitHubApiStatusService = gitHubApiStatusService; _gitHubGraphQLApiService = gitHubGraphQLApiService; - _gitHubApiExceptionService = gitHubApiExceptionService; } [FunctionName(nameof(GetReadmeQueueTriggerFunction))] @@ -41,52 +41,39 @@ public async Task GetReadmeQueueTriggerFunction([QueueTrigger(QueueConstants.Rep { const int minimumApiRequests = 2000; - int gitHubRestApiRequestsRemaining, gitHubGraphQLApiRequestsRemaining; - log.LogInformation($"{nameof(GetReadmeFunction)} Stared"); var (repository, gitHubUser) = data; - try - { - var restApiResponse = await _gitHubRestApiService.GetResponseMessage().ConfigureAwait(false); - var graphQLApiResponse = await _gitHubGraphQLApiService.GetViewerInformation().ConfigureAwait(false); + var (gitHubRestApiRequestsRemaining, gitHubGraphQLApiRequestsRemaining) = await _gitHubApiStatusService.GetRemaininRequestCount().ConfigureAwait(false); - gitHubRestApiRequestsRemaining = _gitHubApiExceptionService.GetRemainingRequestCount(restApiResponse.Headers); - gitHubGraphQLApiRequestsRemaining = _gitHubApiExceptionService.GetRemainingRequestCount(graphQLApiResponse.Headers); - } - catch (Exception e) when (_gitHubApiExceptionService.HasReachedMaximimApiCallLimit(e)) - { - gitHubRestApiRequestsRemaining = gitHubGraphQLApiRequestsRemaining = 0; - } - log.LogInformation($"{nameof(gitHubRestApiRequestsRemaining)}: {gitHubRestApiRequestsRemaining}"); - log.LogInformation($"{nameof(gitHubGraphQLApiRequestsRemaining)}: {gitHubGraphQLApiRequestsRemaining}"); - - try + log.LogInformation($"{nameof(gitHubGraphQLApiRequestsRemaining)}: {gitHubGraphQLApiRequestsRemaining}"); + + + // The GitHub API Limits requests to 5,0000 per hour https://docs.github.com/en/free-pro-team@latest/rest#rate-limiting + // If the API Limit is approaching, output to RemainingRepositoriesQueue, where it will be handled by GetReadmeTimerTriggerFunction which runs once an hour + // Otherwise, process the data and place it on VerifyWebTrendsQueue + if (gitHubRestApiRequestsRemaining < minimumApiRequests + || gitHubGraphQLApiRequestsRemaining < minimumApiRequests) { - - // The GitHub API Limits requests to 5,0000 per hour https://docs.github.com/en/free-pro-team@latest/rest#rate-limiting - // If the API Limit is approaching, output to RemainingRepositoriesQueue, where it will be handled by GetReadmeTimerTriggerFunction which runs once an hour - // Otherwise, process the data and place it on VerifyWebTrendsQueue - if (gitHubRestApiRequestsRemaining < minimumApiRequests - || gitHubGraphQLApiRequestsRemaining < minimumApiRequests) - { - log.LogInformation($"Maximum API Requests Reached"); - - remainingRepositoriesData.Add(data); - - log.LogInformation($"Added Data to RemainingRepositoriesQueue"); - } - else - { - await RetrieveReadme(repository, gitHubUser, log, completedRepositoriesData).ConfigureAwait(false); - } + log.LogInformation($"Maximum API Requests Reached"); + + remainingRepositoriesData.Add(data); + + log.LogInformation($"Added Data to RemainingRepositoriesQueue"); } - catch (Refit.ApiException e) when (e.StatusCode is System.Net.HttpStatusCode.NotFound) + else { - //If a Readme doesn't exist, GitHubApiService.GetReadme will return a 404 Not Found response - } + try + { + await RetrieveReadme(repository, gitHubUser, log, completedRepositoriesData).ConfigureAwait(false); + } + catch (Refit.ApiException e) when (e.StatusCode is System.Net.HttpStatusCode.NotFound) + { + //If a Readme doesn't exist, GitHubApiService.GetReadme will return a 404 Not Found response + } + } log.LogInformation($"{nameof(GetReadmeFunction)} Completed"); } @@ -116,9 +103,10 @@ public async Task GetReadmeTimerTriggerFunction([TimerTrigger(_runEveryHour, Run try { - var response = await _gitHubRestApiService.GetResponseMessage().ConfigureAwait(false); + var (gitHubRestApiRequestsRemaining, gitHubGraphQLApiRequestsRemaining) = await _gitHubApiStatusService.GetRemaininRequestCount().ConfigureAwait(false); - if (GitHubApiExceptionService.GetNumberOfApiRequestsRemaining(response.Headers) >= 2000) + if (gitHubRestApiRequestsRemaining > 2000 + && gitHubGraphQLApiRequestsRemaining > 2000) { await RetrieveReadme(repository, gitHubUser, log, completedRepositoriesData).ConfigureAwait(false); await remainingRepositoriesQueue.DeleteMessageAsync(queueMessage).ConfigureAwait(false);