From e1fd174d1f7f8c89b88c0333a831e722a3e8c5fa Mon Sep 17 00:00:00 2001 From: Ekaterina_Kazachkova Date: Wed, 3 May 2023 19:12:49 +0300 Subject: [PATCH 1/3] Issue #3219: Invalid gitlab listing --- .../epam/pipeline/manager/git/GitLabApi.java | 12 +++++-- .../pipeline/manager/git/GitlabClient.java | 33 +++++++++++++------ .../pipeline/manager/git/RestApiUtils.java | 11 +++++++ 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/api/src/main/java/com/epam/pipeline/manager/git/GitLabApi.java b/api/src/main/java/com/epam/pipeline/manager/git/GitLabApi.java index 59ac6cd9b1..34912ff90b 100644 --- a/api/src/main/java/com/epam/pipeline/manager/git/GitLabApi.java +++ b/api/src/main/java/com/epam/pipeline/manager/git/GitLabApi.java @@ -68,6 +68,8 @@ public interface GitLabApi { String PRIVATE_TOKEN = "PRIVATE-TOKEN"; String API_VERSION = "api_version"; String ISSUE_ID = "issue_id"; + String PER_PAGE = "per_page"; + String PAGE = "page"; /** * @param userName The name of the GitLab user @@ -136,12 +138,16 @@ Call deleteProject(@Path(API_VERSION) String apiVersion, * @param path (optional) - The path inside repository. Used to get contend of subdirectories * @param reference (optional) - The name of a repository branch or tag or if not given the default branch * @param recursive (optional) - Boolean value used to get a recursive tree (false by default) + * @param perPage (optional) - Number of results to show per page. If not specified, defaults to 20. + * @param page (optional) - Page number (default: 1). Deprecated in GitLab 14.3. */ @GET("api/v4/projects/{project}/repository/tree") Call> getRepositoryTree(@Path(PROJECT) String idOrName, @Query(PATH) String path, @Query(REF) String reference, - @Query(RECURSIVE) Boolean recursive); + @Query(RECURSIVE) Boolean recursive, + @Query(PER_PAGE) Integer perPage, + @Query(PAGE) Integer page); /** * Allows you to receive information about file in repository like name, size, content. @@ -380,8 +386,8 @@ Call> getIssues(@Path(API_VERSION) String apiVersion, @Path(PROJECT) String idOrName, @Query("labels") String labels, @Query("not[labels]") String notLabels, - @Query("page") Integer page, - @Query("per_page") Integer pageSize, + @Query(PAGE) Integer page, + @Query(PER_PAGE) Integer pageSize, @Query("search") String search); @GET("api/{api_version}/projects/{project}/issues/{issue_id}") diff --git a/api/src/main/java/com/epam/pipeline/manager/git/GitlabClient.java b/api/src/main/java/com/epam/pipeline/manager/git/GitlabClient.java index f32ba84c03..46eadd45f2 100644 --- a/api/src/main/java/com/epam/pipeline/manager/git/GitlabClient.java +++ b/api/src/main/java/com/epam/pipeline/manager/git/GitlabClient.java @@ -93,6 +93,7 @@ import java.util.stream.Stream; import static com.epam.pipeline.manager.git.RestApiUtils.execute; +import static com.epam.pipeline.manager.git.RestApiUtils.fetchPage; import static com.epam.pipeline.manager.git.RestApiUtils.getResponse; @Wither @@ -241,11 +242,11 @@ public GitCredentials buildCloneCredentials(boolean useEnvVars, Long duration) t * @return a list of {@link GitRepositoryEntry}, representing files and folders * @throws GitClientException if something goes wrong */ - public List getRepositoryContents(String path, - String revision, - boolean recursive) throws GitClientException { - String projectId = makeProjectId(namespace, projectName); - return execute(gitLabApi.getRepositoryTree(projectId, path, revision, recursive)); + public List getRepositoryContents(final String path, + final String revision, + final boolean recursive) throws GitClientException { + final String projectId = makeProjectId(namespace, projectName); + return getAllRepositoryContentPages(projectId, path, revision, recursive); } public GitProject createTemplateRepository(Template template, String name, String description, @@ -800,11 +801,23 @@ private Response> fetchIssuePage(final String project, final int page, final int pageSize, final List result) { - final Response> response = getResponse(gitLabApi.getIssues(apiVersion, project, labels, - null, page, pageSize, search)); - if (Objects.nonNull(response.body())) { - result.addAll(response.body()); + return fetchPage( + gitLabApi.getIssues(apiVersion, project, labels, null, page, pageSize, search), result); + } + + private List getAllRepositoryContentPages(final String projectId, + final String path, + final String revision, + final boolean recursive) { + int page = 1; + final List result = new ArrayList<>(); + final Response> response = fetchPage( + gitLabApi.getRepositoryTree(projectId, path, revision, recursive, MAX_PAGE_SIZE, page), result); + final int totalPages = Integer.parseInt(Objects.requireNonNull(response.headers().get(TOTAL_HEADER))); + while (totalPages > result.size()) { + page++; + fetchPage(gitLabApi.getRepositoryTree(projectId, path, revision, recursive, MAX_PAGE_SIZE, page), result); } - return response; + return result; } } diff --git a/api/src/main/java/com/epam/pipeline/manager/git/RestApiUtils.java b/api/src/main/java/com/epam/pipeline/manager/git/RestApiUtils.java index abcfe8dc8f..53dccdfb16 100644 --- a/api/src/main/java/com/epam/pipeline/manager/git/RestApiUtils.java +++ b/api/src/main/java/com/epam/pipeline/manager/git/RestApiUtils.java @@ -25,6 +25,8 @@ import java.io.IOException; import java.io.InputStream; +import java.util.List; +import java.util.Objects; public final class RestApiUtils { @@ -49,6 +51,15 @@ public static Response getResponse(Call call) throws GitClientExceptio } } + public static Response> fetchPage(final Call> call, + final List results) throws GitClientException { + final Response> response = getResponse(call); + if (Objects.nonNull(response.body())) { + results.addAll(response.body()); + } + return response; + } + public static byte[] getFileContent(final Call filesRawContent, final int byteLimit) { try { final ResponseBody body = filesRawContent.execute().body(); From b4c7ef980e58fbe53238781b571063f1cb8d4fac Mon Sep 17 00:00:00 2001 From: Ekaterina_Kazachkova Date: Thu, 4 May 2023 14:07:05 +0300 Subject: [PATCH 2/3] Issue #3219: Invalid gitlab listing - tests fix --- .../epam/pipeline/manager/git/GitManagerTest.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/api/src/test/java/com/epam/pipeline/manager/git/GitManagerTest.java b/api/src/test/java/com/epam/pipeline/manager/git/GitManagerTest.java index 741c7f9637..4d0744f15e 100644 --- a/api/src/test/java/com/epam/pipeline/manager/git/GitManagerTest.java +++ b/api/src/test/java/com/epam/pipeline/manager/git/GitManagerTest.java @@ -38,6 +38,7 @@ import com.epam.pipeline.manager.preference.PreferenceManager; import com.epam.pipeline.manager.preference.SystemPreferences; import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder; import com.github.tomakehurst.wiremock.client.WireMock; import com.github.tomakehurst.wiremock.junit.WireMockRule; import lombok.SneakyThrows; @@ -319,7 +320,7 @@ public void shouldFetchPipelineDocs() throws GitClientException { .withQueryParam(REF, equalTo(TEST_REVISION)) .withQueryParam(PATH, equalTo(DOCS + "/")) .withQueryParam(RECURSIVE, equalTo(String.valueOf(false))) - .willReturn(okJson(with(tree))) + .willReturn(treeResponse(tree)) ); final GitTagEntry tag = new GitTagEntry(); givenThat( @@ -404,7 +405,7 @@ public void shouldRemoveFolder() throws GitClientException { .withQueryParam(REF, equalTo(GIT_MASTER_REPOSITORY)) .withQueryParam(PATH, equalTo(DOCS)) .withQueryParam(RECURSIVE, equalTo(String.valueOf(true))) - .willReturn(okJson(with(tree))) + .willReturn(treeResponse(tree)) ); final GitCommitEntry expectedCommit = mockGitCommitRequest(); final Pipeline pipeline = testingPipeline(); @@ -494,7 +495,7 @@ public void createFolderShouldFailIfDirectoryExists() throws GitClientException givenThat( get(urlPathEqualTo(apiV4(REPOSITORY_TREE))) .withQueryParam(PATH, equalTo(DOCS)) - .willReturn(okJson(with(tree))) + .willReturn(treeResponse(tree)) ); givenThat( get(urlPathEqualTo(api(REPOSITORY_FILES + "/" + encodeUrlPath(DOCS)))) @@ -518,7 +519,7 @@ public void createFolderShouldFailIfFileWithTheSameNameExists() throws GitClient givenThat( get(urlPathEqualTo(apiV4(REPOSITORY_TREE))) .withQueryParam(PATH, equalTo(DOCS)) - .willReturn(okJson(with(tree))) + .willReturn(treeResponse(tree)) ); mockFileContentRequest(DOCS, GIT_MASTER_REPOSITORY, FILE_CONTENT); gitManager.createOrRenameFolder(pipeline.getId(), folder); @@ -770,4 +771,8 @@ private GitCommitEntry mockGitCommitRequest() { ); return expectedCommit; } + + private ResponseDefinitionBuilder treeResponse(final List tree) { + return okJson(with(tree)).withHeader("X-Total", String.valueOf(tree.size())); + } } From 396295dce922be70097180d087b282fe41440702 Mon Sep 17 00:00:00 2001 From: Ekaterina_Kazachkova Date: Thu, 4 May 2023 14:18:53 +0300 Subject: [PATCH 3/3] Issue #3219: Invalid gitlab listing - cleanups --- .../main/java/com/epam/pipeline/manager/git/RestApiUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/com/epam/pipeline/manager/git/RestApiUtils.java b/api/src/main/java/com/epam/pipeline/manager/git/RestApiUtils.java index 53dccdfb16..280ce27bf4 100644 --- a/api/src/main/java/com/epam/pipeline/manager/git/RestApiUtils.java +++ b/api/src/main/java/com/epam/pipeline/manager/git/RestApiUtils.java @@ -52,7 +52,7 @@ public static Response getResponse(Call call) throws GitClientExceptio } public static Response> fetchPage(final Call> call, - final List results) throws GitClientException { + final List results) throws GitClientException { final Response> response = getResponse(call); if (Objects.nonNull(response.body())) { results.addAll(response.body());