Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Issue #3219: Invalid gitlab listing #3223

Open
wants to merge 3 commits into
base: develop
Choose a base branch
from
Open

Issue #3219: Invalid gitlab listing #3223

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e1fd174
Issue #3219: Invalid gitlab listing
ekazachkova May 3, 2023
b4c7ef9
Issue #3219: Invalid gitlab listing - tests fix
ekazachkova May 4, 2023
396295d
Issue #3219: Invalid gitlab listing - cleanups
ekazachkova May 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions api/src/main/java/com/epam/pipeline/manager/git/GitLabApi.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@ public interface GitLabApi {
String PRIVATE_TOKEN = "PRIVATE-TOKEN";
String API_VERSION = "api_version";
String ISSUE_ID = "issue_id";
String PER_PAGE = "per_page";
String PAGE = "page";

/**
* @param userName The name of the GitLab user
Expand Down Expand Up @@ -136,12 +138,16 @@ Call<Boolean> deleteProject(@Path(API_VERSION) String apiVersion,
* @param path (optional) - The path inside repository. Used to get contend of subdirectories
* @param reference (optional) - The name of a repository branch or tag or if not given the default branch
* @param recursive (optional) - Boolean value used to get a recursive tree (false by default)
* @param perPage (optional) - Number of results to show per page. If not specified, defaults to 20.
* @param page (optional) - Page number (default: 1). Deprecated in GitLab 14.3.
*/
@GET("api/v4/projects/{project}/repository/tree")
Call<List<GitRepositoryEntry>> getRepositoryTree(@Path(PROJECT) String idOrName,
@Query(PATH) String path,
@Query(REF) String reference,
@Query(RECURSIVE) Boolean recursive);
@Query(RECURSIVE) Boolean recursive,
@Query(PER_PAGE) Integer perPage,
@Query(PAGE) Integer page);

/**
* Allows you to receive information about file in repository like name, size, content.
Expand Down Expand Up @@ -380,8 +386,8 @@ Call<List<GitlabIssue>> getIssues(@Path(API_VERSION) String apiVersion,
@Path(PROJECT) String idOrName,
@Query("labels") String labels,
@Query("not[labels]") String notLabels,
@Query("page") Integer page,
@Query("per_page") Integer pageSize,
@Query(PAGE) Integer page,
@Query(PER_PAGE) Integer pageSize,
@Query("search") String search);

@GET("api/{api_version}/projects/{project}/issues/{issue_id}")
Expand Down
33 changes: 23 additions & 10 deletions api/src/main/java/com/epam/pipeline/manager/git/GitlabClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@
import java.util.stream.Stream;

import static com.epam.pipeline.manager.git.RestApiUtils.execute;
import static com.epam.pipeline.manager.git.RestApiUtils.fetchPage;
import static com.epam.pipeline.manager.git.RestApiUtils.getResponse;

@Wither
Expand Down Expand Up @@ -241,11 +242,11 @@ public GitCredentials buildCloneCredentials(boolean useEnvVars, Long duration) t
* @return a list of {@link GitRepositoryEntry}, representing files and folders
* @throws GitClientException if something goes wrong
*/
public List<GitRepositoryEntry> getRepositoryContents(String path,
String revision,
boolean recursive) throws GitClientException {
String projectId = makeProjectId(namespace, projectName);
return execute(gitLabApi.getRepositoryTree(projectId, path, revision, recursive));
public List<GitRepositoryEntry> getRepositoryContents(final String path,
final String revision,
final boolean recursive) throws GitClientException {
final String projectId = makeProjectId(namespace, projectName);
return getAllRepositoryContentPages(projectId, path, revision, recursive);
}

public GitProject createTemplateRepository(Template template, String name, String description,
Expand Down Expand Up @@ -800,11 +801,23 @@ private Response<List<GitlabIssue>> fetchIssuePage(final String project,
final int page,
final int pageSize,
final List<GitlabIssue> result) {
final Response<List<GitlabIssue>> response = getResponse(gitLabApi.getIssues(apiVersion, project, labels,
null, page, pageSize, search));
if (Objects.nonNull(response.body())) {
result.addAll(response.body());
return fetchPage(
gitLabApi.getIssues(apiVersion, project, labels, null, page, pageSize, search), result);
}

private List<GitRepositoryEntry> getAllRepositoryContentPages(final String projectId,
final String path,
final String revision,
final boolean recursive) {
int page = 1;
final List<GitRepositoryEntry> result = new ArrayList<>();
final Response<List<GitRepositoryEntry>> response = fetchPage(
gitLabApi.getRepositoryTree(projectId, path, revision, recursive, MAX_PAGE_SIZE, page), result);
final int totalPages = Integer.parseInt(Objects.requireNonNull(response.headers().get(TOTAL_HEADER)));
while (totalPages > result.size()) {
page++;
fetchPage(gitLabApi.getRepositoryTree(projectId, path, revision, recursive, MAX_PAGE_SIZE, page), result);
}
return response;
return result;
}
}
11 changes: 11 additions & 0 deletions api/src/main/java/com/epam/pipeline/manager/git/RestApiUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@

import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.Objects;

public final class RestApiUtils {

Expand All @@ -49,6 +51,15 @@ public static <R> Response<R> getResponse(Call<R> call) throws GitClientExceptio
}
}

public static <T> Response<List<T>> fetchPage(final Call<List<T>> call,
final List<T> results) throws GitClientException {
final Response<List<T>> response = getResponse(call);
if (Objects.nonNull(response.body())) {
results.addAll(response.body());
}
return response;
}

public static byte[] getFileContent(final Call<ResponseBody> filesRawContent, final int byteLimit) {
try {
final ResponseBody body = filesRawContent.execute().body();
Expand Down
13 changes: 9 additions & 4 deletions api/src/test/java/com/epam/pipeline/manager/git/GitManagerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import com.epam.pipeline.manager.preference.PreferenceManager;
import com.epam.pipeline.manager.preference.SystemPreferences;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder;
import com.github.tomakehurst.wiremock.client.WireMock;
import com.github.tomakehurst.wiremock.junit.WireMockRule;
import lombok.SneakyThrows;
Expand Down Expand Up @@ -319,7 +320,7 @@ public void shouldFetchPipelineDocs() throws GitClientException {
.withQueryParam(REF, equalTo(TEST_REVISION))
.withQueryParam(PATH, equalTo(DOCS + "/"))
.withQueryParam(RECURSIVE, equalTo(String.valueOf(false)))
.willReturn(okJson(with(tree)))
.willReturn(treeResponse(tree))
);
final GitTagEntry tag = new GitTagEntry();
givenThat(
Expand Down Expand Up @@ -404,7 +405,7 @@ public void shouldRemoveFolder() throws GitClientException {
.withQueryParam(REF, equalTo(GIT_MASTER_REPOSITORY))
.withQueryParam(PATH, equalTo(DOCS))
.withQueryParam(RECURSIVE, equalTo(String.valueOf(true)))
.willReturn(okJson(with(tree)))
.willReturn(treeResponse(tree))
);
final GitCommitEntry expectedCommit = mockGitCommitRequest();
final Pipeline pipeline = testingPipeline();
Expand Down Expand Up @@ -494,7 +495,7 @@ public void createFolderShouldFailIfDirectoryExists() throws GitClientException
givenThat(
get(urlPathEqualTo(apiV4(REPOSITORY_TREE)))
.withQueryParam(PATH, equalTo(DOCS))
.willReturn(okJson(with(tree)))
.willReturn(treeResponse(tree))
);
givenThat(
get(urlPathEqualTo(api(REPOSITORY_FILES + "/" + encodeUrlPath(DOCS))))
Expand All @@ -518,7 +519,7 @@ public void createFolderShouldFailIfFileWithTheSameNameExists() throws GitClient
givenThat(
get(urlPathEqualTo(apiV4(REPOSITORY_TREE)))
.withQueryParam(PATH, equalTo(DOCS))
.willReturn(okJson(with(tree)))
.willReturn(treeResponse(tree))
);
mockFileContentRequest(DOCS, GIT_MASTER_REPOSITORY, FILE_CONTENT);
gitManager.createOrRenameFolder(pipeline.getId(), folder);
Expand Down Expand Up @@ -770,4 +771,8 @@ private GitCommitEntry mockGitCommitRequest() {
);
return expectedCommit;
}

private ResponseDefinitionBuilder treeResponse(final List<GitRepositoryEntry> tree) {
return okJson(with(tree)).withHeader("X-Total", String.valueOf(tree.size()));
}
}