Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[IGNORE ME] just want to see the test results :) #805

Closed
wants to merge 15 commits into from
Closed

[IGNORE ME] just want to see the test results :) #805

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
8807bbd
wip
brig Aug 18, 2023
e0df2eb
wip
brig Aug 19, 2023
5daac91
wip
brig Aug 21, 2023
b7f051b
merge with master
brig Aug 21, 2023
2d33e6b
cleanup
brig Aug 21, 2023
8ab5999
up
brig Aug 21, 2023
6ec0bff
wip
brig Aug 23, 2023
a8e6067
up
brig Aug 27, 2023
d352bb7
wip
brig Aug 28, 2023
3d9a8b7
up
brig Aug 28, 2023
33d8c50
up
brig Aug 28, 2023
9528b14
up
brig Aug 28, 2023
5a6d75c
client impl
brig Sep 2, 2023
e59680b
wip
brig Sep 7, 2023
f6022f7
multipart as inputstream
brig Sep 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 3 additions & 26 deletions agent/src/main/java/com/walmartlabs/concord/agent/DefaultStateFetcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,18 +23,13 @@
import com.walmartlabs.concord.client.ClientUtils;
import com.walmartlabs.concord.client.ProcessApi;
import com.walmartlabs.concord.common.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.inject.Inject;
import java.io.File;
import java.nio.file.Path;
import java.io.InputStream;
import java.nio.file.StandardCopyOption;

public class DefaultStateFetcher implements StateFetcher {

private static final Logger log = LoggerFactory.getLogger(DefaultStateFetcher.class);

private final ProcessApi processApi;

@Inject
Expand All @@ -44,26 +39,8 @@ public DefaultStateFetcher(ProcessApi processApi) {

@Override
public void downloadState(JobRequest job) throws Exception {
File payload = null;
try {
payload = ClientUtils.withRetry(AgentConstants.API_CALL_MAX_RETRIES, AgentConstants.API_CALL_RETRY_DELAY, () -> processApi.downloadState(job.getInstanceId()));
IOUtils.unzip(payload.toPath(), job.getPayloadDir(), StandardCopyOption.REPLACE_EXISTING);
} finally {
if (payload != null) {
delete(payload.toPath());
}
}
}

private static void delete(Path dir) {
if (dir == null) {
return;
}

try {
IOUtils.deleteRecursively(dir);
} catch (Exception e) {
log.warn("delete ['{}'] -> error", dir, e);
try (InputStream is = ClientUtils.withRetry(AgentConstants.API_CALL_MAX_RETRIES, AgentConstants.API_CALL_RETRY_DELAY, () -> processApi.downloadState(job.getInstanceId()))){
IOUtils.unzip(is, job.getPayloadDir(), StandardCopyOption.REPLACE_EXISTING);
}
}
}
17 changes: 7 additions & 10 deletions agent/src/main/java/com/walmartlabs/concord/agent/logging/RemoteLogAppender.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.slf4j.LoggerFactory;

import javax.inject.Inject;
import java.io.ByteArrayInputStream;
import java.util.UUID;

public class RemoteLogAppender implements LogAppender {
Expand All @@ -48,11 +49,9 @@ public RemoteLogAppender(ApiClient apiClient) {

@Override
public void appendLog(UUID instanceId, byte[] ab) {
String path = "/api/v1/process/" + instanceId + "/log";

try {
ClientUtils.withRetry(AgentConstants.API_CALL_MAX_RETRIES, AgentConstants.API_CALL_RETRY_DELAY, () -> {
ClientUtils.postData(processApi.getApiClient(), path, ab);
processApi.appendProcessLog(instanceId, new ByteArrayInputStream(ab));
return null;
});
} catch (ApiException e) {
Expand All @@ -63,11 +62,9 @@ public void appendLog(UUID instanceId, byte[] ab) {

@Override
public boolean appendLog(UUID instanceId, long segmentId, byte[] ab) {
String path = "/api/v2/process/" + instanceId + "/log/segment/" + segmentId + "/data";

try {
ClientUtils.withRetry(AgentConstants.API_CALL_MAX_RETRIES, AgentConstants.API_CALL_RETRY_DELAY, () -> {
ClientUtils.postData(processApi.getApiClient(), path, ab);
processLogV2Api.appendProcessLogSegment(instanceId, segmentId, new ByteArrayInputStream(ab));
return null;
});
return true;
Expand All @@ -81,13 +78,13 @@ public boolean appendLog(UUID instanceId, long segmentId, byte[] ab) {
@Override
public boolean updateSegment(UUID instanceId, long segmentId, LogSegmentStats stats) {
LogSegmentUpdateRequest request = new LogSegmentUpdateRequest()
.setStatus(stats.status())
.setWarnings(stats.warnings())
.setErrors(stats.errors());
.status(stats.status())
.warnings(stats.warnings())
.errors(stats.errors());

try {
ClientUtils.withRetry(AgentConstants.API_CALL_MAX_RETRIES, AgentConstants.API_CALL_RETRY_DELAY,
() -> processLogV2Api.updateSegment(instanceId, segmentId, request));
() -> processLogV2Api.updateProcessLogSegment(instanceId, segmentId, request));
return true;
} catch (Exception e) {
log.warn("updateSegment ['{}', '{}', '{}'] -> error: {}", instanceId, segmentId, stats, e.getMessage());
Expand Down
76 changes: 20 additions & 56 deletions agent/src/main/java/com/walmartlabs/concord/agent/remote/ApiClientFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,24 +20,19 @@
* =====
*/

import com.squareup.okhttp.Interceptor;
import com.squareup.okhttp.OkHttpClient;
import com.squareup.okhttp.Request;
import com.squareup.okhttp.Response;
import com.walmartlabs.concord.ApiClient;
import com.walmartlabs.concord.agent.cfg.ServerConfiguration;
import com.walmartlabs.concord.client.ConcordApiClient;
import com.walmartlabs.concord.common.IOUtils;

import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Singleton;
import java.io.IOException;
import java.nio.file.Path;
import java.time.Duration;
import java.time.temporal.ChronoUnit;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;

@Named
@Singleton
Expand All @@ -46,77 +41,46 @@ public class ApiClientFactory {
private static final String SESSION_COOKIE_NAME = "JSESSIONID";

private final ServerConfiguration cfg;
private final Path tmpPath;

@Inject
public ApiClientFactory(ServerConfiguration cfg) throws IOException {
public ApiClientFactory(ServerConfiguration cfg) {
this.cfg = cfg;
this.tmpPath = IOUtils.createTempDir("agent-client");
}

public ApiClient create(String sessionToken) throws IOException {
OkHttpClient ok = new OkHttpClient();
ok.setReadTimeout(cfg.getReadTimeout(), TimeUnit.MILLISECONDS);
ok.setConnectTimeout(cfg.getConnectTimeout(), TimeUnit.MILLISECONDS);

Map<String, String> cookieJar = new HashMap<>();
ok.interceptors().add(new AddCookiesInterceptor(cookieJar));
ok.interceptors().add(new ReceivedCookiesInterceptor(cookieJar));

ConcordApiClient client = new ConcordApiClient(cfg.getApiBaseUrl(), ok);
client.setTempFolderPath(tmpPath.toString());
ConcordApiClient client = new ConcordApiClient(cfg.getApiBaseUrl());
client.setReadTimeout(Duration.of(cfg.getReadTimeout(), ChronoUnit.MILLIS));
client.setConnectTimeout(Duration.of(cfg.getConnectTimeout(), ChronoUnit.MILLIS));
if (sessionToken != null) {
client.setSessionToken(sessionToken);
} else {
client.setApiKey(cfg.getApiKey());
}
client.setUserAgent(cfg.getUserAgent());
client.setVerifyingSsl(cfg.isVerifySsl());
return client;
}

private static class AddCookiesInterceptor implements Interceptor {

private final Map<String, String> cookieJar;

private AddCookiesInterceptor(Map<String, String> cookieJar) {
this.cookieJar = cookieJar;
}

@Override
public Response intercept(Chain chain) throws IOException {
Request.Builder builder = chain.request().newBuilder();
for (Map.Entry<String, String> cookie : cookieJar.entrySet()) {
builder.addHeader("Cookie", cookie.getValue());
}
return chain.proceed(builder.build());
}
}

private static class ReceivedCookiesInterceptor implements Interceptor {

private final Map<String, String> cookieJar;

private ReceivedCookiesInterceptor(Map<String, String> cookieJar) {
this.cookieJar = cookieJar;
}

@Override
public Response intercept(Chain chain) throws IOException {
Response resp = chain.proceed(chain.request());
// TODO: brig: impl
// client.setVerifyingSsl(cfg.isVerifySsl());

List<String> cookies = resp.headers("Set-Cookie");
Map<String, String> cookieJar = new HashMap<>();
client.setResponseInterceptor(response -> {
List<String> cookies = response.headers().allValues("Set-Cookie");
if (cookies.isEmpty()) {
return resp;
return;
}

for (String cookie : cookies) {
if (cookie.startsWith(SESSION_COOKIE_NAME)) {
cookieJar.put(SESSION_COOKIE_NAME, cookie);
}
}
});

return resp;
}
client.setRequestInterceptor(builder -> {
for (Map.Entry<String, String> cookie : cookieJar.entrySet()) {
builder.header("Cookie", cookie.getValue());
}
});

return client;
}
}
9 changes: 4 additions & 5 deletions agent/src/main/java/com/walmartlabs/concord/agent/remote/AttachmentsUploader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import com.walmartlabs.concord.ApiClient;
import com.walmartlabs.concord.agent.AgentConstants;
import com.walmartlabs.concord.client.ClientUtils;
import com.walmartlabs.concord.client.ProcessApi;
import com.walmartlabs.concord.common.IOUtils;
import com.walmartlabs.concord.common.TemporaryPath;
import com.walmartlabs.concord.sdk.Constants;
Expand All @@ -35,11 +36,11 @@

public class AttachmentsUploader {

private final ApiClient apiClient;
private final ProcessApi api;

@Inject
public AttachmentsUploader(ApiClient apiClient) {
this.apiClient = apiClient;
this.api = new ProcessApi(apiClient);
}

public void upload(UUID instanceId, Path workDir) throws Exception {
Expand All @@ -53,10 +54,8 @@ public void upload(UUID instanceId, Path workDir) throws Exception {
IOUtils.zip(zip, attachmentsDir);
}

String path = "/api/v1/process/" + instanceId + "/attachment";

ClientUtils.withRetry(AgentConstants.API_CALL_MAX_RETRIES, AgentConstants.API_CALL_RETRY_DELAY, () -> {
ClientUtils.postData(apiClient, path, tmp.path().toFile());
api.uploadProcessAttachments(instanceId,Files.newInputStream( tmp.path()));
return null;
});
}
Expand Down
107 changes: 55 additions & 52 deletions client/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,32 +29,25 @@
<artifactId>concord-sdk</artifactId>
</dependency>
<dependency>
<groupId>com.squareup.okhttp</groupId>
<artifactId>okhttp</artifactId>
</dependency>
<dependency>
<groupId>com.squareup.okhttp</groupId>
<artifactId>logging-interceptor</artifactId>
</dependency>
<dependency>
<groupId>com.squareup.okio</groupId>
<artifactId>okio</artifactId>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<!-- JSON processing: jackson -->
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</dependency>
<dependency>
<groupId>io.gsonfire</groupId>
<artifactId>gson-fire</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>
<dependency>
<groupId>io.swagger</groupId>
<artifactId>swagger-annotations</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
</dependency>

<!-- Immutables -->
Expand Down Expand Up @@ -95,6 +88,11 @@
<artifactId>wiremock-jre8</artifactId>
<scope>test</scope>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.apache.httpcomponents.client5</groupId>-->
<!-- <artifactId>httpclient5</artifactId>-->
<!-- <version>5.1</version>-->
<!-- </dependency>-->
</dependencies>

<build>
Expand All @@ -119,45 +117,50 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>cobertura-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>io.swagger</groupId>
<artifactId>swagger-codegen-maven-plugin</artifactId>
<executions>
<execution>
<id>server</id>
<goals>
<goal>generate</goal>
</goals>
<configuration>
<inputSpec>
${project.basedir}/../server/impl/target/classes/com/walmartlabs/concord/server/swagger/swagger.json
</inputSpec>
<language>java</language>
<languageSpecificPrimitives>
<supportJava6>false</supportJava6>
</languageSpecificPrimitives>
<templateDirectory>${project.basedir}/src/main/template</templateDirectory>
<library>okhttp-gson</library>
<apiPackage>com.walmartlabs.concord.client</apiPackage>
<modelPackage>com.walmartlabs.concord.client</modelPackage>
<configOptions>
<sourceFolder>src/gen/java/main</sourceFolder>
<dateLibrary>java8</dateLibrary>
<serializableModel>true</serializableModel>
</configOptions>
<generateApiTests>false</generateApiTests>
<generateModelTests>false</generateModelTests>
<generateApiDocumentation>false</generateApiDocumentation>
</configuration>
</execution>
</executions>
</plugin>
</plugin> <plugin>
<groupId>org.openapitools</groupId>
<artifactId>openapi-generator-maven-plugin</artifactId>
<version>6.6.0</version>
<executions>
<execution>
<goals>
<goal>generate</goal>
</goals>
<configuration>
<inputSpec>${project.basedir}/../server/impl/target/classes/com/walmartlabs/concord/server/swagger/swagger.yaml</inputSpec>
<generatorName>java</generatorName>
<apiPackage>com.walmartlabs.concord.client</apiPackage>
<modelPackage>com.walmartlabs.concord.client</modelPackage>
<configOptions>
<sourceFolder>src/gen/java/main</sourceFolder>
<dateLibrary>java8</dateLibrary>
<serializableModel>true</serializableModel>
<openApiNullable>false</openApiNullable>
<supportUrlQuery>false</supportUrlQuery>
</configOptions>
<skipValidateSpec>false</skipValidateSpec>
<library>native</library>
<generateApiTests>false</generateApiTests>
<generateModelTests>false</generateModelTests>
<generateApiDocumentation>false</generateApiDocumentation>
<generateModelDocumentation>false</generateModelDocumentation>
<generateSupportingFiles>true</generateSupportingFiles>
<supportingFilesToGenerate>ApiClient.java,ApiResponse.java,ApiException.java,Pair.java</supportingFilesToGenerate>
<templateDirectory>${project.basedir}/src/main/template</templateDirectory>
<cleanupOutput>true</cleanupOutput>
<typeMappings>string+binary=InputStream</typeMappings>
<importMappings>InputStream=java.io.InputStream</importMappings>
</configuration>
</execution>
</executions>
</plugin>


<plugin>
<groupId>org.revapi</groupId>
<artifactId>revapi-maven-plugin</artifactId>
<configuration>
<skip>true</skip>
<analysisConfiguration>
<revapi.ignore>
<item>
Expand Down
Loading