Arachne #61 Add config option to specify docker pull policy (#401)

OHDSI · Oct 9, 2024 · 50f5673 · 50f5673
1 parent cc5549e
commit 50f5673
Show file tree

Hide file tree

Showing 3 changed files with 74 additions and 13 deletions.
diff --git a/README.md b/README.md
@@ -11,12 +11,21 @@ ARACHNE Execution Engine is able to use local or Docker Image/tar ball pre-built
 
 ### Important configuration options
 
-docker.image.default - default docker image to be used for execution. If specified, this enables docker execution support. 
-docker.image.filter - a regex used to scan for docker images to be listed. Specifying enables docker execution support and image scanning.
-runtimeservice.dist.archive - Default image to use for running executions
-runtime.local - true/false, enables using local R runtime.
+`docker.image.default` - default docker image to be used for execution. If specified, this enables docker execution support.
 
-docker.enable - outdated option. This is only kept for backward compatibility with implementations that don't support dybamic mode detection. 
+`docker.image.filter` - a regex used to scan for docker images to be listed. Specifying enables docker execution support and image scanning.
+
+`docker.image.pull` - policy that defines when docker images should be pulled. The possible values are as follows: 
+ `NEVER` Never pull any images. If image does not exist in local repository, fail analysis. 
+ `MISSING` If image exists in local repository, use it. Do not check for updated image. 
+ `ALWAYS` Always attempt to pull. If pull failed but image exists in local repository, proceed with that image. 
+ `FORCE` Always attempt to pull. If pull fails, analysis will fail as well, even if local image exists. 
+
+`runtimeservice.dist.archive` - Default image to use for running executions
+
+`runtime.local` - true/false, enables using local R runtime.
+
+`docker.enable` - outdated option. This is only kept for backward compatibility with implementations that don't support dynamic mode detection. 
 
 ### Sample options for creating a container for running locally
 

diff --git a/...e/src/main/java/com/odysseusinc/arachne/executionengine/execution/r/DockerPullPolicy.java b/...e/src/main/java/com/odysseusinc/arachne/executionengine/execution/r/DockerPullPolicy.java
@@ -0,0 +1,12 @@
+package com.odysseusinc.arachne.executionengine.execution.r;
+
+public enum DockerPullPolicy {
+ /** Never pull any images. If image does not exist in local registry, fail analysis. */
+ NEVER,
+ /** If image exists in local repository, use it. Do not check for updated image. Pull if image is missing.*/
+ MISSING,
+ /** Always attempt to pull. If pull failed but image exists in local repository, proceed with that image. */
+ ALWAYS,
+ /** Always attempt to pull. If pull fails, analysis will fail as well, even if local image exists. */
+ FORCE,
+}
diff --git a/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/r/DockerService.java b/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/r/DockerService.java
@@ -3,6 +3,8 @@
 import com.github.dockerjava.api.DockerClient;
 import com.github.dockerjava.api.command.CreateContainerResponse;
 import com.github.dockerjava.api.command.PullImageResultCallback;
+import com.github.dockerjava.api.exception.DockerException;
+import com.github.dockerjava.api.exception.NotFoundException;
 import com.github.dockerjava.api.model.Bind;
 import com.github.dockerjava.api.model.HostConfig;
 import com.github.dockerjava.api.model.Image;
@@ -27,6 +29,7 @@
 import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
 import org.springframework.stereotype.Service;
 
+import javax.annotation.PostConstruct;
 import java.io.File;
 import java.net.URI;
 import java.time.Instant;
@@ -40,6 +43,9 @@
 import java.util.stream.Stream;
 
 import static com.github.dockerjava.api.model.ResponseItem.ProgressDetail;
+import static com.odysseusinc.arachne.executionengine.execution.r.DockerPullPolicy.ALWAYS;
+import static com.odysseusinc.arachne.executionengine.execution.r.DockerPullPolicy.FORCE;
+import static com.odysseusinc.arachne.executionengine.execution.r.DockerPullPolicy.NEVER;
 
 @Service
 @Slf4j
@@ -66,14 +72,14 @@ public class DockerService extends RService implements AutoCloseable {
  @Value("${docker.image.filter:#{null}}")
  private String filterRegex;
 
- private final boolean pull;
+ @Value("${docker.image.pull:#{T(com.odysseusinc.arachne.executionengine.execution.r.DockerPullPolicy).MISSING}}")
+ private DockerPullPolicy pull;
 
  @Autowired
  @Qualifier("analysisTaskExecutor")
  private ThreadPoolTaskExecutor executor;
 
  public DockerService(DockerRegistryProperties properties) {
- pull = properties.getUrl() != null;
  DefaultDockerClientConfig.Builder builder = DefaultDockerClientConfig.createDefaultConfigBuilder()
  .withDockerTlsVerify(certPath != null)
  .withDockerCertPath(certPath)
@@ -90,7 +96,11 @@ public DockerService(DockerRegistryProperties properties) {
  .maxConnections(50)
  .build();
  client = DockerClientImpl.getInstance(config, dockerHttpClient);
- log.info("Initialized Docker interface [{}]", host);
+ }
+
+ @PostConstruct
+ public void init() {
+ log.info("Initialized Docker interface [{}], pull policy: {}", host, pull);
  }
 
  @Override
@@ -113,13 +123,35 @@ protected Overseer analyze(
 
  CompletableFuture<String> init = CompletableFuture.supplyAsync(
  () -> {
- log.info("Execution [{}] use Docker image [{}], pull: {}", id, image, pull);
- if (pull) {
+ log.info("Execution [{}] use Docker image [{}], pull policy: {}", id, image, pull);
+ if (pull == FORCE || pull == ALWAYS) {
+ log.info("Execution [{}] force pull image image {}", id, image);
+ callback.accept(Stage.INITIALIZE, "Force pull image [" + image + "]\r\n");
+ try {
+ pullImage(callback, id, image, stdout, client);
+ callback.accept(Stage.INITIALIZE, "Pull complete, creating container\r\n");
+ } catch (DockerException e) {
+ if (pull == ALWAYS && imageExists(image)) {
+ log.warn("Execution [{}] pull failed: {}, execution will proceed with image found locally", id, e.getMessage());
+ callback.accept(Stage.INITIALIZE, "Execution [" + id + "] pull failed: " + e.getMessage() + ", proceed with local image");
+ } else {
+ log.error("Execution [{}] pull failed: ", id, e);
+ callback.accept(Stage.INITIALIZE, ExceptionUtils.getStackTrace(e));
+ throw e;
+ }
+ }
+ } else if (imageExists(image)) {
+ log.info("Execution [{}] image found, will proceed without pull", id);
+ callback.accept(Stage.INITIALIZE, "Image found, proceed to execution\r\n");
+ } else if (pull == NEVER) {
+ log.info("Execution [{}] image not found, aborting", id);
+ callback.accept(Stage.INITIALIZE, "Image image not found, aborting\r\n");
+ throw new NotFoundException("Image [" + image + "] not found");
+ } else {
+ log.info("Execution [{}] image not found, proceed with pull", id);
+ callback.accept(Stage.INITIALIZE, "Image [" + image + "] not found, proceed to pull\r\n");
  pullImage(callback, id, image, stdout, client);
  callback.accept(Stage.INITIALIZE, "Pull complete, creating container\r\n");
- } else {
- log.info("Execution [{}] pull skipped, no docker registry configured", id);
- callback.accept(Stage.INITIALIZE, "Pull skipped, no docker registry configured\r\n");
  }
  log.info("Execution [{}] creating container with image [{}]", id, image);
  String containerId = createContainer(analysisDir, env, image, script);
@@ -136,6 +168,14 @@ protected Overseer analyze(
  return new DockerOverseer(id, client, started, runtimeTimeOutSec, stdout, init, updateInterval, sendCallback, imageTagOrId, killTimeoutSec);
  }
 
+ private boolean imageExists(String image) {
+ try {
+ return client.inspectImageCmd(image).exec() != null;
+ } catch (NotFoundException e) {
+ return false;
+ }
+ }
+
  private void pullImage(BiConsumer<String, String> callback, Long id, String image, StringBuffer stdout, DockerClient client) {
  try {
  callback.accept(Stage.INITIALIZE, "Pulling image [" + image + "]\r\n");