Docker updates (#3790)

* Add support for JDK17 in the docker containers
* Docker compose for redis, postgres and mysql with Elasticsearch7

co-author: @manan164

docker/README.md

@@ -1 +1,51 @@
-[Docker Instructions](/docs/docs/gettingstarted/docker.md)
+
+# Conductor Docker Builds
+
+## Pre-built docker images
+
+Conductor server with support for the following backend:
+1. Redis
+2. Postgres
+3. Mysql
+4. Cassandra
+
+### Docker File for Server and UI
+
+[Docker Image Source for Server with UI](docker/server/DockerFile)
+
+### Configuration Guide for Conductor Server
+Conductor uses a persistent store for managing state.  
+The choice of backend is quite flexible and can be configured at runtime using `conductor.db.type` property.
+
+Refer to the table below for various supported backend and required configurations to enable each of them.
+
+> [!IMPORTANT]
+> 
+> See [config.properties](docker/server/config/config.properties) for the required properties for each of the backends.
+>
+> | Backend    | Property                           |
+> |------------|------------------------------------|
+> | postgres   | conductor.db.type=postgres         |
+> | redis      | conductor.db.type=redis_standalone |
+> | mysql      | conductor.db.type=mysql            |
+> | cassandra  | conductor.db.type=cassandra        |    
+>
+
+Conductor using Elasticsearch for indexing the workflow data.  
+Currently, Elasticsearch 6 and 7 are supported.
+
+We welcome community contributions for other indexing backends.
+
+**Note:** Docker images use Elasticsearch 7.
+
+## Helm Charts
+TODO: Link to the helm charts
+
+## Run Docker Compose Locally
+### Use the docker-compose to bring up the local conductor server.
+
+| Docker Compose                                               | Description                |
+|--------------------------------------------------------------|----------------------------|
+| [docker-compose.yaml](docker-compose.yaml)                   | Redis + Elasticsearch 7    |
+| [docker-compose-postgres.yaml](docker-compose-postgres.yaml) | Postgres + Elasticsearch 7 |
+| [docker-compose-postgres.yaml](docker-compose-mysql.yaml)    | Mysql + Elasticsearch 7    |

docker/grpc/docker-compose.yaml → docker/docker-compose-mysql.yaml

@@ -4,43 +4,41 @@ services:
 
   conductor-server:
     environment:
-      - CONFIG_PROP=config-mysql-grpc.properties
+      - CONFIG_PROP=config-mysql.properties
     image: conductor:server
+    container_name: conductor-server
     build:
-      context: ../../
+      context: ../
       dockerfile: docker/server/Dockerfile
     networks:
       - internal
     ports:
       - 8080:8080
-      - 8090:8090
+      - 5000:5000
+    healthcheck:
+      test: [ "CMD", "curl","-I" ,"-XGET", "http://localhost:8080/health" ]
+      interval: 60s
+      timeout: 30s
+      retries: 12
     links:
-      - elasticsearch:es
-      - redis:rs
+      - conductor-elasticsearch:es
+      - conductor-mysql:mysql
+      - conductor-redis:rs
     depends_on:
-      elasticsearch:
+      conductor-elasticsearch:
         condition: service_healthy
-      mysql:
+      conductor-mysql:
         condition: service_healthy
-      redis:
+      conductor-redis:
         condition: service_healthy
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "1k"
+        max-file: "3"
 
-  conductor-ui:
-    environment:
-      - WF_SERVER=http://conductor-server:8080
-    image: conductor:ui
-    build:
-      context: ../../
-      dockerfile: docker/ui/Dockerfile
-    networks:
-      - internal
-    ports:
-      - 5000:5000
-    depends_on:
-      - conductor-server
-
-  mysql:
-    image: mysql:5.7
+  conductor-mysql:
+    image: mysql:latest
     environment:
       MYSQL_ROOT_PASSWORD: 12345
       MYSQL_DATABASE: conductor
@@ -60,31 +58,31 @@ services:
       timeout: 5s
       retries: 12
 
-  redis:
+  conductor-redis:
     image: redis:6.2.3-alpine
     volumes:
       - ./redis.conf:/usr/local/etc/redis/redis.conf
     networks:
       - internal
     ports:
-      - 6379:6379
+      - 7379:6379
     healthcheck:
       test: [ "CMD", "redis-cli","ping" ]
 
-  elasticsearch:
-    image: elasticsearch:6.8.15
+  conductor-elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.11
     environment:
       - "ES_JAVA_OPTS=-Xms512m -Xmx1024m"
-      - transport.host=0.0.0.0
-      - discovery.type=single-node
       - xpack.security.enabled=false
+      - discovery.type=single-node
+    volumes:
+      - esdata-conductor:/usr/share/elasticsearch/data
     networks:
       - internal
     ports:
-      - 9200:9200
-      - 9300:9300
+      - 9201:9200
     healthcheck:
-      test: timeout 5 bash -c 'cat < /dev/null > /dev/tcp/localhost/9300'
+      test: curl http://localhost:9200/_cluster/health -o /dev/null
       interval: 5s
       timeout: 5s
       retries: 12
@@ -96,6 +94,9 @@ services:
 
 volumes:
   conductor_mysql:
+    driver: local
+  esdata-conductor:
+    driver: local
 
 networks:
   internal:

docker/docker-compose-postgres.yaml

@@ -13,29 +13,27 @@ services:
       - internal
     ports:
       - 8080:8080
+      - 5000:5000
     healthcheck:
       test: [ "CMD", "curl","-I" ,"-XGET", "http://localhost:8080/health" ]
       interval: 60s
       timeout: 30s
       retries: 12
     links:
-      - elasticsearch:es
-      - redis:rs
-      - postgres:postgresdb
+      - conductor-elasticsearch:es
+      - conductor-postgres:postgresdb
     depends_on:
-      elasticsearch:
+      conductor-elasticsearch:
         condition: service_healthy
-      redis:
-        condition: service_healthy
-      postgres:
+      conductor-postgres:
         condition: service_healthy
     logging:
       driver: "json-file"
       options:
         max-size: "1k"
         max-file: "3"
 
-  postgres:
+  conductor-postgres:
     image: postgres
     environment:
       - POSTGRES_USER=conductor
@@ -45,7 +43,7 @@ services:
     networks:
       - internal
     ports:
-      - 5432:5432
+      - 6432:5432
     healthcheck:
       test: timeout 5 bash -c 'cat < /dev/null > /dev/tcp/localhost/5432'
       interval: 5s
@@ -57,32 +55,20 @@ services:
         max-size: "1k"
         max-file: "3"
 
-  redis:
-    image: redis:6.2.3-alpine
-    volumes:
-      - ./redis.conf:/usr/local/etc/redis/redis.conf
-    networks:
-      - internal
-    ports:
-      - 6379:6379
-    healthcheck:
-      test: [ "CMD", "redis-cli","ping" ]
-
-  elasticsearch:
-    image: elasticsearch:6.8.15
-    container_name: elasticsearch
+  conductor-elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.11
     environment:
       - "ES_JAVA_OPTS=-Xms512m -Xmx1024m"
-      - transport.host=0.0.0.0
-      - discovery.type=single-node
       - xpack.security.enabled=false
+      - discovery.type=single-node
+    volumes:
+      - esdata-conductor:/usr/share/elasticsearch/data
     networks:
       - internal
     ports:
-      - 9200:9200
-      - 9300:9300
+      - 9201:9200
     healthcheck:
-      test: wget http://localhost:9200/ -O /dev/null
+      test: curl http://localhost:9200/_cluster/health -o /dev/null
       interval: 5s
       timeout: 5s
       retries: 12
@@ -95,6 +81,8 @@ services:
 volumes:
   pgdata-conductor:
     driver: local
+  esdata-conductor:
+    driver: local
 
 networks:
   internal: