Merge pull request #887 from davidwadden/es5-rest-client

Elasticsearch 5 REST Client

docker/server/config/config-local.properties

@@ -35,7 +35,7 @@ queues.dynomite.nonQuorum.port=22122
 # memory: The instance is created in memory and lost when the server dies. Useful for development and testing.
 # external: Elastic search instance runs outside of the server. Data is persisted and does not get lost when
 #           the server dies. Useful for more stable environments like staging or production.
-workflow.elasticsearch.instanceType=external
+workflow.elasticsearch.instanceType=memory
 
 # Transport address to elasticsearch
 workflow.elasticsearch.url=localhost:9300

es5-persistence/README.md

@@ -1,2 +1,67 @@
+# ES5 Persistence
+
+This module provides ES5 persistence when indexing workflows and tasks.
+
 ## Usage
 
+This module uses the following configuration options:
+
+* `workflow.elasticsearch.instanceType` - This determines the type of ES instance we are using with conductor.
+The two values are either `MEMORY` or `EXTERNAL`.
+If `MEMORY`, then an embedded server will be run.
+Default is `MEMORY`.
+* `workflow.elasticsearch.url` - A comma separated list of schema/host/port of the ES nodes to communicate with.
+Schema can be ignored when using `tcp` transport; otherwise, you must specify `http` or `https`.
+If using the `http` or `https`, then conductor will use the REST transport protocol.
+* `workflow.elasticsearch.index.name` - The name of the workflow and task index.
+Defaults to `conductor`
+* `workflow.elasticsearch.tasklog.index.name` - The name of the task log index.
+Defaults to `task_log`
+
+### Embedded Configuration
+
+If `workflow.elasticsearch.instanceType=MEMORY`, then you can configure the embedded server using the following configurations: 
+
+* `workflow.elasticsearch.embedded.port` - The starting port of the embedded server.
+This is the port used for the TCP transport.
+It will also use this + 100 in order to setup the http transport.
+Default is `9200`
+* `workflow.elasticsearch.embedded.cluster.name` - The name of the embedded cluster name.
+Default is `elasticsearch_test`
+* `workflow.elasticsearch.embedded.host` - The host of the embedded server.
+Default is `127.0.0.1`
+
+### REST Transport
+
+If you are using AWS ElasticSearch, you should use the `rest` transport as that's the only version transport that they support.
+However, this module currently only works with open IAM, VPC version of ElasticSearch.
+Eventually, we should create ES modules that can be loaded in to support authentication and request signing, but this currently does not support that.
+
+### Example Configurations
+
+**In-memory ES with TCP transport**
+
+```
+workflow.elasticsearch.instanceType=MEMORY
+```
+
+**In-memory ES with REST transport**
+
+```
+workflow.elasticsearch.instanceType=MEMORY
+workflow.elasticsearch.url=http://localhost:9300
+```
+
+**ES with TCP transport**
+
+```
+workflow.elasticsearch.instanceType=EXTERNAL
+workflow.elasticsearch.url=127.0.0.1:9300
+```
+
+**ES with REST transport**
+
+```
+workflow.elasticsearch.instanceType=EXTERNAL
+workflow.elasticsearch.url=http://127.0.0.1:9200
+```

es5-persistence/build.gradle

@@ -5,9 +5,13 @@ dependencies {
 
 	compile "org.elasticsearch:elasticsearch:${revElasticSearch5}"
 	compile "org.elasticsearch.client:transport:${revElasticSearch5}"
+    compile "org.elasticsearch.client:elasticsearch-rest-client:${revElasticSearch5}"
     compile "org.elasticsearch.client:elasticsearch-rest-high-level-client:${revElasticSearch5}"
 
     //ES5 Dependency
     compile "org.apache.logging.log4j:log4j-api:${revLog4jApi}"
     compile "org.apache.logging.log4j:log4j-core:${revLog4jCore}"
+
+    testCompile "org.slf4j:slf4j-log4j12:${revSlf4jlog4j}"
+    testCompile "org.awaitility:awaitility:${revAwaitility}"
 }

es5-persistence/dependencies.lock

@@ -113,6 +113,10 @@
             "locked": "2.9.1",
             "requested": "2.9.1"
         },
+        "org.elasticsearch.client:elasticsearch-rest-client": {
+            "locked": "5.6.8",
+            "requested": "5.6.8"
+        },
         "org.elasticsearch.client:elasticsearch-rest-high-level-client": {
             "locked": "5.6.8",
             "requested": "5.6.8"
@@ -246,6 +250,10 @@
             "locked": "2.9.1",
             "requested": "2.9.1"
         },
+        "org.elasticsearch.client:elasticsearch-rest-client": {
+            "locked": "5.6.8",
+            "requested": "5.6.8"
+        },
         "org.elasticsearch.client:elasticsearch-rest-high-level-client": {
             "locked": "5.6.8",
             "requested": "5.6.8"
@@ -379,6 +387,10 @@
             "locked": "2.9.1",
             "requested": "2.9.1"
         },
+        "org.elasticsearch.client:elasticsearch-rest-client": {
+            "locked": "5.6.8",
+            "requested": "5.6.8"
+        },
         "org.elasticsearch.client:elasticsearch-rest-high-level-client": {
             "locked": "5.6.8",
             "requested": "5.6.8"
@@ -522,6 +534,10 @@
             "locked": "2.9.1",
             "requested": "2.9.1"
         },
+        "org.elasticsearch.client:elasticsearch-rest-client": {
+            "locked": "5.6.8",
+            "requested": "5.6.8"
+        },
         "org.elasticsearch.client:elasticsearch-rest-high-level-client": {
             "locked": "5.6.8",
             "requested": "5.6.8"
@@ -655,6 +671,10 @@
             "locked": "2.9.1",
             "requested": "2.9.1"
         },
+        "org.elasticsearch.client:elasticsearch-rest-client": {
+            "locked": "5.6.8",
+            "requested": "5.6.8"
+        },
         "org.elasticsearch.client:elasticsearch-rest-high-level-client": {
             "locked": "5.6.8",
             "requested": "5.6.8"
@@ -792,6 +812,14 @@
             "locked": "2.9.1",
             "requested": "2.9.1"
         },
+        "org.awaitility:awaitility": {
+            "locked": "3.1.2",
+            "requested": "3.1.2"
+        },
+        "org.elasticsearch.client:elasticsearch-rest-client": {
+            "locked": "5.6.8",
+            "requested": "5.6.8"
+        },
         "org.elasticsearch.client:elasticsearch-rest-high-level-client": {
             "locked": "5.6.8",
             "requested": "5.6.8"
@@ -812,7 +840,11 @@
             "firstLevelTransitive": [
                 "com.netflix.conductor:conductor-common"
             ],
-            "locked": "1.7.25"
+            "locked": "1.8.0-alpha1"
+        },
+        "org.slf4j:slf4j-log4j12": {
+            "locked": "1.8.0-alpha1",
+            "requested": "1.8.0-alpha1"
         }
     },
     "testCompileClasspath": {
@@ -933,6 +965,14 @@
             "locked": "2.9.1",
             "requested": "2.9.1"
         },
+        "org.awaitility:awaitility": {
+            "locked": "3.1.2",
+            "requested": "3.1.2"
+        },
+        "org.elasticsearch.client:elasticsearch-rest-client": {
+            "locked": "5.6.8",
+            "requested": "5.6.8"
+        },
         "org.elasticsearch.client:elasticsearch-rest-high-level-client": {
             "locked": "5.6.8",
             "requested": "5.6.8"
@@ -953,7 +993,11 @@
             "firstLevelTransitive": [
                 "com.netflix.conductor:conductor-common"
             ],
-            "locked": "1.7.25"
+            "locked": "1.8.0-alpha1"
+        },
+        "org.slf4j:slf4j-log4j12": {
+            "locked": "1.8.0-alpha1",
+            "requested": "1.8.0-alpha1"
         }
     },
     "testRuntime": {
@@ -1074,6 +1118,14 @@
             "locked": "2.9.1",
             "requested": "2.9.1"
         },
+        "org.awaitility:awaitility": {
+            "locked": "3.1.2",
+            "requested": "3.1.2"
+        },
+        "org.elasticsearch.client:elasticsearch-rest-client": {
+            "locked": "5.6.8",
+            "requested": "5.6.8"
+        },
         "org.elasticsearch.client:elasticsearch-rest-high-level-client": {
             "locked": "5.6.8",
             "requested": "5.6.8"
@@ -1094,7 +1146,11 @@
             "firstLevelTransitive": [
                 "com.netflix.conductor:conductor-common"
             ],
-            "locked": "1.7.25"
+            "locked": "1.8.0-alpha1"
+        },
+        "org.slf4j:slf4j-log4j12": {
+            "locked": "1.8.0-alpha1",
+            "requested": "1.8.0-alpha1"
         }
     },
     "testRuntimeClasspath": {
@@ -1215,6 +1271,14 @@
             "locked": "2.9.1",
             "requested": "2.9.1"
         },
+        "org.awaitility:awaitility": {
+            "locked": "3.1.2",
+            "requested": "3.1.2"
+        },
+        "org.elasticsearch.client:elasticsearch-rest-client": {
+            "locked": "5.6.8",
+            "requested": "5.6.8"
+        },
         "org.elasticsearch.client:elasticsearch-rest-high-level-client": {
             "locked": "5.6.8",
             "requested": "5.6.8"
@@ -1235,7 +1299,11 @@
             "firstLevelTransitive": [
                 "com.netflix.conductor:conductor-common"
             ],
-            "locked": "1.7.25"
+            "locked": "1.8.0-alpha1"
+        },
+        "org.slf4j:slf4j-log4j12": {
+            "locked": "1.8.0-alpha1",
+            "requested": "1.8.0-alpha1"
         }
     }
 }