Merge pull request #227 from danielklim/dlim/dev

minor usability improvements
sat-utils · Jul 28, 2020 · 63dd520 · 63dd520
2 parents df878e7 + 66fba17
commit 63dd520
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 9 deletions.
diff --git a/packages/api-lib/libs/es.js b/packages/api-lib/libs/es.js
@@ -19,10 +19,20 @@ async function connect() {
   let esConfig
   let client
 
-  // use local client
-  if (!process.env.ES_HOST) {
-    client = new elasticsearch.Client({ host: 'localhost:9200' })
-  } else {
+  // non-AWS ES 
+  if(!process.env.AWS_ACCESS_KEY_ID || process.env.AWS_SECRET_ACCESS_KEY) {
+  	// https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/16.x/configuration.html
+    esConfig = {
+      hosts: process.env.ES_HOST || 'localhost:9200',
+      apiVersion: process.env.ES_API_VERSION || '6.8',
+      httpAuth: process.env.ES_HTTP_AUTH || null,
+      // add further params here as needed. alternatively move to a kwargs/splat type arrangement
+      requestTimeout: 120000, // milliseconds
+    }
+    client = new elasticsearch.Client(esConfig)
+  } 
+  // AWS managed ES
+  else {
     await new Promise((resolve, reject) => AWS.config.getCredentials((err) => {
       if (err) return reject(err)
       return resolve()
@@ -109,6 +119,7 @@ async function prepare(index) {
       index,
       body: {
         mappings: {
+          // TODO: this structure different for ElasticSearch 7+,  
           doc: {
             /*'_all': {
                 enabled: true
@@ -132,10 +143,12 @@ async function prepare(index) {
       await client.indices.create(payload)
       logger.info(`Created index: ${JSON.stringify(payload)}`)
     } catch (error) {
-      const debugMessage = `Error creating index, already created: ${error}`
-      logger.debug(debugMessage)
+      logger.debug(`Error creating index '${index}': ${error}`)
     }
   }
+  else {
+      logger.debug(`Index '${index}' exists`)
+  }
 }
 
 // Given an input stream and a transform, write records to an elasticsearch instance

diff --git a/packages/api-lib/libs/ingest.js b/packages/api-lib/libs/ingest.js
@@ -40,7 +40,7 @@ async function fetchChildren(node, links, basePath) {
     if (!selfHref || !link.href) {
       return Promise.reject(new Error(`${node.id} has invalid links`))
     }
-    if (path.isAbsolute(link.href)) {
+    if (isUrl(link.href) || path.isAbsolute(link.href)) {
       urlPath = link.href
     } else {
       // eslint-disable-next-line

diff --git a/packages/api-lib/package.json b/packages/api-lib/package.json
@@ -28,9 +28,9 @@
     "@mapbox/extent": "^0.4.0",
     "@turf/helpers": "^6.1.4",
     "bottleneck": "^2.13.1",
-    "elasticsearch": "^14.2.2",
+    "elasticsearch": "^16",
     "geojson-validation": "^0.1.6",
-    "http-aws-es": "^4.0.0",
+    "http-aws-es": "^6",
     "is-url": "^1.2.4",
     "memorystream": "^0.3.1",
     "pump": "^3.0.0",