add qgis geometry cleaning modules

the 'delete holes' module has been added to the pipeline. next to be added is the initial cleaning module

processing/db/connection.js → db/connection.js

@@ -1,9 +1,9 @@
 'use strict';
 var assert = require('assert');
 
-// set the db urls base on environment
+// set the db urls based on environment
 var DEFAULT_ENVIRONMENT = 'development';
-var environment = process.env.MACROCOSM_ENV || DEFAULT_ENVIRONMENT;
+var environment = process.env.ORMA_ENV || DEFAULT_ENVIRONMENT;
 var connection = process.env.DATABASE_URL || require('./local').connection[environment];
 
 assert.ok(connection, 'Connection is undefined; check DATABASE_URL or local.js');

db/local/index.js

@@ -0,0 +1,7 @@
+module.exports = {
+  connection: {
+    'development': '',
+    'staging': '',
+    'production': ''
+  }
+}

docker/clean-geometries/Dockerfile

@@ -0,0 +1,17 @@
+# Synopsys: Dockerfile that builds an image that allows headless use of qgis processing module
+# uses https://hub.docker.com/r/nuest/qgis-model/~/dockerfile/
+# note, the entrypoint of the image is a custom script, hence the forced use of /bin/bash
+# entrypoint in docker run cmd
+
+FROM nuest/qgis-model:xenial-multimodel
+# copy over python scripts and initial data.
+COPY ./clean-geom.py /workspace/clean-geom.py
+COPY ./vietnam-communes.cpg /workspace/vietnam-communes.cpg
+COPY ./vietnam-communes.dbf /workspace/vietnam-communes.dbf
+COPY ./vietnam-communes.prj /workspace/vietnam-communes.prj
+COPY ./vietnam-communes.qpj /workspace/vietnam-communes.qpj
+COPY ./vietnam-communes.shp /workspace/vietnam-communes.shp
+COPY ./vietnam-communes.shx /workspace/vietnam-communes.shx
+COPY ./run.sh /workspace/run.sh
+# on entry into container, run the run.sh script
+ENTRYPOINT ["/bin/bash", "/workspace/run.sh"]

docker/clean-geometries/clean-geom.py

@@ -0,0 +1,23 @@
+# Synopysis: cleans admin geometries using the grass gis v.clean alg available with qgis install
+# sys is used primarily for adding qgis utils to path
+import sys
+import os
+# the following qgis modules import and order reference the following
+# https://github.com/nuest/docker-qgis-model/blob/master/workspace/example/model.py#L20
+from qgis.core import *
+import qgis.utilsinput_communes,
+# to use processing script a qgis app needs to be initialized
+app = QgsApplication([], True)
+QgsApplication.setPrefixPath('/usr', True)
+QgsApplication.initQgis()
+# append processing plugin to system path
+sys.path.append('/usr/share/qgis/python/plugins')
+# import, then initalize the processing pobject
+from processing.core.Processing import Processing
+Processing.initialize()
+import processing
+# set path to inputs and outputs
+input_communes = os.path.join(os.getcwd(), sys.argv[1])
+output_communes = os.path.join(os.getcwd(), sys.argv[2] + '.shp')
+# clean the geometries
+processing.runalg('grass:v.clean',input_communes,0, 0.1,'205952.54923,985984.624375,929508.401261,2586975.43865',-1, 0.0001,output_communes)

docker/clean-geometries/run.sh

@@ -0,0 +1,6 @@
+# cd to workspace dir
+cd /workspace
+# run qgis process in python while keeping qgis 'headless', or put otherwise access and use qgis processing modules
+# without running the qgis gui
+# clean the geometry for the initial communes file
+xvfb-run -e ${XVFB_LOGFILE} python clean-geom.py vietnam-communes.shp vietnam-communes-clean-geom

docker/delete-holes/Dockerfile

@@ -0,0 +1,8 @@
+FROM nuest/qgis-model:xenial-multimodel
+COPY ./delete-holes.py /workspace/delete-holes.py
+COPY ./run.sh /workspace/run.sh
+COPY ./vietnam-communes.geojson /workspace/vietnam-communes.geojson
+COPY ./vietnam-district.geojson /workspace/vietnam-district.geojson
+COPY ./vietnam-province.geojson /workspace/vietnam-province.geojson
+# on entry into container, run the run.sh script
+ENTRYPOINT ["/bin/bash", "/workspace/run.sh"]

docker/delete-holes/delete-holes.py

@@ -0,0 +1,23 @@
+# Synopysis: cleans admin geometries using the grass gis v.clean alg available with qgis install
+# sys is used primarily for adding qgis utils to path
+import sys
+import os
+# the following qgis modules import and order reference the following
+# https://github.com/nuest/docker-qgis-model/blob/master/workspace/example/model.py#L20
+from qgis.core import *
+import qgis.utils
+# to use processing script a qgis app needs to be initialized
+app = QgsApplication([], True)
+QgsApplication.setPrefixPath('/usr', True)
+QgsApplication.initQgis()
+# append processing plugin to system path
+sys.path.append('/usr/share/qgis/python/plugins')
+# import, then initalize the processing pobject
+from processing.core.Processing import Processing
+Processing.initialize()
+import processing
+# set path to inputs and outputs
+input_communes = os.path.join(os.getcwd(), sys.argv[1])
+output_communes = os.path.join(os.getcwd(), sys.argv[2] + '.geojson')
+# clean the geometries
+processing.runalg('qgis:fillholes',input_communes, 100000, output_communes)

docker/delete-holes/run.sh

@@ -0,0 +1,7 @@
+# run qgis process in python while keeping qgis 'headless', or put otherwise access and use qgis processing modules
+# without running the qgis gui
+cd /workspace
+# remove holes for all three admin files
+xvfb-run -e ${XVFB_LOGFILE} python delete-holes.py vietnam-province.geojson vietnam-province-filled-holes
+xvfb-run -e ${XVFB_LOGFILE} python delete-holes.py vietnam-communes.geojson vietnam-communes-filled-holes
+xvfb-run -e ${XVFB_LOGFILE} python delete-holes.py vietnam-district.geojson vietnam-district-filled-holes

package.json

@@ -7,7 +7,7 @@
   "author": "maxgrossman <[email protected]>",
   "license": "MIT",
   "scripts": {
-    "start": "chmod +x ./admin-tables-pipe.sh"
+    "start": "chmod +x ./admin-tables-pipe.sh && ./admin-tables-pipe.sh"
   },
   "dependencies": {
     "async": "^2.5.0",

processing/a-dissolve.sh → processing/b-dissolve.sh

@@ -15,13 +15,17 @@ do
   OUTPUT=${1}/output/vietnam-${ADMIN_ARRAY[0]}.geojson
   # make ${DISSOLVE_FIELD} per the current admin's dissolve field
   DISSOLVE_FIELD=${ADMIN_ARRAY[1]}
-  # dissolve on admin field with ogr2ogr and write output as a geojson
+  # dissolve on admin field with ogr2ogr and write output as a geojson; also reproject from UTM to wgs84
   # this comman creates a new geojson where features are geometries that share the same ${DISSOLVE_FIELD}
   # 'ST_UNION' merges geometries.
   # 'GROUP BY' tells gdal which gemetries to merge together
-  ogr2ogr -f 'GeoJSON' "${OUTPUT}" "${INPUT}" -dialect sqlite -sql $'SELECT ST_Union(geometry), * FROM "'"$INPUT_NAME"$'" GROUP BY '"$DISSOLVE_FIELD"
+  # -t_srs is a flag for reprojection
+  # EPSG:4326 is the WGS84 EPSG code
+  # http://spatialreference.org/ref/epsg/wgs-84/
+  ogr2ogr -t_srs EPSG:4326 -f 'GeoJSON' "${OUTPUT}" "${INPUT}" -dialect sqlite -sql $'SELECT ST_Union(geometry), * FROM "'"$INPUT_NAME"$'" GROUP BY '"$DISSOLVE_FIELD"
 done
 # name of geojson output file
 OUT_GJSN=${1}/output/${INPUT_NAME}.geojson
 # since communes don't need to be dissolved, do a simple shp->geojson conversion
-ogr2ogr -f 'GeoJSON' "${INPUT}" "${IN_SHP}"
+# make sure also to reproject
+ogr2ogr -t_srs EPSG:4326 -f 'GeoJSON' "${OUT_GJSN}" "${INPUT}"

processing/c-delete-holes.sh

@@ -0,0 +1,16 @@
+# Synopysis: remoevs holes from dissolved polygons
+
+# take what is in the delete-holes tmp directory and copy it into the the ./processing/docker/delete-holes folder
+cp ${1}/tmp/* ./docker/delete-holes
+# build delete holes container
+docker build -t 'qgis_headless' ./docker/delete-holes
+# run the docker contianer, entering at run.sh
+docker run -it qgis_headless
+# get admin areas from container add copy them over to the output folder of the process
+# using the docker cp command, copying from the most recently built container to the output folder
+# `docker ps --latest -q` grabs the most recent container
+docker cp `docker ps --latest -q`:workspace/vietnam-communes-filled-holes.geojson ${1}/output/vietnam-communes-filled-holes.geojson
+docker cp `docker ps --latest -q`:workspace/vietnam-district-filled-holes.geojson ${1}/output/vietnam-district-filled-holes.geojson
+docker cp `docker ps --latest -q`:workspace/vietnam-province-filled-holes.geojson ${1}/output/vietnam-province-filled-holes.geojson
+# clean up the docker directory
+rm -f ./docker/delete-holes/vietnam*

processing/f-insert-tables.js

@@ -0,0 +1,85 @@
+/**
+ * @file reads streaming admin geojson and 'inserts' each feature 'into' matching admin postgis table
+ */
+
+// // these modules are needed for streaming geojsons
+// var createReadStream = require('fs').createReadStream;
+// var createWriteStream = require('fs').createWriteStream;
+// var readdirSync = require('fs').readdirSync;
+// var geojsonStream = require('geojson-stream');
+// var parser = geojsonStream.parse();
+// var stringifier = geojsonStream.stringify();
+// // module to read path
+// var path = require('path');
+// // parallel allows for reading each admin geojson stream asynchronously
+// var parallel = require('async').parallel;
+// // knex creates a knex obj that links to the current environmnets database
+// var knex = require('./db/connection/.js')
+// // postgis is a knex extension to generate postgis statements
+// var postgis = require('knex-postgis');
+// // helps split single-line json into chunked-by-line geojson as mentinoed in d-simplify-props.js
+// var split = require('split');
+// // directory with geojsons
+// var adminPath = `data/processing/d-simplify-props/tmp`;
+// // array including elements with each file in that directory
+// var admins = readdirSync(adminPath)
+// // st is short for spatial type. spatial type is the prefix for postgis functions that allow for spatial sql statements
+// // see https://postgis.net/docs/reference.html
+// var st = postgis(knex);
+//
+// // create list of async functions to pass to parallel
+// const adminTasks = admins.map((admin) => {
+//   return function(cb) {
+//     // base name mirrors admin name
+//     var basename = admin.split('-')[1]
+//     // here's the path to the current admin file
+//     var adminFile = path.join(adminPath, admin)
+//     // stream of this admin file
+//     var adminFileStream = createReadStream(adminFile)
+//     // pipe split for the lines needed to send along to the geojson parser
+//     .pipe(split())
+//     // the geojson parser for parsing the feature collection
+//     .pipe(parser)
+//     .on('data', (feature) => {
+//       // for each feature, insert it into the table using the insertIntoTable function
+//       insertIntoTable(feature, basename)
+//     })
+//     // fire a callback on end event
+//     .on('end', () => { cb(null, null) })
+//   }
+// });
+//
+// /**
+//  * transforms feature into postgis table row and inserts it into the proper admin table
+//  *
+//  * @param {object} feature geojson feature
+//  * @param {string} admin admin name
+//  */
+// function insertIntoTable (feature, admin) {
+//   // generate properties and geometry objects from feature object
+//   const properties = feature.properties;
+//   const geometry = feature.geometry;
+//   const statement = db.insert({
+//     // shared identifier for each row in admin table
+//     type: admin,
+//     // numeric id for current admin unit
+//     id: properties.id,
+//     // numeric id for currrent admin unit's parent (for instance a commune's parent district)
+//     // this is helpful for future spatial analysis
+//     parent_id: properties.p_id,
+//     // admin unit geometry
+//     geo: st.geomFromGeoJSON(geometry),
+//     // english name of admin unit
+//     name_en: properties.en_name,
+//     // vietnamese name of admin unit
+//     name_vn: ''
+//   })
+//   // method that inserts the insert statement into its correct table
+//   .into(`${admin}-table`).toString();
+// }
+//
+// // run tasks in parallel
+// parallel(adminTasks, (err, res) => {
+//   // do nothing on result
+//   if (!err) {}
+// });