diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ce1b56d --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.DS_Store +node_modules +**/*/.DS_Store +data diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 0000000..e0ea36f --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +6.0 diff --git a/README.md b/README.md index 4278812..e253c02 100644 --- a/README.md +++ b/README.md @@ -1 +1,37 @@ # openroads-vn-boundaries + +a pipeline to take vietnam admin unit shapefiles and insert them as postgis tables into openroads-vn-api's database + +### install + +#### node packages +`$ yarn install` + +#### s3 cli +`$ pip install awscli` + +### docker +[mac](https://docs.docker.com/docker-for-mac/install/#where-to-go-next) + +[pc](https://docs.docker.com/docker-for-windows/install/) + +#### data + +data is downloaded from an s3 bucket. There no need to add input data as it is added while the pipeline is running. + +#### database + +add a file `./db/local/index.js` of the following spec + +```javascript +module.exports = { + connection: { + development: `development.db.url`, + production: `production.db.url` + } +} +``` + +# run + +`yarn start` diff --git a/admin-tables-pipe.sh b/admin-tables-pipe.sh new file mode 100755 index 0000000..70bb0a5 --- /dev/null +++ b/admin-tables-pipe.sh @@ -0,0 +1,74 @@ +# Synopysis: links a set of I/O geoprocessing scripts that transform a commune level shapefile of Vietnam admin areas +########### into postgis table that includes data from commune, district, and commune level admin areas + + +# input directory that holds to which the initial shapefiles are downloaded from s3 +INPUT_DIR=./data/input +# output directory that holds the final output of linked processes +OUT_DIR=./data/output +# the base processing directory that includes sub directories that I/O data for each process +PROCESSING_BASE_DIR=./data/processing +# a special directory used to handoff data between each process +HNDF_DIR=./data/handoff + +# delete handoff or process directories from previous runs that may have errored. +rm -rf ${HNDF_DIR} +rm -rf ${PROCESSING_BASE_DIR} + +# make handoff and process directories for current pipeline run +mkdir -p ${PROCESSING_BASE_DIR} +mkdir -p ${HNDF_DIR} +mkdir -p ${INPUT_DIR} + +sh source.sh ${INPUT_DIR} + +# make directories in ${PROCESSING_BASE_DIR} for each process's I/O these process scripts live in ./processing +for FILE in ./processing/* +do + # get base filename from its path to generate the process's ${PROCESS_DIR} IN ${PROCESS_BASE_DIR} + FILEBASE=${FILE##*/} + FILESPLIT=(${FILEBASE//./ }) + FILENAME=${FILESPLIT[0]} + PROCESS_DIR=${PROCESSING_BASE_DIR}/${FILENAME} + # make process dir + mkdir ${PROCESS_DIR} + # IN ${PROCESS_DIR} generate the input, tmp, and output ${PROCESS_SUBDIR}s needed to handle process specific I/O + for SUBDIR in input tmp output + do + PROCESS_SUBDIR=${PROCESS_DIR}/${SUBDIR} + mkdir ${PROCESS_SUBDIR} + # if the current ${PROCESS_SUBDIR} is input, and the process is the first dissolve process, copy the pipeline's only input, the commune shapefile, into it + if [[ $SUBDIR == *"input"* ]] + then + if [[ $PROCESS_SUBDIR == *"dissolve"* ]] + then + cp -R ./data/input/ ${PROCESS_SUBDIR}/ + fi + fi + done + # for all processes except the first dissolve process, first copy the data inside the ${HNDF_DIR} into the process's input dir, then delete that process's content from handoff + # the reason for removal is to make sure only proper files exist there as some process scripts read in all of input and not files of a specific nomenclature + if [[ $PROCESS_SUBDIR != *"dissolve"* ]] + then + cp -R ${HNDF_DIR}/. ${PROCESS_DIR}/input/ + rm -f ${HNDF_DIR}/* + fi + # move input data to process's tmp dir so that any pipeline process errors allow for original input to be inspected. + cp -R ${PROCESS_DIR}/input/. ${PROCESS_DIR}/tmp/ + + # run process with command specific to if it is a shell process or javascript process + echo --- running ${FILENAME} --- + if [[ $FILE == *".sh"* ]] + then + ${FILE} ${PROCESS_DIR} + else + node ${FILE} ${PROCESS_DIR} + fi + # copy output contents to handoff directory for the next process to grab + cp -R ${PROCESS_DIR}/output/. ${HNDF_DIR}/ +done +# clean up temp directories and remove the input data +# rm -rf ${HNDF_DIR} +# rm -rf ${PROCESSING_BASE_DIR} +# rm -R ${INPUT_DIR} + diff --git a/data/input/vietnam-communes.cpg b/data/input/vietnam-communes.cpg deleted file mode 100644 index 3ad133c..0000000 --- a/data/input/vietnam-communes.cpg +++ /dev/null @@ -1 +0,0 @@ -UTF-8 \ No newline at end of file diff --git a/data/input/vietnam-communes.dbf b/data/input/vietnam-communes.dbf deleted file mode 100644 index 3661dfd..0000000 Binary files a/data/input/vietnam-communes.dbf and /dev/null differ diff --git a/data/input/vietnam-communes.prj b/data/input/vietnam-communes.prj deleted file mode 100644 index 0202b8e..0000000 --- a/data/input/vietnam-communes.prj +++ /dev/null @@ -1 +0,0 @@ -PROJCS["WGS_1984_UTM_Zone_48N",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",105],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",0],UNIT["Meter",1]] \ No newline at end of file diff --git a/data/input/vietnam-communes.qpj b/data/input/vietnam-communes.qpj deleted file mode 100644 index 18131bc..0000000 --- a/data/input/vietnam-communes.qpj +++ /dev/null @@ -1 +0,0 @@ -PROJCS["WGS 84 / UTM zone 48N",GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",105],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AXIS["Easting",EAST],AXIS["Northing",NORTH],AUTHORITY["EPSG","32648"]] diff --git a/data/input/vietnam-communes.shp b/data/input/vietnam-communes.shp deleted file mode 100644 index a230230..0000000 Binary files a/data/input/vietnam-communes.shp and /dev/null differ diff --git a/data/input/vietnam-communes.shx b/data/input/vietnam-communes.shx deleted file mode 100644 index d656a1d..0000000 Binary files a/data/input/vietnam-communes.shx and /dev/null differ diff --git a/db/connection.js b/db/connection.js new file mode 100644 index 0000000..ade539e --- /dev/null +++ b/db/connection.js @@ -0,0 +1,23 @@ +'use strict'; +var assert = require('assert'); + +// set the db urls based on environment +var DEFAULT_ENVIRONMENT = 'development'; +var environment = process.env.ORMA_ENV || DEFAULT_ENVIRONMENT; +var connection = process.env.DATABASE_URL || require('./local').connection[environment]; + +assert.ok(connection, 'Connection is undefined; check DATABASE_URL or local.js'); + +// connect knex to the current env's db. +var knex = require('knex')({ + client: 'pg', + connection: connection, + debug: false, + pool: { + min: 2, + max: 10 + }, + acquireConnectionTimeout: 100000 +}); + +module.exports = knex; diff --git a/db/local/index.js b/db/local/index.js new file mode 100644 index 0000000..48d6b24 --- /dev/null +++ b/db/local/index.js @@ -0,0 +1,7 @@ +module.exports = { + connection: { + 'development': '', + 'staging': '', + 'production': '' + } +} diff --git a/docker/delete-holes/Dockerfile b/docker/delete-holes/Dockerfile new file mode 100644 index 0000000..44b4019 --- /dev/null +++ b/docker/delete-holes/Dockerfile @@ -0,0 +1,9 @@ +FROM nuest/qgis-model:xenial-multimodel +RUN apt-get update +COPY ./delete-holes.py /workspace/delete-holes.py +COPY ./join-geojsons.js /workspace/join-geojsons.js +COPY ./main.sh /workspace/main.sh +COPY ./vietnam-district.geojson /workspace/vietnam-district.geojson +COPY ./vietnam-province.geojson /workspace/vietnam-province.geojson +# on entry into container, run the run.sh script +ENTRYPOINT ["/bin/bash", "/workspace/main.sh"] diff --git a/docker/delete-holes/delete-holes.py b/docker/delete-holes/delete-holes.py new file mode 100644 index 0000000..c78c0f5 --- /dev/null +++ b/docker/delete-holes/delete-holes.py @@ -0,0 +1,23 @@ +# Synopysis: cleans admin geometries using the grass gis v.clean alg available with qgis install +# sys is used primarily for adding qgis utils to path +import sys +import os +# the following qgis modules import and order reference the following +# https://github.com/nuest/docker-qgis-model/blob/master/workspace/example/model.py#L20 +from qgis.core import * +import qgis.utils +# to use processing script a qgis app needs to be initialized +app = QgsApplication([], True) +QgsApplication.setPrefixPath('/usr', True) +QgsApplication.initQgis() +# append processing plugin to system path +sys.path.append('/usr/share/qgis/python/plugins') +# import, then initalize the processing pobject +from processing.core.Processing import Processing +Processing.initialize() +import processing +# set path to inputs and outputs +input_communes = os.path.join(os.getcwd(), sys.argv[1]) +output_communes = os.path.join(os.getcwd(), sys.argv[2] + '.geojson') +# clean the geometries +processing.runalg('qgis:fillholes',input_communes, 100000, output_communes) diff --git a/docker/delete-holes/join-geojsons.js b/docker/delete-holes/join-geojsons.js new file mode 100644 index 0000000..bed79d1 --- /dev/null +++ b/docker/delete-holes/join-geojsons.js @@ -0,0 +1,46 @@ +var geojsonStream = require('geojson-stream') +var readFile = require('fs').readFile; +var through2 = require('through2'); +var createReadStream = require('fs').createReadStream; +var _ = require('underscore'); + +/** + * reads in feature collection, then (in a stream) joins it to a different fc + * @param {FeatureCollection} fc feature collection we are joining (to another fc) + * @param {function} cb a callback! + */ +function joiner (fc, cb) { + readFile(fc, 'utf8', (err, res) => { + if (err) { throw err } + cb(null, JSON.parse(res)); + }); +} + +joiner(process.argv[2], (err, fc) => { + // create obj w/keys === join field val + const joinField = process.argv[4]; + // make sure index is a string + if (typeof fc.features[0].properties[joinField] !== 'string') { + fc.features = fc.features.map((f) => { + f.properties[joinField] = f.properties[joinField].toString(); + return f; + }); + }; + const joiningIndex = _.indexBy(fc.features.map(f => f.properties), joinField); + // stream read fc to join, joining each match, then adding to features. + createReadStream(process.argv[3]) + .pipe(geojsonStream.parse()) + .pipe(through2.obj((feature, _, callback) => { + // see if joiningIndex includes match and if so pipe it through. + var toJoinVal = feature.properties[joinField]; + // only join on matches + if (toJoinVal) { + if (typeof toJoinVal !== 'string') { toJoinVal = toJoinVal.toString(); } + var joinableVal = joiningIndex[toJoinVal]; + feature.properties = Object.assign(feature.properties, joinableVal); + } + callback(null, feature) + })) + .pipe(geojsonStream.stringify()) + .pipe(process.stdout) +}) \ No newline at end of file diff --git a/docker/delete-holes/main.sh b/docker/delete-holes/main.sh new file mode 100644 index 0000000..a7fe0d6 --- /dev/null +++ b/docker/delete-holes/main.sh @@ -0,0 +1,17 @@ +# run qgis process in python while keeping qgis 'headless', or put otherwise access and use qgis processing modules +# without running the qgis gui +cd /workspace +# down node/npm, link them, then setup package w/dependencies for join-geojsons.js +apt-get install -qy nodejs +apt-get install -qy npm +ln -s /usr/bin/nodejs /usr/bin/node +npm init -y +npm install geojson-stream fs through2 underscore +# make geojson with just id and geojson for cleaning. reason for this is the whole deletion seems to be messing with the vietnamese unicode +ogr2ogr -f 'GeoJSON' vietnam-province-id.geojson vietnam-province.geojson -select PROCODE02 +ogr2ogr -f 'GeoJSON' vietnam-district-id.geojson vietnam-district.geojson -select DISTCODE02 +xvfb-run -a python delete-holes.py vietnam-province-id.geojson vietnam-province-filled +xvfb-run -a python delete-holes.py vietnam-district-id.geojson vietnam-district-filled +node join-geojsons.js vietnam-province.geojson vietnam-province-filled.geojson PROCODE02 > vietnam-province-filled-holes.geojson +node join-geojsons.js vietnam-district.geojson vietnam-district-filled.geojson PROCODE02 > vietnam-district-filled-holes.geojson + diff --git a/docker/dissolve/Dockerfile b/docker/dissolve/Dockerfile new file mode 100644 index 0000000..ac9f17c --- /dev/null +++ b/docker/dissolve/Dockerfile @@ -0,0 +1,15 @@ +# Synopsys: Dockerfile that builds an image that includes gdal +# uses https://github.com/geo-data/gdal-docker + +FROM geodata/gdal +# make a workspace directory +RUN mkdir -p /workspace/ +# copy over python scripts and initial data. +COPY ./vietnam-communes.dbf /workspace/vietnam-communes.dbf +COPY ./vietnam-communes.prj /workspace/vietnam-communes.prj +COPY ./vietnam-communes.qpj /workspace/vietnam-communes.qpj +COPY ./vietnam-communes.shp /workspace/vietnam-communes.shp +COPY ./vietnam-communes.shx /workspace/vietnam-communes.shx +COPY ./main.sh /workspace/main.sh +# on entry into container, run the main.sh script +ENTRYPOINT ["/bin/bash", "/workspace/main.sh"] diff --git a/docker/dissolve/main.sh b/docker/dissolve/main.sh new file mode 100644 index 0000000..2f4c02b --- /dev/null +++ b/docker/dissolve/main.sh @@ -0,0 +1,24 @@ +# cd to workspace dir +cd /workspace +INPUT=./vietnam-communes.shp +INPUT_NAME=vietnam-communes + +for ADMIN in 'communes;COMCODE02' 'district;DISTCODE02' 'province;PROCODE02' +do + # split ${ADMIN} string on the semi-colon to grab the admin name and field id + ADMIN_ARRAY=(${ADMIN//;/ }) + # make the unique output file per the current admin name + OUTPUT=./vietnam-${ADMIN_ARRAY[0]}.geojson + # make ${DISSOLVE_FIELD} per the current admin's dissolve field + DISSOLVE_FIELD=${ADMIN_ARRAY[1]} + # dissolve on admin field with ogr2ogr and write output as a geojson; also reproject from UTM to wgs84 + # this comman creates a new geojson where features are geometries that share the same ${DISSOLVE_FIELD} + # 'ST_UNION' merges geometries. + # 'GROUP BY' tells gdal which gemetries to merge together + # -t_srs is a flag for reprojection + # EPSG:4326 is the WGS84 EPSG code + # http://spatialreference.org/ref/epsg/wgs-84/ + # echo ${QUERY} + ogr2ogr -t_srs EPSG:4326 -f 'GeoJSON' "${OUTPUT}" "${INPUT}" -dialect sqlite -sql $'SELECT ST_Union(geometry), * FROM "'"$INPUT_NAME"$'" GROUP BY '"$DISSOLVE_FIELD" +done + diff --git a/docker/update-geojson-spec/Dockerfile b/docker/update-geojson-spec/Dockerfile new file mode 100644 index 0000000..470c881 --- /dev/null +++ b/docker/update-geojson-spec/Dockerfile @@ -0,0 +1,12 @@ +# Synopsys: Dockerfile that builds an image that includes gdal +# uses https://github.com/geo-data/gdal-docker + +FROM node:6.11-slim +# istall geojson rewind +RUN npm install -g geojson-rewind +RUN mkdir -p /workspace/ +COPY ./main.sh /workspace/main.sh +COPY ./vietnam-communes-filled-holes.geojson /workspace/vietnam-communes-filled-holes.geojson +COPY ./vietnam-district-filled-holes.geojson /workspace/vietnam-district-filled-holes.geojson +COPY ./vietnam-province-filled-holes.geojson /workspace/vietnam-province-filled-holes.geojson +ENTRYPOINT ["bin/bash", "/workspace/main.sh"] diff --git a/docker/update-geojson-spec/main.sh b/docker/update-geojson-spec/main.sh new file mode 100644 index 0000000..b4b8f0b --- /dev/null +++ b/docker/update-geojson-spec/main.sh @@ -0,0 +1,17 @@ +# cd to workspace dir +cd /workspace + +for ADMIN in communes district province +do + # generate unique input and output files as it has been done in previous examples + INPUT_FILE=./vietnam-${ADMIN}-filled-holes.geojson + OUTPUT_FILE=./vietnam-${ADMIN}-cleaned.geojson + # remove crs object to match current GeoJSON spec using sed. + # the below command was found in following place + # https://stackoverflow.com/questions/38028600/how-to-delete-a-json-object-from-json-file-by-sed-command-in-bash (see the mailer example) + # the `-i .org` allows inplace convserion so the ${INPUT_FILE} effectively has its crs removed. + # sed -e '/\"crs\"/ d; /^$/d' ${INPUT_FILE} > ${INPUT_FILE} + # geojson-rewind winds left-to right wound geojsons right-to-left. the right-to-left output is saved to ${OUTPUT_FILE} + geojson-rewind ${INPUT_FILE} > ${OUTPUT_FILE} +done + diff --git a/js/insert-into-table.js b/js/insert-into-table.js new file mode 100755 index 0000000..aea482e --- /dev/null +++ b/js/insert-into-table.js @@ -0,0 +1,95 @@ +/** + * @file reads streaming admin geojson and 'inserts' each feature 'into' matching admin postgis table + */ + +// these modules are needed for streaming geojsons +var createReadStream = require('fs').createReadStream; +var createWriteStream = require('fs').createWriteStream; +var readdirSync = require('fs').readdirSync; +var geojsonStream = require('geojson-stream'); +var parser = geojsonStream.parse(); +var stringifier = geojsonStream.stringify(); +Promise = require('bluebird'); +// module to read path +var path = require('path'); +// parallel allows for reading each admin geojson stream asynchronously +var parallel = require('async').parallel; +// knex creates a knex obj that links to the current environmnets database +// var knex = require('./db/connection/.js') +var db = require('../db/connection'); +// postgis is a knex extension to generate postgis statements +var postgis = require('knex-postgis'); +// helps split single-line json into chunked-by-line geojson as mentinoed in d-simplify-props.js +var split = require('split'); +// directory with geojsons +var baseDir = process.argv[2]; +var adminInPath = `${baseDir}/tmp` +// array including elements with each file in that directory +// st is short for spatial type. spatial type is the prefix for postgis functions that allow for spatial sql statements +// see https://postgis.net/docs/reference.html +var st = postgis(db); + +// return current admin +var admin = readdirSync(adminInPath).find((admin) => new RegExp(process.argv[3]).test(admin)); +// base name mirrors admin name +var basename = admin.split('-')[1] +// here's the path to the current admin file +var adminFile = path.join(adminInPath, admin) + +function insertAdmin (knex, Promise, adminFile) { + return new Promise ((res, rej) => { + createReadStream(adminFile) + .on('error', rej) + .pipe(split()) + .on('error', rej) + .pipe(parser()) + .on('error', rej) + .pipe(through((feature, _, next) => { + insertIntoTable(feature, admin, st, db, next) + .then(() => re()) + .catch((e) => { console.log(e); rej(); }) + })) + }) +} + +/** + * transforms feature into postgis table row and inserts it into the proper admin table + * + * @param {object} feature geojson feature + * @param {string} admin admin name + * @param {object} st spatial type object (generated by knex postgis extension) that allows for making st/postgis statements + * @param {object} db kenx object for connecting to the database + * + */ +function insertIntoTable (feature, admin, st, db, next) { + // generate properties and geometry objects from feature object + const properties = feature.properties; + const geometry = feature.geometry; + if (admin === 'communes') { admin = 'commune'; } + if (![0, 11317, 11715, 40906, 71714].includes(properties.id)) { next() } + db.transaction((t) => { + return db('admin_boundaries') + .transacting(t) + .insert({ + // shared identifier for each row in admin table + type: admin, + // numeric id for current admin unit + id: properties.id, + // numeric id for currrent admin unit's parent (for instance a commune's parent district) + // this is helpful for future spatial analysis + parent_id: properties.p_id, + // admin unit geometry + geom: st.geomFromGeoJSON(geometry), + // english name of admin unit + name_en: properties.en_name, + // vietnamese name of admin unit + name_vn: properties.vn_name + }) + .then(t.commit) + .catch(t.rollback); + }) + .catch((e) => { console.log(e); done(); }) + then(() => done()); +}; + +insertAdmin(db, Promise, adminFile) \ No newline at end of file diff --git a/js/simplify-props.js b/js/simplify-props.js new file mode 100755 index 0000000..66fc76b --- /dev/null +++ b/js/simplify-props.js @@ -0,0 +1,114 @@ +/** + * @file reads streaming admin geojson and reduces properties to match the schema of the table to which it is going to be written + */ + +// these modules are needed for streaming geojsons +var createReadStream = require('fs').createReadStream; +var createWriteStream = require('fs').createWriteStream; +var readdirSync = require('fs').readdirSync; +var geojsonStream = require('geojson-stream'); +var parser = geojsonStream.parse(); +var stringifier = geojsonStream.stringify(); +// module to read path +var path = require('path'); +// since the output of `c-update-geojson-spec.sh` writes geojsons to a single line, the stream needs to be broken up into lines, otherwise it will not work +// split is a module that does just this. +var split = require('split'); +// directory for geojson input and output +var baseDir = process.argv[2]; +var adminInPath = `${baseDir}/tmp`; +var adminOutPath = `${baseDir}/output` +// read in files as a list usable in the parallel function +var admin = readdirSync(adminInPath).find((admin) => new RegExp(process.argv[3]).test(admin)); +writeSimplifiedProps(admin); + +/** + * simplifies input properties to spec needed to make admin postgis tables + * @func makeNewProperties + * @param {object} properties original properties from streaming geojson + * @param {string} admin admin unit name, like 'commune', 'district,' + * @return {object} newProperties simplified properties generated from properties + */ +function makeNewProperties (properties, admin) { + const newProperties = {}; + if (new RegExp(/commune/).test(admin)) { + newProperties.en_name = properties.EN_name + newProperties.vn_name = properties.COMNAME + newProperties.id = properties.COMCODE02; + newProperties.p_id = properties.DISTCODE02 + } else if (new RegExp(/district/).test(admin)) { + newProperties.en_name = properties.D_EName + newProperties.vn_name = properties.DISTNAME + newProperties.id = properties.DISTCODE02 + newProperties.p_id = properties.PROCODE02 + } else if (new RegExp(/province/).test(admin)) { + newProperties.en_name = properties.P_EName + newProperties.vn_name = properties.PRONAME + newProperties.id = properties.PROCODE02 + } + newProperties.en_name = cleanName(newProperties.en_name, admin); + newProperties.vn_name = cleanName(newProperties.vn_name, admin); + return newProperties; +} + +/** + * reads in raw geojson and writes out simplified geojson for provided admin level + * @func writeSimplifiedProps + * @param {string} admin string representation of admin type + * + */ +function writeSimplifiedProps(adminPath) { + // the basename, really the admin level name, of the current admin + var basename = admin.split('-')[1]; + // the relative path to the current admin file + var adminInFile = path.join(adminInPath, admin) + // a read stream of admin file + createReadStream(adminInFile) + // piping split makes the new lines mentioned to be neccessary above + .pipe(split()) + // parser is a transform stream that parses geojson feature collections (the form of the input geojson) + .pipe(parser) + .on('data', (feature) => { + // make and pass feature's properties to the make makeNewProperties function that correctly transforms + // the properties to uniform spec needed to insert into the postgis tables + const properties = feature.properties; + // reset the feature properties as the returj from makeNewProperties + feature.properties = makeNewProperties(properties, basename) + }) + // stringify the geojson to send to createWriteStream, then write it to fiel + .pipe(stringifier) + .pipe(createWriteStream(`${adminOutPath}/vietnam-${basename}-simplified.geojson`)) +} + +/** + * returns cleaned version of place name + * @func cleanName + * @param {string} name admin unit name + * @return {string} cleaned admin unit name + */ +function cleanName(name, admin) { + let cleanName; + if (name) { + if (new RegExp(/X. /).test(name)) { + cleanName = name.replace('X. ',''); + } else if (new RegExp(/P. /).test(name)) { + cleanName = name.replace('P. ', '') + } else if (new RegExp(/Tt. /).test(name)) { + cleanName = name.replace('Tt. ', '') + } else if (new RegExp(/TP./).test(name)) { + cleanName = name.replace(/TP./, '') + } else if (new RegExp(/P. /).test(name)) { + cleanName = name.replace('P. ', '') + } else if (new RegExp(/ D./).test(name)){ + cleanName = name.replace(' D.', '') + } else if (new RegExp(/\\?/).test(name)) { + cleanName = name.replace('?', 'ỉ') + } + if (Boolean(Number(cleanName))) { + cleanName = `${admin} ${cleanName}` + } + cleanName.trim() + } else { cleanName = ''} + // make it titlecase + return cleanName; +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..fb62131 --- /dev/null +++ b/package.json @@ -0,0 +1,24 @@ +{ + "name": "openroads-vn-boundaries", + "version": "0.0.1", + "description": "processing scripts and data for OpenRoads Vietnam admin boundary data", + "main": "index.js", + "repository": "https://github.com/orma/openroads-vn-boundaries.git", + "author": "maxgrossman ", + "license": "MIT", + "scripts": { + "start": "./admin-tables-pipe.sh" + }, + "dependencies": { + "async": "^2.5.0", + "bluebird": "^3.5.0", + "dirty-json": "^0.4.1", + "geojson-rewind": "^0.2.0", + "geojson-stream": "^0.0.1", + "iconv-lite": "^0.4.19", + "knex": "^0.13.0", + "knex-postgis": "^0.2.2", + "pg": "^7.3.0", + "split": "^1.0.1" + } +} diff --git a/processing/a-dissolve.sh b/processing/a-dissolve.sh new file mode 100755 index 0000000..4f12084 --- /dev/null +++ b/processing/a-dissolve.sh @@ -0,0 +1,16 @@ +# Synopysis: creates district and province level admins by dissolving (merging geometrise) of features with same uniq id. + +# copy over admin files to tmp docker folder +cp ${1}/tmp/* ./docker/dissolve +# build gdal docker container +docker build -t 'gdal' ./docker/dissolve +# run the gdal container +docker run -it gdal + +# cp over each admin to the admin data folder. +docker cp `docker ps --latest -q`:workspace/vietnam-communes.geojson ${1}/output/vietnam-communes.geojson +docker cp `docker ps --latest -q`:workspace/vietnam-district.geojson ${1}/output/vietnam-district.geojson +docker cp `docker ps --latest -q`:workspace/vietnam-province.geojson ${1}/output/vietnam-province.geojson + +# remove shapefiles from docer folder +rm -f ./docker/dissolve/vietnam* \ No newline at end of file diff --git a/processing/b-delete-holes.sh b/processing/b-delete-holes.sh new file mode 100755 index 0000000..66f78e1 --- /dev/null +++ b/processing/b-delete-holes.sh @@ -0,0 +1,18 @@ +# Synopysis: remoevs holes from dissolved polygons + +# take what is in the delete-holes tmp directory and copy it into the the ./processing/docker/delete-holes folder +cp ${1}/tmp/* ./docker/delete-holes +# build delete holes container +docker build -t 'qgis_headless' ./docker/delete-holes +# run the docker contianer, entering at run.sh +docker run -it qgis_headless +# get admin areas from container add copy them over to the output folder of the process +# using the docker cp command, copying from the most recently built container to the output folder +`docker ps --latest -q` grabs the most recent container +docker cp `docker ps --latest -q`:workspace/vietnam-province-filled-holes.geojson ${1}/output/vietnam-province-filled-holes.geojson +docker cp `docker ps --latest -q`:workspace/vietnam-district-filled-holes.geojson ${1}/output/vietnam-district-filled-holes.geojson +# communes do not need filled holes. so they are just copied directly from the tmp to output folderope +cp ${1}/tmp/vietnam-communes.geojson ${1}/output/vietnam-communes-filled-holes.geojson +# clean up the docker directory +rm -f ./docker/delete-holes/vietnam* +# \ No newline at end of file diff --git a/processing/c-update-geojson-spec.sh b/processing/c-update-geojson-spec.sh new file mode 100755 index 0000000..e8684a4 --- /dev/null +++ b/processing/c-update-geojson-spec.sh @@ -0,0 +1,17 @@ +# Synopysis: removes the crs object within geojsons outputed by b-reproject as well as enforces right hand rule for polygon draw orders. +# info about right hand rules and the new GeoJSON spec below +# for right hand, see the winding section here: https://macwright.org/2015/03/23/geojson-second-bite.html +# for the brave, here's the actual spec https://tools.ietf.org/html/rfc7946 + +# cp files into docker folder + # copy over admin files to tmp docker folder +cp ${1}/tmp/* ./docker/update-geojson-spec +# build gdal docker container +docker build -t 'update-geojson-spec' ./docker/update-geojson-spec +docker run -it 'update-geojson-spec' +# copy geojsons back over for the new folder +docker cp `docker ps --latest -q`:workspace/vietnam-communes-cleaned.geojson ${1}/output/vietnam-communes-cleaned.geojson +docker cp `docker ps --latest -q`:workspace/vietnam-district-cleaned.geojson ${1}/output/vietnam-district-cleaned.geojson +docker cp `docker ps --latest -q`:workspace/vietnam-province-cleaned.geojson ${1}/output/vietnam-province-cleaned.geojson +# rm files from the docker file +rm -f ./docker/update-geojson-spec/vietnam* \ No newline at end of file diff --git a/processing/d-simplify-props.sh b/processing/d-simplify-props.sh new file mode 100755 index 0000000..2361c4a --- /dev/null +++ b/processing/d-simplify-props.sh @@ -0,0 +1,9 @@ +# Synopsys: reduces geojson properties to a uniform spec used to insert data into postgis tables + +# javascipt code that does the props simplifying +SIMPLIFY_PROPS=./js/simplify-props.js +# loop over each admin level, simplifying each's props. +for ADMIN in province district commune +do + node ${SIMPLIFY_PROPS} ${1} ${ADMIN} +done \ No newline at end of file diff --git a/processing/dissolve.sh b/processing/dissolve.sh deleted file mode 100644 index 1f56bc8..0000000 --- a/processing/dissolve.sh +++ /dev/null @@ -1,19 +0,0 @@ -# input, output files -INPUT=./data/tmp/vietnam-communes.shp -INPUT_NAME=vietnam-communes - -# copy input shapefile into tmp directory -cp ./data/input/* ./data/tmp - -# for districts and provinces + their uniq field -for ADMIN in 'district;DISTCODE02' 'province;PROCODE02' -do - # split ADMIN into array including admin name and its field - ADMIN_ARRAY=(${ADMIN//;/ }) - # use admin name to generate output file name - OUTPUT=./data/output/vietnam-${ADMIN_ARRAY[0]}.geojson - # set DISSOLVE_FIELD to admin field - DISSOLVE_FIELD=${ADMIN_ARRAY[1]} - # dissolve on admin field and write to file - ogr2ogr -f 'GeoJSON' "${OUTPUT}" "${INPUT}" -dialect sqlite -sql $'SELECT ST_Union(geometry), * FROM "'"$INPUT_NAME"$'" GROUP BY '"$DISSOLVE_FIELD" -done diff --git a/processing/e-insert-into-table.sh b/processing/e-insert-into-table.sh new file mode 100755 index 0000000..ae8082d --- /dev/null +++ b/processing/e-insert-into-table.sh @@ -0,0 +1,9 @@ +# Synopsis: Insert each feature as a row into admin_boundaries table + +# javascipt code that does the props simplifying +INSERT_INTO_TABLE=./js/insert-into-table.js +# loop over each admin level, simplifying each's props. +for ADMIN in province district commune +do + node ${INSERT_INTO_TABLE} ${1} ${ADMIN} +done \ No newline at end of file diff --git a/processing/reproject.sh b/processing/reproject.sh deleted file mode 100644 index 52e0b0f..0000000 --- a/processing/reproject.sh +++ /dev/null @@ -1,12 +0,0 @@ -# copy input geojsons into temp folder -cp ./data/input/* ./data/tmp - - -for ADMIN in district province -do - # use admin name to generate output and input file names - INPUT=./data/tmp/vietnam-${ADMIN}.geojson - OUTPUT=./data/output/vietnam-${ADMIN}-wgs84.geojson - # reproject to wgs84 - ogr2ogr -t_srs EPSG:4326 -f 'GeoJSON' "${OUTPUT}" "${INPUT}" -done diff --git a/source.sh b/source.sh new file mode 100644 index 0000000..edf2845 --- /dev/null +++ b/source.sh @@ -0,0 +1,3 @@ +# copy input shapefiles from the s3 bucket in which they live +echo --- downloading input boundaries from s3 --- +aws s3 cp s3://openroads-vn-boundaries ${1} --recursive diff --git a/yarn.lock b/yarn.lock new file mode 100644 index 0000000..d0e9495 --- /dev/null +++ b/yarn.lock @@ -0,0 +1,750 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@mapbox/geojsonhint@^1.0.0": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@mapbox/geojsonhint/-/geojsonhint-1.2.1.tgz#2afe435e8d567aa51873eb1e1ba65cc4d463c7a0" + dependencies: + chalk "^1.1.0" + concat-stream "~1.4.4" + jsonlint-lines "1.7.1" + minimist "1.1.1" + text-table "^0.2.0" + +JSONStream@^1.0.0: + version "1.3.1" + resolved "https://registry.yarnpkg.com/JSONStream/-/JSONStream-1.3.1.tgz#707f761e01dae9e16f1bcf93703b78c70966579a" + dependencies: + jsonparse "^1.2.0" + through ">=2.2.7 <3" + +"JSV@>= 4.0.x": + version "4.0.2" + resolved "https://registry.yarnpkg.com/JSV/-/JSV-4.0.2.tgz#d077f6825571f82132f9dffaed587b4029feff57" + +ansi-regex@^2.0.0: + version "2.1.1" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df" + +ansi-styles@^2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" + +ansi-styles@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-1.0.0.tgz#cb102df1c56f5123eab8b67cd7b98027a0279178" + +arr-diff@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/arr-diff/-/arr-diff-2.0.0.tgz#8f3b827f955a8bd669697e4a4256ac3ceae356cf" + dependencies: + arr-flatten "^1.0.1" + +arr-flatten@^1.0.1: + version "1.1.0" + resolved "https://registry.yarnpkg.com/arr-flatten/-/arr-flatten-1.1.0.tgz#36048bbff4e7b47e136644316c99669ea5ae91f1" + +array-unique@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.2.1.tgz#a1d97ccafcbc2625cc70fadceb36a50c58b01a53" + +async@^2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/async/-/async-2.5.0.tgz#843190fd6b7357a0b9e1c956edddd5ec8462b54d" + dependencies: + lodash "^4.14.0" + +babel-runtime@^6.11.6: + version "6.26.0" + resolved "https://registry.yarnpkg.com/babel-runtime/-/babel-runtime-6.26.0.tgz#965c7058668e82b55d7bfe04ff2337bc8b5647fe" + dependencies: + core-js "^2.4.0" + regenerator-runtime "^0.11.0" + +base64-js@0.0.2: + version "0.0.2" + resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-0.0.2.tgz#024f0f72afa25b75f9c0ee73cd4f55ec1bed9784" + +bluebird@^3.4.6, bluebird@^3.5.0: + version "3.5.0" + resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.0.tgz#791420d7f551eea2897453a8a77653f96606d67c" + +bops@0.0.6: + version "0.0.6" + resolved "https://registry.yarnpkg.com/bops/-/bops-0.0.6.tgz#082d1d55fa01e60dbdc2ebc2dba37f659554cf3a" + dependencies: + base64-js "0.0.2" + to-utf8 "0.0.1" + +braces@^1.8.2: + version "1.8.5" + resolved "https://registry.yarnpkg.com/braces/-/braces-1.8.5.tgz#ba77962e12dff969d6b76711e914b737857bf6a7" + dependencies: + expand-range "^1.8.1" + preserve "^0.2.0" + repeat-element "^1.1.2" + +buffer-writer@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/buffer-writer/-/buffer-writer-1.0.1.tgz#22a936901e3029afcd7547eb4487ceb697a3bf08" + +chalk@^1.0.0, chalk@^1.1.0: + version "1.1.3" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98" + dependencies: + ansi-styles "^2.2.1" + escape-string-regexp "^1.0.2" + has-ansi "^2.0.0" + strip-ansi "^3.0.0" + supports-color "^2.0.0" + +chalk@~0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-0.4.0.tgz#5199a3ddcd0c1efe23bc08c1b027b06176e0c64f" + dependencies: + ansi-styles "~1.0.0" + has-color "~0.1.0" + strip-ansi "~0.1.0" + +commander@^2.2.0: + version "2.11.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.11.0.tgz#157152fd1e7a6c8d98a5b715cf376df928004563" + +concat-stream@~1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/concat-stream/-/concat-stream-1.2.1.tgz#f35100b6c46378bfba8b6b80f9f0d0ccdf13dc60" + dependencies: + bops "0.0.6" + +concat-stream@~1.4.4: + version "1.4.10" + resolved "https://registry.yarnpkg.com/concat-stream/-/concat-stream-1.4.10.tgz#acc3bbf5602cb8cc980c6ac840fa7d8603e3ef36" + dependencies: + inherits "~2.0.1" + readable-stream "~1.1.9" + typedarray "~0.0.5" + +core-js@^2.4.0: + version "2.5.1" + resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.5.1.tgz#ae6874dc66937789b80754ff5428df66819ca50b" + +core-util-is@~1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" + +debug@^2.1.3: + version "2.6.8" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.8.tgz#e731531ca2ede27d188222427da17821d68ff4fc" + dependencies: + ms "2.0.0" + +detect-file@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/detect-file/-/detect-file-0.1.0.tgz#4935dedfd9488648e006b0129566e9386711ea63" + dependencies: + fs-exists-sync "^0.1.0" + +dirty-json@^0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/dirty-json/-/dirty-json-0.4.1.tgz#a46d71d08008578c5a9d26b9c1441c47c8c1c64d" + dependencies: + lex "^1.7.9" + q "^1.4.1" + +escape-string-regexp@^1.0.2: + version "1.0.5" + resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" + +expand-brackets@^0.1.4: + version "0.1.5" + resolved "https://registry.yarnpkg.com/expand-brackets/-/expand-brackets-0.1.5.tgz#df07284e342a807cd733ac5af72411e581d1177b" + dependencies: + is-posix-bracket "^0.1.0" + +expand-range@^1.8.1: + version "1.8.2" + resolved "https://registry.yarnpkg.com/expand-range/-/expand-range-1.8.2.tgz#a299effd335fe2721ebae8e257ec79644fc85337" + dependencies: + fill-range "^2.1.0" + +expand-tilde@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/expand-tilde/-/expand-tilde-1.2.2.tgz#0b81eba897e5a3d31d1c3d102f8f01441e559449" + dependencies: + os-homedir "^1.0.1" + +extend@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444" + +extglob@^0.3.1: + version "0.3.2" + resolved "https://registry.yarnpkg.com/extglob/-/extglob-0.3.2.tgz#2e18ff3d2f49ab2765cec9023f011daa8d8349a1" + dependencies: + is-extglob "^1.0.0" + +filename-regex@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/filename-regex/-/filename-regex-2.0.1.tgz#c1c4b9bee3e09725ddb106b75c1e301fe2f18b26" + +fill-range@^2.1.0: + version "2.2.3" + resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-2.2.3.tgz#50b77dfd7e469bc7492470963699fe7a8485a723" + dependencies: + is-number "^2.1.0" + isobject "^2.0.0" + randomatic "^1.1.3" + repeat-element "^1.1.2" + repeat-string "^1.5.2" + +findup-sync@^0.4.2: + version "0.4.3" + resolved "https://registry.yarnpkg.com/findup-sync/-/findup-sync-0.4.3.tgz#40043929e7bc60adf0b7f4827c4c6e75a0deca12" + dependencies: + detect-file "^0.1.0" + is-glob "^2.0.1" + micromatch "^2.3.7" + resolve-dir "^0.1.0" + +flagged-respawn@^0.3.2: + version "0.3.2" + resolved "https://registry.yarnpkg.com/flagged-respawn/-/flagged-respawn-0.3.2.tgz#ff191eddcd7088a675b2610fffc976be9b8074b5" + +for-in@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80" + +for-own@^0.1.4: + version "0.1.5" + resolved "https://registry.yarnpkg.com/for-own/-/for-own-0.1.5.tgz#5265c681a4f294dabbf17c9509b6763aa84510ce" + dependencies: + for-in "^1.0.1" + +fs-exists-sync@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/fs-exists-sync/-/fs-exists-sync-0.1.0.tgz#982d6893af918e72d08dec9e8673ff2b5a8d6add" + +generic-pool@^2.4.2: + version "2.5.4" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.5.4.tgz#38c6188513e14030948ec6e5cf65523d9779299b" + +geojson-area@0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/geojson-area/-/geojson-area-0.1.0.tgz#d48d807082cfadf4a78df1349be50f38bf1894ae" + dependencies: + wgs84 "0.0.0" + +geojson-rewind@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/geojson-rewind/-/geojson-rewind-0.2.0.tgz#ea558e9e44ff03b8655d0a08b75078dc33a15e79" + dependencies: + concat-stream "~1.2.1" + geojson-area "0.1.0" + minimist "0.0.5" + +geojson-stream@^0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/geojson-stream/-/geojson-stream-0.0.1.tgz#1a1d617167041c7302a45470318164699761502f" + dependencies: + JSONStream "^1.0.0" + through "^2.3.4" + +glob-base@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/glob-base/-/glob-base-0.3.0.tgz#dbb164f6221b1c0b1ccf82aea328b497df0ea3c4" + dependencies: + glob-parent "^2.0.0" + is-glob "^2.0.0" + +glob-parent@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-2.0.0.tgz#81383d72db054fcccf5336daa902f182f6edbb28" + dependencies: + is-glob "^2.0.0" + +global-modules@^0.2.3: + version "0.2.3" + resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-0.2.3.tgz#ea5a3bed42c6d6ce995a4f8a1269b5dae223828d" + dependencies: + global-prefix "^0.1.4" + is-windows "^0.2.0" + +global-prefix@^0.1.4: + version "0.1.5" + resolved "https://registry.yarnpkg.com/global-prefix/-/global-prefix-0.1.5.tgz#8d3bc6b8da3ca8112a160d8d496ff0462bfef78f" + dependencies: + homedir-polyfill "^1.0.0" + ini "^1.3.4" + is-windows "^0.2.0" + which "^1.2.12" + +has-ansi@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91" + dependencies: + ansi-regex "^2.0.0" + +has-color@~0.1.0: + version "0.1.7" + resolved "https://registry.yarnpkg.com/has-color/-/has-color-0.1.7.tgz#67144a5260c34fc3cca677d041daf52fe7b78b2f" + +homedir-polyfill@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/homedir-polyfill/-/homedir-polyfill-1.0.1.tgz#4c2bbc8a758998feebf5ed68580f76d46768b4bc" + dependencies: + parse-passwd "^1.0.0" + +iconv-lite@^0.4.19: + version "0.4.19" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.19.tgz#f7468f60135f5e5dad3399c0a81be9a1603a082b" + +inherits@~2.0.1: + version "2.0.3" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" + +ini@^1.3.4: + version "1.3.4" + resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.4.tgz#0537cb79daf59b59a1a517dff706c86ec039162e" + +interpret@^0.6.5: + version "0.6.6" + resolved "https://registry.yarnpkg.com/interpret/-/interpret-0.6.6.tgz#fecd7a18e7ce5ca6abfb953e1f86213a49f1625b" + +is-buffer@^1.1.5: + version "1.1.5" + resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.5.tgz#1f3b26ef613b214b88cbca23cc6c01d87961eecc" + +is-dotfile@^1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/is-dotfile/-/is-dotfile-1.0.3.tgz#a6a2f32ffd2dfb04f5ca25ecd0f6b83cf798a1e1" + +is-equal-shallow@^0.1.3: + version "0.1.3" + resolved "https://registry.yarnpkg.com/is-equal-shallow/-/is-equal-shallow-0.1.3.tgz#2238098fc221de0bcfa5d9eac4c45d638aa1c534" + dependencies: + is-primitive "^2.0.0" + +is-extendable@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/is-extendable/-/is-extendable-0.1.1.tgz#62b110e289a471418e3ec36a617d472e301dfc89" + +is-extglob@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-1.0.0.tgz#ac468177c4943405a092fc8f29760c6ffc6206c0" + +is-glob@^2.0.0, is-glob@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-2.0.1.tgz#d096f926a3ded5600f3fdfd91198cb0888c2d863" + dependencies: + is-extglob "^1.0.0" + +is-number@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/is-number/-/is-number-2.1.0.tgz#01fcbbb393463a548f2f466cce16dece49db908f" + dependencies: + kind-of "^3.0.2" + +is-number@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/is-number/-/is-number-3.0.0.tgz#24fd6201a4782cf50561c810276afc7d12d71195" + dependencies: + kind-of "^3.0.2" + +is-posix-bracket@^0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/is-posix-bracket/-/is-posix-bracket-0.1.1.tgz#3334dc79774368e92f016e6fbc0a88f5cd6e6bc4" + +is-primitive@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/is-primitive/-/is-primitive-2.0.0.tgz#207bab91638499c07b2adf240a41a87210034575" + +is-windows@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/is-windows/-/is-windows-0.2.0.tgz#de1aa6d63ea29dd248737b69f1ff8b8002d2108c" + +isarray@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf" + +isarray@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" + +isexe@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" + +isobject@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/isobject/-/isobject-2.1.0.tgz#f065561096a3f1da2ef46272f815c840d87e0c89" + dependencies: + isarray "1.0.0" + +js-string-escape@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/js-string-escape/-/js-string-escape-1.0.1.tgz#e2625badbc0d67c7533e9edc1068c587ae4137ef" + +jsonlint-lines@1.7.1: + version "1.7.1" + resolved "https://registry.yarnpkg.com/jsonlint-lines/-/jsonlint-lines-1.7.1.tgz#507de680d3fb8c4be1641cc57d6f679f29f178ff" + dependencies: + JSV ">= 4.0.x" + nomnom ">= 1.5.x" + +jsonparse@^1.2.0: + version "1.3.1" + resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" + +kind-of@^3.0.2: + version "3.2.2" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" + dependencies: + is-buffer "^1.1.5" + +kind-of@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-4.0.0.tgz#20813df3d712928b207378691a45066fae72dd57" + dependencies: + is-buffer "^1.1.5" + +knex-postgis@^0.2.2: + version "0.2.2" + resolved "https://registry.yarnpkg.com/knex-postgis/-/knex-postgis-0.2.2.tgz#ad269d9e48d2a9e9e95a7c9961c3f7b01517d95b" + dependencies: + "@mapbox/geojsonhint" "^1.0.0" + +knex@^0.13.0: + version "0.13.0" + resolved "https://registry.yarnpkg.com/knex/-/knex-0.13.0.tgz#08dd494f6bb64928934eec9dac34787a14ca5fa4" + dependencies: + babel-runtime "^6.11.6" + bluebird "^3.4.6" + chalk "^1.0.0" + commander "^2.2.0" + debug "^2.1.3" + generic-pool "^2.4.2" + inherits "~2.0.1" + interpret "^0.6.5" + liftoff "~2.2.0" + lodash "^4.6.0" + minimist "~1.1.0" + mkdirp "^0.5.0" + pg-connection-string "^0.1.3" + readable-stream "^1.1.12" + safe-buffer "^5.0.1" + tildify "~1.0.0" + uuid "^3.0.0" + v8flags "^2.0.2" + +lex@^1.7.9: + version "1.7.9" + resolved "https://registry.yarnpkg.com/lex/-/lex-1.7.9.tgz#5d5636ccef574348362938b79a47f0eed8ed0d43" + +liftoff@~2.2.0: + version "2.2.5" + resolved "https://registry.yarnpkg.com/liftoff/-/liftoff-2.2.5.tgz#998c2876cff484b103e4423b93d356da44734c91" + dependencies: + extend "^3.0.0" + findup-sync "^0.4.2" + flagged-respawn "^0.3.2" + rechoir "^0.6.2" + resolve "^1.1.7" + +lodash@^4.14.0, lodash@^4.6.0: + version "4.17.4" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae" + +micromatch@^2.3.7: + version "2.3.11" + resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-2.3.11.tgz#86677c97d1720b363431d04d0d15293bd38c1565" + dependencies: + arr-diff "^2.0.0" + array-unique "^0.2.1" + braces "^1.8.2" + expand-brackets "^0.1.4" + extglob "^0.3.1" + filename-regex "^2.0.0" + is-extglob "^1.0.0" + is-glob "^2.0.1" + kind-of "^3.0.2" + normalize-path "^2.0.1" + object.omit "^2.0.0" + parse-glob "^3.0.4" + regex-cache "^0.4.2" + +minimist@0.0.5: + version "0.0.5" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.5.tgz#d7aa327bcecf518f9106ac6b8f003fa3bcea8566" + +minimist@0.0.8: + version "0.0.8" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" + +minimist@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.1.1.tgz#1bc2bc71658cdca5712475684363615b0b4f695b" + +minimist@~1.1.0: + version "1.1.3" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.1.3.tgz#3bedfd91a92d39016fcfaa1c681e8faa1a1efda8" + +mkdirp@^0.5.0: + version "0.5.1" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" + dependencies: + minimist "0.0.8" + +ms@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" + +"nomnom@>= 1.5.x": + version "1.8.1" + resolved "https://registry.yarnpkg.com/nomnom/-/nomnom-1.8.1.tgz#2151f722472ba79e50a76fc125bb8c8f2e4dc2a7" + dependencies: + chalk "~0.4.0" + underscore "~1.6.0" + +normalize-path@^2.0.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-2.1.1.tgz#1ab28b556e198363a8c1a6f7e6fa20137fe6aed9" + dependencies: + remove-trailing-separator "^1.0.1" + +object.omit@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/object.omit/-/object.omit-2.0.1.tgz#1a9c744829f39dbb858c76ca3579ae2a54ebd1fa" + dependencies: + for-own "^0.1.4" + is-extendable "^0.1.1" + +os-homedir@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3" + +packet-reader@0.3.1: + version "0.3.1" + resolved "https://registry.yarnpkg.com/packet-reader/-/packet-reader-0.3.1.tgz#cd62e60af8d7fea8a705ec4ff990871c46871f27" + +parse-glob@^3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/parse-glob/-/parse-glob-3.0.4.tgz#b2c376cfb11f35513badd173ef0bb6e3a388391c" + dependencies: + glob-base "^0.3.0" + is-dotfile "^1.0.0" + is-extglob "^1.0.0" + is-glob "^2.0.0" + +parse-passwd@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/parse-passwd/-/parse-passwd-1.0.0.tgz#6d5b934a456993b23d37f40a382d6f1666a8e5c6" + +path-parse@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.5.tgz#3c1adf871ea9cd6c9431b6ea2bd74a0ff055c4c1" + +pg-connection-string@0.1.3, pg-connection-string@^0.1.3: + version "0.1.3" + resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7" + +pg-pool@~2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-2.0.3.tgz#c022032c8949f312a4f91fb6409ce04076be3257" + +pg-types@~1.12.1: + version "1.12.1" + resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.12.1.tgz#d64087e3903b58ffaad279e7595c52208a14c3d2" + dependencies: + postgres-array "~1.0.0" + postgres-bytea "~1.0.0" + postgres-date "~1.0.0" + postgres-interval "^1.1.0" + +pg@^7.3.0: + version "7.3.0" + resolved "https://registry.yarnpkg.com/pg/-/pg-7.3.0.tgz#275e27466e54a645f6b4a16f6acadf6b849ad83b" + dependencies: + buffer-writer "1.0.1" + js-string-escape "1.0.1" + packet-reader "0.3.1" + pg-connection-string "0.1.3" + pg-pool "~2.0.3" + pg-types "~1.12.1" + pgpass "1.x" + semver "4.3.2" + +pgpass@1.x: + version "1.0.2" + resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.2.tgz#2a7bb41b6065b67907e91da1b07c1847c877b306" + dependencies: + split "^1.0.0" + +postgres-array@~1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/postgres-array/-/postgres-array-1.0.2.tgz#8e0b32eb03bf77a5c0a7851e0441c169a256a238" + +postgres-bytea@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/postgres-bytea/-/postgres-bytea-1.0.0.tgz#027b533c0aa890e26d172d47cf9ccecc521acd35" + +postgres-date@~1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.3.tgz#e2d89702efdb258ff9d9cee0fe91bd06975257a8" + +postgres-interval@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.1.1.tgz#acdb0f897b4b1c6e496d9d4e0a853e1c428f06f0" + dependencies: + xtend "^4.0.0" + +preserve@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/preserve/-/preserve-0.2.0.tgz#815ed1f6ebc65926f865b310c0713bcb3315ce4b" + +q@^1.4.1: + version "1.5.0" + resolved "https://registry.yarnpkg.com/q/-/q-1.5.0.tgz#dd01bac9d06d30e6f219aecb8253ee9ebdc308f1" + +randomatic@^1.1.3: + version "1.1.7" + resolved "https://registry.yarnpkg.com/randomatic/-/randomatic-1.1.7.tgz#c7abe9cc8b87c0baa876b19fde83fd464797e38c" + dependencies: + is-number "^3.0.0" + kind-of "^4.0.0" + +readable-stream@^1.1.12, readable-stream@~1.1.9: + version "1.1.14" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "0.0.1" + string_decoder "~0.10.x" + +rechoir@^0.6.2: + version "0.6.2" + resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.6.2.tgz#85204b54dba82d5742e28c96756ef43af50e3384" + dependencies: + resolve "^1.1.6" + +regenerator-runtime@^0.11.0: + version "0.11.0" + resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.11.0.tgz#7e54fe5b5ccd5d6624ea6255c3473be090b802e1" + +regex-cache@^0.4.2: + version "0.4.4" + resolved "https://registry.yarnpkg.com/regex-cache/-/regex-cache-0.4.4.tgz#75bdc58a2a1496cec48a12835bc54c8d562336dd" + dependencies: + is-equal-shallow "^0.1.3" + +remove-trailing-separator@^1.0.1: + version "1.1.0" + resolved "https://registry.yarnpkg.com/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz#c24bce2a283adad5bc3f58e0d48249b92379d8ef" + +repeat-element@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/repeat-element/-/repeat-element-1.1.2.tgz#ef089a178d1483baae4d93eb98b4f9e4e11d990a" + +repeat-string@^1.5.2: + version "1.6.1" + resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" + +resolve-dir@^0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/resolve-dir/-/resolve-dir-0.1.1.tgz#b219259a5602fac5c5c496ad894a6e8cc430261e" + dependencies: + expand-tilde "^1.2.2" + global-modules "^0.2.3" + +resolve@^1.1.6, resolve@^1.1.7: + version "1.4.0" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.4.0.tgz#a75be01c53da25d934a98ebd0e4c4a7312f92a86" + dependencies: + path-parse "^1.0.5" + +safe-buffer@^5.0.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" + +semver@4.3.2: + version "4.3.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.2.tgz#c7a07158a80bedd052355b770d82d6640f803be7" + +split@^1.0.0, split@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9" + dependencies: + through "2" + +string_decoder@~0.10.x: + version "0.10.31" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" + +strip-ansi@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf" + dependencies: + ansi-regex "^2.0.0" + +strip-ansi@~0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-0.1.1.tgz#39e8a98d044d150660abe4a6808acf70bb7bc991" + +supports-color@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" + +text-table@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" + +through@2, "through@>=2.2.7 <3", through@^2.3.4: + version "2.3.8" + resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" + +tildify@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/tildify/-/tildify-1.0.0.tgz#2a021db5e8fbde0a8f8b4df37adaa8fb1d39d7dd" + dependencies: + user-home "^1.0.0" + +to-utf8@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/to-utf8/-/to-utf8-0.0.1.tgz#d17aea72ff2fba39b9e43601be7b3ff72e089852" + +typedarray@~0.0.5: + version "0.0.6" + resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" + +underscore@~1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.6.0.tgz#8b38b10cacdef63337b8b24e4ff86d45aea529a8" + +user-home@^1.0.0, user-home@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/user-home/-/user-home-1.1.1.tgz#2b5be23a32b63a7c9deb8d0f28d485724a3df190" + +uuid@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04" + +v8flags@^2.0.2: + version "2.1.1" + resolved "https://registry.yarnpkg.com/v8flags/-/v8flags-2.1.1.tgz#aab1a1fa30d45f88dd321148875ac02c0b55e5b4" + dependencies: + user-home "^1.1.1" + +wgs84@0.0.0: + version "0.0.0" + resolved "https://registry.yarnpkg.com/wgs84/-/wgs84-0.0.0.tgz#34fdc555917b6e57cf2a282ed043710c049cdc76" + +which@^1.2.12: + version "1.3.0" + resolved "https://registry.yarnpkg.com/which/-/which-1.3.0.tgz#ff04bdfc010ee547d780bec38e1ac1c2777d253a" + dependencies: + isexe "^2.0.0" + +xtend@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af"