Merge pull request #5 from evansiroky/2016i

2016i
evansiroky · Nov 7, 2016 · 263766e · 263766e
2 parents 81af15c + b57a5b9
commit 263766e
Show file tree

Hide file tree

Showing 5 changed files with 177 additions and 108 deletions.
diff --git a/README.md b/README.md
@@ -8,7 +8,7 @@ There are two config files that describe the boundary building process.  The `os
 
 ### Special note regarding the Overpass API
 
-Although the code currently queries the publicly available overpass API, this is not recommended because you will get throttled really quickly!  I was able to get away with this because I worked on extracting pieces of data over a few months.  In the future, I hope to make a docker file that downloads a planet dump and starts a local version of the Overpass API to extract the data.
+The code does query the publicly available overpass API, but it self-throttles the making of requests to have a minimum of 4 seconds gap between requests.  If the Overpass API throttles the download, then the gap will be increased exponentionally.
 
 ### Running the project
 

diff --git a/index.js b/index.js
@@ -1,37 +1,36 @@
-var exec = require('child_process').exec,
-  fs = require('fs'),
-  http = require('http')
-
-var async = require('async'),
-  jsts = require('jsts'),
-  multiPolygon = require('turf-multipolygon'),
-  overpass = require('query-overpass'),
-  polygon = require('turf-polygon'),
-  shp = require('shpjs')
-
-
-var osmBoundarySources = require('./osmBoundarySources.json'),
-  zoneCfg = require('./timezones.json'),
-  geoJsonReader = new jsts.io.GeoJSONReader(),
-  geoJsonWriter = new jsts.io.GeoJSONWriter(),
-  distZones = {}
-
-var safeMkdir = function(dirname, callback) {
-  fs.mkdir(dirname, function(err) {
-    if(err && err.code === 'EEXIST') {
+var exec = require('child_process').exec
+var fs = require('fs')
+
+var asynclib = require('async')
+var jsts = require('jsts')
+var rimraf = require('rimraf')
+var multiPolygon = require('turf-multipolygon')
+var overpass = require('query-overpass')
+var polygon = require('turf-polygon')
+
+var osmBoundarySources = require('./osmBoundarySources.json')
+var zoneCfg = require('./timezones.json')
+var geoJsonReader = new jsts.io.GeoJSONReader()
+var geoJsonWriter = new jsts.io.GeoJSONWriter()
+var distZones = {}
+var minRequestGap = 4
+var curRequestGap = 4
+
+var safeMkdir = function (dirname, callback) {
+  fs.mkdir(dirname, function (err) {
+    if (err && err.code === 'EEXIST') {
       callback()
     } else {
       callback(err)
     }
   })
 }
 
-debugGeo = function(op, a, b) {
-
+var debugGeo = function (op, a, b) {
   var result
 
   try {
-    switch(op) {
+    switch (op) {
       case 'union':
         result = a.union(b)
         break
@@ -41,11 +40,11 @@ debugGeo = function(op, a, b) {
       case 'diff':
         try {
           result = a.difference(b)
-        } catch(e) {
-          if(e.name === 'TopologyException') {
+        } catch (e) {
+          if (e.name === 'TopologyException') {
             console.log('retry with GeometryPrecisionReducer')
-            var precisionModel = new jsts.geom.PrecisionModel(10000),
-              precisionReducer = new jsts.precision.GeometryPrecisionReducer(precisionModel)
+            var precisionModel = new jsts.geom.PrecisionModel(10000)
+            var precisionReducer = new jsts.precision.GeometryPrecisionReducer(precisionModel)
 
             a = precisionReducer.reduce(a)
             b = precisionReducer.reduce(b)
@@ -60,7 +59,7 @@ debugGeo = function(op, a, b) {
         var err = new Error('invalid op: ' + op)
         throw err
     }
-  } catch(e) {
+  } catch (e) {
     console.log('op err')
     console.log(e)
     console.log(e.stack)
@@ -72,70 +71,87 @@ debugGeo = function(op, a, b) {
   return result
 }
 
-var fetchIfNeeded = function(file, superCallback, fetchFn) {
-  fs.stat(file, function(err) {
-    if(!err) { return superCallback() }
+var fetchIfNeeded = function (file, superCallback, fetchFn) {
+  fs.stat(file, function (err) {
+    if (!err) { return superCallback() }
     fetchFn()
   })
 }
 
-var geoJsonToGeom = function(geoJson) {
+var geoJsonToGeom = function (geoJson) {
   return geoJsonReader.read(JSON.stringify(geoJson))
 }
 
 var geomToGeoJson = function (geom) {
   return geoJsonWriter.write(geom)
 }
 
-var geomToGeoJsonString = function(geom) {
+var geomToGeoJsonString = function (geom) {
   return JSON.stringify(geoJsonWriter.write(geom))
 }
 
-var downloadOsmBoundary = function(boundaryId, boundaryCallback) {
-  var cfg = osmBoundarySources[boundaryId],
-    query = '[out:json][timeout:60];(relation',
-    boundaryFilename = './downloads/' + boundaryId + '.json',
-    debug = 'getting data for ' + boundaryId,
-    queryKeys = Object.keys(cfg)
+var downloadOsmBoundary = function (boundaryId, boundaryCallback) {
+  var cfg = osmBoundarySources[boundaryId]
+  var query = '[out:json][timeout:60];(relation'
+  var boundaryFilename = './downloads/' + boundaryId + '.json'
+  var debug = 'getting data for ' + boundaryId
+  var queryKeys = Object.keys(cfg)
 
   for (var i = queryKeys.length - 1; i >= 0; i--) {
-    var k = queryKeys[i],
-      v = cfg[k]
+    var k = queryKeys[i]
+    var v = cfg[k]
 
     query += '["' + k + '"="' + v + '"]'
-
   }
 
   query += ');out body;>;out meta qt;'
 
   console.log(debug)
 
-  async.auto({
-    downloadFromOverpass: function(cb) {
+  asynclib.auto({
+    downloadFromOverpass: function (cb) {
       console.log('downloading from overpass')
-      fetchIfNeeded(boundaryFilename, boundaryCallback, function() {
-        overpass(query, cb, { flatProperties: true })
+      fetchIfNeeded(boundaryFilename, boundaryCallback, function () {
+        var overpassResponseHandler = function (err, data) {
+          if (err) {
+            console.log(err)
+            console.log('Increasing overpass request gap')
+            curRequestGap *= 2
+            makeQuery()
+          } else {
+            console.log('Success, decreasing overpass request gap')
+            curRequestGap = Math.max(minRequestGap, curRequestGap / 2)
+            cb(null, data)
+          }
+        }
+        var makeQuery = function () {
+          console.log('waiting ' + curRequestGap + ' seconds')
+          setTimeout(function () {
+            overpass(query, overpassResponseHandler, { flatProperties: true })
+          }, curRequestGap * 1000)
+        }
+        makeQuery()
       })
     },
-    validateOverpassResult: ['downloadFromOverpass', function(results, cb) {
+    validateOverpassResult: ['downloadFromOverpass', function (results, cb) {
       var data = results.downloadFromOverpass
-      if(!data.features || data.features.length == 0) {
-        err = new Error('Invalid geojson for boundary: ' + boundaryId)
+      if (!data.features || data.features.length === 0) {
+        var err = new Error('Invalid geojson for boundary: ' + boundaryId)
         return cb(err)
       }
       cb()
     }],
-    saveSingleMultiPolygon: ['validateOverpassResult', function(results, cb) {
-      var data = results.downloadFromOverpass,
-        combined
+    saveSingleMultiPolygon: ['validateOverpassResult', function (results, cb) {
+      var data = results.downloadFromOverpass
+      var combined
 
       // union all multi-polygons / polygons into one
       for (var i = data.features.length - 1; i >= 0; i--) {
         var curOsmGeom = data.features[i].geometry
-        if(curOsmGeom.type === 'Polygon' || curOsmGeom.type === 'MultiPolygon') {
+        if (curOsmGeom.type === 'Polygon' || curOsmGeom.type === 'MultiPolygon') {
           console.log('combining border')
           var curGeom = geoJsonToGeom(curOsmGeom)
-          if(!combined) {
+          if (!combined) {
             combined = curGeom
           } else {
             combined = debugGeo('union', curGeom, combined)
@@ -160,15 +176,15 @@ var getTzDistFilename = function (tzid) {
  *     - `id` if from a file
  *     - `id` if from a file
  */
-var getDataSource = function(source) {
+var getDataSource = function (source) {
   var geoJson
-  if(source.source === 'overpass') {
+  if (source.source === 'overpass') {
     geoJson = require('./downloads/' + source.id + '.json')
-  } else if(source.source === 'manual-polygon') {
+  } else if (source.source === 'manual-polygon') {
     geoJson = polygon(source.data).geometry
-  } else if(source.source === 'manual-multipolygon') {
+  } else if (source.source === 'manual-multipolygon') {
     geoJson = multiPolygon(source.data).geometry
-  } else if(source.source === 'dist') {
+  } else if (source.source === 'dist') {
     geoJson = require(getTzDistFilename(source.id))
   } else {
     var err = new Error('unknown source: ' + source.source)
@@ -177,33 +193,33 @@ var getDataSource = function(source) {
   return geoJsonToGeom(geoJson)
 }
 
-var makeTimezoneBoundary = function(tzid, callback) {
+var makeTimezoneBoundary = function (tzid, callback) {
   console.log('makeTimezoneBoundary for', tzid)
 
-  var ops = zoneCfg[tzid],
-    geom
+  var ops = zoneCfg[tzid]
+  var geom
 
-  async.eachSeries(ops, function(task, cb) {
+  asynclib.eachSeries(ops, function (task, cb) {
     var taskData = getDataSource(task)
     console.log('-', task.op, task.id)
-    if(task.op === 'init') {
+    if (task.op === 'init') {
       geom = taskData
-    } else if(task.op === 'intersect') {
+    } else if (task.op === 'intersect') {
       geom = debugGeo('intersection', geom, taskData)
-    } else if(task.op === 'difference') {
+    } else if (task.op === 'difference') {
       geom = debugGeo('diff', geom, taskData)
-    } else if(task.op === 'difference-reverse-order') {
+    } else if (task.op === 'difference-reverse-order') {
       geom = debugGeo('diff', taskData, geom)
-    } else if(task.op === 'union') {
+    } else if (task.op === 'union') {
       geom = debugGeo('union', geom, taskData)
     } else {
       var err = new Error('unknown op: ' + task.op)
       return cb(err)
     }
     cb()
   },
-  function(err) {
-    if(err) { return callback(err) }
+  function (err) {
+    if (err) { return callback(err) }
     fs.writeFile(getTzDistFilename(tzid),
       geomToGeoJsonString(geom),
       callback)
@@ -212,8 +228,8 @@ var makeTimezoneBoundary = function(tzid, callback) {
 
 var loadDistZonesIntoMemory = function () {
   console.log('load zones into memory')
-  var zones = Object.keys(zoneCfg),
-    tzid
+  var zones = Object.keys(zoneCfg)
+  var tzid
 
   for (var i = 0; i < zones.length; i++) {
     tzid = zones[i]
@@ -227,9 +243,9 @@ var getDistZoneGeom = function (tzid) {
 
 var validateTimezoneBoundaries = function () {
   console.log('do validation')
-  var allZonesOk = true,
-    zones = Object.keys(zoneCfg),
-    compareTzid, tzid, zoneGeom
+  var allZonesOk = true
+  var zones = Object.keys(zoneCfg)
+  var compareTzid, tzid, zoneGeom
 
   for (var i = 0; i < zones.length; i++) {
     tzid = zones[i]
@@ -239,11 +255,11 @@ var validateTimezoneBoundaries = function () {
       compareTzid = zones[j]
 
       var compareZoneGeom = getDistZoneGeom(compareTzid)
-      if(zoneGeom.intersects(compareZoneGeom)) {
-        var intersectedGeom = debugGeo('intersection', zoneGeom, compareZoneGeom),
-          intersectedArea = intersectedGeom.getArea()
+      if (zoneGeom.intersects(compareZoneGeom)) {
+        var intersectedGeom = debugGeo('intersection', zoneGeom, compareZoneGeom)
+        var intersectedArea = intersectedGeom.getArea()
 
-        if(intersectedArea > 0.0001) {
+        if (intersectedArea > 0.0001) {
           console.log('Validation error: ' + tzid + ' intersects ' + compareTzid + ' area: ' + intersectedArea)
           allZonesOk = false
         }
@@ -252,17 +268,16 @@ var validateTimezoneBoundaries = function () {
   }
 
   return allZonesOk ? null : 'Zone validation unsuccessful'
-
 }
 
-var combineAndWriteZones = function(callback) {
+var combineAndWriteZones = function (callback) {
   var stream = fs.createWriteStream('./dist/combined.json')
   var zones = Object.keys(zoneCfg)
 
   stream.write('{"type":"FeatureCollection","features":[')
 
   for (var i = 0; i < zones.length; i++) {
-    if(i > 0) {
+    if (i > 0) {
       stream.write(',')
     }
     var feature = {
@@ -275,29 +290,29 @@ var combineAndWriteZones = function(callback) {
   stream.end(']}', callback)
 }
 
-async.auto({
-  makeDownloadsDir: function(cb) {
+asynclib.auto({
+  makeDownloadsDir: function (cb) {
     console.log('creating downloads dir')
     safeMkdir('./downloads', cb)
   },
-  makeDistDir: function(cb) {
+  makeDistDir: function (cb) {
     console.log('createing dist dir')
     safeMkdir('./dist', cb)
   },
-  getOsmBoundaries: ['makeDownloadsDir', function(results, cb) {
+  getOsmBoundaries: ['makeDownloadsDir', function (results, cb) {
     console.log('downloading osm boundaries')
-    async.eachSeries(Object.keys(osmBoundarySources), downloadOsmBoundary, cb)
+    asynclib.eachSeries(Object.keys(osmBoundarySources), downloadOsmBoundary, cb)
   }],
-  createZones: ['makeDistDir', 'getOsmBoundaries', function(results, cb) {
+  createZones: ['makeDistDir', 'getOsmBoundaries', function (results, cb) {
     console.log('createZones')
-    async.each(Object.keys(zoneCfg), makeTimezoneBoundary, cb)
+    asynclib.each(Object.keys(zoneCfg), makeTimezoneBoundary, cb)
   }],
-  validateZones: ['createZones', function(results, cb) {
+  validateZones: ['createZones', function (results, cb) {
     console.log('validating zones')
     loadDistZonesIntoMemory()
     cb(validateTimezoneBoundaries())
   }],
-  mergeZones: ['validateZones', function(results, cb) {
+  mergeZones: ['validateZones', function (results, cb) {
     console.log('merge zones')
     combineAndWriteZones(cb)
   }],
@@ -307,14 +322,16 @@ async.auto({
   }],
   makeShapefile: ['mergeZones', function (results, cb) {
     console.log('convert from geojson to shapefile')
+    rimraf.sync('dist/dist')
+    rimraf.sync('dist/combined_shapefile.*')
     exec('ogr2ogr -nlt MULTIPOLYGON dist/combined_shapefile.shp dist/combined.json OGRGeoJSON', function (err, stdout, stderr) {
-      if(err) { return cb(err) }
+      if (err) { return cb(err) }
       exec('zip dist/timezones.shapefile.zip dist/combined_shapefile.*', cb)
     })
   }]
-}, function(err, results) {
+}, function (err, results) {
   console.log('done')
-  if(err) {
+  if (err) {
     console.log('error!', err)
     return
   }