Bring in Output Area census zones from popgetter, with vehicle owners…

…hip stats. Bake into the map model. #116

Manually add England data.

Cargo.toml

@@ -37,12 +37,12 @@ opt-level = 3
 # Specify the versions for common dependencies just once here, instead of
 # repeating in a bunch of crates
 [workspace.dependencies]
-anyhow = "1.0.38"
+anyhow = "1.0.69"
 bincode = "1.3.1"
 colorous = "1.0.9"
 contour = "0.7.0"
 csv = "1.2.0"
-fs-err = "2.6.0"
+fs-err = "2.9.0"
 geo = "0.24.0"
 geojson = { version = "0.24.0", features = ["geo-types"] }
 getrandom = "0.2.3"

convert_osm/Cargo.toml

@@ -14,6 +14,7 @@ geom = { path = "../geom" }
 kml = { path = "../kml" }
 log = { workspace = true }
 osm2streets = { git = "https://github.com/a-b-street/osm2streets" }
+popgetter = { git = "https://github.com/dabreegster/popgetter/" }
 raw_map = { path = "../raw_map" }
 serde = { workspace = true }
 streets_reader = { git = "https://github.com/a-b-street/osm2streets" }

convert_osm/src/lib.rs

@@ -152,6 +152,12 @@ pub fn convert(
         gtfs::import(&mut map).unwrap();
     }
 
+    timer.start("Add census data");
+    if let Err(err) = add_census(&mut map) {
+        error!("Skipping census data: {err}");
+    }
+    timer.stop("Add census data");
+
     if map.name == MapName::new("gb", "bristol", "east") {
         bristol_hack(&mut map);
     }
@@ -379,3 +385,23 @@ fn filter_crosswalks(
         }
     }
 }
+
+fn add_census(map: &mut RawMap) -> Result<()> {
+    // TODO Fixed to one area right now. Assumes the file exists.
+    let input_path = "data/input/shared/popgetter/england.topojson";
+    let boundary = map
+        .streets
+        .boundary_polygon
+        .to_geo_wgs84(&map.streets.gps_bounds);
+    for (geo_polygon, census_zone) in popgetter::clip_zones(input_path, boundary)? {
+        match Polygon::from_geo_wgs84(geo_polygon, &map.streets.gps_bounds) {
+            Ok(polygon) => {
+                map.census_zones.push((polygon, census_zone));
+            }
+            Err(err) => {
+                warn!("Skipping census zone {}: {}", census_zone.id, err);
+            }
+        }
+    }
+    Ok(())
+}

data/MANIFEST.json

@@ -216,8 +216,8 @@
       "compressed_size_bytes": 13642164
     },
     "data/input/fr/paris/raw_maps/center.bin": {
-      "checksum": "6f2811ae79e8b71cb2f596fb456d7dfe",
-      "uncompressed_size_bytes": 25065354,
+      "checksum": "896d1a1ae5a7dacd63d32b5fd6d931f6",
+      "uncompressed_size_bytes": 25065338,
       "compressed_size_bytes": 6777329
     },
     "data/input/fr/paris/raw_maps/east.bin": {
@@ -1530,6 +1530,11 @@
       "uncompressed_size_bytes": 272419737,
       "compressed_size_bytes": 271888733
     },
+    "data/input/shared/popgetter/england.topojson": {
+      "checksum": "a894795a5bfadf60899f4ba892a32789",
+      "uncompressed_size_bytes": 52802533,
+      "compressed_size_bytes": 7976809
+    },
     "data/input/shared/wu03ew_v2.csv": {
       "checksum": "6614880a64e448b4e9a7255ab5355d00",
       "uncompressed_size_bytes": 108977615,

geom/src/polygon.rs

@@ -2,7 +2,9 @@ use std::collections::BTreeMap;
 use std::fmt;
 
 use anyhow::Result;
-use geo::{Area, BooleanOps, Contains, ConvexHull, Intersects, SimplifyVwPreserve};
+use geo::{
+    Area, BooleanOps, Contains, ConvexHull, Intersects, MapCoordsInPlace, SimplifyVwPreserve,
+};
 use serde::{Deserialize, Serialize};
 
 use crate::{
@@ -410,8 +412,6 @@ impl Polygon {
 
     /// Optionally map the world-space points back to GPS.
     pub fn to_geojson(&self, gps: Option<&GPSBounds>) -> geojson::Geometry {
-        use geo::MapCoordsInPlace;
-
         let mut geom: geo::Geometry = self.to_geo().into();
         if let Some(ref gps_bounds) = gps {
             geom.map_coords_in_place(|c| {
@@ -498,6 +498,24 @@ impl Polygon {
     fn to_geo(&self) -> geo::Polygon {
         self.clone().into()
     }
+
+    /// Convert to `geo` and also map from world-space to WGS84
+    pub fn to_geo_wgs84(&self, gps_bounds: &GPSBounds) -> geo::Polygon<f64> {
+        let mut p = self.to_geo();
+        p.map_coords_in_place(|c| {
+            let gps = Pt2D::new(c.x, c.y).to_gps(gps_bounds);
+            (gps.x(), gps.y()).into()
+        });
+        p
+    }
+
+    pub fn from_geo_wgs84(mut polygon: geo::Polygon<f64>, gps_bounds: &GPSBounds) -> Result<Self> {
+        polygon.map_coords_in_place(|c| {
+            let pt = LonLat::new(c.x, c.y).to_pt(gps_bounds);
+            (pt.x(), pt.y()).into()
+        });
+        polygon.try_into()
+    }
 }
 
 impl fmt::Display for Polygon {

map_model/Cargo.toml

@@ -17,6 +17,7 @@ log = { workspace = true }
 lyon = "1.0.1"
 md5 = "0.7.0"
 petgraph = { version = "0.6.3", features=["serde-1"] }
+popgetter = { git = "https://github.com/dabreegster/popgetter/" }
 rand = { workspace = true }
 rand_xorshift = { workspace = true }
 raw_map = { path = "../raw_map" }

map_model/src/lib.rs

@@ -30,6 +30,7 @@ extern crate log;
 
 use std::collections::BTreeMap;
 
+use popgetter::CensusZone;
 use serde::{Deserialize, Serialize};
 
 use abstio::MapName;
@@ -115,6 +116,7 @@ pub struct Map {
     routing_params: RoutingParams,
     // Not the source of truth, just cached.
     zones: Vec<Zone>,
+    census_zones: Vec<(Polygon, CensusZone)>,
 
     name: MapName,
 

map_model/src/make/mod.rs

@@ -54,6 +54,7 @@ impl Map {
             areas: Vec::new(),
             parking_lots: Vec::new(),
             zones: Vec::new(),
+            census_zones: raw.census_zones.clone(),
             boundary_polygon: raw.streets.boundary_polygon.clone(),
             stop_signs: BTreeMap::new(),
             traffic_signals: BTreeMap::new(),

map_model/src/map.rs

@@ -4,6 +4,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
 
 use anyhow::Result;
 use petgraph::graphmap::{DiGraphMap, UnGraphMap};
+use popgetter::CensusZone;
 
 use abstio::{CityName, MapName};
 use abstutil::{prettyprint_usize, serialized_size_bytes, MultiMap, Tags, Timer};
@@ -104,6 +105,11 @@ impl Map {
                     self.zones.len(),
                     serialized_size_bytes(&self.zones),
                 ),
+                (
+                    "census_zones",
+                    self.census_zones.len(),
+                    serialized_size_bytes(&self.census_zones),
+                ),
                 ("pathfinder", 1, serialized_size_bytes(&self.pathfinder)),
             ];
             costs.sort_by_key(|(_, _, bytes)| *bytes);
@@ -135,6 +141,7 @@ impl Map {
             areas: Vec::new(),
             parking_lots: Vec::new(),
             zones: Vec::new(),
+            census_zones: Vec::new(),
             boundary_polygon: Ring::must_new(vec![
                 Pt2D::new(0.0, 0.0),
                 Pt2D::new(1.0, 0.0),
@@ -257,6 +264,10 @@ impl Map {
         &self.zones
     }
 
+    pub fn all_census_zones(&self) -> &Vec<(Polygon, CensusZone)> {
+        &self.census_zones
+    }
+
     pub fn maybe_get_r(&self, id: RoadID) -> Option<&Road> {
         self.roads.get(id.0)
     }

map_model/src/pathfind/mod.rs

@@ -193,8 +193,6 @@ pub struct RoutingParams {
     /// TODO The route may cross one of these roads if it's the start or end!
     pub avoid_roads: BTreeSet<RoadID>,
     /// Related to `avoid_roads`, but used as an optimization in map construction
-    // TODO Serialize this normally. In the meantime, safe because the default is indeed empty.
-    #[serde(skip_serializing, skip_deserializing)]
     pub only_use_roads: BTreeSet<RoadID>,
 
     /// Don't allow movements between these roads at all. Only affects vehicle routing, not

raw_map/Cargo.toml

@@ -10,5 +10,6 @@ abstutil = { path = "../abstutil" }
 geom = { path = "../geom" }
 serde = { workspace = true }
 osm2streets = { git = "https://github.com/a-b-street/osm2streets" }
+popgetter = { git = "https://github.com/dabreegster/popgetter/" }
 strum = "0.24.1"
 strum_macros = "0.24.3"

raw_map/src/lib.rs

@@ -5,6 +5,7 @@
 use std::collections::BTreeMap;
 
 use osm2streets::{osm, IntersectionID, RoadID, StreetNetwork};
+use popgetter::CensusZone;
 use serde::{Deserialize, Serialize};
 
 use abstio::{CityName, MapName};
@@ -31,6 +32,7 @@ pub struct RawMap {
     pub parking_lots: Vec<RawParkingLot>,
     pub parking_aisles: Vec<(osm::WayID, Vec<Pt2D>)>,
     pub transit_routes: Vec<RawTransitRoute>,
+    pub census_zones: Vec<(Polygon, CensusZone)>,
     #[serde(
         serialize_with = "serialize_btreemap",
         deserialize_with = "deserialize_btreemap"
@@ -74,6 +76,7 @@ impl RawMap {
             parking_lots: Vec::new(),
             parking_aisles: Vec::new(),
             transit_routes: Vec::new(),
+            census_zones: Vec::new(),
             transit_stops: BTreeMap::new(),
             bus_routes_on_roads: MultiMap::new(),
             osm_tags: BTreeMap::new(),