From 4ce5f62141e1cdbf9a16d731d20d9604c6736190 Mon Sep 17 00:00:00 2001 From: Dustin Carlino Date: Wed, 13 Dec 2023 14:00:12 +0900 Subject: [PATCH] Switch to osm-reader to handle XML and PBF uniformly --- Cargo.lock | 27 +- experimental/src/io.rs | 2 +- osm2lanes/Cargo.toml | 1 + osm2lanes/src/osm.rs | 52 +- osm2streets-java/src/StreetNetwork.java | 6 +- osm2streets-java/src/lib.rs | 13 +- .../src/org_osm2streets_StreetNetwork.h | 4 +- osm2streets-js/src/lib.rs | 9 +- streets_reader/Cargo.toml | 3 +- streets_reader/src/lib.rs | 54 +- streets_reader/src/osm_reader/mod.rs | 1 - streets_reader/src/osm_reader/reader.rs | 488 ++++-------------- tests/src/lib.rs | 6 +- web/src/osm2streets-svelte/Geocoder.svelte | 4 +- .../import/ImportControls.svelte | 3 +- .../osm_input/BuiltInSelector.svelte | 2 +- 16 files changed, 146 insertions(+), 529 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 16857f89..ad5c68b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -862,6 +862,17 @@ dependencies = [ "serde", ] +[[package]] +name = "osm-reader" +version = "0.1.0" +source = "git+https://github.com/a-b-street/osm-reader#e8e7539d77042e4b9cda38ae3ced97a5e4f5503c" +dependencies = [ + "anyhow", + "osmpbf", + "roxmltree", + "serde", +] + [[package]] name = "osm2lanes" version = "0.1.0" @@ -871,6 +882,7 @@ dependencies = [ "enumset", "env_logger", "geom", + "osm-reader", "serde", ] @@ -1138,6 +1150,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cbf4a6aa5f6d6888f39e980649f3ad6b666acdce1d78e95b8a2cb076e687ae30" +[[package]] +name = "roxmltree" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cd14fd5e3b777a7422cca79358c57a8f6e3a703d9ac187448d0daf220c2407f" + [[package]] name = "rstar" version = "0.11.0" @@ -1326,9 +1344,8 @@ dependencies = [ "country-geocoder", "geom", "log", + "osm-reader", "osm2streets", - "osmpbf", - "xmlparser", ] [[package]] @@ -1771,12 +1788,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" -[[package]] -name = "xmlparser" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d25c75bf9ea12c4040a97f829154768bbbce366287e2dc044af160cd79a13fd" - [[package]] name = "zerocopy" version = "0.7.26" diff --git a/experimental/src/io.rs b/experimental/src/io.rs index 11a14c9c..53c2e72d 100644 --- a/experimental/src/io.rs +++ b/experimental/src/io.rs @@ -28,7 +28,7 @@ pub fn load_road_network(osm_path: String, timer: &mut Timer) -> Result fmt::Result { - write!(f, "https://www.openstreetmap.org/node/{}", self.0) - } -} -impl fmt::Display for WayID { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "https://www.openstreetmap.org/way/{}", self.0) - } -} -impl fmt::Display for RelationID { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "https://www.openstreetmap.org/relation/{}", self.0) - } -} - -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, PartialOrd, Ord, Serialize, Deserialize)] -pub enum OsmID { - Node(NodeID), - Way(WayID), - Relation(RelationID), -} -impl fmt::Display for OsmID { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - OsmID::Node(n) => write!(f, "{}", n), - OsmID::Way(w) => write!(f, "{}", w), - OsmID::Relation(r) => write!(f, "{}", r), - } - } -} -impl OsmID { - pub fn inner(self) -> i64 { - match self { - OsmID::Node(n) => n.0, - OsmID::Way(w) => w.0, - OsmID::Relation(r) => r.0, - } - } -} diff --git a/osm2streets-java/src/StreetNetwork.java b/osm2streets-java/src/StreetNetwork.java index ac66803b..a567de9d 100644 --- a/osm2streets-java/src/StreetNetwork.java +++ b/osm2streets-java/src/StreetNetwork.java @@ -36,15 +36,15 @@ public class StreetNetwork { } } - public static native StreetNetwork create(String osmXmlInput); + public static native StreetNetwork create(byte[] osmInput); public native List getSurfaces(); public native List getPaintAreas(); public static void main(String[] args) throws Exception { - String osmXmlInput = new String(Files.readAllBytes(Paths.get("../tests/src/aurora_sausage_link/input.osm"))); - StreetNetwork network = create(osmXmlInput); + byte[] osmInput = Files.readAllBytes(Paths.get("../tests/src/aurora_sausage_link/input.osm")); + StreetNetwork network = create(osmInput); System.out.println(network.getSurfaces()); System.out.println(network.getPaintAreas()); } diff --git a/osm2streets-java/src/lib.rs b/osm2streets-java/src/lib.rs index 015249fe..8fc02422 100644 --- a/osm2streets-java/src/lib.rs +++ b/osm2streets-java/src/lib.rs @@ -1,5 +1,5 @@ use abstutil::Timer; -use jni::objects::{JClass, JObject, JString, JValue}; +use jni::objects::{JClass, JObject, JValue}; use jni::sys::{jlong, jobject}; use jni::JNIEnv; @@ -10,14 +10,13 @@ struct StreetNetwork { } impl StreetNetwork { - fn new(osm_xml_input: String) -> Self { + fn new(input_bytes: &[u8]) -> Self { let cfg = MapConfig::default(); let clip_pts = None; let mut timer = Timer::throwaway(); let (mut network, _) = - streets_reader::osm_to_street_network(&osm_xml_input, clip_pts, cfg, &mut timer) - .unwrap(); + streets_reader::osm_to_street_network(input_bytes, clip_pts, cfg, &mut timer).unwrap(); let transformations = Transformation::standard_for_clipped_areas(); network.apply_transformations(transformations, &mut timer); @@ -29,10 +28,10 @@ impl StreetNetwork { pub extern "system" fn Java_org_osm2streets_StreetNetwork_create( env: JNIEnv, _: JClass, - osm_xml_input: JString, + input_bytes: jni::sys::jbyteArray, ) -> jobject { - let osm_xml_input: String = env.get_string(osm_xml_input).unwrap().into(); - let network = StreetNetwork::new(osm_xml_input); + let input_bytes: Vec = env.convert_byte_array(input_bytes).unwrap(); + let network = StreetNetwork::new(&input_bytes); let pointer = Box::into_raw(Box::new(network)) as jlong; let obj_class = env.find_class("org/osm2streets/StreetNetwork").unwrap(); diff --git a/osm2streets-java/src/org_osm2streets_StreetNetwork.h b/osm2streets-java/src/org_osm2streets_StreetNetwork.h index 2e02af5e..9d34c871 100644 --- a/osm2streets-java/src/org_osm2streets_StreetNetwork.h +++ b/osm2streets-java/src/org_osm2streets_StreetNetwork.h @@ -10,10 +10,10 @@ extern "C" { /* * Class: org_osm2streets_StreetNetwork * Method: create - * Signature: (Ljava/lang/String;)Lorg/osm2streets/StreetNetwork; + * Signature: ([B)Lorg/osm2streets/StreetNetwork; */ JNIEXPORT jobject JNICALL Java_org_osm2streets_StreetNetwork_create - (JNIEnv *, jclass, jstring); + (JNIEnv *, jclass, jbyteArray); /* * Class: org_osm2streets_StreetNetwork diff --git a/osm2streets-js/src/lib.rs b/osm2streets-js/src/lib.rs index af9cf4b0..70b59322 100644 --- a/osm2streets-js/src/lib.rs +++ b/osm2streets-js/src/lib.rs @@ -32,7 +32,7 @@ impl JsStreetNetwork { // TODO clip_pts_geojson should be Option. Empty means None. #[wasm_bindgen(constructor)] pub fn new( - osm_xml_input: &str, + osm_input: &[u8], clip_pts_geojson: &str, input: JsValue, ) -> Result { @@ -60,7 +60,7 @@ impl JsStreetNetwork { let mut timer = Timer::throwaway(); let (mut street_network, doc) = - streets_reader::osm_to_street_network(osm_xml_input, clip_pts, cfg, &mut timer) + streets_reader::osm_to_street_network(osm_input, clip_pts, cfg, &mut timer) .map_err(|err| JsValue::from_str(&err.to_string()))?; let mut transformations = Transformation::standard_for_clipped_areas(); if input.dual_carriageway_experiment { @@ -229,9 +229,10 @@ impl JsStreetNetwork { pub fn way_to_xml(&self, id: i64) -> String { let way = &self.ways[&osm::WayID(id)]; let mut out = format!(r#"\n"); for node in &way.nodes { out.push_str(&format!(r#" "#, node.0)); diff --git a/streets_reader/Cargo.toml b/streets_reader/Cargo.toml index ce112c4f..621d5eec 100644 --- a/streets_reader/Cargo.toml +++ b/streets_reader/Cargo.toml @@ -10,6 +10,5 @@ anyhow = { workspace = true } country-geocoder = { git = "https://github.com/a-b-street/country-geocoder" } geom = { workspace = true } log = "0.4.14" +osm-reader = { git = "https://github.com/a-b-street/osm-reader" } osm2streets = { path = "../osm2streets" } -osmpbf = "0.3.2" -xmlparser = "0.13.5" diff --git a/streets_reader/src/lib.rs b/streets_reader/src/lib.rs index 3d6d0bc3..13b6aa48 100644 --- a/streets_reader/src/lib.rs +++ b/streets_reader/src/lib.rs @@ -17,14 +17,14 @@ pub mod extract; pub mod osm_reader; pub mod split_ways; -/// Create a `StreetNetwork` from the contents of an `.osm.xml` file. If `clip_pts` is specified, -/// use these as a boundary polygon. (Use `LonLat::read_geojson_polygon` or similar to produce -/// these.) +/// Create a `StreetNetwork` from the contents of an `.osm.xml` or `.pbf` file. If `clip_pts` is +/// specified, use these as a boundary polygon. (Use `LonLat::read_geojson_polygon` or similar to +/// produce these.) /// /// You probably want to do `StreetNetwork::apply_transformations` on the result to get a useful /// result. pub fn osm_to_street_network( - osm_xml_input: &str, + input_bytes: &[u8], clip_pts: Option>, cfg: MapConfig, timer: &mut Timer, @@ -34,7 +34,7 @@ pub fn osm_to_street_network( // happens in split_ways. streets.config = cfg; - let (extract, doc) = extract_osm(&mut streets, osm_xml_input, clip_pts, timer)?; + let (extract, doc) = extract_osm(&mut streets, input_bytes, clip_pts, timer)?; split_ways::split_up_roads(&mut streets, extract, timer); // Cul-de-sacs aren't supported yet. @@ -43,31 +43,6 @@ pub fn osm_to_street_network( Ok((streets, doc)) } -pub fn pbf_to_street_network( - input: &[u8], - clip_pts: Option>, - cfg: MapConfig, - timer: &mut Timer, -) -> Result<(StreetNetwork, Document)> { - let mut streets = StreetNetwork::blank(); - // Note that DrivingSide is still incorrect. It'll be set in extract_osm, before Road::new - // happens in split_ways. - streets.config = cfg; - - let mut doc = Document::read_pbf( - input, - clip_pts.as_ref().map(|pts| GPSBounds::from(pts.clone())), - timer, - )?; - - let out = process_data(&mut streets, clip_pts, timer, &mut doc)?; - split_ways::split_up_roads(&mut streets, out, timer); - - // Cul-de-sacs aren't supported yet. - streets.retain_roads(|r| r.src_i != r.dst_i); - Ok((streets, doc)) -} - /// Set up country code and driving side, using an arbitrary point. This must be called after /// `gps_bounds` is set. pub fn detect_country_code(streets: &mut StreetNetwork) { @@ -88,27 +63,15 @@ pub fn detect_country_code(streets: &mut StreetNetwork) { fn extract_osm( streets: &mut StreetNetwork, - osm_xml_input: &str, + input_bytes: &[u8], clip_pts: Option>, timer: &mut Timer, ) -> Result<(OsmExtract, Document)> { let mut doc = Document::read( - osm_xml_input, + input_bytes, clip_pts.as_ref().map(|pts| GPSBounds::from(pts.clone())), timer, )?; - // If GPSBounds aren't provided above, they'll be computed in the Document - let out = process_data(streets, clip_pts, timer, &mut doc)?; - - Ok((out, doc)) -} - -fn process_data( - streets: &mut StreetNetwork, - clip_pts: Option>, - timer: &mut Timer, - doc: &mut Document, -) -> Result { // If GPSBounds aren't provided, they'll be computed in the Document streets.gps_bounds = doc.gps_bounds.clone().unwrap(); @@ -150,5 +113,6 @@ fn process_data( timer.next(); out.handle_relation(*id, rel); } - Ok(out) + + Ok((out, doc)) } diff --git a/streets_reader/src/osm_reader/mod.rs b/streets_reader/src/osm_reader/mod.rs index 3d88b264..f9fa1ebb 100644 --- a/streets_reader/src/osm_reader/mod.rs +++ b/streets_reader/src/osm_reader/mod.rs @@ -32,7 +32,6 @@ pub struct Way { pub nodes: Vec, pub pts: Vec, pub tags: Tags, - pub version: Option, } pub struct Relation { diff --git a/streets_reader/src/osm_reader/reader.rs b/streets_reader/src/osm_reader/reader.rs index 7574b6a1..52616021 100644 --- a/streets_reader/src/osm_reader/reader.rs +++ b/streets_reader/src/osm_reader/reader.rs @@ -1,14 +1,10 @@ use std::collections::{BTreeMap, HashMap}; -use std::io::Cursor; -use std::iter::Peekable; use abstutil::{Tags, Timer}; use anyhow::Result; use geom::{GPSBounds, LonLat}; -use osmpbf::{BlobDecode, BlobReader, Element as PbfElement, IndexedReader, RelMemberType}; -use xmlparser::Token; +use osm_reader::{Element, OsmID}; -use osm2streets::osm::{NodeID, OsmID, RelationID, WayID}; use osm2streets::utils::prettyprint_usize; use super::{Document, Node, Relation, Way}; @@ -23,154 +19,9 @@ use super::{Document, Node, Relation, Way}; // TODO Replicate IDs in each object, and change members to just hold a reference to the object // (which is guaranteed to exist). impl Document { - /// Parses raw pbf from buffer and extracts all objects - pub fn read_pbf( - input: &[u8], - gps_bounds: Option, - timer: &mut Timer, - ) -> Result { - let mut doc = Self { - gps_bounds, - nodes: BTreeMap::new(), - ways: BTreeMap::new(), - relations: BTreeMap::new(), - clipped_copied_ways: Vec::new(), - }; - - let mut blob_reader = BlobReader::new(Cursor::new(input)); - // par_map_reduce would be faster, but would not allow us to read the bbox + header - timer.start("scrape objects"); - while let Some(Ok(blob)) = blob_reader.next() { - match blob.decode().unwrap() { - BlobDecode::OsmHeader(head) => { - // if we find a bounding box - // in the blob header, use it. - if let Some(bbox) = head.bbox() { - doc.gps_bounds = Some(GPSBounds { - min_lon: bbox.left, - min_lat: bbox.top, - max_lon: bbox.right, - max_lat: bbox.bottom, - }); - } else if doc.gps_bounds.is_none() { - doc.gps_bounds = Option::from(scrape_bounds_pbf( - IndexedReader::new(Cursor::new(input)).unwrap(), - )) - } - } - BlobDecode::OsmData(block) => { - block.elements().for_each(|element| { - match element { - PbfElement::Node(node) => { - let pt = LonLat::new(node.lon(), node.lat()) - .to_pt(&doc.gps_bounds.clone().unwrap()); - let mut tags = Tags::new(BTreeMap::new()); - for (k, v) in node.tags() { - tags.insert(k, v); - } - doc.nodes.insert(NodeID(node.id()), Node { pt, tags }); - } - PbfElement::DenseNode(node) => { - let pt = LonLat::new(node.lon(), node.lat()) - .to_pt(&doc.gps_bounds.clone().unwrap()); - - let mut tags = Tags::new(BTreeMap::new()); - for (k, v) in node.tags() { - tags.insert(k, v); - } - - doc.nodes.insert(NodeID(node.id()), Node { pt, tags }); - } - PbfElement::Way(way) => { - let mut tags = Tags::new(BTreeMap::new()); - for (k, v) in way.tags() { - tags.insert(k, v); - } - - let mut nodes = Vec::new(); - let mut pts = Vec::new(); - for nd in way.refs() { - let n = NodeID(nd); - // Just skip missing nodes - if let Some(node) = doc.nodes.get(&n) { - nodes.push(n); - pts.push(node.pt); - } - } - let version = way.info().version().map(|x| x as usize); - - if !nodes.is_empty() { - doc.ways.insert( - WayID(way.id()), - Way { - nodes, - pts, - tags, - version, - }, - ); - } - } - PbfElement::Relation(relation) => { - let mut tags = Tags::new(BTreeMap::new()); - for (k, v) in relation.tags() { - tags.insert(k, v); - } - let id = RelationID(relation.id()); - if doc.relations.contains_key(&id) { - error!("Duplicate IDs detected. Your PBF is corrupt."); - return; - } - let mut members = Vec::new(); - for member in relation.members() { - let osm_id = match member.member_type { - RelMemberType::Node => { - let n = NodeID(member.member_id); - if !doc.nodes.contains_key(&n) { - continue; - } - OsmID::Node(n) - } - RelMemberType::Way => { - let w = WayID(member.member_id); - if !doc.ways.contains_key(&w) { - continue; - } - OsmID::Way(w) - } - RelMemberType::Relation => { - let r = RelationID(member.member_id); - if !doc.relations.contains_key(&r) { - continue; - } - OsmID::Relation(r) - } - }; - members.push((member.role().unwrap().to_string(), osm_id)); - } - - doc.relations.insert(id, Relation { tags, members }); - } - } - }); - } - // Just skip unrecognizable data. - BlobDecode::Unknown(_) => {} - } - } - timer.stop("scrape objects"); - info!( - "Found {} nodes, {} ways, {} relations", - prettyprint_usize(doc.nodes.len()), - prettyprint_usize(doc.ways.len()), - prettyprint_usize(doc.relations.len()) - ); - Ok(doc) - } - - /// Parses raw OSM XML and extracts all objects. + /// Parses xml or pbf bytes and extracts all objects pub fn read( - raw_string: &str, + input_bytes: &[u8], gps_bounds: Option, timer: &mut Timer, ) -> Result { @@ -182,132 +33,93 @@ impl Document { clipped_copied_ways: Vec::new(), }; - // We use the lower-level xmlparser instead of roxmltree to reduce peak memory usage in - // large files. - let mut reader = ElementReader { - tokenizer: xmlparser::Tokenizer::from(raw_string), - } - .peekable(); - timer.start("scrape objects"); - while let Some(obj) = reader.next() { - match obj.name { - "bounds" => { - // If we weren't provided with GPSBounds, use this. - if doc.gps_bounds.is_some() { - continue; - } + osm_reader::parse(input_bytes, |elem| match elem { + Element::Bounds { + min_lon, + min_lat, + max_lon, + max_lat, + } => { + // If we weren't provided with GPSBounds, use this. + if doc.gps_bounds.is_none() { doc.gps_bounds = Some(GPSBounds::from(vec![ - LonLat::new( - obj.attribute("minlon").parse::().unwrap(), - obj.attribute("minlat").parse::().unwrap(), - ), - LonLat::new( - obj.attribute("maxlon").parse::().unwrap(), - obj.attribute("maxlat").parse::().unwrap(), - ), + LonLat::new(min_lon, min_lat), + LonLat::new(max_lon, max_lat), ])); } - "node" => { - if doc.gps_bounds.is_none() { - warn!( - "No clipping polygon provided and the .osm is missing a element, \ - so figuring out the bounds manually." - ); - doc.gps_bounds = Some(scrape_bounds(raw_string)); - } - - let id = NodeID(obj.attribute("id").parse::().unwrap()); - if doc.nodes.contains_key(&id) { - bail!("Duplicate {}, your .osm is corrupt", id); - } - let pt = LonLat::new( - obj.attribute("lon").parse::().unwrap(), - obj.attribute("lat").parse::().unwrap(), - ) - .to_pt(doc.gps_bounds.as_ref().unwrap()); - let tags = read_tags(&mut reader); - doc.nodes.insert(id, Node { pt, tags }); + } + Element::Node { id, lon, lat, tags } => { + if doc.gps_bounds.is_none() { + warn!( + "No clipping polygon provided and the .osm is missing a element, \ + so figuring out the bounds manually." + ); + doc.gps_bounds = Some(scrape_bounds(input_bytes).unwrap()); } - "way" => { - let id = WayID(obj.attribute("id").parse::().unwrap()); - if doc.ways.contains_key(&id) { - bail!("Duplicate {}, your .osm is corrupt", id); - } - let version = obj - .attributes - .get("version") - .and_then(|x| x.parse::().ok()); - let mut nodes = Vec::new(); - let mut pts = Vec::new(); - while reader.peek().map(|x| x.name == "nd").unwrap_or(false) { - let node_ref = reader.next().unwrap(); - let n = NodeID(node_ref.attribute("ref").parse::().unwrap()); - // Just skip missing nodes - if let Some(node) = doc.nodes.get(&n) { - nodes.push(n); - pts.push(node.pt); - } - } - - // We assume 's come before 's - let tags = read_tags(&mut reader); - if !nodes.is_empty() { - doc.ways.insert( - id, - Way { - nodes, - pts, - tags, - version, - }, - ); - } + if doc.nodes.contains_key(&id) { + // TODO Make osm_reader API take fallible callbacks + panic!("Duplicate {id}, your .osm is corrupt"); } - "relation" => { - let id = RelationID(obj.attribute("id").parse::().unwrap()); - if doc.relations.contains_key(&id) { - bail!("Duplicate {}, your .osm is corrupt", id); - } - let mut members = Vec::new(); - while reader.peek().map(|x| x.name == "member").unwrap_or(false) { - let child = reader.next().unwrap(); - let member = match child.attribute("type") { - "node" => { - let n = NodeID(child.attribute("ref").parse::().unwrap()); - if !doc.nodes.contains_key(&n) { - continue; - } - OsmID::Node(n) - } - "way" => { - let w = WayID(child.attribute("ref").parse::().unwrap()); - if !doc.ways.contains_key(&w) { - continue; - } - OsmID::Way(w) - } - "relation" => { - let r = RelationID(child.attribute("ref").parse::().unwrap()); - if !doc.relations.contains_key(&r) { - continue; - } - OsmID::Relation(r) - } - _ => continue, - }; - members.push((child.attribute("role").to_string(), member)); + let pt = LonLat::new(lon, lat).to_pt(doc.gps_bounds.as_ref().unwrap()); + doc.nodes.insert( + id, + Node { + pt, + tags: make_tags(tags), + }, + ); + } + Element::Way { id, node_ids, tags } => { + if doc.ways.contains_key(&id) { + panic!("Duplicate {id}, your .osm is corrupt"); + } + let mut pts = Vec::new(); + let mut nodes = Vec::new(); + for n in node_ids { + // Just skip missing nodes + if let Some(node) = doc.nodes.get(&n) { + nodes.push(n); + pts.push(node.pt); } + } - // We assume 's come before 's - let tags = read_tags(&mut reader); - - doc.relations.insert(id, Relation { tags, members }); + if !nodes.is_empty() { + doc.ways.insert( + id, + Way { + nodes, + pts, + tags: make_tags(tags), + }, + ); } - _ => {} } - } + Element::Relation { + id, + tags, + mut members, + } => { + if doc.relations.contains_key(&id) { + panic!("Duplicate {id}, your .osm is corrupt"); + } + // Filter out missing members + members.retain(|(_, member)| match member { + OsmID::Node(n) => doc.nodes.contains_key(n), + OsmID::Way(w) => doc.ways.contains_key(w), + OsmID::Relation(r) => doc.relations.contains_key(r), + }); + + doc.relations.insert( + id, + Relation { + tags: make_tags(tags), + members, + }, + ); + } + })?; timer.stop("scrape objects"); info!( "Found {} nodes, {} ways, {} relations", @@ -320,140 +132,18 @@ impl Document { } } -fn read_tags(reader: &mut Peekable) -> Tags { - let mut tags = Tags::empty(); - - while reader.peek().map(|x| x.name == "tag").unwrap_or(false) { - let obj = reader.next().unwrap(); - let key = obj.attribute("k"); - let value = obj.attribute("v"); - tags.insert(key, unescape(value).unwrap()); - } - - tags -} - -fn scrape_bounds(raw_string: &str) -> GPSBounds { - let mut b = GPSBounds::new(); - for obj in (ElementReader { - tokenizer: xmlparser::Tokenizer::from(raw_string), - }) { - if obj.name == "node" { - b.update(LonLat::new( - obj.attribute("lon").parse::().unwrap(), - obj.attribute("lat").parse::().unwrap(), - )); - } - } - b -} - -fn scrape_bounds_pbf(mut reader: IndexedReader>) -> GPSBounds { +fn scrape_bounds(input_bytes: &[u8]) -> Result { let mut b = GPSBounds::new(); - reader - .for_each_node(|el| match el { - PbfElement::Node(node) => { - b.update(LonLat::new(node.lon(), node.lat())); - } - PbfElement::DenseNode(node) => { - b.update(LonLat::new(node.lon(), node.lat())); - } - _ => {} - }) - .expect("Failed to scrape bounds from nodes."); - b -} - -// Reads one element with attributes at a time. Ignores/flattens nested elements. -struct ElementReader<'a> { - tokenizer: xmlparser::Tokenizer<'a>, -} - -struct Element<'a> { - name: &'a str, - attributes: HashMap<&'a str, &'a str>, -} - -impl<'a> Element<'a> { - fn attribute(&self, key: &str) -> &str { - self.attributes.get(key).unwrap() - } -} - -impl<'a> Iterator for ElementReader<'a> { - type Item = Element<'a>; - - fn next(&mut self) -> Option { - let mut name: Option<&'a str> = None; - let mut attributes = HashMap::new(); - loop { - match self.tokenizer.next()?.unwrap() { - Token::ElementStart { local, .. } => { - assert!(name.is_none()); - assert!(attributes.is_empty()); - name = Some(local.as_str()); - } - Token::Attribute { local, value, .. } => { - assert!(name.is_some()); - attributes.insert(local.as_str(), value.as_str()); - } - Token::ElementEnd { .. } => { - if name.is_none() { - assert!(attributes.is_empty()); - continue; - } - - return Some(Element { - name: name.unwrap(), - attributes, - }); - } - _ => {} - } + osm_reader::parse(input_bytes, |elem| match elem { + Element::Node { lon, lat, .. } => { + b.update(LonLat::new(lon, lat)); } - } + _ => {} + })?; + Ok(b) } -// Copied from https://github.com/Florob/RustyXML, Apache licensed. Unescapes all valid XML -// entities in a string. -fn unescape(input: &str) -> Result { - let mut result = String::with_capacity(input.len()); - - let mut it = input.split('&'); - - // Push everything before the first '&' - if let Some(sub) = it.next() { - result.push_str(sub); - } - - for sub in it { - match sub.find(';') { - Some(idx) => { - let ent = &sub[..idx]; - match ent { - "quot" => result.push('"'), - "apos" => result.push('\''), - "gt" => result.push('>'), - "lt" => result.push('<'), - "amp" => result.push('&'), - ent => { - let val = if ent.starts_with("#x") { - u32::from_str_radix(&ent[2..], 16).ok() - } else if ent.starts_with('#') { - u32::from_str_radix(&ent[1..], 10).ok() - } else { - None - }; - match val.and_then(char::from_u32) { - Some(c) => result.push(c), - None => bail!("&{};", ent), - } - } - } - result.push_str(&sub[idx + 1..]); - } - None => bail!("&".to_owned() + sub), - } - } - Ok(result) +// Temporary shim to build from hashmap +fn make_tags(tags: HashMap) -> Tags { + Tags::new(tags.into_iter().collect()) } diff --git a/tests/src/lib.rs b/tests/src/lib.rs index 8d765286..7ca0a301 100644 --- a/tests/src/lib.rs +++ b/tests/src/lib.rs @@ -39,14 +39,14 @@ mod tests { let (mut street_network, _) = if Path::new(format!("{path}/input.osm").as_str()).exists() { streets_reader::osm_to_street_network( - &std::fs::read_to_string(format!("{path}/input.osm"))?, + &std::fs::read(format!("{path}/input.osm"))?, clip_pts, MapConfig::default(), &mut timer, )? } else { - streets_reader::pbf_to_street_network( - &std::fs::read(format!("{path}/input.osm.pbf").as_str())?, + streets_reader::osm_to_street_network( + &std::fs::read(format!("{path}/input.osm.pbf"))?, clip_pts, MapConfig::default(), &mut timer, diff --git a/web/src/osm2streets-svelte/Geocoder.svelte b/web/src/osm2streets-svelte/Geocoder.svelte index 410cc74c..b5589162 100644 --- a/web/src/osm2streets-svelte/Geocoder.svelte +++ b/web/src/osm2streets-svelte/Geocoder.svelte @@ -10,7 +10,9 @@ // TODO HMR is broken onMount(() => { - mapController = createMapLibreGlMapController($map, maplibregl); + if ($map) { + mapController = createMapLibreGlMapController($map, maplibregl); + } }); // TODO Show markers diff --git a/web/src/osm2streets-svelte/import/ImportControls.svelte b/web/src/osm2streets-svelte/import/ImportControls.svelte index 31212a98..74580897 100644 --- a/web/src/osm2streets-svelte/import/ImportControls.svelte +++ b/web/src/osm2streets-svelte/import/ImportControls.svelte @@ -65,7 +65,8 @@ try { imported = { kind: "loading", msg: "Running osm2streets" }; let network = new JsStreetNetwork( - osmXml, + // TODO Can we avoid this? + new Uint8Array(new TextEncoder().encode(osmXml)), JSON.stringify(boundaryGj), settings ); diff --git a/web/src/osm2streets-svelte/osm_input/BuiltInSelector.svelte b/web/src/osm2streets-svelte/osm_input/BuiltInSelector.svelte index e2997da5..86efda4b 100644 --- a/web/src/osm2streets-svelte/osm_input/BuiltInSelector.svelte +++ b/web/src/osm2streets-svelte/osm_input/BuiltInSelector.svelte @@ -47,7 +47,7 @@ boundaryGj, osmXml, }); - } catch (err) { + } catch (err: any) { dispatch("error", err.toString()); } }