From 21888334dd1e0fbbd25cc62c725eb33bb7968147 Mon Sep 17 00:00:00 2001 From: Siarhei Fedartsou Date: Fri, 30 Sep 2022 14:29:10 +0200 Subject: [PATCH] Do not generate intermediate .osrm file in osrm-extract. (#6354) --- .github/workflows/osrm-backend.yml | 12 +- CHANGELOG.md | 3 +- README.md | 2 +- docs/nodejs/api.md | 6 +- include/extractor/extraction_containers.hpp | 8 +- include/extractor/extractor.hpp | 21 ++- include/extractor/extractor_config.hpp | 3 +- include/extractor/files.hpp | 9 +- .../extractor/node_based_graph_factory.hpp | 17 +- include/partitioner/partitioner_config.hpp | 2 +- src/extractor/extraction_containers.cpp | 158 +++++++----------- src/extractor/extractor.cpp | 83 +++++---- src/extractor/node_based_graph_factory.cpp | 24 ++- src/nodejs/node_osrm.cpp | 6 +- src/tools/components.cpp | 6 +- src/tools/customize.cpp | 2 +- src/tools/extract.cpp | 7 +- src/tools/partition.cpp | 2 +- test/data/Makefile | 6 +- 19 files changed, 193 insertions(+), 184 deletions(-) diff --git a/.github/workflows/osrm-backend.yml b/.github/workflows/osrm-backend.yml index 04729ec1ba2..ed8e855c60e 100644 --- a/.github/workflows/osrm-backend.yml +++ b/.github/workflows/osrm-backend.yml @@ -124,7 +124,17 @@ jobs: # when `--memory-swap` value equals `--memory` it means container won't use swap # see https://docs.docker.com/config/containers/resource_constraints/#--memory-swap-details MEMORY_ARGS="--memory=1g --memory-swap=1g" - docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-extract -p /opt/car.lua /data/berlin-latest.osm.pbf + docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-extract --dump-nbg-graph -p /opt/car.lua /data/berlin-latest.osm.pbf + docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-components /data/berlin-latest.osrm.nbg /data/berlin-latest.geojson + if [ ! -s "${PWD}/berlin-latest.geojson" ] + then + >&2 echo "No berlin-latest.geojson found" + exit 1 + fi + + # removing `.osrm.nbg` to check that whole pipeline works without it + rm -rf "${PWD}/berlin-latest.osrm.nbg" + docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-partition /data/berlin-latest.osrm docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-customize /data/berlin-latest.osrm docker run $MEMORY_ARGS --name=osrm-container -t -p 5000:5000 -v "${PWD}:/data" "${TAG}" osrm-routed --algorithm mld /data/berlin-latest.osrm & diff --git a/CHANGELOG.md b/CHANGELOG.md index 526a273b928..2c4e128538d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,7 +64,8 @@ - FIXED: Ensure u-turn exists in intersection view. [#6376](https://github.com/Project-OSRM/osrm-backend/pull/6376) - Profile: - CHANGED: Bicycle surface speeds [#6212](https://github.com/Project-OSRM/osrm-backend/pull/6212) - + - Tools: + - CHANGED: Do not generate intermediate .osrm file in osrm-extract. [#6354](https://github.com/Project-OSRM/osrm-backend/pull/6354) # 5.26.0 - Changes from 5.25.0 diff --git a/README.md b/README.md index 1f6b096cb49..3f06d88525c 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ The flag `-v "${PWD}:/data"` creates the directory `/data` inside the docker con docker run -t -v "${PWD}:/data" osrm/osrm-backend osrm-partition /data/berlin-latest.osrm docker run -t -v "${PWD}:/data" osrm/osrm-backend osrm-customize /data/berlin-latest.osrm -Note that `berlin-latest.osrm` has a different file extension. +Note there is no `berlin-latest.osrm` file, but multiple `berlin-latest.osrm.*` files, i.e. `berlin-latest.osrm` is not file path, but "base" path referring to set of files and there is an option to omit this `.osrm` suffix completely(e.g. `osrm-partition /data/berlin-latest`). docker run -t -i -p 5000:5000 -v "${PWD}:/data" osrm/osrm-backend osrm-routed --algorithm mld /data/berlin-latest.osrm diff --git a/docs/nodejs/api.md b/docs/nodejs/api.md index 39d0a9786a1..ef52ed79020 100644 --- a/docs/nodejs/api.md +++ b/docs/nodejs/api.md @@ -3,8 +3,8 @@ ## OSRM The `OSRM` method is the main constructor for creating an OSRM instance. -An OSRM instance requires a `.osrm` dataset, which is prepared by the OSRM toolchain. -You can create such a `.osrm` file by running the OSRM binaries we ship in `node_modules/osrm/lib/binding/` and default +An OSRM instance requires a `.osrm.*` dataset(`.osrm.*` because it contains several files), which is prepared by the OSRM toolchain. +You can create such a `.osrm.*` dataset by running the OSRM binaries we ship in `node_modules/osrm/lib/binding/` and default profiles (e.g. for setting speeds and determining road types to route on) in `node_modules/osrm/profiles/`: node_modules/osrm/lib/binding/osrm-extract data.osm.pbf -p node_modules/osrm/profiles/car.lua @@ -12,7 +12,7 @@ profiles (e.g. for setting speeds and determining road types to route on) in `no Consult the [osrm-backend](https://github.com/Project-OSRM/osrm-backend) documentation for further details. -Once you have a complete `network.osrm` file, you can calculate routes in javascript with this object. +Once you have a complete `network.osrm.*` dataset, you can calculate routes in javascript with this object. ```javascript var osrm = new OSRM('network.osrm'); diff --git a/include/extractor/extraction_containers.hpp b/include/extractor/extraction_containers.hpp index b630763fbf9..6f643aed009 100644 --- a/include/extractor/extraction_containers.hpp +++ b/include/extractor/extraction_containers.hpp @@ -43,9 +43,6 @@ class ExtractionContainers void PrepareTrafficSignals(const ReferencedTrafficSignals &referenced_traffic_signals); void PrepareEdges(ScriptingEnvironment &scripting_environment); - void WriteNodes(storage::tar::FileWriter &file_out) const; - void WriteEdges(storage::tar::FileWriter &file_out) const; - void WriteMetadata(storage::tar::FileWriter &file_out) const; void WriteCharData(const std::string &file_name); public: @@ -75,6 +72,8 @@ class ExtractionContainers std::vector external_traffic_signals; TrafficSignals internal_traffic_signals; + std::vector used_edges; + // List of restrictions (conditional and unconditional) before we transform them into the // output types. Input containers reference OSMNodeIDs. We can only transform them to the // correct internal IDs after we've read everything. Without a multi-parse approach, @@ -84,11 +83,12 @@ class ExtractionContainers std::vector external_maneuver_overrides_list; std::vector internal_maneuver_overrides; + std::unordered_set used_barrier_nodes; + NodeVector used_nodes; ExtractionContainers(); void PrepareData(ScriptingEnvironment &scripting_environment, - const std::string &osrm_path, const std::string &names_data_path); }; } // namespace extractor diff --git a/include/extractor/extractor.hpp b/include/extractor/extractor.hpp index fc103fe6972..fb32af8a096 100644 --- a/include/extractor/extractor.hpp +++ b/include/extractor/extractor.hpp @@ -60,14 +60,25 @@ class Extractor Extractor(ExtractorConfig extractor_config) : config(std::move(extractor_config)) {} int run(ScriptingEnvironment &scripting_environment); + private: + struct ParsedOSMData + { + LaneDescriptionMap turn_lane_map; + std::vector turn_restrictions; + std::vector unresolved_maneuver_overrides; + TrafficSignals traffic_signals; + std::unordered_set barriers; + std::vector osm_coordinates; + extractor::PackedOSMIDs osm_node_ids; + std::vector edge_list; + std::vector annotation_data; + }; + private: ExtractorConfig config; - std::tuple, - std::vector, - TrafficSignals> - ParseOSMData(ScriptingEnvironment &scripting_environment, const unsigned number_of_threads); + ParsedOSMData ParseOSMData(ScriptingEnvironment &scripting_environment, + const unsigned number_of_threads); EdgeID BuildEdgeExpandedGraph( // input data diff --git a/include/extractor/extractor_config.hpp b/include/extractor/extractor_config.hpp index 848d97addeb..f25e3ebfc54 100644 --- a/include/extractor/extractor_config.hpp +++ b/include/extractor/extractor_config.hpp @@ -48,7 +48,7 @@ struct ExtractorConfig final : storage::IOConfig "", }, {}, - {".osrm", + {".osrm.nbg", ".osrm.restrictions", ".osrm.names", ".osrm.tls", @@ -89,6 +89,7 @@ struct ExtractorConfig final : storage::IOConfig bool use_metadata = false; bool parse_conditionals = false; bool use_locations_cache = true; + bool dump_nbg_graph = false; }; } // namespace extractor } // namespace osrm diff --git a/include/extractor/files.hpp b/include/extractor/files.hpp index 5c71e9de9f9..8cd4f1e9594 100644 --- a/include/extractor/files.hpp +++ b/include/extractor/files.hpp @@ -444,13 +444,11 @@ inline void readConditionalRestrictions(const boost::filesystem::path &path, } // reads .osrm file which is a temporary file of osrm-extract -template +template void readRawNBGraph(const boost::filesystem::path &path, - BarrierOutIter barriers, std::vector &coordinates, PackedOSMIDsT &osm_node_ids, - std::vector &edge_list, - std::vector &annotations) + std::vector &edge_list) { const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; storage::tar::FileReader reader{path, fingerprint}; @@ -468,10 +466,7 @@ void readRawNBGraph(const boost::filesystem::path &path, reader.ReadStreaming("/extractor/nodes", boost::make_function_output_iterator(decode)); - reader.ReadStreaming("/extractor/barriers", barriers); - storage::serialization::read(reader, "/extractor/edges", edge_list); - storage::serialization::read(reader, "/extractor/annotations", annotations); } template diff --git a/include/extractor/node_based_graph_factory.hpp b/include/extractor/node_based_graph_factory.hpp index 9e07513c2be..7c0a70f2e92 100644 --- a/include/extractor/node_based_graph_factory.hpp +++ b/include/extractor/node_based_graph_factory.hpp @@ -34,15 +34,19 @@ namespace extractor class NodeBasedGraphFactory { public: - // The node-based graph factory loads the *.osrm file and transforms the data within into the + // The node-based graph factory transforms the graph data into the // node-based graph to represent the OSM network. This includes geometry compression, annotation // data optimisation and many other aspects. After this step, the edge-based graph factory can // turn the graph into the routing graph to be used with the navigation algorithms. - NodeBasedGraphFactory(const boost::filesystem::path &input_file, - ScriptingEnvironment &scripting_environment, + NodeBasedGraphFactory(ScriptingEnvironment &scripting_environment, std::vector &turn_restrictions, std::vector &maneuver_overrides, - const TrafficSignals &traffic_signals); + const TrafficSignals &traffic_signals, + std::unordered_set &&barriers, + std::vector &&coordinates, + extractor::PackedOSMIDs &&osm_node_ids, + const std::vector &edge_list, + std::vector &&annotation_data); auto const &GetGraph() const { return compressed_output_graph; } auto const &GetBarriers() const { return barriers; } @@ -60,9 +64,8 @@ class NodeBasedGraphFactory void ReleaseOsmNodes(); private: - // Get the information from the *.osrm file (direct product of the extractor callback/extraction - // containers) and prepare the graph creation process - void LoadDataFromFile(const boost::filesystem::path &input_file); + // Build and validate compressed output graph + void BuildCompressedOutputGraph(const std::vector &edge_list); // Compress the node-based graph into a compact representation of itself. This removes storing a // single edge for every part of the geometry and might also combine meta-data for multiple diff --git a/include/partitioner/partitioner_config.hpp b/include/partitioner/partitioner_config.hpp index 01189b78062..6eb3581daf4 100644 --- a/include/partitioner/partitioner_config.hpp +++ b/include/partitioner/partitioner_config.hpp @@ -16,7 +16,7 @@ namespace partitioner struct PartitionerConfig final : storage::IOConfig { PartitionerConfig() - : IOConfig({".osrm", ".osrm.fileIndex", ".osrm.ebg_nodes", ".osrm.enw"}, + : IOConfig({".osrm.fileIndex", ".osrm.ebg_nodes", ".osrm.enw"}, {".osrm.hsgr", ".osrm.cnbg"}, {".osrm.ebg", ".osrm.cnbg", diff --git a/src/extractor/extraction_containers.cpp b/src/extractor/extraction_containers.cpp index 5c3a073093a..553968d6f18 100644 --- a/src/extractor/extraction_containers.cpp +++ b/src/extractor/extraction_containers.cpp @@ -5,8 +5,8 @@ #include "extractor/name_table.hpp" #include "extractor/restriction.hpp" #include "extractor/serialization.hpp" - #include "util/coordinate_calculation.hpp" +#include "util/integer_range.hpp" #include "util/exception.hpp" #include "util/exception_utils.hpp" @@ -16,6 +16,7 @@ #include "util/timing_util.hpp" #include +#include #include #include @@ -407,22 +408,14 @@ ExtractionContainers::ExtractionContainers() * */ void ExtractionContainers::PrepareData(ScriptingEnvironment &scripting_environment, - const std::string &osrm_path, const std::string &name_file_name) { - storage::tar::FileWriter writer(osrm_path, storage::tar::FileWriter::GenerateFingerprint); - const auto restriction_ways = IdentifyRestrictionWays(); const auto maneuver_override_ways = IdentifyManeuverOverrideWays(); const auto traffic_signals = IdentifyTrafficSignals(); PrepareNodes(); - WriteNodes(writer); PrepareEdges(scripting_environment); - all_nodes_list.clear(); // free all_nodes_list before allocation of normal_edges - all_nodes_list.shrink_to_fit(); - WriteEdges(writer); - WriteMetadata(writer); PrepareTrafficSignals(traffic_signals); PrepareManeuverOverrides(maneuver_override_ways); @@ -519,6 +512,60 @@ void ExtractionContainers::PrepareNodes() TIMER_STOP(id_map); log << "ok, after " << TIMER_SEC(id_map) << "s"; } + { + util::UnbufferedLog log; + log << "Confirming/Writing used nodes ... "; + TIMER_START(write_nodes); + // identify all used nodes by a merging step of two sorted lists + auto node_iterator = all_nodes_list.begin(); + auto node_id_iterator = used_node_id_list.begin(); + const auto all_nodes_list_end = all_nodes_list.end(); + + for (const auto index : util::irange(0, used_node_id_list.size())) + { + boost::ignore_unused(index); + BOOST_ASSERT(node_id_iterator != used_node_id_list.end()); + BOOST_ASSERT(node_iterator != all_nodes_list_end); + BOOST_ASSERT(*node_id_iterator >= node_iterator->node_id); + while (*node_id_iterator > node_iterator->node_id && + node_iterator != all_nodes_list_end) + { + ++node_iterator; + } + if (node_iterator == all_nodes_list_end || *node_id_iterator < node_iterator->node_id) + { + throw util::exception( + "Invalid OSM data: Referenced non-existing node with ID " + + std::to_string(static_cast(*node_id_iterator))); + } + BOOST_ASSERT(*node_id_iterator == node_iterator->node_id); + + ++node_id_iterator; + + used_nodes.emplace_back(*node_iterator++); + } + + TIMER_STOP(write_nodes); + log << "ok, after " << TIMER_SEC(write_nodes) << "s"; + } + + { + util::UnbufferedLog log; + log << "Writing barrier nodes ... "; + TIMER_START(write_nodes); + for (const auto osm_id : barrier_nodes) + { + const auto node_id = mapExternalToInternalNodeID( + used_node_id_list.begin(), used_node_id_list.end(), osm_id); + if (node_id != SPECIAL_NODEID) + { + used_barrier_nodes.emplace(node_id); + } + } + log << "ok, after " << TIMER_SEC(write_nodes) << "s"; + } + + util::Log() << "Processed " << max_internal_node_id << " nodes"; } void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environment) @@ -804,12 +851,11 @@ void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environm all_edges_list[j].result.target = SPECIAL_NODEID; } } -} -void ExtractionContainers::WriteEdges(storage::tar::FileWriter &writer) const -{ - std::vector normal_edges; - normal_edges.reserve(all_edges_list.size()); + all_nodes_list.clear(); // free all_nodes_list before allocation of used_edges + all_nodes_list.shrink_to_fit(); + + used_edges.reserve(all_edges_list.size()); { util::UnbufferedLog log; log << "Writing used edges ... " << std::flush; @@ -825,98 +871,20 @@ void ExtractionContainers::WriteEdges(storage::tar::FileWriter &writer) const // IMPORTANT: here, we're using slicing to only write the data from the base // class of NodeBasedEdgeWithOSM - normal_edges.push_back(edge.result); + used_edges.push_back(edge.result); } - if (normal_edges.size() > std::numeric_limits::max()) + if (used_edges.size() > std::numeric_limits::max()) { throw util::exception("There are too many edges, OSRM only supports 2^32" + SOURCE_REF); } - storage::serialization::write(writer, "/extractor/edges", normal_edges); - TIMER_STOP(write_edges); log << "ok, after " << TIMER_SEC(write_edges) << "s"; - log << " -- Processed " << normal_edges.size() << " edges"; + log << " -- Processed " << used_edges.size() << " edges"; } } -void ExtractionContainers::WriteMetadata(storage::tar::FileWriter &writer) const -{ - util::UnbufferedLog log; - log << "Writing way meta-data ... " << std::flush; - TIMER_START(write_meta_data); - - storage::serialization::write(writer, "/extractor/annotations", all_edges_annotation_data_list); - - TIMER_STOP(write_meta_data); - log << "ok, after " << TIMER_SEC(write_meta_data) << "s"; - log << " -- Metadata contains << " << all_edges_annotation_data_list.size() << " entries."; -} - -void ExtractionContainers::WriteNodes(storage::tar::FileWriter &writer) const -{ - { - util::UnbufferedLog log; - log << "Confirming/Writing used nodes ... "; - TIMER_START(write_nodes); - // identify all used nodes by a merging step of two sorted lists - auto node_iterator = all_nodes_list.begin(); - auto node_id_iterator = used_node_id_list.begin(); - const auto all_nodes_list_end = all_nodes_list.end(); - - const std::function encode_function = [&]() -> QueryNode { - BOOST_ASSERT(node_id_iterator != used_node_id_list.end()); - BOOST_ASSERT(node_iterator != all_nodes_list_end); - BOOST_ASSERT(*node_id_iterator >= node_iterator->node_id); - while (*node_id_iterator > node_iterator->node_id && - node_iterator != all_nodes_list_end) - { - ++node_iterator; - } - if (node_iterator == all_nodes_list_end || *node_id_iterator < node_iterator->node_id) - { - throw util::exception( - "Invalid OSM data: Referenced non-existing node with ID " + - std::to_string(static_cast(*node_id_iterator))); - } - BOOST_ASSERT(*node_id_iterator == node_iterator->node_id); - - ++node_id_iterator; - return *node_iterator++; - }; - - writer.WriteElementCount64("/extractor/nodes", used_node_id_list.size()); - writer.WriteStreaming( - "/extractor/nodes", - boost::make_function_input_iterator(encode_function, boost::infinite()), - used_node_id_list.size()); - - TIMER_STOP(write_nodes); - log << "ok, after " << TIMER_SEC(write_nodes) << "s"; - } - - { - util::UnbufferedLog log; - log << "Writing barrier nodes ... "; - TIMER_START(write_nodes); - std::vector internal_barrier_nodes; - for (const auto osm_id : barrier_nodes) - { - const auto node_id = mapExternalToInternalNodeID( - used_node_id_list.begin(), used_node_id_list.end(), osm_id); - if (node_id != SPECIAL_NODEID) - { - internal_barrier_nodes.push_back(node_id); - } - } - storage::serialization::write(writer, "/extractor/barriers", internal_barrier_nodes); - log << "ok, after " << TIMER_SEC(write_nodes) << "s"; - } - - util::Log() << "Processed " << max_internal_node_id << " nodes"; -} - ExtractionContainers::ReferencedWays ExtractionContainers::IdentifyManeuverOverrideWays() { ReferencedWays maneuver_override_ways; diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index 388ee866dc5..687b35b5352 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -204,12 +204,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) tbb::global_control gc(tbb::global_control::max_allowed_parallelism, config.requested_num_threads); - LaneDescriptionMap turn_lane_map; - std::vector turn_restrictions; - std::vector unresolved_maneuver_overrides; - TrafficSignals traffic_signals; - std::tie(turn_lane_map, turn_restrictions, unresolved_maneuver_overrides, traffic_signals) = - ParseOSMData(scripting_environment, number_of_threads); + auto parsed_osm_data = ParseOSMData(scripting_environment, number_of_threads); // Transform the node-based graph that OSM is based on into an edge-based graph // that is better for routing. Every edge becomes a node, and every valid @@ -227,11 +222,15 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) std::uint32_t ebg_connectivity_checksum = 0; // Create a node-based graph from the OSRM file - NodeBasedGraphFactory node_based_graph_factory(config.GetPath(".osrm"), - scripting_environment, - turn_restrictions, - unresolved_maneuver_overrides, - traffic_signals); + NodeBasedGraphFactory node_based_graph_factory(scripting_environment, + parsed_osm_data.turn_restrictions, + parsed_osm_data.unresolved_maneuver_overrides, + parsed_osm_data.traffic_signals, + std::move(parsed_osm_data.barriers), + std::move(parsed_osm_data.osm_coordinates), + std::move(parsed_osm_data.osm_node_ids), + parsed_osm_data.edge_list, + std::move(parsed_osm_data.annotation_data)); NameTable name_table; files::readNames(config.GetPath(".osrm.names"), name_table); @@ -270,10 +269,11 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) edge_based_nodes_container = EdgeBasedNodeDataContainer({}, std::move(node_based_graph_factory.GetAnnotationData())); - turn_restrictions = removeInvalidTurnPaths(std::move(turn_restrictions), node_based_graph); - unresolved_maneuver_overrides = - removeInvalidTurnPaths(std::move(unresolved_maneuver_overrides), node_based_graph); - auto restriction_graph = constructRestrictionGraph(turn_restrictions); + parsed_osm_data.turn_restrictions = + removeInvalidTurnPaths(std::move(parsed_osm_data.turn_restrictions), node_based_graph); + parsed_osm_data.unresolved_maneuver_overrides = removeInvalidTurnPaths( + std::move(parsed_osm_data.unresolved_maneuver_overrides), node_based_graph); + auto restriction_graph = constructRestrictionGraph(parsed_osm_data.turn_restrictions); const auto number_of_node_based_nodes = node_based_graph.GetNumberOfNodes(); @@ -282,12 +282,12 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) coordinates, node_based_graph_factory.GetCompressedEdges(), barrier_nodes, - traffic_signals, + parsed_osm_data.traffic_signals, restriction_graph, segregated_edges, name_table, - unresolved_maneuver_overrides, - turn_lane_map, + parsed_osm_data.unresolved_maneuver_overrides, + parsed_osm_data.turn_lane_map, scripting_environment, edge_based_nodes_container, edge_based_node_segments, @@ -304,7 +304,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) barrier_nodes, restriction_graph, name_table, - std::move(turn_lane_map), + std::move(parsed_osm_data.turn_lane_map), scripting_environment); TIMER_STOP(expansion); @@ -356,17 +356,13 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) util::Log() << "Expansion: " << nodes_per_second << " nodes/sec and " << edges_per_second << " edges/sec"; util::Log() << "To prepare the data for routing, run: " - << "./osrm-contract " << config.GetPath(".osrm"); + << "./osrm-contract " << config.base_path; return 0; } -std::tuple, - std::vector, - TrafficSignals> -Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, - const unsigned number_of_threads) +Extractor::ParsedOSMData Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, + const unsigned number_of_threads) { TIMER_START(extracting); @@ -617,7 +613,6 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, } extraction_containers.PrepareData(scripting_environment, - config.GetPath(".osrm").string(), config.GetPath(".osrm.names").string()); auto profile_properties = scripting_environment.GetProfileProperties(); @@ -629,10 +624,36 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, TIMER_STOP(extracting); util::Log() << "extraction finished after " << TIMER_SEC(extracting) << "s"; - return std::make_tuple(std::move(turn_lane_map), - std::move(extraction_containers.turn_restrictions), - std::move(extraction_containers.internal_maneuver_overrides), - std::move(extraction_containers.internal_traffic_signals)); + std::vector osm_coordinates; + extractor::PackedOSMIDs osm_node_ids; + + osm_coordinates.resize(extraction_containers.used_nodes.size()); + osm_node_ids.reserve(extraction_containers.used_nodes.size()); + for (size_t index = 0; index < extraction_containers.used_nodes.size(); ++index) + { + const auto ¤t_node = extraction_containers.used_nodes[index]; + osm_coordinates[index].lon = current_node.lon; + osm_coordinates[index].lat = current_node.lat; + osm_node_ids.push_back(current_node.node_id); + } + + if (config.dump_nbg_graph) + { + storage::tar::FileWriter writer(config.GetPath(".osrm.nbg").string(), + storage::tar::FileWriter::GenerateFingerprint); + storage::serialization::write(writer, "/extractor/nodes", extraction_containers.used_nodes); + storage::serialization::write(writer, "/extractor/edges", extraction_containers.used_edges); + } + + return ParsedOSMData{std::move(turn_lane_map), + std::move(extraction_containers.turn_restrictions), + std::move(extraction_containers.internal_maneuver_overrides), + std::move(extraction_containers.internal_traffic_signals), + std::move(extraction_containers.used_barrier_nodes), + std::move(osm_coordinates), + std::move(osm_node_ids), + std::move(extraction_containers.used_edges), + std::move(extraction_containers.all_edges_annotation_data_list)}; } void Extractor::FindComponents(unsigned number_of_edge_based_nodes, diff --git a/src/extractor/node_based_graph_factory.cpp b/src/extractor/node_based_graph_factory.cpp index 887523de98d..5edc447154f 100644 --- a/src/extractor/node_based_graph_factory.cpp +++ b/src/extractor/node_based_graph_factory.cpp @@ -16,32 +16,30 @@ namespace extractor { NodeBasedGraphFactory::NodeBasedGraphFactory( - const boost::filesystem::path &input_file, ScriptingEnvironment &scripting_environment, std::vector &turn_restrictions, std::vector &maneuver_overrides, - const TrafficSignals &traffic_signals) + const TrafficSignals &traffic_signals, + std::unordered_set &&barriers, + std::vector &&coordinates, + extractor::PackedOSMIDs &&osm_node_ids, + const std::vector &edge_list, + std::vector &&annotation_data) + : annotation_data(std::move(annotation_data)), barriers(std::move(barriers)), + coordinates(std::move(coordinates)), osm_node_ids(std::move(osm_node_ids)) { - LoadDataFromFile(input_file); + BuildCompressedOutputGraph(edge_list); Compress(scripting_environment, turn_restrictions, maneuver_overrides, traffic_signals); CompressGeometry(); CompressAnnotationData(); } -// load the data serialised during the extraction run -void NodeBasedGraphFactory::LoadDataFromFile(const boost::filesystem::path &input_file) +void NodeBasedGraphFactory::BuildCompressedOutputGraph(const std::vector &edge_list) { - auto barriers_iter = inserter(barriers, end(barriers)); - std::vector edge_list; - - files::readRawNBGraph( - input_file, barriers_iter, coordinates, osm_node_ids, edge_list, annotation_data); - const auto number_of_node_based_nodes = coordinates.size(); if (edge_list.empty()) { - throw util::exception("Node-based-graph (" + input_file.string() + ") contains no edges." + - SOURCE_REF); + throw util::exception("Node-based-graph contains no edges." + SOURCE_REF); } // at this point, the data isn't compressed, but since we update the graph in-place, we assign diff --git a/src/nodejs/node_osrm.cpp b/src/nodejs/node_osrm.cpp index be475787991..91fd5f4f566 100644 --- a/src/nodejs/node_osrm.cpp +++ b/src/nodejs/node_osrm.cpp @@ -55,8 +55,8 @@ NAN_MODULE_INIT(Engine::Init) // clang-format off /** * The `OSRM` method is the main constructor for creating an OSRM instance. - * An OSRM instance requires a `.osrm` dataset, which is prepared by the OSRM toolchain. - * You can create such a `.osrm` file by running the OSRM binaries we ship in `node_modules/osrm/lib/binding/` and default + * An OSRM instance requires a `.osrm.*` dataset(`.osrm.*` because it contains several files), which is prepared by the OSRM toolchain. + * You can create such a `.osrm.*` dataset by running the OSRM binaries we ship in `node_modules/osrm/lib/binding/` and default * profiles (e.g. for setting speeds and determining road types to route on) in `node_modules/osrm/profiles/`: * * node_modules/osrm/lib/binding/osrm-extract data.osm.pbf -p node_modules/osrm/profiles/car.lua @@ -64,7 +64,7 @@ NAN_MODULE_INIT(Engine::Init) * * Consult the [osrm-backend](https://github.com/Project-OSRM/osrm-backend) documentation for further details. * - * Once you have a complete `network.osrm` file, you can calculate routes in javascript with this object. + * Once you have a complete `network.osrm.*` dataset, you can calculate routes in javascript with this object. * * ```javascript * var osrm = new OSRM('network.osrm'); diff --git a/src/tools/components.cpp b/src/tools/components.cpp index 1b43cc215b6..ea4f21dbff7 100644 --- a/src/tools/components.cpp +++ b/src/tools/components.cpp @@ -39,12 +39,8 @@ std::size_t loadGraph(const std::string &path, std::vector &graph_edge_list) { std::vector edge_list; - std::vector annotation_data; - auto nop = boost::make_function_output_iterator([](auto) {}); - - extractor::files::readRawNBGraph( - path, nop, coordinate_list, osm_node_ids, edge_list, annotation_data); + extractor::files::readRawNBGraph(path, coordinate_list, osm_node_ids, edge_list); // Building a node-based graph for (const auto &input_edge : edge_list) diff --git a/src/tools/customize.cpp b/src/tools/customize.cpp index 3a63eb3b365..f77b88795c6 100644 --- a/src/tools/customize.cpp +++ b/src/tools/customize.cpp @@ -77,7 +77,7 @@ return_code parseArguments(int argc, hidden_options.add_options()( "input,i", boost::program_options::value(&customization_config.base_path), - "Input file in .osrm format"); + "Input base file path"); // positional option boost::program_options::positional_options_description positional_options; diff --git a/src/tools/extract.cpp b/src/tools/extract.cpp index 0cd1c7b4f75..0fc615c492b 100644 --- a/src/tools/extract.cpp +++ b/src/tools/extract.cpp @@ -74,7 +74,12 @@ return_code parseArguments(int argc, boost::program_options::bool_switch(&extractor_config.use_locations_cache) ->implicit_value(false) ->default_value(true), - "Use internal nodes locations cache for location-dependent data lookups"); + "Use internal nodes locations cache for location-dependent data lookups")( + "dump-nbg-graph", + boost::program_options::bool_switch(&extractor_config.dump_nbg_graph) + ->implicit_value(true) + ->default_value(false), + "Dump raw node-based graph to *.osrm file for debug purposes."); bool dummy; // hidden options, will be allowed on command line, but will not be diff --git a/src/tools/partition.cpp b/src/tools/partition.cpp index 6d1e82007c8..44f06fad73b 100644 --- a/src/tools/partition.cpp +++ b/src/tools/partition.cpp @@ -119,7 +119,7 @@ return_code parseArguments(int argc, hidden_options.add_options()( "input,i", boost::program_options::value(&config.base_path), - "Input file in .osrm format"); + "Input base file path"); // positional option boost::program_options::positional_options_description positional_options; diff --git a/test/data/Makefile b/test/data/Makefile index 89b241c129f..87cd75b1d31 100755 --- a/test/data/Makefile +++ b/test/data/Makefile @@ -22,15 +22,15 @@ clean: ch/$(DATA_NAME).osrm: $(DATA_NAME).osrm mkdir -p ch - cp $(DATA_NAME).osrm $(DATA_NAME).osrm.* ch/ + cp $(DATA_NAME).osrm.* ch/ corech/$(DATA_NAME).osrm: $(DATA_NAME).osrm mkdir -p corech - cp $(DATA_NAME).osrm $(DATA_NAME).osrm.* corech/ + cp $(DATA_NAME).osrm.* corech/ mld/$(DATA_NAME).osrm: $(DATA_NAME).osrm mkdir -p mld - cp $(DATA_NAME).osrm $(DATA_NAME).osrm.* mld/ + cp $(DATA_NAME).osrm.* mld/ $(DATA_NAME).osrm: $(DATA_NAME).osm.pbf $(DATA_NAME).poly $(PROFILE) $(OSRM_EXTRACT) @echo "Verifiyng data file integrity..."