diff --git a/include/extractor/edge_based_graph_factory.hpp b/include/extractor/edge_based_graph_factory.hpp index e2a0a2cb1c1..d1fd1634dd6 100644 --- a/include/extractor/edge_based_graph_factory.hpp +++ b/include/extractor/edge_based_graph_factory.hpp @@ -7,18 +7,16 @@ #include "extractor/edge_based_edge.hpp" #include "extractor/edge_based_node.hpp" #include "extractor/extraction_turn.hpp" +#include "extractor/guidance/turn_analysis.hpp" +#include "extractor/guidance/turn_instruction.hpp" +#include "extractor/guidance/turn_lane_types.hpp" #include "extractor/original_edge_data.hpp" #include "extractor/profile_properties.hpp" #include "extractor/query_node.hpp" #include "extractor/restriction_map.hpp" - -#include "extractor/guidance/turn_analysis.hpp" -#include "extractor/guidance/turn_instruction.hpp" -#include "extractor/guidance/turn_lane_types.hpp" +#include "util/deallocating_vector.hpp" #include "util/guidance/bearing_class.hpp" #include "util/guidance/entry_class.hpp" - -#include "util/deallocating_vector.hpp" #include "util/name_table.hpp" #include "util/node_based_graph.hpp" #include "util/typedefs.hpp" @@ -35,6 +33,7 @@ #include #include +#include namespace osrm { @@ -86,7 +85,9 @@ struct TurnIndexBlock #pragma pack(pop) static_assert(std::is_trivial::value, "TurnIndexBlock is not trivial"); static_assert(sizeof(TurnIndexBlock) == 24, "TurnIndexBlock is not packed correctly"); -} +} // ns lookup + +struct NodeBasedGraphToEdgeBasedGraphMappingWriter; // fwd. decl class EdgeBasedGraphFactory { @@ -113,7 +114,9 @@ class EdgeBasedGraphFactory const std::string &turn_weight_penalties_filename, const std::string &turn_duration_penalties_filename, const std::string &turn_penalties_index_filename, - const bool generate_edge_lookup); + const bool generate_edge_lookup, + const bool generate_nbg_ebg_mapping, + const std::string &nbg_ebg_mapping_path); // The following get access functions destroy the content in the factory void GetEdgeBasedEdges(util::DeallocatingVector &edges); @@ -172,7 +175,10 @@ class EdgeBasedGraphFactory guidance::LaneDescriptionMap &lane_description_map; unsigned RenumberEdges(); - void GenerateEdgeExpandedNodes(); + + void GenerateEdgeExpandedNodes(const bool generate_nbg_ebg_mapping, + const std::string &nbg_ebg_mapping_path); + void GenerateEdgeExpandedEdges(ScriptingEnvironment &scripting_environment, const std::string &original_edge_data_filename, const std::string &turn_lane_data_filename, @@ -182,7 +188,15 @@ class EdgeBasedGraphFactory const std::string &turn_penalties_index_filename, const bool generate_edge_lookup); - void InsertEdgeBasedNode(const NodeID u, const NodeID v); + // Mapping betweenn the node based graph u,v nodes and the edge based graph head,tail edge ids. + // Required in the osrm-partition tool to translate from a nbg partition to a ebg partition. + struct Mapping + { + NodeID u, v; + EdgeID head, tail; + }; + + boost::optional InsertEdgeBasedNode(const NodeID u, const NodeID v); void FlushVectorToStream(std::ofstream &edge_data_file, std::vector &original_edge_data_vector) const; diff --git a/include/extractor/extractor_config.hpp b/include/extractor/extractor_config.hpp index 24c761bec82..04fb11fcadb 100644 --- a/include/extractor/extractor_config.hpp +++ b/include/extractor/extractor_config.hpp @@ -78,6 +78,7 @@ struct ExtractorConfig profile_properties_output_path = basepath + ".osrm.properties"; intersection_class_data_output_path = basepath + ".osrm.icd"; compressed_node_based_graph_output_path = basepath + ".osrm.cnbg"; + nbg_ebg_graph_mapping_output_path = basepath + ".osrm.nbg_to_ebg"; } boost::filesystem::path input_path; @@ -110,8 +111,10 @@ struct ExtractorConfig bool use_metadata; + // Auxiliary data for osrm-partition bool dump_compressed_node_based_graph; std::string compressed_node_based_graph_output_path; + std::string nbg_ebg_graph_mapping_output_path; }; } } diff --git a/include/extractor/node_based_graph_to_edge_based_graph_mapping_writer.hpp b/include/extractor/node_based_graph_to_edge_based_graph_mapping_writer.hpp new file mode 100644 index 00000000000..df6a91c49b7 --- /dev/null +++ b/include/extractor/node_based_graph_to_edge_based_graph_mapping_writer.hpp @@ -0,0 +1,63 @@ +#ifndef NODE_BASED_GRAPH_TO_EDGE_BASED_GRAPH_MAPPING_WRITER_HPP +#define NODE_BASED_GRAPH_TO_EDGE_BASED_GRAPH_MAPPING_WRITER_HPP + +#include "storage/io.hpp" +#include "util/log.hpp" +#include "util/typedefs.hpp" + +#include + +#include + +#include + +namespace osrm +{ +namespace extractor +{ + +// Writes: | Fingerprint | #mappings | u v fwd_node bkw_node | u v fwd_node bkw_node | .. +// - uint64: number of mappings (u, v, fwd_node bkw_node) chunks +// - NodeID u, NodeID v, EdgeID fwd_node, EdgeID bkw_node + +struct NodeBasedGraphToEdgeBasedGraphMappingWriter +{ + NodeBasedGraphToEdgeBasedGraphMappingWriter(const std::string &path) + : writer{path, storage::io::FileWriter::GenerateFingerprint}, num_written{0} + { + const std::uint64_t dummy{0}; // filled in later + writer.WriteElementCount64(dummy); + } + + void WriteMapping(NodeID u, NodeID v, EdgeID fwd_ebg_node, EdgeID bkw_ebg_node) + { + BOOST_ASSERT(u != SPECIAL_NODEID); + BOOST_ASSERT(v != SPECIAL_NODEID); + BOOST_ASSERT(fwd_ebg_node != SPECIAL_EDGEID || bkw_ebg_node != SPECIAL_EDGEID); + + writer.WriteOne(u); + writer.WriteOne(v); + writer.WriteOne(fwd_ebg_node); + writer.WriteOne(bkw_ebg_node); + + num_written += 1; + } + + ~NodeBasedGraphToEdgeBasedGraphMappingWriter() + { + if (num_written != 0) + { + writer.SkipToBeginning(); + writer.WriteOne(num_written); + } + } + + private: + storage::io::FileWriter writer; + std::uint64_t num_written; +}; + +} // ns extractor +} // ns osrm + +#endif // NODE_BASED_GRAPH_TO_EDGE_BASED_GRAPH_MAPPING_WRITER_HPP diff --git a/include/partition/partition_config.hpp b/include/partition/partition_config.hpp index 155d0c111e6..a6cfa65ebda 100644 --- a/include/partition/partition_config.hpp +++ b/include/partition/partition_config.hpp @@ -32,6 +32,7 @@ struct PartitionConfig edge_based_graph_path = basepath + ".osrm.ebg"; compressed_node_based_graph_path = basepath + ".osrm.cnbg"; + nbg_ebg_mapping_path = basepath + ".osrm.nbg_to_ebg"; partition_path = basepath + ".osrm.partition"; } @@ -39,6 +40,7 @@ struct PartitionConfig boost::filesystem::path base_path; boost::filesystem::path edge_based_graph_path; boost::filesystem::path compressed_node_based_graph_path; + boost::filesystem::path nbg_ebg_mapping_path; boost::filesystem::path partition_path; unsigned requested_num_threads; diff --git a/include/storage/io.hpp b/include/storage/io.hpp index 07a2a024411..34bc83fc474 100644 --- a/include/storage/io.hpp +++ b/include/storage/io.hpp @@ -43,6 +43,7 @@ class FileReader VerifyFingerprint, HasNoFingerprint }; + FileReader(const std::string &filename, const FingerprintFlag flag) : FileReader(boost::filesystem::path(filename), flag) { @@ -194,10 +195,6 @@ class FileReader class FileWriter { - private: - const boost::filesystem::path filepath; - boost::filesystem::ofstream output_stream; - public: enum FingerprintFlag { @@ -211,7 +208,7 @@ class FileWriter } FileWriter(const boost::filesystem::path &filepath_, const FingerprintFlag flag) - : filepath(filepath_) + : filepath(filepath_), fingerprint(flag) { output_stream.open(filepath, std::ios::binary); if (!output_stream) @@ -262,9 +259,30 @@ class FileWriter const auto fingerprint = util::FingerPrint::GetValid(); return WriteOne(fingerprint); } + + template void Skip(const std::size_t element_count) + { + boost::iostreams::seek(output_stream, element_count * sizeof(T), BOOST_IOS::cur); + } + + void SkipToBeginning() + { + boost::iostreams::seek(output_stream, 0, std::ios::beg); + + // If we wrote a Fingerprint, skip over it + if (fingerprint == FingerprintFlag::GenerateFingerprint) + Skip(1); + + // Should probably return a functor for jumping back to the current pos. + } + + private: + const boost::filesystem::path filepath; + boost::filesystem::ofstream output_stream; + FingerprintFlag fingerprint; }; -} -} -} +} // ns io +} // ns storage +} // ns osrm #endif diff --git a/src/extractor/edge_based_graph_factory.cpp b/src/extractor/edge_based_graph_factory.cpp index d4ce0329cb3..3c60201f422 100644 --- a/src/extractor/edge_based_graph_factory.cpp +++ b/src/extractor/edge_based_graph_factory.cpp @@ -1,5 +1,10 @@ #include "extractor/edge_based_graph_factory.hpp" #include "extractor/edge_based_edge.hpp" +#include "extractor/guidance/turn_analysis.hpp" +#include "extractor/guidance/turn_lane_handler.hpp" +#include "extractor/node_based_graph_to_edge_based_graph_mapping_writer.hpp" +#include "extractor/scripting_environment.hpp" +#include "extractor/suffix_table.hpp" #include "util/bearing.hpp" #include "util/coordinate.hpp" #include "util/coordinate_calculation.hpp" @@ -10,11 +15,6 @@ #include "util/percent.hpp" #include "util/timing_util.hpp" -#include "extractor/guidance/turn_analysis.hpp" -#include "extractor/guidance/turn_lane_handler.hpp" -#include "extractor/scripting_environment.hpp" -#include "extractor/suffix_table.hpp" - #include #include @@ -92,7 +92,8 @@ void EdgeBasedGraphFactory::GetEdgeBasedNodeWeights(std::vector &out EdgeID EdgeBasedGraphFactory::GetHighestEdgeID() { return m_max_edge_id; } -void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeID node_v) +boost::optional +EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeID node_v) { // merge edges together into one EdgeBasedNode BOOST_ASSERT(node_u != SPECIAL_NODEID); @@ -112,7 +113,7 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeI if (forward_data.edge_id == SPECIAL_NODEID && reverse_data.edge_id == SPECIAL_NODEID) { - return; + return boost::none; } if (forward_data.edge_id != SPECIAL_NODEID && reverse_data.edge_id == SPECIAL_NODEID) @@ -172,6 +173,8 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeI } BOOST_ASSERT(current_edge_source_coordinate_id == node_v); + + return Mapping{node_u, node_v, forward_data.edge_id, reverse_data.edge_id}; } void EdgeBasedGraphFactory::FlushVectorToStream( @@ -193,7 +196,9 @@ void EdgeBasedGraphFactory::Run(ScriptingEnvironment &scripting_environment, const std::string &turn_weight_penalties_filename, const std::string &turn_duration_penalties_filename, const std::string &turn_penalties_index_filename, - const bool generate_edge_lookup) + const bool generate_edge_lookup, + const bool generate_nbg_ebg_mapping, + const std::string &nbg_ebg_mapping_path) { TIMER_START(renumber); m_max_edge_id = RenumberEdges() - 1; @@ -201,7 +206,7 @@ void EdgeBasedGraphFactory::Run(ScriptingEnvironment &scripting_environment, TIMER_START(generate_nodes); m_edge_based_node_weights.reserve(m_max_edge_id + 1); - GenerateEdgeExpandedNodes(); + GenerateEdgeExpandedNodes(generate_nbg_ebg_mapping, nbg_ebg_mapping_path); TIMER_STOP(generate_nodes); TIMER_START(generate_edges); @@ -255,8 +260,16 @@ unsigned EdgeBasedGraphFactory::RenumberEdges() } /// Creates the nodes in the edge expanded graph from edges in the node-based graph. -void EdgeBasedGraphFactory::GenerateEdgeExpandedNodes() +void EdgeBasedGraphFactory::GenerateEdgeExpandedNodes(const bool generate_nbg_ebg_mapping, + const std::string &nbg_ebg_mapping_path) { + // Optional writer, for writing out a mapping. Neither default ctor not boost::optional work + // with the underlying FileWriter, so hack around that limitation with a unique_ptr. + std::unique_ptr writer; + if (generate_nbg_ebg_mapping) + writer = + std::make_unique(nbg_ebg_mapping_path); + util::Log() << "Generating edge expanded nodes ... "; { util::UnbufferedLog log; @@ -286,15 +299,20 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedNodes() BOOST_ASSERT(node_u < node_v); + boost::optional mapping; + // if we found a non-forward edge reverse and try again if (edge_data.edge_id == SPECIAL_NODEID) { - InsertEdgeBasedNode(node_v, node_u); + mapping = InsertEdgeBasedNode(node_v, node_u); } else { - InsertEdgeBasedNode(node_u, node_v); + mapping = InsertEdgeBasedNode(node_u, node_v); } + + if (generate_nbg_ebg_mapping) + writer->WriteMapping(mapping->u, mapping->v, mapping->head, mapping->tail); } } } diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index cb8d79f2c50..f5f04fc2491 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -503,7 +503,9 @@ Extractor::BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment, config.turn_weight_penalties_path, config.turn_duration_penalties_path, config.turn_penalties_index_path, - config.generate_edge_lookup); + config.generate_edge_lookup, + config.dump_compressed_node_based_graph, + config.nbg_ebg_graph_mapping_output_path); // The osrm-partition tool requires the compressed node based graph with an embedding. // diff --git a/src/partition/partitioner.cpp b/src/partition/partitioner.cpp index 58382aea743..1c9ec9efda7 100644 --- a/src/partition/partitioner.cpp +++ b/src/partition/partitioner.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -39,6 +40,8 @@ struct CompressedNodeBasedGraph // - uint64: number of nodes and therefore also coordinates // - (uint32_t, uint32_t): num_edges * edges // - (int32_t, int32_t: num_nodes * coordinates (lon, lat) + // + // Gets written in Extractor::WriteCompressedNodeBasedGraph const auto num_edges = reader.ReadElementCount64(); const auto num_nodes = reader.ReadElementCount64(); @@ -57,13 +60,69 @@ struct CompressedNodeBasedGraph CompressedNodeBasedGraph LoadCompressedNodeBasedGraph(const std::string &path) { const auto fingerprint = storage::io::FileReader::VerifyFingerprint; - storage::io::FileReader reader(path, fingerprint); CompressedNodeBasedGraph graph{reader}; return graph; } +struct NodeBasedGraphToEdgeBasedGraphMapping +{ + NodeBasedGraphToEdgeBasedGraphMapping(storage::io::FileReader &reader) + { + // Reads: | Fingerprint | #mappings | u v fwd_node bkw_node | u v fwd_node bkw_node | .. + // - uint64: number of mappings (u, v, fwd_node, bkw_node) chunks + // - NodeID u, NodeID v, EdgeID fwd_node, EdgeID bkw_node + // + // Gets written in NodeBasedGraphToEdgeBasedGraphMappingWriter + + const auto num_mappings = reader.ReadElementCount64(); + + edge_based_node_to_node_based_nodes.reserve(num_mappings * 2); + + for (std::uint64_t i{0}; i < num_mappings; ++i) + { + + const auto u = reader.ReadOne(); // node based graph `from` node + const auto v = reader.ReadOne(); // node based graph `to` node + const auto fwd_ebg_node = reader.ReadOne(); // edge based graph forward node + const auto bkw_ebg_node = reader.ReadOne(); // edge based graph backward node + + edge_based_node_to_node_based_nodes.insert({fwd_ebg_node, {u, v}}); + edge_based_node_to_node_based_nodes.insert({bkw_ebg_node, {v, u}}); + } + } + + struct NodeBasedNodes + { + NodeID u, v; + }; + + NodeBasedNodes Lookup(EdgeID edge_based_node) const + { + auto it = edge_based_node_to_node_based_nodes.find(edge_based_node); + + if (it != end(edge_based_node_to_node_based_nodes)) + return it->second; + + BOOST_ASSERT_MSG(false, "unable to fine edge based node, graph <-> mapping out of sync"); + return NodeBasedNodes{SPECIAL_NODEID, SPECIAL_NODEID}; + } + + private: + std::unordered_map edge_based_node_to_node_based_nodes; +}; + +NodeBasedGraphToEdgeBasedGraphMapping +LoadNodeBasedGraphToEdgeBasedGraphMapping(const std::string &path) +{ + const auto fingerprint = storage::io::FileReader::VerifyFingerprint; + storage::io::FileReader reader(path, fingerprint); + + NodeBasedGraphToEdgeBasedGraphMapping mapping{reader}; + return mapping; +} + void LogGeojson(const std::string &filename, const std::vector &bisection_ids) { // reload graph, since we destroyed the old one @@ -156,6 +215,10 @@ int Partitioner::Run(const PartitionConfig &config) LogGeojson(config.compressed_node_based_graph_path.string(), recursive_bisection.BisectionIDs()); + auto mapping = LoadNodeBasedGraphToEdgeBasedGraphMapping(config.nbg_ebg_mapping_path.string()); + + util::Log() << "Loaded node based graph to edge based graph mapping"; + return 0; }