From c96dfefd197a4f9cc48097e58fc4ee45232cf3e0 Mon Sep 17 00:00:00 2001 From: Daniel Patterson Date: Mon, 14 Mar 2016 23:03:19 -0700 Subject: [PATCH] Enables the use of multiple segment-speed-files on the osrm-contract command line, and exposes the file name used for each edge in the debug tiles. --- include/contractor/contractor.hpp | 4 +- include/contractor/contractor_config.hpp | 10 +- include/engine/datafacade/datafacade_base.hpp | 8 ++ .../engine/datafacade/internal_datafacade.hpp | 83 +++++++++++++ .../engine/datafacade/shared_datafacade.hpp | 83 +++++++++++++ include/engine/engine_config.hpp | 2 + .../extractor/edge_based_graph_factory.hpp | 9 -- include/storage/shared_datatype.hpp | 4 + include/util/routed_options.hpp | 4 + src/contractor/contractor.cpp | 114 +++++++++++++----- src/engine/plugins/tile.cpp | 42 ++++++- src/extractor/extractor.cpp | 7 +- src/storage/storage.cpp | 99 +++++++++++++++ src/tools/contract.cpp | 7 +- src/tools/store.cpp | 20 ++- 15 files changed, 442 insertions(+), 54 deletions(-) diff --git a/include/contractor/contractor.hpp b/include/contractor/contractor.hpp index adf05011add..7cfc3dc664d 100644 --- a/include/contractor/contractor.hpp +++ b/include/contractor/contractor.hpp @@ -83,9 +83,11 @@ class Contractor util::DeallocatingVector &edge_based_edge_list, const std::string &edge_segment_lookup_path, const std::string &edge_penalty_path, - const std::string &segment_speed_path, + const std::vector &segment_speed_path, const std::string &nodes_filename, const std::string &geometry_filename, + const std::string &datasource_names_filename, + const std::string &datasource_indexes_filename, const std::string &rtree_leaf_filename); }; } diff --git a/include/contractor/contractor_config.hpp b/include/contractor/contractor_config.hpp index 48393ab2c9e..f557279caed 100644 --- a/include/contractor/contractor_config.hpp +++ b/include/contractor/contractor_config.hpp @@ -53,6 +53,8 @@ struct ContractorConfig node_based_graph_path = osrm_input_path.string() + ".nodes"; geometry_path = osrm_input_path.string() + ".geometry"; rtree_leaf_path = osrm_input_path.string() + ".fileIndex"; + datasource_names_path = osrm_input_path.string() + ".datasource_names"; + datasource_indexes_path = osrm_input_path.string() + ".datasource_indexes"; } boost::filesystem::path config_file_path; @@ -78,11 +80,9 @@ struct ContractorConfig //(e.g. 0.8 contracts 80 percent of the hierarchy, leaving a core of 20%) double core_factor; - std::string segment_speed_lookup_path; - -#ifdef DEBUG_GEOMETRY - std::string debug_geometry_path; -#endif + std::vector segment_speed_lookup_paths; + std::string datasource_indexes_path; + std::string datasource_names_path; }; } } diff --git a/include/engine/datafacade/datafacade_base.hpp b/include/engine/datafacade/datafacade_base.hpp index 690c53146c5..f261d902fab 100644 --- a/include/engine/datafacade/datafacade_base.hpp +++ b/include/engine/datafacade/datafacade_base.hpp @@ -78,6 +78,14 @@ class BaseDataFacade virtual void GetUncompressedWeights(const EdgeID id, std::vector &result_weights) const = 0; + // Returns the data source ids that were used to supply the edge + // weights. Will return an empty array when only the base profile is used. + virtual void GetUncompressedDatasources(const EdgeID id, + std::vector &data_sources) const = 0; + + // Gets the name of a datasource + virtual std::string GetDatasourceName(const uint8_t datasource_name_id) const = 0; + virtual extractor::guidance::TurnInstruction GetTurnInstructionForEdgeID(const unsigned id) const = 0; diff --git a/include/engine/datafacade/internal_datafacade.hpp b/include/engine/datafacade/internal_datafacade.hpp index a0fbfa88848..0fc623c6435 100644 --- a/include/engine/datafacade/internal_datafacade.hpp +++ b/include/engine/datafacade/internal_datafacade.hpp @@ -78,6 +78,8 @@ class InternalDataFacade final : public BaseDataFacade util::ShM::vector m_geometry_list; util::ShM::vector m_is_core_node; util::ShM::vector m_segment_weights; + util::ShM::vector m_datasource_list; + util::ShM::vector m_datasource_names; boost::thread_specific_ptr m_static_rtree; boost::thread_specific_ptr m_geospatial_query; @@ -222,6 +224,34 @@ class InternalDataFacade final : public BaseDataFacade } } + void LoadDatasourceInfo(const boost::filesystem::path &datasource_names_file, + const boost::filesystem::path &datasource_indexes_file) + { + std::ifstream datasources_stream(datasource_indexes_file.c_str(), std::ios::binary); + if (datasources_stream) + { + std::size_t number_of_datasources = 0; + datasources_stream.read(reinterpret_cast(&number_of_datasources), + sizeof(std::size_t)); + if (number_of_datasources > 0) + { + m_datasource_list.resize(number_of_datasources); + datasources_stream.read(reinterpret_cast(&(m_datasource_list[0])), + number_of_datasources * sizeof(uint8_t)); + } + } + + std::ifstream datasourcenames_stream(datasource_names_file.c_str(), std::ios::binary); + if (datasourcenames_stream) + { + std::string name; + while (std::getline(datasourcenames_stream, name)) + { + m_datasource_names.push_back(name); + } + } + } + void LoadRTree() { BOOST_ASSERT_MSG(!m_coordinate_list->empty(), "coordinates must be loaded before r-tree"); @@ -269,6 +299,14 @@ class InternalDataFacade final : public BaseDataFacade return it->second; }; + const auto optional_file_for = [&server_paths, &end_it](const std::string &path) + { + const auto it = server_paths.find(path); + if (it == end_it) + throw util::exception("no valid " + path + " file given in ini file"); + return it->second; + }; + ram_index_path = file_for("ramindex"); file_index_path = file_for("fileindex"); @@ -284,6 +322,10 @@ class InternalDataFacade final : public BaseDataFacade util::SimpleLogger().Write() << "loading geometries"; LoadGeometries(file_for("geometries")); + util::SimpleLogger().Write() << "loading datasource info"; + LoadDatasourceInfo(optional_file_for("datasource_names"), + optional_file_for("datasource_indexes")); + util::SimpleLogger().Write() << "loading timestamp"; LoadTimestamp(file_for("timestamp")); @@ -592,6 +634,47 @@ class InternalDataFacade final : public BaseDataFacade }); } + // Returns the data source ids that were used to supply the edge + // weights. + virtual void + GetUncompressedDatasources(const EdgeID id, + std::vector &result_datasources) const override final + { + const unsigned begin = m_geometry_indices.at(id); + const unsigned end = m_geometry_indices.at(id + 1); + + result_datasources.clear(); + result_datasources.reserve(end - begin); + + // If there was no datasource info, return an array of 0's. + if (m_datasource_list.empty()) + { + for (unsigned i = 0; i < end - begin; ++i) + { + result_datasources.push_back(0); + } + } + else + { + std::for_each(m_datasource_list.begin() + begin, m_datasource_list.begin() + end, + [&](const uint8_t &datasource_id) + { + result_datasources.push_back(datasource_id); + }); + } + } + + virtual std::string GetDatasourceName(const uint8_t datasource_name_id) const override final + { + if (m_datasource_names.empty() || datasource_name_id > m_datasource_names.size()) + { + if (datasource_name_id == 0) + return "lua profile"; + return "UNKNOWN"; + } + return m_datasource_names[datasource_name_id]; + } + std::string GetTimestamp() const override final { return m_timestamp; } }; } diff --git a/include/engine/datafacade/shared_datafacade.hpp b/include/engine/datafacade/shared_datafacade.hpp index 1601de8edf8..0722e61cb0e 100644 --- a/include/engine/datafacade/shared_datafacade.hpp +++ b/include/engine/datafacade/shared_datafacade.hpp @@ -80,6 +80,11 @@ class SharedDataFacade final : public BaseDataFacade util::ShM::vector m_geometry_indices; util::ShM::vector m_geometry_list; util::ShM::vector m_is_core_node; + util::ShM::vector m_datasource_list; + + util::ShM::vector m_datasource_name_data; + util::ShM::vector m_datasource_name_offsets; + util::ShM::vector m_datasource_name_lengths; boost::thread_specific_ptr>> m_static_rtree; boost::thread_specific_ptr m_geospatial_query; @@ -231,6 +236,34 @@ class SharedDataFacade final : public BaseDataFacade geometry_list(geometries_list_ptr, data_layout->num_entries[storage::SharedDataLayout::GEOMETRIES_LIST]); m_geometry_list = std::move(geometry_list); + + auto datasources_list_ptr = data_layout->GetBlockPtr( + shared_memory, storage::SharedDataLayout::DATASOURCES_LIST); + typename util::ShM::vector datasources_list( + datasources_list_ptr, + data_layout->num_entries[storage::SharedDataLayout::DATASOURCES_LIST]); + m_datasource_list = std::move(datasources_list); + + auto datasource_name_data_ptr = data_layout->GetBlockPtr( + shared_memory, storage::SharedDataLayout::DATASOURCE_NAME_DATA); + typename util::ShM::vector datasource_name_data( + datasource_name_data_ptr, + data_layout->num_entries[storage::SharedDataLayout::DATASOURCE_NAME_DATA]); + m_datasource_name_data = std::move(datasource_name_data); + + auto datasource_name_offsets_ptr = data_layout->GetBlockPtr( + shared_memory, storage::SharedDataLayout::DATASOURCE_NAME_OFFSETS); + typename util::ShM::vector datasource_name_offsets( + datasource_name_offsets_ptr, + data_layout->num_entries[storage::SharedDataLayout::DATASOURCE_NAME_OFFSETS]); + m_datasource_name_offsets = std::move(datasource_name_offsets); + + auto datasource_name_lengths_ptr = data_layout->GetBlockPtr( + shared_memory, storage::SharedDataLayout::DATASOURCE_NAME_LENGTHS); + typename util::ShM::vector datasource_name_lengths( + datasource_name_lengths_ptr, + data_layout->num_entries[storage::SharedDataLayout::DATASOURCE_NAME_LENGTHS]); + m_datasource_name_lengths = std::move(datasource_name_lengths); } public: @@ -632,6 +665,56 @@ class SharedDataFacade final : public BaseDataFacade virtual std::size_t GetCoreSize() const override final { return m_is_core_node.size(); } + // Returns the data source ids that were used to supply the edge + // weights. + virtual void + GetUncompressedDatasources(const EdgeID id, + std::vector &result_datasources) const override final + { + const unsigned begin = m_geometry_indices.at(id); + const unsigned end = m_geometry_indices.at(id + 1); + + result_datasources.clear(); + result_datasources.reserve(end - begin); + + // If there was no datasource info, return an array of 0's. + if (m_datasource_list.empty()) + { + for (unsigned i = 0; i < end - begin; ++i) + { + result_datasources.push_back(0); + } + } + else + { + std::for_each(m_datasource_list.begin() + begin, m_datasource_list.begin() + end, + [&](const uint8_t &datasource_id) + { + result_datasources.push_back(datasource_id); + }); + } + } + + virtual std::string GetDatasourceName(const uint8_t datasource_name_id) const override final + { + std::string result; + + if (m_datasource_name_offsets.empty() || + datasource_name_id > m_datasource_name_offsets.size()) + { + if (datasource_name_id == 0) + return "lua profile"; + return "UNKNOWN"; + } + + std::copy(m_datasource_name_data.begin() + m_datasource_name_offsets[datasource_name_id], + m_datasource_name_data.begin() + m_datasource_name_offsets[datasource_name_id] + + m_datasource_name_lengths[datasource_name_id], + std::back_inserter(result)); + + return result; + } + std::string GetTimestamp() const override final { return m_timestamp; } }; } diff --git a/include/engine/engine_config.hpp b/include/engine/engine_config.hpp index c54cd52cb90..182ef805ad7 100644 --- a/include/engine/engine_config.hpp +++ b/include/engine/engine_config.hpp @@ -52,6 +52,8 @@ struct EngineConfig {"coredata", base.string() + ".core"}, {"geometries", base.string() + ".geometry"}, {"timestamp", base.string() + ".timestamp"}, + {"datasource_names", base.string() + ".datasource_names"}, + {"datasource_indexes", base.string() + ".datasource_indexes"}, {"namesdata", base.string() + ".names"}} { } diff --git a/include/extractor/edge_based_graph_factory.hpp b/include/extractor/edge_based_graph_factory.hpp index 4e448ebf381..ede4deb024d 100644 --- a/include/extractor/edge_based_graph_factory.hpp +++ b/include/extractor/edge_based_graph_factory.hpp @@ -109,20 +109,11 @@ class EdgeBasedGraphFactory void CompressGeometry(); unsigned RenumberEdges(); void GenerateEdgeExpandedNodes(); -#ifdef DEBUG_GEOMETRY - void GenerateEdgeExpandedEdges(const std::string &original_edge_data_filename, - lua_State *lua_state, - const std::string &edge_segment_lookup_filename, - const std::string &edge_fixed_penalties_filename, - const bool generate_edge_lookup, - const std::string &debug_turns_path); -#else void GenerateEdgeExpandedEdges(const std::string &original_edge_data_filename, lua_State *lua_state, const std::string &edge_segment_lookup_filename, const std::string &edge_fixed_penalties_filename, const bool generate_edge_lookup); -#endif void InsertEdgeBasedNode(const NodeID u, const NodeID v); diff --git a/include/storage/shared_datatype.hpp b/include/storage/shared_datatype.hpp index 22f7bdec2c0..20c25d699fa 100644 --- a/include/storage/shared_datatype.hpp +++ b/include/storage/shared_datatype.hpp @@ -38,6 +38,10 @@ struct SharedDataLayout TIMESTAMP, FILE_INDEX_PATH, CORE_MARKER, + DATASOURCES_LIST, + DATASOURCE_NAME_DATA, + DATASOURCE_NAME_OFFSETS, + DATASOURCE_NAME_LENGTHS, NUM_BLOCKS }; diff --git a/include/util/routed_options.hpp b/include/util/routed_options.hpp index 0a23bdd2f5d..2ea65f82e00 100644 --- a/include/util/routed_options.hpp +++ b/include/util/routed_options.hpp @@ -53,6 +53,10 @@ populate_base_path(std::unordered_map &ser BOOST_ASSERT(server_paths.find("namesdata") != server_paths.end()); server_paths["timestamp"] = base_string + ".timestamp"; BOOST_ASSERT(server_paths.find("timestamp") != server_paths.end()); + server_paths["datasource_indexes"] = base_string + ".datasource_indexes"; + BOOST_ASSERT(server_paths.find("timestamp") != server_paths.end()); + server_paths["datasource_names"] = base_string + ".datasource_names"; + BOOST_ASSERT(server_paths.find("timestamp") != server_paths.end()); } // check if files are give and whether they exist at all diff --git a/src/contractor/contractor.cpp b/src/contractor/contractor.cpp index 8df0f923729..824adb99e32 100644 --- a/src/contractor/contractor.cpp +++ b/src/contractor/contractor.cpp @@ -71,8 +71,9 @@ int Contractor::Run() std::size_t max_edge_id = LoadEdgeExpandedGraph( config.edge_based_graph_path, edge_based_edge_list, config.edge_segment_lookup_path, - config.edge_penalty_path, config.segment_speed_lookup_path, config.node_based_graph_path, - config.geometry_path, config.rtree_leaf_path); + config.edge_penalty_path, config.segment_speed_lookup_paths, config.node_based_graph_path, + config.geometry_path, config.datasource_names_path, config.datasource_indexes_path, + config.rtree_leaf_path); // Contracting the edge-expanded graph @@ -127,15 +128,17 @@ std::size_t Contractor::LoadEdgeExpandedGraph( util::DeallocatingVector &edge_based_edge_list, const std::string &edge_segment_lookup_filename, const std::string &edge_penalty_filename, - const std::string &segment_speed_filename, + const std::vector &segment_speed_filenames, const std::string &nodes_filename, const std::string &geometry_filename, + const std::string &datasource_names_filename, + const std::string &datasource_indexes_filename, const std::string &rtree_leaf_filename) { util::SimpleLogger().Write() << "Opening " << edge_based_graph_filename; boost::filesystem::ifstream input_stream(edge_based_graph_filename, std::ios::binary); - const bool update_edge_weights = segment_speed_filename != ""; + const bool update_edge_weights = !segment_speed_filenames.empty(); boost::filesystem::ifstream edge_segment_input_stream; boost::filesystem::ifstream edge_fixed_penalties_input_stream; @@ -167,22 +170,36 @@ std::size_t Contractor::LoadEdgeExpandedGraph( util::SimpleLogger().Write() << "Reading " << number_of_edges << " edges from the edge based graph"; - std::unordered_map, unsigned> segment_speed_lookup; + std::unordered_map, std::pair> + segment_speed_lookup; if (update_edge_weights) { - util::SimpleLogger().Write() - << "Segment speed data supplied, will update edge weights from " - << segment_speed_filename; - io::CSVReader<3> csv_in(segment_speed_filename); - csv_in.set_header("from_node", "to_node", "speed"); - uint64_t from_node_id{}; - uint64_t to_node_id{}; - unsigned speed{}; - while (csv_in.read_row(from_node_id, to_node_id, speed)) + uint8_t file_id = 1; + for (auto segment_speed_filename : segment_speed_filenames) { - segment_speed_lookup[std::make_pair(OSMNodeID(from_node_id), OSMNodeID(to_node_id))] = - speed; + util::SimpleLogger().Write() + << "Segment speed data supplied, will update edge weights from " + << segment_speed_filename; + io::CSVReader<3> csv_in(segment_speed_filename); + csv_in.set_header("from_node", "to_node", "speed"); + uint64_t from_node_id{}; + uint64_t to_node_id{}; + unsigned speed{}; + while (csv_in.read_row(from_node_id, to_node_id, speed)) + { + segment_speed_lookup[std::make_pair(OSMNodeID(from_node_id), + OSMNodeID(to_node_id))] = + std::make_pair(speed, file_id); + } + ++file_id; + + // Check for overflow + if (file_id == 0) + { + throw util::exception( + "Sorry, there's a limit of 254 segment speed files, you supplied too many"); + } } std::vector internal_to_external_node_map; @@ -241,6 +258,14 @@ std::size_t Contractor::LoadEdgeExpandedGraph( } } + // This is a list of the "data source id" for every segment in the compressed + // geometry container. We assume that everything so far has come from the + // profile (data source 0). Here, we replace the 0's with the index of the + // CSV file that supplied the value that gets used for that segment, then + // we write out this list so that it can be returned by the debugging + // vector tiles later on. + std::vector m_geometry_datasource(m_geometry_list.size(), 0); + // Now, we iterate over all the segments stored in the StaticRTree, updating // the packed geometry weights in the `.geometries` file (note: we do not // update the RTree itself, we just use the leaf nodes to iterate over all segments) @@ -297,12 +322,16 @@ std::size_t Contractor::LoadEdgeExpandedGraph( segment_speed_lookup.find(std::make_pair(u->node_id, v->node_id)); if (forward_speed_iter != segment_speed_lookup.end()) { - int new_segment_weight = std::max( - 1, static_cast(std::floor( - (segment_length * 10.) / (forward_speed_iter->second / 3.6) + - .5))); + int new_segment_weight = + std::max(1, static_cast(std::floor( + (segment_length * 10.) / + (forward_speed_iter->second.first / 3.6) + + .5))); m_geometry_list[forward_begin + leaf_object.fwd_segment_position] .weight = new_segment_weight; + m_geometry_datasource[forward_begin + + leaf_object.fwd_segment_position] = + forward_speed_iter->second.second; } } if (leaf_object.reverse_packed_geometry_id != SPECIAL_EDGEID) @@ -338,12 +367,15 @@ std::size_t Contractor::LoadEdgeExpandedGraph( segment_speed_lookup.find(std::make_pair(u->node_id, v->node_id)); if (reverse_speed_iter != segment_speed_lookup.end()) { - int new_segment_weight = std::max( - 1, static_cast(std::floor( - (segment_length * 10.) / (reverse_speed_iter->second / 3.6) + - .5))); + int new_segment_weight = + std::max(1, static_cast(std::floor( + (segment_length * 10.) / + (reverse_speed_iter->second.first / 3.6) + + .5))); m_geometry_list[reverse_begin + rev_segment_position].weight = new_segment_weight; + m_geometry_datasource[reverse_begin + rev_segment_position] = + reverse_speed_iter->second.second; } } } @@ -370,6 +402,34 @@ std::size_t Contractor::LoadEdgeExpandedGraph( number_of_compressed_geometries * sizeof(extractor::CompressedEdgeContainer::CompressedEdge)); } + + { + std::ofstream datasource_stream(datasource_indexes_filename, std::ios::binary); + if (!datasource_stream) + { + throw util::exception("Failed to open " + datasource_indexes_filename + + " for writing"); + } + auto number_of_datasource_entries = m_geometry_datasource.size(); + datasource_stream.write(reinterpret_cast(&number_of_datasource_entries), + sizeof(number_of_datasource_entries)); + datasource_stream.write(reinterpret_cast(&(m_geometry_datasource[0])), + number_of_datasource_entries * sizeof(uint8_t)); + } + + { + std::ofstream datasource_stream(datasource_names_filename, std::ios::binary); + if (!datasource_stream) + { + throw util::exception("Failed to open " + datasource_names_filename + + " for writing"); + } + datasource_stream << "lua profile" << std::endl; + for (auto const &name : segment_speed_filenames) + { + datasource_stream << name << std::endl; + } + } } // TODO: can we read this in bulk? util::DeallocatingVector isn't necessarily @@ -413,9 +473,9 @@ std::size_t Contractor::LoadEdgeExpandedGraph( // This sets the segment weight using the same formula as the // EdgeBasedGraphFactory for consistency. The *why* of this formula // is lost in the annals of time. - int new_segment_weight = - std::max(1, static_cast(std::floor( - (segment_length * 10.) / (speed_iter->second / 3.6) + .5))); + int new_segment_weight = std::max( + 1, static_cast(std::floor( + (segment_length * 10.) / (speed_iter->second.first / 3.6) + .5))); new_weight += new_segment_weight; } else diff --git a/src/engine/plugins/tile.cpp b/src/engine/plugins/tile.cpp index 4db4008f728..026bc00571b 100644 --- a/src/engine/plugins/tile.cpp +++ b/src/engine/plugins/tile.cpp @@ -191,6 +191,8 @@ Status TilePlugin::HandleRequest(const api::TileParameters ¶meters, std::str xyz2mercator(parameters.x, parameters.y, parameters.z, min_lon, min_lat, max_lon, max_lat); const bbox tile_bbox{min_lon, min_lat, max_lon, max_lat}; + uint8_t max_datasource_id = 0; + // Protobuf serialized blocks when objects go out of scope, hence // the extra scoping below. protozero::pbf_writer tile_writer{pbf_buffer}; @@ -221,12 +223,20 @@ Status TilePlugin::HandleRequest(const api::TileParameters ¶meters, std::str int forward_weight = 0; int reverse_weight = 0; + uint8_t forward_datasource = 0; + uint8_t reverse_datasource = 0; + if (edge.forward_packed_geometry_id != SPECIAL_EDGEID) { std::vector forward_weight_vector; facade.GetUncompressedWeights(edge.forward_packed_geometry_id, forward_weight_vector); forward_weight = forward_weight_vector[edge.fwd_segment_position]; + + std::vector forward_datasource_vector; + facade.GetUncompressedDatasources(edge.forward_packed_geometry_id, + forward_datasource_vector); + forward_datasource = forward_datasource_vector[edge.fwd_segment_position]; } if (edge.reverse_packed_geometry_id != SPECIAL_EDGEID) @@ -239,8 +249,20 @@ Status TilePlugin::HandleRequest(const api::TileParameters ¶meters, std::str reverse_weight = reverse_weight_vector[reverse_weight_vector.size() - edge.fwd_segment_position - 1]; + + std::vector reverse_datasource_vector; + facade.GetUncompressedDatasources(edge.reverse_packed_geometry_id, + reverse_datasource_vector); + reverse_datasource = + reverse_datasource_vector[reverse_datasource_vector.size() - + edge.fwd_segment_position - 1]; } + // Keep track of the highest datasource seen so that we don't write unnecessary + // data to the layer attribute values + max_datasource_id = std::max(max_datasource_id, forward_datasource); + max_datasource_id = std::max(max_datasource_id, reverse_datasource); + // If this is a valid forward edge, go ahead and add it to the tile if (forward_weight != 0 && edge.forward_edge_based_node_id != SPECIAL_NODEID) { @@ -296,7 +318,10 @@ Status TilePlugin::HandleRequest(const api::TileParameters ¶meters, std::str field.add_element( std::min(speed, 127u)); // save the speed value, capped at 127 field.add_element(1); // "is_small" tag key offset - field.add_element(edge.component.is_tiny ? 0 : 1); // is_small feature + field.add_element(128 + + (edge.component.is_tiny ? 0 : 1)); // is_small feature + field.add_element(2); // "datasource" tag key offset + field.add_element(130 + forward_datasource); // datasource value offset } { // Encode the geometry for the feature @@ -346,7 +371,10 @@ Status TilePlugin::HandleRequest(const api::TileParameters ¶meters, std::str field.add_element( std::min(speed, 127u)); // save the speed value, capped at 127 field.add_element(1); // "is_small" tag key offset - field.add_element(edge.component.is_tiny ? 0 : 1); // is_small feature + field.add_element(128 + + (edge.component.is_tiny ? 0 : 1)); // is_small feature + field.add_element(2); // "datasource" tag key offset + field.add_element(130 + reverse_datasource); // datasource value offset } { protozero::packed_field_uint32 geometry(feature_writer, 4); @@ -361,6 +389,7 @@ Status TilePlugin::HandleRequest(const api::TileParameters ¶meters, std::str // earlier layer_writer.add_string(3, "speed"); layer_writer.add_string(3, "is_small"); + layer_writer.add_string(3, "datasource"); // Now, we write out the possible speed value arrays and possible is_tiny // values. Field type 4 is the "values" field. It's a variable type field, @@ -384,6 +413,15 @@ Status TilePlugin::HandleRequest(const api::TileParameters ¶meters, std::str // Attribute value 7 == bool type values_writer.add_bool(7, false); } + for (std::size_t i = 0; i <= max_datasource_id; i++) + { + { + // Writing field type 4 == variant type + protozero::pbf_writer values_writer(layer_writer, 4); + // Attribute value 1 == string type + values_writer.add_string(1, facade.GetDatasourceName(i)); + } + } } return Status::Ok; diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index 7ac29534062..7123bfce731 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -528,12 +528,7 @@ Extractor::BuildEdgeExpandedGraph(std::vector &internal_to_external_n edge_based_graph_factory.Run(config.edge_output_path, lua_state, config.edge_segment_lookup_path, config.edge_penalty_path, - config.generate_edge_lookup -#ifdef DEBUG_GEOMETRY - , - config.debug_turns_path -#endif - ); + config.generate_edge_lookup); lua_close(lua_state); diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index ccf9f174547..c40bcbacfe5 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -131,6 +131,14 @@ int Storage::Run() { throw util::exception("no core file found"); } + if (paths.find("datasource_indexes") == paths.end()) + { + throw util::exception("no datasource_indexes file found"); + } + if (paths.find("datasource_names") == paths.end()) + { + throw util::exception("no datasource_names file found"); + } auto paths_iterator = paths.find("hsgrdata"); BOOST_ASSERT(paths.end() != paths_iterator); @@ -170,6 +178,14 @@ int Storage::Run() BOOST_ASSERT(paths.end() != paths_iterator); BOOST_ASSERT(!paths_iterator->second.empty()); const boost::filesystem::path &core_marker_path = paths_iterator->second; + paths_iterator = paths.find("datasource_indexes"); + BOOST_ASSERT(paths.end() != paths_iterator); + BOOST_ASSERT(!paths_iterator->second.empty()); + const boost::filesystem::path &datasource_indexes_path = paths_iterator->second; + paths_iterator = paths.find("datasource_names"); + BOOST_ASSERT(paths.end() != paths_iterator); + BOOST_ASSERT(!paths_iterator->second.empty()); + const boost::filesystem::path &datasource_names_path = paths_iterator->second; // determine segment to use bool segment2_in_use = SharedMemory::RegionExists(LAYOUT_2); @@ -325,6 +341,45 @@ int Storage::Run() geometry_input_stream.read((char *)&number_of_compressed_geometries, sizeof(unsigned)); shared_layout_ptr->SetBlockSize( SharedDataLayout::GEOMETRIES_LIST, number_of_compressed_geometries); + + // load datasource sizes. This file is optional, and it's non-fatal if it doesn't + // exist. + std::ifstream geometry_datasource_input_stream(datasource_indexes_path.c_str(), + std::ios::binary); + if (geometry_datasource_input_stream) + { + std::size_t number_of_compressed_datasources = 0; + geometry_datasource_input_stream.read( + reinterpret_cast(&number_of_compressed_datasources), sizeof(std::size_t)); + shared_layout_ptr->SetBlockSize(SharedDataLayout::DATASOURCES_LIST, + number_of_compressed_datasources); + } + + // Load datasource name sizes. This file is optional, and it's non-fatal if it doesn't + // exist + std::ifstream datasource_names_input_stream(datasource_names_path.c_str(), std::ios::binary); + std::vector m_datasource_name_data; + std::vector m_datasource_name_offsets; + std::vector m_datasource_name_lengths; + if (datasource_names_input_stream) + { + std::string name; + while (std::getline(datasource_names_input_stream, name)) + { + m_datasource_name_offsets.push_back(m_datasource_name_data.size()); + std::copy(name.c_str(), name.c_str() + name.size(), + std::back_inserter(m_datasource_name_data)); + m_datasource_name_lengths.push_back(name.size()); + } + + shared_layout_ptr->SetBlockSize(SharedDataLayout::DATASOURCE_NAME_DATA, + m_datasource_name_data.size()); + shared_layout_ptr->SetBlockSize(SharedDataLayout::DATASOURCE_NAME_OFFSETS, + m_datasource_name_offsets.size()); + shared_layout_ptr->SetBlockSize(SharedDataLayout::DATASOURCE_NAME_LENGTHS, + m_datasource_name_lengths.size()); + } + // allocate shared memory block util::SimpleLogger().Write() << "allocating shared memory of " << shared_layout_ptr->GetSizeOfLayout() << " bytes"; @@ -457,6 +512,50 @@ int Storage::Run() shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_LIST)); } + // load datasource information (if it exists) + if (geometry_datasource_input_stream) + { + uint8_t *datasources_list_ptr = shared_layout_ptr->GetBlockPtr( + shared_memory_ptr, SharedDataLayout::DATASOURCES_LIST); + if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCES_LIST) > 0) + { + geometry_datasource_input_stream.read( + reinterpret_cast(datasources_list_ptr), + shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCES_LIST)); + } + } + + // load datasource name information (if it exists) + if (!m_datasource_name_data.empty()) + { + char *datasource_name_data_ptr = shared_layout_ptr->GetBlockPtr( + shared_memory_ptr, SharedDataLayout::DATASOURCE_NAME_DATA); + if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCE_NAME_DATA) > 0) + { + std::cout << "Copying " + << (m_datasource_name_data.end() - m_datasource_name_data.begin()) + << " chars into name data ptr\n"; + std::copy(m_datasource_name_data.begin(), m_datasource_name_data.end(), + datasource_name_data_ptr); + } + + auto datasource_name_offsets_ptr = shared_layout_ptr->GetBlockPtr( + shared_memory_ptr, SharedDataLayout::DATASOURCE_NAME_OFFSETS); + if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCE_NAME_OFFSETS) > 0) + { + std::copy(m_datasource_name_offsets.begin(), m_datasource_name_offsets.end(), + datasource_name_offsets_ptr); + } + + auto datasource_name_lengths_ptr = shared_layout_ptr->GetBlockPtr( + shared_memory_ptr, SharedDataLayout::DATASOURCE_NAME_LENGTHS); + if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCE_NAME_LENGTHS) > 0) + { + std::copy(m_datasource_name_lengths.begin(), m_datasource_name_lengths.end(), + datasource_name_lengths_ptr); + } + } + // Loading list of coordinates util::Coordinate *coordinates_ptr = shared_layout_ptr->GetBlockPtr( shared_memory_ptr, SharedDataLayout::COORDINATE_LIST); diff --git a/src/tools/contract.cpp b/src/tools/contract.cpp index 3b8672be366..5bc65c5ed57 100644 --- a/src/tools/contract.cpp +++ b/src/tools/contract.cpp @@ -39,9 +39,10 @@ return_code parseArguments(int argc, char *argv[], contractor::ContractorConfig "core,k", boost::program_options::value(&contractor_config.core_factor)->default_value(1.0), "Percentage of the graph (in vertices) to contract [0..1]")( - "segment-speed-file", - boost::program_options::value(&contractor_config.segment_speed_lookup_path), - "Lookup file containing nodeA,nodeB,speed data to adjust edge weights")( + "segment-speed-file", boost::program_options::value>( + &contractor_config.segment_speed_lookup_paths) + ->composing(), + "Lookup files containing nodeA, nodeB, speed data to adjust edge weights")( "level-cache,o", boost::program_options::value(&contractor_config.use_cached_priority) ->default_value(false), "Use .level file to retain the contaction level for each node from the last run."); diff --git a/src/tools/store.cpp b/src/tools/store.cpp index ba01423fa08..b2a589e5895 100644 --- a/src/tools/store.cpp +++ b/src/tools/store.cpp @@ -38,7 +38,13 @@ bool generateDataStoreOptions(const int argc, const char *argv[], storage::DataP "namesdata", boost::program_options::value(&paths["namesdata"]), ".names file")("timestamp", boost::program_options::value(&paths["timestamp"]), - ".timestamp file"); + ".timestamp file")( + "datasource_names", + boost::program_options::value(&paths["datasource_names"]), + ".datasource_names file")( + "datasource_indexes", + boost::program_options::value(&paths["datasource_indexes"]), + ".datasource_indexes file"); // hidden options, will be allowed on command line but will not be shown to the user boost::program_options::options_description hidden_options("Hidden options"); @@ -145,6 +151,18 @@ bool generateDataStoreOptions(const int argc, const char *argv[], storage::DataP path_iterator->second = base_string + ".timestamp"; } + path_iterator = paths.find("datasource_indexes"); + if (path_iterator != paths.end()) + { + path_iterator->second = base_string + ".datasource_indexes"; + } + + path_iterator = paths.find("datasource_names"); + if (path_iterator != paths.end()) + { + path_iterator->second = base_string + ".datasource_names"; + } + path_iterator = paths.find("hsgrdata"); if (path_iterator == paths.end() || path_iterator->second.string().empty() || !boost::filesystem::is_regular_file(path_iterator->second))