From e8c737dffa4ed39e4249af1cb398159adb5e18a6 Mon Sep 17 00:00:00 2001 From: Daniel Patterson Date: Tue, 15 Mar 2016 13:01:22 -0700 Subject: [PATCH] Include the names of datasources in the vector tile outputs. --- include/contractor/contractor.hpp | 2 + include/contractor/contractor_config.hpp | 7 +- include/engine/datafacade/datafacade_base.hpp | 5 +- .../engine/datafacade/internal_datafacade.hpp | 40 +++++++++- .../engine/datafacade/shared_datafacade.hpp | 44 +++++++++++ include/engine/engine_config.hpp | 2 + .../extractor/edge_based_graph_factory.hpp | 9 --- include/storage/shared_datatype.hpp | 4 +- include/util/routed_options.hpp | 4 + src/contractor/contractor.cpp | 22 +++++- src/engine/plugins/tile.cpp | 2 +- src/extractor/extractor.cpp | 7 +- src/storage/storage.cpp | 73 ++++++++++++++++++- src/tools/store.cpp | 20 ++++- 14 files changed, 210 insertions(+), 31 deletions(-) diff --git a/include/contractor/contractor.hpp b/include/contractor/contractor.hpp index 9bb8ec40c0c..7cfc3dc664d 100644 --- a/include/contractor/contractor.hpp +++ b/include/contractor/contractor.hpp @@ -86,6 +86,8 @@ class Contractor const std::vector &segment_speed_path, const std::string &nodes_filename, const std::string &geometry_filename, + const std::string &datasource_names_filename, + const std::string &datasource_indexes_filename, const std::string &rtree_leaf_filename); }; } diff --git a/include/contractor/contractor_config.hpp b/include/contractor/contractor_config.hpp index 42fb79e763f..9bf087f3ae3 100644 --- a/include/contractor/contractor_config.hpp +++ b/include/contractor/contractor_config.hpp @@ -53,6 +53,8 @@ struct ContractorConfig node_based_graph_path = osrm_input_path.string() + ".nodes"; geometry_path = osrm_input_path.string() + ".geometry"; rtree_leaf_path = osrm_input_path.string() + ".fileIndex"; + datasource_names_path = osrm_input_path.string() + ".datasource_names"; + datasource_indexes_path = osrm_input_path.string() + ".datasource_indexes"; } boost::filesystem::path config_file_path; @@ -79,10 +81,9 @@ struct ContractorConfig double core_factor; std::vector segment_speed_lookup_paths; + std::string datasource_indexes_path; + std::string datasource_names_path; -#ifdef DEBUG_GEOMETRY - std::string debug_geometry_path; -#endif }; } } diff --git a/include/engine/datafacade/datafacade_base.hpp b/include/engine/datafacade/datafacade_base.hpp index fb71623d266..f261d902fab 100644 --- a/include/engine/datafacade/datafacade_base.hpp +++ b/include/engine/datafacade/datafacade_base.hpp @@ -80,9 +80,12 @@ class BaseDataFacade // Returns the data source ids that were used to supply the edge // weights. Will return an empty array when only the base profile is used. - virtual void GetUncompressedDatasources(const EdgeID, + virtual void GetUncompressedDatasources(const EdgeID id, std::vector &data_sources) const = 0; + // Gets the name of a datasource + virtual std::string GetDatasourceName(const uint8_t datasource_name_id) const = 0; + virtual extractor::guidance::TurnInstruction GetTurnInstructionForEdgeID(const unsigned id) const = 0; diff --git a/include/engine/datafacade/internal_datafacade.hpp b/include/engine/datafacade/internal_datafacade.hpp index 9d278c39826..181a625b6c3 100644 --- a/include/engine/datafacade/internal_datafacade.hpp +++ b/include/engine/datafacade/internal_datafacade.hpp @@ -79,6 +79,7 @@ class InternalDataFacade final : public BaseDataFacade util::ShM::vector m_is_core_node; util::ShM::vector m_segment_weights; util::ShM::vector m_datasource_list; + util::ShM::vector m_datasource_names; boost::thread_specific_ptr m_static_rtree; boost::thread_specific_ptr m_geospatial_query; @@ -221,20 +222,33 @@ class InternalDataFacade final : public BaseDataFacade number_of_compressed_geometries * sizeof(extractor::CompressedEdgeContainer::CompressedEdge)); } + } - - std::ifstream datasources_stream((geometry_file.string() + ".sources").c_str(), std::ios::binary); + void LoadDatasourceInfo(const boost::filesystem::path &datasource_names_file, + const boost::filesystem::path &datasource_indexes_file) + { + std::ifstream datasources_stream(datasource_indexes_file.c_str(), std::ios::binary); if (datasources_stream) { size_t number_of_datasources = 0; datasources_stream.read(reinterpret_cast(&number_of_datasources), sizeof(size_t)); - if (number_of_compressed_geometries > 0) + if (number_of_datasources > 0) { m_datasource_list.resize(number_of_datasources); datasources_stream.read(reinterpret_cast(&(m_datasource_list[0])), number_of_datasources * sizeof(uint8_t)); } } + + std::ifstream datasourcenames_stream(datasource_names_file.c_str(), std::ios::binary); + if (datasourcenames_stream) + { + std::string name; + while (std::getline(datasourcenames_stream, name)) + { + m_datasource_names.push_back(name); + } + } } void LoadRTree() @@ -284,6 +298,14 @@ class InternalDataFacade final : public BaseDataFacade return it->second; }; + const auto optional_file_for = [&server_paths, &end_it](const std::string &path) + { + const auto it = server_paths.find(path); + if (it == end_it) + throw util::exception("no valid " + path + " file given in ini file"); + return it->second; + }; + ram_index_path = file_for("ramindex"); file_index_path = file_for("fileindex"); @@ -299,6 +321,9 @@ class InternalDataFacade final : public BaseDataFacade util::SimpleLogger().Write() << "loading geometries"; LoadGeometries(file_for("geometries")); + util::SimpleLogger().Write() << "loading datasource info"; + LoadDatasourceInfo(optional_file_for("datasource_names"), optional_file_for("datasource_indexes")); + util::SimpleLogger().Write() << "loading timestamp"; LoadTimestamp(file_for("timestamp")); @@ -636,6 +661,15 @@ class InternalDataFacade final : public BaseDataFacade } } + virtual std::string GetDatasourceName(const uint8_t datasource_name_id) const override final + { + if (datasource_name_id > m_datasource_names.size()) + { + return "lua profile"; + } + return m_datasource_names[datasource_name_id]; + } + std::string GetTimestamp() const override final { return m_timestamp; } }; } diff --git a/include/engine/datafacade/shared_datafacade.hpp b/include/engine/datafacade/shared_datafacade.hpp index 81433faf7c8..d38ff6392b6 100644 --- a/include/engine/datafacade/shared_datafacade.hpp +++ b/include/engine/datafacade/shared_datafacade.hpp @@ -82,6 +82,10 @@ class SharedDataFacade final : public BaseDataFacade util::ShM::vector m_is_core_node; util::ShM::vector m_datasource_list; + util::ShM::vector m_datasource_name_data; + util::ShM::vector m_datasource_name_offsets; + util::ShM::vector m_datasource_name_lengths; + boost::thread_specific_ptr>> m_static_rtree; boost::thread_specific_ptr m_geospatial_query; boost::filesystem::path file_index_path; @@ -240,6 +244,30 @@ class SharedDataFacade final : public BaseDataFacade datasources_list(datasources_list_ptr, data_layout->num_entries[storage::SharedDataLayout::DATASOURCES_LIST]); m_datasource_list = std::move(datasources_list); + + auto datasource_name_data_ptr = + data_layout->GetBlockPtr( + shared_memory, storage::SharedDataLayout::DATASOURCE_NAME_DATA); + typename util::ShM::vector + datasource_name_data(datasource_name_data_ptr, + data_layout->num_entries[storage::SharedDataLayout::DATASOURCE_NAME_DATA]); + m_datasource_name_data = std::move(datasource_name_data); + + auto datasource_name_offsets_ptr = + data_layout->GetBlockPtr( + shared_memory, storage::SharedDataLayout::DATASOURCE_NAME_OFFSETS); + typename util::ShM::vector + datasource_name_offsets(datasource_name_offsets_ptr, + data_layout->num_entries[storage::SharedDataLayout::DATASOURCE_NAME_OFFSETS]); + m_datasource_name_offsets = std::move(datasource_name_offsets); + + auto datasource_name_lengths_ptr = + data_layout->GetBlockPtr( + shared_memory, storage::SharedDataLayout::DATASOURCE_NAME_LENGTHS); + typename util::ShM::vector + datasource_name_lengths(datasource_name_lengths_ptr, + data_layout->num_entries[storage::SharedDataLayout::DATASOURCE_NAME_LENGTHS]); + m_datasource_name_lengths = std::move(datasource_name_lengths); } public: @@ -670,6 +698,22 @@ class SharedDataFacade final : public BaseDataFacade } } + virtual std::string GetDatasourceName(const uint8_t datasource_name_id) const override final + { + std::string result; + + if (datasource_name_id > m_datasource_name_offsets.size()) + { + return "lua profile"; + } + + std::copy(m_datasource_name_data.begin() + m_datasource_name_offsets[datasource_name_id], + m_datasource_name_data.begin() + m_datasource_name_offsets[datasource_name_id] + m_datasource_name_lengths[datasource_name_id], + std::back_inserter(result)); + + return result; + } + std::string GetTimestamp() const override final { return m_timestamp; } }; } diff --git a/include/engine/engine_config.hpp b/include/engine/engine_config.hpp index c54cd52cb90..182ef805ad7 100644 --- a/include/engine/engine_config.hpp +++ b/include/engine/engine_config.hpp @@ -52,6 +52,8 @@ struct EngineConfig {"coredata", base.string() + ".core"}, {"geometries", base.string() + ".geometry"}, {"timestamp", base.string() + ".timestamp"}, + {"datasource_names", base.string() + ".datasource_names"}, + {"datasource_indexes", base.string() + ".datasource_indexes"}, {"namesdata", base.string() + ".names"}} { } diff --git a/include/extractor/edge_based_graph_factory.hpp b/include/extractor/edge_based_graph_factory.hpp index 4e448ebf381..ede4deb024d 100644 --- a/include/extractor/edge_based_graph_factory.hpp +++ b/include/extractor/edge_based_graph_factory.hpp @@ -109,20 +109,11 @@ class EdgeBasedGraphFactory void CompressGeometry(); unsigned RenumberEdges(); void GenerateEdgeExpandedNodes(); -#ifdef DEBUG_GEOMETRY - void GenerateEdgeExpandedEdges(const std::string &original_edge_data_filename, - lua_State *lua_state, - const std::string &edge_segment_lookup_filename, - const std::string &edge_fixed_penalties_filename, - const bool generate_edge_lookup, - const std::string &debug_turns_path); -#else void GenerateEdgeExpandedEdges(const std::string &original_edge_data_filename, lua_State *lua_state, const std::string &edge_segment_lookup_filename, const std::string &edge_fixed_penalties_filename, const bool generate_edge_lookup); -#endif void InsertEdgeBasedNode(const NodeID u, const NodeID v); diff --git a/include/storage/shared_datatype.hpp b/include/storage/shared_datatype.hpp index e41a9fbaebd..20c25d699fa 100644 --- a/include/storage/shared_datatype.hpp +++ b/include/storage/shared_datatype.hpp @@ -40,8 +40,8 @@ struct SharedDataLayout CORE_MARKER, DATASOURCES_LIST, DATASOURCE_NAME_DATA, - DATASOURCE_NAME_INDEXES, - DATASOURCE_NAME_LENGTHS + DATASOURCE_NAME_OFFSETS, + DATASOURCE_NAME_LENGTHS, NUM_BLOCKS }; diff --git a/include/util/routed_options.hpp b/include/util/routed_options.hpp index 0a23bdd2f5d..2ea65f82e00 100644 --- a/include/util/routed_options.hpp +++ b/include/util/routed_options.hpp @@ -53,6 +53,10 @@ populate_base_path(std::unordered_map &ser BOOST_ASSERT(server_paths.find("namesdata") != server_paths.end()); server_paths["timestamp"] = base_string + ".timestamp"; BOOST_ASSERT(server_paths.find("timestamp") != server_paths.end()); + server_paths["datasource_indexes"] = base_string + ".datasource_indexes"; + BOOST_ASSERT(server_paths.find("timestamp") != server_paths.end()); + server_paths["datasource_names"] = base_string + ".datasource_names"; + BOOST_ASSERT(server_paths.find("timestamp") != server_paths.end()); } // check if files are give and whether they exist at all diff --git a/src/contractor/contractor.cpp b/src/contractor/contractor.cpp index 1ff6fdf6ed3..33c2157608f 100644 --- a/src/contractor/contractor.cpp +++ b/src/contractor/contractor.cpp @@ -72,7 +72,8 @@ int Contractor::Run() std::size_t max_edge_id = LoadEdgeExpandedGraph( config.edge_based_graph_path, edge_based_edge_list, config.edge_segment_lookup_path, config.edge_penalty_path, config.segment_speed_lookup_paths, config.node_based_graph_path, - config.geometry_path, config.rtree_leaf_path); + config.geometry_path, config.datasource_names_path, config.datasource_indexes_path, + config.rtree_leaf_path); // Contracting the edge-expanded graph @@ -130,6 +131,8 @@ std::size_t Contractor::LoadEdgeExpandedGraph( const std::vector &segment_speed_filenames, const std::string &nodes_filename, const std::string &geometry_filename, + const std::string &datasource_names_filename, + const std::string &datasource_indexes_filename, const std::string &rtree_leaf_filename) { util::SimpleLogger().Write() << "Opening " << edge_based_graph_filename; @@ -395,10 +398,10 @@ std::size_t Contractor::LoadEdgeExpandedGraph( } { - std::ofstream datasource_stream(geometry_filename + ".sources", std::ios::binary); + std::ofstream datasource_stream(datasource_indexes_filename, std::ios::binary); if (!datasource_stream) { - throw util::exception("Failed to open " + geometry_filename + ".sources for writing"); + throw util::exception("Failed to open " + datasource_indexes_filename + " for writing"); } auto number_of_datasource_entries = m_geometry_datasource.size(); datasource_stream.write(reinterpret_cast(&number_of_datasource_entries), @@ -406,6 +409,19 @@ std::size_t Contractor::LoadEdgeExpandedGraph( datasource_stream.write(reinterpret_cast(&(m_geometry_datasource[0])), number_of_datasource_entries * sizeof(uint8_t)); } + + { + std::ofstream datasource_stream(datasource_names_filename, std::ios::binary); + if (!datasource_stream) + { + throw util::exception("Failed to open " + datasource_names_filename + " for writing"); + } + datasource_stream << "lua profile" << std::endl; + for (auto const &name : segment_speed_filenames) + { + datasource_stream << name << std::endl; + } + } } // TODO: can we read this in bulk? util::DeallocatingVector isn't necessarily diff --git a/src/engine/plugins/tile.cpp b/src/engine/plugins/tile.cpp index f9dc57e6809..c8ff86b25f3 100644 --- a/src/engine/plugins/tile.cpp +++ b/src/engine/plugins/tile.cpp @@ -416,7 +416,7 @@ Status TilePlugin::HandleRequest(const api::TileParameters ¶meters, std::str // Writing field type 4 == variant type protozero::pbf_writer values_writer(layer_writer, 4); // Attribute value 1 == string type - values_writer.add_string(1, "datasource " + std::to_string(i)); + values_writer.add_string(1, facade.GetDatasourceName(i)); } } } diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index 7ac29534062..7123bfce731 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -528,12 +528,7 @@ Extractor::BuildEdgeExpandedGraph(std::vector &internal_to_external_n edge_based_graph_factory.Run(config.edge_output_path, lua_state, config.edge_segment_lookup_path, config.edge_penalty_path, - config.generate_edge_lookup -#ifdef DEBUG_GEOMETRY - , - config.debug_turns_path -#endif - ); + config.generate_edge_lookup); lua_close(lua_state); diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 88c8402d43d..f9281dd6cdf 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -131,6 +131,14 @@ int Storage::Run() { throw util::exception("no core file found"); } + if (paths.find("datasource_indexes") == paths.end()) + { + throw util::exception("no datasource_indexes file found"); + } + if (paths.find("datasource_names") == paths.end()) + { + throw util::exception("no datasource_names file found"); + } auto paths_iterator = paths.find("hsgrdata"); BOOST_ASSERT(paths.end() != paths_iterator); @@ -170,6 +178,14 @@ int Storage::Run() BOOST_ASSERT(paths.end() != paths_iterator); BOOST_ASSERT(!paths_iterator->second.empty()); const boost::filesystem::path &core_marker_path = paths_iterator->second; + paths_iterator = paths.find("datasource_indexes"); + BOOST_ASSERT(paths.end() != paths_iterator); + BOOST_ASSERT(!paths_iterator->second.empty()); + const boost::filesystem::path &datasource_indexes_path = paths_iterator->second; + paths_iterator = paths.find("datasource_names"); + BOOST_ASSERT(paths.end() != paths_iterator); + BOOST_ASSERT(!paths_iterator->second.empty()); + const boost::filesystem::path &datasource_names_path = paths_iterator->second; // determine segment to use bool segment2_in_use = SharedMemory::RegionExists(LAYOUT_2); @@ -328,7 +344,7 @@ int Storage::Run() // load datasource sizes. This file is optional, and it's non-fatal if it doesn't // exist. - std::ifstream geometry_datasource_input_stream((geometries_data_path.string()+".sources").c_str(), std::ios::binary); + std::ifstream geometry_datasource_input_stream(datasource_indexes_path.c_str(), std::ios::binary); if (geometry_datasource_input_stream) { size_t number_of_compressed_datasources = 0; @@ -336,6 +352,27 @@ int Storage::Run() shared_layout_ptr->SetBlockSize(SharedDataLayout::DATASOURCES_LIST, number_of_compressed_datasources); } + // Load datasource name sizes. This file is optional, and it's non-fatal if it doesn't + // exist + std::ifstream datasource_names_input_stream(datasource_names_path.c_str(), std::ios::binary); + std::vector m_datasource_name_data; + std::vector m_datasource_name_offsets; + std::vector m_datasource_name_lengths; + if (datasource_names_input_stream) + { + std::string name; + while (std::getline(datasource_names_input_stream, name)) + { + m_datasource_name_offsets.push_back(m_datasource_name_data.size()); + std::copy(name.c_str(), name.c_str() + name.size(), std::back_inserter(m_datasource_name_data)); + m_datasource_name_lengths.push_back(name.size()); + } + + shared_layout_ptr->SetBlockSize(SharedDataLayout::DATASOURCE_NAME_DATA, m_datasource_name_data.size()); + shared_layout_ptr->SetBlockSize(SharedDataLayout::DATASOURCE_NAME_OFFSETS, m_datasource_name_offsets.size()); + shared_layout_ptr->SetBlockSize(SharedDataLayout::DATASOURCE_NAME_LENGTHS, m_datasource_name_lengths.size()); + } + // allocate shared memory block util::SimpleLogger().Write() << "allocating shared memory of " << shared_layout_ptr->GetSizeOfLayout() << " bytes"; @@ -478,11 +515,43 @@ int Storage::Run() if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCES_LIST) > 0) { geometry_datasource_input_stream.read( - (char *)datasources_list_ptr, + reinterpret_cast(datasources_list_ptr), shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCES_LIST)); } } + // load datasource name information (if it exists) + if (!m_datasource_name_data.empty()) + { + char *datasource_name_data_ptr = + shared_layout_ptr->GetBlockPtr( + shared_memory_ptr, SharedDataLayout::DATASOURCE_NAME_DATA); + if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCE_NAME_DATA) > 0) + { + std::cout << "Copying " << (m_datasource_name_data.end() - m_datasource_name_data.begin()) << " chars into name data ptr\n"; + std::copy(m_datasource_name_data.begin(), m_datasource_name_data.end(), + datasource_name_data_ptr); + } + + size_t *datasource_name_offsets_ptr = + shared_layout_ptr->GetBlockPtr( + shared_memory_ptr, SharedDataLayout::DATASOURCE_NAME_OFFSETS); + if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCE_NAME_OFFSETS) > 0) + { + std::copy(m_datasource_name_offsets.begin(), m_datasource_name_offsets.end(), + datasource_name_offsets_ptr); + } + + size_t *datasource_name_lengths_ptr = + shared_layout_ptr->GetBlockPtr( + shared_memory_ptr, SharedDataLayout::DATASOURCE_NAME_LENGTHS); + if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCE_NAME_LENGTHS) > 0) + { + std::copy(m_datasource_name_lengths.begin(), m_datasource_name_lengths.end(), + datasource_name_lengths_ptr); + } + } + // Loading list of coordinates util::Coordinate *coordinates_ptr = shared_layout_ptr->GetBlockPtr( shared_memory_ptr, SharedDataLayout::COORDINATE_LIST); diff --git a/src/tools/store.cpp b/src/tools/store.cpp index ba01423fa08..fb3458f2571 100644 --- a/src/tools/store.cpp +++ b/src/tools/store.cpp @@ -38,7 +38,13 @@ bool generateDataStoreOptions(const int argc, const char *argv[], storage::DataP "namesdata", boost::program_options::value(&paths["namesdata"]), ".names file")("timestamp", boost::program_options::value(&paths["timestamp"]), - ".timestamp file"); + ".timestamp file") + ("datasource_names", + boost::program_options::value(&paths["datasource_names"]), + ".datasource_names file") + ("datasource_indexes", + boost::program_options::value(&paths["datasource_indexes"]), + ".datasource_indexes file"); // hidden options, will be allowed on command line but will not be shown to the user boost::program_options::options_description hidden_options("Hidden options"); @@ -145,6 +151,18 @@ bool generateDataStoreOptions(const int argc, const char *argv[], storage::DataP path_iterator->second = base_string + ".timestamp"; } + path_iterator = paths.find("datasource_indexes"); + if (path_iterator != paths.end()) + { + path_iterator->second = base_string + ".datasource_indexes"; + } + + path_iterator = paths.find("datasource_names"); + if (path_iterator != paths.end()) + { + path_iterator->second = base_string + ".datasource_names"; + } + path_iterator = paths.find("hsgrdata"); if (path_iterator == paths.end() || path_iterator->second.string().empty() || !boost::filesystem::is_regular_file(path_iterator->second))