Merge pull request #2098 from Project-OSRM/tile_datasource_info

Support multiple traffic update files, and include filename in debug tile metadata

include/contractor/contractor.hpp

@@ -83,9 +83,11 @@ class Contractor
                           util::DeallocatingVector<extractor::EdgeBasedEdge> &edge_based_edge_list,
                           const std::string &edge_segment_lookup_path,
                           const std::string &edge_penalty_path,
-                          const std::string &segment_speed_path,
+                          const std::vector<std::string> &segment_speed_path,
                           const std::string &nodes_filename,
                           const std::string &geometry_filename,
+                          const std::string &datasource_names_filename,
+                          const std::string &datasource_indexes_filename,
                           const std::string &rtree_leaf_filename);
 };
 }

include/contractor/contractor_config.hpp

@@ -53,6 +53,8 @@ struct ContractorConfig
         node_based_graph_path = osrm_input_path.string() + ".nodes";
         geometry_path = osrm_input_path.string() + ".geometry";
         rtree_leaf_path = osrm_input_path.string() + ".fileIndex";
+        datasource_names_path = osrm_input_path.string() + ".datasource_names";
+        datasource_indexes_path = osrm_input_path.string() + ".datasource_indexes";
     }
 
     boost::filesystem::path config_file_path;
@@ -78,11 +80,9 @@ struct ContractorConfig
     //(e.g. 0.8 contracts 80 percent of the hierarchy, leaving a core of 20%)
     double core_factor;
 
-    std::string segment_speed_lookup_path;
-
-#ifdef DEBUG_GEOMETRY
-    std::string debug_geometry_path;
-#endif
+    std::vector<std::string> segment_speed_lookup_paths;
+    std::string datasource_indexes_path;
+    std::string datasource_names_path;
 };
 }
 }

include/engine/datafacade/datafacade_base.hpp

@@ -78,6 +78,14 @@ class BaseDataFacade
     virtual void GetUncompressedWeights(const EdgeID id,
                                         std::vector<EdgeWeight> &result_weights) const = 0;
 
+    // Returns the data source ids that were used to supply the edge
+    // weights.  Will return an empty array when only the base profile is used.
+    virtual void GetUncompressedDatasources(const EdgeID id,
+                                            std::vector<uint8_t> &data_sources) const = 0;
+
+    // Gets the name of a datasource
+    virtual std::string GetDatasourceName(const uint8_t datasource_name_id) const = 0;
+
     virtual extractor::guidance::TurnInstruction
     GetTurnInstructionForEdgeID(const unsigned id) const = 0;
 

include/engine/datafacade/internal_datafacade.hpp

@@ -78,6 +78,8 @@ class InternalDataFacade final : public BaseDataFacade
     util::ShM<extractor::CompressedEdgeContainer::CompressedEdge, false>::vector m_geometry_list;
     util::ShM<bool, false>::vector m_is_core_node;
     util::ShM<unsigned, false>::vector m_segment_weights;
+    util::ShM<uint8_t, false>::vector m_datasource_list;
+    util::ShM<std::string, false>::vector m_datasource_names;
 
     boost::thread_specific_ptr<InternalRTree> m_static_rtree;
     boost::thread_specific_ptr<InternalGeospatialQuery> m_geospatial_query;
@@ -222,6 +224,34 @@ class InternalDataFacade final : public BaseDataFacade
         }
     }
 
+    void LoadDatasourceInfo(const boost::filesystem::path &datasource_names_file,
+                            const boost::filesystem::path &datasource_indexes_file)
+    {
+        std::ifstream datasources_stream(datasource_indexes_file.c_str(), std::ios::binary);
+        if (datasources_stream)
+        {
+            std::size_t number_of_datasources = 0;
+            datasources_stream.read(reinterpret_cast<char *>(&number_of_datasources),
+                                    sizeof(std::size_t));
+            if (number_of_datasources > 0)
+            {
+                m_datasource_list.resize(number_of_datasources);
+                datasources_stream.read(reinterpret_cast<char *>(&(m_datasource_list[0])),
+                                        number_of_datasources * sizeof(uint8_t));
+            }
+        }
+
+        std::ifstream datasourcenames_stream(datasource_names_file.c_str(), std::ios::binary);
+        if (datasourcenames_stream)
+        {
+            std::string name;
+            while (std::getline(datasourcenames_stream, name))
+            {
+                m_datasource_names.push_back(name);
+            }
+        }
+    }
+
     void LoadRTree()
     {
         BOOST_ASSERT_MSG(!m_coordinate_list->empty(), "coordinates must be loaded before r-tree");
@@ -269,6 +299,14 @@ class InternalDataFacade final : public BaseDataFacade
             return it->second;
         };
 
+        const auto optional_file_for = [&server_paths, &end_it](const std::string &path)
+        {
+            const auto it = server_paths.find(path);
+            if (it == end_it)
+                throw util::exception("no valid " + path + " file given in ini file");
+            return it->second;
+        };
+
         ram_index_path = file_for("ramindex");
         file_index_path = file_for("fileindex");
 
@@ -284,6 +322,10 @@ class InternalDataFacade final : public BaseDataFacade
         util::SimpleLogger().Write() << "loading geometries";
         LoadGeometries(file_for("geometries"));
 
+        util::SimpleLogger().Write() << "loading datasource info";
+        LoadDatasourceInfo(optional_file_for("datasource_names"),
+                           optional_file_for("datasource_indexes"));
+
         util::SimpleLogger().Write() << "loading timestamp";
         LoadTimestamp(file_for("timestamp"));
 
@@ -592,6 +634,47 @@ class InternalDataFacade final : public BaseDataFacade
                       });
     }
 
+    // Returns the data source ids that were used to supply the edge
+    // weights.
+    virtual void
+    GetUncompressedDatasources(const EdgeID id,
+                               std::vector<uint8_t> &result_datasources) const override final
+    {
+        const unsigned begin = m_geometry_indices.at(id);
+        const unsigned end = m_geometry_indices.at(id + 1);
+
+        result_datasources.clear();
+        result_datasources.reserve(end - begin);
+
+        // If there was no datasource info, return an array of 0's.
+        if (m_datasource_list.empty())
+        {
+            for (unsigned i = 0; i < end - begin; ++i)
+            {
+                result_datasources.push_back(0);
+            }
+        }
+        else
+        {
+            std::for_each(m_datasource_list.begin() + begin, m_datasource_list.begin() + end,
+                          [&](const uint8_t &datasource_id)
+                          {
+                              result_datasources.push_back(datasource_id);
+                          });
+        }
+    }
+
+    virtual std::string GetDatasourceName(const uint8_t datasource_name_id) const override final
+    {
+        if (m_datasource_names.empty() || datasource_name_id > m_datasource_names.size())
+        {
+            if (datasource_name_id == 0)
+                return "lua profile";
+            return "UNKNOWN";
+        }
+        return m_datasource_names[datasource_name_id];
+    }
+
     std::string GetTimestamp() const override final { return m_timestamp; }
 };
 }

include/engine/datafacade/shared_datafacade.hpp

@@ -80,6 +80,11 @@ class SharedDataFacade final : public BaseDataFacade
     util::ShM<unsigned, true>::vector m_geometry_indices;
     util::ShM<extractor::CompressedEdgeContainer::CompressedEdge, true>::vector m_geometry_list;
     util::ShM<bool, true>::vector m_is_core_node;
+    util::ShM<uint8_t, true>::vector m_datasource_list;
+
+    util::ShM<char, true>::vector m_datasource_name_data;
+    util::ShM<std::size_t, true>::vector m_datasource_name_offsets;
+    util::ShM<std::size_t, true>::vector m_datasource_name_lengths;
 
     boost::thread_specific_ptr<std::pair<unsigned, std::shared_ptr<SharedRTree>>> m_static_rtree;
     boost::thread_specific_ptr<SharedGeospatialQuery> m_geospatial_query;
@@ -231,6 +236,34 @@ class SharedDataFacade final : public BaseDataFacade
             geometry_list(geometries_list_ptr,
                           data_layout->num_entries[storage::SharedDataLayout::GEOMETRIES_LIST]);
         m_geometry_list = std::move(geometry_list);
+
+        auto datasources_list_ptr = data_layout->GetBlockPtr<uint8_t>(
+            shared_memory, storage::SharedDataLayout::DATASOURCES_LIST);
+        typename util::ShM<uint8_t, true>::vector datasources_list(
+            datasources_list_ptr,
+            data_layout->num_entries[storage::SharedDataLayout::DATASOURCES_LIST]);
+        m_datasource_list = std::move(datasources_list);
+
+        auto datasource_name_data_ptr = data_layout->GetBlockPtr<char>(
+            shared_memory, storage::SharedDataLayout::DATASOURCE_NAME_DATA);
+        typename util::ShM<char, true>::vector datasource_name_data(
+            datasource_name_data_ptr,
+            data_layout->num_entries[storage::SharedDataLayout::DATASOURCE_NAME_DATA]);
+        m_datasource_name_data = std::move(datasource_name_data);
+
+        auto datasource_name_offsets_ptr = data_layout->GetBlockPtr<std::size_t>(
+            shared_memory, storage::SharedDataLayout::DATASOURCE_NAME_OFFSETS);
+        typename util::ShM<std::size_t, true>::vector datasource_name_offsets(
+            datasource_name_offsets_ptr,
+            data_layout->num_entries[storage::SharedDataLayout::DATASOURCE_NAME_OFFSETS]);
+        m_datasource_name_offsets = std::move(datasource_name_offsets);
+
+        auto datasource_name_lengths_ptr = data_layout->GetBlockPtr<std::size_t>(
+            shared_memory, storage::SharedDataLayout::DATASOURCE_NAME_LENGTHS);
+        typename util::ShM<std::size_t, true>::vector datasource_name_lengths(
+            datasource_name_lengths_ptr,
+            data_layout->num_entries[storage::SharedDataLayout::DATASOURCE_NAME_LENGTHS]);
+        m_datasource_name_lengths = std::move(datasource_name_lengths);
     }
 
   public:
@@ -632,6 +665,56 @@ class SharedDataFacade final : public BaseDataFacade
 
     virtual std::size_t GetCoreSize() const override final { return m_is_core_node.size(); }
 
+    // Returns the data source ids that were used to supply the edge
+    // weights.
+    virtual void
+    GetUncompressedDatasources(const EdgeID id,
+                               std::vector<uint8_t> &result_datasources) const override final
+    {
+        const unsigned begin = m_geometry_indices.at(id);
+        const unsigned end = m_geometry_indices.at(id + 1);
+
+        result_datasources.clear();
+        result_datasources.reserve(end - begin);
+
+        // If there was no datasource info, return an array of 0's.
+        if (m_datasource_list.empty())
+        {
+            for (unsigned i = 0; i < end - begin; ++i)
+            {
+                result_datasources.push_back(0);
+            }
+        }
+        else
+        {
+            std::for_each(m_datasource_list.begin() + begin, m_datasource_list.begin() + end,
+                          [&](const uint8_t &datasource_id)
+                          {
+                              result_datasources.push_back(datasource_id);
+                          });
+        }
+    }
+
+    virtual std::string GetDatasourceName(const uint8_t datasource_name_id) const override final
+    {
+        std::string result;
+
+        if (m_datasource_name_offsets.empty() ||
+            datasource_name_id > m_datasource_name_offsets.size())
+        {
+            if (datasource_name_id == 0)
+                return "lua profile";
+            return "UNKNOWN";
+        }
+
+        std::copy(m_datasource_name_data.begin() + m_datasource_name_offsets[datasource_name_id],
+                  m_datasource_name_data.begin() + m_datasource_name_offsets[datasource_name_id] +
+                      m_datasource_name_lengths[datasource_name_id],
+                  std::back_inserter(result));
+
+        return result;
+    }
+
     std::string GetTimestamp() const override final { return m_timestamp; }
 };
 }

include/engine/engine_config.hpp

@@ -52,6 +52,8 @@ struct EngineConfig
                        {"coredata", base.string() + ".core"},
                        {"geometries", base.string() + ".geometry"},
                        {"timestamp", base.string() + ".timestamp"},
+                       {"datasource_names", base.string() + ".datasource_names"},
+                       {"datasource_indexes", base.string() + ".datasource_indexes"},
                        {"namesdata", base.string() + ".names"}}
     {
     }

include/extractor/edge_based_graph_factory.hpp

@@ -109,20 +109,11 @@ class EdgeBasedGraphFactory
     void CompressGeometry();
     unsigned RenumberEdges();
     void GenerateEdgeExpandedNodes();
-#ifdef DEBUG_GEOMETRY
-    void GenerateEdgeExpandedEdges(const std::string &original_edge_data_filename,
-                                   lua_State *lua_state,
-                                   const std::string &edge_segment_lookup_filename,
-                                   const std::string &edge_fixed_penalties_filename,
-                                   const bool generate_edge_lookup,
-                                   const std::string &debug_turns_path);
-#else
     void GenerateEdgeExpandedEdges(const std::string &original_edge_data_filename,
                                    lua_State *lua_state,
                                    const std::string &edge_segment_lookup_filename,
                                    const std::string &edge_fixed_penalties_filename,
                                    const bool generate_edge_lookup);
-#endif
 
     void InsertEdgeBasedNode(const NodeID u, const NodeID v);
 

include/storage/shared_datatype.hpp

@@ -38,6 +38,10 @@ struct SharedDataLayout
         TIMESTAMP,
         FILE_INDEX_PATH,
         CORE_MARKER,
+        DATASOURCES_LIST,
+        DATASOURCE_NAME_DATA,
+        DATASOURCE_NAME_OFFSETS,
+        DATASOURCE_NAME_LENGTHS,
         NUM_BLOCKS
     };
 

include/util/routed_options.hpp

@@ -53,6 +53,10 @@ populate_base_path(std::unordered_map<std::string, boost::filesystem::path> &ser
         BOOST_ASSERT(server_paths.find("namesdata") != server_paths.end());
         server_paths["timestamp"] = base_string + ".timestamp";
         BOOST_ASSERT(server_paths.find("timestamp") != server_paths.end());
+        server_paths["datasource_indexes"] = base_string + ".datasource_indexes";
+        BOOST_ASSERT(server_paths.find("timestamp") != server_paths.end());
+        server_paths["datasource_names"] = base_string + ".datasource_names";
+        BOOST_ASSERT(server_paths.find("timestamp") != server_paths.end());
     }
 
     // check if files are give and whether they exist at all