Add documentation to extractor

data_structures/restriction.hpp

@@ -77,11 +77,15 @@ struct TurnRestriction
     }
 };
 
+/**
+ * This is just a wrapper around TurnRestriction used in the extractor.
+ * 
+ * Could be merged with TurnRestriction. For now the type-destiction makes sense
+ * as the format in which the restriction is presented in the extractor and in the
+ * preprocessing is different. (see restriction_parser.cpp)
+ */
 struct InputRestrictionContainer
 {
-    // EdgeID fromWay;
-    // EdgeID toWay;
-    // NodeID via_node;
     TurnRestriction restriction;
 
     InputRestrictionContainer(EdgeID fromWay, EdgeID toWay, EdgeID vw)

extractor/extraction_containers.cpp

@@ -62,6 +62,17 @@ ExtractionContainers::~ExtractionContainers()
     way_start_end_id_list.clear();
 }
 
+/**
+ * Processes the collected data and serializes it.
+ * At this point nodes are still referenced by their OSM id.
+ *
+ * - map start-end nodes of ways to ways used int restrictions to compute compressed
+ *   trippe representation
+ * - filter nodes list to nodes that are referenced by ways
+ * - merge edges with nodes to include location of start/end points and serialize
+ *
+ * FIXME: Each of this step should be an own function for readability.
+ */
 void ExtractionContainers::PrepareData(const std::string &output_file_name,
                                        const std::string &restrictions_file_name)
 {
@@ -301,6 +312,7 @@ void ExtractionContainers::PrepareData(const std::string &output_file_name,
         node_iterator = all_nodes_list.begin();
         edge_iterator = all_edges_list.begin();
 
+        // Also serializes the edges
         while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end())
         {
             if (edge_iterator->target < node_iterator->node_id)
@@ -329,6 +341,8 @@ void ExtractionContainers::PrepareData(const std::string &output_file_name,
                 int integer_weight = std::max(
                     1, (int)std::floor(
                            (edge_iterator->is_duration_set ? edge_iterator->speed : weight) + .5));
+                // FIXME: This means we have a _minimum_ edge length of 1m
+                // maybe use dm as base unit?
                 const int integer_distance = std::max(1, (int)distance);
                 const short zero = 0;
                 const short one = 1;

extractor/extraction_containers.hpp

@@ -36,6 +36,10 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <stxxl/vector>
 
+/**
+ * Uses external memory containers from stxxl to store all the data that
+ * is collected by the extractor callbacks.
+ */
 class ExtractionContainers
 {
 #ifndef _MSC_VER

extractor/extraction_way.hpp

@@ -34,6 +34,12 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <string>
 #include <vector>
 
+/**
+ * This struct is the direct result of the call to ```way_function```
+ * in the lua based profile.
+ *
+ * It is split into multiple edge segments in the ExtractorCallback.
+ */
 struct ExtractionWay
 {
     ExtractionWay() { clear(); }

extractor/extractor.cpp

@@ -64,6 +64,24 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <unordered_map>
 #include <vector>
 
+/**
+ * TODO: Refactor this function into smaller functions for better readability.
+ *
+ * This function is the entry point for the whole extraction process. The goal of the extraction
+ * step is to filter and convert the OSM geometry to something more fitting for routing.
+ * That includes:
+ *  - extracting turn restrictions
+ *  - splitting ways into (directional!) edge segments
+ *  - checking if nodes are barriers or traffic signal
+ *  - discarding all tag information: All relevant type information for nodes/ways
+ *    is extracted at this point.
+ *
+ * The result of this process are the following files:
+ *  .names : Names of all streets, stored as long consecutive string with prefix sum based index
+ *  .osrm  : Nodes and edges in a intermediate format that easy to digest for osrm-prepare
+ *  .restrictions : Turn restrictions that are used my osrm-prepare to construct the edge-expanded graph
+ *
+ */
 int extractor::run(const ExtractorConfig &extractor_config)
 {
     try
@@ -83,12 +101,8 @@ int extractor::run(const ExtractorConfig &extractor_config)
         // setup scripting environment
         ScriptingEnvironment scripting_environment(extractor_config.profile_path.string().c_str());
 
-        std::unordered_map<std::string, NodeID> string_map;
-        string_map[""] = 0;
-
         ExtractionContainers extraction_containers;
-        auto extractor_callbacks =
-            osrm::make_unique<ExtractorCallbacks>(extraction_containers, string_map);
+        auto extractor_callbacks = osrm::make_unique<ExtractorCallbacks>(extraction_containers);
 
         const osmium::io::File input_file(extractor_config.input_path.string());
         osmium::io::Reader reader(input_file);

extractor/extractor_callbacks.cpp

@@ -41,13 +41,18 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <string>
 #include <vector>
 
-ExtractorCallbacks::ExtractorCallbacks(ExtractionContainers &extraction_containers,
-                                       std::unordered_map<std::string, NodeID> &string_map)
-    : string_map(string_map), external_memory(extraction_containers)
+ExtractorCallbacks::ExtractorCallbacks(ExtractionContainers &extraction_containers)
+    : external_memory(extraction_containers)
 {
+    string_map[""] = 0;
 }
 
-/** warning: caller needs to take care of synchronization! */
+/**
+ * Takes the node position from osmium and the filtered properties from the lua
+ * profile and saves them to external memory.
+ *
+ * warning: caller needs to take care of synchronization!
+ */
 void ExtractorCallbacks::ProcessNode(const osmium::Node &input_node,
                                      const ExtractionNode &result_node)
 {
@@ -72,7 +77,15 @@ void ExtractorCallbacks::ProcessRestriction(
         //                           "y" : "n");
     }
 }
-/** warning: caller needs to take care of synchronization! */
+/**
+ * Takes the geometry contained in the ```input_way``` and the tags computed
+ * by the lua profile inside ```parsed_way``` and computes all edge segments.
+ *
+ * Depending on the forward/backwards weights the edges are split into forward
+ * and backward edges.
+ *
+ * warning: caller needs to take care of synchronization!
+ */
 void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const ExtractionWay &parsed_way)
 {
     if (((0 >= parsed_way.forward_speed) ||

extractor/extractor_callbacks.hpp

@@ -43,17 +43,24 @@ class ExtractionContainers;
 struct InputRestrictionContainer;
 struct ExtractionNode;
 
+/**
+ * This class is uses by the extractor with the results of the
+ * osmium based parsing and the customization through the lua profile.
+ *
+ * It mediates between the multi-threaded extraction process and the external memory containers.
+ * Thus the synchronization is handled inside of the extractor.
+ */
 class ExtractorCallbacks
 {
   private:
-    std::unordered_map<std::string, NodeID> &string_map;
+    // used to deduplicate street names: actually maps to name ids
+    std::unordered_map<std::string, NodeID> string_map;
     ExtractionContainers &external_memory;
 
   public:
     ExtractorCallbacks() = delete;
     ExtractorCallbacks(const ExtractorCallbacks &) = delete;
-    explicit ExtractorCallbacks(ExtractionContainers &extraction_containers,
-                                std::unordered_map<std::string, NodeID> &string_map);
+    explicit ExtractorCallbacks(ExtractionContainers &extraction_containers);
 
     // warning: caller needs to take care of synchronization!
     void ProcessNode(const osmium::Node &current_node, const ExtractionNode &result_node);

extractor/restriction_parser.cpp

@@ -27,7 +27,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "restriction_parser.hpp"
 #include "extraction_way.hpp"
-#include "scripting_environment.hpp"
 
 #include "../data_structures/external_memory_node.hpp"
 #include "../util/lua_util.hpp"
@@ -53,7 +52,7 @@ int lua_error_callback(lua_State *lua_state)
 }
 
 RestrictionParser::RestrictionParser(lua_State *lua_state)
-    : /*lua_state(scripting_environment.getLuaState()),*/ use_turn_restrictions(true)
+    : use_turn_restrictions(true)
 {
     ReadUseRestrictionsSetting(lua_state);
 
@@ -103,6 +102,13 @@ void RestrictionParser::ReadRestrictionExceptions(lua_State *lua_state)
     }
 }
 
+/**
+ * Tries to parse an relation as turn restriction. This can fail for a number of
+ * reasons, this the return type is a mapbox::util::optional<>.
+ *
+ * Some restrictions can also be ignored: See the ```get_exceptions``` function
+ * in the corresponding profile.
+ */
 mapbox::util::optional<InputRestrictionContainer>
 RestrictionParser::TryParse(const osmium::Relation &relation) const
 {

extractor/restriction_parser.hpp

@@ -41,10 +41,27 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 struct lua_State;
 class ScriptingEnvironment;
 
+/**
+ * Parses the relations that represents turn restrictions.
+ *
+ * Currently only restrictions where the via objects is a node are supported.
+ *  from   via   to
+ * ------->(x)-------->
+ *
+ * While this class does not directly invoke any lua code _per relation_ it does
+ * load configuration values from the profile, that are saved in variables.
+ * Namely ```use_turn_restrictions``` and ```get_exceptions```.
+ *
+ * The restriction is represented by the osm id of the from way, the osm id of the
+ * to way and the osm id of the via node. This representation must be post-processed
+ * in the extractor to work with the edge-based data-model of OSRM:
+ * Since the from and to way share the via-node a turn will have the following form:
+ * ...----(a)-----(via)------(b)----...
+ * So it can be represented by the tripe (a, via, b).
+ */
 class RestrictionParser
 {
   public:
-    // RestrictionParser(ScriptingEnvironment &scripting_environment);
     RestrictionParser(lua_State *lua_state);
     mapbox::util::optional<InputRestrictionContainer>
     TryParse(const osmium::Relation &relation) const;
@@ -54,7 +71,6 @@ class RestrictionParser
     void ReadRestrictionExceptions(lua_State *lua_state);
     bool ShouldIgnoreRestriction(const std::string &except_tag_string) const;
 
-    // lua_State *lua_state;
     std::vector<std::string> restriction_exceptions;
     bool use_turn_restrictions;
 };

extractor/scripting_environment.hpp

@@ -35,6 +35,13 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 struct lua_State;
 
+/**
+ * Creates a lua context and binds osmium way, node and relation objects and
+ * ExtractionWay and ExtractionNode to lua objects.
+ *
+ * Each thread has its own lua state which is implemented with thread specific
+ * storage from TBB.
+ */
 class ScriptingEnvironment
 {
   public: