progress updates for GTFS parser (#27)

base/core/include/motis/core/common/projection.h

@@ -0,0 +1,26 @@
+#pragma once
+
+namespace motis {
+
+struct projection {
+  projection(float const low, float const high,
+             float const scale_factor = 100.0,  // percent
+             float const input_low = 0.0, float const input_high = 1.0)
+      : low_{low},
+        high_{high},
+        scale_factor_{scale_factor},
+        input_low_{input_low},
+        input_high_{input_high} {}
+
+  int operator()(float const in) const {
+    auto const normalized = (in - input_low_) / (input_high_ - input_low_);
+    return static_cast<int>(scale_factor_ *
+                            (low_ + normalized * (high_ - low_)));
+  }
+
+  float low_, high_;
+  float scale_factor_;
+  float input_low_, input_high_;
+};
+
+}  // namespace motis

base/loader/src/gtfs/calendar_date.cc

@@ -1,7 +1,10 @@
 #include "motis/loader/gtfs/calendar_date.h"
 
+#include "utl/enumerate.h"
 #include "utl/parser/csv.h"
 
+#include "motis/core/common/logging.h"
+#include "motis/core/common/projection.h"
 #include "motis/loader/util.h"
 
 using namespace utl;
@@ -27,9 +30,14 @@ calendar_date read_date(gtfs_calendar_date const& gtfs_date) {
 
 std::map<std::string, std::vector<calendar_date>> read_calendar_date(
     loaded_file f) {
+  motis::logging::scoped_timer timer{"calendar dates"};
+  std::clog << '\0' << 'S' << "Parse Calendar Dates" << '\0';
+  auto const p = projection{0.0, 0.05};
+
   std::map<std::string, std::vector<calendar_date>> services;
-  for (auto const& d :
-       read<gtfs_calendar_date>(f.content(), calendar_columns)) {
+  auto const entries = read<gtfs_calendar_date>(f.content(), calendar_columns);
+  for (auto const& [i, d] : utl::enumerate(entries)) {
+    std::clog << '\0' << p(i / static_cast<float>(entries.size())) << '\0';
     services[get<service_id>(d).to_str()].push_back(read_date(d));
   }
   return services;

base/loader/src/gtfs/feed_info.cc

@@ -25,7 +25,7 @@ feed_map read_feed_publisher(loaded_file file) {
                   feed{get<feed_publisher_name>(f).to_str(),
                        get<feed_version>(f).to_str()});
   }
-
   return feeds;
 }
+
 }  // namespace motis::loader::gtfs

base/loader/src/gtfs/gtfs_parser.cc

@@ -17,6 +17,7 @@
 
 #include "motis/core/common/date_time_util.h"
 #include "motis/core/common/logging.h"
+#include "motis/core/common/projection.h"
 #include "motis/loader/gtfs/agency.h"
 #include "motis/loader/gtfs/calendar.h"
 #include "motis/loader/gtfs/calendar_date.h"
@@ -98,6 +99,8 @@ std::time_t to_unix_time(boost::gregorian::date const& date) {
 }
 
 void gtfs_parser::parse(fs::path const& root, FlatBufferBuilder& fbb) {
+  motis::logging::scoped_timer global_timer{"gtfs parser"};
+
   auto const load = [&](char const* file) {
     return fs::is_regular_file(root / file) ? loaded_file{root / file}
                                             : loaded_file{};
@@ -157,12 +160,18 @@ void gtfs_parser::parse(fs::path const& root, FlatBufferBuilder& fbb) {
                               [&]() { return fbb.CreateString(s); });
   };
 
+  motis::logging::scoped_timer export_timer{"export"};
+  auto const p = projection{0.4, 0.8};
+  std::clog << '\0' << 'S' << "Export schedule.raw" << '\0';
   auto const interval =
       Interval{static_cast<uint64_t>(to_unix_time(services.first_day_)),
                static_cast<uint64_t>(to_unix_time(services.last_day_))};
+  auto i = 0.0;
   auto const output_services = fbb.CreateVector(utl::to_vec(
       begin(trips), end(trips),
       [&](std::pair<std::string const, std::unique_ptr<trip>> const& entry) {
+        std::clog << '\0' << p((i += 1.) / trips.size()) << '\0';
+
         auto const& t = entry.second;
         auto const stop_seq = t->stops();
         return CreateService(

base/loader/src/gtfs/route.cc

@@ -5,6 +5,7 @@
 
 #include "utl/parser/csv.h"
 
+#include "motis/core/common/logging.h"
 #include "motis/loader/util.h"
 
 using namespace utl;
@@ -165,6 +166,8 @@ static const column_mapping<gtfs_route> columns = {
      "route_desc", "route_type"}};
 
 route_map read_routes(loaded_file file, agency_map const& agencies) {
+  motis::logging::scoped_timer timer{"read routes"};
+
   route_map routes;
   for (auto const& r : read<gtfs_route>(file.content(), columns)) {
     auto agency_it = agencies.find(get<agency_id>(r).to_str());

base/loader/src/gtfs/services.cc

@@ -2,6 +2,8 @@
 
 #include "boost/date_time/gregorian/gregorian.hpp"
 
+#include "motis/core/common/logging.h"
+
 namespace greg = boost::gregorian;
 
 namespace motis::loader::gtfs {
@@ -55,6 +57,8 @@ void add_exception(greg::date const& start, calendar_date const& exception,
 services traffic_days(
     std::map<std::string, calendar> const& base,
     std::map<std::string, std::vector<calendar_date>> const& exceptions) {
+  motis::logging::scoped_timer timer{"traffic days"};
+
   services s;
   s.first_day_ = bound_date(base, true);
   s.last_day_ = bound_date(base, false);

base/loader/src/gtfs/stop.cc

@@ -6,6 +6,7 @@
 
 #include "utl/parser/csv.h"
 
+#include "motis/core/common/logging.h"
 #include "motis/loader/gtfs/common.h"
 #include "motis/loader/util.h"
 
@@ -21,6 +22,8 @@ static const column_mapping<gtfs_stop> columns = {
     {"stop_id", "stop_name", "stop_timezone", "stop_lat", "stop_lon"}};
 
 stop_map read_stops(loaded_file file) {
+  motis::logging::scoped_timer timer{"read stops"};
+
   stop_map stops;
   for (auto const& s : read<gtfs_stop>(file.content(), columns)) {
     // load the swiss dataset without track information

base/loader/src/gtfs/stop_time.cc

@@ -3,9 +3,12 @@
 #include <algorithm>
 #include <tuple>
 
+#include "utl/enumerate.h"
 #include "utl/parser/arg_parser.h"
 #include "utl/parser/csv.h"
 
+#include "motis/core/common/logging.h"
+#include "motis/core/common/projection.h"
 #include "motis/loader/gtfs/common.h"
 #include "motis/loader/util.h"
 
@@ -51,11 +54,17 @@ int hhmm_to_min(cstr s) {
 
 void read_stop_times(loaded_file const& file, trip_map& trips,
                      stop_map const& stops) {
+  motis::logging::scoped_timer timer{"read stop times"};
+  std::clog << '\0' << 'S' << "Parse Stop Times" << '\0';
+
   std::string last_trip_id;
   trip* last_trip = nullptr;
 
-  for (auto const& s :
-       read<gtfs_stop_time>(file.content(), stop_time_columns)) {
+  auto const p = projection{0.2, 0.4};
+  auto const entries = read<gtfs_stop_time>(file.content(), stop_time_columns);
+  for (auto const& [i, s] : utl::enumerate(entries)) {
+    std::clog << '\0' << p(i / static_cast<float>(entries.size())) << '\0';
+
     trip* t = nullptr;
     auto t_id = get<trip_id>(s).to_str();
     if (last_trip != nullptr && t_id == last_trip_id) {

base/loader/src/gtfs/transfers.cc

@@ -5,6 +5,7 @@
 
 #include "utl/parser/csv.h"
 
+#include "motis/core/common/logging.h"
 #include "motis/loader/gtfs/common.h"
 #include "motis/loader/util.h"
 
@@ -22,6 +23,8 @@ static const column_mapping<gtfs_transfer> columns = {
 
 std::map<stop_pair, transfer> read_transfers(loaded_file f,
                                              stop_map const& stops) {
+  motis::logging::scoped_timer timer{"read transfers"};
+
   std::map<stop_pair, transfer> transfers;
   for (auto const& t : read<gtfs_transfer>(f.content(), columns)) {
     if (parse_stop_id(get<from_stop_id>(t).to_str()) !=

base/loader/src/gtfs/trip.cc

@@ -3,8 +3,11 @@
 #include <algorithm>
 #include <tuple>
 
+#include "utl/enumerate.h"
 #include "utl/parser/csv.h"
 
+#include "motis/core/common/logging.h"
+#include "motis/core/common/projection.h"
 #include "motis/loader/util.h"
 
 using namespace utl;
@@ -19,9 +22,14 @@ static const column_mapping<gtfs_trip> columns = {
 
 trip_map read_trips(loaded_file file, route_map const& routes,
                     services const& services) {
+  motis::logging::scoped_timer timer{"read trips"};
+
   trip_map trips;
   auto line = 1U;
-  for (auto const& t : read<gtfs_trip>(file.content(), columns)) {
+  auto const p = projection{0.05, 0.2};
+  auto const entries = read<gtfs_trip>(file.content(), columns);
+  for (auto const& [i, t] : utl::enumerate(entries)) {
+    std::clog << '\0' << p(i / static_cast<float>(entries.size())) << '\0';
     trips.emplace(
         get<trip_id>(t).to_str(),
         std::make_unique<trip>(