From d8d9ac8686f8753953c9a9fe76caa4ce7ca13074 Mon Sep 17 00:00:00 2001
From: "Tomonobu.Saito (Desktop PC)" <Tomonobu.Saito@sony.com>
Date: Wed, 2 Oct 2019 15:47:16 +0900
Subject: [PATCH 01/16] Add assert to check memory allocation

---
 include/extractor/raster_source.hpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/include/extractor/raster_source.hpp b/include/extractor/raster_source.hpp
index 57a082d0e40..dd924abc0cb 100644
--- a/include/extractor/raster_source.hpp
+++ b/include/extractor/raster_source.hpp
@@ -43,6 +43,7 @@ class RasterGrid
         xdim = _xdim;
         ydim = _ydim;
         _data.reserve(ydim * xdim);
+        BOOST_ASSERT(_data.capacity() >= ydim * xdim);
 
         storage::io::FileReader file_reader(filepath, storage::io::FileReader::HasNoFingerprint);
 

From 62c8b70f78c215bb2ef24a7a87de35b5cf596cf4 Mon Sep 17 00:00:00 2001
From: Tomonobu Saito <Tomnobu.Saito@gmail.com>
Date: Wed, 2 Oct 2019 19:04:01 +0900
Subject: [PATCH 02/16] use boost::filesystem::file_size() to get the file size
 (instead of seeking the file).

---
 include/storage/io.hpp | 11 +++++++++++
 1 file changed, 11 insertions(+)
 mode change 100644 => 100755 include/storage/io.hpp

diff --git a/include/storage/io.hpp b/include/storage/io.hpp
old mode 100644
new mode 100755
index 7702c0ddb77..bee8c966b7c
--- a/include/storage/io.hpp
+++ b/include/storage/io.hpp
@@ -10,6 +10,7 @@
 #include "util/log.hpp"
 #include "util/version.hpp"
 
+#include <boost/filesystem.hpp>
 #include <boost/filesystem/fstream.hpp>
 #include <boost/iostreams/device/array.hpp>
 #include <boost/iostreams/seek.hpp>
@@ -60,6 +61,15 @@ class FileReader
 
     std::size_t GetSize()
     {
+        const boost::filesystem::path path(filepath);
+        try {
+            return std::size_t(boost::filesystem::file_size(path)) - ((fingerprint == FingerprintFlag::VerifyFingerprint) ? sizeof(util::FingerPrint) : 0);
+        }
+        catch (boost::filesystem::filesystem_error& ex) {
+            std::cout << ex.what() << std::endl;
+            throw;
+        }
+/*
         const boost::filesystem::ifstream::pos_type position = input_stream.tellg();
         input_stream.seekg(0, std::ios::end);
         const boost::filesystem::ifstream::pos_type file_size = input_stream.tellg();
@@ -84,6 +94,7 @@ class FileReader
         {
             return file_size;
         }
+ */
     }
 
     /* Read count objects of type T into pointer dest */

From d316ff9d41a82abe553cbde06d2d3dbc7b889678 Mon Sep 17 00:00:00 2001
From: Tomonobu Saito <Tomnobu.Saito@gmail.com>
Date: Thu, 3 Oct 2019 16:44:27 +0900
Subject: [PATCH 03/16] Improvement of raster source loading to reduce consumed
 memory size.

---
 include/extractor/raster_source.hpp | 35 ++++++++++++++++++++++++++---
 include/storage/io.hpp              | 14 +++++++++++-
 src/extractor/raster_source.cpp     | 20 +++++++++++++++--
 3 files changed, 63 insertions(+), 6 deletions(-)
 mode change 100644 => 100755 include/extractor/raster_source.hpp
 mode change 100644 => 100755 src/extractor/raster_source.cpp

diff --git a/include/extractor/raster_source.hpp b/include/extractor/raster_source.hpp
old mode 100644
new mode 100755
index dd924abc0cb..4f7c3171316
--- a/include/extractor/raster_source.hpp
+++ b/include/extractor/raster_source.hpp
@@ -5,15 +5,22 @@
 #include "util/exception.hpp"
 
 #include <boost/algorithm/string/trim.hpp>
+#include <boost/algorithm/string.hpp>
 #include <boost/assert.hpp>
 #include <boost/filesystem.hpp>
 #include <boost/filesystem/fstream.hpp>
 #include <boost/spirit/include/qi.hpp>
 #include <boost/spirit/include/qi_int.hpp>
+#include <boost/foreach.hpp>
+
 #include <storage/io.hpp>
 
 #include <iterator>
 #include <unordered_map>
+#include <string>
+#include <list>
+#include <iostream>
+using namespace std;
 
 namespace osrm
 {
@@ -45,8 +52,29 @@ class RasterGrid
         _data.reserve(ydim * xdim);
         BOOST_ASSERT(_data.capacity() >= ydim * xdim);
 
+        // Construct FileReader
         storage::io::FileReader file_reader(filepath, storage::io::FileReader::HasNoFingerprint);
-
+        std::string buf;
+        buf.resize(xdim * 11); // INT32_MAX = 2147483647 = 10 chars + 1 white space = 11
+        BOOST_ASSERT(buf.size() >= xdim * 11);
+
+        for (unsigned int y = 0 ; y < ydim ; y++) {
+            // read one line from file.
+            file_reader.ReadLine(&buf[0], xdim * 11);
+            boost::algorithm::trim(buf);
+
+            std::vector<std::string> result;
+            std::string delim (" ");
+            //boost::split(result, buf, boost::is_any_of(delim), boost::algorithm::token_compress_on);
+            boost::split(result, buf, boost::is_any_of(delim));
+            unsigned int x = 0;
+            BOOST_FOREACH(std::string s, result) {
+                _data[(y * xdim) + x] = atoi(s.c_str());
+                ++x;
+            }
+            BOOST_ASSERT(x == xdim);
+        }
+/*
         std::string buffer;
         buffer.resize(file_reader.GetSize());
 
@@ -76,6 +104,7 @@ class RasterGrid
             throw util::exception("Failed to parse raster source: " + filepath.string() +
                                   SOURCE_REF);
         }
+*/
     }
 
     RasterGrid(const RasterGrid &) = default;
@@ -144,8 +173,8 @@ class RasterContainer
     RasterDatum GetRasterInterpolateFromSource(unsigned int source_id, double lon, double lat);
 
   private:
-    std::vector<RasterSource> LoadedSources;
-    std::unordered_map<std::string, int> LoadedSourcePaths;
+    static std::vector<RasterSource> LoadedSources;
+    static std::unordered_map<std::string, int> LoadedSourcePaths;
 };
 }
 }
diff --git a/include/storage/io.hpp b/include/storage/io.hpp
index bee8c966b7c..fcb4dedd7f6 100755
--- a/include/storage/io.hpp
+++ b/include/storage/io.hpp
@@ -69,7 +69,10 @@ class FileReader
             std::cout << ex.what() << std::endl;
             throw;
         }
+    }
 /*
+    std::size_t GetSize()
+    {
         const boost::filesystem::ifstream::pos_type position = input_stream.tellg();
         input_stream.seekg(0, std::ios::end);
         const boost::filesystem::ifstream::pos_type file_size = input_stream.tellg();
@@ -94,7 +97,16 @@ class FileReader
         {
             return file_size;
         }
- */
+    }
+*/
+    /* Read one line */
+    template <typename T> void ReadLine(T *dest, const std::size_t count) {
+        if (0 < count) {
+            const auto &ios = input_stream.getline(reinterpret_cast<char *>(dest), count * sizeof(T));
+            for (std::size_t n = ios.gcount(); n < count; ++n) {
+                reinterpret_cast<char *>(dest)[n] = '\0';
+            }
+        }
     }
 
     /* Read count objects of type T into pointer dest */
diff --git a/src/extractor/raster_source.cpp b/src/extractor/raster_source.cpp
old mode 100644
new mode 100755
index 012f347a8ff..0788ac71cce
--- a/src/extractor/raster_source.cpp
+++ b/src/extractor/raster_source.cpp
@@ -78,6 +78,10 @@ RasterDatum RasterSource::GetRasterInterpolate(const int lon, const int lat) con
                                       raster_data(right, bottom) * (fromLeft * fromTop))};
 }
 
+// static member of Raster Container
+std::vector<RasterSource> RasterContainer::LoadedSources;
+std::unordered_map<std::string, int> RasterContainer::LoadedSourcePaths;
+
 // Load raster source into memory
 int RasterContainer::LoadRasterSource(const std::string &path_string,
                                       double xmin,
@@ -91,7 +95,13 @@ int RasterContainer::LoadRasterSource(const std::string &path_string,
     const auto _xmax = static_cast<std::int32_t>(util::toFixed(util::FloatLongitude{xmax}));
     const auto _ymin = static_cast<std::int32_t>(util::toFixed(util::FloatLatitude{ymin}));
     const auto _ymax = static_cast<std::int32_t>(util::toFixed(util::FloatLatitude{ymax}));
-
+/*
+    // for debug : list up all keys and values
+    util::Log() << "Num of Raster Sources : " << LoadedSourcePaths.size();
+    for (auto i = LoadedSourcePaths.begin(); i != LoadedSourcePaths.end(); ++i) {
+        util::Log() << "Key : " << i->first << " Value: " << i->second;
+    }
+*/
     const auto itr = LoadedSourcePaths.find(path_string);
     if (itr != LoadedSourcePaths.end())
     {
@@ -120,7 +130,13 @@ int RasterContainer::LoadRasterSource(const std::string &path_string,
     LoadedSources.push_back(std::move(source));
 
     util::Log() << "[source loader] ok, after " << TIMER_SEC(loading_source) << "s";
-
+/*
+    // for debug : list up all keys and values
+    util::Log() << "Num of Raster Sources : " << LoadedSourcePaths.size();
+    for (auto i = LoadedSourcePaths.begin(); i != LoadedSourcePaths.end(); ++i) {
+        util::Log() << "Key : " << i->first << " Value: " << i->second;
+    }
+*/
     return source_id;
 }
 

From e4aaf07879a6edd0d9f60d1413f14851d6a49ccf Mon Sep 17 00:00:00 2001
From: Tomonobu Saito <Tomnobu.Saito@gmail.com>
Date: Thu, 3 Oct 2019 17:24:42 +0900
Subject: [PATCH 04/16] remove unused lines

---
 include/extractor/raster_source.hpp | 35 ++---------------------------
 include/storage/io.hpp              | 28 -----------------------
 src/extractor/raster_source.cpp     | 16 ++-----------
 3 files changed, 4 insertions(+), 75 deletions(-)

diff --git a/include/extractor/raster_source.hpp b/include/extractor/raster_source.hpp
index 4f7c3171316..4f39314f30e 100755
--- a/include/extractor/raster_source.hpp
+++ b/include/extractor/raster_source.hpp
@@ -50,13 +50,13 @@ class RasterGrid
         xdim = _xdim;
         ydim = _ydim;
         _data.reserve(ydim * xdim);
-        BOOST_ASSERT(_data.capacity() >= ydim * xdim);
+        BOOST_ASSERT(ydim * xdim <= _data.capacity());
 
         // Construct FileReader
         storage::io::FileReader file_reader(filepath, storage::io::FileReader::HasNoFingerprint);
         std::string buf;
         buf.resize(xdim * 11); // INT32_MAX = 2147483647 = 10 chars + 1 white space = 11
-        BOOST_ASSERT(buf.size() >= xdim * 11);
+        BOOST_ASSERT(xdim * 11 <= buf.size());
 
         for (unsigned int y = 0 ; y < ydim ; y++) {
             // read one line from file.
@@ -74,37 +74,6 @@ class RasterGrid
             }
             BOOST_ASSERT(x == xdim);
         }
-/*
-        std::string buffer;
-        buffer.resize(file_reader.GetSize());
-
-        BOOST_ASSERT(buffer.size() > 1);
-
-        file_reader.ReadInto(&buffer[0], buffer.size());
-
-        boost::algorithm::trim(buffer);
-
-        auto itr = buffer.begin();
-        auto end = buffer.end();
-
-        bool r = false;
-        try
-        {
-            r = boost::spirit::qi::parse(
-                itr, end, +boost::spirit::qi::int_ % +boost::spirit::qi::space, _data);
-        }
-        catch (std::exception const &ex)
-        {
-            throw util::exception("Failed to read from raster source " + filepath.string() + ": " +
-                                  ex.what() + SOURCE_REF);
-        }
-
-        if (!r || itr != end)
-        {
-            throw util::exception("Failed to parse raster source: " + filepath.string() +
-                                  SOURCE_REF);
-        }
-*/
     }
 
     RasterGrid(const RasterGrid &) = default;
diff --git a/include/storage/io.hpp b/include/storage/io.hpp
index fcb4dedd7f6..0a6eb0775ba 100755
--- a/include/storage/io.hpp
+++ b/include/storage/io.hpp
@@ -70,35 +70,7 @@ class FileReader
             throw;
         }
     }
-/*
-    std::size_t GetSize()
-    {
-        const boost::filesystem::ifstream::pos_type position = input_stream.tellg();
-        input_stream.seekg(0, std::ios::end);
-        const boost::filesystem::ifstream::pos_type file_size = input_stream.tellg();
-
-        if (file_size == boost::filesystem::ifstream::pos_type(-1))
-        {
-            throw util::RuntimeError("Unable to determine file size for " +
-                                         std::string(filepath.string()),
-                                     ErrorCode::FileIOError,
-                                     SOURCE_REF,
-                                     std::strerror(errno));
-        }
-
-        // restore the current position
-        input_stream.seekg(position, std::ios::beg);
 
-        if (fingerprint == FingerprintFlag::VerifyFingerprint)
-        {
-            return std::size_t(file_size) - sizeof(util::FingerPrint);
-        }
-        else
-        {
-            return file_size;
-        }
-    }
-*/
     /* Read one line */
     template <typename T> void ReadLine(T *dest, const std::size_t count) {
         if (0 < count) {
diff --git a/src/extractor/raster_source.cpp b/src/extractor/raster_source.cpp
index 0788ac71cce..befae77b76f 100755
--- a/src/extractor/raster_source.cpp
+++ b/src/extractor/raster_source.cpp
@@ -95,13 +95,7 @@ int RasterContainer::LoadRasterSource(const std::string &path_string,
     const auto _xmax = static_cast<std::int32_t>(util::toFixed(util::FloatLongitude{xmax}));
     const auto _ymin = static_cast<std::int32_t>(util::toFixed(util::FloatLatitude{ymin}));
     const auto _ymax = static_cast<std::int32_t>(util::toFixed(util::FloatLatitude{ymax}));
-/*
-    // for debug : list up all keys and values
-    util::Log() << "Num of Raster Sources : " << LoadedSourcePaths.size();
-    for (auto i = LoadedSourcePaths.begin(); i != LoadedSourcePaths.end(); ++i) {
-        util::Log() << "Key : " << i->first << " Value: " << i->second;
-    }
-*/
+
     const auto itr = LoadedSourcePaths.find(path_string);
     if (itr != LoadedSourcePaths.end())
     {
@@ -130,13 +124,7 @@ int RasterContainer::LoadRasterSource(const std::string &path_string,
     LoadedSources.push_back(std::move(source));
 
     util::Log() << "[source loader] ok, after " << TIMER_SEC(loading_source) << "s";
-/*
-    // for debug : list up all keys and values
-    util::Log() << "Num of Raster Sources : " << LoadedSourcePaths.size();
-    for (auto i = LoadedSourcePaths.begin(); i != LoadedSourcePaths.end(); ++i) {
-        util::Log() << "Key : " << i->first << " Value: " << i->second;
-    }
-*/
+
     return source_id;
 }
 

From 432d49e23d5343d0f7f157076c04f64656386d67 Mon Sep 17 00:00:00 2001
From: Tomonobu Saito <Tomnobu.Saito@gmail.com>
Date: Fri, 4 Oct 2019 13:50:13 +0900
Subject: [PATCH 05/16] bugfix: support multiple delimitor.

---
 include/extractor/raster_source.hpp | 6 ++++--
 src/extractor/raster_source.cpp     | 8 +++++++-
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/include/extractor/raster_source.hpp b/include/extractor/raster_source.hpp
index 4f39314f30e..c1c8d138dc9 100755
--- a/include/extractor/raster_source.hpp
+++ b/include/extractor/raster_source.hpp
@@ -47,6 +47,8 @@ class RasterGrid
   public:
     RasterGrid(const boost::filesystem::path &filepath, std::size_t _xdim, std::size_t _ydim)
     {
+        util::Log() << " filepath : " << filepath;
+
         xdim = _xdim;
         ydim = _ydim;
         _data.reserve(ydim * xdim);
@@ -65,8 +67,8 @@ class RasterGrid
 
             std::vector<std::string> result;
             std::string delim (" ");
-            //boost::split(result, buf, boost::is_any_of(delim), boost::algorithm::token_compress_on);
-            boost::split(result, buf, boost::is_any_of(delim));
+            boost::split(result, buf, boost::is_any_of(delim), boost::algorithm::token_compress_on);
+            //boost::split(result, buf, boost::is_any_of(delim));
             unsigned int x = 0;
             BOOST_FOREACH(std::string s, result) {
                 _data[(y * xdim) + x] = atoi(s.c_str());
diff --git a/src/extractor/raster_source.cpp b/src/extractor/raster_source.cpp
index befae77b76f..88291983aec 100755
--- a/src/extractor/raster_source.cpp
+++ b/src/extractor/raster_source.cpp
@@ -124,7 +124,13 @@ int RasterContainer::LoadRasterSource(const std::string &path_string,
     LoadedSources.push_back(std::move(source));
 
     util::Log() << "[source loader] ok, after " << TIMER_SEC(loading_source) << "s";
-
+/*
+    // for debug : list up all keys and values
+    util::Log() << "Num of Raster Sources : " << LoadedSourcePaths.size();
+    for (auto i = LoadedSourcePaths.begin(); i != LoadedSourcePaths.end(); ++i) {
+        util::Log() << "Key : " << i->first << " Value: " << i->second;
+    }
+*/
     return source_id;
 }
 

From a9fce74e6345d74dd0c27376ff06611d34b7f35d Mon Sep 17 00:00:00 2001
From: Tomonobu Saito <Tomnobu.Saito@gmail.com>
Date: Fri, 4 Oct 2019 14:02:57 +0900
Subject: [PATCH 06/16] remove unused code (debug code)

---
 src/extractor/raster_source.cpp | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/src/extractor/raster_source.cpp b/src/extractor/raster_source.cpp
index 88291983aec..befae77b76f 100755
--- a/src/extractor/raster_source.cpp
+++ b/src/extractor/raster_source.cpp
@@ -124,13 +124,7 @@ int RasterContainer::LoadRasterSource(const std::string &path_string,
     LoadedSources.push_back(std::move(source));
 
     util::Log() << "[source loader] ok, after " << TIMER_SEC(loading_source) << "s";
-/*
-    // for debug : list up all keys and values
-    util::Log() << "Num of Raster Sources : " << LoadedSourcePaths.size();
-    for (auto i = LoadedSourcePaths.begin(); i != LoadedSourcePaths.end(); ++i) {
-        util::Log() << "Key : " << i->first << " Value: " << i->second;
-    }
-*/
+
     return source_id;
 }
 

From eef072234ecefae509f72c136fc6ed3529be3376 Mon Sep 17 00:00:00 2001
From: Tomonobu Saito <Tomnobu.Saito@gmail.com>
Date: Fri, 4 Oct 2019 15:14:20 +0900
Subject: [PATCH 07/16] update delim chars

---
 include/extractor/raster_source.hpp | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/include/extractor/raster_source.hpp b/include/extractor/raster_source.hpp
index c1c8d138dc9..2ed08345be1 100755
--- a/include/extractor/raster_source.hpp
+++ b/include/extractor/raster_source.hpp
@@ -47,8 +47,6 @@ class RasterGrid
   public:
     RasterGrid(const boost::filesystem::path &filepath, std::size_t _xdim, std::size_t _ydim)
     {
-        util::Log() << " filepath : " << filepath;
-
         xdim = _xdim;
         ydim = _ydim;
         _data.reserve(ydim * xdim);
@@ -66,7 +64,7 @@ class RasterGrid
             boost::algorithm::trim(buf);
 
             std::vector<std::string> result;
-            std::string delim (" ");
+            std::string delim (" \r\n\0");
             boost::split(result, buf, boost::is_any_of(delim), boost::algorithm::token_compress_on);
             //boost::split(result, buf, boost::is_any_of(delim));
             unsigned int x = 0;

From f9ee74d78e599143750ae365b5f268ff07caeeaa Mon Sep 17 00:00:00 2001
From: Tomonobu Saito <Tomnobu.Saito@gmail.com>
Date: Fri, 4 Oct 2019 16:47:31 +0900
Subject: [PATCH 08/16] Add x range check to avoid data corruption.

---
 include/extractor/raster_source.hpp | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/include/extractor/raster_source.hpp b/include/extractor/raster_source.hpp
index 2ed08345be1..633c0c66140 100755
--- a/include/extractor/raster_source.hpp
+++ b/include/extractor/raster_source.hpp
@@ -64,12 +64,11 @@ class RasterGrid
             boost::algorithm::trim(buf);
 
             std::vector<std::string> result;
-            std::string delim (" \r\n\0");
-            boost::split(result, buf, boost::is_any_of(delim), boost::algorithm::token_compress_on);
-            //boost::split(result, buf, boost::is_any_of(delim));
+            boost::split(result, buf, boost::is_any_of(" \r\n\0"), boost::algorithm::token_compress_on);
+            //boost::split(result, buf, boost::is_any_of(" \r\n\0"));
             unsigned int x = 0;
             BOOST_FOREACH(std::string s, result) {
-                _data[(y * xdim) + x] = atoi(s.c_str());
+                if (x < xdim) _data[(y * xdim) + x] = atoi(s.c_str());
                 ++x;
             }
             BOOST_ASSERT(x == xdim);

From a587b140068de141b808a4b3b5aa77c0e3cc0bb5 Mon Sep 17 00:00:00 2001
From: Tomonobu Saito <Tomnobu.Saito@gmail.com>
Date: Fri, 4 Oct 2019 17:30:35 +0900
Subject: [PATCH 09/16] destruct static vector and unorderd_map when last
 RasterContainer is destructed.

---
 include/extractor/raster_source.hpp | 10 +++++++++-
 src/extractor/raster_source.cpp     |  1 +
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/include/extractor/raster_source.hpp b/include/extractor/raster_source.hpp
index 633c0c66140..f85144a5558 100755
--- a/include/extractor/raster_source.hpp
+++ b/include/extractor/raster_source.hpp
@@ -126,7 +126,14 @@ class RasterSource
 class RasterContainer
 {
   public:
-    RasterContainer() = default;
+    RasterContainer() { ++count; }
+    ~RasterContainer() {
+        --count;
+        if (0 == count) {
+            LoadedSources.clear();
+            LoadedSourcePaths.clear();
+        }
+    }
 
     int LoadRasterSource(const std::string &path_string,
                          double xmin,
@@ -143,6 +150,7 @@ class RasterContainer
   private:
     static std::vector<RasterSource> LoadedSources;
     static std::unordered_map<std::string, int> LoadedSourcePaths;
+    static int count;
 };
 }
 }
diff --git a/src/extractor/raster_source.cpp b/src/extractor/raster_source.cpp
index befae77b76f..8c62cc8b2d0 100755
--- a/src/extractor/raster_source.cpp
+++ b/src/extractor/raster_source.cpp
@@ -81,6 +81,7 @@ RasterDatum RasterSource::GetRasterInterpolate(const int lon, const int lat) con
 // static member of Raster Container
 std::vector<RasterSource> RasterContainer::LoadedSources;
 std::unordered_map<std::string, int> RasterContainer::LoadedSourcePaths;
+int RasterContainer::count = 0;
 
 // Load raster source into memory
 int RasterContainer::LoadRasterSource(const std::string &path_string,

From f36707d1fbbf997f7ea0e4254e1c56a0167d24d1 Mon Sep 17 00:00:00 2001
From: Tomonobu Saito <Tomnobu.Saito@gmail.com>
Date: Fri, 4 Oct 2019 17:39:05 +0900
Subject: [PATCH 10/16] revert file mode to 664

---
 include/extractor/raster_source.hpp | 0
 include/storage/io.hpp              | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 mode change 100755 => 100644 include/extractor/raster_source.hpp
 mode change 100755 => 100644 include/storage/io.hpp

diff --git a/include/extractor/raster_source.hpp b/include/extractor/raster_source.hpp
old mode 100755
new mode 100644
diff --git a/include/storage/io.hpp b/include/storage/io.hpp
old mode 100755
new mode 100644

From 542c3ba87261e4939c6842e2c912bf5a747a9be9 Mon Sep 17 00:00:00 2001
From: Tomonobu Saito <Tomnobu.Saito@gmail.com>
Date: Wed, 9 Oct 2019 13:02:59 +0900
Subject: [PATCH 11/16] Add singletone class RasterCache to handle global cache
 data.

---
 include/extractor/raster_source.hpp | 36 ++++++++++++++++++++---------
 src/extractor/raster_source.cpp     | 29 +++++++++++------------
 2 files changed, 38 insertions(+), 27 deletions(-)
 mode change 100644 => 100755 include/extractor/raster_source.hpp

diff --git a/include/extractor/raster_source.hpp b/include/extractor/raster_source.hpp
old mode 100644
new mode 100755
index f85144a5558..48d69d4c484
--- a/include/extractor/raster_source.hpp
+++ b/include/extractor/raster_source.hpp
@@ -126,14 +126,7 @@ class RasterSource
 class RasterContainer
 {
   public:
-    RasterContainer() { ++count; }
-    ~RasterContainer() {
-        --count;
-        if (0 == count) {
-            LoadedSources.clear();
-            LoadedSourcePaths.clear();
-        }
-    }
+    RasterContainer() = default;
 
     int LoadRasterSource(const std::string &path_string,
                          double xmin,
@@ -148,9 +141,30 @@ class RasterContainer
     RasterDatum GetRasterInterpolateFromSource(unsigned int source_id, double lon, double lat);
 
   private:
-    static std::vector<RasterSource> LoadedSources;
-    static std::unordered_map<std::string, int> LoadedSourcePaths;
-    static int count;
+};
+
+// << singletone >> RasterCache
+class RasterCache
+{
+public:
+    // class method to get the instance
+    static RasterCache& getInstance() {
+        if (NULL == g_instance) {
+            g_instance = new RasterCache();
+        }
+        return *g_instance;
+    }
+    // get reference of cache
+    std::vector<RasterSource>& getLoadedSources() { return LoadedSources; }
+    std::unordered_map<std::string, int>& getLoadedSourcePaths() { return LoadedSourcePaths; }
+private:
+    // constructor
+    RasterCache() = default;
+    // member
+    std::vector<RasterSource> LoadedSources;
+    std::unordered_map<std::string, int> LoadedSourcePaths;
+    // the instance
+    static RasterCache *g_instance;
 };
 }
 }
diff --git a/src/extractor/raster_source.cpp b/src/extractor/raster_source.cpp
index 8c62cc8b2d0..5b69ebd5692 100755
--- a/src/extractor/raster_source.cpp
+++ b/src/extractor/raster_source.cpp
@@ -78,11 +78,6 @@ RasterDatum RasterSource::GetRasterInterpolate(const int lon, const int lat) con
                                       raster_data(right, bottom) * (fromLeft * fromTop))};
 }
 
-// static member of Raster Container
-std::vector<RasterSource> RasterContainer::LoadedSources;
-std::unordered_map<std::string, int> RasterContainer::LoadedSourcePaths;
-int RasterContainer::count = 0;
-
 // Load raster source into memory
 int RasterContainer::LoadRasterSource(const std::string &path_string,
                                       double xmin,
@@ -97,15 +92,15 @@ int RasterContainer::LoadRasterSource(const std::string &path_string,
     const auto _ymin = static_cast<std::int32_t>(util::toFixed(util::FloatLatitude{ymin}));
     const auto _ymax = static_cast<std::int32_t>(util::toFixed(util::FloatLatitude{ymax}));
 
-    const auto itr = LoadedSourcePaths.find(path_string);
-    if (itr != LoadedSourcePaths.end())
+    const auto itr = RasterCache::getInstance().getLoadedSourcePaths().find(path_string);
+    if (itr != RasterCache::getInstance().getLoadedSourcePaths().end())
     {
         util::Log() << "[source loader] Already loaded source '" << path_string << "' at source_id "
                     << itr->second;
         return itr->second;
     }
 
-    int source_id = static_cast<int>(LoadedSources.size());
+    int source_id = static_cast<int>(RasterCache::getInstance().getLoadedSources().size());
 
     util::Log() << "[source loader] Loading from " << path_string << "  ... ";
     TIMER_START(loading_source);
@@ -121,8 +116,8 @@ int RasterContainer::LoadRasterSource(const std::string &path_string,
 
     RasterSource source{std::move(rasterData), ncols, nrows, _xmin, _xmax, _ymin, _ymax};
     TIMER_STOP(loading_source);
-    LoadedSourcePaths.emplace(path_string, source_id);
-    LoadedSources.push_back(std::move(source));
+    RasterCache::getInstance().getLoadedSourcePaths().emplace(path_string, source_id);
+    RasterCache::getInstance().getLoadedSources().push_back(std::move(source));
 
     util::Log() << "[source loader] ok, after " << TIMER_SEC(loading_source) << "s";
 
@@ -132,10 +127,10 @@ int RasterContainer::LoadRasterSource(const std::string &path_string,
 // External function for looking up nearest data point from a specified source
 RasterDatum RasterContainer::GetRasterDataFromSource(unsigned int source_id, double lon, double lat)
 {
-    if (LoadedSources.size() < source_id + 1)
+    if (RasterCache::getInstance().getLoadedSources().size() < source_id + 1)
     {
         throw util::exception("Attempted to access source " + std::to_string(source_id) +
-                              ", but there are only " + std::to_string(LoadedSources.size()) +
+                              ", but there are only " + std::to_string(RasterCache::getInstance().getLoadedSources().size()) +
                               " loaded" + SOURCE_REF);
     }
 
@@ -144,7 +139,7 @@ RasterDatum RasterContainer::GetRasterDataFromSource(unsigned int source_id, dou
     BOOST_ASSERT(lon < 180);
     BOOST_ASSERT(lon > -180);
 
-    const auto &found = LoadedSources[source_id];
+    const auto &found = RasterCache::getInstance().getLoadedSources()[source_id];
     return found.GetRasterData(static_cast<std::int32_t>(util::toFixed(util::FloatLongitude{lon})),
                                static_cast<std::int32_t>(util::toFixed(util::FloatLatitude{lat})));
 }
@@ -153,10 +148,10 @@ RasterDatum RasterContainer::GetRasterDataFromSource(unsigned int source_id, dou
 RasterDatum
 RasterContainer::GetRasterInterpolateFromSource(unsigned int source_id, double lon, double lat)
 {
-    if (LoadedSources.size() < source_id + 1)
+    if (RasterCache::getInstance().getLoadedSources().size() < source_id + 1)
     {
         throw util::exception("Attempted to access source " + std::to_string(source_id) +
-                              ", but there are only " + std::to_string(LoadedSources.size()) +
+                              ", but there are only " + std::to_string(RasterCache::getInstance().getLoadedSources().size()) +
                               " loaded" + SOURCE_REF);
     }
 
@@ -165,10 +160,12 @@ RasterContainer::GetRasterInterpolateFromSource(unsigned int source_id, double l
     BOOST_ASSERT(lon < 180);
     BOOST_ASSERT(lon > -180);
 
-    const auto &found = LoadedSources[source_id];
+    const auto &found = RasterCache::getInstance().getLoadedSources()[source_id];
     return found.GetRasterInterpolate(
         static_cast<std::int32_t>(util::toFixed(util::FloatLongitude{lon})),
         static_cast<std::int32_t>(util::toFixed(util::FloatLatitude{lat})));
 }
+
+RasterCache *RasterCache::g_instance = NULL;
 }
 }

From 17f32f4ca14fac451c54a9336a1167f45537e03e Mon Sep 17 00:00:00 2001
From: Tomonobu Saito <Tomnobu.Saito@gmail.com>
Date: Wed, 9 Oct 2019 13:35:19 +0900
Subject: [PATCH 12/16] fix coding format

---
 include/extractor/raster_source.hpp | 38 +++++++++++++++++------------
 include/storage/io.hpp              | 22 +++++++++++------
 src/extractor/raster_source.cpp     |  6 +++--
 3 files changed, 41 insertions(+), 25 deletions(-)
 mode change 100755 => 100644 include/extractor/raster_source.hpp
 mode change 100755 => 100644 src/extractor/raster_source.cpp

diff --git a/include/extractor/raster_source.hpp b/include/extractor/raster_source.hpp
old mode 100755
new mode 100644
index 48d69d4c484..939b3ee602b
--- a/include/extractor/raster_source.hpp
+++ b/include/extractor/raster_source.hpp
@@ -4,22 +4,22 @@
 #include "util/coordinate.hpp"
 #include "util/exception.hpp"
 
-#include <boost/algorithm/string/trim.hpp>
 #include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/trim.hpp>
 #include <boost/assert.hpp>
 #include <boost/filesystem.hpp>
 #include <boost/filesystem/fstream.hpp>
+#include <boost/foreach.hpp>
 #include <boost/spirit/include/qi.hpp>
 #include <boost/spirit/include/qi_int.hpp>
-#include <boost/foreach.hpp>
 
 #include <storage/io.hpp>
 
+#include <iostream>
 #include <iterator>
-#include <unordered_map>
-#include <string>
 #include <list>
-#include <iostream>
+#include <string>
+#include <unordered_map>
 using namespace std;
 
 namespace osrm
@@ -58,17 +58,21 @@ class RasterGrid
         buf.resize(xdim * 11); // INT32_MAX = 2147483647 = 10 chars + 1 white space = 11
         BOOST_ASSERT(xdim * 11 <= buf.size());
 
-        for (unsigned int y = 0 ; y < ydim ; y++) {
+        for (unsigned int y = 0; y < ydim; y++)
+        {
             // read one line from file.
             file_reader.ReadLine(&buf[0], xdim * 11);
             boost::algorithm::trim(buf);
 
             std::vector<std::string> result;
-            boost::split(result, buf, boost::is_any_of(" \r\n\0"), boost::algorithm::token_compress_on);
-            //boost::split(result, buf, boost::is_any_of(" \r\n\0"));
+            boost::split(
+                result, buf, boost::is_any_of(" \r\n\0"), boost::algorithm::token_compress_on);
+            // boost::split(result, buf, boost::is_any_of(" \r\n\0"));
             unsigned int x = 0;
-            BOOST_FOREACH(std::string s, result) {
-                if (x < xdim) _data[(y * xdim) + x] = atoi(s.c_str());
+            BOOST_FOREACH (std::string s, result)
+            {
+                if (x < xdim)
+                    _data[(y * xdim) + x] = atoi(s.c_str());
                 ++x;
             }
             BOOST_ASSERT(x == xdim);
@@ -146,18 +150,20 @@ class RasterContainer
 // << singletone >> RasterCache
 class RasterCache
 {
-public:
+  public:
     // class method to get the instance
-    static RasterCache& getInstance() {
-        if (NULL == g_instance) {
+    static RasterCache &getInstance()
+    {
+        if (NULL == g_instance)
+        {
             g_instance = new RasterCache();
         }
         return *g_instance;
     }
     // get reference of cache
-    std::vector<RasterSource>& getLoadedSources() { return LoadedSources; }
-    std::unordered_map<std::string, int>& getLoadedSourcePaths() { return LoadedSourcePaths; }
-private:
+    std::vector<RasterSource> &getLoadedSources() { return LoadedSources; }
+    std::unordered_map<std::string, int> &getLoadedSourcePaths() { return LoadedSourcePaths; }
+  private:
     // constructor
     RasterCache() = default;
     // member
diff --git a/include/storage/io.hpp b/include/storage/io.hpp
index 0a6eb0775ba..8635bd2c910 100644
--- a/include/storage/io.hpp
+++ b/include/storage/io.hpp
@@ -62,20 +62,28 @@ class FileReader
     std::size_t GetSize()
     {
         const boost::filesystem::path path(filepath);
-        try {
-            return std::size_t(boost::filesystem::file_size(path)) - ((fingerprint == FingerprintFlag::VerifyFingerprint) ? sizeof(util::FingerPrint) : 0);
+        try
+        {
+            return std::size_t(boost::filesystem::file_size(path)) -
+                   ((fingerprint == FingerprintFlag::VerifyFingerprint) ? sizeof(util::FingerPrint)
+                                                                        : 0);
         }
-        catch (boost::filesystem::filesystem_error& ex) {
+        catch (boost::filesystem::filesystem_error &ex)
+        {
             std::cout << ex.what() << std::endl;
             throw;
         }
     }
 
     /* Read one line */
-    template <typename T> void ReadLine(T *dest, const std::size_t count) {
-        if (0 < count) {
-            const auto &ios = input_stream.getline(reinterpret_cast<char *>(dest), count * sizeof(T));
-            for (std::size_t n = ios.gcount(); n < count; ++n) {
+    template <typename T> void ReadLine(T *dest, const std::size_t count)
+    {
+        if (0 < count)
+        {
+            const auto &ios =
+                input_stream.getline(reinterpret_cast<char *>(dest), count * sizeof(T));
+            for (std::size_t n = ios.gcount(); n < count; ++n)
+            {
                 reinterpret_cast<char *>(dest)[n] = '\0';
             }
         }
diff --git a/src/extractor/raster_source.cpp b/src/extractor/raster_source.cpp
old mode 100755
new mode 100644
index 5b69ebd5692..10b517ef7ae
--- a/src/extractor/raster_source.cpp
+++ b/src/extractor/raster_source.cpp
@@ -130,7 +130,8 @@ RasterDatum RasterContainer::GetRasterDataFromSource(unsigned int source_id, dou
     if (RasterCache::getInstance().getLoadedSources().size() < source_id + 1)
     {
         throw util::exception("Attempted to access source " + std::to_string(source_id) +
-                              ", but there are only " + std::to_string(RasterCache::getInstance().getLoadedSources().size()) +
+                              ", but there are only " +
+                              std::to_string(RasterCache::getInstance().getLoadedSources().size()) +
                               " loaded" + SOURCE_REF);
     }
 
@@ -151,7 +152,8 @@ RasterContainer::GetRasterInterpolateFromSource(unsigned int source_id, double l
     if (RasterCache::getInstance().getLoadedSources().size() < source_id + 1)
     {
         throw util::exception("Attempted to access source " + std::to_string(source_id) +
-                              ", but there are only " + std::to_string(RasterCache::getInstance().getLoadedSources().size()) +
+                              ", but there are only " +
+                              std::to_string(RasterCache::getInstance().getLoadedSources().size()) +
                               " loaded" + SOURCE_REF);
     }
 

From 9c1c842b79cb566e1d94e2002fac1933db008703 Mon Sep 17 00:00:00 2001
From: Tomonobu Saito <Tomnobu.Saito@gmail.com>
Date: Thu, 10 Oct 2019 12:26:13 +0900
Subject: [PATCH 13/16] update CHANGELOG.md

---
 CHANGELOG.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c9ca7a8c234..32befa035a0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,8 @@
       - CHANGED: allow routing past `barrier=arch` [#5352](https://github.com/Project-OSRM/osrm-backend/pull/5352)
       - CHANGED: default car weight was reduced to 2000 kg. [#5371](https://github.com/Project-OSRM/osrm-backend/pull/5371)
       - CHANGED: default car height was reduced to 2 meters. [#5389](https://github.com/Project-OSRM/osrm-backend/pull/5389)
+    - Misc:
+      - CHANGED: Reduce memory usage for raster source handling. [#5572](https://github.com/Project-OSRM/osrm-backend/pull/5572)
 
 # 5.21.0
   - Changes from 5.20.0

From fd0f1b60bb0de000cd321cd09b3041ec23bdcd96 Mon Sep 17 00:00:00 2001
From: Tomonobu Saito <Tomnobu.Saito@gmail.com>
Date: Wed, 6 Nov 2019 11:06:29 +0900
Subject: [PATCH 14/16] fix by revier comments

---
 include/extractor/raster_source.hpp | 15 +++++++--------
 include/storage/io.hpp              |  8 ++------
 2 files changed, 9 insertions(+), 14 deletions(-)
 mode change 100644 => 100755 include/storage/io.hpp

diff --git a/include/extractor/raster_source.hpp b/include/extractor/raster_source.hpp
index 939b3ee602b..16f8e57bec6 100644
--- a/include/extractor/raster_source.hpp
+++ b/include/extractor/raster_source.hpp
@@ -54,22 +54,21 @@ class RasterGrid
 
         // Construct FileReader
         storage::io::FileReader file_reader(filepath, storage::io::FileReader::HasNoFingerprint);
-        std::string buf;
-        buf.resize(xdim * 11); // INT32_MAX = 2147483647 = 10 chars + 1 white space = 11
-        BOOST_ASSERT(xdim * 11 <= buf.size());
+        std::string buffer;
+        buffer.resize(xdim * 11); // INT32_MAX = 2147483647 = 10 chars + 1 white space = 11
+        BOOST_ASSERT(xdim * 11 <= buffer.size());
 
         for (unsigned int y = 0; y < ydim; y++)
         {
             // read one line from file.
-            file_reader.ReadLine(&buf[0], xdim * 11);
-            boost::algorithm::trim(buf);
+            file_reader.ReadLine(&buffer[0], xdim * 11);
+            boost::algorithm::trim(buffer);
 
             std::vector<std::string> result;
             boost::split(
-                result, buf, boost::is_any_of(" \r\n\0"), boost::algorithm::token_compress_on);
-            // boost::split(result, buf, boost::is_any_of(" \r\n\0"));
+                result, buffer, boost::is_any_of(" \r\n\0"), boost::algorithm::token_compress_on);
             unsigned int x = 0;
-            BOOST_FOREACH (std::string s, result)
+            for (const auto &s : result)
             {
                 if (x < xdim)
                     _data[(y * xdim) + x] = atoi(s.c_str());
diff --git a/include/storage/io.hpp b/include/storage/io.hpp
old mode 100644
new mode 100755
index 8635bd2c910..2ba84530115
--- a/include/storage/io.hpp
+++ b/include/storage/io.hpp
@@ -80,12 +80,8 @@ class FileReader
     {
         if (0 < count)
         {
-            const auto &ios =
-                input_stream.getline(reinterpret_cast<char *>(dest), count * sizeof(T));
-            for (std::size_t n = ios.gcount(); n < count; ++n)
-            {
-                reinterpret_cast<char *>(dest)[n] = '\0';
-            }
+            memset(dest, 0, count * sizeof(T));
+            input_stream.getline(reinterpret_cast<char *>(dest), count * sizeof(T));
         }
     }
 

From ee177efe41a6da6ee8c2e9ed6bb3847eba387545 Mon Sep 17 00:00:00 2001
From: Tomonobu Saito <Tomnobu.Saito@gmail.com>
Date: Fri, 8 Nov 2019 16:15:38 +0900
Subject: [PATCH 15/16] fix: remove unused include / add const for necessity
 portion

---
 include/extractor/raster_source.hpp | 2 --
 include/storage/io.hpp              | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)
 mode change 100644 => 100755 include/extractor/raster_source.hpp

diff --git a/include/extractor/raster_source.hpp b/include/extractor/raster_source.hpp
old mode 100644
new mode 100755
index 16f8e57bec6..f426984a3f5
--- a/include/extractor/raster_source.hpp
+++ b/include/extractor/raster_source.hpp
@@ -15,9 +15,7 @@
 
 #include <storage/io.hpp>
 
-#include <iostream>
 #include <iterator>
-#include <list>
 #include <string>
 #include <unordered_map>
 using namespace std;
diff --git a/include/storage/io.hpp b/include/storage/io.hpp
index 2ba84530115..750605ac1bc 100755
--- a/include/storage/io.hpp
+++ b/include/storage/io.hpp
@@ -68,7 +68,7 @@ class FileReader
                    ((fingerprint == FingerprintFlag::VerifyFingerprint) ? sizeof(util::FingerPrint)
                                                                         : 0);
         }
-        catch (boost::filesystem::filesystem_error &ex)
+        catch (const boost::filesystem::filesystem_error &ex)
         {
             std::cout << ex.what() << std::endl;
             throw;

From 9da6cf876434480e28fc722c39e1acc8cbdb44c0 Mon Sep 17 00:00:00 2001
From: Tomonobu Saito <Tomnobu.Saito@gmail.com>
Date: Wed, 13 Nov 2019 11:01:07 +0900
Subject: [PATCH 16/16] add: comment for RasterCache class

---
 include/extractor/raster_source.hpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/include/extractor/raster_source.hpp b/include/extractor/raster_source.hpp
index f426984a3f5..eebe6297993 100755
--- a/include/extractor/raster_source.hpp
+++ b/include/extractor/raster_source.hpp
@@ -145,6 +145,10 @@ class RasterContainer
 };
 
 // << singletone >> RasterCache
+// The instance of RasterContainer is created for every threads osrm-extract uses.
+// To avoid multiple load of same file on each RasterContainer,
+// The LoadedSources and LoadedSourcePaths are separated to RasterCache class
+// and handled as the singletone pattern to avoid duplicate creation.
 class RasterCache
 {
   public: