diff --git a/CHANGELOG.md b/CHANGELOG.md index c9ca7a8c234..32befa035a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ - CHANGED: allow routing past `barrier=arch` [#5352](https://github.com/Project-OSRM/osrm-backend/pull/5352) - CHANGED: default car weight was reduced to 2000 kg. [#5371](https://github.com/Project-OSRM/osrm-backend/pull/5371) - CHANGED: default car height was reduced to 2 meters. [#5389](https://github.com/Project-OSRM/osrm-backend/pull/5389) + - Misc: + - CHANGED: Reduce memory usage for raster source handling. [#5572](https://github.com/Project-OSRM/osrm-backend/pull/5572) # 5.21.0 - Changes from 5.20.0 diff --git a/include/extractor/raster_source.hpp b/include/extractor/raster_source.hpp old mode 100644 new mode 100755 index 57a082d0e40..eebe6297993 --- a/include/extractor/raster_source.hpp +++ b/include/extractor/raster_source.hpp @@ -4,16 +4,21 @@ #include "util/coordinate.hpp" #include "util/exception.hpp" +#include #include #include #include #include +#include #include #include + #include #include +#include #include +using namespace std; namespace osrm { @@ -43,37 +48,31 @@ class RasterGrid xdim = _xdim; ydim = _ydim; _data.reserve(ydim * xdim); + BOOST_ASSERT(ydim * xdim <= _data.capacity()); + // Construct FileReader storage::io::FileReader file_reader(filepath, storage::io::FileReader::HasNoFingerprint); - std::string buffer; - buffer.resize(file_reader.GetSize()); - - BOOST_ASSERT(buffer.size() > 1); - - file_reader.ReadInto(&buffer[0], buffer.size()); - - boost::algorithm::trim(buffer); - - auto itr = buffer.begin(); - auto end = buffer.end(); - - bool r = false; - try - { - r = boost::spirit::qi::parse( - itr, end, +boost::spirit::qi::int_ % +boost::spirit::qi::space, _data); - } - catch (std::exception const &ex) - { - throw util::exception("Failed to read from raster source " + filepath.string() + ": " + - ex.what() + SOURCE_REF); - } + buffer.resize(xdim * 11); // INT32_MAX = 2147483647 = 10 chars + 1 white space = 11 + BOOST_ASSERT(xdim * 11 <= buffer.size()); - if (!r || itr != end) + for (unsigned int y = 0; y < ydim; y++) { - throw util::exception("Failed to parse raster source: " + filepath.string() + - SOURCE_REF); + // read one line from file. + file_reader.ReadLine(&buffer[0], xdim * 11); + boost::algorithm::trim(buffer); + + std::vector result; + boost::split( + result, buffer, boost::is_any_of(" \r\n\0"), boost::algorithm::token_compress_on); + unsigned int x = 0; + for (const auto &s : result) + { + if (x < xdim) + _data[(y * xdim) + x] = atoi(s.c_str()); + ++x; + } + BOOST_ASSERT(x == xdim); } } @@ -143,8 +142,36 @@ class RasterContainer RasterDatum GetRasterInterpolateFromSource(unsigned int source_id, double lon, double lat); private: +}; + +// << singletone >> RasterCache +// The instance of RasterContainer is created for every threads osrm-extract uses. +// To avoid multiple load of same file on each RasterContainer, +// The LoadedSources and LoadedSourcePaths are separated to RasterCache class +// and handled as the singletone pattern to avoid duplicate creation. +class RasterCache +{ + public: + // class method to get the instance + static RasterCache &getInstance() + { + if (NULL == g_instance) + { + g_instance = new RasterCache(); + } + return *g_instance; + } + // get reference of cache + std::vector &getLoadedSources() { return LoadedSources; } + std::unordered_map &getLoadedSourcePaths() { return LoadedSourcePaths; } + private: + // constructor + RasterCache() = default; + // member std::vector LoadedSources; std::unordered_map LoadedSourcePaths; + // the instance + static RasterCache *g_instance; }; } } diff --git a/include/storage/io.hpp b/include/storage/io.hpp old mode 100644 new mode 100755 index 7702c0ddb77..750605ac1bc --- a/include/storage/io.hpp +++ b/include/storage/io.hpp @@ -10,6 +10,7 @@ #include "util/log.hpp" #include "util/version.hpp" +#include #include #include #include @@ -60,29 +61,27 @@ class FileReader std::size_t GetSize() { - const boost::filesystem::ifstream::pos_type position = input_stream.tellg(); - input_stream.seekg(0, std::ios::end); - const boost::filesystem::ifstream::pos_type file_size = input_stream.tellg(); - - if (file_size == boost::filesystem::ifstream::pos_type(-1)) + const boost::filesystem::path path(filepath); + try { - throw util::RuntimeError("Unable to determine file size for " + - std::string(filepath.string()), - ErrorCode::FileIOError, - SOURCE_REF, - std::strerror(errno)); + return std::size_t(boost::filesystem::file_size(path)) - + ((fingerprint == FingerprintFlag::VerifyFingerprint) ? sizeof(util::FingerPrint) + : 0); } - - // restore the current position - input_stream.seekg(position, std::ios::beg); - - if (fingerprint == FingerprintFlag::VerifyFingerprint) + catch (const boost::filesystem::filesystem_error &ex) { - return std::size_t(file_size) - sizeof(util::FingerPrint); + std::cout << ex.what() << std::endl; + throw; } - else + } + + /* Read one line */ + template void ReadLine(T *dest, const std::size_t count) + { + if (0 < count) { - return file_size; + memset(dest, 0, count * sizeof(T)); + input_stream.getline(reinterpret_cast(dest), count * sizeof(T)); } } diff --git a/src/extractor/raster_source.cpp b/src/extractor/raster_source.cpp index 012f347a8ff..10b517ef7ae 100644 --- a/src/extractor/raster_source.cpp +++ b/src/extractor/raster_source.cpp @@ -92,15 +92,15 @@ int RasterContainer::LoadRasterSource(const std::string &path_string, const auto _ymin = static_cast(util::toFixed(util::FloatLatitude{ymin})); const auto _ymax = static_cast(util::toFixed(util::FloatLatitude{ymax})); - const auto itr = LoadedSourcePaths.find(path_string); - if (itr != LoadedSourcePaths.end()) + const auto itr = RasterCache::getInstance().getLoadedSourcePaths().find(path_string); + if (itr != RasterCache::getInstance().getLoadedSourcePaths().end()) { util::Log() << "[source loader] Already loaded source '" << path_string << "' at source_id " << itr->second; return itr->second; } - int source_id = static_cast(LoadedSources.size()); + int source_id = static_cast(RasterCache::getInstance().getLoadedSources().size()); util::Log() << "[source loader] Loading from " << path_string << " ... "; TIMER_START(loading_source); @@ -116,8 +116,8 @@ int RasterContainer::LoadRasterSource(const std::string &path_string, RasterSource source{std::move(rasterData), ncols, nrows, _xmin, _xmax, _ymin, _ymax}; TIMER_STOP(loading_source); - LoadedSourcePaths.emplace(path_string, source_id); - LoadedSources.push_back(std::move(source)); + RasterCache::getInstance().getLoadedSourcePaths().emplace(path_string, source_id); + RasterCache::getInstance().getLoadedSources().push_back(std::move(source)); util::Log() << "[source loader] ok, after " << TIMER_SEC(loading_source) << "s"; @@ -127,10 +127,11 @@ int RasterContainer::LoadRasterSource(const std::string &path_string, // External function for looking up nearest data point from a specified source RasterDatum RasterContainer::GetRasterDataFromSource(unsigned int source_id, double lon, double lat) { - if (LoadedSources.size() < source_id + 1) + if (RasterCache::getInstance().getLoadedSources().size() < source_id + 1) { throw util::exception("Attempted to access source " + std::to_string(source_id) + - ", but there are only " + std::to_string(LoadedSources.size()) + + ", but there are only " + + std::to_string(RasterCache::getInstance().getLoadedSources().size()) + " loaded" + SOURCE_REF); } @@ -139,7 +140,7 @@ RasterDatum RasterContainer::GetRasterDataFromSource(unsigned int source_id, dou BOOST_ASSERT(lon < 180); BOOST_ASSERT(lon > -180); - const auto &found = LoadedSources[source_id]; + const auto &found = RasterCache::getInstance().getLoadedSources()[source_id]; return found.GetRasterData(static_cast(util::toFixed(util::FloatLongitude{lon})), static_cast(util::toFixed(util::FloatLatitude{lat}))); } @@ -148,10 +149,11 @@ RasterDatum RasterContainer::GetRasterDataFromSource(unsigned int source_id, dou RasterDatum RasterContainer::GetRasterInterpolateFromSource(unsigned int source_id, double lon, double lat) { - if (LoadedSources.size() < source_id + 1) + if (RasterCache::getInstance().getLoadedSources().size() < source_id + 1) { throw util::exception("Attempted to access source " + std::to_string(source_id) + - ", but there are only " + std::to_string(LoadedSources.size()) + + ", but there are only " + + std::to_string(RasterCache::getInstance().getLoadedSources().size()) + " loaded" + SOURCE_REF); } @@ -160,10 +162,12 @@ RasterContainer::GetRasterInterpolateFromSource(unsigned int source_id, double l BOOST_ASSERT(lon < 180); BOOST_ASSERT(lon > -180); - const auto &found = LoadedSources[source_id]; + const auto &found = RasterCache::getInstance().getLoadedSources()[source_id]; return found.GetRasterInterpolate( static_cast(util::toFixed(util::FloatLongitude{lon})), static_cast(util::toFixed(util::FloatLatitude{lat}))); } + +RasterCache *RasterCache::g_instance = NULL; } }