Cache deferred json input stream

CleverRaven · Dec 27, 2020 · 4c9eb7b · 4c9eb7b
1 parent 30fc616
commit 4c9eb7b
Show file tree

Hide file tree

Showing 4 changed files with 49 additions and 7 deletions.
diff --git a/src/init.cpp b/src/init.cpp
@@ -44,6 +44,7 @@
 #include "item_factory.h"
 #include "json.h"
 #include "loading_ui.h"
+#include "lru_cache.h"
 #include "magic.h"
 #include "magic_enchantment.h"
 #include "magic_ter_furn_transform.h"
@@ -119,6 +120,30 @@ void DynamicDataLoader::load_object( const JsonObject &jo, const std::string &sr
     it->second( jo, src, base_path, full_path );
 }
 
+struct DynamicDataLoader::cached_streams {
+    lru_cache<std::string, shared_ptr_fast<std::istringstream>> cache;
+};
+
+shared_ptr_fast<std::istream> DynamicDataLoader::get_cached_stream( const std::string &path )
+{
+    cata_assert( !finalized &&
+                 "Cannot open data file after finalization." );
+    if( !stream_cache ) {
+        stream_cache = std::make_unique<cached_streams>();
+    }
+    shared_ptr_fast<std::istringstream> cached = stream_cache->cache.get( path, nullptr );
+    // Create a new stream if the file is not opened yet, or if some code is still
+    // using the previous stream (in such case, `cached` and `stream_cache` have
+    // two references to the stream, hence the test for > 2).
+    if( !cached ) {
+        cached = make_shared_fast<std::istringstream>( read_entire_file( path ) );
+    } else if( cached.use_count() > 2 ) {
+        cached = make_shared_fast<std::istringstream>( cached->str() );
+    }
+    stream_cache->cache.insert( 8, path, cached );
+    return cached;
+}
+
 void DynamicDataLoader::load_deferred( deferred_json &data )
 {
     while( !data.empty() ) {
@@ -129,8 +154,8 @@ void DynamicDataLoader::load_deferred( deferred_json &data )
                 debugmsg( "JSON source location has null path, data may load incorrectly" );
             } else {
                 try {
-                    std::ifstream str( *it->first.path, std::ifstream::in | std::ifstream::binary );
-                    JsonIn jsin( str, it->first );
+                    shared_ptr_fast<std::istream> stream = get_cached_stream( *it->first.path );
+                    JsonIn jsin( *stream, it->first );
                     JsonObject jo = jsin.get_object();
                     load_object( jo, it->second );
                 } catch( const JsonError &err ) {
@@ -146,8 +171,8 @@ void DynamicDataLoader::load_deferred( deferred_json &data )
                     debugmsg( "JSON source location has null path when reporting circular dependency" );
                 } else {
                     try {
-                        std::ifstream str( *elem.first.path, std::ifstream::in | std::ifstream::binary );
-                        JsonIn jsin( str, elem.first );
+                        shared_ptr_fast<std::istream> stream = get_cached_stream( *it->first.path );
+                        JsonIn jsin( *stream, elem.first );
                         jsin.error( "JSON contains circular dependency, this object is discarded" );
                     } catch( const JsonError &err ) {
                         debugmsg( "(json-error)\n%s", err.what() );
@@ -651,6 +676,7 @@ void DynamicDataLoader::finalize_loaded_data( loading_ui &ui )
     }
 
     check_consistency( ui );
+    stream_cache.reset();
     finalized = true;
 }
 

diff --git a/src/init.h b/src/init.h
@@ -10,6 +10,7 @@
 #include <utility>
 
 #include "json.h"
+#include "memory_fast.h"
 
 class loading_ui;
 class JsonObject;
@@ -68,6 +69,9 @@ class DynamicDataLoader
     private:
         bool finalized = false;
 
+        struct cached_streams;
+        std::unique_ptr<cached_streams> stream_cache;
+
     protected:
         /**
          * Maps the type string (coming from json) to the
@@ -163,6 +167,14 @@ class DynamicDataLoader
         bool is_data_finalized() const {
             return finalized;
         }
+
+        /**
+         * Get a possibly cached stream for deferred data loading. If the cached
+         * stream is still in use by outside code, this returns a new stream to
+         * avoid conflict of stream cursor. The stream cursor is not reset if a
+         * cached stream is returned.
+         */
+        shared_ptr_fast<std::istream> get_cached_stream( const std::string &path );
 };
 
 #endif // CATA_SRC_INIT_H
diff --git a/src/lru_cache.cpp b/src/lru_cache.cpp
@@ -1,10 +1,13 @@
 #include "lru_cache.h"
 
 #include <cstddef>
+#include <sstream>
 #include <iterator>
 #include <memory>
+#include <string>
 
 #include "map_memory.h"
+#include "memory_fast.h"
 #include "point.h"
 
 template<typename Key, typename Value>
@@ -73,3 +76,4 @@ const std::list<typename lru_cache<Key, Value>::Pair> &lru_cache<Key, Value>::li
 template class lru_cache<tripoint, memorized_terrain_tile>;
 template class lru_cache<tripoint, int>;
 template class lru_cache<point, char>;
+template class lru_cache<std::string, shared_ptr_fast<std::istringstream>>;
diff --git a/src/mapgen.cpp b/src/mapgen.cpp
@@ -4,7 +4,6 @@
 #include <array>
 #include <cmath>
 #include <cstdlib>
-#include <fstream>
 #include <functional>
 #include <list>
 #include <map>
@@ -2439,8 +2438,9 @@ void mapgen_function_json_base::setup_common()
         debugmsg( "null json source location path" );
         return;
     }
-    std::ifstream ifs( *jsrcloc.path, std::ifstream::in | std::ifstream::binary );
-    JsonIn jsin( ifs, jsrcloc );
+    shared_ptr_fast<std::istream> stream = DynamicDataLoader::get_instance().get_cached_stream(
+            *jsrcloc.path );
+    JsonIn jsin( *stream, jsrcloc );
     JsonObject jo = jsin.get_object();
     mapgen_defer::defer = false;
     if( !setup_common( jo ) ) {