Show source location of deferred json in debug messages

src/generic_factory.h

@@ -221,7 +221,8 @@ class generic_factory
                         def = ab->second;
                     } else {
                         def.was_loaded = false;
-                        deferred.emplace_back( jo.str(), src );
+                        deferred.emplace_back( jo.get_source_location(), src );
+                        jo.allow_omitted_members();
                         return false;
                     }
                 }

src/init.cpp

@@ -125,24 +125,35 @@ void DynamicDataLoader::load_deferred( deferred_json &data )
         const size_t n = data.size();
         auto it = data.begin();
         for( size_t idx = 0; idx != n; ++idx ) {
-            try {
-                std::istringstream str( it->first );
-                JsonIn jsin( str );
-                JsonObject jo = jsin.get_object();
-                load_object( jo, it->second );
-            } catch( const std::exception &err ) {
-                debugmsg( "Error loading data from json: %s", err.what() );
+            if( !it->first.path ) {
+                debugmsg( "JSON source location has null path, data may load incorrectly" );
+            } else {
+                try {
+                    std::ifstream str( *it->first.path, std::ifstream::in | std::ifstream::binary );
+                    JsonIn jsin( str, it->first );
+                    JsonObject jo = jsin.get_object();
+                    load_object( jo, it->second );
+                } catch( const JsonError &err ) {
+                    debugmsg( "(json-error)\n%s", err.what() );
+                }
             }
             ++it;
         }
         data.erase( data.begin(), it );
         if( data.size() == n ) {
-            std::string discarded;
             for( const auto &elem : data ) {
-                discarded += elem.first;
+                if( !elem.first.path ) {
+                    debugmsg( "JSON source location has null path when reporting circular dependency" );
+                } else {
+                    try {
+                        std::ifstream str( *elem.first.path, std::ifstream::in | std::ifstream::binary );
+                        JsonIn jsin( str, elem.first );
+                        jsin.error( "JSON contains circular dependency, this object is discarded" );
+                    } catch( const JsonError &err ) {
+                        debugmsg( "(json-error)\n%s", err.what() );
+                    }
+                }
             }
-            debugmsg( "JSON contains circular dependency.  Discarded %i objects:\n%s",
-                      data.size(), discarded );
             data.clear();
             return; // made no progress on this cycle so abort
         }

src/init.h

@@ -9,6 +9,8 @@
 #include <vector>
 #include <utility>
 
+#include "json.h"
+
 class loading_ui;
 class JsonObject;
 class JsonIn;
@@ -59,9 +61,9 @@ class DynamicDataLoader
 
         /**
          * JSON data dependent upon as-yet unparsed definitions
-         * first: JSON data, second: source identifier
+         * first: JSON source location, second: source identifier
          */
-        using deferred_json = std::list<std::pair<std::string, std::string>>;
+        using deferred_json = std::list<std::pair<json_source_location, std::string>>;
 
     private:
         bool finalized = false;

src/item_factory.cpp

@@ -1611,7 +1611,8 @@ bool Item_factory::load_definition( const JsonObject &jo, const std::string &src
         return true;
     }
 
-    deferred.emplace_back( jo.str(), src );
+    deferred.emplace_back( jo.get_source_location(), src );
+    jo.allow_omitted_members();
     return false;
 }
 

src/json.cpp

@@ -252,6 +252,21 @@ JsonIn *JsonObject::get_raw( const std::string &name ) const
     return jsin;
 }
 
+json_source_location JsonObject::get_source_location() const
+{
+    if( !jsin ) {
+        throw JsonError( "JsonObject::get_source_location called when stream is null" );
+    }
+    json_source_location loc;
+    loc.path = jsin->get_path();
+    if( !loc.path ) {
+        jsin->seek( start );
+        jsin->error( "JsonObject::get_source_location called but the path is unknown" );
+    }
+    loc.offset = start;
+    return loc;
+}
+
 /* returning values by name */
 
 bool JsonObject::get_bool( const std::string &name ) const
@@ -1629,6 +1644,7 @@ bool JsonIn::read( JsonDeserializer &j, bool throw_on_error )
 // WARNING: for occasional use only.
 std::string JsonIn::line_number( int offset_modifier )
 {
+    const std::string &name = path ? *path : "<unknown source file>";
     if( stream && stream->eof() ) {
         return name + ":EOF";
     } else if( !stream || stream->fail() ) {

src/json.h

@@ -17,6 +17,7 @@
 
 #include "colony.h"
 #include "enum_conversions.h"
+#include "memory_fast.h"
 #include "string_id.h"
 
 /* Cataclysm-DDA homegrown JSON tools
@@ -93,6 +94,11 @@ struct number_sci_notation {
     int64_t exp = 0;
 };
 
+struct json_source_location {
+    shared_ptr_fast<std::string> path;
+    int offset = 0;
+};
+
 /* JsonIn
  * ======
  *
@@ -177,8 +183,7 @@ class JsonIn
 {
     private:
         std::istream *stream;
-        // Used for error message and thus intentionally untranslated
-        std::string name = "<unknown source file>";
+        shared_ptr_fast<std::string> path;
         bool ate_separator = false;
 
         void skip_separator();
@@ -187,10 +192,19 @@ class JsonIn
 
     public:
         JsonIn( std::istream &s ) : stream( &s ) {}
-        JsonIn( std::istream &s, const std::string &name ) : stream( &s ), name( name ) {}
+        JsonIn( std::istream &s, const std::string &path )
+            : stream( &s ), path( make_shared_fast<std::string>( path ) ) {}
+        JsonIn( std::istream &s, const json_source_location &loc )
+            : stream( &s ), path( loc.path ) {
+            seek( loc.offset );
+        }
         JsonIn( const JsonIn & ) = delete;
         JsonIn &operator=( const JsonIn & ) = delete;
 
+        shared_ptr_fast<std::string> get_path() const {
+            return path;
+        }
+
         bool get_ate_separator() {
             return ate_separator;
         }
@@ -895,6 +909,7 @@ class JsonObject
         // seek to a value and return a pointer to the JsonIn (member must exist)
         JsonIn *get_raw( const std::string &name ) const;
         JsonValue get_member( const std::string &name ) const;
+        json_source_location get_source_location() const;
 
         // values by name
         // variants with no fallback throw an error if the name is not found.

src/mapgen.cpp

@@ -4,12 +4,12 @@
 #include <array>
 #include <cmath>
 #include <cstdlib>
+#include <fstream>
 #include <functional>
 #include <list>
 #include <map>
 #include <memory>
 #include <set>
-#include <sstream>
 #include <stdexcept>
 #include <unordered_map>
 
@@ -443,9 +443,11 @@ load_mapgen_function( const JsonObject &jio, const std::string &id_base, const p
             jio.throw_error( "function does not exist", "name" );
         }
     } else if( mgtype == "json" ) {
-        const std::string jstr = jio.get_object( "object" ).str();
+        JsonObject jo = jio.get_object( "object" );
+        const json_source_location jsrc = jo.get_source_location();
+        jo.allow_omitted_members();
         ret = std::make_shared<mapgen_function_json>(
-                  jstr, mgweight, "mapgen " + id_base, offset );
+                  jsrc, mgweight, "mapgen " + id_base, offset );
         oter_mapgen.add( id_base, ret );
     } else {
         jio.throw_error( R"(invalid value: must be "builtin" or "json")", "method" );
@@ -460,9 +462,10 @@ static void load_nested_mapgen( const JsonObject &jio, const std::string &id_bas
         if( jio.has_object( "object" ) ) {
             int weight = jio.get_int( "weight", 1000 );
             JsonObject jo = jio.get_object( "object" );
-            std::string jstr = jo.str();
+            const json_source_location jsrc = jo.get_source_location();
+            jo.allow_omitted_members();
             nested_mapgen[id_base].add(
-                std::make_shared<mapgen_function_json_nested>( jstr, "nested mapgen " + id_base ),
+                std::make_shared<mapgen_function_json_nested>( jsrc, "nested mapgen " + id_base ),
                 weight );
         } else {
             debugmsg( "Nested mapgen: Invalid mapgen function (missing \"object\" object)", id_base.c_str() );
@@ -479,10 +482,11 @@ static void load_update_mapgen( const JsonObject &jio, const std::string &id_bas
     if( mgtype == "json" ) {
         if( jio.has_object( "object" ) ) {
             JsonObject jo = jio.get_object( "object" );
-            std::string jstr = jo.str();
+            const json_source_location jsrc = jo.get_source_location();
+            jo.allow_omitted_members();
             update_mapgen[id_base].push_back(
                 std::make_unique<update_mapgen_function_json>(
-                    jstr, "update mapgen " + id_base ) );
+                    jsrc, "update mapgen " + id_base ) );
         } else {
             debugmsg( "Update mapgen: Invalid mapgen function (missing \"object\" object)",
                       id_base.c_str() );
@@ -588,8 +592,8 @@ bool mapgen_function_json_base::check_inbounds( const jmapgen_int &x, const jmap
 }
 
 mapgen_function_json_base::mapgen_function_json_base(
-    const std::string &s, const std::string &context )
-    : jdata( s )
+    const json_source_location &jsrcloc, const std::string &context )
+    : jsrcloc( jsrcloc )
     , context_( context )
     , do_format( false )
     , is_ready( false )
@@ -600,10 +604,10 @@ mapgen_function_json_base::mapgen_function_json_base(
 
 mapgen_function_json_base::~mapgen_function_json_base() = default;
 
-mapgen_function_json::mapgen_function_json( const std::string &s, const int w,
+mapgen_function_json::mapgen_function_json( const json_source_location &jsrcloc, const int w,
         const std::string &context, const point &grid_offset )
     : mapgen_function( w )
-    , mapgen_function_json_base( s, context )
+    , mapgen_function_json_base( jsrcloc, context )
     , fill_ter( t_null )
     , rotation( 0 )
 {
@@ -613,8 +617,8 @@ mapgen_function_json::mapgen_function_json( const std::string &s, const int w,
 }
 
 mapgen_function_json_nested::mapgen_function_json_nested(
-    const std::string &s, const std::string &context )
-    : mapgen_function_json_base( s, context )
+    const json_source_location &jsrcloc, const std::string &context )
+    : mapgen_function_json_base( jsrcloc, context )
     , rotation( 0 )
 {
 }
@@ -2431,8 +2435,12 @@ void mapgen_function_json_base::setup_common()
     if( is_ready ) {
         return;
     }
-    std::istringstream iss( jdata );
-    JsonIn jsin( iss );
+    if( !jsrcloc.path ) {
+        debugmsg( "null json source location path" );
+        return;
+    }
+    std::ifstream ifs( *jsrcloc.path, std::ifstream::in | std::ifstream::binary );
+    JsonIn jsin( ifs, jsrcloc );
     JsonObject jo = jsin.get_object();
     mapgen_defer::defer = false;
     if( !setup_common( jo ) ) {
@@ -6484,8 +6492,8 @@ void add_corpse( map *m, const point &p )
 
 //////////////////// mapgen update
 update_mapgen_function_json::update_mapgen_function_json(
-    const std::string &s, const std::string &context ) :
-    mapgen_function_json_base( s, context )
+    const json_source_location &jsrcloc, const std::string &context ) :
+    mapgen_function_json_base( jsrcloc, context )
 {
 }
 
@@ -6578,7 +6586,8 @@ mapgen_update_func add_mapgen_update_func( const JsonObject &jo, bool &defer )
         return update_function;
     }
 
-    update_mapgen_function_json json_data( "", "unknown object in add_mapgen_update_func" );
+    update_mapgen_function_json json_data( json_source_location {},
+                                           "unknown object in add_mapgen_update_func" );
     mapgen_defer::defer = defer;
     if( !json_data.setup_update( jo ) ) {
         const auto null_function = []( const tripoint_abs_omt &, mission * ) {

src/mapgen.h

@@ -319,11 +319,11 @@ class mapgen_function_json_base
         bool has_vehicle_collision( const mapgendata &dat, const point &offset ) const;
 
     private:
-        std::string jdata;
+        json_source_location jsrcloc;
         std::string context_;
 
     protected:
-        mapgen_function_json_base( const std::string &s, const std::string &context );
+        mapgen_function_json_base( const json_source_location &jsrcloc, const std::string &context );
         virtual ~mapgen_function_json_base();
 
         void setup_common();
@@ -354,7 +354,7 @@ class mapgen_function_json : public mapgen_function_json_base, public virtual ma
         void setup() override;
         void check() const override;
         void generate( mapgendata & ) override;
-        mapgen_function_json( const std::string &s, int w, const std::string &context,
+        mapgen_function_json( const json_source_location &jsrcloc, int w, const std::string &context,
                               const point &grid_offset = point_zero );
         ~mapgen_function_json() override = default;
 
@@ -371,7 +371,7 @@ class mapgen_function_json : public mapgen_function_json_base, public virtual ma
 class update_mapgen_function_json : public mapgen_function_json_base
 {
     public:
-        update_mapgen_function_json( const std::string &s, const std::string &context );
+        update_mapgen_function_json( const json_source_location &jsrcloc, const std::string &context );
         ~update_mapgen_function_json() override = default;
 
         void setup();
@@ -392,7 +392,7 @@ class mapgen_function_json_nested : public mapgen_function_json_base
     public:
         void setup();
         void check() const;
-        mapgen_function_json_nested( const std::string &s, const std::string &context );
+        mapgen_function_json_nested( const json_source_location &jsrcloc, const std::string &context );
         ~mapgen_function_json_nested() override = default;
 
         void nest( const mapgendata &dat, const point &offset ) const;

src/missiondef.cpp

@@ -287,7 +287,8 @@ void mission_type::load( const JsonObject &jo, const std::string &src )
         } else if( jo.has_member( phase ) ) {
             JsonObject j_start = jo.get_object( phase );
             if( !parse_funcs( j_start, phase_func ) ) {
-                deferred.emplace_back( jo.str(), src );
+                deferred.emplace_back( jo.get_source_location(), src );
+                jo.allow_omitted_members();
                 j_start.allow_omitted_members();
                 return false;
             }

src/recipe_dictionary.cpp

@@ -324,7 +324,8 @@ recipe &recipe_dictionary::load( const JsonObject &jo, const std::string &src,
     if( jo.has_string( "copy-from" ) ) {
         auto base = recipe_id( jo.get_string( "copy-from" ) );
         if( !out.count( base ) ) {
-            deferred.emplace_back( jo.str(), src );
+            deferred.emplace_back( jo.get_source_location(), src );
+            jo.allow_omitted_members();
             return null_recipe;
         }
         r = out[ base ];

src/veh_type.cpp

@@ -357,7 +357,8 @@ void vpart_info::load( const JsonObject &jo, const std::string &src )
             }
             def.categories = ab->second.categories;
         } else {
-            deferred.emplace_back( jo.str(), src );
+            deferred.emplace_back( jo.get_source_location(), src );
+            jo.allow_omitted_members();
             return;
         }
     }