Fix MA mod / loading gzipped json with json_loader (#61601)

* Fix move constructors/operators for cata::{i,o}fstream

* Fix loading gzipped json with json_loader

src/cata_utility.cpp

@@ -353,12 +353,90 @@ std::istream &safe_getline( std::istream &ins, std::string &str )
     }
 }
 
+namespace
+{
+
+std::string read_compressed_file_to_string( std::istream &fin )
+{
+    std::string outstring;
+
+    std::ostringstream deflated_contents_stream;
+    std::string str;
+
+    deflated_contents_stream << fin.rdbuf();
+    str = deflated_contents_stream.str();
+
+    z_stream zs;
+    memset( &zs, 0, sizeof( zs ) );
+
+    if( inflateInit2( &zs, MAX_WBITS | 16 ) != Z_OK ) {
+        throw std::runtime_error( "inflateInit failed while decompressing." );
+    }
+
+    zs.next_in = reinterpret_cast<unsigned char *>( const_cast<char *>( str.data() ) );
+    zs.avail_in = str.size();
+
+    int ret;
+    std::array<char, 32768> outbuffer;
+
+    // get the decompressed bytes blockwise using repeated calls to inflate
+    do {
+        zs.next_out = reinterpret_cast<Bytef *>( outbuffer.data() );
+        zs.avail_out = sizeof( outbuffer );
+
+        ret = inflate( &zs, 0 );
+
+        if( outstring.size() < static_cast<size_t>( zs.total_out ) ) {
+            outstring.append( outbuffer.data(),
+                              zs.total_out - outstring.size() );
+        }
+
+    } while( ret == Z_OK );
+
+    inflateEnd( &zs );
+
+    if( ret != Z_STREAM_END ) { // an error occurred that was not EOF
+        std::ostringstream oss;
+        oss << "Exception during zlib decompression: (" << ret << ") "
+            << zs.msg;
+        throw std::runtime_error( oss.str() );
+    }
+    return outstring;
+}
+
+} // namespace
+
 bool read_from_file( const cata_path &path, const std::function<void( std::istream & )> &reader )
 {
     return read_from_file( path.get_unrelative_path(), reader );
 }
 
 bool read_from_file( const fs::path &path, const std::function<void( std::istream & )> &reader )
+{
+    std::unique_ptr<std::istream> finp = read_maybe_compressed_file( path );
+    if( !finp ) {
+        return false;
+    }
+    try {
+        reader( *finp );
+    } catch( const std::exception &err ) {
+        debugmsg( _( "Failed to read from \"%1$s\": %2$s" ), path.generic_u8string().c_str(), err.what() );
+        return false;
+    }
+    return true;
+}
+
+bool read_from_file( const std::string &path, const std::function<void( std::istream & )> &reader )
+{
+    return read_from_file( fs::u8path( path ), reader );
+}
+
+std::unique_ptr<std::istream> read_maybe_compressed_file( const std::string &path )
+{
+    return read_maybe_compressed_file( fs::u8path( path ) );
+}
+
+std::unique_ptr<std::istream> read_maybe_compressed_file( const fs::path &path )
 {
     try {
         cata::ifstream fin( path, std::ios::binary );
@@ -374,72 +452,39 @@ bool read_from_file( const fs::path &path, const std::function<void( std::istrea
         fin.seekg( 0, std::ios::beg ); // reset read position
 
         if( ( header[0] == '\x1f' ) && ( header[1] == '\x8b' ) ) {
-            std::ostringstream deflated_contents_stream;
-            std::string str;
-
-            deflated_contents_stream << fin.rdbuf();
-            str = deflated_contents_stream.str();
-
-            z_stream zs;
-            memset( &zs, 0, sizeof( zs ) );
-
-            if( inflateInit2( &zs, MAX_WBITS | 16 ) != Z_OK ) {
-                throw( std::runtime_error( "inflateInit failed while decompressing." ) );
-            }
-
-            zs.next_in = reinterpret_cast<unsigned char *>( const_cast<char *>( str.data() ) );
-            zs.avail_in = str.size();
-
-            int ret;
-            std::array<char, 32768> outbuffer;
-            std::string outstring;
-
-            // get the decompressed bytes blockwise using repeated calls to inflate
-            do {
-                zs.next_out = reinterpret_cast<Bytef *>( outbuffer.data() );
-                zs.avail_out = sizeof( outbuffer );
-
-                ret = inflate( &zs, 0 );
-
-                if( outstring.size() < static_cast<size_t>( zs.total_out ) ) {
-                    outstring.append( outbuffer.data(),
-                                      zs.total_out - outstring.size() );
-                }
-
-            } while( ret == Z_OK );
-
-            inflateEnd( &zs );
-
-            if( ret != Z_STREAM_END ) { // an error occurred that was not EOF
-                std::ostringstream oss;
-                oss << "Exception during zlib decompression: (" << ret << ") "
-                    << zs.msg;
-                throw( std::runtime_error( oss.str() ) );
-            }
-
+            std::string outstring = read_compressed_file_to_string( fin );
             std::stringstream inflated_contents_stream;
             inflated_contents_stream.write( outstring.data(), outstring.size() );
-
-            reader( inflated_contents_stream );
+            return std::make_unique<std::stringstream>( std::move( inflated_contents_stream ) );
         } else {
-            reader( fin );
+            return std::make_unique<cata::ifstream>( std::move( fin ) );
         }
         if( fin.bad() ) {
             throw std::runtime_error( "reading file failed" );
         }
-        return true;
 
     } catch( const std::exception &err ) {
         debugmsg( _( "Failed to read from \"%1$s\": %2$s" ), path.generic_u8string().c_str(), err.what() );
-        return false;
     }
+    return nullptr;
 }
 
+std::unique_ptr<std::istream> read_maybe_compressed_file( const cata_path &path )
+{
+    return read_maybe_compressed_file( path.get_unrelative_path() );
+}
 
-bool read_from_file( const std::string &path, const std::function<void( std::istream & )> &reader )
+
+cata::optional<std::string> read_whole_file( const std::string &path )
+{
+    return read_whole_file( fs::u8path( path ) );
+}
+
+cata::optional<std::string> read_whole_file( const fs::path &path )
 {
+    std::string outstring;
     try {
-        cata::ifstream fin( fs::u8path( path ), std::ios::binary );
+        cata::ifstream fin( path, std::ios::binary );
         if( !fin ) {
             throw std::runtime_error( "opening file failed" );
         }
@@ -452,67 +497,32 @@ bool read_from_file( const std::string &path, const std::function<void( std::ist
         fin.seekg( 0, std::ios::beg ); // reset read position
 
         if( ( header[0] == '\x1f' ) && ( header[1] == '\x8b' ) ) {
-            std::ostringstream deflated_contents_stream;
-            std::string str;
-
-            deflated_contents_stream << fin.rdbuf();
-            str = deflated_contents_stream.str();
-
-            z_stream zs;
-            memset( &zs, 0, sizeof( zs ) );
-
-            if( inflateInit2( &zs, MAX_WBITS | 16 ) != Z_OK ) {
-                throw( std::runtime_error( "inflateInit failed while decompressing." ) );
-            }
-
-            zs.next_in = reinterpret_cast<unsigned char *>( const_cast<char *>( str.data() ) );
-            zs.avail_in = str.size();
-
-            int ret;
-            std::array<char, 32768> outbuffer;
-            std::string outstring;
-
-            // get the decompressed bytes blockwise using repeated calls to inflate
-            do {
-                zs.next_out = reinterpret_cast<Bytef *>( outbuffer.data() );
-                zs.avail_out = sizeof( outbuffer );
-
-                ret = inflate( &zs, 0 );
-
-                if( outstring.size() < static_cast<size_t>( zs.total_out ) ) {
-                    outstring.append( outbuffer.data(),
-                                      zs.total_out - outstring.size() );
-                }
-
-            } while( ret == Z_OK );
-
-            inflateEnd( &zs );
-
-            if( ret != Z_STREAM_END ) { // an error occurred that was not EOF
-                std::ostringstream oss;
-                oss << "Exception during zlib decompression: (" << ret << ") "
-                    << zs.msg;
-                throw( std::runtime_error( oss.str() ) );
-            }
-
-            std::stringstream inflated_contents_stream;
-            inflated_contents_stream.write( outstring.data(), outstring.size() );
-
-            reader( inflated_contents_stream );
+            outstring = read_compressed_file_to_string( fin );
         } else {
-            reader( fin );
+            fin.seekg( 0, std::ios_base::end );
+            std::streamoff size = fin.tellg();
+            fin.seekg( 0 );
+
+            outstring.resize( size );
+            fin.read( &outstring[0], size );
         }
         if( fin.bad() ) {
             throw std::runtime_error( "reading file failed" );
         }
-        return true;
 
+        return cata::optional<std::string>( std::move( outstring ) );
     } catch( const std::exception &err ) {
-        debugmsg( _( "Failed to read from \"%1$s\": %2$s" ), path.c_str(), err.what() );
-        return false;
+        debugmsg( _( "Failed to read from \"%1$s\": %2$s" ), path.generic_u8string().c_str(), err.what() );
     }
+    return cata::nullopt;
 }
 
+cata::optional<std::string> read_whole_file( const cata_path &path )
+{
+    return read_whole_file( path.get_unrelative_path() );
+}
+
+
 bool read_from_file_json( const cata_path &path,
                           const std::function<void( const JsonValue & )> &reader )
 {

src/cata_utility.h

@@ -8,6 +8,7 @@
 #include <functional>
 #include <iosfwd>
 #include <map>
+#include <memory>
 #include <string> // IWYU pragma: keep
 #include <type_traits>
 #include <unordered_set>
@@ -338,6 +339,36 @@ bool read_from_file_optional_json( const cata_path &path,
                                    const std::function<void( const JsonValue & )> &reader );
 /**@}*/
 
+/**
+ * Try to open and provide a std::istream for the given, possibly, gzipped, file.
+ *
+ * The file is opened for reading (binary mode) and tested to see if it is compressed.
+ * Compressed files are decompressed into a std::stringstream and returned.
+ * Uncompressed files are returned as normal lazy ifstreams.
+ * Any exceptions during reading, including failing to open the file, are reported as dbgmsg.
+ *
+ * @return A unique_ptr of the appropriate istream, or nullptr on failure.
+ */
+/**@{*/
+std::unique_ptr<std::istream> read_maybe_compressed_file( const std::string &path );
+std::unique_ptr<std::istream> read_maybe_compressed_file( const fs::path &path );
+std::unique_ptr<std::istream> read_maybe_compressed_file( const cata_path &path );
+/**@}*/
+
+/**
+ * Try to open and read the entire file to a string.
+ *
+ * The file is opened for reading (binary mode), read into a string, and then closed.
+ * Any exceptions during reading, including failing to open the file, are reported as dbgmsg.
+ *
+ * @return A nonempty optional with the contents of the file on success, or cata::nullopt on failure.
+ */
+/**@{*/
+cata::optional<std::string> read_whole_file( const std::string &path );
+cata::optional<std::string> read_whole_file( const fs::path &path );
+cata::optional<std::string> read_whole_file( const cata_path &path );
+/**@}*/
+
 std::istream &safe_getline( std::istream &ins, std::string &str );
 
 /** Apply fuzzy effect to a string like:

src/filesystem.h

@@ -125,9 +125,13 @@ class basic_ifstream : public std::basic_ifstream<charT, traits>
         basic_ifstream( const basic_ifstream & ) = delete;
         const basic_ifstream &operator=( const basic_ifstream & ) = delete;
         // NOLINTNEXTLINE(performance-noexcept-move-constructor)
-        basic_ifstream( basic_ifstream && ) noexcept( basic_ifstream_is_noexcept ) = default;
+        basic_ifstream( basic_ifstream &&rhs ) noexcept( basic_ifstream_is_noexcept ) :
+            std::basic_ifstream<charT, traits>( std::move( rhs ) ) {};
         // NOLINTNEXTLINE(performance-noexcept-move-constructor)
-        basic_ifstream &operator=( basic_ifstream && ) noexcept( basic_ifstream_is_noexcept ) = default;
+        basic_ifstream &operator=( basic_ifstream &&rhs ) noexcept( basic_ifstream_is_noexcept ) {
+            std::basic_ifstream<charT, traits>::operator=( std::move( rhs ) );
+            return *this;
+        };
         ~basic_ifstream() override = default;
 };
 
@@ -145,9 +149,13 @@ class basic_ofstream : public std::basic_ofstream<charT, traits>
         basic_ofstream( const basic_ofstream & ) = delete;
         const basic_ofstream &operator=( const basic_ofstream & ) = delete;
         // NOLINTNEXTLINE(performance-noexcept-move-constructor)
-        basic_ofstream( basic_ofstream && ) noexcept( basic_ofstream_is_noexcept ) = default;
+        basic_ofstream( basic_ofstream &&rhs ) noexcept( basic_ofstream_is_noexcept ) :
+            std::basic_ofstream<charT, traits>( std::move( rhs ) ) {};
         // NOLINTNEXTLINE(performance-noexcept-move-constructor)
-        basic_ofstream &operator=( basic_ofstream && ) noexcept( basic_ofstream_is_noexcept ) = default;
+        basic_ofstream &operator=( basic_ofstream &&rhs ) noexcept( basic_ofstream_is_noexcept ) {
+            std::basic_ofstream<charT, traits>::operator=( std::move( rhs ) );
+            return *this;
+        };
         ~basic_ofstream() override = default;
 };
 

src/flexbuffer_cache.cpp

@@ -14,6 +14,7 @@
 
 #include <ghc/fs_std_fwd.hpp>
 
+#include "cata_utility.h"
 #include "filesystem.h"
 #include "json.h"
 #include "json_error.h"
@@ -131,9 +132,8 @@ struct file_flexbuffer : parsed_flexbuffer {
         }
 
         std::unique_ptr<std::istream> get_source_stream() const noexcept( false ) override {
-            auto ifs = std::make_unique<cata::ifstream>( source_file_path_,
-                       std::ifstream::in | std::ifstream::binary );
-            if( !ifs->good() ) {
+            std::unique_ptr<std::istream> ifs = read_maybe_compressed_file( source_file_path_ );
+            if( !ifs || !ifs->good() ) {
                 return nullptr;
             }
             ifs->seekg( offset_ );
@@ -382,10 +382,12 @@ std::shared_ptr<parsed_flexbuffer> flexbuffer_cache::parse_and_cache(
     }
 
     std::string json_source_path_string = lexically_normal_json_source_path.generic_u8string();
-    std::string json_source = read_entire_file( lexically_normal_json_source_path );
-    if( json_source.empty() ) {
+    cata::optional<std::string> json_file_contents = read_whole_file(
+                lexically_normal_json_source_path );
+    if( !json_file_contents.has_value() || json_file_contents->empty() ) {
         throw std::runtime_error( "Failed to read " + json_source_path_string );
     }
+    std::string &json_source = *json_file_contents;
 
     const char *json_text = reinterpret_cast<const char *>( json_source.c_str() ) + offset;
     std::vector<uint8_t> fb = parse_json_to_flexbuffer_( json_text, json_source_path_string.c_str() );