diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 216d6b9c722d..b4a3810ad816 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -50,7 +50,8 @@ jobs: sudo apt-get install \ cmake gettext ninja-build mold ccache jq \ clang-18 libclang-18-dev llvm-18 llvm-18-dev clang-tidy-18 \ - libsdl2-dev libsdl2-ttf-dev libsdl2-image-dev libsdl2-mixer-dev libpulse-dev libflac-dev + libsdl2-dev libsdl2-ttf-dev libsdl2-image-dev libsdl2-mixer-dev libpulse-dev libflac-dev \ + sqlite3 libsqlite3-dev zlib1g-dev - name: ensure clang 18 is installed run: | diff --git a/.github/workflows/msys2-cmake.yml b/.github/workflows/msys2-cmake.yml index a74970c8ccc2..9e768b4931ac 100644 --- a/.github/workflows/msys2-cmake.yml +++ b/.github/workflows/msys2-cmake.yml @@ -55,6 +55,8 @@ jobs: mingw-w64-x86_64-SDL2_ttf mingw-w64-x86_64-toolchain mingw-w64-x86_64-cmake + mingw-w64-x86_64-sqlite3 + mingw-w64-x86_64-zstd - name: Create build directory run: mkdir build diff --git a/CMakeLists.txt b/CMakeLists.txt index 80337602f1a9..98deb39cc185 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -113,6 +113,8 @@ include(CheckCXXCompilerFlag) #SET(CMAKE_SHARED_LIBRARY_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CXX_FLAGS} -m32") find_package(PkgConfig) +find_package(SQLite3) +find_package(ZLIB) if (NOT DYNAMIC_LINKING) if(NOT MSVC) set(CMAKE_FIND_LIBRARY_SUFFIXES ".a;.dll.a") diff --git a/data/raw/keybindings/keybindings.json b/data/raw/keybindings/keybindings.json index ea2a1c246a98..7ba5f7f7463f 100644 --- a/data/raw/keybindings/keybindings.json +++ b/data/raw/keybindings/keybindings.json @@ -744,6 +744,13 @@ "name": "Pick random world name", "bindings": [ { "input_method": "keyboard", "key": "*" } ] }, + { + "type": "keybinding", + "id": "TOGGLE_V2_SAVE_FORMAT", + "category": "WORLDGEN_CONFIRM_DIALOG", + "name": "Toggle new world save format", + "bindings": [ { "input_method": "keyboard", "key": "=" } ] + }, { "type": "keybinding", "id": "QUIT", diff --git a/msvc-full-features/vcpkg.json b/msvc-full-features/vcpkg.json index da73cbcffe94..744eba6e35d7 100644 --- a/msvc-full-features/vcpkg.json +++ b/msvc-full-features/vcpkg.json @@ -11,7 +11,9 @@ "name": "sdl2-mixer", "features": [ "libflac", "mpg123", "libmodplug" ] }, - "sdl2-ttf" + "sdl2-ttf", + "sqlite3", + "zlib" ], "builtin-baseline": "c9aba300923c8ec0ab190e2bff23085209925c97", "vcpkg-configuration": { diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 474a0f9a10c6..a54ff26fe2da 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -61,6 +61,8 @@ if (TILES) add_dependencies(cataclysm-bn-tiles-common get_version) + target_link_libraries(cataclysm-bn-tiles-common PUBLIC ZLIB::ZLIB) + target_link_libraries(cataclysm-bn-tiles-common PUBLIC SQLite::SQLite3) target_link_libraries(cataclysm-bn-tiles PRIVATE cataclysm-bn-tiles-common) target_compile_definitions(cataclysm-bn-tiles-common PUBLIC TILES ) @@ -176,9 +178,12 @@ if (CURSES) endif () add_dependencies(cataclysm-bn-common get_version) + target_link_libraries(cataclysm-bn PRIVATE cataclysm-bn-common) target_include_directories(cataclysm-bn-common PUBLIC ${CURSES_INCLUDE_DIR}) + target_link_libraries(cataclysm-bn-common PUBLIC ZLIB::ZLIB) + target_link_libraries(cataclysm-bn-common PUBLIC SQLite::SQLite3) target_link_libraries(cataclysm-bn-common PUBLIC ${CURSES_LIBRARIES}) if (CMAKE_USE_PTHREADS_INIT) diff --git a/src/compress.cpp b/src/compress.cpp new file mode 100644 index 000000000000..6bba1c9b13ab --- /dev/null +++ b/src/compress.cpp @@ -0,0 +1,53 @@ +#include "compress.h" + +#include +#include +#include +#include +#include + +void zlib_compress( const std::string &input, std::vector &output ) +{ + uLongf compressedSize = compressBound( input.size() ); + output.resize( compressedSize ); + + int result = compress2( + reinterpret_cast( output.data() ), + &compressedSize, + reinterpret_cast( input.data() ), + input.size(), + Z_BEST_SPEED + ); + + if( result != Z_OK ) { + throw std::runtime_error( "Zlib compression error" ); + } + + output.resize( compressedSize ); +} + +void zlib_decompress( const void *compressed_data, int compressed_size, std::string &output ) +{ + // We need to guess at the decompressed size - we expect things to compress fairly well. + uLongf decompressedSize = compressed_size * 8; + output.resize( decompressedSize ); + + int result; + do { + result = uncompress( + reinterpret_cast( &output[0] ), + &decompressedSize, + reinterpret_cast( compressed_data ), + compressed_size + ); + + if( result == Z_BUF_ERROR ) { + decompressedSize *= 2; // Double the buffer size and retry + output.resize( decompressedSize ); + } else if( result != Z_OK ) { + throw std::runtime_error( "Zlib decompression failed" ); + } + } while( result == Z_BUF_ERROR ); + + output.resize( decompressedSize ); +} \ No newline at end of file diff --git a/src/compress.h b/src/compress.h new file mode 100644 index 000000000000..ef4392c2a965 --- /dev/null +++ b/src/compress.h @@ -0,0 +1,12 @@ +#pragma once +#ifndef CATA_SRC_COMPRESS_H +#define CATA_SRC_COMPRESS_H + +#include + +#include "fstream_utils.h" + +void zlib_compress( const std::string &input, std::vector &output ); +void zlib_decompress( const void *compressed_data, int compressed_size, std::string &output ); + +#endif // CATA_SRC_COMPRESS_H diff --git a/src/main_menu.cpp b/src/main_menu.cpp index ffc420780957..e0156f251cb7 100644 --- a/src/main_menu.cpp +++ b/src/main_menu.cpp @@ -500,6 +500,7 @@ void main_menu::init_strings() vWorldSubItems.emplace_back( pgettext( "Main Menu|World", "Character to Template" ) ); vWorldSubItems.emplace_back( pgettext( "Main Menu|World", "Reset World" ) ); vWorldSubItems.emplace_back( pgettext( "Main Menu|World", "Delete World" ) ); + vWorldSubItems.emplace_back( pgettext( "Main Menu|World", "Convert to V2 Save Format" ) ); vWorldSubItems.emplace_back( pgettext( "Main Menu|World", "<= Return" ) ); vWorldHotkeys = { 'm', 'e', 's', 't', 'r', 'd', 'q' }; @@ -1061,7 +1062,21 @@ void main_menu::world_tab( const std::string &worldname ) } }; + auto convert_v2 = [this, &worldname]() { + world_generator->set_active_world( nullptr ); + savegames.clear(); + MAPBUFFER.clear(); + overmap_buffer.clear(); + world_generator->convert_to_v2( worldname ); + }; + switch( opt_val ) { + case 6: // Convert to V2 Save Format + if( query_yn( + _( "Convert to V2 Save Format? A backup will be created. Conversion may take several minutes." ) ) ) { + convert_v2(); + } + break; case 5: // Delete World if( query_yn( _( "Delete the world and all saves within?" ) ) ) { clear_world( true ); diff --git a/src/world.cpp b/src/world.cpp index 52e4ff16b39f..02f8a79e1a77 100644 --- a/src/world.cpp +++ b/src/world.cpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include "game.h" #include "avatar.h" @@ -13,9 +15,44 @@ #include "worldfactory.h" #include "mod_manager.h" #include "path_info.h" +#include "compress.h" #define dbg(x) DebugLogFL((x),DC::Main) +static sqlite3 *open_db( const std::string &path ) +{ + sqlite3 *db = nullptr; + int ret; + + if( SQLITE_OK != ( ret = sqlite3_initialize() ) ) { + dbg( DL::Error ) << "Failed to initialize sqlite3 (Error " << ret << ")"; + throw std::runtime_error( "Failed to initialize sqlite3" ); + } + + if( SQLITE_OK != ( ret = sqlite3_open_v2( path.c_str(), &db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL ) ) ) { + dbg( DL::Error ) << "Failed to open db" << path << " (Error " << ret << ")"; + throw std::runtime_error( "Failed to open db" ); + } + + auto sql = R"sql( + CREATE TABLE IF NOT EXISTS files ( + path TEXT PRIMARY KEY NOT NULL, + parent TEXT NOT NULL, + compression TEXT DEFAULT NULL, + data BLOB NOT NULL + ); + )sql"; + + char *sqlErrMsg = 0; + if( SQLITE_OK != ( ret = sqlite3_exec( db, sql, NULL, NULL, &sqlErrMsg ) ) ) { + dbg( DL::Error ) << "Failed to init db" << path << " (" << sqlErrMsg << ")"; + throw std::runtime_error( "Failed to open db" ); + } + + return db; +} + save_t::save_t( const std::string &name ): name( name ) {} std::string save_t::decoded_name() const @@ -172,6 +209,16 @@ bool WORLDINFO::save( const bool is_conversion ) const } world_generator->get_mod_manager().save_mods_list( const_cast( this ) ); + + // If the world is a V2 world and there's no SQLite3 file yet, create a blank one. + // We infer that the world is V2 if there's a map.sqlite3 file in the world directory. + // When a world is freshly created, we need to create a file here to remember the users' + // choice of world save format. + if( world_save_format == save_format::V2_COMPRESSED_SQLITE3 && + !file_exist( folder_path() + "/map.sqlite3" ) ) { + sqlite3 *db = open_db( folder_path() + "/map.sqlite3" ); + sqlite3_close( db ); + } return true; } @@ -218,18 +265,178 @@ void load_external_option( const JsonObject &jo ) } } +static bool file_exist_in_db( sqlite3 *db, const std::string &path ) +{ + int fileCount = 0; + const char *sql = "SELECT count() FROM files WHERE path = :path"; + sqlite3_stmt *stmt = nullptr; + + if( sqlite3_prepare_v2( db, sql, -1, &stmt, nullptr ) != SQLITE_OK ) { + dbg( DL::Error ) << "Failed to prepare statement: " << sqlite3_errmsg( db ) << std::endl; + throw std::runtime_error( "DB query failed" ); + } + + if( sqlite3_bind_text( stmt, sqlite3_bind_parameter_index( stmt, ":path" ), path.c_str(), -1, + SQLITE_TRANSIENT ) != SQLITE_OK ) { + dbg( DL::Error ) << "Failed to bind parameter: " << sqlite3_errmsg( db ) << std::endl; + sqlite3_finalize( stmt ); + throw std::runtime_error( "DB query failed" ); + } + + if( sqlite3_step( stmt ) == SQLITE_ROW ) { + // Retrieve the count result + fileCount = sqlite3_column_int( stmt, 0 ); + } else { + dbg( DL::Error ) << "Failed to execute query: " << sqlite3_errmsg( db ) << std::endl; + sqlite3_finalize( stmt ); + throw std::runtime_error( "DB query failed" ); + } + + sqlite3_finalize( stmt ); + + return fileCount > 0; +} + +static void write_to_db( sqlite3 *db, const std::string &path, file_write_fn writer ) +{ + std::ostringstream oss; + writer( oss ); + auto data = oss.str(); + + std::vector compressedData; + zlib_compress( data, compressedData ); + + size_t basePos = path.find_last_of( "/\\" ); + auto parent = ( basePos == std::string::npos ) ? "" : path.substr( 0, basePos ); + + auto sql = R"sql( + INSERT INTO files(path, parent, data, compression) + VALUES (:path, :parent, :data, 'zlib') + ON CONFLICT(path) DO UPDATE + SET data = excluded.data, + parent = excluded.parent, + compression = excluded.compression; + )sql"; + + sqlite3_stmt *stmt = nullptr; + + if( sqlite3_prepare_v2( db, sql, -1, &stmt, nullptr ) != SQLITE_OK ) { + dbg( DL::Error ) << "Failed to prepare statement: " << sqlite3_errmsg( db ) << std::endl; + throw std::runtime_error( "DB query failed" ); + } + + if( sqlite3_bind_text( stmt, sqlite3_bind_parameter_index( stmt, ":path" ), path.c_str(), -1, + SQLITE_TRANSIENT ) != SQLITE_OK || + sqlite3_bind_text( stmt, sqlite3_bind_parameter_index( stmt, ":parent" ), parent.c_str(), -1, + SQLITE_TRANSIENT ) != SQLITE_OK || + sqlite3_bind_blob( stmt, sqlite3_bind_parameter_index( stmt, ":data" ), compressedData.data(), + compressedData.size(), SQLITE_TRANSIENT ) != SQLITE_OK ) { + dbg( DL::Error ) << "Failed to bind parameters: " << sqlite3_errmsg( db ) << std::endl; + sqlite3_finalize( stmt ); + throw std::runtime_error( "DB query failed" ); + } + + if( sqlite3_step( stmt ) != SQLITE_DONE ) { + dbg( DL::Error ) << "Failed to execute query: " << sqlite3_errmsg( db ) << std::endl; + } + sqlite3_finalize( stmt ); +} + +static bool read_from_db( sqlite3 *db, const std::string &path, file_read_fn reader, + bool optional ) +{ + const char *sql = "SELECT data, compression FROM files WHERE path = :path LIMIT 1"; + + sqlite3_stmt *stmt = nullptr; + + if( sqlite3_prepare_v2( db, sql, -1, &stmt, nullptr ) != SQLITE_OK ) { + dbg( DL::Error ) << "Failed to prepare statement: " << sqlite3_errmsg( db ) << std::endl; + throw std::runtime_error( "DB query failed" ); + } + + if( sqlite3_bind_text( stmt, sqlite3_bind_parameter_index( stmt, ":path" ), path.c_str(), -1, + SQLITE_TRANSIENT ) != SQLITE_OK ) { + dbg( DL::Error ) << "Failed to bind parameter: " << sqlite3_errmsg( db ) << std::endl; + sqlite3_finalize( stmt ); + throw std::runtime_error( "DB query failed" ); + } + + if( sqlite3_step( stmt ) == SQLITE_ROW ) { + // Retrieve the count result + const void *blobData = sqlite3_column_blob( stmt, 0 ); + int blobSize = sqlite3_column_bytes( stmt, 0 ); + auto compression_raw = sqlite3_column_text( stmt, 1 ); + std::string compression = compression_raw ? reinterpret_cast( compression_raw ) : ""; + + if( blobData == nullptr ) { + return false; // Return an empty string if there's no data + } + + std::string dataString; + if( compression == "" ) { + dataString = std::string( static_cast( blobData ), blobSize ); + } else if( compression == "zlib" ) { + zlib_decompress( blobData, blobSize, dataString ); + } else { + throw std::runtime_error( "Unknown compression format: " + compression ); + } + + std::istringstream stream( dataString ); + reader( stream ); + sqlite3_finalize( stmt ); + } else { + auto err = sqlite3_errmsg( db ); + sqlite3_finalize( stmt ); + + if( !optional ) { + dbg( DL::Error ) << "Failed to execute query: " << err << std::endl; + throw std::runtime_error( "DB query failed" ); + } + return false; + } + + return true; +} + +static bool read_from_db_json( sqlite3 *db, const std::string &path, file_read_json_fn reader, + bool optional ) +{ + return read_from_db( db, path, [&]( std::istream & fin ) { + JsonIn jsin( fin, path ); + reader( jsin ); + }, optional ); +} + world::world( WORLDINFO *info ) : info( info ) , save_tx_start_ts( 0 ) { - if( !assure_dir_exist( "" ) - || !assure_dir_exist( "/maps" ) ) { + if( !assure_dir_exist( "" ) ) { dbg( DL::Error ) << "Unable to create or open world directory structure: " << info->folder_path(); } + + if( info->world_save_format == save_format::V2_COMPRESSED_SQLITE3 ) { + map_db = open_db( info->folder_path() + "/map.sqlite3" ); + } else { + if( !assure_dir_exist( "/maps" ) ) { + dbg( DL::Error ) << "Unable to create or open world directory structure: " << info->folder_path(); + } + } } world::~world() { + if( save_tx_start_ts != 0 ) { + dbg( DL::Error ) << "Save transaction was not committed before world destruction"; + } + + if( map_db ) { + sqlite3_close( map_db ); + } + + if( save_db ) { + sqlite3_close( save_db ); + } } void world::start_save_tx() @@ -240,6 +447,14 @@ void world::start_save_tx() save_tx_start_ts = std::chrono::duration_cast< std::chrono::milliseconds >( std::chrono::system_clock::now().time_since_epoch() ).count(); + + if( map_db ) { + sqlite3_exec( map_db, "BEGIN TRANSACTION", NULL, NULL, NULL ); + } + + if( save_db ) { + sqlite3_exec( save_db, "BEGIN TRANSACTION", NULL, NULL, NULL ); + } } int64_t world::commit_save_tx() @@ -247,6 +462,15 @@ int64_t world::commit_save_tx() if( save_tx_start_ts == 0 ) { throw std::runtime_error( "Attempted to commit a save transaction while none was in progress" ); } + + if( map_db ) { + sqlite3_exec( map_db, "COMMIT", NULL, NULL, NULL ); + } + + if( save_db ) { + sqlite3_exec( save_db, "COMMIT", NULL, NULL, NULL ); + } + int64_t now = std::chrono::duration_cast< std::chrono::milliseconds >( std::chrono::system_clock::now().time_since_epoch() ).count(); @@ -275,19 +499,24 @@ bool world::read_map_quad( const tripoint &om_addr, file_read_json_fn reader ) c const std::string dirname = get_quad_dirname( om_addr ); std::string quad_path = dirname + "/" + get_quad_filename( om_addr ); - if( !file_exist( quad_path ) ) { - // Fix for old saves where the path was generated using std::stringstream, which - // did format the number using the current locale. That formatting may insert - // thousands separators, so the resulting path is "map/1,234.7.8.map" instead - // of "map/1234.7.8.map". - std::ostringstream buffer; - buffer << dirname << "/" << om_addr.x << "." << om_addr.y << "." << om_addr.z << ".map"; - if( file_exist( buffer.str() ) ) { - quad_path = buffer.str(); + // V2 logic + if( info->world_save_format == save_format::V2_COMPRESSED_SQLITE3 ) { + return read_from_db_json( map_db, quad_path, reader, true ); + } else { + if( !file_exist( quad_path ) ) { + // Fix for old saves where the path was generated using std::stringstream, which + // did format the number using the current locale. That formatting may insert + // thousands separators, so the resulting path is "map/1,234.7.8.map" instead + // of "map/1234.7.8.map". + std::ostringstream buffer; + buffer << dirname << "/" << om_addr.x << "." << om_addr.y << "." << om_addr.z << ".map"; + if( file_exist( buffer.str() ) ) { + quad_path = buffer.str(); + } } - } - return read_from_file_json( quad_path, reader, true ); + return read_from_file_json( quad_path, reader, true ); + } } bool world::write_map_quad( const tripoint &om_addr, file_write_fn writer ) const @@ -295,8 +524,14 @@ bool world::write_map_quad( const tripoint &om_addr, file_write_fn writer ) cons const std::string dirname = get_quad_dirname( om_addr ); std::string quad_path = dirname + "/" + get_quad_filename( om_addr ); - assure_dir_exist( dirname ); - return write_to_file( quad_path, writer ); + // V2 logic + if( info->world_save_format == save_format::V2_COMPRESSED_SQLITE3 ) { + write_to_db( map_db, quad_path, writer ); + return true; + } else { + assure_dir_exist( dirname ); + return write_to_file( quad_path, writer ); + } } /** @@ -315,27 +550,51 @@ std::string world::overmap_player_filename( const point_abs_om &p ) const bool world::overmap_exists( const point_abs_om &p ) const { - return file_exist( overmap_terrain_filename( p ) ); + if( info->world_save_format == save_format::V2_COMPRESSED_SQLITE3 ) { + return file_exist_in_db( map_db, overmap_terrain_filename( p ) ); + } else { + return file_exist( overmap_terrain_filename( p ) ); + } } bool world::read_overmap( const point_abs_om &p, file_read_fn reader ) const { - return read_from_file( overmap_terrain_filename( p ), reader, true ); + if( info->world_save_format == save_format::V2_COMPRESSED_SQLITE3 ) { + return read_from_db( map_db, overmap_terrain_filename( p ), reader, true ); + } else { + return read_from_file( overmap_terrain_filename( p ), reader, true ); + } } bool world::read_overmap_player_visibility( const point_abs_om &p, file_read_fn reader ) { - return read_from_player_file( overmap_player_filename( p ), reader, true ); + if( info->world_save_format == save_format::V2_COMPRESSED_SQLITE3 ) { + auto playerdb = get_player_db(); + return read_from_db( playerdb, overmap_player_filename( p ), reader, true ); + } else { + return read_from_player_file( overmap_player_filename( p ), reader, true ); + } } bool world::write_overmap( const point_abs_om &p, file_write_fn writer ) const { - return write_to_file( overmap_terrain_filename( p ), writer ); + if( info->world_save_format == save_format::V2_COMPRESSED_SQLITE3 ) { + write_to_db( map_db, overmap_terrain_filename( p ), writer ); + return true; + } else { + return write_to_file( overmap_terrain_filename( p ), writer ); + } } bool world::write_overmap_player_visibility( const point_abs_om &p, file_write_fn writer ) { - return write_to_player_file( overmap_player_filename( p ), writer ); + if( info->world_save_format == save_format::V2_COMPRESSED_SQLITE3 ) { + auto playerdb = get_player_db(); + write_to_db( playerdb, overmap_player_filename( p ), writer ); + return true; + } else { + return write_to_player_file( overmap_player_filename( p ), writer ); + } } /** @@ -348,17 +607,28 @@ static std::string get_mm_filename( const tripoint &p ) bool world::read_player_mm_quad( const tripoint &p, file_read_json_fn reader ) { - return read_from_player_file_json( ".mm1/" + get_mm_filename( p ), reader, true ); + if( info->world_save_format == save_format::V2_COMPRESSED_SQLITE3 ) { + auto playerdb = get_player_db(); + return read_from_db_json( playerdb, get_mm_filename( p ), reader, true ); + } else { + return read_from_player_file_json( ".mm1/" + get_mm_filename( p ), reader, true ); + } } bool world::write_player_mm_quad( const tripoint &p, file_write_fn writer ) { - const std::string descr = string_format( - _( "memory map region for (%d,%d,%d)" ), - p.x, p.y, p.z - ); - assure_dir_exist( get_player_path() + ".mm1" ); - return write_to_player_file( ".mm1/" + get_mm_filename( p ), writer, descr.c_str() ); + if( info->world_save_format == save_format::V2_COMPRESSED_SQLITE3 ) { + auto playerdb = get_player_db(); + write_to_db( playerdb, get_mm_filename( p ), writer ); + return true; + } else { + const std::string descr = string_format( + _( "memory map region for (%d,%d,%d)" ), + p.x, p.y, p.z + ); + assure_dir_exist( get_player_path() + ".mm1" ); + return write_to_player_file( ".mm1/" + get_mm_filename( p ), writer, descr.c_str() ); + } } /** @@ -370,6 +640,20 @@ std::string world::get_player_path() const return base64_encode( g->u.get_save_id() ); } +sqlite3 *world::get_player_db() +{ + if( !save_db ) { + save_db = open_db( info->folder_path() + "/" + get_player_path() + ".sqlite3" ); + last_save_id = g->u.get_save_id(); + } + + if( last_save_id != g->u.get_save_id() ) { + throw std::runtime_error( "Save ID changed without reloading the world object" ); + } + + return save_db; +} + bool world::player_file_exist( const std::string &path ) { return file_exist( get_player_path() + path ); @@ -423,4 +707,121 @@ bool world::read_from_file_json( const std::string &path, file_read_json_fn read bool optional ) const { return ::read_from_file_json( info->folder_path() + "/" + path, reader, optional ); -} \ No newline at end of file +} + +void replaceBackslashes( std::string &input ) +{ + std::size_t pos = 0; + while( ( pos = input.find( '\\', pos ) ) != std::string::npos ) { + input.replace( pos, 1, "/" ); + pos++; // Move past the replaced character + } +} + +/** + * Save Conversion + */ +void world::convert_from_v1( const std::unique_ptr &old_world ) +{ + dbg( DL::Info ) << "Converting world '" << info->world_name << "' from v1 to v2 format"; + + // The map database should already be loaded via the constructor. + // The save database(s) will need to be created separately here. + // Transactions are mostly being used for performance reasons rather than consistency. + sqlite3_exec( map_db, "BEGIN TRANSACTION", NULL, NULL, NULL ); + + // Keep track of the last used save DB + sqlite3 *last_save_db = nullptr; + std::string last_save_id = ""; + + // Begin copying files to the new world folder. + // This method is BFS, so we'll need to run two passes to keep player-specific + // files together. + auto old_world_path = old_world->folder_path() + "/"; + auto root_paths = get_files_from_path( "", old_world->folder_path(), false, true ); + for( auto &file_path : root_paths ) { + // Remove the old world path prefix from the file path + std::string part = file_path.substr( old_world_path.size() ); + replaceBackslashes( part ); + + // Migrate contents of the maps/ directory into the map database + if( part == "maps" ) { + // Recurse down the directory tree and migrate files into sqlite. + auto subpaths = get_files_from_path( "", file_path, true, true ); + for( auto &subpath : subpaths ) { + std::string map_path = "maps/" + subpath.substr( file_path.size() + 1 ); + replaceBackslashes( map_path ); + if( !map_path.ends_with( ".map" ) ) { + continue; + } + ::read_from_file( subpath, [&]( std::istream & fin ) { + write_to_db( map_db, map_path, [&]( std::ostream & fout ) { + fout << fin.rdbuf(); + } ); + } ); + } + continue; + } + + // Migrate o.* files into the map database + if( part.starts_with( "o." ) ) { + ::read_from_file( file_path, [&]( std::istream & fin ) { + write_to_db( map_db, part, [&]( std::ostream & fout ) { + fout << fin.rdbuf(); + } ); + } ); + continue; + } + + // Handle player-specific prefixed files + if( part.find( ".seen." ) != std::string::npos || part.find( ".mm1" ) != std::string::npos ) { + auto save_id = part.substr( 0, part.find( "." ) ); + if( save_id != last_save_id ) { + if( last_save_db ) { + sqlite3_exec( last_save_db, "COMMIT", NULL, NULL, NULL ); + sqlite3_close( last_save_db ); + } + last_save_db = open_db( info->folder_path() + "/" + save_id + ".sqlite3" ); + last_save_id = save_id; + sqlite3_exec( last_save_db, "BEGIN TRANSACTION", NULL, NULL, NULL ); + } + + if( part.find( ".seen." ) != std::string::npos ) { + ::read_from_file( file_path, [&]( std::istream & fin ) { + write_to_db( last_save_db, part.substr( save_id.size() ), [&]( std::ostream & fout ) { + fout << fin.rdbuf(); + } ); + } ); + } else { + // Recurse down the directory tree and migrate files into sqlite. + auto subpaths = get_files_from_path( "", file_path, true, true ); + for( auto &subpath : subpaths ) { + std::string map_path = subpath.substr( file_path.size() + 1 ); + replaceBackslashes( map_path ); + if( map_path.ends_with( '/' ) ) { + continue; + } + ::read_from_file( subpath, [&]( std::istream & fin ) { + write_to_db( last_save_db, map_path, [&]( std::ostream & fout ) { + fout << fin.rdbuf(); + } ); + } ); + } + } + + continue; + } + + // Copy all other files as-is + if( !part.ends_with( "/" ) ) { + copy_file( file_path, info->folder_path() + "/" + part ); + } + } + + if( last_save_db ) { + sqlite3_exec( last_save_db, "COMMIT", NULL, NULL, NULL ); + sqlite3_close( last_save_db ); + } + + sqlite3_exec( map_db, "COMMIT", NULL, NULL, NULL ); +} diff --git a/src/world.h b/src/world.h index 4081a77dca8a..0e929aee6716 100644 --- a/src/world.h +++ b/src/world.h @@ -11,6 +11,7 @@ #include "fstream_utils.h" class avatar; +class sqlite3; class save_t { @@ -39,6 +40,9 @@ class save_t enum save_format : int { /** Original save layout; uncompressed JSON as loose files */ V1 = 0, + + /** V2 format - compressed tuples in SQLite3 */ + V2_COMPRESSED_SQLITE3 = 1, }; /** @@ -157,6 +161,12 @@ class world bool read_from_file_json( const std::string &path, file_read_json_fn reader, bool optional = true ) const; + /** + * Convert (copy) the save data from the old format to the new format. + * This should only be called from `worldfactory`. + */ + void convert_from_v1( const std::unique_ptr &old_world ); + private: /** If non-zero, indicates we're in the middle of a save event */ int64_t save_tx_start_ts = 0; @@ -165,6 +175,11 @@ class world std::string overmap_player_filename( const point_abs_om &p ) const; std::string get_player_path() const; + sqlite3 *map_db = nullptr; + + sqlite3 *save_db = nullptr; + std::string last_save_id = ""; + sqlite3 *get_player_db(); }; #endif // CATA_SRC_WORLD_H diff --git a/src/worldfactory.cpp b/src/worldfactory.cpp index ae40414898dd..3bb0a6991941 100644 --- a/src/worldfactory.cpp +++ b/src/worldfactory.cpp @@ -196,8 +196,12 @@ void worldfactory::init() all_worlds[worldname] = std::make_unique(); // give the world a name all_worlds[worldname]->world_name = worldname; - // Record the world save format. Only one exists at this time. - all_worlds[worldname]->world_save_format = save_format::V1; + // Record the world save format. V2 is identified by the presence of a map.sqlite3 file. + if( file_exist( world_dir + "/map.sqlite3" ) ) { + all_worlds[worldname]->world_save_format = save_format::V2_COMPRESSED_SQLITE3; + } else { + all_worlds[worldname]->world_save_format = save_format::V1; + } // add sav files for( auto &world_sav_file : world_sav_files ) { all_worlds[worldname]->world_saves.push_back( save_t::from_base_path( world_sav_file ) ); @@ -1275,6 +1279,7 @@ int worldfactory::show_worldgen_tab_confirm( const catacurses::window &win, WORL ctxt.register_action( "NEXT_TAB" ); ctxt.register_action( "PREV_TAB" ); ctxt.register_action( "PICK_RANDOM_WORLDNAME" ); + ctxt.register_action( "TOGGLE_V2_SAVE_FORMAT" ); // string input popup actions ctxt.register_action( "TEXT.LEFT" ); ctxt.register_action( "TEXT.RIGHT" ); @@ -1324,6 +1329,22 @@ int worldfactory::show_worldgen_tab_confirm( const catacurses::window &win, WORL fold_and_print( w_confirmation, point( 2, 3 ), getmaxx( w_confirmation ) - 2, c_light_gray, _( "Press [%s] to pick a random name for your world." ), ctxt.get_desc( "PICK_RANDOM_WORLDNAME" ) ); + + if( world->world_save_format == save_format::V2_COMPRESSED_SQLITE3 ) { + mvwprintz( w_confirmation, point( 2, 6 ), c_cyan, + _( "Save Format: Experimental V2 save format" ) ); + } else { + mvwprintz( w_confirmation, point( 2, 6 ), c_white, + _( "Save Format: Standard (V1) save format" ) ); + } + + fold_and_print( w_confirmation, point( 2, 8 ), getmaxx( w_confirmation ) - 2, c_light_gray, + _( "[Experimental] Press [%s] to toggle save format.\n" + "The new format shrinks save files and reduces save corruption, at the cost of " + "slightly slower saves. You can opt into this later by converting an existing world to V2" + " from the main menu. V2 worlds cannot currently be converted back to V1." ), + ctxt.get_desc( "TOGGLE_V2_SAVE_FORMAT" ) ); + fold_and_print( w_confirmation, point( 2, TERMY / 2 - 2 ), getmaxx( w_confirmation ) - 2, c_light_gray, _( "Press [%s] when you are satisfied with the world as it is and are ready " @@ -1372,6 +1393,12 @@ int worldfactory::show_worldgen_tab_confirm( const catacurses::window &win, WORL return -1; } else if( action == "PICK_RANDOM_WORLDNAME" ) { world->world_name = worldname = pick_random_name(); + } else if( action == "TOGGLE_V2_SAVE_FORMAT" ) { + if( world->world_save_format == save_format::V2_COMPRESSED_SQLITE3 ) { + world->world_save_format = save_format::V1; + } else { + world->world_save_format = save_format::V2_COMPRESSED_SQLITE3; + } } else if( action == "QUIT" && ( !on_quit || on_quit() ) ) { world->world_name = worldname; return -999; @@ -1580,3 +1607,45 @@ void worldfactory::delete_world( const std::string &worldname, const bool delete get_world( worldname )->world_saves.clear(); } } + +void worldfactory::convert_to_v2( const std::string &worldname ) +{ + // Ensure we're ready to convert + WORLDINFO *worldinfo = get_world( worldname ); + if( worldinfo == nullptr ) { + popup( _( "Tried to convert non-existing world %s to v2" ), worldname ); + return; + } + + if( worldinfo->world_save_format != save_format::V1 ) { + popup( _( "World %s is already at savefile version 2" ), worldname ); + return; + } + + // Backup the world by renaming it to a new name + std::string backup_name = worldname + " (V2 Conversion Backup)"; + if( has_world( backup_name ) ) { + popup( _( "Backup world '%s' already exists, aborting conversion" ), backup_name ); + return; + } + + std::unique_ptr old_world = std::make_unique(); + old_world->COPY_WORLD( world_generator->get_world( worldname ) ); + old_world->world_name = backup_name; + + // Deep copy the world saves + std::vector world_saves_copy( worldinfo->world_saves ); + old_world->world_saves = worldinfo->world_saves; + + // Rename the world folder perform the move + rename_file( worldinfo->folder_path(), old_world->folder_path() ); + worldinfo->world_save_format = save_format::V2_COMPRESSED_SQLITE3; + world new_world( worldinfo ); + new_world.convert_from_v1( old_world ); + add_world( std::move( old_world ) ); + + // Save the world + worldinfo->save(); + + popup( _( "Conversion Complete!" ) ); +} \ No newline at end of file diff --git a/src/worldfactory.h b/src/worldfactory.h index d8cef9486538..3c34615ac0a3 100644 --- a/src/worldfactory.h +++ b/src/worldfactory.h @@ -79,6 +79,8 @@ class worldfactory void show_active_world_mods( const std::vector &world_mods ); void edit_active_world_mods( WORLDINFO *world ); + void convert_to_v2( const std::string &worldname ); + private: std::map> all_worlds;