feat: SQLite+Compressed save format (Phase 2 of v2 save format) (#5777)

* fix: clang-tidy should ignore changes to non-source files The `clang-tidy` CI step is breaking because it's trying to interpret all files in src/ and tests/ as CPP files. Thus when src/CMakeLists.txt is changed, it tries to parse that file as C++ and fails. Signed-off-by: David Li <[email protected]> * feat: V2 save format Implement new SQLite-based save format. New worlds can be configured to use V2. Existing worlds can be converted from the main menu. Existing V1 worlds should load exactly the same as before until converted. Signed-off-by: David Li <[email protected]> # Conflicts: # .github/workflows/manual-release.yml # .github/workflows/release.yml # Conflicts: # msvc-full-features/vcpkg.json * fix: Compiler warnings, OSX+Android Build Signed-off-by: David Li <[email protected]> * docs: Update docs with build instructions for v2 saves Signed-off-by: David Li <[email protected]> * Avoid ever-growing world names from the V2 conversion Ask the user to rename the backup world before they attempt another conversion. Signed-off-by: David Li <[email protected]> --------- Signed-off-by: David Li <[email protected]> Co-authored-by: Chaosvolt <[email protected]> Co-authored-by: scarf <[email protected]>
cataclysmbnteam · Jan 19, 2025 · cac9b08 · cac9b08
1 parent e075571
commit cac9b08
Show file tree

Hide file tree

Showing 27 changed files with 724 additions and 49 deletions.
diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml
@@ -42,7 +42,7 @@ jobs:
       - name: Setup build and dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install gettext
+          sudo apt-get install gettext libsqlite3-dev zlib1g-dev
 
       - name: Build
         working-directory: ./android

diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml
@@ -50,7 +50,8 @@ jobs:
           sudo apt-get install \
             cmake gettext ninja-build mold ccache jq \
             clang-18 libclang-18-dev llvm-18 llvm-18-dev clang-tidy-18 \
-            libsdl2-dev libsdl2-ttf-dev libsdl2-image-dev libsdl2-mixer-dev libpulse-dev libflac-dev
+            libsdl2-dev libsdl2-ttf-dev libsdl2-image-dev libsdl2-mixer-dev libpulse-dev libflac-dev \
+            sqlite3 libsqlite3-dev zlib1g-dev
 
       - name: ensure clang 18 is installed
         run: |

diff --git a/.github/workflows/manual-release.yml b/.github/workflows/manual-release.yml
@@ -228,11 +228,11 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install libncursesw5-dev libsdl2-dev libsdl2-ttf-dev libsdl2-image-dev \
-            libsdl2-mixer-dev libpulse-dev ccache gettext parallel
+            libsdl2-mixer-dev libpulse-dev ccache gettext parallel libsqlite3-dev zlib1g-dev
       - name: Install dependencies (mac)
         if: runner.os == 'macOS'
         run: |
-          HOMEBREW_NO_AUTO_UPDATE=yes HOMEBREW_NO_INSTALL_CLEANUP=yes brew install sdl2 sdl2_image sdl2_ttf sdl2_mixer gettext ccache parallel llvm astyle
+          HOMEBREW_NO_AUTO_UPDATE=yes HOMEBREW_NO_INSTALL_CLEANUP=yes brew install sdl2 sdl2_image sdl2_ttf sdl2_mixer gettext ccache parallel llvm astyle sqlite3 zlib
           python3 -m venv ./venv
           source ./venv/bin/activate
           pip3 install mac_alias==2.2.0 dmgbuild==1.6.1 biplist polib luaparser

diff --git a/.github/workflows/matrix.yml b/.github/workflows/matrix.yml
@@ -156,7 +156,7 @@ jobs:
         if: ${{ env.SKIP == 'false' && runner.os == 'Linux' }}
         run: |
           sudo apt-get update
-          sudo apt-get install libncursesw5-dev ccache gettext parallel
+          sudo apt-get install libncursesw5-dev ccache gettext parallel libsqlite3-dev zlib1g-dev
           sudo locale-gen en_US.UTF-8 fr_FR.UTF-8 ru_RU.UTF-8
 
       - name: install SDL2 dependencies (ubuntu)
@@ -189,7 +189,7 @@ jobs:
           # Ensure that it is actually needed version
           ${{ matrix.compiler }} --version | grep -q -E "${{ matrix.grep_clang_version_rxp }}"
 
-          HOMEBREW_NO_AUTO_UPDATE=yes HOMEBREW_NO_INSTALL_CLEANUP=yes brew install gettext ccache parallel
+          HOMEBREW_NO_AUTO_UPDATE=yes HOMEBREW_NO_INSTALL_CLEANUP=yes brew install gettext ccache parallel sqlite3 zlib
 
       # problem matchers are only ran on GCC to reduce error spam
       - name: add problem matcher

diff --git a/.github/workflows/msys2-cmake.yml b/.github/workflows/msys2-cmake.yml
@@ -55,6 +55,8 @@ jobs:
           mingw-w64-x86_64-SDL2_ttf
           mingw-w64-x86_64-toolchain
           mingw-w64-x86_64-cmake
+          mingw-w64-x86_64-sqlite3
+          mingw-w64-x86_64-zstd
 
     - name: Create build directory
       run: mkdir build

diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml
@@ -80,7 +80,7 @@ jobs:
 
       - name: Install build dependencies
         run: |
-          HOMEBREW_NO_AUTO_UPDATE=yes HOMEBREW_NO_INSTALL_CLEANUP=yes brew install sdl2 sdl2_image sdl2_ttf sdl2_mixer gettext ccache parallel llvm astyle
+          HOMEBREW_NO_AUTO_UPDATE=yes HOMEBREW_NO_INSTALL_CLEANUP=yes brew install sdl2 sdl2_image sdl2_ttf sdl2_mixer gettext ccache parallel llvm astyle sqlite3 zlib
           python3 -m venv ./venv
           source ./venv/bin/activate
           pip3 install mac_alias==2.2.0 dmgbuild==1.6.1 biplist polib luaparser

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -262,7 +262,7 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install libncursesw5-dev libsdl2-dev libsdl2-ttf-dev libsdl2-image-dev \
-            libsdl2-mixer-dev libpulse-dev ccache gettext parallel
+            libsdl2-mixer-dev libpulse-dev ccache gettext parallel libsqlite3-dev zlib1g-dev
 
       - name: Install runtime dependencies (mac)
         if: runner.os == 'macOS'
@@ -275,7 +275,7 @@ jobs:
       - name: Install build dependencies (mac)
         if: runner.os == 'macOS'
         run: |
-          HOMEBREW_NO_AUTO_UPDATE=yes HOMEBREW_NO_INSTALL_CLEANUP=yes brew install gettext ccache parallel llvm astyle
+          HOMEBREW_NO_AUTO_UPDATE=yes HOMEBREW_NO_INSTALL_CLEANUP=yes brew install gettext ccache parallel llvm astyle sqlite3 zlib
           python3 -m venv ./venv
           source ./venv/bin/activate
           pip3 install mac_alias==2.2.0 dmgbuild==1.6.1 biplist polib luaparser

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -113,6 +113,8 @@ include(CheckCXXCompilerFlag)
 #SET(CMAKE_SHARED_LIBRARY_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CXX_FLAGS} -m32")
 
 find_package(PkgConfig)
+find_package(SQLite3)
+find_package(ZLIB)
 if (NOT DYNAMIC_LINKING)
     if(NOT MSVC)
         set(CMAKE_FIND_LIBRARY_SUFFIXES ".a;.dll.a")

diff --git a/Makefile b/Makefile
@@ -607,6 +607,11 @@ endif
 
 PKG_CONFIG = $(CROSS)pkg-config
 
+CXXFLAGS += $(shell $(PKG_CONFIG) --cflags sqlite3)
+LDFLAGS += $(shell $(PKG_CONFIG) --libs sqlite3)
+CXXFLAGS += $(shell $(PKG_CONFIG) --cflags zlib)
+LDFLAGS += $(shell $(PKG_CONFIG) --libs zlib)
+
 ifeq ($(SOUND), 1)
   ifneq ($(TILES),1)
     $(error "SOUND=1 only works with TILES=1")

diff --git a/android/.gitignore b/android/.gitignore
@@ -11,6 +11,8 @@
 /app/jni/SDL2_image
 /app/jni/SDL2_mixer
 /app/jni/SDL2_ttf
+/app/jni/sqlite3/*.c
+/app/jni/sqlite3/*.h
 /app/jni/libintl-lite
 /app/jni/libhidapi
 /app/jni/lua

diff --git a/android/app/build.gradle b/android/app/build.gradle
@@ -142,6 +142,29 @@ task unzipDeps(type: Copy) {
     into outputDir
 }
 
+task fetchSqlite {
+    doLast {
+        def url = new URL('https://www.sqlite.org/2025/sqlite-amalgamation-3480000.zip')
+        def file = new File('/tmp/sqlite-amalgamation-3480000.zip')
+        file.withOutputStream { out ->
+            url.withInputStream { from -> out << from }
+        }
+    }
+}
+
+task unzipSqlite(type: Copy) {
+    dependsOn fetchSqlite
+    from(zipTree(file('/tmp/sqlite-amalgamation-3480000.zip'))) {
+        includeEmptyDirs = false
+        eachFile { fileCopyDetails ->
+            if (fileCopyDetails.path.startsWith('sqlite-amalgamation-3480000/')) {
+                fileCopyDetails.path = fileCopyDetails.path - 'sqlite-amalgamation-3480000/'
+            }
+        }
+    }
+    into 'jni/sqlite3'
+}
+
 task makeLocalization(type: Exec) {
     if (localize) {
         println("Building with localization'")
@@ -168,6 +191,7 @@ task makeLocalization(type: Exec) {
 makeLocalization
 unzipDeps.dependsOn makeLocalization
 preBuild.dependsOn unzipDeps
+preBuild.dependsOn unzipSqlite
 
 android {
     namespace "com.cleverraven.cataclysmdda"

diff --git a/android/app/jni/sqlite3/Android.mk b/android/app/jni/sqlite3/Android.mk
@@ -0,0 +1,49 @@
+# Reference: https://developer.android.com/ndk/guides/android_mk.html
+
+# Based on https://sqlite.org/android/file?name=sqlite3/src/main/jni/sqlite/Android.mk&ci=tip
+
+LOCAL_PATH:= $(call my-dir)
+include $(CLEAR_VARS)
+
+# If using SEE, uncomment the following:
+# LOCAL_CFLAGS += -DSQLITE_HAS_CODEC
+
+#Define HAVE_USLEEP, otherwise ALL sleep() calls take at least 1000ms
+LOCAL_CFLAGS += -DHAVE_USLEEP=1
+
+# Enable SQLite extensions.
+LOCAL_CFLAGS += -DSQLITE_ENABLE_FTS5 
+LOCAL_CFLAGS += -DSQLITE_ENABLE_RTREE
+LOCAL_CFLAGS += -DSQLITE_ENABLE_FTS3
+LOCAL_CFLAGS += -DSQLITE_ENABLE_BATCH_ATOMIC_WRITE
+
+# This is important - it causes SQLite to use memory for temp files. Since 
+# Android has no globally writable temp directory, if this is not defined the
+# application throws an exception when it tries to create a temp file.
+#
+LOCAL_CFLAGS += -DSQLITE_TEMP_STORE=3
+
+LOCAL_CFLAGS += -DHAVE_CONFIG_H -DKHTML_NO_EXCEPTIONS -DGKWQ_NO_JAVA
+LOCAL_CFLAGS += -DNO_SUPPORT_JS_BINDING -DQT_NO_WHEELEVENT -DKHTML_NO_XBL
+LOCAL_CFLAGS += -U__APPLE__
+LOCAL_CFLAGS += -DHAVE_STRCHRNUL=0
+LOCAL_CFLAGS += -DSQLITE_USE_URI=1
+LOCAL_CFLAGS += -Wno-unused-parameter -Wno-int-to-pointer-cast
+LOCAL_CFLAGS += -Wno-uninitialized -Wno-parentheses
+LOCAL_CPPFLAGS += -Wno-conversion-null
+
+ifeq ($(TARGET_ARCH), arm)
+	LOCAL_CFLAGS += -DPACKED="__attribute__ ((packed))"
+else
+	LOCAL_CFLAGS += -DPACKED=""
+endif
+
+LOCAL_SRC_FILES := sqlite3.c
+
+LOCAL_C_INCLUDES += $(LOCAL_PATH)
+LOCAL_EXPORT_C_INCLUDES := $(LOCAL_PATH)
+
+LOCAL_MODULE := libsqlite3
+LOCAL_LDLIBS += -ldl -llog 
+
+include $(BUILD_SHARED_LIBRARY)
diff --git a/data/raw/keybindings/keybindings.json b/data/raw/keybindings/keybindings.json
@@ -744,6 +744,13 @@
     "name": "Pick random world name",
     "bindings": [ { "input_method": "keyboard", "key": "*" } ]
   },
+  {
+    "type": "keybinding",
+    "id": "TOGGLE_V2_SAVE_FORMAT",
+    "category": "WORLDGEN_CONFIRM_DIALOG",
+    "name": "Toggle new world save format",
+    "bindings": [ { "input_method": "keyboard", "key": "=" } ]
+  },
   {
     "type": "keybinding",
     "id": "QUIT",

diff --git a/doc/src/content/docs/en/dev/guides/building/atomic_cmake.md b/doc/src/content/docs/en/dev/guides/building/atomic_cmake.md
@@ -59,7 +59,8 @@ installation script. For Bazzite, that script looks like:
 ```sh
 $ sudo dnf install git cmake clang ninja-build mold ccache \
   SDL2-devel SDL2_image-devel SDL2_ttf-devel SDL2_mixer-devel \
-  freetype glibc bzip2 zlib-ng libvorbis ncurses gettext flac-devel
+  freetype glibc bzip2 zlib-ng libvorbis ncurses gettext flac-devel \
+  sqlite-devel zlib-devel
 ```
 
 After this, your container is set up for all your building needs in the future! Now, onto the steps

diff --git a/doc/src/content/docs/en/dev/guides/building/cmake.md b/doc/src/content/docs/en/dev/guides/building/cmake.md
@@ -21,6 +21,7 @@ CataclysmBN:
   - `glibc`
   - `zlib`
   - `bzip2`
+  - `sqlite3`
 - Curses
   - `ncurses`
 - Tiles
@@ -50,15 +51,17 @@ Obtain packages specified above with your system package manager.
 ```sh
 $ sudo apt install git cmake ninja-build mold clang ccache \ 
   libsdl2-dev libsdl2-image-dev libsdl2-ttf-dev libsdl2-mixer-dev \ 
-  freetype glibc bzip2 zlib libvorbis ncurses gettext libflac++-dev
+  freetype glibc bzip2 zlib libvorbis ncurses gettext libflac++-dev \
+  libsqlite3-dev zlib1g-dev
 ```
 
 - For Fedora-based distros:
 
 ```sh
 $ sudo dnf install git cmake ninja-build mold clang ccache \
   SDL2-devel SDL2_image-devel SDL2_ttf-devel SDL2_mixer-devel \
-  freetype glibc bzip2 zlib-ng libvorbis ncurses gettext flac-devel
+  freetype glibc bzip2 zlib-ng libvorbis ncurses gettext flac-devel \
+  sqlite-devel zlib-devel
 ```
 
 ### Windows Environment (MSYS2)

diff --git a/doc/src/content/docs/en/dev/guides/building/makefile.md b/doc/src/content/docs/en/dev/guides/building/makefile.md
@@ -58,7 +58,7 @@ your distro packages libraries and their development files separately (e.g. Debi
 
 Rough list based on building on Arch:
 
-- General: `gcc-libs`, `glibc`, `zlib`, `bzip2`
+- General: `gcc-libs`, `glibc`, `zlib`, `bzip2`, `sqlite3`
 - Optional: `intltool`
 - Curses: `ncurses`
 - Tiles: `sdl2`, `sdl2_image`, `sdl2_ttf`, `sdl2_mixer`, `freetype2`
@@ -137,7 +137,7 @@ Dependencies:
 Install:
 
 ```sh
-sudo apt-get install libncurses5-dev libncursesw5-dev build-essential astyle
+sudo apt-get install libncurses5-dev libncursesw5-dev build-essential astyle libsqlite3-dev zlib1g-dev
 ```
 
 ### Building

diff --git a/msvc-full-features/vcpkg.json b/msvc-full-features/vcpkg.json
@@ -5,7 +5,8 @@
     "sdl2",
     { "name": "sdl2-image", "features": [ "libjpeg-turbo" ] },
     { "name": "sdl2-mixer", "features": [ "libflac", "mpg123", "libmodplug" ] },
-    "sdl2-ttf"
+    "sdl2-ttf",
+    "sqlite3", "zlib"
   ],
   "builtin-baseline": "b322364f06308bdd24823f9d8f03fe0cc86fd46f",
   "overrides": [

diff --git a/scripts/affected_files.ts b/scripts/affected_files.ts
@@ -25,7 +25,7 @@ const getDiffs = async (pr: number) => {
     octokit.rest.pulls.listFiles,
     { owner: "cataclysmbnteam", repo: "Cataclysm-BN", pull_number: pr },
   )
-  const pathsFilter = new RegExp(`(^${paths.join("|")})/`)
+  const pathsFilter = new RegExp(`(^${paths.join("|")})/.*\\.(cpp|h|hpp)`)
 
   return res
     .filter((x) => ACMRT.has(x.status))
@@ -79,7 +79,7 @@ const main = new Command()
       getDiffs(pr).then((xs) => partition(xs, (x) => x.endsWith(".h"))),
     ])
     // console.log(deps)
-    // console.log({ src, headers })
+    console.log({ src, headers })
 
     const affected = new Set(src.concat(headers.flatMap((x) => Array.from(deps.get(x) ?? []))))
     console.log(affected)

diff --git a/src/Android.mk b/src/Android.mk
@@ -17,9 +17,9 @@ LOCAL_SRC_FILES := $(FILE_LIST:$(LOCAL_PATH)/%=%)
 FILE_LIST_LUA := $(wildcard $(LOCAL_PATH)/lua/*.c)
 LOCAL_SRC_FILES += $(FILE_LIST_LUA:$(LOCAL_PATH)/%=%)
 
-LOCAL_SHARED_LIBRARIES := libhidapi SDL2 SDL2_mixer SDL2_image SDL2_ttf mpg123
+LOCAL_SHARED_LIBRARIES := libhidapi SDL2 SDL2_mixer SDL2_image SDL2_ttf mpg123 libsqlite3
 
-LOCAL_LDLIBS := -lGLESv1_CM -lGLESv2 -llog
+LOCAL_LDLIBS := -lGLESv1_CM -lGLESv2 -ldl -llog -lz
 
 LOCAL_CFLAGS += -DTILES=1 -DSDL_SOUND=1 -DLUA=1 -DBACKTRACE=1 -Wextra -Wall -fsigned-char -ffast-math
 

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -61,6 +61,8 @@ if (TILES)
 
     add_dependencies(cataclysm-bn-tiles-common get_version)
 
+    target_link_libraries(cataclysm-bn-tiles-common PUBLIC ZLIB::ZLIB)
+    target_link_libraries(cataclysm-bn-tiles-common PUBLIC SQLite::SQLite3)
     target_link_libraries(cataclysm-bn-tiles PRIVATE cataclysm-bn-tiles-common)
     target_compile_definitions(cataclysm-bn-tiles-common PUBLIC TILES )
 
@@ -176,9 +178,12 @@ if (CURSES)
     endif ()
 
     add_dependencies(cataclysm-bn-common get_version)
+
     target_link_libraries(cataclysm-bn PRIVATE cataclysm-bn-common)
 
     target_include_directories(cataclysm-bn-common PUBLIC ${CURSES_INCLUDE_DIR})
+    target_link_libraries(cataclysm-bn-common PUBLIC ZLIB::ZLIB)
+    target_link_libraries(cataclysm-bn-common PUBLIC SQLite::SQLite3)
     target_link_libraries(cataclysm-bn-common PUBLIC ${CURSES_LIBRARIES})
 
     if (CMAKE_USE_PTHREADS_INIT)

diff --git a/src/compress.cpp b/src/compress.cpp
@@ -0,0 +1,53 @@
+#include "compress.h"
+
+#include <zlib.h>
+#include <vector>
+#include <string>
+#include <stdexcept>
+#include <cstddef>
+
+void zlib_compress( const std::string &input, std::vector<std::byte> &output )
+{
+    uLongf compressedSize = compressBound( input.size() );
+    output.resize( compressedSize );
+
+    int result = compress2(
+                     reinterpret_cast<Bytef *>( output.data() ),
+                     &compressedSize,
+                     reinterpret_cast<const Bytef *>( input.data() ),
+                     input.size(),
+                     Z_BEST_SPEED
+                 );
+
+    if( result != Z_OK ) {
+        throw std::runtime_error( "Zlib compression error" );
+    }
+
+    output.resize( compressedSize );
+}
+
+void zlib_decompress( const void *compressed_data, int compressed_size, std::string &output )
+{
+    // We need to guess at the decompressed size - we expect things to compress fairly well.
+    uLongf decompressedSize = static_cast<uLongf>( compressed_size ) * 8;
+    output.resize( decompressedSize );
+
+    int result;
+    do {
+        result = uncompress(
+                     reinterpret_cast<Bytef *>( output.data() ),
+                     &decompressedSize,
+                     reinterpret_cast<const Bytef *>( compressed_data ),
+                     compressed_size
+                 );
+
+        if( result == Z_BUF_ERROR ) {
+            decompressedSize *= 2; // Double the buffer size and retry
+            output.resize( decompressedSize );
+        } else if( result != Z_OK ) {
+            throw std::runtime_error( "Zlib decompression failed" );
+        }
+    } while( result == Z_BUF_ERROR );
+
+    output.resize( decompressedSize );
+}