Merge pull request fedimint#4018 from m1sterc001guy/dummy_migration_t…

…ests

test: Run Dummy Migration Tests

db/migrations/dummy-server-v0/CURRENT

@@ -0,0 +1 @@
+MANIFEST-000005

db/migrations/dummy-server-v0/IDENTITY

@@ -0,0 +1 @@
+5ed017c3-3f04-4cd2-a6d7-404197b2b652

db/migrations/dummy-server-v0/OPTIONS-000007

@@ -0,0 +1,200 @@
+# This is a RocksDB option file.
+#
+# For detailed file format spec, please refer to the example file
+# in examples/rocksdb_option_file_example.ini
+#
+
+[Version]
+  rocksdb_version=8.3.2
+  options_file_version=1.1
+
+[DBOptions]
+  compaction_readahead_size=0
+  strict_bytes_per_sync=false
+  bytes_per_sync=0
+  max_background_jobs=2
+  avoid_flush_during_shutdown=false
+  max_background_flushes=-1
+  delayed_write_rate=16777216
+  max_open_files=-1
+  max_subcompactions=1
+  writable_file_max_buffer_size=1048576
+  wal_bytes_per_sync=0
+  max_background_compactions=-1
+  max_total_wal_size=0
+  delete_obsolete_files_period_micros=21600000000
+  stats_dump_period_sec=600
+  stats_history_buffer_size=1048576
+  stats_persist_period_sec=600
+  enforce_single_del_contracts=true
+  lowest_used_cache_tier=kNonVolatileBlockTier
+  bgerror_resume_retry_interval=1000000
+  best_efforts_recovery=false
+  log_readahead_size=0
+  write_dbid_to_manifest=false
+  wal_compression=kNoCompression
+  manual_wal_flush=false
+  db_host_id=__hostname__
+  two_write_queues=false
+  random_access_max_buffer_size=1048576
+  avoid_unnecessary_blocking_io=false
+  skip_checking_sst_file_sizes_on_db_open=false
+  flush_verify_memtable_count=true
+  fail_if_options_file_error=false
+  atomic_flush=false
+  verify_sst_unique_id_in_manifest=true
+  skip_stats_update_on_db_open=false
+  track_and_verify_wals_in_manifest=false
+  paranoid_checks=true
+  create_if_missing=true
+  max_write_batch_group_size_bytes=1048576
+  avoid_flush_during_recovery=false
+  file_checksum_gen_factory=nullptr
+  enable_thread_tracking=false
+  allow_fallocate=true
+  allow_data_in_errors=false
+  error_if_exists=false
+  use_direct_io_for_flush_and_compaction=false
+  create_missing_column_families=false
+  WAL_size_limit_MB=0
+  use_direct_reads=false
+  persist_stats_to_disk=false
+  allow_mmap_reads=false
+  allow_mmap_writes=false
+  use_adaptive_mutex=false
+  allow_2pc=false
+  is_fd_close_on_exec=true
+  max_log_file_size=0
+  access_hint_on_compaction_start=NORMAL
+  max_file_opening_threads=16
+  wal_filter=nullptr
+  use_fsync=false
+  table_cache_numshardbits=6
+  dump_malloc_stats=false
+  db_write_buffer_size=0
+  allow_ingest_behind=false
+  keep_log_file_num=1000
+  max_bgerror_resume_count=2147483647
+  allow_concurrent_memtable_write=true
+  recycle_log_file_num=0
+  log_file_time_to_roll=0
+  manifest_preallocation_size=4194304
+  enable_write_thread_adaptive_yield=true
+  WAL_ttl_seconds=0
+  max_manifest_file_size=1073741824
+  wal_recovery_mode=kPointInTimeRecovery
+  enable_pipelined_write=false
+  write_thread_slow_yield_usec=3
+  unordered_write=false
+  write_thread_max_yield_usec=100
+  advise_random_on_open=true
+  info_log_level=INFO_LEVEL
+
+
+[CFOptions "default"]
+  compression_opts={max_dict_buffer_bytes=0;enabled=false;max_dict_bytes=0;max_compressed_bytes_per_kb=896;parallel_threads=1;zstd_max_train_bytes=0;level=32767;use_zstd_dict_trainer=true;strategy=0;window_bits=-14;}
+  block_protection_bytes_per_key=0
+  memtable_protection_bytes_per_key=0
+  target_file_size_multiplier=1
+  report_bg_io_stats=false
+  write_buffer_size=67108864
+  memtable_huge_page_size=0
+  max_successive_merges=0
+  max_write_buffer_number=2
+  prefix_extractor=nullptr
+  bottommost_compression_opts={max_dict_buffer_bytes=0;enabled=false;max_dict_bytes=0;max_compressed_bytes_per_kb=896;parallel_threads=1;zstd_max_train_bytes=0;level=32767;use_zstd_dict_trainer=true;strategy=0;window_bits=-14;}
+  paranoid_file_checks=false
+  blob_garbage_collection_force_threshold=1.000000
+  enable_blob_files=false
+  blob_file_starting_level=0
+  memtable_prefix_bloom_size_ratio=0.000000
+  inplace_update_num_locks=10000
+  blob_compaction_readahead_size=0
+  ignore_max_compaction_bytes_for_input=true
+  arena_block_size=1048576
+  level0_stop_writes_trigger=36
+  blob_compression_type=kNoCompression
+  level0_slowdown_writes_trigger=20
+  hard_pending_compaction_bytes_limit=274877906944
+  soft_pending_compaction_bytes_limit=68719476736
+  target_file_size_base=67108864
+  level0_file_num_compaction_trigger=4
+  max_compaction_bytes=1677721600
+  disable_auto_compactions=false
+  check_flush_compaction_key_order=true
+  min_blob_size=0
+  memtable_whole_key_filtering=false
+  max_bytes_for_level_base=268435456
+  last_level_temperature=kUnknown
+  compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;}
+  max_bytes_for_level_multiplier=10.000000
+  max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
+  max_sequential_skip_in_iterations=8
+  prepopulate_blob_cache=kDisable
+  compression=kSnappyCompression
+  compaction_options_universal={incremental=false;compression_size_percent=-1;allow_trivial_move=false;max_size_amplification_percent=200;max_merge_width=4294967295;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;size_ratio=1;}
+  blob_garbage_collection_age_cutoff=0.250000
+  ttl=2592000
+  periodic_compaction_seconds=0
+  sample_for_compression=0
+  blob_file_size=268435456
+  enable_blob_garbage_collection=false
+  experimental_mempurge_threshold=0.000000
+  bottommost_compression=kDisableCompressionOption
+  persist_user_defined_timestamps=true
+  min_write_buffer_number_to_merge=1
+  preserve_internal_time_seconds=0
+  preclude_last_level_data_seconds=0
+  sst_partitioner_factory=nullptr
+  num_levels=7
+  force_consistency_checks=true
+  memtable_insert_with_hint_prefix_extractor=nullptr
+  memtable_factory=SkipListFactory
+  level_compaction_dynamic_file_size=true
+  max_write_buffer_number_to_maintain=0
+  optimize_filters_for_hits=false
+  level_compaction_dynamic_level_bytes=false
+  compaction_style=kCompactionStyleLevel
+  compaction_filter=nullptr
+  inplace_update_support=false
+  merge_operator=nullptr
+  table_factory=BlockBasedTable
+  bloom_locality=0
+  comparator=leveldb.BytewiseComparator
+  compaction_filter_factory=nullptr
+  max_write_buffer_size_to_maintain=134217728
+  compaction_pri=kMinOverlappingRatio
+
+[TableOptions/BlockBasedTable "default"]
+  initial_auto_readahead_size=8192
+  pin_top_level_index_and_filter=true
+  block_align=false
+  block_size_deviation=10
+  checksum=kXXH3
+  index_shortening=kShortenSeparators
+  num_file_reads_for_auto_readahead=2
+  whole_key_filtering=true
+  data_block_index_type=kDataBlockBinarySearch
+  index_type=kBinarySearch
+  no_block_cache=false
+  index_block_restart_interval=1
+  data_block_hash_table_util_ratio=0.750000
+  prepopulate_block_cache=kDisable
+  pin_l0_filter_and_index_blocks_in_cache=false
+  filter_policy=nullptr
+  cache_index_and_filter_blocks_with_high_priority=true
+  verify_compression=false
+  block_restart_interval=16
+  max_auto_readahead_size=262144
+  flush_block_policy_factory=FlushBlockBySizePolicyFactory
+  partition_filters=false
+  cache_index_and_filter_blocks=false
+  block_size=4096
+  metadata_block_size=4096
+  optimize_filters_for_memory=false
+  detect_filter_construct_corruption=false
+  format_version=5
+  metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;}
+  read_amp_bytes_per_bit=0
+  enable_index_compression=true
+

fedimint-core/src/db/mod.rs

@@ -1835,6 +1835,7 @@ pub async fn apply_migrations(
 
         while current_db_version < target_db_version {
             if let Some(migration) = migrations.get(&current_db_version) {
+                info!(target: LOG_DB, "Migrating module {kind} from {current_db_version} to {target_db_version}");
                 migration(&mut dbtx.to_ref_nc()).await?;
             } else {
                 panic!("Missing migration for version {current_db_version}");

fedimint-server/src/db.rs

@@ -147,6 +147,7 @@ mod fedimint_migration_tests {
     use fedimint_core::{Amount, PeerId, ServerModule, TransactionId};
     use fedimint_dummy_common::{DummyCommonInit, DummyInput, DummyOutput};
     use fedimint_dummy_server::Dummy;
+    use fedimint_logging::TracingSetup;
     use fedimint_testing::db::{prepare_db_migration_snapshot, validate_migrations, BYTE_32};
     use futures::StreamExt;
     use rand::rngs::OsRng;
@@ -167,6 +168,9 @@ mod fedimint_migration_tests {
     /// database keys/values change - instead a new function should be added
     /// that creates a new database backup that can be tested.
     async fn create_server_db_with_v0_data(mut dbtx: DatabaseTransaction<'_>) {
+        dbtx.insert_new_entry(&DatabaseVersionKey, &DatabaseVersion(0))
+            .await;
+
         let accepted_tx_id = AcceptedTransactionKey(TransactionId::from_slice(&BYTE_32).unwrap());
 
         dbtx.insert_new_entry(&DatabaseVersionKey, &DatabaseVersion(0))
@@ -247,6 +251,8 @@ mod fedimint_migration_tests {
 
     #[tokio::test(flavor = "multi_thread")]
     async fn test_migrations() -> anyhow::Result<()> {
+        TracingSetup::default().init()?;
+
         validate_migrations(
             "fedimint-server",
             |db| async move {

justfile.fedimint.just

@@ -41,6 +41,7 @@ check-wasm:
 # ex: `just prepare_server_db_migration_snapshots fedimint-mint-tests`
 # ex: `just prepare_server_db_migration_snapshots fedimint-ln-tests`
 # ex: `just prepare_server_db_migration_snapshots fedimint-wallet-tests`
+# ex: `just prepare_server_db_migration_snapshots fedimint-dummy-tests`
 prepare_server_db_migration_snapshots +extra_args:
   env FM_PREPARE_DB_MIGRATION_SNAPSHOTS=force cargo test ${CARGO_PROFILE:+--profile ${CARGO_PROFILE}} -p {{extra_args}} prepare_server_db_migration_snapshots
 

modules/fedimint-dummy-server/src/lib.rs

@@ -35,7 +35,7 @@ use crate::db::{
     DummyOutcomePrefix,
 };
 
-mod db;
+pub mod db;
 
 /// Generates the module
 #[derive(Debug, Clone)]

modules/fedimint-dummy-tests/Cargo.toml

@@ -21,8 +21,11 @@ fedimint-dummy-server = { path = "../fedimint-dummy-server" }
 fedimint-logging = { path = "../../fedimint-logging" }
 fedimint-server = { path = "../../fedimint-server" }
 fedimint-testing = { path = "../../fedimint-testing" }
+futures = "0.3.24"
 rand = "0.8"
 secp256k1 = "0.24.2"
+strum = "0.24"
+strum_macros = "0.24"
 tokio = { version = "1.26.0", features = ["sync"] }
 tracing = "0.1.37"