Make it easier to migrate filters for non-devs operators (WalletWasab…

…i#13405)

Contrib/Migration/README.md

@@ -0,0 +1,36 @@
+# Filters Migration
+
+Starting with Wasabi v2.2.0.0, the backend now utilizes SQLite to store and retrieve compact filters. As a result, operators need to migrate their old plain text filters to the new SQLite format.
+
+## Migration Guide
+
+### Using Nix
+
+If you're deploying with `Nix`, migrating is straightforward. Simply run the following command on your backend server:
+
+```bash
+$ nix run github:WalletWasabi/WalletWasabi#migrateFilters
+
+Database already exists. Skipping creation.
+.....................................
+Completed. Total processed: 371888, Total inserted: 371888
+Max Block Height in DB: 853711
+```
+The migration tool will automatically process your filters and insert them into the new SQLite database.
+The old filters will still be there untouched. We recommend to keep them for a while just in case a rollback
+to a previous version is needed.
+
+
+## Using dotnet
+
+For those using dotnet, follow these steps:
+
+* Clone the repository
+* Navigate to the migration directory:
+  ```
+  $ cd <your wasabi repo dir>/Contrib/Migration
+  ```
+* Run the migration script:
+  ```
+  dotnet fsi migrateBackendFilters.fsx
+  ```

Contrib/migrateBackendFilters.fsx → Contrib/Migration/migrateBackendFilters.fsx

@@ -7,8 +7,9 @@ open System
 open System.IO
 open Microsoft.Data.Sqlite
 
-let inputFilePath = "IndexMain.dat"
-let outputDbPath = "IndexMain.sqlite"
+let indexServiceDirectory = Path.Combine (Environment.GetEnvironmentVariable("HOME"), ".walletwasabi/backend/IndexBuilderService")
+let inputFilePath = Path.Combine(indexServiceDirectory, "IndexMain.dat")
+let outputDbPath = Path.Combine(indexServiceDirectory, "IndexMain.sqlite");
 let batchSize = 1000
 
 type Filter = { Height: int; BlockHash: byte[]; Filter: byte[]; BlockTime: int64; PrevBlockHash: byte[] }
@@ -110,10 +111,12 @@ while not reader.EndOfStream do
     totalProcessed <- totalProcessed + 1
 
     if batch.Length = batchSize || reader.EndOfStream then
+        if totalProcessed % 10_000 = 0 then
+            printf "."
         let inserted = insertFiltersBatch conn batch
         totalInserted <- totalInserted + inserted
         batch <- []
 
-printfn $"Completed. Total processed: %d{totalProcessed}, Total inserted: %d{totalInserted}"
+printfn $"\nCompleted. Total processed: %d{totalProcessed}, Total inserted: %d{totalInserted}"
 let maxHeight = getMaxBlockHeight conn
 printfn $"Max Block Height in DB: %d{maxHeight}"

flake.nix

@@ -99,10 +99,18 @@
              export PS1='\n\[\033[1;34m\][Wasabi:\w]\$\[\033[0m\] '
            '';
         };
+        migrateBackendFilters = {
+           type = "app";
+           program = "${(pkgs.writeShellScript "migrateBackendFilters" ''
+              ${pkgs.dotnetCorePackages.sdk_8_0}/bin/dotnet fsi ${./.}/Contrib/Migration/migrateBackendFilters.fsx;
+              '')}";
+        };
     in
     {
       packages.x86_64-linux.default = buildBackend;
       packages.x86_64-linux.all = buildEverything;
       devShells.x86_64-linux.default = wasabi-shell;
+
+      apps.x86_64-linux.migrateFilters = migrateBackendFilters;
     };
 }