Persitence Plugin for exporting storage changes

StatesDumper/PersistActions.cs

@@ -0,0 +1,10 @@
+using System;
+
+namespace Neo.Plugins
+{
+    [Flags]
+    internal enum PersistActions : byte
+    {
+        StorageChanges = 0b00000001
+    }
+}

StatesDumper/Settings.cs

@@ -0,0 +1,48 @@
+using Microsoft.Extensions.Configuration;
+using System;
+using System.Reflection;
+
+namespace Neo.Plugins
+{
+    internal class Settings
+    {
+        /// <summary>
+        /// Amount of storages states (heights) to be dump in a given json file
+        /// </summary>
+        public uint BlockCacheSize { get; }
+        /// <summary>
+        /// Height to begin storage dump
+        /// </summary>
+        public uint HeightToBegin { get; }
+        /// <summary>
+        /// Height to begin real-time syncing and dumping on, consequently, dumping every block into a single files
+        /// </summary>
+        public uint HeightToStartRealTimeSyncing { get; }
+        /// <summary>
+        /// Persisting actions
+        /// </summary>
+        public PersistActions PersistAction { get; }
+
+        public static Settings Default { get; }
+
+        static Settings()
+        {
+            Default = new Settings(Assembly.GetExecutingAssembly().GetConfiguration());
+        }
+
+        public Settings(IConfigurationSection section)
+        {
+            /// Geting settings for storage changes state dumper
+            this.BlockCacheSize = GetValueOrDefault(section.GetSection("BlockCacheSize"), 1000u, p => uint.Parse(p));
+            this.HeightToBegin = GetValueOrDefault(section.GetSection("HeightToBegin"), 0u, p => uint.Parse(p));
+            this.HeightToStartRealTimeSyncing = GetValueOrDefault(section.GetSection("HeightToStartRealTimeSyncing"), 2883000u, p => uint.Parse(p));
+            this.PersistAction = GetValueOrDefault(section.GetSection("PersistAction"), PersistActions.StorageChanges, p => (PersistActions)Enum.Parse(typeof(PersistActions), p));
+        }
+
+        public T GetValueOrDefault<T>(IConfigurationSection section, T defaultValue, Func<string, T> selector)
+        {
+            if (section.Value == null) return defaultValue;
+            return selector(section.Value);
+        }
+    }
+}

StatesDumper/StatesDumper.cs

@@ -1,15 +1,19 @@
 using Neo.IO;
+using Neo.IO.Caching;
 using Neo.IO.Json;
 using Neo.Ledger;
+using Neo.Persistence;
 using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 
 namespace Neo.Plugins
 {
-    public class StatesDumper : Plugin
+    public class StatesDumper : Plugin, IPersistencePlugin
     {
+        private readonly JArray bs_cache = new JArray();
+
         private static void Dump<TKey, TValue>(IEnumerable<KeyValuePair<TKey, TValue>> states)
             where TKey : ISerializable
             where TValue : ISerializable
@@ -63,5 +67,74 @@ private bool OnHelp(string[] args)
             Console.Write($"{Name} Commands:\n" + "\tdump storage <key>\n");
             return true;
         }
+
+        public void OnPersist(Snapshot snapshot)
+        {
+            if (Settings.Default.PersistAction.HasFlag(PersistActions.StorageChanges))
+                OnPersistStorage(snapshot);
+        }
+
+        private void OnPersistStorage(Snapshot snapshot)
+        {
+            uint blockIndex = snapshot.Height;
+            if (blockIndex >= Settings.Default.HeightToBegin)
+            {
+                string dirPath = "./Storage";
+                Directory.CreateDirectory(dirPath);
+                string path = $"{HandlePaths(dirPath, blockIndex)}/dump-block-{blockIndex.ToString()}.json";
+
+                JArray array = new JArray();
+
+                foreach (DataCache<StorageKey, StorageItem>.Trackable trackable in snapshot.Storages.GetChangeSet())
+                {
+                    JObject state = new JObject();
+
+                    switch (trackable.State)
+                    {
+
+                        case TrackState.Added:
+                            state["state"] = "Added";
+                            state["key"] = trackable.Key.ToArray().ToHexString();
+                            state["value"] = trackable.Item.ToArray().ToHexString();
+                            // Here we have a new trackable.Key and trackable.Item
+                            break;
+                        case TrackState.Changed:
+                            state["state"] = "Changed";
+                            state["key"] = trackable.Key.ToArray().ToHexString();
+                            state["value"] = trackable.Item.ToArray().ToHexString();
+                            break;
+                        case TrackState.Deleted:
+                            state["state"] = "Deleted";
+                            state["key"] = trackable.Key.ToArray().ToHexString();
+                            break;
+                    }
+                    array.Add(state);
+                }
+
+                JObject bs_item = new JObject();
+                bs_item["block"] = blockIndex;
+                bs_item["size"] = array.Count;
+                bs_item["storage"] = array;
+                bs_cache.Add(bs_item);
+
+                if ((blockIndex % Settings.Default.BlockCacheSize == 0) || (blockIndex > Settings.Default.HeightToStartRealTimeSyncing))
+                {
+                    File.WriteAllText(path, bs_cache.ToString());
+                    bs_cache.Clear();
+                }
+            }
+        }
+
+        private static string HandlePaths(string dirPath, uint blockIndex)
+        {
+            //Default Parameter
+            uint storagePerFolder = 100000;
+            uint folder = (((blockIndex - 1) / storagePerFolder) + 1) * storagePerFolder;
+            if (blockIndex == 0)
+                folder = 0;
+            string dirPathWithBlock = $"{dirPath}/BlockStorage_{folder}";
+            Directory.CreateDirectory(dirPathWithBlock);
+            return dirPathWithBlock;
+        }
     }
 }

StatesDumper/StatesDumper.csproj

@@ -6,6 +6,13 @@
     <RootNamespace>Neo.Plugins</RootNamespace>
   </PropertyGroup>
 
+  <ItemGroup>
+    <None Update="StatesDumper\config.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <CopyToPublishDirectory>PreserveNewest</CopyToPublishDirectory>
+    </None>
+  </ItemGroup>
+
   <ItemGroup>
     <PackageReference Include="Neo" Version="2.9.2" />
   </ItemGroup>

StatesDumper/StatesDumper/config.json

@@ -0,0 +1,8 @@
+{
+  "PluginConfiguration": {
+    "PersistAction": "StorageChanges",
+    "BlockCacheSize": 1000,
+    "HeightToBegin": 0,
+    "HeightToRealTimeSyncing": 2883000
+  }
+}