[StateService] get historical state

src/StateService/MPT/Helper.cs

@@ -0,0 +1,18 @@
+using System;
+
+namespace Neo.Plugins.MPT
+{
+    public static class Helper
+    {
+        public static int CompareTo(this byte[] arr1, byte[] arr2)
+        {
+            if (arr1 is null || arr2 is null) throw new ArgumentNullException();
+            for (int i = 0; i < arr1.Length && i < arr2.Length; i++)
+            {
+                var r = arr1[i].CompareTo(arr2[i]);
+                if (r != 0) return r;
+            }
+            return arr2.Length < arr1.Length ? 1 : arr2.Length == arr1.Length ? 0 : -1;
+        }
+    }
+}

src/StateService/MPT/MPTTrie.Find.cs

@@ -62,22 +62,33 @@ private ReadOnlySpan<byte> Seek(ref MPTNode node, ReadOnlySpan<byte> path, out M
             return ReadOnlySpan<byte>.Empty;
         }
 
-        public IEnumerable<(TKey Key, TValue Value)> Find(ReadOnlySpan<byte> prefix)
+        public IEnumerable<(TKey Key, TValue Value)> Find(ReadOnlySpan<byte> prefix, byte[] from = null)
         {
             var path = ToNibbles(prefix);
+            int offset = 0;
+            if (from is null) from = Array.Empty<byte>();
+            if (0 < from.Length)
+            {
+                if (!from.AsSpan().StartsWith(prefix))
+                    throw new InvalidOperationException("invalid from key");
+                offset = prefix.Length * 2;
+                from = ToNibbles(from.AsSpan());
+            }
             path = Seek(ref root, path, out MPTNode start).ToArray();
-            return Travers(start, path)
+            return Travers(start, path, from, offset)
                 .Select(p => (FromNibbles(p.Key).AsSerializable<TKey>(), p.Value.AsSerializable<TValue>()));
         }
 
-        private IEnumerable<(byte[] Key, byte[] Value)> Travers(MPTNode node, byte[] path)
+        private IEnumerable<(byte[] Key, byte[] Value)> Travers(MPTNode node, byte[] path, byte[] from, int offset)
         {
             if (node is null) yield break;
+            if (offset < 0) throw new InvalidOperationException("invalid offset");
             switch (node.Type)
             {
                 case NodeType.LeafNode:
                     {
-                        yield return (path, (byte[])node.Value.Clone());
+                        if (from.Length <= offset && !path.SequenceEqual(from))
+                            yield return (path, (byte[])node.Value.Clone());
                         break;
                     }
                 case NodeType.Empty:
@@ -87,23 +98,44 @@ private ReadOnlySpan<byte> Seek(ref MPTNode node, ReadOnlySpan<byte> path, out M
                         var newNode = cache.Resolve(node.Hash);
                         if (newNode is null) throw new InvalidOperationException("Internal error, can't resolve hash when mpt find");
                         node = newNode;
-                        foreach (var item in Travers(node, path))
+                        foreach (var item in Travers(node, path, from, offset))
                             yield return item;
                         break;
                     }
                 case NodeType.BranchNode:
                     {
-                        for (int i = 0; i < MPTNode.BranchChildCount; i++)
+                        if (offset < from.Length)
+                        {
+                            for (int i = 0; i < MPTNode.BranchChildCount - 1; i++)
+                            {
+                                if (from[offset] < i)
+                                    foreach (var item in Travers(node.Children[i], Concat(path, new byte[] { (byte)i }), from, from.Length))
+                                        yield return item;
+                                else if (i == from[offset])
+                                    foreach (var item in Travers(node.Children[i], Concat(path, new byte[] { (byte)i }), from, offset + 1))
+                                        yield return item;
+                            }
+                        }
+                        else
                         {
-                            foreach (var item in Travers(node.Children[i], i == MPTNode.BranchChildCount - 1 ? path : Concat(path, new byte[] { (byte)i })))
+                            foreach (var item in Travers(node.Children[MPTNode.BranchChildCount - 1], path, from, offset))
                                 yield return item;
+                            for (int i = 0; i < MPTNode.BranchChildCount - 1; i++)
+                            {
+                                foreach (var item in Travers(node.Children[i], Concat(path, new byte[] { (byte)i }), from, offset))
+                                    yield return item;
+                            }
                         }
                         break;
                     }
                 case NodeType.ExtensionNode:
                     {
-                        foreach (var item in Travers(node.Next, Concat(path, node.Key)))
-                            yield return item;
+                        if (offset < from.Length && from.AsSpan()[offset..].StartsWith(node.Key))
+                            foreach (var item in Travers(node.Next, Concat(path, node.Key), from, offset + node.Key.Length))
+                                yield return item;
+                        else if (from.Length <= offset || 0 < node.Key.CompareTo(from[offset..]))
+                            foreach (var item in Travers(node.Next, Concat(path, node.Key), from, from.Length))
+                                yield return item;
                         break;
                     }
             }

src/StateService/README.md

@@ -0,0 +1,70 @@
+# StateService
+
+## RPC API
+
+### GetStateRoot
+#### Params
+|Name|Type|Summary|Required|
+|-|-|-|-|
+|Index|uint|index|true|
+#### Result
+StateRoot Object
+|Name|Type|Summary|
+|-|-|-|
+|version|number|version|
+|index|number|index|
+|roothash|string|version|
+|witness|Object|witness from validators|
+
+### GetProof
+#### Params
+|Name|Type|Summary|Required|
+|-|-|-|-|
+|RootHash|UInt256|state root|true|
+|ScriptHash|UInt160|contract script hash|true|
+|Key|base64 string|key|true|
+#### Result
+Proof in base64 string
+
+### VerifyProof
+#### Params
+|Name|Type|Summary|
+|-|-|-|
+|RootHash|UInt256|state root|true|
+|Proof|base64 string|proof|true|
+#### Result
+Value in base64 string
+
+### GetStateheight
+#### Result
+|Name|Type|Summary|
+|-|-|-|
+|localrootindex|number|root hash index calculated locally|
+|validatedrootindex|number|root hash index verified by validators|
+
+### GetState
+#### Params
+|Name|Type|Summary|Required|
+|-|-|-|-|
+|RootHash|UInt256|specify state|true|
+|ScriptHash|UInt160|contract script hash|true|
+|Key|base64 string|key|true|
+#### Result
+Value in base64 string or `null`
+
+### FindStates
+#### Params
+|Name|Type|Summary|Required|
+|-|-|-|-|
+|RootHash|UInt256|specify state|true|
+|ScriptHash|UInt160|contract script hash|true|
+|Prefix|base64 string|key prefix|true|
+|From|base64 string|start key, default `Empty`|optional|
+|Count|number|count of results in one request, default `MaxFindResultItems`|optional|
+#### Result
+|Name|Type|Summary|
+|-|-|-|
+|firstProof|string|proof of first value in results|
+|lastProof|string|proof of last value in results|
+|truncated|bool|whether the results is truncated because of limitation|
+|results|array|key-values found|

src/StateService/Settings.cs

@@ -8,6 +8,7 @@ internal class Settings
         public bool FullState { get; }
         public uint Network { get; }
         public bool AutoVerify { get; }
+        public int MaxFindResultItems { get; }
 
         public static Settings Default { get; private set; }
 
@@ -17,6 +18,7 @@ private Settings(IConfigurationSection section)
             FullState = section.GetValue("FullState", false);
             Network = section.GetValue("Network", 5195086u);
             AutoVerify = section.GetValue("AutoVerify", false);
+            MaxFindResultItems = section.GetValue("MaxFindResultItems", 100);
         }
 
         public static void Load(IConfigurationSection section)

src/StateService/StatePlugin.cs

@@ -177,7 +177,10 @@ private string GetProof(UInt256 root_hash, UInt160 script_hash, byte[] key)
                 Id = contract.Id,
                 Key = key,
             };
-            var result = StateStore.Singleton.TryGetProof(root_hash, skey, out var proof);
+
+            using ISnapshot store = StateStore.Singleton.GetStoreSnapshot();
+            var trie = new MPTTrie<StorageKey, StorageItem>(store, root_hash);
+            var result = trie.TryGetProof(skey, out var proof);
             if (!result) throw new RpcException(-100, "Unknown value");
 
             using MemoryStream ms = new MemoryStream();
@@ -239,5 +242,91 @@ public JObject GetStateHeight(JArray _params)
             json["validatedrootindex"] = StateStore.Singleton.ValidatedRootIndex;
             return json;
         }
+
+        private ContractState GetHistoricalContractState(MPTTrie<StorageKey, StorageItem> trie, UInt160 script_hash)
+        {
+            const byte prefix = 8;
+            StorageKey skey = new KeyBuilder(NativeContract.ContractManagement.Id, prefix).Add(script_hash);
+            return trie.TryGetValue(skey, out var value) ? value.GetInteroperable<ContractState>() : null;
+        }
+
+        [RpcMethod]
+        public JObject FindStates(JArray _params)
+        {
+            var root_hash = UInt256.Parse(_params[0].AsString());
+            if (!Settings.Default.FullState && StateStore.Singleton.CurrentLocalRootHash != root_hash)
+                throw new RpcException(-100, "Old state not supported");
+            var script_hash = UInt160.Parse(_params[1].AsString());
+            var prefix = Convert.FromBase64String(_params[2].AsString());
+            byte[] key = Array.Empty<byte>();
+            if (3 < _params.Count)
+                key = Convert.FromBase64String(_params[3].AsString());
+            int count = Settings.Default.MaxFindResultItems;
+            if (4 < _params.Count)
+                count = int.Parse(_params[4].AsString());
+            if (Settings.Default.MaxFindResultItems < count)
+                count = Settings.Default.MaxFindResultItems;
+            using var store = StateStore.Singleton.GetStoreSnapshot();
+            var trie = new MPTTrie<StorageKey, StorageItem>(store, root_hash);
+            var contract = GetHistoricalContractState(trie, script_hash);
+            if (contract is null) throw new RpcException(-100, "Unknown contract");
+            StorageKey pkey = new()
+            {
+                Id = contract.Id,
+                Key = prefix,
+            };
+            StorageKey fkey = new()
+            {
+                Id = pkey.Id,
+                Key = key,
+            };
+            JObject json = new();
+            JArray jarr = new();
+            int i = 0;
+            foreach (var (ikey, ivalue) in trie.Find(pkey.ToArray(), 0 < key.Length ? fkey.ToArray() : null))
+            {
+                if (count < i) break;
+                if (i < count)
+                {
+                    JObject j = new();
+                    j["key"] = Convert.ToBase64String(ikey.Key);
+                    j["value"] = Convert.ToBase64String(ivalue.Value);
+                    jarr.Add(j);
+                }
+                i++;
+            };
+            if (0 < jarr.Count)
+            {
+                json["firstProof"] = GetProof(root_hash, script_hash, Convert.FromBase64String(jarr.First()["key"].AsString()));
+            }
+            if (1 < jarr.Count)
+            {
+                json["lastProof"] = GetProof(root_hash, script_hash, Convert.FromBase64String(jarr.Last()["key"].AsString()));
+            }
+            json["truncated"] = count < i;
+            json["results"] = jarr;
+            return json;
+        }
+
+        [RpcMethod]
+        public JObject GetState(JArray _params)
+        {
+            var root_hash = UInt256.Parse(_params[0].AsString());
+            if (!Settings.Default.FullState && StateStore.Singleton.CurrentLocalRootHash != root_hash)
+                throw new RpcException(-100, "Old state not supported");
+            var script_hash = UInt160.Parse(_params[1].AsString());
+            var key = Convert.FromBase64String(_params[2].AsString());
+            using var store = StateStore.Singleton.GetStoreSnapshot();
+            var trie = new MPTTrie<StorageKey, StorageItem>(store, root_hash);
+
+            var contract = GetHistoricalContractState(trie, script_hash);
+            if (contract is null) throw new RpcException(-100, "Unknown contract");
+            StorageKey skey = new()
+            {
+                Id = contract.Id,
+                Key = key,
+            };
+            return Convert.ToBase64String(trie[skey].Value);
+        }
     }
 }

src/StateService/StateService/config.json

@@ -3,7 +3,8 @@
     "Path": "Data_MPT_{0}",
     "FullState": false,
     "Network": 860833102,
-    "AutoVerify": false
+    "AutoVerify": false,
+    "MaxFindResultItems": 100
   },
   "Dependency": [
     "RpcServer"

src/StateService/Storage/StateStore.cs

@@ -55,11 +55,9 @@ public StateSnapshot GetSnapshot()
             return new StateSnapshot(store);
         }
 
-        public bool TryGetProof(UInt256 root, StorageKey skey, out HashSet<byte[]> proof)
+        public ISnapshot GetStoreSnapshot()
         {
-            using ISnapshot snapshot = store.GetSnapshot();
-            var trie = new MPTTrie<StorageKey, StorageItem>(snapshot, root);
-            return trie.TryGetProof(skey, out proof);
+            return store.GetSnapshot();
         }
 
         protected override void OnReceive(object message)

tests/Neo.Plugins.StateService.Tests/MPT/UT_Helper.cs

@@ -0,0 +1,28 @@
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System;
+using static Neo.Plugins.MPT.Helper;
+
+namespace Neo.Plugins.StateService.Tests
+{
+    [TestClass]
+    public class UT_Helper
+    {
+        [TestMethod]
+        public void TestCompareTo()
+        {
+            var arr1 = new byte[] { 0, 1, 2 };
+            var arr2 = new byte[] { 0, 1, 2 };
+            Assert.AreEqual(0, arr1.CompareTo(arr2));
+            arr1 = new byte[] { 0, 1 };
+            Assert.AreEqual(-1, arr1.CompareTo(arr2));
+            arr2 = new byte[] { 0 };
+            Assert.AreEqual(1, arr1.CompareTo(arr2));
+            arr2 = new byte[] { 0, 2 };
+            Assert.AreEqual(-1, arr1.CompareTo(arr2));
+            arr1 = new byte[] { 0, 3, 1 };
+            Assert.AreEqual(1, arr1.CompareTo(arr2));
+            Assert.AreEqual(0, Array.Empty<byte>().CompareTo(Array.Empty<byte>()));
+            Assert.ThrowsException<ArgumentNullException>(() => arr1.CompareTo(null));
+        }
+    }
+}

tests/Neo.Plugins.StateService.Tests/MPT/UT_MPTTrie.cs

@@ -601,5 +601,23 @@ public void TestEmptyValueIssue633()
             Assert.IsNotNull(val);
             Assert.AreEqual(0, val.Size);
         }
+
+        [TestMethod]
+        public void TestFindWithFrom()
+        {
+            var snapshot = new TestSnapshot();
+            var mpt = new MPTTrie<TestKey, TestValue>(snapshot, null);
+            mpt.Put("aa".HexToBytes(), "02".HexToBytes());
+            mpt.Put("aa10".HexToBytes(), "03".HexToBytes());
+            mpt.Put("aa50".HexToBytes(), "04".HexToBytes());
+            var r = mpt.Find("aa".HexToBytes()).ToList();
+            Assert.AreEqual(3, r.Count);
+            r = mpt.Find("aa".HexToBytes(), "aa30".HexToBytes()).ToList();
+            Assert.AreEqual(1, r.Count);
+            r = mpt.Find("aa".HexToBytes(), "aa60".HexToBytes()).ToList();
+            Assert.AreEqual(0, r.Count);
+            r = mpt.Find("aa".HexToBytes(), "aa10".HexToBytes()).ToList();
+            Assert.AreEqual(1, r.Count);//without from key
+        }
     }
 }