From 8d48da70a1d850716b69e1d0685613d33c735624 Mon Sep 17 00:00:00 2001 From: Shargon Date: Tue, 16 Apr 2019 10:31:07 +0200 Subject: [PATCH 01/45] First draft of P2P protocol --- neo/Consensus/ConsensusService.cs | 2 +- neo/Network/P2P/Helper.cs | 33 ++++++++ neo/Network/P2P/LocalNode.cs | 4 +- neo/Network/P2P/Message.cs | 118 +++++++++++++++++++++-------- neo/Network/P2P/MessageCommand.cs | 36 +++++++++ neo/Network/P2P/MessageFlags.cs | 11 +++ neo/Network/P2P/ProtocolHandler.cs | 99 ++++++++++++------------ neo/Network/P2P/RemoteNode.cs | 85 ++++++++++++--------- neo/Network/P2P/TaskManager.cs | 10 +-- 9 files changed, 272 insertions(+), 126 deletions(-) create mode 100644 neo/Network/P2P/MessageCommand.cs create mode 100644 neo/Network/P2P/MessageFlags.cs diff --git a/neo/Consensus/ConsensusService.cs b/neo/Consensus/ConsensusService.cs index 1e3e15ad97..5e252a8ffd 100644 --- a/neo/Consensus/ConsensusService.cs +++ b/neo/Consensus/ConsensusService.cs @@ -581,7 +581,7 @@ private void SendPrepareRequest() if (context.TransactionHashes.Length > 1) { foreach (InvPayload payload in InvPayload.CreateGroup(InventoryType.TX, context.TransactionHashes.Skip(1).ToArray())) - localNode.Tell(Message.Create("inv", payload)); + localNode.Tell(Message.Create(MessageCommand.inv, payload)); } ChangeTimer(TimeSpan.FromSeconds((Blockchain.SecondsPerBlock << (context.ViewNumber + 1)) - (context.ViewNumber == 0 ? Blockchain.SecondsPerBlock : 0))); } diff --git a/neo/Network/P2P/Helper.cs b/neo/Network/P2P/Helper.cs index 43ded70d6a..cfd99bc8ca 100644 --- a/neo/Network/P2P/Helper.cs +++ b/neo/Network/P2P/Helper.cs @@ -1,10 +1,43 @@ using Neo.Network.P2P.Payloads; using System.IO; +using System.IO.Compression; namespace Neo.Network.P2P { public static class Helper { + public static byte[] Uncompress(this byte[] data) + { + using (var output = new MemoryStream()) + using (var input = new MemoryStream(data)) + using (var gzip = new GZipStream(input, CompressionMode.Decompress)) + { + int nRead; + byte[] buffer = new byte[1024]; + + while ((nRead = gzip.Read(buffer, 0, buffer.Length)) > 0) + { + output.Write(buffer, 0, nRead); + } + + return output.ToArray(); + } + } + + public static byte[] Compress(this byte[] data) + { + using (var stream = new MemoryStream()) + { + using (var gzip = new GZipStream(stream, CompressionLevel.Optimal, true)) + { + gzip.Write(data, 0, data.Length); + gzip.Flush(); + } + + return stream.ToArray(); + } + } + public static byte[] GetHashData(this IVerifiable verifiable) { using (MemoryStream ms = new MemoryStream()) diff --git a/neo/Network/P2P/LocalNode.cs b/neo/Network/P2P/LocalNode.cs index a0154ffe33..a1dd282c9c 100644 --- a/neo/Network/P2P/LocalNode.cs +++ b/neo/Network/P2P/LocalNode.cs @@ -58,7 +58,7 @@ public LocalNode(NeoSystem system) } } - private void BroadcastMessage(string command, ISerializable payload = null) + private void BroadcastMessage(MessageCommand command, ISerializable payload = null) { BroadcastMessage(Message.Create(command, payload)); } @@ -126,7 +126,7 @@ protected override void NeedMorePeers(int count) count = Math.Max(count, 5); if (ConnectedPeers.Count > 0) { - BroadcastMessage("getaddr"); + BroadcastMessage(MessageCommand.getaddr); } else { diff --git a/neo/Network/P2P/Message.cs b/neo/Network/P2P/Message.cs index 189c496b2a..16d44173e3 100644 --- a/neo/Network/P2P/Message.cs +++ b/neo/Network/P2P/Message.cs @@ -1,79 +1,135 @@ -using Neo.Cryptography; +using System; +using System.IO; +using Akka.IO; +using Neo.Cryptography; using Neo.IO; using Neo.Network.P2P.Payloads; -using System; -using System.IO; namespace Neo.Network.P2P { public class Message : ISerializable { - public const int HeaderSize = sizeof(uint) + 12 + sizeof(int) + sizeof(uint); public const int PayloadMaxSize = 0x02000000; + public const int CompressionMinSize = 180; + public const int CompressionThreshold = 100; - public static readonly uint Magic = ProtocolSettings.Default.Magic; - public string Command; - public uint Checksum; + public MessageFlags Flags; + public MessageCommand Command; public byte[] Payload; - public int Size => HeaderSize + Payload.Length; + public int Size => 2 + IO.Helper.GetVarSize(Payload.Length) + Payload.Length; - public static Message Create(string command, ISerializable payload = null) + public static Message Create(MessageCommand command, ISerializable payload = null) { return Create(command, payload == null ? new byte[0] : payload.ToArray()); } - public static Message Create(string command, byte[] payload) + public static Message Create(MessageCommand command, byte[] payload) { + var flags = MessageFlags.None; + + // Try compression + + if (payload.Length > CompressionMinSize) + { + var compressed = payload.Compress(); + + if (compressed.Length < payload.Length - CompressionThreshold) + { + payload = compressed; + flags |= MessageFlags.Compressed; + } + } + return new Message { + Flags = flags, Command = command, - Checksum = GetChecksum(payload), Payload = payload }; } void ISerializable.Deserialize(BinaryReader reader) { - if (reader.ReadUInt32() != Magic) - throw new FormatException(); - this.Command = reader.ReadFixedString(12); - uint length = reader.ReadUInt32(); - if (length > PayloadMaxSize) - throw new FormatException(); - this.Checksum = reader.ReadUInt32(); - this.Payload = reader.ReadBytes((int)length); - if (GetChecksum(Payload) != Checksum) - throw new FormatException(); + this.Flags = (MessageFlags)reader.ReadByte(); + this.Command = (MessageCommand)reader.ReadByte(); + var length = (int)reader.ReadVarInt(int.MaxValue); + + if (length > PayloadMaxSize) throw new FormatException(); + this.Payload = reader.ReadBytes(length); } - private static uint GetChecksum(byte[] value) + public static int TryDeserialize(ByteString data, out Message msg) { - return Crypto.Default.Hash256(value).ToUInt32(0); + msg = null; + if (data.Count < 3) return 0; + + var header = data.Slice(0, 3).ToArray(); + ulong length = header[2]; + int payloadIndex = 3; + + if (length == 0xFD) + { + if (data.Count < 5) return 0; + length = data.Slice(payloadIndex, 2).ToArray().ToUInt16(0); + payloadIndex += 2; + } + else if (length == 0xFE) + { + if (data.Count < 7) return 0; + length = data.Slice(payloadIndex, 4).ToArray().ToUInt32(0); + payloadIndex += 4; + } + else if (length == 0xFF) + { + if (data.Count < 11) return 0; + length = data.Slice(payloadIndex, 8).ToArray().ToUInt64(0); + payloadIndex += 8; + } + + if (length > PayloadMaxSize) throw new FormatException(); + if (data.Count < (int)length) return 0; + + msg = new Message() + { + Flags = (MessageFlags)header[0], + Command = (MessageCommand)header[1], + Payload = data.Slice(payloadIndex, (int)length).ToArray() + }; + + return payloadIndex + (int)length; + } + + public byte[] GetPayload() + { + if (this.Flags.HasFlag(MessageFlags.Compressed)) + { + return this.Payload.Uncompress(); + } + + return this.Payload; } private ISerializable _payload_deserialized = null; public T GetPayload() where T : ISerializable, new() { if (_payload_deserialized is null) - _payload_deserialized = Payload.AsSerializable(); + _payload_deserialized = GetPayload().AsSerializable(); return (T)_payload_deserialized; } public Transaction GetTransaction() { if (_payload_deserialized is null) - _payload_deserialized = Transaction.DeserializeFrom(Payload); + _payload_deserialized = Transaction.DeserializeFrom(GetPayload()); return (Transaction)_payload_deserialized; } void ISerializable.Serialize(BinaryWriter writer) { - writer.Write(Magic); - writer.WriteFixedString(Command, 12); - writer.Write(Payload.Length); - writer.Write(Checksum); - writer.Write(Payload); + writer.Write((byte)Flags); + writer.Write((byte)Command); + writer.WriteVarBytes(Payload); } } -} +} \ No newline at end of file diff --git a/neo/Network/P2P/MessageCommand.cs b/neo/Network/P2P/MessageCommand.cs new file mode 100644 index 0000000000..003b2d04b0 --- /dev/null +++ b/neo/Network/P2P/MessageCommand.cs @@ -0,0 +1,36 @@ +namespace Neo.Network.P2P +{ + public enum MessageCommand : byte + { + // Same value as InventoryType + tx = 0x01, + // Same value as InventoryType + block = 0x02, + mempool = 0x03, + addr = 0x04, + inv = 0x05, + headers = 0x06, + merkleblock = 0x07, + version = 0x08, + verack = 0x09, + alert = 0x0A, + reject = 0x0B, + + ping = 0x10, + pong = 0x11, + + getaddr = 0x20, + getblocks = 0x21, + getdata = 0x22, + getheaders = 0x23, + + filteradd = 0x30, + filterclear = 0x31, + filterload = 0x32, + + // Same value as InventoryType + consensus = 0xE0, + + notfound = 0xFF, + } +} \ No newline at end of file diff --git a/neo/Network/P2P/MessageFlags.cs b/neo/Network/P2P/MessageFlags.cs new file mode 100644 index 0000000000..a608f5bfef --- /dev/null +++ b/neo/Network/P2P/MessageFlags.cs @@ -0,0 +1,11 @@ +using System; + +namespace Neo.Network.P2P +{ + [Flags] + public enum MessageFlags : byte + { + None = 0, + Compressed = 1 << 0 + } +} \ No newline at end of file diff --git a/neo/Network/P2P/ProtocolHandler.cs b/neo/Network/P2P/ProtocolHandler.cs index 0765054ff5..79f4b892a3 100644 --- a/neo/Network/P2P/ProtocolHandler.cs +++ b/neo/Network/P2P/ProtocolHandler.cs @@ -42,79 +42,78 @@ protected override void OnReceive(object message) return; if (version == null) { - if (msg.Command != "version") + if (msg.Command != MessageCommand.version) throw new ProtocolViolationException(); OnVersionMessageReceived(msg.GetPayload()); return; } if (!verack) { - if (msg.Command != "verack") + if (msg.Command != MessageCommand.verack) throw new ProtocolViolationException(); OnVerackMessageReceived(); return; } switch (msg.Command) { - case "addr": + case MessageCommand.addr: OnAddrMessageReceived(msg.GetPayload()); break; - case "block": + case MessageCommand.block: OnInventoryReceived(msg.GetPayload()); break; - case "consensus": + case MessageCommand.consensus: OnInventoryReceived(msg.GetPayload()); break; - case "filteradd": + case MessageCommand.filteradd: OnFilterAddMessageReceived(msg.GetPayload()); break; - case "filterclear": + case MessageCommand.filterclear: OnFilterClearMessageReceived(); break; - case "filterload": + case MessageCommand.filterload: OnFilterLoadMessageReceived(msg.GetPayload()); break; - case "getaddr": + case MessageCommand.getaddr: OnGetAddrMessageReceived(); break; - case "getblocks": + case MessageCommand.getblocks: OnGetBlocksMessageReceived(msg.GetPayload()); break; - case "getdata": + case MessageCommand.getdata: OnGetDataMessageReceived(msg.GetPayload()); break; - case "getheaders": + case MessageCommand.getheaders: OnGetHeadersMessageReceived(msg.GetPayload()); break; - case "headers": + case MessageCommand.headers: OnHeadersMessageReceived(msg.GetPayload()); break; - case "inv": + case MessageCommand.inv: OnInvMessageReceived(msg.GetPayload()); break; - case "mempool": + case MessageCommand.mempool: OnMemPoolMessageReceived(); break; - case "ping": + case MessageCommand.ping: OnPingMessageReceived(msg.GetPayload()); break; - case "pong": + case MessageCommand.pong: OnPongMessageReceived(msg.GetPayload()); break; - case "tx": - if (msg.Payload.Length <= Transaction.MaxTransactionSize) - OnInventoryReceived(msg.GetTransaction()); + case MessageCommand.tx: + var tx = msg.GetTransaction(); + if (tx.Size > Transaction.MaxTransactionSize) return; + OnInventoryReceived(tx); break; - case "verack": - case "version": + case MessageCommand.verack: + case MessageCommand.version: throw new ProtocolViolationException(); - case "alert": - case "merkleblock": - case "notfound": - case "reject": - default: - //暂时忽略 - break; + case MessageCommand.alert: + case MessageCommand.merkleblock: + case MessageCommand.notfound: + case MessageCommand.reject: + default: break; } } @@ -154,7 +153,7 @@ private void OnGetAddrMessageReceived() .Take(AddrPayload.MaxCountToSend); NetworkAddressWithTime[] networkAddresses = peers.Select(p => NetworkAddressWithTime.Create(p.Listener, p.Version.Services, p.Version.Timestamp)).ToArray(); if (networkAddresses.Length == 0) return; - Context.Parent.Tell(Message.Create("addr", AddrPayload.Create(networkAddresses))); + Context.Parent.Tell(Message.Create(MessageCommand.addr, AddrPayload.Create(networkAddresses))); } private void OnGetBlocksMessageReceived(GetBlocksPayload payload) @@ -175,7 +174,7 @@ private void OnGetBlocksMessageReceived(GetBlocksPayload payload) hashes.Add(hash); } if (hashes.Count == 0) return; - Context.Parent.Tell(Message.Create("inv", InvPayload.Create(InventoryType.Block, hashes.ToArray()))); + Context.Parent.Tell(Message.Create(MessageCommand.inv, InvPayload.Create(InventoryType.Block, hashes.ToArray()))); } private void OnGetDataMessageReceived(InvPayload payload) @@ -190,7 +189,7 @@ private void OnGetDataMessageReceived(InvPayload payload) if (inventory == null) inventory = Blockchain.Singleton.GetTransaction(hash); if (inventory is Transaction) - Context.Parent.Tell(Message.Create("tx", inventory)); + Context.Parent.Tell(Message.Create(MessageCommand.tx, inventory)); break; case InventoryType.Block: if (inventory == null) @@ -199,18 +198,18 @@ private void OnGetDataMessageReceived(InvPayload payload) { if (bloom_filter == null) { - Context.Parent.Tell(Message.Create("block", inventory)); + Context.Parent.Tell(Message.Create(MessageCommand.block, inventory)); } else { BitArray flags = new BitArray(block.Transactions.Select(p => bloom_filter.Test(p)).ToArray()); - Context.Parent.Tell(Message.Create("merkleblock", MerkleBlockPayload.Create(block, flags))); + Context.Parent.Tell(Message.Create(MessageCommand.merkleblock, MerkleBlockPayload.Create(block, flags))); } } break; case InventoryType.Consensus: if (inventory != null) - Context.Parent.Tell(Message.Create("consensus", inventory)); + Context.Parent.Tell(Message.Create(MessageCommand.consensus, inventory)); break; } } @@ -235,7 +234,7 @@ private void OnGetHeadersMessageReceived(GetBlocksPayload payload) headers.Add(header); } if (headers.Count == 0) return; - Context.Parent.Tell(Message.Create("headers", HeadersPayload.Create(headers))); + Context.Parent.Tell(Message.Create(MessageCommand.headers, HeadersPayload.Create(headers))); } private void OnHeadersMessageReceived(HeadersPayload payload) @@ -273,13 +272,13 @@ private void OnInvMessageReceived(InvPayload payload) private void OnMemPoolMessageReceived() { foreach (InvPayload payload in InvPayload.CreateGroup(InventoryType.TX, Blockchain.Singleton.MemPool.GetVerifiedTransactions().Select(p => p.Hash).ToArray())) - Context.Parent.Tell(Message.Create("inv", payload)); + Context.Parent.Tell(Message.Create(MessageCommand.inv, payload)); } private void OnPingMessageReceived(PingPayload payload) { Context.Parent.Tell(payload); - Context.Parent.Tell(Message.Create("pong", PingPayload.Create(Blockchain.Singleton.Height, payload.Nonce))); + Context.Parent.Tell(Message.Create(MessageCommand.pong, PingPayload.Create(Blockchain.Singleton.Height, payload.Nonce))); } private void OnPongMessageReceived(PingPayload payload) @@ -317,13 +316,13 @@ protected override bool IsHighPriority(object message) if (!(message is Message msg)) return true; switch (msg.Command) { - case "consensus": - case "filteradd": - case "filterclear": - case "filterload": - case "verack": - case "version": - case "alert": + case MessageCommand.consensus: + case MessageCommand.filteradd: + case MessageCommand.filterclear: + case MessageCommand.filterload: + case MessageCommand.verack: + case MessageCommand.version: + case MessageCommand.alert: return true; default: return false; @@ -335,11 +334,11 @@ protected override bool ShallDrop(object message, IEnumerable queue) if (!(message is Message msg)) return false; switch (msg.Command) { - case "getaddr": - case "getblocks": - case "getdata": - case "getheaders": - case "mempool": + case MessageCommand.getaddr: + case MessageCommand.getblocks: + case MessageCommand.getdata: + case MessageCommand.getheaders: + case MessageCommand.mempool: return queue.OfType().Any(p => p.Command == msg.Command); default: return false; diff --git a/neo/Network/P2P/RemoteNode.cs b/neo/Network/P2P/RemoteNode.cs index 65a384188a..b6d6dc2676 100644 --- a/neo/Network/P2P/RemoteNode.cs +++ b/neo/Network/P2P/RemoteNode.cs @@ -1,4 +1,8 @@ -using Akka.Actor; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using Akka.Actor; using Akka.Configuration; using Akka.IO; using Neo.Cryptography; @@ -6,10 +10,6 @@ using Neo.IO.Actors; using Neo.Ledger; using Neo.Network.P2P.Payloads; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net; namespace Neo.Network.P2P { @@ -22,9 +22,12 @@ internal class Relay { public IInventory Inventory; } private readonly Queue message_queue_high = new Queue(); private readonly Queue message_queue_low = new Queue(); private ByteString msg_buffer = ByteString.Empty; - private bool ack = true; private BloomFilter bloom_filter; + private bool ack = true; private bool verack = false; + private bool _validateConnection = true; + + public static readonly uint Magic = ProtocolSettings.Default.Magic; public IPEndPoint Listener => new IPEndPoint(Remote.Address, ListenerPort); public override int ListenerPort => Version?.Port ?? 0; @@ -37,7 +40,9 @@ public RemoteNode(NeoSystem system, object connection, IPEndPoint remote, IPEndP this.system = system; this.protocol = Context.ActorOf(ProtocolHandler.Props(system)); LocalNode.Singleton.RemoteNodes.TryAdd(Self, this); - SendMessage(Message.Create("version", VersionPayload.Create(LocalNode.Singleton.ListenerPort, LocalNode.Nonce, LocalNode.UserAgent, Blockchain.Singleton.Height))); + + SendData(ByteString.FromBytes(BitConverter.GetBytes(Magic))); + SendMessage(Message.Create(MessageCommand.version, VersionPayload.Create(LocalNode.Singleton.ListenerPort, LocalNode.Nonce, LocalNode.UserAgent, Blockchain.Singleton.Height))); } private void CheckMessageQueue() @@ -49,7 +54,7 @@ private void CheckMessageQueue() SendMessage(queue.Dequeue()); } - private void EnqueueMessage(string command, ISerializable payload = null) + private void EnqueueMessage(MessageCommand command, ISerializable payload = null) { EnqueueMessage(Message.Create(command, payload)); } @@ -59,26 +64,26 @@ private void EnqueueMessage(Message message) bool is_single = false; switch (message.Command) { - case "addr": - case "getaddr": - case "getblocks": - case "getheaders": - case "mempool": - case "ping": - case "pong": + case MessageCommand.addr: + case MessageCommand.getaddr: + case MessageCommand.getblocks: + case MessageCommand.getheaders: + case MessageCommand.mempool: + case MessageCommand.ping: + case MessageCommand.pong: is_single = true; break; } Queue message_queue; switch (message.Command) { - case "alert": - case "consensus": - case "filteradd": - case "filterclear": - case "filterload": - case "getaddr": - case "mempool": + case MessageCommand.alert: + case MessageCommand.consensus: + case MessageCommand.filteradd: + case MessageCommand.filterclear: + case MessageCommand.filterload: + case MessageCommand.getaddr: + case MessageCommand.mempool: message_queue = message_queue_high; break; default: @@ -99,6 +104,20 @@ protected override void OnAck() protected override void OnData(ByteString data) { msg_buffer = msg_buffer.Concat(data); + + if (_validateConnection) + { + // We need to check the magic header + + if (msg_buffer.Count < sizeof(uint)) return; + + uint magic = msg_buffer.Slice(0, sizeof(uint)).ToArray().ToUInt32(0); + if (magic != Magic) throw new FormatException(); + + msg_buffer = msg_buffer.Slice(sizeof(uint)).Compact(); + _validateConnection = false; + } + for (Message message = TryParseMessage(); message != null; message = TryParseMessage()) protocol.Tell(message); } @@ -146,7 +165,7 @@ private void OnRelay(IInventory inventory) if (bloom_filter != null && !bloom_filter.Test((Transaction)inventory)) return; } - EnqueueMessage("inv", InvPayload.Create(inventory.InventoryType, inventory.Hash)); + EnqueueMessage(MessageCommand.inv, InvPayload.Create(inventory.InventoryType, inventory.Hash)); } private void OnSend(IInventory inventory) @@ -157,7 +176,7 @@ private void OnSend(IInventory inventory) if (bloom_filter != null && !bloom_filter.Test((Transaction)inventory)) return; } - EnqueueMessage(inventory.InventoryType.ToString().ToLower(), inventory); + EnqueueMessage((MessageCommand)inventory.InventoryType, inventory); } private void OnSetFilter(BloomFilter filter) @@ -186,7 +205,7 @@ private void OnVersionPayload(VersionPayload version) Disconnect(true); return; } - SendMessage(Message.Create("verack")); + SendMessage(Message.Create(MessageCommand.verack)); } protected override void PostStop() @@ -217,19 +236,11 @@ protected override SupervisorStrategy SupervisorStrategy() private Message TryParseMessage() { - if (msg_buffer.Count < sizeof(uint)) return null; - uint magic = msg_buffer.Slice(0, sizeof(uint)).ToArray().ToUInt32(0); - if (magic != Message.Magic) - throw new FormatException(); - if (msg_buffer.Count < Message.HeaderSize) return null; - int length = msg_buffer.Slice(16, sizeof(int)).ToArray().ToInt32(0); - if (length > Message.PayloadMaxSize) - throw new FormatException(); - length += Message.HeaderSize; - if (msg_buffer.Count < length) return null; - Message message = msg_buffer.Slice(0, length).ToArray().AsSerializable(); + var length = Message.TryDeserialize(msg_buffer, out var msg); + if (length <= 0) return null; + msg_buffer = msg_buffer.Slice(length).Compact(); - return message; + return msg; } } diff --git a/neo/Network/P2P/TaskManager.cs b/neo/Network/P2P/TaskManager.cs index 558f1a2eee..536e390603 100644 --- a/neo/Network/P2P/TaskManager.cs +++ b/neo/Network/P2P/TaskManager.cs @@ -74,7 +74,7 @@ private void OnNewTasks(InvPayload payload) } foreach (InvPayload group in InvPayload.CreateGroup(payload.Type, hashes.ToArray())) - Sender.Tell(Message.Create("getdata", group)); + Sender.Tell(Message.Create(MessageCommand.getdata, group)); } protected override void OnReceive(object message) @@ -119,7 +119,7 @@ private void OnRestartTasks(InvPayload payload) foreach (UInt256 hash in payload.Hashes) globalTasks.Remove(hash); foreach (InvPayload group in InvPayload.CreateGroup(payload.Type, payload.Hashes)) - system.LocalNode.Tell(Message.Create("getdata", group)); + system.LocalNode.Tell(Message.Create(MessageCommand.getdata, group)); } private void OnTaskCompleted(UInt256 hash) @@ -215,7 +215,7 @@ private void RequestTasks(TaskSession session) foreach (UInt256 hash in hashes) session.Tasks[hash] = DateTime.UtcNow; foreach (InvPayload group in InvPayload.CreateGroup(InventoryType.Block, hashes.ToArray())) - session.RemoteNode.Tell(Message.Create("getdata", group)); + session.RemoteNode.Tell(Message.Create(MessageCommand.getdata, group)); return; } } @@ -223,7 +223,7 @@ private void RequestTasks(TaskSession session) { session.Tasks[HeaderTaskHash] = DateTime.UtcNow; IncrementGlobalTask(HeaderTaskHash); - session.RemoteNode.Tell(Message.Create("getheaders", GetBlocksPayload.Create(Blockchain.Singleton.CurrentHeaderHash))); + session.RemoteNode.Tell(Message.Create(MessageCommand.getheaders, GetBlocksPayload.Create(Blockchain.Singleton.CurrentHeaderHash))); } else if (Blockchain.Singleton.Height < session.Version.StartHeight) { @@ -237,7 +237,7 @@ private void RequestTasks(TaskSession session) break; } } - session.RemoteNode.Tell(Message.Create("getblocks", GetBlocksPayload.Create(hash))); + session.RemoteNode.Tell(Message.Create(MessageCommand.getblocks, GetBlocksPayload.Create(hash))); } } } From 4865963db3b5f0ea2597aadad94f6edbe42726fd Mon Sep 17 00:00:00 2001 From: Shargon Date: Tue, 16 Apr 2019 12:41:58 +0200 Subject: [PATCH 02/45] Unit test --- neo.UnitTests/UT_P2PMessage.cs | 85 ++++++++++++++++++++++++++++++++++ neo/Network/P2P/Message.cs | 22 +++++---- 2 files changed, 98 insertions(+), 9 deletions(-) create mode 100644 neo.UnitTests/UT_P2PMessage.cs diff --git a/neo.UnitTests/UT_P2PMessage.cs b/neo.UnitTests/UT_P2PMessage.cs new file mode 100644 index 0000000000..916932a173 --- /dev/null +++ b/neo.UnitTests/UT_P2PMessage.cs @@ -0,0 +1,85 @@ +using Akka.IO; +using FluentAssertions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Neo.IO; +using Neo.Network.P2P; +using Neo.Network.P2P.Payloads; + +namespace Neo.UnitTests +{ + [TestClass] + public class UT_P2PMessage + { + [TestMethod] + public void Serialize_Deserialize() + { + var payload = PingPayload.Create(uint.MaxValue); + var msg = Message.Create(MessageCommand.ping, payload); + var buffer = msg.ToArray(); + var copy = buffer.AsSerializable(); + var payloadCopy = copy.GetPayload(); + + copy.Command.Should().Be(MessageCommand.ping); + copy.Flags.Should().Be(MessageFlags.None); + + payloadCopy.LastBlockIndex.Should().Be(payload.LastBlockIndex); + payloadCopy.Nonce.Should().Be(payload.Nonce); + payloadCopy.Timestamp.Should().Be(payload.Timestamp); + } + + [TestMethod] + public void Serialize_Deserialize_ByteString() + { + var payload = PingPayload.Create(uint.MaxValue); + var msg = Message.Create(MessageCommand.ping, payload); + var buffer = ByteString.CopyFrom(msg.ToArray()); + var length = Message.TryDeserialize(buffer, out var copy); + + length.Should().Be(buffer.Count); + + var payloadCopy = copy.GetPayload(); + + copy.Command.Should().Be(MessageCommand.ping); + copy.Flags.Should().Be(MessageFlags.None); + + payloadCopy.LastBlockIndex.Should().Be(payload.LastBlockIndex); + payloadCopy.Nonce.Should().Be(payload.Nonce); + payloadCopy.Timestamp.Should().Be(payload.Timestamp); + } + + [TestMethod] + public void Compression() + { + var payload = new VersionPayload() + { + Relay = true, + UserAgent = "".PadLeft(1024, '0'), + Nonce = 1, + Port = 2, + Services = 3, + StartHeight = 4, + Timestamp = 5, + Version = 6 + }; + var msg = Message.Create(MessageCommand.version, payload); + var buffer = msg.ToArray(); + + buffer.Length.Should().BeLessThan(80); + + var copy = buffer.AsSerializable(); + var payloadCopy = copy.GetPayload(); + + copy.Command.Should().Be(MessageCommand.version); + copy.Flags.Should().Be(MessageFlags.Compressed); + + payloadCopy.Relay.Should().Be(payload.Relay); + payloadCopy.UserAgent.Should().Be(payload.UserAgent); + payloadCopy.Nonce.Should().Be(payload.Nonce); + payloadCopy.Port.Should().Be(payload.Port); + payloadCopy.Services.Should().Be(payload.Services); + payloadCopy.StartHeight.Should().Be(payload.StartHeight); + payloadCopy.Timestamp.Should().Be(payload.Timestamp); + payloadCopy.Version.Should().Be(payload.Version); + } + } +} diff --git a/neo/Network/P2P/Message.cs b/neo/Network/P2P/Message.cs index 16d44173e3..16f6ee8178 100644 --- a/neo/Network/P2P/Message.cs +++ b/neo/Network/P2P/Message.cs @@ -17,11 +17,16 @@ public class Message : ISerializable public MessageCommand Command; public byte[] Payload; + private ISerializable _payload_deserialized = null; + public int Size => 2 + IO.Helper.GetVarSize(Payload.Length) + Payload.Length; public static Message Create(MessageCommand command, ISerializable payload = null) { - return Create(command, payload == null ? new byte[0] : payload.ToArray()); + var ret = Create(command, payload == null ? new byte[0] : payload.ToArray()); + ret._payload_deserialized = payload; + + return ret; } public static Message Create(MessageCommand command, byte[] payload) @@ -49,6 +54,13 @@ public static Message Create(MessageCommand command, byte[] payload) }; } + void ISerializable.Serialize(BinaryWriter writer) + { + writer.Write((byte)Flags); + writer.Write((byte)Command); + writer.WriteVarBytes(Payload); + } + void ISerializable.Deserialize(BinaryReader reader) { this.Flags = (MessageFlags)reader.ReadByte(); @@ -110,7 +122,6 @@ public byte[] GetPayload() return this.Payload; } - private ISerializable _payload_deserialized = null; public T GetPayload() where T : ISerializable, new() { if (_payload_deserialized is null) @@ -124,12 +135,5 @@ public Transaction GetTransaction() _payload_deserialized = Transaction.DeserializeFrom(GetPayload()); return (Transaction)_payload_deserialized; } - - void ISerializable.Serialize(BinaryWriter writer) - { - writer.Write((byte)Flags); - writer.Write((byte)Command); - writer.WriteVarBytes(Payload); - } } } \ No newline at end of file From cde68d41ab05010fa02a259b00d0e72443aa1893 Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 22 Apr 2019 09:27:23 +0200 Subject: [PATCH 03/45] VersionServices --- neo/Network/P2P/Payloads/NetworkAddressWithTime.cs | 14 ++++++-------- neo/Network/P2P/Payloads/VersionPayload.cs | 12 ++++++------ neo/Network/P2P/Payloads/VersionServices.cs | 11 +++++++++++ 3 files changed, 23 insertions(+), 14 deletions(-) create mode 100644 neo/Network/P2P/Payloads/VersionServices.cs diff --git a/neo/Network/P2P/Payloads/NetworkAddressWithTime.cs b/neo/Network/P2P/Payloads/NetworkAddressWithTime.cs index 32b995cba6..fee0cca399 100644 --- a/neo/Network/P2P/Payloads/NetworkAddressWithTime.cs +++ b/neo/Network/P2P/Payloads/NetworkAddressWithTime.cs @@ -1,22 +1,20 @@ -using Neo.IO; -using System; +using System; using System.IO; using System.Linq; using System.Net; +using Neo.IO; namespace Neo.Network.P2P.Payloads { public class NetworkAddressWithTime : ISerializable { - public const ulong NODE_NETWORK = 1; - public uint Timestamp; - public ulong Services; + public VersionServices Services; public IPEndPoint EndPoint; public int Size => sizeof(uint) + sizeof(ulong) + 16 + sizeof(ushort); - public static NetworkAddressWithTime Create(IPEndPoint endpoint, ulong services, uint timestamp) + public static NetworkAddressWithTime Create(IPEndPoint endpoint, VersionServices services, uint timestamp) { return new NetworkAddressWithTime { @@ -29,7 +27,7 @@ public static NetworkAddressWithTime Create(IPEndPoint endpoint, ulong services, void ISerializable.Deserialize(BinaryReader reader) { Timestamp = reader.ReadUInt32(); - Services = reader.ReadUInt64(); + Services = (VersionServices)reader.ReadUInt64(); byte[] data = reader.ReadBytes(16); if (data.Length != 16) throw new FormatException(); IPAddress address = new IPAddress(data).Unmap(); @@ -42,7 +40,7 @@ void ISerializable.Deserialize(BinaryReader reader) void ISerializable.Serialize(BinaryWriter writer) { writer.Write(Timestamp); - writer.Write(Services); + writer.Write((ulong)Services); writer.Write(EndPoint.Address.MapToIPv6().GetAddressBytes()); writer.Write(BitConverter.GetBytes((ushort)EndPoint.Port).Reverse().ToArray()); } diff --git a/neo/Network/P2P/Payloads/VersionPayload.cs b/neo/Network/P2P/Payloads/VersionPayload.cs index ddfdfc7d69..624e6f1b8d 100644 --- a/neo/Network/P2P/Payloads/VersionPayload.cs +++ b/neo/Network/P2P/Payloads/VersionPayload.cs @@ -1,13 +1,13 @@ -using Neo.IO; -using System; +using System; using System.IO; +using Neo.IO; namespace Neo.Network.P2P.Payloads { public class VersionPayload : ISerializable { public uint Version; - public ulong Services; + public VersionServices Services; public uint Timestamp; public ushort Port; public uint Nonce; @@ -22,7 +22,7 @@ public static VersionPayload Create(int port, uint nonce, string userAgent, uint return new VersionPayload { Version = LocalNode.ProtocolVersion, - Services = NetworkAddressWithTime.NODE_NETWORK, + Services = VersionServices.NodeNetwork, Timestamp = DateTime.Now.ToTimestamp(), Port = (ushort)port, Nonce = nonce, @@ -35,7 +35,7 @@ public static VersionPayload Create(int port, uint nonce, string userAgent, uint void ISerializable.Deserialize(BinaryReader reader) { Version = reader.ReadUInt32(); - Services = reader.ReadUInt64(); + Services = (VersionServices)reader.ReadUInt64(); Timestamp = reader.ReadUInt32(); Port = reader.ReadUInt16(); Nonce = reader.ReadUInt32(); @@ -47,7 +47,7 @@ void ISerializable.Deserialize(BinaryReader reader) void ISerializable.Serialize(BinaryWriter writer) { writer.Write(Version); - writer.Write(Services); + writer.Write((ulong)Services); writer.Write(Timestamp); writer.Write(Port); writer.Write(Nonce); diff --git a/neo/Network/P2P/Payloads/VersionServices.cs b/neo/Network/P2P/Payloads/VersionServices.cs new file mode 100644 index 0000000000..9c93d0bf29 --- /dev/null +++ b/neo/Network/P2P/Payloads/VersionServices.cs @@ -0,0 +1,11 @@ +using System; + +namespace Neo.Network.P2P.Payloads +{ + [Flags] + public enum VersionServices : ulong + { + None = 0x00, + NodeNetwork = 0x01, + } +} \ No newline at end of file From a115abb1ad7705b5115ead5f4eecd59727ac93e4 Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 22 Apr 2019 09:37:47 +0200 Subject: [PATCH 04/45] Remove string --- neo/Network/P2P/ProtocolHandler.cs | 2 +- neo/Network/P2P/RemoteNode.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/neo/Network/P2P/ProtocolHandler.cs b/neo/Network/P2P/ProtocolHandler.cs index 79f4b892a3..69575d13f2 100644 --- a/neo/Network/P2P/ProtocolHandler.cs +++ b/neo/Network/P2P/ProtocolHandler.cs @@ -289,7 +289,7 @@ private void OnPongMessageReceived(PingPayload payload) private void OnVerackMessageReceived() { verack = true; - Context.Parent.Tell("verack"); + Context.Parent.Tell(MessageCommand.verack); } private void OnVersionMessageReceived(VersionPayload payload) diff --git a/neo/Network/P2P/RemoteNode.cs b/neo/Network/P2P/RemoteNode.cs index b6d6dc2676..a0e22a67ba 100644 --- a/neo/Network/P2P/RemoteNode.cs +++ b/neo/Network/P2P/RemoteNode.cs @@ -139,7 +139,7 @@ protected override void OnReceive(object message) case VersionPayload payload: OnVersionPayload(payload); break; - case "verack": + case MessageCommand.verack: OnVerack(); break; case ProtocolHandler.SetFilter setFilter: From 5eb05924add7691f6ec348fd3776560e99af3114 Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 22 Apr 2019 10:10:30 +0200 Subject: [PATCH 05/45] Optimization --- neo/Network/P2P/RemoteNode.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/neo/Network/P2P/RemoteNode.cs b/neo/Network/P2P/RemoteNode.cs index a0e22a67ba..61b4f95c27 100644 --- a/neo/Network/P2P/RemoteNode.cs +++ b/neo/Network/P2P/RemoteNode.cs @@ -49,8 +49,11 @@ private void CheckMessageQueue() { if (!verack || !ack) return; Queue queue = message_queue_high; - if (queue.Count == 0) queue = message_queue_low; - if (queue.Count == 0) return; + if (queue.Count == 0) + { + queue = message_queue_low; + if (queue.Count == 0) return; + } SendMessage(queue.Dequeue()); } From 89ac44784b1bbc3f9fc83858c15381bddf86aa4a Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 22 Apr 2019 11:31:35 +0200 Subject: [PATCH 06/45] Fix test --- neo.UnitTests/UT_P2PMessage.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neo.UnitTests/UT_P2PMessage.cs b/neo.UnitTests/UT_P2PMessage.cs index 916932a173..4d447356df 100644 --- a/neo.UnitTests/UT_P2PMessage.cs +++ b/neo.UnitTests/UT_P2PMessage.cs @@ -1,4 +1,4 @@ -using Akka.IO; +using Akka.IO; using FluentAssertions; using Microsoft.VisualStudio.TestTools.UnitTesting; using Neo.IO; @@ -56,7 +56,7 @@ public void Compression() UserAgent = "".PadLeft(1024, '0'), Nonce = 1, Port = 2, - Services = 3, + Services = VersionServices.NodeNetwork, StartHeight = 4, Timestamp = 5, Version = 6 From 964bd05d862c533831de6f52f0096e0a42a836d3 Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 22 Apr 2019 19:31:53 +0200 Subject: [PATCH 07/45] Rename compression flag --- neo/Network/P2P/Helper.cs | 4 ++-- neo/Network/P2P/Message.cs | 8 ++++---- neo/Network/P2P/MessageFlags.cs | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/neo/Network/P2P/Helper.cs b/neo/Network/P2P/Helper.cs index cfd99bc8ca..c02bd126eb 100644 --- a/neo/Network/P2P/Helper.cs +++ b/neo/Network/P2P/Helper.cs @@ -6,7 +6,7 @@ namespace Neo.Network.P2P { public static class Helper { - public static byte[] Uncompress(this byte[] data) + public static byte[] UncompressGzip(this byte[] data) { using (var output = new MemoryStream()) using (var input = new MemoryStream(data)) @@ -24,7 +24,7 @@ public static byte[] Uncompress(this byte[] data) } } - public static byte[] Compress(this byte[] data) + public static byte[] CompressGzip(this byte[] data) { using (var stream = new MemoryStream()) { diff --git a/neo/Network/P2P/Message.cs b/neo/Network/P2P/Message.cs index 16f6ee8178..dd01059cf5 100644 --- a/neo/Network/P2P/Message.cs +++ b/neo/Network/P2P/Message.cs @@ -37,12 +37,12 @@ public static Message Create(MessageCommand command, byte[] payload) if (payload.Length > CompressionMinSize) { - var compressed = payload.Compress(); + var compressed = payload.CompressGzip(); if (compressed.Length < payload.Length - CompressionThreshold) { payload = compressed; - flags |= MessageFlags.Compressed; + flags |= MessageFlags.CompressedGzip; } } @@ -114,9 +114,9 @@ public static int TryDeserialize(ByteString data, out Message msg) public byte[] GetPayload() { - if (this.Flags.HasFlag(MessageFlags.Compressed)) + if (this.Flags.HasFlag(MessageFlags.CompressedGzip)) { - return this.Payload.Uncompress(); + return this.Payload.UncompressGzip(); } return this.Payload; diff --git a/neo/Network/P2P/MessageFlags.cs b/neo/Network/P2P/MessageFlags.cs index a608f5bfef..37958d4b7b 100644 --- a/neo/Network/P2P/MessageFlags.cs +++ b/neo/Network/P2P/MessageFlags.cs @@ -6,6 +6,6 @@ namespace Neo.Network.P2P public enum MessageFlags : byte { None = 0, - Compressed = 1 << 0 + CompressedGzip = 1 << 0 } } \ No newline at end of file From 23a53a30ad4f040e8202567369d5d438b6dd547c Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 22 Apr 2019 19:34:45 +0200 Subject: [PATCH 08/45] Fix test --- neo.UnitTests/UT_P2PMessage.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo.UnitTests/UT_P2PMessage.cs b/neo.UnitTests/UT_P2PMessage.cs index 4d447356df..af1c87c126 100644 --- a/neo.UnitTests/UT_P2PMessage.cs +++ b/neo.UnitTests/UT_P2PMessage.cs @@ -70,7 +70,7 @@ public void Compression() var payloadCopy = copy.GetPayload(); copy.Command.Should().Be(MessageCommand.version); - copy.Flags.Should().Be(MessageFlags.Compressed); + copy.Flags.Should().Be(MessageFlags.CompressedGzip); payloadCopy.Relay.Should().Be(payload.Relay); payloadCopy.UserAgent.Should().Be(payload.UserAgent); From 2e8fa8fae3b1b939aea854674c75a0dec69ca12c Mon Sep 17 00:00:00 2001 From: Shargon Date: Sun, 28 Apr 2019 21:57:28 +0200 Subject: [PATCH 09/45] Move magic to VersionPayload --- neo/Network/P2P/Payloads/VersionPayload.cs | 6 +++++- neo/Network/P2P/RemoteNode.cs | 22 ++-------------------- 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/neo/Network/P2P/Payloads/VersionPayload.cs b/neo/Network/P2P/Payloads/VersionPayload.cs index 624e6f1b8d..2ce8641bb2 100644 --- a/neo/Network/P2P/Payloads/VersionPayload.cs +++ b/neo/Network/P2P/Payloads/VersionPayload.cs @@ -6,6 +6,7 @@ namespace Neo.Network.P2P.Payloads { public class VersionPayload : ISerializable { + public uint Magic; public uint Version; public VersionServices Services; public uint Timestamp; @@ -15,12 +16,13 @@ public class VersionPayload : ISerializable public uint StartHeight; public bool Relay; - public int Size => sizeof(uint) + sizeof(ulong) + sizeof(uint) + sizeof(ushort) + sizeof(uint) + UserAgent.GetVarSize() + sizeof(uint) + sizeof(bool); + public int Size => (sizeof(uint) * 5) + sizeof(ulong) + sizeof(ushort) + UserAgent.GetVarSize() + sizeof(bool); public static VersionPayload Create(int port, uint nonce, string userAgent, uint startHeight) { return new VersionPayload { + Magic = ProtocolSettings.Default.Magic, Version = LocalNode.ProtocolVersion, Services = VersionServices.NodeNetwork, Timestamp = DateTime.Now.ToTimestamp(), @@ -34,6 +36,7 @@ public static VersionPayload Create(int port, uint nonce, string userAgent, uint void ISerializable.Deserialize(BinaryReader reader) { + Magic = reader.ReadUInt32(); Version = reader.ReadUInt32(); Services = (VersionServices)reader.ReadUInt64(); Timestamp = reader.ReadUInt32(); @@ -46,6 +49,7 @@ void ISerializable.Deserialize(BinaryReader reader) void ISerializable.Serialize(BinaryWriter writer) { + writer.Write(Magic); writer.Write(Version); writer.Write((ulong)Services); writer.Write(Timestamp); diff --git a/neo/Network/P2P/RemoteNode.cs b/neo/Network/P2P/RemoteNode.cs index 61b4f95c27..fef9edc6fb 100644 --- a/neo/Network/P2P/RemoteNode.cs +++ b/neo/Network/P2P/RemoteNode.cs @@ -1,5 +1,4 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; using System.Net; using Akka.Actor; @@ -25,9 +24,6 @@ internal class Relay { public IInventory Inventory; } private BloomFilter bloom_filter; private bool ack = true; private bool verack = false; - private bool _validateConnection = true; - - public static readonly uint Magic = ProtocolSettings.Default.Magic; public IPEndPoint Listener => new IPEndPoint(Remote.Address, ListenerPort); public override int ListenerPort => Version?.Port ?? 0; @@ -41,7 +37,6 @@ public RemoteNode(NeoSystem system, object connection, IPEndPoint remote, IPEndP this.protocol = Context.ActorOf(ProtocolHandler.Props(system)); LocalNode.Singleton.RemoteNodes.TryAdd(Self, this); - SendData(ByteString.FromBytes(BitConverter.GetBytes(Magic))); SendMessage(Message.Create(MessageCommand.version, VersionPayload.Create(LocalNode.Singleton.ListenerPort, LocalNode.Nonce, LocalNode.UserAgent, Blockchain.Singleton.Height))); } @@ -108,19 +103,6 @@ protected override void OnData(ByteString data) { msg_buffer = msg_buffer.Concat(data); - if (_validateConnection) - { - // We need to check the magic header - - if (msg_buffer.Count < sizeof(uint)) return; - - uint magic = msg_buffer.Slice(0, sizeof(uint)).ToArray().ToUInt32(0); - if (magic != Magic) throw new FormatException(); - - msg_buffer = msg_buffer.Slice(sizeof(uint)).Compact(); - _validateConnection = false; - } - for (Message message = TryParseMessage(); message != null; message = TryParseMessage()) protocol.Tell(message); } @@ -198,7 +180,7 @@ private void OnVersionPayload(VersionPayload version) { this.Version = version; this.LastBlockIndex = Version.StartHeight; - if (version.Nonce == LocalNode.Nonce) + if (version.Nonce == LocalNode.Nonce || version.Magic != ProtocolSettings.Default.Magic) { Disconnect(true); return; From f5185962d07a3c361f4e4e7bf580d544dd7fc83c Mon Sep 17 00:00:00 2001 From: Shargon Date: Sun, 28 Apr 2019 21:58:57 +0200 Subject: [PATCH 10/45] faster check --- neo/Network/P2P/ProtocolHandler.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/neo/Network/P2P/ProtocolHandler.cs b/neo/Network/P2P/ProtocolHandler.cs index 69575d13f2..c72153cd51 100644 --- a/neo/Network/P2P/ProtocolHandler.cs +++ b/neo/Network/P2P/ProtocolHandler.cs @@ -102,9 +102,8 @@ protected override void OnReceive(object message) OnPongMessageReceived(msg.GetPayload()); break; case MessageCommand.tx: - var tx = msg.GetTransaction(); - if (tx.Size > Transaction.MaxTransactionSize) return; - OnInventoryReceived(tx); + if (msg.Payload.Length > Transaction.MaxTransactionSize) return; + OnInventoryReceived(msg.GetTransaction()); break; case MessageCommand.verack: case MessageCommand.version: From 96f510b5a098bf630496a8eb50583a38491c3c22 Mon Sep 17 00:00:00 2001 From: Shargon Date: Sun, 28 Apr 2019 22:01:02 +0200 Subject: [PATCH 11/45] Reduce changes --- neo/Network/P2P/ProtocolHandler.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neo/Network/P2P/ProtocolHandler.cs b/neo/Network/P2P/ProtocolHandler.cs index c72153cd51..e718e565d6 100644 --- a/neo/Network/P2P/ProtocolHandler.cs +++ b/neo/Network/P2P/ProtocolHandler.cs @@ -102,8 +102,8 @@ protected override void OnReceive(object message) OnPongMessageReceived(msg.GetPayload()); break; case MessageCommand.tx: - if (msg.Payload.Length > Transaction.MaxTransactionSize) return; - OnInventoryReceived(msg.GetTransaction()); + if (msg.Payload.Length <= Transaction.MaxTransactionSize) + OnInventoryReceived(msg.GetTransaction()); break; case MessageCommand.verack: case MessageCommand.version: From a9302ddc420f4ef515779c3e54ec1170475d232b Mon Sep 17 00:00:00 2001 From: Shargon Date: Sun, 28 Apr 2019 22:04:25 +0200 Subject: [PATCH 12/45] Capitalize enum --- neo.UnitTests/UT_P2PMessage.cs | 12 ++-- neo/Consensus/ConsensusService.cs | 2 +- neo/Network/P2P/LocalNode.cs | 2 +- neo/Network/P2P/MessageCommand.cs | 44 +++++++------- neo/Network/P2P/ProtocolHandler.cs | 92 +++++++++++++++--------------- neo/Network/P2P/RemoteNode.cs | 36 ++++++------ neo/Network/P2P/TaskManager.cs | 10 ++-- 7 files changed, 99 insertions(+), 99 deletions(-) diff --git a/neo.UnitTests/UT_P2PMessage.cs b/neo.UnitTests/UT_P2PMessage.cs index af1c87c126..4c21b8e1eb 100644 --- a/neo.UnitTests/UT_P2PMessage.cs +++ b/neo.UnitTests/UT_P2PMessage.cs @@ -14,12 +14,12 @@ public class UT_P2PMessage public void Serialize_Deserialize() { var payload = PingPayload.Create(uint.MaxValue); - var msg = Message.Create(MessageCommand.ping, payload); + var msg = Message.Create(MessageCommand.Ping, payload); var buffer = msg.ToArray(); var copy = buffer.AsSerializable(); var payloadCopy = copy.GetPayload(); - copy.Command.Should().Be(MessageCommand.ping); + copy.Command.Should().Be(MessageCommand.Ping); copy.Flags.Should().Be(MessageFlags.None); payloadCopy.LastBlockIndex.Should().Be(payload.LastBlockIndex); @@ -31,7 +31,7 @@ public void Serialize_Deserialize() public void Serialize_Deserialize_ByteString() { var payload = PingPayload.Create(uint.MaxValue); - var msg = Message.Create(MessageCommand.ping, payload); + var msg = Message.Create(MessageCommand.Ping, payload); var buffer = ByteString.CopyFrom(msg.ToArray()); var length = Message.TryDeserialize(buffer, out var copy); @@ -39,7 +39,7 @@ public void Serialize_Deserialize_ByteString() var payloadCopy = copy.GetPayload(); - copy.Command.Should().Be(MessageCommand.ping); + copy.Command.Should().Be(MessageCommand.Ping); copy.Flags.Should().Be(MessageFlags.None); payloadCopy.LastBlockIndex.Should().Be(payload.LastBlockIndex); @@ -61,7 +61,7 @@ public void Compression() Timestamp = 5, Version = 6 }; - var msg = Message.Create(MessageCommand.version, payload); + var msg = Message.Create(MessageCommand.Version, payload); var buffer = msg.ToArray(); buffer.Length.Should().BeLessThan(80); @@ -69,7 +69,7 @@ public void Compression() var copy = buffer.AsSerializable(); var payloadCopy = copy.GetPayload(); - copy.Command.Should().Be(MessageCommand.version); + copy.Command.Should().Be(MessageCommand.Version); copy.Flags.Should().Be(MessageFlags.CompressedGzip); payloadCopy.Relay.Should().Be(payload.Relay); diff --git a/neo/Consensus/ConsensusService.cs b/neo/Consensus/ConsensusService.cs index ff1f3beffe..12acc43b5c 100644 --- a/neo/Consensus/ConsensusService.cs +++ b/neo/Consensus/ConsensusService.cs @@ -624,7 +624,7 @@ private void SendPrepareRequest() if (context.TransactionHashes.Length > 1) { foreach (InvPayload payload in InvPayload.CreateGroup(InventoryType.TX, context.TransactionHashes.Skip(1).ToArray())) - localNode.Tell(Message.Create(MessageCommand.inv, payload)); + localNode.Tell(Message.Create(MessageCommand.Inv, payload)); } ChangeTimer(TimeSpan.FromSeconds((Blockchain.SecondsPerBlock << (context.ViewNumber + 1)) - (context.ViewNumber == 0 ? Blockchain.SecondsPerBlock : 0))); } diff --git a/neo/Network/P2P/LocalNode.cs b/neo/Network/P2P/LocalNode.cs index a1dd282c9c..489ca68d72 100644 --- a/neo/Network/P2P/LocalNode.cs +++ b/neo/Network/P2P/LocalNode.cs @@ -126,7 +126,7 @@ protected override void NeedMorePeers(int count) count = Math.Max(count, 5); if (ConnectedPeers.Count > 0) { - BroadcastMessage(MessageCommand.getaddr); + BroadcastMessage(MessageCommand.GetAddr); } else { diff --git a/neo/Network/P2P/MessageCommand.cs b/neo/Network/P2P/MessageCommand.cs index 003b2d04b0..3fee26db45 100644 --- a/neo/Network/P2P/MessageCommand.cs +++ b/neo/Network/P2P/MessageCommand.cs @@ -3,34 +3,34 @@ public enum MessageCommand : byte { // Same value as InventoryType - tx = 0x01, + Tx = 0x01, // Same value as InventoryType - block = 0x02, - mempool = 0x03, - addr = 0x04, - inv = 0x05, - headers = 0x06, - merkleblock = 0x07, - version = 0x08, - verack = 0x09, - alert = 0x0A, - reject = 0x0B, + Block = 0x02, + Mempool = 0x03, + Addr = 0x04, + Inv = 0x05, + Headers = 0x06, + MerklebBock = 0x07, + Version = 0x08, + Verack = 0x09, + Alert = 0x0A, + Reject = 0x0B, - ping = 0x10, - pong = 0x11, + Ping = 0x10, + Pong = 0x11, - getaddr = 0x20, - getblocks = 0x21, - getdata = 0x22, - getheaders = 0x23, + GetAddr = 0x20, + GetBlocks = 0x21, + GetData = 0x22, + GetHeaders = 0x23, - filteradd = 0x30, - filterclear = 0x31, - filterload = 0x32, + FilterAdd = 0x30, + FilterClear = 0x31, + FilterLoad = 0x32, // Same value as InventoryType - consensus = 0xE0, + Consensus = 0xE0, - notfound = 0xFF, + NotFound = 0xFF, } } \ No newline at end of file diff --git a/neo/Network/P2P/ProtocolHandler.cs b/neo/Network/P2P/ProtocolHandler.cs index e718e565d6..a181e5b681 100644 --- a/neo/Network/P2P/ProtocolHandler.cs +++ b/neo/Network/P2P/ProtocolHandler.cs @@ -42,76 +42,76 @@ protected override void OnReceive(object message) return; if (version == null) { - if (msg.Command != MessageCommand.version) + if (msg.Command != MessageCommand.Version) throw new ProtocolViolationException(); OnVersionMessageReceived(msg.GetPayload()); return; } if (!verack) { - if (msg.Command != MessageCommand.verack) + if (msg.Command != MessageCommand.Verack) throw new ProtocolViolationException(); OnVerackMessageReceived(); return; } switch (msg.Command) { - case MessageCommand.addr: + case MessageCommand.Addr: OnAddrMessageReceived(msg.GetPayload()); break; - case MessageCommand.block: + case MessageCommand.Block: OnInventoryReceived(msg.GetPayload()); break; - case MessageCommand.consensus: + case MessageCommand.Consensus: OnInventoryReceived(msg.GetPayload()); break; - case MessageCommand.filteradd: + case MessageCommand.FilterAdd: OnFilterAddMessageReceived(msg.GetPayload()); break; - case MessageCommand.filterclear: + case MessageCommand.FilterClear: OnFilterClearMessageReceived(); break; - case MessageCommand.filterload: + case MessageCommand.FilterLoad: OnFilterLoadMessageReceived(msg.GetPayload()); break; - case MessageCommand.getaddr: + case MessageCommand.GetAddr: OnGetAddrMessageReceived(); break; - case MessageCommand.getblocks: + case MessageCommand.GetBlocks: OnGetBlocksMessageReceived(msg.GetPayload()); break; - case MessageCommand.getdata: + case MessageCommand.GetData: OnGetDataMessageReceived(msg.GetPayload()); break; - case MessageCommand.getheaders: + case MessageCommand.GetHeaders: OnGetHeadersMessageReceived(msg.GetPayload()); break; - case MessageCommand.headers: + case MessageCommand.Headers: OnHeadersMessageReceived(msg.GetPayload()); break; - case MessageCommand.inv: + case MessageCommand.Inv: OnInvMessageReceived(msg.GetPayload()); break; - case MessageCommand.mempool: + case MessageCommand.Mempool: OnMemPoolMessageReceived(); break; - case MessageCommand.ping: + case MessageCommand.Ping: OnPingMessageReceived(msg.GetPayload()); break; - case MessageCommand.pong: + case MessageCommand.Pong: OnPongMessageReceived(msg.GetPayload()); break; - case MessageCommand.tx: + case MessageCommand.Tx: if (msg.Payload.Length <= Transaction.MaxTransactionSize) OnInventoryReceived(msg.GetTransaction()); break; - case MessageCommand.verack: - case MessageCommand.version: + case MessageCommand.Verack: + case MessageCommand.Version: throw new ProtocolViolationException(); - case MessageCommand.alert: - case MessageCommand.merkleblock: - case MessageCommand.notfound: - case MessageCommand.reject: + case MessageCommand.Alert: + case MessageCommand.MerklebBock: + case MessageCommand.NotFound: + case MessageCommand.Reject: default: break; } } @@ -152,7 +152,7 @@ private void OnGetAddrMessageReceived() .Take(AddrPayload.MaxCountToSend); NetworkAddressWithTime[] networkAddresses = peers.Select(p => NetworkAddressWithTime.Create(p.Listener, p.Version.Services, p.Version.Timestamp)).ToArray(); if (networkAddresses.Length == 0) return; - Context.Parent.Tell(Message.Create(MessageCommand.addr, AddrPayload.Create(networkAddresses))); + Context.Parent.Tell(Message.Create(MessageCommand.Addr, AddrPayload.Create(networkAddresses))); } private void OnGetBlocksMessageReceived(GetBlocksPayload payload) @@ -173,7 +173,7 @@ private void OnGetBlocksMessageReceived(GetBlocksPayload payload) hashes.Add(hash); } if (hashes.Count == 0) return; - Context.Parent.Tell(Message.Create(MessageCommand.inv, InvPayload.Create(InventoryType.Block, hashes.ToArray()))); + Context.Parent.Tell(Message.Create(MessageCommand.Inv, InvPayload.Create(InventoryType.Block, hashes.ToArray()))); } private void OnGetDataMessageReceived(InvPayload payload) @@ -188,7 +188,7 @@ private void OnGetDataMessageReceived(InvPayload payload) if (inventory == null) inventory = Blockchain.Singleton.GetTransaction(hash); if (inventory is Transaction) - Context.Parent.Tell(Message.Create(MessageCommand.tx, inventory)); + Context.Parent.Tell(Message.Create(MessageCommand.Tx, inventory)); break; case InventoryType.Block: if (inventory == null) @@ -197,18 +197,18 @@ private void OnGetDataMessageReceived(InvPayload payload) { if (bloom_filter == null) { - Context.Parent.Tell(Message.Create(MessageCommand.block, inventory)); + Context.Parent.Tell(Message.Create(MessageCommand.Block, inventory)); } else { BitArray flags = new BitArray(block.Transactions.Select(p => bloom_filter.Test(p)).ToArray()); - Context.Parent.Tell(Message.Create(MessageCommand.merkleblock, MerkleBlockPayload.Create(block, flags))); + Context.Parent.Tell(Message.Create(MessageCommand.MerklebBock, MerkleBlockPayload.Create(block, flags))); } } break; case InventoryType.Consensus: if (inventory != null) - Context.Parent.Tell(Message.Create(MessageCommand.consensus, inventory)); + Context.Parent.Tell(Message.Create(MessageCommand.Consensus, inventory)); break; } } @@ -233,7 +233,7 @@ private void OnGetHeadersMessageReceived(GetBlocksPayload payload) headers.Add(header); } if (headers.Count == 0) return; - Context.Parent.Tell(Message.Create(MessageCommand.headers, HeadersPayload.Create(headers))); + Context.Parent.Tell(Message.Create(MessageCommand.Headers, HeadersPayload.Create(headers))); } private void OnHeadersMessageReceived(HeadersPayload payload) @@ -271,13 +271,13 @@ private void OnInvMessageReceived(InvPayload payload) private void OnMemPoolMessageReceived() { foreach (InvPayload payload in InvPayload.CreateGroup(InventoryType.TX, Blockchain.Singleton.MemPool.GetVerifiedTransactions().Select(p => p.Hash).ToArray())) - Context.Parent.Tell(Message.Create(MessageCommand.inv, payload)); + Context.Parent.Tell(Message.Create(MessageCommand.Inv, payload)); } private void OnPingMessageReceived(PingPayload payload) { Context.Parent.Tell(payload); - Context.Parent.Tell(Message.Create(MessageCommand.pong, PingPayload.Create(Blockchain.Singleton.Height, payload.Nonce))); + Context.Parent.Tell(Message.Create(MessageCommand.Pong, PingPayload.Create(Blockchain.Singleton.Height, payload.Nonce))); } private void OnPongMessageReceived(PingPayload payload) @@ -288,7 +288,7 @@ private void OnPongMessageReceived(PingPayload payload) private void OnVerackMessageReceived() { verack = true; - Context.Parent.Tell(MessageCommand.verack); + Context.Parent.Tell(MessageCommand.Verack); } private void OnVersionMessageReceived(VersionPayload payload) @@ -315,13 +315,13 @@ protected override bool IsHighPriority(object message) if (!(message is Message msg)) return true; switch (msg.Command) { - case MessageCommand.consensus: - case MessageCommand.filteradd: - case MessageCommand.filterclear: - case MessageCommand.filterload: - case MessageCommand.verack: - case MessageCommand.version: - case MessageCommand.alert: + case MessageCommand.Consensus: + case MessageCommand.FilterAdd: + case MessageCommand.FilterClear: + case MessageCommand.FilterLoad: + case MessageCommand.Verack: + case MessageCommand.Version: + case MessageCommand.Alert: return true; default: return false; @@ -333,11 +333,11 @@ protected override bool ShallDrop(object message, IEnumerable queue) if (!(message is Message msg)) return false; switch (msg.Command) { - case MessageCommand.getaddr: - case MessageCommand.getblocks: - case MessageCommand.getdata: - case MessageCommand.getheaders: - case MessageCommand.mempool: + case MessageCommand.GetAddr: + case MessageCommand.GetBlocks: + case MessageCommand.GetData: + case MessageCommand.GetHeaders: + case MessageCommand.Mempool: return queue.OfType().Any(p => p.Command == msg.Command); default: return false; diff --git a/neo/Network/P2P/RemoteNode.cs b/neo/Network/P2P/RemoteNode.cs index fef9edc6fb..482cb557f2 100644 --- a/neo/Network/P2P/RemoteNode.cs +++ b/neo/Network/P2P/RemoteNode.cs @@ -37,7 +37,7 @@ public RemoteNode(NeoSystem system, object connection, IPEndPoint remote, IPEndP this.protocol = Context.ActorOf(ProtocolHandler.Props(system)); LocalNode.Singleton.RemoteNodes.TryAdd(Self, this); - SendMessage(Message.Create(MessageCommand.version, VersionPayload.Create(LocalNode.Singleton.ListenerPort, LocalNode.Nonce, LocalNode.UserAgent, Blockchain.Singleton.Height))); + SendMessage(Message.Create(MessageCommand.Version, VersionPayload.Create(LocalNode.Singleton.ListenerPort, LocalNode.Nonce, LocalNode.UserAgent, Blockchain.Singleton.Height))); } private void CheckMessageQueue() @@ -62,26 +62,26 @@ private void EnqueueMessage(Message message) bool is_single = false; switch (message.Command) { - case MessageCommand.addr: - case MessageCommand.getaddr: - case MessageCommand.getblocks: - case MessageCommand.getheaders: - case MessageCommand.mempool: - case MessageCommand.ping: - case MessageCommand.pong: + case MessageCommand.Addr: + case MessageCommand.GetAddr: + case MessageCommand.GetBlocks: + case MessageCommand.GetHeaders: + case MessageCommand.Mempool: + case MessageCommand.Ping: + case MessageCommand.Pong: is_single = true; break; } Queue message_queue; switch (message.Command) { - case MessageCommand.alert: - case MessageCommand.consensus: - case MessageCommand.filteradd: - case MessageCommand.filterclear: - case MessageCommand.filterload: - case MessageCommand.getaddr: - case MessageCommand.mempool: + case MessageCommand.Alert: + case MessageCommand.Consensus: + case MessageCommand.FilterAdd: + case MessageCommand.FilterClear: + case MessageCommand.FilterLoad: + case MessageCommand.GetAddr: + case MessageCommand.Mempool: message_queue = message_queue_high; break; default: @@ -124,7 +124,7 @@ protected override void OnReceive(object message) case VersionPayload payload: OnVersionPayload(payload); break; - case MessageCommand.verack: + case MessageCommand.Verack: OnVerack(); break; case ProtocolHandler.SetFilter setFilter: @@ -150,7 +150,7 @@ private void OnRelay(IInventory inventory) if (bloom_filter != null && !bloom_filter.Test((Transaction)inventory)) return; } - EnqueueMessage(MessageCommand.inv, InvPayload.Create(inventory.InventoryType, inventory.Hash)); + EnqueueMessage(MessageCommand.Inv, InvPayload.Create(inventory.InventoryType, inventory.Hash)); } private void OnSend(IInventory inventory) @@ -190,7 +190,7 @@ private void OnVersionPayload(VersionPayload version) Disconnect(true); return; } - SendMessage(Message.Create(MessageCommand.verack)); + SendMessage(Message.Create(MessageCommand.Verack)); } protected override void PostStop() diff --git a/neo/Network/P2P/TaskManager.cs b/neo/Network/P2P/TaskManager.cs index 536e390603..bd9481f2ea 100644 --- a/neo/Network/P2P/TaskManager.cs +++ b/neo/Network/P2P/TaskManager.cs @@ -74,7 +74,7 @@ private void OnNewTasks(InvPayload payload) } foreach (InvPayload group in InvPayload.CreateGroup(payload.Type, hashes.ToArray())) - Sender.Tell(Message.Create(MessageCommand.getdata, group)); + Sender.Tell(Message.Create(MessageCommand.GetData, group)); } protected override void OnReceive(object message) @@ -119,7 +119,7 @@ private void OnRestartTasks(InvPayload payload) foreach (UInt256 hash in payload.Hashes) globalTasks.Remove(hash); foreach (InvPayload group in InvPayload.CreateGroup(payload.Type, payload.Hashes)) - system.LocalNode.Tell(Message.Create(MessageCommand.getdata, group)); + system.LocalNode.Tell(Message.Create(MessageCommand.GetData, group)); } private void OnTaskCompleted(UInt256 hash) @@ -215,7 +215,7 @@ private void RequestTasks(TaskSession session) foreach (UInt256 hash in hashes) session.Tasks[hash] = DateTime.UtcNow; foreach (InvPayload group in InvPayload.CreateGroup(InventoryType.Block, hashes.ToArray())) - session.RemoteNode.Tell(Message.Create(MessageCommand.getdata, group)); + session.RemoteNode.Tell(Message.Create(MessageCommand.GetData, group)); return; } } @@ -223,7 +223,7 @@ private void RequestTasks(TaskSession session) { session.Tasks[HeaderTaskHash] = DateTime.UtcNow; IncrementGlobalTask(HeaderTaskHash); - session.RemoteNode.Tell(Message.Create(MessageCommand.getheaders, GetBlocksPayload.Create(Blockchain.Singleton.CurrentHeaderHash))); + session.RemoteNode.Tell(Message.Create(MessageCommand.GetHeaders, GetBlocksPayload.Create(Blockchain.Singleton.CurrentHeaderHash))); } else if (Blockchain.Singleton.Height < session.Version.StartHeight) { @@ -237,7 +237,7 @@ private void RequestTasks(TaskSession session) break; } } - session.RemoteNode.Tell(Message.Create(MessageCommand.getblocks, GetBlocksPayload.Create(hash))); + session.RemoteNode.Tell(Message.Create(MessageCommand.GetBlocks, GetBlocksPayload.Create(hash))); } } } From 2b8f13fb469b0fa2d6ed1ac6aa1ce1e6c8604dfb Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 29 Apr 2019 09:48:50 +0200 Subject: [PATCH 13/45] Relay inside flag and FullNode flag --- neo.UnitTests/UT_P2PMessage.cs | 4 +--- neo/Network/P2P/Payloads/VersionPayload.cs | 10 +++------- neo/Network/P2P/Payloads/VersionServices.cs | 7 +++++-- neo/Network/P2P/RemoteNode.cs | 4 ++-- 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/neo.UnitTests/UT_P2PMessage.cs b/neo.UnitTests/UT_P2PMessage.cs index 4c21b8e1eb..b0c2cc5100 100644 --- a/neo.UnitTests/UT_P2PMessage.cs +++ b/neo.UnitTests/UT_P2PMessage.cs @@ -52,11 +52,10 @@ public void Compression() { var payload = new VersionPayload() { - Relay = true, UserAgent = "".PadLeft(1024, '0'), Nonce = 1, Port = 2, - Services = VersionServices.NodeNetwork, + Services = VersionServices.FullNode, StartHeight = 4, Timestamp = 5, Version = 6 @@ -72,7 +71,6 @@ public void Compression() copy.Command.Should().Be(MessageCommand.Version); copy.Flags.Should().Be(MessageFlags.CompressedGzip); - payloadCopy.Relay.Should().Be(payload.Relay); payloadCopy.UserAgent.Should().Be(payload.UserAgent); payloadCopy.Nonce.Should().Be(payload.Nonce); payloadCopy.Port.Should().Be(payload.Port); diff --git a/neo/Network/P2P/Payloads/VersionPayload.cs b/neo/Network/P2P/Payloads/VersionPayload.cs index 2ce8641bb2..d25c5532d5 100644 --- a/neo/Network/P2P/Payloads/VersionPayload.cs +++ b/neo/Network/P2P/Payloads/VersionPayload.cs @@ -14,9 +14,8 @@ public class VersionPayload : ISerializable public uint Nonce; public string UserAgent; public uint StartHeight; - public bool Relay; - public int Size => (sizeof(uint) * 5) + sizeof(ulong) + sizeof(ushort) + UserAgent.GetVarSize() + sizeof(bool); + public int Size => (sizeof(uint) * 5) + sizeof(ulong) + sizeof(ushort) + UserAgent.GetVarSize(); public static VersionPayload Create(int port, uint nonce, string userAgent, uint startHeight) { @@ -24,13 +23,12 @@ public static VersionPayload Create(int port, uint nonce, string userAgent, uint { Magic = ProtocolSettings.Default.Magic, Version = LocalNode.ProtocolVersion, - Services = VersionServices.NodeNetwork, + Services = VersionServices.FullNode, Timestamp = DateTime.Now.ToTimestamp(), Port = (ushort)port, Nonce = nonce, UserAgent = userAgent, StartHeight = startHeight, - Relay = true }; } @@ -44,7 +42,6 @@ void ISerializable.Deserialize(BinaryReader reader) Nonce = reader.ReadUInt32(); UserAgent = reader.ReadVarString(1024); StartHeight = reader.ReadUInt32(); - Relay = reader.ReadBoolean(); } void ISerializable.Serialize(BinaryWriter writer) @@ -57,7 +54,6 @@ void ISerializable.Serialize(BinaryWriter writer) writer.Write(Nonce); writer.WriteVarString(UserAgent); writer.Write(StartHeight); - writer.Write(Relay); } } -} +} \ No newline at end of file diff --git a/neo/Network/P2P/Payloads/VersionServices.cs b/neo/Network/P2P/Payloads/VersionServices.cs index 9c93d0bf29..036bedaa27 100644 --- a/neo/Network/P2P/Payloads/VersionServices.cs +++ b/neo/Network/P2P/Payloads/VersionServices.cs @@ -5,7 +5,10 @@ namespace Neo.Network.P2P.Payloads [Flags] public enum VersionServices : ulong { - None = 0x00, - NodeNetwork = 0x01, + None = 0, + NodeNetwork = 1 << 0, + Relay = 1 << 1, + + FullNode = NodeNetwork | Relay } } \ No newline at end of file diff --git a/neo/Network/P2P/RemoteNode.cs b/neo/Network/P2P/RemoteNode.cs index 482cb557f2..dc7009e35e 100644 --- a/neo/Network/P2P/RemoteNode.cs +++ b/neo/Network/P2P/RemoteNode.cs @@ -144,7 +144,7 @@ private void OnPingPayload(PingPayload payload) private void OnRelay(IInventory inventory) { - if (Version?.Relay != true) return; + if (Version?.Services.HasFlag(VersionServices.Relay) != true) return; if (inventory.InventoryType == InventoryType.TX) { if (bloom_filter != null && !bloom_filter.Test((Transaction)inventory)) @@ -155,7 +155,7 @@ private void OnRelay(IInventory inventory) private void OnSend(IInventory inventory) { - if (Version?.Relay != true) return; + if (Version?.Services.HasFlag(VersionServices.Relay) != true) return; if (inventory.InventoryType == InventoryType.TX) { if (bloom_filter != null && !bloom_filter.Test((Transaction)inventory)) From a8ce6dd7fe54d101699a8ce417aa69bf098269c5 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Mon, 29 Apr 2019 10:04:03 +0200 Subject: [PATCH 14/45] Update neo/Network/P2P/Payloads/VersionServices.cs Co-Authored-By: shargon --- neo/Network/P2P/Payloads/VersionServices.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neo/Network/P2P/Payloads/VersionServices.cs b/neo/Network/P2P/Payloads/VersionServices.cs index 036bedaa27..38b0ec62c4 100644 --- a/neo/Network/P2P/Payloads/VersionServices.cs +++ b/neo/Network/P2P/Payloads/VersionServices.cs @@ -7,8 +7,8 @@ public enum VersionServices : ulong { None = 0, NodeNetwork = 1 << 0, - Relay = 1 << 1, + AcceptRelay = 1 << 1, FullNode = NodeNetwork | Relay } -} \ No newline at end of file +} From e2581035ddacbc39bab6fd298e31ab677570131f Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 29 Apr 2019 10:06:56 +0200 Subject: [PATCH 15/45] Fix --- neo/Network/P2P/Message.cs | 2 +- neo/Network/P2P/Payloads/VersionServices.cs | 2 +- neo/Network/P2P/RemoteNode.cs | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/neo/Network/P2P/Message.cs b/neo/Network/P2P/Message.cs index dd01059cf5..5957dca0ee 100644 --- a/neo/Network/P2P/Message.cs +++ b/neo/Network/P2P/Message.cs @@ -100,7 +100,7 @@ public static int TryDeserialize(ByteString data, out Message msg) } if (length > PayloadMaxSize) throw new FormatException(); - if (data.Count < (int)length) return 0; + if (data.Count < (int)length + payloadIndex) return 0; msg = new Message() { diff --git a/neo/Network/P2P/Payloads/VersionServices.cs b/neo/Network/P2P/Payloads/VersionServices.cs index 38b0ec62c4..4eb76d5d2b 100644 --- a/neo/Network/P2P/Payloads/VersionServices.cs +++ b/neo/Network/P2P/Payloads/VersionServices.cs @@ -9,6 +9,6 @@ public enum VersionServices : ulong NodeNetwork = 1 << 0, AcceptRelay = 1 << 1, - FullNode = NodeNetwork | Relay + FullNode = NodeNetwork | AcceptRelay } } diff --git a/neo/Network/P2P/RemoteNode.cs b/neo/Network/P2P/RemoteNode.cs index dc7009e35e..b04179aa87 100644 --- a/neo/Network/P2P/RemoteNode.cs +++ b/neo/Network/P2P/RemoteNode.cs @@ -144,7 +144,7 @@ private void OnPingPayload(PingPayload payload) private void OnRelay(IInventory inventory) { - if (Version?.Services.HasFlag(VersionServices.Relay) != true) return; + if (Version?.Services.HasFlag(VersionServices.AcceptRelay) != true) return; if (inventory.InventoryType == InventoryType.TX) { if (bloom_filter != null && !bloom_filter.Test((Transaction)inventory)) @@ -155,7 +155,7 @@ private void OnRelay(IInventory inventory) private void OnSend(IInventory inventory) { - if (Version?.Services.HasFlag(VersionServices.Relay) != true) return; + if (Version?.Services.HasFlag(VersionServices.AcceptRelay) != true) return; if (inventory.InventoryType == InventoryType.TX) { if (bloom_filter != null && !bloom_filter.Test((Transaction)inventory)) From 935f06849296b9bd26437486d1913cc8664fd594 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Mon, 29 Apr 2019 10:28:08 +0200 Subject: [PATCH 16/45] Update neo/Network/P2P/Message.cs Co-Authored-By: shargon --- neo/Network/P2P/Message.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neo/Network/P2P/Message.cs b/neo/Network/P2P/Message.cs index 5957dca0ee..832db15133 100644 --- a/neo/Network/P2P/Message.cs +++ b/neo/Network/P2P/Message.cs @@ -19,7 +19,7 @@ public class Message : ISerializable private ISerializable _payload_deserialized = null; - public int Size => 2 + IO.Helper.GetVarSize(Payload.Length) + Payload.Length; + public int Size => sizeof(MessageFlags) + sizeof(MessageCommand) + Payload.GetVarSize(); public static Message Create(MessageCommand command, ISerializable payload = null) { @@ -136,4 +136,4 @@ public Transaction GetTransaction() return (Transaction)_payload_deserialized; } } -} \ No newline at end of file +} From 7d55288d500abd49169bcc5fb3521ffb33b6e330 Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 29 Apr 2019 14:05:16 +0200 Subject: [PATCH 17/45] Allow Checksum --- neo.UnitTests/UT_P2PMessage.cs | 62 ++++++++++++++++++++++++++------- neo/Network/P2P/Message.cs | 54 +++++++++++++++++++++------- neo/Network/P2P/MessageFlags.cs | 3 +- 3 files changed, 94 insertions(+), 25 deletions(-) diff --git a/neo.UnitTests/UT_P2PMessage.cs b/neo.UnitTests/UT_P2PMessage.cs index b0c2cc5100..eb314c3a39 100644 --- a/neo.UnitTests/UT_P2PMessage.cs +++ b/neo.UnitTests/UT_P2PMessage.cs @@ -10,43 +10,80 @@ namespace Neo.UnitTests [TestClass] public class UT_P2PMessage { - [TestMethod] - public void Serialize_Deserialize() + public void Serialize_Deserialize(bool checksum) { var payload = PingPayload.Create(uint.MaxValue); - var msg = Message.Create(MessageCommand.Ping, payload); + var msg = Message.Create(MessageCommand.Ping, payload, checksum); var buffer = msg.ToArray(); var copy = buffer.AsSerializable(); var payloadCopy = copy.GetPayload(); - copy.Command.Should().Be(MessageCommand.Ping); - copy.Flags.Should().Be(MessageFlags.None); + copy.CheckSum.Should().Be(msg.CheckSum); + copy.Command.Should().Be(msg.Command); + copy.Flags.Should().Be(msg.Flags); payloadCopy.LastBlockIndex.Should().Be(payload.LastBlockIndex); payloadCopy.Nonce.Should().Be(payload.Nonce); payloadCopy.Timestamp.Should().Be(payload.Timestamp); + + if (checksum) + { + copy.Flags.Should().HaveFlag(MessageFlags.Checksum); + copy.CheckSum.Should().BeGreaterThan(0); + } + else + { + copy.Flags.Should().NotHaveFlag(MessageFlags.Checksum); + copy.CheckSum.Should().Be(0); + } } [TestMethod] - public void Serialize_Deserialize_ByteString() + public void Serialize_Deserialize_Checksum() => Serialize_Deserialize(true); + + [TestMethod] + public void Serialize_Deserialize_WithoutChecksum() => Serialize_Deserialize(false); + + public void Serialize_Deserialize_ByteString(bool checksum) { var payload = PingPayload.Create(uint.MaxValue); - var msg = Message.Create(MessageCommand.Ping, payload); + var msg = Message.Create(MessageCommand.Ping, payload, checksum); var buffer = ByteString.CopyFrom(msg.ToArray()); var length = Message.TryDeserialize(buffer, out var copy); - length.Should().Be(buffer.Count); var payloadCopy = copy.GetPayload(); - copy.Command.Should().Be(MessageCommand.Ping); - copy.Flags.Should().Be(MessageFlags.None); + copy.CheckSum.Should().Be(msg.CheckSum); + copy.Command.Should().Be(msg.Command); + copy.Flags.Should().Be(msg.Flags); payloadCopy.LastBlockIndex.Should().Be(payload.LastBlockIndex); payloadCopy.Nonce.Should().Be(payload.Nonce); payloadCopy.Timestamp.Should().Be(payload.Timestamp); + + if (checksum) + { + buffer.Count.Should().Be(length + 2); + + copy.Flags.Should().HaveFlag(MessageFlags.Checksum); + copy.CheckSum.Should().BeGreaterThan(0); + } + else + { + buffer.Count.Should().Be(length); + + copy.Flags.Should().NotHaveFlag(MessageFlags.Checksum); + copy.CheckSum.Should().Be(0); + } } + [TestMethod] + public void Serialize_Deserialize_ByteString_Checksum() => Serialize_Deserialize_ByteString(true); + + [TestMethod] + public void Serialize_Deserialize_ByteString_WithoutChecksum() => Serialize_Deserialize_ByteString(false); + [TestMethod] public void Compression() { @@ -68,7 +105,8 @@ public void Compression() var copy = buffer.AsSerializable(); var payloadCopy = copy.GetPayload(); - copy.Command.Should().Be(MessageCommand.Version); + copy.CheckSum.Should().Be(msg.CheckSum); + copy.Command.Should().Be(msg.Command); copy.Flags.Should().Be(MessageFlags.CompressedGzip); payloadCopy.UserAgent.Should().Be(payload.UserAgent); @@ -80,4 +118,4 @@ public void Compression() payloadCopy.Version.Should().Be(payload.Version); } } -} +} \ No newline at end of file diff --git a/neo/Network/P2P/Message.cs b/neo/Network/P2P/Message.cs index 832db15133..75d377fb77 100644 --- a/neo/Network/P2P/Message.cs +++ b/neo/Network/P2P/Message.cs @@ -16,22 +16,23 @@ public class Message : ISerializable public MessageFlags Flags; public MessageCommand Command; public byte[] Payload; + public short CheckSum; private ISerializable _payload_deserialized = null; - public int Size => sizeof(MessageFlags) + sizeof(MessageCommand) + Payload.GetVarSize(); + public int Size => sizeof(MessageFlags) + sizeof(MessageCommand) + Payload.GetVarSize() + (Flags.HasFlag(MessageFlags.Checksum) ? sizeof(short) : 0); - public static Message Create(MessageCommand command, ISerializable payload = null) + public static Message Create(MessageCommand command, ISerializable payload = null, bool checksum = true) { - var ret = Create(command, payload == null ? new byte[0] : payload.ToArray()); + var ret = Create(command, payload == null ? new byte[0] : payload.ToArray(), checksum); ret._payload_deserialized = payload; return ret; } - public static Message Create(MessageCommand command, byte[] payload) + public static Message Create(MessageCommand command, byte[] payload, bool checksum = true) { - var flags = MessageFlags.None; + var flags = checksum ? MessageFlags.Checksum : MessageFlags.None; // Try compression @@ -50,25 +51,40 @@ public static Message Create(MessageCommand command, byte[] payload) { Flags = flags, Command = command, - Payload = payload + Payload = payload, + CheckSum = checksum ? ComputeChecksum(payload) : (short)0 }; } + public static short ComputeChecksum(byte[] data) => BitConverter.ToInt16(data.Sha256(), 0); + void ISerializable.Serialize(BinaryWriter writer) { writer.Write((byte)Flags); writer.Write((byte)Command); writer.WriteVarBytes(Payload); + + if (Flags.HasFlag(MessageFlags.Checksum)) + { + writer.Write(CheckSum); + } } void ISerializable.Deserialize(BinaryReader reader) { - this.Flags = (MessageFlags)reader.ReadByte(); - this.Command = (MessageCommand)reader.ReadByte(); + Flags = (MessageFlags)reader.ReadByte(); + Command = (MessageCommand)reader.ReadByte(); var length = (int)reader.ReadVarInt(int.MaxValue); if (length > PayloadMaxSize) throw new FormatException(); - this.Payload = reader.ReadBytes(length); + Payload = reader.ReadBytes(length); + + if (Flags.HasFlag(MessageFlags.Checksum)) + { + CheckSum = reader.ReadInt16(); + + if (CheckSum != ComputeChecksum(Payload)) throw new FormatException(); + } } public static int TryDeserialize(ByteString data, out Message msg) @@ -100,13 +116,27 @@ public static int TryDeserialize(ByteString data, out Message msg) } if (length > PayloadMaxSize) throw new FormatException(); - if (data.Count < (int)length + payloadIndex) return 0; + + short checksum = 0; + var flags = (MessageFlags)header[0]; + + if (flags.HasFlag(MessageFlags.Checksum)) + { + if (data.Count < (int)length + payloadIndex + 2) return 0; + + checksum = BitConverter.ToInt16(data.Slice(payloadIndex + (int)length, 2).ToArray(), 0); + } + else + { + if (data.Count < (int)length + payloadIndex) return 0; + } msg = new Message() { - Flags = (MessageFlags)header[0], + Flags = flags, Command = (MessageCommand)header[1], - Payload = data.Slice(payloadIndex, (int)length).ToArray() + Payload = data.Slice(payloadIndex, (int)length).ToArray(), + CheckSum = checksum, }; return payloadIndex + (int)length; diff --git a/neo/Network/P2P/MessageFlags.cs b/neo/Network/P2P/MessageFlags.cs index 37958d4b7b..b48d5efc43 100644 --- a/neo/Network/P2P/MessageFlags.cs +++ b/neo/Network/P2P/MessageFlags.cs @@ -6,6 +6,7 @@ namespace Neo.Network.P2P public enum MessageFlags : byte { None = 0, - CompressedGzip = 1 << 0 + Checksum = 1 << 0, + CompressedGzip = 1 << 1 } } \ No newline at end of file From fa4536b977612c1eb67351f4440e06fcc1b98d17 Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 29 Apr 2019 14:08:12 +0200 Subject: [PATCH 18/45] Clean code --- neo/Network/P2P/Message.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/neo/Network/P2P/Message.cs b/neo/Network/P2P/Message.cs index 75d377fb77..d789b6f972 100644 --- a/neo/Network/P2P/Message.cs +++ b/neo/Network/P2P/Message.cs @@ -82,7 +82,6 @@ void ISerializable.Deserialize(BinaryReader reader) if (Flags.HasFlag(MessageFlags.Checksum)) { CheckSum = reader.ReadInt16(); - if (CheckSum != ComputeChecksum(Payload)) throw new FormatException(); } } From 055e3f9ccc3f753883546644bf2f6ff043d35c19 Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 29 Apr 2019 14:09:13 +0200 Subject: [PATCH 19/45] Clean code --- neo/Network/P2P/Message.cs | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/neo/Network/P2P/Message.cs b/neo/Network/P2P/Message.cs index d789b6f972..1da385eb3d 100644 --- a/neo/Network/P2P/Message.cs +++ b/neo/Network/P2P/Message.cs @@ -122,7 +122,6 @@ public static int TryDeserialize(ByteString data, out Message msg) if (flags.HasFlag(MessageFlags.Checksum)) { if (data.Count < (int)length + payloadIndex + 2) return 0; - checksum = BitConverter.ToInt16(data.Slice(payloadIndex + (int)length, 2).ToArray(), 0); } else @@ -141,15 +140,7 @@ public static int TryDeserialize(ByteString data, out Message msg) return payloadIndex + (int)length; } - public byte[] GetPayload() - { - if (this.Flags.HasFlag(MessageFlags.CompressedGzip)) - { - return this.Payload.UncompressGzip(); - } - - return this.Payload; - } + public byte[] GetPayload() => Flags.HasFlag(MessageFlags.CompressedGzip) ? Payload.UncompressGzip() : Payload; public T GetPayload() where T : ISerializable, new() { @@ -165,4 +156,4 @@ public Transaction GetTransaction() return (Transaction)_payload_deserialized; } } -} +} \ No newline at end of file From 7c7cd9350eae438b56c120b0dce87e7c72d93004 Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 29 Apr 2019 14:11:07 +0200 Subject: [PATCH 20/45] Fix tests --- neo.UnitTests/UT_P2PMessage.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/neo.UnitTests/UT_P2PMessage.cs b/neo.UnitTests/UT_P2PMessage.cs index eb314c3a39..ca9f0c123b 100644 --- a/neo.UnitTests/UT_P2PMessage.cs +++ b/neo.UnitTests/UT_P2PMessage.cs @@ -29,7 +29,7 @@ public void Serialize_Deserialize(bool checksum) if (checksum) { copy.Flags.Should().HaveFlag(MessageFlags.Checksum); - copy.CheckSum.Should().BeGreaterThan(0); + copy.CheckSum.Should().NotBe(0); } else { @@ -67,7 +67,7 @@ public void Serialize_Deserialize_ByteString(bool checksum) buffer.Count.Should().Be(length + 2); copy.Flags.Should().HaveFlag(MessageFlags.Checksum); - copy.CheckSum.Should().BeGreaterThan(0); + copy.CheckSum.Should().NotBe(0); } else { @@ -107,7 +107,7 @@ public void Compression() copy.CheckSum.Should().Be(msg.CheckSum); copy.Command.Should().Be(msg.Command); - copy.Flags.Should().Be(MessageFlags.CompressedGzip); + copy.Flags.Should().HaveFlag(MessageFlags.CompressedGzip); payloadCopy.UserAgent.Should().Be(payload.UserAgent); payloadCopy.Nonce.Should().Be(payload.Nonce); From eb18c9ac639de0d25491503171e09af1335d87db Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 29 Apr 2019 14:16:44 +0200 Subject: [PATCH 21/45] Fix and clean --- neo/Network/P2P/Helper.cs | 6 +++++- neo/Network/P2P/Message.cs | 10 +++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/neo/Network/P2P/Helper.cs b/neo/Network/P2P/Helper.cs index c02bd126eb..b8595d0beb 100644 --- a/neo/Network/P2P/Helper.cs +++ b/neo/Network/P2P/Helper.cs @@ -1,6 +1,8 @@ -using Neo.Network.P2P.Payloads; +using System; using System.IO; using System.IO.Compression; +using Neo.Cryptography; +using Neo.Network.P2P.Payloads; namespace Neo.Network.P2P { @@ -24,6 +26,8 @@ public static byte[] UncompressGzip(this byte[] data) } } + public static short Checksum(this byte[] data) => BitConverter.ToInt16(data.Sha256(), 0); + public static byte[] CompressGzip(this byte[] data) { using (var stream = new MemoryStream()) diff --git a/neo/Network/P2P/Message.cs b/neo/Network/P2P/Message.cs index 1da385eb3d..3c36214340 100644 --- a/neo/Network/P2P/Message.cs +++ b/neo/Network/P2P/Message.cs @@ -52,12 +52,10 @@ public static Message Create(MessageCommand command, byte[] payload, bool checks Flags = flags, Command = command, Payload = payload, - CheckSum = checksum ? ComputeChecksum(payload) : (short)0 + CheckSum = checksum ? payload.Checksum() : (short)0 }; } - public static short ComputeChecksum(byte[] data) => BitConverter.ToInt16(data.Sha256(), 0); - void ISerializable.Serialize(BinaryWriter writer) { writer.Write((byte)Flags); @@ -82,7 +80,7 @@ void ISerializable.Deserialize(BinaryReader reader) if (Flags.HasFlag(MessageFlags.Checksum)) { CheckSum = reader.ReadInt16(); - if (CheckSum != ComputeChecksum(Payload)) throw new FormatException(); + if (CheckSum != Payload.Checksum()) throw new FormatException(); } } @@ -118,11 +116,13 @@ public static int TryDeserialize(ByteString data, out Message msg) short checksum = 0; var flags = (MessageFlags)header[0]; + var ret = payloadIndex + (int)length; if (flags.HasFlag(MessageFlags.Checksum)) { if (data.Count < (int)length + payloadIndex + 2) return 0; checksum = BitConverter.ToInt16(data.Slice(payloadIndex + (int)length, 2).ToArray(), 0); + ret += 2; } else { @@ -137,7 +137,7 @@ public static int TryDeserialize(ByteString data, out Message msg) CheckSum = checksum, }; - return payloadIndex + (int)length; + return ret; } public byte[] GetPayload() => Flags.HasFlag(MessageFlags.CompressedGzip) ? Payload.UncompressGzip() : Payload; From 0c8cb422f617c1ebba954bb32ba42b997d34e139 Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 29 Apr 2019 14:17:58 +0200 Subject: [PATCH 22/45] Fix ut --- neo.UnitTests/UT_P2PMessage.cs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/neo.UnitTests/UT_P2PMessage.cs b/neo.UnitTests/UT_P2PMessage.cs index ca9f0c123b..b816a591fd 100644 --- a/neo.UnitTests/UT_P2PMessage.cs +++ b/neo.UnitTests/UT_P2PMessage.cs @@ -51,7 +51,6 @@ public void Serialize_Deserialize_ByteString(bool checksum) var buffer = ByteString.CopyFrom(msg.ToArray()); var length = Message.TryDeserialize(buffer, out var copy); - var payloadCopy = copy.GetPayload(); copy.CheckSum.Should().Be(msg.CheckSum); @@ -62,17 +61,15 @@ public void Serialize_Deserialize_ByteString(bool checksum) payloadCopy.Nonce.Should().Be(payload.Nonce); payloadCopy.Timestamp.Should().Be(payload.Timestamp); + buffer.Count.Should().Be(length); + if (checksum) { - buffer.Count.Should().Be(length + 2); - copy.Flags.Should().HaveFlag(MessageFlags.Checksum); copy.CheckSum.Should().NotBe(0); } else { - buffer.Count.Should().Be(length); - copy.Flags.Should().NotHaveFlag(MessageFlags.Checksum); copy.CheckSum.Should().Be(0); } From 9d89b6f1c8ba46d9387161f79c2599f7bd9b99d2 Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 29 Apr 2019 17:47:51 +0200 Subject: [PATCH 23/45] Move checksum --- neo/Network/P2P/Message.cs | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/neo/Network/P2P/Message.cs b/neo/Network/P2P/Message.cs index 3c36214340..874c65feb6 100644 --- a/neo/Network/P2P/Message.cs +++ b/neo/Network/P2P/Message.cs @@ -15,12 +15,12 @@ public class Message : ISerializable public MessageFlags Flags; public MessageCommand Command; - public byte[] Payload; public short CheckSum; + public byte[] Payload; private ISerializable _payload_deserialized = null; - public int Size => sizeof(MessageFlags) + sizeof(MessageCommand) + Payload.GetVarSize() + (Flags.HasFlag(MessageFlags.Checksum) ? sizeof(short) : 0); + public int Size => sizeof(MessageFlags) + sizeof(MessageCommand) + (Flags.HasFlag(MessageFlags.Checksum) ? sizeof(short) : 0) + Payload.GetVarSize(); public static Message Create(MessageCommand command, ISerializable payload = null, bool checksum = true) { @@ -60,12 +60,13 @@ void ISerializable.Serialize(BinaryWriter writer) { writer.Write((byte)Flags); writer.Write((byte)Command); - writer.WriteVarBytes(Payload); if (Flags.HasFlag(MessageFlags.Checksum)) { writer.Write(CheckSum); } + + writer.WriteVarBytes(Payload); } void ISerializable.Deserialize(BinaryReader reader) @@ -75,13 +76,18 @@ void ISerializable.Deserialize(BinaryReader reader) var length = (int)reader.ReadVarInt(int.MaxValue); if (length > PayloadMaxSize) throw new FormatException(); - Payload = reader.ReadBytes(length); if (Flags.HasFlag(MessageFlags.Checksum)) { CheckSum = reader.ReadInt16(); + Payload = reader.ReadBytes(length); + if (CheckSum != Payload.Checksum()) throw new FormatException(); } + else + { + Payload = reader.ReadBytes(length); + } } public static int TryDeserialize(ByteString data, out Message msg) @@ -116,13 +122,13 @@ public static int TryDeserialize(ByteString data, out Message msg) short checksum = 0; var flags = (MessageFlags)header[0]; - var ret = payloadIndex + (int)length; if (flags.HasFlag(MessageFlags.Checksum)) { - if (data.Count < (int)length + payloadIndex + 2) return 0; - checksum = BitConverter.ToInt16(data.Slice(payloadIndex + (int)length, 2).ToArray(), 0); - ret += 2; + payloadIndex += 2; + if (data.Count < (int)length + payloadIndex) return 0; + + checksum = BitConverter.ToInt16(data.Slice(payloadIndex, 2).ToArray(), 0); } else { @@ -137,7 +143,7 @@ public static int TryDeserialize(ByteString data, out Message msg) CheckSum = checksum, }; - return ret; + return payloadIndex + (int)length; } public byte[] GetPayload() => Flags.HasFlag(MessageFlags.CompressedGzip) ? Payload.UncompressGzip() : Payload; From 5d18814493af0a135439ee0a681d9da4c66267ee Mon Sep 17 00:00:00 2001 From: Shargon Date: Tue, 30 Apr 2019 09:35:29 +0200 Subject: [PATCH 24/45] Fix checksum and ut --- neo/Network/P2P/Message.cs | 39 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/neo/Network/P2P/Message.cs b/neo/Network/P2P/Message.cs index 874c65feb6..894d1b58fc 100644 --- a/neo/Network/P2P/Message.cs +++ b/neo/Network/P2P/Message.cs @@ -73,19 +73,21 @@ void ISerializable.Deserialize(BinaryReader reader) { Flags = (MessageFlags)reader.ReadByte(); Command = (MessageCommand)reader.ReadByte(); - var length = (int)reader.ReadVarInt(int.MaxValue); - - if (length > PayloadMaxSize) throw new FormatException(); if (Flags.HasFlag(MessageFlags.Checksum)) { CheckSum = reader.ReadInt16(); + + var length = (int)reader.ReadVarInt(int.MaxValue); + if (length > PayloadMaxSize) throw new FormatException(); Payload = reader.ReadBytes(length); if (CheckSum != Payload.Checksum()) throw new FormatException(); } else { + var length = (int)reader.ReadVarInt(int.MaxValue); + if (length > PayloadMaxSize) throw new FormatException(); Payload = reader.ReadBytes(length); } } @@ -93,11 +95,19 @@ void ISerializable.Deserialize(BinaryReader reader) public static int TryDeserialize(ByteString data, out Message msg) { msg = null; - if (data.Count < 3) return 0; + if (data.Count < 5) return 0; - var header = data.Slice(0, 3).ToArray(); - ulong length = header[2]; - int payloadIndex = 3; + short checksum = 0; + var header = data.Slice(0, 5).ToArray(); + var flags = (MessageFlags)header[0]; + + if (flags.HasFlag(MessageFlags.Checksum)) + { + checksum = BitConverter.ToInt16(header, 2); + } + + ulong length = header[4]; + int payloadIndex = 5; if (length == 0xFD) { @@ -120,20 +130,7 @@ public static int TryDeserialize(ByteString data, out Message msg) if (length > PayloadMaxSize) throw new FormatException(); - short checksum = 0; - var flags = (MessageFlags)header[0]; - - if (flags.HasFlag(MessageFlags.Checksum)) - { - payloadIndex += 2; - if (data.Count < (int)length + payloadIndex) return 0; - - checksum = BitConverter.ToInt16(data.Slice(payloadIndex, 2).ToArray(), 0); - } - else - { - if (data.Count < (int)length + payloadIndex) return 0; - } + if (data.Count < (int)length + payloadIndex) return 0; msg = new Message() { From 63c1c16aaedfe93a254c6ac8d3c055a1abfca7a0 Mon Sep 17 00:00:00 2001 From: Shargon Date: Tue, 30 Apr 2019 09:46:10 +0200 Subject: [PATCH 25/45] More fix :P --- neo/Network/P2P/Message.cs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/neo/Network/P2P/Message.cs b/neo/Network/P2P/Message.cs index 894d1b58fc..46b11a2fe9 100644 --- a/neo/Network/P2P/Message.cs +++ b/neo/Network/P2P/Message.cs @@ -97,17 +97,24 @@ public static int TryDeserialize(ByteString data, out Message msg) msg = null; if (data.Count < 5) return 0; - short checksum = 0; + ulong length; + short checksum; + int payloadIndex; var header = data.Slice(0, 5).ToArray(); var flags = (MessageFlags)header[0]; - + if (flags.HasFlag(MessageFlags.Checksum)) { checksum = BitConverter.ToInt16(header, 2); + length = header[4]; + payloadIndex = 5; + } + else + { + checksum = 0; + length = header[2]; + payloadIndex = 3; } - - ulong length = header[4]; - int payloadIndex = 5; if (length == 0xFD) { From 6cf939559c22901ec89e2e0e8c0f11391b7d4eda Mon Sep 17 00:00:00 2001 From: Shargon Date: Tue, 30 Apr 2019 12:17:33 +0200 Subject: [PATCH 26/45] Remove Checksum --- neo.UnitTests/UT_P2PMessage.cs | 45 +++---------------------- neo/Network/P2P/Helper.cs | 6 +--- neo/Network/P2P/Message.cs | 60 +++++++-------------------------- neo/Network/P2P/MessageFlags.cs | 3 +- 4 files changed, 19 insertions(+), 95 deletions(-) diff --git a/neo.UnitTests/UT_P2PMessage.cs b/neo.UnitTests/UT_P2PMessage.cs index b816a591fd..191f974e87 100644 --- a/neo.UnitTests/UT_P2PMessage.cs +++ b/neo.UnitTests/UT_P2PMessage.cs @@ -10,50 +10,33 @@ namespace Neo.UnitTests [TestClass] public class UT_P2PMessage { - public void Serialize_Deserialize(bool checksum) + [TestMethod] + public void Serialize_Deserialize() { var payload = PingPayload.Create(uint.MaxValue); - var msg = Message.Create(MessageCommand.Ping, payload, checksum); + var msg = Message.Create(MessageCommand.Ping, payload); var buffer = msg.ToArray(); var copy = buffer.AsSerializable(); var payloadCopy = copy.GetPayload(); - copy.CheckSum.Should().Be(msg.CheckSum); copy.Command.Should().Be(msg.Command); copy.Flags.Should().Be(msg.Flags); payloadCopy.LastBlockIndex.Should().Be(payload.LastBlockIndex); payloadCopy.Nonce.Should().Be(payload.Nonce); payloadCopy.Timestamp.Should().Be(payload.Timestamp); - - if (checksum) - { - copy.Flags.Should().HaveFlag(MessageFlags.Checksum); - copy.CheckSum.Should().NotBe(0); - } - else - { - copy.Flags.Should().NotHaveFlag(MessageFlags.Checksum); - copy.CheckSum.Should().Be(0); - } } [TestMethod] - public void Serialize_Deserialize_Checksum() => Serialize_Deserialize(true); - - [TestMethod] - public void Serialize_Deserialize_WithoutChecksum() => Serialize_Deserialize(false); - - public void Serialize_Deserialize_ByteString(bool checksum) + public void Serialize_Deserialize_ByteString() { var payload = PingPayload.Create(uint.MaxValue); - var msg = Message.Create(MessageCommand.Ping, payload, checksum); + var msg = Message.Create(MessageCommand.Ping, payload); var buffer = ByteString.CopyFrom(msg.ToArray()); var length = Message.TryDeserialize(buffer, out var copy); var payloadCopy = copy.GetPayload(); - copy.CheckSum.Should().Be(msg.CheckSum); copy.Command.Should().Be(msg.Command); copy.Flags.Should().Be(msg.Flags); @@ -62,25 +45,8 @@ public void Serialize_Deserialize_ByteString(bool checksum) payloadCopy.Timestamp.Should().Be(payload.Timestamp); buffer.Count.Should().Be(length); - - if (checksum) - { - copy.Flags.Should().HaveFlag(MessageFlags.Checksum); - copy.CheckSum.Should().NotBe(0); - } - else - { - copy.Flags.Should().NotHaveFlag(MessageFlags.Checksum); - copy.CheckSum.Should().Be(0); - } } - [TestMethod] - public void Serialize_Deserialize_ByteString_Checksum() => Serialize_Deserialize_ByteString(true); - - [TestMethod] - public void Serialize_Deserialize_ByteString_WithoutChecksum() => Serialize_Deserialize_ByteString(false); - [TestMethod] public void Compression() { @@ -102,7 +68,6 @@ public void Compression() var copy = buffer.AsSerializable(); var payloadCopy = copy.GetPayload(); - copy.CheckSum.Should().Be(msg.CheckSum); copy.Command.Should().Be(msg.Command); copy.Flags.Should().HaveFlag(MessageFlags.CompressedGzip); diff --git a/neo/Network/P2P/Helper.cs b/neo/Network/P2P/Helper.cs index b8595d0beb..be08a3ee5c 100644 --- a/neo/Network/P2P/Helper.cs +++ b/neo/Network/P2P/Helper.cs @@ -1,7 +1,5 @@ -using System; -using System.IO; +using System.IO; using System.IO.Compression; -using Neo.Cryptography; using Neo.Network.P2P.Payloads; namespace Neo.Network.P2P @@ -26,8 +24,6 @@ public static byte[] UncompressGzip(this byte[] data) } } - public static short Checksum(this byte[] data) => BitConverter.ToInt16(data.Sha256(), 0); - public static byte[] CompressGzip(this byte[] data) { using (var stream = new MemoryStream()) diff --git a/neo/Network/P2P/Message.cs b/neo/Network/P2P/Message.cs index 46b11a2fe9..5e9dd2b0d0 100644 --- a/neo/Network/P2P/Message.cs +++ b/neo/Network/P2P/Message.cs @@ -15,24 +15,23 @@ public class Message : ISerializable public MessageFlags Flags; public MessageCommand Command; - public short CheckSum; public byte[] Payload; private ISerializable _payload_deserialized = null; - public int Size => sizeof(MessageFlags) + sizeof(MessageCommand) + (Flags.HasFlag(MessageFlags.Checksum) ? sizeof(short) : 0) + Payload.GetVarSize(); + public int Size => sizeof(MessageFlags) + sizeof(MessageCommand) + Payload.GetVarSize(); - public static Message Create(MessageCommand command, ISerializable payload = null, bool checksum = true) + public static Message Create(MessageCommand command, ISerializable payload = null) { - var ret = Create(command, payload == null ? new byte[0] : payload.ToArray(), checksum); + var ret = Create(command, payload == null ? new byte[0] : payload.ToArray()); ret._payload_deserialized = payload; return ret; } - public static Message Create(MessageCommand command, byte[] payload, bool checksum = true) + public static Message Create(MessageCommand command, byte[] payload) { - var flags = checksum ? MessageFlags.Checksum : MessageFlags.None; + var flags = MessageFlags.None; // Try compression @@ -51,8 +50,7 @@ public static Message Create(MessageCommand command, byte[] payload, bool checks { Flags = flags, Command = command, - Payload = payload, - CheckSum = checksum ? payload.Checksum() : (short)0 + Payload = payload }; } @@ -60,12 +58,6 @@ void ISerializable.Serialize(BinaryWriter writer) { writer.Write((byte)Flags); writer.Write((byte)Command); - - if (Flags.HasFlag(MessageFlags.Checksum)) - { - writer.Write(CheckSum); - } - writer.WriteVarBytes(Payload); } @@ -74,22 +66,9 @@ void ISerializable.Deserialize(BinaryReader reader) Flags = (MessageFlags)reader.ReadByte(); Command = (MessageCommand)reader.ReadByte(); - if (Flags.HasFlag(MessageFlags.Checksum)) - { - CheckSum = reader.ReadInt16(); - - var length = (int)reader.ReadVarInt(int.MaxValue); - if (length > PayloadMaxSize) throw new FormatException(); - Payload = reader.ReadBytes(length); - - if (CheckSum != Payload.Checksum()) throw new FormatException(); - } - else - { - var length = (int)reader.ReadVarInt(int.MaxValue); - if (length > PayloadMaxSize) throw new FormatException(); - Payload = reader.ReadBytes(length); - } + var length = (int)reader.ReadVarInt(int.MaxValue); + if (length > PayloadMaxSize) throw new FormatException(); + Payload = reader.ReadBytes(length); } public static int TryDeserialize(ByteString data, out Message msg) @@ -97,24 +76,10 @@ public static int TryDeserialize(ByteString data, out Message msg) msg = null; if (data.Count < 5) return 0; - ulong length; - short checksum; - int payloadIndex; var header = data.Slice(0, 5).ToArray(); var flags = (MessageFlags)header[0]; - - if (flags.HasFlag(MessageFlags.Checksum)) - { - checksum = BitConverter.ToInt16(header, 2); - length = header[4]; - payloadIndex = 5; - } - else - { - checksum = 0; - length = header[2]; - payloadIndex = 3; - } + ulong length = header[2]; + var payloadIndex = 3; if (length == 0xFD) { @@ -143,8 +108,7 @@ public static int TryDeserialize(ByteString data, out Message msg) { Flags = flags, Command = (MessageCommand)header[1], - Payload = data.Slice(payloadIndex, (int)length).ToArray(), - CheckSum = checksum, + Payload = data.Slice(payloadIndex, (int)length).ToArray() }; return payloadIndex + (int)length; diff --git a/neo/Network/P2P/MessageFlags.cs b/neo/Network/P2P/MessageFlags.cs index b48d5efc43..37958d4b7b 100644 --- a/neo/Network/P2P/MessageFlags.cs +++ b/neo/Network/P2P/MessageFlags.cs @@ -6,7 +6,6 @@ namespace Neo.Network.P2P public enum MessageFlags : byte { None = 0, - Checksum = 1 << 0, - CompressedGzip = 1 << 1 + CompressedGzip = 1 << 0 } } \ No newline at end of file From eb1555446fe0808146435376abfe2213eb21e674 Mon Sep 17 00:00:00 2001 From: Shargon Date: Tue, 30 Apr 2019 12:21:10 +0200 Subject: [PATCH 27/45] Update Message.cs --- neo/Network/P2P/Message.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neo/Network/P2P/Message.cs b/neo/Network/P2P/Message.cs index 5e9dd2b0d0..f54c86989c 100644 --- a/neo/Network/P2P/Message.cs +++ b/neo/Network/P2P/Message.cs @@ -74,7 +74,7 @@ void ISerializable.Deserialize(BinaryReader reader) public static int TryDeserialize(ByteString data, out Message msg) { msg = null; - if (data.Count < 5) return 0; + if (data.Count < 3) return 0; var header = data.Slice(0, 5).ToArray(); var flags = (MessageFlags)header[0]; @@ -130,4 +130,4 @@ public Transaction GetTransaction() return (Transaction)_payload_deserialized; } } -} \ No newline at end of file +} From d08a059bc439a5c9a0f9847386e93c6b4254dd24 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Wed, 1 May 2019 14:16:41 +0800 Subject: [PATCH 28/45] Simplify `Message.Deserialize()` --- neo/Network/P2P/Message.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/neo/Network/P2P/Message.cs b/neo/Network/P2P/Message.cs index f54c86989c..83934a2e1a 100644 --- a/neo/Network/P2P/Message.cs +++ b/neo/Network/P2P/Message.cs @@ -65,10 +65,7 @@ void ISerializable.Deserialize(BinaryReader reader) { Flags = (MessageFlags)reader.ReadByte(); Command = (MessageCommand)reader.ReadByte(); - - var length = (int)reader.ReadVarInt(int.MaxValue); - if (length > PayloadMaxSize) throw new FormatException(); - Payload = reader.ReadBytes(length); + Payload = reader.ReadVarBytes(PayloadMaxSize); } public static int TryDeserialize(ByteString data, out Message msg) From f4de5fbc63399b4d59a85b3215fa4384fc10c2a7 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Wed, 1 May 2019 14:46:09 +0800 Subject: [PATCH 29/45] Add `InventoryType.ToMessageCommand()` --- neo/Network/P2P/Helper.cs | 20 ++++++++++++++++++-- neo/Network/P2P/RemoteNode.cs | 2 +- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/neo/Network/P2P/Helper.cs b/neo/Network/P2P/Helper.cs index be08a3ee5c..7daa13d1dd 100644 --- a/neo/Network/P2P/Helper.cs +++ b/neo/Network/P2P/Helper.cs @@ -1,6 +1,7 @@ -using System.IO; +using Neo.Network.P2P.Payloads; +using System; +using System.IO; using System.IO.Compression; -using Neo.Network.P2P.Payloads; namespace Neo.Network.P2P { @@ -48,5 +49,20 @@ public static byte[] GetHashData(this IVerifiable verifiable) return ms.ToArray(); } } + + internal static MessageCommand ToMessageCommand(this InventoryType inventoryType) + { + switch (inventoryType) + { + case InventoryType.TX: + return MessageCommand.Tx; + case InventoryType.Block: + return MessageCommand.Block; + case InventoryType.Consensus: + return MessageCommand.Consensus; + default: + throw new ArgumentOutOfRangeException(nameof(inventoryType)); + } + } } } diff --git a/neo/Network/P2P/RemoteNode.cs b/neo/Network/P2P/RemoteNode.cs index b04179aa87..7f93b5bba2 100644 --- a/neo/Network/P2P/RemoteNode.cs +++ b/neo/Network/P2P/RemoteNode.cs @@ -161,7 +161,7 @@ private void OnSend(IInventory inventory) if (bloom_filter != null && !bloom_filter.Test((Transaction)inventory)) return; } - EnqueueMessage((MessageCommand)inventory.InventoryType, inventory); + EnqueueMessage(inventory.InventoryType.ToMessageCommand(), inventory); } private void OnSetFilter(BloomFilter filter) From 1caa825571f027a5098f182d05f56a7910b25321 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Wed, 1 May 2019 15:04:09 +0800 Subject: [PATCH 30/45] Reorder the members of `MessageCommand`. --- neo/Network/P2P/Helper.cs | 2 +- neo/Network/P2P/MessageCommand.cs | 55 +++++++++++++++--------------- neo/Network/P2P/ProtocolHandler.cs | 8 ++--- 3 files changed, 33 insertions(+), 32 deletions(-) diff --git a/neo/Network/P2P/Helper.cs b/neo/Network/P2P/Helper.cs index 7daa13d1dd..fac7cfcf01 100644 --- a/neo/Network/P2P/Helper.cs +++ b/neo/Network/P2P/Helper.cs @@ -55,7 +55,7 @@ internal static MessageCommand ToMessageCommand(this InventoryType inventoryType switch (inventoryType) { case InventoryType.TX: - return MessageCommand.Tx; + return MessageCommand.Transaction; case InventoryType.Block: return MessageCommand.Block; case InventoryType.Consensus: diff --git a/neo/Network/P2P/MessageCommand.cs b/neo/Network/P2P/MessageCommand.cs index 3fee26db45..ce0527e136 100644 --- a/neo/Network/P2P/MessageCommand.cs +++ b/neo/Network/P2P/MessageCommand.cs @@ -2,35 +2,36 @@ { public enum MessageCommand : byte { - // Same value as InventoryType - Tx = 0x01, - // Same value as InventoryType - Block = 0x02, - Mempool = 0x03, - Addr = 0x04, - Inv = 0x05, - Headers = 0x06, - MerklebBock = 0x07, - Version = 0x08, - Verack = 0x09, - Alert = 0x0A, - Reject = 0x0B, + //handshaking + Version = 0x00, + Verack = 0x01, - Ping = 0x10, - Pong = 0x11, + //connectivity + GetAddr = 0x10, + Addr = 0x11, + Ping = 0x18, + Pong = 0x19, - GetAddr = 0x20, - GetBlocks = 0x21, - GetData = 0x22, - GetHeaders = 0x23, + //synchronization + GetHeaders = 0x20, + Headers = 0x21, + GetBlocks = 0x24, + Mempool = 0x25, + Inv = 0x27, + GetData = 0x28, + NotFound = 0x2a, + Transaction = 0x2b, + Block = 0x2c, + Consensus = 0x2d, + Reject = 0x2f, - FilterAdd = 0x30, - FilterClear = 0x31, - FilterLoad = 0x32, + //SPV protocol + FilterLoad = 0x30, + FilterAdd = 0x31, + FilterClear = 0x32, + MerkleBlock = 0x38, - // Same value as InventoryType - Consensus = 0xE0, - - NotFound = 0xFF, + //others + Alert = 0x40, } -} \ No newline at end of file +} diff --git a/neo/Network/P2P/ProtocolHandler.cs b/neo/Network/P2P/ProtocolHandler.cs index a181e5b681..708fcb35bd 100644 --- a/neo/Network/P2P/ProtocolHandler.cs +++ b/neo/Network/P2P/ProtocolHandler.cs @@ -101,7 +101,7 @@ protected override void OnReceive(object message) case MessageCommand.Pong: OnPongMessageReceived(msg.GetPayload()); break; - case MessageCommand.Tx: + case MessageCommand.Transaction: if (msg.Payload.Length <= Transaction.MaxTransactionSize) OnInventoryReceived(msg.GetTransaction()); break; @@ -109,7 +109,7 @@ protected override void OnReceive(object message) case MessageCommand.Version: throw new ProtocolViolationException(); case MessageCommand.Alert: - case MessageCommand.MerklebBock: + case MessageCommand.MerkleBlock: case MessageCommand.NotFound: case MessageCommand.Reject: default: break; @@ -188,7 +188,7 @@ private void OnGetDataMessageReceived(InvPayload payload) if (inventory == null) inventory = Blockchain.Singleton.GetTransaction(hash); if (inventory is Transaction) - Context.Parent.Tell(Message.Create(MessageCommand.Tx, inventory)); + Context.Parent.Tell(Message.Create(MessageCommand.Transaction, inventory)); break; case InventoryType.Block: if (inventory == null) @@ -202,7 +202,7 @@ private void OnGetDataMessageReceived(InvPayload payload) else { BitArray flags = new BitArray(block.Transactions.Select(p => bloom_filter.Test(p)).ToArray()); - Context.Parent.Tell(Message.Create(MessageCommand.MerklebBock, MerkleBlockPayload.Create(block, flags))); + Context.Parent.Tell(Message.Create(MessageCommand.MerkleBlock, MerkleBlockPayload.Create(block, flags))); } } break; From dcdabd846266f5cc6f67b0cdb4c6a64b49432c5f Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 1 May 2019 09:42:04 +0200 Subject: [PATCH 31/45] Lz4 --- neo.UnitTests/UT_P2PMessage.cs | 2 +- neo/Network/P2P/Helper.cs | 47 ++++++++++++++++++++++++++++----- neo/Network/P2P/Message.cs | 10 +++---- neo/Network/P2P/MessageFlags.cs | 2 +- neo/neo.csproj | 2 ++ 5 files changed, 50 insertions(+), 13 deletions(-) diff --git a/neo.UnitTests/UT_P2PMessage.cs b/neo.UnitTests/UT_P2PMessage.cs index 191f974e87..deedaf914f 100644 --- a/neo.UnitTests/UT_P2PMessage.cs +++ b/neo.UnitTests/UT_P2PMessage.cs @@ -69,7 +69,7 @@ public void Compression() var payloadCopy = copy.GetPayload(); copy.Command.Should().Be(msg.Command); - copy.Flags.Should().HaveFlag(MessageFlags.CompressedGzip); + copy.Flags.Should().HaveFlag(MessageFlags.Compressed); payloadCopy.UserAgent.Should().Be(payload.UserAgent); payloadCopy.Nonce.Should().Be(payload.Nonce); diff --git a/neo/Network/P2P/Helper.cs b/neo/Network/P2P/Helper.cs index be08a3ee5c..74dad34f70 100644 --- a/neo/Network/P2P/Helper.cs +++ b/neo/Network/P2P/Helper.cs @@ -1,21 +1,45 @@ using System.IO; using System.IO.Compression; +using K4os.Compression.LZ4; +using K4os.Compression.LZ4.Streams; using Neo.Network.P2P.Payloads; namespace Neo.Network.P2P { public static class Helper { - public static byte[] UncompressGzip(this byte[] data) + private static readonly LZ4EncoderSettings CompressSettings = new LZ4EncoderSettings() { CompressionLevel = LZ4Level.L04_HC }; + + private static readonly LZ4DecoderSettings DecompressSettings = new LZ4DecoderSettings() { }; + + public static byte[] DecompressGzip(this byte[] data) + { + using (var output = new MemoryStream()) + using (var input = new MemoryStream(data)) + using (var decoder = new GZipStream(input, CompressionMode.Decompress)) + { + int nRead; + byte[] buffer = new byte[1024]; + + while ((nRead = decoder.Read(buffer, 0, buffer.Length)) > 0) + { + output.Write(buffer, 0, nRead); + } + + return output.ToArray(); + } + } + + public static byte[] DecompressLz4(this byte[] data) { using (var output = new MemoryStream()) using (var input = new MemoryStream(data)) - using (var gzip = new GZipStream(input, CompressionMode.Decompress)) + using (var decoder = LZ4Stream.Decode(input, DecompressSettings, true)) { int nRead; byte[] buffer = new byte[1024]; - while ((nRead = gzip.Read(buffer, 0, buffer.Length)) > 0) + while ((nRead = decoder.Read(buffer, 0, buffer.Length)) > 0) { output.Write(buffer, 0, nRead); } @@ -24,14 +48,25 @@ public static byte[] UncompressGzip(this byte[] data) } } + public static byte[] CompressLz4(this byte[] data) + { + using (var stream = new MemoryStream()) + using (var encoder = LZ4Stream.Encode(stream, CompressSettings, true)) + { + encoder.Write(data, 0, data.Length); + encoder.Flush(); + return stream.ToArray(); + } + } + public static byte[] CompressGzip(this byte[] data) { using (var stream = new MemoryStream()) { - using (var gzip = new GZipStream(stream, CompressionLevel.Optimal, true)) + using (var encoder = new GZipStream(stream, CompressionLevel.Optimal, true)) { - gzip.Write(data, 0, data.Length); - gzip.Flush(); + encoder.Write(data, 0, data.Length); + encoder.Flush(); } return stream.ToArray(); diff --git a/neo/Network/P2P/Message.cs b/neo/Network/P2P/Message.cs index f54c86989c..80fa48cd01 100644 --- a/neo/Network/P2P/Message.cs +++ b/neo/Network/P2P/Message.cs @@ -10,8 +10,8 @@ namespace Neo.Network.P2P public class Message : ISerializable { public const int PayloadMaxSize = 0x02000000; - public const int CompressionMinSize = 180; - public const int CompressionThreshold = 100; + public const int CompressionMinSize = 128; + public const int CompressionThreshold = 64; public MessageFlags Flags; public MessageCommand Command; @@ -37,12 +37,12 @@ public static Message Create(MessageCommand command, byte[] payload) if (payload.Length > CompressionMinSize) { - var compressed = payload.CompressGzip(); + var compressed = payload.CompressLz4(); if (compressed.Length < payload.Length - CompressionThreshold) { payload = compressed; - flags |= MessageFlags.CompressedGzip; + flags |= MessageFlags.Compressed; } } @@ -114,7 +114,7 @@ public static int TryDeserialize(ByteString data, out Message msg) return payloadIndex + (int)length; } - public byte[] GetPayload() => Flags.HasFlag(MessageFlags.CompressedGzip) ? Payload.UncompressGzip() : Payload; + public byte[] GetPayload() => Flags.HasFlag(MessageFlags.Compressed) ? Payload.DecompressLz4() : Payload; public T GetPayload() where T : ISerializable, new() { diff --git a/neo/Network/P2P/MessageFlags.cs b/neo/Network/P2P/MessageFlags.cs index 37958d4b7b..a608f5bfef 100644 --- a/neo/Network/P2P/MessageFlags.cs +++ b/neo/Network/P2P/MessageFlags.cs @@ -6,6 +6,6 @@ namespace Neo.Network.P2P public enum MessageFlags : byte { None = 0, - CompressedGzip = 1 << 0 + Compressed = 1 << 0 } } \ No newline at end of file diff --git a/neo/neo.csproj b/neo/neo.csproj index db7ed335f1..a731d352c1 100644 --- a/neo/neo.csproj +++ b/neo/neo.csproj @@ -21,6 +21,8 @@ + + From 6df5c67a845659ab399efaef46b9eb3f428925cd Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 1 May 2019 10:04:46 +0200 Subject: [PATCH 32/45] Fix lz4 --- neo/Network/P2P/Helper.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/neo/Network/P2P/Helper.cs b/neo/Network/P2P/Helper.cs index c497c23c2e..024a43ed7a 100644 --- a/neo/Network/P2P/Helper.cs +++ b/neo/Network/P2P/Helper.cs @@ -53,10 +53,12 @@ public static byte[] DecompressLz4(this byte[] data) public static byte[] CompressLz4(this byte[] data) { using (var stream = new MemoryStream()) - using (var encoder = LZ4Stream.Encode(stream, CompressSettings, true)) { - encoder.Write(data, 0, data.Length); - encoder.Flush(); + using (var encoder = LZ4Stream.Encode(stream, CompressSettings, true)) + { + encoder.Write(data, 0, data.Length); + } + return stream.ToArray(); } } @@ -68,7 +70,6 @@ public static byte[] CompressGzip(this byte[] data) using (var encoder = new GZipStream(stream, CompressionLevel.Optimal, true)) { encoder.Write(data, 0, data.Length); - encoder.Flush(); } return stream.ToArray(); From 985a5ea296fd91a31f9ff1f526dd561efe492e8a Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 1 May 2019 10:13:23 +0200 Subject: [PATCH 33/45] Clean --- neo/Network/P2P/Helper.cs | 33 +-------------------------------- 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/neo/Network/P2P/Helper.cs b/neo/Network/P2P/Helper.cs index 024a43ed7a..e9ff11afc6 100644 --- a/neo/Network/P2P/Helper.cs +++ b/neo/Network/P2P/Helper.cs @@ -10,28 +10,10 @@ namespace Neo.Network.P2P { public static class Helper { - private static readonly LZ4EncoderSettings CompressSettings = new LZ4EncoderSettings() { CompressionLevel = LZ4Level.L04_HC }; + private static readonly LZ4EncoderSettings CompressSettings = new LZ4EncoderSettings() { CompressionLevel = LZ4Level.L00_FAST }; private static readonly LZ4DecoderSettings DecompressSettings = new LZ4DecoderSettings() { }; - public static byte[] DecompressGzip(this byte[] data) - { - using (var output = new MemoryStream()) - using (var input = new MemoryStream(data)) - using (var decoder = new GZipStream(input, CompressionMode.Decompress)) - { - int nRead; - byte[] buffer = new byte[1024]; - - while ((nRead = decoder.Read(buffer, 0, buffer.Length)) > 0) - { - output.Write(buffer, 0, nRead); - } - - return output.ToArray(); - } - } - public static byte[] DecompressLz4(this byte[] data) { using (var output = new MemoryStream()) @@ -63,19 +45,6 @@ public static byte[] CompressLz4(this byte[] data) } } - public static byte[] CompressGzip(this byte[] data) - { - using (var stream = new MemoryStream()) - { - using (var encoder = new GZipStream(stream, CompressionLevel.Optimal, true)) - { - encoder.Write(data, 0, data.Length); - } - - return stream.ToArray(); - } - } - public static byte[] GetHashData(this IVerifiable verifiable) { using (MemoryStream ms = new MemoryStream()) From d072a96e40400a03dabfef54485424263b06dbf6 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 1 May 2019 10:39:22 +0200 Subject: [PATCH 34/45] Clean usings --- neo/Network/P2P/Helper.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/neo/Network/P2P/Helper.cs b/neo/Network/P2P/Helper.cs index e9ff11afc6..e2eac08c5a 100644 --- a/neo/Network/P2P/Helper.cs +++ b/neo/Network/P2P/Helper.cs @@ -1,7 +1,5 @@ -using Neo.Network.P2P.Payloads; -using System; +using System; using System.IO; -using System.IO.Compression; using K4os.Compression.LZ4; using K4os.Compression.LZ4.Streams; using Neo.Network.P2P.Payloads; From 222eb2ee1e7a001c2607191defaaeef7e5a91308 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Wed, 1 May 2019 17:43:04 +0800 Subject: [PATCH 35/45] Simplify Lz4 compression/decompression --- neo/Network/P2P/Helper.cs | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/neo/Network/P2P/Helper.cs b/neo/Network/P2P/Helper.cs index e2eac08c5a..7ce613ed8b 100644 --- a/neo/Network/P2P/Helper.cs +++ b/neo/Network/P2P/Helper.cs @@ -1,31 +1,19 @@ -using System; -using System.IO; -using K4os.Compression.LZ4; -using K4os.Compression.LZ4.Streams; +using K4os.Compression.LZ4.Streams; using Neo.Network.P2P.Payloads; +using System; +using System.IO; namespace Neo.Network.P2P { public static class Helper { - private static readonly LZ4EncoderSettings CompressSettings = new LZ4EncoderSettings() { CompressionLevel = LZ4Level.L00_FAST }; - - private static readonly LZ4DecoderSettings DecompressSettings = new LZ4DecoderSettings() { }; - public static byte[] DecompressLz4(this byte[] data) { + using (var input = new MemoryStream(data, false)) + using (var decoder = LZ4Stream.Decode(input, leaveOpen: true)) using (var output = new MemoryStream()) - using (var input = new MemoryStream(data)) - using (var decoder = LZ4Stream.Decode(input, DecompressSettings, true)) { - int nRead; - byte[] buffer = new byte[1024]; - - while ((nRead = decoder.Read(buffer, 0, buffer.Length)) > 0) - { - output.Write(buffer, 0, nRead); - } - + decoder.CopyTo(output); return output.ToArray(); } } @@ -34,7 +22,7 @@ public static byte[] CompressLz4(this byte[] data) { using (var stream = new MemoryStream()) { - using (var encoder = LZ4Stream.Encode(stream, CompressSettings, true)) + using (var encoder = LZ4Stream.Encode(stream, leaveOpen: true)) { encoder.Write(data, 0, data.Length); } From 3fdbf2e15ea2eb091b330bc8ac43de0ab8ee78a3 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Wed, 1 May 2019 18:40:29 +0800 Subject: [PATCH 36/45] Deserialize the payloads when deserializing `Message`s --- neo.UnitTests/UT_P2PMessage.cs | 6 +- neo/Network/P2P/Helper.cs | 13 +++- neo/Network/P2P/Message.cs | 115 ++++++++++++++++++----------- neo/Network/P2P/ProtocolHandler.cs | 30 ++++---- 4 files changed, 100 insertions(+), 64 deletions(-) diff --git a/neo.UnitTests/UT_P2PMessage.cs b/neo.UnitTests/UT_P2PMessage.cs index deedaf914f..407e3692e7 100644 --- a/neo.UnitTests/UT_P2PMessage.cs +++ b/neo.UnitTests/UT_P2PMessage.cs @@ -17,7 +17,7 @@ public void Serialize_Deserialize() var msg = Message.Create(MessageCommand.Ping, payload); var buffer = msg.ToArray(); var copy = buffer.AsSerializable(); - var payloadCopy = copy.GetPayload(); + var payloadCopy = (PingPayload)copy.Payload; copy.Command.Should().Be(msg.Command); copy.Flags.Should().Be(msg.Flags); @@ -35,7 +35,7 @@ public void Serialize_Deserialize_ByteString() var buffer = ByteString.CopyFrom(msg.ToArray()); var length = Message.TryDeserialize(buffer, out var copy); - var payloadCopy = copy.GetPayload(); + var payloadCopy = (PingPayload)copy.Payload; copy.Command.Should().Be(msg.Command); copy.Flags.Should().Be(msg.Flags); @@ -66,7 +66,7 @@ public void Compression() buffer.Length.Should().BeLessThan(80); var copy = buffer.AsSerializable(); - var payloadCopy = copy.GetPayload(); + var payloadCopy = (VersionPayload)copy.Payload; copy.Command.Should().Be(msg.Command); copy.Flags.Should().HaveFlag(MessageFlags.Compressed); diff --git a/neo/Network/P2P/Helper.cs b/neo/Network/P2P/Helper.cs index 7ce613ed8b..3eded3f84c 100644 --- a/neo/Network/P2P/Helper.cs +++ b/neo/Network/P2P/Helper.cs @@ -7,13 +7,22 @@ namespace Neo.Network.P2P { public static class Helper { - public static byte[] DecompressLz4(this byte[] data) + public static byte[] DecompressLz4(this byte[] data, int maxOutput = int.MaxValue) { using (var input = new MemoryStream(data, false)) using (var decoder = LZ4Stream.Decode(input, leaveOpen: true)) using (var output = new MemoryStream()) { - decoder.CopyTo(output); + int nRead; + byte[] buffer = new byte[1024]; + + while ((nRead = decoder.Read(buffer, 0, buffer.Length)) > 0) + { + maxOutput -= nRead; + if (maxOutput < 0) throw new FormatException(); + output.Write(buffer, 0, nRead); + } + return output.ToArray(); } } diff --git a/neo/Network/P2P/Message.cs b/neo/Network/P2P/Message.cs index 7bed6b43ab..f21160f3d1 100644 --- a/neo/Network/P2P/Message.cs +++ b/neo/Network/P2P/Message.cs @@ -1,9 +1,9 @@ -using System; -using System.IO; -using Akka.IO; +using Akka.IO; using Neo.Cryptography; using Neo.IO; using Neo.Network.P2P.Payloads; +using System; +using System.IO; namespace Neo.Network.P2P { @@ -15,57 +15,49 @@ public class Message : ISerializable public MessageFlags Flags; public MessageCommand Command; - public byte[] Payload; + public ISerializable Payload; - private ISerializable _payload_deserialized = null; + private byte[] _payload_compressed; - public int Size => sizeof(MessageFlags) + sizeof(MessageCommand) + Payload.GetVarSize(); + public int Size => sizeof(MessageFlags) + sizeof(MessageCommand) + _payload_compressed.GetVarSize(); public static Message Create(MessageCommand command, ISerializable payload = null) { - var ret = Create(command, payload == null ? new byte[0] : payload.ToArray()); - ret._payload_deserialized = payload; - - return ret; - } - - public static Message Create(MessageCommand command, byte[] payload) - { - var flags = MessageFlags.None; + Message message = new Message + { + Flags = MessageFlags.None, + Command = command, + Payload = payload, + _payload_compressed = payload?.ToArray() ?? new byte[0] + }; // Try compression - - if (payload.Length > CompressionMinSize) + if (message._payload_compressed.Length > CompressionMinSize) { - var compressed = payload.CompressLz4(); - - if (compressed.Length < payload.Length - CompressionThreshold) + var compressed = message._payload_compressed.CompressLz4(); + if (compressed.Length < message._payload_compressed.Length - CompressionThreshold) { - payload = compressed; - flags |= MessageFlags.Compressed; + message._payload_compressed = compressed; + message.Flags |= MessageFlags.Compressed; } } - return new Message - { - Flags = flags, - Command = command, - Payload = payload - }; + return message; } void ISerializable.Serialize(BinaryWriter writer) { writer.Write((byte)Flags); writer.Write((byte)Command); - writer.WriteVarBytes(Payload); + writer.WriteVarBytes(_payload_compressed); } void ISerializable.Deserialize(BinaryReader reader) { Flags = (MessageFlags)reader.ReadByte(); Command = (MessageCommand)reader.ReadByte(); - Payload = reader.ReadVarBytes(PayloadMaxSize); + _payload_compressed = reader.ReadVarBytes(PayloadMaxSize); + SetPayload(); } public static int TryDeserialize(ByteString data, out Message msg) @@ -105,26 +97,61 @@ public static int TryDeserialize(ByteString data, out Message msg) { Flags = flags, Command = (MessageCommand)header[1], - Payload = data.Slice(payloadIndex, (int)length).ToArray() + _payload_compressed = data.Slice(payloadIndex, (int)length).ToArray() }; + msg.SetPayload(); return payloadIndex + (int)length; } - public byte[] GetPayload() => Flags.HasFlag(MessageFlags.Compressed) ? Payload.DecompressLz4() : Payload; - - public T GetPayload() where T : ISerializable, new() - { - if (_payload_deserialized is null) - _payload_deserialized = GetPayload().AsSerializable(); - return (T)_payload_deserialized; - } - - public Transaction GetTransaction() + private void SetPayload() { - if (_payload_deserialized is null) - _payload_deserialized = Transaction.DeserializeFrom(GetPayload()); - return (Transaction)_payload_deserialized; + if (_payload_compressed.Length == 0) return; + byte[] decompressed = Flags.HasFlag(MessageFlags.Compressed) + ? _payload_compressed.DecompressLz4(PayloadMaxSize) + : _payload_compressed; + switch (Command) + { + case MessageCommand.Version: + Payload = decompressed.AsSerializable(); + break; + case MessageCommand.Addr: + Payload = decompressed.AsSerializable(); + break; + case MessageCommand.Ping: + case MessageCommand.Pong: + Payload = decompressed.AsSerializable(); + break; + case MessageCommand.GetHeaders: + case MessageCommand.GetBlocks: + Payload = decompressed.AsSerializable(); + break; + case MessageCommand.Headers: + Payload = decompressed.AsSerializable(); + break; + case MessageCommand.Inv: + case MessageCommand.GetData: + Payload = decompressed.AsSerializable(); + break; + case MessageCommand.Transaction: + Payload = Transaction.DeserializeFrom(decompressed); + break; + case MessageCommand.Block: + Payload = decompressed.AsSerializable(); + break; + case MessageCommand.Consensus: + Payload = decompressed.AsSerializable(); + break; + case MessageCommand.FilterLoad: + Payload = decompressed.AsSerializable(); + break; + case MessageCommand.FilterAdd: + Payload = decompressed.AsSerializable(); + break; + case MessageCommand.MerkleBlock: + Payload = decompressed.AsSerializable(); + break; + } } } } diff --git a/neo/Network/P2P/ProtocolHandler.cs b/neo/Network/P2P/ProtocolHandler.cs index 708fcb35bd..6bf0b05f10 100644 --- a/neo/Network/P2P/ProtocolHandler.cs +++ b/neo/Network/P2P/ProtocolHandler.cs @@ -44,7 +44,7 @@ protected override void OnReceive(object message) { if (msg.Command != MessageCommand.Version) throw new ProtocolViolationException(); - OnVersionMessageReceived(msg.GetPayload()); + OnVersionMessageReceived((VersionPayload)msg.Payload); return; } if (!verack) @@ -57,53 +57,53 @@ protected override void OnReceive(object message) switch (msg.Command) { case MessageCommand.Addr: - OnAddrMessageReceived(msg.GetPayload()); + OnAddrMessageReceived((AddrPayload)msg.Payload); break; case MessageCommand.Block: - OnInventoryReceived(msg.GetPayload()); + OnInventoryReceived((Block)msg.Payload); break; case MessageCommand.Consensus: - OnInventoryReceived(msg.GetPayload()); + OnInventoryReceived((ConsensusPayload)msg.Payload); break; case MessageCommand.FilterAdd: - OnFilterAddMessageReceived(msg.GetPayload()); + OnFilterAddMessageReceived((FilterAddPayload)msg.Payload); break; case MessageCommand.FilterClear: OnFilterClearMessageReceived(); break; case MessageCommand.FilterLoad: - OnFilterLoadMessageReceived(msg.GetPayload()); + OnFilterLoadMessageReceived((FilterLoadPayload)msg.Payload); break; case MessageCommand.GetAddr: OnGetAddrMessageReceived(); break; case MessageCommand.GetBlocks: - OnGetBlocksMessageReceived(msg.GetPayload()); + OnGetBlocksMessageReceived((GetBlocksPayload)msg.Payload); break; case MessageCommand.GetData: - OnGetDataMessageReceived(msg.GetPayload()); + OnGetDataMessageReceived((InvPayload)msg.Payload); break; case MessageCommand.GetHeaders: - OnGetHeadersMessageReceived(msg.GetPayload()); + OnGetHeadersMessageReceived((GetBlocksPayload)msg.Payload); break; case MessageCommand.Headers: - OnHeadersMessageReceived(msg.GetPayload()); + OnHeadersMessageReceived((HeadersPayload)msg.Payload); break; case MessageCommand.Inv: - OnInvMessageReceived(msg.GetPayload()); + OnInvMessageReceived((InvPayload)msg.Payload); break; case MessageCommand.Mempool: OnMemPoolMessageReceived(); break; case MessageCommand.Ping: - OnPingMessageReceived(msg.GetPayload()); + OnPingMessageReceived((PingPayload)msg.Payload); break; case MessageCommand.Pong: - OnPongMessageReceived(msg.GetPayload()); + OnPongMessageReceived((PingPayload)msg.Payload); break; case MessageCommand.Transaction: - if (msg.Payload.Length <= Transaction.MaxTransactionSize) - OnInventoryReceived(msg.GetTransaction()); + if (msg.Payload.Size <= Transaction.MaxTransactionSize) + OnInventoryReceived((Transaction)msg.Payload); break; case MessageCommand.Verack: case MessageCommand.Version: From f22fa21bfce22e4396ee00b84b099ead848a448b Mon Sep 17 00:00:00 2001 From: Shargon Date: Thu, 2 May 2019 12:56:40 +0200 Subject: [PATCH 37/45] Speed up relay process --- neo/Network/P2P/Message.cs | 40 ++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/neo/Network/P2P/Message.cs b/neo/Network/P2P/Message.cs index f21160f3d1..8c60ead481 100644 --- a/neo/Network/P2P/Message.cs +++ b/neo/Network/P2P/Message.cs @@ -1,4 +1,4 @@ -using Akka.IO; +using Akka.IO; using Neo.Cryptography; using Neo.IO; using Neo.Network.P2P.Payloads; @@ -15,9 +15,17 @@ public class Message : ISerializable public MessageFlags Flags; public MessageCommand Command; - public ISerializable Payload; + public ISerializable Payload + { + get + { + if (_payload == null) SetPayload(); + return _payload; + } + } private byte[] _payload_compressed; + private ISerializable _payload; public int Size => sizeof(MessageFlags) + sizeof(MessageCommand) + _payload_compressed.GetVarSize(); @@ -27,7 +35,7 @@ public static Message Create(MessageCommand command, ISerializable payload = nul { Flags = MessageFlags.None, Command = command, - Payload = payload, + _payload = payload, _payload_compressed = payload?.ToArray() ?? new byte[0] }; @@ -57,7 +65,6 @@ void ISerializable.Deserialize(BinaryReader reader) Flags = (MessageFlags)reader.ReadByte(); Command = (MessageCommand)reader.ReadByte(); _payload_compressed = reader.ReadVarBytes(PayloadMaxSize); - SetPayload(); } public static int TryDeserialize(ByteString data, out Message msg) @@ -99,7 +106,6 @@ public static int TryDeserialize(ByteString data, out Message msg) Command = (MessageCommand)header[1], _payload_compressed = data.Slice(payloadIndex, (int)length).ToArray() }; - msg.SetPayload(); return payloadIndex + (int)length; } @@ -113,43 +119,43 @@ private void SetPayload() switch (Command) { case MessageCommand.Version: - Payload = decompressed.AsSerializable(); + _payload = decompressed.AsSerializable(); break; case MessageCommand.Addr: - Payload = decompressed.AsSerializable(); + _payload = decompressed.AsSerializable(); break; case MessageCommand.Ping: case MessageCommand.Pong: - Payload = decompressed.AsSerializable(); + _payload = decompressed.AsSerializable(); break; case MessageCommand.GetHeaders: case MessageCommand.GetBlocks: - Payload = decompressed.AsSerializable(); + _payload = decompressed.AsSerializable(); break; case MessageCommand.Headers: - Payload = decompressed.AsSerializable(); + _payload = decompressed.AsSerializable(); break; case MessageCommand.Inv: case MessageCommand.GetData: - Payload = decompressed.AsSerializable(); + _payload = decompressed.AsSerializable(); break; case MessageCommand.Transaction: - Payload = Transaction.DeserializeFrom(decompressed); + _payload = Transaction.DeserializeFrom(decompressed); break; case MessageCommand.Block: - Payload = decompressed.AsSerializable(); + _payload = decompressed.AsSerializable(); break; case MessageCommand.Consensus: - Payload = decompressed.AsSerializable(); + _payload = decompressed.AsSerializable(); break; case MessageCommand.FilterLoad: - Payload = decompressed.AsSerializable(); + _payload = decompressed.AsSerializable(); break; case MessageCommand.FilterAdd: - Payload = decompressed.AsSerializable(); + _payload = decompressed.AsSerializable(); break; case MessageCommand.MerkleBlock: - Payload = decompressed.AsSerializable(); + _payload = decompressed.AsSerializable(); break; } } From 23044da849580e4b8e4c98924aea027354f33bd3 Mon Sep 17 00:00:00 2001 From: Shargon Date: Thu, 2 May 2019 12:56:40 +0200 Subject: [PATCH 38/45] Revert "Speed up relay process" This reverts commit f22fa21bfce22e4396ee00b84b099ead848a448b. --- neo/Network/P2P/Message.cs | 40 ++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/neo/Network/P2P/Message.cs b/neo/Network/P2P/Message.cs index 8c60ead481..f21160f3d1 100644 --- a/neo/Network/P2P/Message.cs +++ b/neo/Network/P2P/Message.cs @@ -1,4 +1,4 @@ -using Akka.IO; +using Akka.IO; using Neo.Cryptography; using Neo.IO; using Neo.Network.P2P.Payloads; @@ -15,17 +15,9 @@ public class Message : ISerializable public MessageFlags Flags; public MessageCommand Command; - public ISerializable Payload - { - get - { - if (_payload == null) SetPayload(); - return _payload; - } - } + public ISerializable Payload; private byte[] _payload_compressed; - private ISerializable _payload; public int Size => sizeof(MessageFlags) + sizeof(MessageCommand) + _payload_compressed.GetVarSize(); @@ -35,7 +27,7 @@ public static Message Create(MessageCommand command, ISerializable payload = nul { Flags = MessageFlags.None, Command = command, - _payload = payload, + Payload = payload, _payload_compressed = payload?.ToArray() ?? new byte[0] }; @@ -65,6 +57,7 @@ void ISerializable.Deserialize(BinaryReader reader) Flags = (MessageFlags)reader.ReadByte(); Command = (MessageCommand)reader.ReadByte(); _payload_compressed = reader.ReadVarBytes(PayloadMaxSize); + SetPayload(); } public static int TryDeserialize(ByteString data, out Message msg) @@ -106,6 +99,7 @@ public static int TryDeserialize(ByteString data, out Message msg) Command = (MessageCommand)header[1], _payload_compressed = data.Slice(payloadIndex, (int)length).ToArray() }; + msg.SetPayload(); return payloadIndex + (int)length; } @@ -119,43 +113,43 @@ private void SetPayload() switch (Command) { case MessageCommand.Version: - _payload = decompressed.AsSerializable(); + Payload = decompressed.AsSerializable(); break; case MessageCommand.Addr: - _payload = decompressed.AsSerializable(); + Payload = decompressed.AsSerializable(); break; case MessageCommand.Ping: case MessageCommand.Pong: - _payload = decompressed.AsSerializable(); + Payload = decompressed.AsSerializable(); break; case MessageCommand.GetHeaders: case MessageCommand.GetBlocks: - _payload = decompressed.AsSerializable(); + Payload = decompressed.AsSerializable(); break; case MessageCommand.Headers: - _payload = decompressed.AsSerializable(); + Payload = decompressed.AsSerializable(); break; case MessageCommand.Inv: case MessageCommand.GetData: - _payload = decompressed.AsSerializable(); + Payload = decompressed.AsSerializable(); break; case MessageCommand.Transaction: - _payload = Transaction.DeserializeFrom(decompressed); + Payload = Transaction.DeserializeFrom(decompressed); break; case MessageCommand.Block: - _payload = decompressed.AsSerializable(); + Payload = decompressed.AsSerializable(); break; case MessageCommand.Consensus: - _payload = decompressed.AsSerializable(); + Payload = decompressed.AsSerializable(); break; case MessageCommand.FilterLoad: - _payload = decompressed.AsSerializable(); + Payload = decompressed.AsSerializable(); break; case MessageCommand.FilterAdd: - _payload = decompressed.AsSerializable(); + Payload = decompressed.AsSerializable(); break; case MessageCommand.MerkleBlock: - _payload = decompressed.AsSerializable(); + Payload = decompressed.AsSerializable(); break; } } From 2c06bdcc59514a402afd96a5df70ab9a531cfcf9 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Thu, 2 May 2019 23:43:38 +0800 Subject: [PATCH 39/45] Use blocks compression --- neo/Network/P2P/Helper.cs | 47 ++++++++++++++++++--------------------- neo/neo.csproj | 1 - 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/neo/Network/P2P/Helper.cs b/neo/Network/P2P/Helper.cs index 3eded3f84c..0ef8965d56 100644 --- a/neo/Network/P2P/Helper.cs +++ b/neo/Network/P2P/Helper.cs @@ -1,43 +1,40 @@ -using K4os.Compression.LZ4.Streams; +using K4os.Compression.LZ4; using Neo.Network.P2P.Payloads; using System; +using System.Buffers; using System.IO; namespace Neo.Network.P2P { public static class Helper { - public static byte[] DecompressLz4(this byte[] data, int maxOutput = int.MaxValue) + public static byte[] DecompressLz4(this byte[] data, int maxOutput) { - using (var input = new MemoryStream(data, false)) - using (var decoder = LZ4Stream.Decode(input, leaveOpen: true)) - using (var output = new MemoryStream()) + maxOutput = Math.Min(maxOutput, data.Length * 255); + byte[] buffer = ArrayPool.Shared.Rent(maxOutput); + try { - int nRead; - byte[] buffer = new byte[1024]; - - while ((nRead = decoder.Read(buffer, 0, buffer.Length)) > 0) - { - maxOutput -= nRead; - if (maxOutput < 0) throw new FormatException(); - output.Write(buffer, 0, nRead); - } - - return output.ToArray(); + int length = LZ4Codec.Decode(data, 0, data.Length, buffer, 0, buffer.Length); + if (length < 0 || length > maxOutput) throw new FormatException(); + data = new byte[length]; + Buffer.BlockCopy(buffer, 0, data, 0, length); + return data; + } + finally + { + ArrayPool.Shared.Return(buffer); } } public static byte[] CompressLz4(this byte[] data) { - using (var stream = new MemoryStream()) - { - using (var encoder = LZ4Stream.Encode(stream, leaveOpen: true)) - { - encoder.Write(data, 0, data.Length); - } - - return stream.ToArray(); - } + int maxLength = LZ4Codec.MaximumOutputSize(data.Length); + byte[] buffer = ArrayPool.Shared.Rent(maxLength); + int length = LZ4Codec.Encode(data, 0, data.Length, buffer, 0, buffer.Length); + data = new byte[length]; + Buffer.BlockCopy(buffer, 0, data, 0, length); + ArrayPool.Shared.Return(buffer); + return data; } public static byte[] GetHashData(this IVerifiable verifiable) diff --git a/neo/neo.csproj b/neo/neo.csproj index a731d352c1..54b508d8ec 100644 --- a/neo/neo.csproj +++ b/neo/neo.csproj @@ -22,7 +22,6 @@ - From b46ff8acdec3d8bdbaf048ca090486ee90b816ef Mon Sep 17 00:00:00 2001 From: erikzhang Date: Fri, 3 May 2019 19:10:27 +0800 Subject: [PATCH 40/45] format --- neo/Network/P2P/Helper.cs | 22 ++-- neo/Network/P2P/Message.cs | 118 +++++++++--------- neo/Network/P2P/MessageFlags.cs | 2 +- .../P2P/Payloads/NetworkAddressWithTime.cs | 4 +- neo/Network/P2P/Payloads/VersionPayload.cs | 16 ++- neo/Network/P2P/RemoteNode.cs | 8 +- 6 files changed, 89 insertions(+), 81 deletions(-) diff --git a/neo/Network/P2P/Helper.cs b/neo/Network/P2P/Helper.cs index 0ef8965d56..433725c45b 100644 --- a/neo/Network/P2P/Helper.cs +++ b/neo/Network/P2P/Helper.cs @@ -8,6 +8,17 @@ namespace Neo.Network.P2P { public static class Helper { + public static byte[] CompressLz4(this byte[] data) + { + int maxLength = LZ4Codec.MaximumOutputSize(data.Length); + byte[] buffer = ArrayPool.Shared.Rent(maxLength); + int length = LZ4Codec.Encode(data, 0, data.Length, buffer, 0, buffer.Length); + data = new byte[length]; + Buffer.BlockCopy(buffer, 0, data, 0, length); + ArrayPool.Shared.Return(buffer); + return data; + } + public static byte[] DecompressLz4(this byte[] data, int maxOutput) { maxOutput = Math.Min(maxOutput, data.Length * 255); @@ -26,17 +37,6 @@ public static byte[] DecompressLz4(this byte[] data, int maxOutput) } } - public static byte[] CompressLz4(this byte[] data) - { - int maxLength = LZ4Codec.MaximumOutputSize(data.Length); - byte[] buffer = ArrayPool.Shared.Rent(maxLength); - int length = LZ4Codec.Encode(data, 0, data.Length, buffer, 0, buffer.Length); - data = new byte[length]; - Buffer.BlockCopy(buffer, 0, data, 0, length); - ArrayPool.Shared.Return(buffer); - return data; - } - public static byte[] GetHashData(this IVerifiable verifiable) { using (MemoryStream ms = new MemoryStream()) diff --git a/neo/Network/P2P/Message.cs b/neo/Network/P2P/Message.cs index f21160f3d1..aae81f3c78 100644 --- a/neo/Network/P2P/Message.cs +++ b/neo/Network/P2P/Message.cs @@ -10,8 +10,8 @@ namespace Neo.Network.P2P public class Message : ISerializable { public const int PayloadMaxSize = 0x02000000; - public const int CompressionMinSize = 128; - public const int CompressionThreshold = 64; + private const int CompressionMinSize = 128; + private const int CompressionThreshold = 64; public MessageFlags Flags; public MessageCommand Command; @@ -45,11 +45,54 @@ public static Message Create(MessageCommand command, ISerializable payload = nul return message; } - void ISerializable.Serialize(BinaryWriter writer) + private void DecompressPayload() { - writer.Write((byte)Flags); - writer.Write((byte)Command); - writer.WriteVarBytes(_payload_compressed); + if (_payload_compressed.Length == 0) return; + byte[] decompressed = Flags.HasFlag(MessageFlags.Compressed) + ? _payload_compressed.DecompressLz4(PayloadMaxSize) + : _payload_compressed; + switch (Command) + { + case MessageCommand.Version: + Payload = decompressed.AsSerializable(); + break; + case MessageCommand.Addr: + Payload = decompressed.AsSerializable(); + break; + case MessageCommand.Ping: + case MessageCommand.Pong: + Payload = decompressed.AsSerializable(); + break; + case MessageCommand.GetHeaders: + case MessageCommand.GetBlocks: + Payload = decompressed.AsSerializable(); + break; + case MessageCommand.Headers: + Payload = decompressed.AsSerializable(); + break; + case MessageCommand.Inv: + case MessageCommand.GetData: + Payload = decompressed.AsSerializable(); + break; + case MessageCommand.Transaction: + Payload = Transaction.DeserializeFrom(decompressed); + break; + case MessageCommand.Block: + Payload = decompressed.AsSerializable(); + break; + case MessageCommand.Consensus: + Payload = decompressed.AsSerializable(); + break; + case MessageCommand.FilterLoad: + Payload = decompressed.AsSerializable(); + break; + case MessageCommand.FilterAdd: + Payload = decompressed.AsSerializable(); + break; + case MessageCommand.MerkleBlock: + Payload = decompressed.AsSerializable(); + break; + } } void ISerializable.Deserialize(BinaryReader reader) @@ -57,10 +100,17 @@ void ISerializable.Deserialize(BinaryReader reader) Flags = (MessageFlags)reader.ReadByte(); Command = (MessageCommand)reader.ReadByte(); _payload_compressed = reader.ReadVarBytes(PayloadMaxSize); - SetPayload(); + DecompressPayload(); + } + + void ISerializable.Serialize(BinaryWriter writer) + { + writer.Write((byte)Flags); + writer.Write((byte)Command); + writer.WriteVarBytes(_payload_compressed); } - public static int TryDeserialize(ByteString data, out Message msg) + internal static int TryDeserialize(ByteString data, out Message msg) { msg = null; if (data.Count < 3) return 0; @@ -99,59 +149,9 @@ public static int TryDeserialize(ByteString data, out Message msg) Command = (MessageCommand)header[1], _payload_compressed = data.Slice(payloadIndex, (int)length).ToArray() }; - msg.SetPayload(); + msg.DecompressPayload(); return payloadIndex + (int)length; } - - private void SetPayload() - { - if (_payload_compressed.Length == 0) return; - byte[] decompressed = Flags.HasFlag(MessageFlags.Compressed) - ? _payload_compressed.DecompressLz4(PayloadMaxSize) - : _payload_compressed; - switch (Command) - { - case MessageCommand.Version: - Payload = decompressed.AsSerializable(); - break; - case MessageCommand.Addr: - Payload = decompressed.AsSerializable(); - break; - case MessageCommand.Ping: - case MessageCommand.Pong: - Payload = decompressed.AsSerializable(); - break; - case MessageCommand.GetHeaders: - case MessageCommand.GetBlocks: - Payload = decompressed.AsSerializable(); - break; - case MessageCommand.Headers: - Payload = decompressed.AsSerializable(); - break; - case MessageCommand.Inv: - case MessageCommand.GetData: - Payload = decompressed.AsSerializable(); - break; - case MessageCommand.Transaction: - Payload = Transaction.DeserializeFrom(decompressed); - break; - case MessageCommand.Block: - Payload = decompressed.AsSerializable(); - break; - case MessageCommand.Consensus: - Payload = decompressed.AsSerializable(); - break; - case MessageCommand.FilterLoad: - Payload = decompressed.AsSerializable(); - break; - case MessageCommand.FilterAdd: - Payload = decompressed.AsSerializable(); - break; - case MessageCommand.MerkleBlock: - Payload = decompressed.AsSerializable(); - break; - } - } } } diff --git a/neo/Network/P2P/MessageFlags.cs b/neo/Network/P2P/MessageFlags.cs index a608f5bfef..9af64e5fbc 100644 --- a/neo/Network/P2P/MessageFlags.cs +++ b/neo/Network/P2P/MessageFlags.cs @@ -8,4 +8,4 @@ public enum MessageFlags : byte None = 0, Compressed = 1 << 0 } -} \ No newline at end of file +} diff --git a/neo/Network/P2P/Payloads/NetworkAddressWithTime.cs b/neo/Network/P2P/Payloads/NetworkAddressWithTime.cs index fee0cca399..975541be97 100644 --- a/neo/Network/P2P/Payloads/NetworkAddressWithTime.cs +++ b/neo/Network/P2P/Payloads/NetworkAddressWithTime.cs @@ -1,8 +1,8 @@ -using System; +using Neo.IO; +using System; using System.IO; using System.Linq; using System.Net; -using Neo.IO; namespace Neo.Network.P2P.Payloads { diff --git a/neo/Network/P2P/Payloads/VersionPayload.cs b/neo/Network/P2P/Payloads/VersionPayload.cs index d25c5532d5..0125e4eee0 100644 --- a/neo/Network/P2P/Payloads/VersionPayload.cs +++ b/neo/Network/P2P/Payloads/VersionPayload.cs @@ -1,6 +1,6 @@ -using System; +using Neo.IO; +using System; using System.IO; -using Neo.IO; namespace Neo.Network.P2P.Payloads { @@ -15,7 +15,15 @@ public class VersionPayload : ISerializable public string UserAgent; public uint StartHeight; - public int Size => (sizeof(uint) * 5) + sizeof(ulong) + sizeof(ushort) + UserAgent.GetVarSize(); + public int Size => + sizeof(uint) + //Magic + sizeof(uint) + //Version + sizeof(VersionServices) + //Services + sizeof(uint) + //Timestamp + sizeof(ushort) + //Port + sizeof(uint) + //Nonce + UserAgent.GetVarSize() + //UserAgent + sizeof(uint); //StartHeight public static VersionPayload Create(int port, uint nonce, string userAgent, uint startHeight) { @@ -56,4 +64,4 @@ void ISerializable.Serialize(BinaryWriter writer) writer.Write(StartHeight); } } -} \ No newline at end of file +} diff --git a/neo/Network/P2P/RemoteNode.cs b/neo/Network/P2P/RemoteNode.cs index 7f93b5bba2..5a0381e04f 100644 --- a/neo/Network/P2P/RemoteNode.cs +++ b/neo/Network/P2P/RemoteNode.cs @@ -1,7 +1,4 @@ -using System.Collections.Generic; -using System.Linq; -using System.Net; -using Akka.Actor; +using Akka.Actor; using Akka.Configuration; using Akka.IO; using Neo.Cryptography; @@ -9,6 +6,9 @@ using Neo.IO.Actors; using Neo.Ledger; using Neo.Network.P2P.Payloads; +using System.Collections.Generic; +using System.Linq; +using System.Net; namespace Neo.Network.P2P { From 5328c332e1edf7b99a1bda690adb2876f7d6082b Mon Sep 17 00:00:00 2001 From: Belane Date: Fri, 3 May 2019 14:13:15 +0200 Subject: [PATCH 41/45] fix connection issues --- neo/Network/P2P/Message.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neo/Network/P2P/Message.cs b/neo/Network/P2P/Message.cs index aae81f3c78..f49456e7b9 100644 --- a/neo/Network/P2P/Message.cs +++ b/neo/Network/P2P/Message.cs @@ -115,7 +115,7 @@ internal static int TryDeserialize(ByteString data, out Message msg) msg = null; if (data.Count < 3) return 0; - var header = data.Slice(0, 5).ToArray(); + var header = data.Slice(0, 3).ToArray(); var flags = (MessageFlags)header[0]; ulong length = header[2]; var payloadIndex = 3; @@ -147,7 +147,7 @@ internal static int TryDeserialize(ByteString data, out Message msg) { Flags = flags, Command = (MessageCommand)header[1], - _payload_compressed = data.Slice(payloadIndex, (int)length).ToArray() + _payload_compressed = length <= 0 ? new byte[0] : data.Slice(payloadIndex, (int)length).ToArray() }; msg.DecompressPayload(); From 70abc9cfe3b7693d12867c91a3366381a299d2f1 Mon Sep 17 00:00:00 2001 From: Shargon Date: Fri, 3 May 2019 16:58:58 +0200 Subject: [PATCH 42/45] More ut --- neo.UnitTests/UT_P2PMessage.cs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/neo.UnitTests/UT_P2PMessage.cs b/neo.UnitTests/UT_P2PMessage.cs index 407e3692e7..ee27228a2e 100644 --- a/neo.UnitTests/UT_P2PMessage.cs +++ b/neo.UnitTests/UT_P2PMessage.cs @@ -47,6 +47,33 @@ public void Serialize_Deserialize_ByteString() buffer.Count.Should().Be(length); } + [TestMethod] + public void Serialize_Deserialize_WithoutPayload() + { + var msg = Message.Create(MessageCommand.GetAddr); + var buffer = msg.ToArray(); + var copy = buffer.AsSerializable(); + + copy.Command.Should().Be(msg.Command); + copy.Flags.Should().Be(msg.Flags); + copy.Payload.Should().Be(null); + } + + [TestMethod] + public void Serialize_Deserialize_WithoutPayload_ByteString() + { + var msg = Message.Create(MessageCommand.GetAddr); + var buffer = ByteString.CopyFrom(msg.ToArray()); + var length = Message.TryDeserialize(buffer, out var copy); + + copy.Command.Should().Be(msg.Command); + copy.Flags.Should().Be(msg.Flags); + copy.Payload.Should().Be(null); + + buffer.Count.Should().Be(length); + } + + [TestMethod] public void Compression() { From d8ec9e5f968ff6eac1703f0ff30ade61c3a84461 Mon Sep 17 00:00:00 2001 From: Shargon Date: Sun, 5 May 2019 20:50:31 +0200 Subject: [PATCH 43/45] Simplify GetBlocksPayload --- neo/Network/P2P/Payloads/GetBlocksPayload.cs | 24 ++++++++++---------- neo/Network/P2P/ProtocolHandler.cs | 14 +++++------- neo/Network/P2P/TaskManager.cs | 4 ++-- 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/neo/Network/P2P/Payloads/GetBlocksPayload.cs b/neo/Network/P2P/Payloads/GetBlocksPayload.cs index af3bfb7238..17bc2af456 100644 --- a/neo/Network/P2P/Payloads/GetBlocksPayload.cs +++ b/neo/Network/P2P/Payloads/GetBlocksPayload.cs @@ -1,34 +1,34 @@ -using Neo.IO; -using System.IO; +using System.IO; +using Neo.IO; namespace Neo.Network.P2P.Payloads { public class GetBlocksPayload : ISerializable { - public UInt256[] HashStart; - public UInt256 HashStop; + public UInt256 HashStart; + public int Count; - public int Size => HashStart.GetVarSize() + HashStop.Size; + public int Size => 32 + sizeof(int); - public static GetBlocksPayload Create(UInt256 hash_start, UInt256 hash_stop = null) + public static GetBlocksPayload Create(UInt256 hash_start, int count) { return new GetBlocksPayload { - HashStart = new[] { hash_start }, - HashStop = hash_stop ?? UInt256.Zero + HashStart = hash_start, + Count = count }; } void ISerializable.Deserialize(BinaryReader reader) { - HashStart = reader.ReadSerializableArray(16); - HashStop = reader.ReadSerializable(); + HashStart = reader.ReadSerializable(); + Count = reader.ReadInt16(); } void ISerializable.Serialize(BinaryWriter writer) { writer.Write(HashStart); - writer.Write(HashStop); + writer.Write(Count); } } -} +} \ No newline at end of file diff --git a/neo/Network/P2P/ProtocolHandler.cs b/neo/Network/P2P/ProtocolHandler.cs index 6bf0b05f10..30acd79f17 100644 --- a/neo/Network/P2P/ProtocolHandler.cs +++ b/neo/Network/P2P/ProtocolHandler.cs @@ -157,19 +157,18 @@ private void OnGetAddrMessageReceived() private void OnGetBlocksMessageReceived(GetBlocksPayload payload) { - UInt256 hash = payload.HashStart[0]; - if (hash == payload.HashStop) return; + UInt256 hash = payload.HashStart; + if (payload.Count <= 0) return; BlockState state = Blockchain.Singleton.Store.GetBlocks().TryGet(hash); if (state == null) return; List hashes = new List(); - for (uint i = 1; i <= InvPayload.MaxHashesCount; i++) + for (uint i = 1; i <= InvPayload.MaxHashesCount && i <= payload.Count; i++) { uint index = state.TrimmedBlock.Index + i; if (index > Blockchain.Singleton.Height) break; hash = Blockchain.Singleton.GetBlockHash(index); if (hash == null) break; - if (hash == payload.HashStop) break; hashes.Add(hash); } if (hashes.Count == 0) return; @@ -216,18 +215,17 @@ private void OnGetDataMessageReceived(InvPayload payload) private void OnGetHeadersMessageReceived(GetBlocksPayload payload) { - UInt256 hash = payload.HashStart[0]; - if (hash == payload.HashStop) return; + UInt256 hash = payload.HashStart; + if (payload.Count <= 0) return; DataCache cache = Blockchain.Singleton.Store.GetBlocks(); BlockState state = cache.TryGet(hash); if (state == null) return; List
headers = new List
(); - for (uint i = 1; i <= HeadersPayload.MaxHeadersCount; i++) + for (uint i = 1; i <= HeadersPayload.MaxHeadersCount && i <= payload.Count; i++) { uint index = state.TrimmedBlock.Index + i; hash = Blockchain.Singleton.GetBlockHash(index); if (hash == null) break; - if (hash == payload.HashStop) break; Header header = cache.TryGet(hash)?.TrimmedBlock.Header; if (header == null) break; headers.Add(header); diff --git a/neo/Network/P2P/TaskManager.cs b/neo/Network/P2P/TaskManager.cs index bd9481f2ea..a226b3ce3a 100644 --- a/neo/Network/P2P/TaskManager.cs +++ b/neo/Network/P2P/TaskManager.cs @@ -223,7 +223,7 @@ private void RequestTasks(TaskSession session) { session.Tasks[HeaderTaskHash] = DateTime.UtcNow; IncrementGlobalTask(HeaderTaskHash); - session.RemoteNode.Tell(Message.Create(MessageCommand.GetHeaders, GetBlocksPayload.Create(Blockchain.Singleton.CurrentHeaderHash))); + session.RemoteNode.Tell(Message.Create(MessageCommand.GetHeaders, GetBlocksPayload.Create(Blockchain.Singleton.CurrentHeaderHash, HeadersPayload.MaxHeadersCount))); } else if (Blockchain.Singleton.Height < session.Version.StartHeight) { @@ -237,7 +237,7 @@ private void RequestTasks(TaskSession session) break; } } - session.RemoteNode.Tell(Message.Create(MessageCommand.GetBlocks, GetBlocksPayload.Create(hash))); + session.RemoteNode.Tell(Message.Create(MessageCommand.GetBlocks, GetBlocksPayload.Create(hash, InvPayload.MaxHashesCount))); } } } From d0f82a154a4611179a10162036ea1d41d4355566 Mon Sep 17 00:00:00 2001 From: Shargon Date: Sun, 5 May 2019 21:19:15 +0200 Subject: [PATCH 44/45] Update GetBlocksPayload.cs --- neo/Network/P2P/Payloads/GetBlocksPayload.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neo/Network/P2P/Payloads/GetBlocksPayload.cs b/neo/Network/P2P/Payloads/GetBlocksPayload.cs index 17bc2af456..59877bda59 100644 --- a/neo/Network/P2P/Payloads/GetBlocksPayload.cs +++ b/neo/Network/P2P/Payloads/GetBlocksPayload.cs @@ -22,7 +22,7 @@ public static GetBlocksPayload Create(UInt256 hash_start, int count) void ISerializable.Deserialize(BinaryReader reader) { HashStart = reader.ReadSerializable(); - Count = reader.ReadInt16(); + Count = reader.ReadInt32(); } void ISerializable.Serialize(BinaryWriter writer) @@ -31,4 +31,4 @@ void ISerializable.Serialize(BinaryWriter writer) writer.Write(Count); } } -} \ No newline at end of file +} From 164e49a21d053847d34bf16b3c787e37e6cfe552 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Mon, 6 May 2019 15:46:50 +0800 Subject: [PATCH 45/45] Allow `GetBlocksPayload.Count == -1` --- neo/Network/P2P/Payloads/GetBlocksPayload.cs | 14 ++++++++------ neo/Network/P2P/ProtocolHandler.cs | 8 ++++---- neo/Network/P2P/TaskManager.cs | 4 ++-- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/neo/Network/P2P/Payloads/GetBlocksPayload.cs b/neo/Network/P2P/Payloads/GetBlocksPayload.cs index 59877bda59..f7e39927f1 100644 --- a/neo/Network/P2P/Payloads/GetBlocksPayload.cs +++ b/neo/Network/P2P/Payloads/GetBlocksPayload.cs @@ -1,16 +1,17 @@ -using System.IO; -using Neo.IO; +using Neo.IO; +using System; +using System.IO; namespace Neo.Network.P2P.Payloads { public class GetBlocksPayload : ISerializable { public UInt256 HashStart; - public int Count; + public short Count; - public int Size => 32 + sizeof(int); + public int Size => sizeof(short) + HashStart.Size; - public static GetBlocksPayload Create(UInt256 hash_start, int count) + public static GetBlocksPayload Create(UInt256 hash_start, short count = -1) { return new GetBlocksPayload { @@ -22,7 +23,8 @@ public static GetBlocksPayload Create(UInt256 hash_start, int count) void ISerializable.Deserialize(BinaryReader reader) { HashStart = reader.ReadSerializable(); - Count = reader.ReadInt32(); + Count = reader.ReadInt16(); + if (Count < -1 || Count == 0) throw new FormatException(); } void ISerializable.Serialize(BinaryWriter writer) diff --git a/neo/Network/P2P/ProtocolHandler.cs b/neo/Network/P2P/ProtocolHandler.cs index 30acd79f17..a38ad60580 100644 --- a/neo/Network/P2P/ProtocolHandler.cs +++ b/neo/Network/P2P/ProtocolHandler.cs @@ -158,11 +158,11 @@ private void OnGetAddrMessageReceived() private void OnGetBlocksMessageReceived(GetBlocksPayload payload) { UInt256 hash = payload.HashStart; - if (payload.Count <= 0) return; + int count = payload.Count < 0 ? InvPayload.MaxHashesCount : payload.Count; BlockState state = Blockchain.Singleton.Store.GetBlocks().TryGet(hash); if (state == null) return; List hashes = new List(); - for (uint i = 1; i <= InvPayload.MaxHashesCount && i <= payload.Count; i++) + for (uint i = 1; i <= count; i++) { uint index = state.TrimmedBlock.Index + i; if (index > Blockchain.Singleton.Height) @@ -216,12 +216,12 @@ private void OnGetDataMessageReceived(InvPayload payload) private void OnGetHeadersMessageReceived(GetBlocksPayload payload) { UInt256 hash = payload.HashStart; - if (payload.Count <= 0) return; + int count = payload.Count < 0 ? HeadersPayload.MaxHeadersCount : payload.Count; DataCache cache = Blockchain.Singleton.Store.GetBlocks(); BlockState state = cache.TryGet(hash); if (state == null) return; List
headers = new List
(); - for (uint i = 1; i <= HeadersPayload.MaxHeadersCount && i <= payload.Count; i++) + for (uint i = 1; i <= count; i++) { uint index = state.TrimmedBlock.Index + i; hash = Blockchain.Singleton.GetBlockHash(index); diff --git a/neo/Network/P2P/TaskManager.cs b/neo/Network/P2P/TaskManager.cs index a226b3ce3a..bd9481f2ea 100644 --- a/neo/Network/P2P/TaskManager.cs +++ b/neo/Network/P2P/TaskManager.cs @@ -223,7 +223,7 @@ private void RequestTasks(TaskSession session) { session.Tasks[HeaderTaskHash] = DateTime.UtcNow; IncrementGlobalTask(HeaderTaskHash); - session.RemoteNode.Tell(Message.Create(MessageCommand.GetHeaders, GetBlocksPayload.Create(Blockchain.Singleton.CurrentHeaderHash, HeadersPayload.MaxHeadersCount))); + session.RemoteNode.Tell(Message.Create(MessageCommand.GetHeaders, GetBlocksPayload.Create(Blockchain.Singleton.CurrentHeaderHash))); } else if (Blockchain.Singleton.Height < session.Version.StartHeight) { @@ -237,7 +237,7 @@ private void RequestTasks(TaskSession session) break; } } - session.RemoteNode.Tell(Message.Create(MessageCommand.GetBlocks, GetBlocksPayload.Create(hash, InvPayload.MaxHashesCount))); + session.RemoteNode.Tell(Message.Create(MessageCommand.GetBlocks, GetBlocksPayload.Create(hash))); } } }