From a8bcfbba6d80edc4c59b7aa627568b25aa757480 Mon Sep 17 00:00:00 2001 From: Vitor Date: Sat, 5 Oct 2019 19:54:56 -0300 Subject: [PATCH 1/5] Simplify and improve consensus payload serialiation --- neo/Consensus/ConsensusContext.cs | 41 ++++++++++--------------------- 1 file changed, 13 insertions(+), 28 deletions(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index 0d9edbb3dc..2621d801f3 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -107,13 +107,14 @@ public void Deserialize(BinaryReader reader) PreparationPayloads = new ConsensusPayload[reader.ReadVarInt(Blockchain.MaxValidators)]; for (int i = 0; i < PreparationPayloads.Length; i++) PreparationPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; - CommitPayloads = new ConsensusPayload[reader.ReadVarInt(Blockchain.MaxValidators)]; + ulong emptyOrValidators = reader.ReadVarInt(Blockchain.MaxValidators); + CommitPayloads = new ConsensusPayload[emptyOrValidators]; for (int i = 0; i < CommitPayloads.Length; i++) CommitPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; - ChangeViewPayloads = new ConsensusPayload[reader.ReadVarInt(Blockchain.MaxValidators)]; + ChangeViewPayloads = new ConsensusPayload[emptyOrValidators]; for (int i = 0; i < ChangeViewPayloads.Length; i++) ChangeViewPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; - LastChangeViewPayloads = new ConsensusPayload[reader.ReadVarInt(Blockchain.MaxValidators)]; + LastChangeViewPayloads = new ConsensusPayload[emptyOrValidators]; for (int i = 0; i < LastChangeViewPayloads.Length; i++) LastChangeViewPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; if (TransactionHashes.Length == 0 && !RequestSentOrReceived) @@ -417,31 +418,15 @@ public void Serialize(BinaryWriter writer) writer.Write(TransactionHashes ?? new UInt256[0]); writer.Write(Transactions?.Values.ToArray() ?? new Transaction[0]); writer.WriteVarInt(PreparationPayloads.Length); - foreach (var payload in PreparationPayloads) - { - bool hasPayload = !(payload is null); - writer.Write(hasPayload); - if (!hasPayload) continue; - writer.Write(payload); - } - writer.WriteVarInt(CommitPayloads.Length); - foreach (var payload in CommitPayloads) - { - bool hasPayload = !(payload is null); - writer.Write(hasPayload); - if (!hasPayload) continue; - writer.Write(payload); - } - writer.WriteVarInt(ChangeViewPayloads.Length); - foreach (var payload in ChangeViewPayloads) - { - bool hasPayload = !(payload is null); - writer.Write(hasPayload); - if (!hasPayload) continue; - writer.Write(payload); - } - writer.WriteVarInt(LastChangeViewPayloads.Length); - foreach (var payload in LastChangeViewPayloads) + SerializeConsensusPayloadArray(writer, PreparationPayloads); + writer.WriteVarInt(Validators.Length); + SerializeConsensusPayloadArray(writer, CommitPayloads); + SerializeConsensusPayloadArray(writer, ChangeViewPayloads); + SerializeConsensusPayloadArray(writer, LastChangeViewPayloads); + } + private void SerializeConsensusPayloadArray(BinaryWriter writer, ConsensusPayload[] payloadArray) + { + foreach (var payload in payloadArray) { bool hasPayload = !(payload is null); writer.Write(hasPayload); From 9960cbb471797bb4093afa2213c139263a97d66d Mon Sep 17 00:00:00 2001 From: Vitor Date: Sat, 5 Oct 2019 20:33:50 -0300 Subject: [PATCH 2/5] Simplyfing read --- neo/Consensus/ConsensusContext.cs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index 2621d801f3..22b8f8158c 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -104,19 +104,13 @@ public void Deserialize(BinaryReader reader) ViewNumber = reader.ReadByte(); TransactionHashes = reader.ReadSerializableArray(); Transaction[] transactions = reader.ReadSerializableArray(Block.MaxTransactionsPerBlock); - PreparationPayloads = new ConsensusPayload[reader.ReadVarInt(Blockchain.MaxValidators)]; - for (int i = 0; i < PreparationPayloads.Length; i++) - PreparationPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; + + ReadConsensusPayloadArray(reader, out PreparationPayloads, reader.ReadVarInt(Blockchain.MaxValidators)); ulong emptyOrValidators = reader.ReadVarInt(Blockchain.MaxValidators); - CommitPayloads = new ConsensusPayload[emptyOrValidators]; - for (int i = 0; i < CommitPayloads.Length; i++) - CommitPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; - ChangeViewPayloads = new ConsensusPayload[emptyOrValidators]; - for (int i = 0; i < ChangeViewPayloads.Length; i++) - ChangeViewPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; - LastChangeViewPayloads = new ConsensusPayload[emptyOrValidators]; - for (int i = 0; i < LastChangeViewPayloads.Length; i++) - LastChangeViewPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; + ReadConsensusPayloadArray(reader, out CommitPayloads, emptyOrValidators); + ReadConsensusPayloadArray(reader, out ChangeViewPayloads, emptyOrValidators); + ReadConsensusPayloadArray(reader, out LastChangeViewPayloads, emptyOrValidators); + if (TransactionHashes.Length == 0 && !RequestSentOrReceived) TransactionHashes = null; Transactions = transactions.Length == 0 && !RequestSentOrReceived ? null : transactions.ToDictionary(p => p.Hash); @@ -434,5 +428,11 @@ private void SerializeConsensusPayloadArray(BinaryWriter writer, ConsensusPayloa writer.Write(payload); } } + private void ReadConsensusPayloadArray(BinaryReader reader, out ConsensusPayload[] payloadArray, ulong payloadLenght) + { + payloadArray = new ConsensusPayload[payloadLenght]; + for (int i = 0; i < payloadArray.Length; i++) + payloadArray[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; + } } } From f2362437e13c66dd0a617e46f23a046be5678f10 Mon Sep 17 00:00:00 2001 From: Vitor Date: Fri, 11 Oct 2019 23:02:29 -0300 Subject: [PATCH 3/5] Function return instead of out parameter --- neo/Consensus/ConsensusContext.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index 22b8f8158c..3480a9b25a 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -105,11 +105,11 @@ public void Deserialize(BinaryReader reader) TransactionHashes = reader.ReadSerializableArray(); Transaction[] transactions = reader.ReadSerializableArray(Block.MaxTransactionsPerBlock); - ReadConsensusPayloadArray(reader, out PreparationPayloads, reader.ReadVarInt(Blockchain.MaxValidators)); + PreparationPayloads = ReadConsensusPayloadArray(reader, reader.ReadVarInt(Blockchain.MaxValidators)); ulong emptyOrValidators = reader.ReadVarInt(Blockchain.MaxValidators); - ReadConsensusPayloadArray(reader, out CommitPayloads, emptyOrValidators); - ReadConsensusPayloadArray(reader, out ChangeViewPayloads, emptyOrValidators); - ReadConsensusPayloadArray(reader, out LastChangeViewPayloads, emptyOrValidators); + CommitPayloads = ReadConsensusPayloadArray(reader, emptyOrValidators); + ChangeViewPayloads = ReadConsensusPayloadArray(reader, emptyOrValidators); + LastChangeViewPayloads = ReadConsensusPayloadArray(reader, emptyOrValidators); if (TransactionHashes.Length == 0 && !RequestSentOrReceived) TransactionHashes = null; @@ -428,11 +428,12 @@ private void SerializeConsensusPayloadArray(BinaryWriter writer, ConsensusPayloa writer.Write(payload); } } - private void ReadConsensusPayloadArray(BinaryReader reader, out ConsensusPayload[] payloadArray, ulong payloadLenght) + private ConsensusPayload[] ReadConsensusPayloadArray(BinaryReader reader, ulong payloadLenght) { - payloadArray = new ConsensusPayload[payloadLenght]; + ConsensusPayload[] payloadArray = new ConsensusPayload[payloadLenght]; for (int i = 0; i < payloadArray.Length; i++) payloadArray[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; + return payloadArray; } } } From 04766094ec50d00914b020ffd63546d5493cabc5 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Mon, 14 Oct 2019 16:02:00 +0800 Subject: [PATCH 4/5] Reorganize --- neo/Consensus/ConsensusContext.cs | 38 +++++++------------------------ neo/IO/Helper.cs | 20 ++++++++++++++++ 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index 3480a9b25a..ddbf6caeea 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -104,13 +104,10 @@ public void Deserialize(BinaryReader reader) ViewNumber = reader.ReadByte(); TransactionHashes = reader.ReadSerializableArray(); Transaction[] transactions = reader.ReadSerializableArray(Block.MaxTransactionsPerBlock); - - PreparationPayloads = ReadConsensusPayloadArray(reader, reader.ReadVarInt(Blockchain.MaxValidators)); - ulong emptyOrValidators = reader.ReadVarInt(Blockchain.MaxValidators); - CommitPayloads = ReadConsensusPayloadArray(reader, emptyOrValidators); - ChangeViewPayloads = ReadConsensusPayloadArray(reader, emptyOrValidators); - LastChangeViewPayloads = ReadConsensusPayloadArray(reader, emptyOrValidators); - + PreparationPayloads = reader.ReadNullableArray(Blockchain.MaxValidators); + CommitPayloads = reader.ReadNullableArray(Blockchain.MaxValidators); + ChangeViewPayloads = reader.ReadNullableArray(Blockchain.MaxValidators); + LastChangeViewPayloads = reader.ReadNullableArray(Blockchain.MaxValidators); if (TransactionHashes.Length == 0 && !RequestSentOrReceived) TransactionHashes = null; Transactions = transactions.Length == 0 && !RequestSentOrReceived ? null : transactions.ToDictionary(p => p.Hash); @@ -411,29 +408,10 @@ public void Serialize(BinaryWriter writer) writer.Write(ViewNumber); writer.Write(TransactionHashes ?? new UInt256[0]); writer.Write(Transactions?.Values.ToArray() ?? new Transaction[0]); - writer.WriteVarInt(PreparationPayloads.Length); - SerializeConsensusPayloadArray(writer, PreparationPayloads); - writer.WriteVarInt(Validators.Length); - SerializeConsensusPayloadArray(writer, CommitPayloads); - SerializeConsensusPayloadArray(writer, ChangeViewPayloads); - SerializeConsensusPayloadArray(writer, LastChangeViewPayloads); - } - private void SerializeConsensusPayloadArray(BinaryWriter writer, ConsensusPayload[] payloadArray) - { - foreach (var payload in payloadArray) - { - bool hasPayload = !(payload is null); - writer.Write(hasPayload); - if (!hasPayload) continue; - writer.Write(payload); - } - } - private ConsensusPayload[] ReadConsensusPayloadArray(BinaryReader reader, ulong payloadLenght) - { - ConsensusPayload[] payloadArray = new ConsensusPayload[payloadLenght]; - for (int i = 0; i < payloadArray.Length; i++) - payloadArray[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; - return payloadArray; + writer.WriteNullableArray(PreparationPayloads); + writer.WriteNullableArray(CommitPayloads); + writer.WriteNullableArray(ChangeViewPayloads); + writer.WriteNullableArray(LastChangeViewPayloads); } } } diff --git a/neo/IO/Helper.cs b/neo/IO/Helper.cs index c985e676b2..c47878579d 100644 --- a/neo/IO/Helper.cs +++ b/neo/IO/Helper.cs @@ -112,6 +112,14 @@ public static string ReadFixedString(this BinaryReader reader, int length) return Encoding.UTF8.GetString(data.TakeWhile(p => p != 0).ToArray()); } + public static T[] ReadNullableArray(this BinaryReader reader, int max = 0x1000000) where T : class, ISerializable, new() + { + T[] array = new T[reader.ReadVarInt((ulong)max)]; + for (int i = 0; i < array.Length; i++) + array[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; + return array; + } + public static T ReadSerializable(this BinaryReader reader) where T : ISerializable, new() { T obj = new T(); @@ -225,6 +233,18 @@ public static void WriteFixedString(this BinaryWriter writer, string value, int writer.Write(new byte[length - bytes.Length]); } + public static void WriteNullableArray(this BinaryWriter writer, T[] value) where T : class, ISerializable + { + writer.WriteVarInt(value.Length); + foreach (var item in value) + { + bool isNull = item is null; + writer.Write(!isNull); + if (isNull) continue; + item.Serialize(writer); + } + } + public static void WriteVarBytes(this BinaryWriter writer, byte[] value) { writer.WriteVarInt(value.Length); From 35afab4c2a50b6d9b0aa90b420164e4daecbea64 Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 14 Oct 2019 11:18:22 +0200 Subject: [PATCH 5/5] Update UT_IOHelper.cs --- neo.UnitTests/IO/UT_IOHelper.cs | 42 ++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/neo.UnitTests/IO/UT_IOHelper.cs b/neo.UnitTests/IO/UT_IOHelper.cs index 114ceffc8d..d0f63e5c1c 100644 --- a/neo.UnitTests/IO/UT_IOHelper.cs +++ b/neo.UnitTests/IO/UT_IOHelper.cs @@ -23,6 +23,46 @@ public void TestAsSerializableGeneric() Assert.AreEqual(UInt160.Zero, result); } + [TestMethod] + public void TestNullableArray() + { + var caseArray = new UInt160[] + { + null, UInt160.Zero, new UInt160( + new byte[] { + 0xAA,0x00,0x00,0x00,0x00, + 0xBB,0x00,0x00,0x00,0x00, + 0xCC,0x00,0x00,0x00,0x00, + 0xDD,0x00,0x00,0x00,0x00 + }) + }; + + byte[] data; + using (var stream = new MemoryStream()) + using (var writter = new BinaryWriter(stream)) + { + Neo.IO.Helper.WriteNullableArray(writter, caseArray); + data = stream.ToArray(); + } + + // Read Error + + using (var stream = new MemoryStream(data)) + using (var reader = new BinaryReader(stream)) + { + Assert.ThrowsException(() => Neo.IO.Helper.ReadNullableArray(reader, 2)); + } + + // Read 100% + + using (var stream = new MemoryStream(data)) + using (var reader = new BinaryReader(stream)) + { + var read = Neo.IO.Helper.ReadNullableArray(reader); + CollectionAssert.AreEqual(caseArray, read); + } + } + [TestMethod] public void TestAsSerializable() { @@ -48,7 +88,7 @@ public void TestAsSerializable() [TestMethod] public void TestAsSerializableArray() { - byte[] byteArray = Neo.IO.Helper.ToByteArray(new UInt160[] { UInt160.Zero }); + byte[] byteArray = Neo.IO.Helper.ToByteArray(new UInt160[] { UInt160.Zero }); UInt160[] result = Neo.IO.Helper.AsSerializableArray(byteArray); Assert.AreEqual(1, result.Length); Assert.AreEqual(UInt160.Zero, result[0]);