Use HashsetCache instead of FIFOSet for KnownHashes and SentHashes

src/neo/Helper.cs

@@ -83,7 +83,7 @@ internal static void Remove<T>(this HashSet<T> set, ISet<T> other)
  }
  }
 
- internal static void Remove<T>(this HashSet<T> set, FIFOSet<T> other)
+ internal static void Remove<T>(this HashSet<T> set, HashSetCache<T> other)
  where T : IEquatable<T>
  {
  if (set.Count > other.Count)

src/neo/IO/Caching/HashSetCache.cs

@@ -0,0 +1,82 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace Neo.IO.Caching
+{
+ public class HashSetCache<T> : IReadOnlyCollection<T> where T : IEquatable<T>
+ {
+ private readonly int bucketCapacity;
+ private readonly int bucketCount;
+ private readonly LinkedList<HashSet<T>> sets = new LinkedList<HashSet<T>>();
+
+ public int Count
+ {
+ get
+ {
+ int count = 0;
+ foreach (var set in sets)
+ {
+ count += set.Count;
+ }
+ return count;
+ }
+ }
+
+ public HashSetCache(int bucketCapacity, int bucketCount = 10)
+ {
+ if (bucketCapacity <= 0) throw new ArgumentOutOfRangeException(nameof(bucketCapacity));
+ if (bucketCount <= 0 || bucketCount > 20) throw new ArgumentOutOfRangeException($"{nameof(bucketCount)} should between 1 and 20");
+
+ this.bucketCapacity = bucketCapacity;
+ this.bucketCount = bucketCount;
+ sets.AddFirst(new HashSet<T>());
+ }
+
+ public bool Add(T item)
+ {
+ if (Contains(item)) return false;
+ if (sets.First.Value.Count < bucketCapacity) return sets.First.Value.Add(item);
+ var newSet = new HashSet<T>
+ {
+ item
+ };
+ sets.AddFirst(newSet);
+ if (sets.Count > bucketCount) sets.RemoveLast();
+ return true;
+ }
+
+ public bool Contains(T item)
+ {
+ foreach (var set in sets)
+ {
+ if (set.Contains(item)) return true;
+ }
+ return false;
+ }
+
+ public void ExceptWith(IEnumerable<T> items)
+ {
+ foreach (var item in items)
+ {
+ foreach (var set in sets)
+ {
+ if (set.Remove(item)) break;
+ }
+ }
+ }
+
+ public IEnumerator<T> GetEnumerator()
+ {
+ foreach (var set in sets)
+ {
+ foreach (var item in set)
+ {
+ yield return item;
+ }
+ }
+ }
+
+ IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
+ }
+}

src/neo/Network/P2P/ProtocolHandler.cs

@@ -32,8 +32,8 @@ protected override UInt256 GetKeyForItem((UInt256, DateTime) item)
 
  private readonly NeoSystem system;
  private readonly PendingKnownHashesCollection pendingKnownHashes;
- private readonly FIFOSet<UInt256> knownHashes;
- private readonly FIFOSet<UInt256> sentHashes;
+ private readonly HashSetCache<UInt256> knownHashes;
+ private readonly HashSetCache<UInt256> sentHashes;
  private VersionPayload version;
  private bool verack = false;
  private BloomFilter bloom_filter;
@@ -47,8 +47,8 @@ public ProtocolHandler(NeoSystem system)
  {
  this.system = system;
  this.pendingKnownHashes = new PendingKnownHashesCollection();
- this.knownHashes = new FIFOSet<UInt256>(Blockchain.Singleton.MemPool.Capacity * 2);
- this.sentHashes = new FIFOSet<UInt256>(Blockchain.Singleton.MemPool.Capacity * 2);
+ this.knownHashes = new HashSetCache<UInt256>(Blockchain.Singleton.MemPool.Capacity * 2 / 10);
+ this.sentHashes = new HashSetCache<UInt256>(Blockchain.Singleton.MemPool.Capacity * 2 / 10);
  }
 
  protected override void OnReceive(object message)

src/neo/Network/P2P/TaskManager.cs

@@ -32,7 +32,7 @@ private class Timer { }
  /// <summary>
  /// A set of known hashes, of inventories or payloads, already received.
  /// </summary> 
- private readonly FIFOSet<UInt256> knownHashes;
+ private readonly HashSetCache<UInt256> knownHashes;
  private readonly Dictionary<UInt256, int> globalTasks = new Dictionary<UInt256, int>();
  private readonly Dictionary<IActorRef, TaskSession> sessions = new Dictionary<IActorRef, TaskSession>();
  private readonly ICancelable timer = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(TimerInterval, TimerInterval, Context.Self, new Timer(), ActorRefs.NoSender);
@@ -43,7 +43,7 @@ private class Timer { }
  public TaskManager(NeoSystem system)
  {
  this.system = system;
- this.knownHashes = new FIFOSet<UInt256>(Blockchain.Singleton.MemPool.Capacity * 2);
+ this.knownHashes = new HashSetCache<UInt256>(Blockchain.Singleton.MemPool.Capacity * 2 / 10);
  }
 
  private void OnHeaderTaskCompleted()

tests/neo.UnitTests/IO/Caching/UT_FIFOSet.cs → tests/neo.UnitTests/IO/Caching/UT_HashSetCache.cs

@@ -8,62 +8,52 @@
 namespace Neo.UnitTests.IO.Caching
 {
  [TestClass]
- public class UT_FIFOSet
+ public class UT_HashSetCache
  {
  [TestMethod]
- public void FIFOSetTest()
+ public void TestHashSetCache()
  {
- var a = UInt256.Zero;
- var b = new UInt256();
- var c = new UInt256(new byte[32] {
- 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
- 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
- 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
- 0x01, 0x01
- });
-
- var set = new FIFOSet<UInt256>(3);
-
- Assert.IsTrue(set.Add(a));
- Assert.IsFalse(set.Add(a));
- Assert.IsFalse(set.Add(b));
- Assert.IsTrue(set.Add(c));
-
- CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { a, c });
-
- var d = new UInt256(new byte[32] {
- 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
- 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
- 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
- 0x01, 0x02
- });
+ var bucket = new HashSetCache<int>(10);
+ for (int i = 1; i <= 100; i++)
+ {
+ bucket.Add(i);
+ }
+ bucket.Count.Should().Be(100);
 
- // Testing Fifo max size
- Assert.IsTrue(set.Add(d));
- CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { a, c, d });
+ int sum = 0;
+ foreach (var ele in bucket)
+ {
+ sum += ele;
+ }
+ sum.Should().Be(5050);
 
- var e = new UInt256(new byte[32] {
- 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
- 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
- 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
- 0x01, 0x03
- });
+ bucket.Add(101);
+ bucket.Count.Should().Be(91);
 
- Assert.IsTrue(set.Add(e));
- Assert.IsFalse(set.Add(e));
- CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { c, d, e });
+ var items = new int[10];
+ var value = 11;
+ for (int i = 0; i < 10; i++)
+ {
+ items[i] = value;
+ value += 2;
+ }
+ bucket.ExceptWith(items);
+ bucket.Count.Should().Be(81);
+
+ bucket.Contains(13).Should().BeFalse();
+ bucket.Contains(50).Should().BeTrue();
  }
 
  [TestMethod]
  public void TestConstructor()
  {
- Action action1 = () => new FIFOSet<UInt256>(-1);
+ Action action1 = () => new HashSetCache<UInt256>(-1);
  action1.Should().Throw<ArgumentOutOfRangeException>();
 
- Action action2 = () => new FIFOSet<UInt256>(1, -1);
+ Action action2 = () => new HashSetCache<UInt256>(1, -1);
  action2.Should().Throw<ArgumentOutOfRangeException>();
 
- Action action3 = () => new FIFOSet<UInt256>(1, 2);
+ Action action3 = () => new HashSetCache<UInt256>(1, 21);
  action3.Should().Throw<ArgumentOutOfRangeException>();
  }
 
@@ -82,7 +72,7 @@ public void TestAdd()
  0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
  0x01, 0x02
  });
- var set = new FIFOSet<UInt256>(1, 1)
+ var set = new HashSetCache<UInt256>(1, 1)
  {
  a,
  b
@@ -105,7 +95,7 @@ public void TestGetEnumerator()
  0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
  0x01, 0x02
  });
- var set = new FIFOSet<UInt256>(1, 1)
+ var set = new HashSetCache<UInt256>(1, 1)
  {
  a,
  b
@@ -136,7 +126,7 @@ public void TestExceptWith()
  0x01, 0x03
  });
 
- var set = new FIFOSet<UInt256>(10)
+ var set = new HashSetCache<UInt256>(10)
  {
  a,
  b,
@@ -145,7 +135,7 @@ public void TestExceptWith()
  set.ExceptWith(new UInt256[] { b, c });
  CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { a });
 
- set = new FIFOSet<UInt256>(10)
+ set = new HashSetCache<UInt256>(10)
  {
  a,
  b,
@@ -154,7 +144,7 @@ public void TestExceptWith()
  set.ExceptWith(new UInt256[] { a });
  CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { b, c });
 
- set = new FIFOSet<UInt256>(10)
+ set = new HashSetCache<UInt256>(10)
  {
  a,
  b,