From 370693fea165f9b4d02560e4e9daa2cd3e082ec8 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 9 Aug 2024 18:52:20 +0200 Subject: [PATCH 01/38] first draft --- .../Data/BlobAndProofV1.cs | 24 +++++++ .../Data/GetBlobsV1Result.cs | 11 ++++ .../EngineRpcModule.Cancun.cs | 5 ++ .../EngineRpcModule.cs | 3 + .../Handlers/GetBlobsHandler.cs | 62 +++++++++++++++++++ .../IEngineRpcModule.Cancun.cs | 6 ++ .../Nethermind.Merge.Plugin/MergePlugin.cs | 2 + .../Collections/BlobTxDistinctSortedPool.cs | 47 ++++++++++++++ .../PersistentBlobTxDistinctSortedPool.cs | 4 ++ src/Nethermind/Nethermind.TxPool/ITxPool.cs | 2 + .../Nethermind.TxPool/LightTransaction.cs | 6 +- src/Nethermind/Nethermind.TxPool/TxPool.cs | 3 + 12 files changed, 170 insertions(+), 5 deletions(-) create mode 100644 src/Nethermind/Nethermind.Merge.Plugin/Data/BlobAndProofV1.cs create mode 100644 src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs create mode 100644 src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Data/BlobAndProofV1.cs b/src/Nethermind/Nethermind.Merge.Plugin/Data/BlobAndProofV1.cs new file mode 100644 index 00000000000..f6790d7688c --- /dev/null +++ b/src/Nethermind/Nethermind.Merge.Plugin/Data/BlobAndProofV1.cs @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + + +using System; +using Nethermind.Core; + +namespace Nethermind.Merge.Plugin.Data; + +public class BlobAndProofV1 +{ + public BlobAndProofV1(Transaction blobTx, int index) + { + if (blobTx is not { NetworkWrapper: ShardBlobNetworkWrapper wrapper }) + { + throw new ArgumentException("Shard blob transaction should contain network wrapper data"); + } + + Blob = wrapper.Blobs[index]; + Proof = wrapper.Proofs[index]; + } + private byte[] Blob { get; set; } + private byte[] Proof { get; set; } +} diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs b/src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs new file mode 100644 index 00000000000..a98ef421d87 --- /dev/null +++ b/src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; + +namespace Nethermind.Merge.Plugin.Data; + +public class GetBlobsV1Result(BlobAndProofV1?[] blobsAndProofs) +{ + public BlobAndProofV1?[] BlobsAndProofs = blobsAndProofs; +} diff --git a/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.Cancun.cs b/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.Cancun.cs index 4a28ae4ac0d..fb70e4195f7 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.Cancun.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.Cancun.cs @@ -14,6 +14,8 @@ namespace Nethermind.Merge.Plugin; public partial class EngineRpcModule : IEngineRpcModule { private readonly IAsyncHandler _getPayloadHandlerV3; + private readonly IAsyncHandler _getBlobsHandler; + public Task> engine_forkchoiceUpdatedV3(ForkchoiceStateV1 forkchoiceState, PayloadAttributes? payloadAttributes = null) => ForkchoiceUpdated(forkchoiceState, payloadAttributes, EngineApiVersions.Cancun); @@ -23,4 +25,7 @@ public Task> engine_newPayloadV3(ExecutionPayload public async Task> engine_getPayloadV3(byte[] payloadId) => await _getPayloadHandlerV3.HandleAsync(payloadId); + + public async Task> engine_getBlobsV1(byte[][] blobVersionedHashes) => + await _getBlobsHandler.HandleAsync(blobVersionedHashes); } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.cs b/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.cs index 24aaf85a38b..329b0e996fc 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.cs @@ -30,11 +30,13 @@ public EngineRpcModule( IGetPayloadBodiesByRangeV1Handler executionGetPayloadBodiesByRangeV1Handler, IHandler transitionConfigurationHandler, IHandler, IEnumerable> capabilitiesHandler, + IAsyncHandler getBlobsHandler, ISpecProvider specProvider, GCKeeper gcKeeper, ILogManager logManager) { _capabilitiesHandler = capabilitiesHandler ?? throw new ArgumentNullException(nameof(capabilitiesHandler)); + _getBlobsHandler = getBlobsHandler; _getPayloadHandlerV1 = getPayloadHandlerV1; _getPayloadHandlerV2 = getPayloadHandlerV2; _getPayloadHandlerV3 = getPayloadHandlerV3; @@ -43,6 +45,7 @@ public EngineRpcModule( _executionGetPayloadBodiesByHashV1Handler = executionGetPayloadBodiesByHashV1Handler; _executionGetPayloadBodiesByRangeV1Handler = executionGetPayloadBodiesByRangeV1Handler; _transitionConfigurationHandler = transitionConfigurationHandler; + _getBlobsHandler = getBlobsHandler; _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); _gcKeeper = gcKeeper; _logger = logManager.GetClassLogger(); diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs new file mode 100644 index 00000000000..b87a48ec7b1 --- /dev/null +++ b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Nethermind.Core; +using Nethermind.Core.Collections; +using Nethermind.Core.Crypto; +using Nethermind.JsonRpc; +using Nethermind.Merge.Plugin.Data; +using Nethermind.TxPool; + +namespace Nethermind.Merge.Plugin.Handlers; + +public class GetBlobsHandler(ITxPool txPool) : IAsyncHandler +{ + private const int MaxRequest = 128; + + private readonly ConcurrentDictionary> _blobIndex = txPool.GetBlobIndex(); + + public Task> HandleAsync(byte[][] request) + { + if (request.Length > MaxRequest) + { + var error = $"The number of requested blobs must not exceed {MaxRequest}"; + return ResultWrapper.Fail(error, MergeErrorCodes.TooLargeRequest); + } + + ArrayPoolList blobsAndProofs = new(request.Length); + + foreach (byte[] requestedBlobVersionedHash in request) + { + bool isBlobFound = false; + if (_blobIndex.TryGetValue(requestedBlobVersionedHash, out List? txHashes) + && txPool.TryGetPendingBlobTransaction(txHashes.First(), out Transaction? blobTx) + && blobTx?.BlobVersionedHashes?.Length > 0) + { + int indexOfRequestedBlob = 0; + foreach (var txBlobVersionedHash in blobTx.BlobVersionedHashes) + { + if (txBlobVersionedHash == requestedBlobVersionedHash) + { + isBlobFound = true; + blobsAndProofs.Add(new BlobAndProofV1(blobTx, indexOfRequestedBlob)); + break; + } + + indexOfRequestedBlob++; + } + } + + if (!isBlobFound) + { + blobsAndProofs.Add(null); + } + } + + return ResultWrapper.Success(new GetBlobsV1Result(blobsAndProofs.ToArray())); + } +} diff --git a/src/Nethermind/Nethermind.Merge.Plugin/IEngineRpcModule.Cancun.cs b/src/Nethermind/Nethermind.Merge.Plugin/IEngineRpcModule.Cancun.cs index 1510791e28b..552956057da 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/IEngineRpcModule.Cancun.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/IEngineRpcModule.Cancun.cs @@ -30,4 +30,10 @@ public partial interface IEngineRpcModule : IRpcModule IsSharable = true, IsImplemented = true)] public Task> engine_getPayloadV3(byte[] payloadId); + + [JsonRpcMethod( + Description = "Returns requested blobs and proofs.", + IsSharable = true, + IsImplemented = true)] + public Task> engine_getBlobsV1(byte[][] blobVersionedHashes); } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index 6bcc5eb7990..2adf42baa54 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -280,6 +280,7 @@ public Task InitRpcModules() if (_api.Sealer is null) throw new ArgumentNullException(nameof(_api.Sealer)); if (_api.BlockValidator is null) throw new ArgumentNullException(nameof(_api.BlockValidator)); if (_api.BlockProcessingQueue is null) throw new ArgumentNullException(nameof(_api.BlockProcessingQueue)); + if (_api.TxPool is null) throw new ArgumentNullException(nameof(_api.TxPool)); if (_api.SpecProvider is null) throw new ArgumentNullException(nameof(_api.SpecProvider)); if (_api.StateReader is null) throw new ArgumentNullException(nameof(_api.StateReader)); if (_beaconPivot is null) throw new ArgumentNullException(nameof(_beaconPivot)); @@ -354,6 +355,7 @@ public Task InitRpcModules() new GetPayloadBodiesByRangeV1Handler(_api.BlockTree, _api.LogManager), new ExchangeTransitionConfigurationV1Handler(_poSSwitcher, _api.LogManager), new ExchangeCapabilitiesHandler(_api.RpcCapabilitiesProvider, _api.LogManager), + new GetBlobsHandler(_api.TxPool), _api.SpecProvider, new GCKeeper(new NoSyncGcRegionStrategy(_api.SyncModeSelector, _mergeConfig), _api.LogManager), _api.LogManager); diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index 807a03fe519..c2e696d116c 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Collections.Concurrent; using System.Collections.Generic; using Nethermind.Core; using Nethermind.Core.Crypto; @@ -11,11 +12,57 @@ namespace Nethermind.TxPool.Collections; public class BlobTxDistinctSortedPool(int capacity, IComparer comparer, ILogManager logManager) : TxDistinctSortedPool(capacity, comparer, logManager) { + public ConcurrentDictionary> GetBlobIndex => BlobIndex; + + protected readonly ConcurrentDictionary> BlobIndex = new(); + protected override IComparer GetReplacementComparer(IComparer comparer) => comparer.GetBlobReplacementComparer(); protected override string ShortPoolName => "BlobPool"; + protected void AddToBlobIndex(Transaction blobTx) + { + if (blobTx.BlobVersionedHashes?.Length > 0) + { + foreach (var blobVersionedHash in blobTx.BlobVersionedHashes) + { + if (blobVersionedHash?.Length == 32) + { + BlobIndex.AddOrUpdate(blobVersionedHash, + k => [blobTx.Hash!], + (k, b) => + { + b.Add(blobTx.Hash!); + return b; + }); + } + } + } + } + + protected void RemoveFromBlobIndex(Transaction blobTx) + { + if (blobTx.BlobVersionedHashes?.Length > 0) + { + foreach (var blobVersionedHash in blobTx.BlobVersionedHashes) + { + if (blobVersionedHash is not null + && BlobIndex.TryGetValue(blobVersionedHash, out List? txHashes)) + { + if (txHashes.Count < 2) + { + BlobIndex.Remove(blobVersionedHash, out _); + } + else + { + txHashes.Remove(blobTx.Hash!); + } + } + } + } + } + /// /// For tests only - to test sorting /// diff --git a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs index 593266a0de6..52ea6a41bdc 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs @@ -38,6 +38,7 @@ private void RecreateLightTxCollectionAndCache(ITxStorage blobTxStorage) { if (base.TryInsert(lightBlobTx.Hash, lightBlobTx, out _)) { + AddToBlobIndex(lightBlobTx); numberOfTxsInDb++; numberOfBlobsInDb += lightBlobTx.BlobVersionedHashes?.Length ?? 0; } @@ -47,6 +48,7 @@ private void RecreateLightTxCollectionAndCache(ITxStorage blobTxStorage) { long loadingTime = stopwatch.ElapsedMilliseconds; _logger.Info($"Loaded {numberOfTxsInDb} blob txs from persistent db, containing {numberOfBlobsInDb} blobs, in {loadingTime}ms"); + _logger.Info($"There are {BlobIndex.Count} unique blobs indexed"); } stopwatch.Stop(); } @@ -57,6 +59,7 @@ public override bool TryInsert(ValueHash256 hash, Transaction fullBlobTx, out Tr { _blobTxCache.Set(fullBlobTx.Hash, fullBlobTx); _blobTxStorage.Add(fullBlobTx); + AddToBlobIndex(fullBlobTx); return true; } @@ -95,6 +98,7 @@ protected override bool Remove(ValueHash256 hash, out Transaction? tx) if (tx is not null) { _blobTxStorage.Delete(hash, tx.Timestamp); + RemoveFromBlobIndex(tx); } _blobTxCache.Delete(hash); diff --git a/src/Nethermind/Nethermind.TxPool/ITxPool.cs b/src/Nethermind/Nethermind.TxPool/ITxPool.cs index e6ba750b865..65b5dad734f 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPool.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using Nethermind.Core; @@ -15,6 +16,7 @@ public interface ITxPool int GetPendingTransactionsCount(); int GetPendingBlobTransactionsCount(); Transaction[] GetPendingTransactions(); + ConcurrentDictionary> GetBlobIndex(); /// /// Non-blob txs grouped by sender address, sorted by nonce and later tx pool sorting diff --git a/src/Nethermind/Nethermind.TxPool/LightTransaction.cs b/src/Nethermind/Nethermind.TxPool/LightTransaction.cs index 217224f21a6..0ce61b41ec1 100644 --- a/src/Nethermind/Nethermind.TxPool/LightTransaction.cs +++ b/src/Nethermind/Nethermind.TxPool/LightTransaction.cs @@ -15,10 +15,6 @@ namespace Nethermind.TxPool; /// public class LightTransaction : Transaction { - private static readonly Dictionary _blobVersionedHashesCache = - Enumerable.Range(1, Eip4844Constants.GetMaxBlobsPerBlock()).ToDictionary(i => i, i => new byte[i][]); - - public LightTransaction(Transaction fullTx) { Type = TxType.Blob; @@ -30,7 +26,7 @@ public LightTransaction(Transaction fullTx) GasPrice = fullTx.GasPrice; // means MaxPriorityFeePerGas DecodedMaxFeePerGas = fullTx.DecodedMaxFeePerGas; MaxFeePerBlobGas = fullTx.MaxFeePerBlobGas; - BlobVersionedHashes = _blobVersionedHashesCache[fullTx.BlobVersionedHashes!.Length]; + BlobVersionedHashes = fullTx.BlobVersionedHashes; GasBottleneck = fullTx.GasBottleneck; Timestamp = fullTx.Timestamp; PoolIndex = fullTx.PoolIndex; diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 7b78f8d32af..e5cfef0b42f 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; @@ -182,6 +183,8 @@ public Transaction[] GetPendingTransactionsBySender(Address address) => public int GetPendingBlobTransactionsCount() => _blobTransactions.Count; + public ConcurrentDictionary> GetBlobIndex() => _blobTransactions.GetBlobIndex; + private void OnHeadChange(object? sender, BlockReplacementEventArgs e) { try From 4599fe49d00147d22d5dbef4f59ed97057f2bd87 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 12 Aug 2024 09:25:31 +0200 Subject: [PATCH 02/38] improve handler --- .../Handlers/GetBlobsHandler.cs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs index b87a48ec7b1..b86d52c3dc4 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs @@ -35,19 +35,16 @@ public Task> HandleAsync(byte[][] request) bool isBlobFound = false; if (_blobIndex.TryGetValue(requestedBlobVersionedHash, out List? txHashes) && txPool.TryGetPendingBlobTransaction(txHashes.First(), out Transaction? blobTx) - && blobTx?.BlobVersionedHashes?.Length > 0) + && blobTx.BlobVersionedHashes?.Length > 0) { - int indexOfRequestedBlob = 0; - foreach (var txBlobVersionedHash in blobTx.BlobVersionedHashes) + for (int indexOfBlob = 0; indexOfBlob < blobTx.BlobVersionedHashes.Length; indexOfBlob++) { - if (txBlobVersionedHash == requestedBlobVersionedHash) + if (blobTx.BlobVersionedHashes[indexOfBlob] == requestedBlobVersionedHash) { isBlobFound = true; - blobsAndProofs.Add(new BlobAndProofV1(blobTx, indexOfRequestedBlob)); + blobsAndProofs.Add(new BlobAndProofV1(blobTx, indexOfBlob)); break; } - - indexOfRequestedBlob++; } } From 1380390eadd76ef1179d2ec5edd903e8f6422a96 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 12 Aug 2024 09:30:14 +0200 Subject: [PATCH 03/38] adjust other classes to engine module changes --- .../Nethermind.Merge.Plugin.Test/EngineModuleTests.Setup.cs | 1 + .../Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs | 1 + src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs | 2 ++ .../Nethermind.Optimism/OptimismReceiptStorageDecoder.cs | 2 +- src/Nethermind/Nethermind.TxPool/NullTxPool.cs | 5 ++--- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Setup.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Setup.cs index c499adefe37..610c9999981 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Setup.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Setup.cs @@ -128,6 +128,7 @@ private IEngineRpcModule CreateEngineModule(MergeTestBlockchain chain, ISyncConf new GetPayloadBodiesByRangeV1Handler(chain.BlockTree, chain.LogManager), new ExchangeTransitionConfigurationV1Handler(chain.PoSSwitcher, chain.LogManager), new ExchangeCapabilitiesHandler(capabilitiesProvider, chain.LogManager), + new GetBlobsHandler(chain.TxPool), chain.SpecProvider, new GCKeeper(NoGCStrategy.Instance, chain.LogManager), chain.LogManager); diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs index ebc6b40d7c2..82faf4e1b30 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs @@ -361,6 +361,7 @@ public async Task NewPayloadV3_should_verify_blob_versioned_hashes_again Substitute.For(), Substitute.For>(), Substitute.For, IEnumerable>>(), + Substitute.For>(), chain.SpecProvider, new GCKeeper(NoGCStrategy.Instance, chain.LogManager), Substitute.For())); diff --git a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs index b57df364474..9659aee08c2 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs @@ -202,6 +202,7 @@ public async Task InitRpcModules() ArgumentNullException.ThrowIfNull(_api.BlockValidator); ArgumentNullException.ThrowIfNull(_api.RpcModuleProvider); ArgumentNullException.ThrowIfNull(_api.BlockProducer); + ArgumentNullException.ThrowIfNull(_api.TxPool); ArgumentNullException.ThrowIfNull(_beaconSync); ArgumentNullException.ThrowIfNull(_beaconPivot); @@ -268,6 +269,7 @@ public async Task InitRpcModules() new GetPayloadBodiesByRangeV1Handler(_api.BlockTree, _api.LogManager), new ExchangeTransitionConfigurationV1Handler(_api.PoSSwitcher, _api.LogManager), new ExchangeCapabilitiesHandler(_api.RpcCapabilitiesProvider, _api.LogManager), + new GetBlobsHandler(_api.TxPool), _api.SpecProvider, new GCKeeper( initConfig.DisableGcOnNewPayload diff --git a/src/Nethermind/Nethermind.Optimism/OptimismReceiptStorageDecoder.cs b/src/Nethermind/Nethermind.Optimism/OptimismReceiptStorageDecoder.cs index 38d3a9071b0..d3e95e82ef1 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismReceiptStorageDecoder.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismReceiptStorageDecoder.cs @@ -311,7 +311,7 @@ public int GetLength(OptimismTxReceipt item, RlpBehaviors rlpBehaviors) TxReceipt IRlpStreamDecoder.Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors) { - return Decode(rlpStream, rlpBehaviors); ; + return Decode(rlpStream, rlpBehaviors); } public void Encode(RlpStream stream, TxReceipt item, RlpBehaviors rlpBehaviors = RlpBehaviors.None) diff --git a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs index 3df7080af87..d23cbd6fbe2 100644 --- a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using Nethermind.Core; @@ -19,7 +20,7 @@ private NullTxPool() { } public int GetPendingTransactionsCount() => 0; public int GetPendingBlobTransactionsCount() => 0; public Transaction[] GetPendingTransactions() => Array.Empty(); - + public ConcurrentDictionary> GetBlobIndex() => new(); public Transaction[] GetPendingTransactionsBySender(Address address) => Array.Empty(); public IDictionary GetPendingTransactionsBySender() @@ -28,8 +29,6 @@ public IDictionary GetPendingTransactionsBySender() public IDictionary GetPendingLightBlobTransactionsBySender() => new Dictionary(); - public static IEnumerable GetPendingBlobTransactions() => Array.Empty(); - public void AddPeer(ITxPoolPeer peer) { } public void RemovePeer(PublicKey nodeId) { } From 472513ae7fd2d43fc4a1f3f6836f5ff988df25f2 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 12 Aug 2024 09:33:40 +0200 Subject: [PATCH 04/38] cosmetics --- src/Nethermind/Nethermind.TxPool/NullTxPool.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs index d23cbd6fbe2..f3d34cf2709 100644 --- a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs @@ -20,7 +20,7 @@ private NullTxPool() { } public int GetPendingTransactionsCount() => 0; public int GetPendingBlobTransactionsCount() => 0; public Transaction[] GetPendingTransactions() => Array.Empty(); - public ConcurrentDictionary> GetBlobIndex() => new(); + public Transaction[] GetPendingTransactionsBySender(Address address) => Array.Empty(); public IDictionary GetPendingTransactionsBySender() @@ -53,6 +53,8 @@ public bool TryGetPendingBlobTransaction(Hash256 hash, [NotNullWhen(true)] out T return false; } + public ConcurrentDictionary> GetBlobIndex() => new(); + public UInt256 GetLatestPendingNonce(Address address) => 0; From b2b0ec051d145f7b51eb60f2264f1539dbf18ce9 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 12 Aug 2024 13:09:13 +0200 Subject: [PATCH 05/38] refactor to allow indexing blobs in InMemory mode --- .../Data/BlobAndProofV1.cs | 4 ++-- .../Collections/BlobTxDistinctSortedPool.cs | 24 +++++++++++++++++++ .../PersistentBlobTxDistinctSortedPool.cs | 2 -- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Data/BlobAndProofV1.cs b/src/Nethermind/Nethermind.Merge.Plugin/Data/BlobAndProofV1.cs index f6790d7688c..1785442043d 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Data/BlobAndProofV1.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Data/BlobAndProofV1.cs @@ -19,6 +19,6 @@ public BlobAndProofV1(Transaction blobTx, int index) Blob = wrapper.Blobs[index]; Proof = wrapper.Proofs[index]; } - private byte[] Blob { get; set; } - private byte[] Proof { get; set; } + public byte[] Blob { get; set; } + public byte[] Proof { get; set; } } diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index c2e696d116c..b73db8c5556 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -21,6 +21,16 @@ protected override IComparer GetReplacementComparer(IComparer "BlobPool"; + public override bool TryInsert(ValueHash256 hash, Transaction blobTx, out Transaction? removed) + { + if (base.TryInsert(blobTx.Hash, blobTx, out removed)) + { + AddToBlobIndex(blobTx); + return true; + } + + return false; + } protected void AddToBlobIndex(Transaction blobTx) { if (blobTx.BlobVersionedHashes?.Length > 0) @@ -41,6 +51,20 @@ protected void AddToBlobIndex(Transaction blobTx) } } + protected override bool Remove(ValueHash256 hash, out Transaction? tx) + { + if (base.Remove(hash, out tx)) + { + if (tx is not null) + { + RemoveFromBlobIndex(tx); + } + return true; + } + + return false; + } + protected void RemoveFromBlobIndex(Transaction blobTx) { if (blobTx.BlobVersionedHashes?.Length > 0) diff --git a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs index 52ea6a41bdc..d0f7eb34d92 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs @@ -59,7 +59,6 @@ public override bool TryInsert(ValueHash256 hash, Transaction fullBlobTx, out Tr { _blobTxCache.Set(fullBlobTx.Hash, fullBlobTx); _blobTxStorage.Add(fullBlobTx); - AddToBlobIndex(fullBlobTx); return true; } @@ -98,7 +97,6 @@ protected override bool Remove(ValueHash256 hash, out Transaction? tx) if (tx is not null) { _blobTxStorage.Delete(hash, tx.Timestamp); - RemoveFromBlobIndex(tx); } _blobTxCache.Delete(hash); From 0ec36a94c7bacb8269d56a06c3886ed12fd9d3e0 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 12 Aug 2024 13:09:22 +0200 Subject: [PATCH 06/38] add test --- .../EngineModuleTests.V3.cs | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs index 82faf4e1b30..616b9c05049 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs @@ -27,6 +27,7 @@ using Nethermind.Serialization.Json; using Nethermind.Serialization.Rlp; using Nethermind.Specs.Forks; +using Nethermind.TxPool; using NSubstitute; using NUnit.Framework; @@ -509,6 +510,43 @@ static void MarkAsUnprocessed(MergeTestBlockchain chain, int blockNumber) Assert.That(res2.Data.PayloadStatus.Status, Is.EqualTo(PayloadStatus.Valid)); } + [Test] + public async Task GetBlobsV1_should_return_requested_blobs([Values(1, 2, 3, 4, 5, 6)] int numberOfBlobs) + { + MergeTestBlockchain chain = await CreateBlockchain(releaseSpec: Cancun.Instance); + IEngineRpcModule rpcModule = CreateEngineModule(chain, null, TimeSpan.FromDays(1)); + var ethereumEcdsa = new EthereumEcdsa(BlockchainIds.Mainnet, LimboLogs.Instance); + + Transaction blobTx = Build.A.Transaction + .WithShardBlobTxTypeAndFields(numberOfBlobs) + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) + .WithMaxFeePerBlobGas(1000.Wei()) + .SignedAndResolved(ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + chain.TxPool.SubmitTx(blobTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + + List blobsAndProofs = new(); + + for (int i = 0; i < blobTx.BlobVersionedHashes!.Length; i++) + { + blobsAndProofs.Add(new BlobAndProofV1(blobTx, i)); + } + + GetBlobsV1Result expected = new(blobsAndProofs.ToArray()); + + ResultWrapper result = await rpcModule.engine_getBlobsV1(blobTx.BlobVersionedHashes!); + + result.Data.Should().BeEquivalentTo(expected); + result.Data.BlobsAndProofs.Length.Should().Be(numberOfBlobs); + for (int i = 0; i < result.Data.BlobsAndProofs.Length; i++) + { + result.Data.BlobsAndProofs[i]!.Blob.Should().BeEquivalentTo(((ShardBlobNetworkWrapper)blobTx.NetworkWrapper!).Blobs[i]); + result.Data.BlobsAndProofs[i]!.Proof.Should().BeEquivalentTo(((ShardBlobNetworkWrapper)blobTx.NetworkWrapper!).Proofs[i]); + } + } + + public static IEnumerable ForkchoiceUpdatedV3DeclinedTestCaseSource { get From 31424d27a368fa0c37b551b191e99d45c883f45d Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 12 Aug 2024 13:12:09 +0200 Subject: [PATCH 07/38] fix file encodings --- src/Nethermind/Nethermind.Merge.Plugin/Data/BlobAndProofV1.cs | 2 +- src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs | 2 +- .../Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Data/BlobAndProofV1.cs b/src/Nethermind/Nethermind.Merge.Plugin/Data/BlobAndProofV1.cs index 1785442043d..6dd84dfd6dd 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Data/BlobAndProofV1.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Data/BlobAndProofV1.cs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs b/src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs index a98ef421d87..938b59a1c05 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only using System.Collections.Generic; diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs index b86d52c3dc4..f3a22ef014c 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only using System.Collections.Concurrent; From 09850c43b2b33842c168c5c592eae7ca1a4dccf7 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 12 Aug 2024 13:12:32 +0200 Subject: [PATCH 08/38] cosmetics --- src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs b/src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs index 938b59a1c05..543f94c25f3 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs @@ -7,5 +7,5 @@ namespace Nethermind.Merge.Plugin.Data; public class GetBlobsV1Result(BlobAndProofV1?[] blobsAndProofs) { - public BlobAndProofV1?[] BlobsAndProofs = blobsAndProofs; + public readonly BlobAndProofV1?[] BlobsAndProofs = blobsAndProofs; } From 4447ccba0697aae1f10f9d0927bd84bbb467fad0 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 12 Aug 2024 14:09:43 +0200 Subject: [PATCH 09/38] fix build? --- .../Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs index 616b9c05049..9f66fec5fe6 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs @@ -515,7 +515,7 @@ public async Task GetBlobsV1_should_return_requested_blobs([Values(1, 2, 3, 4, 5 { MergeTestBlockchain chain = await CreateBlockchain(releaseSpec: Cancun.Instance); IEngineRpcModule rpcModule = CreateEngineModule(chain, null, TimeSpan.FromDays(1)); - var ethereumEcdsa = new EthereumEcdsa(BlockchainIds.Mainnet, LimboLogs.Instance); + EthereumEcdsa ethereumEcdsa = new((ulong)BlockchainIds.Mainnet, LimboLogs.Instance); Transaction blobTx = Build.A.Transaction .WithShardBlobTxTypeAndFields(numberOfBlobs) From 33404d9e8af03c46e376d5264402b5a8854cb9e6 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 12 Aug 2024 14:16:40 +0200 Subject: [PATCH 10/38] fix build? --- .../Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs index 9f66fec5fe6..de953be564a 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs @@ -515,14 +515,13 @@ public async Task GetBlobsV1_should_return_requested_blobs([Values(1, 2, 3, 4, 5 { MergeTestBlockchain chain = await CreateBlockchain(releaseSpec: Cancun.Instance); IEngineRpcModule rpcModule = CreateEngineModule(chain, null, TimeSpan.FromDays(1)); - EthereumEcdsa ethereumEcdsa = new((ulong)BlockchainIds.Mainnet, LimboLogs.Instance); Transaction blobTx = Build.A.Transaction .WithShardBlobTxTypeAndFields(numberOfBlobs) .WithMaxFeePerGas(1.GWei()) .WithMaxPriorityFeePerGas(1.GWei()) .WithMaxFeePerBlobGas(1000.Wei()) - .SignedAndResolved(ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + .SignedAndResolved(chain.EthereumEcdsa, TestItem.PrivateKeyA).TestObject; chain.TxPool.SubmitTx(blobTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); From fd4fd374807117015e2a6612ad4049c4645d7aeb Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 12 Aug 2024 14:26:09 +0200 Subject: [PATCH 11/38] add engine capability --- .../Handlers/EngineRpcCapabilitiesProvider.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/EngineRpcCapabilitiesProvider.cs b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/EngineRpcCapabilitiesProvider.cs index 729abe4ce86..dd87a1ab60c 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/EngineRpcCapabilitiesProvider.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/EngineRpcCapabilitiesProvider.cs @@ -45,6 +45,7 @@ public EngineRpcCapabilitiesProvider(ISpecProvider specProvider) _capabilities[nameof(IEngineRpcModule.engine_getPayloadV3)] = (spec.IsEip4844Enabled, spec.IsEip4844Enabled); _capabilities[nameof(IEngineRpcModule.engine_forkchoiceUpdatedV3)] = (spec.IsEip4844Enabled, spec.IsEip4844Enabled); _capabilities[nameof(IEngineRpcModule.engine_newPayloadV3)] = (spec.IsEip4844Enabled, spec.IsEip4844Enabled); + _capabilities[nameof(IEngineRpcModule.engine_getBlobsV1)] = (spec.IsEip4844Enabled, spec.IsEip4844Enabled); #endregion } From fbd031c3736ab77f9e82a37b8d5ee5fa4852d021 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 12 Aug 2024 14:30:38 +0200 Subject: [PATCH 12/38] add engine_getBlobsV1 as expected cap for mainnet --- .../Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs index 9f3f8db4439..9f75c4d6fd8 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs @@ -1569,7 +1569,8 @@ public void Should_return_expected_capabilities_for_mainnet() nameof(IEngineRpcModule.engine_getPayloadV3), nameof(IEngineRpcModule.engine_forkchoiceUpdatedV3), - nameof(IEngineRpcModule.engine_newPayloadV3) + nameof(IEngineRpcModule.engine_newPayloadV3), + nameof(IEngineRpcModule.engine_getBlobsV1) }; Assert.That(result, Is.EquivalentTo(expectedMethods)); } From c9ff2fc273b143ad839ba327aebaece5e40087f2 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 12 Aug 2024 15:13:50 +0200 Subject: [PATCH 13/38] cosmetics --- .../Nethermind.Merge.Plugin/EngineRpcModule.Cancun.cs | 1 - .../Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.Cancun.cs b/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.Cancun.cs index fb70e4195f7..5392161337b 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.Cancun.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.Cancun.cs @@ -16,7 +16,6 @@ public partial class EngineRpcModule : IEngineRpcModule private readonly IAsyncHandler _getPayloadHandlerV3; private readonly IAsyncHandler _getBlobsHandler; - public Task> engine_forkchoiceUpdatedV3(ForkchoiceStateV1 forkchoiceState, PayloadAttributes? payloadAttributes = null) => ForkchoiceUpdated(forkchoiceState, payloadAttributes, EngineApiVersions.Cancun); diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index b73db8c5556..f2fc4ecb923 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -12,6 +12,8 @@ namespace Nethermind.TxPool.Collections; public class BlobTxDistinctSortedPool(int capacity, IComparer comparer, ILogManager logManager) : TxDistinctSortedPool(capacity, comparer, logManager) { + protected override string ShortPoolName => "BlobPool"; + public ConcurrentDictionary> GetBlobIndex => BlobIndex; protected readonly ConcurrentDictionary> BlobIndex = new(); @@ -19,8 +21,6 @@ public class BlobTxDistinctSortedPool(int capacity, IComparer compa protected override IComparer GetReplacementComparer(IComparer comparer) => comparer.GetBlobReplacementComparer(); - protected override string ShortPoolName => "BlobPool"; - public override bool TryInsert(ValueHash256 hash, Transaction blobTx, out Transaction? removed) { if (base.TryInsert(blobTx.Hash, blobTx, out removed)) From 457029853ea1e6dd7edb7cb9511cfe9fdaff0c51 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 13 Aug 2024 11:54:16 +0200 Subject: [PATCH 14/38] add test --- .../EngineModuleTests.V3.cs | 39 ++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs index de953be564a..9d56e536f03 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs @@ -525,26 +525,55 @@ public async Task GetBlobsV1_should_return_requested_blobs([Values(1, 2, 3, 4, 5 chain.TxPool.SubmitTx(blobTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); - List blobsAndProofs = new(); - - for (int i = 0; i < blobTx.BlobVersionedHashes!.Length; i++) + List blobsAndProofs = new(numberOfBlobs); + for (int i = 0; i < numberOfBlobs; i++) { blobsAndProofs.Add(new BlobAndProofV1(blobTx, i)); } - GetBlobsV1Result expected = new(blobsAndProofs.ToArray()); ResultWrapper result = await rpcModule.engine_getBlobsV1(blobTx.BlobVersionedHashes!); result.Data.Should().BeEquivalentTo(expected); result.Data.BlobsAndProofs.Length.Should().Be(numberOfBlobs); - for (int i = 0; i < result.Data.BlobsAndProofs.Length; i++) + for (int i = 0; i < numberOfBlobs; i++) { result.Data.BlobsAndProofs[i]!.Blob.Should().BeEquivalentTo(((ShardBlobNetworkWrapper)blobTx.NetworkWrapper!).Blobs[i]); result.Data.BlobsAndProofs[i]!.Proof.Should().BeEquivalentTo(((ShardBlobNetworkWrapper)blobTx.NetworkWrapper!).Proofs[i]); } } + [Test] + public async Task GetBlobsV1_should_return_nulls_when_blobs_not_found([Values(1, 2, 3, 4, 5, 6)] int numberOfRequestedBlobs) + { + MergeTestBlockchain chain = await CreateBlockchain(releaseSpec: Cancun.Instance); + IEngineRpcModule rpcModule = CreateEngineModule(chain, null, TimeSpan.FromDays(1)); + + // we are not adding this tx + Transaction blobTx = Build.A.Transaction + .WithShardBlobTxTypeAndFields(numberOfRequestedBlobs) + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) + .WithMaxFeePerBlobGas(1000.Wei()) + .SignedAndResolved(chain.EthereumEcdsa, TestItem.PrivateKeyA).TestObject; + + List blobsAndProofs = new(numberOfRequestedBlobs); + for (int i = 0; i < numberOfRequestedBlobs; i++) + { + blobsAndProofs.Add(null); + } + GetBlobsV1Result expected = new(blobsAndProofs.ToArray()); + + // requesting hashes that are not present in TxPool + ResultWrapper result = await rpcModule.engine_getBlobsV1(blobTx.BlobVersionedHashes!); + + result.Data.Should().BeEquivalentTo(expected); + result.Data.BlobsAndProofs.Length.Should().Be(numberOfRequestedBlobs); + for (int i = 0; i < numberOfRequestedBlobs; i++) + { + result.Data.BlobsAndProofs[i]!.Should().BeNull(); + } + } public static IEnumerable ForkchoiceUpdatedV3DeclinedTestCaseSource { From 9caf0d46232f6f32863b2770976896316baa9ede Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 14 Aug 2024 11:44:42 +0200 Subject: [PATCH 15/38] refactor index to have string as a key --- .../Handlers/GetBlobsHandler.cs | 5 +++-- .../Collections/BlobTxDistinctSortedPool.cs | 16 +++++++++------- src/Nethermind/Nethermind.TxPool/ITxPool.cs | 2 +- src/Nethermind/Nethermind.TxPool/NullTxPool.cs | 2 +- src/Nethermind/Nethermind.TxPool/TxPool.cs | 2 +- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs index f3a22ef014c..5d3c269a066 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs @@ -8,6 +8,7 @@ using Nethermind.Core; using Nethermind.Core.Collections; using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; using Nethermind.JsonRpc; using Nethermind.Merge.Plugin.Data; using Nethermind.TxPool; @@ -18,7 +19,7 @@ public class GetBlobsHandler(ITxPool txPool) : IAsyncHandler> _blobIndex = txPool.GetBlobIndex(); + private readonly ConcurrentDictionary> _blobIndex = txPool.GetBlobIndex(); public Task> HandleAsync(byte[][] request) { @@ -33,7 +34,7 @@ public Task> HandleAsync(byte[][] request) foreach (byte[] requestedBlobVersionedHash in request) { bool isBlobFound = false; - if (_blobIndex.TryGetValue(requestedBlobVersionedHash, out List? txHashes) + if (_blobIndex.TryGetValue(requestedBlobVersionedHash.ToHexString(), out List? txHashes) && txPool.TryGetPendingBlobTransaction(txHashes.First(), out Transaction? blobTx) && blobTx.BlobVersionedHashes?.Length > 0) { diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index f2fc4ecb923..1c7b4db52aa 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; using Nethermind.Logging; namespace Nethermind.TxPool.Collections; @@ -14,9 +15,9 @@ public class BlobTxDistinctSortedPool(int capacity, IComparer compa { protected override string ShortPoolName => "BlobPool"; - public ConcurrentDictionary> GetBlobIndex => BlobIndex; + public ConcurrentDictionary> GetBlobIndex => BlobIndex; - protected readonly ConcurrentDictionary> BlobIndex = new(); + protected readonly ConcurrentDictionary> BlobIndex = new(); protected override IComparer GetReplacementComparer(IComparer comparer) => comparer.GetBlobReplacementComparer(); @@ -31,6 +32,7 @@ public override bool TryInsert(ValueHash256 hash, Transaction blobTx, out Transa return false; } + protected void AddToBlobIndex(Transaction blobTx) { if (blobTx.BlobVersionedHashes?.Length > 0) @@ -39,7 +41,7 @@ protected void AddToBlobIndex(Transaction blobTx) { if (blobVersionedHash?.Length == 32) { - BlobIndex.AddOrUpdate(blobVersionedHash, + BlobIndex.AddOrUpdate(blobVersionedHash.ToHexString(), k => [blobTx.Hash!], (k, b) => { @@ -65,18 +67,18 @@ protected override bool Remove(ValueHash256 hash, out Transaction? tx) return false; } - protected void RemoveFromBlobIndex(Transaction blobTx) + private void RemoveFromBlobIndex(Transaction blobTx) { if (blobTx.BlobVersionedHashes?.Length > 0) { foreach (var blobVersionedHash in blobTx.BlobVersionedHashes) { - if (blobVersionedHash is not null - && BlobIndex.TryGetValue(blobVersionedHash, out List? txHashes)) + if (blobVersionedHash?.ToHexString() is { } hexString + && BlobIndex.TryGetValue(hexString, out List? txHashes)) { if (txHashes.Count < 2) { - BlobIndex.Remove(blobVersionedHash, out _); + BlobIndex.Remove(hexString, out _); } else { diff --git a/src/Nethermind/Nethermind.TxPool/ITxPool.cs b/src/Nethermind/Nethermind.TxPool/ITxPool.cs index 65b5dad734f..1428ab438d9 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPool.cs @@ -16,7 +16,7 @@ public interface ITxPool int GetPendingTransactionsCount(); int GetPendingBlobTransactionsCount(); Transaction[] GetPendingTransactions(); - ConcurrentDictionary> GetBlobIndex(); + ConcurrentDictionary> GetBlobIndex(); /// /// Non-blob txs grouped by sender address, sorted by nonce and later tx pool sorting diff --git a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs index f3d34cf2709..30393f9b1cc 100644 --- a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs @@ -53,7 +53,7 @@ public bool TryGetPendingBlobTransaction(Hash256 hash, [NotNullWhen(true)] out T return false; } - public ConcurrentDictionary> GetBlobIndex() => new(); + public ConcurrentDictionary> GetBlobIndex() => new(); public UInt256 GetLatestPendingNonce(Address address) => 0; diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index e5cfef0b42f..146b2f419e7 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -183,7 +183,7 @@ public Transaction[] GetPendingTransactionsBySender(Address address) => public int GetPendingBlobTransactionsCount() => _blobTransactions.Count; - public ConcurrentDictionary> GetBlobIndex() => _blobTransactions.GetBlobIndex; + public ConcurrentDictionary> GetBlobIndex() => _blobTransactions.GetBlobIndex; private void OnHeadChange(object? sender, BlockReplacementEventArgs e) { From dbbeb3a8a4f848f3774a9465dbd4792116484173 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 14 Aug 2024 11:45:12 +0200 Subject: [PATCH 16/38] add TxPool test for indexing blobs --- .../TxPoolTests.Blobs.cs | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index f4d2b2092ae..59a93624661 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -1,9 +1,12 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; +using System.Collections.Generic; using System.Threading.Tasks; using FluentAssertions; using Nethermind.Core; +using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Core.Test.Builders; using Nethermind.Crypto; @@ -599,6 +602,76 @@ public async Task should_bring_back_reorganized_blob_txs() .Excluding(t => t.PoolIndex)); // ...as well as PoolIndex } + [Test] + public void should_index_blobs_when_adding_txs([Values(true, false)] bool isPersistentStorage, [Values(true, false)] bool uniqueBlobs) + { + const int poolSize = 10; + TxPoolConfig txPoolConfig = new() + { + BlobsSupport = isPersistentStorage ? BlobsSupportMode.Storage : BlobsSupportMode.InMemory, + PersistentBlobStorageSize = isPersistentStorage ? poolSize : 0, + InMemoryBlobPoolSize = isPersistentStorage ? 0 : poolSize + }; + + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); + + Transaction[] blobTxs = new Transaction[poolSize * 2]; + + // adding 2x more txs than pool capacity. First half will be evicted + for (int i = 0; i < poolSize * 2; i++) + { + EnsureSenderBalance(TestItem.Addresses[i], UInt256.MaxValue); + + blobTxs[i] = Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .WithMaxFeePerGas(1.GWei() + (UInt256)i) + .WithMaxPriorityFeePerGas(1.GWei() + (UInt256)i) + .WithMaxFeePerBlobGas(1000.Wei() + (UInt256)i) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeys[i]).TestObject; + + // making blobs unique. Otherwise, all txs have the same blob + if (uniqueBlobs + && blobTxs[i].NetworkWrapper is ShardBlobNetworkWrapper wrapper) + { + wrapper.Blobs[0] = new byte[Ckzg.Ckzg.BytesPerBlob]; + wrapper.Blobs[0][0] = (byte)(i % 256); + + KzgPolynomialCommitments.KzgifyBlob( + wrapper.Blobs[0], + wrapper.Commitments[0], + wrapper.Proofs[0], + blobTxs[i].BlobVersionedHashes[0].AsSpan()); + } + + _txPool.SubmitTx(blobTxs[i], TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + } + + _txPool.GetBlobIndex().Count.Should().Be(uniqueBlobs ? poolSize : 1); + + // first half of txs (0, poolSize - 1) was evicted and should be removed from index + // second half (poolSize, 2x poolSize - 1) should be indexed + for (int i = 0; i < poolSize * 2; i++) + { + // if blobs are unique, we expect index to have 10 keys (poolSize, 2x poolSize - 1) with 1 value each + if (uniqueBlobs) + { + _txPool.GetBlobIndex().TryGetValue(blobTxs[i].BlobVersionedHashes[0]!.ToHexString(), out List txHashes).Should().Be(i >= poolSize); + if (i >= poolSize) + { + txHashes.Count.Should().Be(1); + txHashes[0].Should().Be(blobTxs[i].Hash); + } + } + // if blobs are not unique, we expect index to have 1 key with 10 values (poolSize, 2x poolSize - 1) + else + { + _txPool.GetBlobIndex().TryGetValue(blobTxs[i].BlobVersionedHashes[0]!.ToHexString(), out List values).Should().BeTrue(); + values.Count.Should().Be(poolSize); + values.Contains(blobTxs[i].Hash).Should().Be(i >= poolSize); + } + } + } + private Transaction GetTx(PrivateKey sender) { return Build.A.Transaction From f5ec9a375a51e8e197e50862be68560705db80a3 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 14 Aug 2024 11:45:35 +0200 Subject: [PATCH 17/38] add one more engine test --- .../EngineModuleTests.V3.cs | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs index 9d56e536f03..977420c01c9 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs @@ -575,6 +575,54 @@ public async Task GetBlobsV1_should_return_nulls_when_blobs_not_found([Values(1, } } + [Test] + public async Task GetBlobsV1_should_return_mix_of_blobs_and_nulls([Values(1, 2, 3, 4, 5, 6)] int numberOfBlobs) + { + int requestSize = 10 * numberOfBlobs; + + MergeTestBlockchain chain = await CreateBlockchain(releaseSpec: Cancun.Instance); + IEngineRpcModule rpcModule = CreateEngineModule(chain, null, TimeSpan.FromDays(1)); + + Transaction blobTx = Build.A.Transaction + .WithShardBlobTxTypeAndFields(numberOfBlobs) + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) + .WithMaxFeePerBlobGas(1000.Wei()) + .SignedAndResolved(chain.EthereumEcdsa, TestItem.PrivateKeyA).TestObject; + + chain.TxPool.SubmitTx(blobTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + + List blobVersionedHashesRequest = new List(requestSize); + List blobsAndProofs = new(requestSize); + + int actualIndex = 0; + for (int i = 0; i < requestSize; i++) + { + bool addActualHash = i % 10 == 0; + + blobsAndProofs.Add(addActualHash ? new BlobAndProofV1(blobTx, actualIndex) : null); + blobVersionedHashesRequest.Add(addActualHash ? blobTx.BlobVersionedHashes![actualIndex++]! : Bytes.FromHexString(i.ToString("X64"))); + } + GetBlobsV1Result expected = new(blobsAndProofs.ToArray()); + + ResultWrapper result = await rpcModule.engine_getBlobsV1(blobVersionedHashesRequest.ToArray()); + + result.Data.Should().BeEquivalentTo(expected); + result.Data.BlobsAndProofs.Length.Should().Be(requestSize); + for (int i = 0; i < requestSize; i++) + { + if (i % 10 == 0) + { + result.Data.BlobsAndProofs[i]!.Blob.Should().BeEquivalentTo(((ShardBlobNetworkWrapper)blobTx.NetworkWrapper!).Blobs[i / 10]); + result.Data.BlobsAndProofs[i]!.Proof.Should().BeEquivalentTo(((ShardBlobNetworkWrapper)blobTx.NetworkWrapper!).Proofs[i / 10]); + } + else + { + result.Data.BlobsAndProofs[i].Should().BeNull(); + } + } + } + public static IEnumerable ForkchoiceUpdatedV3DeclinedTestCaseSource { get From 9cf95be0e2cf26870a5d9ca0993617a9af2db06b Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 14 Aug 2024 12:50:43 +0200 Subject: [PATCH 18/38] add test for max request size --- .../EngineModuleTests.V3.cs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs index 977420c01c9..aed6daec6c5 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs @@ -510,6 +510,32 @@ static void MarkAsUnprocessed(MergeTestBlockchain chain, int blockNumber) Assert.That(res2.Data.PayloadStatus.Status, Is.EqualTo(PayloadStatus.Valid)); } + [Test] + public async Task GetBlobsV1_should_throw_if_more_than_128_requested_blobs([Values(128, 129)] int requestSize) + { + MergeTestBlockchain chain = await CreateBlockchain(releaseSpec: Cancun.Instance); + IEngineRpcModule rpcModule = CreateEngineModule(chain, null, TimeSpan.FromDays(1)); + + List request = new List(requestSize); + for (int i = 0; i < requestSize; i++) + { + request.Add(Bytes.FromHexString(i.ToString("X64"))); + } + + ResultWrapper result = await rpcModule.engine_getBlobsV1(request.ToArray()); + + if (requestSize > 128) + { + result.Result.Should().BeEquivalentTo(Result.Fail($"The number of requested blobs must not exceed 128")); + result.ErrorCode.Should().Be(MergeErrorCodes.TooLargeRequest); + } + else + { + result.Result.Should().Be(Result.Success); + result.Data.BlobsAndProofs.Length.Should().Be(requestSize); + } + } + [Test] public async Task GetBlobsV1_should_return_requested_blobs([Values(1, 2, 3, 4, 5, 6)] int numberOfBlobs) { From c42837815ffcac00309ffa685490042fc1db4678 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 14 Aug 2024 13:30:55 +0200 Subject: [PATCH 19/38] cosmetics --- .../Nethermind.Merge.Plugin/EngineRpcModule.cs | 1 - .../Collections/BlobTxDistinctSortedPool.cs | 12 ++++++------ .../PersistentBlobTxDistinctSortedPool.cs | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.cs b/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.cs index 329b0e996fc..ace48261d8a 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.cs @@ -36,7 +36,6 @@ public EngineRpcModule( ILogManager logManager) { _capabilitiesHandler = capabilitiesHandler ?? throw new ArgumentNullException(nameof(capabilitiesHandler)); - _getBlobsHandler = getBlobsHandler; _getPayloadHandlerV1 = getPayloadHandlerV1; _getPayloadHandlerV2 = getPayloadHandlerV2; _getPayloadHandlerV3 = getPayloadHandlerV3; diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index 1c7b4db52aa..f7b76b91517 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -15,9 +15,9 @@ public class BlobTxDistinctSortedPool(int capacity, IComparer compa { protected override string ShortPoolName => "BlobPool"; - public ConcurrentDictionary> GetBlobIndex => BlobIndex; + public ConcurrentDictionary> GetBlobIndex => _blobIndex; - protected readonly ConcurrentDictionary> BlobIndex = new(); + private readonly ConcurrentDictionary> _blobIndex = new(); protected override IComparer GetReplacementComparer(IComparer comparer) => comparer.GetBlobReplacementComparer(); @@ -41,7 +41,7 @@ protected void AddToBlobIndex(Transaction blobTx) { if (blobVersionedHash?.Length == 32) { - BlobIndex.AddOrUpdate(blobVersionedHash.ToHexString(), + _blobIndex.AddOrUpdate(blobVersionedHash.ToHexString(), k => [blobTx.Hash!], (k, b) => { @@ -73,12 +73,12 @@ private void RemoveFromBlobIndex(Transaction blobTx) { foreach (var blobVersionedHash in blobTx.BlobVersionedHashes) { - if (blobVersionedHash?.ToHexString() is { } hexString - && BlobIndex.TryGetValue(hexString, out List? txHashes)) + if (blobVersionedHash?.ToHexString() is { } key + && _blobIndex.TryGetValue(key, out List? txHashes)) { if (txHashes.Count < 2) { - BlobIndex.Remove(hexString, out _); + _blobIndex.Remove(key, out _); } else { diff --git a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs index d0f7eb34d92..8933d014f16 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs @@ -48,7 +48,7 @@ private void RecreateLightTxCollectionAndCache(ITxStorage blobTxStorage) { long loadingTime = stopwatch.ElapsedMilliseconds; _logger.Info($"Loaded {numberOfTxsInDb} blob txs from persistent db, containing {numberOfBlobsInDb} blobs, in {loadingTime}ms"); - _logger.Info($"There are {BlobIndex.Count} unique blobs indexed"); + _logger.Info($"There are {GetBlobIndex.Count} unique blobs indexed"); } stopwatch.Stop(); } From 361be1042908355edf1192f85d9be239ed7a31d0 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 14 Aug 2024 13:40:34 +0200 Subject: [PATCH 20/38] cosmetic --- .../Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index f7b76b91517..e43c31a8a6d 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -6,6 +6,7 @@ using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; +using Nethermind.Crypto; using Nethermind.Logging; namespace Nethermind.TxPool.Collections; @@ -39,7 +40,7 @@ protected void AddToBlobIndex(Transaction blobTx) { foreach (var blobVersionedHash in blobTx.BlobVersionedHashes) { - if (blobVersionedHash?.Length == 32) + if (blobVersionedHash?.Length == KzgPolynomialCommitments.BytesPerBlobVersionedHash) { _blobIndex.AddOrUpdate(blobVersionedHash.ToHexString(), k => [blobTx.Hash!], From bcfce3032c21055e9ba23ac599eabd541745324a Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 14 Aug 2024 15:01:59 +0200 Subject: [PATCH 21/38] move back to byte[] as a key and just use value comparer --- .../Handlers/GetBlobsHandler.cs | 5 ++--- .../Nethermind.TxPool.Test/TxPoolTests.Blobs.cs | 4 ++-- .../Collections/BlobTxDistinctSortedPool.cs | 12 ++++++------ src/Nethermind/Nethermind.TxPool/ITxPool.cs | 2 +- src/Nethermind/Nethermind.TxPool/NullTxPool.cs | 2 +- src/Nethermind/Nethermind.TxPool/TxPool.cs | 2 +- 6 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs index 5d3c269a066..f3a22ef014c 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs @@ -8,7 +8,6 @@ using Nethermind.Core; using Nethermind.Core.Collections; using Nethermind.Core.Crypto; -using Nethermind.Core.Extensions; using Nethermind.JsonRpc; using Nethermind.Merge.Plugin.Data; using Nethermind.TxPool; @@ -19,7 +18,7 @@ public class GetBlobsHandler(ITxPool txPool) : IAsyncHandler> _blobIndex = txPool.GetBlobIndex(); + private readonly ConcurrentDictionary> _blobIndex = txPool.GetBlobIndex(); public Task> HandleAsync(byte[][] request) { @@ -34,7 +33,7 @@ public Task> HandleAsync(byte[][] request) foreach (byte[] requestedBlobVersionedHash in request) { bool isBlobFound = false; - if (_blobIndex.TryGetValue(requestedBlobVersionedHash.ToHexString(), out List? txHashes) + if (_blobIndex.TryGetValue(requestedBlobVersionedHash, out List? txHashes) && txPool.TryGetPendingBlobTransaction(txHashes.First(), out Transaction? blobTx) && blobTx.BlobVersionedHashes?.Length > 0) { diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index 59a93624661..a4f74ca09eb 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -655,7 +655,7 @@ public void should_index_blobs_when_adding_txs([Values(true, false)] bool isPers // if blobs are unique, we expect index to have 10 keys (poolSize, 2x poolSize - 1) with 1 value each if (uniqueBlobs) { - _txPool.GetBlobIndex().TryGetValue(blobTxs[i].BlobVersionedHashes[0]!.ToHexString(), out List txHashes).Should().Be(i >= poolSize); + _txPool.GetBlobIndex().TryGetValue(blobTxs[i].BlobVersionedHashes[0]!, out List txHashes).Should().Be(i >= poolSize); if (i >= poolSize) { txHashes.Count.Should().Be(1); @@ -665,7 +665,7 @@ public void should_index_blobs_when_adding_txs([Values(true, false)] bool isPers // if blobs are not unique, we expect index to have 1 key with 10 values (poolSize, 2x poolSize - 1) else { - _txPool.GetBlobIndex().TryGetValue(blobTxs[i].BlobVersionedHashes[0]!.ToHexString(), out List values).Should().BeTrue(); + _txPool.GetBlobIndex().TryGetValue(blobTxs[i].BlobVersionedHashes[0]!, out List values).Should().BeTrue(); values.Count.Should().Be(poolSize); values.Contains(blobTxs[i].Hash).Should().Be(i >= poolSize); } diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index e43c31a8a6d..82113d32676 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -16,9 +16,9 @@ public class BlobTxDistinctSortedPool(int capacity, IComparer compa { protected override string ShortPoolName => "BlobPool"; - public ConcurrentDictionary> GetBlobIndex => _blobIndex; + public ConcurrentDictionary> GetBlobIndex => _blobIndex; - private readonly ConcurrentDictionary> _blobIndex = new(); + private readonly ConcurrentDictionary> _blobIndex = new(Bytes.EqualityComparer); protected override IComparer GetReplacementComparer(IComparer comparer) => comparer.GetBlobReplacementComparer(); @@ -42,7 +42,7 @@ protected void AddToBlobIndex(Transaction blobTx) { if (blobVersionedHash?.Length == KzgPolynomialCommitments.BytesPerBlobVersionedHash) { - _blobIndex.AddOrUpdate(blobVersionedHash.ToHexString(), + _blobIndex.AddOrUpdate(blobVersionedHash, k => [blobTx.Hash!], (k, b) => { @@ -74,12 +74,12 @@ private void RemoveFromBlobIndex(Transaction blobTx) { foreach (var blobVersionedHash in blobTx.BlobVersionedHashes) { - if (blobVersionedHash?.ToHexString() is { } key - && _blobIndex.TryGetValue(key, out List? txHashes)) + if (blobVersionedHash is not null + && _blobIndex.TryGetValue(blobVersionedHash, out List? txHashes)) { if (txHashes.Count < 2) { - _blobIndex.Remove(key, out _); + _blobIndex.Remove(blobVersionedHash, out _); } else { diff --git a/src/Nethermind/Nethermind.TxPool/ITxPool.cs b/src/Nethermind/Nethermind.TxPool/ITxPool.cs index 1428ab438d9..65b5dad734f 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPool.cs @@ -16,7 +16,7 @@ public interface ITxPool int GetPendingTransactionsCount(); int GetPendingBlobTransactionsCount(); Transaction[] GetPendingTransactions(); - ConcurrentDictionary> GetBlobIndex(); + ConcurrentDictionary> GetBlobIndex(); /// /// Non-blob txs grouped by sender address, sorted by nonce and later tx pool sorting diff --git a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs index 30393f9b1cc..f3d34cf2709 100644 --- a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs @@ -53,7 +53,7 @@ public bool TryGetPendingBlobTransaction(Hash256 hash, [NotNullWhen(true)] out T return false; } - public ConcurrentDictionary> GetBlobIndex() => new(); + public ConcurrentDictionary> GetBlobIndex() => new(); public UInt256 GetLatestPendingNonce(Address address) => 0; diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 146b2f419e7..e5cfef0b42f 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -183,7 +183,7 @@ public Transaction[] GetPendingTransactionsBySender(Address address) => public int GetPendingBlobTransactionsCount() => _blobTransactions.Count; - public ConcurrentDictionary> GetBlobIndex() => _blobTransactions.GetBlobIndex; + public ConcurrentDictionary> GetBlobIndex() => _blobTransactions.GetBlobIndex; private void OnHeadChange(object? sender, BlockReplacementEventArgs e) { From 48351424ad2193864deef413b94896774b765a15 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 20 Aug 2024 11:38:52 +0200 Subject: [PATCH 22/38] refactor --- .../EngineModuleTests.V3.cs | 23 ++++++----- .../Data/GetBlobsV1Result.cs | 5 ++- .../Handlers/GetBlobsHandler.cs | 37 +----------------- .../BlobAndProofV1.cs | 2 +- .../Nethermind.TxPool/BlobCatcher.cs | 39 +++++++++++++++++++ src/Nethermind/Nethermind.TxPool/ITxPool.cs | 3 +- .../Nethermind.TxPool/NullTxPool.cs | 2 +- src/Nethermind/Nethermind.TxPool/TxPool.cs | 9 ++++- 8 files changed, 67 insertions(+), 53 deletions(-) rename src/Nethermind/{Nethermind.Merge.Plugin/Data => Nethermind.TxPool}/BlobAndProofV1.cs (93%) create mode 100644 src/Nethermind/Nethermind.TxPool/BlobCatcher.cs diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs index aed6daec6c5..5f42e02e9ed 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs @@ -532,7 +532,7 @@ public async Task GetBlobsV1_should_throw_if_more_than_128_requested_blobs([Valu else { result.Result.Should().Be(Result.Success); - result.Data.BlobsAndProofs.Length.Should().Be(requestSize); + result.Data.BlobsAndProofs.ToArray().Length.Should().Be(requestSize); } } @@ -561,11 +561,12 @@ public async Task GetBlobsV1_should_return_requested_blobs([Values(1, 2, 3, 4, 5 ResultWrapper result = await rpcModule.engine_getBlobsV1(blobTx.BlobVersionedHashes!); result.Data.Should().BeEquivalentTo(expected); - result.Data.BlobsAndProofs.Length.Should().Be(numberOfBlobs); + BlobAndProofV1?[] resultBlobsAndProofs = result.Data.BlobsAndProofs.ToArray(); + resultBlobsAndProofs.Length.Should().Be(numberOfBlobs); for (int i = 0; i < numberOfBlobs; i++) { - result.Data.BlobsAndProofs[i]!.Blob.Should().BeEquivalentTo(((ShardBlobNetworkWrapper)blobTx.NetworkWrapper!).Blobs[i]); - result.Data.BlobsAndProofs[i]!.Proof.Should().BeEquivalentTo(((ShardBlobNetworkWrapper)blobTx.NetworkWrapper!).Proofs[i]); + resultBlobsAndProofs[i]!.Blob.Should().BeEquivalentTo(((ShardBlobNetworkWrapper)blobTx.NetworkWrapper!).Blobs[i]); + resultBlobsAndProofs[i]!.Proof.Should().BeEquivalentTo(((ShardBlobNetworkWrapper)blobTx.NetworkWrapper!).Proofs[i]); } } @@ -594,10 +595,11 @@ public async Task GetBlobsV1_should_return_nulls_when_blobs_not_found([Values(1, ResultWrapper result = await rpcModule.engine_getBlobsV1(blobTx.BlobVersionedHashes!); result.Data.Should().BeEquivalentTo(expected); - result.Data.BlobsAndProofs.Length.Should().Be(numberOfRequestedBlobs); + BlobAndProofV1?[] resultBlobsAndProofs = result.Data.BlobsAndProofs.ToArray(); + resultBlobsAndProofs.Length.Should().Be(numberOfRequestedBlobs); for (int i = 0; i < numberOfRequestedBlobs; i++) { - result.Data.BlobsAndProofs[i]!.Should().BeNull(); + resultBlobsAndProofs[i]!.Should().BeNull(); } } @@ -634,17 +636,18 @@ public async Task GetBlobsV1_should_return_mix_of_blobs_and_nulls([Values(1, 2, ResultWrapper result = await rpcModule.engine_getBlobsV1(blobVersionedHashesRequest.ToArray()); result.Data.Should().BeEquivalentTo(expected); - result.Data.BlobsAndProofs.Length.Should().Be(requestSize); + BlobAndProofV1?[] resultBlobsAndProofs = result.Data.BlobsAndProofs.ToArray(); + resultBlobsAndProofs.Length.Should().Be(requestSize); for (int i = 0; i < requestSize; i++) { if (i % 10 == 0) { - result.Data.BlobsAndProofs[i]!.Blob.Should().BeEquivalentTo(((ShardBlobNetworkWrapper)blobTx.NetworkWrapper!).Blobs[i / 10]); - result.Data.BlobsAndProofs[i]!.Proof.Should().BeEquivalentTo(((ShardBlobNetworkWrapper)blobTx.NetworkWrapper!).Proofs[i / 10]); + resultBlobsAndProofs[i]!.Blob.Should().BeEquivalentTo(((ShardBlobNetworkWrapper)blobTx.NetworkWrapper!).Blobs[i / 10]); + resultBlobsAndProofs[i]!.Proof.Should().BeEquivalentTo(((ShardBlobNetworkWrapper)blobTx.NetworkWrapper!).Proofs[i / 10]); } else { - result.Data.BlobsAndProofs[i].Should().BeNull(); + result.Data.BlobsAndProofs.ToArray()[i].Should().BeNull(); } } } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs b/src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs index 543f94c25f3..3ed3015399e 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs @@ -2,10 +2,11 @@ // SPDX-License-Identifier: LGPL-3.0-only using System.Collections.Generic; +using Nethermind.TxPool; namespace Nethermind.Merge.Plugin.Data; -public class GetBlobsV1Result(BlobAndProofV1?[] blobsAndProofs) +public class GetBlobsV1Result(IEnumerable blobsAndProofs) { - public readonly BlobAndProofV1?[] BlobsAndProofs = blobsAndProofs; + public readonly IEnumerable BlobsAndProofs = blobsAndProofs; } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs index f3a22ef014c..6ad612daf85 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs @@ -1,13 +1,7 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; using System.Threading.Tasks; -using Nethermind.Core; -using Nethermind.Core.Collections; -using Nethermind.Core.Crypto; using Nethermind.JsonRpc; using Nethermind.Merge.Plugin.Data; using Nethermind.TxPool; @@ -17,9 +11,6 @@ namespace Nethermind.Merge.Plugin.Handlers; public class GetBlobsHandler(ITxPool txPool) : IAsyncHandler { private const int MaxRequest = 128; - - private readonly ConcurrentDictionary> _blobIndex = txPool.GetBlobIndex(); - public Task> HandleAsync(byte[][] request) { if (request.Length > MaxRequest) @@ -28,32 +19,6 @@ public Task> HandleAsync(byte[][] request) return ResultWrapper.Fail(error, MergeErrorCodes.TooLargeRequest); } - ArrayPoolList blobsAndProofs = new(request.Length); - - foreach (byte[] requestedBlobVersionedHash in request) - { - bool isBlobFound = false; - if (_blobIndex.TryGetValue(requestedBlobVersionedHash, out List? txHashes) - && txPool.TryGetPendingBlobTransaction(txHashes.First(), out Transaction? blobTx) - && blobTx.BlobVersionedHashes?.Length > 0) - { - for (int indexOfBlob = 0; indexOfBlob < blobTx.BlobVersionedHashes.Length; indexOfBlob++) - { - if (blobTx.BlobVersionedHashes[indexOfBlob] == requestedBlobVersionedHash) - { - isBlobFound = true; - blobsAndProofs.Add(new BlobAndProofV1(blobTx, indexOfBlob)); - break; - } - } - } - - if (!isBlobFound) - { - blobsAndProofs.Add(null); - } - } - - return ResultWrapper.Success(new GetBlobsV1Result(blobsAndProofs.ToArray())); + return ResultWrapper.Success(new GetBlobsV1Result(txPool.GetBlobsAndProofs(request))); } } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Data/BlobAndProofV1.cs b/src/Nethermind/Nethermind.TxPool/BlobAndProofV1.cs similarity index 93% rename from src/Nethermind/Nethermind.Merge.Plugin/Data/BlobAndProofV1.cs rename to src/Nethermind/Nethermind.TxPool/BlobAndProofV1.cs index 6dd84dfd6dd..fb7870841d4 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Data/BlobAndProofV1.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobAndProofV1.cs @@ -5,7 +5,7 @@ using System; using Nethermind.Core; -namespace Nethermind.Merge.Plugin.Data; +namespace Nethermind.TxPool; public class BlobAndProofV1 { diff --git a/src/Nethermind/Nethermind.TxPool/BlobCatcher.cs b/src/Nethermind/Nethermind.TxPool/BlobCatcher.cs new file mode 100644 index 00000000000..b96c5658054 --- /dev/null +++ b/src/Nethermind/Nethermind.TxPool/BlobCatcher.cs @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; +using System.Linq; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.TxPool.Collections; + +namespace Nethermind.TxPool; + +public class BlobCatcher(BlobTxDistinctSortedPool blobPool) +{ + public IEnumerable GetBlobsAndProofs(byte[][] request) + { + foreach (byte[] requestedBlobVersionedHash in request) + { + yield return GetBlobAndProofV1(requestedBlobVersionedHash); + } + } + + private BlobAndProofV1? GetBlobAndProofV1(byte[] requestedBlobVersionedHash) + { + if (blobPool.GetBlobIndex.TryGetValue(requestedBlobVersionedHash, out List? txHashes) + && blobPool.TryGetValue(txHashes.First(), out Transaction? blobTx) + && blobTx.BlobVersionedHashes?.Length > 0) + { + for (int indexOfBlob = 0; indexOfBlob < blobTx.BlobVersionedHashes.Length; indexOfBlob++) + { + if (blobTx.BlobVersionedHashes[indexOfBlob] == requestedBlobVersionedHash) + { + return new BlobAndProofV1(blobTx, indexOfBlob); + } + } + } + + return null; + } +} diff --git a/src/Nethermind/Nethermind.TxPool/ITxPool.cs b/src/Nethermind/Nethermind.TxPool/ITxPool.cs index 65b5dad734f..2be5d8cc099 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPool.cs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using Nethermind.Core; @@ -16,7 +15,7 @@ public interface ITxPool int GetPendingTransactionsCount(); int GetPendingBlobTransactionsCount(); Transaction[] GetPendingTransactions(); - ConcurrentDictionary> GetBlobIndex(); + IEnumerable GetBlobsAndProofs(byte[][] blobVersionedHashes); /// /// Non-blob txs grouped by sender address, sorted by nonce and later tx pool sorting diff --git a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs index f3d34cf2709..c47fb20106d 100644 --- a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs @@ -53,7 +53,7 @@ public bool TryGetPendingBlobTransaction(Hash256 hash, [NotNullWhen(true)] out T return false; } - public ConcurrentDictionary> GetBlobIndex() => new(); + public IEnumerable GetBlobsAndProofs(byte[][] blobVersionedHashes) => Array.Empty(); public UInt256 GetLatestPendingNonce(Address address) => 0; diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index e5cfef0b42f..275a96a9ffb 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -43,6 +43,7 @@ public class TxPool : ITxPool, IDisposable private readonly TxDistinctSortedPool _transactions; private readonly BlobTxDistinctSortedPool _blobTransactions; + private readonly BlobCatcher _blobCatcher; private readonly IChainHeadSpecProvider _specProvider; private readonly IAccountStateProvider _accounts; @@ -120,6 +121,8 @@ public TxPool(IEthereumEcdsa ecdsa, : new BlobTxDistinctSortedPool(txPoolConfig.BlobsSupport == BlobsSupportMode.InMemory ? _txPoolConfig.InMemoryBlobPoolSize : 0, comparer, logManager); if (_blobTransactions.Count > 0) _blobTransactions.UpdatePool(_accounts, _updateBucket); + _blobCatcher = new BlobCatcher(_blobTransactions); + _headInfo.HeadChanged += OnHeadChange; _preHashFilters = new IIncomingTxFilter[] @@ -181,9 +184,13 @@ public Transaction[] GetPendingTransactionsBySender(Address address) => // only for testing reasons internal Transaction[] GetOwnPendingTransactions() => _broadcaster.GetSnapshot(); + // only for testing reasons + internal ConcurrentDictionary> GetBlobIndex() => _blobTransactions.GetBlobIndex; + public int GetPendingBlobTransactionsCount() => _blobTransactions.Count; - public ConcurrentDictionary> GetBlobIndex() => _blobTransactions.GetBlobIndex; + public IEnumerable GetBlobsAndProofs(byte[][] blobVersionedHashes) => + _blobCatcher.GetBlobsAndProofs(blobVersionedHashes); private void OnHeadChange(object? sender, BlockReplacementEventArgs e) { From ac25e53e45991dd83272579bcd9f6f2e3dda8edd Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 20 Aug 2024 11:44:34 +0200 Subject: [PATCH 23/38] fix file encoding --- src/Nethermind/Nethermind.TxPool/BlobCatcher.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool/BlobCatcher.cs b/src/Nethermind/Nethermind.TxPool/BlobCatcher.cs index b96c5658054..9249f1137f6 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobCatcher.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobCatcher.cs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only using System.Collections.Generic; From 05c4d0edfc07b3f395faa827dc73bbb5f464fc7b Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 20 Aug 2024 12:49:31 +0200 Subject: [PATCH 24/38] refactor --- .../TxPoolTests.Blobs.cs | 18 +++++++++++++----- .../Collections/BlobTxDistinctSortedPool.cs | 2 +- src/Nethermind/Nethermind.TxPool/TxPool.cs | 3 --- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index a4f74ca09eb..b7ff68b64eb 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Threading.Tasks; using FluentAssertions; +using Nethermind.Consensus.Comparers; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; @@ -12,6 +13,9 @@ using Nethermind.Crypto; using Nethermind.Evm; using Nethermind.Int256; +using Nethermind.Logging; +using Nethermind.Specs; +using Nethermind.TxPool.Collections; using NUnit.Framework; namespace Nethermind.TxPool.Test @@ -613,7 +617,11 @@ public void should_index_blobs_when_adding_txs([Values(true, false)] bool isPers InMemoryBlobPoolSize = isPersistentStorage ? 0 : poolSize }; - _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); + IComparer comparer = new TransactionComparerProvider(_specProvider, _blockTree).GetDefaultComparer(); + + BlobTxDistinctSortedPool blobPool = isPersistentStorage + ? new PersistentBlobTxDistinctSortedPool(new BlobTxStorage(), txPoolConfig, comparer, LimboLogs.Instance) + : new BlobTxDistinctSortedPool(txPoolConfig.InMemoryBlobPoolSize, comparer, LimboLogs.Instance); Transaction[] blobTxs = new Transaction[poolSize * 2]; @@ -643,10 +651,10 @@ public void should_index_blobs_when_adding_txs([Values(true, false)] bool isPers blobTxs[i].BlobVersionedHashes[0].AsSpan()); } - _txPool.SubmitTx(blobTxs[i], TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + blobPool.TryInsert(blobTxs[i].Hash, blobTxs[i], out _).Should().BeTrue(); } - _txPool.GetBlobIndex().Count.Should().Be(uniqueBlobs ? poolSize : 1); + blobPool.GetBlobIndex.Count.Should().Be(uniqueBlobs ? poolSize : 1); // first half of txs (0, poolSize - 1) was evicted and should be removed from index // second half (poolSize, 2x poolSize - 1) should be indexed @@ -655,7 +663,7 @@ public void should_index_blobs_when_adding_txs([Values(true, false)] bool isPers // if blobs are unique, we expect index to have 10 keys (poolSize, 2x poolSize - 1) with 1 value each if (uniqueBlobs) { - _txPool.GetBlobIndex().TryGetValue(blobTxs[i].BlobVersionedHashes[0]!, out List txHashes).Should().Be(i >= poolSize); + blobPool.GetBlobIndex.TryGetValue(blobTxs[i].BlobVersionedHashes[0]!, out List txHashes).Should().Be(i >= poolSize); if (i >= poolSize) { txHashes.Count.Should().Be(1); @@ -665,7 +673,7 @@ public void should_index_blobs_when_adding_txs([Values(true, false)] bool isPers // if blobs are not unique, we expect index to have 1 key with 10 values (poolSize, 2x poolSize - 1) else { - _txPool.GetBlobIndex().TryGetValue(blobTxs[i].BlobVersionedHashes[0]!, out List values).Should().BeTrue(); + blobPool.GetBlobIndex.TryGetValue(blobTxs[i].BlobVersionedHashes[0]!, out List values).Should().BeTrue(); values.Count.Should().Be(poolSize); values.Contains(blobTxs[i].Hash).Should().Be(i >= poolSize); } diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index 82113d32676..e9b10e0269c 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -16,7 +16,7 @@ public class BlobTxDistinctSortedPool(int capacity, IComparer compa { protected override string ShortPoolName => "BlobPool"; - public ConcurrentDictionary> GetBlobIndex => _blobIndex; + internal ConcurrentDictionary> GetBlobIndex => _blobIndex; private readonly ConcurrentDictionary> _blobIndex = new(Bytes.EqualityComparer); diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 275a96a9ffb..ee541fc548f 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -184,9 +184,6 @@ public Transaction[] GetPendingTransactionsBySender(Address address) => // only for testing reasons internal Transaction[] GetOwnPendingTransactions() => _broadcaster.GetSnapshot(); - // only for testing reasons - internal ConcurrentDictionary> GetBlobIndex() => _blobTransactions.GetBlobIndex; - public int GetPendingBlobTransactionsCount() => _blobTransactions.Count; public IEnumerable GetBlobsAndProofs(byte[][] blobVersionedHashes) => From 70d1b17dffd0975ce483e9e09f7c4a46517beb37 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 20 Aug 2024 13:10:59 +0200 Subject: [PATCH 25/38] improve validation --- .../EngineModuleTests.V3.cs | 12 ++++++++--- .../Nethermind.TxPool/BlobAndProofV1.cs | 20 +++---------------- .../Nethermind.TxPool/BlobCatcher.cs | 5 +++-- 3 files changed, 15 insertions(+), 22 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs index 5f42e02e9ed..e185a908525 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs @@ -552,10 +552,14 @@ public async Task GetBlobsV1_should_return_requested_blobs([Values(1, 2, 3, 4, 5 chain.TxPool.SubmitTx(blobTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); List blobsAndProofs = new(numberOfBlobs); - for (int i = 0; i < numberOfBlobs; i++) + if (blobTx.NetworkWrapper is ShardBlobNetworkWrapper wrapper) { - blobsAndProofs.Add(new BlobAndProofV1(blobTx, i)); + for (int i = 0; i < numberOfBlobs; i++) + { + blobsAndProofs.Add(new BlobAndProofV1(wrapper.Blobs[i], wrapper.Proofs[i])); + } } + GetBlobsV1Result expected = new(blobsAndProofs.ToArray()); ResultWrapper result = await rpcModule.engine_getBlobsV1(blobTx.BlobVersionedHashes!); @@ -628,7 +632,9 @@ public async Task GetBlobsV1_should_return_mix_of_blobs_and_nulls([Values(1, 2, { bool addActualHash = i % 10 == 0; - blobsAndProofs.Add(addActualHash ? new BlobAndProofV1(blobTx, actualIndex) : null); + blobsAndProofs.Add(addActualHash && blobTx.NetworkWrapper is ShardBlobNetworkWrapper wrapper + ? new BlobAndProofV1(wrapper.Blobs[actualIndex], wrapper.Proofs[actualIndex]) + : null); blobVersionedHashesRequest.Add(addActualHash ? blobTx.BlobVersionedHashes![actualIndex++]! : Bytes.FromHexString(i.ToString("X64"))); } GetBlobsV1Result expected = new(blobsAndProofs.ToArray()); diff --git a/src/Nethermind/Nethermind.TxPool/BlobAndProofV1.cs b/src/Nethermind/Nethermind.TxPool/BlobAndProofV1.cs index fb7870841d4..03a4b974921 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobAndProofV1.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobAndProofV1.cs @@ -1,24 +1,10 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only - -using System; -using Nethermind.Core; - namespace Nethermind.TxPool; -public class BlobAndProofV1 +public class BlobAndProofV1(byte[] blob, byte[] proof) { - public BlobAndProofV1(Transaction blobTx, int index) - { - if (blobTx is not { NetworkWrapper: ShardBlobNetworkWrapper wrapper }) - { - throw new ArgumentException("Shard blob transaction should contain network wrapper data"); - } - - Blob = wrapper.Blobs[index]; - Proof = wrapper.Proofs[index]; - } - public byte[] Blob { get; set; } - public byte[] Proof { get; set; } + public byte[] Blob { get; set; } = blob; + public byte[] Proof { get; set; } = proof; } diff --git a/src/Nethermind/Nethermind.TxPool/BlobCatcher.cs b/src/Nethermind/Nethermind.TxPool/BlobCatcher.cs index 9249f1137f6..519f882b614 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobCatcher.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobCatcher.cs @@ -27,9 +27,10 @@ public class BlobCatcher(BlobTxDistinctSortedPool blobPool) { for (int indexOfBlob = 0; indexOfBlob < blobTx.BlobVersionedHashes.Length; indexOfBlob++) { - if (blobTx.BlobVersionedHashes[indexOfBlob] == requestedBlobVersionedHash) + if (blobTx.BlobVersionedHashes[indexOfBlob] == requestedBlobVersionedHash + && blobTx.NetworkWrapper is ShardBlobNetworkWrapper wrapper) { - return new BlobAndProofV1(blobTx, indexOfBlob); + return new BlobAndProofV1(wrapper.Blobs[indexOfBlob], wrapper.Proofs[indexOfBlob]); } } } From 24ced8aeccb627caaaef8ebd654eb722ff93b0e8 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 20 Aug 2024 13:30:19 +0200 Subject: [PATCH 26/38] drop linq --- src/Nethermind/Nethermind.TxPool/BlobCatcher.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/BlobCatcher.cs b/src/Nethermind/Nethermind.TxPool/BlobCatcher.cs index 519f882b614..cfdd559ff2c 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobCatcher.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobCatcher.cs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: LGPL-3.0-only using System.Collections.Generic; -using System.Linq; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.TxPool.Collections; @@ -22,7 +21,8 @@ public class BlobCatcher(BlobTxDistinctSortedPool blobPool) private BlobAndProofV1? GetBlobAndProofV1(byte[] requestedBlobVersionedHash) { if (blobPool.GetBlobIndex.TryGetValue(requestedBlobVersionedHash, out List? txHashes) - && blobPool.TryGetValue(txHashes.First(), out Transaction? blobTx) + && txHashes[0] is not null + && blobPool.TryGetValue(txHashes[0], out Transaction? blobTx) && blobTx.BlobVersionedHashes?.Length > 0) { for (int indexOfBlob = 0; indexOfBlob < blobTx.BlobVersionedHashes.Length; indexOfBlob++) From 853cc50eaca5e543f7e04f6f8fd5242b8d871e93 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 20 Aug 2024 13:30:42 +0200 Subject: [PATCH 27/38] add test --- .../EngineModuleTests.V3.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs index e185a908525..ca26398b8e3 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs @@ -536,6 +536,18 @@ public async Task GetBlobsV1_should_throw_if_more_than_128_requested_blobs([Valu } } + [Test] + public async Task GetBlobsV1_should_handle_empty_request() + { + MergeTestBlockchain chain = await CreateBlockchain(releaseSpec: Cancun.Instance); + IEngineRpcModule rpcModule = CreateEngineModule(chain, null, TimeSpan.FromDays(1)); + + ResultWrapper result = await rpcModule.engine_getBlobsV1([]); + + result.Result.Should().Be(Result.Success); + result.Data.Should().BeEquivalentTo(new GetBlobsV1Result(ArraySegment.Empty)); + } + [Test] public async Task GetBlobsV1_should_return_requested_blobs([Values(1, 2, 3, 4, 5, 6)] int numberOfBlobs) { From 1bcf4ad9a109315c6b9468fd54bc7fb806bcab88 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 20 Aug 2024 13:37:42 +0200 Subject: [PATCH 28/38] cosmetics --- src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs | 1 - src/Nethermind/Nethermind.TxPool/NullTxPool.cs | 1 - src/Nethermind/Nethermind.TxPool/TxPool.cs | 1 - 3 files changed, 3 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index b7ff68b64eb..da6f2aa0ca6 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -14,7 +14,6 @@ using Nethermind.Evm; using Nethermind.Int256; using Nethermind.Logging; -using Nethermind.Specs; using Nethermind.TxPool.Collections; using NUnit.Framework; diff --git a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs index c47fb20106d..9e735a7bd2d 100644 --- a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using Nethermind.Core; diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index ee541fc548f..22dfa155ab3 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; From 1f107dd8dcbc3fe61476200e10807b51408ee684 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak <77129288+marcindsobczak@users.noreply.github.com> Date: Wed, 21 Aug 2024 10:05:38 +0200 Subject: [PATCH 29/38] Update src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs Co-authored-by: Lukasz Rozmej --- .../Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs index ca26398b8e3..c37aceb2736 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs @@ -532,7 +532,7 @@ public async Task GetBlobsV1_should_throw_if_more_than_128_requested_blobs([Valu else { result.Result.Should().Be(Result.Success); - result.Data.BlobsAndProofs.ToArray().Length.Should().Be(requestSize); + result.Data.BlobsAndProofs.Should().HaveCount(requestSize); } } From 6dfcb9bdb85f4b504a36b6930c4599498039b4ce Mon Sep 17 00:00:00 2001 From: Marcin Sobczak <77129288+marcindsobczak@users.noreply.github.com> Date: Wed, 21 Aug 2024 10:05:50 +0200 Subject: [PATCH 30/38] Update src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs Co-authored-by: Lukasz Rozmej --- .../Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs index c37aceb2736..f85729bb96f 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs @@ -578,12 +578,9 @@ public async Task GetBlobsV1_should_return_requested_blobs([Values(1, 2, 3, 4, 5 result.Data.Should().BeEquivalentTo(expected); BlobAndProofV1?[] resultBlobsAndProofs = result.Data.BlobsAndProofs.ToArray(); - resultBlobsAndProofs.Length.Should().Be(numberOfBlobs); - for (int i = 0; i < numberOfBlobs; i++) - { - resultBlobsAndProofs[i]!.Blob.Should().BeEquivalentTo(((ShardBlobNetworkWrapper)blobTx.NetworkWrapper!).Blobs[i]); - resultBlobsAndProofs[i]!.Proof.Should().BeEquivalentTo(((ShardBlobNetworkWrapper)blobTx.NetworkWrapper!).Proofs[i]); - } + ShardBlobNetworkWrapper wrapper = ((ShardBlobNetworkWrapper)blobTx.NetworkWrapper!); + resultBlobsAndProofs.Select(b => b.Blob).Should().BeEquivalentTo(wrapper.Blobs); + resultBlobsAndProofs.Select(b => b.Proof).Should().BeEquivalentTo(wrapper.Proofs); } [Test] From 6e2cb34b1efcce4392f349ab99aa9e4e80ff2a6b Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 21 Aug 2024 10:06:37 +0200 Subject: [PATCH 31/38] move BlobAndProofV1 back to merge plugin --- .../Data}/BlobAndProofV1.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/Nethermind/{Nethermind.TxPool => Nethermind.Merge.Plugin/Data}/BlobAndProofV1.cs (86%) diff --git a/src/Nethermind/Nethermind.TxPool/BlobAndProofV1.cs b/src/Nethermind/Nethermind.Merge.Plugin/Data/BlobAndProofV1.cs similarity index 86% rename from src/Nethermind/Nethermind.TxPool/BlobAndProofV1.cs rename to src/Nethermind/Nethermind.Merge.Plugin/Data/BlobAndProofV1.cs index 03a4b974921..af2fdc4e7a5 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobAndProofV1.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Data/BlobAndProofV1.cs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -namespace Nethermind.TxPool; +namespace Nethermind.Merge.Plugin.Data; public class BlobAndProofV1(byte[] blob, byte[] proof) { From 9d59e4b60707e1d211f1d07a2fb7531f21bfe4c8 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 21 Aug 2024 10:07:48 +0200 Subject: [PATCH 32/38] refactor blob collecting logic --- .../Handlers/GetBlobsHandler.cs | 13 +++++++++- .../{BlobCatcher.cs => BlobFinder.cs} | 26 +++++++++---------- src/Nethermind/Nethermind.TxPool/ITxPool.cs | 4 ++- .../Nethermind.TxPool/NullTxPool.cs | 9 ++++++- src/Nethermind/Nethermind.TxPool/TxPool.cs | 10 ++++--- 5 files changed, 42 insertions(+), 20 deletions(-) rename src/Nethermind/Nethermind.TxPool/{BlobCatcher.cs => BlobFinder.cs} (56%) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs index 6ad612daf85..e6065ad83b1 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Collections.Generic; using System.Threading.Tasks; using Nethermind.JsonRpc; using Nethermind.Merge.Plugin.Data; @@ -19,6 +20,16 @@ public Task> HandleAsync(byte[][] request) return ResultWrapper.Fail(error, MergeErrorCodes.TooLargeRequest); } - return ResultWrapper.Success(new GetBlobsV1Result(txPool.GetBlobsAndProofs(request))); + return ResultWrapper.Success(new GetBlobsV1Result(GetBlobsAndProofs(request))); + } + + private IEnumerable GetBlobsAndProofs(byte[][] request) + { + foreach (byte[] requestedBlobVersionedHash in request) + { + yield return txPool.TryGetBlobAndProof(requestedBlobVersionedHash, out byte[]? blob, out byte[]? proof) + ? new BlobAndProofV1(blob, proof) + : null; + } } } diff --git a/src/Nethermind/Nethermind.TxPool/BlobCatcher.cs b/src/Nethermind/Nethermind.TxPool/BlobFinder.cs similarity index 56% rename from src/Nethermind/Nethermind.TxPool/BlobCatcher.cs rename to src/Nethermind/Nethermind.TxPool/BlobFinder.cs index cfdd559ff2c..3106688e4cf 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobCatcher.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobFinder.cs @@ -2,23 +2,19 @@ // SPDX-License-Identifier: LGPL-3.0-only using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; using Nethermind.TxPool.Collections; namespace Nethermind.TxPool; -public class BlobCatcher(BlobTxDistinctSortedPool blobPool) +public class BlobFinder(BlobTxDistinctSortedPool blobPool) { - public IEnumerable GetBlobsAndProofs(byte[][] request) - { - foreach (byte[] requestedBlobVersionedHash in request) - { - yield return GetBlobAndProofV1(requestedBlobVersionedHash); - } - } - - private BlobAndProofV1? GetBlobAndProofV1(byte[] requestedBlobVersionedHash) + public bool TryGetBlobAndProof(byte[] requestedBlobVersionedHash, + [NotNullWhen(true)] out byte[]? blob, + [NotNullWhen(true)] out byte[]? proof) { if (blobPool.GetBlobIndex.TryGetValue(requestedBlobVersionedHash, out List? txHashes) && txHashes[0] is not null @@ -27,14 +23,18 @@ public class BlobCatcher(BlobTxDistinctSortedPool blobPool) { for (int indexOfBlob = 0; indexOfBlob < blobTx.BlobVersionedHashes.Length; indexOfBlob++) { - if (blobTx.BlobVersionedHashes[indexOfBlob] == requestedBlobVersionedHash + if (Bytes.AreEqual(blobTx.BlobVersionedHashes[indexOfBlob], requestedBlobVersionedHash) && blobTx.NetworkWrapper is ShardBlobNetworkWrapper wrapper) { - return new BlobAndProofV1(wrapper.Blobs[indexOfBlob], wrapper.Proofs[indexOfBlob]); + blob = wrapper.Blobs[indexOfBlob]; + proof = wrapper.Proofs[indexOfBlob]; + return true; } } } - return null; + blob = default; + proof = default; + return false; } } diff --git a/src/Nethermind/Nethermind.TxPool/ITxPool.cs b/src/Nethermind/Nethermind.TxPool/ITxPool.cs index 2be5d8cc099..7fbb8102149 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPool.cs @@ -15,7 +15,6 @@ public interface ITxPool int GetPendingTransactionsCount(); int GetPendingBlobTransactionsCount(); Transaction[] GetPendingTransactions(); - IEnumerable GetBlobsAndProofs(byte[][] blobVersionedHashes); /// /// Non-blob txs grouped by sender address, sorted by nonce and later tx pool sorting @@ -42,6 +41,9 @@ public interface ITxPool bool IsKnown(Hash256 hash); bool TryGetPendingTransaction(Hash256 hash, [NotNullWhen(true)] out Transaction? transaction); bool TryGetPendingBlobTransaction(Hash256 hash, [NotNullWhen(true)] out Transaction? blobTransaction); + bool TryGetBlobAndProof(byte[] blobVersionedHash, + [NotNullWhen(true)] out byte[]? blob, + [NotNullWhen(true)] out byte[]? proof); UInt256 GetLatestPendingNonce(Address address); event EventHandler NewDiscovered; event EventHandler NewPending; diff --git a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs index 9e735a7bd2d..57d830e3b52 100644 --- a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs @@ -52,7 +52,14 @@ public bool TryGetPendingBlobTransaction(Hash256 hash, [NotNullWhen(true)] out T return false; } - public IEnumerable GetBlobsAndProofs(byte[][] blobVersionedHashes) => Array.Empty(); + public bool TryGetBlobAndProof(byte[] blobVersionedHash, + [NotNullWhen(true)] out byte[]? blob, + [NotNullWhen(true)] out byte[]? proof) + { + blob = null; + proof = null; + return false; + } public UInt256 GetLatestPendingNonce(Address address) => 0; diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 22dfa155ab3..694fa14b712 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -42,7 +42,7 @@ public class TxPool : ITxPool, IDisposable private readonly TxDistinctSortedPool _transactions; private readonly BlobTxDistinctSortedPool _blobTransactions; - private readonly BlobCatcher _blobCatcher; + private readonly BlobFinder _blobFinder; private readonly IChainHeadSpecProvider _specProvider; private readonly IAccountStateProvider _accounts; @@ -120,7 +120,7 @@ public TxPool(IEthereumEcdsa ecdsa, : new BlobTxDistinctSortedPool(txPoolConfig.BlobsSupport == BlobsSupportMode.InMemory ? _txPoolConfig.InMemoryBlobPoolSize : 0, comparer, logManager); if (_blobTransactions.Count > 0) _blobTransactions.UpdatePool(_accounts, _updateBucket); - _blobCatcher = new BlobCatcher(_blobTransactions); + _blobFinder = new BlobFinder(_blobTransactions); _headInfo.HeadChanged += OnHeadChange; @@ -185,8 +185,10 @@ public Transaction[] GetPendingTransactionsBySender(Address address) => public int GetPendingBlobTransactionsCount() => _blobTransactions.Count; - public IEnumerable GetBlobsAndProofs(byte[][] blobVersionedHashes) => - _blobCatcher.GetBlobsAndProofs(blobVersionedHashes); + public bool TryGetBlobAndProof(byte[] blobVersionedHash, + [NotNullWhen(true)] out byte[]? blob, + [NotNullWhen(true)] out byte[]? proof) + => _blobFinder.TryGetBlobAndProof(blobVersionedHash, out blob, out proof); private void OnHeadChange(object? sender, BlockReplacementEventArgs e) { From 83789f821b93db1384caa833a78144f00bbe7d2e Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 21 Aug 2024 10:43:40 +0200 Subject: [PATCH 33/38] refactor tests --- .../EngineModuleTests.V3.cs | 34 +++---------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs index f85729bb96f..f24bfd65fea 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs @@ -563,24 +563,12 @@ public async Task GetBlobsV1_should_return_requested_blobs([Values(1, 2, 3, 4, 5 chain.TxPool.SubmitTx(blobTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); - List blobsAndProofs = new(numberOfBlobs); - if (blobTx.NetworkWrapper is ShardBlobNetworkWrapper wrapper) - { - for (int i = 0; i < numberOfBlobs; i++) - { - blobsAndProofs.Add(new BlobAndProofV1(wrapper.Blobs[i], wrapper.Proofs[i])); - } - } - - GetBlobsV1Result expected = new(blobsAndProofs.ToArray()); - ResultWrapper result = await rpcModule.engine_getBlobsV1(blobTx.BlobVersionedHashes!); - result.Data.Should().BeEquivalentTo(expected); BlobAndProofV1?[] resultBlobsAndProofs = result.Data.BlobsAndProofs.ToArray(); - ShardBlobNetworkWrapper wrapper = ((ShardBlobNetworkWrapper)blobTx.NetworkWrapper!); - resultBlobsAndProofs.Select(b => b.Blob).Should().BeEquivalentTo(wrapper.Blobs); - resultBlobsAndProofs.Select(b => b.Proof).Should().BeEquivalentTo(wrapper.Proofs); + ShardBlobNetworkWrapper wrapper = (ShardBlobNetworkWrapper)blobTx.NetworkWrapper!; + resultBlobsAndProofs.Select(b => b!.Blob).Should().BeEquivalentTo(wrapper.Blobs); + resultBlobsAndProofs.Select(b => b!.Proof).Should().BeEquivalentTo(wrapper.Proofs); } [Test] @@ -597,23 +585,11 @@ public async Task GetBlobsV1_should_return_nulls_when_blobs_not_found([Values(1, .WithMaxFeePerBlobGas(1000.Wei()) .SignedAndResolved(chain.EthereumEcdsa, TestItem.PrivateKeyA).TestObject; - List blobsAndProofs = new(numberOfRequestedBlobs); - for (int i = 0; i < numberOfRequestedBlobs; i++) - { - blobsAndProofs.Add(null); - } - GetBlobsV1Result expected = new(blobsAndProofs.ToArray()); - // requesting hashes that are not present in TxPool ResultWrapper result = await rpcModule.engine_getBlobsV1(blobTx.BlobVersionedHashes!); - result.Data.Should().BeEquivalentTo(expected); - BlobAndProofV1?[] resultBlobsAndProofs = result.Data.BlobsAndProofs.ToArray(); - resultBlobsAndProofs.Length.Should().Be(numberOfRequestedBlobs); - for (int i = 0; i < numberOfRequestedBlobs; i++) - { - resultBlobsAndProofs[i]!.Should().BeNull(); - } + result.Data.BlobsAndProofs.Should().HaveCount(numberOfRequestedBlobs); + result.Data.BlobsAndProofs.Should().AllBeEquivalentTo(null); } [Test] From 9ec8f2383e90137d2fefbb557c3058a3f6a66b00 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 21 Aug 2024 11:17:43 +0200 Subject: [PATCH 34/38] drop GetBlobIndex --- .../Nethermind.TxPool.Test/TxPoolTests.Blobs.cs | 6 +++--- .../Collections/BlobTxDistinctSortedPool.cs | 10 ++++------ .../Collections/PersistentBlobTxDistinctSortedPool.cs | 2 +- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index da6f2aa0ca6..42fdaf69b59 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -653,7 +653,7 @@ public void should_index_blobs_when_adding_txs([Values(true, false)] bool isPers blobPool.TryInsert(blobTxs[i].Hash, blobTxs[i], out _).Should().BeTrue(); } - blobPool.GetBlobIndex.Count.Should().Be(uniqueBlobs ? poolSize : 1); + blobPool.BlobIndex.Count.Should().Be(uniqueBlobs ? poolSize : 1); // first half of txs (0, poolSize - 1) was evicted and should be removed from index // second half (poolSize, 2x poolSize - 1) should be indexed @@ -662,7 +662,7 @@ public void should_index_blobs_when_adding_txs([Values(true, false)] bool isPers // if blobs are unique, we expect index to have 10 keys (poolSize, 2x poolSize - 1) with 1 value each if (uniqueBlobs) { - blobPool.GetBlobIndex.TryGetValue(blobTxs[i].BlobVersionedHashes[0]!, out List txHashes).Should().Be(i >= poolSize); + blobPool.BlobIndex.TryGetValue(blobTxs[i].BlobVersionedHashes[0]!, out List txHashes).Should().Be(i >= poolSize); if (i >= poolSize) { txHashes.Count.Should().Be(1); @@ -672,7 +672,7 @@ public void should_index_blobs_when_adding_txs([Values(true, false)] bool isPers // if blobs are not unique, we expect index to have 1 key with 10 values (poolSize, 2x poolSize - 1) else { - blobPool.GetBlobIndex.TryGetValue(blobTxs[i].BlobVersionedHashes[0]!, out List values).Should().BeTrue(); + blobPool.BlobIndex.TryGetValue(blobTxs[i].BlobVersionedHashes[0]!, out List values).Should().BeTrue(); values.Count.Should().Be(poolSize); values.Contains(blobTxs[i].Hash).Should().Be(i >= poolSize); } diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index e9b10e0269c..7d73d0cef28 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -16,9 +16,7 @@ public class BlobTxDistinctSortedPool(int capacity, IComparer compa { protected override string ShortPoolName => "BlobPool"; - internal ConcurrentDictionary> GetBlobIndex => _blobIndex; - - private readonly ConcurrentDictionary> _blobIndex = new(Bytes.EqualityComparer); + internal readonly ConcurrentDictionary> BlobIndex = new(Bytes.EqualityComparer); protected override IComparer GetReplacementComparer(IComparer comparer) => comparer.GetBlobReplacementComparer(); @@ -42,7 +40,7 @@ protected void AddToBlobIndex(Transaction blobTx) { if (blobVersionedHash?.Length == KzgPolynomialCommitments.BytesPerBlobVersionedHash) { - _blobIndex.AddOrUpdate(blobVersionedHash, + BlobIndex.AddOrUpdate(blobVersionedHash, k => [blobTx.Hash!], (k, b) => { @@ -75,11 +73,11 @@ private void RemoveFromBlobIndex(Transaction blobTx) foreach (var blobVersionedHash in blobTx.BlobVersionedHashes) { if (blobVersionedHash is not null - && _blobIndex.TryGetValue(blobVersionedHash, out List? txHashes)) + && BlobIndex.TryGetValue(blobVersionedHash, out List? txHashes)) { if (txHashes.Count < 2) { - _blobIndex.Remove(blobVersionedHash, out _); + BlobIndex.Remove(blobVersionedHash, out _); } else { diff --git a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs index 8933d014f16..d0f7eb34d92 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs @@ -48,7 +48,7 @@ private void RecreateLightTxCollectionAndCache(ITxStorage blobTxStorage) { long loadingTime = stopwatch.ElapsedMilliseconds; _logger.Info($"Loaded {numberOfTxsInDb} blob txs from persistent db, containing {numberOfBlobsInDb} blobs, in {loadingTime}ms"); - _logger.Info($"There are {GetBlobIndex.Count} unique blobs indexed"); + _logger.Info($"There are {BlobIndex.Count} unique blobs indexed"); } stopwatch.Stop(); } From 5ae5fcf78778ce87c8c997a854a338dac6e77afc Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 21 Aug 2024 11:21:16 +0200 Subject: [PATCH 35/38] fix: rename in one more place --- src/Nethermind/Nethermind.TxPool/BlobFinder.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool/BlobFinder.cs b/src/Nethermind/Nethermind.TxPool/BlobFinder.cs index 3106688e4cf..dadcfd3ecb2 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobFinder.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobFinder.cs @@ -16,7 +16,7 @@ public bool TryGetBlobAndProof(byte[] requestedBlobVersionedHash, [NotNullWhen(true)] out byte[]? blob, [NotNullWhen(true)] out byte[]? proof) { - if (blobPool.GetBlobIndex.TryGetValue(requestedBlobVersionedHash, out List? txHashes) + if (blobPool.BlobIndex.TryGetValue(requestedBlobVersionedHash, out List? txHashes) && txHashes[0] is not null && blobPool.TryGetValue(txHashes[0], out Transaction? blobTx) && blobTx.BlobVersionedHashes?.Length > 0) From e4fadd7e5d874eeda5baef3bbb6f69ab5f8297d2 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 21 Aug 2024 11:35:58 +0200 Subject: [PATCH 36/38] drop BlobFinder --- .../Nethermind.TxPool/BlobFinder.cs | 40 ------------------- .../Collections/BlobTxDistinctSortedPool.cs | 27 +++++++++++++ src/Nethermind/Nethermind.TxPool/TxPool.cs | 5 +-- 3 files changed, 28 insertions(+), 44 deletions(-) delete mode 100644 src/Nethermind/Nethermind.TxPool/BlobFinder.cs diff --git a/src/Nethermind/Nethermind.TxPool/BlobFinder.cs b/src/Nethermind/Nethermind.TxPool/BlobFinder.cs deleted file mode 100644 index dadcfd3ecb2..00000000000 --- a/src/Nethermind/Nethermind.TxPool/BlobFinder.cs +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using Nethermind.Core; -using Nethermind.Core.Crypto; -using Nethermind.Core.Extensions; -using Nethermind.TxPool.Collections; - -namespace Nethermind.TxPool; - -public class BlobFinder(BlobTxDistinctSortedPool blobPool) -{ - public bool TryGetBlobAndProof(byte[] requestedBlobVersionedHash, - [NotNullWhen(true)] out byte[]? blob, - [NotNullWhen(true)] out byte[]? proof) - { - if (blobPool.BlobIndex.TryGetValue(requestedBlobVersionedHash, out List? txHashes) - && txHashes[0] is not null - && blobPool.TryGetValue(txHashes[0], out Transaction? blobTx) - && blobTx.BlobVersionedHashes?.Length > 0) - { - for (int indexOfBlob = 0; indexOfBlob < blobTx.BlobVersionedHashes.Length; indexOfBlob++) - { - if (Bytes.AreEqual(blobTx.BlobVersionedHashes[indexOfBlob], requestedBlobVersionedHash) - && blobTx.NetworkWrapper is ShardBlobNetworkWrapper wrapper) - { - blob = wrapper.Blobs[indexOfBlob]; - proof = wrapper.Proofs[indexOfBlob]; - return true; - } - } - } - - blob = default; - proof = default; - return false; - } -} diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index 7d73d0cef28..d447cf4af9f 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -3,6 +3,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; @@ -21,6 +22,32 @@ public class BlobTxDistinctSortedPool(int capacity, IComparer compa protected override IComparer GetReplacementComparer(IComparer comparer) => comparer.GetBlobReplacementComparer(); + public bool TryGetBlobAndProof(byte[] requestedBlobVersionedHash, + [NotNullWhen(true)] out byte[]? blob, + [NotNullWhen(true)] out byte[]? proof) + { + if (BlobIndex.TryGetValue(requestedBlobVersionedHash, out List? txHashes) + && txHashes[0] is not null + && TryGetValue(txHashes[0], out Transaction? blobTx) + && blobTx.BlobVersionedHashes?.Length > 0) + { + for (int indexOfBlob = 0; indexOfBlob < blobTx.BlobVersionedHashes.Length; indexOfBlob++) + { + if (Bytes.AreEqual(blobTx.BlobVersionedHashes[indexOfBlob], requestedBlobVersionedHash) + && blobTx.NetworkWrapper is ShardBlobNetworkWrapper wrapper) + { + blob = wrapper.Blobs[indexOfBlob]; + proof = wrapper.Proofs[indexOfBlob]; + return true; + } + } + } + + blob = default; + proof = default; + return false; + } + public override bool TryInsert(ValueHash256 hash, Transaction blobTx, out Transaction? removed) { if (base.TryInsert(blobTx.Hash, blobTx, out removed)) diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 694fa14b712..05055748f69 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -42,7 +42,6 @@ public class TxPool : ITxPool, IDisposable private readonly TxDistinctSortedPool _transactions; private readonly BlobTxDistinctSortedPool _blobTransactions; - private readonly BlobFinder _blobFinder; private readonly IChainHeadSpecProvider _specProvider; private readonly IAccountStateProvider _accounts; @@ -120,8 +119,6 @@ public TxPool(IEthereumEcdsa ecdsa, : new BlobTxDistinctSortedPool(txPoolConfig.BlobsSupport == BlobsSupportMode.InMemory ? _txPoolConfig.InMemoryBlobPoolSize : 0, comparer, logManager); if (_blobTransactions.Count > 0) _blobTransactions.UpdatePool(_accounts, _updateBucket); - _blobFinder = new BlobFinder(_blobTransactions); - _headInfo.HeadChanged += OnHeadChange; _preHashFilters = new IIncomingTxFilter[] @@ -188,7 +185,7 @@ public Transaction[] GetPendingTransactionsBySender(Address address) => public bool TryGetBlobAndProof(byte[] blobVersionedHash, [NotNullWhen(true)] out byte[]? blob, [NotNullWhen(true)] out byte[]? proof) - => _blobFinder.TryGetBlobAndProof(blobVersionedHash, out blob, out proof); + => _blobTransactions.TryGetBlobAndProof(blobVersionedHash, out blob, out proof); private void OnHeadChange(object? sender, BlockReplacementEventArgs e) { From cd1bbeecee1762786b45346cb280192550bb8849 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 21 Aug 2024 11:56:01 +0200 Subject: [PATCH 37/38] cosmetics --- .../Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs | 7 +++---- .../Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs index f24bfd65fea..7bd893b0865 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs @@ -565,10 +565,9 @@ public async Task GetBlobsV1_should_return_requested_blobs([Values(1, 2, 3, 4, 5 ResultWrapper result = await rpcModule.engine_getBlobsV1(blobTx.BlobVersionedHashes!); - BlobAndProofV1?[] resultBlobsAndProofs = result.Data.BlobsAndProofs.ToArray(); ShardBlobNetworkWrapper wrapper = (ShardBlobNetworkWrapper)blobTx.NetworkWrapper!; - resultBlobsAndProofs.Select(b => b!.Blob).Should().BeEquivalentTo(wrapper.Blobs); - resultBlobsAndProofs.Select(b => b!.Proof).Should().BeEquivalentTo(wrapper.Proofs); + result.Data.BlobsAndProofs.Select(b => b!.Blob).Should().BeEquivalentTo(wrapper.Blobs); + result.Data.BlobsAndProofs.Select(b => b!.Proof).Should().BeEquivalentTo(wrapper.Proofs); } [Test] @@ -638,7 +637,7 @@ public async Task GetBlobsV1_should_return_mix_of_blobs_and_nulls([Values(1, 2, } else { - result.Data.BlobsAndProofs.ToArray()[i].Should().BeNull(); + resultBlobsAndProofs[i].Should().BeNull(); } } } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs index e6065ad83b1..27fbdad96a3 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs @@ -12,6 +12,7 @@ namespace Nethermind.Merge.Plugin.Handlers; public class GetBlobsHandler(ITxPool txPool) : IAsyncHandler { private const int MaxRequest = 128; + public Task> HandleAsync(byte[][] request) { if (request.Length > MaxRequest) From c9797b0b361e852d641566caa51d0e2aa66f265f Mon Sep 17 00:00:00 2001 From: Marcin Sobczak <77129288+marcindsobczak@users.noreply.github.com> Date: Wed, 21 Aug 2024 17:40:36 +0200 Subject: [PATCH 38/38] Update src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs Co-authored-by: Ruben Buniatyan --- src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs b/src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs index 3ed3015399e..fd4b6215b94 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Data/GetBlobsV1Result.cs @@ -8,5 +8,5 @@ namespace Nethermind.Merge.Plugin.Data; public class GetBlobsV1Result(IEnumerable blobsAndProofs) { - public readonly IEnumerable BlobsAndProofs = blobsAndProofs; + public IEnumerable BlobsAndProofs { get; } = blobsAndProofs; }