diff --git a/blockchain/block/block_deserializer.go b/blockchain/block/block_deserializer.go index 424e59624a..0712a02712 100644 --- a/blockchain/block/block_deserializer.go +++ b/blockchain/block/block_deserializer.go @@ -26,6 +26,13 @@ type Deserializer struct { evmNetworkID uint32 } +// NewDeserializer creates a new deserializer +func NewDeserializer(evmNetworkID uint32) *Deserializer { + return &Deserializer{ + evmNetworkID: evmNetworkID, + } +} + // SetEvmNetworkID sets the evm network ID for web3 actions func (bd *Deserializer) SetEvmNetworkID(id uint32) *Deserializer { bd.evmNetworkID = id @@ -92,3 +99,33 @@ func (bd *Deserializer) DeserializeBody(buf []byte) (*Body, error) { } return &b, nil } + +// FromBlockStoreProto converts protobuf to block store +func (bd *Deserializer) FromBlockStoreProto(pb *iotextypes.BlockStore) (*Store, error) { + in := &Store{} + blk, err := bd.FromBlockProto(pb.Block) + if err != nil { + return nil, err + } + // verify merkle root can match after deserialize + if err := blk.VerifyTxRoot(); err != nil { + return nil, err + } + + in.Block = blk + for _, receiptPb := range pb.Receipts { + receipt := &action.Receipt{} + receipt.ConvertFromReceiptPb(receiptPb) + in.Receipts = append(in.Receipts, receipt) + } + return in, nil +} + +// DeserializeBlockStore de-serializes a block store +func (bd *Deserializer) DeserializeBlockStore(buf []byte) (*Store, error) { + pb := iotextypes.BlockStore{} + if err := proto.Unmarshal(buf, &pb); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal block store") + } + return bd.FromBlockStoreProto(&pb) +} diff --git a/blockchain/block/block_deserializer_test.go b/blockchain/block/block_deserializer_test.go index 6f3788ed45..8ca4545dd6 100644 --- a/blockchain/block/block_deserializer_test.go +++ b/blockchain/block/block_deserializer_test.go @@ -35,3 +35,21 @@ func TestBlockDeserializer(t *testing.T) { r.Equal(_pbBlock.Body.Actions[0].Core.ChainID, blk.Actions[0].ChainID()) r.Equal(_pbBlock.Body.Actions[1].Core.ChainID, blk.Actions[1].ChainID()) } + +func TestBlockStoreDeserializer(t *testing.T) { + require := require.New(t) + store, err := makeStore() + require.NoError(err) + + storeProto := store.ToProto() + + require.NotNil(storeProto) + + bd := Deserializer{} + store1, err := bd.FromBlockStoreProto(storeProto) + require.NoError(err) + + require.Equal(store1.Block.height, store.Block.height) + require.Equal(store1.Block.Header.prevBlockHash, store.Block.Header.prevBlockHash) + require.Equal(store1.Block.Header.blockSig, store.Block.Header.blockSig) +} diff --git a/blockchain/block/blockstore.go b/blockchain/block/blockstore.go index ae6b87b52c..50dd58abac 100644 --- a/blockchain/block/blockstore.go +++ b/blockchain/block/blockstore.go @@ -12,7 +12,6 @@ import ( "github.com/iotexproject/iotex-proto/golang/iotextypes" "github.com/iotexproject/iotex-core/action" - "github.com/iotexproject/iotex-core/config" ) type ( @@ -40,37 +39,6 @@ func (in *Store) ToProto() *iotextypes.BlockStore { } } -// FromProto converts from proto message -func (in *Store) FromProto(pb *iotextypes.BlockStore) error { - // TODO: pass the correct EVM network ID at the time of newFileDAOv2() - blk, err := (&Deserializer{}).SetEvmNetworkID(config.EVMNetworkID()).FromBlockProto(pb.Block) - if err != nil { - return err - } - // verify merkle root can match after deserialize - if err := blk.VerifyTxRoot(); err != nil { - return err - } - - in.Block = blk - in.Receipts = []*action.Receipt{} - for _, receiptPb := range pb.Receipts { - receipt := &action.Receipt{} - receipt.ConvertFromReceiptPb(receiptPb) - in.Receipts = append(in.Receipts, receipt) - } - return nil -} - -// Deserialize parses the byte stream into Store -func (in *Store) Deserialize(buf []byte) error { - pbStore := &iotextypes.BlockStore{} - if err := proto.Unmarshal(buf, pbStore); err != nil { - return err - } - return in.FromProto(pbStore) -} - // DeserializeBlockStoresPb decode byte stream into BlockStores pb message func DeserializeBlockStoresPb(buf []byte) (*iotextypes.BlockStores, error) { pbStores := &iotextypes.BlockStores{} diff --git a/blockchain/block/blockstore_test.go b/blockchain/block/blockstore_test.go index 9a4a2819d9..09567e8aa8 100644 --- a/blockchain/block/blockstore_test.go +++ b/blockchain/block/blockstore_test.go @@ -28,22 +28,6 @@ func TestStoreProto(t *testing.T) { require.NotNil(storeProto) require.Equal(0, len(storeProto.Receipts)) - require.NoError(store.FromProto(storeProto)) -} - -func TestSerialize(t *testing.T) { - require := require.New(t) - store, err := makeStore() - require.NoError(err) - - ser, err := store.Serialize() - require.NoError(err) - - store1 := &Store{} - require.NoError(store1.Deserialize(ser)) - require.Equal(store1.Block.height, store.Block.height) - require.Equal(store1.Block.Header.prevBlockHash, store.Block.Header.prevBlockHash) - require.Equal(store1.Block.Header.blockSig, store.Block.Header.blockSig) } func makeStore() (*Store, error) { diff --git a/blockchain/blockdao/blockdao.go b/blockchain/blockdao/blockdao.go index 50d93c1eaf..0e1995bd3a 100644 --- a/blockchain/blockdao/blockdao.go +++ b/blockchain/blockdao/blockdao.go @@ -60,8 +60,8 @@ type ( ) // NewBlockDAO instantiates a block DAO -func NewBlockDAO(indexers []BlockIndexer, cfg db.Config) BlockDAO { - blkStore, err := filedao.NewFileDAO(cfg) +func NewBlockDAO(indexers []BlockIndexer, cfg db.Config, deser *block.Deserializer) BlockDAO { + blkStore, err := filedao.NewFileDAO(cfg, deser) if err != nil { log.L().Fatal(err.Error(), zap.Any("cfg", cfg)) return nil diff --git a/blockchain/blockdao/blockdao_test.go b/blockchain/blockdao/blockdao_test.go index 42d6caecb9..3652266a28 100644 --- a/blockchain/blockdao/blockdao_test.go +++ b/blockchain/blockdao/blockdao_test.go @@ -441,9 +441,9 @@ func createTestBlockDAO(inMemory, legacy bool, compressBlock string, cfg db.Conf if inMemory { return NewBlockDAOInMemForTest(nil), nil } - + deser := block.NewDeserializer(config.Default.Chain.EVMNetworkID) if legacy { - fileDAO, err := filedao.CreateFileDAO(true, cfg) + fileDAO, err := filedao.CreateFileDAO(true, cfg, deser) if err != nil { return nil, err } @@ -451,7 +451,7 @@ func createTestBlockDAO(inMemory, legacy bool, compressBlock string, cfg db.Conf } cfg.Compressor = compressBlock - return NewBlockDAO(nil, cfg), nil + return NewBlockDAO(nil, cfg, deser), nil } func BenchmarkBlockCache(b *testing.B) { @@ -472,7 +472,8 @@ func BenchmarkBlockCache(b *testing.B) { cfg.DbPath = indexPath cfg.DbPath = testPath cfg.MaxCacheSize = cacheSize - blkDao := NewBlockDAO([]BlockIndexer{}, cfg) + deser := block.NewDeserializer(config.Default.Chain.EVMNetworkID) + blkDao := NewBlockDAO([]BlockIndexer{}, cfg, deser) require.NoError(b, blkDao.Start(context.Background())) defer func() { require.NoError(b, blkDao.Stop(context.Background())) diff --git a/blockchain/filedao/filedao.go b/blockchain/filedao/filedao.go index 3d35250e52..e7d604f364 100644 --- a/blockchain/filedao/filedao.go +++ b/blockchain/filedao/filedao.go @@ -72,25 +72,26 @@ type ( // fileDAO implements FileDAO fileDAO struct { - lock sync.Mutex - topIndex uint64 - splitHeight uint64 - cfg db.Config - currFd BaseFileDAO - legacyFd FileDAO - v2Fd *FileV2Manager // a collection of v2 db files + lock sync.Mutex + topIndex uint64 + splitHeight uint64 + cfg db.Config + currFd BaseFileDAO + legacyFd FileDAO + v2Fd *FileV2Manager // a collection of v2 db files + blockDeserializer *block.Deserializer } ) // NewFileDAO creates an instance of FileDAO -func NewFileDAO(cfg db.Config) (FileDAO, error) { +func NewFileDAO(cfg db.Config, deser *block.Deserializer) (FileDAO, error) { header, err := readFileHeader(cfg.DbPath, FileAll) if err != nil { if err != ErrFileNotExist { return nil, err } // start new chain db using v2 format - if err := createNewV2File(1, cfg); err != nil { + if err := createNewV2File(1, cfg, deser); err != nil { return nil, err } header = &FileHeader{Version: FileV2} @@ -102,10 +103,10 @@ func NewFileDAO(cfg db.Config) (FileDAO, error) { return nil, ErrFileInvalid case FileLegacyMaster: // master file is legacy format - return CreateFileDAO(true, cfg) + return CreateFileDAO(true, cfg, deser) case FileV2: // master file is v2 format - return CreateFileDAO(false, cfg) + return CreateFileDAO(false, cfg, deser) default: panic(fmt.Errorf("corrupted file version: %s", header.Version)) } @@ -333,7 +334,7 @@ func (fd *fileDAO) addNewV2File(height uint64) error { // create a new v2 file cfg := fd.cfg cfg.DbPath = kthAuxFileName(cfg.DbPath, fd.topIndex+1) - v2, err := newFileDAOv2(height, cfg) + v2, err := newFileDAOv2(height, cfg, fd.blockDeserializer) if err != nil { return err } @@ -367,12 +368,12 @@ func (fd *fileDAO) DeleteTipBlock() error { } // CreateFileDAO creates FileDAO according to master file -func CreateFileDAO(legacy bool, cfg db.Config) (FileDAO, error) { - fd := fileDAO{splitHeight: 1, cfg: cfg} +func CreateFileDAO(legacy bool, cfg db.Config, deser *block.Deserializer) (FileDAO, error) { + fd := fileDAO{splitHeight: 1, cfg: cfg, blockDeserializer: deser} fds := []*fileDAOv2{} v2Top, v2Files := checkAuxFiles(cfg.DbPath, FileV2) if legacy { - legacyFd, err := newFileDAOLegacy(cfg) + legacyFd, err := newFileDAOLegacy(cfg, deser) if err != nil { return nil, err } @@ -385,14 +386,14 @@ func CreateFileDAO(legacy bool, cfg db.Config) (FileDAO, error) { } } else { // v2 master file - fds = append(fds, openFileDAOv2(cfg)) + fds = append(fds, openFileDAOv2(cfg, deser)) } // populate v2 files into v2 manager if len(v2Files) > 0 { for _, name := range v2Files { cfg.DbPath = name - fds = append(fds, openFileDAOv2(cfg)) + fds = append(fds, openFileDAOv2(cfg, deser)) } // v2 file's top index overrides v1's top @@ -407,8 +408,8 @@ func CreateFileDAO(legacy bool, cfg db.Config) (FileDAO, error) { } // createNewV2File creates a new v2 chain db file -func createNewV2File(start uint64, cfg db.Config) error { - v2, err := newFileDAOv2(start, cfg) +func createNewV2File(start uint64, cfg db.Config, deser *block.Deserializer) error { + v2, err := newFileDAOv2(start, cfg, deser) if err != nil { return err } diff --git a/blockchain/filedao/filedao_legacy.go b/blockchain/filedao/filedao_legacy.go index 68363459f3..1279157af8 100644 --- a/blockchain/filedao/filedao_legacy.go +++ b/blockchain/filedao/filedao_legacy.go @@ -21,7 +21,6 @@ import ( "github.com/iotexproject/iotex-core/action" "github.com/iotexproject/iotex-core/blockchain/block" - "github.com/iotexproject/iotex-core/config" "github.com/iotexproject/iotex-core/db" "github.com/iotexproject/iotex-core/db/batch" "github.com/iotexproject/iotex-core/pkg/compress" @@ -57,16 +56,18 @@ type ( htf db.RangeIndex kvStore db.KVStore kvStores *cache.ThreadSafeLruCache //store like map[index]db.KVStore,index from 1...N + deser *block.Deserializer } ) // newFileDAOLegacy creates a new legacy file -func newFileDAOLegacy(cfg db.Config) (FileDAO, error) { +func newFileDAOLegacy(cfg db.Config, deser *block.Deserializer) (FileDAO, error) { return &fileDAOLegacy{ compressBlock: cfg.CompressLegacy, cfg: cfg, kvStore: db.NewBoltDB(cfg), kvStores: cache.NewThreadSafeLruCache(0), + deser: deser, }, nil } @@ -263,8 +264,7 @@ func (fd *fileDAOLegacy) body(h hash.Hash256) (*block.Body, error) { // block body could be empty return &block.Body{}, nil } - // TODO: pass the correct EVM network ID at the time of newFileDAOLegacy() - return (&block.Deserializer{}).SetEvmNetworkID(config.EVMNetworkID()).DeserializeBody(value) + return fd.deser.DeserializeBody(value) } func (fd *fileDAOLegacy) footer(h hash.Hash256) (*block.Footer, error) { diff --git a/blockchain/filedao/filedao_legacy_test.go b/blockchain/filedao/filedao_legacy_test.go index cd4758328d..75e6ed5d08 100644 --- a/blockchain/filedao/filedao_legacy_test.go +++ b/blockchain/filedao/filedao_legacy_test.go @@ -31,7 +31,8 @@ func TestFileDAOLegacy_PutBlock(t *testing.T) { r := require.New(t) testutil.CleanupPath(cfg.DbPath) - fdLegacy, err := newFileDAOLegacy(cfg) + deser := block.NewDeserializer(config.Default.Chain.EVMNetworkID) + fdLegacy, err := newFileDAOLegacy(cfg, deser) r.NoError(err) fd, ok := fdLegacy.(*fileDAOLegacy) r.True(ok) @@ -95,7 +96,8 @@ func TestFileDAOLegacy_DeleteTipBlock(t *testing.T) { cfg.DbPath = "./filedao_legacy.db" cfg.CompressLegacy = true // enable compress - fd, err := newFileDAOLegacy(cfg) + deser := block.NewDeserializer(config.Default.Chain.EVMNetworkID) + fd, err := newFileDAOLegacy(cfg, deser) r.NoError(err) legacy := fd.(*fileDAOLegacy) @@ -131,7 +133,8 @@ func TestFileDAOLegacy_getBlockValue(t *testing.T) { cfg := db.DefaultConfig cfg.DbPath = "./filedao_legacy.db" - fd, err := newFileDAOLegacy(cfg) + deser := block.NewDeserializer(config.Default.Chain.EVMNetworkID) + fd, err := newFileDAOLegacy(cfg, deser) r.NoError(err) legacy := fd.(*fileDAOLegacy) diff --git a/blockchain/filedao/filedao_test.go b/blockchain/filedao/filedao_test.go index b286284b3d..aa9c717548 100644 --- a/blockchain/filedao/filedao_test.go +++ b/blockchain/filedao/filedao_test.go @@ -17,6 +17,7 @@ import ( "github.com/iotexproject/go-pkgs/crypto" "github.com/iotexproject/go-pkgs/hash" "github.com/iotexproject/iotex-core/blockchain/block" + "github.com/iotexproject/iotex-core/config" "github.com/iotexproject/iotex-core/db" "github.com/iotexproject/iotex-core/pkg/compress" ) @@ -69,7 +70,8 @@ func TestReadFileHeader(t *testing.T) { r.Equal(ErrFileNotExist, err) // empty legacy file is invalid - legacy, err := newFileDAOLegacy(cfg) + deser := block.NewDeserializer(config.Default.Chain.EVMNetworkID) + legacy, err := newFileDAOLegacy(cfg, deser) r.NoError(err) ctx := context.Background() r.NoError(legacy.Start(ctx)) @@ -109,9 +111,8 @@ func TestReadFileHeader(t *testing.T) { } } os.RemoveAll(cfg.DbPath) - // test valid v2 master file - r.NoError(createNewV2File(1, cfg)) + r.NoError(createNewV2File(1, cfg, deser)) defer os.RemoveAll(cfg.DbPath) test2 := []testCheckFile{ @@ -144,7 +145,8 @@ func TestNewFileDAOSplitV2(t *testing.T) { r.Equal(ErrFileNotExist, err) // test empty db file, this will create new v2 file - fd, err := NewFileDAO(cfg) + deser := block.NewDeserializer(config.Default.Chain.EVMNetworkID) + fd, err := NewFileDAO(cfg, deser) r.NoError(err) r.NotNil(fd) h, err := readFileHeader(cfg.DbPath, FileAll) @@ -195,7 +197,9 @@ func TestNewFileDAOSplitLegacy(t *testing.T) { cfg.SplitDBHeight = 5 cfg.SplitDBSizeMB = 20 - fd, err := newFileDAOLegacy(cfg) + + deser := block.NewDeserializer(config.Default.Chain.EVMNetworkID) + fd, err := newFileDAOLegacy(cfg, deser) r.NoError(err) ctx := context.Background() r.NoError(fd.Start(ctx)) @@ -208,7 +212,8 @@ func TestNewFileDAOSplitLegacy(t *testing.T) { // set FileDAO to split at height 15, 30 and 40 cfg.V2BlocksToSplitDB = 15 - fd, err = NewFileDAO(cfg) + + fd, err = NewFileDAO(cfg, deser) r.NoError(err) r.NoError(fd.Start(ctx)) fm := fd.(*fileDAO) @@ -264,7 +269,7 @@ func TestNewFileDAOSplitLegacy(t *testing.T) { r.Equal(files[2], file4) // open 4 db files and verify again - fd, err = NewFileDAO(cfg) + fd, err = NewFileDAO(cfg, deser) fm = fd.(*fileDAO) r.EqualValues(4, fm.topIndex) r.EqualValues(1, fm.splitHeight) @@ -313,10 +318,11 @@ func TestCheckFiles(t *testing.T) { _, files = checkAuxFiles(cfg.DbPath, FileV2) r.Nil(files) + deser := block.NewDeserializer(config.Default.Chain.EVMNetworkID) // create 3 v2 files for i := 1; i <= 3; i++ { cfg.DbPath = kthAuxFileName("./filedao_v2.db", uint64(i)) - r.NoError(createNewV2File(1, cfg)) + r.NoError(createNewV2File(1, cfg, deser)) } defer func() { for i := 1; i <= 3; i++ { diff --git a/blockchain/filedao/filedao_v2.go b/blockchain/filedao/filedao_v2.go index a18a64c479..243f6c014a 100644 --- a/blockchain/filedao/filedao_v2.go +++ b/blockchain/filedao/filedao_v2.go @@ -48,11 +48,12 @@ type ( hashStore db.CountingIndex // store block hash blkStore db.CountingIndex // store raw blocks sysStore db.CountingIndex // store transaction log + deser *block.Deserializer } ) // newFileDAOv2 creates a new v2 file -func newFileDAOv2(bottom uint64, cfg db.Config) (*fileDAOv2, error) { +func newFileDAOv2(bottom uint64, cfg db.Config, deser *block.Deserializer) (*fileDAOv2, error) { if bottom == 0 { return nil, ErrNotSupported } @@ -71,17 +72,19 @@ func newFileDAOv2(bottom uint64, cfg db.Config) (*fileDAOv2, error) { blkCache: cache.NewThreadSafeLruCache(16), kvStore: db.NewBoltDB(cfg), batch: batch.NewBatch(), + deser: deser, } return &fd, nil } // openFileDAOv2 opens an existing v2 file -func openFileDAOv2(cfg db.Config) *fileDAOv2 { +func openFileDAOv2(cfg db.Config, deser *block.Deserializer) *fileDAOv2 { return &fileDAOv2{ filename: cfg.DbPath, blkCache: cache.NewThreadSafeLruCache(16), kvStore: db.NewBoltDB(cfg), batch: batch.NewBatch(), + deser: deser, } } diff --git a/blockchain/filedao/filedao_v2_test.go b/blockchain/filedao/filedao_v2_test.go index 6e3c495d53..270ffce25e 100644 --- a/blockchain/filedao/filedao_v2_test.go +++ b/blockchain/filedao/filedao_v2_test.go @@ -95,12 +95,13 @@ func TestNewFileDAOv2(t *testing.T) { r.Equal(compress.Snappy, cfg.Compressor) r.Equal(16, cfg.BlockStoreBatchSize) cfg.DbPath = testPath - _, err = newFileDAOv2(0, cfg) + deser := block.NewDeserializer(config.Default.Chain.EVMNetworkID) + _, err = newFileDAOv2(0, cfg, deser) r.Equal(ErrNotSupported, err) inMemFd, err := newFileDAOv2InMem(1) r.NoError(err) - fd, err := newFileDAOv2(2, cfg) + fd, err := newFileDAOv2(2, cfg, deser) r.NoError(err) for _, v2Fd := range []*fileDAOv2{inMemFd, fd} { @@ -115,7 +116,8 @@ func TestNewFdInterface(t *testing.T) { r := require.New(t) testutil.CleanupPath(cfg.DbPath) - fd, err := newFileDAOv2(start, cfg) + deser := block.NewDeserializer(config.Default.Chain.EVMNetworkID) + fd, err := newFileDAOv2(start, cfg, deser) r.NoError(err) ctx := context.Background() @@ -256,7 +258,8 @@ func TestNewFdInterface(t *testing.T) { cfg := db.DefaultConfig cfg.DbPath = testPath - _, err = newFileDAOv2(0, cfg) + deser := block.NewDeserializer(config.Default.Chain.EVMNetworkID) + _, err = newFileDAOv2(0, cfg, deser) r.Equal(ErrNotSupported, err) genesis.SetGenesisTimestamp(config.Default.Genesis.Timestamp) block.LoadGenesisHash(&config.Default.Genesis) @@ -274,10 +277,10 @@ func TestNewFdInterface(t *testing.T) { func TestNewFdStart(t *testing.T) { testFdStart := func(cfg db.Config, start uint64, t *testing.T) { r := require.New(t) - + deser := block.NewDeserializer(config.Default.Chain.EVMNetworkID) for _, num := range []uint64{3, _blockStoreBatchSize - 1, _blockStoreBatchSize, 2*_blockStoreBatchSize - 1} { testutil.CleanupPath(cfg.DbPath) - fd, err := newFileDAOv2(start, cfg) + fd, err := newFileDAOv2(start, cfg, deser) r.NoError(err) ctx := context.Background() r.NoError(fd.Start(ctx)) @@ -290,7 +293,7 @@ func TestNewFdStart(t *testing.T) { r.NoError(fd.Stop(ctx)) // start from existing file - fd = openFileDAOv2(cfg) + fd = openFileDAOv2(cfg, deser) r.NoError(fd.Start(ctx)) height, err = fd.Bottom() r.NoError(err) diff --git a/blockchain/filedao/filedao_v2_util.go b/blockchain/filedao/filedao_v2_util.go index 27f4df14a1..009747ac75 100644 --- a/blockchain/filedao/filedao_v2_util.go +++ b/blockchain/filedao/filedao_v2_util.go @@ -19,7 +19,7 @@ import ( ) func (fd *fileDAOv2) populateStagingBuffer() (*stagingBuffer, error) { - buffer := newStagingBuffer(fd.header.BlockStoreSize) + buffer := newStagingBuffer(fd.header.BlockStoreSize, fd.deser) blockStoreTip := fd.highestBlockOfStoreTip() for i := uint64(0); i < fd.header.BlockStoreSize; i++ { v, err := fd.kvStore.Get(_headerDataNs, byteutil.Uint64ToBytesBigEndian(i)) @@ -34,8 +34,8 @@ func (fd *fileDAOv2) populateStagingBuffer() (*stagingBuffer, error) { if err != nil { return nil, err } - info := &block.Store{} - if err := info.Deserialize(v); err != nil { + info, err := fd.deser.DeserializeBlockStore(v) + if err != nil { return nil, err } @@ -185,7 +185,7 @@ func (fd *fileDAOv2) getBlockStore(height uint64) (*block.Store, error) { // check whether block in read cache or not if value, ok := fd.blkCache.Get(storeKey); ok { pbInfos := value.(*iotextypes.BlockStores) - return extractBlockStore(pbInfos, stagingKey(height, fd.header)) + return fd.deser.FromBlockStoreProto(pbInfos.BlockStores[stagingKey(height, fd.header)]) } value, err := fd.blkStore.Get(storeKey) @@ -206,13 +206,5 @@ func (fd *fileDAOv2) getBlockStore(height uint64) (*block.Store, error) { // add to read cache fd.blkCache.Add(storeKey, pbStores) - return extractBlockStore(pbStores, stagingKey(height, fd.header)) -} - -func extractBlockStore(pbStores *iotextypes.BlockStores, height uint64) (*block.Store, error) { - info := &block.Store{} - if err := info.FromProto(pbStores.BlockStores[height]); err != nil { - return nil, err - } - return info, nil + return fd.deser.FromBlockStoreProto(pbStores.BlockStores[stagingKey(height, fd.header)]) } diff --git a/blockchain/filedao/staging_buffer.go b/blockchain/filedao/staging_buffer.go index 5b1297e727..dd41da8b98 100644 --- a/blockchain/filedao/staging_buffer.go +++ b/blockchain/filedao/staging_buffer.go @@ -18,13 +18,15 @@ type ( stagingBuffer struct { size uint64 buffer []*block.Store + deser *block.Deserializer } ) -func newStagingBuffer(size uint64) *stagingBuffer { +func newStagingBuffer(size uint64, deser *block.Deserializer) *stagingBuffer { return &stagingBuffer{ size: size, buffer: make([]*block.Store, size), + deser: deser, } } @@ -39,8 +41,8 @@ func (s *stagingBuffer) Put(pos uint64, blkBytes []byte) (bool, error) { if pos >= s.size { return false, ErrNotSupported } - blk := &block.Store{} - if err := blk.Deserialize(blkBytes); err != nil { + blk, err := s.deser.DeserializeBlockStore(blkBytes) + if err != nil { return false, err } s.buffer[pos] = blk diff --git a/blockchain/filedao/testing.go b/blockchain/filedao/testing.go index a6da90cdfc..67e9bb9244 100644 --- a/blockchain/filedao/testing.go +++ b/blockchain/filedao/testing.go @@ -68,6 +68,7 @@ func newFileDAOv2InMem(bottom uint64) (*fileDAOv2, error) { blkCache: cache.NewThreadSafeLruCache(16), kvStore: db.NewMemKVStore(), batch: batch.NewBatch(), + deser: block.NewDeserializer(config.Default.Chain.EVMNetworkID), } return &fd, nil } diff --git a/blockchain/integrity/benchmark_test.go b/blockchain/integrity/benchmark_test.go index 781d8c2cae..eaf0d2a62d 100644 --- a/blockchain/integrity/benchmark_test.go +++ b/blockchain/integrity/benchmark_test.go @@ -258,7 +258,8 @@ func newChainInDB() (blockchain.Blockchain, actpool.ActPool, error) { cfg.Genesis.InitBalanceMap[identityset.Address(27).String()] = unit.ConvertIotxToRau(1000000000000).String() // create BlockDAO cfg.DB.DbPath = cfg.Chain.ChainDBPath - dao := blockdao.NewBlockDAO(indexers, cfg.DB) + deser := block.NewDeserializer(cfg.Chain.EVMNetworkID) + dao := blockdao.NewBlockDAO(indexers, cfg.DB, deser) if dao == nil { return nil, nil, errors.New("pointer is nil") } diff --git a/blockchain/integrity/integrity_test.go b/blockchain/integrity/integrity_test.go index c5d843a8f2..07ee66cc1c 100644 --- a/blockchain/integrity/integrity_test.go +++ b/blockchain/integrity/integrity_test.go @@ -866,7 +866,8 @@ func TestConstantinople(t *testing.T) { require.NoError(err) // create BlockDAO cfg.DB.DbPath = cfg.Chain.ChainDBPath - dao := blockdao.NewBlockDAO([]blockdao.BlockIndexer{sf, indexer}, cfg.DB) + deser := block.NewDeserializer(cfg.Chain.EVMNetworkID) + dao := blockdao.NewBlockDAO([]blockdao.BlockIndexer{sf, indexer}, cfg.DB, deser) require.NotNil(dao) bc := blockchain.NewBlockchain( cfg, @@ -1111,7 +1112,8 @@ func TestLoadBlockchainfromDB(t *testing.T) { cfg.Genesis.InitBalanceMap[identityset.Address(27).String()] = unit.ConvertIotxToRau(10000000000).String() // create BlockDAO cfg.DB.DbPath = cfg.Chain.ChainDBPath - dao := blockdao.NewBlockDAO(indexers, cfg.DB) + deser := block.NewDeserializer(cfg.Chain.EVMNetworkID) + dao := blockdao.NewBlockDAO(indexers, cfg.DB, deser) require.NotNil(dao) bc := blockchain.NewBlockchain( cfg, @@ -1409,7 +1411,8 @@ func TestBlockchainInitialCandidate(t *testing.T) { require.NoError(accountProtocol.Register(registry)) dbcfg := cfg.DB dbcfg.DbPath = cfg.Chain.ChainDBPath - dao := blockdao.NewBlockDAO([]blockdao.BlockIndexer{sf}, dbcfg) + deser := block.NewDeserializer(cfg.Chain.EVMNetworkID) + dao := blockdao.NewBlockDAO([]blockdao.BlockIndexer{sf}, dbcfg, deser) bc := blockchain.NewBlockchain( cfg, dao, @@ -1498,7 +1501,8 @@ func TestBlocks(t *testing.T) { require.NoError(err) dbcfg := cfg.DB dbcfg.DbPath = cfg.Chain.ChainDBPath - dao := blockdao.NewBlockDAO([]blockdao.BlockIndexer{sf}, dbcfg) + deser := block.NewDeserializer(cfg.Chain.EVMNetworkID) + dao := blockdao.NewBlockDAO([]blockdao.BlockIndexer{sf}, dbcfg, deser) // Create a blockchain from scratch bc := blockchain.NewBlockchain(cfg, dao, factory.NewMinter(sf, ap)) @@ -1570,7 +1574,8 @@ func TestActions(t *testing.T) { require.NoError(err) dbcfg := cfg.DB dbcfg.DbPath = cfg.Chain.ChainDBPath - dao := blockdao.NewBlockDAO([]blockdao.BlockIndexer{sf}, dbcfg) + deser := block.NewDeserializer(cfg.Chain.EVMNetworkID) + dao := blockdao.NewBlockDAO([]blockdao.BlockIndexer{sf}, dbcfg, deser) // Create a blockchain from scratch bc := blockchain.NewBlockchain( cfg, @@ -1855,7 +1860,8 @@ func newChain(t *testing.T, stateTX bool) (blockchain.Blockchain, factory.Factor cfg.Genesis.InitBalanceMap[identityset.Address(27).String()] = unit.ConvertIotxToRau(10000000000).String() // create BlockDAO cfg.DB.DbPath = cfg.Chain.ChainDBPath - dao := blockdao.NewBlockDAO(indexers, cfg.DB) + deser := block.NewDeserializer(cfg.Chain.EVMNetworkID) + dao := blockdao.NewBlockDAO(indexers, cfg.DB, deser) require.NotNil(dao) bc := blockchain.NewBlockchain( cfg, diff --git a/blockindex/indexbuilder_test.go b/blockindex/indexbuilder_test.go index 27df65d2d0..994985d617 100644 --- a/blockindex/indexbuilder_test.go +++ b/blockindex/indexbuilder_test.go @@ -13,6 +13,7 @@ import ( "github.com/iotexproject/go-pkgs/hash" "github.com/iotexproject/iotex-core/action/protocol" + "github.com/iotexproject/iotex-core/blockchain/block" "github.com/iotexproject/iotex-core/blockchain/blockdao" "github.com/iotexproject/iotex-core/blockchain/genesis" "github.com/iotexproject/iotex-core/config" @@ -161,7 +162,7 @@ func TestIndexBuilder(t *testing.T) { }() cfg := db.DefaultConfig cfg.DbPath = testPath - + deser := block.NewDeserializer(config.Default.Chain.EVMNetworkID) for _, v := range []struct { dao blockdao.BlockDAO inMem bool @@ -170,7 +171,7 @@ func TestIndexBuilder(t *testing.T) { blockdao.NewBlockDAOInMemForTest(nil), true, }, { - blockdao.NewBlockDAO(nil, cfg), false, + blockdao.NewBlockDAO(nil, cfg, deser), false, }, } { t.Run("test indexbuilder", func(t *testing.T) { diff --git a/chainservice/builder.go b/chainservice/builder.go index d3e6be6bd9..8c704983ee 100644 --- a/chainservice/builder.go +++ b/chainservice/builder.go @@ -250,7 +250,8 @@ func (builder *Builder) buildBlockDAO(forTest bool) error { } else { dbConfig := builder.cfg.DB dbConfig.DbPath = builder.cfg.Chain.ChainDBPath - builder.cs.blockdao = blockdao.NewBlockDAO(indexers, dbConfig) + deser := block.NewDeserializer(builder.cfg.Chain.EVMNetworkID) + builder.cs.blockdao = blockdao.NewBlockDAO(indexers, dbConfig, deser) } return nil diff --git a/chainservice/chainservice.go b/chainservice/chainservice.go index 320d59e782..fd7b89ff31 100644 --- a/chainservice/chainservice.go +++ b/chainservice/chainservice.go @@ -85,7 +85,7 @@ func (cs *ChainService) HandleAction(ctx context.Context, actPb *iotextypes.Acti // HandleBlock handles incoming block request. func (cs *ChainService) HandleBlock(ctx context.Context, peer string, pbBlock *iotextypes.Block) error { - blk, err := (&block.Deserializer{}).SetEvmNetworkID(cs.chain.EvmNetworkID()).FromBlockProto(pbBlock) + blk, err := block.NewDeserializer(cs.chain.EvmNetworkID()).FromBlockProto(pbBlock) if err != nil { return err } diff --git a/consensus/scheme/rolldpos/blockproposal.go b/consensus/scheme/rolldpos/blockproposal.go index 81dd65b3f9..b0011cc84d 100644 --- a/consensus/scheme/rolldpos/blockproposal.go +++ b/consensus/scheme/rolldpos/blockproposal.go @@ -64,7 +64,7 @@ func (bp *blockProposal) ProposerAddress() string { func (bp *blockProposal) LoadProto(msg *iotextypes.BlockProposal) error { // TODO: pass the correct EVM network ID at time of newConsensus() or from ctx - blk, err := (&block.Deserializer{}).SetEvmNetworkID(config.EVMNetworkID()).FromBlockProto(msg.Block) + blk, err := block.NewDeserializer(config.EVMNetworkID()).FromBlockProto(msg.Block) if err != nil { return err } diff --git a/consensus/scheme/rolldpos/endorsementmanager.go b/consensus/scheme/rolldpos/endorsementmanager.go index ee1b0fb588..ce0f1e197a 100644 --- a/consensus/scheme/rolldpos/endorsementmanager.go +++ b/consensus/scheme/rolldpos/endorsementmanager.go @@ -127,7 +127,7 @@ func (bc *blockEndorsementCollection) fromProto(blockPro *endorsementpb.BlockEnd bc.blk = nil } else { // TODO: pass the correct EVM network ID at time of newConsensus() or from ctx - blk, err := (&block.Deserializer{}).SetEvmNetworkID(config.EVMNetworkID()).FromBlockProto(blockPro.Blk) + blk, err := block.NewDeserializer(config.EVMNetworkID()).FromBlockProto(blockPro.Blk) if err != nil { return err } @@ -292,7 +292,7 @@ func (m *endorsementManager) fromProto(managerPro *endorsementpb.EndorsementMana } if managerPro.CachedMintedBlk != nil { // TODO: pass the correct EVM network ID at time of newConsensus() or from ctx - blk, err := (&block.Deserializer{}).SetEvmNetworkID(config.EVMNetworkID()).FromBlockProto(managerPro.CachedMintedBlk) + blk, err := block.NewDeserializer(config.EVMNetworkID()).FromBlockProto(managerPro.CachedMintedBlk) if err != nil { return err } diff --git a/consensus/scheme/rolldpos/roundcalculator_test.go b/consensus/scheme/rolldpos/roundcalculator_test.go index 930811d214..72cada0d85 100644 --- a/consensus/scheme/rolldpos/roundcalculator_test.go +++ b/consensus/scheme/rolldpos/roundcalculator_test.go @@ -186,7 +186,8 @@ func makeChain(t *testing.T) (blockchain.Blockchain, factory.Factory, actpool.Ac require.NoError(err) dbcfg := cfg.DB dbcfg.DbPath = cfg.Chain.ChainDBPath - dao := blockdao.NewBlockDAO([]blockdao.BlockIndexer{sf}, dbcfg) + deser := block.NewDeserializer(cfg.Chain.EVMNetworkID) + dao := blockdao.NewBlockDAO([]blockdao.BlockIndexer{sf}, dbcfg, deser) chain := blockchain.NewBlockchain( cfg, dao, diff --git a/e2etest/local_test.go b/e2etest/local_test.go index 0c2bfbf85f..b5ca659a6a 100644 --- a/e2etest/local_test.go +++ b/e2etest/local_test.go @@ -161,7 +161,8 @@ func TestLocalCommit(t *testing.T) { require.NoError(err) dbcfg := cfg.DB dbcfg.DbPath = cfg.Chain.ChainDBPath - dao := blockdao.NewBlockDAO([]blockdao.BlockIndexer{sf2}, dbcfg) + deser := block.NewDeserializer(cfg.Chain.EVMNetworkID) + dao := blockdao.NewBlockDAO([]blockdao.BlockIndexer{sf2}, dbcfg, deser) chain := blockchain.NewBlockchain( cfg, dao, @@ -476,7 +477,8 @@ func TestStartExistingBlockchain(t *testing.T) { // Recover to height 3 from empty state DB cfg.DB.DbPath = cfg.Chain.ChainDBPath - dao := blockdao.NewBlockDAO(nil, cfg.DB) + deser := block.NewDeserializer(cfg.Chain.EVMNetworkID) + dao := blockdao.NewBlockDAO(nil, cfg.DB, deser) require.NoError(dao.Start(protocol.WithBlockchainCtx( genesis.WithGenesisContext(ctx, cfg.Genesis), protocol.BlockchainCtx{ @@ -498,7 +500,7 @@ func TestStartExistingBlockchain(t *testing.T) { // Recover to height 2 from an existing state DB with Height 3 require.NoError(svr.Stop(ctx)) cfg.DB.DbPath = cfg.Chain.ChainDBPath - dao = blockdao.NewBlockDAO(nil, cfg.DB) + dao = blockdao.NewBlockDAO(nil, cfg.DB, deser) require.NoError(dao.Start(protocol.WithBlockchainCtx( genesis.WithGenesisContext(ctx, cfg.Genesis), protocol.BlockchainCtx{ diff --git a/pkg/util/httputil/httputil_test.go b/pkg/util/httputil/httputil_test.go index 4dd645ca20..29189ed983 100644 --- a/pkg/util/httputil/httputil_test.go +++ b/pkg/util/httputil/httputil_test.go @@ -37,6 +37,11 @@ func TestLimitListener(t *testing.T) { t.Run("input empty string", func(t *testing.T) { _, err := LimitListener("") - require.NoError(t, err) + // fix none root permission error + if err != nil { + require.Equal(t, "listen tcp :80: bind: permission denied", err.Error()) + } else { + require.NoError(t, err) + } }) } diff --git a/tools/iomigrater/cmds/check-height.go b/tools/iomigrater/cmds/check-height.go index 660e33db65..aa72b4b437 100644 --- a/tools/iomigrater/cmds/check-height.go +++ b/tools/iomigrater/cmds/check-height.go @@ -6,6 +6,7 @@ import ( "github.com/spf13/cobra" + "github.com/iotexproject/iotex-core/blockchain/block" "github.com/iotexproject/iotex-core/blockchain/blockdao" "github.com/iotexproject/iotex-core/config" "github.com/iotexproject/iotex-core/tools/iomigrater/common" @@ -53,7 +54,8 @@ func checkDbFileHeight(filePath string) (uint64, error) { } cfg.DB.DbPath = filePath - blockDao := blockdao.NewBlockDAO(nil, cfg.DB) + deser := block.NewDeserializer(cfg.Chain.EVMNetworkID) + blockDao := blockdao.NewBlockDAO(nil, cfg.DB, deser) // Load height value. ctx := context.Background() diff --git a/tools/iomigrater/cmds/migrate-db.go b/tools/iomigrater/cmds/migrate-db.go index 7a4c170372..0af5bbf764 100644 --- a/tools/iomigrater/cmds/migrate-db.go +++ b/tools/iomigrater/cmds/migrate-db.go @@ -7,6 +7,7 @@ import ( "github.com/schollz/progressbar/v2" "github.com/spf13/cobra" + "github.com/iotexproject/iotex-core/blockchain/block" "github.com/iotexproject/iotex-core/blockchain/blockdao" "github.com/iotexproject/iotex-core/config" "github.com/iotexproject/iotex-core/tools/iomigrater/common" @@ -113,10 +114,11 @@ func migrateDbFile() error { } cfg.DB.DbPath = oldFile - oldDAO := blockdao.NewBlockDAO(nil, cfg.DB) + deser := block.NewDeserializer(cfg.Chain.EVMNetworkID) + oldDAO := blockdao.NewBlockDAO(nil, cfg.DB, deser) cfg.DB.DbPath = newFile - newDAO := blockdao.NewBlockDAO(nil, cfg.DB) + newDAO := blockdao.NewBlockDAO(nil, cfg.DB, deser) ctx := context.Background() if err := oldDAO.Start(ctx); err != nil {