diff --git a/Makefile b/Makefile index 94ea62b5e1..e92f965247 100644 --- a/Makefile +++ b/Makefile @@ -74,6 +74,10 @@ ifeq (boltdb,$(findstring boltdb,$(COSMOS_BUILD_OPTIONS))) BUILD_TAGS += boltdb endif +ifeq (mdbx,$(findstring mdbx,$(COSMOS_BUILD_OPTIONS))) + BUILD_TAGS += mdbx +endif + ifeq (,$(findstring nostrip,$(COSMOS_BUILD_OPTIONS))) ldflags += -w -s endif diff --git a/app/app.go b/app/app.go index 7cd77b35e4..12de0b73fc 100644 --- a/app/app.go +++ b/app/app.go @@ -5,12 +5,15 @@ import ( "net/http" "os" "path/filepath" + "strings" "sync" + "github.com/crypto-org-chain/cronos/x/cronos" "github.com/crypto-org-chain/cronos/x/cronos/middleware" "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/codec/types" + "github.com/cosmos/cosmos-sdk/server" "github.com/gorilla/mux" "github.com/rakyll/statik/fs" "github.com/spf13/cast" @@ -122,7 +125,8 @@ import ( // this line is used by starport scaffolding # stargate/app/moduleImport cronosappclient "github.com/crypto-org-chain/cronos/client" - "github.com/crypto-org-chain/cronos/x/cronos" + "github.com/crypto-org-chain/cronos/versiondb" + "github.com/crypto-org-chain/cronos/versiondb/tmdb" cronosclient "github.com/crypto-org-chain/cronos/x/cronos/client" cronoskeeper "github.com/crypto-org-chain/cronos/x/cronos/keeper" evmhandlers "github.com/crypto-org-chain/cronos/x/cronos/keeper/evmhandlers" @@ -350,7 +354,8 @@ func New( // configure state listening capabilities using AppOptions // we are doing nothing with the returned streamingServices and waitGroup in this case // Only support file streamer right now. - if cast.ToString(appOpts.Get(cronosappclient.FlagStreamers)) == "file" { + streamers := cast.ToString(appOpts.Get(cronosappclient.FlagStreamers)) + if strings.Contains(streamers, "file") { streamingDir := filepath.Join(cast.ToString(appOpts.Get(flags.FlagHome)), "data", FileStreamerDirectory) if err := os.MkdirAll(streamingDir, os.ModePerm); err != nil { panic(err) @@ -373,6 +378,37 @@ func New( } } + if strings.Contains(streamers, "versiondb") { + rootDir := cast.ToString(appOpts.Get(flags.FlagHome)) + dataDir := filepath.Join(rootDir, "data", "versiondb") + if err := os.MkdirAll(dataDir, os.ModePerm); err != nil { + panic(err) + } + backendType := server.GetAppDBBackend(appOpts) + plainDB, err := dbm.NewDB("plain", backendType, dataDir) + if err != nil { + panic(err) + } + historyDB, err := dbm.NewDB("history", backendType, dataDir) + if err != nil { + panic(err) + } + changesetDB, err := dbm.NewDB("changeset", backendType, dataDir) + if err != nil { + panic(err) + } + versionDB := tmdb.NewStore(plainDB, historyDB, changesetDB) + + // default to exposing all + exposeStoreKeys := make([]storetypes.StoreKey, 0, len(keys)) + for _, storeKey := range keys { + exposeStoreKeys = append(exposeStoreKeys, storeKey) + } + service := versiondb.NewStreamingService(versionDB, exposeStoreKeys) + bApp.SetStreamingService(service) + bApp.SetQueryMultiStore(versiondb.NewMultiStore(versionDB, exposeStoreKeys)) + } + app := &App{ BaseApp: bApp, cdc: cdc, diff --git a/default.nix b/default.nix index fcbab2154a..846e83eea0 100644 --- a/default.nix +++ b/default.nix @@ -8,7 +8,7 @@ let version = "v0.9.0"; pname = "cronosd"; - tags = [ "ledger" "netgo" network ] + tags = [ "ledger" "netgo" network "mdbx" ] ++ lib.lists.optionals (rocksdb != null) [ "rocksdb" "rocksdb_build" ]; ldflags = lib.concatStringsSep "\n" ([ "-X github.com/cosmos/cosmos-sdk/version.Name=cronos" @@ -27,6 +27,7 @@ buildGoApplication rec { "!/app/" "!/cmd/" "!/client/" + "!/versiondb/" "!go.mod" "!go.sum" "!gomod2nix.toml" diff --git a/go.mod b/go.mod index f8b0fbef90..c8255dadbf 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,10 @@ go 1.18 require ( cosmossdk.io/math v1.0.0-beta.3 + github.com/RoaringBitmap/roaring v1.2.1 github.com/armon/go-metrics v0.4.1 github.com/cosmos/cosmos-sdk v0.46.2 + github.com/cosmos/gogoproto v1.4.2 github.com/cosmos/ibc-go/v5 v5.0.0 github.com/ethereum/go-ethereum v1.10.19 github.com/evmos/ethermint v0.6.1-0.20221003153722-491c3da7ebd7 @@ -45,10 +47,12 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/bgentry/speakeasy v0.1.0 // indirect + github.com/bits-and-blooms/bitset v1.2.0 // indirect github.com/btcsuite/btcd v0.22.1 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce // indirect + github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect @@ -69,9 +73,8 @@ require ( github.com/deckarep/golang-set v1.8.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect - github.com/dgraph-io/badger/v2 v2.2007.4 // indirect + github.com/dgraph-io/badger/v3 v3.2103.2 // indirect github.com/dgraph-io/ristretto v0.1.0 // indirect - github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect github.com/dop251/goja v0.0.0-20220405120441-9037c2b61cbf // indirect github.com/dustin/go-humanize v1.0.0 // indirect @@ -91,7 +94,8 @@ require ( github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/btree v1.0.1 // indirect + github.com/google/btree v1.1.2 // indirect + github.com/google/flatbuffers v2.0.0+incompatible // indirect github.com/google/go-cmp v0.5.8 // indirect github.com/google/orderedcode v0.0.1 // indirect github.com/google/uuid v1.3.0 // indirect @@ -135,6 +139,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mschoch/smat v0.2.0 // indirect github.com/mtibben/percent v0.2.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pelletier/go-toml v1.9.5 // indirect @@ -165,6 +170,7 @@ require ( github.com/tendermint/go-amino v0.16.0 // indirect github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect + github.com/torquem-ch/mdbx-go v0.26.0 // indirect github.com/tyler-smith/go-bip39 v1.1.0 // indirect github.com/ulikunitz/xz v0.5.8 // indirect github.com/zondax/hid v0.9.1-0.20220302062450-5552068d2266 // indirect @@ -189,7 +195,7 @@ require ( ) replace ( - github.com/cosmos/cosmos-sdk => github.com/cosmos/cosmos-sdk v0.46.2 + github.com/cosmos/cosmos-sdk => github.com/yihuang/cosmos-sdk v0.43.0-beta1.0.20221007070449-b35ded805600 github.com/ethereum/go-ethereum => github.com/ethereum/go-ethereum v1.10.19 // Fix upstream GHSA-h395-qcrw-5vmq vulnerability. @@ -200,6 +206,9 @@ replace ( // TODO: remove when gravity update dependencies github.com/peggyjv/gravity-bridge/module/v2 => github.com/crypto-org-chain/gravity-bridge/module/v2 v2.0.1-0.20221004054346-17ff40b2b361 + // https://github.com/tendermint/tm-db/pull/297 + github.com/tendermint/tm-db => github.com/yihuang/tm-db v0.0.0-20221006023748-f6214ae9454d + // TODO: remove after fixed https://github.com/cosmos/cosmos-sdk/issues/11364 github.com/zondax/hid => github.com/zondax/hid v0.9.0 ) diff --git a/go.sum b/go.sum index 8d5f8bb3ae..757e3da998 100644 --- a/go.sum +++ b/go.sum @@ -239,6 +239,8 @@ github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/RoaringBitmap/roaring v1.2.1 h1:58/LJlg/81wfEHd5L9qsHduznOIhyv4qb1yWcSvVq9A= +github.com/RoaringBitmap/roaring v1.2.1/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= @@ -347,6 +349,7 @@ github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bkielbasa/cyclop v1.2.0/go.mod h1:qOI0yy6A7dYC4Zgsa72Ppm9kONl0RoIlPbzot9mhmeI= @@ -403,6 +406,8 @@ github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3k github.com/butuzov/ireturn v0.1.1/go.mod h1:Wh6Zl3IMtTpaIKbmwzqi6olnM9ptYQxxVacMsOEFPoc= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= +github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b h1:6+ZFm0flnudZzdSE0JxlhR2hKnGPcNB35BjQf4RYQDY= +github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= github.com/caarlos0/ctrlc v1.0.0/go.mod h1:CdXpj4rmq0q/1Eb44M9zi2nKB0QraNKuRGYGrrHhcQw= github.com/campoy/unique v0.0.0-20180121183637-88950e537e7e/go.mod h1:9IOqJGCPMSc6E5ydlp5NIonxObaeu/Iub/X03EKPVYo= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= @@ -619,12 +624,12 @@ github.com/cosmos/btcutil v1.0.4 h1:n7C2ngKXo7UC9gNyMNLbzqz7Asuf+7Qv4gnX/rOdQ44= github.com/cosmos/btcutil v1.0.4/go.mod h1:Ffqc8Hn6TJUdDgHBwIZLtrLQC1KdJ9jGJl/TvgUaxbU= github.com/cosmos/cosmos-proto v1.0.0-alpha7 h1:yqYUOHF2jopwZh4dVQp3xgqwftE5/2hkrwIV6vkUbO0= github.com/cosmos/cosmos-proto v1.0.0-alpha7/go.mod h1:dosO4pSAbJF8zWCzCoTWP7nNsjcvSUBQmniFxDg5daw= -github.com/cosmos/cosmos-sdk v0.46.2 h1:3dUNqbLas94ud5aTcJKCwxVOmNXpuGBtVQTMrYczTwY= -github.com/cosmos/cosmos-sdk v0.46.2/go.mod h1:0aUPGPU6PWaDEaHNjtgrpNhgxo9bAUrQ7BO7XCvFOfs= github.com/cosmos/cosmos-sdk/db v1.0.0-beta.1/go.mod h1:JUMM2MxF9wuwzRWZJjb8BjXsn1BmPmdBd3a75pIct4I= github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY= github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw= +github.com/cosmos/gogoproto v1.4.2 h1:UeGRcmFW41l0G0MiefWhkPEVEwvu78SZsHBvI78dAYw= +github.com/cosmos/gogoproto v1.4.2/go.mod h1:cLxOsn1ljAHSV527CHOtaIP91kK6cCrZETRBrkzItWU= github.com/cosmos/gorocksdb v1.2.0 h1:d0l3jJG8M4hBouIZq0mDUHZ+zjOx044J3nGRskwTb4Y= github.com/cosmos/gorocksdb v1.2.0/go.mod h1:aaKvKItm514hKfNJpUJXnnOWeBnk2GL4+Qw9NHizILw= github.com/cosmos/iavl v0.19.0/go.mod h1:l5h9pAB3m5fihB3pXVgwYqdY8aBsMagqz7T0MUjxZeA= @@ -685,8 +690,8 @@ github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= -github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= +github.com/dgraph-io/badger/v3 v3.2103.2 h1:dpyM5eCJAtQCBcMCZcT4UBZchuTJgCywerHHgmxfxM8= github.com/dgraph-io/badger/v3 v3.2103.2/go.mod h1:RHo4/GmYcKKh5Lxu63wLEMHJ70Pac2JqZRYGhlyAo2M= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= @@ -1044,13 +1049,15 @@ github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg= github.com/google/certificate-transparency-go v1.1.1/go.mod h1:FDKqPvSXawb2ecErVRrD+nfy23RCzyl7eqVCEmlT1Zs= github.com/google/crfs v0.0.0-20191108021818-71d77da419c9/go.mod h1:etGhoOqfwPkooV6aqoX3eBGQOJblqdoc9XvWOeuxpPw= github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v2.0.0+incompatible h1:dicJ2oXwypfwUGnB2/TYWYEKiuk9eYQlQO/AnOHl5mI= github.com/google/flatbuffers v2.0.0+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -1631,6 +1638,8 @@ github.com/mozilla/tls-observatory v0.0.0-20210609171429-7bc42856d2e5/go.mod h1: github.com/mrunalp/fileutils v0.0.0-20200520151820-abd8a0e76976/go.mod h1:x8F1gnqOkIEiO4rqoeEEEqQbo7HjGMTvyoq3gej4iT0= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -2122,9 +2131,6 @@ github.com/tendermint/tendermint v0.34.19/go.mod h1:R5+wgIwSxMdKQcmOaeudL0Cjkr3H github.com/tendermint/tendermint v0.34.20/go.mod h1:KtOwCLYJcsS1ymtAfnjjAtXfXClbqcqjdqzFt2Em1Ac= github.com/tendermint/tendermint v0.34.21 h1:UiGGnBFHVrZhoQVQ7EfwSOLuCtarqCSsRf8VrklqB7s= github.com/tendermint/tendermint v0.34.21/go.mod h1:XDvfg6U7grcFTDx7VkzxnhazQ/bspGJAn4DZ6DcLLjQ= -github.com/tendermint/tm-db v0.6.6/go.mod h1:wP8d49A85B7/erz/r4YbKssKw6ylsO/hKtFk7E1aWZI= -github.com/tendermint/tm-db v0.6.7 h1:fE00Cbl0jayAoqlExN6oyQJ7fR/ZtoVOmvPJ//+shu8= -github.com/tendermint/tm-db v0.6.7/go.mod h1:byQDzFkZV1syXr/ReXS808NxA2xvyuuVgXOJ/088L6I= github.com/tenntenn/modver v1.0.1/go.mod h1:bePIyQPb7UeioSRkw3Q0XeMhYZSMx9B8ePqg6SAMGH0= github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3/go.mod h1:ON8b8w4BN/kE1EOhwT0o+d62W65a6aPw1nouo9LMgyY= github.com/tetafro/godot v0.3.7/go.mod h1:/7NLHhv08H1+8DNj0MElpAACw1ajsCuf3TKNQxA5S+0= @@ -2166,6 +2172,8 @@ github.com/tonistiigi/go-actions-cache v0.0.0-20220404170428-0bdeb6e1eac7/go.mod github.com/tonistiigi/go-archvariant v1.0.0/go.mod h1:TxFmO5VS6vMq2kvs3ht04iPXtu2rUT/erOnGFYfk5Ho= github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea/go.mod h1:WPnis/6cRcDZSUvVmezrxJPkiO87ThFYsoUiMwWNDJk= github.com/tonistiigi/vt100 v0.0.0-20210615222946-8066bb97264f/go.mod h1:ulncasL3N9uLrVann0m+CDlJKWsIAP34MPcOJF6VRvc= +github.com/torquem-ch/mdbx-go v0.26.0 h1:d8ph2MsVZoBZr0eFWHRiSYjoCXggED6XzcspUX/HsZM= +github.com/torquem-ch/mdbx-go v0.26.0/go.mod h1:T2fsoJDVppxfAPTLd1svUgH1kpPmeXdPESmroSHcL1E= github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= @@ -2240,6 +2248,10 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/yagipy/maintidx v1.0.0/go.mod h1:0qNf/I/CCZXSMhsRsrEPDZ+DkekpKLXAJfsTACwgXLk= github.com/ybbus/jsonrpc v2.1.2+incompatible/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= github.com/yeya24/promlinter v0.2.0/go.mod h1:u54lkmBOZrpEbQQ6gox2zWKKLKu2SGe+2KOiextY+IA= +github.com/yihuang/cosmos-sdk v0.43.0-beta1.0.20221007070449-b35ded805600 h1:Fs1/gXC9FIeJjLrp2noVEquVO6vYDETwAzmv+ChdK/g= +github.com/yihuang/cosmos-sdk v0.43.0-beta1.0.20221007070449-b35ded805600/go.mod h1:0aUPGPU6PWaDEaHNjtgrpNhgxo9bAUrQ7BO7XCvFOfs= +github.com/yihuang/tm-db v0.0.0-20221006023748-f6214ae9454d h1:+HeYUyE9B0Zg/eYVj9LIZfriP7B71akpRvCmv1oIIbw= +github.com/yihuang/tm-db v0.0.0-20221006023748-f6214ae9454d/go.mod h1:a8QLaVn+zo5zgX2qMrNSETPO/8uBRPxvLsqL+urelEU= github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= diff --git a/gomod2nix.toml b/gomod2nix.toml index ba0829d602..961206f7cc 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -31,6 +31,9 @@ schema = 3 [mod."github.com/ChainSafe/go-schnorrkel"] version = "v0.0.0-20200405005733-88cbf1b4c40d" hash = "sha256-i8RXZemJGlSjBT35oPm0SawFiBoIU5Pkq5xp4n/rzCY=" + [mod."github.com/RoaringBitmap/roaring"] + version = "v1.2.1" + hash = "sha256-0/R956wrCW71eOE36CbxGJJRuQjKwvvIQ/D8QTn2A6w=" [mod."github.com/StackExchange/wmi"] version = "v1.2.1" hash = "sha256-1BoEeWAWyebH+1mMuyPhWZut8nWHb6r73MgcqlGuUEY=" @@ -58,6 +61,9 @@ schema = 3 [mod."github.com/bgentry/speakeasy"] version = "v0.1.0" hash = "sha256-Gt1vj6CFovLnO6wX5u2O4UfecY9V2J9WGw1ez4HMrgk=" + [mod."github.com/bits-and-blooms/bitset"] + version = "v1.2.0" + hash = "sha256-IxNmtELycM+XVzg4qBv04hAJUT3nSWuyP9R+8zc9LmU=" [mod."github.com/btcsuite/btcd"] version = "v0.22.1" hash = "sha256-hBU+roIELcmbW2Gz7eGZzL9qNA1bakq5wNxqCgs4TKc=" @@ -70,6 +76,9 @@ schema = 3 [mod."github.com/btcsuite/btcutil"] version = "v1.0.3-0.20201208143702-a53e38424cce" hash = "sha256-4kasJReFcj25JRHx9dJMct3yDkHqVoHGUx5cu45Msfo=" + [mod."github.com/c2h5oh/datasize"] + version = "v0.0.0-20220606134207-859f65c6625b" + hash = "sha256-1uH+D3w0Y/B3poXm545XGrT4S4c+msTbj7gKgu9pbPM=" [mod."github.com/cenkalti/backoff/v4"] version = "v4.1.3" hash = "sha256-u6MEDopHoTWAZoVvvXOKnAg++xre53YgQx0gmf6t2KU=" @@ -98,12 +107,15 @@ schema = 3 version = "v1.0.0-alpha7" hash = "sha256-2wCH+toTF2A6MfFjOa13muEH5oBCcxAhZEqirNOrBA0=" [mod."github.com/cosmos/cosmos-sdk"] - version = "v0.46.2" - hash = "sha256-Lgn4+Vd5PUUkfHc+lTdK2G6/nymZekFVTe1FxWRqh2w=" - replaced = "github.com/cosmos/cosmos-sdk" + version = "v0.43.0-beta1.0.20221007070449-b35ded805600" + hash = "sha256-MriHtStuBGsr7DjQYEYCU9fF8g98kqYpWgy+pbN8m1Y=" + replaced = "github.com/yihuang/cosmos-sdk" [mod."github.com/cosmos/go-bip39"] version = "v1.0.0" hash = "sha256-Qm2aC2vaS8tjtMUbHmlBSagOSqbduEEDwc51qvQaBmA=" + [mod."github.com/cosmos/gogoproto"] + version = "v1.4.2" + hash = "sha256-hOY+mhPDYWcSYSdth2AW7IONdgicqQir0z/1XrXt9NY=" [mod."github.com/cosmos/gorocksdb"] version = "v1.2.0" hash = "sha256-209TcVuXc5s/TcOvNlaQ1HEJAUDTEK3nxPhs+d8TEcY=" @@ -137,15 +149,12 @@ schema = 3 [mod."github.com/desertbit/timer"] version = "v0.0.0-20180107155436-c41aec40b27f" hash = "sha256-abLOtEcomAqCWLphd2X6WkD/ED764w6sa6unox4BXss=" - [mod."github.com/dgraph-io/badger/v2"] - version = "v2.2007.4" - hash = "sha256-+KwqZJZpViv8S3TqUVvPXrFoMgWFyS3NoLsi4RR5fGk=" + [mod."github.com/dgraph-io/badger/v3"] + version = "v3.2103.2" + hash = "sha256-F6pvsaSKwXOl9RfnUQFqAl6xpCVu9+rthQgOxhKVk1g=" [mod."github.com/dgraph-io/ristretto"] version = "v0.1.0" hash = "sha256-01jneg1+1x8tTfUTBZ+6mHkQaqXVnPYxLJyJhJQcvt4=" - [mod."github.com/dgryski/go-farm"] - version = "v0.0.0-20200201041132-a6ae2369ad13" - hash = "sha256-aOMlPwFY36bLiiIx4HonbCYRAhagk5N6HAWN7Ygif+E=" [mod."github.com/dlclark/regexp2"] version = "v1.4.1-0.20201116162257-a2a8dda75c91" hash = "sha256-VNNMZIc7NkDg3DVLnqeJNM/KZqkkaZu2/HTLBL8X2xE=" @@ -164,6 +173,7 @@ schema = 3 [mod."github.com/ethereum/go-ethereum"] version = "v1.10.19" hash = "sha256-7FPnTGcCb8Xd1QVR+6PmGTaHdTY1mm/8osFTW1JLuG8=" + replaced = "github.com/ethereum/go-ethereum" [mod."github.com/evmos/ethermint"] version = "v0.6.1-0.20221003153722-491c3da7ebd7" hash = "sha256-vnfjk57gYa+F8nn0LByX/B1LV8PY2Jvm8vXV6be4ufc=" @@ -217,8 +227,11 @@ schema = 3 version = "v0.0.4" hash = "sha256-Umx+5xHAQCN/Gi4HbtMhnDCSPFAXSsjVbXd8n5LhjAA=" [mod."github.com/google/btree"] - version = "v1.0.1" - hash = "sha256-1PIeFGgUL4BK/StL/D12pg9bEQ5HfMT/fMLdus4pZTs=" + version = "v1.1.2" + hash = "sha256-K7V2obq3pLM71Mg0vhhHtZ+gtaubwXPQx3xcIyZDCjM=" + [mod."github.com/google/flatbuffers"] + version = "v2.0.0+incompatible" + hash = "sha256-4Db9FdOL60Da4H1+K4Qv02w4omxdsh3uzpmY1vtqHeA=" [mod."github.com/google/go-cmp"] version = "v0.5.8" hash = "sha256-8zkIo+Sr1NXMnj3PNmvjX2sZKnAKWXOFvmnX7D9bwxQ=" @@ -354,6 +367,9 @@ schema = 3 [mod."github.com/mitchellh/mapstructure"] version = "v1.5.0" hash = "sha256-ztVhGQXs67MF8UadVvG72G3ly0ypQW0IRDdOOkjYwoE=" + [mod."github.com/mschoch/smat"] + version = "v0.2.0" + hash = "sha256-DZvUJXjIcta3U+zxzgU3wpoGn/V4lpBY7Xme8aQUi+E=" [mod."github.com/mtibben/percent"] version = "v0.2.1" hash = "sha256-Zj1lpCP6mKQ0UUTMs2By4LC414ou+iJzKkK+eBHfEcc=" @@ -461,14 +477,18 @@ schema = 3 version = "v0.34.21" hash = "sha256-C1KW6Wd4TwsFC09IHt8Jo3dozEk/Q2ZVdTapM8btEJ0=" [mod."github.com/tendermint/tm-db"] - version = "v0.6.7" - hash = "sha256-hl/3RrBrpkk2zA6dmrNlIYKs1/GfqegSscDSkA5Pjlo=" + version = "v0.0.0-20221006023748-f6214ae9454d" + hash = "sha256-mtTVR3f3A9CmcyJBXTeurQuHGiVA5hIzqlxskz1M1qk=" + replaced = "github.com/yihuang/tm-db" [mod."github.com/tklauser/go-sysconf"] version = "v0.3.10" hash = "sha256-Zf2NsgM9+HeM949vCce4HQtSbfUiFpeiQ716yKcFyx4=" [mod."github.com/tklauser/numcpus"] version = "v0.4.0" hash = "sha256-ndE82nOb3agubhEV7aRzEqqTlN4DPbKFHEm2+XZLn8k=" + [mod."github.com/torquem-ch/mdbx-go"] + version = "v0.26.0" + hash = "sha256-HBDflmCuxe/fBKSmasfEsAf1VEdH52sPjSIhJNOm+qE=" [mod."github.com/tyler-smith/go-bip39"] version = "v1.1.0" hash = "sha256-3YhWBtSwRLGwm7vNwqumphZG3uLBW1vwT9QkQ8JuSjU=" @@ -478,6 +498,7 @@ schema = 3 [mod."github.com/zondax/hid"] version = "v0.9.0" hash = "sha256-PvXtxXo/3C+DS9ZeGBlr4zXbIpaYNtMqLzxYhusFXNY=" + replaced = "github.com/zondax/hid" [mod."go.etcd.io/bbolt"] version = "v1.3.6" hash = "sha256-DenVAmyN22xUiivk6fdJp4C9ZnUJXCMDUf8E0goRRV4=" diff --git a/integration_tests/configs/default.jsonnet b/integration_tests/configs/default.jsonnet index 12ca83f47d..23b183f7d5 100644 --- a/integration_tests/configs/default.jsonnet +++ b/integration_tests/configs/default.jsonnet @@ -2,7 +2,7 @@ dotenv: '../../scripts/.env', 'cronos_777-1': { cmd: 'cronosd', - 'start-flags': '--trace --streamers file', + 'start-flags': '--trace --streamers versiondb,file', config: { mempool: { version: 'v1', diff --git a/integration_tests/configs/state_benchmark.jsonnet b/integration_tests/configs/state_benchmark.jsonnet new file mode 100644 index 0000000000..ba8dbf1741 --- /dev/null +++ b/integration_tests/configs/state_benchmark.jsonnet @@ -0,0 +1,36 @@ +local config = import 'default.jsonnet'; + +config { + 'cronos_777-1'+: { + 'start-flags': '--trace --streamers file,versiondb', + 'app-config'+: { + 'app-db-backend': 'rocksdb', + 'state-sync'+: { + 'snapshot-interval': 0, + }, + }, + validators: [ + super.validators[0], + super.validators[1] { + 'app-config'+: { + pruning: 'everything', + }, + }, + ] + super.validators[2:], + genesis+: { + consensus_params+: { + block+: { + max_gas: '163000000', + }, + }, + app_state+: { + feemarket+: { + params+: { + no_base_fee: true, + min_gas_multiplier: '0', + }, + }, + }, + }, + }, +} diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py index 980d4a9627..a496fe7a12 100644 --- a/integration_tests/conftest.py +++ b/integration_tests/conftest.py @@ -13,6 +13,28 @@ def pytest_configure(config): config.addinivalue_line("markers", "slow: marks tests as slow") config.addinivalue_line("markers", "gravity: gravity bridge test cases") + config.addinivalue_line( + "markers", "benchmark: benchmarks, only run if '--run-benchmark' is passed" + ) + + +def pytest_addoption(parser): + parser.addoption( + "--run-benchmark", + action="store_true", + default=False, + help="include benchmark cases", + ) + + +def pytest_collection_modifyitems(config, items): + if config.getoption("--run-benchmark"): + # run benchmarks + return + skip = pytest.mark.skip(reason="need --run-benchmark option to run") + for item in items: + if "benchmark" in item.keywords: + item.add_marker(skip) @pytest.fixture(scope="session") diff --git a/integration_tests/contracts/contracts/BenchmarkStorage.sol b/integration_tests/contracts/contracts/BenchmarkStorage.sol new file mode 100644 index 0000000000..69dcfde451 --- /dev/null +++ b/integration_tests/contracts/contracts/BenchmarkStorage.sol @@ -0,0 +1,15 @@ +pragma solidity 0.8.10; + +contract BenchmarkStorage { + uint seed; + mapping(uint => uint) state; + function random(uint i) private view returns (uint) { + return uint(keccak256(abi.encodePacked(i, seed))); + } + function batch_set(uint _seed, uint n, uint range) public { + seed = _seed; + for (uint i=0; i< n; i++) { + state[random(i) % range] = random(i+i); + } + } +} diff --git a/integration_tests/poetry.lock b/integration_tests/poetry.lock index 9b0b08e93d..ef584333b8 100644 --- a/integration_tests/poetry.lock +++ b/integration_tests/poetry.lock @@ -263,10 +263,10 @@ python-versions = ">=3.6, <4" eth-hash = {version = ">=0.3.1,<0.4.0", extras = ["pycryptodome"]} [package.extras] -deploy = ["bumpversion (>=0.5.3,<1.0.0)", "wheel (>=0.30.0,<1.0.0)"] -dev = ["bumpversion (>=0.5.3,<1.0.0)", "wheel (>=0.30.0,<1.0.0)", "twine", "pytest (==3.0.7)", "hypothesis (==3.7.0)", "tox (==2.6.0)", "flake8 (>=3.5.0,<4.0.0)", "mypy (<0.600)"] -lint = ["flake8 (>=3.5.0,<4.0.0)", "mypy (<0.600)"] -test = ["pytest (==3.0.7)", "hypothesis (==3.7.0)", "tox (==2.6.0)"] +test = ["tox (==2.6.0)", "hypothesis (==3.7.0)", "pytest (==3.0.7)"] +lint = ["mypy (<0.600)", "flake8 (>=3.5.0,<4.0.0)"] +dev = ["mypy (<0.600)", "flake8 (>=3.5.0,<4.0.0)", "tox (==2.6.0)", "hypothesis (==3.7.0)", "pytest (==3.0.7)", "twine", "wheel (>=0.30.0,<1.0.0)", "bumpversion (>=0.5.3,<1.0.0)"] +deploy = ["wheel (>=0.30.0,<1.0.0)", "bumpversion (>=0.5.3,<1.0.0)"] [[package]] name = "eth-hash" @@ -280,12 +280,12 @@ python-versions = ">=3.5, <4" pycryptodome = {version = ">=3.6.6,<4", optional = true, markers = "extra == \"pycryptodome\""} [package.extras] -dev = ["bumpversion (>=0.5.3,<1)", "pytest-watch (>=4.1.0,<5)", "wheel", "twine", "ipython", "pytest (==5.4.1)", "pytest-xdist", "tox (==3.14.6)", "flake8 (==3.7.9)", "isort (>=4.2.15,<5)", "mypy (==0.770)", "pydocstyle (>=5.0.0,<6)", "Sphinx (>=1.6.5,<2)", "sphinx-rtd-theme (>=0.1.9,<1)", "towncrier (>=19.2.0,<20)"] -doc = ["Sphinx (>=1.6.5,<2)", "sphinx-rtd-theme (>=0.1.9,<1)", "towncrier (>=19.2.0,<20)"] -lint = ["flake8 (==3.7.9)", "isort (>=4.2.15,<5)", "mypy (==0.770)", "pydocstyle (>=5.0.0,<6)"] -pycryptodome = ["pycryptodome (>=3.6.6,<4)"] +test = ["tox (==3.14.6)", "pytest-xdist", "pytest (==5.4.1)"] pysha3 = ["pysha3 (>=1.0.0,<2.0.0)"] -test = ["pytest (==5.4.1)", "pytest-xdist", "tox (==3.14.6)"] +pycryptodome = ["pycryptodome (>=3.6.6,<4)"] +lint = ["pydocstyle (>=5.0.0,<6)", "mypy (==0.770)", "isort (>=4.2.15,<5)", "flake8 (==3.7.9)"] +doc = ["towncrier (>=19.2.0,<20)", "sphinx-rtd-theme (>=0.1.9,<1)", "Sphinx (>=1.6.5,<2)"] +dev = ["towncrier (>=19.2.0,<20)", "sphinx-rtd-theme (>=0.1.9,<1)", "Sphinx (>=1.6.5,<2)", "pydocstyle (>=5.0.0,<6)", "mypy (==0.770)", "isort (>=4.2.15,<5)", "flake8 (==3.7.9)", "tox (==3.14.6)", "pytest-xdist", "pytest (==5.4.1)", "ipython", "twine", "wheel", "pytest-watch (>=4.1.0,<5)", "bumpversion (>=0.5.3,<1)"] [[package]] name = "eth-keyfile" @@ -314,11 +314,11 @@ eth-typing = ">=2.2.1,<3.0.0" eth-utils = ">=1.8.2,<2.0.0" [package.extras] +test = ["eth-hash", "eth-hash", "hypothesis (>=5.10.3,<6.0.0)", "pytest (==5.4.1)", "pyasn1 (>=0.4.5,<0.5)", "factory-boy (>=3.0.1,<3.1)", "asn1tools (>=0.146.2,<0.147)"] +lint = ["mypy (==0.782)", "flake8 (==3.0.4)"] +eth-keys = ["eth-typing (>=2.2.1,<3.0.0)", "eth-utils (>=1.8.2,<2.0.0)"] +dev = ["eth-hash", "eth-hash", "hypothesis (>=5.10.3,<6.0.0)", "pytest (==5.4.1)", "pyasn1 (>=0.4.5,<0.5)", "factory-boy (>=3.0.1,<3.1)", "asn1tools (>=0.146.2,<0.147)", "mypy (==0.782)", "flake8 (==3.0.4)", "eth-typing (>=2.2.1,<3.0.0)", "eth-utils (>=1.8.2,<2.0.0)", "twine", "bumpversion (==0.5.3)", "tox (==3.20.0)"] coincurve = ["coincurve (>=7.0.0,<13.0.0)"] -dev = ["tox (==3.20.0)", "bumpversion (==0.5.3)", "twine", "eth-utils (>=1.8.2,<2.0.0)", "eth-typing (>=2.2.1,<3.0.0)", "flake8 (==3.0.4)", "mypy (==0.782)", "asn1tools (>=0.146.2,<0.147)", "factory-boy (>=3.0.1,<3.1)", "pyasn1 (>=0.4.5,<0.5)", "pytest (==5.4.1)", "hypothesis (>=5.10.3,<6.0.0)", "eth-hash", "eth-hash"] -eth-keys = ["eth-utils (>=1.8.2,<2.0.0)", "eth-typing (>=2.2.1,<3.0.0)"] -lint = ["flake8 (==3.0.4)", "mypy (==0.782)"] -test = ["asn1tools (>=0.146.2,<0.147)", "factory-boy (>=3.0.1,<3.1)", "pyasn1 (>=0.4.5,<0.5)", "pytest (==5.4.1)", "hypothesis (>=5.10.3,<6.0.0)", "eth-hash", "eth-hash"] [[package]] name = "eth-rlp" @@ -334,10 +334,10 @@ hexbytes = ">=0.1.0,<1" rlp = ">=0.6.0,<3" [package.extras] -dev = ["Sphinx (>=1.6.5,<2)", "bumpversion (>=0.5.3,<1)", "eth-hash", "flake8 (==3.7.9)", "ipython", "isort (>=4.2.15,<5)", "mypy (==0.770)", "pydocstyle (>=3.0.0,<4)", "pytest-watch (>=4.1.0,<5)", "pytest-xdist", "pytest (==5.4.1)", "sphinx-rtd-theme (>=0.1.9)", "towncrier (>=19.2.0,<20)", "tox (==3.14.6)", "twine", "wheel"] -doc = ["Sphinx (>=1.6.5,<2)", "sphinx-rtd-theme (>=0.1.9)", "towncrier (>=19.2.0,<20)"] -lint = ["flake8 (==3.7.9)", "isort (>=4.2.15,<5)", "mypy (==0.770)", "pydocstyle (>=3.0.0,<4)"] -test = ["eth-hash", "pytest-xdist", "pytest (==5.4.1)", "tox (==3.14.6)"] +test = ["tox (==3.14.6)", "pytest (==5.4.1)", "pytest-xdist", "eth-hash"] +lint = ["pydocstyle (>=3.0.0,<4)", "mypy (==0.770)", "isort (>=4.2.15,<5)", "flake8 (==3.7.9)"] +doc = ["towncrier (>=19.2.0,<20)", "sphinx-rtd-theme (>=0.1.9)", "Sphinx (>=1.6.5,<2)"] +dev = ["wheel", "twine", "tox (==3.14.6)", "towncrier (>=19.2.0,<20)", "sphinx-rtd-theme (>=0.1.9)", "pytest (==5.4.1)", "pytest-xdist", "pytest-watch (>=4.1.0,<5)", "pydocstyle (>=3.0.0,<4)", "mypy (==0.770)", "isort (>=4.2.15,<5)", "ipython", "flake8 (==3.7.9)", "eth-hash", "bumpversion (>=0.5.3,<1)", "Sphinx (>=1.6.5,<2)"] [[package]] name = "eth-typing" @@ -348,10 +348,10 @@ optional = false python-versions = ">=3.5, <4" [package.extras] -dev = ["bumpversion (>=0.5.3,<1)", "pytest-watch (>=4.1.0,<5)", "wheel", "twine", "ipython", "pytest (>=4.4,<4.5)", "pytest-xdist", "tox (>=2.9.1,<3)", "flake8 (==3.8.3)", "isort (>=4.2.15,<5)", "mypy (==0.782)", "pydocstyle (>=3.0.0,<4)", "Sphinx (>=1.6.5,<2)", "sphinx-rtd-theme (>=0.1.9)"] -doc = ["Sphinx (>=1.6.5,<2)", "sphinx-rtd-theme (>=0.1.9)"] -lint = ["flake8 (==3.8.3)", "isort (>=4.2.15,<5)", "mypy (==0.782)", "pydocstyle (>=3.0.0,<4)"] -test = ["pytest (>=4.4,<4.5)", "pytest-xdist", "tox (>=2.9.1,<3)"] +test = ["tox (>=2.9.1,<3)", "pytest-xdist", "pytest (>=4.4,<4.5)"] +lint = ["pydocstyle (>=3.0.0,<4)", "mypy (==0.782)", "isort (>=4.2.15,<5)", "flake8 (==3.8.3)"] +doc = ["sphinx-rtd-theme (>=0.1.9)", "Sphinx (>=1.6.5,<2)"] +dev = ["sphinx-rtd-theme (>=0.1.9)", "Sphinx (>=1.6.5,<2)", "pydocstyle (>=3.0.0,<4)", "mypy (==0.782)", "isort (>=4.2.15,<5)", "flake8 (==3.8.3)", "tox (>=2.9.1,<3)", "pytest-xdist", "pytest (>=4.4,<4.5)", "ipython", "twine", "wheel", "pytest-watch (>=4.1.0,<5)", "bumpversion (>=0.5.3,<1)"] [[package]] name = "eth-utils" @@ -469,10 +469,10 @@ optional = false python-versions = ">=3.6, <4" [package.extras] -dev = ["Sphinx (>=1.6.5,<2)", "bumpversion (>=0.5.3,<1)", "eth-utils (>=1.0.1,<2)", "flake8 (==3.7.9)", "hypothesis (>=3.44.24,<4)", "ipython", "isort (>=4.2.15,<5)", "mypy (==0.770)", "pydocstyle (>=5.0.0,<6)", "pytest-watch (>=4.1.0,<5)", "pytest-xdist", "pytest (==5.4.1)", "sphinx-rtd-theme (>=0.1.9,<1)", "towncrier (>=19.2.0,<20)", "tox (==3.14.6)", "twine", "wheel"] -doc = ["Sphinx (>=1.6.5,<2)", "sphinx-rtd-theme (>=0.1.9,<1)", "towncrier (>=19.2.0,<20)"] -lint = ["flake8 (==3.7.9)", "isort (>=4.2.15,<5)", "mypy (==0.770)", "pydocstyle (>=5.0.0,<6)"] -test = ["eth-utils (>=1.0.1,<2)", "hypothesis (>=3.44.24,<4)", "pytest-xdist", "pytest (==5.4.1)", "tox (==3.14.6)"] +test = ["tox (==3.14.6)", "pytest (==5.4.1)", "pytest-xdist", "hypothesis (>=3.44.24,<4)", "eth-utils (>=1.0.1,<2)"] +lint = ["pydocstyle (>=5.0.0,<6)", "mypy (==0.770)", "isort (>=4.2.15,<5)", "flake8 (==3.7.9)"] +doc = ["towncrier (>=19.2.0,<20)", "sphinx-rtd-theme (>=0.1.9,<1)", "Sphinx (>=1.6.5,<2)"] +dev = ["wheel", "twine", "tox (==3.14.6)", "towncrier (>=19.2.0,<20)", "sphinx-rtd-theme (>=0.1.9,<1)", "pytest (==5.4.1)", "pytest-xdist", "pytest-watch (>=4.1.0,<5)", "pydocstyle (>=5.0.0,<6)", "mypy (==0.770)", "isort (>=4.2.15,<5)", "ipython", "hypothesis (>=3.44.24,<4)", "flake8 (==3.7.9)", "eth-utils (>=1.0.1,<2)", "bumpversion (>=0.5.3,<1)", "Sphinx (>=1.6.5,<2)"] [[package]] name = "idna" @@ -694,8 +694,8 @@ optional = false python-versions = ">=3.6" [package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] +testing = ["pytest-benchmark", "pytest"] +dev = ["tox", "pre-commit"] [[package]] name = "protobuf" @@ -876,8 +876,8 @@ python-versions = ">=3.5" PyYAML = ">=5.1,<7.0" [package.extras] -all = ["toml"] toml = ["toml"] +all = ["toml"] [[package]] name = "requests" @@ -909,11 +909,30 @@ python-versions = "*" eth-utils = ">=1.0.2,<2" [package.extras] -dev = ["Sphinx (>=1.6.5,<2)", "bumpversion (>=0.5.3,<1)", "flake8 (==3.4.1)", "hypothesis (==5.19.0)", "ipython", "pytest-watch (>=4.1.0,<5)", "pytest-xdist", "pytest (==5.4.3)", "setuptools (>=36.2.0)", "sphinx-rtd-theme (>=0.1.9)", "tox (>=2.9.1,<3)", "twine", "wheel"] -doc = ["Sphinx (>=1.6.5,<2)", "sphinx-rtd-theme (>=0.1.9)"] -lint = ["flake8 (==3.4.1)"] +test = ["tox (>=2.9.1,<3)", "pytest (==5.4.3)", "hypothesis (==5.19.0)"] rust-backend = ["rusty-rlp (>=0.1.15,<0.2)"] -test = ["hypothesis (==5.19.0)", "pytest (==5.4.3)", "tox (>=2.9.1,<3)"] +lint = ["flake8 (==3.4.1)"] +doc = ["sphinx-rtd-theme (>=0.1.9)", "Sphinx (>=1.6.5,<2)"] +dev = ["wheel", "twine", "tox (>=2.9.1,<3)", "sphinx-rtd-theme (>=0.1.9)", "setuptools (>=36.2.0)", "pytest (==5.4.3)", "pytest-xdist", "pytest-watch (>=4.1.0,<5)", "ipython", "hypothesis (==5.19.0)", "flake8 (==3.4.1)", "bumpversion (>=0.5.3,<1)", "Sphinx (>=1.6.5,<2)"] + +[[package]] +name = "rocksdb" +version = "0.9.1" +description = "" +category = "main" +optional = false +python-versions = "*" +develop = false + +[package.extras] +doc = ["sphinx", "sphinx-rtd-theme"] +test = ["pytest"] + +[package.source] +type = "git" +url = "https://github.com/HathorNetwork/python-rocksdb.git" +reference = "master" +resolved_reference = "947f68a80d97c4a5621ee681ae01602ebd883f3a" [[package]] name = "six" @@ -1055,9 +1074,9 @@ optional = false python-versions = ">=3.7" [package.extras] -docs = ["Sphinx (>=3.4)", "sphinx-rtd-theme (>=0.5)"] -optional = ["python-socks", "wsaccel"] test = ["websockets"] +optional = ["wsaccel", "python-socks"] +docs = ["sphinx-rtd-theme (>=0.5)", "Sphinx (>=3.4)"] [[package]] name = "websockets" @@ -1094,7 +1113,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "0dd8419b0eba0ed6f49c4f485e1894f5e24127acf71beb7c8d31aa8bce6388e2" +content-hash = "f834c50bf1ecb20adab3eaed22d5a7f71f3913e64969e9a6f6c5cf57372e5f98" [metadata.files] aiohttp = [ @@ -1237,6 +1256,7 @@ click = [ ] colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, + {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, ] cprotobuf = [] cytoolz = [ @@ -1752,6 +1772,7 @@ rlp = [ {file = "rlp-2.0.1-py2.py3-none-any.whl", hash = "sha256:52a57c9f53f03c88b189283734b397314288250cc4a3c4113e9e36e2ac6bdd16"}, {file = "rlp-2.0.1.tar.gz", hash = "sha256:665e8312750b3fc5f7002e656d05b9dcb6e93b6063df40d95c49ad90c19d1f0e"}, ] +rocksdb = [] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, diff --git a/integration_tests/pyproject.toml b/integration_tests/pyproject.toml index f5b3e565b2..3ed67d3c0f 100644 --- a/integration_tests/pyproject.toml +++ b/integration_tests/pyproject.toml @@ -27,6 +27,7 @@ pysha3 = "^1.0.2" jsonnet = "^0.18.0" eth-account = { git = "https://github.com/mmsqe/eth-account.git", branch = "v0.5.8-rc0" } cprotobuf = { git = "https://github.com/yihuang/cprotobuf.git" } +rocksdb = { git = "https://github.com/HathorNetwork/python-rocksdb.git", branch = "master" } [tool.poetry.dev-dependencies] diff --git a/integration_tests/test_benchmark_storage.py b/integration_tests/test_benchmark_storage.py new file mode 100644 index 0000000000..29ed85f3c7 --- /dev/null +++ b/integration_tests/test_benchmark_storage.py @@ -0,0 +1,83 @@ +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import pytest +from web3 import Web3 + +from .network import setup_custom_cronos +from .utils import ( + ACCOUNTS, + CONTRACTS, + deploy_contract, + send_transaction, + w3_wait_for_block, +) + + +@pytest.fixture(scope="module") +def custom_cronos(tmp_path_factory): + path = tmp_path_factory.mktemp("benchmark") + yield from setup_custom_cronos( + path, 26200, Path(__file__).parent / "configs/state_benchmark.jsonnet" + ) + + +@pytest.mark.benchmark +def test_benchmark_storage(custom_cronos): + w3: Web3 = custom_cronos.w3 + w3_wait_for_block(w3, 1) + contract = deploy_contract(w3, CONTRACTS["BenchmarkStorage"]) + + n = 3000 + gas = 81500000 + iterations = 200 + parity = 100 + + def task(acct, acct_i): + for i in range(iterations): + seed = i * 10 + acct_i + tx = contract.functions.batch_set(seed, n, n * parity).buildTransaction( + {"from": acct.address, "gas": gas} + ) + print(send_transaction(w3, tx, acct.key)) + + accounts = [ + ACCOUNTS["validator"], + ACCOUNTS["community"], + ACCOUNTS["signer1"], + ACCOUNTS["signer2"], + ] + with ThreadPoolExecutor(len(accounts)) as exec: + tasks = [exec.submit(task, acct, i) for i, acct in enumerate(accounts)] + for t in tasks: + t.result() + + +def rocksdb_stats(path): + import rocksdb + + db = rocksdb.DB(str(path), rocksdb.Options()) + for field in ["rocksdb.stats", "rocksdb.sstables"]: + print(f"############# {field}") + print(db.get_property(field.encode()).decode()) + + # space amplification + it = db.iteritems() + it.seek_to_first() + count = 0 + size = 0 + for k, v in it: + count += 1 + size += len(k) + len(v) + # directory size + fsize = sum(f.stat().st_size for f in path.glob("**/*") if f.is_file()) + print( + f"space_amplification: {fsize / size:.2f}, kv pairs: {count}, " + f"data size: {size}, file size: {fsize}" + ) + + +if __name__ == "__main__": + import sys + + rocksdb_stats(Path(sys.argv[1])) diff --git a/integration_tests/utils.py b/integration_tests/utils.py index 79b2bbbff5..4e02f6a6fc 100644 --- a/integration_tests/utils.py +++ b/integration_tests/utils.py @@ -47,6 +47,7 @@ "TestBlackListERC20": "TestBlackListERC20.sol", "CroBridge": "CroBridge.sol", "CronosGravityCancellation": "CronosGravityCancellation.sol", + "BenchmarkStorage": "BenchmarkStorage.sol", } diff --git a/nix/default.nix b/nix/default.nix index 09bbf95631..9232c5258c 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -50,7 +50,7 @@ import sources.nixpkgs { }; hermes = pkgs.callPackage ./hermes.nix { src = sources.ibc-rs; }; }) - (_: pkgs: { test-env = import ./testenv.nix { inherit pkgs; }; }) + (final: _: { test-env = import ./testenv.nix { pkgs = final; }; }) (_: pkgs: { rocksdb = (pkgs.rocksdb.override { enableJemalloc = true; }).overrideAttrs (old: rec { pname = "rocksdb"; diff --git a/nix/testenv.nix b/nix/testenv.nix index c5b9ef023a..028e692c2a 100644 --- a/nix/testenv.nix +++ b/nix/testenv.nix @@ -20,5 +20,12 @@ pkgs.poetry2nix.mkPoetryEnv { nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ self.cython ]; } ); + + rocksdb = super.rocksdb.overridePythonAttrs ( + old: { + nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ self.cython self.pkgconfig ]; + buildInputs = (old.buildInputs or [ ]) ++ [ pkgs.rocksdb ]; + } + ); }); } diff --git a/scripts/cronos-devnet.yaml b/scripts/cronos-devnet.yaml index 0b9b865e1a..ec2f3c9fe0 100644 --- a/scripts/cronos-devnet.yaml +++ b/scripts/cronos-devnet.yaml @@ -1,7 +1,7 @@ dotenv: .env cronos_777-1: cmd: cronosd - start-flags: "--trace" + start-flags: "--trace --streamers versiondb,file" app-config: minimum-gas-prices: 0basetcro index-events: diff --git a/versiondb/dbutils.go b/versiondb/dbutils.go new file mode 100644 index 0000000000..2334d0f2c1 --- /dev/null +++ b/versiondb/dbutils.go @@ -0,0 +1,84 @@ +package versiondb + +import ( + "encoding/binary" + "sort" + + "github.com/RoaringBitmap/roaring/roaring64" +) + +var ChunkLimit = uint64(1950) // threshold beyond which MDBX overflow pages appear: 4096 / 2 - (keySize + 8) + +// CutLeft - cut from bitmap `targetSize` bytes from left +// removing lft part from `bm` +// returns nil on zero cardinality +func CutLeft64(bm *roaring64.Bitmap, sizeLimit uint64) *roaring64.Bitmap { + if bm.GetCardinality() == 0 { + return nil + } + + sz := bm.GetSerializedSizeInBytes() + if sz <= sizeLimit { + lft := roaring64.New() + lft.AddRange(bm.Minimum(), bm.Maximum()+1) + lft.And(bm) + lft.RunOptimize() + bm.Clear() + return lft + } + + from := bm.Minimum() + minMax := bm.Maximum() - bm.Minimum() + to := sort.Search(int(minMax), func(i int) bool { // can be optimized to avoid "too small steps", but let's leave it for readability + lft := roaring64.New() // bitmap.Clear() method intentionally not used here, because then serialized size of bitmap getting bigger + lft.AddRange(from, from+uint64(i)+1) + lft.And(bm) + lft.RunOptimize() + return lft.GetSerializedSizeInBytes() > sizeLimit + }) + + lft := roaring64.New() + lft.AddRange(from, from+uint64(to)) // no +1 because sort.Search returns element which is just higher threshold - but we need lower + lft.And(bm) + bm.RemoveRange(from, from+uint64(to)) + lft.RunOptimize() + return lft +} + +func WalkChunks64(bm *roaring64.Bitmap, sizeLimit uint64, f func(chunk *roaring64.Bitmap, isLast bool) error) error { + for bm.GetCardinality() > 0 { + if err := f(CutLeft64(bm, sizeLimit), bm.GetCardinality() == 0); err != nil { + return err + } + } + return nil +} + +func WalkChunkWithKeys64(k []byte, m *roaring64.Bitmap, sizeLimit uint64, f func(chunkKey []byte, chunk *roaring64.Bitmap) error) error { + return WalkChunks64(m, sizeLimit, func(chunk *roaring64.Bitmap, isLast bool) error { + chunkKey := make([]byte, len(k)+8) + copy(chunkKey, k) + if isLast { + binary.BigEndian.PutUint64(chunkKey[len(k):], ^uint64(0)) + } else { + binary.BigEndian.PutUint64(chunkKey[len(k):], chunk.Maximum()) + } + return f(chunkKey, chunk) + }) +} + +// SeekInBitmap64 - returns value in bitmap which is >= n +func SeekInBitmap64(m *roaring64.Bitmap, n uint64) (found uint64, ok bool) { + if m == nil || m.IsEmpty() { + return 0, false + } + if n == 0 { + return m.Minimum(), true + } + searchRank := m.Rank(n - 1) + if searchRank >= m.GetCardinality() { + return 0, false + } + found, _ = m.Select(searchRank) + return found, true +} diff --git a/versiondb/multistore.go b/versiondb/multistore.go new file mode 100644 index 0000000000..c60948087f --- /dev/null +++ b/versiondb/multistore.go @@ -0,0 +1,116 @@ +package versiondb + +import ( + "io" + "sync" + + "github.com/cosmos/cosmos-sdk/store/cachemulti" + "github.com/cosmos/cosmos-sdk/store/types" + sdk "github.com/cosmos/cosmos-sdk/types" +) + +var _ sdk.MultiStore = (*MultiStore)(nil) + +type MultiStore struct { + versionDB VersionStore + storeKeys []types.StoreKey + + traceWriter io.Writer + traceContext types.TraceContext + traceContextMutex sync.Mutex +} + +func NewMultiStore(versionDB VersionStore, storeKeys []types.StoreKey) sdk.MultiStore { + return &MultiStore{versionDB: versionDB, storeKeys: storeKeys} +} + +func (s *MultiStore) GetStoreType() types.StoreType { + return types.StoreTypeMulti +} + +func (s *MultiStore) cacheMultiStore(version *int64) sdk.CacheMultiStore { + stores := make(map[types.StoreKey]types.CacheWrapper) + for _, k := range s.storeKeys { + stores[k] = NewKVStore(s.versionDB, k, version) + } + return cachemulti.NewStore(nil, stores, nil, s.traceWriter, s.getTracingContext(), nil) +} + +func (s *MultiStore) CacheMultiStore() sdk.CacheMultiStore { + return s.cacheMultiStore(nil) +} + +func (s *MultiStore) CacheMultiStoreWithVersion(version int64) (sdk.CacheMultiStore, error) { + return s.cacheMultiStore(&version), nil +} + +// CacheWrap implements CacheWrapper/MultiStore/CommitStore. +func (s *MultiStore) CacheWrap() types.CacheWrap { + return s.CacheMultiStore().(types.CacheWrap) +} + +// CacheWrapWithTrace implements the CacheWrapper interface. +func (s *MultiStore) CacheWrapWithTrace(_ io.Writer, _ types.TraceContext) types.CacheWrap { + return s.CacheWrap() +} + +// CacheWrapWithListeners implements the CacheWrapper interface. +func (s *MultiStore) CacheWrapWithListeners(_ types.StoreKey, _ []types.WriteListener) types.CacheWrap { + return s.CacheWrap() +} + +func (s *MultiStore) GetStore(storeKey types.StoreKey) sdk.Store { + return s.GetKVStore(storeKey) +} + +func (s *MultiStore) GetKVStore(storeKey types.StoreKey) sdk.KVStore { + return NewKVStore(s.versionDB, storeKey, nil) +} + +// SetTracer sets the tracer for the MultiStore that the underlying +// stores will utilize to trace operations. A MultiStore is returned. +func (s *MultiStore) SetTracer(w io.Writer) types.MultiStore { + s.traceWriter = w + return s +} + +// SetTracingContext updates the tracing context for the MultiStore by merging +// the given context with the existing context by key. Any existing keys will +// be overwritten. It is implied that the caller should update the context when +// necessary between tracing operations. It returns a modified MultiStore. +func (s *MultiStore) SetTracingContext(tc types.TraceContext) types.MultiStore { + s.traceContextMutex.Lock() + defer s.traceContextMutex.Unlock() + s.traceContext = s.traceContext.Merge(tc) + + return s +} + +func (s *MultiStore) getTracingContext() types.TraceContext { + s.traceContextMutex.Lock() + defer s.traceContextMutex.Unlock() + + if s.traceContext == nil { + return nil + } + + ctx := types.TraceContext{} + for k, v := range s.traceContext { + ctx[k] = v + } + + return ctx +} + +// TracingEnabled returns if tracing is enabled for the MultiStore. +func (s *MultiStore) TracingEnabled() bool { + return s.traceWriter != nil +} + +func (s *MultiStore) ListeningEnabled(key types.StoreKey) bool { + return false +} + +func (s *MultiStore) AddListeners(key types.StoreKey, listeners []types.WriteListener) { + panic("not supported") +} diff --git a/versiondb/store.go b/versiondb/store.go new file mode 100644 index 0000000000..41905a041a --- /dev/null +++ b/versiondb/store.go @@ -0,0 +1,86 @@ +package versiondb + +import ( + "io" + "time" + + "github.com/cosmos/cosmos-sdk/store/cachekv" + "github.com/cosmos/cosmos-sdk/store/listenkv" + "github.com/cosmos/cosmos-sdk/store/tracekv" + "github.com/cosmos/cosmos-sdk/store/types" + "github.com/cosmos/cosmos-sdk/telemetry" +) + +var _ types.KVStore = (*Store)(nil) + +// Store Implements types.KVStore +type Store struct { + store VersionStore + storeKey types.StoreKey + version *int64 +} + +func NewKVStore(store VersionStore, storeKey types.StoreKey, version *int64) *Store { + return &Store{store, storeKey, version} +} + +// Implements Store. +func (st *Store) GetStoreType() types.StoreType { + // FIXME + return types.StoreTypeIAVL +} + +// Implements Store. +func (st *Store) CacheWrap() types.CacheWrap { + return cachekv.NewStore(st) +} + +// CacheWrapWithTrace implements the Store interface. +func (st *Store) CacheWrapWithTrace(w io.Writer, tc types.TraceContext) types.CacheWrap { + return cachekv.NewStore(tracekv.NewStore(st, w, tc)) +} + +// CacheWrapWithListeners implements the CacheWrapper interface. +func (st *Store) CacheWrapWithListeners(storeKey types.StoreKey, listeners []types.WriteListener) types.CacheWrap { + return cachekv.NewStore(listenkv.NewStore(st, storeKey, listeners)) +} + +// Implements types.KVStore. +func (st *Store) Get(key []byte) []byte { + defer telemetry.MeasureSince(time.Now(), "store", "iavl", "get") + value, err := st.store.GetAtVersion(st.storeKey.Name(), key, st.version) + if err != nil { + panic(err) + } + return value +} + +// Implements types.KVStore. +func (st *Store) Has(key []byte) (exists bool) { + defer telemetry.MeasureSince(time.Now(), "store", "iavl", "has") + has, err := st.store.HasAtVersion(st.storeKey.Name(), key, st.version) + if err != nil { + panic(err) + } + return has +} + +// Implements types.KVStore. +func (st *Store) Iterator(start, end []byte) types.Iterator { + return st.store.IteratorAtVersion(st.storeKey.Name(), start, end, st.version) +} + +// Implements types.KVStore. +func (st *Store) ReverseIterator(start, end []byte) types.Iterator { + return st.store.ReverseIteratorAtVersion(st.storeKey.Name(), start, end, st.version) +} + +// Implements types.KVStore. +func (st *Store) Set(key, value []byte) { + panic("write operation is not supported") +} + +// Implements types.KVStore. +func (st *Store) Delete(key []byte) { + panic("write operation is not supported") +} diff --git a/versiondb/streaming_service.go b/versiondb/streaming_service.go new file mode 100644 index 0000000000..ae833fd1eb --- /dev/null +++ b/versiondb/streaming_service.go @@ -0,0 +1,119 @@ +package versiondb + +import ( + "sort" + "strings" + "sync" + + abci "github.com/tendermint/tendermint/abci/types" + + "github.com/cosmos/cosmos-sdk/baseapp" + "github.com/cosmos/cosmos-sdk/store/types" + sdk "github.com/cosmos/cosmos-sdk/types" +) + +var _ baseapp.StreamingService = &StreamingService{} + +// FlattenListener listens to the state writes and flatten them in memory. +// One listener only listens to a single storeKey. +type FlattenListener struct { + stateCache map[string]types.StoreKVPair +} + +func NewFlattenListener() *FlattenListener { + return &FlattenListener{ + stateCache: make(map[string]types.StoreKVPair), + } +} + +func (fl *FlattenListener) OnWrite(storeKey types.StoreKey, key []byte, value []byte, delete bool) error { + fl.stateCache[string(key)] = types.StoreKVPair{ + StoreKey: storeKey.Name(), + Delete: delete, + Key: key, + Value: value, + } + return nil +} + +// StreamingService is a concrete implementation of StreamingService that accumulate the state changes in current block, +// writes the ordered changeset out to version storage. +type StreamingService struct { + listeners map[types.StoreKey]*FlattenListener // the listeners that will be initialized with BaseApp + versionStore VersionStore + currentBlockNumber int64 // the current block number +} + +// NewStreamingService creates a new StreamingService for the provided writeDir, (optional) filePrefix, and storeKeys +func NewStreamingService(versionStore VersionStore, storeKeys []types.StoreKey) *StreamingService { + listeners := make(map[types.StoreKey]*FlattenListener, len(storeKeys)) + // in this case, we are using the same listener for each Store + for _, key := range storeKeys { + listeners[key] = NewFlattenListener() + } + return &StreamingService{listeners, versionStore, 0} +} + +// Listeners satisfies the baseapp.StreamingService interface +func (fss *StreamingService) Listeners() map[types.StoreKey][]types.WriteListener { + listeners := make(map[types.StoreKey][]types.WriteListener, len(fss.listeners)) + for storeKey, listener := range fss.listeners { + listeners[storeKey] = []types.WriteListener{listener} + } + return listeners +} + +// ListenBeginBlock satisfies the baseapp.ABCIListener interface +// It sets the currentBlockNumber. +func (fss *StreamingService) ListenBeginBlock(ctx sdk.Context, req abci.RequestBeginBlock, res abci.ResponseBeginBlock) error { + fss.currentBlockNumber = req.GetHeader().Height + return nil +} + +// ListenDeliverTx satisfies the baseapp.ABCIListener interface +func (fss *StreamingService) ListenDeliverTx(ctx sdk.Context, req abci.RequestDeliverTx, res abci.ResponseDeliverTx) error { + return nil +} + +// ListenEndBlock satisfies the baseapp.ABCIListener interface +// It merge the state caches of all the listeners together, and write out to the versionStore. +func (fss *StreamingService) ListenEndBlock(ctx sdk.Context, req abci.RequestEndBlock, res abci.ResponseEndBlock) error { + // sort by the storeKeys first + storeKeys := make([]types.StoreKey, 0, len(fss.listeners)) + for storeKey := range fss.listeners { + storeKeys = append(storeKeys, storeKey) + } + sort.SliceStable(storeKeys, func(i, j int) bool { + return strings.Compare(storeKeys[i].Name(), storeKeys[j].Name()) < 0 + }) + + // concat the state caches + var changeSet []types.StoreKVPair + for _, storeKey := range storeKeys { + cache := fss.listeners[storeKey].stateCache + fss.listeners[storeKey].stateCache = make(map[string]types.StoreKVPair) + + // sort the cache by key + keys := make([]string, 0, len(cache)) + for key := range cache { + keys = append(keys, key) + } + sort.Strings(keys) + + for _, key := range keys { + changeSet = append(changeSet, cache[key]) + } + } + + return fss.versionStore.PutAtVersion(fss.currentBlockNumber, changeSet) +} + +// Stream satisfies the baseapp.StreamingService interface +func (fss *StreamingService) Stream(wg *sync.WaitGroup) error { + return nil +} + +// Close satisfies the io.Closer interface, which satisfies the baseapp.StreamingService interface +func (fss *StreamingService) Close() error { + return nil +} diff --git a/versiondb/tmdb/history.go b/versiondb/tmdb/history.go new file mode 100644 index 0000000000..7f6a829d57 --- /dev/null +++ b/versiondb/tmdb/history.go @@ -0,0 +1,93 @@ +package tmdb + +import ( + "bytes" + + "github.com/RoaringBitmap/roaring/roaring64" + + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/crypto-org-chain/cronos/versiondb" + dbm "github.com/tendermint/tm-db" +) + +const LastChunkId = ^uint64(0) + +func HistoryIndexKey(key []byte, height uint64) []byte { + return append(key, sdk.Uint64ToBigEndian(height)...) +} + +// GetHistoryIndex returns the history index bitmap chunk which covers the target version. +func GetHistoryIndex(db dbm.DB, key []byte, height uint64) (*roaring64.Bitmap, error) { + // try to seek the first chunk whose maximum is bigger or equal to the target height. + it, err := db.Iterator( + HistoryIndexKey(key, height), + sdk.PrefixEndBytes(key), + ) + if err != nil { + return nil, err + } + defer it.Close() // nolint: errcheck + + if !it.Valid() { + return nil, nil + } + + m := roaring64.New() + _, err = m.ReadFrom(bytes.NewReader(it.Value())) + if err != nil { + return nil, err + } + return m, nil +} + +// SeekHistoryIndex locate the minimal version that changed the key and is larger than the target version, +// using the returned version can find the value for the target version in changeset table. +// If not found, return -1 +func SeekHistoryIndex(db dbm.DB, key []byte, version uint64) (int64, error) { + // either m.Maximum() >= version + 1, or is the last chunk. + m, err := GetHistoryIndex(db, key, version+1) + if err != nil { + return -1, err + } + found, ok := versiondb.SeekInBitmap64(m, version+1) + if !ok { + return -1, nil + } + return int64(found), nil +} + +// WriteHistoryIndex set the block height to the history bitmap. +// it try to set to the last chunk, if the last chunk exceeds chunk limit, split it. +func WriteHistoryIndex(db dbm.DB, batch dbm.Batch, key []byte, height uint64) error { + lastKey := HistoryIndexKey(key, LastChunkId) + bz, err := db.Get(lastKey) + if err != nil { + return err + } + + m := roaring64.New() + if len(bz) > 0 { + _, err = m.ReadFrom(bytes.NewReader(bz)) + if err != nil { + return err + } + } + m.Add(height) + + // chunking + if err = versiondb.WalkChunks64(m, versiondb.ChunkLimit, func(chunk *roaring64.Bitmap, isLast bool) error { + chunkKey := lastKey + if !isLast { + chunkKey = HistoryIndexKey(key, chunk.Maximum()) + } + bz, err := chunk.ToBytes() + if err != nil { + return err + } + return batch.Set(chunkKey, bz) + }); err != nil { + return err + } + + return nil +} diff --git a/versiondb/tmdb/store.go b/versiondb/tmdb/store.go new file mode 100644 index 0000000000..34606f1afe --- /dev/null +++ b/versiondb/tmdb/store.go @@ -0,0 +1,190 @@ +package tmdb + +import ( + "bytes" + "errors" + + "github.com/cosmos/cosmos-sdk/store/types" + sdk "github.com/cosmos/cosmos-sdk/types" + gogotypes "github.com/cosmos/gogoproto/types" + "github.com/crypto-org-chain/cronos/versiondb" + dbm "github.com/tendermint/tm-db" +) + +const latestVersionKey = "s/latest" + +var _ versiondb.VersionStore = (*Store)(nil) + +// Store implements `VersionStore`. +type Store struct { + // latest key-value pairs + plainDB dbm.DB + // history bitmap index of keys + historyDB dbm.DB + // changesets of each blocks + changesetDB dbm.DB +} + +func NewStore(plainDB, historyDB, changesetDB dbm.DB) *Store { + return &Store{plainDB, historyDB, changesetDB} +} + +// PutAtVersion implements VersionStore interface +// TODO reduce allocation within iterations. +func (s *Store) PutAtVersion(version int64, changeSet []types.StoreKVPair) error { + plainBatch := s.plainDB.NewBatch() + defer plainBatch.Close() // nolint: errcheck + historyBatch := s.historyDB.NewBatch() + defer historyBatch.Close() // nolint: errcheck + changesetBatch := s.changesetDB.NewBatch() + defer changesetBatch.Close() // nolint: errcheck + + for _, pair := range changeSet { + key := prependStoreKey(pair.StoreKey, pair.Key) + + if version == 0 { + // genesis state is written into plain state directly + if pair.Delete { + return errors.New("can't delete at genesis") + } else { + if err := plainBatch.Set(key, pair.Value); err != nil { + return err + } + } + continue + } + + original, err := s.plainDB.Get(key) + if err != nil { + return err + } + if bytes.Equal(original, pair.Value) { + // do nothing if the value is not changed + continue + } + + // write history index + if err := WriteHistoryIndex(s.historyDB, historyBatch, key, uint64(version)); err != nil { + return err + } + + // write the old value to changeset + if len(original) > 0 { + changesetKey := append(sdk.Uint64ToBigEndian(uint64(version)), key...) + if err := changesetBatch.Set(changesetKey, original); err != nil { + return err + } + } + + // write the new value to plain state + if pair.Delete { + if err := plainBatch.Delete(key); err != nil { + return err + } + } else { + if err := plainBatch.Set(key, pair.Value); err != nil { + return err + } + } + } + + // write latest version to plain state + bz, err := gogotypes.StdInt64Marshal(version) + if err != nil { + return err + } + if err := plainBatch.Set([]byte(latestVersionKey), bz); err != nil { + return err + } + + if err := changesetBatch.WriteSync(); err != nil { + return err + } + if err := historyBatch.WriteSync(); err != nil { + return err + } + return plainBatch.WriteSync() +} + +// GetAtVersion implements VersionStore interface +func (s *Store) GetAtVersion(storeKey string, key []byte, version *int64) ([]byte, error) { + rawKey := prependStoreKey(storeKey, key) + if version == nil { + return s.plainDB.Get(rawKey) + } + height := uint64(*version) + found, err := SeekHistoryIndex(s.historyDB, rawKey, height) + if err != nil { + return nil, err + } + if found < 0 { + // there's no change records found after the target version, query the latest state. + return s.plainDB.Get(rawKey) + } + // get from changeset + changesetKey := ChangesetKey(uint64(found), rawKey) + return s.changesetDB.Get(changesetKey) +} + +// HasAtVersion implements VersionStore interface +func (s *Store) HasAtVersion(storeKey string, key []byte, version *int64) (bool, error) { + rawKey := prependStoreKey(storeKey, key) + if version == nil { + return s.plainDB.Has(rawKey) + } + height := uint64(*version) + found, err := SeekHistoryIndex(s.historyDB, rawKey, height) + if err != nil { + return false, err + } + if found < 0 { + // there's no change records after the target version, query the latest state. + return s.plainDB.Has(rawKey) + } + // get from changeset + changesetKey := ChangesetKey(uint64(found), rawKey) + return s.changesetDB.Has(changesetKey) +} + +// IteratorAtVersion implements VersionStore interface +func (s *Store) IteratorAtVersion(storeKey string, start, end []byte, version *int64) types.Iterator { + // TODO + return nil +} + +// ReverseIteratorAtVersion implements VersionStore interface +func (s *Store) ReverseIteratorAtVersion(storeKey string, start, end []byte, version *int64) types.Iterator { + // TODO + return nil +} + +// GetLatestVersion returns the latest version stored in plain state, +// it's committed after the changesets, so the data for this version is guaranteed to be persisted. +// returns -1 if the key don't exists. +func (s *Store) GetLatestVersion() (int64, error) { + bz, err := s.plainDB.Get([]byte(latestVersionKey)) + if err != nil { + return -1, err + } else if bz == nil { + return -1, nil + } + + var latestVersion int64 + + if err := gogotypes.StdInt64Unmarshal(&latestVersion, bz); err != nil { + return -1, err + } + + return latestVersion, nil +} + +// ChangesetKey build key changeset db +func ChangesetKey(version uint64, key []byte) []byte { + return append(sdk.Uint64ToBigEndian(version), key...) +} + +// prependStoreKey prepends storeKey to the key +func prependStoreKey(storeKey string, key []byte) []byte { + prefix := "s/k:" + storeKey + "/" + return append([]byte(prefix), key...) +} diff --git a/versiondb/tmdb/store_test.go b/versiondb/tmdb/store_test.go new file mode 100644 index 0000000000..cc597247ff --- /dev/null +++ b/versiondb/tmdb/store_test.go @@ -0,0 +1,88 @@ +package tmdb + +import ( + "testing" + + "github.com/cosmos/cosmos-sdk/store/types" + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/crypto-org-chain/cronos/versiondb" + "github.com/stretchr/testify/require" + dbm "github.com/tendermint/tm-db" +) + +func TestBasics(t *testing.T) { + var v int64 + key1 := []byte("key1") + key2 := []byte("key2") + value1 := []byte("value1") + value2 := []byte("value2") + + store := NewStore(dbm.NewMemDB(), dbm.NewMemDB(), dbm.NewMemDB()) + require.NoError(t, store.PutAtVersion(0, []types.StoreKVPair{ + {StoreKey: "bank", Key: key1, Value: value1}, + {StoreKey: "bank", Key: key2, Value: value2}, + {StoreKey: "staking", Key: key1, Value: value1}, + {StoreKey: "evm", Key: key1, Value: value1}, + })) + require.NoError(t, store.PutAtVersion(1, []types.StoreKVPair{ + {StoreKey: "bank", Key: key1, Value: value2}, + })) + require.NoError(t, store.PutAtVersion(2, []types.StoreKVPair{ + {StoreKey: "staking", Delete: true, Key: key1}, + })) + require.NoError(t, store.PutAtVersion(3, []types.StoreKVPair{ + {StoreKey: "staking", Key: key1, Value: value2}, + })) + + value, err := store.GetAtVersion("staking", key1, nil) + require.NoError(t, err) + require.Equal(t, value, value2) + + v = 2 + ok, err := store.HasAtVersion("staking", key1, &v) + require.NoError(t, err) + require.False(t, ok) + value, err = store.GetAtVersion("staking", key1, &v) + require.NoError(t, err) + require.Empty(t, value) + + v = 1 + ok, err = store.HasAtVersion("staking", key1, &v) + require.NoError(t, err) + require.True(t, ok) + value, err = store.GetAtVersion("staking", key1, &v) + require.NoError(t, err) + require.Equal(t, value, value1) + + // never changed since genesis + ok, err = store.HasAtVersion("bank", key2, nil) + require.NoError(t, err) + require.True(t, ok) + value, err = store.GetAtVersion("bank", key2, nil) + require.NoError(t, err) + require.Equal(t, value2, value) + for i := int64(1); i < 4; i++ { + // never changed + ok, err = store.HasAtVersion("bank", key2, &i) + require.NoError(t, err) + require.True(t, ok) + + value, err = store.GetAtVersion("bank", key2, &i) + require.NoError(t, err) + require.Equal(t, value2, value) + } +} + +func TestBitmapChunking(t *testing.T) { + oldChunkLimit := versiondb.ChunkLimit + versiondb.ChunkLimit = uint64(100) + key1 := []byte("key1") + store := NewStore(dbm.NewMemDB(), dbm.NewMemDB(), dbm.NewMemDB()) + for i := int64(0); i < 100000; i++ { + require.NoError(t, store.PutAtVersion(i, []types.StoreKVPair{ + {StoreKey: "bank", Key: key1, Value: sdk.Uint64ToBigEndian(uint64(i))}, + })) + } + versiondb.ChunkLimit = oldChunkLimit + // TODO +} diff --git a/versiondb/types.go b/versiondb/types.go new file mode 100644 index 0000000000..e1ca397677 --- /dev/null +++ b/versiondb/types.go @@ -0,0 +1,21 @@ +package versiondb + +import ( + "github.com/cosmos/cosmos-sdk/store/types" +) + +// VersionStore is a versioned storage of a flat key-value pairs. +// it don't need to support merkle proof, so could be implemented in a much more efficient way. +// `nil` version means the latest version. +type VersionStore interface { + GetAtVersion(storeKey string, key []byte, version *int64) ([]byte, error) + HasAtVersion(storeKey string, key []byte, version *int64) (bool, error) + IteratorAtVersion(storeKey string, start, end []byte, version *int64) types.Iterator + ReverseIteratorAtVersion(storeKey string, start, end []byte, version *int64) types.Iterator + GetLatestVersion() (int64, error) + + // Persist the change set of a block, + // the `changeSet` should be ordered by (storeKey, key), + // the version should be latest version plus one. + PutAtVersion(version int64, changeSet []types.StoreKVPair) error +}