Skip to content

Commit

Permalink
Feature/update go geth sharding (#13)
Browse files Browse the repository at this point in the history
* Upgrade Geth, Go and add CICD

* update ipld-eth-server version

* Track validation progress on a channel

* Add integration tests

* Setup validator config and update instructions to run tests locally

* Update readme and tests

* Update test to use v4 Infrastructure

* Inlcude the env file

* Fix config file write

* Update DB configuration

Co-authored-by: prathamesh0 <[email protected]>
Co-authored-by: Ashwin Phatak <[email protected]>
  • Loading branch information
3 people authored Jun 3, 2022
1 parent cc935dc commit d111dd8
Show file tree
Hide file tree
Showing 15 changed files with 428 additions and 360 deletions.
6 changes: 6 additions & 0 deletions .github/workflows/on-pr.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
name: Run all tests
on: [pull_request]

jobs:
run-tests:
uses: ./.github/workflows/tests.yml
113 changes: 113 additions & 0 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
name: Test the stack.
on:
workflow_call:

env:
STACK_ORCHESTRATOR_REF: 418957a1f745c921b21286c13bb033f922a91ae9
GO_ETHEREUM_REF: "v1.10.18-statediff-4.0.2-alpha"
IPLD_ETH_DB_REF: 91d30b9ea1acecd0a7f4307390a98bf3e289b8d7

jobs:
integrationtest:
name: Run integration tests
env:
GOPATH: /tmp/go
DB_WRITE: true
runs-on: ubuntu-latest
steps:
- name: Create GOPATH
run: mkdir -p /tmp/go
- uses: actions/setup-go@v3
with:
go-version: ">=1.18.0"
check-latest: true
- uses: actions/checkout@v2
with:
path: "./ipld-eth-db-validator"
- uses: actions/checkout@v2
with:
ref: ${{ env.STACK_ORCHESTRATOR_REF }}
path: "./stack-orchestrator/"
repository: vulcanize/stack-orchestrator
- uses: actions/checkout@v2
with:
ref: ${{ env.GO_ETHEREUM_REF }}
repository: vulcanize/go-ethereum
path: "./go-ethereum/"
- uses: actions/checkout@v2
with:
ref: ${{ env.IPLD_ETH_DB_REF }}
repository: vulcanize/ipld-eth-db
path: "./ipld-eth-db/"
- name: Create config file
run: |
echo vulcanize_go_ethereum=$GITHUB_WORKSPACE/go-ethereum/ >> ./config.sh
echo vulcanize_test_contract=$GITHUB_WORKSPACE/ipld-eth-db-validator/test/contract >> ./config.sh
echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ >> ./config.sh
echo db_write=$DB_WRITE >> ./config.sh
echo go_ethereum_db_dependency=access-node >> ./config.sh
echo connecting_db_name=vulcanize_testing_v4 >> ./config.sh
cat ./config.sh
- name: Build geth
run: |
cd $GITHUB_WORKSPACE/stack-orchestrator/helper-scripts
./compile-geth.sh \
-p "$GITHUB_WORKSPACE/config.sh" \
-e docker
- name: Run docker compose
run: |
docker-compose \
-f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-migration.yml" \
-f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" \
-f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-contract.yml" \
-f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-timescale-db.yml" \
--env-file "$GITHUB_WORKSPACE/config.sh" \
up -d --build
- name: Run integration test.
run: |
cd $GITHUB_WORKSPACE/ipld-eth-db-validator
./scripts/run_integration_test.sh
unittest:
name: Run unit tests
env:
GOPATH: /tmp/go
DB_WRITE: true
runs-on: ubuntu-latest
steps:
- name: Create GOPATH
run: mkdir -p /tmp/go
- uses: actions/setup-go@v3
with:
go-version: ">=1.18.0"
check-latest: true
- uses: actions/checkout@v2
with:
path: "./ipld-eth-db-validator"
- uses: actions/checkout@v2
with:
ref: ${{ env.STACK_ORCHESTRATOR_REF }}
path: "./stack-orchestrator/"
repository: vulcanize/stack-orchestrator
- uses: actions/checkout@v2
with:
ref: ${{ env.IPLD_ETH_DB_REF }}
repository: vulcanize/ipld-eth-db
path: "./ipld-eth-db/"
- name: Create config file
run: |
echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ >> ./config.sh
echo db_write=$DB_WRITE >> ./config.sh
echo connecting_db_name=vulcanize_testing_v4 >> ./config.sh
cat ./config.sh
- name: Run docker compose
run: |
docker-compose \
-f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-migration.yml" \
-f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-timescale-db.yml" \
--env-file "$GITHUB_WORKSPACE/config.sh" \
up -d --build
- name: Run unit test.
run: |
cd $GITHUB_WORKSPACE/ipld-eth-db-validator
PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8066 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing_v4 make test
12 changes: 4 additions & 8 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -3,22 +3,18 @@ BASE = $(GOPATH)/src/$(PACKAGE)
PKGS = go list ./... | grep -v "^vendor/"

# Tools
## Testing library
GINKGO = $(BIN)/ginkgo
$(BIN)/ginkgo:
go install github.com/onsi/ginkgo/ginkgo

.PHONY: integrationtest
integrationtest: | $(GINKGO) $(GOOSE)
integrationtest: | $(GOOSE)
go vet ./...
go fmt ./...
$(GINKGO) -r test/ -v
go run github.com/onsi/ginkgo/ginkgo -r test/ -v

.PHONY: test
test: | $(GINKGO) $(GOOSE)
test: | $(GOOSE)
go vet ./...
go fmt ./...
$(GINKGO) -r pkg/validator/ validator_test/ -v
go run github.com/onsi/ginkgo/ginkgo -r validator_test/ -v

build:
go fmt ./...
Expand Down
16 changes: 1 addition & 15 deletions cmd/state_validator.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"os/signal"
"sync"

"github.com/ethereum/go-ethereum/statediff"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
Expand All @@ -34,20 +33,7 @@ func stateValidator() {
logWithCommand.Fatal(err)
}

height := viper.GetUint64("validate.block-height")
if height < 1 {
logWithCommand.Fatalf("block height cannot be less the 1")
}
trail := viper.GetUint64("validate.trail")
sleepInterval := viper.GetUint("validate.sleepInterval")

chainConfigPath := viper.GetString("ethereum.chainConfig")
chainCfg, err := statediff.LoadConfig(chainConfigPath)
if err != nil {
logWithCommand.Fatal(err)
}

service := validator.NewService(cfg.DB, height, trail, sleepInterval, chainCfg)
service := validator.NewService(cfg, nil)

wg := new(sync.WaitGroup)
wg.Add(1)
Expand Down
22 changes: 12 additions & 10 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,17 +1,18 @@
module github.com/vulcanize/ipld-eth-db-validator

go 1.17
go 1.18

require (
github.com/ethereum/go-ethereum v1.10.17
github.com/ethereum/go-ethereum v1.10.18
github.com/jmoiron/sqlx v1.3.5
github.com/onsi/ginkgo v1.16.5
github.com/onsi/gomega v1.19.0
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.4.0
github.com/spf13/viper v1.11.0
github.com/vulcanize/ipfs-ethdb/v4 v4.0.0-alpha
github.com/vulcanize/ipld-eth-server/v4 v4.0.1-alpha
github.com/vulcanize/ipfs-ethdb/v4 v4.0.1-alpha
github.com/vulcanize/ipld-eth-server/v3 v3.2.1
github.com/vulcanize/ipld-eth-server/v4 v4.0.2-alpha
)

require (
Expand Down Expand Up @@ -60,7 +61,7 @@ require (
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
github.com/holiman/uint256 v1.2.0 // indirect
github.com/huin/goupnp v1.0.3-0.20220313090229-ca81a64b4204 // indirect
github.com/huin/goupnp v1.0.3 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/ipfs/bbloom v0.0.4 // indirect
github.com/ipfs/go-bitswap v0.4.0 // indirect
Expand Down Expand Up @@ -171,11 +172,12 @@ require (
github.com/libp2p/go-ws-transport v0.5.0 // indirect
github.com/libp2p/go-yamux/v2 v2.2.0 // indirect
github.com/libp2p/zeroconf/v2 v2.0.0 // indirect
github.com/lucas-clemente/quic-go v0.23.0 // indirect
github.com/lucas-clemente/quic-go v0.26.0 // indirect
github.com/magiconair/properties v1.8.6 // indirect
github.com/mailgun/groupcache/v2 v2.3.0 // indirect
github.com/marten-seemann/qtls-go1-16 v0.1.4 // indirect
github.com/marten-seemann/qtls-go1-17 v0.1.0 // indirect
github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect
github.com/marten-seemann/qtls-go1-18 v0.1.1 // indirect
github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
Expand Down Expand Up @@ -233,7 +235,7 @@ require (
github.com/tklauser/go-sysconf v0.3.6 // indirect
github.com/tklauser/numcpus v0.2.2 // indirect
github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef // indirect
github.com/vulcanize/eth-ipfs-state-validator/v4 v4.0.0-alpha // indirect
github.com/vulcanize/eth-ipfs-state-validator/v4 v4.0.1-alpha // indirect
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2 // indirect
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
Expand Down Expand Up @@ -270,4 +272,4 @@ require (
lukechampine.com/blake3 v1.1.6 // indirect
)

replace github.com/ethereum/go-ethereum v1.10.17 => github.com/vulcanize/go-ethereum v1.10.17-statediff-4.0.1-alpha
replace github.com/ethereum/go-ethereum v1.10.18 => github.com/vulcanize/go-ethereum v1.10.18-statediff-4.0.2-alpha
Loading

0 comments on commit d111dd8

Please sign in to comment.