test(p2p): regression test for kademlia

ChainSafe · Aug 6, 2024 · 8680331 · 8680331
1 parent 55c8beb
commit 8680331
Show file tree

Hide file tree

Showing 10 changed files with 1,255 additions and 119 deletions.
diff --git a/Makefile b/Makefile
@@ -115,7 +115,8 @@ test-release: go-mod
 test-all: test test-release
 
 go-mod:
- (cd $(PWD)/src/libp2p_bitswap/tests/go-app && go mod vendor && go build -o /tmp/forest-go-compat-test) || \
+ (cd $(PWD)/src/libp2p_bitswap/tests/go-app && go mod vendor && go build -o /tmp/forest-go-bitswap-compat-test) && \
+ (cd $(PWD)/src/libp2p/tests/go-kad && go mod vendor && go build -o /tmp/forest-go-kad-compat-test) || \
  (echo "Some tests require Go 1.21.x to be installed, follow instructions at https://go.dev/dl/" && exit 1)
 
 # Checks if all headers are present and adds if not

diff --git a/src/libp2p/discovery.rs b/src/libp2p/discovery.rs
@@ -26,7 +26,7 @@ use libp2p::{
  upnp, StreamProtocol,
 };
 use tokio::time::Interval;
-use tracing::{debug, info, trace, warn};
+use tracing::{debug, info, trace};
 
 use crate::utils::version::FOREST_VERSION_STRING;
 
@@ -133,35 +133,17 @@ impl<'a> DiscoveryConfig<'a> {
  } = self;
 
  let mut peers = HashSet::new();
-
  let kademlia_opt = if enable_kademlia {
- // This is a workaround for <https:/ChainSafe/forest/issues/4576>
- // `go-libp2p-kad-dht` sends the remote peer id as query key during bootstrap,
- // however, `rust-libp2p` returns empty peer list when the key it receives
- // matches the local one. It's fine to use a random peer id as a workaround
- // since Kademlia not used as value providers in filecoin p2p network.
- let kad_random_peer_id = PeerId::random();
- let store = MemoryStore::new(kad_random_peer_id);
- let kad_config = {
- let mut cfg = kad::Config::default();
- cfg.set_protocol_names(vec![StreamProtocol::try_from_owned(format!(
+ let mut kademlia = new_kademlia(
+ local_peer_id,
+ vec![StreamProtocol::try_from_owned(format!(
  "/fil/kad/{network_name}/kad/1.0.0"
- ))?]);
- cfg
- };
- let mut kademlia = kad::Behaviour::with_config(kad_random_peer_id, store, kad_config);
- // `set_mode(Server)` fixes https:/ChainSafe/forest/issues/3620
- // but it should not be required as the behaviour should automatically switch to server mode
- // according to the doc. It might be a bug in `libp2p`.
- // We should fix the bug or report with a minimal reproduction.
- kademlia.set_mode(Some(kad::Mode::Server));
+ ))?],
+ );
  for (peer_id, addr) in &user_defined {
  kademlia.add_address(peer_id, addr.clone());
  peers.insert(*peer_id);
  }
- if let Err(e) = kademlia.bootstrap() {
- warn!("Kademlia bootstrap failed: {}", e);
- }
  Some(kademlia)
  } else {
  None
@@ -198,6 +180,31 @@ impl<'a> DiscoveryConfig<'a> {
  }
 }
 
+pub fn new_kademlia(
+ _peer_id: PeerId,
+ protocols: Vec<StreamProtocol>,
+) -> kad::Behaviour<kad::store::MemoryStore> {
+ // This is a workaround for <https:/ChainSafe/forest/issues/4576>
+ // `go-libp2p-kad-dht` sends the remote peer id as query key during bootstrap,
+ // however, `rust-libp2p` returns empty peer list when the key it receives
+ // matches the local one. It's fine to use a random peer id as a workaround
+ // since Kademlia not used as value providers in filecoin p2p network.
+ let peer_id = PeerId::random();
+ let store = MemoryStore::new(peer_id);
+ let kad_config = {
+ let mut cfg = kad::Config::default();
+ cfg.set_protocol_names(protocols);
+ cfg
+ };
+ let mut kademlia = kad::Behaviour::with_config(peer_id, store, kad_config);
+ // `set_mode(Server)` fixes https:/ChainSafe/forest/issues/3620
+ // but it should not be required as the behaviour should automatically switch to server mode
+ // according to the doc. It might be a bug in `libp2p`.
+ // We should fix the bug or report with a minimal reproduction.
+ kademlia.set_mode(Some(kad::Mode::Server));
+ kademlia
+}
+
 /// Implementation of `NetworkBehaviour` that discovers the nodes on the
 /// network.
 // Behaviours that manage connections should come first, to get rid of some panics in debug build.

diff --git a/src/libp2p/mod.rs b/src/libp2p/mod.rs
@@ -26,4 +26,5 @@ pub use self::{config::*, peer_manager::*, service::*};
 #[cfg(test)]
 mod tests {
  mod decode_test;
+ mod kad_go_compat;
 }
diff --git a/src/libp2p/tests/go-kad/.gitignore b/src/libp2p/tests/go-kad/.gitignore
@@ -0,0 +1,27 @@
+# Created by https://www.toptal.com/developers/gitignore/api/go
+# Edit at https://www.toptal.com/developers/gitignore?templates=go
+
+### Go ###
+# If you prefer the allow list template instead of the deny list, see community template:
+# https:/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
+#
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+vendor/
+
+# Go workspace file
+go.work
+
+# End of https://www.toptal.com/developers/gitignore/api/go
diff --git a/src/libp2p/tests/go-kad/go.mod b/src/libp2p/tests/go-kad/go.mod
@@ -0,0 +1,134 @@
+module test/v2
+
+go 1.21
+
+require (
+ github.com/libp2p/go-libp2p v0.36.0
+ github.com/libp2p/go-libp2p-kad-dht v0.25.2
+ github.com/multiformats/go-multiaddr v0.13.0
+)
+
+require (
+ github.com/benbjohnson/clock v1.3.5 // indirect
+ github.com/beorn7/perks v1.0.1 // indirect
+ github.com/cespare/xxhash/v2 v2.3.0 // indirect
+ github.com/containerd/cgroups v1.1.0 // indirect
+ github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+ github.com/davecgh/go-spew v1.1.1 // indirect
+ github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
+ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
+ github.com/docker/go-units v0.5.0 // indirect
+ github.com/elastic/gosigar v0.14.3 // indirect
+ github.com/flynn/noise v1.1.0 // indirect
+ github.com/francoispqt/gojay v1.2.13 // indirect
+ github.com/go-logr/logr v1.4.2 // indirect
+ github.com/go-logr/stdr v1.2.2 // indirect
+ github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
+ github.com/godbus/dbus/v5 v5.1.0 // indirect
+ github.com/gogo/protobuf v1.3.2 // indirect
+ github.com/google/gopacket v1.1.19 // indirect
+ github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect
+ github.com/google/uuid v1.6.0 // indirect
+ github.com/gorilla/websocket v1.5.3 // indirect
+ github.com/hashicorp/errwrap v1.1.0 // indirect
+ github.com/hashicorp/go-multierror v1.1.1 // indirect
+ github.com/hashicorp/golang-lru v1.0.2 // indirect
+ github.com/huin/goupnp v1.3.0 // indirect
+ github.com/ipfs/boxo v0.21.0 // indirect
+ github.com/ipfs/go-cid v0.4.1 // indirect
+ github.com/ipfs/go-datastore v0.6.0 // indirect
+ github.com/ipfs/go-log v1.0.5 // indirect
+ github.com/ipfs/go-log/v2 v2.5.1 // indirect
+ github.com/ipld/go-ipld-prime v0.21.0 // indirect
+ github.com/jackpal/go-nat-pmp v1.0.2 // indirect
+ github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
+ github.com/jbenet/goprocess v0.1.4 // indirect
+ github.com/klauspost/compress v1.17.9 // indirect
+ github.com/klauspost/cpuid/v2 v2.2.8 // indirect
+ github.com/koron/go-ssdp v0.0.4 // indirect
+ github.com/libp2p/go-buffer-pool v0.1.0 // indirect
+ github.com/libp2p/go-cidranger v1.1.0 // indirect
+ github.com/libp2p/go-flow-metrics v0.1.0 // indirect
+ github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect
+ github.com/libp2p/go-libp2p-kbucket v0.6.3 // indirect
+ github.com/libp2p/go-libp2p-record v0.2.0 // indirect
+ github.com/libp2p/go-libp2p-routing-helpers v0.7.4 // indirect
+ github.com/libp2p/go-msgio v0.3.0 // indirect
+ github.com/libp2p/go-nat v0.2.0 // indirect
+ github.com/libp2p/go-netroute v0.2.1 // indirect
+ github.com/libp2p/go-reuseport v0.4.0 // indirect
+ github.com/libp2p/go-yamux/v4 v4.0.1 // indirect
+ github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
+ github.com/mattn/go-isatty v0.0.20 // indirect
+ github.com/miekg/dns v1.1.61 // indirect
+ github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
+ github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
+ github.com/minio/sha256-simd v1.0.1 // indirect
+ github.com/mr-tron/base58 v1.2.0 // indirect
+ github.com/multiformats/go-base32 v0.1.0 // indirect
+ github.com/multiformats/go-base36 v0.2.0 // indirect
+ github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
+ github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
+ github.com/multiformats/go-multibase v0.2.0 // indirect
+ github.com/multiformats/go-multicodec v0.9.0 // indirect
+ github.com/multiformats/go-multihash v0.2.3 // indirect
+ github.com/multiformats/go-multistream v0.5.0 // indirect
+ github.com/multiformats/go-varint v0.0.7 // indirect
+ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+ github.com/onsi/ginkgo/v2 v2.19.1 // indirect
+ github.com/opencontainers/runtime-spec v1.2.0 // indirect
+ github.com/opentracing/opentracing-go v1.2.0 // indirect
+ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
+ github.com/pion/datachannel v1.5.8 // indirect
+ github.com/pion/dtls/v2 v2.2.12 // indirect
+ github.com/pion/ice/v2 v2.3.33 // indirect
+ github.com/pion/interceptor v0.1.29 // indirect
+ github.com/pion/logging v0.2.2 // indirect
+ github.com/pion/mdns v0.0.12 // indirect
+ github.com/pion/randutil v0.1.0 // indirect
+ github.com/pion/rtcp v1.2.14 // indirect
+ github.com/pion/rtp v1.8.8 // indirect
+ github.com/pion/sctp v1.8.20 // indirect
+ github.com/pion/sdp/v3 v3.0.9 // indirect
+ github.com/pion/srtp/v2 v2.0.20 // indirect
+ github.com/pion/stun v0.6.1 // indirect
+ github.com/pion/transport/v2 v2.2.10 // indirect
+ github.com/pion/turn/v2 v2.1.6 // indirect
+ github.com/pion/webrtc/v3 v3.2.50 // indirect
+ github.com/pkg/errors v0.9.1 // indirect
+ github.com/pmezard/go-difflib v1.0.0 // indirect
+ github.com/polydawn/refmt v0.89.0 // indirect
+ github.com/prometheus/client_golang v1.19.1 // indirect
+ github.com/prometheus/client_model v0.6.1 // indirect
+ github.com/prometheus/common v0.55.0 // indirect
+ github.com/prometheus/procfs v0.15.1 // indirect
+ github.com/quic-go/qpack v0.4.0 // indirect
+ github.com/quic-go/quic-go v0.45.2 // indirect
+ github.com/quic-go/webtransport-go v0.8.0 // indirect
+ github.com/raulk/go-watchdog v1.3.0 // indirect
+ github.com/spaolacci/murmur3 v1.1.0 // indirect
+ github.com/stretchr/testify v1.9.0 // indirect
+ github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
+ github.com/wlynxg/anet v0.0.3 // indirect
+ go.opencensus.io v0.24.0 // indirect
+ go.opentelemetry.io/otel v1.28.0 // indirect
+ go.opentelemetry.io/otel/metric v1.28.0 // indirect
+ go.opentelemetry.io/otel/trace v1.28.0 // indirect
+ go.uber.org/dig v1.17.1 // indirect
+ go.uber.org/fx v1.22.1 // indirect
+ go.uber.org/mock v0.4.0 // indirect
+ go.uber.org/multierr v1.11.0 // indirect
+ go.uber.org/zap v1.27.0 // indirect
+ golang.org/x/crypto v0.25.0 // indirect
+ golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
+ golang.org/x/mod v0.20.0 // indirect
+ golang.org/x/net v0.27.0 // indirect
+ golang.org/x/sync v0.8.0 // indirect
+ golang.org/x/sys v0.23.0 // indirect
+ golang.org/x/text v0.16.0 // indirect
+ golang.org/x/tools v0.23.0 // indirect
+ gonum.org/v1/gonum v0.15.0 // indirect
+ google.golang.org/protobuf v1.34.2 // indirect
+ gopkg.in/yaml.v3 v3.0.1 // indirect
+ lukechampine.com/blake3 v1.3.0 // indirect
+)