feat: Add cannon support to docker-compose

cmd/cannon.go

@@ -14,6 +14,84 @@ var (
  cannonCfgFile string
 )
 
+type CannonOverride struct {
+ FlagHelper func(cmd *cobra.Command)
+ Setter func(cmd *cobra.Command, overrides *cannon.Override) error
+}
+
+type CannonOverrideConfig struct {
+ FlagName string
+ EnvName string
+ Description string
+ OverrideFunc func(val string, overrides *cannon.Override)
+}
+
+func createCannonOverride(config CannonOverrideConfig) CannonOverride {
+ return CannonOverride{
+ FlagHelper: func(cmd *cobra.Command) {
+ cmd.Flags().String(config.FlagName, "", config.Description+` (env: `+config.EnvName+`)`)
+ },
+ Setter: func(cmd *cobra.Command, overrides *cannon.Override) error {
+ val := ""
+
+ if cmd.Flags().Changed(config.FlagName) {
+ val = cmd.Flags().Lookup(config.FlagName).Value.String()
+ }
+
+ if os.Getenv(config.EnvName) != "" {
+ val = os.Getenv(config.EnvName)
+ }
+
+ if val == "" {
+ return nil
+ }
+
+ config.OverrideFunc(val, overrides)
+
+ return nil
+ },
+ }
+}
+
+var CannonOverrides = []CannonOverride{
+ createCannonOverride(CannonOverrideConfig{
+ FlagName: "cannon-xatu-output-authorization",
+ EnvName: "CANNON_XATU_OUTPUT_AUTHORIZATION",
+ Description: "sets the authorization secret for all xatu outputs",
+ OverrideFunc: func(val string, overrides *cannon.Override) {
+ overrides.XatuOutputAuth.Enabled = true
+ overrides.XatuOutputAuth.Value = val
+ },
+ }),
+ createCannonOverride(CannonOverrideConfig{
+ FlagName: "cannon-xatu-coordinator-authorization",
+ EnvName: "CANNON_XATU_COORDINATOR_AUTHORIZATION",
+ Description: "sets the authorization secret for the xatu coordinator",
+ OverrideFunc: func(val string, overrides *cannon.Override) {
+ overrides.XatuCoordinatorAuth.Enabled = true
+ overrides.XatuCoordinatorAuth.Value = val
+ },
+ }),
+ createCannonOverride(CannonOverrideConfig{
+ FlagName: "cannon-beacon-node-url",
+ EnvName: "CANNON_BEACON_NODE_URL",
+ Description: "sets the beacon node url",
+ OverrideFunc: func(val string, overrides *cannon.Override) {
+ overrides.BeaconNodeURL.Enabled = true
+ overrides.BeaconNodeURL.Value = val
+ },
+ }),
+ createCannonOverride(CannonOverrideConfig{
+ FlagName: "cannon-beacon-node-authorization-header",
+ EnvName: "CANNON_BEACON_NODE_AUTHORIZATION_HEADER",
+ Description: "sets the beacon node authorization header",
+ OverrideFunc: func(val string, overrides *cannon.Override) {
+ overrides.BeaconNodeAuthorizationHeader.Enabled = true
+ overrides.BeaconNodeAuthorizationHeader.Value = val
+ },
+ }),
+}
+
 // cannonCmd represents the cannon command
 var cannonCmd = &cobra.Command{
  Use: "cannon",
@@ -32,7 +110,15 @@ var cannonCmd = &cobra.Command{
 
  log.WithField("location", cannonCfgFile).Info("Loaded config")
 
- cannon, err := cannon.New(cmd.Context(), log, config)
+ overrides := &cannon.Override{}
+
+ for _, override := range CannonOverrides {
+ if errr := override.Setter(cmd, overrides); errr != nil {
+ log.Fatal(errr)
+ }
+ }
+
+ cannon, err := cannon.New(cmd.Context(), log, config, overrides)
  if err != nil {
  log.Fatal(err)
  }
@@ -49,6 +135,10 @@ func init() {
  rootCmd.AddCommand(cannonCmd)
 
  cannonCmd.Flags().StringVar(&cannonCfgFile, "config", "cannon.yaml", "config file (default is cannon.yaml)")
+
+ for _, override := range CannonOverrides {
+ override.FlagHelper(cannonCmd)
+ }
 }
 
 func loadcannonConfigFromFile(file string) (*cannon.Config, error) {

cmd/server.go

@@ -13,12 +13,78 @@ var (
  serverCfgFile string
 )
 
+type ServerOverride struct {
+ FlagHelper func(cmd *cobra.Command)
+ Setter func(cmd *cobra.Command, overrides *server.Override) error
+}
+
+type ServerOverrideConfig struct {
+ FlagName string
+ EnvName string
+ Description string
+ OverrideFunc func(val string, overrides *server.Override)
+}
+
+func createServerOverride(config ServerOverrideConfig) ServerOverride {
+ return ServerOverride{
+ FlagHelper: func(cmd *cobra.Command) {
+ cmd.Flags().String(config.FlagName, "", config.Description+` (env: `+config.EnvName+`)`)
+ },
+ Setter: func(cmd *cobra.Command, overrides *server.Override) error {
+ val := ""
+
+ if cmd.Flags().Changed(config.FlagName) {
+ val = cmd.Flags().Lookup(config.FlagName).Value.String()
+ }
+
+ if os.Getenv(config.EnvName) != "" {
+ val = os.Getenv(config.EnvName)
+ }
+
+ if val == "" {
+ return nil
+ }
+
+ config.OverrideFunc(val, overrides)
+
+ return nil
+ },
+ }
+}
+
+var ServerOverrides = []ServerOverride{
+ createServerOverride(ServerOverrideConfig{
+ FlagName: "server-event-ingester-basic-auth-username",
+ EnvName: "SERVER_EVENT_INGESTER_BASIC_AUTH_USERNAME",
+ Description: "sets the basic auth username for the event ingester",
+ OverrideFunc: func(val string, overrides *server.Override) {
+ overrides.EventIngesterBasicAuth.Username = val
+ },
+ }),
+ createServerOverride(ServerOverrideConfig{
+ FlagName: "server-event-ingester-basic-auth-password",
+ EnvName: "SERVER_EVENT_INGESTER_BASIC_AUTH_PASSWORD",
+ Description: "sets the basic auth password for the event ingester",
+ OverrideFunc: func(val string, overrides *server.Override) {
+ overrides.EventIngesterBasicAuth.Password = val
+ },
+ }),
+ createServerOverride(ServerOverrideConfig{
+ FlagName: "server-coordinator-auth-secret",
+ EnvName: "SERVER_COORDINATOR_AUTH_SECRET",
+ Description: "sets the auth secret for the coordinator",
+ OverrideFunc: func(val string, overrides *server.Override) {
+ overrides.CoordinatorAuth.AuthSecret = val
+ },
+ }),
+}
+
 // serverCmd represents the server command
 var serverCmd = &cobra.Command{
  Use: "server",
  Short: "Runs Xatu in Server mode.",
  Long: `Runs Xatu in Server mode, which means it will listen to gRPC requests from
- Xatu Sentry nodes and forward the data on to the configured sinks.`,
+ Xatu modules and forward the data on to the configured sinks.`,
  Run: func(cmd *cobra.Command, args []string) {
  initCommon()
 
@@ -31,7 +97,15 @@ var serverCmd = &cobra.Command{
 
  log.WithField("location", serverCfgFile).Info("Loaded config")
 
- server, err := server.NewXatu(cmd.Context(), log, config)
+ o := &server.Override{}
+
+ for _, override := range ServerOverrides {
+ if errr := override.Setter(cmd, o); errr != nil {
+ log.Fatal(errr)
+ }
+ }
+
+ server, err := server.NewXatu(cmd.Context(), log, config, o)
  if err != nil {
  log.Fatal(err)
  }
@@ -48,6 +122,10 @@ func init() {
  rootCmd.AddCommand(serverCmd)
 
  serverCmd.Flags().StringVar(&serverCfgFile, "config", "server.yaml", "config file (default is server.yaml)")
+
+ for _, override := range ServerOverrides {
+ override.FlagHelper(serverCmd)
+ }
 }
 
 func loadServerConfigFromFile(file string) (*server.Config, error) {

deploy/local/docker-compose/xatu-cannon.yaml

@@ -0,0 +1,80 @@
+logging: "info" # panic,fatal,warn,info,debug,trace
+metricsAddr: ":9097"
+pprofAddr: ":6061" # optional. if supplied it enables pprof server
+
+name: xatu-cannon
+
+# derivers:
+# attesterSlashing:
+# enabled: false
+# blsToExecutionChange:
+# enabled: false
+# deposit:
+# enabled: false
+# withdrawal:
+# enabled: false
+# executionTransaction:
+# enabled: false
+# proposerSlashing:
+# enabled: false
+# voluntaryExit:
+# enabled: false
+# beaconBlock:
+# enabled: true
+# beaconBlobSidecar:
+# enabled: false
+# proposerDuty:
+# enabled: true
+# elaboratedAttestation:
+# enabled: false
+# beaconValidators:
+# enabled: false
+# elaboratedAttestation:
+# enabled: false
+# beaconCommittee:
+# enabled: false
+
+ # blockClassification:
+ # enabled: false
+
+
+
+# Better to use a NTP server close eg.
+# time.aws.com - AWS
+# time.windows.com - Azure
+# time.google.com - GCP
+# pool.ntp.org - https://www.pool.ntp.org/zone/@
+ntpServer: time.google.com
+
+tracing:
+ enabled: true
+ endpoint: tempo:4318
+ insecure: true
+ sampling:
+ rate: 0.1
+
+ethereum:
+ beaconNodeAddress: http://SET_ME:5052
+ # overrideNetworkName: mainnet
+ # blockCacheSize: 10000
+ # blockCacheTtl: 3h
+ # blockPreloadWorkers: 25
+
+coordinator:
+ address: xatu-server:8080
+ headers:
+ Authorization: "Bearer SET_ME"
+
+outputs:
+ - name: xatu
+ type: xatu
+ config:
+ address: xatu-server:8080
+ tls: false
+ maxQueueSize: 51200
+ batchTimeout: 0.5s
+ exportTimeout: 30s
+ maxExportBatchSize: 32
+ workers: 50
+ headers:
+ Authorization: "Bearer SET_ME"

deploy/local/docker-compose/xatu-server.yaml

@@ -38,7 +38,7 @@ services:
  enabled: true # requires persistence to be enabled
  auth:
  enabled: false
- secret: "super-secret-token"
+ # secret: "super-secret-token"
  nodeRecord:
  maxQueueSize: 51200
  batchTimeout: 5s
@@ -48,27 +48,27 @@ services:
  enabled: true
  clientNameSalt: "example_salt"
  authorization:
- enabled: false
- groups:
- default:
- obscureClientNames: true
- users:
- user_a:
- password: password
- shane:
- password: warne
- restricted:
- eventFilter:
- eventNames:
- - BEACON_API_ETH_V2_BEACON_BLOCK_V2
- - BEACON_API_ETH_V1_EVENTS_BLOB_SIDECAR
- - BEACON_API_ETH_V1_EVENTS_BLOCK_V2
- - BEACON_API_ETH_V1_EVENTS_CHAIN_REORG_V2
- - BEACON_API_ETH_V1_EVENTS_FINALIZED_CHECKPOINT_V2
- - BEACON_API_ETH_V1_EVENTS_HEAD_V2
- users:
- user_b:
- password: password
+ # enabled: false
+ # groups:
+ #  default:
+ #  obscureClientNames: true
+ #  users:
+ #  user_a:
+ #  password: password
+ #  shane:
+ #  password: warne
+ #  restricted:
+ #  eventFilter:
+ #  eventNames:
+ #  - BEACON_API_ETH_V2_BEACON_BLOCK_V2
+ #  - BEACON_API_ETH_V1_EVENTS_BLOB_SIDECAR
+ #  - BEACON_API_ETH_V1_EVENTS_BLOCK_V2
+ #  - BEACON_API_ETH_V1_EVENTS_CHAIN_REORG_V2
+ #  - BEACON_API_ETH_V1_EVENTS_FINALIZED_CHECKPOINT_V2
+ #  - BEACON_API_ETH_V1_EVENTS_HEAD_V2
+ #  users:
+ #  user_b:
+ #  password: password
  outputs:
  - name: general
  type: http