diff --git a/cmd/varlogadm/cli.go b/cmd/varlogadm/cli.go index c9196cfbd..ed073fa4e 100644 --- a/cmd/varlogadm/cli.go +++ b/cmd/varlogadm/cli.go @@ -58,12 +58,16 @@ func newStartCommand() *cli.Command { flagSNWatcherReportDeadline.DurationFlag(false, snwatcher.DefaultReportDeadline), flags.GRPCServerReadBufferSize, + flags.GRPCServerRecvBufferPool, flags.GRPCServerWriteBufferSize, + flags.GRPCServerSharedWriteBuffer, flags.GRPCServerMaxRecvMsgSize, flags.GRPCServerInitialConnWindowSize, flags.GRPCServerInitialWindowSize, flags.GRPCClientReadBufferSize, + flags.GRPCClientRecvBufferPool, flags.GRPCClientWriteBufferSize, + flags.GRPCClientSharedWriteBuffer, flags.GRPCClientInitialConnWindowSize, flags.GRPCClientInitialWindowSize, diff --git a/cmd/varlogadm/testdata/varlogadm.ct b/cmd/varlogadm/testdata/varlogadm.ct index 6f1596abf..c8e410b24 100644 --- a/cmd/varlogadm/testdata/varlogadm.ct +++ b/cmd/varlogadm/testdata/varlogadm.ct @@ -68,11 +68,15 @@ OPTIONS: --grpc-client-initial-conn-window-size value Set the gRPC client's initial window size for a connection. If not set, the default value of 64KiB defined by gRPC will be used. [$GRPC_CLIENT_INITIAL_CONN_WINDOW_SIZE] --grpc-client-initial-window-size value Set the gRPC client's initial window size for a stream. If not set, the default value of 64KiB defined by gRPC will be used. [$GRPC_CLIENT_INITIAL_WINDOW_SIZE] --grpc-client-read-buffer-size value Set the gRPC client's read buffer size for a single read syscall. If not set, the default value of 32KiB defined by gRPC will be used. [$GRPC_CLIENT_READ_BUFFER_SIZE] + --grpc-client-recv-buffer-pool Use the gRPC client's shared buffer pool for parsing incoming messages. If not set, the buffer pool will not be used. (default: false) [$GRPC_CLIENT_RECV_BUFFER_POOL] + --grpc-client-shared-write-buffer Enable sharing gRPC client's transport write buffer across connections. If not set, each connection will allocate its own write buffer. (default: false) [$GRPC_CLIENT_SHARED_WRITE_BUFFER] --grpc-client-write-buffer-size value Set the gRPC client's write buffer size for a single write syscall. If not set, the default value of 32KiB defined by gRPC will be used. [$GRPC_CLIENT_WRITE_BUFFER_SIZE] --grpc-server-initial-conn-window-size value Set the gRPC server's initial window size for a connection. If not set, the default value of 64KiB defined by gRPC will be used. [$GRPC_SERVER_INITIAL_CONN_WINDOW_SIZE] --grpc-server-initial-window-size value Set the gRPC server's initial window size for a stream. If not set, the default value of 64KiB defined by gRPC will be used. [$GRPC_SERVER_INITIAL_WINDOW_SIZE] --grpc-server-max-recv-msg-size value Set the maximum message size in bytes that the gRPC server can receive. If not set, the default value of 4MiB defined by gRPC will be used. [$GRPC_SERVER_MAX_RECV_MSG_SIZE] --grpc-server-read-buffer-size value Set the gRPC server's read buffer size for a single read syscall. If not set, the default value of 32KiB defined by gRPC will be used. [$GRPC_SERVER_READ_BUFFER_SIZE] + --grpc-server-recv-buffer-pool Use the gRPC server's shared buffer pool for parsing incoming messages. If not set, the buffer pool will not be used. (default: false) [$GRPC_SERVER_RECV_BUFFER_POOL] + --grpc-server-shared-write-buffer Enable sharing gRPC server's transport write buffer across connections. If not set, each connection will allocate its own write buffer. (default: false) [$GRPC_SERVER_SHARED_WRITE_BUFFER] --grpc-server-write-buffer-size value Set the gRPC server's write buffer size for a single write syscall. If not set, the default value of 32KiB defined by gRPC will be used. [$GRPC_SERVER_WRITE_BUFFER_SIZE] varlogadm: Required flag "metadata-repository-address" not set diff --git a/cmd/varlogmr/metadata_repository.go b/cmd/varlogmr/metadata_repository.go index db96e71bf..cb456cecc 100644 --- a/cmd/varlogmr/metadata_repository.go +++ b/cmd/varlogmr/metadata_repository.go @@ -137,12 +137,16 @@ func initCLI() *cli.App { flagRaftDir.StringFlag(false, metarepos.DefaultRaftDir), flagPeers.StringSliceFlag(false, nil), flags.GRPCServerReadBufferSize, + flags.GRPCServerRecvBufferPool, flags.GRPCServerWriteBufferSize, + flags.GRPCServerSharedWriteBuffer, flags.GRPCServerMaxRecvMsgSize, flags.GRPCServerInitialConnWindowSize, flags.GRPCServerInitialWindowSize, flags.GRPCClientReadBufferSize, + flags.GRPCClientRecvBufferPool, flags.GRPCClientWriteBufferSize, + flags.GRPCClientSharedWriteBuffer, flags.GRPCClientInitialConnWindowSize, flags.GRPCClientInitialWindowSize, flagMaxTopicsCount, diff --git a/cmd/varlogsn/cli.go b/cmd/varlogsn/cli.go index 1122b5dfa..ce611dd31 100644 --- a/cmd/varlogsn/cli.go +++ b/cmd/varlogsn/cli.go @@ -47,12 +47,16 @@ func newStartCommand() *cli.Command { flagVolumes.StringSliceFlag(true, nil), flags.GRPCServerReadBufferSize, + flags.GRPCServerRecvBufferPool, flags.GRPCServerWriteBufferSize, + flags.GRPCServerSharedWriteBuffer, flags.GRPCServerMaxRecvMsgSize, flags.GRPCServerInitialConnWindowSize, flags.GRPCServerInitialWindowSize, flags.GRPCClientReadBufferSize, + flags.GRPCClientRecvBufferPool, flags.GRPCClientWriteBufferSize, + flags.GRPCClientSharedWriteBuffer, flags.GRPCClientInitialConnWindowSize, flags.GRPCClientInitialWindowSize, diff --git a/cmd/varlogsn/testdata/varlogsn.ct b/cmd/varlogsn/testdata/varlogsn.ct index 8c1d2231e..1aac5bc71 100644 --- a/cmd/varlogsn/testdata/varlogsn.ct +++ b/cmd/varlogsn/testdata/varlogsn.ct @@ -87,11 +87,15 @@ OPTIONS: --grpc-client-initial-conn-window-size value Set the gRPC client's initial window size for a connection. If not set, the default value of 64KiB defined by gRPC will be used. [$GRPC_CLIENT_INITIAL_CONN_WINDOW_SIZE] --grpc-client-initial-window-size value Set the gRPC client's initial window size for a stream. If not set, the default value of 64KiB defined by gRPC will be used. [$GRPC_CLIENT_INITIAL_WINDOW_SIZE] --grpc-client-read-buffer-size value Set the gRPC client's read buffer size for a single read syscall. If not set, the default value of 32KiB defined by gRPC will be used. [$GRPC_CLIENT_READ_BUFFER_SIZE] + --grpc-client-recv-buffer-pool Use the gRPC client's shared buffer pool for parsing incoming messages. If not set, the buffer pool will not be used. (default: false) [$GRPC_CLIENT_RECV_BUFFER_POOL] + --grpc-client-shared-write-buffer Enable sharing gRPC client's transport write buffer across connections. If not set, each connection will allocate its own write buffer. (default: false) [$GRPC_CLIENT_SHARED_WRITE_BUFFER] --grpc-client-write-buffer-size value Set the gRPC client's write buffer size for a single write syscall. If not set, the default value of 32KiB defined by gRPC will be used. [$GRPC_CLIENT_WRITE_BUFFER_SIZE] --grpc-server-initial-conn-window-size value Set the gRPC server's initial window size for a connection. If not set, the default value of 64KiB defined by gRPC will be used. [$GRPC_SERVER_INITIAL_CONN_WINDOW_SIZE] --grpc-server-initial-window-size value Set the gRPC server's initial window size for a stream. If not set, the default value of 64KiB defined by gRPC will be used. [$GRPC_SERVER_INITIAL_WINDOW_SIZE] --grpc-server-max-recv-msg-size value Set the maximum message size in bytes that the gRPC server can receive. If not set, the default value of 4MiB defined by gRPC will be used. [$GRPC_SERVER_MAX_RECV_MSG_SIZE] --grpc-server-read-buffer-size value Set the gRPC server's read buffer size for a single read syscall. If not set, the default value of 32KiB defined by gRPC will be used. [$GRPC_SERVER_READ_BUFFER_SIZE] + --grpc-server-recv-buffer-pool Use the gRPC server's shared buffer pool for parsing incoming messages. If not set, the buffer pool will not be used. (default: false) [$GRPC_SERVER_RECV_BUFFER_POOL] + --grpc-server-shared-write-buffer Enable sharing gRPC server's transport write buffer across connections. If not set, each connection will allocate its own write buffer. (default: false) [$GRPC_SERVER_SHARED_WRITE_BUFFER] --grpc-server-write-buffer-size value Set the gRPC server's write buffer size for a single write syscall. If not set, the default value of 32KiB defined by gRPC will be used. [$GRPC_SERVER_WRITE_BUFFER_SIZE] varlogsn: Required flag "volumes" not set diff --git a/internal/flags/grpc.go b/internal/flags/grpc.go index 22e81ecf5..4eef615b8 100644 --- a/internal/flags/grpc.go +++ b/internal/flags/grpc.go @@ -6,6 +6,7 @@ import ( "github.com/urfave/cli/v2" "google.golang.org/grpc" + "google.golang.org/grpc/experimental" "github.com/kakao/varlog/pkg/util/units" ) @@ -32,6 +33,16 @@ var ( return nil }, } + // GRPCServerRecvBufferPool is a flag to use the gRPC server's shared buffer pool for parsing incoming messages. + // + // See: + // - https://pkg.go.dev/google.golang.org/grpc@v1.64.0/experimental#RecvBufferPool + GRPCServerRecvBufferPool = &cli.BoolFlag{ + Name: "grpc-server-recv-buffer-pool", + Category: CategoryGRPC, + EnvVars: []string{"GRPC_SERVER_RECV_BUFFER_POOL"}, + Usage: "Use the gRPC server's shared buffer pool for parsing incoming messages. If not set, the buffer pool will not be used.", + } // GRPCServerWriteBufferSize is a flag to set the gRPC server's write // buffer size for a single write syscall. // @@ -49,6 +60,17 @@ var ( return nil }, } + // GRPCServerSharedWriteBuffer is a flag to enable sharing gRPC server's transport write buffer across connections. + // + // See: + // - https://pkg.go.dev/google.golang.org/grpc#WithSharedWriteBuffer + // - https://github.com/grpc/grpc-go/pull/6309 + GRPCServerSharedWriteBuffer = &cli.BoolFlag{ + Name: "grpc-server-shared-write-buffer", + Category: CategoryGRPC, + EnvVars: []string{"GRPC_SERVER_SHARED_WRITE_BUFFER"}, + Usage: "Enable sharing gRPC server's transport write buffer across connections. If not set, each connection will allocate its own write buffer.", + } // GRPCServerMaxRecvMsgSize is a flag to set the maximum message size the server can receive. // // See: @@ -114,6 +136,16 @@ var ( return nil }, } + // GRPCClientRecvBufferPool is a flag to use the gRPC client's shared buffer pool for parsing incoming messages. + // + // See: + // - https://pkg.go.dev/google.golang.org/grpc/experimental#WithRecvBufferPool + GRPCClientRecvBufferPool = &cli.BoolFlag{ + Name: "grpc-client-recv-buffer-pool", + Category: CategoryGRPC, + EnvVars: []string{"GRPC_CLIENT_RECV_BUFFER_POOL"}, + Usage: "Use the gRPC client's shared buffer pool for parsing incoming messages. If not set, the buffer pool will not be used.", + } // GRPCClientWriteBufferSize is a flag to set the gRPC client's write // buffer size for a single write syscall. // @@ -131,6 +163,17 @@ var ( return nil }, } + // GRPCClientSharedWriteBuffer is a flag to enable sharing gRPC client's transport write buffer across connections. + // + // See: + // - https://pkg.go.dev/google.golang.org/grpc#WithSharedWriteBuffer + // - https://github.com/grpc/grpc-go/pull/6309 + GRPCClientSharedWriteBuffer = &cli.BoolFlag{ + Name: "grpc-client-shared-write-buffer", + Category: CategoryGRPC, + EnvVars: []string{"GRPC_CLIENT_SHARED_WRITE_BUFFER"}, + Usage: "Enable sharing gRPC client's transport write buffer across connections. If not set, each connection will allocate its own write buffer.", + } // GRPCClientInitialConnWindowSize is a flag to set the gRPC client's initial window size for a connection. // // See: @@ -173,6 +216,9 @@ func ParseGRPCServerOptionFlags(c *cli.Context) (opts []grpc.ServerOption, _ err } opts = append(opts, grpc.ReadBufferSize(int(readBufferSize))) } + if c.IsSet(GRPCServerRecvBufferPool.Name) { + opts = append(opts, experimental.RecvBufferPool(grpc.NewSharedBufferPool())) + } if c.IsSet(GRPCServerWriteBufferSize.Name) { writeBufferSize, err := units.FromByteSizeString(c.String(GRPCServerWriteBufferSize.Name)) if err != nil { @@ -180,6 +226,7 @@ func ParseGRPCServerOptionFlags(c *cli.Context) (opts []grpc.ServerOption, _ err } opts = append(opts, grpc.WriteBufferSize(int(writeBufferSize))) } + opts = append(opts, grpc.SharedWriteBuffer(c.Bool(GRPCServerSharedWriteBuffer.Name))) if c.IsSet(GRPCServerMaxRecvMsgSize.Name) { maxRecvMsgSize, err := units.FromByteSizeString(c.String(GRPCServerMaxRecvMsgSize.Name)) if err != nil { @@ -212,6 +259,9 @@ func ParseGRPCDialOptionFlags(c *cli.Context) (opts []grpc.DialOption, err error } opts = append(opts, grpc.WithReadBufferSize(int(readBufferSize))) } + if c.IsSet(GRPCClientRecvBufferPool.Name) { + opts = append(opts, experimental.WithRecvBufferPool(grpc.NewSharedBufferPool())) + } if c.IsSet(GRPCClientWriteBufferSize.Name) { writeBufferSize, err := units.FromByteSizeString(c.String(GRPCClientWriteBufferSize.Name)) if err != nil { @@ -219,6 +269,7 @@ func ParseGRPCDialOptionFlags(c *cli.Context) (opts []grpc.DialOption, err error } opts = append(opts, grpc.WithWriteBufferSize(int(writeBufferSize))) } + opts = append(opts, grpc.WithSharedWriteBuffer(c.Bool(GRPCClientSharedWriteBuffer.Name))) if c.IsSet(GRPCClientInitialConnWindowSize.Name) { initialConnWindowSize, err := units.FromByteSizeString(c.String(GRPCClientInitialConnWindowSize.Name), 0, math.MaxInt32) if err != nil {