feat(kic 2.0) add profiling

CHANGELOG.md

@@ -37,6 +37,8 @@ released and the release notes may change significantly before then.
 
 #### Added
 
+- profiling using `pprof` is now a standalone HTTP server listening on port 10256.
+ [#1417](https:/Kong/kubernetes-ingress-controller/pull/1417)
 - support for [UDP][kong-udp] via the new `UDPIngress` API.
 - `--watch-namespace` now supports multiple distinct namespaces versus simply
  supporting all namespaces (the default) or a single namespace. To watch

railgun/cmd/rootcmd/run.go

@@ -12,5 +12,8 @@ func Run(ctx context.Context, c *config.Config) error {
  if err := StartAdmissionServer(ctx, c); err != nil {
  return fmt.Errorf("StartAdmissionServer: %w", err)
  }
+ if err := StartProfilingServer(ctx, c); err != nil {
+ return fmt.Errorf("StartProfilingServer: %w", err)
+ }
  return manager.Run(ctx, c)
 }

railgun/cmd/rootcmd/admission.go → railgun/cmd/rootcmd/servers.go

@@ -3,8 +3,10 @@ package rootcmd
 import (
  "context"
 
+ "github.com/bombsimon/logrusr"
  "github.com/kong/kubernetes-ingress-controller/pkg/admission"
  "github.com/kong/kubernetes-ingress-controller/pkg/util"
+ "github.com/kong/kubernetes-ingress-controller/railgun/internal/diagnostics"
  "github.com/kong/kubernetes-ingress-controller/railgun/pkg/config"
 )
 
@@ -42,3 +44,24 @@ func StartAdmissionServer(ctx context.Context, c *config.Config) error {
  }()
  return nil
 }
+
+func StartProfilingServer(ctx context.Context, c *config.Config) error {
+ deprecatedLogger, err := util.MakeLogger(c.LogLevel, c.LogFormat)
+ if err != nil {
+ return err
+ }
+ logger := logrusr.NewLogger(deprecatedLogger)
+
+ if !c.EnableProfiling {
+ logger.Info("profiling server disabled")
+ return nil
+ }
+
+ s := diagnostics.Server{Logger: logger}
+ go func() {
+ if err := s.Listen(ctx); err != nil {
+ logger.Error(err, "unable to start diagnostics server")
+ }
+ }()
+ return nil
+}

railgun/internal/diagnostics/server.go

@@ -0,0 +1,69 @@
+package diagnostics
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/http/pprof"
+
+ "github.com/go-logr/logr"
+ "github.com/kong/kubernetes-ingress-controller/railgun/pkg/config"
+)
+
+// Server is an HTTP server running exposing the pprof profiling tool.
+type Server struct {
+ Logger logr.Logger
+}
+
+// Listen starts up the HTTP server and blocks until ctx expires.
+func (s *Server) Listen(ctx context.Context) error {
+ mux := http.NewServeMux()
+ installHandlers(mux)
+
+ httpServer := &http.Server{Addr: fmt.Sprintf(":%d", config.DiagnosticsPort), Handler: mux}
+ errChan := make(chan error)
+ go func() {
+ err := httpServer.ListenAndServe()
+ if err != nil {
+ switch err {
+ case http.ErrServerClosed:
+ s.Logger.Info("shutting down diagnostics server")
+ default:
+ s.Logger.Error(err, "could not start diagnostics server")
+ errChan <- err
+ }
+ }
+ }()
+
+ s.Logger.Info("diagnostics server is starting to listen", "addr", config.DiagnosticsPort)
+
+ select {
+ case <-ctx.Done():
+ s.Logger.Info("shutting down diagnostics server")
+ return httpServer.Shutdown(context.Background())
+ case err := <-errChan:
+ return err
+ }
+}
+
+// installHandlers adds the Profiling webservice to the given mux.
+func installHandlers(mux *http.ServeMux) {
+ mux.HandleFunc("/debug/pprof", redirectTo("/debug/pprof/"))
+ mux.HandleFunc("/debug/pprof/", pprof.Index)
+ mux.HandleFunc("/debug/pprof/heap", pprof.Index)
+ mux.HandleFunc("/debug/pprof/mutex", pprof.Index)
+ mux.HandleFunc("/debug/pprof/goroutine", pprof.Index)
+ mux.HandleFunc("/debug/pprof/threadcreate", pprof.Index)
+ mux.HandleFunc("/debug/pprof/block", pprof.Index)
+ mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+ mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+ mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+ mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+}
+
+// redirectTo redirects request to a certain destination.
+func redirectTo(to string) func(http.ResponseWriter, *http.Request) {
+ return func(rw http.ResponseWriter, req *http.Request) {
+ http.Redirect(rw, req, to, http.StatusFound)
+ }
+}

railgun/manager/run.go

@@ -267,7 +267,6 @@ func Run(ctx context.Context, c *config.Config) error {
  } else {
  setupLog.Info("anonymous reports disabled, skipping")
  }
-
  setupLog.Info("starting manager")
  return mgr.Start(ctx)
 }

railgun/pkg/config/config.go

@@ -68,6 +68,9 @@ type Config struct {
 
  // Admission Webhook server config
  AdmissionServer admission.ServerConfig
+
+ // Performance monitoring
+ EnableProfiling bool
 }
 
 // -----------------------------------------------------------------------------
@@ -157,6 +160,9 @@ func (c *Config) FlagSet() *pflag.FlagSet {
  flagSet.StringVar(&c.AdmissionServer.Key, "admission-webhook-key", "",
  `admission server PEM private key value`)
 
+ // Misc
+ flagSet.BoolVar(&c.EnableProfiling, "profiling", false, "Enable profiling via web interface host:10256/debug/pprof/")
+
  return &flagSet.FlagSet
 }
 

railgun/pkg/config/consts.go

@@ -16,3 +16,6 @@ const HealthzPort = 10254
 // Similar to HealthzPort, it may be used in existing user deployment configurations, and its
 // literal value is used in several stock manifests, which must be updated along with this value.
 const MetricsPort = 10255
+
+// DiagnosticsPort is the default port of the manager's diagnostics service listens on.
+const DiagnosticsPort = 10256

railgun/test/integration/controller_test.go

@@ -57,3 +57,20 @@ func TestMetricsEndpoint(t *testing.T) {
  return len(v) > 0
  }, ingressWait, waitTick)
 }
+
+func TestProfilingEndpoint(t *testing.T) {
+ if useLegacyKIC() {
+ t.Skip("profiling endpoint behaves differently in legacy KIC")
+ }
+ _ = proxyReady()
+ assert.Eventually(t, func() bool {
+ profilingURL := fmt.Sprintf("http://localhost:%v/debug/pprof/", config.DiagnosticsPort)
+ resp, err := httpc.Get(profilingURL)
+ if err != nil {
+ t.Logf("WARNING: error while waiting for %s: %v", profilingURL, err)
+ return false
+ }
+ defer resp.Body.Close()
+ return resp.StatusCode == http.StatusOK
+ }, ingressWait, waitTick)
+}

railgun/test/integration/suite_test.go

@@ -246,6 +246,7 @@ func deployControllers(ctx context.Context, ready chan ktfkind.ProxyReadinessEve
  "--admission-webhook-listen=172.17.0.1:49023",
  fmt.Sprintf("--admission-webhook-cert=%s", admissionWebhookCert),
  fmt.Sprintf("--admission-webhook-key=%s", admissionWebhookKey),
+ "--profiling",
  })
  fmt.Fprintf(os.Stderr, "config: %+v\n", config)