diff --git a/CHANGELOG.md b/CHANGELOG.md index 315545b6b5..3ca5a64a38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,17 @@ ## [2.0.5] - TBD +#### Added + +- [Feature Gates][k8s-fg] have been added to the controller manager in order to + enable alpha/beta/experimental features and provide documentation about those + features and their maturity over time. For more information see the + [KIC Feature Gates Documentation][kic-fg]. + [#1970](https://github.com/Kong/kubernetes-ingress-controller/pull/1970) + +[k8s-fg]:https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/ +[kic-fg]:https://github.com/Kong/kubernetes-ingress-controller/blob/main/FEATURE_GATES.md + #### Fixed - Fixed a bug where version reported for the controller manager was missing diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f68c809b1b..2c7e0b8f7c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -86,6 +86,12 @@ In general the maintainers here feel establishing these things in a KEP should b [template]:https://github.com/kubernetes/enhancements/blob/master/keps/NNNN-kep-template/README.md +## Feature Gates + +New features should be added to the [Feature Gates][kic-fg] documentation and `internal/manager/feature_gates.go`. + +[kic-fg]:https://github.com/Kong/kubernetes-ingress-controller/blob/main/FEATURE_GATES.md + ## Development environment ## Environment diff --git a/FEATURE_GATES.md b/FEATURE_GATES.md new file mode 100644 index 0000000000..68a17b9c60 --- /dev/null +++ b/FEATURE_GATES.md @@ -0,0 +1,61 @@ +# Feature Gates + +Upstream [Kubernetes][k8s] includes [Feature Gates][gates] to enable or disable features with flags and track the maturity of a feature using [Feature Stages][stages]. Here in the Kubernetes Ingress Controller (KIC) we use the same definitions of `Feature Gates` and `Feature Stages` from upstream Kubernetes, but with our own list of features. + +Using `Feature Gates` enables contributors to add and manage new (and potentially) experimental functionality to the KIC in a controlled manner: the features will be "hidden" until generally available (GA) and the progress and maturity of features on their path to GA will be documented. Feature gates also create a clear path for deprecating features. + +See below for current features and their statuses, and follow the links to the relevant feature documentation. + +[k8s]:https://kubernetes.io +[gates]:https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/ +[stages]:https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-stages + +## Feature gates + +Below you will find the overviews of features at various maturity levels: + +- [Feature gates for graduated or deprecated features](/#feature-gates-for-graduated-or-deprecated-features) +- [Feature gates for Alpha or Beta features](/#feature-gates-for-alpha-or-beta-features) + +Please read the [Important Notes](/#important-notes) section before using any `Alpha` or `Beta` features. + +### Important notes + +- Most features will be planned and detailed using [Kubernetes Enhancement Proposals (KEP)][k8s-kep]: If you're interested in the development side of features familiarize yourself with our [KEPs][kic-keps] +- The `Since` and `Until` rows in below tables refer to [KIC Releases][releases] +- For `GA` features the documentation exists in the main [Kong Documentation][kong-docs], see the [API reference][api-ref] and [Guides][kic-guides] + +An additional **warning** for end-users who are reading this documentation and trying to enable `Alpha` or `Beta` features: it is **very important** to understand that features that are currently in an `Alpha` or `Beta` state may **become `Deprecated` at any time** and **may be removed as part of the next consecutive minor release**. This is especially true for `Alpha` maturity features. In other words, **until a feature becomes GA there are no guarantees that it's going to continue being available**. To avoid disruption to your services engage with the community and read the [CHANGELOG](/CHANGELOG.md) carefully to track progress. Alternatively **do not use features until they have reached a GA status**. + +[k8s-keps]:https://github.com/kubernetes/enhancements +[kic-keps]:https://github.com/Kong/kubernetes-ingress-controller/tree/main/keps +[releases]:https://github.com/Kong/kubernetes-ingress-controller/releases +[kong-docs]:https://github.com/Kong/docs.konghq.com +[api-ref]:https://docs.konghq.com/kubernetes-ingress-controller/latest/references/custom-resources/ +[kic-guides]:https://docs.konghq.com/kubernetes-ingress-controller/latest/guides/overview/ + +### Feature gates for graduated or deprecated features + +{{< table caption="Feature Gates for Graduated or Deprecated Features" >}} + +| Feature | Default | Stage | Since | Until | +|----------------------------|---------|------------|-------|-------| + +{{< /table >}} + +Features that reach GA and over time become stable will be removed from this table, they can be found in the main [KIC CRD Documentation][specs] and [Guides][guides]. + +[specs]:https://docs.konghq.com/kubernetes-ingress-controller/latest/references/custom-resources/ +[guides]:https://docs.konghq.com/kubernetes-ingress-controller/latest/guides/overview/ + +### Feature gates for Alpha or Beta features + +{{< table caption="Feature gates for features in Alpha or Beta states" >}} + +| Feature | Default | Stage | Since | Until | +|----------------------------|---------|-------|-------|-------| +| [Knative](#Knative) | `true` | Alpha | 0.8.0 | TBD | +| [GatewayAPI](#Gateway-API) | `false` | Alpha | TBD | TBD | + +{{< /table > }} + diff --git a/README.md b/README.md index 6eda3db47c..36379041e4 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/Kong/kong/blob/master/LICENSE) [![Twitter](https://img.shields.io/twitter/follow/thekonginc.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=thekonginc) -# Kong Ingress Controller for Kubernetes +# Kong Ingress Controller for Kubernetes (KIC) Use [Kong][kong] for Kubernetes [Ingress][ingress]. Configure [plugins][kong-hub], health checking, diff --git a/go.mod b/go.mod index b906cd4aa2..21f1613470 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( k8s.io/apiextensions-apiserver v0.22.3 k8s.io/apimachinery v0.22.3 k8s.io/client-go v0.22.3 + k8s.io/component-base v0.22.3 knative.dev/networking v0.0.0-20210914225408-69ad45454096 knative.dev/pkg v0.0.0-20210919202233-5ae482141474 knative.dev/serving v0.26.0 @@ -123,7 +124,6 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect - k8s.io/component-base v0.22.3 // indirect k8s.io/klog/v2 v2.9.0 // indirect k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e // indirect k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a // indirect diff --git a/internal/manager/config.go b/internal/manager/config.go index 3578a04836..8246bba337 100644 --- a/internal/manager/config.go +++ b/internal/manager/config.go @@ -9,6 +9,7 @@ import ( "github.com/spf13/pflag" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + cliflag "k8s.io/component-base/cli/flag" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/kong/kubernetes-ingress-controller/v2/internal/adminapi" @@ -81,6 +82,9 @@ type Config struct { EnableProfiling bool EnableConfigDumps bool DumpSensitiveConfig bool + + // Feature Gates + FeatureGates map[string]bool } // ----------------------------------------------------------------------------- @@ -175,6 +179,10 @@ func (c *Config) FlagSet() *pflag.FlagSet { flagSet.BoolVar(&c.EnableConfigDumps, "dump-config", false, fmt.Sprintf("Enable config dumps via web interface host:%v/debug/config", DiagnosticsPort)) flagSet.BoolVar(&c.DumpSensitiveConfig, "dump-sensitive-config", false, "Include credentials and TLS secrets in configs exposed with --dump-config") + // Feature Gates (see FEATURE_GATES.md) + flagSet.Var(cliflag.NewMapStringBool(&c.FeatureGates), "feature-gates", "A set of key=value pairs that describe feature gates for alpha/beta/experimental features. "+ + fmt.Sprintf("See the Feature Gates documentation for information and available options: %s", featureGatesDocsURL)) + // Deprecated (to be removed in future releases) flagSet.Float32Var(&c.ProxySyncSeconds, "sync-rate-limit", proxy.DefaultSyncSeconds, "Define the rate (in seconds) in which configuration updates will be applied to the Kong Admin API (DEPRECATED, use --proxy-sync-seconds instead)", diff --git a/internal/manager/controllerdef.go b/internal/manager/controllerdef.go index 05e967e985..fad20d3f12 100644 --- a/internal/manager/controllerdef.go +++ b/internal/manager/controllerdef.go @@ -59,7 +59,7 @@ func (c *ControllerDef) MaybeSetupWithManager(mgr ctrl.Manager) error { // Controller Manager - Controller Setup Functions // ----------------------------------------------------------------------------- -func setupControllers(mgr manager.Manager, proxy proxy.Proxy, c *Config) ([]ControllerDef, error) { +func setupControllers(mgr manager.Manager, proxy proxy.Proxy, c *Config, featureGates map[string]bool) ([]ControllerDef, error) { // Choose the best API version of Ingress to inform which ingress controller to enable. var ingressPicker ingressControllerStrategy if err := ingressPicker.Initialize(c, mgr.GetClient()); err != nil { @@ -196,8 +196,14 @@ func setupControllers(mgr manager.Manager, proxy proxy.Proxy, c *Config) ([]Cont IngressClassName: c.IngressClassName, }, }, + // --------------------------------------------------------------------------- + // Other Controllers + // --------------------------------------------------------------------------- { - Enabled: c.KnativeIngressEnabled, + // knative is a special case because it existed before we added feature gates functionality + // for this controller (only) the existing --enable-controller-knativeingress flag overrides + // any feature gate configuration. See FEATURE_GATES.md for more information. + Enabled: featureGates["Knative"] || c.KnativeIngressEnabled, AutoHandler: crdExistsChecker{GVR: schema.GroupVersionResource{ Group: knativev1alpha1.SchemeGroupVersion.Group, Version: knativev1alpha1.SchemeGroupVersion.Version, diff --git a/internal/manager/feature_gates.go b/internal/manager/feature_gates.go new file mode 100644 index 0000000000..6af829cda8 --- /dev/null +++ b/internal/manager/feature_gates.go @@ -0,0 +1,39 @@ +package manager + +import ( + "fmt" + + "github.com/go-logr/logr" +) + +// featureGatesDocsURL provides a link to the documentation for feature gates in the KIC repository +const featureGatesDocsURL = "https://github.com/Kong/kubernetes-ingress-controller/blob/main/FEATURE_GATES.md" + +// setupFeatureGates converts feature gates to controller enablement +func setupFeatureGates(setupLog logr.Logger, c *Config) (map[string]bool, error) { + // generate a map of feature gates by string names to their controller enablement + ctrlMap := getFeatureGatesDefaults() + + // override the default settings + for feature, enabled := range c.FeatureGates { + setupLog.Info("found configuration option for gated feature", "feature", feature, "enabled", enabled) + _, ok := ctrlMap[feature] + if !ok { + return ctrlMap, fmt.Errorf("%s is not a valid feature, please see the documentation: %s", feature, featureGatesDocsURL) + } + ctrlMap[feature] = enabled + } + + return ctrlMap, nil +} + +// getFeatureGatesDefaults initializes a feature gate map given the currently +// supported feature gates options and derives defaults for them based on +// manager configuration options if present. +// +// NOTE: if you're adding a new feature gate, it needs to be added here. +func getFeatureGatesDefaults() map[string]bool { + return map[string]bool{ + "Knative": false, + } +} diff --git a/internal/manager/feature_gates_test.go b/internal/manager/feature_gates_test.go new file mode 100644 index 0000000000..633fc6b982 --- /dev/null +++ b/internal/manager/feature_gates_test.go @@ -0,0 +1,39 @@ +package manager + +import ( + "bytes" + "testing" + + "github.com/bombsimon/logrusr" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" +) + +func TestFeatureGates(t *testing.T) { + t.Log("setting up configurations and logging for feature gates testing") + out := new(bytes.Buffer) + baseLogger := logrus.New() + baseLogger.SetOutput(out) + baseLogger.SetLevel(logrus.DebugLevel) + setupLog := logrusr.NewLogger(baseLogger) + config := new(Config) + + t.Log("verifying feature gates setup defaults when no feature gates are configured") + fgs, err := setupFeatureGates(setupLog, config) + assert.NoError(t, err) + assert.Len(t, fgs, len(getFeatureGatesDefaults())) + + t.Log("verifying feature gates setup results when valid feature gates options are present") + config.FeatureGates = map[string]bool{"Knative": true} + fgs, err = setupFeatureGates(setupLog, config) + assert.NoError(t, err) + assert.True(t, fgs["Knative"]) + + t.Log("configuring several invalid feature gates options") + config.FeatureGates = map[string]bool{"Batman": true} + + t.Log("verifying feature gates setup results when invalid feature gates options are present") + _, err = setupFeatureGates(setupLog, config) + assert.Error(t, err) + assert.Contains(t, err.Error(), "Batman is not a valid feature") +} diff --git a/internal/manager/run.go b/internal/manager/run.go index d7ed062944..4f9d46af79 100644 --- a/internal/manager/run.go +++ b/internal/manager/run.go @@ -42,6 +42,12 @@ func Run(ctx context.Context, c *Config, diagnostic util.ConfigDumpDiagnostic) e utilruntime.Must(configurationv1beta1.AddToScheme(scheme)) utilruntime.Must(knativev1alpha1.AddToScheme(scheme)) + setupLog.Info("getting enabled options and features") + featureGates, err := setupFeatureGates(setupLog, c) + if err != nil { + return fmt.Errorf("failed to configure feature gates: %w", err) + } + setupLog.Info("getting the kubernetes client configuration") kubeconfig, err := c.GetKubeconfig() if err != nil { @@ -68,7 +74,7 @@ func Run(ctx context.Context, c *Config, diagnostic util.ConfigDumpDiagnostic) e } setupLog.Info("Starting Enabled Controllers") - controllers, err := setupControllers(mgr, proxy, c) + controllers, err := setupControllers(mgr, proxy, c, featureGates) if err != nil { return fmt.Errorf("unable to setup controller as expected %w", err) } @@ -97,7 +103,7 @@ func Run(ctx context.Context, c *Config, diagnostic util.ConfigDumpDiagnostic) e if c.AnonymousReports { setupLog.Info("Starting anonymous reports") - if err := mgrutils.RunReport(ctx, kubeconfig, kongConfig, metadata.Release); err != nil { + if err := mgrutils.RunReport(ctx, kubeconfig, kongConfig, metadata.Release, featureGates); err != nil { setupLog.Error(err, "anonymous reporting failed") } } else { diff --git a/internal/mgrutils/reports.go b/internal/mgrutils/reports.go index 4f34e713e6..ac85411a1d 100644 --- a/internal/mgrutils/reports.go +++ b/internal/mgrutils/reports.go @@ -15,7 +15,7 @@ import ( ) // RunReport runs the anonymous data report and reports any errors that have occurred. -func RunReport(ctx context.Context, kubeCfg *rest.Config, kongCfg sendconfig.Kong, kicVersion string) error { +func RunReport(ctx context.Context, kubeCfg *rest.Config, kongCfg sendconfig.Kong, kicVersion string, featureGates map[string]bool) error { // if anonymous reports are enabled this helps provide Kong with insights about usage of the ingress controller // which is non-sensitive and predominantly informs us of the controller and cluster versions in use. // This data helps inform us what versions, features, e.t.c. end-users are actively using which helps to inform @@ -67,6 +67,7 @@ func RunReport(ctx context.Context, kubeCfg *rest.Config, kongCfg sendconfig.Kon Hostname: hostname, ID: uuid, KongDB: kongDB, + FeatureGates: featureGates, } // run the reporter in the background diff --git a/internal/util/reports.go b/internal/util/reports.go index 33f5eef6ed..847626bf4c 100644 --- a/internal/util/reports.go +++ b/internal/util/reports.go @@ -26,6 +26,7 @@ type Info struct { Hostname string KongDB string ID string + FeatureGates map[string]bool } // Reporter sends anonymous reports of runtime properties and