diff --git a/cli/ingress-controller/main.go b/cli/ingress-controller/main.go index 7f4410975b..945b409b31 100644 --- a/cli/ingress-controller/main.go +++ b/cli/ingress-controller/main.go @@ -42,6 +42,7 @@ import ( configuration "github.com/kong/kubernetes-ingress-controller/pkg/apis/configuration/v1" configclientv1 "github.com/kong/kubernetes-ingress-controller/pkg/client/configuration/clientset/versioned" configinformer "github.com/kong/kubernetes-ingress-controller/pkg/client/configuration/informers/externalversions" + "github.com/kong/kubernetes-ingress-controller/pkg/sendconfig" "github.com/kong/kubernetes-ingress-controller/pkg/store" "github.com/kong/kubernetes-ingress-controller/pkg/util" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -82,7 +83,7 @@ var ( func controllerConfigFromCLIConfig(cliConfig cliConfig) controller.Configuration { return controller.Configuration{ - Kong: controller.Kong{ + Kong: sendconfig.Kong{ URL: cliConfig.KongAdminURL, FilterTags: cliConfig.KongAdminFilterTags, Concurrency: cliConfig.KongAdminConcurrency, diff --git a/internal/ingress/controller/controller.go b/internal/ingress/controller/controller.go index f4620f3310..b6b921f2c6 100644 --- a/internal/ingress/controller/controller.go +++ b/internal/ingress/controller/controller.go @@ -23,14 +23,13 @@ import ( "sync/atomic" "time" - "github.com/blang/semver" "github.com/eapache/channels" - "github.com/kong/go-kong/kong" "github.com/kong/kubernetes-ingress-controller/internal/ingress/election" "github.com/kong/kubernetes-ingress-controller/internal/ingress/status" "github.com/kong/kubernetes-ingress-controller/internal/ingress/task" configClientSet "github.com/kong/kubernetes-ingress-controller/pkg/client/configuration/clientset/versioned" "github.com/kong/kubernetes-ingress-controller/pkg/parser" + "github.com/kong/kubernetes-ingress-controller/pkg/sendconfig" "github.com/kong/kubernetes-ingress-controller/pkg/store" "github.com/kong/kubernetes-ingress-controller/pkg/util" "github.com/sirupsen/logrus" @@ -44,26 +43,10 @@ import ( knativeClientSet "knative.dev/networking/pkg/client/clientset/versioned" ) -// Kong Represents a Kong client and connection information -type Kong struct { - URL string - FilterTags []string - // Headers are injected into every request to Kong's Admin API - // to help with authorization/authentication. - Client *kong.Client - - InMemory bool - HasTagSupport bool - Enterprise bool - - Version semver.Version - - Concurrency int -} - // Configuration contains all the settings required by an Ingress controller type Configuration struct { - Kong + sendconfig.Kong + KongCustomEntitiesSecret string KubeClient clientset.Interface @@ -143,7 +126,7 @@ func NewKongController(ctx context.Context, updateCh: updateCh, stopLock: &sync.Mutex{}, - PluginSchemaStore: *NewPluginSchemaStore(config.Kong.Client), + PluginSchemaStore: *util.NewPluginSchemaStore(config.Kong.Client), Logger: config.Logger, } @@ -228,7 +211,7 @@ type KongController struct { store store.Storer - PluginSchemaStore PluginSchemaStore + PluginSchemaStore util.PluginSchemaStore Logger logrus.FieldLogger } diff --git a/internal/ingress/controller/kong.go b/internal/ingress/controller/kong.go index a860147df3..eb3a5f37d2 100644 --- a/internal/ingress/controller/kong.go +++ b/internal/ingress/controller/kong.go @@ -17,24 +17,12 @@ limitations under the License. package controller import ( - "bytes" "context" - "crypto/sha256" - "encoding/json" "fmt" - "net/http" - "reflect" - "sort" - "strings" - "github.com/kong/deck/diff" - "github.com/kong/deck/dump" - "github.com/kong/deck/file" - "github.com/kong/deck/solver" - "github.com/kong/deck/state" - deckutils "github.com/kong/deck/utils" - "github.com/kong/go-kong/kong" + "github.com/kong/kubernetes-ingress-controller/pkg/deckgen" "github.com/kong/kubernetes-ingress-controller/pkg/kongstate" + "github.com/kong/kubernetes-ingress-controller/pkg/sendconfig" "github.com/kong/kubernetes-ingress-controller/pkg/util" ) @@ -42,8 +30,6 @@ import ( // returning nil implies the synchronization finished correctly. // Returning an error means requeue the update. func (n *KongController) OnUpdate(ctx context.Context, state *kongstate.KongState) error { - targetContent := n.toDeckContent(ctx, state) - var customEntities []byte var err error // process any custom entities @@ -54,139 +40,21 @@ func (n *KongController) OnUpdate(ctx context.Context, state *kongstate.KongStat n.Logger.Errorf("failed to fetch custom entities: %v", err) } } + targetContent := deckgen.ToDeckContent(ctx, n.Logger, state, &n.PluginSchemaStore, n.getIngressControllerTags()) - var shaSum []byte - // disable optimization if reverse sync is enabled - if !n.cfg.EnableReverseSync { - shaSum, err = generateSHA(targetContent, customEntities) - if err != nil { - return err - } - if reflect.DeepEqual(n.runningConfigHash, shaSum) { - n.Logger.Info("no configuration change, skipping sync to kong") - return nil - } - } - if n.cfg.InMemory { - err = n.onUpdateInMemoryMode(ctx, targetContent, customEntities) - } else { - err = n.onUpdateDBMode(targetContent) - } - if err != nil { - return err - } - n.runningConfigHash = shaSum - n.Logger.Info("successfully synced configuration to kong") - return nil -} - -func generateSHA(targetContent *file.Content, - customEntities []byte) ([]byte, error) { - - var buffer bytes.Buffer - - jsonConfig, err := json.Marshal(targetContent) - if err != nil { - return nil, fmt.Errorf("marshaling Kong declarative configuration to JSON: %w", err) - } - buffer.Write(jsonConfig) - - if customEntities != nil { - buffer.Write(customEntities) - } - - shaSum := sha256.Sum256(buffer.Bytes()) - return shaSum[:], nil -} - -func cleanUpNullsInPluginConfigs(state *file.Content) { - - for _, s := range state.Services { - for _, p := range s.Plugins { - for k, v := range p.Config { - if v == nil { - delete(p.Config, k) - } - } - } - for _, r := range state.Routes { - for _, p := range r.Plugins { - for k, v := range p.Config { - if v == nil { - delete(p.Config, k) - } - } - } - } - } - - for _, c := range state.Consumers { - for _, p := range c.Plugins { - for k, v := range p.Config { - if v == nil { - delete(p.Config, k) - } - } - } - } - - for _, p := range state.Plugins { - for k, v := range p.Config { - if v == nil { - delete(p.Config, k) - } - } - } -} - -func (n *KongController) renderConfigWithCustomEntities(state *file.Content, - customEntitiesJSONBytes []byte) ([]byte, error) { - - var kongCoreConfig []byte - var err error - - kongCoreConfig, err = json.Marshal(state) - if err != nil { - return nil, fmt.Errorf("marshaling kong config into json: %w", err) - } - - // fast path - if len(customEntitiesJSONBytes) == 0 { - return kongCoreConfig, nil - } + newSHA, err := sendconfig.PerformUpdate(ctx, + n.Logger, + &n.cfg.Kong, + n.cfg.InMemory, + n.cfg.EnableReverseSync, + targetContent, + n.getIngressControllerTags(), + customEntities, + n.runningConfigHash, + ) - // slow path - mergeMap := map[string]interface{}{} - var result []byte - var customEntities map[string]interface{} - - // unmarshal core config into the merge map - err = json.Unmarshal(kongCoreConfig, &mergeMap) - if err != nil { - return nil, fmt.Errorf("unmarshalling kong config into map[string]interface{}: %w", err) - } - - // unmarshal custom entities config into the merge map - err = json.Unmarshal(customEntitiesJSONBytes, &customEntities) - if err != nil { - // do not error out when custom entities are messed up - n.Logger.Errorf("failed to unmarshal custom entities from secret data: %v", err) - } else { - for k, v := range customEntities { - if _, exists := mergeMap[k]; !exists { - mergeMap[k] = v - } - } - } - - // construct the final configuration - result, err = json.Marshal(mergeMap) - if err != nil { - err = fmt.Errorf("marshaling final config into JSON: %w", err) - return nil, err - } - - return result, nil + n.runningConfigHash = newSHA + return err } func (n *KongController) fetchCustomEntities() ([]byte, error) { @@ -206,82 +74,6 @@ func (n *KongController) fetchCustomEntities() ([]byte, error) { return config, nil } -func (n *KongController) onUpdateInMemoryMode(ctx context.Context, - state *file.Content, - customEntities []byte) error { - client := n.cfg.Kong.Client - - // Kong will error out if this is set - state.Info = nil - // Kong errors out if `null`s are present in `config` of plugins - cleanUpNullsInPluginConfigs(state) - - config, err := n.renderConfigWithCustomEntities(state, customEntities) - if err != nil { - return fmt.Errorf("constructing kong configuration: %w", err) - } - - req, err := http.NewRequest("POST", n.cfg.Kong.URL+"/config", - bytes.NewReader(config)) - if err != nil { - return fmt.Errorf("creating new HTTP request for /config: %w", err) - } - req.Header.Add("content-type", "application/json") - - queryString := req.URL.Query() - queryString.Add("check_hash", "1") - - req.URL.RawQuery = queryString.Encode() - - _, err = client.Do(ctx, req, nil) - if err != nil { - return fmt.Errorf("posting new config to /config: %w", err) - } - - return err -} - -func (n *KongController) onUpdateDBMode(targetContent *file.Content) error { - client := n.cfg.Kong.Client - - // read the current state - rawState, err := dump.Get(client, dump.Config{ - SelectorTags: n.getIngressControllerTags(), - }) - if err != nil { - return fmt.Errorf("loading configuration from kong: %w", err) - } - currentState, err := state.Get(rawState) - if err != nil { - return err - } - - // read the target state - rawState, err = file.Get(targetContent, file.RenderConfig{ - CurrentState: currentState, - KongVersion: n.cfg.Kong.Version, - }) - if err != nil { - return err - } - targetState, err := state.Get(rawState) - if err != nil { - return err - } - - syncer, err := diff.NewSyncer(currentState, targetState) - if err != nil { - return fmt.Errorf("creating a new syncer: %w", err) - } - syncer.SilenceWarnings = true - //client.SetDebugMode(true) - _, errs := solver.Solve(nil, syncer, client, n.cfg.Kong.Concurrency, false) - if errs != nil { - return deckutils.ErrArray{Errors: errs} - } - return nil -} - // getIngressControllerTags returns a tag to use if the current // Kong entity supports tagging. func (n *KongController) getIngressControllerTags() []string { @@ -291,230 +83,3 @@ func (n *KongController) getIngressControllerTags() []string { } return res } - -func (n *KongController) toDeckContent( - ctx context.Context, - k8sState *kongstate.KongState) *file.Content { - var content file.Content - content.FormatVersion = "1.1" - var err error - - for _, s := range k8sState.Services { - service := file.FService{Service: s.Service} - for _, p := range s.Plugins { - plugin := file.FPlugin{ - Plugin: *p.DeepCopy(), - } - err = n.fillPlugin(ctx, &plugin) - if err != nil { - n.Logger.Errorf("failed to fill-in defaults for plugin: %s", *plugin.Name) - } - service.Plugins = append(service.Plugins, &plugin) - sort.SliceStable(service.Plugins, func(i, j int) bool { - return strings.Compare(*service.Plugins[i].Name, *service.Plugins[j].Name) > 0 - }) - } - - for _, r := range s.Routes { - route := file.FRoute{Route: r.Route} - n.fillRoute(&route.Route) - - for _, p := range r.Plugins { - plugin := file.FPlugin{ - Plugin: *p.DeepCopy(), - } - err = n.fillPlugin(ctx, &plugin) - if err != nil { - n.Logger.Errorf("failed to fill-in defaults for plugin: %s", *plugin.Name) - } - route.Plugins = append(route.Plugins, &plugin) - sort.SliceStable(route.Plugins, func(i, j int) bool { - return strings.Compare(*route.Plugins[i].Name, *route.Plugins[j].Name) > 0 - }) - } - service.Routes = append(service.Routes, &route) - } - sort.SliceStable(service.Routes, func(i, j int) bool { - return strings.Compare(*service.Routes[i].Name, *service.Routes[j].Name) > 0 - }) - content.Services = append(content.Services, service) - } - sort.SliceStable(content.Services, func(i, j int) bool { - return strings.Compare(*content.Services[i].Name, *content.Services[j].Name) > 0 - }) - - for _, plugin := range k8sState.Plugins { - plugin := file.FPlugin{ - Plugin: plugin.Plugin, - } - err = n.fillPlugin(ctx, &plugin) - if err != nil { - n.Logger.Errorf("failed to fill-in defaults for plugin: %s", *plugin.Name) - } - content.Plugins = append(content.Plugins, plugin) - } - sort.SliceStable(content.Plugins, func(i, j int) bool { - return strings.Compare(pluginString(content.Plugins[i]), - pluginString(content.Plugins[j])) > 0 - }) - - for _, u := range k8sState.Upstreams { - n.fillUpstream(&u.Upstream) - upstream := file.FUpstream{Upstream: u.Upstream} - for _, t := range u.Targets { - target := file.FTarget{Target: t.Target} - upstream.Targets = append(upstream.Targets, &target) - } - sort.SliceStable(upstream.Targets, func(i, j int) bool { - return strings.Compare(*upstream.Targets[i].Target.Target, *upstream.Targets[j].Target.Target) > 0 - }) - content.Upstreams = append(content.Upstreams, upstream) - } - sort.SliceStable(content.Upstreams, func(i, j int) bool { - return strings.Compare(*content.Upstreams[i].Name, *content.Upstreams[j].Name) > 0 - }) - - for _, c := range k8sState.Certificates { - cert := getFCertificateFromKongCert(c.Certificate) - content.Certificates = append(content.Certificates, cert) - } - sort.SliceStable(content.Certificates, func(i, j int) bool { - return strings.Compare(*content.Certificates[i].Cert, *content.Certificates[j].Cert) > 0 - }) - - for _, c := range k8sState.CACertificates { - content.CACertificates = append(content.CACertificates, - file.FCACertificate{CACertificate: c}) - } - sort.SliceStable(content.CACertificates, func(i, j int) bool { - return strings.Compare(*content.CACertificates[i].Cert, *content.CACertificates[j].Cert) > 0 - }) - - for _, c := range k8sState.Consumers { - consumer := file.FConsumer{Consumer: c.Consumer} - for _, p := range c.Plugins { - consumer.Plugins = append(consumer.Plugins, &file.FPlugin{Plugin: p}) - } - - for _, v := range c.KeyAuths { - consumer.KeyAuths = append(consumer.KeyAuths, &v.KeyAuth) - } - for _, v := range c.HMACAuths { - consumer.HMACAuths = append(consumer.HMACAuths, &v.HMACAuth) - } - for _, v := range c.BasicAuths { - consumer.BasicAuths = append(consumer.BasicAuths, &v.BasicAuth) - } - for _, v := range c.JWTAuths { - consumer.JWTAuths = append(consumer.JWTAuths, &v.JWTAuth) - } - for _, v := range c.ACLGroups { - consumer.ACLGroups = append(consumer.ACLGroups, &v.ACLGroup) - } - for _, v := range c.Oauth2Creds { - consumer.Oauth2Creds = append(consumer.Oauth2Creds, &v.Oauth2Credential) - } - content.Consumers = append(content.Consumers, consumer) - } - sort.SliceStable(content.Consumers, func(i, j int) bool { - return strings.Compare(*content.Consumers[i].Username, *content.Consumers[j].Username) > 0 - }) - selectorTags := n.getIngressControllerTags() - if len(selectorTags) > 0 { - content.Info = &file.Info{ - SelectorTags: selectorTags, - } - } - - return &content -} - -func getFCertificateFromKongCert(kongCert kong.Certificate) file.FCertificate { - var res file.FCertificate - if kongCert.ID != nil { - res.ID = kong.String(*kongCert.ID) - } - if kongCert.Key != nil { - res.Key = kong.String(*kongCert.Key) - } - if kongCert.Cert != nil { - res.Cert = kong.String(*kongCert.Cert) - } - res.SNIs = getSNIs(kongCert.SNIs) - return res -} - -func getSNIs(names []*string) []kong.SNI { - var snis []kong.SNI - for _, name := range names { - snis = append(snis, kong.SNI{ - Name: kong.String(*name), - }) - } - return snis -} - -func pluginString(plugin file.FPlugin) string { - result := "" - if plugin.Name != nil { - result = *plugin.Name - } - if plugin.Consumer != nil && plugin.Consumer.ID != nil { - result += *plugin.Consumer.ID - } - if plugin.Route != nil && plugin.Route.ID != nil { - result += *plugin.Route.ID - } - if plugin.Service != nil && plugin.Service.ID != nil { - result += *plugin.Service.ID - } - return result -} - -func (n *KongController) fillRoute(route *kong.Route) { - if route.HTTPSRedirectStatusCode == nil { - route.HTTPSRedirectStatusCode = kong.Int(426) - } - if route.PathHandling == nil { - route.PathHandling = kong.String("v0") - } -} - -func (n *KongController) fillUpstream(upstream *kong.Upstream) { - if upstream.Algorithm == nil { - upstream.Algorithm = kong.String("round-robin") - } -} - -func (n *KongController) fillPlugin(ctx context.Context, plugin *file.FPlugin) error { - if plugin == nil { - return fmt.Errorf("plugin is nil") - } - if plugin.Name == nil || *plugin.Name == "" { - return fmt.Errorf("plugin doesn't have a name") - } - schema, err := n.PluginSchemaStore.Schema(ctx, *plugin.Name) - if err != nil { - return fmt.Errorf("error retrieveing schema for plugin %s: %w", *plugin.Name, err) - } - if plugin.Config == nil { - plugin.Config = make(kong.Configuration) - } - newConfig, err := fill(schema, plugin.Config) - if err != nil { - return fmt.Errorf("error filling in default for plugin %s: %w", *plugin.Name, err) - } - plugin.Config = newConfig - if plugin.RunOn == nil { - plugin.RunOn = kong.String("first") - } - if plugin.Enabled == nil { - plugin.Enabled = kong.Bool(true) - } - if len(plugin.Protocols) == 0 { - // TODO read this from the schema endpoint - plugin.Protocols = kong.StringSlice("http", "https") - } - plugin.RunOn = nil - return nil -} diff --git a/internal/ingress/controller/plugin_schema_helper.go b/internal/ingress/controller/plugin_schema_helper.go deleted file mode 100644 index a61699ab0f..0000000000 --- a/internal/ingress/controller/plugin_schema_helper.go +++ /dev/null @@ -1,113 +0,0 @@ -package controller - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/kong/go-kong/kong" - "github.com/tidwall/gjson" -) - -// FIXME -// decK will release this official API soon, use that and remove this code. - -// PluginSchemaStore retrives a schema of a Plugin from Kong. -type PluginSchemaStore struct { - client *kong.Client - schemas map[string]map[string]interface{} -} - -// NewPluginSchemaStore creates a PluginSchemaStore. -func NewPluginSchemaStore(client *kong.Client) *PluginSchemaStore { - return &PluginSchemaStore{ - client: client, - schemas: make(map[string]map[string]interface{}), - } -} - -// Schema retrives schema of a plugin. -// A cache is used to save the responses and subsequent queries are served from -// the cache. -func (p *PluginSchemaStore) Schema(ctx context.Context, pluginName string) (map[string]interface{}, error) { - if pluginName == "" { - return nil, fmt.Errorf("pluginName can not be empty") - } - - // lookup in cache - if schema, ok := p.schemas[pluginName]; ok { - return schema, nil - } - - // not present in cache, lookup - req, err := p.client.NewRequest("GET", "/plugins/schema/"+pluginName, - nil, nil) - if err != nil { - return nil, err - } - schema := make(map[string]interface{}) - _, err = p.client.Do(ctx, req, &schema) - if err != nil { - return nil, err - } - p.schemas[pluginName] = schema - return schema, nil -} - -func fill(schema map[string]interface{}, - config kong.Configuration) (kong.Configuration, error) { - jsonb, err := json.Marshal(&schema) - if err != nil { - return nil, err - } - // Get all in the schema - value := gjson.ParseBytes((jsonb)) - return fillRecord(value, config) -} - -func fillRecord(schema gjson.Result, config kong.Configuration) (kong.Configuration, error) { - if config == nil { - return nil, nil - } - res := config.DeepCopy() - value := schema.Get("fields") - - value.ForEach(func(key, value gjson.Result) bool { - // get the key name - ms := value.Map() - fname := "" - for k := range ms { - fname = k - break - } - ftype := value.Get(fname + ".type") - if ftype.String() == "record" { - subConfig := config[fname] - if subConfig == nil { - subConfig = make(map[string]interface{}) - } - newSubConfig, err := fillRecord(value.Get(fname), subConfig.(map[string]interface{})) - if err != nil { - panic(err) - } - res[fname] = map[string]interface{}(newSubConfig) - return true - } - // check if key is already set in the config - if _, ok := config[fname]; ok { - // yes, don't set it - return true - } - // no, set it - value = value.Get(fname + ".default") - if value.Exists() { - res[fname] = value.Value() - } else { - // if no default exists, set an explicit nil - res[fname] = nil - } - return true - }) - - return res, nil -} diff --git a/pkg/deckgen/deckgen.go b/pkg/deckgen/deckgen.go new file mode 100644 index 0000000000..94994a46ff --- /dev/null +++ b/pkg/deckgen/deckgen.go @@ -0,0 +1,178 @@ +package deckgen + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "fmt" + + "github.com/kong/deck/file" + "github.com/kong/go-kong/kong" + "github.com/tidwall/gjson" +) + +// GenerateSHA generates a SHA256 checksum of the (targetContent, customEntities) tuple, with the purpose of change +// detection. +func GenerateSHA(targetContent *file.Content, + customEntities []byte) ([]byte, error) { + + var buffer bytes.Buffer + + jsonConfig, err := json.Marshal(targetContent) + if err != nil { + return nil, fmt.Errorf("marshaling Kong declarative configuration to JSON: %w", err) + } + buffer.Write(jsonConfig) + + if customEntities != nil { + buffer.Write(customEntities) + } + + shaSum := sha256.Sum256(buffer.Bytes()) + return shaSum[:], nil +} + +// CleanUpNullsInPluginConfigs modifies `state` by deleting plugin config map keys that have nil as their value. +func CleanUpNullsInPluginConfigs(state *file.Content) { + for _, s := range state.Services { + for _, p := range s.Plugins { + for k, v := range p.Config { + if v == nil { + delete(p.Config, k) + } + } + } + for _, r := range state.Routes { + for _, p := range r.Plugins { + for k, v := range p.Config { + if v == nil { + delete(p.Config, k) + } + } + } + } + } + + for _, c := range state.Consumers { + for _, p := range c.Plugins { + for k, v := range p.Config { + if v == nil { + delete(p.Config, k) + } + } + } + } + + for _, p := range state.Plugins { + for k, v := range p.Config { + if v == nil { + delete(p.Config, k) + } + } + } +} + +// GetFCertificateFromKongCert converts a kong.Certificate to a file.FCertificate. +func GetFCertificateFromKongCert(kongCert kong.Certificate) file.FCertificate { + var res file.FCertificate + if kongCert.ID != nil { + res.ID = kong.String(*kongCert.ID) + } + if kongCert.Key != nil { + res.Key = kong.String(*kongCert.Key) + } + if kongCert.Cert != nil { + res.Cert = kong.String(*kongCert.Cert) + } + res.SNIs = getSNIs(kongCert.SNIs) + return res +} + +func getSNIs(names []*string) []kong.SNI { + var snis []kong.SNI + for _, name := range names { + snis = append(snis, kong.SNI{ + Name: kong.String(*name), + }) + } + return snis +} + +// PluginString returns a string representation of a FPlugin suitable as a sorting key. +// +// Deprecated. To be replaced by a predicate that compares two FPlugins. +func PluginString(plugin file.FPlugin) string { + result := "" + if plugin.Name != nil { + result = *plugin.Name + } + if plugin.Consumer != nil && plugin.Consumer.ID != nil { + result += *plugin.Consumer.ID + } + if plugin.Route != nil && plugin.Route.ID != nil { + result += *plugin.Route.ID + } + if plugin.Service != nil && plugin.Service.ID != nil { + result += *plugin.Service.ID + } + return result +} + +// FillPluginConfig returns a copy of `config` that has default values filled in from `schema`. +func FillPluginConfig(schema map[string]interface{}, + config kong.Configuration) (kong.Configuration, error) { + jsonb, err := json.Marshal(&schema) + if err != nil { + return nil, err + } + // Get all in the schema + value := gjson.ParseBytes((jsonb)) + return fillRecord(value, config) +} + +func fillRecord(schema gjson.Result, config kong.Configuration) (kong.Configuration, error) { + if config == nil { + return nil, nil + } + res := config.DeepCopy() + value := schema.Get("fields") + + value.ForEach(func(key, value gjson.Result) bool { + // get the key name + ms := value.Map() + fname := "" + for k := range ms { + fname = k + break + } + ftype := value.Get(fname + ".type") + if ftype.String() == "record" { + subConfig := config[fname] + if subConfig == nil { + subConfig = make(map[string]interface{}) + } + newSubConfig, err := fillRecord(value.Get(fname), subConfig.(map[string]interface{})) + if err != nil { + panic(err) + } + res[fname] = map[string]interface{}(newSubConfig) + return true + } + // check if key is already set in the config + if _, ok := config[fname]; ok { + // yes, don't set it + return true + } + // no, set it + value = value.Get(fname + ".default") + if value.Exists() { + res[fname] = value.Value() + } else { + // if no default exists, set an explicit nil + res[fname] = nil + } + return true + }) + + return res, nil +} diff --git a/internal/ingress/controller/plugin_schema_helper_test.go b/pkg/deckgen/deckgen_test.go similarity index 97% rename from internal/ingress/controller/plugin_schema_helper_test.go rename to pkg/deckgen/deckgen_test.go index de64833da8..64e08a0661 100644 --- a/internal/ingress/controller/plugin_schema_helper_test.go +++ b/pkg/deckgen/deckgen_test.go @@ -1,4 +1,4 @@ -package controller +package deckgen import ( "encoding/json" @@ -573,7 +573,7 @@ var ( func TestFillNil(t *testing.T) { assert := assert.New(t) - assert.Nil(fill(nil, nil)) + assert.Nil(FillPluginConfig(nil, nil)) } func TestFillKeyAuth(t *testing.T) { @@ -589,7 +589,7 @@ func TestFillKeyAuth(t *testing.T) { err = json.Unmarshal([]byte(KeyAuthDefaultConfig), &def) assert.Nil(err) - res, err := fill(schema, config) + res, err := FillPluginConfig(schema, config) assert.Equal(def, res) } @@ -606,7 +606,7 @@ func TestFillStatsD(t *testing.T) { err = json.Unmarshal([]byte(StatsDDefaultConfig), &def) assert.Nil(err) - res, err := fill(schema, config) + res, err := FillPluginConfig(schema, config) assert.Equal(def, res) } @@ -624,7 +624,7 @@ func TestKeyAuthSetKeys(t *testing.T) { err = json.Unmarshal([]byte(KeyAuthDefaultConfig), &def) assert.Nil(err) - res, err := fill(schema, config) + res, err := FillPluginConfig(schema, config) assert.NotEqual(def, res) assert.Equal(true, res["key_in_body"]) } @@ -640,7 +640,7 @@ func TestFillReqeustTransformer(t *testing.T) { err = json.Unmarshal([]byte(RequestTransformerConfig), &def) assert.Nil(err) - res, err := fill(schema, config) + res, err := FillPluginConfig(schema, config) assert.Equal(def, res) } @@ -655,7 +655,7 @@ func TestFillReqeustTransformerNestedConfig(t *testing.T) { assert.Nil(err) want := make(kong.Configuration) err = json.Unmarshal([]byte(RequestTransformerNonEmptyFilledConfig), &want) - res, err := fill(schema, config) + res, err := FillPluginConfig(schema, config) assert.Equal(want, res) assert.Nil(err) } diff --git a/pkg/deckgen/generate.go b/pkg/deckgen/generate.go new file mode 100644 index 0000000000..eebd1988f3 --- /dev/null +++ b/pkg/deckgen/generate.go @@ -0,0 +1,203 @@ +package deckgen + +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/kong/deck/file" + "github.com/kong/go-kong/kong" + "github.com/kong/kubernetes-ingress-controller/pkg/kongstate" + "github.com/kong/kubernetes-ingress-controller/pkg/util" + "github.com/sirupsen/logrus" +) + +// ToDeckContent generates a decK configuration from `k8sState` and auxiliary parameters. +func ToDeckContent( + ctx context.Context, + log logrus.FieldLogger, + k8sState *kongstate.KongState, + schemas *util.PluginSchemaStore, + selectorTags []string, +) *file.Content { + var content file.Content + content.FormatVersion = "1.1" + var err error + + for _, s := range k8sState.Services { + service := file.FService{Service: s.Service} + for _, p := range s.Plugins { + plugin := file.FPlugin{ + Plugin: *p.DeepCopy(), + } + err = fillPlugin(ctx, &plugin, schemas) + if err != nil { + log.Errorf("failed to fill-in defaults for plugin: %s", *plugin.Name) + } + service.Plugins = append(service.Plugins, &plugin) + sort.SliceStable(service.Plugins, func(i, j int) bool { + return strings.Compare(*service.Plugins[i].Name, *service.Plugins[j].Name) > 0 + }) + } + + for _, r := range s.Routes { + route := file.FRoute{Route: r.Route} + fillRoute(&route.Route) + + for _, p := range r.Plugins { + plugin := file.FPlugin{ + Plugin: *p.DeepCopy(), + } + err = fillPlugin(ctx, &plugin, schemas) + if err != nil { + log.Errorf("failed to fill-in defaults for plugin: %s", *plugin.Name) + } + route.Plugins = append(route.Plugins, &plugin) + sort.SliceStable(route.Plugins, func(i, j int) bool { + return strings.Compare(*route.Plugins[i].Name, *route.Plugins[j].Name) > 0 + }) + } + service.Routes = append(service.Routes, &route) + } + sort.SliceStable(service.Routes, func(i, j int) bool { + return strings.Compare(*service.Routes[i].Name, *service.Routes[j].Name) > 0 + }) + content.Services = append(content.Services, service) + } + sort.SliceStable(content.Services, func(i, j int) bool { + return strings.Compare(*content.Services[i].Name, *content.Services[j].Name) > 0 + }) + + for _, plugin := range k8sState.Plugins { + plugin := file.FPlugin{ + Plugin: plugin.Plugin, + } + err = fillPlugin(ctx, &plugin, schemas) + if err != nil { + log.Errorf("failed to fill-in defaults for plugin: %s", *plugin.Name) + } + content.Plugins = append(content.Plugins, plugin) + } + sort.SliceStable(content.Plugins, func(i, j int) bool { + return strings.Compare(PluginString(content.Plugins[i]), + PluginString(content.Plugins[j])) > 0 + }) + + for _, u := range k8sState.Upstreams { + fillUpstream(&u.Upstream) + upstream := file.FUpstream{Upstream: u.Upstream} + for _, t := range u.Targets { + target := file.FTarget{Target: t.Target} + upstream.Targets = append(upstream.Targets, &target) + } + sort.SliceStable(upstream.Targets, func(i, j int) bool { + return strings.Compare(*upstream.Targets[i].Target.Target, *upstream.Targets[j].Target.Target) > 0 + }) + content.Upstreams = append(content.Upstreams, upstream) + } + sort.SliceStable(content.Upstreams, func(i, j int) bool { + return strings.Compare(*content.Upstreams[i].Name, *content.Upstreams[j].Name) > 0 + }) + + for _, c := range k8sState.Certificates { + cert := GetFCertificateFromKongCert(c.Certificate) + content.Certificates = append(content.Certificates, cert) + } + sort.SliceStable(content.Certificates, func(i, j int) bool { + return strings.Compare(*content.Certificates[i].Cert, *content.Certificates[j].Cert) > 0 + }) + + for _, c := range k8sState.CACertificates { + content.CACertificates = append(content.CACertificates, + file.FCACertificate{CACertificate: c}) + } + sort.SliceStable(content.CACertificates, func(i, j int) bool { + return strings.Compare(*content.CACertificates[i].Cert, *content.CACertificates[j].Cert) > 0 + }) + + for _, c := range k8sState.Consumers { + consumer := file.FConsumer{Consumer: c.Consumer} + for _, p := range c.Plugins { + consumer.Plugins = append(consumer.Plugins, &file.FPlugin{Plugin: p}) + } + + for _, v := range c.KeyAuths { + consumer.KeyAuths = append(consumer.KeyAuths, &v.KeyAuth) + } + for _, v := range c.HMACAuths { + consumer.HMACAuths = append(consumer.HMACAuths, &v.HMACAuth) + } + for _, v := range c.BasicAuths { + consumer.BasicAuths = append(consumer.BasicAuths, &v.BasicAuth) + } + for _, v := range c.JWTAuths { + consumer.JWTAuths = append(consumer.JWTAuths, &v.JWTAuth) + } + for _, v := range c.ACLGroups { + consumer.ACLGroups = append(consumer.ACLGroups, &v.ACLGroup) + } + for _, v := range c.Oauth2Creds { + consumer.Oauth2Creds = append(consumer.Oauth2Creds, &v.Oauth2Credential) + } + content.Consumers = append(content.Consumers, consumer) + } + sort.SliceStable(content.Consumers, func(i, j int) bool { + return strings.Compare(*content.Consumers[i].Username, *content.Consumers[j].Username) > 0 + }) + if len(selectorTags) > 0 { + content.Info = &file.Info{ + SelectorTags: selectorTags, + } + } + + return &content +} + +func fillRoute(route *kong.Route) { + if route.HTTPSRedirectStatusCode == nil { + route.HTTPSRedirectStatusCode = kong.Int(426) + } + if route.PathHandling == nil { + route.PathHandling = kong.String("v0") + } +} + +func fillUpstream(upstream *kong.Upstream) { + if upstream.Algorithm == nil { + upstream.Algorithm = kong.String("round-robin") + } +} + +func fillPlugin(ctx context.Context, plugin *file.FPlugin, schemas *util.PluginSchemaStore) error { + if plugin == nil { + return fmt.Errorf("plugin is nil") + } + if plugin.Name == nil || *plugin.Name == "" { + return fmt.Errorf("plugin doesn't have a name") + } + schema, err := schemas.Schema(ctx, *plugin.Name) + if err != nil { + return fmt.Errorf("error retrieveing schema for plugin %s: %w", *plugin.Name, err) + } + if plugin.Config == nil { + plugin.Config = make(kong.Configuration) + } + newConfig, err := FillPluginConfig(schema, plugin.Config) + if err != nil { + return fmt.Errorf("error filling in default for plugin %s: %w", *plugin.Name, err) + } + plugin.Config = newConfig + if plugin.RunOn == nil { + plugin.RunOn = kong.String("first") + } + if plugin.Enabled == nil { + plugin.Enabled = kong.Bool(true) + } + if len(plugin.Protocols) == 0 { + // TODO read this from the schema endpoint + plugin.Protocols = kong.StringSlice("http", "https") + } + plugin.RunOn = nil + return nil +} diff --git a/pkg/sendconfig/kong.go b/pkg/sendconfig/kong.go new file mode 100644 index 0000000000..2f4f7336b4 --- /dev/null +++ b/pkg/sendconfig/kong.go @@ -0,0 +1,23 @@ +package sendconfig + +import ( + "github.com/blang/semver" + "github.com/kong/go-kong/kong" +) + +// Kong Represents a Kong client and connection information +type Kong struct { + URL string + FilterTags []string + // Headers are injected into every request to Kong's Admin API + // to help with authorization/authentication. + Client *kong.Client + + InMemory bool + HasTagSupport bool + Enterprise bool + + Version semver.Version + + Concurrency int +} diff --git a/pkg/sendconfig/sendconfig.go b/pkg/sendconfig/sendconfig.go new file mode 100644 index 0000000000..7a64d6c1ea --- /dev/null +++ b/pkg/sendconfig/sendconfig.go @@ -0,0 +1,187 @@ +package sendconfig + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "reflect" + + "github.com/kong/deck/diff" + "github.com/kong/deck/dump" + "github.com/kong/deck/file" + "github.com/kong/deck/solver" + "github.com/kong/deck/state" + deckutils "github.com/kong/deck/utils" + "github.com/kong/kubernetes-ingress-controller/pkg/deckgen" + "github.com/sirupsen/logrus" +) + +func equalSHA(a, b []byte) bool { + return reflect.DeepEqual(a, b) +} + +// PerformUpdate writes `targetContent` and `customEntities` to Kong Admin API specified by `kongConfig`. +func PerformUpdate(ctx context.Context, + log logrus.FieldLogger, + kongConfig *Kong, + inMemory bool, + reverseSync bool, + targetContent *file.Content, + selectorTags []string, + customEntities []byte, + oldSHA []byte, +) ([]byte, error) { + + newSHA, err := deckgen.GenerateSHA(targetContent, customEntities) + if err != nil { + return oldSHA, err + } + // disable optimization if reverse sync is enabled + if !reverseSync { + if equalSHA(oldSHA, newSHA) { + log.Info("no configuration change, skipping sync to kong") + return oldSHA, nil + } + } + + if inMemory { + err = onUpdateInMemoryMode(ctx, log, targetContent, customEntities, kongConfig) + } else { + err = onUpdateDBMode(targetContent, kongConfig, selectorTags) + } + if err != nil { + return nil, err + } + log.Info("successfully synced configuration to kong") + return newSHA, nil +} + +func renderConfigWithCustomEntities(log logrus.FieldLogger, state *file.Content, + customEntitiesJSONBytes []byte) ([]byte, error) { + + var kongCoreConfig []byte + var err error + + kongCoreConfig, err = json.Marshal(state) + if err != nil { + return nil, fmt.Errorf("marshaling kong config into json: %w", err) + } + + // fast path + if len(customEntitiesJSONBytes) == 0 { + return kongCoreConfig, nil + } + + // slow path + mergeMap := map[string]interface{}{} + var result []byte + var customEntities map[string]interface{} + + // unmarshal core config into the merge map + err = json.Unmarshal(kongCoreConfig, &mergeMap) + if err != nil { + return nil, fmt.Errorf("unmarshalling kong config into map[string]interface{}: %w", err) + } + + // unmarshal custom entities config into the merge map + err = json.Unmarshal(customEntitiesJSONBytes, &customEntities) + if err != nil { + // do not error out when custom entities are messed up + log.Errorf("failed to unmarshal custom entities from secret data: %v", err) + } else { + for k, v := range customEntities { + if _, exists := mergeMap[k]; !exists { + mergeMap[k] = v + } + } + } + + // construct the final configuration + result, err = json.Marshal(mergeMap) + if err != nil { + err = fmt.Errorf("marshaling final config into JSON: %w", err) + return nil, err + } + + return result, nil +} + +func onUpdateInMemoryMode(ctx context.Context, + log logrus.FieldLogger, + state *file.Content, + customEntities []byte, + kongConfig *Kong, +) error { + // Kong will error out if this is set + state.Info = nil + // Kong errors out if `null`s are present in `config` of plugins + deckgen.CleanUpNullsInPluginConfigs(state) + + config, err := renderConfigWithCustomEntities(log, state, customEntities) + if err != nil { + return fmt.Errorf("constructing kong configuration: %w", err) + } + + req, err := http.NewRequest("POST", kongConfig.URL+"/config", + bytes.NewReader(config)) + if err != nil { + return fmt.Errorf("creating new HTTP request for /config: %w", err) + } + req.Header.Add("content-type", "application/json") + + queryString := req.URL.Query() + queryString.Add("check_hash", "1") + + req.URL.RawQuery = queryString.Encode() + + _, err = kongConfig.Client.Do(ctx, req, nil) + if err != nil { + return fmt.Errorf("posting new config to /config: %w", err) + } + + return err +} + +func onUpdateDBMode( + targetContent *file.Content, + kongConfig *Kong, + selectorTags []string, +) error { + // read the current state + rawState, err := dump.Get(kongConfig.Client, dump.Config{ + SelectorTags: selectorTags, + }) + if err != nil { + return fmt.Errorf("loading configuration from kong: %w", err) + } + currentState, err := state.Get(rawState) + if err != nil { + return err + } + + // read the target state + rawState, err = file.Get(targetContent, file.RenderConfig{ + CurrentState: currentState, + KongVersion: kongConfig.Version, + }) + if err != nil { + return err + } + targetState, err := state.Get(rawState) + if err != nil { + return err + } + + syncer, err := diff.NewSyncer(currentState, targetState) + if err != nil { + return fmt.Errorf("creating a new syncer: %w", err) + } + syncer.SilenceWarnings = true + _, errs := solver.Solve(nil, syncer, kongConfig.Client, kongConfig.Concurrency, false) + if errs != nil { + return deckutils.ErrArray{Errors: errs} + } + return nil +} diff --git a/internal/ingress/controller/kong_test.go b/pkg/sendconfig/sendconfig_test.go similarity index 94% rename from internal/ingress/controller/kong_test.go rename to pkg/sendconfig/sendconfig_test.go index dadec14298..5758484619 100644 --- a/internal/ingress/controller/kong_test.go +++ b/pkg/sendconfig/sendconfig_test.go @@ -1,4 +1,4 @@ -package controller +package sendconfig import ( "reflect" @@ -104,9 +104,7 @@ func Test_renderConfigWithCustomEntities(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - var n KongController - n.Logger = logrus.New() - got, err := n.renderConfigWithCustomEntities(tt.args.state, tt.args.customEntitiesJSONBytes) + got, err := renderConfigWithCustomEntities(logrus.New(), tt.args.state, tt.args.customEntitiesJSONBytes) if (err != nil) != tt.wantErr { t.Errorf("renderConfigWithCustomEntities() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/pkg/util/plugin_schema_helper.go b/pkg/util/plugin_schema_helper.go new file mode 100644 index 0000000000..50971a754d --- /dev/null +++ b/pkg/util/plugin_schema_helper.go @@ -0,0 +1,53 @@ +package util + +import ( + "context" + "fmt" + + "github.com/kong/go-kong/kong" +) + +// FIXME +// decK will release this official API soon, use that and remove this code. + +// PluginSchemaStore retrives a schema of a Plugin from Kong. +type PluginSchemaStore struct { + client *kong.Client + schemas map[string]map[string]interface{} +} + +// NewPluginSchemaStore creates a PluginSchemaStore. +func NewPluginSchemaStore(client *kong.Client) *PluginSchemaStore { + return &PluginSchemaStore{ + client: client, + schemas: make(map[string]map[string]interface{}), + } +} + +// Schema retrives schema of a plugin. +// A cache is used to save the responses and subsequent queries are served from +// the cache. +func (p *PluginSchemaStore) Schema(ctx context.Context, pluginName string) (map[string]interface{}, error) { + if pluginName == "" { + return nil, fmt.Errorf("pluginName can not be empty") + } + + // lookup in cache + if schema, ok := p.schemas[pluginName]; ok { + return schema, nil + } + + // not present in cache, lookup + req, err := p.client.NewRequest("GET", "/plugins/schema/"+pluginName, + nil, nil) + if err != nil { + return nil, err + } + schema := make(map[string]interface{}) + _, err = p.client.Do(ctx, req, &schema) + if err != nil { + return nil, err + } + p.schemas[pluginName] = schema + return schema, nil +}