From 1f21ffa45b194b1440135c8d956b7bdec28fd733 Mon Sep 17 00:00:00 2001 From: Yi Tao Date: Thu, 12 Oct 2023 17:16:20 +0800 Subject: [PATCH 01/14] enable gateway discovery in db mode --- .../gateway_admin_service.yaml | 14 + .../multi-gw-postgres/gateway_deployment.yaml | 113 + .../gateway_service_patch.yaml | 8 + .../multi-gw-postgres/kustomization.yaml | 24 + .../manager_multi_gateway_patch.yaml | 17 + .../remove_proxy_container.yaml | 2 + ...all-in-one-postgres-multiple-gateways.yaml | 2365 +++++++++++++++++ internal/dataplane/kong_client.go | 45 +- internal/manager/run.go | 10 +- test/e2e/all_in_one_test.go | 20 + 10 files changed, 2605 insertions(+), 13 deletions(-) create mode 100644 config/variants/multi-gw-postgres/gateway_admin_service.yaml create mode 100644 config/variants/multi-gw-postgres/gateway_deployment.yaml create mode 100644 config/variants/multi-gw-postgres/gateway_service_patch.yaml create mode 100644 config/variants/multi-gw-postgres/kustomization.yaml create mode 100644 config/variants/multi-gw-postgres/manager_multi_gateway_patch.yaml create mode 100644 config/variants/multi-gw-postgres/remove_proxy_container.yaml create mode 100644 deploy/single/all-in-one-postgres-multiple-gateways.yaml diff --git a/config/variants/multi-gw-postgres/gateway_admin_service.yaml b/config/variants/multi-gw-postgres/gateway_admin_service.yaml new file mode 100644 index 0000000000..df62a158c8 --- /dev/null +++ b/config/variants/multi-gw-postgres/gateway_admin_service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: kong-admin + namespace: kong +spec: + clusterIP: "None" + selector: + app: proxy-kong + ports: + - name: admin + port: 8444 + targetPort: 8444 + protocol: TCP diff --git a/config/variants/multi-gw-postgres/gateway_deployment.yaml b/config/variants/multi-gw-postgres/gateway_deployment.yaml new file mode 100644 index 0000000000..be1f3d5383 --- /dev/null +++ b/config/variants/multi-gw-postgres/gateway_deployment.yaml @@ -0,0 +1,113 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: proxy-kong + name: proxy-kong + namespace: kong +spec: + replicas: 2 + selector: + matchLabels: + app: proxy-kong + template: + metadata: + annotations: + traffic.sidecar.istio.io/includeInboundPorts: "" + kuma.io/gateway: enabled + kuma.io/service-account-token-volume: kong-serviceaccount-token + labels: + app: proxy-kong + spec: + serviceAccountName: kong-serviceaccount + automountServiceAccountToken: false + volumes: + - name: kong-serviceaccount-token + projected: + sources: + - serviceAccountToken: + expirationSeconds: 3607 + path: token + - configMap: + name: kube-root-ca.crt + items: + - key: ca.crt + path: ca.crt + - downwardAPI: + items: + - fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + path: namespace + containers: + - name: proxy + image: kong-placeholder:placeholder # This is replaced by the config/image.yaml component + env: + # servers + - name: KONG_PROXY_LISTEN + value: 0.0.0.0:8000 reuseport backlog=16384, 0.0.0.0:8443 http2 ssl reuseport backlog=16384 + - name: KONG_PORT_MAPS + value: "80:8000, 443:8443" + - name: KONG_ADMIN_LISTEN + value: 0.0.0.0:8444 http2 ssl reuseport backlog=16384 + - name: KONG_STATUS_LISTEN + value: 0.0.0.0:8100 + # DB + - name: KONG_DATABASE + value: "postgres" + - name: KONG_PG_HOST + value: postgres + - name: KONG_PG_PASSWORD + value: kong + # runtime tweaks + - name: KONG_NGINX_WORKER_PROCESSES + value: "2" + - name: KONG_KIC + value: "on" + # logging + - name: KONG_ADMIN_ACCESS_LOG + value: /dev/stdout + - name: KONG_ADMIN_ERROR_LOG + value: /dev/stderr + # - name: KONG_PROXY_ACCESS_LOG + # - value: /dev/stdout + - name: KONG_PROXY_ERROR_LOG + value: /dev/stderr + # router mode in 3.0.0. use `traditional` here for full compatibility. + - name: KONG_ROUTER_FLAVOR + value: traditional + lifecycle: + preStop: + exec: + command: [ "/bin/bash", "-c", "kong quit" ] + ports: + - name: proxy + containerPort: 8000 + protocol: TCP + - name: proxy-ssl + containerPort: 8443 + protocol: TCP + - name: metrics + containerPort: 8100 + protocol: TCP + livenessProbe: + httpGet: + path: /status + port: 8100 + scheme: HTTP + initialDelaySeconds: 5 + timeoutSeconds: 1 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /status/ready + port: 8100 + scheme: HTTP + initialDelaySeconds: 5 + timeoutSeconds: 1 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 diff --git a/config/variants/multi-gw-postgres/gateway_service_patch.yaml b/config/variants/multi-gw-postgres/gateway_service_patch.yaml new file mode 100644 index 0000000000..9f2ec6b5a7 --- /dev/null +++ b/config/variants/multi-gw-postgres/gateway_service_patch.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Service +metadata: + name: kong-proxy + namespace: kong +spec: + selector: + app: proxy-kong diff --git a/config/variants/multi-gw-postgres/kustomization.yaml b/config/variants/multi-gw-postgres/kustomization.yaml new file mode 100644 index 0000000000..f89f98e0cc --- /dev/null +++ b/config/variants/multi-gw-postgres/kustomization.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: kong + +resources: +- ../postgres +- gateway_admin_service.yaml +- gateway_deployment.yaml + +components: + - ../../image/oss + +patchesStrategicMerge: +- manager_multi_gateway_patch.yaml +- gateway_service_patch.yaml + +patchesJson6902: +- target: + group: apps + version: v1 + kind: Deployment + name: ingress-kong + path: ./remove_proxy_container.yaml diff --git a/config/variants/multi-gw-postgres/manager_multi_gateway_patch.yaml b/config/variants/multi-gw-postgres/manager_multi_gateway_patch.yaml new file mode 100644 index 0000000000..49bc5dbd5b --- /dev/null +++ b/config/variants/multi-gw-postgres/manager_multi_gateway_patch.yaml @@ -0,0 +1,17 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: ingress-kong + name: ingress-kong + namespace: kong +spec: + template: + spec: + containers: + - name: ingress-controller + env: + - name: CONTROLLER_KONG_ADMIN_SVC + value: kong/kong-admin + - name: CONTROLLER_KONG_ADMIN_URL + $patch: delete diff --git a/config/variants/multi-gw-postgres/remove_proxy_container.yaml b/config/variants/multi-gw-postgres/remove_proxy_container.yaml new file mode 100644 index 0000000000..f9b75861c1 --- /dev/null +++ b/config/variants/multi-gw-postgres/remove_proxy_container.yaml @@ -0,0 +1,2 @@ +- op: remove + path: "/spec/template/spec/containers/1" diff --git a/deploy/single/all-in-one-postgres-multiple-gateways.yaml b/deploy/single/all-in-one-postgres-multiple-gateways.yaml new file mode 100644 index 0000000000..6ba9ee8340 --- /dev/null +++ b/deploy/single/all-in-one-postgres-multiple-gateways.yaml @@ -0,0 +1,2365 @@ +# Generated by build-single-manifest.sh. DO NOT EDIT. + +apiVersion: v1 +kind: Namespace +metadata: + name: kong +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: ingressclassparameterses.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + kind: IngressClassParameters + listKind: IngressClassParametersList + plural: ingressclassparameterses + singular: ingressclassparameters + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: IngressClassParameters is the Schema for the IngressClassParameters + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec is the IngressClassParameters specification. + properties: + enableLegacyRegexDetection: + default: false + description: EnableLegacyRegexDetection automatically detects if ImplementationSpecific + Ingress paths are regular expression paths using the legacy 2.x + heuristic. The controller adds the "~" prefix to those paths if + the Kong version is 3.0 or higher. + type: boolean + serviceUpstream: + default: false + description: Offload load-balancing to kube-proxy or sidecar. + type: boolean + type: object + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: kongclusterplugins.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: KongClusterPlugin + listKind: KongClusterPluginList + plural: kongclusterplugins + shortNames: + - kcp + singular: kongclusterplugin + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Name of the plugin + jsonPath: .plugin + name: Plugin-Type + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Indicates if the plugin is disabled + jsonPath: .disabled + name: Disabled + priority: 1 + type: boolean + - description: Configuration of the plugin + jsonPath: .config + name: Config + priority: 1 + type: string + - jsonPath: .status.conditions[?(@.type=="Programmed")].status + name: Programmed + type: string + name: v1 + schema: + openAPIV3Schema: + description: KongClusterPlugin is the Schema for the kongclusterplugins API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + config: + description: Config contains the plugin configuration. It's a list of + keys and values required to configure the plugin. Please read the documentation + of the plugin being configured to set values in here. For any plugin + in Kong, anything that goes in the `config` JSON key in the Admin API + request, goes into this property. Only one of `config` or `configFrom` + may be used in a KongClusterPlugin, not both at once. + type: object + x-kubernetes-preserve-unknown-fields: true + configFrom: + description: ConfigFrom references a secret containing the plugin configuration. + This should be used when the plugin configuration contains sensitive + information, such as AWS credentials in the Lambda plugin or the client + secret in the OIDC plugin. Only one of `config` or `configFrom` may + be used in a KongClusterPlugin, not both at once. + properties: + secretKeyRef: + description: Specifies a name, a namespace, and a key of a secret + to refer to. + properties: + key: + description: The key containing the value. + type: string + name: + description: The secret containing the key. + type: string + namespace: + description: The namespace containing the secret. + type: string + required: + - key + - name + - namespace + type: object + type: object + consumerRef: + description: ConsumerRef is a reference to a particular consumer. + type: string + disabled: + description: Disabled set if the plugin is disabled or not. + type: boolean + instance_name: + description: InstanceName is an optional custom name to identify an instance + of the plugin. This is useful when running the same plugin in multiple + contexts, for example, on multiple services. + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + ordering: + description: 'Ordering overrides the normal plugin execution order. It''s + only available on Kong Enterprise. `` is a request processing + phase (for example, `access` or `body_filter`) and `` is the + name of the plugin that will run before or after the KongPlugin. For + example, a KongPlugin with `plugin: rate-limiting` and `before.access: + ["key-auth"]` will create a rate limiting plugin that limits requests + _before_ they are authenticated.' + properties: + after: + additionalProperties: + items: + type: string + type: array + description: PluginOrderingPhase indicates which plugins in a phase + should affect the target plugin's order + type: object + before: + additionalProperties: + items: + type: string + type: array + description: PluginOrderingPhase indicates which plugins in a phase + should affect the target plugin's order + type: object + type: object + plugin: + description: PluginName is the name of the plugin to which to apply the + config. + type: string + protocols: + description: Protocols configures plugin to run on requests received on + specific protocols. + items: + description: KongProtocol is a valid Kong protocol. This alias is necessary + to deal with https://github.com/kubernetes-sigs/controller-tools/issues/342 + enum: + - http + - https + - grpc + - grpcs + - tcp + - tls + - udp + type: string + type: array + run_on: + description: RunOn configures the plugin to run on the first or the second + or both nodes in case of a service mesh deployment. + enum: + - first + - second + - all + type: string + status: + description: Status represents the current status of the KongClusterPlugin + resource. + properties: + conditions: + default: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + description: "Conditions describe the current conditions of the KongClusterPluginStatus. + \n Known condition types are: \n * \"Programmed\"" + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + \n type FooStatus struct{ // Represents the observations of a + foo's current state. // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge + // +listType=map // +listMapKey=type Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - plugin + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: kongconsumergroups.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: KongConsumerGroup + listKind: KongConsumerGroupList + plural: kongconsumergroups + shortNames: + - kcg + singular: kongconsumergroup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Programmed")].status + name: Programmed + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: KongConsumerGroup is the Schema for the kongconsumergroups API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + status: + description: Status represents the current status of the KongConsumer + resource. + properties: + conditions: + default: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + description: "Conditions describe the current conditions of the KongConsumerGroup. + \n Known condition types are: \n * \"Programmed\"" + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + \n type FooStatus struct{ // Represents the observations of a + foo's current state. // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge + // +listType=map // +listMapKey=type Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: kongconsumers.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: KongConsumer + listKind: KongConsumerList + plural: kongconsumers + shortNames: + - kc + singular: kongconsumer + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Username of a Kong Consumer + jsonPath: .username + name: Username + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Programmed")].status + name: Programmed + type: string + name: v1 + schema: + openAPIV3Schema: + description: KongConsumer is the Schema for the kongconsumers API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + consumerGroups: + description: ConsumerGroups are references to consumer groups (that consumer + wants to be part of) provisioned in Kong. + items: + type: string + type: array + credentials: + description: Credentials are references to secrets containing a credential + to be provisioned in Kong. + items: + type: string + type: array + custom_id: + description: CustomID is a Kong cluster-unique existing ID for the consumer + - useful for mapping Kong with users in your existing database. + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + status: + description: Status represents the current status of the KongConsumer + resource. + properties: + conditions: + default: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + description: "Conditions describe the current conditions of the KongConsumer. + \n Known condition types are: \n * \"Programmed\"" + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + \n type FooStatus struct{ // Represents the observations of a + foo's current state. // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge + // +listType=map // +listMapKey=type Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + username: + description: Username is a Kong cluster-unique username of the consumer. + type: string + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: kongingresses.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: KongIngress + listKind: KongIngressList + plural: kongingresses + shortNames: + - ki + singular: kongingress + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: KongIngress is the Schema for the kongingresses API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + proxy: + description: Proxy defines additional connection options for the routes + to be configured in the Kong Gateway, e.g. `connection_timeout`, `retries`, + etc. + properties: + connect_timeout: + description: "The timeout in milliseconds for\testablishing a connection + to the upstream server. Deprecated: use Service's \"konghq.com/connect-timeout\" + annotation instead." + minimum: 0 + type: integer + path: + description: '(optional) The path to be used in requests to the upstream + server. Deprecated: use Service''s "konghq.com/path" annotation + instead.' + pattern: ^/.*$ + type: string + protocol: + description: 'The protocol used to communicate with the upstream. + Deprecated: use Service''s "konghq.com/protocol" annotation instead.' + enum: + - http + - https + - grpc + - grpcs + - tcp + - tls + - udp + type: string + read_timeout: + description: 'The timeout in milliseconds between two successive read + operations for transmitting a request to the upstream server. Deprecated: + use Service''s "konghq.com/read-timeout" annotation instead.' + minimum: 0 + type: integer + retries: + description: 'The number of retries to execute upon failure to proxy. + Deprecated: use Service''s "konghq.com/retries" annotation instead.' + minimum: 0 + type: integer + write_timeout: + description: 'The timeout in milliseconds between two successive write + operations for transmitting a request to the upstream server. Deprecated: + use Service''s "konghq.com/write-timeout" annotation instead.' + minimum: 0 + type: integer + type: object + route: + description: Route define rules to match client requests. Each Route is + associated with a Service, and a Service may have multiple Routes associated + to it. + properties: + headers: + additionalProperties: + items: + type: string + type: array + description: 'Headers contains one or more lists of values indexed + by header name that will cause this Route to match if present in + the request. The Host header cannot be used with this attribute. + Deprecated: use Ingress'' "konghq.com/headers" annotation instead.' + type: object + https_redirect_status_code: + description: 'HTTPSRedirectStatusCode is the status code Kong responds + with when all properties of a Route match except the protocol. Deprecated: + use Ingress'' "ingress.kubernetes.io/force-ssl-redirect" or "konghq.com/https-redirect-status-code" + annotations instead.' + type: integer + methods: + description: 'Methods is a list of HTTP methods that match this Route. + Deprecated: use Ingress'' "konghq.com/methods" annotation instead.' + items: + type: string + type: array + path_handling: + description: 'PathHandling controls how the Service path, Route path + and requested path are combined when sending a request to the upstream. + Deprecated: use Ingress'' "konghq.com/path-handling" annotation + instead.' + enum: + - v0 + - v1 + type: string + preserve_host: + description: 'PreserveHost sets When matching a Route via one of the + hosts domain names, use the request Host header in the upstream + request headers. If set to false, the upstream Host header will + be that of the Service’s host. Deprecated: use Ingress'' "konghq.com/preserve-host" + annotation instead.' + type: boolean + protocols: + description: 'Protocols is an array of the protocols this Route should + allow. Deprecated: use Ingress'' "konghq.com/protocols" annotation + instead.' + items: + description: KongProtocol is a valid Kong protocol. This alias is + necessary to deal with https://github.com/kubernetes-sigs/controller-tools/issues/342 + enum: + - http + - https + - grpc + - grpcs + - tcp + - tls + - udp + type: string + type: array + regex_priority: + description: 'RegexPriority is a number used to choose which route + resolves a given request when several routes match it using regexes + simultaneously. Deprecated: use Ingress'' "konghq.com/regex-priority" + annotation instead.' + type: integer + request_buffering: + description: 'RequestBuffering sets whether to enable request body + buffering or not. Deprecated: use Ingress'' "konghq.com/request-buffering" + annotation instead.' + type: boolean + response_buffering: + description: 'ResponseBuffering sets whether to enable response body + buffering or not. Deprecated: use Ingress'' "konghq.com/response-buffering" + annotation instead.' + type: boolean + snis: + description: 'SNIs is a list of SNIs that match this Route when using + stream routing. Deprecated: use Ingress'' "konghq.com/snis" annotation + instead.' + items: + type: string + type: array + strip_path: + description: 'StripPath sets When matching a Route via one of the + paths strip the matching prefix from the upstream request URL. Deprecated: + use Ingress'' "konghq.com/strip-path" annotation instead.' + type: boolean + type: object + upstream: + description: Upstream represents a virtual hostname and can be used to + loadbalance incoming requests over multiple targets (e.g. Kubernetes + `Services` can be a target, OR `Endpoints` can be targets). + properties: + algorithm: + description: 'Algorithm is the load balancing algorithm to use. Accepted + values are: "round-robin", "consistent-hashing", "least-connections", + "latency".' + enum: + - round-robin + - consistent-hashing + - least-connections + - latency + type: string + hash_fallback: + description: 'HashFallback defines What to use as hashing input if + the primary hash_on does not return a hash. Accepted values are: + "none", "consumer", "ip", "header", "cookie".' + type: string + hash_fallback_header: + description: HashFallbackHeader is the header name to take the value + from as hash input. Only required when "hash_fallback" is set to + "header". + type: string + hash_fallback_query_arg: + description: HashFallbackQueryArg is the "hash_fallback" version of + HashOnQueryArg. + type: string + hash_fallback_uri_capture: + description: HashFallbackURICapture is the "hash_fallback" version + of HashOnURICapture. + type: string + hash_on: + description: 'HashOn defines what to use as hashing input. Accepted + values are: "none", "consumer", "ip", "header", "cookie", "path", + "query_arg", "uri_capture".' + type: string + hash_on_cookie: + description: The cookie name to take the value from as hash input. + Only required when "hash_on" or "hash_fallback" is set to "cookie". + type: string + hash_on_cookie_path: + description: The cookie path to set in the response headers. Only + required when "hash_on" or "hash_fallback" is set to "cookie". + type: string + hash_on_header: + description: HashOnHeader defines the header name to take the value + from as hash input. Only required when "hash_on" is set to "header". + type: string + hash_on_query_arg: + description: HashOnQueryArg is the query string parameter whose value + is the hash input when "hash_on" is set to "query_arg". + type: string + hash_on_uri_capture: + description: HashOnURICapture is the name of the capture group whose + value is the hash input when "hash_on" is set to "uri_capture". + type: string + healthchecks: + description: Healthchecks defines the health check configurations + in Kong. + properties: + active: + description: ActiveHealthcheck configures active health check + probing. + properties: + concurrency: + minimum: 1 + type: integer + headers: + additionalProperties: + items: + type: string + type: array + type: object + healthy: + description: Healthy configures thresholds and HTTP status + codes to mark targets healthy for an upstream. + properties: + http_statuses: + items: + type: integer + type: array + interval: + minimum: 0 + type: integer + successes: + minimum: 0 + type: integer + type: object + http_path: + pattern: ^/.*$ + type: string + https_sni: + type: string + https_verify_certificate: + type: boolean + timeout: + minimum: 0 + type: integer + type: + type: string + unhealthy: + description: Unhealthy configures thresholds and HTTP status + codes to mark targets unhealthy. + properties: + http_failures: + minimum: 0 + type: integer + http_statuses: + items: + type: integer + type: array + interval: + minimum: 0 + type: integer + tcp_failures: + minimum: 0 + type: integer + timeouts: + minimum: 0 + type: integer + type: object + type: object + passive: + description: PassiveHealthcheck configures passive checks around + passive health checks. + properties: + healthy: + description: Healthy configures thresholds and HTTP status + codes to mark targets healthy for an upstream. + properties: + http_statuses: + items: + type: integer + type: array + interval: + minimum: 0 + type: integer + successes: + minimum: 0 + type: integer + type: object + type: + type: string + unhealthy: + description: Unhealthy configures thresholds and HTTP status + codes to mark targets unhealthy. + properties: + http_failures: + minimum: 0 + type: integer + http_statuses: + items: + type: integer + type: array + interval: + minimum: 0 + type: integer + tcp_failures: + minimum: 0 + type: integer + timeouts: + minimum: 0 + type: integer + type: object + type: object + threshold: + type: number + type: object + host_header: + description: HostHeader is The hostname to be used as Host header + when proxying requests through Kong. + type: string + slots: + description: Slots is the number of slots in the load balancer algorithm. + minimum: 10 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: kongplugins.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: KongPlugin + listKind: KongPluginList + plural: kongplugins + shortNames: + - kp + singular: kongplugin + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Name of the plugin + jsonPath: .plugin + name: Plugin-Type + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Indicates if the plugin is disabled + jsonPath: .disabled + name: Disabled + priority: 1 + type: boolean + - description: Configuration of the plugin + jsonPath: .config + name: Config + priority: 1 + type: string + - jsonPath: .status.conditions[?(@.type=="Programmed")].status + name: Programmed + type: string + name: v1 + schema: + openAPIV3Schema: + description: KongPlugin is the Schema for the kongplugins API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + config: + description: Config contains the plugin configuration. It's a list of + keys and values required to configure the plugin. Please read the documentation + of the plugin being configured to set values in here. For any plugin + in Kong, anything that goes in the `config` JSON key in the Admin API + request, goes into this property. Only one of `config` or `configFrom` + may be used in a KongPlugin, not both at once. + type: object + x-kubernetes-preserve-unknown-fields: true + configFrom: + description: ConfigFrom references a secret containing the plugin configuration. + This should be used when the plugin configuration contains sensitive + information, such as AWS credentials in the Lambda plugin or the client + secret in the OIDC plugin. Only one of `config` or `configFrom` may + be used in a KongPlugin, not both at once. + properties: + secretKeyRef: + description: Specifies a name and a key of a secret to refer to. The + namespace is implicitly set to the one of referring object. + properties: + key: + description: The key containing the value. + type: string + name: + description: The secret containing the key. + type: string + required: + - key + - name + type: object + type: object + consumerRef: + description: ConsumerRef is a reference to a particular consumer. + type: string + disabled: + description: Disabled set if the plugin is disabled or not. + type: boolean + instance_name: + description: InstanceName is an optional custom name to identify an instance + of the plugin. This is useful when running the same plugin in multiple + contexts, for example, on multiple services. + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + ordering: + description: 'Ordering overrides the normal plugin execution order. It''s + only available on Kong Enterprise. `` is a request processing + phase (for example, `access` or `body_filter`) and `` is the + name of the plugin that will run before or after the KongPlugin. For + example, a KongPlugin with `plugin: rate-limiting` and `before.access: + ["key-auth"]` will create a rate limiting plugin that limits requests + _before_ they are authenticated.' + properties: + after: + additionalProperties: + items: + type: string + type: array + description: PluginOrderingPhase indicates which plugins in a phase + should affect the target plugin's order + type: object + before: + additionalProperties: + items: + type: string + type: array + description: PluginOrderingPhase indicates which plugins in a phase + should affect the target plugin's order + type: object + type: object + plugin: + description: PluginName is the name of the plugin to which to apply the + config. + type: string + protocols: + description: Protocols configures plugin to run on requests received on + specific protocols. + items: + description: KongProtocol is a valid Kong protocol. This alias is necessary + to deal with https://github.com/kubernetes-sigs/controller-tools/issues/342 + enum: + - http + - https + - grpc + - grpcs + - tcp + - tls + - udp + type: string + type: array + run_on: + description: RunOn configures the plugin to run on the first or the second + or both nodes in case of a service mesh deployment. + enum: + - first + - second + - all + type: string + status: + description: Status represents the current status of the KongPlugin resource. + properties: + conditions: + default: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + description: "Conditions describe the current conditions of the KongPluginStatus. + \n Known condition types are: \n * \"Programmed\"" + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + \n type FooStatus struct{ // Represents the observations of a + foo's current state. // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge + // +listType=map // +listMapKey=type Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - plugin + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: tcpingresses.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: TCPIngress + listKind: TCPIngressList + plural: tcpingresses + singular: tcpingress + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Address of the load balancer + jsonPath: .status.loadBalancer.ingress[*].ip + name: Address + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: TCPIngress is the Schema for the tcpingresses API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec is the TCPIngress specification. + properties: + rules: + description: A list of rules used to configure the Ingress. + items: + description: IngressRule represents a rule to apply against incoming + requests. Matching is performed based on an (optional) SNI and + port. + properties: + backend: + description: Backend defines the referenced service endpoint + to which the traffic will be forwarded to. + properties: + serviceName: + description: Specifies the name of the referenced service. + minLength: 1 + type: string + servicePort: + description: Specifies the port of the referenced service. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - serviceName + - servicePort + type: object + host: + description: Host is the fully qualified domain name of a network + host, as defined by RFC 3986. If a Host is not specified, + then port-based TCP routing is performed. Kong doesn't care + about the content of the TCP stream in this case. If a Host + is specified, the protocol must be TLS over TCP. A plain-text + TCP request cannot be routed based on Host. It can only be + routed based on Port. + type: string + port: + description: Port is the port on which to accept TCP or TLS + over TCP sessions and route. It is a required field. If a + Host is not specified, the requested are routed based only + on Port. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - backend + - port + type: object + type: array + tls: + description: TLS configuration. This is similar to the `tls` section + in the Ingress resource in networking.v1beta1 group. The mapping + of SNIs to TLS cert-key pair defined here will be used for HTTP + Ingress rules as well. Once can define the mapping in this resource + or the original Ingress resource, both have the same effect. + items: + description: IngressTLS describes the transport layer security. + properties: + hosts: + description: Hosts are a list of hosts included in the TLS certificate. + The values in this list must match the name/s used in the + tlsSecret. Defaults to the wildcard host setting for the loadbalancer + controller fulfilling this Ingress, if left unspecified. + items: + type: string + type: array + secretName: + description: SecretName is the name of the secret used to terminate + SSL traffic. + type: string + type: object + type: array + type: object + status: + description: TCPIngressStatus defines the observed state of TCPIngress. + properties: + loadBalancer: + description: LoadBalancer contains the current status of the load-balancer. + properties: + ingress: + description: Ingress is a list containing ingress points for the + load-balancer. Traffic intended for the service should be sent + to these ingress points. + items: + description: 'LoadBalancerIngress represents the status of a + load-balancer ingress point: traffic intended for the service + should be sent to an ingress point.' + properties: + hostname: + description: Hostname is set for load-balancer ingress points + that are DNS based (typically AWS load-balancers) + type: string + ip: + description: IP is set for load-balancer ingress points + that are IP based (typically GCE or OpenStack load-balancers) + type: string + ports: + description: Ports is a list of records of service ports + If used, every port defined in the service should have + an entry in it + items: + properties: + error: + description: 'Error is to record the problem with + the service port The format of the error shall comply + with the following rules: - built-in error values + shall be specified in this file and those shall + use CamelCase names - cloud provider specific error + values must have names that comply with the format + foo.example.com/CamelCase. --- The regex it matches + is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number of the service + port of which status is recorded here + format: int32 + type: integer + protocol: + default: TCP + description: 'Protocol is the protocol of the service + port of which status is recorded here The supported + values are: "TCP", "UDP", "SCTP"' + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: udpingresses.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: UDPIngress + listKind: UDPIngressList + plural: udpingresses + singular: udpingress + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Address of the load balancer + jsonPath: .status.loadBalancer.ingress[*].ip + name: Address + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: UDPIngress is the Schema for the udpingresses API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec is the UDPIngress specification. + properties: + rules: + description: A list of rules used to configure the Ingress. + items: + description: UDPIngressRule represents a rule to apply against incoming + requests wherein no Host matching is available for request routing, + only the port is used to match requests. + properties: + backend: + description: Backend defines the Kubernetes service which accepts + traffic from the listening Port defined above. + properties: + serviceName: + description: Specifies the name of the referenced service. + minLength: 1 + type: string + servicePort: + description: Specifies the port of the referenced service. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - serviceName + - servicePort + type: object + port: + description: Port indicates the port for the Kong proxy to accept + incoming traffic on, which will then be routed to the service + Backend. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - backend + - port + type: object + type: array + type: object + status: + description: UDPIngressStatus defines the observed state of UDPIngress. + properties: + loadBalancer: + description: LoadBalancer contains the current status of the load-balancer. + properties: + ingress: + description: Ingress is a list containing ingress points for the + load-balancer. Traffic intended for the service should be sent + to these ingress points. + items: + description: 'LoadBalancerIngress represents the status of a + load-balancer ingress point: traffic intended for the service + should be sent to an ingress point.' + properties: + hostname: + description: Hostname is set for load-balancer ingress points + that are DNS based (typically AWS load-balancers) + type: string + ip: + description: IP is set for load-balancer ingress points + that are IP based (typically GCE or OpenStack load-balancers) + type: string + ports: + description: Ports is a list of records of service ports + If used, every port defined in the service should have + an entry in it + items: + properties: + error: + description: 'Error is to record the problem with + the service port The format of the error shall comply + with the following rules: - built-in error values + shall be specified in this file and those shall + use CamelCase names - cloud provider specific error + values must have names that comply with the format + foo.example.com/CamelCase. --- The regex it matches + is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number of the service + port of which status is recorded here + format: int32 + type: integer + protocol: + default: TCP + description: 'Protocol is the protocol of the service + port of which status is recorded here The supported + values are: "TCP", "UDP", "SCTP"' + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kong-leader-election + namespace: kong +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kong-ingress +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services/status + verbs: + - get + - patch + - update +- apiGroups: + - configuration.konghq.com + resources: + - ingressclassparameterses + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongclusterplugins + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongclusterplugins/status + verbs: + - get + - patch + - update +- apiGroups: + - configuration.konghq.com + resources: + - kongconsumergroups + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongconsumergroups/status + verbs: + - get + - patch + - update +- apiGroups: + - configuration.konghq.com + resources: + - kongconsumers + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongconsumers/status + verbs: + - get + - patch + - update +- apiGroups: + - configuration.konghq.com + resources: + - kongingresses + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongingresses/status + verbs: + - get + - patch + - update +- apiGroups: + - configuration.konghq.com + resources: + - kongplugins + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongplugins/status + verbs: + - get + - patch + - update +- apiGroups: + - configuration.konghq.com + resources: + - tcpingresses + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - tcpingresses/status + verbs: + - get + - patch + - update +- apiGroups: + - configuration.konghq.com + resources: + - udpingresses + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - udpingresses/status + verbs: + - get + - patch + - update +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kong-ingress-gateway +rules: +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses/status + verbs: + - get + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - gateways + verbs: + - get + - list + - update + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gateways/status + verbs: + - get + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - grpcroutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - grpcroutes/status + verbs: + - get + - patch + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - httproutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - httproutes/status + verbs: + - get + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - referencegrants + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - referencegrants/status + verbs: + - get +- apiGroups: + - gateway.networking.k8s.io + resources: + - tcproutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - tcproutes/status + verbs: + - get + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - tlsroutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - tlsroutes/status + verbs: + - get + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - udproutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - udproutes/status + verbs: + - get + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kong-leader-election + namespace: kong +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kong-leader-election +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kong-ingress +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kong-ingress-gateway +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-gateway +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: v1 +kind: Service +metadata: + name: kong-admin + namespace: kong +spec: + clusterIP: None + ports: + - name: admin + port: 8444 + protocol: TCP + targetPort: 8444 + selector: + app: proxy-kong +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp + service.beta.kubernetes.io/aws-load-balancer-type: nlb + name: kong-proxy + namespace: kong +spec: + ports: + - name: proxy + port: 80 + protocol: TCP + targetPort: 8000 + - name: proxy-ssl + port: 443 + protocol: TCP + targetPort: 8443 + selector: + app: proxy-kong + type: LoadBalancer +--- +apiVersion: v1 +kind: Service +metadata: + name: kong-validation-webhook + namespace: kong +spec: + ports: + - name: webhook + port: 443 + protocol: TCP + targetPort: 8080 + selector: + app: ingress-kong +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: kong +spec: + ports: + - name: pgql + port: 5432 + protocol: TCP + targetPort: 5432 + selector: + app: postgres +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: ingress-kong + name: ingress-kong + namespace: kong +spec: + replicas: 1 + selector: + matchLabels: + app: ingress-kong + template: + metadata: + annotations: + kuma.io/gateway: enabled + kuma.io/service-account-token-volume: kong-serviceaccount-token + traffic.sidecar.istio.io/includeInboundPorts: "" + labels: + app: ingress-kong + spec: + automountServiceAccountToken: false + containers: + - env: + - name: CONTROLLER_KONG_ADMIN_SVC + value: kong/kong-admin + - name: CONTROLLER_KONG_ADMIN_TLS_SKIP_VERIFY + value: "true" + - name: CONTROLLER_PUBLISH_SERVICE + value: kong/kong-proxy + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + image: kong/kubernetes-ingress-controller:2.12 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: ingress-controller + ports: + - containerPort: 8080 + name: webhook + protocol: TCP + - containerPort: 10255 + name: cmetrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: 10254 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + volumeMounts: + - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + name: kong-serviceaccount-token + readOnly: true + initContainers: + - command: + - /bin/bash + - -c + - while true; do kong migrations list; if [[ 0 -eq $? ]]; then exit 0; fi; + sleep 2; done; + env: + - name: KONG_PG_HOST + value: postgres + - name: KONG_PG_PASSWORD + value: kong + image: kong:3.4 + name: wait-for-migrations + serviceAccountName: kong-serviceaccount + volumes: + - name: kong-serviceaccount-token + projected: + sources: + - serviceAccountToken: + expirationSeconds: 3607 + path: token + - configMap: + items: + - key: ca.crt + path: ca.crt + name: kube-root-ca.crt + - downwardAPI: + items: + - fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + path: namespace +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: proxy-kong + name: proxy-kong + namespace: kong +spec: + replicas: 2 + selector: + matchLabels: + app: proxy-kong + template: + metadata: + annotations: + kuma.io/gateway: enabled + kuma.io/service-account-token-volume: kong-serviceaccount-token + traffic.sidecar.istio.io/includeInboundPorts: "" + labels: + app: proxy-kong + spec: + automountServiceAccountToken: false + containers: + - env: + - name: KONG_PROXY_LISTEN + value: 0.0.0.0:8000 reuseport backlog=16384, 0.0.0.0:8443 http2 ssl reuseport + backlog=16384 + - name: KONG_PORT_MAPS + value: 80:8000, 443:8443 + - name: KONG_ADMIN_LISTEN + value: 0.0.0.0:8444 http2 ssl reuseport backlog=16384 + - name: KONG_STATUS_LISTEN + value: 0.0.0.0:8100 + - name: KONG_DATABASE + value: postgres + - name: KONG_PG_HOST + value: postgres + - name: KONG_PG_PASSWORD + value: kong + - name: KONG_NGINX_WORKER_PROCESSES + value: "2" + - name: KONG_KIC + value: "on" + - name: KONG_ADMIN_ACCESS_LOG + value: /dev/stdout + - name: KONG_ADMIN_ERROR_LOG + value: /dev/stderr + - name: KONG_PROXY_ERROR_LOG + value: /dev/stderr + - name: KONG_ROUTER_FLAVOR + value: traditional + image: kong:3.4 + lifecycle: + preStop: + exec: + command: + - /bin/bash + - -c + - kong quit + livenessProbe: + failureThreshold: 3 + httpGet: + path: /status + port: 8100 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: proxy + ports: + - containerPort: 8000 + name: proxy + protocol: TCP + - containerPort: 8443 + name: proxy-ssl + protocol: TCP + - containerPort: 8100 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /status/ready + port: 8100 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + serviceAccountName: kong-serviceaccount + volumes: + - name: kong-serviceaccount-token + projected: + sources: + - serviceAccountToken: + expirationSeconds: 3607 + path: token + - configMap: + items: + - key: ca.crt + path: ca.crt + name: kube-root-ca.crt + - downwardAPI: + items: + - fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + path: namespace +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres + namespace: kong +spec: + replicas: 1 + selector: + matchLabels: + app: postgres + serviceName: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - env: + - name: POSTGRES_USER + value: kong + - name: POSTGRES_PASSWORD + value: kong + - name: POSTGRES_DB + value: kong + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + image: postgres:9.5 + name: postgres + ports: + - containerPort: 5432 + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: datadir + subPath: pgdata + terminationGracePeriodSeconds: 60 + volumeClaimTemplates: + - metadata: + name: datadir + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: kong-migrations + namespace: kong +spec: + template: + metadata: + name: kong-migrations + spec: + containers: + - command: + - /bin/bash + - -c + - kong migrations bootstrap && kong migrations up && kong migrations finish + env: + - name: KONG_PG_PASSWORD + value: kong + - name: KONG_PG_HOST + value: postgres + - name: KONG_PG_PORT + value: "5432" + image: kong:3.4 + name: kong-migrations + initContainers: + - command: + - /bin/bash + - -c + - until timeout 1 bash 9<>/dev/tcp/${KONG_PG_HOST}/${KONG_PG_PORT}; do echo + 'waiting for db'; sleep 1; done + env: + - name: KONG_PG_HOST + value: postgres + - name: KONG_PG_PORT + value: "5432" + image: kong:3.4 + name: wait-for-postgres + restartPolicy: OnFailure +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: kong +spec: + controller: ingress-controllers.konghq.com/kong diff --git a/internal/dataplane/kong_client.go b/internal/dataplane/kong_client.go index 9b5cad625d..56b406aa2b 100644 --- a/internal/dataplane/kong_client.go +++ b/internal/dataplane/kong_client.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "math/rand" "reflect" "sort" "sync" @@ -454,17 +455,43 @@ func (c *KongClient) sendOutToGatewayClients( ctx context.Context, s *kongstate.KongState, config sendconfig.Config, ) ([]string, error) { gatewayClients := c.clientsProvider.GatewayClients() - c.logger.V(util.DebugLevel).Info("sending configuration to gateway clients", "count", len(gatewayClients)) - shas, err := iter.MapErr(gatewayClients, func(client **adminapi.Client) (string, error) { - return c.sendToClient(ctx, *client, s, config) - }) - if err != nil { - return nil, err - } previousSHAs := c.SHAs + gatewayClientURLs := []string{} + for _, cl := range gatewayClients { + gatewayClientURLs = append(gatewayClientURLs, cl.BaseRootURL()) + } - sort.Strings(shas) - c.SHAs = shas + if dataplaneutil.IsDBLessMode(c.dbmode) { + c.logger.V(util.DebugLevel).Info("sending configuration to gateway clients", "urls", gatewayClientURLs) + shas, err := iter.MapErr(gatewayClients, func(client **adminapi.Client) (string, error) { + return c.sendToClient(ctx, *client, s, config) + }) + if err != nil { + return nil, err + } + + sort.Strings(shas) + c.SHAs = shas + } else { + // directly return if no gateway clients found. + if len(gatewayClients) == 0 { + c.logger.Info("no active gateway clients found, skip sending") + return previousSHAs, nil + } + // randomly choose one client. + // REVIEW: The order of GatewayClients() is random, could we just use the "first" client? + clientIndex := rand.Intn(len(gatewayClients)) //nolint:gosec + c.logger.V(util.DebugLevel).Info( + fmt.Sprintf("sending configuration to gateway client %d/%d", clientIndex+1, len(gatewayClients)), + "chosen_url", gatewayClientURLs[clientIndex], + "urls", gatewayClientURLs, + ) + sha, err := c.sendToClient(ctx, gatewayClients[clientIndex], s, config) + if err != nil { + return nil, err + } + c.SHAs = []string{sha} + } c.kongConfigFetcher.StoreLastValidConfig(s) diff --git a/internal/manager/run.go b/internal/manager/run.go index fc1c521dcf..6c00c11bb0 100644 --- a/internal/manager/run.go +++ b/internal/manager/run.go @@ -111,10 +111,12 @@ func Run( routerFlavor := kongStartUpConfig.RouterFlavor v := kongStartUpConfig.Version - err = c.ValidateGatewayDiscovery(kongStartUpConfig.DBMode) - if err != nil { - return err - } + /* + err = c.ValidateGatewayDiscovery(kongStartUpConfig.DBMode) + if err != nil { + return err + } + */ kongSemVersion := semver.Version{Major: v.Major(), Minor: v.Minor(), Patch: v.Patch()} diff --git a/test/e2e/all_in_one_test.go b/test/e2e/all_in_one_test.go index f447b13e7a..5405125651 100644 --- a/test/e2e/all_in_one_test.go +++ b/test/e2e/all_in_one_test.go @@ -256,6 +256,26 @@ func TestDeployAllInOneEnterprisePostgres(t *testing.T) { verifyEnterpriseWithPostgres(ctx, t, env, adminPassword) } +func TestDeployAllInOnePostgresGatewayDiscovery(t *testing.T) { + t.Parallel() + + const ( + manifestFileName = "all-in-one-postgres-multiple-gateways.yaml" + manifestFilePath = "../../deploy/single/" + manifestFileName + ) + + t.Logf("configuring %s manifest test", manifestFileName) + ctx, env := setupE2ETest(t) + + t.Log("deploying kong components") + deployments := ManifestDeploy{Path: manifestFilePath}.Run(ctx, t, env) + + t.Log("running ingress tests to verify all-in-one deployed ingress controller and proxy are functional") + deployIngressWithEchoBackends(ctx, t, env, numberOfEchoBackends) + verifyIngressWithEchoBackends(ctx, t, env, numberOfEchoBackends) + ensureAllProxyReplicasAreConfigured(ctx, t, env, deployments.ProxyNN) +} + func TestDeployAllInOneDBLESS(t *testing.T) { t.Parallel() From 8feb584b29391c858638d79dea0c30d073bafbfb Mon Sep 17 00:00:00 2001 From: Yi Tao Date: Fri, 13 Oct 2023 15:15:20 +0800 Subject: [PATCH 02/14] remove ValidateGatewayDiscovery and update CHANGELOG --- CHANGELOG.md | 4 ++ internal/dataplane/kong_client.go | 2 + internal/manager/config_validation.go | 14 ------ internal/manager/config_validation_test.go | 50 ---------------------- internal/manager/run.go | 7 --- 5 files changed, 6 insertions(+), 71 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d05f77a09..ae7f8a40a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -236,6 +236,10 @@ Adding a new version? You'll need three changes: - KIC now specifies its `UserAgent` when communicating with kube-apiserver as `kong-ingress-controller/${VERSION}` where version is the version of KIC. [#5019](https://github.com/Kong/kubernetes-ingress-controller/pull/5019) +- Allow Gateway Discovery with database backed Kong. Currently KIC will send + Kong configurations to the admin API endpoints of one of the Kong pods behind + the Kong Admin Service specified by `--kong-admin-svc`. + [#4828](https://github.com/Kong/kubernetes-ingress-controller/pull/4828) [KIC Annotations reference]: https://docs.konghq.com/kubernetes-ingress-controller/latest/references/annotations/ [KIC CRDs reference]: https://docs.konghq.com/kubernetes-ingress-controller/latest/references/custom-resources/ diff --git a/internal/dataplane/kong_client.go b/internal/dataplane/kong_client.go index 56b406aa2b..91104837c8 100644 --- a/internal/dataplane/kong_client.go +++ b/internal/dataplane/kong_client.go @@ -480,6 +480,8 @@ func (c *KongClient) sendOutToGatewayClients( } // randomly choose one client. // REVIEW: The order of GatewayClients() is random, could we just use the "first" client? + // TODO: if multiple gateway instances uses different DBs, choose one client for each DB: + // https://github.com/Kong/kubernetes-ingress-controller/issues/4845 clientIndex := rand.Intn(len(gatewayClients)) //nolint:gosec c.logger.V(util.DebugLevel).Info( fmt.Sprintf("sending configuration to gateway client %d/%d", clientIndex+1, len(gatewayClients)), diff --git a/internal/manager/config_validation.go b/internal/manager/config_validation.go index abfcc500d8..62e45f7f17 100644 --- a/internal/manager/config_validation.go +++ b/internal/manager/config_validation.go @@ -11,7 +11,6 @@ import ( "github.com/kong/kubernetes-ingress-controller/v3/internal/adminapi" cfgtypes "github.com/kong/kubernetes-ingress-controller/v3/internal/manager/config/types" - dataplaneutil "github.com/kong/kubernetes-ingress-controller/v3/internal/util/dataplane" ) // https://github.com/kubernetes-sigs/gateway-api/blob/547122f7f55ac0464685552898c560658fb40073/apis/v1beta1/shared_types.go#L448-L463 @@ -127,16 +126,3 @@ func validateClientTLS(clientTLS adminapi.TLSClientConfig) error { return nil } - -// ValidateGatewayDiscovery returns error if dbMode is not configured to db-less mode. -// gateway discovery is only supported in db-less mode in its initial release: -// https://github.com/Kong/kubernetes-ingress-controller/issues/3401 -func (c *Config) ValidateGatewayDiscovery(dbMode string) error { - if c.KongAdminSvc.IsAbsent() { - return nil - } - if !dataplaneutil.IsDBLessMode(dbMode) { - return fmt.Errorf("gateway discovery is only supported in dbless mode, not db %s", dbMode) - } - return nil -} diff --git a/internal/manager/config_validation_test.go b/internal/manager/config_validation_test.go index 0cec19138a..c57224e535 100644 --- a/internal/manager/config_validation_test.go +++ b/internal/manager/config_validation_test.go @@ -285,53 +285,3 @@ func TestConfigValidate(t *testing.T) { }) }) } - -func TestConfigValidateGatewayDiscovery(t *testing.T) { - testCases := []struct { - name string - gatewayDiscovery bool - dbMode string - expectError bool - }{ - { - name: "gateway discovery disabled should pass in db-less mode", - gatewayDiscovery: false, - dbMode: "off", - expectError: false, - }, - { - name: "gateway discovery disabled should pass in db-backed mode", - gatewayDiscovery: false, - dbMode: "postgres", - expectError: false, - }, - { - name: "gateway discovery enabled should pass in db-less mode", - gatewayDiscovery: true, - dbMode: "", - expectError: false, - }, - { - name: "gateway discovery enabled should not pass in db-backed mode", - gatewayDiscovery: true, - dbMode: "postgres", - expectError: true, - }, - } - - for _, tc := range testCases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - c := &manager.Config{} - if tc.gatewayDiscovery { - c.KongAdminSvc = mo.Some(k8stypes.NamespacedName{Name: "admin-svc", Namespace: "ns"}) - } - err := c.ValidateGatewayDiscovery(tc.dbMode) - if !tc.expectError { - require.NoError(t, err) - } else { - require.Error(t, err) - } - }) - } -} diff --git a/internal/manager/run.go b/internal/manager/run.go index 6c00c11bb0..435023d8c8 100644 --- a/internal/manager/run.go +++ b/internal/manager/run.go @@ -111,13 +111,6 @@ func Run( routerFlavor := kongStartUpConfig.RouterFlavor v := kongStartUpConfig.Version - /* - err = c.ValidateGatewayDiscovery(kongStartUpConfig.DBMode) - if err != nil { - return err - } - */ - kongSemVersion := semver.Version{Major: v.Major(), Minor: v.Minor(), Patch: v.Patch()} kongConfig := sendconfig.Config{ From 5664a61a87e67033cd6ad8cee2cb17e14f0332d7 Mon Sep 17 00:00:00 2001 From: Yi Tao Date: Fri, 13 Oct 2023 16:33:24 +0800 Subject: [PATCH 03/14] update e2e --- test/e2e/helpers_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/helpers_test.go b/test/e2e/helpers_test.go index 79f9513211..83568b79e8 100644 --- a/test/e2e/helpers_test.go +++ b/test/e2e/helpers_test.go @@ -421,7 +421,7 @@ func getProxyDeploymentName(manifestPath string) string { return singlePodDeploymentName } // all non-legacy dbless manifests use a multi-pod deployment - if strings.Contains(manifestPath, "dbless") { + if strings.Contains(manifestPath, "dbless") || strings.Contains(manifestPath, "multiple-gateways") { return multiPodDeploymentName } From 5be2044939344e29e4c3d230b401efb671cdabe9 Mon Sep 17 00:00:00 2001 From: Yi Tao Date: Tue, 24 Oct 2023 15:31:26 +0800 Subject: [PATCH 04/14] client manager return one client when db backed --- internal/clients/manager.go | 67 ++++++++++++++++++++++++++++++- internal/clients/manager_test.go | 24 +++++++++-- internal/dataplane/kong_client.go | 47 +++++----------------- internal/manager/run.go | 2 + 4 files changed, 98 insertions(+), 42 deletions(-) diff --git a/internal/clients/manager.go b/internal/clients/manager.go index cabcd6fe9f..998bccf8d4 100644 --- a/internal/clients/manager.go +++ b/internal/clients/manager.go @@ -3,12 +3,14 @@ package clients import ( "context" "errors" + "fmt" "sync" "time" "github.com/go-logr/logr" "github.com/samber/lo" "golang.org/x/exp/maps" + k8stypes "k8s.io/apimachinery/pkg/types" "github.com/kong/kubernetes-ingress-controller/v3/internal/adminapi" "github.com/kong/kubernetes-ingress-controller/v3/internal/util" @@ -19,6 +21,8 @@ import ( // It's the same as the default interval of a Kubernetes container's readiness probe. const DefaultReadinessReconciliationInterval = 10 * time.Second +const DefaultAdminAPIServicePort = 8444 + // ClientFactory is responsible for creating Admin API clients. type ClientFactory interface { CreateAdminAPIClient(ctx context.Context, address adminapi.DiscoveredAdminAPI) (*adminapi.Client, error) @@ -47,6 +51,12 @@ type AdminAPIClientsManager struct { // endpoints list that should be used for configuring the dataplane. discoveredAdminAPIsNotifyChan chan []adminapi.DiscoveredAdminAPI gatewayClientsChangesSubscribers []chan struct{} + // adminAPIServiceName is the name of admin API service. + // When Kong gateway is running with DB, the manager will return the address of admin API service + // since KIC need only to send the requests to the admin API once with DB backed Kong gateway. + adminAPIServiceName k8stypes.NamespacedName + adminAPIServicePort int + dbMode string ctx context.Context onceNotifyLoopRunning sync.Once @@ -60,6 +70,9 @@ type AdminAPIClientsManager struct { // configured. pendingGatewayClients map[string]adminapi.DiscoveredAdminAPI + // clientFactory is used to create admin API client pointing to the admin API service. + clientFactory ClientFactory + // readinessChecker is used to check readiness of the clients. readinessChecker ReadinessChecker @@ -85,10 +98,30 @@ func WithReadinessReconciliationTicker(ticker Ticker) AdminAPIClientsManagerOpti } } +func WithAdminAPIServiceName(serviceNN k8stypes.NamespacedName) AdminAPIClientsManagerOption { + return func(m *AdminAPIClientsManager) { + m.adminAPIServiceName = serviceNN + } +} + +func WithAdminAPIServicePort(port int) AdminAPIClientsManagerOption { + return func(m *AdminAPIClientsManager) { + m.adminAPIServicePort = port + } +} + +// WithDBMode allows to set the DBMode of the Kong gateway instances behind the admin API service. +func WithDBMode(dbMode string) AdminAPIClientsManagerOption { + return func(m *AdminAPIClientsManager) { + m.dbMode = dbMode + } +} + func NewAdminAPIClientsManager( ctx context.Context, logger logr.Logger, initialClients []*adminapi.Client, + clientFactory ClientFactory, readinessChecker ReadinessChecker, opts ...AdminAPIClientsManagerOption, ) (*AdminAPIClientsManager, error) { @@ -102,6 +135,8 @@ func NewAdminAPIClientsManager( c := &AdminAPIClientsManager{ readyGatewayClients: readyClients, pendingGatewayClients: make(map[string]adminapi.DiscoveredAdminAPI), + clientFactory: clientFactory, + adminAPIServicePort: DefaultAdminAPIServicePort, readinessChecker: readinessChecker, readinessReconciliationTicker: clock.NewTicker(), discoveredAdminAPIsNotifyChan: make(chan []adminapi.DiscoveredAdminAPI), @@ -170,7 +205,37 @@ func (c *AdminAPIClientsManager) KonnectClient() *adminapi.KonnectClient { func (c *AdminAPIClientsManager) GatewayClients() []*adminapi.Client { c.lock.RLock() defer c.lock.RUnlock() - return lo.Values(c.readyGatewayClients) + readyGatewayClients := lo.Values(c.readyGatewayClients) + // With dbless mode, we should send configuration to ALL Kong gateway instances. + if dataplaneutil.IsDBLessMode(c.dbMode) { + return readyGatewayClients + } + // if there are no ready gateway clients, we should return an empty list. + if len(readyGatewayClients) == 0 { + return []*adminapi.Client{} + } + // return ONE client if Kong gateway is DB backed. + // if the name of admin API service is given, return the address of the service. + if c.adminAPIServiceName.Name != "" && c.adminAPIServiceName.Namespace != "" { + cl, err := c.clientFactory.CreateAdminAPIClient(c.ctx, adminapi.DiscoveredAdminAPI{ + Address: fmt.Sprintf("https://%s.%s:%d", + c.adminAPIServiceName.Name, + c.adminAPIServiceName.Namespace, + c.adminAPIServicePort, + ), + }) + if err != nil { + // Fallback to choose a single instance if we failed to create a client from admin API service. + c.logger.Error(err, "failed to create admin API client from admin API service address", + "service_name", c.adminAPIServiceName.String(), + "service_port", c.adminAPIServicePort, + ) + } else { + return []*adminapi.Client{cl} + } + } + // if no admin API service is given, choose one discovered client to send configurations. + return readyGatewayClients[:1] } func (c *AdminAPIClientsManager) GatewayClientsCount() int { diff --git a/internal/clients/manager_test.go b/internal/clients/manager_test.go index aa10363816..e8a7ff9e72 100644 --- a/internal/clients/manager_test.go +++ b/internal/clients/manager_test.go @@ -97,6 +97,7 @@ func TestAdminAPIClientsManager_OnNotifyClientsAreUpdatedAccordingly(t *testing. ctx, logger, []*adminapi.Client{initialClient}, + mocks.NewAdminAPIClientFactory(map[string]error{}), readinessChecker, ) require.NoError(t, err) @@ -158,7 +159,13 @@ func TestAdminAPIClientsManager_OnNotifyClientsAreUpdatedAccordingly(t *testing. } func TestNewAdminAPIClientsManager_NoInitialClientsDisallowed(t *testing.T) { - _, err := clients.NewAdminAPIClientsManager(context.Background(), zapr.NewLogger(zap.NewNop()), nil, &mockReadinessChecker{}) + _, err := clients.NewAdminAPIClientsManager( + context.Background(), + zapr.NewLogger(zap.NewNop()), + nil, + mocks.NewAdminAPIClientFactory(map[string]error{}), + &mockReadinessChecker{}, + ) require.ErrorContains(t, err, "at least one initial client must be provided") } @@ -171,6 +178,7 @@ func TestAdminAPIClientsManager_NotRunningNotifyLoop(t *testing.T) { context.Background(), zapr.NewLogger(zap.NewNop()), []*adminapi.Client{testClient}, + mocks.NewAdminAPIClientFactory(map[string]error{}), &mockReadinessChecker{}, ) require.NoError(t, err) @@ -191,6 +199,7 @@ func TestAdminAPIClientsManager_Clients(t *testing.T) { context.Background(), zapr.NewLogger(zap.NewNop()), []*adminapi.Client{testClient}, + mocks.NewAdminAPIClientFactory(map[string]error{}), &mockReadinessChecker{}, ) require.NoError(t, err) @@ -212,7 +221,13 @@ func TestAdminAPIClientsManager_SubscribeToGatewayClientsChanges(t *testing.T) { require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) - m, err := clients.NewAdminAPIClientsManager(ctx, zapr.NewLogger(zap.NewNop()), []*adminapi.Client{testClient}, readinessChecker) + m, err := clients.NewAdminAPIClientsManager( + ctx, + zapr.NewLogger(zap.NewNop()), + []*adminapi.Client{testClient}, + mocks.NewAdminAPIClientFactory(map[string]error{}), + readinessChecker) + require.NoError(t, err) t.Run("no notify loop running should return false when subscribing", func(t *testing.T) { @@ -297,7 +312,7 @@ func TestAdminAPIClientsManager_ConcurrentNotify(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - m, err := clients.NewAdminAPIClientsManager(ctx, zapr.NewLogger(zap.NewNop()), []*adminapi.Client{testClient}, readinessChecker) + m, err := clients.NewAdminAPIClientsManager(ctx, zapr.NewLogger(zap.NewNop()), []*adminapi.Client{testClient}, mocks.NewAdminAPIClientFactory(map[string]error{}), readinessChecker) require.NoError(t, err) m.Run() @@ -331,7 +346,7 @@ func TestAdminAPIClientsManager_GatewayClientsChanges(t *testing.T) { readinessChecker := &mockReadinessChecker{} ctx, cancel := context.WithCancel(context.Background()) defer cancel() - m, err := clients.NewAdminAPIClientsManager(ctx, zapr.NewLogger(zap.NewNop()), []*adminapi.Client{testClient}, readinessChecker) + m, err := clients.NewAdminAPIClientsManager(ctx, zapr.NewLogger(zap.NewNop()), []*adminapi.Client{testClient}, mocks.NewAdminAPIClientFactory(map[string]error{}), readinessChecker) require.NoError(t, err) m.Run() @@ -430,6 +445,7 @@ func TestAdminAPIClientsManager_PeriodicReadinessReconciliation(t *testing.T) { ctx, zapr.NewLogger(zap.NewNop()), []*adminapi.Client{testClient}, + mocks.NewAdminAPIClientFactory(map[string]error{}), readinessChecker, clients.WithReadinessReconciliationTicker(readinessTicker), ) diff --git a/internal/dataplane/kong_client.go b/internal/dataplane/kong_client.go index 91104837c8..60eb20a261 100644 --- a/internal/dataplane/kong_client.go +++ b/internal/dataplane/kong_client.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "math/rand" "reflect" "sort" "sync" @@ -456,45 +455,19 @@ func (c *KongClient) sendOutToGatewayClients( ) ([]string, error) { gatewayClients := c.clientsProvider.GatewayClients() previousSHAs := c.SHAs - gatewayClientURLs := []string{} - for _, cl := range gatewayClients { - gatewayClientURLs = append(gatewayClientURLs, cl.BaseRootURL()) - } - - if dataplaneutil.IsDBLessMode(c.dbmode) { - c.logger.V(util.DebugLevel).Info("sending configuration to gateway clients", "urls", gatewayClientURLs) - shas, err := iter.MapErr(gatewayClients, func(client **adminapi.Client) (string, error) { - return c.sendToClient(ctx, *client, s, config) - }) - if err != nil { - return nil, err - } + gatewayClientURLs := lo.Map(gatewayClients, func(cl *adminapi.Client, _ int) string { return cl.BaseRootURL() }) + c.logger.V(util.DebugLevel).Info("sending configuration to gateway clients", "urls", gatewayClientURLs) - sort.Strings(shas) - c.SHAs = shas - } else { - // directly return if no gateway clients found. - if len(gatewayClients) == 0 { - c.logger.Info("no active gateway clients found, skip sending") - return previousSHAs, nil - } - // randomly choose one client. - // REVIEW: The order of GatewayClients() is random, could we just use the "first" client? - // TODO: if multiple gateway instances uses different DBs, choose one client for each DB: - // https://github.com/Kong/kubernetes-ingress-controller/issues/4845 - clientIndex := rand.Intn(len(gatewayClients)) //nolint:gosec - c.logger.V(util.DebugLevel).Info( - fmt.Sprintf("sending configuration to gateway client %d/%d", clientIndex+1, len(gatewayClients)), - "chosen_url", gatewayClientURLs[clientIndex], - "urls", gatewayClientURLs, - ) - sha, err := c.sendToClient(ctx, gatewayClients[clientIndex], s, config) - if err != nil { - return nil, err - } - c.SHAs = []string{sha} + shas, err := iter.MapErr(gatewayClients, func(client **adminapi.Client) (string, error) { + return c.sendToClient(ctx, *client, s, config) + }) + if err != nil { + return nil, err } + sort.Strings(shas) + c.SHAs = shas + c.kongConfigFetcher.StoreLastValidConfig(s) return previousSHAs, nil diff --git a/internal/manager/run.go b/internal/manager/run.go index 435023d8c8..72ff8575e2 100644 --- a/internal/manager/run.go +++ b/internal/manager/run.go @@ -147,12 +147,14 @@ func Run( ctx, logger, initialKongClients, + adminAPIClientsFactory, readinessChecker, ) if err != nil { return fmt.Errorf("failed to create AdminAPIClientsManager: %w", err) } if c.KongAdminSvc.IsPresent() { + clients.WithAdminAPIServiceName(c.KongAdminSvc.MustGet())(clientsManager) setupLog.Info("Running AdminAPIClientsManager loop") clientsManager.Run() } From f4c7e64fb39368a73fec6adb5ef05e43ba1c0c2f Mon Sep 17 00:00:00 2001 From: Yi Tao Date: Tue, 24 Oct 2023 16:14:50 +0800 Subject: [PATCH 05/14] update CHANGELOG --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae7f8a40a3..ef5b5bf7ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -237,8 +237,8 @@ Adding a new version? You'll need three changes: as `kong-ingress-controller/${VERSION}` where version is the version of KIC. [#5019](https://github.com/Kong/kubernetes-ingress-controller/pull/5019) - Allow Gateway Discovery with database backed Kong. Currently KIC will send - Kong configurations to the admin API endpoints of one of the Kong pods behind - the Kong Admin Service specified by `--kong-admin-svc`. + Kong configurations to the address of Kong Admin Service specified by + `--kong-admin-svc`. [#4828](https://github.com/Kong/kubernetes-ingress-controller/pull/4828) [KIC Annotations reference]: https://docs.konghq.com/kubernetes-ingress-controller/latest/references/annotations/ From 9b9641f40ab284b4c0876bb0f7fe437fbb9fb250 Mon Sep 17 00:00:00 2001 From: Yi Tao Date: Fri, 27 Oct 2023 18:13:43 +0800 Subject: [PATCH 06/14] add logs when 0 gateway clients detected --- CHANGELOG.md | 6 ++-- internal/clients/manager.go | 48 ++++--------------------------- internal/dataplane/kong_client.go | 9 ++++++ internal/manager/run.go | 1 - 4 files changed, 17 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef5b5bf7ea..6c6079863c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -236,9 +236,9 @@ Adding a new version? You'll need three changes: - KIC now specifies its `UserAgent` when communicating with kube-apiserver as `kong-ingress-controller/${VERSION}` where version is the version of KIC. [#5019](https://github.com/Kong/kubernetes-ingress-controller/pull/5019) -- Allow Gateway Discovery with database backed Kong. Currently KIC will send - Kong configurations to the address of Kong Admin Service specified by - `--kong-admin-svc`. +- Allow Gateway Discovery with database backed Kong. KIC will send Kong + configurations to one of the backend pods of the service specified by the + flag `--kong-admin-svc` if Kong gateway is DB backed. [#4828](https://github.com/Kong/kubernetes-ingress-controller/pull/4828) [KIC Annotations reference]: https://docs.konghq.com/kubernetes-ingress-controller/latest/references/annotations/ diff --git a/internal/clients/manager.go b/internal/clients/manager.go index 998bccf8d4..7d7dc85b78 100644 --- a/internal/clients/manager.go +++ b/internal/clients/manager.go @@ -3,14 +3,12 @@ package clients import ( "context" "errors" - "fmt" "sync" "time" "github.com/go-logr/logr" "github.com/samber/lo" "golang.org/x/exp/maps" - k8stypes "k8s.io/apimachinery/pkg/types" "github.com/kong/kubernetes-ingress-controller/v3/internal/adminapi" "github.com/kong/kubernetes-ingress-controller/v3/internal/util" @@ -51,12 +49,8 @@ type AdminAPIClientsManager struct { // endpoints list that should be used for configuring the dataplane. discoveredAdminAPIsNotifyChan chan []adminapi.DiscoveredAdminAPI gatewayClientsChangesSubscribers []chan struct{} - // adminAPIServiceName is the name of admin API service. - // When Kong gateway is running with DB, the manager will return the address of admin API service - // since KIC need only to send the requests to the admin API once with DB backed Kong gateway. - adminAPIServiceName k8stypes.NamespacedName - adminAPIServicePort int - dbMode string + + dbMode string ctx context.Context onceNotifyLoopRunning sync.Once @@ -98,18 +92,6 @@ func WithReadinessReconciliationTicker(ticker Ticker) AdminAPIClientsManagerOpti } } -func WithAdminAPIServiceName(serviceNN k8stypes.NamespacedName) AdminAPIClientsManagerOption { - return func(m *AdminAPIClientsManager) { - m.adminAPIServiceName = serviceNN - } -} - -func WithAdminAPIServicePort(port int) AdminAPIClientsManagerOption { - return func(m *AdminAPIClientsManager) { - m.adminAPIServicePort = port - } -} - // WithDBMode allows to set the DBMode of the Kong gateway instances behind the admin API service. func WithDBMode(dbMode string) AdminAPIClientsManagerOption { return func(m *AdminAPIClientsManager) { @@ -136,7 +118,6 @@ func NewAdminAPIClientsManager( readyGatewayClients: readyClients, pendingGatewayClients: make(map[string]adminapi.DiscoveredAdminAPI), clientFactory: clientFactory, - adminAPIServicePort: DefaultAdminAPIServicePort, readinessChecker: readinessChecker, readinessReconciliationTicker: clock.NewTicker(), discoveredAdminAPIsNotifyChan: make(chan []adminapi.DiscoveredAdminAPI), @@ -210,31 +191,12 @@ func (c *AdminAPIClientsManager) GatewayClients() []*adminapi.Client { if dataplaneutil.IsDBLessMode(c.dbMode) { return readyGatewayClients } - // if there are no ready gateway clients, we should return an empty list. + // When Kong gateway is DB backed, we return a random admin API client + // since KIC only need to send requests to one instance. + // If there are no ready gateway clients, we return an empty list. if len(readyGatewayClients) == 0 { return []*adminapi.Client{} } - // return ONE client if Kong gateway is DB backed. - // if the name of admin API service is given, return the address of the service. - if c.adminAPIServiceName.Name != "" && c.adminAPIServiceName.Namespace != "" { - cl, err := c.clientFactory.CreateAdminAPIClient(c.ctx, adminapi.DiscoveredAdminAPI{ - Address: fmt.Sprintf("https://%s.%s:%d", - c.adminAPIServiceName.Name, - c.adminAPIServiceName.Namespace, - c.adminAPIServicePort, - ), - }) - if err != nil { - // Fallback to choose a single instance if we failed to create a client from admin API service. - c.logger.Error(err, "failed to create admin API client from admin API service address", - "service_name", c.adminAPIServiceName.String(), - "service_port", c.adminAPIServicePort, - ) - } else { - return []*adminapi.Client{cl} - } - } - // if no admin API service is given, choose one discovered client to send configurations. return readyGatewayClients[:1] } diff --git a/internal/dataplane/kong_client.go b/internal/dataplane/kong_client.go index 60eb20a261..ff3a296b6e 100644 --- a/internal/dataplane/kong_client.go +++ b/internal/dataplane/kong_client.go @@ -455,6 +455,15 @@ func (c *KongClient) sendOutToGatewayClients( ) ([]string, error) { gatewayClients := c.clientsProvider.GatewayClients() previousSHAs := c.SHAs + if len(gatewayClients) == 0 { + c.logger.Error( + errors.New("no ready gateway clients"), + "could not send configuration to gateways because no ready gateway clients", + ) + // Should not store the configuration in last valid config because the configuration is not validated on Kong gateway. + return previousSHAs, nil + } + gatewayClientURLs := lo.Map(gatewayClients, func(cl *adminapi.Client, _ int) string { return cl.BaseRootURL() }) c.logger.V(util.DebugLevel).Info("sending configuration to gateway clients", "urls", gatewayClientURLs) diff --git a/internal/manager/run.go b/internal/manager/run.go index 72ff8575e2..b8e1b161cd 100644 --- a/internal/manager/run.go +++ b/internal/manager/run.go @@ -154,7 +154,6 @@ func Run( return fmt.Errorf("failed to create AdminAPIClientsManager: %w", err) } if c.KongAdminSvc.IsPresent() { - clients.WithAdminAPIServiceName(c.KongAdminSvc.MustGet())(clientsManager) setupLog.Info("Running AdminAPIClientsManager loop") clientsManager.Run() } From 297b4b525fa45910a9e181f6f0557c500d065ece Mon Sep 17 00:00:00 2001 From: Yi Tao Date: Mon, 30 Oct 2023 17:56:50 +0800 Subject: [PATCH 07/14] remove manifests in deploy/single and address comments --- ...all-in-one-postgres-multiple-gateways.yaml | 2365 ----------------- internal/clients/manager.go | 2 - internal/clients/manager_test.go | 20 + internal/dataplane/kong_client.go | 4 +- internal/manager/run.go | 2 + test/e2e/all_in_one_test.go | 7 +- 6 files changed, 26 insertions(+), 2374 deletions(-) delete mode 100644 deploy/single/all-in-one-postgres-multiple-gateways.yaml diff --git a/deploy/single/all-in-one-postgres-multiple-gateways.yaml b/deploy/single/all-in-one-postgres-multiple-gateways.yaml deleted file mode 100644 index 6ba9ee8340..0000000000 --- a/deploy/single/all-in-one-postgres-multiple-gateways.yaml +++ /dev/null @@ -1,2365 +0,0 @@ -# Generated by build-single-manifest.sh. DO NOT EDIT. - -apiVersion: v1 -kind: Namespace -metadata: - name: kong ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.13.0 - name: ingressclassparameterses.configuration.konghq.com -spec: - group: configuration.konghq.com - names: - kind: IngressClassParameters - listKind: IngressClassParametersList - plural: ingressclassparameterses - singular: ingressclassparameters - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: IngressClassParameters is the Schema for the IngressClassParameters - API. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: Spec is the IngressClassParameters specification. - properties: - enableLegacyRegexDetection: - default: false - description: EnableLegacyRegexDetection automatically detects if ImplementationSpecific - Ingress paths are regular expression paths using the legacy 2.x - heuristic. The controller adds the "~" prefix to those paths if - the Kong version is 3.0 or higher. - type: boolean - serviceUpstream: - default: false - description: Offload load-balancing to kube-proxy or sidecar. - type: boolean - type: object - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.13.0 - name: kongclusterplugins.configuration.konghq.com -spec: - group: configuration.konghq.com - names: - categories: - - kong-ingress-controller - kind: KongClusterPlugin - listKind: KongClusterPluginList - plural: kongclusterplugins - shortNames: - - kcp - singular: kongclusterplugin - scope: Cluster - versions: - - additionalPrinterColumns: - - description: Name of the plugin - jsonPath: .plugin - name: Plugin-Type - type: string - - description: Age - jsonPath: .metadata.creationTimestamp - name: Age - type: date - - description: Indicates if the plugin is disabled - jsonPath: .disabled - name: Disabled - priority: 1 - type: boolean - - description: Configuration of the plugin - jsonPath: .config - name: Config - priority: 1 - type: string - - jsonPath: .status.conditions[?(@.type=="Programmed")].status - name: Programmed - type: string - name: v1 - schema: - openAPIV3Schema: - description: KongClusterPlugin is the Schema for the kongclusterplugins API. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - config: - description: Config contains the plugin configuration. It's a list of - keys and values required to configure the plugin. Please read the documentation - of the plugin being configured to set values in here. For any plugin - in Kong, anything that goes in the `config` JSON key in the Admin API - request, goes into this property. Only one of `config` or `configFrom` - may be used in a KongClusterPlugin, not both at once. - type: object - x-kubernetes-preserve-unknown-fields: true - configFrom: - description: ConfigFrom references a secret containing the plugin configuration. - This should be used when the plugin configuration contains sensitive - information, such as AWS credentials in the Lambda plugin or the client - secret in the OIDC plugin. Only one of `config` or `configFrom` may - be used in a KongClusterPlugin, not both at once. - properties: - secretKeyRef: - description: Specifies a name, a namespace, and a key of a secret - to refer to. - properties: - key: - description: The key containing the value. - type: string - name: - description: The secret containing the key. - type: string - namespace: - description: The namespace containing the secret. - type: string - required: - - key - - name - - namespace - type: object - type: object - consumerRef: - description: ConsumerRef is a reference to a particular consumer. - type: string - disabled: - description: Disabled set if the plugin is disabled or not. - type: boolean - instance_name: - description: InstanceName is an optional custom name to identify an instance - of the plugin. This is useful when running the same plugin in multiple - contexts, for example, on multiple services. - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - ordering: - description: 'Ordering overrides the normal plugin execution order. It''s - only available on Kong Enterprise. `` is a request processing - phase (for example, `access` or `body_filter`) and `` is the - name of the plugin that will run before or after the KongPlugin. For - example, a KongPlugin with `plugin: rate-limiting` and `before.access: - ["key-auth"]` will create a rate limiting plugin that limits requests - _before_ they are authenticated.' - properties: - after: - additionalProperties: - items: - type: string - type: array - description: PluginOrderingPhase indicates which plugins in a phase - should affect the target plugin's order - type: object - before: - additionalProperties: - items: - type: string - type: array - description: PluginOrderingPhase indicates which plugins in a phase - should affect the target plugin's order - type: object - type: object - plugin: - description: PluginName is the name of the plugin to which to apply the - config. - type: string - protocols: - description: Protocols configures plugin to run on requests received on - specific protocols. - items: - description: KongProtocol is a valid Kong protocol. This alias is necessary - to deal with https://github.com/kubernetes-sigs/controller-tools/issues/342 - enum: - - http - - https - - grpc - - grpcs - - tcp - - tls - - udp - type: string - type: array - run_on: - description: RunOn configures the plugin to run on the first or the second - or both nodes in case of a service mesh deployment. - enum: - - first - - second - - all - type: string - status: - description: Status represents the current status of the KongClusterPlugin - resource. - properties: - conditions: - default: - - lastTransitionTime: "1970-01-01T00:00:00Z" - message: Waiting for controller - reason: Pending - status: Unknown - type: Programmed - description: "Conditions describe the current conditions of the KongClusterPluginStatus. - \n Known condition types are: \n * \"Programmed\"" - items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - \n type FooStatus struct{ // Represents the observations of a - foo's current state. // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge - // +listType=map // +listMapKey=type Conditions []metav1.Condition - `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" - protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: message is a human readable message indicating - details about the transition. This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers - of specific condition types may define expected values and - meanings for this field, and whether the values are considered - a guaranteed API. The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - type: object - required: - - plugin - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.13.0 - name: kongconsumergroups.configuration.konghq.com -spec: - group: configuration.konghq.com - names: - categories: - - kong-ingress-controller - kind: KongConsumerGroup - listKind: KongConsumerGroupList - plural: kongconsumergroups - shortNames: - - kcg - singular: kongconsumergroup - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Age - jsonPath: .metadata.creationTimestamp - name: Age - type: date - - jsonPath: .status.conditions[?(@.type=="Programmed")].status - name: Programmed - type: string - name: v1beta1 - schema: - openAPIV3Schema: - description: KongConsumerGroup is the Schema for the kongconsumergroups API. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - status: - description: Status represents the current status of the KongConsumer - resource. - properties: - conditions: - default: - - lastTransitionTime: "1970-01-01T00:00:00Z" - message: Waiting for controller - reason: Pending - status: Unknown - type: Programmed - description: "Conditions describe the current conditions of the KongConsumerGroup. - \n Known condition types are: \n * \"Programmed\"" - items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - \n type FooStatus struct{ // Represents the observations of a - foo's current state. // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge - // +listType=map // +listMapKey=type Conditions []metav1.Condition - `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" - protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: message is a human readable message indicating - details about the transition. This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers - of specific condition types may define expected values and - meanings for this field, and whether the values are considered - a guaranteed API. The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.13.0 - name: kongconsumers.configuration.konghq.com -spec: - group: configuration.konghq.com - names: - categories: - - kong-ingress-controller - kind: KongConsumer - listKind: KongConsumerList - plural: kongconsumers - shortNames: - - kc - singular: kongconsumer - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Username of a Kong Consumer - jsonPath: .username - name: Username - type: string - - description: Age - jsonPath: .metadata.creationTimestamp - name: Age - type: date - - jsonPath: .status.conditions[?(@.type=="Programmed")].status - name: Programmed - type: string - name: v1 - schema: - openAPIV3Schema: - description: KongConsumer is the Schema for the kongconsumers API. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - consumerGroups: - description: ConsumerGroups are references to consumer groups (that consumer - wants to be part of) provisioned in Kong. - items: - type: string - type: array - credentials: - description: Credentials are references to secrets containing a credential - to be provisioned in Kong. - items: - type: string - type: array - custom_id: - description: CustomID is a Kong cluster-unique existing ID for the consumer - - useful for mapping Kong with users in your existing database. - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - status: - description: Status represents the current status of the KongConsumer - resource. - properties: - conditions: - default: - - lastTransitionTime: "1970-01-01T00:00:00Z" - message: Waiting for controller - reason: Pending - status: Unknown - type: Programmed - description: "Conditions describe the current conditions of the KongConsumer. - \n Known condition types are: \n * \"Programmed\"" - items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - \n type FooStatus struct{ // Represents the observations of a - foo's current state. // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge - // +listType=map // +listMapKey=type Conditions []metav1.Condition - `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" - protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: message is a human readable message indicating - details about the transition. This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers - of specific condition types may define expected values and - meanings for this field, and whether the values are considered - a guaranteed API. The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - type: object - username: - description: Username is a Kong cluster-unique username of the consumer. - type: string - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.13.0 - name: kongingresses.configuration.konghq.com -spec: - group: configuration.konghq.com - names: - categories: - - kong-ingress-controller - kind: KongIngress - listKind: KongIngressList - plural: kongingresses - shortNames: - - ki - singular: kongingress - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - description: KongIngress is the Schema for the kongingresses API. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - proxy: - description: Proxy defines additional connection options for the routes - to be configured in the Kong Gateway, e.g. `connection_timeout`, `retries`, - etc. - properties: - connect_timeout: - description: "The timeout in milliseconds for\testablishing a connection - to the upstream server. Deprecated: use Service's \"konghq.com/connect-timeout\" - annotation instead." - minimum: 0 - type: integer - path: - description: '(optional) The path to be used in requests to the upstream - server. Deprecated: use Service''s "konghq.com/path" annotation - instead.' - pattern: ^/.*$ - type: string - protocol: - description: 'The protocol used to communicate with the upstream. - Deprecated: use Service''s "konghq.com/protocol" annotation instead.' - enum: - - http - - https - - grpc - - grpcs - - tcp - - tls - - udp - type: string - read_timeout: - description: 'The timeout in milliseconds between two successive read - operations for transmitting a request to the upstream server. Deprecated: - use Service''s "konghq.com/read-timeout" annotation instead.' - minimum: 0 - type: integer - retries: - description: 'The number of retries to execute upon failure to proxy. - Deprecated: use Service''s "konghq.com/retries" annotation instead.' - minimum: 0 - type: integer - write_timeout: - description: 'The timeout in milliseconds between two successive write - operations for transmitting a request to the upstream server. Deprecated: - use Service''s "konghq.com/write-timeout" annotation instead.' - minimum: 0 - type: integer - type: object - route: - description: Route define rules to match client requests. Each Route is - associated with a Service, and a Service may have multiple Routes associated - to it. - properties: - headers: - additionalProperties: - items: - type: string - type: array - description: 'Headers contains one or more lists of values indexed - by header name that will cause this Route to match if present in - the request. The Host header cannot be used with this attribute. - Deprecated: use Ingress'' "konghq.com/headers" annotation instead.' - type: object - https_redirect_status_code: - description: 'HTTPSRedirectStatusCode is the status code Kong responds - with when all properties of a Route match except the protocol. Deprecated: - use Ingress'' "ingress.kubernetes.io/force-ssl-redirect" or "konghq.com/https-redirect-status-code" - annotations instead.' - type: integer - methods: - description: 'Methods is a list of HTTP methods that match this Route. - Deprecated: use Ingress'' "konghq.com/methods" annotation instead.' - items: - type: string - type: array - path_handling: - description: 'PathHandling controls how the Service path, Route path - and requested path are combined when sending a request to the upstream. - Deprecated: use Ingress'' "konghq.com/path-handling" annotation - instead.' - enum: - - v0 - - v1 - type: string - preserve_host: - description: 'PreserveHost sets When matching a Route via one of the - hosts domain names, use the request Host header in the upstream - request headers. If set to false, the upstream Host header will - be that of the Service’s host. Deprecated: use Ingress'' "konghq.com/preserve-host" - annotation instead.' - type: boolean - protocols: - description: 'Protocols is an array of the protocols this Route should - allow. Deprecated: use Ingress'' "konghq.com/protocols" annotation - instead.' - items: - description: KongProtocol is a valid Kong protocol. This alias is - necessary to deal with https://github.com/kubernetes-sigs/controller-tools/issues/342 - enum: - - http - - https - - grpc - - grpcs - - tcp - - tls - - udp - type: string - type: array - regex_priority: - description: 'RegexPriority is a number used to choose which route - resolves a given request when several routes match it using regexes - simultaneously. Deprecated: use Ingress'' "konghq.com/regex-priority" - annotation instead.' - type: integer - request_buffering: - description: 'RequestBuffering sets whether to enable request body - buffering or not. Deprecated: use Ingress'' "konghq.com/request-buffering" - annotation instead.' - type: boolean - response_buffering: - description: 'ResponseBuffering sets whether to enable response body - buffering or not. Deprecated: use Ingress'' "konghq.com/response-buffering" - annotation instead.' - type: boolean - snis: - description: 'SNIs is a list of SNIs that match this Route when using - stream routing. Deprecated: use Ingress'' "konghq.com/snis" annotation - instead.' - items: - type: string - type: array - strip_path: - description: 'StripPath sets When matching a Route via one of the - paths strip the matching prefix from the upstream request URL. Deprecated: - use Ingress'' "konghq.com/strip-path" annotation instead.' - type: boolean - type: object - upstream: - description: Upstream represents a virtual hostname and can be used to - loadbalance incoming requests over multiple targets (e.g. Kubernetes - `Services` can be a target, OR `Endpoints` can be targets). - properties: - algorithm: - description: 'Algorithm is the load balancing algorithm to use. Accepted - values are: "round-robin", "consistent-hashing", "least-connections", - "latency".' - enum: - - round-robin - - consistent-hashing - - least-connections - - latency - type: string - hash_fallback: - description: 'HashFallback defines What to use as hashing input if - the primary hash_on does not return a hash. Accepted values are: - "none", "consumer", "ip", "header", "cookie".' - type: string - hash_fallback_header: - description: HashFallbackHeader is the header name to take the value - from as hash input. Only required when "hash_fallback" is set to - "header". - type: string - hash_fallback_query_arg: - description: HashFallbackQueryArg is the "hash_fallback" version of - HashOnQueryArg. - type: string - hash_fallback_uri_capture: - description: HashFallbackURICapture is the "hash_fallback" version - of HashOnURICapture. - type: string - hash_on: - description: 'HashOn defines what to use as hashing input. Accepted - values are: "none", "consumer", "ip", "header", "cookie", "path", - "query_arg", "uri_capture".' - type: string - hash_on_cookie: - description: The cookie name to take the value from as hash input. - Only required when "hash_on" or "hash_fallback" is set to "cookie". - type: string - hash_on_cookie_path: - description: The cookie path to set in the response headers. Only - required when "hash_on" or "hash_fallback" is set to "cookie". - type: string - hash_on_header: - description: HashOnHeader defines the header name to take the value - from as hash input. Only required when "hash_on" is set to "header". - type: string - hash_on_query_arg: - description: HashOnQueryArg is the query string parameter whose value - is the hash input when "hash_on" is set to "query_arg". - type: string - hash_on_uri_capture: - description: HashOnURICapture is the name of the capture group whose - value is the hash input when "hash_on" is set to "uri_capture". - type: string - healthchecks: - description: Healthchecks defines the health check configurations - in Kong. - properties: - active: - description: ActiveHealthcheck configures active health check - probing. - properties: - concurrency: - minimum: 1 - type: integer - headers: - additionalProperties: - items: - type: string - type: array - type: object - healthy: - description: Healthy configures thresholds and HTTP status - codes to mark targets healthy for an upstream. - properties: - http_statuses: - items: - type: integer - type: array - interval: - minimum: 0 - type: integer - successes: - minimum: 0 - type: integer - type: object - http_path: - pattern: ^/.*$ - type: string - https_sni: - type: string - https_verify_certificate: - type: boolean - timeout: - minimum: 0 - type: integer - type: - type: string - unhealthy: - description: Unhealthy configures thresholds and HTTP status - codes to mark targets unhealthy. - properties: - http_failures: - minimum: 0 - type: integer - http_statuses: - items: - type: integer - type: array - interval: - minimum: 0 - type: integer - tcp_failures: - minimum: 0 - type: integer - timeouts: - minimum: 0 - type: integer - type: object - type: object - passive: - description: PassiveHealthcheck configures passive checks around - passive health checks. - properties: - healthy: - description: Healthy configures thresholds and HTTP status - codes to mark targets healthy for an upstream. - properties: - http_statuses: - items: - type: integer - type: array - interval: - minimum: 0 - type: integer - successes: - minimum: 0 - type: integer - type: object - type: - type: string - unhealthy: - description: Unhealthy configures thresholds and HTTP status - codes to mark targets unhealthy. - properties: - http_failures: - minimum: 0 - type: integer - http_statuses: - items: - type: integer - type: array - interval: - minimum: 0 - type: integer - tcp_failures: - minimum: 0 - type: integer - timeouts: - minimum: 0 - type: integer - type: object - type: object - threshold: - type: number - type: object - host_header: - description: HostHeader is The hostname to be used as Host header - when proxying requests through Kong. - type: string - slots: - description: Slots is the number of slots in the load balancer algorithm. - minimum: 10 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.13.0 - name: kongplugins.configuration.konghq.com -spec: - group: configuration.konghq.com - names: - categories: - - kong-ingress-controller - kind: KongPlugin - listKind: KongPluginList - plural: kongplugins - shortNames: - - kp - singular: kongplugin - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Name of the plugin - jsonPath: .plugin - name: Plugin-Type - type: string - - description: Age - jsonPath: .metadata.creationTimestamp - name: Age - type: date - - description: Indicates if the plugin is disabled - jsonPath: .disabled - name: Disabled - priority: 1 - type: boolean - - description: Configuration of the plugin - jsonPath: .config - name: Config - priority: 1 - type: string - - jsonPath: .status.conditions[?(@.type=="Programmed")].status - name: Programmed - type: string - name: v1 - schema: - openAPIV3Schema: - description: KongPlugin is the Schema for the kongplugins API. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - config: - description: Config contains the plugin configuration. It's a list of - keys and values required to configure the plugin. Please read the documentation - of the plugin being configured to set values in here. For any plugin - in Kong, anything that goes in the `config` JSON key in the Admin API - request, goes into this property. Only one of `config` or `configFrom` - may be used in a KongPlugin, not both at once. - type: object - x-kubernetes-preserve-unknown-fields: true - configFrom: - description: ConfigFrom references a secret containing the plugin configuration. - This should be used when the plugin configuration contains sensitive - information, such as AWS credentials in the Lambda plugin or the client - secret in the OIDC plugin. Only one of `config` or `configFrom` may - be used in a KongPlugin, not both at once. - properties: - secretKeyRef: - description: Specifies a name and a key of a secret to refer to. The - namespace is implicitly set to the one of referring object. - properties: - key: - description: The key containing the value. - type: string - name: - description: The secret containing the key. - type: string - required: - - key - - name - type: object - type: object - consumerRef: - description: ConsumerRef is a reference to a particular consumer. - type: string - disabled: - description: Disabled set if the plugin is disabled or not. - type: boolean - instance_name: - description: InstanceName is an optional custom name to identify an instance - of the plugin. This is useful when running the same plugin in multiple - contexts, for example, on multiple services. - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - ordering: - description: 'Ordering overrides the normal plugin execution order. It''s - only available on Kong Enterprise. `` is a request processing - phase (for example, `access` or `body_filter`) and `` is the - name of the plugin that will run before or after the KongPlugin. For - example, a KongPlugin with `plugin: rate-limiting` and `before.access: - ["key-auth"]` will create a rate limiting plugin that limits requests - _before_ they are authenticated.' - properties: - after: - additionalProperties: - items: - type: string - type: array - description: PluginOrderingPhase indicates which plugins in a phase - should affect the target plugin's order - type: object - before: - additionalProperties: - items: - type: string - type: array - description: PluginOrderingPhase indicates which plugins in a phase - should affect the target plugin's order - type: object - type: object - plugin: - description: PluginName is the name of the plugin to which to apply the - config. - type: string - protocols: - description: Protocols configures plugin to run on requests received on - specific protocols. - items: - description: KongProtocol is a valid Kong protocol. This alias is necessary - to deal with https://github.com/kubernetes-sigs/controller-tools/issues/342 - enum: - - http - - https - - grpc - - grpcs - - tcp - - tls - - udp - type: string - type: array - run_on: - description: RunOn configures the plugin to run on the first or the second - or both nodes in case of a service mesh deployment. - enum: - - first - - second - - all - type: string - status: - description: Status represents the current status of the KongPlugin resource. - properties: - conditions: - default: - - lastTransitionTime: "1970-01-01T00:00:00Z" - message: Waiting for controller - reason: Pending - status: Unknown - type: Programmed - description: "Conditions describe the current conditions of the KongPluginStatus. - \n Known condition types are: \n * \"Programmed\"" - items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - \n type FooStatus struct{ // Represents the observations of a - foo's current state. // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge - // +listType=map // +listMapKey=type Conditions []metav1.Condition - `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" - protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: message is a human readable message indicating - details about the transition. This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers - of specific condition types may define expected values and - meanings for this field, and whether the values are considered - a guaranteed API. The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - type: object - required: - - plugin - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.13.0 - name: tcpingresses.configuration.konghq.com -spec: - group: configuration.konghq.com - names: - categories: - - kong-ingress-controller - kind: TCPIngress - listKind: TCPIngressList - plural: tcpingresses - singular: tcpingress - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Address of the load balancer - jsonPath: .status.loadBalancer.ingress[*].ip - name: Address - type: string - - description: Age - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: TCPIngress is the Schema for the tcpingresses API. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: Spec is the TCPIngress specification. - properties: - rules: - description: A list of rules used to configure the Ingress. - items: - description: IngressRule represents a rule to apply against incoming - requests. Matching is performed based on an (optional) SNI and - port. - properties: - backend: - description: Backend defines the referenced service endpoint - to which the traffic will be forwarded to. - properties: - serviceName: - description: Specifies the name of the referenced service. - minLength: 1 - type: string - servicePort: - description: Specifies the port of the referenced service. - format: int32 - maximum: 65535 - minimum: 1 - type: integer - required: - - serviceName - - servicePort - type: object - host: - description: Host is the fully qualified domain name of a network - host, as defined by RFC 3986. If a Host is not specified, - then port-based TCP routing is performed. Kong doesn't care - about the content of the TCP stream in this case. If a Host - is specified, the protocol must be TLS over TCP. A plain-text - TCP request cannot be routed based on Host. It can only be - routed based on Port. - type: string - port: - description: Port is the port on which to accept TCP or TLS - over TCP sessions and route. It is a required field. If a - Host is not specified, the requested are routed based only - on Port. - format: int32 - maximum: 65535 - minimum: 1 - type: integer - required: - - backend - - port - type: object - type: array - tls: - description: TLS configuration. This is similar to the `tls` section - in the Ingress resource in networking.v1beta1 group. The mapping - of SNIs to TLS cert-key pair defined here will be used for HTTP - Ingress rules as well. Once can define the mapping in this resource - or the original Ingress resource, both have the same effect. - items: - description: IngressTLS describes the transport layer security. - properties: - hosts: - description: Hosts are a list of hosts included in the TLS certificate. - The values in this list must match the name/s used in the - tlsSecret. Defaults to the wildcard host setting for the loadbalancer - controller fulfilling this Ingress, if left unspecified. - items: - type: string - type: array - secretName: - description: SecretName is the name of the secret used to terminate - SSL traffic. - type: string - type: object - type: array - type: object - status: - description: TCPIngressStatus defines the observed state of TCPIngress. - properties: - loadBalancer: - description: LoadBalancer contains the current status of the load-balancer. - properties: - ingress: - description: Ingress is a list containing ingress points for the - load-balancer. Traffic intended for the service should be sent - to these ingress points. - items: - description: 'LoadBalancerIngress represents the status of a - load-balancer ingress point: traffic intended for the service - should be sent to an ingress point.' - properties: - hostname: - description: Hostname is set for load-balancer ingress points - that are DNS based (typically AWS load-balancers) - type: string - ip: - description: IP is set for load-balancer ingress points - that are IP based (typically GCE or OpenStack load-balancers) - type: string - ports: - description: Ports is a list of records of service ports - If used, every port defined in the service should have - an entry in it - items: - properties: - error: - description: 'Error is to record the problem with - the service port The format of the error shall comply - with the following rules: - built-in error values - shall be specified in this file and those shall - use CamelCase names - cloud provider specific error - values must have names that comply with the format - foo.example.com/CamelCase. --- The regex it matches - is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - port: - description: Port is the port number of the service - port of which status is recorded here - format: int32 - type: integer - protocol: - default: TCP - description: 'Protocol is the protocol of the service - port of which status is recorded here The supported - values are: "TCP", "UDP", "SCTP"' - type: string - required: - - port - - protocol - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: array - type: object - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.13.0 - name: udpingresses.configuration.konghq.com -spec: - group: configuration.konghq.com - names: - categories: - - kong-ingress-controller - kind: UDPIngress - listKind: UDPIngressList - plural: udpingresses - singular: udpingress - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Address of the load balancer - jsonPath: .status.loadBalancer.ingress[*].ip - name: Address - type: string - - description: Age - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: UDPIngress is the Schema for the udpingresses API. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: Spec is the UDPIngress specification. - properties: - rules: - description: A list of rules used to configure the Ingress. - items: - description: UDPIngressRule represents a rule to apply against incoming - requests wherein no Host matching is available for request routing, - only the port is used to match requests. - properties: - backend: - description: Backend defines the Kubernetes service which accepts - traffic from the listening Port defined above. - properties: - serviceName: - description: Specifies the name of the referenced service. - minLength: 1 - type: string - servicePort: - description: Specifies the port of the referenced service. - format: int32 - maximum: 65535 - minimum: 1 - type: integer - required: - - serviceName - - servicePort - type: object - port: - description: Port indicates the port for the Kong proxy to accept - incoming traffic on, which will then be routed to the service - Backend. - format: int32 - maximum: 65535 - minimum: 1 - type: integer - required: - - backend - - port - type: object - type: array - type: object - status: - description: UDPIngressStatus defines the observed state of UDPIngress. - properties: - loadBalancer: - description: LoadBalancer contains the current status of the load-balancer. - properties: - ingress: - description: Ingress is a list containing ingress points for the - load-balancer. Traffic intended for the service should be sent - to these ingress points. - items: - description: 'LoadBalancerIngress represents the status of a - load-balancer ingress point: traffic intended for the service - should be sent to an ingress point.' - properties: - hostname: - description: Hostname is set for load-balancer ingress points - that are DNS based (typically AWS load-balancers) - type: string - ip: - description: IP is set for load-balancer ingress points - that are IP based (typically GCE or OpenStack load-balancers) - type: string - ports: - description: Ports is a list of records of service ports - If used, every port defined in the service should have - an entry in it - items: - properties: - error: - description: 'Error is to record the problem with - the service port The format of the error shall comply - with the following rules: - built-in error values - shall be specified in this file and those shall - use CamelCase names - cloud provider specific error - values must have names that comply with the format - foo.example.com/CamelCase. --- The regex it matches - is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - port: - description: Port is the port number of the service - port of which status is recorded here - format: int32 - type: integer - protocol: - default: TCP - description: 'Protocol is the protocol of the service - port of which status is recorded here The supported - values are: "TCP", "UDP", "SCTP"' - type: string - required: - - port - - protocol - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: array - type: object - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kong-serviceaccount - namespace: kong ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: kong-leader-election - namespace: kong -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: kong-ingress -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - "" - resources: - - nodes - verbs: - - list - - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - secrets - verbs: - - list - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - services/status - verbs: - - get - - patch - - update -- apiGroups: - - configuration.konghq.com - resources: - - ingressclassparameterses - verbs: - - get - - list - - watch -- apiGroups: - - configuration.konghq.com - resources: - - kongclusterplugins - verbs: - - get - - list - - watch -- apiGroups: - - configuration.konghq.com - resources: - - kongclusterplugins/status - verbs: - - get - - patch - - update -- apiGroups: - - configuration.konghq.com - resources: - - kongconsumergroups - verbs: - - get - - list - - watch -- apiGroups: - - configuration.konghq.com - resources: - - kongconsumergroups/status - verbs: - - get - - patch - - update -- apiGroups: - - configuration.konghq.com - resources: - - kongconsumers - verbs: - - get - - list - - watch -- apiGroups: - - configuration.konghq.com - resources: - - kongconsumers/status - verbs: - - get - - patch - - update -- apiGroups: - - configuration.konghq.com - resources: - - kongingresses - verbs: - - get - - list - - watch -- apiGroups: - - configuration.konghq.com - resources: - - kongingresses/status - verbs: - - get - - patch - - update -- apiGroups: - - configuration.konghq.com - resources: - - kongplugins - verbs: - - get - - list - - watch -- apiGroups: - - configuration.konghq.com - resources: - - kongplugins/status - verbs: - - get - - patch - - update -- apiGroups: - - configuration.konghq.com - resources: - - tcpingresses - verbs: - - get - - list - - watch -- apiGroups: - - configuration.konghq.com - resources: - - tcpingresses/status - verbs: - - get - - patch - - update -- apiGroups: - - configuration.konghq.com - resources: - - udpingresses - verbs: - - get - - list - - watch -- apiGroups: - - configuration.konghq.com - resources: - - udpingresses/status - verbs: - - get - - patch - - update -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - get - - patch - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: kong-ingress-crds -rules: -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: kong-ingress-gateway -rules: -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get - - list - - watch -- apiGroups: - - gateway.networking.k8s.io - resources: - - gatewayclasses - verbs: - - get - - list - - watch -- apiGroups: - - gateway.networking.k8s.io - resources: - - gatewayclasses/status - verbs: - - get - - update -- apiGroups: - - gateway.networking.k8s.io - resources: - - gateways - verbs: - - get - - list - - update - - watch -- apiGroups: - - gateway.networking.k8s.io - resources: - - gateways/status - verbs: - - get - - update -- apiGroups: - - gateway.networking.k8s.io - resources: - - grpcroutes - verbs: - - get - - list - - watch -- apiGroups: - - gateway.networking.k8s.io - resources: - - grpcroutes/status - verbs: - - get - - patch - - update -- apiGroups: - - gateway.networking.k8s.io - resources: - - httproutes - verbs: - - get - - list - - watch -- apiGroups: - - gateway.networking.k8s.io - resources: - - httproutes/status - verbs: - - get - - update -- apiGroups: - - gateway.networking.k8s.io - resources: - - referencegrants - verbs: - - get - - list - - watch -- apiGroups: - - gateway.networking.k8s.io - resources: - - referencegrants/status - verbs: - - get -- apiGroups: - - gateway.networking.k8s.io - resources: - - tcproutes - verbs: - - get - - list - - watch -- apiGroups: - - gateway.networking.k8s.io - resources: - - tcproutes/status - verbs: - - get - - update -- apiGroups: - - gateway.networking.k8s.io - resources: - - tlsroutes - verbs: - - get - - list - - watch -- apiGroups: - - gateway.networking.k8s.io - resources: - - tlsroutes/status - verbs: - - get - - update -- apiGroups: - - gateway.networking.k8s.io - resources: - - udproutes - verbs: - - get - - list - - watch -- apiGroups: - - gateway.networking.k8s.io - resources: - - udproutes/status - verbs: - - get - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: kong-leader-election - namespace: kong -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: kong-leader-election -subjects: -- kind: ServiceAccount - name: kong-serviceaccount - namespace: kong ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kong-ingress -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kong-ingress -subjects: -- kind: ServiceAccount - name: kong-serviceaccount - namespace: kong ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kong-ingress-crds -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kong-ingress-crds -subjects: -- kind: ServiceAccount - name: kong-serviceaccount - namespace: kong ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kong-ingress-gateway -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kong-ingress-gateway -subjects: -- kind: ServiceAccount - name: kong-serviceaccount - namespace: kong ---- -apiVersion: v1 -kind: Service -metadata: - name: kong-admin - namespace: kong -spec: - clusterIP: None - ports: - - name: admin - port: 8444 - protocol: TCP - targetPort: 8444 - selector: - app: proxy-kong ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp - service.beta.kubernetes.io/aws-load-balancer-type: nlb - name: kong-proxy - namespace: kong -spec: - ports: - - name: proxy - port: 80 - protocol: TCP - targetPort: 8000 - - name: proxy-ssl - port: 443 - protocol: TCP - targetPort: 8443 - selector: - app: proxy-kong - type: LoadBalancer ---- -apiVersion: v1 -kind: Service -metadata: - name: kong-validation-webhook - namespace: kong -spec: - ports: - - name: webhook - port: 443 - protocol: TCP - targetPort: 8080 - selector: - app: ingress-kong ---- -apiVersion: v1 -kind: Service -metadata: - name: postgres - namespace: kong -spec: - ports: - - name: pgql - port: 5432 - protocol: TCP - targetPort: 5432 - selector: - app: postgres ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: ingress-kong - name: ingress-kong - namespace: kong -spec: - replicas: 1 - selector: - matchLabels: - app: ingress-kong - template: - metadata: - annotations: - kuma.io/gateway: enabled - kuma.io/service-account-token-volume: kong-serviceaccount-token - traffic.sidecar.istio.io/includeInboundPorts: "" - labels: - app: ingress-kong - spec: - automountServiceAccountToken: false - containers: - - env: - - name: CONTROLLER_KONG_ADMIN_SVC - value: kong/kong-admin - - name: CONTROLLER_KONG_ADMIN_TLS_SKIP_VERIFY - value: "true" - - name: CONTROLLER_PUBLISH_SERVICE - value: kong/kong-proxy - - name: POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - image: kong/kubernetes-ingress-controller:2.12 - imagePullPolicy: IfNotPresent - livenessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 5 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - name: ingress-controller - ports: - - containerPort: 8080 - name: webhook - protocol: TCP - - containerPort: 10255 - name: cmetrics - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /readyz - port: 10254 - scheme: HTTP - initialDelaySeconds: 5 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - volumeMounts: - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount - name: kong-serviceaccount-token - readOnly: true - initContainers: - - command: - - /bin/bash - - -c - - while true; do kong migrations list; if [[ 0 -eq $? ]]; then exit 0; fi; - sleep 2; done; - env: - - name: KONG_PG_HOST - value: postgres - - name: KONG_PG_PASSWORD - value: kong - image: kong:3.4 - name: wait-for-migrations - serviceAccountName: kong-serviceaccount - volumes: - - name: kong-serviceaccount-token - projected: - sources: - - serviceAccountToken: - expirationSeconds: 3607 - path: token - - configMap: - items: - - key: ca.crt - path: ca.crt - name: kube-root-ca.crt - - downwardAPI: - items: - - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - path: namespace ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: proxy-kong - name: proxy-kong - namespace: kong -spec: - replicas: 2 - selector: - matchLabels: - app: proxy-kong - template: - metadata: - annotations: - kuma.io/gateway: enabled - kuma.io/service-account-token-volume: kong-serviceaccount-token - traffic.sidecar.istio.io/includeInboundPorts: "" - labels: - app: proxy-kong - spec: - automountServiceAccountToken: false - containers: - - env: - - name: KONG_PROXY_LISTEN - value: 0.0.0.0:8000 reuseport backlog=16384, 0.0.0.0:8443 http2 ssl reuseport - backlog=16384 - - name: KONG_PORT_MAPS - value: 80:8000, 443:8443 - - name: KONG_ADMIN_LISTEN - value: 0.0.0.0:8444 http2 ssl reuseport backlog=16384 - - name: KONG_STATUS_LISTEN - value: 0.0.0.0:8100 - - name: KONG_DATABASE - value: postgres - - name: KONG_PG_HOST - value: postgres - - name: KONG_PG_PASSWORD - value: kong - - name: KONG_NGINX_WORKER_PROCESSES - value: "2" - - name: KONG_KIC - value: "on" - - name: KONG_ADMIN_ACCESS_LOG - value: /dev/stdout - - name: KONG_ADMIN_ERROR_LOG - value: /dev/stderr - - name: KONG_PROXY_ERROR_LOG - value: /dev/stderr - - name: KONG_ROUTER_FLAVOR - value: traditional - image: kong:3.4 - lifecycle: - preStop: - exec: - command: - - /bin/bash - - -c - - kong quit - livenessProbe: - failureThreshold: 3 - httpGet: - path: /status - port: 8100 - scheme: HTTP - initialDelaySeconds: 5 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - name: proxy - ports: - - containerPort: 8000 - name: proxy - protocol: TCP - - containerPort: 8443 - name: proxy-ssl - protocol: TCP - - containerPort: 8100 - name: metrics - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /status/ready - port: 8100 - scheme: HTTP - initialDelaySeconds: 5 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - serviceAccountName: kong-serviceaccount - volumes: - - name: kong-serviceaccount-token - projected: - sources: - - serviceAccountToken: - expirationSeconds: 3607 - path: token - - configMap: - items: - - key: ca.crt - path: ca.crt - name: kube-root-ca.crt - - downwardAPI: - items: - - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - path: namespace ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: postgres - namespace: kong -spec: - replicas: 1 - selector: - matchLabels: - app: postgres - serviceName: postgres - template: - metadata: - labels: - app: postgres - spec: - containers: - - env: - - name: POSTGRES_USER - value: kong - - name: POSTGRES_PASSWORD - value: kong - - name: POSTGRES_DB - value: kong - - name: PGDATA - value: /var/lib/postgresql/data/pgdata - image: postgres:9.5 - name: postgres - ports: - - containerPort: 5432 - volumeMounts: - - mountPath: /var/lib/postgresql/data - name: datadir - subPath: pgdata - terminationGracePeriodSeconds: 60 - volumeClaimTemplates: - - metadata: - name: datadir - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: kong-migrations - namespace: kong -spec: - template: - metadata: - name: kong-migrations - spec: - containers: - - command: - - /bin/bash - - -c - - kong migrations bootstrap && kong migrations up && kong migrations finish - env: - - name: KONG_PG_PASSWORD - value: kong - - name: KONG_PG_HOST - value: postgres - - name: KONG_PG_PORT - value: "5432" - image: kong:3.4 - name: kong-migrations - initContainers: - - command: - - /bin/bash - - -c - - until timeout 1 bash 9<>/dev/tcp/${KONG_PG_HOST}/${KONG_PG_PORT}; do echo - 'waiting for db'; sleep 1; done - env: - - name: KONG_PG_HOST - value: postgres - - name: KONG_PG_PORT - value: "5432" - image: kong:3.4 - name: wait-for-postgres - restartPolicy: OnFailure ---- -apiVersion: networking.k8s.io/v1 -kind: IngressClass -metadata: - name: kong -spec: - controller: ingress-controllers.konghq.com/kong diff --git a/internal/clients/manager.go b/internal/clients/manager.go index 7d7dc85b78..91ccaf367a 100644 --- a/internal/clients/manager.go +++ b/internal/clients/manager.go @@ -19,8 +19,6 @@ import ( // It's the same as the default interval of a Kubernetes container's readiness probe. const DefaultReadinessReconciliationInterval = 10 * time.Second -const DefaultAdminAPIServicePort = 8444 - // ClientFactory is responsible for creating Admin API clients. type ClientFactory interface { CreateAdminAPIClient(ctx context.Context, address adminapi.DiscoveredAdminAPI) (*adminapi.Client, error) diff --git a/internal/clients/manager_test.go b/internal/clients/manager_test.go index e8a7ff9e72..3640b2a1a5 100644 --- a/internal/clients/manager_test.go +++ b/internal/clients/manager_test.go @@ -211,6 +211,26 @@ func TestAdminAPIClientsManager_Clients(t *testing.T) { require.Len(t, m.GatewayClients(), 1, "konnect client should not be returned from GatewayClients") require.Equal(t, m.GatewayClientsCount(), 1, "konnect client should not be counted in GatewayClientsCount") require.Equal(t, konnectTestClient, m.KonnectClient(), "konnect client should be returned from KonnectClient") + + testClient2, err := adminapi.NewTestClient("localhost:8081") + initialClients := []*adminapi.Client{testClient, testClient2} + require.NoError(t, err) + clientManagerWithDB, err := clients.NewAdminAPIClientsManager( + context.Background(), + zapr.NewLogger(zap.NewNop()), + initialClients, + mocks.NewAdminAPIClientFactory(map[string]error{}), + &mockReadinessChecker{}, + ) + require.NoError(t, err) + clients.WithDBMode("postgres")(clientManagerWithDB) + clients := clientManagerWithDB.GatewayClients() + require.Len(t, clients, 1, "Expecting one client returned with DB mode") + require.Equal(t, clientManagerWithDB.GatewayClientsCount(), 2, "Expecting 2 initial clients") + require.Truef(t, lo.ContainsBy(initialClients, func(cl *adminapi.Client) bool { + return cl.BaseRootURL() == clients[0].BaseRootURL() + }), "Returned client root URL %s should in root URLs of initial clients") + } func TestAdminAPIClientsManager_SubscribeToGatewayClientsChanges(t *testing.T) { diff --git a/internal/dataplane/kong_client.go b/internal/dataplane/kong_client.go index ff3a296b6e..c78ce48655 100644 --- a/internal/dataplane/kong_client.go +++ b/internal/dataplane/kong_client.go @@ -458,14 +458,14 @@ func (c *KongClient) sendOutToGatewayClients( if len(gatewayClients) == 0 { c.logger.Error( errors.New("no ready gateway clients"), - "could not send configuration to gateways because no ready gateway clients", + "Could not send configuration to gateways", ) // Should not store the configuration in last valid config because the configuration is not validated on Kong gateway. return previousSHAs, nil } gatewayClientURLs := lo.Map(gatewayClients, func(cl *adminapi.Client, _ int) string { return cl.BaseRootURL() }) - c.logger.V(util.DebugLevel).Info("sending configuration to gateway clients", "urls", gatewayClientURLs) + c.logger.V(util.DebugLevel).Info("Sending configuration to gateway clients", "urls", gatewayClientURLs) shas, err := iter.MapErr(gatewayClients, func(client **adminapi.Client) (string, error) { return c.sendToClient(ctx, *client, s, config) diff --git a/internal/manager/run.go b/internal/manager/run.go index b8e1b161cd..3e8b90c77f 100644 --- a/internal/manager/run.go +++ b/internal/manager/run.go @@ -153,6 +153,8 @@ func Run( if err != nil { return fmt.Errorf("failed to create AdminAPIClientsManager: %w", err) } + clients.WithDBMode(dbMode)(clientsManager) + if c.KongAdminSvc.IsPresent() { setupLog.Info("Running AdminAPIClientsManager loop") clientsManager.Run() diff --git a/test/e2e/all_in_one_test.go b/test/e2e/all_in_one_test.go index 5405125651..91260834f8 100644 --- a/test/e2e/all_in_one_test.go +++ b/test/e2e/all_in_one_test.go @@ -259,12 +259,9 @@ func TestDeployAllInOneEnterprisePostgres(t *testing.T) { func TestDeployAllInOnePostgresGatewayDiscovery(t *testing.T) { t.Parallel() - const ( - manifestFileName = "all-in-one-postgres-multiple-gateways.yaml" - manifestFilePath = "../../deploy/single/" + manifestFileName - ) + const manifestFilePath = "manifests/all-in-one-postgres-multiple-gateways.yaml" - t.Logf("configuring %s manifest test", manifestFileName) + t.Logf("configuring %s manifest test", manifestFilePath) ctx, env := setupE2ETest(t) t.Log("deploying kong components") From 612b07ad8418cb66497a6bb5208dc7c0658d9470 Mon Sep 17 00:00:00 2001 From: Yi Tao Date: Wed, 1 Nov 2023 17:21:29 +0800 Subject: [PATCH 08/14] update script to build manifests --- ...all-in-one-postgres-multiple-gateways.yaml | 18 + internal/clients/manager.go | 1 + internal/dataplane/kong_client.go | 7 + scripts/build-single-manifests.sh | 1 + ...all-in-one-postgres-multiple-gateways.yaml | 2771 +++++++++++++++++ 5 files changed, 2798 insertions(+) create mode 100644 deploy/single/all-in-one-postgres-multiple-gateways.yaml create mode 100644 test/e2e/manifests/all-in-one-postgres-multiple-gateways.yaml diff --git a/deploy/single/all-in-one-postgres-multiple-gateways.yaml b/deploy/single/all-in-one-postgres-multiple-gateways.yaml new file mode 100644 index 0000000000..bde70b8a97 --- /dev/null +++ b/deploy/single/all-in-one-postgres-multiple-gateways.yaml @@ -0,0 +1,18 @@ +# Generated by build-single-manifest.sh. DO NOT EDIT. +# +# DEPRECATED +# +# For Kong Ingress Controller 3.0+, please use Helm instead: +# +# $ helm repo add kong https://charts.konghq.com +# $ helm repo update +# $ helm install kong/kong --generate-name --set ingressController.installCRDs=false +# +# If you intend to use an older version, Helm is recommended but you still have the option +# to install using manifests. In that case, replace the 'main' branch in your link with the +# KIC tag. For example: +# kubectl apply -f https://raw.githubusercontent.com/Kong/kubernetes-ingress-controller/v2.12.0/deploy/single/all-in-one-postgres-multiple-gateways.yaml +# + +apiVersion: please-use-helm-to-install-kong +kind: Deprecated diff --git a/internal/clients/manager.go b/internal/clients/manager.go index 91ccaf367a..71536af610 100644 --- a/internal/clients/manager.go +++ b/internal/clients/manager.go @@ -13,6 +13,7 @@ import ( "github.com/kong/kubernetes-ingress-controller/v3/internal/adminapi" "github.com/kong/kubernetes-ingress-controller/v3/internal/util" "github.com/kong/kubernetes-ingress-controller/v3/internal/util/clock" + dataplaneutil "github.com/kong/kubernetes-ingress-controller/v3/internal/util/dataplane" ) // DefaultReadinessReconciliationInterval is the interval at which the manager will run readiness reconciliation loop. diff --git a/internal/dataplane/kong_client.go b/internal/dataplane/kong_client.go index c78ce48655..8d17cd5b23 100644 --- a/internal/dataplane/kong_client.go +++ b/internal/dataplane/kong_client.go @@ -1,6 +1,7 @@ package dataplane import ( + "bytes" "context" "errors" "fmt" @@ -550,6 +551,12 @@ func (c *KongClient) sendToClient( } return "", fmt.Errorf("performing update for %s failed: %w", client.AdminAPIClient().BaseRootURL(), err) } + oldConfigSHA := client.LastConfigSHA() + if bytes.Equal(oldConfigSHA, newConfigSHA) { + logger.V(util.DebugLevel).Info("Skipped sending configuration to Kong because SHA not changed", "SHA", newConfigSHA) + } else { + logger.V(util.DebugLevel).Info("Sent configuration to Kong", "SHA", newConfigSHA, "old_SHA", oldConfigSHA) + } // update the lastConfigSHA with the new updated checksum client.SetLastConfigSHA(newConfigSHA) diff --git a/scripts/build-single-manifests.sh b/scripts/build-single-manifests.sh index 2ecc50de27..aa9346f4e4 100755 --- a/scripts/build-single-manifests.sh +++ b/scripts/build-single-manifests.sh @@ -49,6 +49,7 @@ MANIFESTS=( 'enterprise all-in-one-dbless-k4k8s-enterprise.yaml' 'enterprise-postgres all-in-one-postgres-enterprise.yaml' 'multi-gw/oss all-in-one-dbless.yaml' + 'multi-gw-postgres all-in-one-postgres-multiple-gateways.yaml' 'konnect/oss all-in-one-dbless-konnect.yaml' 'konnect/enterprise all-in-one-dbless-konnect-enterprise.yaml' ) diff --git a/test/e2e/manifests/all-in-one-postgres-multiple-gateways.yaml b/test/e2e/manifests/all-in-one-postgres-multiple-gateways.yaml new file mode 100644 index 0000000000..21cdb3f834 --- /dev/null +++ b/test/e2e/manifests/all-in-one-postgres-multiple-gateways.yaml @@ -0,0 +1,2771 @@ +# Generated by build-single-manifest.sh. NOT FOR PRODUCTION USE (only used internally for testing). DO NOT EDIT. + +apiVersion: v1 +kind: Namespace +metadata: + name: kong +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: ingressclassparameterses.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + kind: IngressClassParameters + listKind: IngressClassParametersList + plural: ingressclassparameterses + singular: ingressclassparameters + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: IngressClassParameters is the Schema for the IngressClassParameters + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec is the IngressClassParameters specification. + properties: + enableLegacyRegexDetection: + default: false + description: EnableLegacyRegexDetection automatically detects if ImplementationSpecific + Ingress paths are regular expression paths using the legacy 2.x + heuristic. The controller adds the "~" prefix to those paths if + the Kong version is 3.0 or higher. + type: boolean + serviceUpstream: + default: false + description: Offload load-balancing to kube-proxy or sidecar. + type: boolean + type: object + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: kongclusterplugins.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: KongClusterPlugin + listKind: KongClusterPluginList + plural: kongclusterplugins + shortNames: + - kcp + singular: kongclusterplugin + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Name of the plugin + jsonPath: .plugin + name: Plugin-Type + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Indicates if the plugin is disabled + jsonPath: .disabled + name: Disabled + priority: 1 + type: boolean + - description: Configuration of the plugin + jsonPath: .config + name: Config + priority: 1 + type: string + - jsonPath: .status.conditions[?(@.type=="Programmed")].status + name: Programmed + type: string + name: v1 + schema: + openAPIV3Schema: + description: KongClusterPlugin is the Schema for the kongclusterplugins API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + config: + description: Config contains the plugin configuration. It's a list of + keys and values required to configure the plugin. Please read the documentation + of the plugin being configured to set values in here. For any plugin + in Kong, anything that goes in the `config` JSON key in the Admin API + request, goes into this property. Only one of `config` or `configFrom` + may be used in a KongClusterPlugin, not both at once. + type: object + x-kubernetes-preserve-unknown-fields: true + configFrom: + description: ConfigFrom references a secret containing the plugin configuration. + This should be used when the plugin configuration contains sensitive + information, such as AWS credentials in the Lambda plugin or the client + secret in the OIDC plugin. Only one of `config` or `configFrom` may + be used in a KongClusterPlugin, not both at once. + properties: + secretKeyRef: + description: Specifies a name, a namespace, and a key of a secret + to refer to. + properties: + key: + description: The key containing the value. + type: string + name: + description: The secret containing the key. + type: string + namespace: + description: The namespace containing the secret. + type: string + required: + - key + - name + - namespace + type: object + type: object + consumerRef: + description: ConsumerRef is a reference to a particular consumer. + type: string + disabled: + description: Disabled set if the plugin is disabled or not. + type: boolean + instance_name: + description: InstanceName is an optional custom name to identify an instance + of the plugin. This is useful when running the same plugin in multiple + contexts, for example, on multiple services. + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + ordering: + description: 'Ordering overrides the normal plugin execution order. It''s + only available on Kong Enterprise. `` is a request processing + phase (for example, `access` or `body_filter`) and `` is the + name of the plugin that will run before or after the KongPlugin. For + example, a KongPlugin with `plugin: rate-limiting` and `before.access: + ["key-auth"]` will create a rate limiting plugin that limits requests + _before_ they are authenticated.' + properties: + after: + additionalProperties: + items: + type: string + type: array + description: PluginOrderingPhase indicates which plugins in a phase + should affect the target plugin's order + type: object + before: + additionalProperties: + items: + type: string + type: array + description: PluginOrderingPhase indicates which plugins in a phase + should affect the target plugin's order + type: object + type: object + plugin: + description: PluginName is the name of the plugin to which to apply the + config. + type: string + protocols: + description: Protocols configures plugin to run on requests received on + specific protocols. + items: + description: KongProtocol is a valid Kong protocol. This alias is necessary + to deal with https://github.com/kubernetes-sigs/controller-tools/issues/342 + enum: + - http + - https + - grpc + - grpcs + - tcp + - tls + - udp + type: string + type: array + run_on: + description: RunOn configures the plugin to run on the first or the second + or both nodes in case of a service mesh deployment. + enum: + - first + - second + - all + type: string + status: + description: Status represents the current status of the KongClusterPlugin + resource. + properties: + conditions: + default: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + description: "Conditions describe the current conditions of the KongClusterPluginStatus. + \n Known condition types are: \n * \"Programmed\"" + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + \n type FooStatus struct{ // Represents the observations of a + foo's current state. // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge + // +listType=map // +listMapKey=type Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - plugin + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: kongconsumergroups.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: KongConsumerGroup + listKind: KongConsumerGroupList + plural: kongconsumergroups + shortNames: + - kcg + singular: kongconsumergroup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Programmed")].status + name: Programmed + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: KongConsumerGroup is the Schema for the kongconsumergroups API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + status: + description: Status represents the current status of the KongConsumer + resource. + properties: + conditions: + default: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + description: "Conditions describe the current conditions of the KongConsumerGroup. + \n Known condition types are: \n * \"Programmed\"" + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + \n type FooStatus struct{ // Represents the observations of a + foo's current state. // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge + // +listType=map // +listMapKey=type Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: kongconsumers.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: KongConsumer + listKind: KongConsumerList + plural: kongconsumers + shortNames: + - kc + singular: kongconsumer + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Username of a Kong Consumer + jsonPath: .username + name: Username + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Programmed")].status + name: Programmed + type: string + name: v1 + schema: + openAPIV3Schema: + description: KongConsumer is the Schema for the kongconsumers API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + consumerGroups: + description: ConsumerGroups are references to consumer groups (that consumer + wants to be part of) provisioned in Kong. + items: + type: string + type: array + credentials: + description: Credentials are references to secrets containing a credential + to be provisioned in Kong. + items: + type: string + type: array + custom_id: + description: CustomID is a Kong cluster-unique existing ID for the consumer + - useful for mapping Kong with users in your existing database. + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + status: + description: Status represents the current status of the KongConsumer + resource. + properties: + conditions: + default: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + description: "Conditions describe the current conditions of the KongConsumer. + \n Known condition types are: \n * \"Programmed\"" + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + \n type FooStatus struct{ // Represents the observations of a + foo's current state. // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge + // +listType=map // +listMapKey=type Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + username: + description: Username is a Kong cluster-unique username of the consumer. + type: string + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: kongingresses.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: KongIngress + listKind: KongIngressList + plural: kongingresses + shortNames: + - ki + singular: kongingress + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: KongIngress is the Schema for the kongingresses API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + proxy: + description: Proxy defines additional connection options for the routes + to be configured in the Kong Gateway, e.g. `connection_timeout`, `retries`, + etc. + properties: + connect_timeout: + description: "The timeout in milliseconds for\testablishing a connection + to the upstream server. Deprecated: use Service's \"konghq.com/connect-timeout\" + annotation instead." + minimum: 0 + type: integer + path: + description: '(optional) The path to be used in requests to the upstream + server. Deprecated: use Service''s "konghq.com/path" annotation + instead.' + pattern: ^/.*$ + type: string + protocol: + description: 'The protocol used to communicate with the upstream. + Deprecated: use Service''s "konghq.com/protocol" annotation instead.' + enum: + - http + - https + - grpc + - grpcs + - tcp + - tls + - udp + type: string + read_timeout: + description: 'The timeout in milliseconds between two successive read + operations for transmitting a request to the upstream server. Deprecated: + use Service''s "konghq.com/read-timeout" annotation instead.' + minimum: 0 + type: integer + retries: + description: 'The number of retries to execute upon failure to proxy. + Deprecated: use Service''s "konghq.com/retries" annotation instead.' + minimum: 0 + type: integer + write_timeout: + description: 'The timeout in milliseconds between two successive write + operations for transmitting a request to the upstream server. Deprecated: + use Service''s "konghq.com/write-timeout" annotation instead.' + minimum: 0 + type: integer + type: object + route: + description: Route define rules to match client requests. Each Route is + associated with a Service, and a Service may have multiple Routes associated + to it. + properties: + headers: + additionalProperties: + items: + type: string + type: array + description: 'Headers contains one or more lists of values indexed + by header name that will cause this Route to match if present in + the request. The Host header cannot be used with this attribute. + Deprecated: use Ingress'' "konghq.com/headers" annotation instead.' + type: object + https_redirect_status_code: + description: 'HTTPSRedirectStatusCode is the status code Kong responds + with when all properties of a Route match except the protocol. Deprecated: + use Ingress'' "ingress.kubernetes.io/force-ssl-redirect" or "konghq.com/https-redirect-status-code" + annotations instead.' + type: integer + methods: + description: 'Methods is a list of HTTP methods that match this Route. + Deprecated: use Ingress'' "konghq.com/methods" annotation instead.' + items: + type: string + type: array + path_handling: + description: 'PathHandling controls how the Service path, Route path + and requested path are combined when sending a request to the upstream. + Deprecated: use Ingress'' "konghq.com/path-handling" annotation + instead.' + enum: + - v0 + - v1 + type: string + preserve_host: + description: 'PreserveHost sets When matching a Route via one of the + hosts domain names, use the request Host header in the upstream + request headers. If set to false, the upstream Host header will + be that of the Service’s host. Deprecated: use Ingress'' "konghq.com/preserve-host" + annotation instead.' + type: boolean + protocols: + description: 'Protocols is an array of the protocols this Route should + allow. Deprecated: use Ingress'' "konghq.com/protocols" annotation + instead.' + items: + description: KongProtocol is a valid Kong protocol. This alias is + necessary to deal with https://github.com/kubernetes-sigs/controller-tools/issues/342 + enum: + - http + - https + - grpc + - grpcs + - tcp + - tls + - udp + type: string + type: array + regex_priority: + description: 'RegexPriority is a number used to choose which route + resolves a given request when several routes match it using regexes + simultaneously. Deprecated: use Ingress'' "konghq.com/regex-priority" + annotation instead.' + type: integer + request_buffering: + description: 'RequestBuffering sets whether to enable request body + buffering or not. Deprecated: use Ingress'' "konghq.com/request-buffering" + annotation instead.' + type: boolean + response_buffering: + description: 'ResponseBuffering sets whether to enable response body + buffering or not. Deprecated: use Ingress'' "konghq.com/response-buffering" + annotation instead.' + type: boolean + snis: + description: 'SNIs is a list of SNIs that match this Route when using + stream routing. Deprecated: use Ingress'' "konghq.com/snis" annotation + instead.' + items: + type: string + type: array + strip_path: + description: 'StripPath sets When matching a Route via one of the + paths strip the matching prefix from the upstream request URL. Deprecated: + use Ingress'' "konghq.com/strip-path" annotation instead.' + type: boolean + type: object + upstream: + description: Upstream represents a virtual hostname and can be used to + loadbalance incoming requests over multiple targets (e.g. Kubernetes + `Services` can be a target, OR `Endpoints` can be targets). + properties: + algorithm: + description: 'Algorithm is the load balancing algorithm to use. Accepted + values are: "round-robin", "consistent-hashing", "least-connections", + "latency".' + enum: + - round-robin + - consistent-hashing + - least-connections + - latency + type: string + hash_fallback: + description: 'HashFallback defines What to use as hashing input if + the primary hash_on does not return a hash. Accepted values are: + "none", "consumer", "ip", "header", "cookie".' + type: string + hash_fallback_header: + description: HashFallbackHeader is the header name to take the value + from as hash input. Only required when "hash_fallback" is set to + "header". + type: string + hash_fallback_query_arg: + description: HashFallbackQueryArg is the "hash_fallback" version of + HashOnQueryArg. + type: string + hash_fallback_uri_capture: + description: HashFallbackURICapture is the "hash_fallback" version + of HashOnURICapture. + type: string + hash_on: + description: 'HashOn defines what to use as hashing input. Accepted + values are: "none", "consumer", "ip", "header", "cookie", "path", + "query_arg", "uri_capture".' + type: string + hash_on_cookie: + description: The cookie name to take the value from as hash input. + Only required when "hash_on" or "hash_fallback" is set to "cookie". + type: string + hash_on_cookie_path: + description: The cookie path to set in the response headers. Only + required when "hash_on" or "hash_fallback" is set to "cookie". + type: string + hash_on_header: + description: HashOnHeader defines the header name to take the value + from as hash input. Only required when "hash_on" is set to "header". + type: string + hash_on_query_arg: + description: HashOnQueryArg is the query string parameter whose value + is the hash input when "hash_on" is set to "query_arg". + type: string + hash_on_uri_capture: + description: HashOnURICapture is the name of the capture group whose + value is the hash input when "hash_on" is set to "uri_capture". + type: string + healthchecks: + description: Healthchecks defines the health check configurations + in Kong. + properties: + active: + description: ActiveHealthcheck configures active health check + probing. + properties: + concurrency: + minimum: 1 + type: integer + headers: + additionalProperties: + items: + type: string + type: array + type: object + healthy: + description: Healthy configures thresholds and HTTP status + codes to mark targets healthy for an upstream. + properties: + http_statuses: + items: + type: integer + type: array + interval: + minimum: 0 + type: integer + successes: + minimum: 0 + type: integer + type: object + http_path: + pattern: ^/.*$ + type: string + https_sni: + type: string + https_verify_certificate: + type: boolean + timeout: + minimum: 0 + type: integer + type: + type: string + unhealthy: + description: Unhealthy configures thresholds and HTTP status + codes to mark targets unhealthy. + properties: + http_failures: + minimum: 0 + type: integer + http_statuses: + items: + type: integer + type: array + interval: + minimum: 0 + type: integer + tcp_failures: + minimum: 0 + type: integer + timeouts: + minimum: 0 + type: integer + type: object + type: object + passive: + description: PassiveHealthcheck configures passive checks around + passive health checks. + properties: + healthy: + description: Healthy configures thresholds and HTTP status + codes to mark targets healthy for an upstream. + properties: + http_statuses: + items: + type: integer + type: array + interval: + minimum: 0 + type: integer + successes: + minimum: 0 + type: integer + type: object + type: + type: string + unhealthy: + description: Unhealthy configures thresholds and HTTP status + codes to mark targets unhealthy. + properties: + http_failures: + minimum: 0 + type: integer + http_statuses: + items: + type: integer + type: array + interval: + minimum: 0 + type: integer + tcp_failures: + minimum: 0 + type: integer + timeouts: + minimum: 0 + type: integer + type: object + type: object + threshold: + type: number + type: object + host_header: + description: HostHeader is The hostname to be used as Host header + when proxying requests through Kong. + type: string + slots: + description: Slots is the number of slots in the load balancer algorithm. + minimum: 10 + type: integer + type: object + type: object + x-kubernetes-validations: + - message: '''proxy'' field is no longer supported, use Service''s annotations + instead' + rule: '!has(self.proxy)' + - message: '''route'' field is no longer supported, use Ingress'' annotations + instead' + rule: '!has(self.route)' + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: kongplugins.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: KongPlugin + listKind: KongPluginList + plural: kongplugins + shortNames: + - kp + singular: kongplugin + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Name of the plugin + jsonPath: .plugin + name: Plugin-Type + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Indicates if the plugin is disabled + jsonPath: .disabled + name: Disabled + priority: 1 + type: boolean + - description: Configuration of the plugin + jsonPath: .config + name: Config + priority: 1 + type: string + - jsonPath: .status.conditions[?(@.type=="Programmed")].status + name: Programmed + type: string + name: v1 + schema: + openAPIV3Schema: + description: KongPlugin is the Schema for the kongplugins API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + config: + description: Config contains the plugin configuration. It's a list of + keys and values required to configure the plugin. Please read the documentation + of the plugin being configured to set values in here. For any plugin + in Kong, anything that goes in the `config` JSON key in the Admin API + request, goes into this property. Only one of `config` or `configFrom` + may be used in a KongPlugin, not both at once. + type: object + x-kubernetes-preserve-unknown-fields: true + configFrom: + description: ConfigFrom references a secret containing the plugin configuration. + This should be used when the plugin configuration contains sensitive + information, such as AWS credentials in the Lambda plugin or the client + secret in the OIDC plugin. Only one of `config` or `configFrom` may + be used in a KongPlugin, not both at once. + properties: + secretKeyRef: + description: Specifies a name and a key of a secret to refer to. The + namespace is implicitly set to the one of referring object. + properties: + key: + description: The key containing the value. + type: string + name: + description: The secret containing the key. + type: string + required: + - key + - name + type: object + type: object + consumerRef: + description: ConsumerRef is a reference to a particular consumer. + type: string + disabled: + description: Disabled set if the plugin is disabled or not. + type: boolean + instance_name: + description: InstanceName is an optional custom name to identify an instance + of the plugin. This is useful when running the same plugin in multiple + contexts, for example, on multiple services. + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + ordering: + description: 'Ordering overrides the normal plugin execution order. It''s + only available on Kong Enterprise. `` is a request processing + phase (for example, `access` or `body_filter`) and `` is the + name of the plugin that will run before or after the KongPlugin. For + example, a KongPlugin with `plugin: rate-limiting` and `before.access: + ["key-auth"]` will create a rate limiting plugin that limits requests + _before_ they are authenticated.' + properties: + after: + additionalProperties: + items: + type: string + type: array + description: PluginOrderingPhase indicates which plugins in a phase + should affect the target plugin's order + type: object + before: + additionalProperties: + items: + type: string + type: array + description: PluginOrderingPhase indicates which plugins in a phase + should affect the target plugin's order + type: object + type: object + plugin: + description: PluginName is the name of the plugin to which to apply the + config. + type: string + protocols: + description: Protocols configures plugin to run on requests received on + specific protocols. + items: + description: KongProtocol is a valid Kong protocol. This alias is necessary + to deal with https://github.com/kubernetes-sigs/controller-tools/issues/342 + enum: + - http + - https + - grpc + - grpcs + - tcp + - tls + - udp + type: string + type: array + run_on: + description: RunOn configures the plugin to run on the first or the second + or both nodes in case of a service mesh deployment. + enum: + - first + - second + - all + type: string + status: + description: Status represents the current status of the KongPlugin resource. + properties: + conditions: + default: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + description: "Conditions describe the current conditions of the KongPluginStatus. + \n Known condition types are: \n * \"Programmed\"" + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + \n type FooStatus struct{ // Represents the observations of a + foo's current state. // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge + // +listType=map // +listMapKey=type Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - plugin + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + labels: + gateway.networking.k8s.io/policy: direct + name: kongupstreampolicies.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: KongUpstreamPolicy + listKind: KongUpstreamPolicyList + plural: kongupstreampolicies + shortNames: + - kup + singular: kongupstreampolicy + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: "KongUpstreamPolicy allows configuring algorithm that should + be used for load balancing traffic between Kong Upstream's Targets. It also + allows configuring health checks for Kong Upstream's Targets. \n Its configuration + is similar to Kong Upstream object (https://docs.konghq.com/gateway/latest/admin-api/#upstream-object), + and it is applied to Kong Upstream objects created by the controller. \n + It can be attached to Services. To attach it to a Service, it has to be + annotated with `konghq.com/upstream-policy: `, where `` is the + name of the KongUpstreamPolicy object in the same namespace as the Service. + \n When attached to a Service, it will affect all Kong Upstreams created + for the Service. \n When attached to a Service used in a Gateway API *Route + rule with multiple BackendRefs, all of its Services MUST be configured with + the same KongUpstreamPolicy. Otherwise, the controller will *ignore* the + KongUpstreamPolicy. \n Note: KongUpstreamPolicy doesn't implement Gateway + API's GEP-713 strictly. In particular, it doesn't use the TargetRef for + attaching to Services and Gateway API *Routes - annotations are used instead. + This is to allow reusing the same KongUpstreamPolicy for multiple Services + and Gateway API *Routes." + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec contains the configuration of the Kong upstream. + properties: + algorithm: + description: 'Algorithm is the load balancing algorithm to use. Accepted + values are: "round-robin", "consistent-hashing", "least-connections", + "latency".' + enum: + - round-robin + - consistent-hashing + - least-connections + - latency + type: string + hashOn: + description: HashOn defines how to calculate hash for consistent-hashing + load balancing algorithm. Algorithm must be set to "consistent-hashing" + for this field to have effect. + properties: + cookie: + description: Cookie is the name of the cookie to use as hash input. + type: string + cookiePath: + description: CookiePath is cookie path to set in the response + headers. + type: string + header: + description: Header is the name of the header to use as hash input. + type: string + input: + description: Input allows using one of the predefined inputs (ip, + consumer, path). For other parametrized inputs, use one of the + fields below. + enum: + - ip + - consumer + - path + type: string + queryArg: + description: QueryArg is the name of the query argument to use + as hash input. + type: string + uriCapture: + description: URICapture is the name of the URI capture group to + use as hash input. + type: string + type: object + hashOnFallback: + description: HashOnFallback defines how to calculate hash for consistent-hashing + load balancing algorithm if the primary hash function fails. Algorithm + must be set to "consistent-hashing" for this field to have effect. + properties: + cookie: + description: Cookie is the name of the cookie to use as hash input. + type: string + cookiePath: + description: CookiePath is cookie path to set in the response + headers. + type: string + header: + description: Header is the name of the header to use as hash input. + type: string + input: + description: Input allows using one of the predefined inputs (ip, + consumer, path). For other parametrized inputs, use one of the + fields below. + enum: + - ip + - consumer + - path + type: string + queryArg: + description: QueryArg is the name of the query argument to use + as hash input. + type: string + uriCapture: + description: URICapture is the name of the URI capture group to + use as hash input. + type: string + type: object + healthchecks: + description: Healthchecks defines the health check configurations + in Kong. + properties: + active: + description: Active configures active health check probing. + properties: + concurrency: + description: Concurrency is the number of targets to check + concurrently. + minimum: 1 + type: integer + headers: + additionalProperties: + items: + type: string + type: array + description: Headers is a list of HTTP headers to add to the + probe request. + type: object + healthy: + description: Healthy configures thresholds and HTTP status + codes to mark targets healthy for an upstream. + properties: + httpStatuses: + description: HTTPStatuses is a list of HTTP status codes + that Kong considers a success. + items: + description: HTTPStatus is an HTTP status code. + maximum: 599 + minimum: 100 + type: integer + type: array + interval: + description: Interval is the interval between active health + checks for an upstream in seconds when in a healthy + state. + minimum: 0 + type: integer + successes: + description: Successes is the number of successes to consider + a target healthy. + minimum: 0 + type: integer + type: object + httpPath: + description: HTTPPath is the path to use in GET HTTP request + to run as a probe. + pattern: ^/.*$ + type: string + httpsSni: + description: HTTPSSNI is the SNI to use in GET HTTPS request + to run as a probe. + type: string + httpsVerifyCertificate: + description: HTTPSVerifyCertificate is a boolean value that + indicates if the certificate should be verified. + type: boolean + timeout: + description: Timeout is the probe timeout in seconds. + minimum: 0 + type: integer + type: + description: Type determines whether to perform active health + checks using HTTP or HTTPS, or just attempt a TCP connection. + Accepted values are "http", "https", "tcp", "grpc", "grpcs". + enum: + - http + - https + - tcp + - grpc + - grpcs + type: string + unhealthy: + description: Unhealthy configures thresholds and HTTP status + codes to mark targets unhealthy for an upstream. + properties: + httpFailures: + description: HTTPFailures is the number of failures to + consider a target unhealthy. + minimum: 0 + type: integer + httpStatuses: + description: HTTPStatuses is a list of HTTP status codes + that Kong considers a failure. + items: + description: HTTPStatus is an HTTP status code. + maximum: 599 + minimum: 100 + type: integer + type: array + interval: + description: Interval is the interval between active health + checks for an upstream in seconds when in an unhealthy + state. + minimum: 0 + type: integer + tcpFailures: + description: TCPFailures is the number of TCP failures + in a row to consider a target unhealthy. + minimum: 0 + type: integer + timeouts: + description: Timeouts is the number of timeouts in a row + to consider a target unhealthy. + minimum: 0 + type: integer + type: object + type: object + passive: + description: Passive configures passive health check probing. + properties: + healthy: + description: Healthy configures thresholds and HTTP status + codes to mark targets healthy for an upstream. + properties: + httpStatuses: + description: HTTPStatuses is a list of HTTP status codes + that Kong considers a success. + items: + description: HTTPStatus is an HTTP status code. + maximum: 599 + minimum: 100 + type: integer + type: array + interval: + description: Interval is the interval between active health + checks for an upstream in seconds when in a healthy + state. + minimum: 0 + type: integer + successes: + description: Successes is the number of successes to consider + a target healthy. + minimum: 0 + type: integer + type: object + type: + description: Type determines whether to perform passive health + checks interpreting HTTP/HTTPS statuses, or just check for + TCP connection success. Accepted values are "http", "https", + "tcp", "grpc", "grpcs". + enum: + - http + - https + - tcp + - grpc + - grpcs + type: string + unhealthy: + description: Unhealthy configures thresholds and HTTP status + codes to mark targets unhealthy. + properties: + httpFailures: + description: HTTPFailures is the number of failures to + consider a target unhealthy. + minimum: 0 + type: integer + httpStatuses: + description: HTTPStatuses is a list of HTTP status codes + that Kong considers a failure. + items: + description: HTTPStatus is an HTTP status code. + maximum: 599 + minimum: 100 + type: integer + type: array + interval: + description: Interval is the interval between active health + checks for an upstream in seconds when in an unhealthy + state. + minimum: 0 + type: integer + tcpFailures: + description: TCPFailures is the number of TCP failures + in a row to consider a target unhealthy. + minimum: 0 + type: integer + timeouts: + description: Timeouts is the number of timeouts in a row + to consider a target unhealthy. + minimum: 0 + type: integer + type: object + type: object + threshold: + description: Threshold is the minimum percentage of the upstream’s + targets’ weight that must be available for the whole upstream + to be considered healthy. + type: integer + type: object + slots: + description: Slots is the number of slots in the load balancer algorithm. + If not set, the default value in Kong for the algorithm is used. + maximum: 65536 + minimum: 10 + type: integer + type: object + type: object + x-kubernetes-validations: + - message: Only one of spec.hashOn.(input|cookie|header|uriCapture|queryArg) + can be set. + rule: 'has(self.spec.hashOn) ? [has(self.spec.hashOn.input), has(self.spec.hashOn.cookie), + has(self.spec.hashOn.header), has(self.spec.hashOn.uriCapture), has(self.spec.hashOn.queryArg)].filter(fieldSet, + fieldSet == true).size() <= 1 : true' + - message: When spec.hashOn.cookie is set, spec.hashOn.cookiePath is required. + rule: 'has(self.spec.hashOn) && has(self.spec.hashOn.cookie) ? has(self.spec.hashOn.cookiePath) + : true' + - message: When spec.hashOn.cookiePath is set, spec.hashOn.cookie is required. + rule: 'has(self.spec.hashOn) && has(self.spec.hashOn.cookiePath) ? has(self.spec.hashOn.cookie) + : true' + - message: spec.algorithm must be set to "consistent-hashing" when spec.hashOn + is set. + rule: 'has(self.spec.hashOn) ? has(self.spec.algorithm) && self.spec.algorithm + == "consistent-hashing" : true' + - message: Only one of spec.hashOnFallback.(input|header|uriCapture|queryArg) + can be set. + rule: 'has(self.spec.hashOnFallback) ? [has(self.spec.hashOnFallback.input), + has(self.spec.hashOnFallback.header), has(self.spec.hashOnFallback.uriCapture), + has(self.spec.hashOnFallback.queryArg)].filter(fieldSet, fieldSet == true).size() + <= 1 : true' + - message: spec.algorithm must be set to "consistent-hashing" when spec.hashOnFallback + is set. + rule: 'has(self.spec.hashOnFallback) ? has(self.spec.algorithm) && self.spec.algorithm + == "consistent-hashing" : true' + - message: spec.hashOnFallback.cookie must not be set. + rule: 'has(self.spec.hashOnFallback) ? !has(self.spec.hashOnFallback.cookie) + : true' + - message: spec.hashOnFallback.cookiePath must not be set. + rule: 'has(self.spec.hashOnFallback) ? !has(self.spec.hashOnFallback.cookiePath) + : true' + - message: spec.healthchecks.passive.healthy.interval must not be set. + rule: 'has(self.spec.healthchecks) && has(self.spec.healthchecks.passive) + && has(self.spec.healthchecks.passive.healthy) ? !has(self.spec.healthchecks.passive.healthy.interval) + : true' + - message: spec.healthchecks.passive.unhealthy.interval must not be set. + rule: 'has(self.spec.healthchecks) && has(self.spec.healthchecks.passive) + && has(self.spec.healthchecks.passive.unhealthy) ? !has(self.spec.healthchecks.passive.unhealthy.interval) + : true' + - message: spec.hashOnFallback must not be set when spec.hashOn.cookie is + set. + rule: 'has(self.spec.hashOn) && has(self.spec.hashOn.cookie) ? !has(self.spec.hashOnFallback) + : true' + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: tcpingresses.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: TCPIngress + listKind: TCPIngressList + plural: tcpingresses + singular: tcpingress + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Address of the load balancer + jsonPath: .status.loadBalancer.ingress[*].ip + name: Address + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: TCPIngress is the Schema for the tcpingresses API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec is the TCPIngress specification. + properties: + rules: + description: A list of rules used to configure the Ingress. + items: + description: IngressRule represents a rule to apply against incoming + requests. Matching is performed based on an (optional) SNI and + port. + properties: + backend: + description: Backend defines the referenced service endpoint + to which the traffic will be forwarded to. + properties: + serviceName: + description: Specifies the name of the referenced service. + minLength: 1 + type: string + servicePort: + description: Specifies the port of the referenced service. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - serviceName + - servicePort + type: object + host: + description: Host is the fully qualified domain name of a network + host, as defined by RFC 3986. If a Host is not specified, + then port-based TCP routing is performed. Kong doesn't care + about the content of the TCP stream in this case. If a Host + is specified, the protocol must be TLS over TCP. A plain-text + TCP request cannot be routed based on Host. It can only be + routed based on Port. + type: string + port: + description: Port is the port on which to accept TCP or TLS + over TCP sessions and route. It is a required field. If a + Host is not specified, the requested are routed based only + on Port. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - backend + - port + type: object + type: array + tls: + description: TLS configuration. This is similar to the `tls` section + in the Ingress resource in networking.v1beta1 group. The mapping + of SNIs to TLS cert-key pair defined here will be used for HTTP + Ingress rules as well. Once can define the mapping in this resource + or the original Ingress resource, both have the same effect. + items: + description: IngressTLS describes the transport layer security. + properties: + hosts: + description: Hosts are a list of hosts included in the TLS certificate. + The values in this list must match the name/s used in the + tlsSecret. Defaults to the wildcard host setting for the loadbalancer + controller fulfilling this Ingress, if left unspecified. + items: + type: string + type: array + secretName: + description: SecretName is the name of the secret used to terminate + SSL traffic. + type: string + type: object + type: array + type: object + status: + description: TCPIngressStatus defines the observed state of TCPIngress. + properties: + loadBalancer: + description: LoadBalancer contains the current status of the load-balancer. + properties: + ingress: + description: Ingress is a list containing ingress points for the + load-balancer. Traffic intended for the service should be sent + to these ingress points. + items: + description: 'LoadBalancerIngress represents the status of a + load-balancer ingress point: traffic intended for the service + should be sent to an ingress point.' + properties: + hostname: + description: Hostname is set for load-balancer ingress points + that are DNS based (typically AWS load-balancers) + type: string + ip: + description: IP is set for load-balancer ingress points + that are IP based (typically GCE or OpenStack load-balancers) + type: string + ports: + description: Ports is a list of records of service ports + If used, every port defined in the service should have + an entry in it + items: + properties: + error: + description: 'Error is to record the problem with + the service port The format of the error shall comply + with the following rules: - built-in error values + shall be specified in this file and those shall + use CamelCase names - cloud provider specific error + values must have names that comply with the format + foo.example.com/CamelCase. --- The regex it matches + is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number of the service + port of which status is recorded here + format: int32 + type: integer + protocol: + default: TCP + description: 'Protocol is the protocol of the service + port of which status is recorded here The supported + values are: "TCP", "UDP", "SCTP"' + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: udpingresses.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: UDPIngress + listKind: UDPIngressList + plural: udpingresses + singular: udpingress + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Address of the load balancer + jsonPath: .status.loadBalancer.ingress[*].ip + name: Address + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: UDPIngress is the Schema for the udpingresses API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec is the UDPIngress specification. + properties: + rules: + description: A list of rules used to configure the Ingress. + items: + description: UDPIngressRule represents a rule to apply against incoming + requests wherein no Host matching is available for request routing, + only the port is used to match requests. + properties: + backend: + description: Backend defines the Kubernetes service which accepts + traffic from the listening Port defined above. + properties: + serviceName: + description: Specifies the name of the referenced service. + minLength: 1 + type: string + servicePort: + description: Specifies the port of the referenced service. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - serviceName + - servicePort + type: object + port: + description: Port indicates the port for the Kong proxy to accept + incoming traffic on, which will then be routed to the service + Backend. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - backend + - port + type: object + type: array + type: object + status: + description: UDPIngressStatus defines the observed state of UDPIngress. + properties: + loadBalancer: + description: LoadBalancer contains the current status of the load-balancer. + properties: + ingress: + description: Ingress is a list containing ingress points for the + load-balancer. Traffic intended for the service should be sent + to these ingress points. + items: + description: 'LoadBalancerIngress represents the status of a + load-balancer ingress point: traffic intended for the service + should be sent to an ingress point.' + properties: + hostname: + description: Hostname is set for load-balancer ingress points + that are DNS based (typically AWS load-balancers) + type: string + ip: + description: IP is set for load-balancer ingress points + that are IP based (typically GCE or OpenStack load-balancers) + type: string + ports: + description: Ports is a list of records of service ports + If used, every port defined in the service should have + an entry in it + items: + properties: + error: + description: 'Error is to record the problem with + the service port The format of the error shall comply + with the following rules: - built-in error values + shall be specified in this file and those shall + use CamelCase names - cloud provider specific error + values must have names that comply with the format + foo.example.com/CamelCase. --- The regex it matches + is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number of the service + port of which status is recorded here + format: int32 + type: integer + protocol: + default: TCP + description: 'Protocol is the protocol of the service + port of which status is recorded here The supported + values are: "TCP", "UDP", "SCTP"' + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kong-leader-election + namespace: kong +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kong-ingress +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services/status + verbs: + - get + - patch + - update +- apiGroups: + - configuration.konghq.com + resources: + - ingressclassparameterses + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongclusterplugins + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongclusterplugins/status + verbs: + - get + - patch + - update +- apiGroups: + - configuration.konghq.com + resources: + - kongconsumergroups + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongconsumergroups/status + verbs: + - get + - patch + - update +- apiGroups: + - configuration.konghq.com + resources: + - kongconsumers + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongconsumers/status + verbs: + - get + - patch + - update +- apiGroups: + - configuration.konghq.com + resources: + - kongingresses + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongingresses/status + verbs: + - get + - patch + - update +- apiGroups: + - configuration.konghq.com + resources: + - kongplugins + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongplugins/status + verbs: + - get + - patch + - update +- apiGroups: + - configuration.konghq.com + resources: + - kongupstreampolicies + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongupstreampolicies/status + verbs: + - get + - patch + - update +- apiGroups: + - configuration.konghq.com + resources: + - tcpingresses + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - tcpingresses/status + verbs: + - get + - patch + - update +- apiGroups: + - configuration.konghq.com + resources: + - udpingresses + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - udpingresses/status + verbs: + - get + - patch + - update +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kong-ingress-gateway +rules: +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses/status + verbs: + - get + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - gateways + verbs: + - get + - list + - update + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gateways/status + verbs: + - get + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - grpcroutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - grpcroutes/status + verbs: + - get + - patch + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - httproutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - httproutes/status + verbs: + - get + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - referencegrants + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - referencegrants/status + verbs: + - get +- apiGroups: + - gateway.networking.k8s.io + resources: + - tcproutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - tcproutes/status + verbs: + - get + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - tlsroutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - tlsroutes/status + verbs: + - get + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - udproutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - udproutes/status + verbs: + - get + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kong-leader-election + namespace: kong +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kong-leader-election +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kong-ingress +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kong-ingress-gateway +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-gateway +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: v1 +kind: Service +metadata: + name: kong-admin + namespace: kong +spec: + clusterIP: None + ports: + - name: admin + port: 8444 + protocol: TCP + targetPort: 8444 + selector: + app: proxy-kong +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp + service.beta.kubernetes.io/aws-load-balancer-type: nlb + name: kong-proxy + namespace: kong +spec: + ports: + - name: proxy + port: 80 + protocol: TCP + targetPort: 8000 + - name: proxy-ssl + port: 443 + protocol: TCP + targetPort: 8443 + selector: + app: proxy-kong + type: LoadBalancer +--- +apiVersion: v1 +kind: Service +metadata: + name: kong-validation-webhook + namespace: kong +spec: + ports: + - name: webhook + port: 443 + protocol: TCP + targetPort: 8080 + selector: + app: ingress-kong +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: kong +spec: + ports: + - name: pgql + port: 5432 + protocol: TCP + targetPort: 5432 + selector: + app: postgres +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: ingress-kong + name: ingress-kong + namespace: kong +spec: + replicas: 1 + selector: + matchLabels: + app: ingress-kong + template: + metadata: + annotations: + kuma.io/gateway: enabled + kuma.io/service-account-token-volume: kong-serviceaccount-token + traffic.sidecar.istio.io/includeInboundPorts: "" + labels: + app: ingress-kong + spec: + automountServiceAccountToken: false + containers: + - env: + - name: CONTROLLER_KONG_ADMIN_SVC + value: kong/kong-admin + - name: CONTROLLER_KONG_ADMIN_TLS_SKIP_VERIFY + value: "true" + - name: CONTROLLER_PUBLISH_SERVICE + value: kong/kong-proxy + - name: CONTROLLER_LOG_LEVEL + value: "debug" + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + image: kong/kubernetes-ingress-controller:v2.12.0-213-g8c8938369 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: ingress-controller + ports: + - containerPort: 8080 + name: webhook + protocol: TCP + - containerPort: 10255 + name: cmetrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: 10254 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + volumeMounts: + - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + name: kong-serviceaccount-token + readOnly: true + initContainers: + - command: + - /bin/bash + - -c + - while true; do kong migrations list; if [[ 0 -eq $? ]]; then exit 0; fi; + sleep 2; done; + env: + - name: KONG_PG_HOST + value: postgres + - name: KONG_PG_PASSWORD + value: kong + image: kong:3.4 + name: wait-for-migrations + serviceAccountName: kong-serviceaccount + volumes: + - name: kong-serviceaccount-token + projected: + sources: + - serviceAccountToken: + expirationSeconds: 3607 + path: token + - configMap: + items: + - key: ca.crt + path: ca.crt + name: kube-root-ca.crt + - downwardAPI: + items: + - fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + path: namespace +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: proxy-kong + name: proxy-kong + namespace: kong +spec: + replicas: 2 + selector: + matchLabels: + app: proxy-kong + template: + metadata: + annotations: + kuma.io/gateway: enabled + kuma.io/service-account-token-volume: kong-serviceaccount-token + traffic.sidecar.istio.io/includeInboundPorts: "" + labels: + app: proxy-kong + spec: + automountServiceAccountToken: false + containers: + - env: + - name: KONG_PROXY_LISTEN + value: 0.0.0.0:8000 reuseport backlog=16384, 0.0.0.0:8443 http2 ssl reuseport + backlog=16384 + - name: KONG_PORT_MAPS + value: 80:8000, 443:8443 + - name: KONG_ADMIN_LISTEN + value: 0.0.0.0:8444 http2 ssl reuseport backlog=16384 + - name: KONG_STATUS_LISTEN + value: 0.0.0.0:8100 + - name: KONG_DATABASE + value: postgres + - name: KONG_PG_HOST + value: postgres + - name: KONG_PG_PASSWORD + value: kong + - name: KONG_NGINX_WORKER_PROCESSES + value: "2" + - name: KONG_KIC + value: "on" + - name: KONG_ADMIN_ACCESS_LOG + value: /dev/stdout + - name: KONG_ADMIN_ERROR_LOG + value: /dev/stderr + - name: KONG_PROXY_ERROR_LOG + value: /dev/stderr + - name: KONG_ROUTER_FLAVOR + value: traditional + image: kong:3.4 + lifecycle: + preStop: + exec: + command: + - /bin/bash + - -c + - kong quit + livenessProbe: + failureThreshold: 3 + httpGet: + path: /status + port: 8100 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: proxy + ports: + - containerPort: 8000 + name: proxy + protocol: TCP + - containerPort: 8443 + name: proxy-ssl + protocol: TCP + - containerPort: 8100 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /status/ready + port: 8100 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + serviceAccountName: kong-serviceaccount + volumes: + - name: kong-serviceaccount-token + projected: + sources: + - serviceAccountToken: + expirationSeconds: 3607 + path: token + - configMap: + items: + - key: ca.crt + path: ca.crt + name: kube-root-ca.crt + - downwardAPI: + items: + - fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + path: namespace +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres + namespace: kong +spec: + replicas: 1 + selector: + matchLabels: + app: postgres + serviceName: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - env: + - name: POSTGRES_USER + value: kong + - name: POSTGRES_PASSWORD + value: kong + - name: POSTGRES_DB + value: kong + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + image: postgres:9.5 + name: postgres + ports: + - containerPort: 5432 + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: datadir + subPath: pgdata + terminationGracePeriodSeconds: 60 + volumeClaimTemplates: + - metadata: + name: datadir + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: kong-migrations + namespace: kong +spec: + template: + metadata: + name: kong-migrations + spec: + containers: + - command: + - /bin/bash + - -c + - kong migrations bootstrap && kong migrations up && kong migrations finish + env: + - name: KONG_PG_PASSWORD + value: kong + - name: KONG_PG_HOST + value: postgres + - name: KONG_PG_PORT + value: "5432" + image: kong:3.4 + name: kong-migrations + initContainers: + - command: + - /bin/bash + - -c + - until timeout 1 bash 9<>/dev/tcp/${KONG_PG_HOST}/${KONG_PG_PORT}; do echo + 'waiting for db'; sleep 1; done + env: + - name: KONG_PG_HOST + value: postgres + - name: KONG_PG_PORT + value: "5432" + image: kong:3.4 + name: wait-for-postgres + restartPolicy: OnFailure +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: kong +spec: + controller: ingress-controllers.konghq.com/kong From b4041f2152ffceab5ae59e17899f7b2ca416f7b1 Mon Sep 17 00:00:00 2001 From: Yi Tao Date: Thu, 2 Nov 2023 15:52:44 +0800 Subject: [PATCH 09/14] propagate SHAs to all clients in DB mode --- internal/clients/manager.go | 15 ++++++++++---- internal/clients/manager_test.go | 27 ++++++++++++++++++-------- internal/dataplane/kong_client.go | 22 +++++++++++---------- internal/dataplane/kong_client_test.go | 20 +++++++++++++++---- internal/manager/run.go | 2 +- 5 files changed, 59 insertions(+), 27 deletions(-) diff --git a/internal/clients/manager.go b/internal/clients/manager.go index 71536af610..c10c9f7f09 100644 --- a/internal/clients/manager.go +++ b/internal/clients/manager.go @@ -30,6 +30,7 @@ type ClientFactory interface { type AdminAPIClientsProvider interface { KonnectClient() *adminapi.KonnectClient GatewayClients() []*adminapi.Client + GatewayClientsToConfigure() []*adminapi.Client } // Ticker is an interface that allows to control a ticker. @@ -92,10 +93,10 @@ func WithReadinessReconciliationTicker(ticker Ticker) AdminAPIClientsManagerOpti } // WithDBMode allows to set the DBMode of the Kong gateway instances behind the admin API service. -func WithDBMode(dbMode string) AdminAPIClientsManagerOption { - return func(m *AdminAPIClientsManager) { - m.dbMode = dbMode - } + +func (c *AdminAPIClientsManager) WithDBMode(dbMode string) *AdminAPIClientsManager { + c.dbMode = dbMode + return c } func NewAdminAPIClientsManager( @@ -183,6 +184,12 @@ func (c *AdminAPIClientsManager) KonnectClient() *adminapi.KonnectClient { // GatewayClients returns a copy of current client's slice. Konnect client won't be included. // This method can be used when some actions need to be performed only against Kong Gateway clients. func (c *AdminAPIClientsManager) GatewayClients() []*adminapi.Client { + c.lock.RLock() + defer c.lock.RUnlock() + return lo.Values(c.readyGatewayClients) +} + +func (c *AdminAPIClientsManager) GatewayClientsToConfigure() []*adminapi.Client { c.lock.RLock() defer c.lock.RUnlock() readyGatewayClients := lo.Values(c.readyGatewayClients) diff --git a/internal/clients/manager_test.go b/internal/clients/manager_test.go index 3640b2a1a5..1bf1b0c577 100644 --- a/internal/clients/manager_test.go +++ b/internal/clients/manager_test.go @@ -205,17 +205,24 @@ func TestAdminAPIClientsManager_Clients(t *testing.T) { require.NoError(t, err) require.Len(t, m.GatewayClients(), 1, "expecting one initial client") require.Equal(t, m.GatewayClientsCount(), 1, "expecting one initial client") + require.Len(t, m.GatewayClientsToConfigure(), 1, "Expecting one initial client") konnectTestClient := &adminapi.KonnectClient{} m.SetKonnectClient(konnectTestClient) require.Len(t, m.GatewayClients(), 1, "konnect client should not be returned from GatewayClients") require.Equal(t, m.GatewayClientsCount(), 1, "konnect client should not be counted in GatewayClientsCount") require.Equal(t, konnectTestClient, m.KonnectClient(), "konnect client should be returned from KonnectClient") +} +func TestAdminAPIClientsManager_Clients_DBMode(t *testing.T) { + testClient, err := adminapi.NewTestClient("localhost:8080") + require.NoError(t, err) testClient2, err := adminapi.NewTestClient("localhost:8081") + require.NoError(t, err) initialClients := []*adminapi.Client{testClient, testClient2} require.NoError(t, err) - clientManagerWithDB, err := clients.NewAdminAPIClientsManager( + + m, err := clients.NewAdminAPIClientsManager( context.Background(), zapr.NewLogger(zap.NewNop()), initialClients, @@ -223,14 +230,18 @@ func TestAdminAPIClientsManager_Clients(t *testing.T) { &mockReadinessChecker{}, ) require.NoError(t, err) - clients.WithDBMode("postgres")(clientManagerWithDB) - clients := clientManagerWithDB.GatewayClients() - require.Len(t, clients, 1, "Expecting one client returned with DB mode") - require.Equal(t, clientManagerWithDB.GatewayClientsCount(), 2, "Expecting 2 initial clients") - require.Truef(t, lo.ContainsBy(initialClients, func(cl *adminapi.Client) bool { - return cl.BaseRootURL() == clients[0].BaseRootURL() - }), "Returned client root URL %s should in root URLs of initial clients") + m = m.WithDBMode("postgres") + + clients := m.GatewayClients() + require.Len(t, clients, 2, "Expecting 2 clients returned with DB mode") + + configureClients := m.GatewayClientsToConfigure() + require.Len(t, configureClients, 1, "Expecting 1 client to configure") + require.Truef(t, lo.ContainsBy(initialClients, func(c *adminapi.Client) bool { + return c.BaseRootURL() == configureClients[0].BaseRootURL() + }), "Client's address %s should be in initial clients") + require.Equal(t, m.GatewayClientsCount(), 2, "Expecting 2 initial clients") } func TestAdminAPIClientsManager_SubscribeToGatewayClientsChanges(t *testing.T) { diff --git a/internal/dataplane/kong_client.go b/internal/dataplane/kong_client.go index 8d17cd5b23..eab91dd18b 100644 --- a/internal/dataplane/kong_client.go +++ b/internal/dataplane/kong_client.go @@ -1,7 +1,6 @@ package dataplane import ( - "bytes" "context" "errors" "fmt" @@ -465,16 +464,25 @@ func (c *KongClient) sendOutToGatewayClients( return previousSHAs, nil } - gatewayClientURLs := lo.Map(gatewayClients, func(cl *adminapi.Client, _ int) string { return cl.BaseRootURL() }) - c.logger.V(util.DebugLevel).Info("Sending configuration to gateway clients", "urls", gatewayClientURLs) + gatewayClientsToConfigure := c.clientsProvider.GatewayClientsToConfigure() + configureGatewayClientURLs := lo.Map(gatewayClientsToConfigure, func(cl *adminapi.Client, _ int) string { return cl.BaseRootURL() }) + c.logger.V(util.DebugLevel).Info("Sending configuration to gateway clients", "urls", configureGatewayClientURLs) - shas, err := iter.MapErr(gatewayClients, func(client **adminapi.Client) (string, error) { + shas, err := iter.MapErr(gatewayClientsToConfigure, func(client **adminapi.Client) (string, error) { return c.sendToClient(ctx, *client, s, config) }) if err != nil { return nil, err } + // After succeeded to update configurations in DB mode, we should propagate the SHA from + // the chosen gateway client to other clients. + if !dataplaneutil.IsDBLessMode(c.dbmode) && len(shas) > 0 { + for _, client := range gatewayClients { + client.SetLastConfigSHA([]byte(shas[0])) + } + } + sort.Strings(shas) c.SHAs = shas @@ -551,12 +559,6 @@ func (c *KongClient) sendToClient( } return "", fmt.Errorf("performing update for %s failed: %w", client.AdminAPIClient().BaseRootURL(), err) } - oldConfigSHA := client.LastConfigSHA() - if bytes.Equal(oldConfigSHA, newConfigSHA) { - logger.V(util.DebugLevel).Info("Skipped sending configuration to Kong because SHA not changed", "SHA", newConfigSHA) - } else { - logger.V(util.DebugLevel).Info("Sent configuration to Kong", "SHA", newConfigSHA, "old_SHA", oldConfigSHA) - } // update the lastConfigSHA with the new updated checksum client.SetLastConfigSHA(newConfigSHA) diff --git a/internal/dataplane/kong_client_test.go b/internal/dataplane/kong_client_test.go index a5c82ad8a9..086a07c302 100644 --- a/internal/dataplane/kong_client_test.go +++ b/internal/dataplane/kong_client_test.go @@ -38,6 +38,7 @@ import ( "github.com/kong/kubernetes-ingress-controller/v3/internal/metrics" "github.com/kong/kubernetes-ingress-controller/v3/internal/store" "github.com/kong/kubernetes-ingress-controller/v3/internal/util" + dataplaneutil "github.com/kong/kubernetes-ingress-controller/v3/internal/util/dataplane" "github.com/kong/kubernetes-ingress-controller/v3/internal/versions" "github.com/kong/kubernetes-ingress-controller/v3/test/mocks" ) @@ -143,14 +144,25 @@ var ( type mockGatewayClientsProvider struct { gatewayClients []*adminapi.Client konnectClient *adminapi.KonnectClient + dbMode string } -func (f mockGatewayClientsProvider) KonnectClient() *adminapi.KonnectClient { - return f.konnectClient +func (p mockGatewayClientsProvider) KonnectClient() *adminapi.KonnectClient { + return p.konnectClient } -func (f mockGatewayClientsProvider) GatewayClients() []*adminapi.Client { - return f.gatewayClients +func (p mockGatewayClientsProvider) GatewayClients() []*adminapi.Client { + return p.gatewayClients +} + +func (p mockGatewayClientsProvider) GatewayClientsToConfigure() []*adminapi.Client { + if dataplaneutil.IsDBLessMode(p.dbMode) { + return p.gatewayClients + } + if len(p.gatewayClients) == 0 { + return []*adminapi.Client{} + } + return p.gatewayClients[:1] } // mockUpdateStrategy is a mock implementation of sendconfig.UpdateStrategyResolver. diff --git a/internal/manager/run.go b/internal/manager/run.go index 3e8b90c77f..a77bddf6e2 100644 --- a/internal/manager/run.go +++ b/internal/manager/run.go @@ -153,7 +153,7 @@ func Run( if err != nil { return fmt.Errorf("failed to create AdminAPIClientsManager: %w", err) } - clients.WithDBMode(dbMode)(clientsManager) + clientsManager = clientsManager.WithDBMode(dbMode) if c.KongAdminSvc.IsPresent() { setupLog.Info("Running AdminAPIClientsManager loop") From 7b784addf88fd252c5906aa7b0ec66cd02e14e53 Mon Sep 17 00:00:00 2001 From: Yi Tao Date: Thu, 2 Nov 2023 17:43:24 +0800 Subject: [PATCH 10/14] fix manifests and remove useless clientFactory --- .../variants/multi-gw-postgres/kustomization.yaml | 8 +++----- internal/clients/manager.go | 5 ----- internal/clients/manager_test.go | 14 +++++--------- internal/dataplane/kong_client.go | 4 ++-- internal/manager/run.go | 1 - .../all-in-one-postgres-multiple-gateways.yaml | 4 +--- 6 files changed, 11 insertions(+), 25 deletions(-) diff --git a/config/variants/multi-gw-postgres/kustomization.yaml b/config/variants/multi-gw-postgres/kustomization.yaml index f89f98e0cc..3e883be3b1 100644 --- a/config/variants/multi-gw-postgres/kustomization.yaml +++ b/config/variants/multi-gw-postgres/kustomization.yaml @@ -11,11 +11,9 @@ resources: components: - ../../image/oss -patchesStrategicMerge: -- manager_multi_gateway_patch.yaml -- gateway_service_patch.yaml - -patchesJson6902: +patches: +- path: manager_multi_gateway_patch.yaml +- path: gateway_service_patch.yaml - target: group: apps version: v1 diff --git a/internal/clients/manager.go b/internal/clients/manager.go index c10c9f7f09..545a842e60 100644 --- a/internal/clients/manager.go +++ b/internal/clients/manager.go @@ -64,9 +64,6 @@ type AdminAPIClientsManager struct { // configured. pendingGatewayClients map[string]adminapi.DiscoveredAdminAPI - // clientFactory is used to create admin API client pointing to the admin API service. - clientFactory ClientFactory - // readinessChecker is used to check readiness of the clients. readinessChecker ReadinessChecker @@ -103,7 +100,6 @@ func NewAdminAPIClientsManager( ctx context.Context, logger logr.Logger, initialClients []*adminapi.Client, - clientFactory ClientFactory, readinessChecker ReadinessChecker, opts ...AdminAPIClientsManagerOption, ) (*AdminAPIClientsManager, error) { @@ -117,7 +113,6 @@ func NewAdminAPIClientsManager( c := &AdminAPIClientsManager{ readyGatewayClients: readyClients, pendingGatewayClients: make(map[string]adminapi.DiscoveredAdminAPI), - clientFactory: clientFactory, readinessChecker: readinessChecker, readinessReconciliationTicker: clock.NewTicker(), discoveredAdminAPIsNotifyChan: make(chan []adminapi.DiscoveredAdminAPI), diff --git a/internal/clients/manager_test.go b/internal/clients/manager_test.go index 1bf1b0c577..274425b8d1 100644 --- a/internal/clients/manager_test.go +++ b/internal/clients/manager_test.go @@ -97,7 +97,6 @@ func TestAdminAPIClientsManager_OnNotifyClientsAreUpdatedAccordingly(t *testing. ctx, logger, []*adminapi.Client{initialClient}, - mocks.NewAdminAPIClientFactory(map[string]error{}), readinessChecker, ) require.NoError(t, err) @@ -163,7 +162,6 @@ func TestNewAdminAPIClientsManager_NoInitialClientsDisallowed(t *testing.T) { context.Background(), zapr.NewLogger(zap.NewNop()), nil, - mocks.NewAdminAPIClientFactory(map[string]error{}), &mockReadinessChecker{}, ) require.ErrorContains(t, err, "at least one initial client must be provided") @@ -178,7 +176,6 @@ func TestAdminAPIClientsManager_NotRunningNotifyLoop(t *testing.T) { context.Background(), zapr.NewLogger(zap.NewNop()), []*adminapi.Client{testClient}, - mocks.NewAdminAPIClientFactory(map[string]error{}), &mockReadinessChecker{}, ) require.NoError(t, err) @@ -199,7 +196,6 @@ func TestAdminAPIClientsManager_Clients(t *testing.T) { context.Background(), zapr.NewLogger(zap.NewNop()), []*adminapi.Client{testClient}, - mocks.NewAdminAPIClientFactory(map[string]error{}), &mockReadinessChecker{}, ) require.NoError(t, err) @@ -226,7 +222,7 @@ func TestAdminAPIClientsManager_Clients_DBMode(t *testing.T) { context.Background(), zapr.NewLogger(zap.NewNop()), initialClients, - mocks.NewAdminAPIClientFactory(map[string]error{}), + &mockReadinessChecker{}, ) require.NoError(t, err) @@ -256,7 +252,7 @@ func TestAdminAPIClientsManager_SubscribeToGatewayClientsChanges(t *testing.T) { ctx, zapr.NewLogger(zap.NewNop()), []*adminapi.Client{testClient}, - mocks.NewAdminAPIClientFactory(map[string]error{}), + readinessChecker) require.NoError(t, err) @@ -343,7 +339,7 @@ func TestAdminAPIClientsManager_ConcurrentNotify(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - m, err := clients.NewAdminAPIClientsManager(ctx, zapr.NewLogger(zap.NewNop()), []*adminapi.Client{testClient}, mocks.NewAdminAPIClientFactory(map[string]error{}), readinessChecker) + m, err := clients.NewAdminAPIClientsManager(ctx, zapr.NewLogger(zap.NewNop()), []*adminapi.Client{testClient}, readinessChecker) require.NoError(t, err) m.Run() @@ -377,7 +373,7 @@ func TestAdminAPIClientsManager_GatewayClientsChanges(t *testing.T) { readinessChecker := &mockReadinessChecker{} ctx, cancel := context.WithCancel(context.Background()) defer cancel() - m, err := clients.NewAdminAPIClientsManager(ctx, zapr.NewLogger(zap.NewNop()), []*adminapi.Client{testClient}, mocks.NewAdminAPIClientFactory(map[string]error{}), readinessChecker) + m, err := clients.NewAdminAPIClientsManager(ctx, zapr.NewLogger(zap.NewNop()), []*adminapi.Client{testClient}, readinessChecker) require.NoError(t, err) m.Run() @@ -476,7 +472,7 @@ func TestAdminAPIClientsManager_PeriodicReadinessReconciliation(t *testing.T) { ctx, zapr.NewLogger(zap.NewNop()), []*adminapi.Client{testClient}, - mocks.NewAdminAPIClientFactory(map[string]error{}), + readinessChecker, clients.WithReadinessReconciliationTicker(readinessTicker), ) diff --git a/internal/dataplane/kong_client.go b/internal/dataplane/kong_client.go index eab91dd18b..f303b60ec2 100644 --- a/internal/dataplane/kong_client.go +++ b/internal/dataplane/kong_client.go @@ -454,14 +454,13 @@ func (c *KongClient) sendOutToGatewayClients( ctx context.Context, s *kongstate.KongState, config sendconfig.Config, ) ([]string, error) { gatewayClients := c.clientsProvider.GatewayClients() - previousSHAs := c.SHAs if len(gatewayClients) == 0 { c.logger.Error( errors.New("no ready gateway clients"), "Could not send configuration to gateways", ) // Should not store the configuration in last valid config because the configuration is not validated on Kong gateway. - return previousSHAs, nil + return c.SHAs, nil } gatewayClientsToConfigure := c.clientsProvider.GatewayClientsToConfigure() @@ -483,6 +482,7 @@ func (c *KongClient) sendOutToGatewayClients( } } + previousSHAs := c.SHAs sort.Strings(shas) c.SHAs = shas diff --git a/internal/manager/run.go b/internal/manager/run.go index a77bddf6e2..9b6c90b4f3 100644 --- a/internal/manager/run.go +++ b/internal/manager/run.go @@ -147,7 +147,6 @@ func Run( ctx, logger, initialKongClients, - adminAPIClientsFactory, readinessChecker, ) if err != nil { diff --git a/test/e2e/manifests/all-in-one-postgres-multiple-gateways.yaml b/test/e2e/manifests/all-in-one-postgres-multiple-gateways.yaml index 21cdb3f834..a4a0e20673 100644 --- a/test/e2e/manifests/all-in-one-postgres-multiple-gateways.yaml +++ b/test/e2e/manifests/all-in-one-postgres-multiple-gateways.yaml @@ -2490,8 +2490,6 @@ spec: value: "true" - name: CONTROLLER_PUBLISH_SERVICE value: kong/kong-proxy - - name: CONTROLLER_LOG_LEVEL - value: "debug" - name: POD_NAME valueFrom: fieldRef: @@ -2502,7 +2500,7 @@ spec: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - image: kong/kubernetes-ingress-controller:v2.12.0-213-g8c8938369 + image: kong/kubernetes-ingress-controller:2.12 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 From eca994d715b9a9a1b3996373c11fc1f6fae8d145 Mon Sep 17 00:00:00 2001 From: Yi Tao Date: Thu, 2 Nov 2023 22:24:48 +0800 Subject: [PATCH 11/14] address comments --- internal/clients/manager.go | 6 +++--- internal/clients/manager_test.go | 3 --- internal/dataplane/kong_client.go | 7 ++++--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/internal/clients/manager.go b/internal/clients/manager.go index 545a842e60..5693a0a2cf 100644 --- a/internal/clients/manager.go +++ b/internal/clients/manager.go @@ -188,12 +188,12 @@ func (c *AdminAPIClientsManager) GatewayClientsToConfigure() []*adminapi.Client c.lock.RLock() defer c.lock.RUnlock() readyGatewayClients := lo.Values(c.readyGatewayClients) - // With dbless mode, we should send configuration to ALL Kong gateway instances. + // With DB-less mode, we should send the configuration to ALL gateway instances. if dataplaneutil.IsDBLessMode(c.dbMode) { return readyGatewayClients } - // When Kong gateway is DB backed, we return a random admin API client - // since KIC only need to send requests to one instance. + // When a gateway is DB-backed, we return a random client + // since KIC only needs to send requests to one instance. // If there are no ready gateway clients, we return an empty list. if len(readyGatewayClients) == 0 { return []*adminapi.Client{} diff --git a/internal/clients/manager_test.go b/internal/clients/manager_test.go index 274425b8d1..b109ed7821 100644 --- a/internal/clients/manager_test.go +++ b/internal/clients/manager_test.go @@ -222,7 +222,6 @@ func TestAdminAPIClientsManager_Clients_DBMode(t *testing.T) { context.Background(), zapr.NewLogger(zap.NewNop()), initialClients, - &mockReadinessChecker{}, ) require.NoError(t, err) @@ -252,7 +251,6 @@ func TestAdminAPIClientsManager_SubscribeToGatewayClientsChanges(t *testing.T) { ctx, zapr.NewLogger(zap.NewNop()), []*adminapi.Client{testClient}, - readinessChecker) require.NoError(t, err) @@ -472,7 +470,6 @@ func TestAdminAPIClientsManager_PeriodicReadinessReconciliation(t *testing.T) { ctx, zapr.NewLogger(zap.NewNop()), []*adminapi.Client{testClient}, - readinessChecker, clients.WithReadinessReconciliationTicker(readinessTicker), ) diff --git a/internal/dataplane/kong_client.go b/internal/dataplane/kong_client.go index f303b60ec2..63309544b9 100644 --- a/internal/dataplane/kong_client.go +++ b/internal/dataplane/kong_client.go @@ -474,9 +474,10 @@ func (c *KongClient) sendOutToGatewayClients( return nil, err } - // After succeeded to update configurations in DB mode, we should propagate the SHA from - // the chosen gateway client to other clients. - if !dataplaneutil.IsDBLessMode(c.dbmode) && len(shas) > 0 { + // After a successful configuration update in DB mode, we should propagate the SHA from + // the chosen client to other clients as well as they will pick the configuration from the shared database. + if !dataplaneutil.IsDBLessMode(c.dbmode) && + len(gatewayClients) > 1 { for _, client := range gatewayClients { client.SetLastConfigSHA([]byte(shas[0])) } From b201d82d3c3fafd08c4d149362a5c3ce979a3146 Mon Sep 17 00:00:00 2001 From: Yi Tao Date: Thu, 2 Nov 2023 22:55:50 +0800 Subject: [PATCH 12/14] address comments Again --- internal/clients/manager.go | 6 ++++++ internal/dataplane/kong_client.go | 8 +++++--- internal/util/dataplane/mode.go | 6 ++++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/internal/clients/manager.go b/internal/clients/manager.go index 5693a0a2cf..49a77d4f31 100644 --- a/internal/clients/manager.go +++ b/internal/clients/manager.go @@ -184,6 +184,12 @@ func (c *AdminAPIClientsManager) GatewayClients() []*adminapi.Client { return lo.Values(c.readyGatewayClients) } +// GatewayClientsToConfigure returns the gateway clients need to be configured with the new configuration +// in sending generated configurations. +// In DBLess mode, it returns ALL gateway clients +// because we need to update configurations of each gateway instance. +// In DB-backed mode, it returns ONE random gateway client +// because we only need to send configurations to one gateway instance, and other instances could be synced from DB. func (c *AdminAPIClientsManager) GatewayClientsToConfigure() []*adminapi.Client { c.lock.RLock() defer c.lock.RUnlock() diff --git a/internal/dataplane/kong_client.go b/internal/dataplane/kong_client.go index 63309544b9..00c9d0524e 100644 --- a/internal/dataplane/kong_client.go +++ b/internal/dataplane/kong_client.go @@ -474,9 +474,11 @@ func (c *KongClient) sendOutToGatewayClients( return nil, err } - // After a successful configuration update in DB mode, we should propagate the SHA from - // the chosen client to other clients as well as they will pick the configuration from the shared database. - if !dataplaneutil.IsDBLessMode(c.dbmode) && + // After a successful configuration update in DB mode, + // since only ONE gateway client is chosen to send requests and store SHA of latest configurations, + // we should propagate the SHA from the chosen client to other clients + // as well as they will pick the configuration from the shared database. + if dataplaneutil.DBBacked(c.dbmode) && len(gatewayClients) > 1 { for _, client := range gatewayClients { client.SetLastConfigSHA([]byte(shas[0])) diff --git a/internal/util/dataplane/mode.go b/internal/util/dataplane/mode.go index efb6517feb..6aabc12b7b 100644 --- a/internal/util/dataplane/mode.go +++ b/internal/util/dataplane/mode.go @@ -4,3 +4,9 @@ package dataplane func IsDBLessMode(mode string) bool { return mode == "" || mode == "off" } + +// DBBacked returns true if the gateway is DB backed. +// reverse of IsDBLessMode for readability. +func DBBacked(mode string) bool { + return !IsDBLessMode(mode) +} From 18b89bd01af3394048bc02e31102e0854206e65d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Thu, 2 Nov 2023 16:35:47 +0100 Subject: [PATCH 13/14] Apply suggestions from code review --- internal/clients/manager.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/clients/manager.go b/internal/clients/manager.go index 49a77d4f31..13dc796be1 100644 --- a/internal/clients/manager.go +++ b/internal/clients/manager.go @@ -90,7 +90,6 @@ func WithReadinessReconciliationTicker(ticker Ticker) AdminAPIClientsManagerOpti } // WithDBMode allows to set the DBMode of the Kong gateway instances behind the admin API service. - func (c *AdminAPIClientsManager) WithDBMode(dbMode string) *AdminAPIClientsManager { c.dbMode = dbMode return c @@ -184,12 +183,12 @@ func (c *AdminAPIClientsManager) GatewayClients() []*adminapi.Client { return lo.Values(c.readyGatewayClients) } -// GatewayClientsToConfigure returns the gateway clients need to be configured with the new configuration -// in sending generated configurations. +// GatewayClientsToConfigure returns the gateway clients which need to be configured // with the new configuration. // In DBLess mode, it returns ALL gateway clients // because we need to update configurations of each gateway instance. // In DB-backed mode, it returns ONE random gateway client -// because we only need to send configurations to one gateway instance, and other instances could be synced from DB. +// because we only need to send configurations to one gateway instance +// while others will be synced using the DB. func (c *AdminAPIClientsManager) GatewayClientsToConfigure() []*adminapi.Client { c.lock.RLock() defer c.lock.RUnlock() From 56a71a48c17d022cf528457b75452fad3de67efe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Burzy=C5=84ski?= Date: Thu, 2 Nov 2023 16:39:36 +0100 Subject: [PATCH 14/14] Update internal/clients/manager.go --- internal/clients/manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/clients/manager.go b/internal/clients/manager.go index 13dc796be1..a95440c3d1 100644 --- a/internal/clients/manager.go +++ b/internal/clients/manager.go @@ -183,7 +183,7 @@ func (c *AdminAPIClientsManager) GatewayClients() []*adminapi.Client { return lo.Values(c.readyGatewayClients) } -// GatewayClientsToConfigure returns the gateway clients which need to be configured // with the new configuration. +// GatewayClientsToConfigure returns the gateway clients which need to be configured with the new configuration. // In DBLess mode, it returns ALL gateway clients // because we need to update configurations of each gateway instance. // In DB-backed mode, it returns ONE random gateway client