diff --git a/CHANGELOG.md b/CHANGELOG.md index 308216d4c8..29baf6d3d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,6 +81,8 @@ [#2451](https://github.com/Kong/kubernetes-ingress-controller/issues/2451) - Added description of each field of `kongIngresses` CRD. [#1766](https://github.com/Kong/kubernetes-ingress-controller/issues/1766) +- Added support for `TLSRoute` resources. + [#2476](https://github.com/Kong/kubernetes-ingress-controller/issues/2476) #### Fixed diff --git a/PROJECT b/PROJECT index d89c47a2e9..7f8ad0e8a8 100644 --- a/PROJECT +++ b/PROJECT @@ -84,4 +84,9 @@ resources: group: gateway kind: ReferencePolicy version: v1alpha2 +- controller: true + domain: networking.k8s.io + group: gateway + kind: TLSRoute + version: v1alpha2 version: "3" diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 5c6dbc9b4c..1d067dc073 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -273,6 +273,21 @@ rules: verbs: - get - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - tlsroutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - tlsroutes/status + verbs: + - get + - update - apiGroups: - gateway.networking.k8s.io resources: diff --git a/deploy/single/all-in-one-dbless-k4k8s-enterprise.yaml b/deploy/single/all-in-one-dbless-k4k8s-enterprise.yaml index a3be0cf3a0..33298c1202 100644 --- a/deploy/single/all-in-one-dbless-k4k8s-enterprise.yaml +++ b/deploy/single/all-in-one-dbless-k4k8s-enterprise.yaml @@ -1281,6 +1281,21 @@ rules: verbs: - get - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - tlsroutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - tlsroutes/status + verbs: + - get + - update - apiGroups: - gateway.networking.k8s.io resources: diff --git a/deploy/single/all-in-one-dbless.yaml b/deploy/single/all-in-one-dbless.yaml index 60338b0409..ca759477a6 100644 --- a/deploy/single/all-in-one-dbless.yaml +++ b/deploy/single/all-in-one-dbless.yaml @@ -1281,6 +1281,21 @@ rules: verbs: - get - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - tlsroutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - tlsroutes/status + verbs: + - get + - update - apiGroups: - gateway.networking.k8s.io resources: diff --git a/deploy/single/all-in-one-postgres-enterprise.yaml b/deploy/single/all-in-one-postgres-enterprise.yaml index c0962a178d..8c7efcaa76 100644 --- a/deploy/single/all-in-one-postgres-enterprise.yaml +++ b/deploy/single/all-in-one-postgres-enterprise.yaml @@ -1281,6 +1281,21 @@ rules: verbs: - get - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - tlsroutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - tlsroutes/status + verbs: + - get + - update - apiGroups: - gateway.networking.k8s.io resources: diff --git a/deploy/single/all-in-one-postgres.yaml b/deploy/single/all-in-one-postgres.yaml index d7d3738d08..84ac1873cf 100644 --- a/deploy/single/all-in-one-postgres.yaml +++ b/deploy/single/all-in-one-postgres.yaml @@ -1281,6 +1281,21 @@ rules: verbs: - get - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - tlsroutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - tlsroutes/status + verbs: + - get + - update - apiGroups: - gateway.networking.k8s.io resources: diff --git a/examples/gateway-tlsroute.yaml b/examples/gateway-tlsroute.yaml new file mode 100644 index 0000000000..c4f9eb07ea --- /dev/null +++ b/examples/gateway-tlsroute.yaml @@ -0,0 +1,78 @@ +# WARNING: Gateway APIs support is still experimental. Use as your own risk. +# +# NOTE: You need to install the Gateway APIs CRDs before using this example, +# they are external and can be deployed with the following one-liner: +# +# kubectl kustomize https://github.com/kubernetes-sigs/gateway-api.git/config/crd?ref=master | kubectl apply -f - +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tcpecho + labels: + app: tcpecho +spec: + selector: + matchLabels: + app: tcpecho + template: + metadata: + labels: + app: tcpecho + spec: + containers: + - name: tcpecho + image: cjimti/go-echo + ports: + - containerPort: 1025 + env: + - name: POD_NAME + value: tlsroute-example-manifest +--- +apiVersion: v1 +kind: Service +metadata: + name: tcpecho +spec: + ports: + - port: 8888 + protocol: TCP + targetPort: 1025 + selector: + app: tcpecho + type: ClusterIP +--- +kind: GatewayClass +apiVersion: gateway.networking.k8s.io/v1alpha2 +metadata: + name: kong +spec: + controllerName: konghq.com/kic-gateway-controller +--- +kind: Gateway +apiVersion: gateway.networking.k8s.io/v1alpha2 +metadata: + name: kong +spec: + gatewayClassName: kong + listeners: + - name: http + protocol: HTTP + port: 80 + - name: tcp + protocol: TCP + port: 8888 +--- +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: TLSRoute +metadata: + name: tlsecho +spec: + parentRefs: + - name: kong + hostnames: + - tlsecho.kong.example + rules: + - backendRefs: + - name: tcpecho + port: 8888 diff --git a/internal/controllers/gateway/route_utils.go b/internal/controllers/gateway/route_utils.go index fb7e034bae..9163e75de4 100644 --- a/internal/controllers/gateway/route_utils.go +++ b/internal/controllers/gateway/route_utils.go @@ -28,6 +28,8 @@ func parentRefsForRoute(obj client.Object) ([]gatewayv1alpha2.ParentReference, e return v.Spec.ParentRefs, nil case *gatewayv1alpha2.TCPRoute: return v.Spec.ParentRefs, nil + case *gatewayv1alpha2.TLSRoute: + return v.Spec.ParentRefs, nil default: return nil, fmt.Errorf("cant determine parent gateway for unsupported type %s", reflect.TypeOf(obj)) } diff --git a/internal/controllers/gateway/tlsroute_controller.go b/internal/controllers/gateway/tlsroute_controller.go new file mode 100644 index 0000000000..80cefb50bb --- /dev/null +++ b/internal/controllers/gateway/tlsroute_controller.go @@ -0,0 +1,450 @@ +package gateway + +import ( + "context" + "fmt" + "reflect" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + "github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane" +) + +// ----------------------------------------------------------------------------- +// TLSRoute Controller - TLSRouteReconciler +// ----------------------------------------------------------------------------- + +// TLSRouteReconciler reconciles an TLSRoute object +type TLSRouteReconciler struct { + client.Client + + Log logr.Logger + Scheme *runtime.Scheme + DataplaneClient *dataplane.KongClient +} + +// SetupWithManager sets up the controller with the Manager. +func (r *TLSRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { + c, err := controller.New("tlsroute-controller", mgr, controller.Options{ + Reconciler: r, + Log: r.Log, + }) + if err != nil { + return err + } + + // if a GatewayClass updates then we need to enqueue the linked TLSRoutes to + // ensure that any route objects that may have been orphaned by that change get + // removed from data-plane configurations, and any routes that are now supported + // due to that change get added to data-plane configurations. + if err := c.Watch( + &source.Kind{Type: &gatewayv1alpha2.GatewayClass{}}, + handler.EnqueueRequestsFromMapFunc(r.listTLSRoutesForGatewayClass), + predicate.Funcs{ + GenericFunc: func(e event.GenericEvent) bool { return false }, // we don't need to enqueue from generic + CreateFunc: func(e event.CreateEvent) bool { return isGatewayClassEventInClass(r.Log, e) }, + UpdateFunc: func(e event.UpdateEvent) bool { return isGatewayClassEventInClass(r.Log, e) }, + DeleteFunc: func(e event.DeleteEvent) bool { return isGatewayClassEventInClass(r.Log, e) }, + }, + ); err != nil { + return err + } + + // if a Gateway updates then we need to enqueue the linked TLSRoutes to + // ensure that any route objects that may have been orphaned by that change get + // removed from data-plane configurations, and any routes that are now supported + // due to that change get added to data-plane configurations. + if err := c.Watch( + &source.Kind{Type: &gatewayv1alpha2.Gateway{}}, + handler.EnqueueRequestsFromMapFunc(r.listTLSRoutesForGateway), + ); err != nil { + return err + } + + // because of the additional burden of having to manage reference data-plane + // configurations for TLSRoute objects in the underlying Kong Gateway, we + // simply reconcile ALL TLSRoute objects. This allows us to drop the backend + // data-plane config for an TLSRoute if it somehow becomes disconnected from + // a supported Gateway and GatewayClass. + return c.Watch( + &source.Kind{Type: &gatewayv1alpha2.TLSRoute{}}, + &handler.EnqueueRequestForObject{}, + ) +} + +// ----------------------------------------------------------------------------- +// TLSRoute Controller - Event Handlers +// ----------------------------------------------------------------------------- + +// listTLSRoutesForGatewayClass is a controller-runtime event.Handler which +// produces a list of TLSRoutes which were bound to a Gateway which is or was +// bound to this GatewayClass. This implementation effectively does a map-reduce +// to determine the TLSRoutes as the relationship has to be discovered entirely +// by object reference. This relies heavily on the inherent performance benefits of +// the cached manager client to avoid API overhead. +func (r *TLSRouteReconciler) listTLSRoutesForGatewayClass(obj client.Object) []reconcile.Request { + // verify that the object is a GatewayClass + gwc, ok := obj.(*gatewayv1alpha2.GatewayClass) + if !ok { + r.Log.Error(fmt.Errorf("invalid type"), "found invalid type in event handlers", "expected", "GatewayClass", "found", reflect.TypeOf(obj)) + return nil + } + + // map all Gateway objects + gatewayList := gatewayv1alpha2.GatewayList{} + if err := r.Client.List(context.Background(), &gatewayList); err != nil { + r.Log.Error(err, "failed to list gateway objects from the cached client") + return nil + } + + // reduce for in-class Gateway objects + gateways := make(map[string]map[string]struct{}) + for _, gateway := range gatewayList.Items { + if string(gateway.Spec.GatewayClassName) == gwc.Name { + _, ok := gateways[gateway.Namespace] + if !ok { + gateways[gateway.Namespace] = make(map[string]struct{}) + } + gateways[gateway.Namespace][gateway.Name] = struct{}{} + } + } + + // if there are no Gateways associated with this GatewayClass we can stop + if len(gateways) == 0 { + return nil + } + + // map all TLSRoute objects + tlsrouteList := gatewayv1alpha2.TLSRouteList{} + if err := r.Client.List(context.Background(), &tlsrouteList); err != nil { + r.Log.Error(err, "failed to list tlsroute objects from the cached client") + return nil + } + + // reduce for TLSRoute objects bound to an in-class Gateway + queue := make([]reconcile.Request, 0) + for _, tlsroute := range tlsrouteList.Items { + // check the tlsroute's parentRefs + for _, parentRef := range tlsroute.Spec.ParentRefs { + // determine what namespace the parent gateway is in + namespace := tlsroute.Namespace + if parentRef.Namespace != nil { + namespace = string(*parentRef.Namespace) + } + + // if the gateway matches one of our previously filtered gateways, enqueue the route + if gatewaysForNamespace, ok := gateways[namespace]; ok { + if _, ok := gatewaysForNamespace[string(parentRef.Name)]; ok { + queue = append(queue, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: tlsroute.Namespace, + Name: tlsroute.Name, + }, + }) + } + } + } + } + + return queue +} + +// listTLSRoutesForGateway is a controller-runtime event.Handler which enqueues TLSRoute +// objects for changes to Gateway objects. The relationship between TLSRoutes and their +// Gateways (by way of .Spec.ParentRefs) must be discovered by object relation, so this +// implementation effectively does a map reduce to determine inclusion. This relies heavily +// on the inherent performance benefits of the cached manager client to avoid API overhead. +// +// NOTE: due to a race condition where a Gateway and a GatewayClass may be updated at the +// same time and could cause a changed Gateway object to look like it wasn't in-class +// while in reality it may still have active data-plane configurations because it was +// recently in-class, we can't reliably filter Gateway objects based on class as we +// can't verify that didn't change since we received the object. As such the current +// implementation enqueues ALL TLSRoute objects for reconciliation every time a Gateway +// changes. This is not ideal, but after communicating with other members of the +// community this appears to be a standard approach across multiple implementations at +// the moment for v1alpha2. As future releases of Gateway come out we'll need to +// continue iterating on this and perhaps advocating for upstream changes to help avoid +// this kind of problem without having to enqueue extra objects. +func (r *TLSRouteReconciler) listTLSRoutesForGateway(obj client.Object) []reconcile.Request { + // verify that the object is a Gateway + gw, ok := obj.(*gatewayv1alpha2.Gateway) + if !ok { + r.Log.Error(fmt.Errorf("invalid type"), "found invalid type in event handlers", "expected", "Gateway", "found", reflect.TypeOf(obj)) + return nil + } + + // map all TLSRoute objects + tlsrouteList := gatewayv1alpha2.TLSRouteList{} + if err := r.Client.List(context.Background(), &tlsrouteList); err != nil { + r.Log.Error(err, "failed to list tlsroute objects from the cached client") + return nil + } + + // reduce for TLSRoute objects bound to the Gateway + queue := make([]reconcile.Request, 0) + for _, tlsroute := range tlsrouteList.Items { + for _, parentRef := range tlsroute.Spec.ParentRefs { + namespace := tlsroute.Namespace + if parentRef.Namespace != nil { + namespace = string(*parentRef.Namespace) + } + if namespace == gw.Namespace && string(parentRef.Name) == gw.Name { + queue = append(queue, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: tlsroute.Namespace, + Name: tlsroute.Name, + }, + }) + } + } + } + + return queue +} + +// ----------------------------------------------------------------------------- +// TLSRoute Controller - Reconciliation +// ----------------------------------------------------------------------------- + +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=tlsroutes,verbs=get;list;watch +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=tlsroutes/status,verbs=get;update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +func (r *TLSRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := r.Log.WithValues("NetV1Alpha2TLSRoute", req.NamespacedName) + + tlsroute := new(gatewayv1alpha2.TLSRoute) + if err := r.Get(ctx, req.NamespacedName, tlsroute); err != nil { + // if the queued object is no longer present in the proxy cache we need + // to ensure that if it was ever added to the cache, it gets removed. + if errors.IsNotFound(err) { + debug(log, tlsroute, "object does not exist, ensuring it is not present in the proxy cache") + tlsroute.Namespace = req.Namespace + tlsroute.Name = req.Name + return ctrl.Result{}, r.DataplaneClient.DeleteObject(tlsroute) + } + + // for any error other than 404, requeue + return ctrl.Result{}, err + } + debug(log, tlsroute, "processing tlsroute") + + // if there's a present deletion timestamp then we need to update the proxy cache + // to drop all relevant routes from its configuration, regardless of whether or + // not we can find a valid gateway as that gateway may now be deleted but we still + // need to ensure removal of the data-plane configuration. + debug(log, tlsroute, "checking deletion timestamp") + if tlsroute.DeletionTimestamp != nil { + debug(log, tlsroute, "tlsroute is being deleted, re-configuring data-plane") + if err := r.DataplaneClient.DeleteObject(tlsroute); err != nil { + debug(log, tlsroute, "failed to delete object from data-plane, requeuing") + return ctrl.Result{}, err + } + debug(log, tlsroute, "ensured object was removed from the data-plane (if ever present)") + return ctrl.Result{}, r.DataplaneClient.DeleteObject(tlsroute) + } + + // we need to pull the Gateway parent objects for the TLSRoute to verify + // routing behavior and ensure compatibility with Gateway configurations. + debug(log, tlsroute, "retrieving GatewayClass and Gateway for route") + gateways, err := getSupportedGatewayForRoute(ctx, r.Client, tlsroute) + if err != nil { + if err.Error() == unsupportedGW { + debug(log, tlsroute, "unsupported route found, processing to verify whether it was ever supported") + // if there's no supported Gateway then this route could have been previously + // supported by this controller. As such we ensure that no supported Gateway + // references exist in the object status any longer. + statusUpdated, err := r.ensureGatewayReferenceStatusRemoved(ctx, tlsroute) + if err != nil { + // some failure happened so we need to retry to avoid orphaned statuses + return ctrl.Result{}, err + } + if statusUpdated { + // the status did in fact needed to be updated, so no need to requeue + // as the status update will trigger a requeue. + debug(log, tlsroute, "unsupported route was previously supported, status was updated") + return ctrl.Result{}, nil + } + + // if the route doesn't have a supported Gateway+GatewayClass associated with + // it it's possible it became orphaned after becoming queued. In either case + // ensure that it's removed from the proxy cache to avoid orphaned data-plane + // configurations. + debug(log, tlsroute, "ensuring that dataplane is updated to remove unsupported route (if applicable)") + return ctrl.Result{}, r.DataplaneClient.DeleteObject(tlsroute) + } + return ctrl.Result{}, err + } + + // the referenced gateway object(s) for the TLSRoute needs to be ready + // before we'll attempt any configurations of it. If it's not we'll + // requeue the object and wait until all supported gateways are ready. + debug(log, tlsroute, "checking if the tlsroute's gateways are ready") + for _, gateway := range gateways { + if !isGatewayReady(gateway) { + debug(log, tlsroute, "gateway for route was not ready, waiting") + return ctrl.Result{Requeue: true}, nil + } + } + + // if the gateways are ready, and the TLSRoute is destined for them, ensure that + // the object is pushed to the dataplane. + if err := r.DataplaneClient.UpdateObject(tlsroute); err != nil { + debug(log, tlsroute, "failed to update object in data-plane, requeueing") + return ctrl.Result{}, err + } + if r.DataplaneClient.AreKubernetesObjectReportsEnabled() { + // if the dataplane client has reporting enabled (this is the default and is + // tied in with status updates being enabled in the controller manager) then + // we will wait until the object is reported as successfully configured before + // moving on to status updates. + if !r.DataplaneClient.KubernetesObjectIsConfigured(tlsroute) { + return ctrl.Result{Requeue: true}, nil + } + } + + // now that the object has been successfully configured for in the dataplane + // we can update the object status to indicate that it's now properly linked + // to the configured Gateways. + debug(log, tlsroute, "ensuring status contains Gateway associations") + statusUpdated, err := r.ensureGatewayReferenceStatusAdded(ctx, tlsroute, gateways...) + if err != nil { + // don't proceed until the statuses can be updated appropriately + return ctrl.Result{}, err + } + if statusUpdated { + // if the status was updated it will trigger a follow-up reconciliation + // so we don't need to do anything further here. + return ctrl.Result{}, nil + } + + // once the data-plane has accepted the TLSRoute object, we're all set. + info(log, tlsroute, "tlsroute has been configured on the data-plane") + return ctrl.Result{}, nil +} + +// ----------------------------------------------------------------------------- +// TLSRouteReconciler - Status Helpers +// ----------------------------------------------------------------------------- + +// tlsrouteParentKind indicates the only object KIND that this TLSRoute +// implementation supports for route object parent references. +var tlsrouteParentKind = "Gateway" + +// ensureGatewayReferenceStatus takes any number of Gateways that should be +// considered "attached" to a given TLSRoute and ensures that the status +// for the TLSRoute is updated appropriately. +func (r *TLSRouteReconciler) ensureGatewayReferenceStatusAdded(ctx context.Context, tlsroute *gatewayv1alpha2.TLSRoute, gateways ...*gatewayv1alpha2.Gateway) (bool, error) { + // map the existing parentStatues to avoid duplications + parentStatuses := make(map[string]*gatewayv1alpha2.RouteParentStatus) + for _, existingParent := range tlsroute.Status.Parents { + namespace := tlsroute.Namespace + if existingParent.ParentRef.Namespace != nil { + namespace = string(*existingParent.ParentRef.Namespace) + } + existingParentCopy := existingParent + parentStatuses[namespace+string(existingParent.ParentRef.Name)] = &existingParentCopy + } + + // overlay the parent ref statuses for all new gateway references + statusChangesWereMade := false + for _, gateway := range gateways { + // build a new status for the parent Gateway + gatewayParentStatus := &gatewayv1alpha2.RouteParentStatus{ + ParentRef: gatewayv1alpha2.ParentReference{ + Group: (*gatewayv1alpha2.Group)(&gatewayv1alpha2.GroupVersion.Group), + Kind: (*gatewayv1alpha2.Kind)(&tlsrouteParentKind), + Namespace: (*gatewayv1alpha2.Namespace)(&gateway.Namespace), + Name: gatewayv1alpha2.ObjectName(gateway.Name), + }, + ControllerName: ControllerName, + Conditions: []metav1.Condition{{ + Type: string(gatewayv1alpha2.ConditionRouteAccepted), + Status: metav1.ConditionTrue, + ObservedGeneration: tlsroute.Generation, + LastTransitionTime: metav1.Now(), + Reason: string(gatewayv1alpha2.GatewayReasonReady), + }}, + } + + // if the reference already exists and doesn't require any changes + // then just leave it alone. + if existingGatewayParentStatus, exists := parentStatuses[gateway.Namespace+gateway.Name]; exists { + // fake the time of the existing status as this wont be equal + for i := range existingGatewayParentStatus.Conditions { + existingGatewayParentStatus.Conditions[i].LastTransitionTime = gatewayParentStatus.Conditions[0].LastTransitionTime + } + + // other than the condition timestamps, check if the statuses are equal + if reflect.DeepEqual(existingGatewayParentStatus, gatewayParentStatus) { + continue + } + } + + // otherwise overlay the new status on top the list of parentStatuses + parentStatuses[gateway.Namespace+gateway.Name] = gatewayParentStatus + statusChangesWereMade = true + } + + // if we didn't have to actually make any changes, no status update is needed + if !statusChangesWereMade { + return false, nil + } + + // update the tlsroute status with the new status references + tlsroute.Status.Parents = make([]gatewayv1alpha2.RouteParentStatus, 0, len(parentStatuses)) + for _, parent := range parentStatuses { + tlsroute.Status.Parents = append(tlsroute.Status.Parents, *parent) + } + + // update the object status in the API + if err := r.Status().Update(ctx, tlsroute); err != nil { + return false, err + } + + // the status needed an update and it was updated successfully + return true, nil +} + +// ensureGatewayReferenceStatusRemoved uses the ControllerName provided by the Gateway +// implementation to prune status references to Gateways supported by this controller +// in the provided TLSRoute object. +func (r *TLSRouteReconciler) ensureGatewayReferenceStatusRemoved(ctx context.Context, tlsroute *gatewayv1alpha2.TLSRoute) (bool, error) { + // drop all status references to supported Gateway objects + newStatuses := make([]gatewayv1alpha2.RouteParentStatus, 0) + for _, status := range tlsroute.Status.Parents { + if status.ControllerName != ControllerName { + newStatuses = append(newStatuses, status) + } + } + + // if the new list of statuses is the same length as the old + // nothing has changed and we're all done. + if len(newStatuses) == len(tlsroute.Status.Parents) { + return false, nil + } + + // update the object status in the API + tlsroute.Status.Parents = newStatuses + if err := r.Status().Update(ctx, tlsroute); err != nil { + return false, err + } + + // the status needed an update and it was updated successfully + return true, nil +} diff --git a/internal/dataplane/parser/parser.go b/internal/dataplane/parser/parser.go index 41e22f411e..f0dc337719 100644 --- a/internal/dataplane/parser/parser.go +++ b/internal/dataplane/parser/parser.go @@ -70,6 +70,7 @@ func (p *Parser) Build() (*kongstate.KongState, error) { p.ingressRulesFromHTTPRoutes(), p.ingressRulesFromUDPRoutes(), p.ingressRulesFromTCPRoutes(), + p.ingressRulesFromTLSRoutes(), ) // populate any Kubernetes Service objects relevant objects diff --git a/internal/dataplane/parser/translate_tlsroute.go b/internal/dataplane/parser/translate_tlsroute.go new file mode 100644 index 0000000000..35f36480c0 --- /dev/null +++ b/internal/dataplane/parser/translate_tlsroute.go @@ -0,0 +1,126 @@ +package parser + +import ( + "fmt" + + "github.com/kong/go-kong/kong" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + "github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane/kongstate" + "github.com/kong/kubernetes-ingress-controller/v2/internal/util" +) + +// ----------------------------------------------------------------------------- +// Translate TLSRoute - IngressRules Translation +// ----------------------------------------------------------------------------- + +// ingressRulesFromTLSRoutes processes a list of TLSRoute objects and translates +// then into Kong configuration objects. +func (p *Parser) ingressRulesFromTLSRoutes() ingressRules { + result := newIngressRules() + + tlsRouteList, err := p.storer.ListTLSRoutes() + if err != nil { + p.logger.WithError(err).Error("failed to list TLSRoutes") + return result + } + + var errs []error + for _, tlsroute := range tlsRouteList { + if err := p.ingressRulesFromTLSRoute(&result, tlsroute); err != nil { + err = fmt.Errorf("TLSRoute %s/%s can't be routed: %w", tlsroute.Namespace, tlsroute.Name, err) + errs = append(errs, err) + } else { + // at this point the object has been configured and can be + // reported as successfully parsed. + p.ReportKubernetesObjectUpdate(tlsroute) + } + } + + if len(errs) > 0 { + for _, err := range errs { + p.logger.Errorf(err.Error()) + } + } + + return result +} + +func (p *Parser) ingressRulesFromTLSRoute(result *ingressRules, tlsroute *gatewayv1alpha2.TLSRoute) error { + // first we grab the spec and gather some metdata about the object + spec := tlsroute.Spec + + if len(spec.Hostnames) == 0 { + return fmt.Errorf("no hostnames provided") + } + if len(spec.Rules) == 0 { + return fmt.Errorf("no rules provided") + } + + // each rule may represent a different set of backend services that will be accepting + // traffic, so we make separate routes and Kong services for every present rule. + for ruleNumber, rule := range spec.Rules { + // determine the routes needed to route traffic to services for this rule + routes, err := generateKongRoutesFromTLSRouteRule(tlsroute, ruleNumber, rule) + if err != nil { + return err + } + + // create a service and attach the routes to it + service, err := p.generateKongServiceFromBackendRef(result, tlsroute, ruleNumber, "tcp", rule.BackendRefs...) + if err != nil { + return err + } + service.Routes = append(service.Routes, routes...) + + // cache the service to avoid duplicates in further loop iterations + result.ServiceNameToServices[*service.Service.Name] = service + } + + return nil +} + +// ----------------------------------------------------------------------------- +// Translate TLSRoute - Utils +// ----------------------------------------------------------------------------- + +// generateKongRoutesFromTLSRouteRule converts an TLSRoute rule to one or more +// Kong Route objects to route traffic to services. +func generateKongRoutesFromTLSRouteRule( + tlsroute *gatewayv1alpha2.TLSRoute, + ruleNumber int, + rule gatewayv1alpha2.TLSRouteRule, +) ([]kongstate.Route, error) { + // gather the k8s object information and hostnames from the tlsroute + objectInfo := util.FromK8sObject(tlsroute) + + var routes []kongstate.Route + + if len(rule.BackendRefs) == 0 { + return routes, fmt.Errorf("TLSRoute rules must include at least one backendRef") + } + + routeName := kong.String(fmt.Sprintf( + "tlsroute.%s.%s.%d.%d", + tlsroute.Namespace, + tlsroute.Name, + ruleNumber, + 0, + )) + + hostnames := make([]string, len(tlsroute.Spec.Hostnames)) + for i, hostname := range tlsroute.Spec.Hostnames { + hostnames[i] = string(hostname) + } + + r := kongstate.Route{ + Ingress: objectInfo, + Route: kong.Route{ + Name: routeName, + Protocols: kong.StringSlice("tls"), + SNIs: kong.StringSlice(hostnames...), + }, + } + + return append(routes, r), nil +} diff --git a/internal/manager/controllerdef.go b/internal/manager/controllerdef.go index 10919f32af..75d8de1762 100644 --- a/internal/manager/controllerdef.go +++ b/internal/manager/controllerdef.go @@ -329,6 +329,21 @@ func setupControllers( DataplaneClient: dataplaneClient, }, }, + { + Enabled: featureGates[gatewayFeature], + AutoHandler: crdExistsChecker{ + GVR: schema.GroupVersionResource{ + Group: gatewayv1alpha2.SchemeGroupVersion.Group, + Version: gatewayv1alpha2.SchemeGroupVersion.Version, + Resource: "tlsroutes", + }}.CRDExists, + Controller: &gateway.TLSRouteReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("TLSRoute"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + }, + }, } return controllers, nil diff --git a/internal/store/fake_store.go b/internal/store/fake_store.go index e984d07f1c..6a66c06a53 100644 --- a/internal/store/fake_store.go +++ b/internal/store/fake_store.go @@ -36,6 +36,7 @@ type FakeObjects struct { HTTPRoutes []*gatewayv1alpha2.HTTPRoute UDPRoutes []*gatewayv1alpha2.UDPRoute TCPRoutes []*gatewayv1alpha2.TCPRoute + TLSRoutes []*gatewayv1alpha2.TLSRoute ReferencePolicies []*gatewayv1alpha2.ReferencePolicy TCPIngresses []*configurationv1beta1.TCPIngress UDPIngresses []*configurationv1beta1.UDPIngress @@ -90,11 +91,17 @@ func NewFakeStore( } } tcprouteStore := cache.NewStore(keyFunc) - for _, tcproute := range objects.UDPRoutes { + for _, tcproute := range objects.TCPRoutes { if err := tcprouteStore.Add(tcproute); err != nil { return nil, err } } + tlsrouteStore := cache.NewStore(keyFunc) + for _, tlsroute := range objects.TLSRoutes { + if err := tlsrouteStore.Add(tlsroute); err != nil { + return nil, err + } + } referencepolicyStore := cache.NewStore(keyFunc) for _, referencepolicy := range objects.ReferencePolicies { if err := referencepolicyStore.Add(referencepolicy); err != nil { @@ -179,6 +186,7 @@ func NewFakeStore( HTTPRoute: httprouteStore, UDPRoute: udprouteStore, TCPRoute: tcprouteStore, + TLSRoute: tlsrouteStore, ReferencePolicy: referencepolicyStore, TCPIngress: tcpIngressStore, UDPIngress: udpIngressStore, diff --git a/internal/store/fake_store_test.go b/internal/store/fake_store_test.go index 6279dffcf6..144079fac5 100644 --- a/internal/store/fake_store_test.go +++ b/internal/store/fake_store_test.go @@ -765,6 +765,56 @@ func TestFakeStoreUDPRoute(t *testing.T) { assert.Len(routes, 2, "expect two UDPRoutes") } +func TestFakeStoreTCPRoute(t *testing.T) { + assert := assert.New(t) + + classes := []*gatewayv1alpha2.TCPRoute{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: gatewayv1alpha2.TCPRouteSpec{}, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + }, + Spec: gatewayv1alpha2.TCPRouteSpec{}, + }, + } + store, err := NewFakeStore(FakeObjects{TCPRoutes: classes}) + assert.Nil(err) + assert.NotNil(store) + routes, err := store.ListTCPRoutes() + assert.Nil(err) + assert.Len(routes, 2, "expect two TCPRoutes") +} + +func TestFakeStoreTLSRoute(t *testing.T) { + assert := assert.New(t) + + classes := []*gatewayv1alpha2.TLSRoute{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: gatewayv1alpha2.TLSRouteSpec{}, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + }, + Spec: gatewayv1alpha2.TLSRouteSpec{}, + }, + } + store, err := NewFakeStore(FakeObjects{TLSRoutes: classes}) + assert.Nil(err) + assert.NotNil(store) + routes, err := store.ListTLSRoutes() + assert.Nil(err) + assert.Len(routes, 2, "expect two TLSRoutes") +} + func TestFakeStoreReferencePolicy(t *testing.T) { assert := assert.New(t) diff --git a/internal/store/store.go b/internal/store/store.go index f7fd40ac75..ccab7d33c1 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -85,6 +85,7 @@ type Storer interface { ListHTTPRoutes() ([]*gatewayv1alpha2.HTTPRoute, error) ListUDPRoutes() ([]*gatewayv1alpha2.UDPRoute, error) ListTCPRoutes() ([]*gatewayv1alpha2.TCPRoute, error) + ListTLSRoutes() ([]*gatewayv1alpha2.TLSRoute, error) ListReferencePolicies() ([]*gatewayv1alpha2.ReferencePolicy, error) ListTCPIngresses() ([]*kongv1beta1.TCPIngress, error) ListUDPIngresses() ([]*kongv1beta1.UDPIngress, error) @@ -129,6 +130,7 @@ type CacheStores struct { HTTPRoute cache.Store UDPRoute cache.Store TCPRoute cache.Store + TLSRoute cache.Store ReferencePolicy cache.Store // Kong Stores @@ -156,6 +158,7 @@ func NewCacheStores() (c CacheStores) { c.HTTPRoute = cache.NewStore(keyFunc) c.UDPRoute = cache.NewStore(keyFunc) c.TCPRoute = cache.NewStore(keyFunc) + c.TLSRoute = cache.NewStore(keyFunc) c.ReferencePolicy = cache.NewStore(keyFunc) c.KnativeIngress = cache.NewStore(keyFunc) c.Plugin = cache.NewStore(keyFunc) @@ -242,6 +245,8 @@ func (c CacheStores) Get(obj runtime.Object) (item interface{}, exists bool, err return c.UDPRoute.Get(obj) case *gatewayv1alpha2.TCPRoute: return c.TCPRoute.Get(obj) + case *gatewayv1alpha2.TLSRoute: + return c.TLSRoute.Get(obj) case *gatewayv1alpha2.ReferencePolicy: return c.ReferencePolicy.Get(obj) // ---------------------------------------------------------------------------- @@ -301,6 +306,8 @@ func (c CacheStores) Add(obj runtime.Object) error { return c.UDPRoute.Add(obj) case *gatewayv1alpha2.TCPRoute: return c.TCPRoute.Add(obj) + case *gatewayv1alpha2.TLSRoute: + return c.TLSRoute.Add(obj) case *gatewayv1alpha2.ReferencePolicy: return c.ReferencePolicy.Add(obj) // ---------------------------------------------------------------------------- @@ -361,6 +368,8 @@ func (c CacheStores) Delete(obj runtime.Object) error { return c.UDPRoute.Delete(obj) case *gatewayv1alpha2.TCPRoute: return c.TCPRoute.Delete(obj) + case *gatewayv1alpha2.TLSRoute: + return c.TLSRoute.Delete(obj) case *gatewayv1alpha2.ReferencePolicy: return c.ReferencePolicy.Delete(obj) // ---------------------------------------------------------------------------- @@ -575,6 +584,22 @@ func (s Store) ListTCPRoutes() ([]*gatewayv1alpha2.TCPRoute, error) { return tcproutes, nil } +// ListTLSRoutes returns the list of TLSRoutes in the TLSRoute cache store. +func (s Store) ListTLSRoutes() ([]*gatewayv1alpha2.TLSRoute, error) { + var tlsroutes []*gatewayv1alpha2.TLSRoute + if err := cache.ListAll(s.stores.TLSRoute, labels.NewSelector(), + func(ob interface{}) { + tlsroute, ok := ob.(*gatewayv1alpha2.TLSRoute) + if ok { + tlsroutes = append(tlsroutes, tlsroute) + } + }, + ); err != nil { + return nil, err + } + return tlsroutes, nil +} + // ListReferencePolicies returns the list of ReferencePolicies in the ReferencePolicy cache store. func (s Store) ListReferencePolicies() ([]*gatewayv1alpha2.ReferencePolicy, error) { var policies []*gatewayv1alpha2.ReferencePolicy diff --git a/test/integration/examples_test.go b/test/integration/examples_test.go index 15a0a77e39..9cf6841013 100644 --- a/test/integration/examples_test.go +++ b/test/integration/examples_test.go @@ -99,6 +99,10 @@ func TestHTTPRouteExample(t *testing.T) { var udpRouteExampleManifests = fmt.Sprintf("%s/gateway-udproute.yaml", examplesDIR) func TestUDPRouteExample(t *testing.T) { + t.Log("locking Gateway UDP ports") + udpMutex.Lock() + defer udpMutex.Unlock() + t.Logf("applying yaml manifest %s", strings.TrimPrefix(udpRouteExampleManifests, examplesDIR)) b, err := os.ReadFile(udpRouteExampleManifests) // TODO as of 2022-04-01, UDPRoute does not support using a different inbound port than the outbound @@ -134,8 +138,6 @@ func TestUDPRouteExample(t *testing.T) { var tcprouteExampleManifests = fmt.Sprintf("%s/gateway-tcproute.yaml", examplesDIR) func TestTCPRouteExample(t *testing.T) { - t.Parallel() - t.Log("locking Gateway TCP ports") tcpMutex.Lock() defer tcpMutex.Unlock() @@ -157,6 +159,31 @@ func TestTCPRouteExample(t *testing.T) { }, ingressWait, waitTick) } +var tlsrouteExampleManifests = fmt.Sprintf("%s/gateway-tlsroute.yaml", examplesDIR) + +func TestTLSRouteExample(t *testing.T) { + t.Log("locking Gateway TLS ports") + tlsMutex.Lock() + defer tlsMutex.Unlock() + + t.Logf("applying yaml manifest %s", tlsrouteExampleManifests) + b, err := os.ReadFile(tlsrouteExampleManifests) + require.NoError(t, err) + require.NoError(t, clusters.ApplyYAML(ctx, env.Cluster(), string(b))) + + defer func() { + t.Logf("deleting tlsroute example") + require.NoError(t, clusters.DeleteYAML(ctx, env.Cluster(), string(b))) + }() + + t.Log("verifying that TLSRoute becomes routable") + require.Eventually(t, func() bool { + responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), + "tlsroute-example-manifest", "tlsecho.kong.example") + return err == nil && responded + }, ingressWait, waitTick) +} + var ingressExampleManifests = fmt.Sprintf("%s/ingress.yaml", examplesDIR) func TestIngressExample(t *testing.T) { @@ -209,8 +236,6 @@ func TestIngressExample(t *testing.T) { var udpingressExampleManifests = fmt.Sprintf("%s/udpingress.yaml", examplesDIR) func TestUDPIngressExample(t *testing.T) { - t.Parallel() - t.Log("locking Gateway UDP ports") udpMutex.Lock() defer udpMutex.Unlock() diff --git a/test/integration/tlsroute_test.go b/test/integration/tlsroute_test.go new file mode 100644 index 0000000000..9a4c0d6825 --- /dev/null +++ b/test/integration/tlsroute_test.go @@ -0,0 +1,468 @@ +//go:build integration_tests +// +build integration_tests + +package integration + +import ( + "bytes" + "crypto/tls" + "errors" + "fmt" + "io" + "net" + "testing" + "time" + + "github.com/google/uuid" + ktfkong "github.com/kong/kubernetes-testing-framework/pkg/clusters/addons/kong" + "github.com/kong/kubernetes-testing-framework/pkg/utils/kubernetes/generators" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gatewayclient "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" + + "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/gateway" +) + +const ( + tlsRouteHostname = "tlsroute.kong.example" +) + +func TestTLSRouteEssentials(t *testing.T) { + backendPort := gatewayv1alpha2.PortNumber(tcpEchoPort) + t.Log("locking TLS port") + tlsMutex.Lock() + defer func() { + t.Log("unlocking TLS port") + tlsMutex.Unlock() + }() + ns, cleanup := namespace(t) + defer cleanup() + + // TODO consolidate into suite and use for all GW tests? + // https://github.com/Kong/kubernetes-ingress-controller/issues/2461 + t.Log("deploying a supported gatewayclass to the test cluster") + c, err := gatewayclient.NewForConfig(env.Cluster().Config()) + require.NoError(t, err) + gwc := &gatewayv1alpha2.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: uuid.NewString(), + }, + Spec: gatewayv1alpha2.GatewayClassSpec{ + ControllerName: gateway.ControllerName, + }, + } + gwc, err = c.GatewayV1alpha2().GatewayClasses().Create(ctx, gwc, metav1.CreateOptions{}) + require.NoError(t, err) + + defer func() { + t.Log("cleaning up gatewayclasses") + if err := c.GatewayV1alpha2().GatewayClasses().Delete(ctx, gwc.Name, metav1.DeleteOptions{}); err != nil { + if !apierrors.IsNotFound(err) { + assert.NoError(t, err) + } + } + }() + + t.Log("deploying a gateway to the test cluster using unmanaged gateway mode") + gw := &gatewayv1alpha2.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kong", + Annotations: map[string]string{ + unmanagedAnnotation: "true", // trigger the unmanaged gateway mode + }, + }, + Spec: gatewayv1alpha2.GatewaySpec{ + GatewayClassName: gatewayv1alpha2.ObjectName(gwc.Name), + Listeners: []gatewayv1alpha2.Listener{{ + Name: "tls", + Protocol: gatewayv1alpha2.TLSProtocolType, + Port: gatewayv1alpha2.PortNumber(ktfkong.DefaultTLSServicePort), + }}, + }, + } + gw, err = c.GatewayV1alpha2().Gateways(ns.Name).Create(ctx, gw, metav1.CreateOptions{}) + require.NoError(t, err) + + defer func() { + t.Log("cleaning up gateways") + if err := c.GatewayV1alpha2().Gateways(ns.Name).Delete(ctx, gw.Name, metav1.DeleteOptions{}); err != nil { + if !apierrors.IsNotFound(err) { + assert.NoError(t, err) + } + } + }() + + t.Log("creating a tcpecho pod to test TLSRoute traffic routing") + container1 := generators.NewContainer("tcpecho-1", tcpEchoImage, tcpEchoPort) + // go-echo sends a "Running on Pod ." immediately on connecting + testUUID1 := uuid.NewString() + container1.Env = []corev1.EnvVar{ + { + Name: "POD_NAME", + Value: testUUID1, + }, + } + deployment1 := generators.NewDeploymentForContainer(container1) + deployment1, err = env.Cluster().Client().AppsV1().Deployments(ns.Name).Create(ctx, deployment1, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Log("creating an additional tcpecho pod to test TLSRoute multiple backendRef loadbalancing") + container2 := generators.NewContainer("tcpecho-2", tcpEchoImage, tcpEchoPort) + // go-echo sends a "Running on Pod ." immediately on connecting + testUUID2 := uuid.NewString() + container2.Env = []corev1.EnvVar{ + { + Name: "POD_NAME", + Value: testUUID2, + }, + } + deployment2 := generators.NewDeploymentForContainer(container2) + deployment2, err = env.Cluster().Client().AppsV1().Deployments(ns.Name).Create(ctx, deployment2, metav1.CreateOptions{}) + require.NoError(t, err) + + defer func() { + t.Logf("cleaning up the deployments %s/%s and %s/%s", deployment1.Namespace, deployment1.Name, deployment2.Namespace, deployment2.Name) + assert.NoError(t, env.Cluster().Client().AppsV1().Deployments(ns.Name).Delete(ctx, deployment1.Name, metav1.DeleteOptions{})) + assert.NoError(t, env.Cluster().Client().AppsV1().Deployments(ns.Name).Delete(ctx, deployment2.Name, metav1.DeleteOptions{})) + }() + + t.Logf("exposing deployment %s/%s via service", deployment1.Namespace, deployment1.Name) + service1 := generators.NewServiceForDeployment(deployment1, corev1.ServiceTypeLoadBalancer) + service1, err = env.Cluster().Client().CoreV1().Services(ns.Name).Create(ctx, service1, metav1.CreateOptions{}) + assert.NoError(t, err) + + t.Logf("exposing deployment %s/%s via service", deployment2.Namespace, deployment2.Name) + service2 := generators.NewServiceForDeployment(deployment2, corev1.ServiceTypeLoadBalancer) + service2, err = env.Cluster().Client().CoreV1().Services(ns.Name).Create(ctx, service2, metav1.CreateOptions{}) + assert.NoError(t, err) + + defer func() { + t.Logf("cleaning up the service %s", service1.Name) + assert.NoError(t, env.Cluster().Client().CoreV1().Services(ns.Name).Delete(ctx, service1.Name, metav1.DeleteOptions{})) + assert.NoError(t, env.Cluster().Client().CoreV1().Services(ns.Name).Delete(ctx, service2.Name, metav1.DeleteOptions{})) + }() + + t.Logf("creating a tlsroute to access deployment %s via kong", deployment1.Name) + tlsroute := &gatewayv1alpha2.TLSRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: uuid.NewString(), + Annotations: map[string]string{}, + }, + Spec: gatewayv1alpha2.TLSRouteSpec{ + CommonRouteSpec: gatewayv1alpha2.CommonRouteSpec{ + ParentRefs: []gatewayv1alpha2.ParentReference{{ + Name: gatewayv1alpha2.ObjectName(gw.Name), + }}, + }, + Hostnames: []gatewayv1alpha2.Hostname{tlsRouteHostname}, + Rules: []gatewayv1alpha2.TLSRouteRule{{ + BackendRefs: []gatewayv1alpha2.BackendRef{{ + BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ + Name: gatewayv1alpha2.ObjectName(service1.Name), + Port: &backendPort, + }, + }}, + }}, + }, + } + tlsroute, err = c.GatewayV1alpha2().TLSRoutes(ns.Name).Create(ctx, tlsroute, metav1.CreateOptions{}) + require.NoError(t, err) + + defer func() { + t.Logf("cleaning up the tlsroute %s", tlsroute.Name) + if err := c.GatewayV1alpha2().TLSRoutes(ns.Name).Delete(ctx, tlsroute.Name, metav1.DeleteOptions{}); err != nil { + if !apierrors.IsNotFound(err) { + assert.NoError(t, err) + } + } + }() + + t.Log("verifying that the Gateway gets linked to the route via status") + tlseventuallyGatewayIsLinkedInStatus(t, c, ns.Name, tlsroute.Name) + + t.Log("verifying that the tcpecho is responding properly over TLS") + require.Eventually(t, func() bool { + responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), + testUUID1, tlsRouteHostname) + return err == nil && responded == true + }, ingressWait, waitTick) + + t.Log("removing the parentrefs from the TLSRoute") + oldParentRefs := tlsroute.Spec.ParentRefs + require.Eventually(t, func() bool { + tlsroute, err = c.GatewayV1alpha2().TLSRoutes(ns.Name).Get(ctx, tlsroute.Name, metav1.GetOptions{}) + require.NoError(t, err) + tlsroute.Spec.ParentRefs = nil + tlsroute, err = c.GatewayV1alpha2().TLSRoutes(ns.Name).Update(ctx, tlsroute, metav1.UpdateOptions{}) + return err == nil + }, time.Minute, time.Second) + + t.Log("verifying that the Gateway gets unlinked from the route via status") + tlseventuallyGatewayIsUnlinkedInStatus(t, c, ns.Name, tlsroute.Name) + + t.Log("verifying that the tcpecho is no longer responding") + require.Eventually(t, func() bool { + responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), + testUUID1, tlsRouteHostname) + return responded == false && errors.Is(err, io.EOF) + }, ingressWait, waitTick) + + t.Log("putting the parentRefs back") + require.Eventually(t, func() bool { + tlsroute, err = c.GatewayV1alpha2().TLSRoutes(ns.Name).Get(ctx, tlsroute.Name, metav1.GetOptions{}) + require.NoError(t, err) + tlsroute.Spec.ParentRefs = oldParentRefs + tlsroute, err = c.GatewayV1alpha2().TLSRoutes(ns.Name).Update(ctx, tlsroute, metav1.UpdateOptions{}) + return err == nil + }, time.Minute, time.Second) + + t.Log("verifying that the Gateway gets linked to the route via status") + tlseventuallyGatewayIsLinkedInStatus(t, c, ns.Name, tlsroute.Name) + + t.Log("verifying that putting the parentRefs back results in the routes becoming available again") + require.Eventually(t, func() bool { + responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), + testUUID1, tlsRouteHostname) + return err == nil && responded == true + }, ingressWait, waitTick) + + t.Log("deleting the GatewayClass") + oldGWCName := gwc.Name + require.NoError(t, c.GatewayV1alpha2().GatewayClasses().Delete(ctx, gwc.Name, metav1.DeleteOptions{})) + + t.Log("verifying that the Gateway gets unlinked from the route via status") + tlseventuallyGatewayIsUnlinkedInStatus(t, c, ns.Name, tlsroute.Name) + + t.Log("verifying that the data-plane configuration from the TLSRoute gets dropped with the GatewayClass now removed") + require.Eventually(t, func() bool { + responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), + testUUID1, tlsRouteHostname) + return responded == false && errors.Is(err, io.EOF) + }, ingressWait, waitTick) + + t.Log("putting the GatewayClass back") + gwc = &gatewayv1alpha2.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: oldGWCName, + }, + Spec: gatewayv1alpha2.GatewayClassSpec{ + ControllerName: gateway.ControllerName, + }, + } + gwc, err = c.GatewayV1alpha2().GatewayClasses().Create(ctx, gwc, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Log("verifying that the Gateway gets linked to the route via status") + tlseventuallyGatewayIsLinkedInStatus(t, c, ns.Name, tlsroute.Name) + + t.Log("verifying that creating the GatewayClass again triggers reconciliation of TLSRoutes and the route becomes available again") + require.Eventually(t, func() bool { + responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), + testUUID1, tlsRouteHostname) + return err == nil && responded == true + }, ingressWait, waitTick) + + t.Log("deleting the Gateway") + oldGWName := gw.Name + require.NoError(t, c.GatewayV1alpha2().Gateways(ns.Name).Delete(ctx, gw.Name, metav1.DeleteOptions{})) + + t.Log("verifying that the Gateway gets unlinked from the route via status") + tlseventuallyGatewayIsUnlinkedInStatus(t, c, ns.Name, tlsroute.Name) + + t.Log("verifying that the data-plane configuration from the TLSRoute gets dropped with the Gateway now removed") + require.Eventually(t, func() bool { + responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), + testUUID1, tlsRouteHostname) + return responded == false && errors.Is(err, io.EOF) + }, ingressWait, waitTick) + + t.Log("putting the Gateway back") + gw = &gatewayv1alpha2.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: oldGWName, + Annotations: map[string]string{ + unmanagedAnnotation: "true", // trigger the unmanaged gateway mode + }, + }, + Spec: gatewayv1alpha2.GatewaySpec{ + GatewayClassName: gatewayv1alpha2.ObjectName(gwc.Name), + Listeners: []gatewayv1alpha2.Listener{{ + Name: "tls", + Protocol: gatewayv1alpha2.TLSProtocolType, + Port: gatewayv1alpha2.PortNumber(ktfkong.DefaultTLSServicePort), + }}, + }, + } + gw, err = c.GatewayV1alpha2().Gateways(ns.Name).Create(ctx, gw, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Log("verifying that the Gateway gets linked to the route via status") + tlseventuallyGatewayIsLinkedInStatus(t, c, ns.Name, tlsroute.Name) + + t.Log("verifying that creating the Gateway again triggers reconciliation of TLSRoutes and the route becomes available again") + require.Eventually(t, func() bool { + responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), + testUUID1, tlsRouteHostname) + return err == nil && responded == true + }, ingressWait, waitTick) + + t.Log("adding an additional backendRef to the TLSRoute") + require.Eventually(t, func() bool { + tlsroute, err = c.GatewayV1alpha2().TLSRoutes(ns.Name).Get(ctx, tlsroute.Name, metav1.GetOptions{}) + require.NoError(t, err) + + tlsroute.Spec.Rules[0].BackendRefs = []gatewayv1alpha2.BackendRef{ + { + BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ + Name: gatewayv1alpha2.ObjectName(service1.Name), + Port: &backendPort, + }, + }, + { + BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ + Name: gatewayv1alpha2.ObjectName(service2.Name), + Port: &backendPort, + }, + }, + } + + tlsroute, err = c.GatewayV1alpha2().TLSRoutes(ns.Name).Update(ctx, tlsroute, metav1.UpdateOptions{}) + return err == nil + }, ingressWait, waitTick) + + t.Log("verifying that the TLSRoute is now load-balanced between two services") + require.Eventually(t, func() bool { + responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), + testUUID1, tlsRouteHostname) + return err == nil && responded == true + }, ingressWait, waitTick) + require.Eventually(t, func() bool { + responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), + testUUID2, tlsRouteHostname) + return err == nil && responded == true + }, ingressWait, waitTick) + + t.Log("deleting both GatewayClass and Gateway rapidly") + require.NoError(t, c.GatewayV1alpha2().GatewayClasses().Delete(ctx, gwc.Name, metav1.DeleteOptions{})) + require.NoError(t, c.GatewayV1alpha2().Gateways(ns.Name).Delete(ctx, gw.Name, metav1.DeleteOptions{})) + + t.Log("verifying that the Gateway gets unlinked from the route via status") + tlseventuallyGatewayIsUnlinkedInStatus(t, c, ns.Name, tlsroute.Name) + + t.Log("verifying that the data-plane configuration from the TLSRoute does not get orphaned with the GatewayClass and Gateway gone") + require.Eventually(t, func() bool { + responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), + testUUID1, tlsRouteHostname) + return responded == false && errors.Is(err, io.EOF) + }, ingressWait, waitTick) +} + +// TODO consolidate shared util gateway linked funcs +// https://github.com/Kong/kubernetes-ingress-controller/issues/2461 +func tlseventuallyGatewayIsLinkedInStatus(t *testing.T, c *gatewayclient.Clientset, namespace, name string) { + require.Eventually(t, func() bool { + // gather a fresh copy of the TLSRoute + tlsroute, err := c.GatewayV1alpha2().TLSRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) + require.NoError(t, err) + + // determine if there is a link to a supported Gateway + for _, parentStatus := range tlsroute.Status.Parents { + if parentStatus.ControllerName == gateway.ControllerName { + // supported Gateway link was found + return true + } + } + + // if no link was found yet retry + return false + }, ingressWait, waitTick) +} + +// TODO https://github.com/Kong/kubernetes-ingress-controller/issues/2461 +func tlseventuallyGatewayIsUnlinkedInStatus(t *testing.T, c *gatewayclient.Clientset, namespace, name string) { + require.Eventually(t, func() bool { + // gather a fresh copy of the TLSRoute + tlsroute, err := c.GatewayV1alpha2().TLSRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) + require.NoError(t, err) + + // determine if there is a link to a supported Gateway + for _, parentStatus := range tlsroute.Status.Parents { + if parentStatus.ControllerName == gateway.ControllerName { + // a supported Gateway link was found retry + return false + } + } + + // linked gateway is not present, all set + return true + }, ingressWait, waitTick) +} + +// tlsEchoResponds takes a TLS address URL and a Pod name and checks if a +// go-echo instance is running on that Pod at that address using hostname for SNI. +// It compares an expected message and its length against an expected message, returning true +// if it is and false and an error explanation if it is not +func tlsEchoResponds(url string, podName string, hostname string) (bool, error) { + dialer := net.Dialer{Timeout: time.Second * 10} + conn, err := tls.DialWithDialer(&dialer, + "tcp", + url, + &tls.Config{ + InsecureSkipVerify: true, //nolint:gosec + ServerName: hostname, + }) + if err != nil { + return false, err + } + defer conn.Close() + + header := []byte(fmt.Sprintf("Running on Pod %s.", podName)) + message := []byte("testing tlsroute") + + wrote, err := conn.Write(message) + if err != nil { + return false, err + } + + if wrote != len(message) { + return false, fmt.Errorf("wrote message of size %d, expected %d", wrote, len(message)) + } + + if err := conn.SetDeadline(time.Now().Add(time.Second * 5)); err != nil { + return false, err + } + + headerResponse := make([]byte, len(header)+1) + read, err := conn.Read(headerResponse) + if err != nil { + return false, err + } + + if read != len(header)+1 { // add 1 for newline + return false, fmt.Errorf("read %d bytes but expected %d", read, len(header)+1) + } + + if !bytes.Contains(headerResponse, header) { + return false, fmt.Errorf(`expected header response "%s", received: "%s"`, string(header), string(headerResponse)) + } + + messageResponse := make([]byte, wrote+1) + read, err = conn.Read(messageResponse) + if err != nil { + return false, err + } + + if read != len(message) { + return false, fmt.Errorf("read %d bytes but expected %d", read, len(message)) + } + + if !bytes.Contains(messageResponse, message) { + return false, fmt.Errorf(`expected message response "%s", received: "%s"`, string(message), string(messageResponse)) + } + + return true, nil +}