From 5d0bff637c19006d00c6e19c4f33ace4eda733e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Flendrich?= Date: Mon, 2 Nov 2020 21:25:52 +0100 Subject: [PATCH] E2E testing a KIC built from HEAD by CI. (#869) fixes #694 - makes CI build a local KIC image from HEAD - if on `main` or `next`: pushes to bintray. Can (or, maybe, should) be extended to all pushes - spins up a local microk8s cluster - pushes the local KIC to the local image registry - runs an instance of KIC - runs a set of tests: applying a bunch of manifests, waiting (open loop - `sleep 6`) for KIC to do its work, asserting on `curl` results - There are several possible ways to mitigate the `sleep` but none of them is perfect: - watch the `status` field of created resources - watch KIC logs for a successful/failed sync - watch the Kong Admin API `/config` endpoint * test(integration): implement harness and basic tests * test(integration): run on ci * chore(test): improve test cleanup * chore(test): switch from microk8s to kind * chore(makefile): define target `integration-test` * test(integration): patch instead of sed, disable anonymous reports * test(e2e): replace kubectl patch with kustomization * test(e2e): bump kubectl wait timeout to account for slow pulls * test(e2e): remove unused PROXY_IP variable * test(e2e): write README * test(e2e): add shebang to verify.sh files * test(e2e): gitignore leftover stuff --- .github/workflows/build-test.yaml | 80 +++++++++++++++++++ .github/workflows/build.yaml | 30 ------- .gitignore | 4 + CONTRIBUTING.md | 4 +- Makefile | 4 + test/integration/README.md | 68 ++++++++++++++++ .../cases/00-ingress-match-type/ingress.yaml | 44 ++++++++++ .../cases/00-ingress-match-type/verify.sh | 26 ++++++ test/integration/cases/01-https/ingress.yaml | 37 +++++++++ test/integration/cases/01-https/verify.sh | 9 +++ .../cases/02-ingress-apiversions/ingress.yaml | 32 ++++++++ .../cases/02-ingress-apiversions/verify.sh | 9 +++ .../03-plugin-correlation-id/ingress.yaml | 26 ++++++ .../cases/03-plugin-correlation-id/verify.sh | 4 + .../cases/04-plugin-key-auth/ingress.yaml | 43 ++++++++++ .../cases/04-plugin-key-auth/verify.sh | 5 ++ .../cases/05-ingress-override/ingress.yaml | 44 ++++++++++ .../cases/05-ingress-override/verify.sh | 8 ++ .../cases/06-default-backend/ingress.yaml | 22 +++++ .../cases/06-default-backend/verify.sh | 7 ++ test/integration/sut/kustomization.yaml | 9 +++ test/integration/sut/patch-deployment.yaml | 13 +++ test/integration/test.sh | 19 +++++ test/integration/util/create-cluster.sh | 19 +++++ test/integration/util/install-sut.sh | 13 +++ test/integration/util/run-all-tests.sh | 52 ++++++++++++ test/integration/util/run-one-test.sh | 36 +++++++++ test/integration/util/teardown-cluster.sh | 5 ++ 28 files changed, 641 insertions(+), 31 deletions(-) create mode 100644 .github/workflows/build-test.yaml delete mode 100644 .github/workflows/build.yaml create mode 100644 test/integration/README.md create mode 100644 test/integration/cases/00-ingress-match-type/ingress.yaml create mode 100755 test/integration/cases/00-ingress-match-type/verify.sh create mode 100644 test/integration/cases/01-https/ingress.yaml create mode 100755 test/integration/cases/01-https/verify.sh create mode 100644 test/integration/cases/02-ingress-apiversions/ingress.yaml create mode 100755 test/integration/cases/02-ingress-apiversions/verify.sh create mode 100644 test/integration/cases/03-plugin-correlation-id/ingress.yaml create mode 100755 test/integration/cases/03-plugin-correlation-id/verify.sh create mode 100644 test/integration/cases/04-plugin-key-auth/ingress.yaml create mode 100755 test/integration/cases/04-plugin-key-auth/verify.sh create mode 100644 test/integration/cases/05-ingress-override/ingress.yaml create mode 100755 test/integration/cases/05-ingress-override/verify.sh create mode 100644 test/integration/cases/06-default-backend/ingress.yaml create mode 100755 test/integration/cases/06-default-backend/verify.sh create mode 100644 test/integration/sut/kustomization.yaml create mode 100644 test/integration/sut/patch-deployment.yaml create mode 100755 test/integration/test.sh create mode 100755 test/integration/util/create-cluster.sh create mode 100755 test/integration/util/install-sut.sh create mode 100755 test/integration/util/run-all-tests.sh create mode 100755 test/integration/util/run-one-test.sh create mode 100755 test/integration/util/teardown-cluster.sh diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml new file mode 100644 index 0000000000..3de99eeda2 --- /dev/null +++ b/.github/workflows/build-test.yaml @@ -0,0 +1,80 @@ +name: Test + +on: [push] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Setup go + uses: actions/setup-go@v2 + with: + go-version: '^1.15' + + - name: Cache Go modules + uses: actions/cache@v1 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-build-codegen-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go + + - name: 'Check out the repo' + uses: actions/checkout@v2 + + - name: 'make container' + run: | + make REGISTRY=local IMGNAME=kong-ingress-controller TAG=ci container + + - run: docker save local/kong-ingress-controller:ci > image.tar + + - uses: actions/upload-artifact@v2 + with: + name: image + path: image.tar + + push-public: + needs: build + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/next' + + steps: + - uses: actions/download-artifact@v2 + with: + name: image + + - name: 'Arrange: Push the Kong Operator image to the public registry' + env: + BINTRAY_KEY: ${{ secrets.BINTRAY_KEY }} + BINTRAY_USER: ${{ secrets.BINTRAY_USER }} + run: | + set -e + + REPO="kong-docker-kubernetes-ingress-controller.bintray.io" + REF="$(git rev-parse --short HEAD)" + + DOCKER_TAG="$REPO/master:$REF" + docker load < image.tar + docker tag local/kong-ingress-controller:ci "$DOCKER_TAG" + echo "${BINTRAY_KEY}" | docker login -u "${BINTRAY_USER}" ${REPO} --password-stdin + docker push "$DOCKER_TAG" + + test-k8s: + needs: build + runs-on: ubuntu-latest + steps: + - name: 'Arrange: Check out the repo' + uses: actions/checkout@v2 + + - uses: actions/download-artifact@v2 + with: + name: image + + - name: 'Arrange: Push the image to the local registry' + run: | + docker load < image.tar + docker tag local/kong-ingress-controller:ci localhost:32000/kong-ingress-controller:ci + + - name: 'Test: Run integration tests' + run: | + env KIC_IMAGE=local/kong-ingress-controller:ci ./test/integration/test.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml deleted file mode 100644 index ccf77a70e0..0000000000 --- a/.github/workflows/build.yaml +++ /dev/null @@ -1,30 +0,0 @@ -name: Build - -on: - push: - branches: - - 'main' - - 'next' - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Setup go - uses: actions/setup-go@v2 - with: - go-version: '^1.15' - - name: Cache Go modules - uses: actions/cache@v1 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-build-codegen-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go - - name: Checkout repository - uses: actions/checkout@v2 - - name: Build and push - run: ./.ci/build-main.sh - env: - BINTRAY_KEY: ${{ secrets.BINTRAY_KEY }} - BINTRAY_USER: ${{ secrets.BINTRAY_USER }} diff --git a/.gitignore b/.gitignore index 9992fecd8b..c5c3589eeb 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,7 @@ kong-ingress-controller # ignore vendor tree vendor + +# integration test residuals +kind +kubeconfig-test-cluster diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3cb2f95c78..0ebbe25b32 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,7 +41,7 @@ patch](#submitting-a-patch) section. ## Dependencies -The build uses dependencies are managed by [go modules](https://blog.golang.org/using-go-modules) +The build uses dependencies are managed by [go modules](https://blog.golang.org/using-go-modules) ## Running in dev mode @@ -126,6 +126,8 @@ $ cd $GOPATH/src/github.com/kong/kubernetes-ingress-controller $ make test ``` +To run integration tests, see the [integration test readme](test/integration/README.md). + ## Releasing Makefile will produce a release binary, as shown above. To publish this diff --git a/Makefile b/Makefile index 6fc3e99b53..2bc5402935 100644 --- a/Makefile +++ b/Makefile @@ -50,3 +50,7 @@ container: .PHONY: run run: ./hack/dev/start.sh ${DB} ${RUN_VERSION} + +.PHONY: integration-test +integration-test: container + KIC_IMAGE="${IMAGE}:${TAG}" ./test/integration/test.sh diff --git a/test/integration/README.md b/test/integration/README.md new file mode 100644 index 0000000000..8432f2a3d6 --- /dev/null +++ b/test/integration/README.md @@ -0,0 +1,68 @@ +## E2E test + +The end-to-end (E2E) integration test performs the following: + +1. Start up an ephemeral Kubernetes cluster (KIND) +1. Install a container registry (localhost:5000) +1. Push the Kong Ingress Controller image to localhost:5000/kic:local +1. Run Kong Ingress Controller in the local Kubernetes cluster +1. Start up example services (echo and httpbin) +1. Run test cases (`/test/integration/cases/*`): + 1. Apply manifests from `.yaml` files present in the test case directory + 1. Wait for KIC to apply the configuration + 1. Run ./verify.sh to verify assertions + 1. Delete manifests created in the apply step above. + +### How to run the test + +```bash +make integration-test +``` + +This command builds a KIC Docker image and runs the test against that image. + +It is possible to run the test against any prebuilt KIC image, skipping the build step: + +```bash +env KIC_IMAGE=some-kic-image:tag ./test/integration/test.sh +``` + +### Troubleshooting + +If you want to troubleshoot a specific test case, here's how to do it: + +#### Run tests with `SKIP_TEARDOWN=yes` +By passing `SKIP_TEARDOWN=yes` to the test you can inspect the test environment after failure, and run certain test cases manually: + +```bash +make SKIP_TEARDOWN=yes integration-test +# or +env KIC_IMAGE=some-kic-image:tag SKIP_TEARDOWN=yes ./test/integration/test.sh +``` + +#### Access the test cluster with `kubectl` + +After the test invocation command with `SKIP_TEARDOWN=yes` set terminates, KIND will continue running in the cluster. You can access it using `./kubeconfig-test-cluster` as kubeconfig: + +```bash +kubectl --kubeconfig=./kubeconfig-test-cluster get pods --all-namespaces +``` + +#### Run a test case manually + +You can run a test case manually: +```bash +kubectl --kubeconfig=./kubeconfig-test-cluster port-forward -n kong svc/kong-proxy "27080:80" "27443:443" + +# in a separate terminal window: +kubectl --kubeconfig=./kubeconfig-test-cluster apply -f ./test/integration/cases/01-https +env SUT_HTTP_HOST=127.0.0.1:27080 SUT_HTTPS_HOST=127.0.0.1:27443 ./test/integration/cases/01-https/verify.sh +kubectl --kubeconfig=./kubeconfig-test-cluster delete -f ./test/integration/cases/01-https +``` + +#### Manually tear down the test cluster + +At the end of the debugging session, you can tear down the environment like this: +```bash +docker rm -f test-cluster-control-plane test-local-registry +``` diff --git a/test/integration/cases/00-ingress-match-type/ingress.yaml b/test/integration/cases/00-ingress-match-type/ingress.yaml new file mode 100644 index 0000000000..4a5ee7451b --- /dev/null +++ b/test/integration/cases/00-ingress-match-type/ingress.yaml @@ -0,0 +1,44 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: echo + annotations: + kubernetes.io/ingress.class: kong +spec: + rules: + - http: + paths: + - backend: + service: + name: echo + port: + number: 8080 + path: /foo + pathType: Prefix + - http: + paths: + - backend: + service: + name: echo + port: + number: 8080 + path: /bar1 + pathType: Exact + - http: + paths: + - backend: + service: + name: echo + port: + number: 8080 + path: /bar2/ + pathType: Exact + - http: + paths: + - backend: + service: + name: echo + port: + number: 8080 + path: "/baz+/" + pathType: ImplementationSpecific diff --git a/test/integration/cases/00-ingress-match-type/verify.sh b/test/integration/cases/00-ingress-match-type/verify.sh new file mode 100755 index 0000000000..4c3291108b --- /dev/null +++ b/test/integration/cases/00-ingress-match-type/verify.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -ex + +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/)" == 404 ] + +# Match type: Prefix +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/foo)" == 200 ] +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/foo/)" == 200 ] +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/fooo)" == 404 ] +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/fooo/)" == 404 ] +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/fooo/xxx)" == 404 ] +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/foo/xxx)" == 200 ] + +# Match type: Exact +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/bar1)" == 200 ] +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/bar1/)" == 404 ] +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/bar1/xxx)" == 404 ] +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/bar2)" == 404 ] +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/bar2/)" == 200 ] +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/bar2/xxx)" == 404 ] + +# Match type: ImplementationSpecific +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/baz)" == 404 ] +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/baz/)" == 200 ] +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/bazzz/)" == 200 ] +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/bazzz/xxx)" == 200 ] diff --git a/test/integration/cases/01-https/ingress.yaml b/test/integration/cases/01-https/ingress.yaml new file mode 100644 index 0000000000..4ce420d03a --- /dev/null +++ b/test/integration/cases/01-https/ingress.yaml @@ -0,0 +1,37 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: echo-http + annotations: + kubernetes.io/ingress.class: kong + konghq.com/protocols: http +spec: + rules: + - http: + paths: + - backend: + service: + name: echo + port: + number: 8080 + path: /foo + pathType: Prefix +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: echo-https + annotations: + kubernetes.io/ingress.class: kong + konghq.com/protocols: https +spec: + rules: + - http: + paths: + - backend: + service: + name: echo + port: + number: 8080 + path: /bar + pathType: Prefix diff --git a/test/integration/cases/01-https/verify.sh b/test/integration/cases/01-https/verify.sh new file mode 100755 index 0000000000..84509ff485 --- /dev/null +++ b/test/integration/cases/01-https/verify.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -ex + +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/foo)" == 200 ] +[ "$(curl -k -sw '%{http_code}' -o /dev/null https://$SUT_HTTPS_HOST/foo)" == 200 ] + +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/bar)" == 426 ] +[ "$(curl -k -sw '%{http_code}' -o /dev/null https://$SUT_HTTPS_HOST/bar)" == 200 ] + diff --git a/test/integration/cases/02-ingress-apiversions/ingress.yaml b/test/integration/cases/02-ingress-apiversions/ingress.yaml new file mode 100644 index 0000000000..f80ea8bf47 --- /dev/null +++ b/test/integration/cases/02-ingress-apiversions/ingress.yaml @@ -0,0 +1,32 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: echo-networkingv1beta1 + annotations: + kubernetes.io/ingress.class: kong +spec: + rules: + - http: + paths: + - backend: + serviceName: echo + servicePort: 8080 + path: /bar +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: echo-networkingv1 + annotations: + kubernetes.io/ingress.class: kong +spec: + rules: + - http: + paths: + - backend: + service: + name: echo + port: + number: 8080 + path: /baz + pathType: Prefix diff --git a/test/integration/cases/02-ingress-apiversions/verify.sh b/test/integration/cases/02-ingress-apiversions/verify.sh new file mode 100755 index 0000000000..3d5b5accdc --- /dev/null +++ b/test/integration/cases/02-ingress-apiversions/verify.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -ex + +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/)" == 404 ] + +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/bar/)" == 200 ] +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/baz/)" == 200 ] + +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/qux/)" == 404 ] diff --git a/test/integration/cases/03-plugin-correlation-id/ingress.yaml b/test/integration/cases/03-plugin-correlation-id/ingress.yaml new file mode 100644 index 0000000000..b6a0d1f19d --- /dev/null +++ b/test/integration/cases/03-plugin-correlation-id/ingress.yaml @@ -0,0 +1,26 @@ +apiVersion: configuration.konghq.com/v1 +kind: KongPlugin +metadata: + name: request-id +config: + header_name: my-request-id +plugin: correlation-id +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: echo + annotations: + kubernetes.io/ingress.class: kong + konghq.com/plugins: request-id +spec: + rules: + - http: + paths: + - backend: + service: + name: echo + port: + number: 8080 + path: /baz + pathType: Prefix diff --git a/test/integration/cases/03-plugin-correlation-id/verify.sh b/test/integration/cases/03-plugin-correlation-id/verify.sh new file mode 100755 index 0000000000..7c8c81ed28 --- /dev/null +++ b/test/integration/cases/03-plugin-correlation-id/verify.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -ex + +curl -v http://$SUT_HTTP_HOST/baz/ | grep "my-request-id" diff --git a/test/integration/cases/04-plugin-key-auth/ingress.yaml b/test/integration/cases/04-plugin-key-auth/ingress.yaml new file mode 100644 index 0000000000..9f203eefbc --- /dev/null +++ b/test/integration/cases/04-plugin-key-auth/ingress.yaml @@ -0,0 +1,43 @@ +apiVersion: configuration.konghq.com/v1 +kind: KongPlugin +metadata: + name: my-auth-plugin +plugin: key-auth +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: echo + annotations: + kubernetes.io/ingress.class: kong + konghq.com/plugins: my-auth-plugin +spec: + rules: + - http: + paths: + - backend: + service: + name: echo + port: + number: 8080 + path: /baz + pathType: Prefix +--- +apiVersion: configuration.konghq.com/v1 +kind: KongConsumer +metadata: + name: someuser + annotations: + kubernetes.io/ingress.class: kong +username: someuser +credentials: +- someuser-apikey +--- +apiVersion: v1 +kind: Secret +metadata: + name: someuser-apikey +data: + key: bXktc29vcGVyLXNlY3JldC1rZXk= + kongCredType: a2V5LWF1dGg= + diff --git a/test/integration/cases/04-plugin-key-auth/verify.sh b/test/integration/cases/04-plugin-key-auth/verify.sh new file mode 100755 index 0000000000..ac5676e789 --- /dev/null +++ b/test/integration/cases/04-plugin-key-auth/verify.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -ex + +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/baz)" == 401 ] +[ "$(curl -H "apikey: my-sooper-secret-key" -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/baz)" == 200 ] diff --git a/test/integration/cases/05-ingress-override/ingress.yaml b/test/integration/cases/05-ingress-override/ingress.yaml new file mode 100644 index 0000000000..981e9d3098 --- /dev/null +++ b/test/integration/cases/05-ingress-override/ingress.yaml @@ -0,0 +1,44 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: echo-has-override + annotations: + kubernetes.io/ingress.class: kong + konghq.com/override: my-override +spec: + rules: + - http: + paths: + - backend: + service: + name: echo + port: + number: 8080 + path: /foo + pathType: Prefix +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: echo-no-override + annotations: + kubernetes.io/ingress.class: kong +spec: + rules: + - http: + paths: + - backend: + service: + name: echo + port: + number: 8080 + path: /bar + pathType: Prefix +--- +apiVersion: configuration.konghq.com/v1 +kind: KongIngress +metadata: + name: my-override +route: + methods: + - GET diff --git a/test/integration/cases/05-ingress-override/verify.sh b/test/integration/cases/05-ingress-override/verify.sh new file mode 100755 index 0000000000..eaaa44eb20 --- /dev/null +++ b/test/integration/cases/05-ingress-override/verify.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -ex + +[ "$(curl -XPOST -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/foo)" == 404 ] +[ "$(curl -XGET -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/foo)" == 200 ] + +[ "$(curl -XPOST -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/bar)" == 200 ] +[ "$(curl -XGET -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/bar)" == 200 ] diff --git a/test/integration/cases/06-default-backend/ingress.yaml b/test/integration/cases/06-default-backend/ingress.yaml new file mode 100644 index 0000000000..61a1a3fa46 --- /dev/null +++ b/test/integration/cases/06-default-backend/ingress.yaml @@ -0,0 +1,22 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: echo + annotations: + kubernetes.io/ingress.class: kong +spec: + defaultBackend: + service: + name: httpbin + port: + number: 80 + rules: + - http: + paths: + - backend: + service: + name: echo + port: + number: 8080 + path: /foo + pathType: Prefix diff --git a/test/integration/cases/06-default-backend/verify.sh b/test/integration/cases/06-default-backend/verify.sh new file mode 100755 index 0000000000..9c6c2a1a81 --- /dev/null +++ b/test/integration/cases/06-default-backend/verify.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -ex + +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/)" == 200 ] +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/status/204)" == 204 ] +[ "$(curl -sw '%{http_code}' -o /dev/null http://$SUT_HTTP_HOST/foo/)" == 200 ] + diff --git a/test/integration/sut/kustomization.yaml b/test/integration/sut/kustomization.yaml new file mode 100644 index 0000000000..288da633db --- /dev/null +++ b/test/integration/sut/kustomization.yaml @@ -0,0 +1,9 @@ +resources: +- ../../../deploy/single/all-in-one-dbless.yaml +patchesStrategicMerge: +- patch-deployment.yaml +images: + - name: kong-docker-kubernetes-ingress-controller.bintray.io/kong-ingress-controller + newName: test-local-registry:5000/kic + newTag: local + diff --git a/test/integration/sut/patch-deployment.yaml b/test/integration/sut/patch-deployment.yaml new file mode 100644 index 0000000000..090ee03e8e --- /dev/null +++ b/test/integration/sut/patch-deployment.yaml @@ -0,0 +1,13 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ingress-kong + namespace: kong +spec: + template: + spec: + containers: + - name: ingress-controller + env: + - name: CONTROLLER_ANONYMOUS_REPORTS + value: "false" diff --git a/test/integration/test.sh b/test/integration/test.sh new file mode 100755 index 0000000000..5b2248a7e0 --- /dev/null +++ b/test/integration/test.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +SCRIPT_DIR="$(dirname "$BASH_SOURCE")" +export KUBECONFIG=$PWD/kubeconfig-test-cluster +export CLUSTER_NAME="test-cluster" +export REGISTRY_NAME="test-local-registry" + +export KIND_BINARY=./kind +export KIND_URL=https://github.com/kubernetes-sigs/kind/releases/download/v0.9.0/kind-linux-amd64 + +wget "$KIND_URL" -O "$KIND_BINARY" || exit 1 +chmod +x "$KIND_BINARY" || exit 1 + +"$SCRIPT_DIR/util/create-cluster.sh" || exit 1 +if [ -z "$SKIP_TEARDOWN" ]; then + trap "$SCRIPT_DIR/util/teardown-cluster.sh" EXIT +fi +"$SCRIPT_DIR/util/install-sut.sh" || exit 1 +"$SCRIPT_DIR/util/run-all-tests.sh" || exit 1 diff --git a/test/integration/util/create-cluster.sh b/test/integration/util/create-cluster.sh new file mode 100755 index 0000000000..e3ab647031 --- /dev/null +++ b/test/integration/util/create-cluster.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +REGISTRY_PORT=5000 +docker run -d --restart=always -p "$REGISTRY_PORT:5000" --name "$REGISTRY_NAME" registry:2 + +KIND_CONFIG=" +apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +name: $CLUSTER_NAME +containerdConfigPatches: +- |- + [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"$REGISTRY_NAME:$REGISTRY_PORT\"] + endpoint = [\"http://${REGISTRY_NAME}:${REGISTRY_PORT}\"] +" + +"$KIND_BINARY" create cluster --config=<(echo "$KIND_CONFIG") +docker network connect "kind" "$REGISTRY_NAME" diff --git a/test/integration/util/install-sut.sh b/test/integration/util/install-sut.sh new file mode 100755 index 0000000000..1a8543e12d --- /dev/null +++ b/test/integration/util/install-sut.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +REMOTE_IMAGE="localhost:5000/kic:local" + +docker tag "$KIC_IMAGE" "$REMOTE_IMAGE" +docker push "$REMOTE_IMAGE" + +SUT_ROOT="$(dirname "$BASH_SOURCE")/../sut" +kustomize build --load_restrictor none "$SUT_ROOT" | kubectl apply -f - + +kubectl wait --for=condition=Available --namespace=kong deploy/ingress-kong --timeout=300s diff --git a/test/integration/util/run-all-tests.sh b/test/integration/util/run-all-tests.sh new file mode 100755 index 0000000000..1a5451f0a3 --- /dev/null +++ b/test/integration/util/run-all-tests.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +cleanup() { + kill $(jobs -p) +} +trap cleanup EXIT + +CASES_DIR="$(dirname "$BASH_SOURCE")/../cases" +TEST_RUNNER="$(dirname "$BASH_SOURCE")/run-one-test.sh" + +echo ">>> Obtaining Kong proxy IP..." +HTTP_PORT=27080 +HTTPS_PORT=27443 +kubectl port-forward -n kong svc/kong-proxy "$HTTP_PORT:80" "$HTTPS_PORT:443" & +export SUT_HTTP_HOST="127.0.0.1:$HTTP_PORT" +export SUT_HTTPS_HOST="127.0.0.1:$HTTPS_PORT" +echo ">>> Kong proxy host is '$SUT_HTTP_HOST' for HTTP and '$SUT_HTTPS_HOST' for HTTPS." + +echo ">>> Setting up example services..." +setup_example_services() ( + set -ex + + kubectl apply -f https://bit.ly/sample-echo-service + kubectl apply -f https://bit.ly/sample-httpbin-service + + kubectl wait --for=condition=Available deploy echo --timeout=300s + kubectl wait --for=condition=Available deploy httpbin --timeout=300s +) + +setup_example_services || { echo ">>> ERROR: Failed to set up example services."; exit 1; } + +let TESTS_PASSED=0 TESTS_FAILED=0 +for CASE_PATH in "$CASES_DIR"/* +do + CASE_NAME="$(basename "$CASE_PATH")" + + if env \ + CASE_NAME="$CASE_NAME" \ + CASE_PATH="$CASE_PATH" \ + $TEST_RUNNER + then + let TESTS_PASSED++ + else + echo ">>> Test $CASE_NAME exited with status $?" + let TESTS_FAILED++ + fi +done + +echo ">>> Overall tests PASSED: $TESTS_PASSED" +echo ">>> Overall tests FAILED: $TESTS_FAILED" + +[[ $TESTS_FAILED == 0 ]] diff --git a/test/integration/util/run-one-test.sh b/test/integration/util/run-one-test.sh new file mode 100755 index 0000000000..0f2d6465f6 --- /dev/null +++ b/test/integration/util/run-one-test.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +fail_usage() { + echo ">>> ERR: Required environment variable $1 not set." + exit 100 +} + +cleanup() { + echo ">>> Test $CASE_NAME: cleanup" + kubectl delete -f "$CASE_PATH" +} +trap cleanup EXIT + +[ -n "$SUT_HTTP_HOST" ] || fail_usage SUT_HTTP_HOST +[ -n "$SUT_HTTPS_HOST" ] || fail_usage SUT_HTTPS_HOST +[ -n "$CASE_NAME" ] || fail_usage CASE_NAME +[ -n "$CASE_PATH" ] || fail_usage CASE_PATH + +echo ">>> Test $CASE_NAME: apply manifests" +kubectl apply -f "$CASE_PATH" || exit 1 + +echo ">>> Test $CASE_NAME: wait" +sleep 6 + +echo ">>> Test $CASE_NAME: verify" +"$CASE_PATH/verify.sh" +STATUS=$? + +if [ $STATUS != 0 ] +then + echo ">>> Test $CASE_NAME: FAIL (exit code $STATUS)" + exit $STATUS +fi + +echo ">>> Test $CASE_NAME: PASS" +exit 0 diff --git a/test/integration/util/teardown-cluster.sh b/test/integration/util/teardown-cluster.sh new file mode 100755 index 0000000000..ccedf87106 --- /dev/null +++ b/test/integration/util/teardown-cluster.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +"$KIND_BINARY" delete cluster "--name=$CLUSTER_NAME" +docker rm "$REGISTRY_NAME" -f +