Improve test stability

.github/workflows/ci.yml

@@ -24,14 +24,17 @@ jobs:
  run: make test-unit-coverage
  integration:
  strategy:
+ fail-fast: false
  matrix:
  go-version: [1.15.x]
  os: [ubuntu-latest]
  kubernetes:
- - v1.17.17
+ # Only v1.18 is currently enabled because of the flakiness in the tests, specifically API calls failing with "etcdserver: request timed out"
+ #- v1.17.17
  - v1.18.15
- - v1.19.7
- - v1.20.2
+ #- v1.19.7
+ #- v1.20.2
+ max-parallel: 1
  runs-on: ${{ matrix.os }}
  steps:
  - name: Install Go
@@ -64,14 +67,17 @@ jobs:
  run: make test-integration
  e2e:
  strategy:
+ fail-fast: false
  matrix:
  go-version: [1.15.x]
  os: [ubuntu-latest]
  kubernetes:
- - v1.17.17
+ # Only v1.18 is currently enabled because of the flakiness in the tests, specifically API calls failing with "etcdserver: request timed out"
+ #- v1.17.17
  - v1.18.15
- - v1.19.7
- - v1.20.2
+ #- v1.19.7
+ #- v1.20.2
+ max-parallel: 2
  runs-on: ${{ matrix.os }}
  steps:
  - name: Install Go
@@ -107,7 +113,7 @@ jobs:
  kubectl apply -f test/data/registry.yaml
  kubectl -n registry rollout status deployment registry --timeout=1m
  - name: Install ko
- run: curl -fsL https:/google/ko/releases/download/v0.8.0/ko_0.8.0_Linux_x86_64.tar.gz | sudo tar xzf - -C /usr/local/bin ko
+ run: curl -fsL https:/google/ko/releases/download/v0.8.1/ko_0.8.1_Linux_x86_64.tar.gz | sudo tar xzf - -C /usr/local/bin ko
  - name: Install Shipwright Build
  run: |
  make install-operator-kind
@@ -117,5 +123,14 @@ jobs:
  - name: Build operator logs
  if: ${{ failure() }}
  run: |
- POD_NAME=$(kubectl -n build-operator get pod -o json | jq -r '.items[0].metadata.name')
+ echo "# Pods:"
+ kubectl -n build-operator get pod
+ PODS=$(kubectl -n build-operator get pod -o json)
+ POD_NAME=$(echo "${PODS}" | jq -r '.items[] | select(.metadata.name | startswith("build-operator-")) | .metadata.name')
+ RESTART_COUNT=$(echo "${PODS}" | jq -r ".items[] | select(.metadata.name == \"${POD_NAME}\") | .status.containerStatuses[0].restartCount")
+ if [ "${RESTART_COUNT}" != "0" ]; then
+ echo "# Previous logs:"
+ kubectl -n build-operator logs "${POD_NAME}" --previous
+ fi
+ echo "# Logs:"
  kubectl -n build-operator logs "${POD_NAME}"

.travis.yml

@@ -7,7 +7,7 @@ services:
 install:
  - |
  # Install ko
- curl -fsL https:/google/ko/releases/download/v0.8.0/ko_0.8.0_Linux_x86_64.tar.gz | sudo tar xzf - -C /usr/local/bin ko
+ curl -fsL https:/google/ko/releases/download/v0.8.1/ko_0.8.1_Linux_x86_64.tar.gz | sudo tar xzf - -C /usr/local/bin ko
 
 deploy:
  - provider: script

test/clusterbuildstrategy_samples.go

@@ -247,6 +247,29 @@ spec:
  memory: 128Mi
 `
 
+// ClusterBuildStrategySleep30s is a strategy that does only sleep 30 seconds
+const ClusterBuildStrategySleep30s = `
+apiVersion: build.dev/v1alpha1
+kind: ClusterBuildStrategy
+metadata:
+ name: noop
+spec:
+ buildSteps:
+ - name: sleep30
+ image: alpine:latest
+ command:
+ - sleep
+ args:
+ - "30s"
+ resources:
+ limits:
+ cpu: 250m
+ memory: 128Mi
+ requests:
+ cpu: 250m
+ memory: 128Mi
+`
+
 // ClusterBuildStrategyWithAnnotations is a cluster build strategy that contains annotations
 const ClusterBuildStrategyWithAnnotations = `
 apiVersion: build.dev/v1alpha1

test/e2e/samples.go

@@ -18,6 +18,7 @@ import (
  framework "github.com/operator-framework/operator-sdk/pkg/test"
  "knative.dev/pkg/apis"
 
+ corev1 "k8s.io/api/core/v1"
  v1 "k8s.io/api/core/v1"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  "k8s.io/apimachinery/pkg/labels"
@@ -81,7 +82,7 @@ func amendBuild(identifier string, b *operator.Build) {
 }
 
 // CreateBuild loads the builds definition from the file path, unifies the output image based on
-// the identifier and creates it in a namespace
+// the identifier, creates it in a namespace and waits for it to be registered
 func createBuild(ctx *framework.Context, namespace string, identifier string, filePath string, timeout time.Duration, retry time.Duration) {
  Logf("Creating build %s", identifier)
 
@@ -98,6 +99,23 @@ func createBuild(ctx *framework.Context, namespace string, identifier string, fi
  Expect(err).ToNot(HaveOccurred(), "Unable to create build %s", identifier)
 
  Logf("Build %s created", identifier)
+
+ buildName := types.NamespacedName{
+ Namespace: namespace,
+ Name: b.Name,
+ }
+
+ trueCondition := corev1.ConditionTrue
+ falseCondition := corev1.ConditionFalse
+
+ Eventually(func() corev1.ConditionStatus {
+ err = clientGet(buildName, b)
+ Expect(err).ToNot(HaveOccurred(), "Error retrieving a build")
+
+ Expect(b.Status.Registered).ToNot(Equal(falseCondition), "Build registered status is false")
+
+ return b.Status.Registered
+ }, time.Duration(20*getTimeoutMultiplier())*time.Second, time.Second).Should(Equal(trueCondition), "Build was not registered")
 }
 
 // retrieveBuildAndBuildRun will retrieve the build and buildRun

test/integration/build_to_buildruns_test.go

@@ -82,6 +82,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
  Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+ buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+ Expect(err).To(BeNil())
+
  Expect(tb.CreateBR(buildRunObject)).To(BeNil())
 
  br, err := tb.GetBRTillCompletion(buildRunObject.Name)
@@ -104,6 +107,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
  Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+ buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+ Expect(err).To(BeNil())
+
  Expect(tb.CreateBR(buildRunObject)).To(BeNil())
 
  br, err := tb.GetBRTillCompletion(buildRunObject.Name)
@@ -118,6 +124,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
  Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+ buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+ Expect(err).To(BeNil())
+
  buildRun, err := tb.Catalog.LoadBRWithNameAndRef(
  BUILDRUN+tb.Namespace,
  BUILD+tb.Namespace,
@@ -149,6 +158,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
  Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+ buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+ Expect(err).To(BeNil())
+
  Expect(tb.CreateBR(buildRunObject)).To(BeNil())
 
  // Wait for BR to get an Starttime
@@ -176,6 +188,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
  Expect(tb.CreateBuild(build)).To(BeNil())
 
+ buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+ Expect(err).To(BeNil())
+
  Expect(tb.CreateBR(buildRunObject)).To(BeNil())
 
  br, err := tb.GetBRTillStartTime(buildRunObject.Name)
@@ -237,6 +252,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
  Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+ buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+ Expect(err).To(BeNil())
+
  Expect(tb.CreateBR(buildRunObject)).To(BeNil())
 
  br, err := tb.GetBRTillCompletion(buildRunObject.Name)
@@ -259,6 +277,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
  Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+ buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+ Expect(err).To(BeNil())
+
  buildRun, err := tb.Catalog.LoadBRWithNameAndRef(
  BUILDRUN+tb.Namespace,
  BUILD+tb.Namespace+"foobar",
@@ -288,6 +309,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
  Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+ buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+ Expect(err).To(BeNil())
+
  buildRun01, err := tb.Catalog.LoadBRWithNameAndRef(
  BUILDRUN+tb.Namespace+"01",
  BUILD+tb.Namespace,
@@ -340,6 +364,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
  Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+ buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+ Expect(err).To(BeNil())
+
  autoDeleteBuildRun, err := tb.Catalog.LoadBRWithNameAndRef(
  BUILDRUN+tb.Namespace,
  BUILD+tb.Namespace,
@@ -352,7 +379,7 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
  _, err = tb.GetBRTillStartTime(autoDeleteBuildRun.Name)
  Expect(err).To(BeNil())
 
- br, err := tb.GetBR(BUILDRUN + tb.Namespace)
+ br, err := tb.GetBRTillOwner(BUILDRUN+tb.Namespace, buildObject.Name)
  Expect(err).To(BeNil())
  Expect(ownerReferenceNames(br.OwnerReferences)).Should(ContainElement(buildObject.Name))
 
@@ -369,6 +396,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
  Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+ buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+ Expect(err).To(BeNil())
+
  autoDeleteBuildRun, err := tb.Catalog.LoadBRWithNameAndRef(
  BUILDRUN+tb.Namespace,
  BUILD+tb.Namespace,
@@ -389,7 +419,7 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
  err = tb.DeleteBuild(BUILD + tb.Namespace)
  Expect(err).To(BeNil())
 
- br, err := tb.GetBR(BUILDRUN + tb.Namespace)
+ br, err := tb.GetBRTillNotOwner(BUILDRUN+tb.Namespace, buildObject.Name)
  Expect(err).To(BeNil())
  Expect(ownerReferenceNames(br.OwnerReferences)).ShouldNot(ContainElement(buildObject.Name))
 
@@ -398,6 +428,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
  Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+ buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+ Expect(err).To(BeNil())
+
  autoDeleteBuildRun, err := tb.Catalog.LoadBRWithNameAndRef(
  BUILDRUN+tb.Namespace,
  BUILD+tb.Namespace,
@@ -418,7 +451,7 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
  err = tb.DeleteBuild(BUILD + tb.Namespace)
  Expect(err).To(BeNil())
 
- br, err := tb.GetBR(BUILDRUN + tb.Namespace)
+ br, err := tb.GetBRTillNotOwner(BUILDRUN+tb.Namespace, buildObject.Name)
  Expect(err).To(BeNil())
  Expect(ownerReferenceNames(br.OwnerReferences)).ShouldNot(ContainElement(buildObject.Name))
 
@@ -427,6 +460,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
  Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+ buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+ Expect(err).To(BeNil())
+
  autoDeleteBuildRun, err := tb.Catalog.LoadBRWithNameAndRef(
  BUILDRUN+tb.Namespace,
  BUILD+tb.Namespace,
@@ -457,7 +493,7 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
  Expect(err).To(BeNil())
  Expect(patchedBuild.Annotations[v1alpha1.AnnotationBuildRunDeletion]).To(Equal("true"))
 
- br, err := tb.GetBR(BUILDRUN + tb.Namespace)
+ br, err := tb.GetBRTillOwner(BUILDRUN+tb.Namespace, buildObject.Name)
  Expect(err).To(BeNil())
  Expect(ownerReferenceNames(br.OwnerReferences)).Should(ContainElement(buildObject.Name))
 

test/integration/buildruns_to_sa_test.go

@@ -6,6 +6,7 @@ package integration_test
 
 import (
  "fmt"
+
  corev1 "k8s.io/api/core/v1"
 
  . "github.com/onsi/ginkgo"
@@ -71,6 +72,9 @@ var _ = Describe("Integration tests BuildRuns and Service-accounts", func() {
 
  Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+ buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+ Expect(err).To(BeNil())
+
  Expect(tb.CreateBR(buildRunObject)).To(BeNil())
 
  _, err = tb.GetBRTillStartTime(buildRunObject.Name)
@@ -99,6 +103,9 @@ var _ = Describe("Integration tests BuildRuns and Service-accounts", func() {
 
  Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+ buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+ Expect(err).To(BeNil())
+
  Expect(tb.CreateBR(buildRunObject)).To(BeNil())
 
  _, err = tb.GetBRTillStartTime(buildRunObject.Name)
@@ -116,6 +123,9 @@ var _ = Describe("Integration tests BuildRuns and Service-accounts", func() {
  }
  Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+ buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+ Expect(err).To(BeNil())
+
  Expect(tb.CreateBR(buildRunObject)).To(BeNil())
 
  _, err = tb.GetBRTillStartTime(buildRunObject.Name)
@@ -137,6 +147,9 @@ var _ = Describe("Integration tests BuildRuns and Service-accounts", func() {
  It("it fails and updates buildrun conditions if the specified serviceaccount doesn't exist", func() {
  Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+ buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+ Expect(err).To(BeNil())
+
  Expect(tb.CreateBR(buildRunObject)).To(BeNil())
 
  br, err := tb.GetBRTillCompletion(buildRunObject.Name)
@@ -149,4 +162,3 @@ var _ = Describe("Integration tests BuildRuns and Service-accounts", func() {
  })
  })
 })
-