From e5f3c7dfe6a75317101a965500f646eb7c749f17 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Tue, 12 Nov 2019 15:57:52 +0100 Subject: [PATCH 1/5] test/builder: add SecurityContext and PodSecurityContext functions Signed-off-by: Vincent Demeester --- test/builder/step.go | 7 +++++++ test/builder/task.go | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/test/builder/step.go b/test/builder/step.go index e54cc5c27d9..2b5ec0b6151 100644 --- a/test/builder/step.go +++ b/test/builder/step.go @@ -29,6 +29,13 @@ func StepCommand(args ...string) StepOp { } } +// StepSecurityContext sets the SecurityContext to the Step. +func StepSecurityContext(context *corev1.SecurityContext) StepOp { + return func(step *v1alpha1.Step) { + step.SecurityContext = context + } +} + // StepArgs sets the command arguments to the Container (step in this case). func StepArgs(args ...string) StepOp { return func(step *v1alpha1.Step) { diff --git a/test/builder/task.go b/test/builder/task.go index 54e89604614..879a9509d39 100644 --- a/test/builder/task.go +++ b/test/builder/task.go @@ -393,6 +393,13 @@ func TaskRunAffinity(affinity *corev1.Affinity) TaskRunSpecOp { } } +// TaskRunPodSecurityContext sets the SecurityContext to the TaskRunSpec (through PodTemplate). +func TaskRunPodSecurityContext(context *corev1.PodSecurityContext) TaskRunSpecOp { + return func(spec *v1alpha1.TaskRunSpec) { + spec.PodTemplate.SecurityContext = context + } +} + // StateTerminated set Terminated to the StepState. func StateTerminated(exitcode int) StepStateOp { return func(s *v1alpha1.StepState) { From fa14e2da8c0cd510f770a4d38a71f8b65d4b2caa Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Tue, 12 Nov 2019 15:58:04 +0100 Subject: [PATCH 2/5] test/build: fix task docstring Signed-off-by: Vincent Demeester --- test/builder/task.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/builder/task.go b/test/builder/task.go index 879a9509d39..4c552f56b44 100644 --- a/test/builder/task.go +++ b/test/builder/task.go @@ -372,21 +372,21 @@ func TaskRunNilTimeout(spec *v1alpha1.TaskRunSpec) { spec.Timeout = nil } -// TaskRunNodeSelector sets the NodeSelector to the PipelineSpec. +// TaskRunNodeSelector sets the NodeSelector to the TaskRunSpec. func TaskRunNodeSelector(values map[string]string) TaskRunSpecOp { return func(spec *v1alpha1.TaskRunSpec) { spec.PodTemplate.NodeSelector = values } } -// TaskRunTolerations sets the Tolerations to the PipelineSpec. +// TaskRunTolerations sets the Tolerations to the TaskRunSpec. func TaskRunTolerations(values []corev1.Toleration) TaskRunSpecOp { return func(spec *v1alpha1.TaskRunSpec) { spec.PodTemplate.Tolerations = values } } -// TaskRunAffinity sets the Affinity to the PipelineSpec. +// TaskRunAffinity sets the Affinity to the TaskRunSpec. func TaskRunAffinity(affinity *corev1.Affinity) TaskRunSpecOp { return func(spec *v1alpha1.TaskRunSpec) { spec.PodTemplate.Affinity = affinity From f50d04cd17fecea96d01caa44c7f7b26e4126358 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Tue, 12 Nov 2019 15:58:33 +0100 Subject: [PATCH 3/5] =?UTF-8?q?TestKanikoTaskRun:=20explicitely=20run=20as?= =?UTF-8?q?=20root=20=F0=9F=94=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It seems `kaniko` needs to be run as root to work, so be explicit about it. This is especially useful on kubernetes distribution that default to run as user (like OpenShift). Signed-off-by: Vincent Demeester --- test/kaniko_task_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/kaniko_task_test.go b/test/kaniko_task_test.go index d237e7e4a78..7e936ede657 100644 --- a/test/kaniko_task_test.go +++ b/test/kaniko_task_test.go @@ -130,6 +130,7 @@ func getImageResource(namespace, repo string) *v1alpha1.PipelineResource { } func getTask(repo, namespace string) *v1alpha1.Task { + root := int64(0) taskSpecOps := []tb.TaskSpecOp{ tb.TaskInputs(tb.InputsResource("gitsource", v1alpha1.PipelineResourceTypeGit)), tb.TaskOutputs(tb.OutputsResource("builtImage", v1alpha1.PipelineResourceTypeImage)), @@ -144,6 +145,7 @@ func getTask(repo, namespace string) *v1alpha1.Task { "--insecure-pull", "--insecure-registry=registry."+namespace+":5000/", ), + tb.StepSecurityContext(&corev1.SecurityContext{RunAsUser: &root}), } step := tb.Step("kaniko", "gcr.io/kaniko-project/executor:v0.13.0", stepOps...) taskSpecOps = append(taskSpecOps, step) From c009601ed799a0593db8a9b1ce39f8a17ccae400 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Tue, 12 Nov 2019 16:07:13 +0100 Subject: [PATCH 4/5] =?UTF-8?q?TestKanikoTaskRun:=20add=20docstring=20to?= =?UTF-8?q?=20getRemoteDigest=20=F0=9F=94=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Explaining why we are using a pod and skopeo. Signed-off-by: Vincent Demeester --- test/kaniko_task_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/kaniko_task_test.go b/test/kaniko_task_test.go index 7e936ede657..4c450ba1c01 100644 --- a/test/kaniko_task_test.go +++ b/test/kaniko_task_test.go @@ -164,6 +164,11 @@ func getTaskRun(namespace string) *v1alpha1.TaskRun { )) } +// getRemoteDigest starts a pod to query the registry from the namespace itself, using skopeo (and jq). +// The reason we have to do that is because the image is pushed on a local registry that is not exposed +// to the "outside" of the test, this means it can be query by the test itself. It can only be query from +// a pod in the namespace. skopeo is able to do that query and we use jq to extract the digest from its +// output. The image used for this pod is build in the tektoncd/plumbing repository. func getRemoteDigest(t *testing.T, c *clients, namespace, image string) (string, error) { t.Helper() podName := "skopeo-jq" From e301ca1b0e94faf882f5e5ffe818d3d6f919224a Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Tue, 12 Nov 2019 16:21:16 +0100 Subject: [PATCH 5/5] =?UTF-8?q?TestKanikoTaskRun:=20refactor=20withRegistr?= =?UTF-8?q?y=20removing=20sleep=20=F0=9F=98=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Watch the deployment replicas instead and create the service only once the deployment has the correct number of replicas. This removes the need for a random 5s sleep in the middle. Signed-off-by: Vincent Demeester --- test/registry_test.go | 33 ++++++++++++--------------------- test/wait.go | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/test/registry_test.go b/test/registry_test.go index 58eb638643a..32654b62c4f 100644 --- a/test/registry_test.go +++ b/test/registry_test.go @@ -19,40 +19,31 @@ package test import ( "testing" - "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" ) func withRegistry(t *testing.T, c *clients, namespace string) { - if _, err := c.KubeClient.Kube.AppsV1().Deployments(namespace).Create(getRegistryDeployment(namespace)); err != nil { + deployment := getRegistryDeployment(namespace) + if _, err := c.KubeClient.Kube.AppsV1().Deployments(namespace).Create(deployment); err != nil { t.Fatalf("Failed to create the local registry deployment: %v", err) } + if err := WaitForDeploymentState(c, deployment.Name, namespace, func(d *appsv1.Deployment) (bool, error) { + var replicas int32 = 1 + if d.Spec.Replicas != nil { + replicas = *d.Spec.Replicas + } + return d.Status.ReadyReplicas == replicas, nil + }, "DeploymentPodRunning"); err != nil { + t.Fatalf("Error waiting for Deployment %q to be ready: %v", deployment.Name, err) + } + service := getRegistryService(namespace) if _, err := c.KubeClient.Kube.CoreV1().Services(namespace).Create(service); err != nil { t.Fatalf("Failed to create the local registry service: %v", err) } - set := labels.Set(service.Spec.Selector) - - // Give it a little bit of time to at least create the pod - time.Sleep(5 * time.Second) - - if pods, err := c.KubeClient.Kube.CoreV1().Pods(namespace).List(metav1.ListOptions{LabelSelector: set.AsSelector().String()}); err != nil { - t.Fatalf("Failed to list Pods of service[%s] error:%v", service.GetName(), err) - } else { - if len(pods.Items) != 1 { - t.Fatalf("Only 1 pod for service %s should be running: %v", service, pods.Items) - } - - if err := WaitForPodState(c, pods.Items[0].Name, namespace, func(pod *corev1.Pod) (bool, error) { - return pod.Status.Phase == "Running", nil - }, "PodContainersRunning"); err != nil { - t.Fatalf("Error waiting for Pod %q to run: %v", pods.Items[0].Name, err) - } - } } func getRegistryDeployment(namespace string) *appsv1.Deployment { diff --git a/test/wait.go b/test/wait.go index 5eb6935c0a0..4289302695d 100644 --- a/test/wait.go +++ b/test/wait.go @@ -51,6 +51,7 @@ import ( "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1" "go.opencensus.io/trace" "golang.org/x/xerrors" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -86,6 +87,24 @@ func WaitForTaskRunState(c *clients, name string, inState TaskRunStateFn, desc s }) } +// WaitForDeploymentState polls the status of the Deployment called name +// from client every interval until inState returns `true` indicating it is done, +// returns an error or timeout. desc will be used to name the metric that is emitted to +// track how long it took for name to get into the state checked by inState. +func WaitForDeploymentState(c *clients, name string, namespace string, inState func(d *appsv1.Deployment) (bool, error), desc string) error { + metricName := fmt.Sprintf("WaitForDeploymentState/%s/%s", name, desc) + _, span := trace.StartSpan(context.Background(), metricName) + defer span.End() + + return wait.PollImmediate(interval, timeout, func() (bool, error) { + d, err := c.KubeClient.Kube.AppsV1().Deployments(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return true, err + } + return inState(d) + }) +} + // WaitForPodState polls the status of the Pod called name from client every // interval until inState returns `true` indicating it is done, returns an // error or timeout. desc will be used to name the metric that is emitted to