From f603c8b7c365e570d35f2995cfac8a798d357833 Mon Sep 17 00:00:00 2001 From: Abhinav Pathak Date: Mon, 3 May 2021 22:07:51 +0530 Subject: [PATCH] add integration test for ipamd env variables Tests the following environment variables - CLUSTER_NAME - ADDITIONAL_ENI_TAGS - INTROSPECTION_BIND_ADDRESS - DISABLE_INTROSPECTION - DISABLE_METRICS --- test/framework/resources/aws/services/ec2.go | 9 + .../resources/k8s/manifest/container.go | 8 + test/framework/resources/k8s/manifest/job.go | 8 + test/framework/resources/k8s/resources/job.go | 1 + test/integration-new/ipamd/eni_tag_test.go | 166 ++++++++++++++++++ .../ipamd/introspection_test.go | 143 +++++++++++++++ .../integration-new/ipamd/ipamd_suite_test.go | 12 -- test/integration-new/ipamd/metrics_test.go | 102 +++++++++++ 8 files changed, 437 insertions(+), 12 deletions(-) create mode 100644 test/integration-new/ipamd/eni_tag_test.go create mode 100644 test/integration-new/ipamd/introspection_test.go create mode 100644 test/integration-new/ipamd/metrics_test.go diff --git a/test/framework/resources/aws/services/ec2.go b/test/framework/resources/aws/services/ec2.go index 0f65afbfef..2349375704 100644 --- a/test/framework/resources/aws/services/ec2.go +++ b/test/framework/resources/aws/services/ec2.go @@ -25,6 +25,7 @@ import ( type EC2 interface { DescribeInstanceType(instanceType string) ([]*ec2.InstanceTypeInfo, error) DescribeInstance(instanceID string) (*ec2.Instance, error) + DescribeNetworkInterface(interfaceIDs []string) (*ec2.DescribeNetworkInterfacesOutput, error) AuthorizeSecurityGroupIngress(groupID string, protocol string, fromPort int, toPort int, cidrIP string) error RevokeSecurityGroupIngress(groupID string, protocol string, fromPort int, toPort int, cidrIP string) error AuthorizeSecurityGroupEgress(groupID string, protocol string, fromPort int, toPort int, cidrIP string) error @@ -141,6 +142,14 @@ func (d *defaultEC2) RevokeSecurityGroupEgress(groupID string, protocol string, return err } +func (d *defaultEC2) DescribeNetworkInterface(interfaceIDs []string) (*ec2.DescribeNetworkInterfacesOutput, error) { + describeNetworkInterfaceInput := &ec2.DescribeNetworkInterfacesInput{ + NetworkInterfaceIds: aws.StringSlice(interfaceIDs), + } + + return d.EC2API.DescribeNetworkInterfaces(describeNetworkInterfaceInput) +} + func NewEC2(session *session.Session) EC2 { return &defaultEC2{ EC2API: ec2.New(session), diff --git a/test/framework/resources/k8s/manifest/container.go b/test/framework/resources/k8s/manifest/container.go index 2cba02f529..e306eb5104 100644 --- a/test/framework/resources/k8s/manifest/container.go +++ b/test/framework/resources/k8s/manifest/container.go @@ -35,6 +35,14 @@ func NewBusyBoxContainerBuilder() *Container { } } +func NewCurlContainer() *Container { + return &Container{ + name: "curl", + image: "curlimages/curl:latest", + imagePullPolicy: v1.PullIfNotPresent, + } +} + func NewNetCatAlpineContainer() *Container { return &Container{ name: "net-cat", diff --git a/test/framework/resources/k8s/manifest/job.go b/test/framework/resources/k8s/manifest/job.go index 2702910fbb..540ba5a5f9 100644 --- a/test/framework/resources/k8s/manifest/job.go +++ b/test/framework/resources/k8s/manifest/job.go @@ -29,6 +29,7 @@ type JobBuilder struct { labels map[string]string terminationGracePeriod int nodeName string + hostNetwork bool } func NewDefaultJobBuilder() *JobBuilder { @@ -37,6 +38,7 @@ func NewDefaultJobBuilder() *JobBuilder { name: "test-job", parallelism: 1, terminationGracePeriod: 0, + labels: map[string]string{}, } } @@ -75,6 +77,11 @@ func (j *JobBuilder) Parallelism(parallelism int) *JobBuilder { return j } +func (j *JobBuilder) HostNetwork(hostNetwork bool) *JobBuilder { + j.hostNetwork = hostNetwork + return j +} + func (j *JobBuilder) Build() *batchV1.Job { return &batchV1.Job{ ObjectMeta: metaV1.ObjectMeta{ @@ -88,6 +95,7 @@ func (j *JobBuilder) Build() *batchV1.Job { Labels: j.labels, }, Spec: v1.PodSpec{ + HostNetwork: j.hostNetwork, Containers: []v1.Container{j.container}, TerminationGracePeriodSeconds: aws.Int64(int64(j.terminationGracePeriod)), RestartPolicy: v1.RestartPolicyNever, diff --git a/test/framework/resources/k8s/resources/job.go b/test/framework/resources/k8s/resources/job.go index 1b2fcd7bd3..15d9050f56 100644 --- a/test/framework/resources/k8s/resources/job.go +++ b/test/framework/resources/k8s/resources/job.go @@ -68,6 +68,7 @@ func (d *defaultJobManager) DeleteAndWaitTillJobIsDeleted(job *v1.Job) error { if err != nil { return err } + observedJob := &v1.Job{} return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) { if err := d.k8sClient.Get(ctx, utils.NamespacedName(job), observedJob); err != nil { diff --git a/test/integration-new/ipamd/eni_tag_test.go b/test/integration-new/ipamd/eni_tag_test.go new file mode 100644 index 0000000000..aa712f2e81 --- /dev/null +++ b/test/integration-new/ipamd/eni_tag_test.go @@ -0,0 +1,166 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package ipamd + +import ( + "encoding/json" + "fmt" + "time" + + k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +// Verifies that additional ENI tags are added on new Secondary Elastic Network Interface created +// by IPAMD +var _ = Describe("test tags are created on Secondary ENI", func() { + // Subset of tags expected on new Secondary ENI + var expectedTags map[string]string + // List of ENIs created after updating the environment variables + var newENIs []string + // Environment variables to be updated for testing new tags on Secondary ENI + var environmentVariables map[string]string + + // Sets the WARM_ENI_TARGET to 0 to allow for unused ENIs to be deleted by IPAMD and then + // sets the desired environment variables and gets the list of new ENIs created after setting + // the environment variables + JustBeforeEach(func() { + By("creating test namespace") + f.K8sResourceManagers.NamespaceManager(). + CreateNamespace(utils.DefaultTestNamespace) + + // To re-initialize for each test case + newENIs = []string{} + + By("try detaching all ENIs by setting WARM_ENI_TARGET to 0") + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, + utils.AwsNodeName, map[string]string{"WARM_ENI_TARGET": "0"}) + + By("sleeping to allow CNI Plugin delete unused ENIs") + time.Sleep(time.Second * 90) + + By("getting the list of ENIs before setting ADDITIONAL_ENI_TAGS") + instance, err := f.CloudServices.EC2().DescribeInstance(*primaryInstance.InstanceId) + Expect(err).ToNot(HaveOccurred()) + + existingENIs := make(map[string]bool) + for _, nwInterface := range instance.NetworkInterfaces { + existingENIs[*nwInterface.NetworkInterfaceId] = true + } + + By(fmt.Sprintf("adding environment variable: %+v", environmentVariables)) + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, + utils.AwsNodeName, environmentVariables) + + By("sleeping to allow CNI Plugin to create new ENI with additional tags") + time.Sleep(time.Second * 90) + + By("getting the list of current ENIs by describing the instance") + instance, err = f.CloudServices.EC2().DescribeInstance(*primaryInstance.InstanceId) + Expect(err).ToNot(HaveOccurred()) + + for _, nwInterface := range instance.NetworkInterfaces { + if _, ok := existingENIs[*nwInterface.NetworkInterfaceId]; !ok { + newENIs = append(newENIs, *nwInterface.NetworkInterfaceId) + } + } + + By("verifying at least one new Secondary ENI is created") + Expect(len(newENIs)).Should(BeNumerically(">", 0)) + }) + + JustAfterEach(func() { + By("deleting test namespace") + f.K8sResourceManagers.NamespaceManager(). + DeleteAndWaitTillNamespaceDeleted(utils.DefaultTestNamespace) + + envVarToRemove := map[string]struct{}{} + for key, _ := range environmentVariables { + envVarToRemove[key] = struct{}{} + } + + By("removing environment variables set by the test") + k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, + utils.AwsNodeName, envVarToRemove) + }) + + Context("when additional ENI tags are added using ADDITIONAL_ENI_TAGS", func() { + BeforeEach(func() { + suppliedTags := map[string]string{ + "tag_owner": "cni_automation_test", + "k8s.amazonaws.com/tag_owner": "cni_automation_test", + } + tagBytes, err := json.Marshal(suppliedTags) + Expect(err).ToNot(HaveOccurred()) + + environmentVariables = map[string]string{ + "ADDITIONAL_ENI_TAGS": string(tagBytes), + "WARM_ENI_TARGET": "2", + } + + expectedTags = map[string]string{ + "tag_owner": "cni_automation_test", + } + }) + + It("new secondary ENI should get new tags and block reserved tags", func() { + VerifyTagIsPresentOnENIs(newENIs, expectedTags) + }) + }) + + Context("when additional secondary ENI are created after setting CLUSTER_NAME", func() { + BeforeEach(func() { + clusterName := "dummy_cluster_name" + expectedTags = map[string]string{ + "cluster.k8s.amazonaws.com/name": clusterName, + } + + environmentVariables = map[string]string{ + "CLUSTER_NAME": clusterName, + "WARM_ENI_TARGET": "2", + } + }) + + It("new secondary ENI should have cluster name tags", func() { + VerifyTagIsPresentOnENIs(newENIs, expectedTags) + }) + }) +}) + +// VerifyTagIsPresentOnENIs verifies that the list of ENIs have expected tag key-val pair +func VerifyTagIsPresentOnENIs(newENIIds []string, expectedTags map[string]string) { + By(fmt.Sprintf("Describing the list of new ENI created after seeting env variable %v", newENIIds)) + describeNetworkInterfaceOutput, err := f.CloudServices.EC2().DescribeNetworkInterface(newENIIds) + Expect(err).ToNot(HaveOccurred()) + + By("verifying the new tags are present on new ENIs") + + // Initially expected tag should not be 0 + Expect(len(expectedTags)).ShouldNot(Equal(0)) + + // Each time there's a match found, remove the entry from expected tags + for _, nwInterface := range describeNetworkInterfaceOutput.NetworkInterfaces { + for _, tag := range nwInterface.TagSet { + if val, ok := expectedTags[*tag.Key]; ok && *tag.Value == val { + delete(expectedTags, *tag.Key) + } + } + } + + // The expected tags map should be empty indicating that all matches were found + Expect(len(expectedTags)).Should(Equal(0)) +} diff --git a/test/integration-new/ipamd/introspection_test.go b/test/integration-new/ipamd/introspection_test.go new file mode 100644 index 0000000000..bc66aaa421 --- /dev/null +++ b/test/integration-new/ipamd/introspection_test.go @@ -0,0 +1,143 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package ipamd + +import ( + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" + k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + v1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" +) + +// TODO: In future, we should also have verification for individual introspection API +var _ = Describe("test Environment Variables for IPAMD Introspection ", func() { + // default port on which IPAMD listen for introspect API + defaultIntrospectionAddr := "127.0.0.1:61679" + // Container used to curl IPAMD to verify introspection URL is available + var curlContainer corev1.Container + // Job whose output determines if the introspect URL is available or not + var curlJob *v1.Job + + JustBeforeEach(func() { + By("creating test namespace") + f.K8sResourceManagers.NamespaceManager(). + CreateNamespace(utils.DefaultTestNamespace) + + // Initially the host networking job pod should succeed + curlContainer = manifest.NewCurlContainer(). + Command([]string{"curl"}). + Args([]string{"--fail", defaultIntrospectionAddr}). + Build() + + curlJob = manifest.NewDefaultJobBuilder(). + Container(curlContainer). + Name("verify-introspection"). + Parallelism(1). + HostNetwork(true). + Build() + + // Set the ENV variable + By("enabling introspection on the aws-node") + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, + utils.AwsNodeName, map[string]string{"DISABLE_INTROSPECTION": "false"}) + + By("deploying a host networking pod to verify introspection is working on default addr") + _, err = f.K8sResourceManagers.JobManager(). + CreateAndWaitTillJobCompleted(curlJob) + Expect(err).ToNot(HaveOccurred()) + + By("deleting the verification job") + err = f.K8sResourceManagers.JobManager(). + DeleteAndWaitTillJobIsDeleted(curlJob) + Expect(err).ToNot(HaveOccurred()) + }) + + JustAfterEach(func() { + By("deleting test namespace") + f.K8sResourceManagers.NamespaceManager(). + DeleteAndWaitTillNamespaceDeleted(utils.DefaultTestNamespace) + }) + + Context("when disabling introspection by setting DISABLE_INTROSPECTION to true", func() { + It("introspection should not work anymore", func() { + + By("disabling introspection on the aws-node") + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, + utils.AwsNodeNamespace, utils.AwsNodeName, + map[string]string{"DISABLE_INTROSPECTION": "true"}) + + curlJob = manifest.NewDefaultJobBuilder(). + Container(curlContainer). + Name("verify-introspection-fails-on-disabling"). + Parallelism(1). + HostNetwork(true). + Build() + + // It should fail this time + By("creating a new job that should error out") + curlJob, err = f.K8sResourceManagers.JobManager(). + CreateAndWaitTillJobCompleted(curlJob) + Expect(err).To(HaveOccurred()) + + err = f.K8sResourceManagers.JobManager(). + DeleteAndWaitTillJobIsDeleted(curlJob) + Expect(err).ToNot(HaveOccurred()) + + // Set the ENV variable + By("removing the DISABLE_INTROSPECTION on the aws-node") + k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, + utils.AwsNodeName, map[string]struct{}{"DISABLE_INTROSPECTION": {}}) + }) + }) + + Context("when changing introspection bind address", func() { + It("the introspection API be available on new address", func() { + newAddr := "127.0.0.1:61671" + + By("updating introspection bind address by setting INTROSPECTION_BIND_ADDRESS") + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, + utils.AwsNodeName, map[string]string{"INTROSPECTION_BIND_ADDRESS": newAddr}) + + curlContainerUpdatedAddr := curlContainer.DeepCopy() + curlContainerUpdatedAddr.Args = []string{"--fail", newAddr} + + curlJob = manifest.NewDefaultJobBuilder(). + Container(*curlContainerUpdatedAddr). + Name("verify-introspection-new-add"). + Parallelism(1). + HostNetwork(true). + Build() + + // It should fail this time + By("creating a new job that should not error out") + curlJob, err = f.K8sResourceManagers.JobManager(). + CreateAndWaitTillJobCompleted(curlJob) + Expect(err).ToNot(HaveOccurred()) + + err = f.K8sResourceManagers.JobManager(). + DeleteAndWaitTillJobIsDeleted(curlJob) + Expect(err).ToNot(HaveOccurred()) + + // Set the ENV variable + By("removing the INTROSPECTION_BIND_ADDRESS on the aws-node") + k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, + utils.AwsNodeNamespace, utils.AwsNodeName, + map[string]struct{}{"INTROSPECTION_BIND_ADDRESS": {}}) + }) + }) +}) diff --git a/test/integration-new/ipamd/ipamd_suite_test.go b/test/integration-new/ipamd/ipamd_suite_test.go index 1ac39b4934..3dedd6f4da 100644 --- a/test/integration-new/ipamd/ipamd_suite_test.go +++ b/test/integration-new/ipamd/ipamd_suite_test.go @@ -18,8 +18,6 @@ import ( "github.com/aws/amazon-vpc-cni-k8s/test/framework" k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" - "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" - "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -39,10 +37,6 @@ func TestIPAMD(t *testing.T) { var _ = BeforeSuite(func() { f = framework.New(framework.GlobalOptions) - By("creating test namespace") - f.K8sResourceManagers.NamespaceManager(). - CreateNamespace(utils.DefaultTestNamespace) - nodeList, err := f.K8sResourceManagers.NodeManager().GetNodes(f.Options.NgNameLabelKey, f.Options.NgNameLabelVal) Expect(err).ToNot(HaveOccurred()) @@ -55,9 +49,3 @@ var _ = BeforeSuite(func() { primaryInstance, err = f.CloudServices.EC2().DescribeInstance(instanceID) Expect(err).ToNot(HaveOccurred()) }) - -var _ = AfterSuite(func() { - By("deleting test namespace") - f.K8sResourceManagers.NamespaceManager(). - DeleteAndWaitTillNamespaceDeleted(utils.DefaultTestNamespace) -}) diff --git a/test/integration-new/ipamd/metrics_test.go b/test/integration-new/ipamd/metrics_test.go new file mode 100644 index 0000000000..19f875d61d --- /dev/null +++ b/test/integration-new/ipamd/metrics_test.go @@ -0,0 +1,102 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package ipamd + +import ( + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" + k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + v1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" +) + +// Verifies toggling the DISABLE_METRICS works as expected +// TODO: In future, we should also have verification for individual metrics +var _ = Describe("test IPAMD metric environment variable", func() { + // container to curl metric API + var curlContainer corev1.Container + // Job's output determines if the API is reachable or not + var curlJob *v1.Job + + JustBeforeEach(func() { + By("creating test namespace") + f.K8sResourceManagers.NamespaceManager(). + CreateNamespace(utils.DefaultTestNamespace) + }) + + JustAfterEach(func() { + By("deleting test namespace") + f.K8sResourceManagers.NamespaceManager(). + DeleteAndWaitTillNamespaceDeleted(utils.DefaultTestNamespace) + }) + + Context("when metrics is disabled", func() { + metricAddr := "127.0.0.1:61678/metrics" + It("should not be accessible anymore", func() { + curlContainer = manifest.NewCurlContainer(). + Command([]string{"curl"}). + Args([]string{"--fail", metricAddr}). + Build() + + By("enabling metrics on aws-node") + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, + utils.AwsNodeName, map[string]string{"DISABLE_METRICS": "false"}) + + curlJob = manifest.NewDefaultJobBuilder(). + Container(curlContainer). + Name("verify-metrics-works"). + Parallelism(1). + HostNetwork(true). + Build() + + By("verify metric is working before disabling it") + curlJob, err = f.K8sResourceManagers.JobManager(). + CreateAndWaitTillJobCompleted(curlJob) + Expect(err).ToNot(HaveOccurred()) + + By("deleting job used for verification") + err = f.K8sResourceManagers.JobManager(). + DeleteAndWaitTillJobIsDeleted(curlJob) + Expect(err).ToNot(HaveOccurred()) + + By("disabling metrics on aws-node") + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, + utils.AwsNodeName, map[string]string{"DISABLE_METRICS": "true"}) + + curlJob = manifest.NewDefaultJobBuilder(). + Container(curlContainer). + Name("verify-metrics-doesnt-works"). + Parallelism(1). + HostNetwork(true). + Build() + + By("verifying metrics is not working after disabling it") + curlJob, err = f.K8sResourceManagers.JobManager(). + CreateAndWaitTillJobCompleted(curlJob) + Expect(err).To(HaveOccurred()) + + By("deleting job used for verification") + err = f.K8sResourceManagers.JobManager(). + DeleteAndWaitTillJobIsDeleted(curlJob) + Expect(err).ToNot(HaveOccurred()) + + By("reverting to default value for disabling metrics") + k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, + utils.AwsNodeName, map[string]struct{}{"DISABLE_METRICS": {}}) + }) + }) +})