add integration test for ipamd env variables

test/framework/resources/aws/services/ec2.go

@@ -25,6 +25,7 @@ import (
 type EC2 interface {
  DescribeInstanceType(instanceType string) ([]*ec2.InstanceTypeInfo, error)
  DescribeInstance(instanceID string) (*ec2.Instance, error)
+ DescribeNetworkInterface(interfaceIDs []string) (*ec2.DescribeNetworkInterfacesOutput, error)
  AuthorizeSecurityGroupIngress(groupID string, protocol string, fromPort int, toPort int, cidrIP string) error
  RevokeSecurityGroupIngress(groupID string, protocol string, fromPort int, toPort int, cidrIP string) error
  AuthorizeSecurityGroupEgress(groupID string, protocol string, fromPort int, toPort int, cidrIP string) error
@@ -141,6 +142,14 @@ func (d *defaultEC2) RevokeSecurityGroupEgress(groupID string, protocol string,
  return err
 }
 
+func (d *defaultEC2) DescribeNetworkInterface(interfaceIDs []string) (*ec2.DescribeNetworkInterfacesOutput, error) {
+ describeNetworkInterfaceInput := &ec2.DescribeNetworkInterfacesInput{
+ NetworkInterfaceIds: aws.StringSlice(interfaceIDs),
+ }
+
+ return d.EC2API.DescribeNetworkInterfaces(describeNetworkInterfaceInput)
+}
+
 func NewEC2(session *session.Session) EC2 {
  return &defaultEC2{
  EC2API: ec2.New(session),

test/framework/resources/k8s/manifest/container.go

@@ -35,6 +35,14 @@ func NewBusyBoxContainerBuilder() *Container {
  }
 }
 
+func NewCurlContainer() *Container {
+ return &Container{
+ name: "curl",
+ image: "curlimages/curl:latest",
+ imagePullPolicy: v1.PullIfNotPresent,
+ }
+}
+
 func NewNetCatAlpineContainer() *Container {
  return &Container{
  name: "net-cat",

test/framework/resources/k8s/manifest/job.go

@@ -29,6 +29,7 @@ type JobBuilder struct {
  labels map[string]string
  terminationGracePeriod int
  nodeName string
+ hostNetwork bool
 }
 
 func NewDefaultJobBuilder() *JobBuilder {
@@ -37,6 +38,7 @@ func NewDefaultJobBuilder() *JobBuilder {
  name: "test-job",
  parallelism: 1,
  terminationGracePeriod: 0,
+ labels: map[string]string{},
  }
 }
 
@@ -75,6 +77,11 @@ func (j *JobBuilder) Parallelism(parallelism int) *JobBuilder {
  return j
 }
 
+func (j *JobBuilder) HostNetwork(hostNetwork bool) *JobBuilder {
+ j.hostNetwork = hostNetwork
+ return j
+}
+
 func (j *JobBuilder) Build() *batchV1.Job {
  return &batchV1.Job{
  ObjectMeta: metaV1.ObjectMeta{
@@ -88,6 +95,7 @@ func (j *JobBuilder) Build() *batchV1.Job {
  Labels: j.labels,
  },
  Spec: v1.PodSpec{
+ HostNetwork: j.hostNetwork,
  Containers: []v1.Container{j.container},
  TerminationGracePeriodSeconds: aws.Int64(int64(j.terminationGracePeriod)),
  RestartPolicy: v1.RestartPolicyNever,

test/framework/resources/k8s/resources/job.go

@@ -68,6 +68,7 @@ func (d *defaultJobManager) DeleteAndWaitTillJobIsDeleted(job *v1.Job) error {
  if err != nil {
  return err
  }
+
  observedJob := &v1.Job{}
  return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
  if err := d.k8sClient.Get(ctx, utils.NamespacedName(job), observedJob); err != nil {

test/integration-new/ipamd/eni_tag_test.go

@@ -0,0 +1,166 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+// http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package ipamd
+
+import (
+ "encoding/json"
+ "fmt"
+ "time"
+
+ k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
+ "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
+
+ . "github.com/onsi/ginkgo"
+ . "github.com/onsi/gomega"
+)
+
+// Verifies that additional ENI tags are added on new Secondary Elastic Network Interface created
+// by IPAMD
+var _ = Describe("test tags are created on Secondary ENI", func() {
+ // Subset of tags expected on new Secondary ENI
+ var expectedTags map[string]string
+ // List of ENIs created after updating the environment variables
+ var newENIs []string
+ // Environment variables to be updated for testing new tags on Secondary ENI
+ var environmentVariables map[string]string
+
+ // Sets the WARM_ENI_TARGET to 0 to allow for unused ENIs to be deleted by IPAMD and then
+ // sets the desired environment variables and gets the list of new ENIs created after setting
+ // the environment variables
+ JustBeforeEach(func() {
+ By("creating test namespace")
+ f.K8sResourceManagers.NamespaceManager().
+ CreateNamespace(utils.DefaultTestNamespace)
+
+ // To re-initialize for each test case
+ newENIs = []string{}
+
+ By("try detaching all ENIs by setting WARM_ENI_TARGET to 0")
+ k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace,
+ utils.AwsNodeName, map[string]string{"WARM_ENI_TARGET": "0"})
+
+ By("sleeping to allow CNI Plugin delete unused ENIs")
+ time.Sleep(time.Second * 90)
+
+ By("getting the list of ENIs before setting ADDITIONAL_ENI_TAGS")
+ instance, err := f.CloudServices.EC2().DescribeInstance(*primaryInstance.InstanceId)
+ Expect(err).ToNot(HaveOccurred())
+
+ existingENIs := make(map[string]bool)
+ for _, nwInterface := range instance.NetworkInterfaces {
+ existingENIs[*nwInterface.NetworkInterfaceId] = true
+ }
+
+ By(fmt.Sprintf("adding environment variable: %+v", environmentVariables))
+ k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace,
+ utils.AwsNodeName, environmentVariables)
+
+ By("sleeping to allow CNI Plugin to create new ENI with additional tags")
+ time.Sleep(time.Second * 90)
+
+ By("getting the list of current ENIs by describing the instance")
+ instance, err = f.CloudServices.EC2().DescribeInstance(*primaryInstance.InstanceId)
+ Expect(err).ToNot(HaveOccurred())
+
+ for _, nwInterface := range instance.NetworkInterfaces {
+ if _, ok := existingENIs[*nwInterface.NetworkInterfaceId]; !ok {
+ newENIs = append(newENIs, *nwInterface.NetworkInterfaceId)
+ }
+ }
+
+ By("verifying at least one new Secondary ENI is created")
+ Expect(len(newENIs)).Should(BeNumerically(">", 0))
+ })
+
+ JustAfterEach(func() {
+ By("deleting test namespace")
+ f.K8sResourceManagers.NamespaceManager().
+ DeleteAndWaitTillNamespaceDeleted(utils.DefaultTestNamespace)
+
+ envVarToRemove := map[string]struct{}{}
+ for key, _ := range environmentVariables {
+ envVarToRemove[key] = struct{}{}
+ }
+
+ By("removing environment variables set by the test")
+ k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace,
+ utils.AwsNodeName, envVarToRemove)
+ })
+
+ Context("when additional ENI tags are added using ADDITIONAL_ENI_TAGS", func() {
+ BeforeEach(func() {
+ suppliedTags := map[string]string{
+ "tag_owner": "cni_automation_test",
+ "k8s.amazonaws.com/tag_owner": "cni_automation_test",
+ }
+ tagBytes, err := json.Marshal(suppliedTags)
+ Expect(err).ToNot(HaveOccurred())
+
+ environmentVariables = map[string]string{
+ "ADDITIONAL_ENI_TAGS": string(tagBytes),
+ "WARM_ENI_TARGET": "2",
+ }
+
+ expectedTags = map[string]string{
+ "tag_owner": "cni_automation_test",
+ }
+ })
+
+ It("new secondary ENI should get new tags and block reserved tags", func() {
+ VerifyTagIsPresentOnENIs(newENIs, expectedTags)
+ })
+ })
+
+ Context("when additional secondary ENI are created after setting CLUSTER_NAME", func() {
+ BeforeEach(func() {
+ clusterName := "dummy_cluster_name"
+ expectedTags = map[string]string{
+ "cluster.k8s.amazonaws.com/name": clusterName,
+ }
+
+ environmentVariables = map[string]string{
+ "CLUSTER_NAME": clusterName,
+ "WARM_ENI_TARGET": "2",
+ }
+ })
+
+ It("new secondary ENI should have cluster name tags", func() {
+ VerifyTagIsPresentOnENIs(newENIs, expectedTags)
+ })
+ })
+})
+
+// VerifyTagIsPresentOnENIs verifies that the list of ENIs have expected tag key-val pair
+func VerifyTagIsPresentOnENIs(newENIIds []string, expectedTags map[string]string) {
+ By(fmt.Sprintf("Describing the list of new ENI created after seeting env variable %v", newENIIds))
+ describeNetworkInterfaceOutput, err := f.CloudServices.EC2().DescribeNetworkInterface(newENIIds)
+ Expect(err).ToNot(HaveOccurred())
+
+ By("verifying the new tags are present on new ENIs")
+
+ // Initially expected tag should not be 0
+ Expect(len(expectedTags)).ShouldNot(Equal(0))
+
+ // Each time there's a match found, remove the entry from expected tags
+ for _, nwInterface := range describeNetworkInterfaceOutput.NetworkInterfaces {
+ for _, tag := range nwInterface.TagSet {
+ if val, ok := expectedTags[*tag.Key]; ok && *tag.Value == val {
+ delete(expectedTags, *tag.Key)
+ }
+ }
+ }
+
+ // The expected tags map should be empty indicating that all matches were found
+ Expect(len(expectedTags)).Should(Equal(0))
+}

test/integration-new/ipamd/introspection_test.go

@@ -0,0 +1,143 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+// http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package ipamd
+
+import (
+ "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest"
+ k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
+ "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
+
+ . "github.com/onsi/ginkgo"
+ . "github.com/onsi/gomega"
+ v1 "k8s.io/api/batch/v1"
+ corev1 "k8s.io/api/core/v1"
+)
+
+// TODO: In future, we should also have verification for individual introspection API
+var _ = Describe("test Environment Variables for IPAMD Introspection ", func() {
+ // default port on which IPAMD listen for introspect API
+ defaultIntrospectionAddr := "127.0.0.1:61679"
+ // Container used to curl IPAMD to verify introspection URL is available
+ var curlContainer corev1.Container
+ // Job whose output determines if the introspect URL is available or not
+ var curlJob *v1.Job
+
+ JustBeforeEach(func() {
+ By("creating test namespace")
+ f.K8sResourceManagers.NamespaceManager().
+ CreateNamespace(utils.DefaultTestNamespace)
+
+ // Initially the host networking job pod should succeed
+ curlContainer = manifest.NewCurlContainer().
+ Command([]string{"curl"}).
+ Args([]string{"--fail", defaultIntrospectionAddr}).
+ Build()
+
+ curlJob = manifest.NewDefaultJobBuilder().
+ Container(curlContainer).
+ Name("verify-introspection").
+ Parallelism(1).
+ HostNetwork(true).
+ Build()
+
+ // Set the ENV variable
+ By("enabling introspection on the aws-node")
+ k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace,
+ utils.AwsNodeName, map[string]string{"DISABLE_INTROSPECTION": "false"})
+
+ By("deploying a host networking pod to verify introspection is working on default addr")
+ _, err = f.K8sResourceManagers.JobManager().
+ CreateAndWaitTillJobCompleted(curlJob)
+ Expect(err).ToNot(HaveOccurred())
+
+ By("deleting the verification job")
+ err = f.K8sResourceManagers.JobManager().
+ DeleteAndWaitTillJobIsDeleted(curlJob)
+ Expect(err).ToNot(HaveOccurred())
+ })
+
+ JustAfterEach(func() {
+ By("deleting test namespace")
+ f.K8sResourceManagers.NamespaceManager().
+ DeleteAndWaitTillNamespaceDeleted(utils.DefaultTestNamespace)
+ })
+
+ Context("when disabling introspection by setting DISABLE_INTROSPECTION to true", func() {
+ It("introspection should not work anymore", func() {
+
+ By("disabling introspection on the aws-node")
+ k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName,
+ utils.AwsNodeNamespace, utils.AwsNodeName,
+ map[string]string{"DISABLE_INTROSPECTION": "true"})
+
+ curlJob = manifest.NewDefaultJobBuilder().
+ Container(curlContainer).
+ Name("verify-introspection-fails-on-disabling").
+ Parallelism(1).
+ HostNetwork(true).
+ Build()
+
+ // It should fail this time
+ By("creating a new job that should error out")
+ curlJob, err = f.K8sResourceManagers.JobManager().
+ CreateAndWaitTillJobCompleted(curlJob)
+ Expect(err).To(HaveOccurred())
+
+ err = f.K8sResourceManagers.JobManager().
+ DeleteAndWaitTillJobIsDeleted(curlJob)
+ Expect(err).ToNot(HaveOccurred())
+
+ // Set the ENV variable
+ By("removing the DISABLE_INTROSPECTION on the aws-node")
+ k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace,
+ utils.AwsNodeName, map[string]struct{}{"DISABLE_INTROSPECTION": {}})
+ })
+ })
+
+ Context("when changing introspection bind address", func() {
+ It("the introspection API be available on new address", func() {
+ newAddr := "127.0.0.1:61671"
+
+ By("updating introspection bind address by setting INTROSPECTION_BIND_ADDRESS")
+ k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace,
+ utils.AwsNodeName, map[string]string{"INTROSPECTION_BIND_ADDRESS": newAddr})
+
+ curlContainerUpdatedAddr := curlContainer.DeepCopy()
+ curlContainerUpdatedAddr.Args = []string{"--fail", newAddr}
+
+ curlJob = manifest.NewDefaultJobBuilder().
+ Container(*curlContainerUpdatedAddr).
+ Name("verify-introspection-new-add").
+ Parallelism(1).
+ HostNetwork(true).
+ Build()
+
+ // It should fail this time
+ By("creating a new job that should not error out")
+ curlJob, err = f.K8sResourceManagers.JobManager().
+ CreateAndWaitTillJobCompleted(curlJob)
+ Expect(err).ToNot(HaveOccurred())
+
+ err = f.K8sResourceManagers.JobManager().
+ DeleteAndWaitTillJobIsDeleted(curlJob)
+ Expect(err).ToNot(HaveOccurred())
+
+ // Set the ENV variable
+ By("removing the INTROSPECTION_BIND_ADDRESS on the aws-node")
+ k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName,
+ utils.AwsNodeNamespace, utils.AwsNodeName,
+ map[string]struct{}{"INTROSPECTION_BIND_ADDRESS": {}})
+ })
+ })
+})