From 45bcab57d6a6a64a7c710081a0b820363a16d80b Mon Sep 17 00:00:00 2001 From: M00nF1sh Date: Mon, 7 Jun 2021 09:59:39 -0700 Subject: [PATCH] Use symmetric return path for non-VPC traffic - alternate solution (#1475) (#1494) * use symmetric return path for non-VPC traffic * account for custom veth prefix configuration * update host iptables rules on VPC CIDR change * update integration tests to recognize new changes * new integration test: reset aws-node config * update README Co-authored-by: Kishor Joshi --- README.md | 2 +- cmd/routed-eni-cni-plugin/driver/driver.go | 39 +- pkg/ipamd/ipamd.go | 12 +- pkg/ipamd/ipamd_test.go | 3 +- pkg/networkutils/mocks/network_mocks.go | 22 +- pkg/networkutils/network.go | 277 ++++++++++---- pkg/networkutils/network_test.go | 359 ++++++++++++++---- scripts/entrypoint.sh | 12 + .../resources/k8s/utils/daemonset.go | 54 +-- test/framework/utils/const.go | 2 +- .../cni/host_networking_test.go | 22 +- .../cni/pod_networking_suite_test.go | 11 +- 12 files changed, 587 insertions(+), 228 deletions(-) diff --git a/README.md b/README.md index 3ee9bd7c40..7920109616 100644 --- a/README.md +++ b/README.md @@ -376,7 +376,7 @@ Type: String Default: `eni` -Specifies the veth prefix used to generate the host-side veth device name for the CNI. The prefix can be at most 4 characters long. +Specifies the veth prefix used to generate the host-side veth device name for the CNI. The prefix can be at most 4 characters long. The prefixes `eth`, `vlan` and `lo` are reserved by the CNI plugin and cannot be specified. We recommend using prefix name not shared by any other network interfaces on the worker node instance. --- diff --git a/cmd/routed-eni-cni-plugin/driver/driver.go b/cmd/routed-eni-cni-plugin/driver/driver.go index c81ce21e52..78440ea181 100644 --- a/cmd/routed-eni-cni-plugin/driver/driver.go +++ b/cmd/routed-eni-cni-plugin/driver/driver.go @@ -217,40 +217,13 @@ func setupNS(hostVethName string, contVethName string, netnsPath string, addr *n if deviceNumber > 0 { // To be backwards compatible, we will have to keep this off-by one setting tableNumber := deviceNumber + 1 - if useExternalSNAT { - // add rule: 1536: from use table - err = addContainerRule(netLink, false, addr, tableNumber) - if err != nil { - log.Errorf("Failed to add fromContainer rule for %s err: %v", addr.String(), err) - return errors.Wrap(err, "add NS network: failed to add fromContainer rule") - } - log.Infof("Added rule priority %d from %s table %d", fromContainerRulePriority, addr.String(), tableNumber) - } else { - // add rule: 1536: list of from to use table
- for _, cidr := range vpcCIDRs { - podRule := netLink.NewRule() - _, podRule.Dst, _ = net.ParseCIDR(cidr) - podRule.Src = addr - podRule.Table = tableNumber - podRule.Priority = fromContainerRulePriority - - err = netLink.RuleAdd(podRule) - if isRuleExistsError(err) { - log.Warnf("Rule already exists [%v]", podRule) - } else { - if err != nil { - log.Errorf("Failed to add pod IP rule [%v]: %v", podRule, err) - return errors.Wrapf(err, "setupNS: failed to add pod rule [%v]", podRule) - } - } - var toDst string - - if podRule.Dst != nil { - toDst = podRule.Dst.String() - } - log.Infof("Successfully added pod rule[%v] to %s", podRule, toDst) - } + // add rule: 1536: from use table
+ err = addContainerRule(netLink, false, addr, tableNumber) + if err != nil { + log.Errorf("Failed to add fromContainer rule for %s err: %v", addr.String(), err) + return errors.Wrap(err, "add NS network: failed to add fromContainer rule") } + log.Infof("Added rule priority %d from %s table %d", fromContainerRulePriority, addr.String(), tableNumber) } return nil } diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index 974bbaa0a6..4f8053be0e 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -405,7 +405,7 @@ func (c *IPAMContext) nodeInit() error { return err } - if err = c.configureIPRulesForPods(vpcCIDRs); err != nil { + if err = c.configureIPRulesForPods(); err != nil { return err } // Spawning updateCIDRsRulesOnChange go-routine @@ -458,7 +458,7 @@ func (c *IPAMContext) nodeInit() error { return nil } -func (c *IPAMContext) configureIPRulesForPods(pbVPCcidrs []string) error { +func (c *IPAMContext) configureIPRulesForPods() error { rules, err := c.networkClient.GetRuleList() if err != nil { log.Errorf("During ipamd init: failed to retrieve IP rule list %v", err) @@ -471,7 +471,7 @@ func (c *IPAMContext) configureIPRulesForPods(pbVPCcidrs []string) error { // Update ip rules in case there is a change in VPC CIDRs, AWS_VPC_K8S_CNI_EXTERNALSNAT setting srcIPNet := net.IPNet{IP: net.ParseIP(info.IP), Mask: net.IPv4Mask(255, 255, 255, 255)} - err = c.networkClient.UpdateRuleListBySrc(rules, srcIPNet, pbVPCcidrs, !c.networkClient.UseExternalSNAT()) + err = c.networkClient.UpdateRuleListBySrc(rules, srcIPNet) if err != nil { log.Warnf("UpdateRuleListBySrc in nodeInit() failed for IP %s: %v", info.IP, err) } @@ -489,7 +489,11 @@ func (c *IPAMContext) updateCIDRsRulesOnChange(oldVPCCIDRs []string) []string { old := sets.NewString(oldVPCCIDRs...) new := sets.NewString(newVPCCIDRs...) if !old.Equal(new) { - _ = c.configureIPRulesForPods(newVPCCIDRs) + primaryIP := c.awsClient.GetLocalIPv4() + err = c.networkClient.UpdateHostIptablesRules(newVPCCIDRs, c.awsClient.GetPrimaryENImac(), &primaryIP) + if err != nil { + log.Warnf("unable to update host iptables rules for VPC CIDRs due to error: %v", err) + } } return newVPCCIDRs } diff --git a/pkg/ipamd/ipamd_test.go b/pkg/ipamd/ipamd_test.go index 696c60040e..6c12b71e8b 100644 --- a/pkg/ipamd/ipamd_test.go +++ b/pkg/ipamd/ipamd_test.go @@ -152,8 +152,7 @@ func TestNodeInit(t *testing.T) { var rules []netlink.Rule m.network.EXPECT().GetRuleList().Return(rules, nil) - m.network.EXPECT().UseExternalSNAT().Return(false) - m.network.EXPECT().UpdateRuleListBySrc(gomock.Any(), gomock.Any(), gomock.Any(), true) + m.network.EXPECT().UpdateRuleListBySrc(gomock.Any(), gomock.Any()) fakeNode := v1.Node{ TypeMeta: metav1.TypeMeta{Kind: "Node"}, diff --git a/pkg/networkutils/mocks/network_mocks.go b/pkg/networkutils/mocks/network_mocks.go index 283d0dd47e..2bb31bef5a 100644 --- a/pkg/networkutils/mocks/network_mocks.go +++ b/pkg/networkutils/mocks/network_mocks.go @@ -151,18 +151,32 @@ func (mr *MockNetworkAPIsMockRecorder) SetupHostNetwork(arg0, arg1, arg2, arg3 i return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupHostNetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupHostNetwork), arg0, arg1, arg2, arg3) } +// UpdateHostIptablesRules mocks base method +func (m *MockNetworkAPIs) UpdateHostIptablesRules(arg0 []string, arg1 string, arg2 *net.IP) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateHostIptablesRules", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateHostIptablesRules indicates an expected call of UpdateHostIptablesRules +func (mr *MockNetworkAPIsMockRecorder) UpdateHostIptablesRules(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateHostIptablesRules", reflect.TypeOf((*MockNetworkAPIs)(nil).UpdateHostIptablesRules), arg0, arg1, arg2) +} + // UpdateRuleListBySrc mocks base method -func (m *MockNetworkAPIs) UpdateRuleListBySrc(arg0 []netlink.Rule, arg1 net.IPNet, arg2 []string, arg3 bool) error { +func (m *MockNetworkAPIs) UpdateRuleListBySrc(arg0 []netlink.Rule, arg1 net.IPNet) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateRuleListBySrc", arg0, arg1, arg2, arg3) + ret := m.ctrl.Call(m, "UpdateRuleListBySrc", arg0, arg1) ret0, _ := ret[0].(error) return ret0 } // UpdateRuleListBySrc indicates an expected call of UpdateRuleListBySrc -func (mr *MockNetworkAPIsMockRecorder) UpdateRuleListBySrc(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { +func (mr *MockNetworkAPIsMockRecorder) UpdateRuleListBySrc(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateRuleListBySrc", reflect.TypeOf((*MockNetworkAPIs)(nil).UpdateRuleListBySrc), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateRuleListBySrc", reflect.TypeOf((*MockNetworkAPIs)(nil).UpdateRuleListBySrc), arg0, arg1) } // UseExternalSNAT mocks base method diff --git a/pkg/networkutils/network.go b/pkg/networkutils/network.go index 029ddce93c..4947bb2ba8 100644 --- a/pkg/networkutils/network.go +++ b/pkg/networkutils/network.go @@ -28,6 +28,7 @@ import ( "time" "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/retry" + "k8s.io/apimachinery/pkg/util/sets" "github.com/pkg/errors" "golang.org/x/sys/unix" @@ -101,6 +102,12 @@ const ( // envMTU gives a way to configure the MTU size for new ENIs attached. Range is from 576 to 9001. envMTU = "AWS_VPC_ENI_MTU" + // envVethPrefix is the environment variable to configure the prefix of the host side veth device names + envVethPrefix = "AWS_VPC_K8S_CNI_VETHPREFIX" + + // envVethPrefixDefault is the default value for the veth prefix + envVethPrefixDefault = "eni" + // Range of MTU for each ENI and veth pair. Defaults to maximumMTU minimumMTU = 576 maximumMTU = 9001 @@ -124,11 +131,13 @@ type NetworkAPIs interface { SetupHostNetwork(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP, enablePodENI bool) error // SetupENINetwork performs ENI level network configuration. Not needed on the primary ENI SetupENINetwork(eniIP string, mac string, deviceNumber int, subnetCIDR string) error + // UpdateHostIptablesRules updates the nat table iptables rules on the host + UpdateHostIptablesRules(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP) error UseExternalSNAT() bool GetExcludeSNATCIDRs() []string GetRuleList() ([]netlink.Rule, error) GetRuleListBySrc(ruleList []netlink.Rule, src net.IPNet) ([]netlink.Rule, error) - UpdateRuleListBySrc(ruleList []netlink.Rule, src net.IPNet, toCIDRs []string, toFlag bool) error + UpdateRuleListBySrc(ruleList []netlink.Rule, src net.IPNet) error DeleteRuleListBySrc(src net.IPNet) error GetLinkByMac(mac string, retryInterval time.Duration) (netlink.Link, error) } @@ -140,6 +149,7 @@ type linuxNetwork struct { nodePortSupportEnabled bool shouldConfigureRpFilter bool mtu int + vethPrefix string netLink netlinkwrapper.NetLink ns nswrapper.NS @@ -179,6 +189,7 @@ func New() NetworkAPIs { shouldConfigureRpFilter: shouldConfigureRpFilter(), mainENIMark: getConnmark(), mtu: GetEthernetMTU(""), + vethPrefix: getVethPrefixName(), netLink: netlinkwrapper.NewNetLink(), ns: nswrapper.NewNS(), @@ -299,11 +310,42 @@ func (n *linuxNetwork) SetupHostNetwork(vpcCIDRs []string, primaryMAC string, pr } } + return n.updateHostIptablesRules(vpcCIDRs, primaryMAC, primaryAddr) +} + +// UpdateHostIptablesRules updates the NAT table rules based on the VPC CIDRs configuration +func (n *linuxNetwork) UpdateHostIptablesRules(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP) error { + return n.updateHostIptablesRules(vpcCIDRs, primaryMAC, primaryAddr) +} + +func (n *linuxNetwork) updateHostIptablesRules(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP) error { + primaryIntf, err := findPrimaryInterfaceName(primaryMAC) + if err != nil { + return errors.Wrapf(err, "failed to SetupHostNetwork") + } ipt, err := n.newIptables() if err != nil { return errors.Wrap(err, "host network setup: failed to create iptables") } + iptablesSNATRules, err := n.buildIptablesSNATRules(vpcCIDRs, primaryAddr, primaryIntf, ipt) + if err != nil { + return err + } + if err := n.updateIptablesRules(iptablesSNATRules, ipt); err != nil { + return err + } + iptablesConnmarkRules, err := n.buildIptablesConnmarkRules(vpcCIDRs, ipt) + if err != nil { + return err + } + if err := n.updateIptablesRules(iptablesConnmarkRules, ipt); err != nil { + return err + } + return nil +} + +func (n *linuxNetwork) buildIptablesSNATRules(vpcCIDRs []string, primaryAddr *net.IP, primaryIntf string, ipt iptablesIface) ([]iptablesRule, error) { type snatCIDR struct { cidr string isExclusion bool @@ -318,12 +360,6 @@ func (n *linuxNetwork) SetupHostNetwork(vpcCIDRs []string, primaryMAC string, pr allCIDRs = append(allCIDRs, snatCIDR{cidr: cidr, isExclusion: true}) } - // if excludeSNATCIDRs or vpcCIDRs have changed they need to be cleared - snatStaleRulesToCheck, err := listCurrentSNATRules(ipt) - if err != nil { - return errors.Wrapf(err, "host network setup: failed to get SNAT chain rules to clear") - } - log.Debugf("Total CIDRs to program - %d", len(allCIDRs)) // build IPTABLES chain for SNAT of non-VPC outbound traffic and excluded CIDRs var chains []string @@ -332,7 +368,7 @@ func (n *linuxNetwork) SetupHostNetwork(vpcCIDRs []string, primaryMAC string, pr log.Debugf("Setup Host Network: iptables -N %s -t nat", chain) if err := ipt.NewChain("nat", chain); err != nil && !containChainExistErr(err) { log.Errorf("ipt.NewChain error for chain [%s]: %v", chain, err) - return errors.Wrapf(err, "host network setup: failed to add chain") + return []iptablesRule{}, errors.Wrapf(err, "host network setup: failed to add chain") } chains = append(chains, chain) } @@ -396,25 +432,12 @@ func (n *linuxNetwork) SetupHostNetwork(vpcCIDRs []string, primaryMAC string, pr rule: snatRule, }) - var snatStaleRulesToClear []iptablesRule - log.Debugf("Setup Host Network: synchronising SNAT stale rules") - for _, staleRule := range snatStaleRulesToCheck { - keepRule := false - for _, newRule := range iptableRules { - if staleRule.chain == newRule.chain && reflect.DeepEqual(newRule.rule, staleRule.rule) { - log.Debugf("Setup Host Network: active rule found: %s", staleRule) - keepRule = true - break - } - } - if !keepRule { - log.Debugf("Setup Host Network: stale rule found: %s", staleRule) - snatStaleRulesToClear = append(snatStaleRulesToClear, staleRule) - } + snatStaleRules, err := computeStaleIptablesRules(ipt, "nat", "AWS-SNAT-CHAIN", iptableRules, chains) + if err != nil { + return []iptablesRule{}, err } - iptableRules = append(iptableRules, snatStaleRulesToClear...) - log.Debugf("iptableRules: %v", iptableRules) + iptableRules = append(iptableRules, snatStaleRules...) iptableRules = append(iptableRules, iptablesRule{ name: "connmark for primary ENI", @@ -436,7 +459,7 @@ func (n *linuxNetwork) SetupHostNetwork(vpcCIDRs []string, primaryMAC string, pr chain: "PREROUTING", rule: []string{ "-m", "comment", "--comment", "AWS, primary ENI", - "-i", "eni+", "-j", "CONNMARK", "--restore-mark", "--mask", fmt.Sprintf("%#x", n.mainENIMark), + "-i", n.vethPrefix + "+", "-j", "CONNMARK", "--restore-mark", "--mask", fmt.Sprintf("%#x", n.mainENIMark), }, }) @@ -451,10 +474,110 @@ func (n *linuxNetwork) SetupHostNetwork(vpcCIDRs []string, primaryMAC string, pr }, }) + log.Debugf("iptableRules: %v", iptableRules) + return iptableRules, nil +} + +func (n *linuxNetwork) buildIptablesConnmarkRules(vpcCIDRs []string, ipt iptablesIface) ([]iptablesRule, error) { + var allCIDRs []string + allCIDRs = append(allCIDRs, vpcCIDRs...) + allCIDRs = append(allCIDRs, n.excludeSNATCIDRs...) + excludeCIDRs := sets.NewString(n.excludeSNATCIDRs...) + + log.Debugf("Total CIDRs to exempt from connmark rules - %d", len(allCIDRs)) + var chains []string + for i := 0; i <= len(allCIDRs); i++ { + chain := fmt.Sprintf("AWS-CONNMARK-CHAIN-%d", i) + log.Debugf("Setup Host Network: iptables -N %s -t nat", chain) + if err := ipt.NewChain("nat", chain); err != nil && !containChainExistErr(err) { + log.Errorf("ipt.NewChain error for chain [%s]: %v", chain, err) + return []iptablesRule{}, errors.Wrapf(err, "host network setup: failed to add chain") + } + chains = append(chains, chain) + } + + var iptableRules []iptablesRule + log.Debugf("Setup Host Network: iptables -t nat -A PREROUTING -i %s+ -m comment --comment \"AWS, outbound connections\" -m state --state NEW -j AWS-CONNMARK-CHAIN-0", n.vethPrefix) + iptableRules = append(iptableRules, iptablesRule{ + name: "connmark rule for non-VPC outbound traffic", + shouldExist: !n.useExternalSNAT, + table: "nat", + chain: "PREROUTING", + rule: []string{ + "-i", n.vethPrefix + "+", "-m", "comment", "--comment", "AWS, outbound connections", + "-m", "state", "--state", "NEW", "-j", "AWS-CONNMARK-CHAIN-0", + }}) + + for i, cidr := range allCIDRs { + curChain := chains[i] + curName := fmt.Sprintf("[%d] AWS-SNAT-CHAIN", i) + nextChain := chains[i+1] + comment := "AWS CONNMARK CHAIN, VPC CIDR" + if excludeCIDRs.Has(cidr) { + comment = "AWS CONNMARK CHAIN, EXCLUDED CIDR" + } + log.Debugf("Setup Host Network: iptables -A %s ! -d %s -t nat -j %s", curChain, cidr, nextChain) + + iptableRules = append(iptableRules, iptablesRule{ + name: curName, + shouldExist: !n.useExternalSNAT, + table: "nat", + chain: curChain, + rule: []string{ + "!", "-d", cidr, "-m", "comment", "--comment", comment, "-j", nextChain, + }}) + } + + iptableRules = append(iptableRules, iptablesRule{ + name: "connmark rule for external outbound traffic", + shouldExist: !n.useExternalSNAT, + table: "nat", + chain: chains[len(chains)-1], + rule: []string{ + "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", + "--set-xmark", fmt.Sprintf("%#x/%#x", n.mainENIMark, n.mainENIMark), + }, + }) + + // Force delete existing restore mark rule so that the subsequent rule gets added to the end + iptableRules = append(iptableRules, iptablesRule{ + name: "connmark to fwmark copy", + shouldExist: false, + table: "nat", + chain: "PREROUTING", + rule: []string{ + "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", + "--restore-mark", "--mask", fmt.Sprintf("%#x", n.mainENIMark), + }, + }) + + iptableRules = append(iptableRules, iptablesRule{ + name: "connmark to fwmark copy", + shouldExist: !n.useExternalSNAT, + table: "nat", + chain: "PREROUTING", + rule: []string{ + "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", + "--restore-mark", "--mask", fmt.Sprintf("%#x", n.mainENIMark), + }, + }) + + connmarkStaleRules, err := computeStaleIptablesRules(ipt, "nat", "AWS-CONNMARK-CHAIN", iptableRules, chains) + if err != nil { + return []iptablesRule{}, err + } + iptableRules = append(iptableRules, connmarkStaleRules...) + + log.Debugf("iptableRules: %v", iptableRules) + return iptableRules, nil +} + +func (n *linuxNetwork) updateIptablesRules(iptableRules []iptablesRule, ipt iptablesIface) error { for _, rule := range iptableRules { log.Debugf("execute iptable rule : %s", rule.name) exists, err := ipt.Exists(rule.table, rule.chain, rule.rule...) + log.Debugf("rule %v exists %v, err %v", rule, exists, err) if err != nil { log.Errorf("host network setup: failed to check existence of %v, %v", rule, err) return errors.Wrapf(err, "host network setup: failed to check existence of %v", rule) @@ -477,19 +600,18 @@ func (n *linuxNetwork) SetupHostNetwork(vpcCIDRs []string, primaryMAC string, pr return nil } -func listCurrentSNATRules(ipt iptablesIface) ([]iptablesRule, error) { +func listCurrentIptablesRules(ipt iptablesIface, table, chainPrefix string) ([]iptablesRule, error) { var toClear []iptablesRule - log.Debug("Setup Host Network: loading existing iptables nat SNAT exclusion rules") - - existingChains, err := ipt.ListChains("nat") + log.Debugf("Setup Host Network: loading existing iptables %s rules with chain prefix %s", table, chainPrefix) + existingChains, err := ipt.ListChains(table) if err != nil { - return nil, errors.Wrap(err, "host network setup: failed to list iptables nat chains") + return nil, errors.Wrapf(err, "host network setup: failed to list iptables %s chains", table) } for _, chain := range existingChains { - if !strings.HasPrefix(chain, "AWS-SNAT-CHAIN") { + if !strings.HasPrefix(chain, chainPrefix) { continue } - rules, err := ipt.List("nat", chain) + rules, err := ipt.List(table, chain) if err != nil { return nil, errors.Wrap(err, fmt.Sprintf("host network setup: failed to list iptables nat chain %s", chain)) } @@ -504,7 +626,7 @@ func listCurrentSNATRules(ipt iptablesIface) ([]iptablesRule, error) { toClear = append(toClear, iptablesRule{ name: fmt.Sprintf("[%d] %s", i, chain), shouldExist: false, // To trigger ipt.Delete for stale rules - table: "nat", + table: table, chain: chain, rule: ruleSpec[2:], //drop action and chain name }) @@ -513,6 +635,35 @@ func listCurrentSNATRules(ipt iptablesIface) ([]iptablesRule, error) { return toClear, nil } +func computeStaleIptablesRules(ipt iptablesIface, table, chainPrefix string, newRules []iptablesRule, chains []string) ([]iptablesRule, error) { + var staleRules []iptablesRule + existingRules, err := listCurrentIptablesRules(ipt, table, chainPrefix) + if err != nil { + return []iptablesRule{}, errors.Wrapf(err, "host network setup: failed to list rules from table %s with chain prefix %s", table, chainPrefix) + } + activeChains := sets.NewString(chains...) + log.Debugf("Setup Host Network: computing stale iptables rules for %s table with chain prefix %s") + for _, staleRule := range existingRules { + if len(staleRule.rule) == 0 && activeChains.Has(staleRule.chain) { + log.Debugf("Setup Host Network: active chain found: %s", staleRule.chain) + continue + } + keepRule := false + for _, newRule := range newRules { + if staleRule.chain == newRule.chain && reflect.DeepEqual(newRule.rule, staleRule.rule) { + log.Debugf("Setup Host Network: active rule found: %s", staleRule) + keepRule = true + break + } + } + if !keepRule { + log.Debugf("Setup Host Network: stale rule found: %s", staleRule) + staleRules = append(staleRules, staleRule) + } + } + return staleRules, nil +} + func containChainExistErr(err error) bool { return strings.Contains(err.Error(), "Chain already exists") } @@ -525,7 +676,7 @@ type iptablesRule struct { } func (r iptablesRule) String() string { - return fmt.Sprintf("%s/%s rule %s", r.table, r.chain, r.name) + return fmt.Sprintf("%s/%s rule %s shouldExist %v rule %v", r.table, r.chain, r.name, r.shouldExist, r.rule) } func containsNoSuchRule(err error) bool { @@ -550,6 +701,7 @@ func GetConfigForDebug() map[string]interface{} { envExcludeSNATCIDRs: getExcludeSNATCIDRs(), envExternalSNAT: useExternalSNAT(), envMTU: GetEthernetMTU(""), + envVethPrefix: getVethPrefixName(), envNodePortSupport: nodePortSupportEnabled(), envRandomizeSNAT: typeOfSNAT(), } @@ -876,9 +1028,8 @@ func (n *linuxNetwork) DeleteRuleListBySrc(src net.IPNet) error { } // UpdateRuleListBySrc modify IP rules that have a matching source IP -func (n *linuxNetwork) UpdateRuleListBySrc(ruleList []netlink.Rule, src net.IPNet, toCIDRs []string, requiresSNAT bool) error { - log.Debugf("Update Rule List[%v] for source[%v] with toCIDRs[%v], excludeSNATCIDRs[%v], requiresSNAT[%v]", - ruleList, src, toCIDRs, n.excludeSNATCIDRs, requiresSNAT) +func (n *linuxNetwork) UpdateRuleListBySrc(ruleList []netlink.Rule, src net.IPNet) error { + log.Debugf("Update Rule List[%v] for source[%v] ", ruleList, src) srcRuleList, err := n.GetRuleListBySrc(ruleList, src) if err != nil { @@ -906,41 +1057,19 @@ func (n *linuxNetwork) UpdateRuleListBySrc(ruleList []netlink.Rule, src net.IPNe return nil } - if requiresSNAT { - allCIDRs := append(toCIDRs, n.excludeSNATCIDRs...) - for _, cidr := range allCIDRs { - podRule := n.netLink.NewRule() - _, podRule.Dst, _ = net.ParseCIDR(cidr) - podRule.Src = &src - podRule.Table = srcRuleTable - podRule.Priority = fromPodRulePriority - - err = n.netLink.RuleAdd(podRule) - if err != nil { - log.Errorf("Failed to add pod IP rule for external SNAT: %v", err) - return errors.Wrapf(err, "UpdateRuleListBySrc: failed to add pod rule for CIDR %s", cidr) - } - var toDst string - - if podRule.Dst != nil { - toDst = podRule.Dst.String() - } - log.Infof("UpdateRuleListBySrc: Successfully added pod rule[%v] to %s", podRule, toDst) - } - } else { - podRule := n.netLink.NewRule() + podRule := n.netLink.NewRule() - podRule.Src = &src - podRule.Table = srcRuleTable - podRule.Priority = fromPodRulePriority + podRule.Src = &src + podRule.Table = srcRuleTable + podRule.Priority = fromPodRulePriority - err = n.netLink.RuleAdd(podRule) - if err != nil { - log.Errorf("Failed to add pod IP rule: %v", err) - return errors.Wrapf(err, "UpdateRuleListBySrc: failed to add pod rule") - } - log.Infof("UpdateRuleListBySrc: Successfully added pod rule[%v]", podRule) + err = n.netLink.RuleAdd(podRule) + if err != nil { + log.Errorf("Failed to add pod IP rule: %v", err) + return errors.Wrapf(err, "UpdateRuleListBySrc: failed to add pod rule") } + log.Infof("UpdateRuleListBySrc: Successfully added pod rule[%v]", podRule) + return nil } @@ -971,3 +1100,11 @@ func GetEthernetMTU(envMTUValue string) int { } return maximumMTU } + +// getVethPrefixName gets the name prefix of the veth devices based on the AWS_VPC_K8S_CNI_VETHPREFIX environment variable +func getVethPrefixName() string { + if envVal, found := os.LookupEnv(envVethPrefix); found { + return envVal + } + return envVethPrefixDefault +} diff --git a/pkg/networkutils/network_test.go b/pkg/networkutils/network_test.go index e638155111..adc6e946d2 100644 --- a/pkg/networkutils/network_test.go +++ b/pkg/networkutils/network_test.go @@ -45,7 +45,8 @@ const ( testeniSubnet = "10.10.0.0/16" // Default MTU of ENI and veth // defined in plugins/routed-eni/driver/driver.go, pkg/networkutils/network.go - testMTU = 9001 + testMTU = 9001 + eniPrefix = "eni" ) var ( @@ -185,80 +186,35 @@ func TestUpdateRuleListBySrc(t *testing.T) { Table: testTable, } testCases := []struct { - name string - oldRule netlink.Rule - requiresSNAT bool - toCIDRs []string - snatExclusionCIDRs []string - ruleList []netlink.Rule - newRules []netlink.Rule - expDst []*net.IPNet - expTable []int + name string + oldRule netlink.Rule + requiresSNAT bool + ruleList []netlink.Rule + newRule netlink.Rule }{ { "multiple destinations", origRule, true, - []string{"10.10.0.0/16", "10.11.0.0/16"}, - nil, []netlink.Rule{origRule}, - make([]netlink.Rule, 2), - make([]*net.IPNet, 2), - []int{origRule.Table, origRule.Table}, + netlink.Rule{}, }, { "single destination", origRule, false, - []string{""}, - nil, []netlink.Rule{origRule}, - make([]netlink.Rule, 1), - make([]*net.IPNet, 1), - []int{origRule.Table}, - }, - { - "SNAT exclusions", - origRule, - true, - []string{"10.10.0.0/16", "10.11.0.0/16"}, - []string{"10.12.0.0/16", "10.13.0.0/16"}, - []netlink.Rule{origRule}, - make([]netlink.Rule, 4), - make([]*net.IPNet, 4), - []int{origRule.Table, origRule.Table, origRule.Table, origRule.Table}, + netlink.Rule{}, }, } for _, tc := range testCases { - ln.excludeSNATCIDRs = tc.snatExclusionCIDRs - var newRuleSize int - if tc.requiresSNAT { - newRuleSize = len(tc.toCIDRs) + len(tc.snatExclusionCIDRs) - } else { - newRuleSize = 1 - } - - allCIDRs := append(tc.toCIDRs, tc.snatExclusionCIDRs...) - for i := 0; i < newRuleSize; i++ { - _, tc.expDst[i], _ = net.ParseCIDR(allCIDRs[i]) - } - mockNetLink.EXPECT().RuleDel(&tc.oldRule) + mockNetLink.EXPECT().NewRule().Return(&tc.newRule) + mockNetLink.EXPECT().RuleAdd(&tc.newRule) - for i := 0; i < newRuleSize; i++ { - mockNetLink.EXPECT().NewRule().Return(&tc.newRules[i]) - mockNetLink.EXPECT().RuleAdd(&tc.newRules[i]) - } - - err := ln.UpdateRuleListBySrc(tc.ruleList, *testENINetIPNet, tc.toCIDRs, tc.requiresSNAT) + err := ln.UpdateRuleListBySrc(tc.ruleList, *testENINetIPNet) assert.NoError(t, err) - - for i := 0; i < newRuleSize; i++ { - assert.Equal(t, tc.oldRule.Src, tc.newRules[i].Src, tc.name) - assert.Equal(t, tc.expDst[i], tc.newRules[i].Dst, tc.name) - assert.Equal(t, tc.expTable[i], tc.newRules[i].Table, tc.name) - } } } @@ -272,6 +228,7 @@ func TestSetupHostNetworkNodePortEnabled(t *testing.T) { shouldConfigureRpFilter: true, mainENIMark: defaultConnmark, mtu: testMTU, + vethPrefix: eniPrefix, netLink: mockNetLink, ns: mockNS, @@ -346,6 +303,7 @@ func TestSetupHostNetworkWithExcludeSNATCIDRs(t *testing.T) { shouldConfigureRpFilter: true, mainENIMark: defaultConnmark, mtu: testMTU, + vethPrefix: eniPrefix, netLink: mockNetLink, ns: mockNS, @@ -365,12 +323,22 @@ func TestSetupHostNetworkWithExcludeSNATCIDRs(t *testing.T) { assert.Equal(t, map[string]map[string][][]string{ "nat": { - "AWS-SNAT-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-1"}}, - "AWS-SNAT-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2"}}, - "AWS-SNAT-CHAIN-2": [][]string{{"!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-3"}}, - "AWS-SNAT-CHAIN-3": [][]string{{"!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-4"}}, - "AWS-SNAT-CHAIN-4": [][]string{{"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}}, - "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}}, + "AWS-SNAT-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-1"}}, + "AWS-SNAT-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2"}}, + "AWS-SNAT-CHAIN-2": [][]string{{"!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-3"}}, + "AWS-SNAT-CHAIN-3": [][]string{{"!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-4"}}, + "AWS-SNAT-CHAIN-4": [][]string{{"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}}, + "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, + "AWS-CONNMARK-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1"}}, + "AWS-CONNMARK-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2"}}, + "AWS-CONNMARK-CHAIN-2": [][]string{{"!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-3"}}, + "AWS-CONNMARK-CHAIN-3": [][]string{{"!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-4"}}, + "AWS-CONNMARK-CHAIN-4": [][]string{{"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}}, + "PREROUTING": [][]string{ + {"-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-m", "state", "--state", "NEW", "-j", "AWS-CONNMARK-CHAIN-0"}, + {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"}, + }, + }, "mangle": { "PREROUTING": [][]string{ {"-m", "comment", "--comment", "AWS, primary ENI", "-i", "lo", "-m", "addrtype", "--dst-type", "LOCAL", "--limit-iface-in", "-j", "CONNMARK", "--set-mark", "0x80/0x80"}, @@ -392,6 +360,7 @@ func TestSetupHostNetworkCleansUpStaleSNATRules(t *testing.T) { shouldConfigureRpFilter: true, mainENIMark: defaultConnmark, mtu: testMTU, + vethPrefix: eniPrefix, netLink: mockNetLink, ns: mockNS, @@ -411,6 +380,13 @@ func TestSetupHostNetworkCleansUpStaleSNATRules(t *testing.T) { _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-4", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") _ = mockIptables.NewChain("nat", "AWS-SNAT-CHAIN-5") _ = mockIptables.Append("nat", "POSTROUTING", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-1", "!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-2", "!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-3") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-3", "!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-4") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-4", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") + _ = mockIptables.Append("nat", "PREROUTING", "-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-m", "state", "--state", "NEW", "-j", "AWS-CONNMARK-CHAIN-0") + _ = mockIptables.Append("nat", "PREROUTING", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") vpcCIDRs := []string{"10.10.0.0/16", "10.11.0.0/16"} err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false) @@ -419,12 +395,22 @@ func TestSetupHostNetworkCleansUpStaleSNATRules(t *testing.T) { assert.Equal(t, map[string]map[string][][]string{ "nat": { - "AWS-SNAT-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-1"}}, - "AWS-SNAT-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2"}}, - "AWS-SNAT-CHAIN-2": [][]string{{"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}}, - "AWS-SNAT-CHAIN-3": [][]string{}, - "AWS-SNAT-CHAIN-4": [][]string{}, - "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}}, + "AWS-SNAT-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-1"}}, + "AWS-SNAT-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2"}}, + "AWS-SNAT-CHAIN-2": [][]string{{"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}}, + "AWS-SNAT-CHAIN-3": [][]string{}, + "AWS-SNAT-CHAIN-4": [][]string{}, + "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, + "AWS-CONNMARK-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1"}}, + "AWS-CONNMARK-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2"}}, + "AWS-CONNMARK-CHAIN-2": [][]string{{"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}}, + "AWS-CONNMARK-CHAIN-3": [][]string{}, + "AWS-CONNMARK-CHAIN-4": [][]string{}, + "PREROUTING": [][]string{ + {"-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-m", "state", "--state", "NEW", "-j", "AWS-CONNMARK-CHAIN-0"}, + {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"}, + }, + }, "mangle": { "PREROUTING": [][]string{ {"-m", "comment", "--comment", "AWS, primary ENI", "-i", "lo", "-m", "addrtype", "--dst-type", "LOCAL", "--limit-iface-in", "-j", "CONNMARK", "--set-mark", "0x80/0x80"}, @@ -435,6 +421,146 @@ func TestSetupHostNetworkCleansUpStaleSNATRules(t *testing.T) { }, mockIptables.dataplaneState) } +func TestSetupHostNetworkWithDifferentVethPrefix(t *testing.T) { + ctrl, mockNetLink, _, mockNS, mockIptables, mockProcSys := setup(t) + defer ctrl.Finish() + + ln := &linuxNetwork{ + useExternalSNAT: false, + excludeSNATCIDRs: []string{"10.12.0.0/16", "10.13.0.0/16"}, + nodePortSupportEnabled: true, + shouldConfigureRpFilter: true, + mainENIMark: defaultConnmark, + mtu: testMTU, + vethPrefix: "veth", + + netLink: mockNetLink, + ns: mockNS, + newIptables: func() (iptablesIface, error) { + return mockIptables, nil + }, + procSys: mockProcSys, + } + + setupNetLinkMocks(ctrl, mockNetLink) + + mockProcSys.EXPECT().Set("net/ipv4/conf/lo/rp_filter", "2").Return(nil) + + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAN", "-j", "AWS-SNAT-CHAIN-1") //AWS SNAT CHAN proves backwards compatibility + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-1", "!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-2", "!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-3") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-3", "!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-4") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-4", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") + _ = mockIptables.NewChain("nat", "AWS-SNAT-CHAIN-5") + _ = mockIptables.Append("nat", "POSTROUTING", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-1", "!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-2", "!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-3") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-3", "!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-4") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-4", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") + _ = mockIptables.Append("nat", "PREROUTING", "-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-m", "state", "--state", "NEW", "-j", "AWS-CONNMARK-CHAIN-0") + _ = mockIptables.Append("nat", "PREROUTING", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") + + vpcCIDRs := []string{"10.10.0.0/16", "10.11.0.0/16"} + err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false) + assert.NoError(t, err) + assert.Equal(t, + map[string]map[string][][]string{ + "nat": { + "AWS-SNAT-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-1"}}, + "AWS-SNAT-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2"}}, + "AWS-SNAT-CHAIN-2": [][]string{{"!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-3"}}, + "AWS-SNAT-CHAIN-3": [][]string{{"!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-4"}}, + "AWS-SNAT-CHAIN-4": [][]string{{"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}}, + "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, + "AWS-CONNMARK-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1"}}, + "AWS-CONNMARK-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2"}}, + "AWS-CONNMARK-CHAIN-2": [][]string{{"!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-3"}}, + "AWS-CONNMARK-CHAIN-3": [][]string{{"!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-4"}}, + "AWS-CONNMARK-CHAIN-4": [][]string{{"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}}, + "PREROUTING": [][]string{ + {"-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-m", "state", "--state", "NEW", "-j", "AWS-CONNMARK-CHAIN-0"}, + {"-i", "veth+", "-m", "comment", "--comment", "AWS, outbound connections", "-m", "state", "--state", "NEW", "-j", "AWS-CONNMARK-CHAIN-0"}, + {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"}, + }, + }, + "mangle": { + "PREROUTING": [][]string{ + {"-m", "comment", "--comment", "AWS, primary ENI", "-i", "lo", "-m", "addrtype", "--dst-type", "LOCAL", "--limit-iface-in", "-j", "CONNMARK", "--set-mark", "0x80/0x80"}, + {"-m", "comment", "--comment", "AWS, primary ENI", "-i", "veth+", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"}, + {"-m", "comment", "--comment", "AWS, primary ENI", "-i", "vlan+", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"}, + }, + }, + }, mockIptables.dataplaneState) +} +func TestSetupHostNetworkExternalNATCleanupConnmark(t *testing.T) { + ctrl, mockNetLink, _, mockNS, mockIptables, mockProcSys := setup(t) + defer ctrl.Finish() + + ln := &linuxNetwork{ + useExternalSNAT: true, + excludeSNATCIDRs: []string{"10.12.0.0/16", "10.13.0.0/16"}, + nodePortSupportEnabled: true, + shouldConfigureRpFilter: true, + mainENIMark: defaultConnmark, + mtu: testMTU, + vethPrefix: eniPrefix, + + netLink: mockNetLink, + ns: mockNS, + newIptables: func() (iptablesIface, error) { + return mockIptables, nil + }, + procSys: mockProcSys, + } + setupNetLinkMocks(ctrl, mockNetLink) + + mockProcSys.EXPECT().Set("net/ipv4/conf/lo/rp_filter", "2").Return(nil) + + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-1") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-1", "!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-2", "!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-3") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-3", "!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-4") + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-4", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") + _ = mockIptables.Append("nat", "POSTROUTING", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-1", "!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-2", "!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-3") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-3", "!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-4") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-4", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") + _ = mockIptables.Append("nat", "PREROUTING", "-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-m", "state", "--state", "NEW", "-j", "AWS-CONNMARK-CHAIN-0") + _ = mockIptables.Append("nat", "PREROUTING", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") + + // remove exclusions + vpcCIDRs := []string{"10.10.0.0/16", "10.11.0.0/16"} + err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false) + assert.NoError(t, err) + + assert.Equal(t, + map[string]map[string][][]string{ + "nat": { + "AWS-SNAT-CHAIN-0": [][]string{}, + "AWS-SNAT-CHAIN-1": [][]string{}, + "AWS-SNAT-CHAIN-2": [][]string{}, + "AWS-SNAT-CHAIN-3": [][]string{}, + "AWS-SNAT-CHAIN-4": [][]string{}, + "POSTROUTING": [][]string{}, + "AWS-CONNMARK-CHAIN-0": [][]string{}, + "AWS-CONNMARK-CHAIN-1": [][]string{}, + "AWS-CONNMARK-CHAIN-2": [][]string{}, + "AWS-CONNMARK-CHAIN-3": [][]string{}, + "AWS-CONNMARK-CHAIN-4": [][]string{}, + "PREROUTING": [][]string{}, + }, + "mangle": { + "PREROUTING": [][]string{ + {"-m", "comment", "--comment", "AWS, primary ENI", "-i", "lo", "-m", "addrtype", "--dst-type", "LOCAL", "--limit-iface-in", "-j", "CONNMARK", "--set-mark", "0x80/0x80"}, + {"-m", "comment", "--comment", "AWS, primary ENI", "-i", "eni+", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"}, + {"-m", "comment", "--comment", "AWS, primary ENI", "-i", "vlan+", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"}, + }, + }, + }, mockIptables.dataplaneState) +} func TestSetupHostNetworkExcludedSNATCIDRsIdempotent(t *testing.T) { ctrl, mockNetLink, _, mockNS, mockIptables, mockProcSys := setup(t) defer ctrl.Finish() @@ -446,6 +572,7 @@ func TestSetupHostNetworkExcludedSNATCIDRsIdempotent(t *testing.T) { shouldConfigureRpFilter: true, mainENIMark: defaultConnmark, mtu: testMTU, + vethPrefix: eniPrefix, netLink: mockNetLink, ns: mockNS, @@ -464,6 +591,13 @@ func TestSetupHostNetworkExcludedSNATCIDRsIdempotent(t *testing.T) { _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-3", "!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-4") _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-4", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") _ = mockIptables.Append("nat", "POSTROUTING", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-1", "!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-2", "!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-3") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-3", "!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-4") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-4", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") + _ = mockIptables.Append("nat", "PREROUTING", "-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-m", "state", "--state", "NEW", "-j", "AWS-CONNMARK-CHAIN-0") + _ = mockIptables.Append("nat", "PREROUTING", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") // remove exclusions vpcCIDRs := []string{"10.10.0.0/16", "10.11.0.0/16"} @@ -473,12 +607,22 @@ func TestSetupHostNetworkExcludedSNATCIDRsIdempotent(t *testing.T) { assert.Equal(t, map[string]map[string][][]string{ "nat": { - "AWS-SNAT-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-1"}}, - "AWS-SNAT-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2"}}, - "AWS-SNAT-CHAIN-2": [][]string{{"!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-3"}}, - "AWS-SNAT-CHAIN-3": [][]string{{"!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-4"}}, - "AWS-SNAT-CHAIN-4": [][]string{{"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}}, - "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}}, + "AWS-SNAT-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-1"}}, + "AWS-SNAT-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2"}}, + "AWS-SNAT-CHAIN-2": [][]string{{"!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-3"}}, + "AWS-SNAT-CHAIN-3": [][]string{{"!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN EXCLUSION", "-j", "AWS-SNAT-CHAIN-4"}}, + "AWS-SNAT-CHAIN-4": [][]string{{"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}}, + "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, + "AWS-CONNMARK-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1"}}, + "AWS-CONNMARK-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2"}}, + "AWS-CONNMARK-CHAIN-2": [][]string{{"!", "-d", "10.12.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-3"}}, + "AWS-CONNMARK-CHAIN-3": [][]string{{"!", "-d", "10.13.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, EXCLUDED CIDR", "-j", "AWS-CONNMARK-CHAIN-4"}}, + "AWS-CONNMARK-CHAIN-4": [][]string{{"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}}, + "PREROUTING": [][]string{ + {"-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-m", "state", "--state", "NEW", "-j", "AWS-CONNMARK-CHAIN-0"}, + {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"}, + }, + }, "mangle": { "PREROUTING": [][]string{ {"-m", "comment", "--comment", "AWS, primary ENI", "-i", "lo", "-m", "addrtype", "--dst-type", "LOCAL", "--limit-iface-in", "-j", "CONNMARK", "--set-mark", "0x80/0x80"}, @@ -489,6 +633,70 @@ func TestSetupHostNetworkExcludedSNATCIDRsIdempotent(t *testing.T) { }, mockIptables.dataplaneState) } +func TestUpdateHostIptablesRules(t *testing.T) { + ctrl, mockNetLink, _, mockNS, mockIptables, mockProcSys := setup(t) + defer ctrl.Finish() + + ln := &linuxNetwork{ + useExternalSNAT: false, + nodePortSupportEnabled: true, + shouldConfigureRpFilter: true, + mainENIMark: defaultConnmark, + mtu: testMTU, + vethPrefix: "veth", + + netLink: mockNetLink, + ns: mockNS, + newIptables: func() (iptablesIface, error) { + return mockIptables, nil + }, + procSys: mockProcSys, + } + + setupNetLinkMocks(ctrl, mockNetLink) + + mockProcSys.EXPECT().Set("net/ipv4/conf/lo/rp_filter", "2").Return(nil) + + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAN", "-j", "AWS-SNAT-CHAIN-1") //AWS SNAT CHAN proves backwards compatibility + _ = mockIptables.Append("nat", "AWS-SNAT-CHAIN-1", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20") + _ = mockIptables.Append("nat", "POSTROUTING", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-0", "!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1") + _ = mockIptables.Append("nat", "AWS-CONNMARK-CHAIN-1", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80") + _ = mockIptables.Append("nat", "PREROUTING", "-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-m", "state", "--state", "NEW", "-j", "AWS-CONNMARK-CHAIN-0") + _ = mockIptables.Append("nat", "PREROUTING", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") + _ = mockIptables.Append("mangle", "PREROUTING", "-m", "comment", "--comment", "AWS, primary ENI", "-i", "lo", "-m", "addrtype", "--dst-type", "LOCAL", "--limit-iface-in", "-j", "CONNMARK", "--set-mark", "0x80/0x80") + _ = mockIptables.Append("mangle", "PREROUTING", "-m", "comment", "--comment", "AWS, primary ENI", "-i", "eni+", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") + _ = mockIptables.Append("mangle", "PREROUTING", "-m", "comment", "--comment", "AWS, primary ENI", "-i", "vlan+", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") + + vpcCIDRs := []string{"10.10.0.0/16", "10.11.0.0/16"} + err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false) + assert.NoError(t, err) + assert.Equal(t, + map[string]map[string][][]string{ + "nat": { + "AWS-SNAT-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-1"}}, + "AWS-SNAT-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-2"}}, + "AWS-SNAT-CHAIN-2": [][]string{{"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "10.10.10.20"}}, + "POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}, + "AWS-CONNMARK-CHAIN-0": [][]string{{"!", "-d", "10.10.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-1"}}, + "AWS-CONNMARK-CHAIN-1": [][]string{{"!", "-d", "10.11.0.0/16", "-m", "comment", "--comment", "AWS CONNMARK CHAIN, VPC CIDR", "-j", "AWS-CONNMARK-CHAIN-2"}}, + "AWS-CONNMARK-CHAIN-2": [][]string{{"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"}}, + "PREROUTING": [][]string{ + {"-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-m", "state", "--state", "NEW", "-j", "AWS-CONNMARK-CHAIN-0"}, + {"-i", "veth+", "-m", "comment", "--comment", "AWS, outbound connections", "-m", "state", "--state", "NEW", "-j", "AWS-CONNMARK-CHAIN-0"}, + {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"}, + }, + }, + "mangle": { + "PREROUTING": [][]string{ + {"-m", "comment", "--comment", "AWS, primary ENI", "-i", "lo", "-m", "addrtype", "--dst-type", "LOCAL", "--limit-iface-in", "-j", "CONNMARK", "--set-mark", "0x80/0x80"}, + {"-m", "comment", "--comment", "AWS, primary ENI", "-i", "eni+", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"}, + {"-m", "comment", "--comment", "AWS, primary ENI", "-i", "vlan+", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"}, + {"-m", "comment", "--comment", "AWS, primary ENI", "-i", "veth+", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"}, + }, + }, + }, mockIptables.dataplaneState) +} func TestSetupHostNetworkMultipleCIDRs(t *testing.T) { ctrl, mockNetLink, _, mockNS, mockIptables, mockProcSys := setup(t) defer ctrl.Finish() @@ -499,6 +707,7 @@ func TestSetupHostNetworkMultipleCIDRs(t *testing.T) { shouldConfigureRpFilter: true, mainENIMark: defaultConnmark, mtu: testMTU, + vethPrefix: eniPrefix, netLink: mockNetLink, ns: mockNS, @@ -552,6 +761,7 @@ func TestSetupHostNetworkIgnoringRpFilterUpdate(t *testing.T) { shouldConfigureRpFilter: false, mainENIMark: defaultConnmark, mtu: testMTU, + vethPrefix: eniPrefix, netLink: mockNetLink, ns: mockNS, @@ -577,6 +787,7 @@ func TestSetupHostNetworkUpdateLocalRule(t *testing.T) { shouldConfigureRpFilter: false, mainENIMark: defaultConnmark, mtu: testMTU, + vethPrefix: eniPrefix, netLink: mockNetLink, ns: mockNS, diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index c26dc2fef9..3fb78bd0aa 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -42,6 +42,18 @@ validate_env_var() log_in_json error "AWS_VPC_K8S_PLUGIN_LOG_FILE cannot be set to stdout" exit 1 fi + + if [[ ${#AWS_VPC_K8S_CNI_VETHPREFIX} -gt 4 ]]; then + log_in_json error "AWS_VPC_K8S_CNI_VETHPREFIX cannot be longer than 4 characters" + exit 1 + fi + + case ${AWS_VPC_K8S_CNI_VETHPREFIX} in + eth|vlan|lo) + log_in_json error "AWS_VPC_K8S_CNI_VETHPREFIX cannot be set to reserved values eth or vlan or lo" + exit 1 + ;; + esac } # Check for all the required binaries before we go forward diff --git a/test/framework/resources/k8s/utils/daemonset.go b/test/framework/resources/k8s/utils/daemonset.go index ccf204dc21..ea408cbbdc 100644 --- a/test/framework/resources/k8s/utils/daemonset.go +++ b/test/framework/resources/k8s/utils/daemonset.go @@ -25,39 +25,47 @@ import ( func AddEnvVarToDaemonSetAndWaitTillUpdated(f *framework.Framework, dsName string, dsNamespace string, containerName string, envVars map[string]string) { - - ds := getDaemonSet(f, dsName, dsNamespace) - updatedDs := ds.DeepCopy() - By(fmt.Sprintf("setting the environment variables on the ds to %+v", envVars)) - err := AddOrUpdateEnvironmentVariable(updatedDs.Spec.Template.Spec.Containers, - containerName, envVars) - // Check for init containers if the container is not found in list of containers - if err != nil { - err = AddOrUpdateEnvironmentVariable(updatedDs.Spec.Template.Spec.InitContainers, - containerName, envVars) - } - Expect(err).ToNot(HaveOccurred()) - - waitTillDaemonSetUpdated(f, ds, updatedDs) + updateDaemonsetEnvVarsAndWait(f, dsName, dsNamespace, containerName, envVars, nil) } func RemoveVarFromDaemonSetAndWaitTillUpdated(f *framework.Framework, dsName string, dsNamespace string, containerName string, envVars map[string]struct{}) { + By(fmt.Sprintf("removing the environment variables from the ds %+v", envVars)) + updateDaemonsetEnvVarsAndWait(f, dsName, dsNamespace, containerName, nil, envVars) +} + +func UpdateEnvVarOnDaemonSetAndWaitUntilReady(f *framework.Framework, dsName string, dsNamespace string, + containerName string, addOrUpdateEnv map[string]string, removeEnv map[string]struct{}) { + By(fmt.Sprintf("update environment variables %+v, remove %+v", addOrUpdateEnv, removeEnv)) + updateDaemonsetEnvVarsAndWait(f, dsName, dsNamespace, containerName, addOrUpdateEnv, removeEnv) +} +func updateDaemonsetEnvVarsAndWait(f *framework.Framework, dsName string, dsNamespace string, + containerName string, addOrUpdateEnv map[string]string, removeEnv map[string]struct{}) { ds := getDaemonSet(f, dsName, dsNamespace) updatedDs := ds.DeepCopy() - By(fmt.Sprintf("setting the environment variables on the ds to %+v", envVars)) - err := RemoveEnvironmentVariables(updatedDs.Spec.Template.Spec.Containers, - containerName, envVars) - // Check for init containers if the container is not found in list of containers - if err != nil { - err = RemoveEnvironmentVariables(updatedDs.Spec.Template.Spec.InitContainers, - containerName, envVars) + if len(addOrUpdateEnv) > 0 { + err := AddOrUpdateEnvironmentVariable(updatedDs.Spec.Template.Spec.Containers, + containerName, addOrUpdateEnv) + // Check for init containers if the container is not found in list of containers + if err != nil { + err = AddOrUpdateEnvironmentVariable(updatedDs.Spec.Template.Spec.InitContainers, + containerName, addOrUpdateEnv) + } + Expect(err).ToNot(HaveOccurred()) + } + if len(removeEnv) > 0 { + err := RemoveEnvironmentVariables(updatedDs.Spec.Template.Spec.Containers, + containerName, removeEnv) + // Check for init containers if the container is not found in list of containers + if err != nil { + err = RemoveEnvironmentVariables(updatedDs.Spec.Template.Spec.InitContainers, + containerName, removeEnv) + } + Expect(err).ToNot(HaveOccurred()) } - Expect(err).ToNot(HaveOccurred()) - waitTillDaemonSetUpdated(f, ds, updatedDs) } diff --git a/test/framework/utils/const.go b/test/framework/utils/const.go index 517673d99a..2ccbf8ab7b 100644 --- a/test/framework/utils/const.go +++ b/test/framework/utils/const.go @@ -22,7 +22,7 @@ const ( AWSInitContainerName = "aws-vpc-cni-init" // See https://gallery.ecr.aws/r3i6j7b0/aws-vpc-cni-test-helper - TestAgentImage = "public.ecr.aws/r3i6j7b0/aws-vpc-cni-test-helper:143009a3" + TestAgentImage = "public.ecr.aws/r3i6j7b0/aws-vpc-cni-test-helper:66180f40" ) const ( diff --git a/test/integration-new/cni/host_networking_test.go b/test/integration-new/cni/host_networking_test.go index f9761a1e22..03a3abe687 100644 --- a/test/integration-new/cni/host_networking_test.go +++ b/test/integration-new/cni/host_networking_test.go @@ -45,6 +45,8 @@ const ( AWS_VPC_K8S_CNI_VETHPREFIX = "AWS_VPC_K8S_CNI_VETHPREFIX" NEW_MTU_VAL = 1300 NEW_VETH_PREFIX = "veth" + DEFAULT_MTU_VAL = "9001" + DEFAULT_VETH_PREFIX = "eni" ) var _ = Describe("test host networking", func() { @@ -53,6 +55,12 @@ var _ = Describe("test host networking", func() { var podLabelVal = "host-networking-test" Context("when pods using IP from primary and secondary ENI are created", func() { + AfterEach(func() { + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ + AWS_VPC_ENI_MTU: DEFAULT_MTU_VAL, + AWS_VPC_K8S_CNI_VETHPREFIX: DEFAULT_VETH_PREFIX, + }) + }) It("should have correct host networking setup when running and cleaned up once terminated", func() { // Launch enough pods so some pods end up using primary ENI IP and some using secondary // ENI IP @@ -105,12 +113,6 @@ var _ = Describe("test host networking", func() { Build() By("Configuring Veth Prefix and MTU value on aws-node daemonset") - ds, err := f.K8sResourceManagers.DaemonSetManager().GetDaemonSet(utils.AwsNodeNamespace, utils.AwsNodeName) - Expect(err).NotTo(HaveOccurred()) - - oldMTU := utils.GetEnvValueForKeyFromDaemonSet(AWS_VPC_ENI_MTU, ds) - oldVethPrefix := utils.GetEnvValueForKeyFromDaemonSet(AWS_VPC_K8S_CNI_VETHPREFIX, ds) - k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ AWS_VPC_ENI_MTU: strconv.Itoa(NEW_MTU_VAL), AWS_VPC_K8S_CNI_VETHPREFIX: NEW_VETH_PREFIX, @@ -136,13 +138,6 @@ var _ = Describe("test host networking", func() { By("validating host networking setup is setup correctly with MTU check as well") ValidateHostNetworking(NetworkingSetupSucceeds, input) - // Restore MTU and Veth Prefix - By("Restoring MTU value and Veth Prefix to old values") - k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ - AWS_VPC_ENI_MTU: oldMTU, - AWS_VPC_K8S_CNI_VETHPREFIX: oldVethPrefix, - }) - By("deleting the deployment to test teardown") err = f.K8sResourceManagers.DeploymentManager(). DeleteAndWaitTillDeploymentIsDeleted(deployment) @@ -244,7 +239,6 @@ func ValidateHostNetworking(testType TestType, podValidationInputString string) By("creating pod to test host networking setup") testPod, err := f.K8sResourceManagers.PodManager(). CreateAndWaitTillPodCompleted(testPod) - logs, errLogs := f.K8sResourceManagers.PodManager(). PodLogs(testPod.Namespace, testPod.Name) Expect(errLogs).ToNot(HaveOccurred()) diff --git a/test/integration-new/cni/pod_networking_suite_test.go b/test/integration-new/cni/pod_networking_suite_test.go index 73c613a18f..19eb3f1499 100644 --- a/test/integration-new/cni/pod_networking_suite_test.go +++ b/test/integration-new/cni/pod_networking_suite_test.go @@ -95,6 +95,13 @@ var _ = AfterSuite(func() { f.K8sResourceManagers.NamespaceManager(). DeleteAndWaitTillNamespaceDeleted(utils.DefaultTestNamespace) - k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, "aws-node", "kube-system", - "aws-node", map[string]struct{}{"WARM_IP_TARGET": {}, "WARM_ENI_TARGET": {}}) + k8sUtils.UpdateEnvVarOnDaemonSetAndWaitUntilReady(f, "aws-node", "kube-system", + "aws-node", map[string]string{ + AWS_VPC_ENI_MTU: "9001", + AWS_VPC_K8S_CNI_VETHPREFIX: "eni", + }, + map[string]struct{}{ + "WARM_IP_TARGET": {}, + "WARM_ENI_TARGET": {}, + }) })