diff --git a/net/bridge.go b/net/bridge.go index 2f6d641f..e6fb747c 100644 --- a/net/bridge.go +++ b/net/bridge.go @@ -17,6 +17,7 @@ import ( "github.com/weaveworks/weave/ipam/tracker" "github.com/weaveworks/weave/net/address" "github.com/weaveworks/weave/net/ipset" + "github.com/weaveworks/weave/npc" ) /* This code implements three possible configurations to connect @@ -457,14 +458,27 @@ func configureIPTables(config *BridgeConfig, ips ipset.Interface) error { } if config.NPC { - // Steer traffic via the NPC - _ = ipt.NewChain("filter", "WEAVE-NPC") // ignore error because it might already exist - // If WEAVE-NPC chain doesn't exist then creating a rule in the chain will fail + // Steer traffic via the NPC. + + if err = ensureChains(ipt, "filter", npc.MainChain, npc.EgressChain); err != nil { + return err + } + + // Steer egress traffic destined to local node. + if err = ipt.AppendUnique("filter", "INPUT", "-i", config.WeaveBridgeName, "-j", npc.EgressChain); err != nil { + return err + } fwdRules = append(fwdRules, [][]string{ + // Might include ingress traffic which is fine as long as we do not + // ACCEPT in WEAVE-NPC-EGRESS chain + {"-i", config.WeaveBridgeName, + "-m", "comment", "--comment", "NOTE: this must go before '-j KUBE-FORWARD'", + "-j", npc.EgressChain}, + // The following rules are for ingress NPC processing {"-o", config.WeaveBridgeName, "-m", "comment", "--comment", "NOTE: this must go before '-j KUBE-FORWARD'", - "-j", "WEAVE-NPC"}, + "-j", npc.MainChain}, {"-o", config.WeaveBridgeName, "-m", "state", "--state", "NEW", "-j", "NFLOG", "--nflog-group", "86"}, {"-o", config.WeaveBridgeName, "-j", "DROP"}, }...) @@ -570,43 +584,6 @@ func linkSetUpByName(linkName string) error { return netlink.LinkSetUp(link) } -// ensureRulesAtTop ensures the presence of given iptables rules. -// -// If any rule from the list is missing, the function deletes all given -// rules and re-inserts them at the top of the chain to ensure the order of the rules. -func ensureRulesAtTop(table, chain string, rulespecs [][]string, ipt *iptables.IPTables) error { - allFound := true - - for _, rs := range rulespecs { - found, err := ipt.Exists(table, chain, rs...) - if err != nil { - return errors.Wrapf(err, "ipt.Exists(%s, %s, %s)", table, chain, rs) - } - if !found { - allFound = false - break - } - } - - // All rules exist, do nothing. - if allFound { - return nil - } - - for pos, rs := range rulespecs { - // If any is missing, then delete all, as we need to preserve the order of - // given rules. Ignore errors, as rule might not exist. - if !allFound { - ipt.Delete(table, chain, rs...) - } - if err := ipt.Insert(table, chain, pos+1, rs...); err != nil { - return errors.Wrapf(err, "ipt.Append(%s, %s, %s)", table, chain, rs) - } - } - - return nil -} - func reexpose(config *BridgeConfig, log *logrus.Logger) error { // Get existing IP addrs of the weave bridge. // If the bridge hasn't been exposed, then this functions does nothing. diff --git a/net/iptables.go b/net/iptables.go new file mode 100644 index 00000000..a11ca533 --- /dev/null +++ b/net/iptables.go @@ -0,0 +1,106 @@ +package net + +import ( + "strings" + + "github.com/coreos/go-iptables/iptables" + "github.com/pkg/errors" +) + +// AddChainWithRules creates a chain and appends given rules to it. +// +// If the chain exists, but its rules are not the same as the given ones, the +// function will flush the chain and then will append the rules. +func AddChainWithRules(ipt *iptables.IPTables, table, chain string, rulespecs [][]string) error { + if err := ensureChains(ipt, table, chain); err != nil { + return err + } + + currRuleSpecs, err := ipt.List(table, chain) + if err != nil { + return errors.Wrapf(err, "iptables -S. table: %q, chain: %q", table, chain) + } + + // First returned rule is "-N $(chain)", so ignore it + currRules := strings.Join(currRuleSpecs[1:], "\n") + rules := make([]string, 0) + for _, r := range rulespecs { + rules = append(rules, strings.Join(r, " ")) + } + reqRules := strings.Join(rules, "\n") + + if currRules == reqRules { + return nil + } + + if err := ipt.ClearChain(table, chain); err != nil { + return err + } + + for _, r := range rulespecs { + if err := ipt.Append(table, chain, r...); err != nil { + return errors.Wrapf(err, "iptables -A. table: %q, chain: %q, rule: %s", table, chain, r) + } + } + + return nil +} + +// ensureChains creates given chains if they do not exist. +func ensureChains(ipt *iptables.IPTables, table string, chains ...string) error { + existingChains, err := ipt.ListChains(table) + if err != nil { + return errors.Wrapf(err, "ipt.ListChains(%s)", table) + } + chainMap := make(map[string]struct{}) + for _, c := range existingChains { + chainMap[c] = struct{}{} + } + + for _, c := range chains { + if _, found := chainMap[c]; !found { + if err := ipt.NewChain(table, c); err != nil { + return errors.Wrapf(err, "ipt.NewChain(%s, %s)", table, c) + } + } + } + + return nil +} + +// ensureRulesAtTop ensures the presence of given iptables rules. +// +// If any rule from the list is missing, the function deletes all given +// rules and re-inserts them at the top of the chain to ensure the order of the rules. +func ensureRulesAtTop(table, chain string, rulespecs [][]string, ipt *iptables.IPTables) error { + allFound := true + + for _, rs := range rulespecs { + found, err := ipt.Exists(table, chain, rs...) + if err != nil { + return errors.Wrapf(err, "ipt.Exists(%s, %s, %s)", table, chain, rs) + } + if !found { + allFound = false + break + } + } + + // All rules exist, do nothing. + if allFound { + return nil + } + + for pos, rs := range rulespecs { + // If any is missing, then delete all, as we need to preserve the order of + // given rules. Ignore errors, as rule might not exist. + if !allFound { + ipt.Delete(table, chain, rs...) + } + if err := ipt.Insert(table, chain, pos+1, rs...); err != nil { + return errors.Wrapf(err, "ipt.Append(%s, %s, %s)", table, chain, rs) + } + } + + return nil +} diff --git a/npc/analyser.go b/npc/analyser.go index 01cdf5ee..a19bb711 100644 --- a/npc/analyser.go +++ b/npc/analyser.go @@ -4,173 +4,158 @@ import ( "fmt" apiv1 "k8s.io/api/core/v1" - extnapi "k8s.io/api/extensions/v1beta1" networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/util/intstr" "github.com/weaveworks/weave/net/ipset" ) -// analysePolicyLegacy is used to analyse extensions/v1beta1/NetworkPolicy (legacy) and -// implements pre-1.7 k8s netpol semantics, whilst analysePolicy - networking.k8s.io/v1/NetworkPolicy -// and 1.7 semantics. - -func (ns *ns) analysePolicyLegacy(policy *extnapi.NetworkPolicy) ( - rules map[string]*ruleSpec, - nsSelectors, podSelectors map[string]*selectorSpec, - err error) { - - nsSelectors = make(map[string]*selectorSpec) - podSelectors = make(map[string]*selectorSpec) - rules = make(map[string]*ruleSpec) - - dstSelector, err := newSelectorSpec(&policy.Spec.PodSelector, true, ns.name, ipset.HashIP) - if err != nil { - return nil, nil, nil, err - } - podSelectors[dstSelector.key] = dstSelector - - for _, ingressRule := range policy.Spec.Ingress { - if ingressRule.Ports != nil && len(ingressRule.Ports) == 0 { - // Ports is empty, this rule matches no ports (no traffic matches). - continue - } - - if ingressRule.From != nil && len(ingressRule.From) == 0 { - // From is empty, this rule matches no sources (no traffic matches). - continue - } - - if ingressRule.From == nil { - // From is not provided, this rule matches all sources (traffic not restricted by source). - if ingressRule.Ports == nil { - // Ports is not provided, this rule matches all ports (traffic not restricted by port). - rule := newRuleSpec(nil, nil, dstSelector, nil) - rules[rule.key] = rule - } else { - // Ports is present and contains at least one item, then this rule allows traffic - // only if the traffic matches at least one port in the ports list. - withNormalisedProtoAndPortLegacy(ingressRule.Ports, func(proto, port string) { - rule := newRuleSpec(&proto, nil, dstSelector, &port) - rules[rule.key] = rule - }) - } - } else { - // From is present and contains at least on item, this rule allows traffic only if the - // traffic matches at least one item in the from list. - for _, peer := range ingressRule.From { - var srcSelector *selectorSpec - if peer.PodSelector != nil { - srcSelector, err = newSelectorSpec(peer.PodSelector, false, ns.name, ipset.HashIP) - if err != nil { - return nil, nil, nil, err - } - podSelectors[srcSelector.key] = srcSelector - } - if peer.NamespaceSelector != nil { - srcSelector, err = newSelectorSpec(peer.NamespaceSelector, false, "", ipset.ListSet) - if err != nil { - return nil, nil, nil, err - } - nsSelectors[srcSelector.key] = srcSelector - } - - if ingressRule.Ports == nil { - // Ports is not provided, this rule matches all ports (traffic not restricted by port). - rule := newRuleSpec(nil, srcSelector, dstSelector, nil) - rules[rule.key] = rule - } else { - // Ports is present and contains at least one item, then this rule allows traffic - // only if the traffic matches at least one port in the ports list. - withNormalisedProtoAndPortLegacy(ingressRule.Ports, func(proto, port string) { - rule := newRuleSpec(&proto, srcSelector, dstSelector, &port) - rules[rule.key] = rule - }) - } - } - } - } - - return rules, nsSelectors, podSelectors, nil -} - func (ns *ns) analysePolicy(policy *networkingv1.NetworkPolicy) ( rules map[string]*ruleSpec, nsSelectors, podSelectors map[string]*selectorSpec, + ipBlocks map[string]*ipBlockSpec, err error) { nsSelectors = make(map[string]*selectorSpec) podSelectors = make(map[string]*selectorSpec) + ipBlocks = make(map[string]*ipBlockSpec) rules = make(map[string]*ruleSpec) + policyTypes := make([]policyType, 0) + for _, pt := range policy.Spec.PolicyTypes { + if pt == networkingv1.PolicyTypeIngress { + policyTypes = append(policyTypes, policyTypeIngress) + } + if pt == networkingv1.PolicyTypeEgress { + policyTypes = append(policyTypes, policyTypeEgress) + } + } // If empty, matches all pods in a namespace - dstSelector, err := newSelectorSpec(&policy.Spec.PodSelector, true, ns.name, ipset.HashIP) + targetSelector, err := newSelectorSpec(&policy.Spec.PodSelector, policyTypes, ns.name, ipset.HashIP) if err != nil { - return nil, nil, nil, err + return nil, nil, nil, nil, err } - // To prevent dstSelector being overwritten by a subsequent selector with + // To prevent targetSelector being overwritten by a subsequent selector with // the same key, addIfNotExist MUST be used when adding to podSelectors. - // Otherwise, dstSelector' "dst" property might be lost resulting in - // an invalid content of the "default-allow" ipset. - addIfNotExist(dstSelector, podSelectors) + // Otherwise, policyTypes of the selector might be lost resulting in + // an invalid content in any "default-allow" ipset. + addIfNotExist(targetSelector, podSelectors) - // If ingress is empty then this NetworkPolicy does not allow any traffic - if policy.Spec.Ingress == nil || len(policy.Spec.Ingress) == 0 { - return - } - - for _, ingressRule := range policy.Spec.Ingress { - // If Ports is empty or missing, this rule matches all ports - allPorts := ingressRule.Ports == nil || len(ingressRule.Ports) == 0 - // If From is empty or missing, this rule matches all sources - allSources := ingressRule.From == nil || len(ingressRule.From) == 0 - - if allSources { - if allPorts { - rule := newRuleSpec(nil, nil, dstSelector, nil) - rules[rule.key] = rule - } else { - withNormalisedProtoAndPort(ingressRule.Ports, func(proto, port string) { - rule := newRuleSpec(&proto, nil, dstSelector, &port) - rules[rule.key] = rule - }) - } - } else { - for _, peer := range ingressRule.From { - var srcSelector *selectorSpec - - // NetworkPolicyPeer describes a peer to allow traffic from. - // Exactly one of its fields must be specified. - if peer.PodSelector != nil { - srcSelector, err = newSelectorSpec(peer.PodSelector, false, ns.name, ipset.HashIP) - if err != nil { - return nil, nil, nil, err - } - addIfNotExist(srcSelector, podSelectors) - - } else if peer.NamespaceSelector != nil { - srcSelector, err = newSelectorSpec(peer.NamespaceSelector, false, "", ipset.ListSet) - if err != nil { - return nil, nil, nil, err - } - nsSelectors[srcSelector.key] = srcSelector - } + // If ingress is empty then this NetworkPolicy does not allow any ingress traffic + if policy.Spec.Ingress != nil && len(policy.Spec.Ingress) != 0 { + for _, ingressRule := range policy.Spec.Ingress { + // If Ports is empty or missing, this rule matches all ports + allPorts := ingressRule.Ports == nil || len(ingressRule.Ports) == 0 + // If From is empty or missing, this rule matches all sources + allSources := ingressRule.From == nil || len(ingressRule.From) == 0 + if allSources { if allPorts { - rule := newRuleSpec(nil, srcSelector, dstSelector, nil) + rule := newRuleSpec(policyTypeIngress, nil, nil, targetSelector, nil) rules[rule.key] = rule } else { withNormalisedProtoAndPort(ingressRule.Ports, func(proto, port string) { - rule := newRuleSpec(&proto, srcSelector, dstSelector, &port) + rule := newRuleSpec(policyTypeIngress, &proto, nil, targetSelector, &port) rules[rule.key] = rule }) } + } else { + for _, peer := range ingressRule.From { + var srcSelector *selectorSpec + + // NetworkPolicyPeer describes a peer to allow traffic from. + // Exactly one of its fields must be specified. + if peer.PodSelector != nil { + srcSelector, err = newSelectorSpec(peer.PodSelector, nil, ns.name, ipset.HashIP) + if err != nil { + return nil, nil, nil, nil, err + } + addIfNotExist(srcSelector, podSelectors) + + } else if peer.NamespaceSelector != nil { + srcSelector, err = newSelectorSpec(peer.NamespaceSelector, nil, "", ipset.ListSet) + if err != nil { + return nil, nil, nil, nil, err + } + nsSelectors[srcSelector.key] = srcSelector + } + + if allPorts { + rule := newRuleSpec(policyTypeIngress, nil, srcSelector, targetSelector, nil) + rules[rule.key] = rule + } else { + withNormalisedProtoAndPort(ingressRule.Ports, func(proto, port string) { + rule := newRuleSpec(policyTypeIngress, &proto, srcSelector, targetSelector, &port) + rules[rule.key] = rule + }) + } + } } } } - return rules, nsSelectors, podSelectors, nil + // If egress is empty then this NetworkPolicy does not allow any egress traffic + if policy.Spec.Egress != nil && len(policy.Spec.Egress) != 0 { + for _, egressRule := range policy.Spec.Egress { + // If Ports is empty or missing, this rule matches all ports + allPorts := egressRule.Ports == nil || len(egressRule.Ports) == 0 + // If To is empty or missing, this rule matches all destinations + allDestinations := egressRule.To == nil || len(egressRule.To) == 0 + + if allDestinations { + if allPorts { + rule := newRuleSpec(policyTypeEgress, nil, targetSelector, nil, nil) + rules[rule.key] = rule + } else { + withNormalisedProtoAndPort(egressRule.Ports, func(proto, port string) { + rule := newRuleSpec(policyTypeEgress, &proto, targetSelector, nil, &port) + rules[rule.key] = rule + }) + } + } else { + for _, peer := range egressRule.To { + var dstSelector *selectorSpec + var dstRuleHost ruleHost + + // NetworkPolicyPeer describes a peer to allow traffic to. + // Exactly one of its fields must be specified. + if peer.PodSelector != nil { + dstSelector, err = newSelectorSpec(peer.PodSelector, nil, ns.name, ipset.HashIP) + if err != nil { + return nil, nil, nil, nil, err + } + addIfNotExist(dstSelector, podSelectors) + dstRuleHost = dstSelector + + } else if peer.NamespaceSelector != nil { + dstSelector, err = newSelectorSpec(peer.NamespaceSelector, nil, "", ipset.ListSet) + if err != nil { + return nil, nil, nil, nil, err + } + nsSelectors[dstSelector.key] = dstSelector + dstRuleHost = dstSelector + } else if peer.IPBlock != nil { + ipBlock := newIPBlockSpec(peer.IPBlock, ns.name) + ipBlocks[ipBlock.key] = ipBlock + dstRuleHost = ipBlock + } + + if allPorts { + rule := newRuleSpec(policyTypeEgress, nil, targetSelector, dstRuleHost, nil) + rules[rule.key] = rule + } else { + withNormalisedProtoAndPort(egressRule.Ports, func(proto, port string) { + rule := newRuleSpec(policyTypeEgress, &proto, targetSelector, dstRuleHost, &port) + rules[rule.key] = rule + }) + } + } + } + } + } + + return rules, nsSelectors, podSelectors, ipBlocks, nil } func addIfNotExist(s *selectorSpec, ss map[string]*selectorSpec) { @@ -179,12 +164,6 @@ func addIfNotExist(s *selectorSpec, ss map[string]*selectorSpec) { } } -func withNormalisedProtoAndPortLegacy(npps []extnapi.NetworkPolicyPort, f func(proto, port string)) { - for _, npp := range npps { - f(proto(npp.Protocol), port(npp.Port)) - } -} - func withNormalisedProtoAndPort(npps []networkingv1.NetworkPolicyPort, f func(proto, port string)) { for _, npp := range npps { f(proto(npp.Protocol), port(npp.Port)) diff --git a/npc/constants.go b/npc/constants.go index 2722c57f..6c94dcc9 100644 --- a/npc/constants.go +++ b/npc/constants.go @@ -7,6 +7,12 @@ const ( DefaultChain = "WEAVE-NPC-DEFAULT" IngressChain = "WEAVE-NPC-INGRESS" + EgressChain = "WEAVE-NPC-EGRESS" + EgressDefaultChain = "WEAVE-NPC-EGRESS-DEFAULT" + EgressCustomChain = "WEAVE-NPC-EGRESS-CUSTOM" + EgressMarkChain = "WEAVE-NPC-EGRESS-ACCEPT" + EgressMark = "0x40000/0x40000" + IpsetNamePrefix = "weave-" LocalIpset = IpsetNamePrefix + "local-pods" diff --git a/npc/controller.go b/npc/controller.go index 6bd26b1e..e8425993 100644 --- a/npc/controller.go +++ b/npc/controller.go @@ -37,19 +37,16 @@ type controller struct { nss map[string]*ns // ns name -> ns struct nsSelectors *selectorSet // selector string -> nsSelector - - legacy bool // use legacy network policy semantics (k8s pre-1.7) } -func New(nodeName string, legacy bool, ipt iptables.Interface, ips ipset.Interface) NetworkPolicyController { +func New(nodeName string, ipt iptables.Interface, ips ipset.Interface) NetworkPolicyController { c := &controller{ nodeName: nodeName, - legacy: legacy, ipt: ipt, ips: ips, nss: make(map[string]*ns)} - doNothing := func(*selector) error { return nil } + doNothing := func(*selector, policyType) error { return nil } c.nsSelectors = newSelectorSet(ips, c.onNewNsSelector, doNothing, doNothing) return c @@ -71,7 +68,7 @@ func (npc *controller) onNewNsSelector(selector *selector) error { func (npc *controller) withNS(name string, f func(ns *ns) error) error { ns, found := npc.nss[name] if !found { - newNs, err := newNS(name, npc.nodeName, npc.legacy, npc.ipt, npc.ips, npc.nsSelectors) + newNs, err := newNS(name, npc.nodeName, npc.ipt, npc.ips, npc.nsSelectors) if err != nil { return err } diff --git a/npc/controller_test.go b/npc/controller_test.go index f7dd7fdb..21488f03 100644 --- a/npc/controller_test.go +++ b/npc/controller_test.go @@ -1,7 +1,9 @@ package npc import ( + "fmt" "log" + "strings" "testing" "github.com/pkg/errors" @@ -103,18 +105,46 @@ func (i *mockIPSet) List(prefix string) ([]ipset.Name, error) { } type mockIPTables struct { + rules map[string]map[string]struct{} // chain -> rulespec -> struct{} +} + +func newMockIPTables() *mockIPTables { + return &mockIPTables{rules: make(map[string]map[string]struct{})} } func (ipt *mockIPTables) Append(table, chain string, rulespec ...string) error { + if table != TableFilter { + return fmt.Errorf("invalid table: %q", table) + } + + if _, found := ipt.rules[chain]; !found { + ipt.rules[chain] = make(map[string]struct{}) + } + + rule := strings.Join(rulespec, " ") + if _, found := ipt.rules[chain][rule]; found { + return fmt.Errorf("rule already exists. chain: %q, rule: %q", chain, rule) + } + + ipt.rules[chain][rule] = struct{}{} + return nil } func (ipt *mockIPTables) Delete(table, chain string, rulespec ...string) error { + rule := strings.Join(rulespec, " ") + + if _, found := ipt.rules[chain][rule]; !found { + return fmt.Errorf("rule does not exist. chain: %q rule: %q", chain, rule) + } + + delete(ipt.rules[chain], rule) + return nil } func (ipt *mockIPTables) Insert(table, chain string, pos int, rulespec ...string) error { - return nil + return errors.New("Not Implemented") } func TestRegressionPolicyNamespaceOrdering3059(t *testing.T) { @@ -144,6 +174,7 @@ func TestRegressionPolicyNamespaceOrdering3059(t *testing.T) { Namespace: "destination", }, Spec: networkingv1.NetworkPolicySpec{ + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, Ingress: []networkingv1.NetworkPolicyIngressRule{ { From: []networkingv1.NetworkPolicyPeer{ @@ -167,7 +198,7 @@ func TestRegressionPolicyNamespaceOrdering3059(t *testing.T) { // Namespaces first m := newMockIPSet() - controller := New("foo", false, &mockIPTables{}, &m) + controller := New("foo", newMockIPTables(), &m) const ( selectorIPSetName = "weave-I239Zp%sCvoVt*D6u=A!2]YEk" @@ -183,7 +214,7 @@ func TestRegressionPolicyNamespaceOrdering3059(t *testing.T) { // NetworkPolicy first m = newMockIPSet() - controller = New("foo", false, &mockIPTables{}, &m) + controller = New("foo", newMockIPTables(), &m) controller.AddNetworkPolicy(networkPolicy) @@ -193,17 +224,18 @@ func TestRegressionPolicyNamespaceOrdering3059(t *testing.T) { require.Equal(t, true, len(m.sets[selectorIPSetName].subSets[sourceIPSetName]) > 0) } -// Tests default-allow ipset behavior when running in non-legacy mode. +// Tests default-allow ipset behavior func TestDefaultAllow(t *testing.T) { const ( - defaultAllowIPSetName = "weave-E.1.0W^NGSp]0_t5WwH/]gX@L" - fooPodIP = "10.32.0.10" - barPodIP = "10.32.0.11" - barPodNewIP = "10.32.0.12" + ingressDefaultAllowIPSetName = "weave-;rGqyMIl1HN^cfDki~Z$3]6!N" + egressDefaultAllowIPSetName = "weave-s_+ChJId4Uy_$}G;WdH|~TK)I" + fooPodIP = "10.32.0.10" + barPodIP = "10.32.0.11" + barPodNewIP = "10.32.0.12" ) m := newMockIPSet() - controller := New("bar", false, &mockIPTables{}, &m) + controller := New("bar", newMockIPTables(), &m) defaultNamespace := &coreapi.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -213,7 +245,8 @@ func TestDefaultAllow(t *testing.T) { controller.AddNamespace(defaultNamespace) // Should create an ipset for default-allow - require.Contains(t, m.sets, defaultAllowIPSetName) + require.Contains(t, m.sets, ingressDefaultAllowIPSetName) + require.Contains(t, m.sets, egressDefaultAllowIPSetName) podFoo := &coreapi.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -225,7 +258,8 @@ func TestDefaultAllow(t *testing.T) { controller.AddPod(podFoo) // Should add the foo pod to default-allow - require.True(t, m.entryExists(defaultAllowIPSetName, fooPodIP)) + require.True(t, m.entryExists(ingressDefaultAllowIPSetName, fooPodIP)) + require.True(t, m.entryExists(egressDefaultAllowIPSetName, fooPodIP)) podBar := &coreapi.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -236,10 +270,12 @@ func TestDefaultAllow(t *testing.T) { Status: coreapi.PodStatus{PodIP: barPodIP}} podBarNoIP := &coreapi.Pod{ObjectMeta: podBar.ObjectMeta} controller.AddPod(podBarNoIP) + controller.UpdatePod(podBarNoIP, podBar) // Should add the bar pod to default-allow - require.True(t, m.entryExists(defaultAllowIPSetName, barPodIP)) + require.True(t, m.entryExists(ingressDefaultAllowIPSetName, barPodIP)) + require.True(t, m.entryExists(egressDefaultAllowIPSetName, barPodIP)) // Allow access from the bar pod to the foo pod netpol := &networkingv1.NetworkPolicy{ @@ -248,6 +284,10 @@ func TestDefaultAllow(t *testing.T) { Namespace: "default", }, Spec: networkingv1.NetworkPolicySpec{ + PolicyTypes: []networkingv1.PolicyType{ + networkingv1.PolicyTypeIngress, + networkingv1.PolicyTypeEgress, + }, PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{"run": "foo"}}, Ingress: []networkingv1.NetworkPolicyIngressRule{{ From: []networkingv1.NetworkPolicyPeer{{ @@ -256,60 +296,74 @@ func TestDefaultAllow(t *testing.T) { }, }}, }}, + Egress: []networkingv1.NetworkPolicyEgressRule{{ + To: []networkingv1.NetworkPolicyPeer{{ + IPBlock: &networkingv1.IPBlock{CIDR: "192.168.48.0/24"}, + }}, + }}, }, } controller.AddNetworkPolicy(netpol) // Should remove the foo pod from default-allow as the netpol selects it - require.False(t, m.entryExists(defaultAllowIPSetName, fooPodIP)) - require.True(t, m.entryExists(defaultAllowIPSetName, barPodIP)) + require.False(t, m.entryExists(ingressDefaultAllowIPSetName, fooPodIP)) + require.False(t, m.entryExists(egressDefaultAllowIPSetName, fooPodIP)) + require.True(t, m.entryExists(ingressDefaultAllowIPSetName, barPodIP)) + require.True(t, m.entryExists(egressDefaultAllowIPSetName, barPodIP)) podBarWithNewIP := *podBar podBarWithNewIP.Status.PodIP = barPodNewIP controller.UpdatePod(podBar, &podBarWithNewIP) // Should update IP addr of the bar pod in default-allow - require.False(t, m.entryExists(defaultAllowIPSetName, barPodIP)) - require.True(t, m.entryExists(defaultAllowIPSetName, barPodNewIP)) + require.False(t, m.entryExists(ingressDefaultAllowIPSetName, barPodIP)) + require.False(t, m.entryExists(egressDefaultAllowIPSetName, barPodIP)) + require.True(t, m.entryExists(ingressDefaultAllowIPSetName, barPodNewIP)) + require.True(t, m.entryExists(egressDefaultAllowIPSetName, barPodNewIP)) controller.UpdatePod(&podBarWithNewIP, podBarNoIP) // Should remove the bar pod from default-allow as it does not have any IP addr - require.False(t, m.entryExists(defaultAllowIPSetName, barPodNewIP)) + require.False(t, m.entryExists(ingressDefaultAllowIPSetName, barPodNewIP)) + require.False(t, m.entryExists(egressDefaultAllowIPSetName, barPodNewIP)) podFooWithNewLabel := *podFoo podFooWithNewLabel.ObjectMeta.Labels = map[string]string{"run": "new-foo"} controller.UpdatePod(podFoo, &podFooWithNewLabel) // Should bring back the foo pod to default-allow as it does not match dst of any netpol - require.True(t, m.entryExists(defaultAllowIPSetName, fooPodIP)) + require.True(t, m.entryExists(ingressDefaultAllowIPSetName, fooPodIP)) + require.True(t, m.entryExists(egressDefaultAllowIPSetName, fooPodIP)) controller.UpdatePod(&podFooWithNewLabel, podFoo) // Should remove from default-allow as it matches the netpol after the update - require.False(t, m.entryExists(defaultAllowIPSetName, fooPodIP)) + require.False(t, m.entryExists(ingressDefaultAllowIPSetName, fooPodIP)) + require.False(t, m.entryExists(egressDefaultAllowIPSetName, fooPodIP)) controller.DeleteNetworkPolicy(netpol) // Should bring back the foo pod to default-allow as no netpol selects it - require.True(t, m.entryExists(defaultAllowIPSetName, fooPodIP)) + require.True(t, m.entryExists(ingressDefaultAllowIPSetName, fooPodIP)) + require.True(t, m.entryExists(egressDefaultAllowIPSetName, fooPodIP)) controller.DeletePod(podFoo) // Should remove foo pod from default-allow - require.False(t, m.entryExists(defaultAllowIPSetName, fooPodIP)) + require.False(t, m.entryExists(ingressDefaultAllowIPSetName, fooPodIP)) + require.False(t, m.entryExists(egressDefaultAllowIPSetName, fooPodIP)) controller.DeleteNamespace(defaultNamespace) // Should remove default ipset - require.NotContains(t, m.sets, defaultAllowIPSetName) - + require.NotContains(t, m.sets, ingressDefaultAllowIPSetName) + require.NotContains(t, m.sets, egressDefaultAllowIPSetName) } func TestOutOfOrderPodEvents(t *testing.T) { const ( - defaultAllowIPSetName = "weave-E.1.0W^NGSp]0_t5WwH/]gX@L" - runBarIPSetName = "weave-bZ~x=yBgzH)Ht()K*Uv3z{M]Y" - podIP = "10.32.0.10" + ingressDefaultAllowIPSetName = "weave-;rGqyMIl1HN^cfDki~Z$3]6!N" + runBarIPSetName = "weave-bZ~x=yBgzH)Ht()K*Uv3z{M]Y" + podIP = "10.32.0.10" ) m := newMockIPSet() - controller := New("qux", false, &mockIPTables{}, &m) + controller := New("qux", newMockIPTables(), &m) defaultNamespace := &coreapi.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -328,7 +382,7 @@ func TestOutOfOrderPodEvents(t *testing.T) { controller.AddPod(podFoo) // Should be in default-allow as no netpol selects podFoo - require.True(t, m.entryExists(defaultAllowIPSetName, podIP)) + require.True(t, m.entryExists(ingressDefaultAllowIPSetName, podIP)) netpol := &networkingv1.NetworkPolicy{ ObjectMeta: metav1.ObjectMeta{ @@ -336,6 +390,7 @@ func TestOutOfOrderPodEvents(t *testing.T) { Namespace: "default", }, Spec: networkingv1.NetworkPolicySpec{ + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{"run": "foo"}}, Ingress: []networkingv1.NetworkPolicyIngressRule{{ From: []networkingv1.NetworkPolicyPeer{{ @@ -349,7 +404,7 @@ func TestOutOfOrderPodEvents(t *testing.T) { controller.AddNetworkPolicy(netpol) // Shouldn't be in default-allow as netpol above selects podFoo - require.False(t, m.entryExists(defaultAllowIPSetName, podIP)) + require.False(t, m.entryExists(ingressDefaultAllowIPSetName, podIP)) podBar := &coreapi.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -361,8 +416,8 @@ func TestOutOfOrderPodEvents(t *testing.T) { controller.AddPod(podBar) // Should be in default-allow as no netpol selects podBar - require.True(t, m.entryExists(defaultAllowIPSetName, podIP)) - require.True(t, m.Exist(podBar.ObjectMeta.UID, defaultAllowIPSetName, podIP)) + require.True(t, m.entryExists(ingressDefaultAllowIPSetName, podIP)) + require.True(t, m.Exist(podBar.ObjectMeta.UID, ingressDefaultAllowIPSetName, podIP)) // Should be in run=bar ipset require.True(t, m.entryExists(runBarIPSetName, podIP)) @@ -373,24 +428,24 @@ func TestOutOfOrderPodEvents(t *testing.T) { // Should be in default-allow as no netpol selects podBar and podFoo removal // should not affect podBar in default-allow - require.True(t, m.entryExists(defaultAllowIPSetName, podIP)) + require.True(t, m.entryExists(ingressDefaultAllowIPSetName, podIP)) controller.DeletePod(podBar) // Should remove from default-allow and run=bar ipsets - require.Equal(t, 0, len(m.sets[defaultAllowIPSetName].subSets)) + require.Equal(t, 0, len(m.sets[ingressDefaultAllowIPSetName].subSets)) require.False(t, m.entryExists(runBarIPSetName, podIP)) } // Test case for https://github.com/weaveworks/weave/issues/3222 -func TestNewDstSelector(t *testing.T) { +func TestNewTargetSelector(t *testing.T) { const ( - defaultAllowIPSetName = "weave-E.1.0W^NGSp]0_t5WwH/]gX@L" - podIP = "10.32.0.10" + ingressDefaultAllowIPSetName = "weave-;rGqyMIl1HN^cfDki~Z$3]6!N" + podIP = "10.32.0.10" ) m := newMockIPSet() - controller := New("baz", false, &mockIPTables{}, &m) + controller := New("baz", newMockIPTables(), &m) defaultNamespace := &coreapi.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -408,7 +463,7 @@ func TestNewDstSelector(t *testing.T) { Status: coreapi.PodStatus{PodIP: podIP}} controller.AddPod(podFoo) - require.True(t, m.entryExists(defaultAllowIPSetName, podIP)) + require.True(t, m.entryExists(ingressDefaultAllowIPSetName, podIP)) netpolBar := &networkingv1.NetworkPolicy{ ObjectMeta: metav1.ObjectMeta{ @@ -417,6 +472,7 @@ func TestNewDstSelector(t *testing.T) { Namespace: "default", }, Spec: networkingv1.NetworkPolicySpec{ + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, PodSelector: metav1.LabelSelector{}, Ingress: []networkingv1.NetworkPolicyIngressRule{{ From: []networkingv1.NetworkPolicyPeer{{ @@ -427,8 +483,8 @@ func TestNewDstSelector(t *testing.T) { } controller.AddNetworkPolicy(netpolBar) - // netpolBar dst selector selects podFoo - require.False(t, m.entryExists(defaultAllowIPSetName, podIP)) + // netpolBar target selector selects podFoo + require.False(t, m.entryExists(ingressDefaultAllowIPSetName, podIP)) netpolFoo := &networkingv1.NetworkPolicy{ ObjectMeta: metav1.ObjectMeta{ @@ -437,6 +493,7 @@ func TestNewDstSelector(t *testing.T) { Namespace: "default", }, Spec: networkingv1.NetworkPolicySpec{ + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, PodSelector: metav1.LabelSelector{}, Ingress: []networkingv1.NetworkPolicyIngressRule{{ From: []networkingv1.NetworkPolicyPeer{{ @@ -448,9 +505,90 @@ func TestNewDstSelector(t *testing.T) { controller.AddNetworkPolicy(netpolFoo) controller.DeleteNetworkPolicy(netpolBar) - // netpolFoo dst-selects podFoo - require.False(t, m.entryExists(defaultAllowIPSetName, podIP)) + // netpolFoo target-selects podFoo + require.False(t, m.entryExists(ingressDefaultAllowIPSetName, podIP)) controller.DeleteNetworkPolicy(netpolFoo) - // No netpol dst-selects podFoo - require.True(t, m.entryExists(defaultAllowIPSetName, podIP)) + // No netpol target-selects podFoo + require.True(t, m.entryExists(ingressDefaultAllowIPSetName, podIP)) +} + +func TestEgressPolicyWithIPBlock(t *testing.T) { + const ( + fooPodIP = "10.32.0.10" + exceptIPSetName = "weave-j:W$5om!$8JS})buYAD7q^#sX" + exceptIPSetNameInNonDefault = "weave-|*P{aL2C@#N/1IZ!IvQItv_pt" + ) + + m := newMockIPSet() + ipt := newMockIPTables() + controller := New("foo", ipt, &m) + + netpolFoo := &networkingv1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + UID: "netpol-foo", + Name: "allow-from-bar-to-foo", + Namespace: "default", + }, + Spec: networkingv1.NetworkPolicySpec{ + PolicyTypes: []networkingv1.PolicyType{ + networkingv1.PolicyTypeEgress, + }, + PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{"run": "bar"}}, + Egress: []networkingv1.NetworkPolicyEgressRule{{ + To: []networkingv1.NetworkPolicyPeer{{ + IPBlock: &networkingv1.IPBlock{ + CIDR: "192.168.48.0/24", + Except: []string{ + "192.168.48.1/32", + "192.168.48.2/32", + }, + }, + }}, + }}, + }, + } + err := controller.AddNetworkPolicy(netpolFoo) + require.NoError(t, err) + + require.Equal(t, 2, len(m.sets[exceptIPSetName].subSets)) + require.True(t, m.entryExists(exceptIPSetName, "192.168.48.1/32")) + require.True(t, m.entryExists(exceptIPSetName, "192.168.48.2/32")) + + // Each egress rule is represented as two iptables rules (-J MARK and -J RETURN). + require.Equal(t, 2, len(ipt.rules[EgressCustomChain])) + for rule := range ipt.rules[EgressCustomChain] { + require.Contains(t, rule, "-d 192.168.48.0/24 -m set ! --match-set "+exceptIPSetName+" dst") + } + + // Check that we create a new ipset for the ipBlock bellow. An ipset with the + // same content already exists (created by netpolFoo), but we need to create + // a new one, as netpolBar is in a different namespace and weave-npc does + // object refcounting only within namespace boundaries. + netpolBar := &networkingv1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + UID: "netpol-foo-bar", + Name: "allow-from-bar-to-foo-in-non-default", + Namespace: "non-default", + }, + Spec: networkingv1.NetworkPolicySpec{ + PolicyTypes: []networkingv1.PolicyType{ + networkingv1.PolicyTypeEgress, + }, + PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{"run": "bar"}}, + Egress: []networkingv1.NetworkPolicyEgressRule{{ + To: []networkingv1.NetworkPolicyPeer{{ + IPBlock: &networkingv1.IPBlock{ + CIDR: "192.168.48.0/24", + Except: []string{ + "192.168.48.1/32", + "192.168.48.2/32", + }, + }, + }}, + }}, + }, + } + err = controller.AddNetworkPolicy(netpolBar) + require.NoError(t, err) + require.Equal(t, 2, len(m.sets[exceptIPSetNameInNonDefault].subSets)) } diff --git a/npc/ipblock.go b/npc/ipblock.go new file mode 100644 index 00000000..7d2579d0 --- /dev/null +++ b/npc/ipblock.go @@ -0,0 +1,116 @@ +package npc + +import ( + "fmt" + "sort" + "strings" + + networkingv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/weaveworks/weave/common" + "github.com/weaveworks/weave/net/ipset" +) + +type ipBlockSpec struct { + key string + ipsetName ipset.Name // ipset for storing excepted CIDRs + ipBlock *networkingv1.IPBlock + nsName string // Namespace name +} + +func newIPBlockSpec(ipb *networkingv1.IPBlock, nsName string) *ipBlockSpec { + spec := &ipBlockSpec{ipBlock: ipb, nsName: nsName} + + if len(ipb.Except) > 0 { + sort.Strings(ipb.Except) + spec.key = strings.Join(ipb.Except, " ") + spec.ipsetName = ipset.Name(IpsetNamePrefix + shortName(nsName+":"+"ipblock-except:"+spec.key)) + } + + return spec +} + +func (spec *ipBlockSpec) getRuleSpec(src bool) ([]string, string) { + dir, dirOpt := "dst", "-d" + if src { + dir, dirOpt = "src", "-s" + } + + if spec.key == "" { + rule := []string{dirOpt, spec.ipBlock.CIDR} + comment := fmt.Sprintf("cidr: %s", spec.ipBlock.CIDR) + return rule, comment + } + + rule := []string{dirOpt, spec.ipBlock.CIDR, + "-m", "set", "!", "--match-set", string(spec.ipsetName), dir} + comment := fmt.Sprintf("cidr: %s except [%s]", spec.ipBlock.CIDR, spec.key) + return rule, comment +} + +type ipBlockSet struct { + ips ipset.Interface + users map[string]map[types.UID]struct{} +} + +func newIPBlockSet(ips ipset.Interface) *ipBlockSet { + return &ipBlockSet{ + ips: ips, + users: make(map[string]map[types.UID]struct{}), + } +} + +func (s *ipBlockSet) deprovision(user types.UID, current, desired map[string]*ipBlockSpec) error { + for key, spec := range current { + if key == "" { + continue + } + + if _, found := desired[key]; !found { + delete(s.users[key], user) + if len(s.users[key]) == 0 { + common.Log.Infof("destroying ipset: %#v", spec) + if err := s.ips.Destroy(spec.ipsetName); err != nil { + return err + } + + delete(s.users, key) + } + } + } + + return nil +} + +func (s *ipBlockSet) provision(user types.UID, current, desired map[string]*ipBlockSpec) (err error) { + for key, spec := range desired { + if key == "" { + // No need to provision an ipBlock with empty list of excepted CIDRs + // (i.e. no ipset is needed for the related iptables rule). + continue + } + + if _, found := current[key]; !found { + if _, found := s.users[key]; !found { + common.Log.Infof("creating ipset: %#v", spec) + if err := s.ips.Create(spec.ipsetName, ipset.HashNet); err != nil { + return err + } + + // TODO(brb) Pass comment to ips.Create instead. + comment := "excepted from " + spec.ipBlock.CIDR + for _, cidr := range spec.ipBlock.Except { + if err = s.ips.AddEntry(user, spec.ipsetName, cidr, comment); err != nil { + return err + } + } + + s.users[key] = make(map[types.UID]struct{}) + } + s.users[key][user] = struct{}{} + } + } + + return nil +} diff --git a/npc/namespace.go b/npc/namespace.go index 5dc3f46e..9d43f1f8 100644 --- a/npc/namespace.go +++ b/npc/namespace.go @@ -1,7 +1,6 @@ package npc import ( - "encoding/json" "errors" "fmt" @@ -32,20 +31,20 @@ type ns struct { uid types.UID // surrogate UID to own allPods selector allPods *selectorSpec // hash:ip ipset of all pod IPs in this namespace - // stores IP addrs of pods which are not selected by any dst podSelector of - // any netpol; used only in non-legacy mode and is used as a dst in - // the WEAVE-NPC-DEFAULT iptables chain. - defaultAllowIPSet ipset.Name + // stores IP addrs of pods which are not selected by any target podSelector of + // any netpol; used as a target in the WEAVE-NPC-{INGRESS,EGRESS}-DEFAULT + // iptables chains. + ingressDefaultAllowIPSet ipset.Name + egressDefaultAllowIPSet ipset.Name nsSelectors *selectorSet podSelectors *selectorSet + ipBlocks *ipBlockSet rules *ruleSet - - legacy bool } -func newNS(name, nodeName string, legacy bool, ipt iptables.Interface, ips ipset.Interface, nsSelectors *selectorSet) (*ns, error) { - allPods, err := newSelectorSpec(&metav1.LabelSelector{}, false, name, ipset.HashIP) +func newNS(name, nodeName string, ipt iptables.Interface, ips ipset.Interface, nsSelectors *selectorSet) (*ns, error) { + allPods, err := newSelectorSpec(&metav1.LabelSelector{}, nil, name, ipset.HashIP) if err != nil { return nil, err } @@ -60,19 +59,24 @@ func newNS(name, nodeName string, legacy bool, ipt iptables.Interface, ips ipset uid: uuid.NewUUID(), allPods: allPods, nsSelectors: nsSelectors, + ipBlocks: newIPBlockSet(ips), rules: newRuleSet(ipt), - legacy: legacy} - - ns.podSelectors = newSelectorSet(ips, ns.onNewPodSelector, ns.onNewDstPodSelector, ns.onDestroyDstPodSelector) - - if !legacy { - defaultAllowIPSet := ipset.Name(IpsetNamePrefix + shortName("default-allow:"+name)) - if err := ips.Create(defaultAllowIPSet, ipset.HashIP); err != nil { - return nil, err - } - ns.defaultAllowIPSet = defaultAllowIPSet } + ns.podSelectors = newSelectorSet(ips, ns.onNewPodSelector, ns.onNewTargetPodSelector, ns.onDestroyTargetPodSelector) + + ingressDefaultAllowIPSet := ipset.Name(IpsetNamePrefix + shortName("ingress-default-allow:"+name)) + if err := ips.Create(ingressDefaultAllowIPSet, ipset.HashIP); err != nil { + return nil, err + } + ns.ingressDefaultAllowIPSet = ingressDefaultAllowIPSet + + egressDefaultAllowIPSet := ipset.Name(IpsetNamePrefix + shortName("egress-default-allow:"+name)) + if err := ips.Create(egressDefaultAllowIPSet, ipset.HashIP); err != nil { + return nil, err + } + ns.egressDefaultAllowIPSet = egressDefaultAllowIPSet + if err := ns.podSelectors.provision(ns.uid, nil, map[string]*selectorSpec{ns.allPods.key: ns.allPods}); err != nil { return nil, err } @@ -102,16 +106,13 @@ func (ns *ns) onNewPodSelector(selector *selector) error { return nil } -func (ns *ns) onNewDstPodSelector(selector *selector) error { - if ns.legacy { - return nil - } - +func (ns *ns) onNewTargetPodSelector(selector *selector, policyType policyType) error { for _, pod := range ns.pods { if hasIP(pod) { // Remove the pod from default-allow if dst podselector matches the pod + ipset := ns.defaultAllowIPSetName(policyType) if selector.matches(pod.ObjectMeta.Labels) { - if err := ns.ips.DelEntry(pod.ObjectMeta.UID, ns.defaultAllowIPSet, pod.Status.PodIP); err != nil { + if err := ns.ips.DelEntry(pod.ObjectMeta.UID, ipset, pod.Status.PodIP); err != nil { return err } } @@ -121,15 +122,11 @@ func (ns *ns) onNewDstPodSelector(selector *selector) error { return nil } -func (ns *ns) onDestroyDstPodSelector(selector *selector) error { - if ns.legacy { - return nil - } - +func (ns *ns) onDestroyTargetPodSelector(selector *selector, policyType policyType) error { for _, pod := range ns.pods { if hasIP(pod) { if selector.matches(pod.ObjectMeta.Labels) { - if err := ns.addToDefaultAllowIfNoMatching(pod); err != nil { + if err := ns.addToDefaultAllowIfNoMatching(pod, policyType); err != nil { return err } } @@ -139,18 +136,19 @@ func (ns *ns) onDestroyDstPodSelector(selector *selector) error { return nil } -// Add pod IP addr to default-allow ipset if there are no matching dst selectors -func (ns *ns) addToDefaultAllowIfNoMatching(pod *coreapi.Pod) error { +// Add pod IP addr to default-allow ipset if there are no matching target selectors +func (ns *ns) addToDefaultAllowIfNoMatching(pod *coreapi.Pod, policyType policyType) error { found := false // TODO(mp) optimize (avoid iterating over selectors) by ref counting IP addrs. for _, s := range ns.podSelectors.entries { - if ns.podSelectors.dstSelectorExist(s) && s.matches(pod.ObjectMeta.Labels) { + if ns.podSelectors.targetSelectorExist(s, policyType) && s.matches(pod.ObjectMeta.Labels) { found = true break } } if !found { - if err := ns.ips.AddEntry(pod.ObjectMeta.UID, ns.defaultAllowIPSet, pod.Status.PodIP, podComment(pod)); err != nil { + ipset := ns.defaultAllowIPSetName(policyType) + if err := ns.ips.AddEntry(pod.ObjectMeta.UID, ipset, pod.Status.PodIP, podComment(pod)); err != nil { return err } } @@ -175,13 +173,18 @@ func (ns *ns) addPod(obj *coreapi.Pod) error { ns.ips.AddEntry(obj.ObjectMeta.UID, LocalIpset, obj.Status.PodIP, podComment(obj)) } - found, err := ns.podSelectors.addToMatching(obj.ObjectMeta.UID, obj.ObjectMeta.Labels, obj.Status.PodIP, podComment(obj)) + foundIngress, foundEgress, err := ns.podSelectors.addToMatching(obj.ObjectMeta.UID, obj.ObjectMeta.Labels, obj.Status.PodIP, podComment(obj)) if err != nil { return err } - // If there are no matching dst selectors, add the pod to default-allow - if !ns.legacy && !found { - if err := ns.ips.AddEntry(obj.ObjectMeta.UID, ns.defaultAllowIPSet, obj.Status.PodIP, podComment(obj)); err != nil { + // If there are no matching target selectors, add the pod to default-allow + if !foundIngress { + if err := ns.ips.AddEntry(obj.ObjectMeta.UID, ns.ingressDefaultAllowIPSet, obj.Status.PodIP, podComment(obj)); err != nil { + return err + } + } + if !foundEgress { + if err := ns.ips.AddEntry(obj.ObjectMeta.UID, ns.egressDefaultAllowIPSet, obj.Status.PodIP, podComment(obj)); err != nil { return err } } @@ -202,10 +205,11 @@ func (ns *ns) updatePod(oldObj, newObj *coreapi.Pod) error { ns.ips.DelEntry(oldObj.ObjectMeta.UID, LocalIpset, oldObj.Status.PodIP) } - if !ns.legacy { - if err := ns.ips.DelEntry(oldObj.ObjectMeta.UID, ns.defaultAllowIPSet, oldObj.Status.PodIP); err != nil { - return err - } + if err := ns.ips.DelEntry(oldObj.ObjectMeta.UID, ns.ingressDefaultAllowIPSet, oldObj.Status.PodIP); err != nil { + return err + } + if err := ns.ips.DelEntry(oldObj.ObjectMeta.UID, ns.egressDefaultAllowIPSet, oldObj.Status.PodIP); err != nil { + return err } return ns.podSelectors.delFromMatching(oldObj.ObjectMeta.UID, oldObj.ObjectMeta.Labels, oldObj.Status.PodIP) @@ -215,13 +219,18 @@ func (ns *ns) updatePod(oldObj, newObj *coreapi.Pod) error { if ns.checkLocalPod(newObj) { ns.ips.AddEntry(newObj.ObjectMeta.UID, LocalIpset, newObj.Status.PodIP, podComment(newObj)) } - found, err := ns.podSelectors.addToMatching(newObj.ObjectMeta.UID, newObj.ObjectMeta.Labels, newObj.Status.PodIP, podComment(newObj)) + foundIngress, foundEgress, err := ns.podSelectors.addToMatching(newObj.ObjectMeta.UID, newObj.ObjectMeta.Labels, newObj.Status.PodIP, podComment(newObj)) if err != nil { return err } - if !ns.legacy && !found { - if err := ns.ips.AddEntry(newObj.ObjectMeta.UID, ns.defaultAllowIPSet, newObj.Status.PodIP, podComment(newObj)); err != nil { + if !foundIngress { + if err := ns.ips.AddEntry(newObj.ObjectMeta.UID, ns.ingressDefaultAllowIPSet, newObj.Status.PodIP, podComment(newObj)); err != nil { + return err + } + } + if !foundEgress { + if err := ns.ips.AddEntry(newObj.ObjectMeta.UID, ns.egressDefaultAllowIPSet, newObj.Status.PodIP, podComment(newObj)); err != nil { return err } } @@ -229,17 +238,11 @@ func (ns *ns) updatePod(oldObj, newObj *coreapi.Pod) error { return nil } - if !ns.legacy && - oldObj.Status.PodIP != newObj.Status.PodIP && - ns.ips.Exist(oldObj.ObjectMeta.UID, ns.defaultAllowIPSet, oldObj.Status.PodIP) { - // Instead of iterating over all selectors we check whether old pod IP - // has been inserted into default-allow ipset to decide whether the IP - // in the ipset has to be updated. - - if err := ns.ips.DelEntry(oldObj.ObjectMeta.UID, ns.defaultAllowIPSet, oldObj.Status.PodIP); err != nil { + if oldObj.Status.PodIP != newObj.Status.PodIP { + if err := ns.updateDefaultAllowIPSetEntry(oldObj, newObj, ns.ingressDefaultAllowIPSet); err != nil { return err } - if err := ns.ips.AddEntry(newObj.ObjectMeta.UID, ns.defaultAllowIPSet, newObj.Status.PodIP, podComment(newObj)); err != nil { + if err := ns.updateDefaultAllowIPSetEntry(oldObj, newObj, ns.egressDefaultAllowIPSet); err != nil { return err } } @@ -264,17 +267,11 @@ func (ns *ns) updatePod(oldObj, newObj *coreapi.Pod) error { } } - if !ns.legacy && ns.podSelectors.dstSelectorExist(ps) { - switch { - case !oldMatch && newMatch: - if err := ns.ips.DelEntry(oldObj.ObjectMeta.UID, ns.defaultAllowIPSet, oldObj.Status.PodIP); err != nil { - return err - } - case oldMatch && !newMatch: - if err := ns.addToDefaultAllowIfNoMatching(newObj); err != nil { - return err - } - } + if err := ns.addOrRemoveToDefaultAllowIPSet(ps, oldObj, newObj, oldMatch, newMatch, policyTypeIngress); err != nil { + return err + } + if err := ns.addOrRemoveToDefaultAllowIPSet(ps, oldObj, newObj, oldMatch, newMatch, policyTypeEgress); err != nil { + return err } } } @@ -282,6 +279,23 @@ func (ns *ns) updatePod(oldObj, newObj *coreapi.Pod) error { return nil } +func (ns *ns) addOrRemoveToDefaultAllowIPSet(ps *selector, oldObj, newObj *coreapi.Pod, oldMatch, newMatch bool, policyType policyType) error { + ipset := ns.defaultAllowIPSetName(policyType) + if ns.podSelectors.targetSelectorExist(ps, policyType) { + switch { + case !oldMatch && newMatch: + if err := ns.ips.DelEntry(oldObj.ObjectMeta.UID, ipset, oldObj.Status.PodIP); err != nil { + return err + } + case oldMatch && !newMatch: + if err := ns.addToDefaultAllowIfNoMatching(newObj, policyType); err != nil { + return err + } + } + } + return nil +} + func (ns *ns) deletePod(obj *coreapi.Pod) error { delete(ns.pods, obj.ObjectMeta.UID) @@ -293,10 +307,11 @@ func (ns *ns) deletePod(obj *coreapi.Pod) error { ns.ips.DelEntry(obj.ObjectMeta.UID, LocalIpset, obj.Status.PodIP) } - if !ns.legacy { - if err := ns.ips.DelEntry(obj.ObjectMeta.UID, ns.defaultAllowIPSet, obj.Status.PodIP); err != nil { - return err - } + if err := ns.ips.DelEntry(obj.ObjectMeta.UID, ns.ingressDefaultAllowIPSet, obj.Status.PodIP); err != nil { + return err + } + if err := ns.ips.DelEntry(obj.ObjectMeta.UID, ns.egressDefaultAllowIPSet, obj.Status.PodIP); err != nil { + return err } return ns.podSelectors.delFromMatching(obj.ObjectMeta.UID, obj.ObjectMeta.Labels, obj.Status.PodIP) @@ -305,7 +320,7 @@ func (ns *ns) deletePod(obj *coreapi.Pod) error { func (ns *ns) addNetworkPolicy(obj interface{}) error { // Analyse policy, determine which rules and ipsets are required - uid, rules, nsSelectors, podSelectors, err := ns.analyse(obj) + uid, rules, nsSelectors, podSelectors, ipBlocks, err := ns.analyse(obj) if err != nil { return err } @@ -317,16 +332,19 @@ func (ns *ns) addNetworkPolicy(obj interface{}) error { if err := ns.podSelectors.provision(uid, nil, podSelectors); err != nil { return err } + if err := ns.ipBlocks.provision(uid, nil, ipBlocks); err != nil { + return err + } return ns.rules.provision(uid, nil, rules) } func (ns *ns) updateNetworkPolicy(oldObj, newObj interface{}) error { // Analyse the old and the new policy so we can determine differences - oldUID, oldRules, oldNsSelectors, oldPodSelectors, err := ns.analyse(oldObj) + oldUID, oldRules, oldNsSelectors, oldPodSelectors, oldIPBlocks, err := ns.analyse(oldObj) if err != nil { return err } - newUID, newRules, newNsSelectors, newPodSelectors, err := ns.analyse(newObj) + newUID, newRules, newNsSelectors, newPodSelectors, newIPBlocks, err := ns.analyse(newObj) if err != nil { return err } @@ -344,18 +362,24 @@ func (ns *ns) updateNetworkPolicy(oldObj, newObj interface{}) error { if err := ns.podSelectors.deprovision(oldUID, oldPodSelectors, newPodSelectors); err != nil { return err } + if err := ns.ipBlocks.deprovision(oldUID, oldIPBlocks, newIPBlocks); err != nil { + return err + } if err := ns.nsSelectors.provision(oldUID, oldNsSelectors, newNsSelectors); err != nil { return err } if err := ns.podSelectors.provision(oldUID, oldPodSelectors, newPodSelectors); err != nil { return err } + if err := ns.ipBlocks.provision(oldUID, oldIPBlocks, newIPBlocks); err != nil { + return err + } return ns.rules.provision(oldUID, oldRules, newRules) } func (ns *ns) deleteNetworkPolicy(obj interface{}) error { // Analyse network policy to free resources - uid, rules, nsSelectors, podSelectors, err := ns.analyse(obj) + uid, rules, nsSelectors, podSelectors, ipBlocks, err := ns.analyse(obj) if err != nil { return err } @@ -369,73 +393,89 @@ func (ns *ns) deleteNetworkPolicy(obj interface{}) error { if err := ns.nsSelectors.deprovision(uid, nsSelectors, nil); err != nil { return err } - return ns.podSelectors.deprovision(uid, podSelectors, nil) -} - -func bypassRule(nsIpsetName ipset.Name, namespace string) []string { - return []string{"-m", "set", "--match-set", string(nsIpsetName), "dst", "-j", "ACCEPT", "-m", "comment", "--comment", "DefaultAllow isolation for namespace: " + namespace} -} - -func (ns *ns) ensureBypassRule() error { - var ipset ipset.Name - if ns.legacy { - ipset = ns.allPods.ipsetName - } else { - ipset = ns.defaultAllowIPSet + if err := ns.podSelectors.deprovision(uid, podSelectors, nil); err != nil { + return err + } + if err := ns.ipBlocks.deprovision(uid, ipBlocks, nil); err != nil { + return err } - common.Log.Debugf("ensuring rule for DefaultAllow in namespace: %s, set %s", ns.name, ipset) - return ns.ipt.Append(TableFilter, DefaultChain, bypassRule(ipset, ns.name)...) + return nil } -func (ns *ns) deleteBypassRule() error { - var ipset ipset.Name - if ns.legacy { - ipset = ns.allPods.ipsetName - } else { - ipset = ns.defaultAllowIPSet +func (ns *ns) updateDefaultAllowIPSetEntry(oldObj, newObj *coreapi.Pod, ipsetName ipset.Name) error { + // Instead of iterating over all selectors we check whether old pod IP + // has been inserted into default-allow ipset to decide whether the IP + // in the ipset has to be updated. + if ns.ips.Exist(oldObj.ObjectMeta.UID, ipsetName, oldObj.Status.PodIP) { + + if err := ns.ips.DelEntry(oldObj.ObjectMeta.UID, ipsetName, oldObj.Status.PodIP); err != nil { + return err + } + if err := ns.ips.AddEntry(newObj.ObjectMeta.UID, ipsetName, newObj.Status.PodIP, podComment(newObj)); err != nil { + return err + } + } + return nil +} + +func bypassRules(namespace string, ingress, egress ipset.Name) map[string][][]string { + return map[string][][]string{ + DefaultChain: { + {"-m", "set", "--match-set", string(ingress), "dst", "-j", "ACCEPT", + "-m", "comment", "--comment", "DefaultAllow ingress isolation for namespace: " + namespace}, + }, + EgressDefaultChain: { + {"-m", "set", "--match-set", string(egress), "src", "-j", EgressMarkChain, + "-m", "comment", "--comment", "DefaultAllow egress isolation for namespace: " + namespace}, + {"-m", "set", "--match-set", string(egress), "src", "-j", "RETURN", + "-m", "comment", "--comment", "DefaultAllow egress isolation for namespace: " + namespace}, + }, + } +} + +func (ns *ns) ensureBypassRules() error { + for chain, rules := range bypassRules(ns.name, ns.ingressDefaultAllowIPSet, ns.egressDefaultAllowIPSet) { + for _, rule := range rules { + common.Log.Debugf("adding rule for DefaultAllow in namespace: %s, chain: %s, %s", ns.name, chain, rule) + if err := ns.ipt.Append(TableFilter, chain, rule...); err != nil { + return err + } + } } - common.Log.Debugf("removing default rule in namespace: %s, set %s", ns.name, ipset) - return ns.ipt.Delete(TableFilter, DefaultChain, bypassRule(ipset, ns.name)...) + return nil +} + +func (ns *ns) deleteBypassRules() error { + for chain, rules := range bypassRules(ns.name, ns.ingressDefaultAllowIPSet, ns.egressDefaultAllowIPSet) { + for _, rule := range rules { + common.Log.Debugf("removing rule for DefaultAllow in namespace: %s, chain: %s, %s", ns.name, chain, rule) + if err := ns.ipt.Delete(TableFilter, chain, rule...); err != nil { + return err + } + } + } + + return nil } func (ns *ns) addNamespace(obj *coreapi.Namespace) error { ns.namespace = obj - // Insert a rule to bypass policies if namespace is DefaultAllow - if !ns.isDefaultDeny(obj) { - if err := ns.ensureBypassRule(); err != nil { - return err - } + // Insert a rule to bypass policies + if err := ns.ensureBypassRules(); err != nil { + return err } // Add namespace ipset to matching namespace selectors - _, err := ns.nsSelectors.addToMatching(obj.ObjectMeta.UID, obj.ObjectMeta.Labels, string(ns.allPods.ipsetName), namespaceComment(ns)) + _, _, err := ns.nsSelectors.addToMatching(obj.ObjectMeta.UID, obj.ObjectMeta.Labels, string(ns.allPods.ipsetName), namespaceComment(ns)) return err } func (ns *ns) updateNamespace(oldObj, newObj *coreapi.Namespace) error { ns.namespace = newObj - // Update bypass rule if ingress default has changed - oldDefaultDeny := ns.isDefaultDeny(oldObj) - newDefaultDeny := ns.isDefaultDeny(newObj) - - if oldDefaultDeny != newDefaultDeny { - common.Log.Infof("namespace DefaultDeny changed from %t to %t", oldDefaultDeny, newDefaultDeny) - if oldDefaultDeny { - if err := ns.ensureBypassRule(); err != nil { - return err - } - } - if newDefaultDeny { - if err := ns.deleteBypassRule(); err != nil { - return err - } - } - } - // Re-evaluate namespace selector membership if labels have changed if !equals(oldObj.ObjectMeta.Labels, newObj.ObjectMeta.Labels) { for _, selector := range ns.nsSelectors.entries { @@ -464,10 +504,8 @@ func (ns *ns) deleteNamespace(obj *coreapi.Namespace) error { ns.namespace = nil // Remove bypass rule - if !ns.isDefaultDeny(obj) { - if err := ns.deleteBypassRule(); err != nil { - return err - } + if err := ns.deleteBypassRules(); err != nil { + return err } // Remove namespace ipset from any matching namespace selectors @@ -476,36 +514,16 @@ func (ns *ns) deleteNamespace(obj *coreapi.Namespace) error { return err } - if !ns.legacy { - return ns.ips.Destroy(ns.defaultAllowIPSet) + if err := ns.ips.Destroy(ns.ingressDefaultAllowIPSet); err != nil { + return err + } + if err := ns.ips.Destroy(ns.egressDefaultAllowIPSet); err != nil { + return err } return nil } -func (ns *ns) isDefaultDeny(namespace *coreapi.Namespace) bool { - nnpJSON, found := namespace.ObjectMeta.Annotations["net.beta.kubernetes.io/network-policy"] - if !found { - return false - } - - if !ns.legacy { - common.Log.Warn("DefaultDeny annotation is supported only in legacy mode (--use-legacy-netpol)") - return false - } - - var nnp NamespaceNetworkPolicy - if err := json.Unmarshal([]byte(nnpJSON), &nnp); err != nil { - common.Log.Warn("Ignoring network policy annotation: unmarshal failed:", err) - // If we can't understand the annotation, behave as if it isn't present - return false - } - - return nnp.Ingress != nil && - nnp.Ingress.Isolation != nil && - *(nnp.Ingress.Isolation) == DefaultDeny -} - func hasIP(pod *coreapi.Pod) bool { // Ensure pod isn't dead, has an IP address and isn't sharing the host network namespace return pod.Status.Phase != "Succeeded" && pod.Status.Phase != "Failed" && @@ -532,10 +550,19 @@ func podComment(pod *coreapi.Pod) string { return fmt.Sprintf("namespace: %s, pod: %s", pod.ObjectMeta.Namespace, pod.ObjectMeta.Name) } +func (ns *ns) defaultAllowIPSetName(pt policyType) ipset.Name { + ipset := ns.ingressDefaultAllowIPSet + if pt == policyTypeEgress { + ipset = ns.egressDefaultAllowIPSet + } + return ipset +} + func (ns *ns) analyse(obj interface{}) ( uid types.UID, rules map[string]*ruleSpec, nsSelectors, podSelectors map[string]*selectorSpec, + ipBlocks map[string]*ipBlockSpec, err error) { switch p := obj.(type) { @@ -550,16 +577,9 @@ func (ns *ns) analyse(obj interface{}) ( ns.policies[uid] = obj // Analyse policy, determine which rules and ipsets are required - if ns.legacy { - rules, nsSelectors, podSelectors, err = ns.analysePolicyLegacy(obj.(*extnapi.NetworkPolicy)) - if err != nil { - return - } - } else { - rules, nsSelectors, podSelectors, err = ns.analysePolicy(obj.(*networkingv1.NetworkPolicy)) - if err != nil { - return - } + rules, nsSelectors, podSelectors, ipBlocks, err = ns.analysePolicy(obj.(*networkingv1.NetworkPolicy)) + if err != nil { + return } return diff --git a/npc/rule.go b/npc/rule.go index c0a334c0..1edee770 100644 --- a/npc/rule.go +++ b/npc/rule.go @@ -2,6 +2,7 @@ package npc import ( "fmt" + "reflect" "strings" "k8s.io/apimachinery/pkg/types" @@ -10,38 +11,71 @@ import ( "github.com/weaveworks/weave/npc/iptables" ) -type ruleSpec struct { - key string - args []string +type ruleHost interface { + // getRuleSpec returns source or destination specification and comment which + // are used in an iptables rule. + // + // If src=true then the rulespec is for source, otherwise - for destination. + getRuleSpec(src bool) ([]string, string) } -func newRuleSpec(proto *string, srcHost *selectorSpec, dstHost *selectorSpec, dstPort *string) *ruleSpec { +type ruleSpec struct { + key string + args []string + policyType policyType +} + +func newRuleSpec(policyType policyType, proto *string, srcHost, dstHost ruleHost, dstPort *string) *ruleSpec { args := []string{} if proto != nil { args = append(args, "-p", *proto) } srcComment := "anywhere" - if srcHost != nil { - args = append(args, "-m", "set", "--match-set", string(srcHost.ipsetName), "src") - if srcHost.nsName != "" { - srcComment = fmt.Sprintf("pods: namespace: %s, selector: %s", srcHost.nsName, srcHost.key) - } else { - srcComment = fmt.Sprintf("namespaces: selector: %s", srcHost.key) - } + if srcHost != nil && !reflect.ValueOf(srcHost).IsNil() { + rule, comment := srcHost.getRuleSpec(true) + args = append(args, rule...) + srcComment = comment } dstComment := "anywhere" - if dstHost != nil { - args = append(args, "-m", "set", "--match-set", string(dstHost.ipsetName), "dst") - dstComment = fmt.Sprintf("pods: namespace: %s, selector: %s", dstHost.nsName, dstHost.key) + if dstHost != nil && !reflect.ValueOf(dstHost).IsNil() { + rule, comment := dstHost.getRuleSpec(false) + args = append(args, rule...) + dstComment = comment } if dstPort != nil { args = append(args, "--dport", *dstPort) } - args = append(args, "-j", "ACCEPT") - args = append(args, "-m", "comment", "--comment", fmt.Sprintf("%s -> %s", srcComment, dstComment)) + // NOTE: if you remove the comment bellow, then embed `policyType` into `key`. + // Otherwise, the rule won't be provisioned if it exists for other policy type. + args = append(args, "-m", "comment", "--comment", fmt.Sprintf("%s -> %s (%s)", srcComment, dstComment, policyTypeStr(policyType))) key := strings.Join(args, " ") - return &ruleSpec{key, args} + return &ruleSpec{key, args, policyType} +} + +func (spec *ruleSpec) iptChain() string { + if spec.policyType == policyTypeEgress { + return EgressCustomChain + } + return IngressChain +} + +func (spec *ruleSpec) iptRuleSpecs() [][]string { + if spec.policyType == policyTypeIngress { + rule := make([]string, len(spec.args)) + copy(rule, spec.args) + rule = append(rule, "-j", "ACCEPT") + return [][]string{rule} + } + + // policyTypeEgress + ruleMark := make([]string, len(spec.args)) + copy(ruleMark, spec.args) + ruleMark = append(ruleMark, "-j", EgressMarkChain) + ruleReturn := make([]string, len(spec.args)) + copy(ruleReturn, spec.args) + ruleReturn = append(ruleReturn, "-j", "RETURN") + return [][]string{ruleMark, ruleReturn} } type ruleSet struct { @@ -58,10 +92,14 @@ func (rs *ruleSet) deprovision(user types.UID, current, desired map[string]*rule if _, found := desired[key]; !found { delete(rs.users[key], user) if len(rs.users[key]) == 0 { - common.Log.Infof("deleting rule: %v", spec.args) - if err := rs.ipt.Delete(TableFilter, IngressChain, spec.args...); err != nil { - return err + chain := spec.iptChain() + for _, rule := range spec.iptRuleSpecs() { + common.Log.Infof("deleting rule %v from %q chain", rule, chain) + if err := rs.ipt.Delete(TableFilter, chain, rule...); err != nil { + return err + } } + delete(rs.users, key) } } @@ -74,9 +112,12 @@ func (rs *ruleSet) provision(user types.UID, current, desired map[string]*ruleSp for key, spec := range desired { if _, found := current[key]; !found { if _, found := rs.users[key]; !found { - common.Log.Infof("adding rule: %v", spec.args) - if err := rs.ipt.Append(TableFilter, IngressChain, spec.args...); err != nil { - return err + chain := spec.iptChain() + for _, rule := range spec.iptRuleSpecs() { + common.Log.Infof("adding rule %v to %q chain", rule, chain) + if err := rs.ipt.Append(TableFilter, chain, rule...); err != nil { + return err + } } rs.users[key] = make(map[types.UID]struct{}) } diff --git a/npc/rule_test.go b/npc/rule_test.go index dee1802f..15535449 100644 --- a/npc/rule_test.go +++ b/npc/rule_test.go @@ -9,7 +9,7 @@ func TestRegressionHandleNil3095(t *testing.T) { // Test for handling nil values // https://github.com/weaveworks/weave/issues/3095 - rs := newRuleSpec(nil, nil, nil, nil) + rs := newRuleSpec(policyTypeIngress, nil, nil, nil, nil) require.NotEqual(t, rs.key, "") diff --git a/npc/selector.go b/npc/selector.go index 2a40e185..6280f26a 100644 --- a/npc/selector.go +++ b/npc/selector.go @@ -1,6 +1,8 @@ package npc import ( + "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" @@ -10,25 +12,25 @@ import ( ) type selectorSpec struct { - key string // string representation (for hash keying/equality comparison) - selector labels.Selector // k8s Selector object (for matching) - dst bool // destination selector + key string // string representation (for hash keying/equality comparison) + selector labels.Selector // k8s Selector object (for matching) + policyTypes []policyType // If non-empty, then selectorSpec is a target selector for given policyTypes. ipsetType ipset.Type // type of ipset to provision ipsetName ipset.Name // generated ipset name nsName string // Namespace name } -func newSelectorSpec(json *metav1.LabelSelector, dst bool, nsName string, ipsetType ipset.Type) (*selectorSpec, error) { +func newSelectorSpec(json *metav1.LabelSelector, policyType []policyType, nsName string, ipsetType ipset.Type) (*selectorSpec, error) { selector, err := metav1.LabelSelectorAsSelector(json) if err != nil { return nil, err } key := selector.String() return &selectorSpec{ - key: key, - selector: selector, - dst: dst, + key: key, + selector: selector, + policyTypes: policyType, // We prefix the selector string with the namespace name when generating // the shortname because you can specify the same selector in multiple // namespaces - we need those to map to distinct ipsets @@ -37,6 +39,23 @@ func newSelectorSpec(json *metav1.LabelSelector, dst bool, nsName string, ipsetT nsName: nsName}, nil } +func (spec *selectorSpec) getRuleSpec(src bool) ([]string, string) { + dir := "dst" + if src { + dir = "src" + } + rule := []string{"-m", "set", "--match-set", string(spec.ipsetName), dir} + + comment := "anywhere" + if spec.nsName != "" { + comment = fmt.Sprintf("pods: namespace: %s, selector: %s", spec.nsName, spec.key) + } else { + comment = fmt.Sprintf("namespaces: selector: %s", spec.key) + } + + return rule, comment +} + type selector struct { ips ipset.Interface spec *selectorSpec @@ -55,49 +74,54 @@ func (s *selector) delEntry(user types.UID, entry string) error { } type selectorFn func(selector *selector) error +type selectorWithPolicyTypeFn func(selector *selector, policyType policyType) error type selectorSet struct { ips ipset.Interface onNewSelector selectorFn - // invoked after dst selector has been provisioned for the first time - onNewDstSelector selectorFn - // invoked after the last instance of dst selector has been deprovisioned - onDestroyDstSelector selectorFn + // invoked after target selector has been provisioned for the first time + onNewTargetSelector selectorWithPolicyTypeFn + // invoked after the last instance of target selector has been deprovisioned + onDestroyTargetSelector selectorWithPolicyTypeFn users map[string]map[types.UID]struct{} // list of users per selector entries map[string]*selector - // We need to keep track of dst selector instances to be able to invoke - // onNewDstSelector and onDestroyDstSelector callbacks at the right time; - // selectorSpec.Key -> count - dstSelectorsCount map[string]int + // We need to keep track of target selector instances to be able to invoke + // onNewTargetSelector and onDestroyTargetSelector callbacks at the right time; + // selectorSpec.Key -> policyType -> count + targetSelectorsCount map[string]map[policyType]int } -func newSelectorSet(ips ipset.Interface, onNewSelector, onNewDstSelector selectorFn, onDestroyDstSelector selectorFn) *selectorSet { +func newSelectorSet(ips ipset.Interface, onNewSelector selectorFn, onNewTargetSelector, onDestroyTargetSelector selectorWithPolicyTypeFn) *selectorSet { return &selectorSet{ - ips: ips, - onNewSelector: onNewSelector, - onNewDstSelector: onNewDstSelector, - onDestroyDstSelector: onDestroyDstSelector, + ips: ips, + onNewSelector: onNewSelector, + onNewTargetSelector: onNewTargetSelector, + onDestroyTargetSelector: onDestroyTargetSelector, users: make(map[string]map[types.UID]struct{}), entries: make(map[string]*selector), - dstSelectorsCount: make(map[string]int)} + targetSelectorsCount: make(map[string]map[policyType]int)} } -func (ss *selectorSet) addToMatching(user types.UID, labelMap map[string]string, entry string, comment string) (bool, error) { - found := false +func (ss *selectorSet) addToMatching(user types.UID, labelMap map[string]string, entry string, comment string) (bool, bool, error) { + foundIngress := false + foundEgress := false for _, s := range ss.entries { if s.matches(labelMap) { - if ss.dstSelectorExist(s) { - found = true + if ss.targetSelectorExist(s, policyTypeIngress) { + foundIngress = true + } + if ss.targetSelectorExist(s, policyTypeEgress) { + foundEgress = true } if err := s.addEntry(user, entry, comment); err != nil { - return found, err + return foundIngress, foundEgress, err } } } - return found, nil + return foundIngress, foundEgress, nil } func (ss *selectorSet) delFromMatching(user types.UID, labelMap map[string]string, entry string) error { @@ -111,8 +135,8 @@ func (ss *selectorSet) delFromMatching(user types.UID, labelMap map[string]strin return nil } -func (ss *selectorSet) dstSelectorExist(s *selector) bool { - return ss.dstSelectorsCount[s.spec.key] > 0 +func (ss *selectorSet) targetSelectorExist(s *selector, policyType policyType) bool { + return ss.targetSelectorsCount[s.spec.key][policyType] > 0 } func (ss *selectorSet) deprovision(user types.UID, current, desired map[string]*selectorSpec) error { @@ -129,13 +153,14 @@ func (ss *selectorSet) deprovision(user types.UID, current, desired map[string]* delete(ss.users, key) } - if spec.dst { - ss.dstSelectorsCount[key]-- - if ss.dstSelectorsCount[key] == 0 { - if err := ss.onDestroyDstSelector(&selector{ss.ips, spec}); err != nil { + for _, policyType := range spec.policyTypes { + ss.targetSelectorsCount[key][policyType]-- + if ss.targetSelectorsCount[key][policyType] == 0 { + if err := ss.onDestroyTargetSelector(&selector{ss.ips, spec}, policyType); err != nil { return err } } + // TODO(brb) delete(...) } } } @@ -160,10 +185,13 @@ func (ss *selectorSet) provision(user types.UID, current, desired map[string]*se } ss.users[key][user] = struct{}{} - if spec.dst { - ss.dstSelectorsCount[key]++ - if ss.dstSelectorsCount[key] == 1 { - if err := ss.onNewDstSelector(selector); err != nil { + for _, pt := range spec.policyTypes { + if _, found := ss.targetSelectorsCount[key]; !found { + ss.targetSelectorsCount[key] = make(map[policyType]int) + } + ss.targetSelectorsCount[key][pt]++ + if ss.targetSelectorsCount[key][pt] == 1 { + if err := ss.onNewTargetSelector(selector, pt); err != nil { return err } } diff --git a/npc/types.go b/npc/types.go new file mode 100644 index 00000000..608f07f0 --- /dev/null +++ b/npc/types.go @@ -0,0 +1,15 @@ +package npc + +type policyType byte + +const ( + policyTypeIngress policyType = iota + policyTypeEgress +) + +func policyTypeStr(policyType policyType) string { + if policyType == policyTypeEgress { + return "egress" + } + return "ingress" +} diff --git a/prog/weave-kube/weave-daemonset-k8s-1.7.yaml b/prog/weave-kube/weave-daemonset-k8s-1.7.yaml index 301e67ab..6f5ba5fc 100644 --- a/prog/weave-kube/weave-daemonset-k8s-1.7.yaml +++ b/prog/weave-kube/weave-daemonset-k8s-1.7.yaml @@ -25,14 +25,6 @@ items: - get - list - watch - - apiGroups: - - extensions - resources: - - networkpolicies - verbs: - - get - - list - - watch - apiGroups: - 'networking.k8s.io' resources: @@ -163,7 +155,6 @@ items: fieldPath: spec.nodeName image: 'weaveworks/weave-npc:latest' imagePullPolicy: Always -#npc-args resources: requests: cpu: 10m diff --git a/prog/weave-npc/main.go b/prog/weave-npc/main.go index d9922de9..9d054206 100644 --- a/prog/weave-npc/main.go +++ b/prog/weave-npc/main.go @@ -8,7 +8,6 @@ import ( "github.com/coreos/go-iptables/iptables" "github.com/spf13/cobra" coreapi "k8s.io/api/core/v1" - extnapi "k8s.io/api/extensions/v1beta1" networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" @@ -18,6 +17,7 @@ import ( "k8s.io/client-go/tools/cache" "github.com/weaveworks/weave/common" + "github.com/weaveworks/weave/net" "github.com/weaveworks/weave/net/ipset" "github.com/weaveworks/weave/npc" "github.com/weaveworks/weave/npc/metrics" @@ -30,7 +30,6 @@ var ( logLevel string allowMcast bool nodeName string - legacy bool maxList int ) @@ -53,7 +52,27 @@ func resetIPTables(ipt *iptables.IPTables) error { return err } - return ipt.ClearChain(npc.TableFilter, npc.MainChain) + if err := ipt.ClearChain(npc.TableFilter, npc.MainChain); err != nil { + return err + } + + if err := ipt.ClearChain(npc.TableFilter, npc.EgressMarkChain); err != nil { + return err + } + + if err := ipt.ClearChain(npc.TableFilter, npc.EgressCustomChain); err != nil { + return err + } + + if err := ipt.ClearChain(npc.TableFilter, npc.EgressDefaultChain); err != nil { + return err + } + + // We do not clear npc.EgressChain here because otherwise, in the case of restarting + // weave-npc process, all egress traffic is allowed for a short period of time. + // The chain is created in createBaseRules. + + return nil } func resetIPSets(ips ipset.Interface) error { @@ -77,6 +96,11 @@ func resetIPSets(ips ipset.Interface) error { } for _, s := range sets { + // LocalIPset might be used by WEAVE-NPC-EGRESS chain which we do not + // flush, so we cannot destroy it. + if s == npc.LocalIpset { + continue + } common.Log.Debugf("Destroying ipset '%s'", string(s)) if err := ips.Destroy(s); err != nil { common.Log.Errorf("Failed to destroy ipset '%s'", string(s)) @@ -112,11 +136,81 @@ func createBaseRules(ipt *iptables.IPTables, ips ipset.Interface) error { } // If the destination address is not any of the local pods, let it through - if err := ips.Create(npc.LocalIpset, ipset.HashIP); err != nil { + found, err := ipsetExist(ips, npc.LocalIpset) + if err != nil { return err } - return ipt.Append(npc.TableFilter, npc.MainChain, - "-m", "set", "!", "--match-set", npc.LocalIpset, "dst", "-j", "ACCEPT") + if !found { + if err := ips.Create(npc.LocalIpset, ipset.HashIP); err != nil { + return err + } + } + if err := ipt.Append(npc.TableFilter, npc.MainChain, + "-m", "set", "!", "--match-set", npc.LocalIpset, "dst", "-j", "ACCEPT"); err != nil { + return err + } + + if err := ipt.Append(npc.TableFilter, npc.EgressMarkChain, + "-j", "MARK", "--set-xmark", npc.EgressMark); err != nil { + return err + } + + // Egress rules: + // + // -A WEAVE-NPC-EGRESS -m state --state RELATED,ESTABLISHED -j ACCEPT + // -A WEAVE-NPC-EGRESS -m set ! --match-set weave-local-pods src -j RETURN + // -A WEAVE-NPC-EGRESS -d 224.0.0.0/4 -j RETURN + // -A WEAVE-NPC-EGRESS -m state --state NEW -j WEAVE-NPC-EGRESS-DEFAULT + // -A WEAVE-NPC-EGRESS -m state --state NEW -m mark ! --mark 0x40000/0x40000 -j WEAVE-NPC-EGRESS-CUSTOM + // -A WEAVE-NPC-EGRESS -m state --state NEW -m mark ! --mark 0x40000/0x40000 -j NFLOG --nflog-group 86 + // -A WEAVE-NPC-EGRESS -m mark ! --mark 0x40000/0x40000 -j DROP + // + // -A WEAVE-NPC-EGRESS-CUSTOM -j MARK --set-xmark 0x40000/0x40000 + // -A WEAVE-NPC-EGRESS-CUSTOM -j RETURN + // + // -A WEAVE-NPC-EGRESS-DEFAULT -j MARK --set-xmark 0x40000/0x40000 + // -A WEAVE-NPC-EGRESS-DEFAULT -j RETURN + // + // For each rule we create two (mark and return). We cannot just accept + // a packet if it matches any rule, as a packet might need to traverse + // the ingress npc as well which happens later in the chain (in some cases + // we cannot detect whether packet is ingress or egress, so we need to + // check both chains). + + ruleSpecs := [][]string{ + {"-m", "state", "--state", "RELATED,ESTABLISHED", "-j", "ACCEPT"}, + {"-m", "state", "--state", "NEW", "-m", "set", "!", "--match-set", npc.LocalIpset, "src", "-j", "RETURN"}, + } + if allowMcast { + ruleSpecs = append(ruleSpecs, []string{"-d", "224.0.0.0/4", "-j", "RETURN"}) + } + ruleSpecs = append(ruleSpecs, [][]string{ + {"-m", "state", "--state", "NEW", "-j", string(npc.EgressDefaultChain)}, + {"-m", "state", "--state", "NEW", "-m", "mark", "!", "--mark", npc.EgressMark, "-j", string(npc.EgressCustomChain)}, + {"-m", "state", "--state", "NEW", "-m", "mark", "!", "--mark", npc.EgressMark, "-j", "NFLOG", "--nflog-group", "86"}, + {"-m", "mark", "!", "--mark", npc.EgressMark, "-j", "DROP"}, + }...) + if err := net.AddChainWithRules(ipt, npc.TableFilter, npc.EgressChain, ruleSpecs); err != nil { + return err + } + + return nil +} + +// Dummy way to check whether a given ipset exists. +// TODO(brb) Use "ipset -exist create <..>" for our purpose instead (for some reasons +// creating an ipset with -exist fails). +func ipsetExist(ips ipset.Interface, name ipset.Name) (bool, error) { + sets, err := ips.List(string(name)) + if err != nil { + return false, err + } + for _, s := range sets { + if s == name { + return true, nil + } + } + return false, nil } func root(cmd *cobra.Command, args []string) { @@ -132,10 +226,6 @@ func root(cmd *cobra.Command, args []string) { } common.Log.Infof("Starting Weaveworks NPC %s; node name %q", version, nodeName) - if legacy { - common.Log.Info("Running in legacy mode (k8s pre-1.7 network policy semantics)") - } - if err := metrics.Start(metricsAddr); err != nil { common.Log.Fatalf("Failed to start metrics: %v", err) } @@ -159,7 +249,7 @@ func root(cmd *cobra.Command, args []string) { handleError(resetIPSets(ips)) handleError(createBaseRules(ipt, ips)) - npc := npc.New(nodeName, legacy, ipt, ips) + npc := npc.New(nodeName, ipt, ips) nsController := makeController(client.Core().RESTClient(), "namespaces", &coreapi.Namespace{}, cache.ResourceEventHandlerFuncs{ @@ -220,11 +310,7 @@ func root(cmd *cobra.Command, args []string) { handleError(npc.UpdateNetworkPolicy(old, new)) }, } - if legacy { - npController = makeController(client.Extensions().RESTClient(), "networkpolicies", &extnapi.NetworkPolicy{}, npHandlers) - } else { - npController = makeController(client.NetworkingV1().RESTClient(), "networkpolicies", &networkingv1.NetworkPolicy{}, npHandlers) - } + npController = makeController(client.NetworkingV1().RESTClient(), "networkpolicies", &networkingv1.NetworkPolicy{}, npHandlers) go nsController.Run(wait.NeverStop) go podController.Run(wait.NeverStop) @@ -245,7 +331,6 @@ func main() { rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "debug", "logging level (debug, info, warning, error)") rootCmd.PersistentFlags().BoolVar(&allowMcast, "allow-mcast", true, "allow all multicast traffic") rootCmd.PersistentFlags().StringVar(&nodeName, "node-name", "", "only generate rules that apply to this node") - rootCmd.PersistentFlags().BoolVar(&legacy, "use-legacy-netpol", false, "use legacy network policies (pre k8s 1.7 vsn)") rootCmd.PersistentFlags().IntVar(&maxList, "max-list-size", 1024, "maximum size of ipset list (for namespaces)") handleError(rootCmd.Execute()) diff --git a/site/kubernetes/kube-addon.md b/site/kubernetes/kube-addon.md index fbb5c0b6..3c5880a4 100644 --- a/site/kubernetes/kube-addon.md +++ b/site/kubernetes/kube-addon.md @@ -370,7 +370,6 @@ You can customise the YAML you get from `cloud.weave.works` by passing some of W - `disable-npc`: boolean (`true|false`). Default: `false`. - `env.NAME=VALUE`: add environment variable `NAME` and set it to `VALUE`. - `seLinuxOptions.NAME=VALUE`: add SELinux option `NAME` and set it to `VALUE`, e.g. `seLinuxOptions.type=spc_t` - - `use-legacy-netpol`: use [legacy NetworkPolicy semantics](https://v1-6.docs.kubernetes.io/docs/api-reference/v1.6/#networkpolicy-v1beta1-extensions), boolean (`true|false`). Default: `true` for Kubernetes version <= 1.6, `false` for > 1.6. The list of variables you can set is: diff --git a/test/840_weave_kube_3_test.sh b/test/840_weave_kube_3_test.sh index 7ae42224..0efde493 100755 --- a/test/840_weave_kube_3_test.sh +++ b/test/840_weave_kube_3_test.sh @@ -52,7 +52,6 @@ fi # Ensure Kubernetes uses locally built container images and inject code coverage environment variable (or do nothing depending on $COVERAGE): sed -e "s%imagePullPolicy: Always%imagePullPolicy: Never%" \ -e "s%env:%$WEAVE_ENV_VARS%" \ - -e "s%#npc-args% args:\n - '--use-legacy-netpol'%" \ "$(dirname "$0")/../prog/weave-kube/weave-daemonset-k8s-1.8.yaml" | run_on "$HOST1" "$KUBECTL apply -n kube-system -f -" sleep 5 @@ -76,10 +75,19 @@ assert "run_on $HOST1 ps aux | grep -c '[d]efunct'" "0" assert "run_on $HOST2 ps aux | grep -c '[d]efunct'" "0" assert "run_on $HOST3 ps aux | grep -c '[d]efunct'" "0" -# Set up a simple network policy so all our test pods can talk to each other -run_on $HOST1 "$KUBECTL annotate ns default net.beta.kubernetes.io/network-policy='{\"ingress\":{\"isolation\":\"DefaultDeny\"}}'" +# Set up simple network policies so all our test pods can talk to each other run_on $HOST1 "$KUBECTL apply -f -" </dev/null" -# Restart weave-net with npc in non-legacy mode -$SSH $HOST1 "$KUBECTL delete ds weave-net -n=kube-system" -sed -e "s%imagePullPolicy: Always%imagePullPolicy: Never%" \ - -e "s%env:%$WEAVE_ENV_VARS%" \ - "$(dirname "$0")/../prog/weave-kube/weave-daemonset-k8s-1.8.yaml" | run_on "$HOST1" "$KUBECTL apply -n kube-system -f -" - -assert_raises 'wait_for_x check_all_pods_communicate pods' - -# nettest-deny should still not be able to reach nettest pods -assert_raises "! $SSH $HOST1 $KUBECTL exec $denyPodName -- curl -s -S -f -m 2 http://$DOMAIN:8080/status >/dev/null" - # check access via virtual IP VIRTUAL_IP=$($SSH $HOST1 $KUBECTL get service netvirt -o template --template={{.spec.clusterIP}}) assert_raises "$SSH $HOST1 $KUBECTL exec $podName -- curl -s -S -f -m 2 http://$VIRTUAL_IP/status >/dev/null" @@ -269,6 +266,61 @@ EOF assert_raises "$SSH $HOST1 curl -s -S -f -m 2 http://$VIRTUAL_IP/status >/dev/null" assert_raises "$SSH $HOST1 curl -s -S -f -m 2 http://$HOST2:31138/status >/dev/null" +# allow nettest-deny to access only to 8.8.8.0/24 +run_on $HOST1 "$KUBECTL apply -f -" </dev/null" +assert_raises "! $SSH $HOST1 $KUBECTL exec $denyPodName -- dig @8.8.8.4 google.com >/dev/null" +assert_raises "$SSH $HOST1 $KUBECTL exec $denyPodName -- dig @8.8.8.8 google.com >/dev/null" + +# also, allow nettest-deny to access nettest and kube-system (for kube-dns) +run_on $HOST1 "$KUBECTL label namespace kube-system name=kube-system" +run_on $HOST1 "$KUBECTL replace -f -" </dev/null" + # Passing --no-masq-local and setting externalTrafficPolicy to Local must preserve # the client source IP addr diff --git a/test/images/network-tester/Dockerfile b/test/images/network-tester/Dockerfile index 149b92c3..a7be5948 100644 --- a/test/images/network-tester/Dockerfile +++ b/test/images/network-tester/Dockerfile @@ -13,7 +13,7 @@ # limitations under the License. FROM bash -RUN apk add --no-cache curl +RUN apk add --no-cache curl bind-tools ADD webserver webserver EXPOSE 8080 ENTRYPOINT ["/webserver"] diff --git a/weave b/weave index 56b42719..e5bb8b7d 100755 --- a/weave +++ b/weave @@ -492,10 +492,20 @@ destroy_bridge() { run_iptables -t filter -D FORWARD -o $BRIDGE -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || true run_iptables -t filter -D FORWARD -i $BRIDGE -o $BRIDGE -j ACCEPT 2>/dev/null || true run_iptables -F WEAVE-NPC >/dev/null 2>&1 || true - run_iptables -t filter -D FORWARD -o $BRIDGE -j WEAVE-NPC 2>/dev/null || true + run_iptables -t filter -D FORWARD -o $BRIDGE -m comment --comment "NOTE: this must go before '-j KUBE-FORWARD'" -j WEAVE-NPC 2>/dev/null || true run_iptables -t filter -D FORWARD -o $BRIDGE -m state --state NEW -j NFLOG --nflog-group 86 2>/dev/null || true run_iptables -t filter -D FORWARD -o $BRIDGE -j DROP 2>/dev/null || true run_iptables -X WEAVE-NPC >/dev/null 2>&1 || true + run_iptables -F WEAVE-NPC-EGRESS >/dev/null 2>&1 || true + run_iptables -t filter -D FORWARD -i $BRIDGE -m comment --comment "NOTE: this must go before '-j KUBE-FORWARD'" -j WEAVE-NPC-EGRESS 2>/dev/null || true + run_iptables -t filter -D INPUT -i $BRIDGE -j WEAVE-NPC-EGRESS 2>/dev/null || true + run_iptables -X WEAVE-NPC-EGRESS >/dev/null 2>&1 || true + run_iptables -F WEAVE-NPC-EGRESS-DEFAULT >/dev/null 2>&1 || true + run_iptables -X WEAVE-NPC-EGRESS-DEFAULT >/dev/null 2>&1 || true + run_iptables -F WEAVE-NPC-EGRESS-CUSTOM >/dev/null 2>&1 || true + run_iptables -X WEAVE-NPC-EGRESS-CUSTOM >/dev/null 2>&1 || true + run_iptables -F WEAVE-NPC-EGRESS-ACCEPT >/dev/null 2>&1 || true + run_iptables -X WEAVE-NPC-EGRESS-ACCEPT >/dev/null 2>&1 || true run_iptables -F WEAVE-EXPOSE >/dev/null 2>&1 || true run_iptables -t filter -D FORWARD -o $BRIDGE -j WEAVE-EXPOSE 2>/dev/null || true