reweave/weave
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3313 from weaveworks/issues/2624-egress-npc

Browse Source 
Implement Egress NPC
This commit is contained in:
Bryan Boreham authored and GitHub committed 2018-07-24 10:17:45 +01:00
2 parents 88616007b0 e65cf32372
commit 9cf0dd1b87
18 files changed
+1057 -497
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

net/bridge.go
+18 -41
View File
@@ -17,6 +17,7 @@ import (
"github.com/weaveworks/weave/ipam/tracker"
"github.com/weaveworks/weave/net/address"
"github.com/weaveworks/weave/net/ipset"
"github.com/weaveworks/weave/npc"
)
/* This code implements three possible configurations to connect
@@ -457,14 +458,27 @@ func configureIPTables(config *BridgeConfig, ips ipset.Interface) error {
}
if config.NPC {
// Steer traffic via the NPC
_ = ipt.NewChain("filter", "WEAVE-NPC") // ignore error because it might already exist
// If WEAVE-NPC chain doesn't exist then creating a rule in the chain will fail
// Steer traffic via the NPC.
if err = ensureChains(ipt, "filter", npc.MainChain, npc.EgressChain); err != nil {
return err
}
// Steer egress traffic destined to local node.
if err = ipt.AppendUnique("filter", "INPUT", "-i", config.WeaveBridgeName, "-j", npc.EgressChain); err != nil {
return err
}
fwdRules = append(fwdRules,
[][]string{
// Might include ingress traffic which is fine as long as we do not
// ACCEPT in WEAVE-NPC-EGRESS chain
{"-i", config.WeaveBridgeName,
"-m", "comment", "--comment", "NOTE: this must go before '-j KUBE-FORWARD'",
"-j", npc.EgressChain},
// The following rules are for ingress NPC processing
{"-o", config.WeaveBridgeName,
"-m", "comment", "--comment", "NOTE: this must go before '-j KUBE-FORWARD'",
"-j", "WEAVE-NPC"},
"-j", npc.MainChain},
{"-o", config.WeaveBridgeName, "-m", "state", "--state", "NEW", "-j", "NFLOG", "--nflog-group", "86"},
{"-o", config.WeaveBridgeName, "-j", "DROP"},
}...)
@@ -570,43 +584,6 @@ func linkSetUpByName(linkName string) error {
return netlink.LinkSetUp(link)
}
// ensureRulesAtTop ensures the presence of given iptables rules.
//
// If any rule from the list is missing, the function deletes all given
// rules and re-inserts them at the top of the chain to ensure the order of the rules.
func ensureRulesAtTop(table, chain string, rulespecs [][]string, ipt *iptables.IPTables) error {
allFound := true
for _, rs := range rulespecs {
found, err := ipt.Exists(table, chain, rs...)
if err != nil {
return errors.Wrapf(err, "ipt.Exists(%s, %s, %s)", table, chain, rs)
}
if !found {
allFound = false
break
}
}
// All rules exist, do nothing.
if allFound {
return nil
}
for pos, rs := range rulespecs {
// If any is missing, then delete all, as we need to preserve the order of
// given rules. Ignore errors, as rule might not exist.
if !allFound {
ipt.Delete(table, chain, rs...)
}
if err := ipt.Insert(table, chain, pos+1, rs...); err != nil {
return errors.Wrapf(err, "ipt.Append(%s, %s, %s)", table, chain, rs)
}
}
return nil
}
func reexpose(config *BridgeConfig, log *logrus.Logger) error {
// Get existing IP addrs of the weave bridge.
// If the bridge hasn't been exposed, then this functions does nothing.
net/iptables.go
+106
View File
@@ -0,0 +1,106 @@
package net
import (
"strings"
"github.com/coreos/go-iptables/iptables"
"github.com/pkg/errors"
)
// AddChainWithRules creates a chain and appends given rules to it.
//
// If the chain exists, but its rules are not the same as the given ones, the
// function will flush the chain and then will append the rules.
func AddChainWithRules(ipt *iptables.IPTables, table, chain string, rulespecs [][]string) error {
if err := ensureChains(ipt, table, chain); err != nil {
return err
}
currRuleSpecs, err := ipt.List(table, chain)
if err != nil {
return errors.Wrapf(err, "iptables -S. table: %q, chain: %q", table, chain)
}
// First returned rule is "-N $(chain)", so ignore it
currRules := strings.Join(currRuleSpecs[1:], "\n")
rules := make([]string, 0)
for _, r := range rulespecs {
rules = append(rules, strings.Join(r, " "))
}
reqRules := strings.Join(rules, "\n")
if currRules == reqRules {
return nil
}
if err := ipt.ClearChain(table, chain); err != nil {
return err
}
for _, r := range rulespecs {
if err := ipt.Append(table, chain, r...); err != nil {
return errors.Wrapf(err, "iptables -A. table: %q, chain: %q, rule: %s", table, chain, r)
}
}
return nil
}
// ensureChains creates given chains if they do not exist.
func ensureChains(ipt *iptables.IPTables, table string, chains ...string) error {
existingChains, err := ipt.ListChains(table)
if err != nil {
return errors.Wrapf(err, "ipt.ListChains(%s)", table)
}
chainMap := make(map[string]struct{})
for _, c := range existingChains {
chainMap[c] = struct{}{}
}
for _, c := range chains {
if _, found := chainMap[c]; !found {
if err := ipt.NewChain(table, c); err != nil {
return errors.Wrapf(err, "ipt.NewChain(%s, %s)", table, c)
}
}
}
return nil
}
// ensureRulesAtTop ensures the presence of given iptables rules.
//
// If any rule from the list is missing, the function deletes all given
// rules and re-inserts them at the top of the chain to ensure the order of the rules.
func ensureRulesAtTop(table, chain string, rulespecs [][]string, ipt *iptables.IPTables) error {
allFound := true
for _, rs := range rulespecs {
found, err := ipt.Exists(table, chain, rs...)
if err != nil {
return errors.Wrapf(err, "ipt.Exists(%s, %s, %s)", table, chain, rs)
}
if !found {
allFound = false
break
}
}
// All rules exist, do nothing.
if allFound {
return nil
}
for pos, rs := range rulespecs {
// If any is missing, then delete all, as we need to preserve the order of
// given rules. Ignore errors, as rule might not exist.
if !allFound {
ipt.Delete(table, chain, rs...)
}
if err := ipt.Insert(table, chain, pos+1, rs...); err != nil {
return errors.Wrapf(err, "ipt.Append(%s, %s, %s)", table, chain, rs)
}
}
return nil
}
npc/analyser.go
+119 -140
View File
@@ -4,173 +4,158 @@ import (
"fmt"
apiv1 "k8s.io/api/core/v1"
extnapi "k8s.io/api/extensions/v1beta1"
networkingv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"github.com/weaveworks/weave/net/ipset"
)
// analysePolicyLegacy is used to analyse extensions/v1beta1/NetworkPolicy (legacy) and
// implements pre-1.7 k8s netpol semantics, whilst analysePolicy - networking.k8s.io/v1/NetworkPolicy
// and 1.7 semantics.
func (ns *ns) analysePolicyLegacy(policy *extnapi.NetworkPolicy) (
rules map[string]*ruleSpec,
nsSelectors, podSelectors map[string]*selectorSpec,
err error) {
nsSelectors = make(map[string]*selectorSpec)
podSelectors = make(map[string]*selectorSpec)
rules = make(map[string]*ruleSpec)
dstSelector, err := newSelectorSpec(&policy.Spec.PodSelector, true, ns.name, ipset.HashIP)
if err != nil {
return nil, nil, nil, err
}
podSelectors[dstSelector.key] = dstSelector
for _, ingressRule := range policy.Spec.Ingress {
if ingressRule.Ports != nil && len(ingressRule.Ports) == 0 {
// Ports is empty, this rule matches no ports (no traffic matches).
continue
}
if ingressRule.From != nil && len(ingressRule.From) == 0 {
// From is empty, this rule matches no sources (no traffic matches).
continue
}
if ingressRule.From == nil {
// From is not provided, this rule matches all sources (traffic not restricted by source).
if ingressRule.Ports == nil {
// Ports is not provided, this rule matches all ports (traffic not restricted by port).
rule := newRuleSpec(nil, nil, dstSelector, nil)
rules[rule.key] = rule
} else {
// Ports is present and contains at least one item, then this rule allows traffic
// only if the traffic matches at least one port in the ports list.
withNormalisedProtoAndPortLegacy(ingressRule.Ports, func(proto, port string) {
rule := newRuleSpec(&proto, nil, dstSelector, &port)
rules[rule.key] = rule
})
}
} else {
// From is present and contains at least on item, this rule allows traffic only if the
// traffic matches at least one item in the from list.
for _, peer := range ingressRule.From {
var srcSelector *selectorSpec
if peer.PodSelector != nil {
srcSelector, err = newSelectorSpec(peer.PodSelector, false, ns.name, ipset.HashIP)
if err != nil {
return nil, nil, nil, err
}
podSelectors[srcSelector.key] = srcSelector
}
if peer.NamespaceSelector != nil {
srcSelector, err = newSelectorSpec(peer.NamespaceSelector, false, "", ipset.ListSet)
if err != nil {
return nil, nil, nil, err
}
nsSelectors[srcSelector.key] = srcSelector
}
if ingressRule.Ports == nil {
// Ports is not provided, this rule matches all ports (traffic not restricted by port).
rule := newRuleSpec(nil, srcSelector, dstSelector, nil)
rules[rule.key] = rule
} else {
// Ports is present and contains at least one item, then this rule allows traffic
// only if the traffic matches at least one port in the ports list.
withNormalisedProtoAndPortLegacy(ingressRule.Ports, func(proto, port string) {
rule := newRuleSpec(&proto, srcSelector, dstSelector, &port)
rules[rule.key] = rule
})
}
}
}
}
return rules, nsSelectors, podSelectors, nil
}
func (ns *ns) analysePolicy(policy *networkingv1.NetworkPolicy) (
rules map[string]*ruleSpec,
nsSelectors, podSelectors map[string]*selectorSpec,
ipBlocks map[string]*ipBlockSpec,
err error) {
nsSelectors = make(map[string]*selectorSpec)
podSelectors = make(map[string]*selectorSpec)
ipBlocks = make(map[string]*ipBlockSpec)
rules = make(map[string]*ruleSpec)
policyTypes := make([]policyType, 0)
for _, pt := range policy.Spec.PolicyTypes {
if pt == networkingv1.PolicyTypeIngress {
policyTypes = append(policyTypes, policyTypeIngress)
}
if pt == networkingv1.PolicyTypeEgress {
policyTypes = append(policyTypes, policyTypeEgress)
}
}
// If empty, matches all pods in a namespace
dstSelector, err := newSelectorSpec(&policy.Spec.PodSelector, true, ns.name, ipset.HashIP)
targetSelector, err := newSelectorSpec(&policy.Spec.PodSelector, policyTypes, ns.name, ipset.HashIP)
if err != nil {
return nil, nil, nil, err
return nil, nil, nil, nil, err
}
// To prevent dstSelector being overwritten by a subsequent selector with
// To prevent targetSelector being overwritten by a subsequent selector with
// the same key, addIfNotExist MUST be used when adding to podSelectors.
// Otherwise, dstSelector' "dst" property might be lost resulting in
// an invalid content of the "default-allow" ipset.
addIfNotExist(dstSelector, podSelectors)
// Otherwise, policyTypes of the selector might be lost resulting in
// an invalid content in any "default-allow" ipset.
addIfNotExist(targetSelector, podSelectors)
// If ingress is empty then this NetworkPolicy does not allow any traffic
if policy.Spec.Ingress == nil || len(policy.Spec.Ingress) == 0 {
return
}
for _, ingressRule := range policy.Spec.Ingress {
// If Ports is empty or missing, this rule matches all ports
allPorts := ingressRule.Ports == nil || len(ingressRule.Ports) == 0
// If From is empty or missing, this rule matches all sources
allSources := ingressRule.From == nil || len(ingressRule.From) == 0
if allSources {
if allPorts {
rule := newRuleSpec(nil, nil, dstSelector, nil)
rules[rule.key] = rule
} else {
withNormalisedProtoAndPort(ingressRule.Ports, func(proto, port string) {
rule := newRuleSpec(&proto, nil, dstSelector, &port)
rules[rule.key] = rule
})
}
} else {
for _, peer := range ingressRule.From {
var srcSelector *selectorSpec
// NetworkPolicyPeer describes a peer to allow traffic from.
// Exactly one of its fields must be specified.
if peer.PodSelector != nil {
srcSelector, err = newSelectorSpec(peer.PodSelector, false, ns.name, ipset.HashIP)
if err != nil {
return nil, nil, nil, err
}
addIfNotExist(srcSelector, podSelectors)
} else if peer.NamespaceSelector != nil {
srcSelector, err = newSelectorSpec(peer.NamespaceSelector, false, "", ipset.ListSet)
if err != nil {
return nil, nil, nil, err
}
nsSelectors[srcSelector.key] = srcSelector
}
// If ingress is empty then this NetworkPolicy does not allow any ingress traffic
if policy.Spec.Ingress != nil && len(policy.Spec.Ingress) != 0 {
for _, ingressRule := range policy.Spec.Ingress {
// If Ports is empty or missing, this rule matches all ports
allPorts := ingressRule.Ports == nil || len(ingressRule.Ports) == 0
// If From is empty or missing, this rule matches all sources
allSources := ingressRule.From == nil || len(ingressRule.From) == 0
if allSources {
if allPorts {
rule := newRuleSpec(nil, srcSelector, dstSelector, nil)
rule := newRuleSpec(policyTypeIngress, nil, nil, targetSelector, nil)
rules[rule.key] = rule
} else {
withNormalisedProtoAndPort(ingressRule.Ports, func(proto, port string) {
rule := newRuleSpec(&proto, srcSelector, dstSelector, &port)
rule := newRuleSpec(policyTypeIngress, &proto, nil, targetSelector, &port)
rules[rule.key] = rule
})
}
} else {
for _, peer := range ingressRule.From {
var srcSelector *selectorSpec
// NetworkPolicyPeer describes a peer to allow traffic from.
// Exactly one of its fields must be specified.
if peer.PodSelector != nil {
srcSelector, err = newSelectorSpec(peer.PodSelector, nil, ns.name, ipset.HashIP)
if err != nil {
return nil, nil, nil, nil, err
}
addIfNotExist(srcSelector, podSelectors)
} else if peer.NamespaceSelector != nil {
srcSelector, err = newSelectorSpec(peer.NamespaceSelector, nil, "", ipset.ListSet)
if err != nil {
return nil, nil, nil, nil, err
}
nsSelectors[srcSelector.key] = srcSelector
}
if allPorts {
rule := newRuleSpec(policyTypeIngress, nil, srcSelector, targetSelector, nil)
rules[rule.key] = rule
} else {
withNormalisedProtoAndPort(ingressRule.Ports, func(proto, port string) {
rule := newRuleSpec(policyTypeIngress, &proto, srcSelector, targetSelector, &port)
rules[rule.key] = rule
})
}
}
}
}
}
return rules, nsSelectors, podSelectors, nil
// If egress is empty then this NetworkPolicy does not allow any egress traffic
if policy.Spec.Egress != nil && len(policy.Spec.Egress) != 0 {
for _, egressRule := range policy.Spec.Egress {
// If Ports is empty or missing, this rule matches all ports
allPorts := egressRule.Ports == nil || len(egressRule.Ports) == 0
// If To is empty or missing, this rule matches all destinations
allDestinations := egressRule.To == nil || len(egressRule.To) == 0
if allDestinations {
if allPorts {
rule := newRuleSpec(policyTypeEgress, nil, targetSelector, nil, nil)
rules[rule.key] = rule
} else {
withNormalisedProtoAndPort(egressRule.Ports, func(proto, port string) {
rule := newRuleSpec(policyTypeEgress, &proto, targetSelector, nil, &port)
rules[rule.key] = rule
})
}
} else {
for _, peer := range egressRule.To {
var dstSelector *selectorSpec
var dstRuleHost ruleHost
// NetworkPolicyPeer describes a peer to allow traffic to.
// Exactly one of its fields must be specified.
if peer.PodSelector != nil {
dstSelector, err = newSelectorSpec(peer.PodSelector, nil, ns.name, ipset.HashIP)
if err != nil {
return nil, nil, nil, nil, err
}
addIfNotExist(dstSelector, podSelectors)
dstRuleHost = dstSelector
} else if peer.NamespaceSelector != nil {
dstSelector, err = newSelectorSpec(peer.NamespaceSelector, nil, "", ipset.ListSet)
if err != nil {
return nil, nil, nil, nil, err
}
nsSelectors[dstSelector.key] = dstSelector
dstRuleHost = dstSelector
} else if peer.IPBlock != nil {
ipBlock := newIPBlockSpec(peer.IPBlock, ns.name)
ipBlocks[ipBlock.key] = ipBlock
dstRuleHost = ipBlock
}
if allPorts {
rule := newRuleSpec(policyTypeEgress, nil, targetSelector, dstRuleHost, nil)
rules[rule.key] = rule
} else {
withNormalisedProtoAndPort(egressRule.Ports, func(proto, port string) {
rule := newRuleSpec(policyTypeEgress, &proto, targetSelector, dstRuleHost, &port)
rules[rule.key] = rule
})
}
}
}
}
}
return rules, nsSelectors, podSelectors, ipBlocks, nil
}
func addIfNotExist(s *selectorSpec, ss map[string]*selectorSpec) {
@@ -179,12 +164,6 @@ func addIfNotExist(s *selectorSpec, ss map[string]*selectorSpec) {
}
}
func withNormalisedProtoAndPortLegacy(npps []extnapi.NetworkPolicyPort, f func(proto, port string)) {
for _, npp := range npps {
f(proto(npp.Protocol), port(npp.Port))
}
}
func withNormalisedProtoAndPort(npps []networkingv1.NetworkPolicyPort, f func(proto, port string)) {
for _, npp := range npps {
f(proto(npp.Protocol), port(npp.Port))
npc/constants.go
+6
View File
@@ -7,6 +7,12 @@ const (
DefaultChain = "WEAVE-NPC-DEFAULT"
IngressChain = "WEAVE-NPC-INGRESS"
EgressChain = "WEAVE-NPC-EGRESS"
EgressDefaultChain = "WEAVE-NPC-EGRESS-DEFAULT"
EgressCustomChain = "WEAVE-NPC-EGRESS-CUSTOM"
EgressMarkChain = "WEAVE-NPC-EGRESS-ACCEPT"
EgressMark = "0x40000/0x40000"
IpsetNamePrefix = "weave-"
LocalIpset = IpsetNamePrefix + "local-pods"
npc/controller.go
+3 -6
View File
@@ -37,19 +37,16 @@ type controller struct {
nss map[string]*ns // ns name -> ns struct
nsSelectors *selectorSet // selector string -> nsSelector
legacy bool // use legacy network policy semantics (k8s pre-1.7)
}
func New(nodeName string, legacy bool, ipt iptables.Interface, ips ipset.Interface) NetworkPolicyController {
func New(nodeName string, ipt iptables.Interface, ips ipset.Interface) NetworkPolicyController {
c := &controller{
nodeName: nodeName,
legacy: legacy,
ipt: ipt,
ips: ips,
nss: make(map[string]*ns)}
doNothing := func(*selector) error { return nil }
doNothing := func(*selector, policyType) error { return nil }
c.nsSelectors = newSelectorSet(ips, c.onNewNsSelector, doNothing, doNothing)
return c
@@ -71,7 +68,7 @@ func (npc *controller) onNewNsSelector(selector *selector) error {
func (npc *controller) withNS(name string, f func(ns *ns) error) error {
ns, found := npc.nss[name]
if !found {
newNs, err := newNS(name, npc.nodeName, npc.legacy, npc.ipt, npc.ips, npc.nsSelectors)
newNs, err := newNS(name, npc.nodeName, npc.ipt, npc.ips, npc.nsSelectors)
if err != nil {
return err
}
npc/controller_test.go
+182 -44
View File
@@ -1,7 +1,9 @@
package npc
import (
"fmt"
"log"
"strings"
"testing"
"github.com/pkg/errors"
@@ -103,18 +105,46 @@ func (i *mockIPSet) List(prefix string) ([]ipset.Name, error) {
}
type mockIPTables struct {
rules map[string]map[string]struct{} // chain -> rulespec -> struct{}
}
func newMockIPTables() *mockIPTables {
return &mockIPTables{rules: make(map[string]map[string]struct{})}
}
func (ipt *mockIPTables) Append(table, chain string, rulespec ...string) error {
if table != TableFilter {
return fmt.Errorf("invalid table: %q", table)
}
if _, found := ipt.rules[chain]; !found {
ipt.rules[chain] = make(map[string]struct{})
}
rule := strings.Join(rulespec, " ")
if _, found := ipt.rules[chain][rule]; found {
return fmt.Errorf("rule already exists. chain: %q, rule: %q", chain, rule)
}
ipt.rules[chain][rule] = struct{}{}
return nil
}
func (ipt *mockIPTables) Delete(table, chain string, rulespec ...string) error {
rule := strings.Join(rulespec, " ")
if _, found := ipt.rules[chain][rule]; !found {
return fmt.Errorf("rule does not exist. chain: %q rule: %q", chain, rule)
}
delete(ipt.rules[chain], rule)
return nil
}
func (ipt *mockIPTables) Insert(table, chain string, pos int, rulespec ...string) error {
return nil
return errors.New("Not Implemented")
}
func TestRegressionPolicyNamespaceOrdering3059(t *testing.T) {
@@ -144,6 +174,7 @@ func TestRegressionPolicyNamespaceOrdering3059(t *testing.T) {
Namespace: "destination",
},
Spec: networkingv1.NetworkPolicySpec{
PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress},
Ingress: []networkingv1.NetworkPolicyIngressRule{
{
From: []networkingv1.NetworkPolicyPeer{
@@ -167,7 +198,7 @@ func TestRegressionPolicyNamespaceOrdering3059(t *testing.T) {
// Namespaces first
m := newMockIPSet()
controller := New("foo", false, &mockIPTables{}, &m)
controller := New("foo", newMockIPTables(), &m)
const (
selectorIPSetName = "weave-I239Zp%sCvoVt*D6u=A!2]YEk"
@@ -183,7 +214,7 @@ func TestRegressionPolicyNamespaceOrdering3059(t *testing.T) {
// NetworkPolicy first
m = newMockIPSet()
controller = New("foo", false, &mockIPTables{}, &m)
controller = New("foo", newMockIPTables(), &m)
controller.AddNetworkPolicy(networkPolicy)
@@ -193,17 +224,18 @@ func TestRegressionPolicyNamespaceOrdering3059(t *testing.T) {
require.Equal(t, true, len(m.sets[selectorIPSetName].subSets[sourceIPSetName]) > 0)
}
// Tests default-allow ipset behavior when running in non-legacy mode.
// Tests default-allow ipset behavior
func TestDefaultAllow(t *testing.T) {
const (
defaultAllowIPSetName = "weave-E.1.0W^NGSp]0_t5WwH/]gX@L"
fooPodIP = "10.32.0.10"
barPodIP = "10.32.0.11"
barPodNewIP = "10.32.0.12"
ingressDefaultAllowIPSetName = "weave-;rGqyMIl1HN^cfDki~Z$3]6!N"
egressDefaultAllowIPSetName = "weave-s_+ChJId4Uy_$}G;WdH|~TK)I"
fooPodIP = "10.32.0.10"
barPodIP = "10.32.0.11"
barPodNewIP = "10.32.0.12"
)
m := newMockIPSet()
controller := New("bar", false, &mockIPTables{}, &m)
controller := New("bar", newMockIPTables(), &m)
defaultNamespace := &coreapi.Namespace{
ObjectMeta: metav1.ObjectMeta{
@@ -213,7 +245,8 @@ func TestDefaultAllow(t *testing.T) {
controller.AddNamespace(defaultNamespace)
// Should create an ipset for default-allow
require.Contains(t, m.sets, defaultAllowIPSetName)
require.Contains(t, m.sets, ingressDefaultAllowIPSetName)
require.Contains(t, m.sets, egressDefaultAllowIPSetName)
podFoo := &coreapi.Pod{
ObjectMeta: metav1.ObjectMeta{
@@ -225,7 +258,8 @@ func TestDefaultAllow(t *testing.T) {
controller.AddPod(podFoo)
// Should add the foo pod to default-allow
require.True(t, m.entryExists(defaultAllowIPSetName, fooPodIP))
require.True(t, m.entryExists(ingressDefaultAllowIPSetName, fooPodIP))
require.True(t, m.entryExists(egressDefaultAllowIPSetName, fooPodIP))
podBar := &coreapi.Pod{
ObjectMeta: metav1.ObjectMeta{
@@ -236,10 +270,12 @@ func TestDefaultAllow(t *testing.T) {
Status: coreapi.PodStatus{PodIP: barPodIP}}
podBarNoIP := &coreapi.Pod{ObjectMeta: podBar.ObjectMeta}
controller.AddPod(podBarNoIP)
controller.UpdatePod(podBarNoIP, podBar)
// Should add the bar pod to default-allow
require.True(t, m.entryExists(defaultAllowIPSetName, barPodIP))
require.True(t, m.entryExists(ingressDefaultAllowIPSetName, barPodIP))
require.True(t, m.entryExists(egressDefaultAllowIPSetName, barPodIP))
// Allow access from the bar pod to the foo pod
netpol := &networkingv1.NetworkPolicy{
@@ -248,6 +284,10 @@ func TestDefaultAllow(t *testing.T) {
Namespace: "default",
},
Spec: networkingv1.NetworkPolicySpec{
PolicyTypes: []networkingv1.PolicyType{
networkingv1.PolicyTypeIngress,
networkingv1.PolicyTypeEgress,
},
PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{"run": "foo"}},
Ingress: []networkingv1.NetworkPolicyIngressRule{{
From: []networkingv1.NetworkPolicyPeer{{
@@ -256,60 +296,74 @@ func TestDefaultAllow(t *testing.T) {
},
}},
}},
Egress: []networkingv1.NetworkPolicyEgressRule{{
To: []networkingv1.NetworkPolicyPeer{{
IPBlock: &networkingv1.IPBlock{CIDR: "192.168.48.0/24"},
}},
}},
},
}
controller.AddNetworkPolicy(netpol)
// Should remove the foo pod from default-allow as the netpol selects it
require.False(t, m.entryExists(defaultAllowIPSetName, fooPodIP))
require.True(t, m.entryExists(defaultAllowIPSetName, barPodIP))
require.False(t, m.entryExists(ingressDefaultAllowIPSetName, fooPodIP))
require.False(t, m.entryExists(egressDefaultAllowIPSetName, fooPodIP))
require.True(t, m.entryExists(ingressDefaultAllowIPSetName, barPodIP))
require.True(t, m.entryExists(egressDefaultAllowIPSetName, barPodIP))
podBarWithNewIP := *podBar
podBarWithNewIP.Status.PodIP = barPodNewIP
controller.UpdatePod(podBar, &podBarWithNewIP)
// Should update IP addr of the bar pod in default-allow
require.False(t, m.entryExists(defaultAllowIPSetName, barPodIP))
require.True(t, m.entryExists(defaultAllowIPSetName, barPodNewIP))
require.False(t, m.entryExists(ingressDefaultAllowIPSetName, barPodIP))
require.False(t, m.entryExists(egressDefaultAllowIPSetName, barPodIP))
require.True(t, m.entryExists(ingressDefaultAllowIPSetName, barPodNewIP))
require.True(t, m.entryExists(egressDefaultAllowIPSetName, barPodNewIP))
controller.UpdatePod(&podBarWithNewIP, podBarNoIP)
// Should remove the bar pod from default-allow as it does not have any IP addr
require.False(t, m.entryExists(defaultAllowIPSetName, barPodNewIP))
require.False(t, m.entryExists(ingressDefaultAllowIPSetName, barPodNewIP))
require.False(t, m.entryExists(egressDefaultAllowIPSetName, barPodNewIP))
podFooWithNewLabel := *podFoo
podFooWithNewLabel.ObjectMeta.Labels = map[string]string{"run": "new-foo"}
controller.UpdatePod(podFoo, &podFooWithNewLabel)
// Should bring back the foo pod to default-allow as it does not match dst of any netpol
require.True(t, m.entryExists(defaultAllowIPSetName, fooPodIP))
require.True(t, m.entryExists(ingressDefaultAllowIPSetName, fooPodIP))
require.True(t, m.entryExists(egressDefaultAllowIPSetName, fooPodIP))
controller.UpdatePod(&podFooWithNewLabel, podFoo)
// Should remove from default-allow as it matches the netpol after the update
require.False(t, m.entryExists(defaultAllowIPSetName, fooPodIP))
require.False(t, m.entryExists(ingressDefaultAllowIPSetName, fooPodIP))
require.False(t, m.entryExists(egressDefaultAllowIPSetName, fooPodIP))
controller.DeleteNetworkPolicy(netpol)
// Should bring back the foo pod to default-allow as no netpol selects it
require.True(t, m.entryExists(defaultAllowIPSetName, fooPodIP))
require.True(t, m.entryExists(ingressDefaultAllowIPSetName, fooPodIP))
require.True(t, m.entryExists(egressDefaultAllowIPSetName, fooPodIP))
controller.DeletePod(podFoo)
// Should remove foo pod from default-allow
require.False(t, m.entryExists(defaultAllowIPSetName, fooPodIP))
require.False(t, m.entryExists(ingressDefaultAllowIPSetName, fooPodIP))
require.False(t, m.entryExists(egressDefaultAllowIPSetName, fooPodIP))
controller.DeleteNamespace(defaultNamespace)
// Should remove default ipset
require.NotContains(t, m.sets, defaultAllowIPSetName)
require.NotContains(t, m.sets, ingressDefaultAllowIPSetName)
require.NotContains(t, m.sets, egressDefaultAllowIPSetName)
}
func TestOutOfOrderPodEvents(t *testing.T) {
const (
defaultAllowIPSetName = "weave-E.1.0W^NGSp]0_t5WwH/]gX@L"
runBarIPSetName = "weave-bZ~x=yBgzH)Ht()K*Uv3z{M]Y"
podIP = "10.32.0.10"
ingressDefaultAllowIPSetName = "weave-;rGqyMIl1HN^cfDki~Z$3]6!N"
runBarIPSetName = "weave-bZ~x=yBgzH)Ht()K*Uv3z{M]Y"
podIP = "10.32.0.10"
)
m := newMockIPSet()
controller := New("qux", false, &mockIPTables{}, &m)
controller := New("qux", newMockIPTables(), &m)
defaultNamespace := &coreapi.Namespace{
ObjectMeta: metav1.ObjectMeta{
@@ -328,7 +382,7 @@ func TestOutOfOrderPodEvents(t *testing.T) {
controller.AddPod(podFoo)
// Should be in default-allow as no netpol selects podFoo
require.True(t, m.entryExists(defaultAllowIPSetName, podIP))
require.True(t, m.entryExists(ingressDefaultAllowIPSetName, podIP))
netpol := &networkingv1.NetworkPolicy{
ObjectMeta: metav1.ObjectMeta{
@@ -336,6 +390,7 @@ func TestOutOfOrderPodEvents(t *testing.T) {
Namespace: "default",
},
Spec: networkingv1.NetworkPolicySpec{
PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress},
PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{"run": "foo"}},
Ingress: []networkingv1.NetworkPolicyIngressRule{{
From: []networkingv1.NetworkPolicyPeer{{
@@ -349,7 +404,7 @@ func TestOutOfOrderPodEvents(t *testing.T) {
controller.AddNetworkPolicy(netpol)
// Shouldn't be in default-allow as netpol above selects podFoo
require.False(t, m.entryExists(defaultAllowIPSetName, podIP))
require.False(t, m.entryExists(ingressDefaultAllowIPSetName, podIP))
podBar := &coreapi.Pod{
ObjectMeta: metav1.ObjectMeta{
@@ -361,8 +416,8 @@ func TestOutOfOrderPodEvents(t *testing.T) {
controller.AddPod(podBar)
// Should be in default-allow as no netpol selects podBar
require.True(t, m.entryExists(defaultAllowIPSetName, podIP))
require.True(t, m.Exist(podBar.ObjectMeta.UID, defaultAllowIPSetName, podIP))
require.True(t, m.entryExists(ingressDefaultAllowIPSetName, podIP))
require.True(t, m.Exist(podBar.ObjectMeta.UID, ingressDefaultAllowIPSetName, podIP))
// Should be in run=bar ipset
require.True(t, m.entryExists(runBarIPSetName, podIP))
@@ -373,24 +428,24 @@ func TestOutOfOrderPodEvents(t *testing.T) {
// Should be in default-allow as no netpol selects podBar and podFoo removal
// should not affect podBar in default-allow
require.True(t, m.entryExists(defaultAllowIPSetName, podIP))
require.True(t, m.entryExists(ingressDefaultAllowIPSetName, podIP))
controller.DeletePod(podBar)
// Should remove from default-allow and run=bar ipsets
require.Equal(t, 0, len(m.sets[defaultAllowIPSetName].subSets))
require.Equal(t, 0, len(m.sets[ingressDefaultAllowIPSetName].subSets))
require.False(t, m.entryExists(runBarIPSetName, podIP))
}
// Test case for https://github.com/weaveworks/weave/issues/3222
func TestNewDstSelector(t *testing.T) {
func TestNewTargetSelector(t *testing.T) {
const (
defaultAllowIPSetName = "weave-E.1.0W^NGSp]0_t5WwH/]gX@L"
podIP = "10.32.0.10"
ingressDefaultAllowIPSetName = "weave-;rGqyMIl1HN^cfDki~Z$3]6!N"
podIP = "10.32.0.10"
)
m := newMockIPSet()
controller := New("baz", false, &mockIPTables{}, &m)
controller := New("baz", newMockIPTables(), &m)
defaultNamespace := &coreapi.Namespace{
ObjectMeta: metav1.ObjectMeta{
@@ -408,7 +463,7 @@ func TestNewDstSelector(t *testing.T) {
Status: coreapi.PodStatus{PodIP: podIP}}
controller.AddPod(podFoo)
require.True(t, m.entryExists(defaultAllowIPSetName, podIP))
require.True(t, m.entryExists(ingressDefaultAllowIPSetName, podIP))
netpolBar := &networkingv1.NetworkPolicy{
ObjectMeta: metav1.ObjectMeta{
@@ -417,6 +472,7 @@ func TestNewDstSelector(t *testing.T) {
Namespace: "default",
},
Spec: networkingv1.NetworkPolicySpec{
PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress},
PodSelector: metav1.LabelSelector{},
Ingress: []networkingv1.NetworkPolicyIngressRule{{
From: []networkingv1.NetworkPolicyPeer{{
@@ -427,8 +483,8 @@ func TestNewDstSelector(t *testing.T) {
}
controller.AddNetworkPolicy(netpolBar)
// netpolBar dst selector selects podFoo
require.False(t, m.entryExists(defaultAllowIPSetName, podIP))
// netpolBar target selector selects podFoo
require.False(t, m.entryExists(ingressDefaultAllowIPSetName, podIP))
netpolFoo := &networkingv1.NetworkPolicy{
ObjectMeta: metav1.ObjectMeta{
@@ -437,6 +493,7 @@ func TestNewDstSelector(t *testing.T) {
Namespace: "default",
},
Spec: networkingv1.NetworkPolicySpec{
PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress},
PodSelector: metav1.LabelSelector{},
Ingress: []networkingv1.NetworkPolicyIngressRule{{
From: []networkingv1.NetworkPolicyPeer{{
@@ -448,9 +505,90 @@ func TestNewDstSelector(t *testing.T) {
controller.AddNetworkPolicy(netpolFoo)
controller.DeleteNetworkPolicy(netpolBar)
// netpolFoo dst-selects podFoo
require.False(t, m.entryExists(defaultAllowIPSetName, podIP))
// netpolFoo target-selects podFoo
require.False(t, m.entryExists(ingressDefaultAllowIPSetName, podIP))
controller.DeleteNetworkPolicy(netpolFoo)
// No netpol dst-selects podFoo
require.True(t, m.entryExists(defaultAllowIPSetName, podIP))
// No netpol target-selects podFoo
require.True(t, m.entryExists(ingressDefaultAllowIPSetName, podIP))
}
func TestEgressPolicyWithIPBlock(t *testing.T) {
const (
fooPodIP = "10.32.0.10"
exceptIPSetName = "weave-j:W$5om!$8JS})buYAD7q^#sX"
exceptIPSetNameInNonDefault = "weave-|*P{aL2C@#N/1IZ!IvQItv_pt"
)
m := newMockIPSet()
ipt := newMockIPTables()
controller := New("foo", ipt, &m)
netpolFoo := &networkingv1.NetworkPolicy{
ObjectMeta: metav1.ObjectMeta{
UID: "netpol-foo",
Name: "allow-from-bar-to-foo",
Namespace: "default",
},
Spec: networkingv1.NetworkPolicySpec{
PolicyTypes: []networkingv1.PolicyType{
networkingv1.PolicyTypeEgress,
},
PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{"run": "bar"}},
Egress: []networkingv1.NetworkPolicyEgressRule{{
To: []networkingv1.NetworkPolicyPeer{{
IPBlock: &networkingv1.IPBlock{
CIDR: "192.168.48.0/24",
Except: []string{
"192.168.48.1/32",
"192.168.48.2/32",
},
},
}},
}},
},
}
err := controller.AddNetworkPolicy(netpolFoo)
require.NoError(t, err)
require.Equal(t, 2, len(m.sets[exceptIPSetName].subSets))
require.True(t, m.entryExists(exceptIPSetName, "192.168.48.1/32"))
require.True(t, m.entryExists(exceptIPSetName, "192.168.48.2/32"))
// Each egress rule is represented as two iptables rules (-J MARK and -J RETURN).
require.Equal(t, 2, len(ipt.rules[EgressCustomChain]))
for rule := range ipt.rules[EgressCustomChain] {
require.Contains(t, rule, "-d 192.168.48.0/24 -m set ! --match-set "+exceptIPSetName+" dst")
}
// Check that we create a new ipset for the ipBlock bellow. An ipset with the
// same content already exists (created by netpolFoo), but we need to create
// a new one, as netpolBar is in a different namespace and weave-npc does
// object refcounting only within namespace boundaries.
netpolBar := &networkingv1.NetworkPolicy{
ObjectMeta: metav1.ObjectMeta{
UID: "netpol-foo-bar",
Name: "allow-from-bar-to-foo-in-non-default",
Namespace: "non-default",
},
Spec: networkingv1.NetworkPolicySpec{
PolicyTypes: []networkingv1.PolicyType{
networkingv1.PolicyTypeEgress,
},
PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{"run": "bar"}},
Egress: []networkingv1.NetworkPolicyEgressRule{{
To: []networkingv1.NetworkPolicyPeer{{
IPBlock: &networkingv1.IPBlock{
CIDR: "192.168.48.0/24",
Except: []string{
"192.168.48.1/32",
"192.168.48.2/32",
},
},
}},
}},
},
}
err = controller.AddNetworkPolicy(netpolBar)
require.NoError(t, err)
require.Equal(t, 2, len(m.sets[exceptIPSetNameInNonDefault].subSets))
}
npc/ipblock.go
+116
View File
@@ -0,0 +1,116 @@
package npc
import (
"fmt"
"sort"
"strings"
networkingv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/types"
"github.com/weaveworks/weave/common"
"github.com/weaveworks/weave/net/ipset"
)
type ipBlockSpec struct {
key string
ipsetName ipset.Name // ipset for storing excepted CIDRs
ipBlock *networkingv1.IPBlock
nsName string // Namespace name
}
func newIPBlockSpec(ipb *networkingv1.IPBlock, nsName string) *ipBlockSpec {
spec := &ipBlockSpec{ipBlock: ipb, nsName: nsName}
if len(ipb.Except) > 0 {
sort.Strings(ipb.Except)
spec.key = strings.Join(ipb.Except, " ")
spec.ipsetName = ipset.Name(IpsetNamePrefix + shortName(nsName+":"+"ipblock-except:"+spec.key))
}
return spec
}
func (spec *ipBlockSpec) getRuleSpec(src bool) ([]string, string) {
dir, dirOpt := "dst", "-d"
if src {
dir, dirOpt = "src", "-s"
}
if spec.key == "" {
rule := []string{dirOpt, spec.ipBlock.CIDR}
comment := fmt.Sprintf("cidr: %s", spec.ipBlock.CIDR)
return rule, comment
}
rule := []string{dirOpt, spec.ipBlock.CIDR,
"-m", "set", "!", "--match-set", string(spec.ipsetName), dir}
comment := fmt.Sprintf("cidr: %s except [%s]", spec.ipBlock.CIDR, spec.key)
return rule, comment
}
type ipBlockSet struct {
ips ipset.Interface
users map[string]map[types.UID]struct{}
}
func newIPBlockSet(ips ipset.Interface) *ipBlockSet {
return &ipBlockSet{
ips: ips,
users: make(map[string]map[types.UID]struct{}),
}
}
func (s *ipBlockSet) deprovision(user types.UID, current, desired map[string]*ipBlockSpec) error {
for key, spec := range current {
if key == "" {
continue
}
if _, found := desired[key]; !found {
delete(s.users[key], user)
if len(s.users[key]) == 0 {
common.Log.Infof("destroying ipset: %#v", spec)
if err := s.ips.Destroy(spec.ipsetName); err != nil {
return err
}
delete(s.users, key)
}
}
}
return nil
}
func (s *ipBlockSet) provision(user types.UID, current, desired map[string]*ipBlockSpec) (err error) {
for key, spec := range desired {
if key == "" {
// No need to provision an ipBlock with empty list of excepted CIDRs
// (i.e. no ipset is needed for the related iptables rule).
continue
}
if _, found := current[key]; !found {
if _, found := s.users[key]; !found {
common.Log.Infof("creating ipset: %#v", spec)
if err := s.ips.Create(spec.ipsetName, ipset.HashNet); err != nil {
return err
}
// TODO(brb) Pass comment to ips.Create instead.
comment := "excepted from " + spec.ipBlock.CIDR
for _, cidr := range spec.ipBlock.Except {
if err = s.ips.AddEntry(user, spec.ipsetName, cidr, comment); err != nil {
return err
}
}
s.users[key] = make(map[types.UID]struct{})
}
s.users[key][user] = struct{}{}
}
}
return nil
}
npc/namespace.go
+180 -160
View File
@@ -1,7 +1,6 @@
package npc
import (
"encoding/json"
"errors"
"fmt"
@@ -32,20 +31,20 @@ type ns struct {
uid types.UID // surrogate UID to own allPods selector
allPods *selectorSpec // hash:ip ipset of all pod IPs in this namespace
// stores IP addrs of pods which are not selected by any dst podSelector of
// any netpol; used only in non-legacy mode and is used as a dst in
// the WEAVE-NPC-DEFAULT iptables chain.
defaultAllowIPSet ipset.Name
// stores IP addrs of pods which are not selected by any target podSelector of
// any netpol; used as a target in the WEAVE-NPC-{INGRESS,EGRESS}-DEFAULT
// iptables chains.
ingressDefaultAllowIPSet ipset.Name
egressDefaultAllowIPSet ipset.Name
nsSelectors *selectorSet
podSelectors *selectorSet
ipBlocks *ipBlockSet
rules *ruleSet
legacy bool
}
func newNS(name, nodeName string, legacy bool, ipt iptables.Interface, ips ipset.Interface, nsSelectors *selectorSet) (*ns, error) {
allPods, err := newSelectorSpec(&metav1.LabelSelector{}, false, name, ipset.HashIP)
func newNS(name, nodeName string, ipt iptables.Interface, ips ipset.Interface, nsSelectors *selectorSet) (*ns, error) {
allPods, err := newSelectorSpec(&metav1.LabelSelector{}, nil, name, ipset.HashIP)
if err != nil {
return nil, err
}
@@ -60,19 +59,24 @@ func newNS(name, nodeName string, legacy bool, ipt iptables.Interface, ips ipset
uid: uuid.NewUUID(),
allPods: allPods,
nsSelectors: nsSelectors,
ipBlocks: newIPBlockSet(ips),
rules: newRuleSet(ipt),
legacy: legacy}
ns.podSelectors = newSelectorSet(ips, ns.onNewPodSelector, ns.onNewDstPodSelector, ns.onDestroyDstPodSelector)
if !legacy {
defaultAllowIPSet := ipset.Name(IpsetNamePrefix + shortName("default-allow:"+name))
if err := ips.Create(defaultAllowIPSet, ipset.HashIP); err != nil {
return nil, err
}
ns.defaultAllowIPSet = defaultAllowIPSet
}
ns.podSelectors = newSelectorSet(ips, ns.onNewPodSelector, ns.onNewTargetPodSelector, ns.onDestroyTargetPodSelector)
ingressDefaultAllowIPSet := ipset.Name(IpsetNamePrefix + shortName("ingress-default-allow:"+name))
if err := ips.Create(ingressDefaultAllowIPSet, ipset.HashIP); err != nil {
return nil, err
}
ns.ingressDefaultAllowIPSet = ingressDefaultAllowIPSet
egressDefaultAllowIPSet := ipset.Name(IpsetNamePrefix + shortName("egress-default-allow:"+name))
if err := ips.Create(egressDefaultAllowIPSet, ipset.HashIP); err != nil {
return nil, err
}
ns.egressDefaultAllowIPSet = egressDefaultAllowIPSet
if err := ns.podSelectors.provision(ns.uid, nil, map[string]*selectorSpec{ns.allPods.key: ns.allPods}); err != nil {
return nil, err
}
@@ -102,16 +106,13 @@ func (ns *ns) onNewPodSelector(selector *selector) error {
return nil
}
func (ns *ns) onNewDstPodSelector(selector *selector) error {
if ns.legacy {
return nil
}
func (ns *ns) onNewTargetPodSelector(selector *selector, policyType policyType) error {
for _, pod := range ns.pods {
if hasIP(pod) {
// Remove the pod from default-allow if dst podselector matches the pod
ipset := ns.defaultAllowIPSetName(policyType)
if selector.matches(pod.ObjectMeta.Labels) {
if err := ns.ips.DelEntry(pod.ObjectMeta.UID, ns.defaultAllowIPSet, pod.Status.PodIP); err != nil {
if err := ns.ips.DelEntry(pod.ObjectMeta.UID, ipset, pod.Status.PodIP); err != nil {
return err
}
}
@@ -121,15 +122,11 @@ func (ns *ns) onNewDstPodSelector(selector *selector) error {
return nil
}
func (ns *ns) onDestroyDstPodSelector(selector *selector) error {
if ns.legacy {
return nil
}
func (ns *ns) onDestroyTargetPodSelector(selector *selector, policyType policyType) error {
for _, pod := range ns.pods {
if hasIP(pod) {
if selector.matches(pod.ObjectMeta.Labels) {
if err := ns.addToDefaultAllowIfNoMatching(pod); err != nil {
if err := ns.addToDefaultAllowIfNoMatching(pod, policyType); err != nil {
return err
}
}
@@ -139,18 +136,19 @@ func (ns *ns) onDestroyDstPodSelector(selector *selector) error {
return nil
}
// Add pod IP addr to default-allow ipset if there are no matching dst selectors
func (ns *ns) addToDefaultAllowIfNoMatching(pod *coreapi.Pod) error {
// Add pod IP addr to default-allow ipset if there are no matching target selectors
func (ns *ns) addToDefaultAllowIfNoMatching(pod *coreapi.Pod, policyType policyType) error {
found := false
// TODO(mp) optimize (avoid iterating over selectors) by ref counting IP addrs.
for _, s := range ns.podSelectors.entries {
if ns.podSelectors.dstSelectorExist(s) && s.matches(pod.ObjectMeta.Labels) {
if ns.podSelectors.targetSelectorExist(s, policyType) && s.matches(pod.ObjectMeta.Labels) {
found = true
break
}
}
if !found {
if err := ns.ips.AddEntry(pod.ObjectMeta.UID, ns.defaultAllowIPSet, pod.Status.PodIP, podComment(pod)); err != nil {
ipset := ns.defaultAllowIPSetName(policyType)
if err := ns.ips.AddEntry(pod.ObjectMeta.UID, ipset, pod.Status.PodIP, podComment(pod)); err != nil {
return err
}
}
@@ -175,13 +173,18 @@ func (ns *ns) addPod(obj *coreapi.Pod) error {
ns.ips.AddEntry(obj.ObjectMeta.UID, LocalIpset, obj.Status.PodIP, podComment(obj))
}
found, err := ns.podSelectors.addToMatching(obj.ObjectMeta.UID, obj.ObjectMeta.Labels, obj.Status.PodIP, podComment(obj))
foundIngress, foundEgress, err := ns.podSelectors.addToMatching(obj.ObjectMeta.UID, obj.ObjectMeta.Labels, obj.Status.PodIP, podComment(obj))
if err != nil {
return err
}
// If there are no matching dst selectors, add the pod to default-allow
if !ns.legacy && !found {
if err := ns.ips.AddEntry(obj.ObjectMeta.UID, ns.defaultAllowIPSet, obj.Status.PodIP, podComment(obj)); err != nil {
// If there are no matching target selectors, add the pod to default-allow
if !foundIngress {
if err := ns.ips.AddEntry(obj.ObjectMeta.UID, ns.ingressDefaultAllowIPSet, obj.Status.PodIP, podComment(obj)); err != nil {
return err
}
}
if !foundEgress {
if err := ns.ips.AddEntry(obj.ObjectMeta.UID, ns.egressDefaultAllowIPSet, obj.Status.PodIP, podComment(obj)); err != nil {
return err
}
}
@@ -202,10 +205,11 @@ func (ns *ns) updatePod(oldObj, newObj *coreapi.Pod) error {
ns.ips.DelEntry(oldObj.ObjectMeta.UID, LocalIpset, oldObj.Status.PodIP)
}
if !ns.legacy {
if err := ns.ips.DelEntry(oldObj.ObjectMeta.UID, ns.defaultAllowIPSet, oldObj.Status.PodIP); err != nil {
return err
}
if err := ns.ips.DelEntry(oldObj.ObjectMeta.UID, ns.ingressDefaultAllowIPSet, oldObj.Status.PodIP); err != nil {
return err
}
if err := ns.ips.DelEntry(oldObj.ObjectMeta.UID, ns.egressDefaultAllowIPSet, oldObj.Status.PodIP); err != nil {
return err
}
return ns.podSelectors.delFromMatching(oldObj.ObjectMeta.UID, oldObj.ObjectMeta.Labels, oldObj.Status.PodIP)
@@ -215,13 +219,18 @@ func (ns *ns) updatePod(oldObj, newObj *coreapi.Pod) error {
if ns.checkLocalPod(newObj) {
ns.ips.AddEntry(newObj.ObjectMeta.UID, LocalIpset, newObj.Status.PodIP, podComment(newObj))
}
found, err := ns.podSelectors.addToMatching(newObj.ObjectMeta.UID, newObj.ObjectMeta.Labels, newObj.Status.PodIP, podComment(newObj))
foundIngress, foundEgress, err := ns.podSelectors.addToMatching(newObj.ObjectMeta.UID, newObj.ObjectMeta.Labels, newObj.Status.PodIP, podComment(newObj))
if err != nil {
return err
}
if !ns.legacy && !found {
if err := ns.ips.AddEntry(newObj.ObjectMeta.UID, ns.defaultAllowIPSet, newObj.Status.PodIP, podComment(newObj)); err != nil {
if !foundIngress {
if err := ns.ips.AddEntry(newObj.ObjectMeta.UID, ns.ingressDefaultAllowIPSet, newObj.Status.PodIP, podComment(newObj)); err != nil {
return err
}
}
if !foundEgress {
if err := ns.ips.AddEntry(newObj.ObjectMeta.UID, ns.egressDefaultAllowIPSet, newObj.Status.PodIP, podComment(newObj)); err != nil {
return err
}
}
@@ -229,17 +238,11 @@ func (ns *ns) updatePod(oldObj, newObj *coreapi.Pod) error {
return nil
}
if !ns.legacy &&
oldObj.Status.PodIP != newObj.Status.PodIP &&
ns.ips.Exist(oldObj.ObjectMeta.UID, ns.defaultAllowIPSet, oldObj.Status.PodIP) {
// Instead of iterating over all selectors we check whether old pod IP
// has been inserted into default-allow ipset to decide whether the IP
// in the ipset has to be updated.
if err := ns.ips.DelEntry(oldObj.ObjectMeta.UID, ns.defaultAllowIPSet, oldObj.Status.PodIP); err != nil {
if oldObj.Status.PodIP != newObj.Status.PodIP {
if err := ns.updateDefaultAllowIPSetEntry(oldObj, newObj, ns.ingressDefaultAllowIPSet); err != nil {
return err
}
if err := ns.ips.AddEntry(newObj.ObjectMeta.UID, ns.defaultAllowIPSet, newObj.Status.PodIP, podComment(newObj)); err != nil {
if err := ns.updateDefaultAllowIPSetEntry(oldObj, newObj, ns.egressDefaultAllowIPSet); err != nil {
return err
}
}
@@ -264,17 +267,11 @@ func (ns *ns) updatePod(oldObj, newObj *coreapi.Pod) error {
}
}
if !ns.legacy && ns.podSelectors.dstSelectorExist(ps) {
switch {
case !oldMatch && newMatch:
if err := ns.ips.DelEntry(oldObj.ObjectMeta.UID, ns.defaultAllowIPSet, oldObj.Status.PodIP); err != nil {
return err
}
case oldMatch && !newMatch:
if err := ns.addToDefaultAllowIfNoMatching(newObj); err != nil {
return err
}
}
if err := ns.addOrRemoveToDefaultAllowIPSet(ps, oldObj, newObj, oldMatch, newMatch, policyTypeIngress); err != nil {
return err
}
if err := ns.addOrRemoveToDefaultAllowIPSet(ps, oldObj, newObj, oldMatch, newMatch, policyTypeEgress); err != nil {
return err
}
}
}
@@ -282,6 +279,23 @@ func (ns *ns) updatePod(oldObj, newObj *coreapi.Pod) error {
return nil
}
func (ns *ns) addOrRemoveToDefaultAllowIPSet(ps *selector, oldObj, newObj *coreapi.Pod, oldMatch, newMatch bool, policyType policyType) error {
ipset := ns.defaultAllowIPSetName(policyType)
if ns.podSelectors.targetSelectorExist(ps, policyType) {
switch {
case !oldMatch && newMatch:
if err := ns.ips.DelEntry(oldObj.ObjectMeta.UID, ipset, oldObj.Status.PodIP); err != nil {
return err
}
case oldMatch && !newMatch:
if err := ns.addToDefaultAllowIfNoMatching(newObj, policyType); err != nil {
return err
}
}
}
return nil
}
func (ns *ns) deletePod(obj *coreapi.Pod) error {
delete(ns.pods, obj.ObjectMeta.UID)
@@ -293,10 +307,11 @@ func (ns *ns) deletePod(obj *coreapi.Pod) error {
ns.ips.DelEntry(obj.ObjectMeta.UID, LocalIpset, obj.Status.PodIP)
}
if !ns.legacy {
if err := ns.ips.DelEntry(obj.ObjectMeta.UID, ns.defaultAllowIPSet, obj.Status.PodIP); err != nil {
return err
}
if err := ns.ips.DelEntry(obj.ObjectMeta.UID, ns.ingressDefaultAllowIPSet, obj.Status.PodIP); err != nil {
return err
}
if err := ns.ips.DelEntry(obj.ObjectMeta.UID, ns.egressDefaultAllowIPSet, obj.Status.PodIP); err != nil {
return err
}
return ns.podSelectors.delFromMatching(obj.ObjectMeta.UID, obj.ObjectMeta.Labels, obj.Status.PodIP)
@@ -305,7 +320,7 @@ func (ns *ns) deletePod(obj *coreapi.Pod) error {
func (ns *ns) addNetworkPolicy(obj interface{}) error {
// Analyse policy, determine which rules and ipsets are required
uid, rules, nsSelectors, podSelectors, err := ns.analyse(obj)
uid, rules, nsSelectors, podSelectors, ipBlocks, err := ns.analyse(obj)
if err != nil {
return err
}
@@ -317,16 +332,19 @@ func (ns *ns) addNetworkPolicy(obj interface{}) error {
if err := ns.podSelectors.provision(uid, nil, podSelectors); err != nil {
return err
}
if err := ns.ipBlocks.provision(uid, nil, ipBlocks); err != nil {
return err
}
return ns.rules.provision(uid, nil, rules)
}
func (ns *ns) updateNetworkPolicy(oldObj, newObj interface{}) error {
// Analyse the old and the new policy so we can determine differences
oldUID, oldRules, oldNsSelectors, oldPodSelectors, err := ns.analyse(oldObj)
oldUID, oldRules, oldNsSelectors, oldPodSelectors, oldIPBlocks, err := ns.analyse(oldObj)
if err != nil {
return err
}
newUID, newRules, newNsSelectors, newPodSelectors, err := ns.analyse(newObj)
newUID, newRules, newNsSelectors, newPodSelectors, newIPBlocks, err := ns.analyse(newObj)
if err != nil {
return err
}
@@ -344,18 +362,24 @@ func (ns *ns) updateNetworkPolicy(oldObj, newObj interface{}) error {
if err := ns.podSelectors.deprovision(oldUID, oldPodSelectors, newPodSelectors); err != nil {
return err
}
if err := ns.ipBlocks.deprovision(oldUID, oldIPBlocks, newIPBlocks); err != nil {
return err
}
if err := ns.nsSelectors.provision(oldUID, oldNsSelectors, newNsSelectors); err != nil {
return err
}
if err := ns.podSelectors.provision(oldUID, oldPodSelectors, newPodSelectors); err != nil {
return err
}
if err := ns.ipBlocks.provision(oldUID, oldIPBlocks, newIPBlocks); err != nil {
return err
}
return ns.rules.provision(oldUID, oldRules, newRules)
}
func (ns *ns) deleteNetworkPolicy(obj interface{}) error {
// Analyse network policy to free resources
uid, rules, nsSelectors, podSelectors, err := ns.analyse(obj)
uid, rules, nsSelectors, podSelectors, ipBlocks, err := ns.analyse(obj)
if err != nil {
return err
}
@@ -369,73 +393,89 @@ func (ns *ns) deleteNetworkPolicy(obj interface{}) error {
if err := ns.nsSelectors.deprovision(uid, nsSelectors, nil); err != nil {
return err
}
return ns.podSelectors.deprovision(uid, podSelectors, nil)
}
func bypassRule(nsIpsetName ipset.Name, namespace string) []string {
return []string{"-m", "set", "--match-set", string(nsIpsetName), "dst", "-j", "ACCEPT", "-m", "comment", "--comment", "DefaultAllow isolation for namespace: " + namespace}
}
func (ns *ns) ensureBypassRule() error {
var ipset ipset.Name
if ns.legacy {
ipset = ns.allPods.ipsetName
} else {
ipset = ns.defaultAllowIPSet
if err := ns.podSelectors.deprovision(uid, podSelectors, nil); err != nil {
return err
}
if err := ns.ipBlocks.deprovision(uid, ipBlocks, nil); err != nil {
return err
}
common.Log.Debugf("ensuring rule for DefaultAllow in namespace: %s, set %s", ns.name, ipset)
return ns.ipt.Append(TableFilter, DefaultChain, bypassRule(ipset, ns.name)...)
return nil
}
func (ns *ns) deleteBypassRule() error {
var ipset ipset.Name
if ns.legacy {
ipset = ns.allPods.ipsetName
} else {
ipset = ns.defaultAllowIPSet
func (ns *ns) updateDefaultAllowIPSetEntry(oldObj, newObj *coreapi.Pod, ipsetName ipset.Name) error {
// Instead of iterating over all selectors we check whether old pod IP
// has been inserted into default-allow ipset to decide whether the IP
// in the ipset has to be updated.
if ns.ips.Exist(oldObj.ObjectMeta.UID, ipsetName, oldObj.Status.PodIP) {
if err := ns.ips.DelEntry(oldObj.ObjectMeta.UID, ipsetName, oldObj.Status.PodIP); err != nil {
return err
}
if err := ns.ips.AddEntry(newObj.ObjectMeta.UID, ipsetName, newObj.Status.PodIP, podComment(newObj)); err != nil {
return err
}
}
return nil
}
func bypassRules(namespace string, ingress, egress ipset.Name) map[string][][]string {
return map[string][][]string{
DefaultChain: {
{"-m", "set", "--match-set", string(ingress), "dst", "-j", "ACCEPT",
"-m", "comment", "--comment", "DefaultAllow ingress isolation for namespace: " + namespace},
},
EgressDefaultChain: {
{"-m", "set", "--match-set", string(egress), "src", "-j", EgressMarkChain,
"-m", "comment", "--comment", "DefaultAllow egress isolation for namespace: " + namespace},
{"-m", "set", "--match-set", string(egress), "src", "-j", "RETURN",
"-m", "comment", "--comment", "DefaultAllow egress isolation for namespace: " + namespace},
},
}
}
func (ns *ns) ensureBypassRules() error {
for chain, rules := range bypassRules(ns.name, ns.ingressDefaultAllowIPSet, ns.egressDefaultAllowIPSet) {
for _, rule := range rules {
common.Log.Debugf("adding rule for DefaultAllow in namespace: %s, chain: %s, %s", ns.name, chain, rule)
if err := ns.ipt.Append(TableFilter, chain, rule...); err != nil {
return err
}
}
}
common.Log.Debugf("removing default rule in namespace: %s, set %s", ns.name, ipset)
return ns.ipt.Delete(TableFilter, DefaultChain, bypassRule(ipset, ns.name)...)
return nil
}
func (ns *ns) deleteBypassRules() error {
for chain, rules := range bypassRules(ns.name, ns.ingressDefaultAllowIPSet, ns.egressDefaultAllowIPSet) {
for _, rule := range rules {
common.Log.Debugf("removing rule for DefaultAllow in namespace: %s, chain: %s, %s", ns.name, chain, rule)
if err := ns.ipt.Delete(TableFilter, chain, rule...); err != nil {
return err
}
}
}
return nil
}
func (ns *ns) addNamespace(obj *coreapi.Namespace) error {
ns.namespace = obj
// Insert a rule to bypass policies if namespace is DefaultAllow
if !ns.isDefaultDeny(obj) {
if err := ns.ensureBypassRule(); err != nil {
return err
}
// Insert a rule to bypass policies
if err := ns.ensureBypassRules(); err != nil {
return err
}
// Add namespace ipset to matching namespace selectors
_, err := ns.nsSelectors.addToMatching(obj.ObjectMeta.UID, obj.ObjectMeta.Labels, string(ns.allPods.ipsetName), namespaceComment(ns))
_, _, err := ns.nsSelectors.addToMatching(obj.ObjectMeta.UID, obj.ObjectMeta.Labels, string(ns.allPods.ipsetName), namespaceComment(ns))
return err
}
func (ns *ns) updateNamespace(oldObj, newObj *coreapi.Namespace) error {
ns.namespace = newObj
// Update bypass rule if ingress default has changed
oldDefaultDeny := ns.isDefaultDeny(oldObj)
newDefaultDeny := ns.isDefaultDeny(newObj)
if oldDefaultDeny != newDefaultDeny {
common.Log.Infof("namespace DefaultDeny changed from %t to %t", oldDefaultDeny, newDefaultDeny)
if oldDefaultDeny {
if err := ns.ensureBypassRule(); err != nil {
return err
}
}
if newDefaultDeny {
if err := ns.deleteBypassRule(); err != nil {
return err
}
}
}
// Re-evaluate namespace selector membership if labels have changed
if !equals(oldObj.ObjectMeta.Labels, newObj.ObjectMeta.Labels) {
for _, selector := range ns.nsSelectors.entries {
@@ -464,10 +504,8 @@ func (ns *ns) deleteNamespace(obj *coreapi.Namespace) error {
ns.namespace = nil
// Remove bypass rule
if !ns.isDefaultDeny(obj) {
if err := ns.deleteBypassRule(); err != nil {
return err
}
if err := ns.deleteBypassRules(); err != nil {
return err
}
// Remove namespace ipset from any matching namespace selectors
@@ -476,36 +514,16 @@ func (ns *ns) deleteNamespace(obj *coreapi.Namespace) error {
return err
}
if !ns.legacy {
return ns.ips.Destroy(ns.defaultAllowIPSet)
if err := ns.ips.Destroy(ns.ingressDefaultAllowIPSet); err != nil {
return err
}
if err := ns.ips.Destroy(ns.egressDefaultAllowIPSet); err != nil {
return err
}
return nil
}
func (ns *ns) isDefaultDeny(namespace *coreapi.Namespace) bool {
nnpJSON, found := namespace.ObjectMeta.Annotations["net.beta.kubernetes.io/network-policy"]
if !found {
return false
}
if !ns.legacy {
common.Log.Warn("DefaultDeny annotation is supported only in legacy mode (--use-legacy-netpol)")
return false
}
var nnp NamespaceNetworkPolicy
if err := json.Unmarshal([]byte(nnpJSON), &nnp); err != nil {
common.Log.Warn("Ignoring network policy annotation: unmarshal failed:", err)
// If we can't understand the annotation, behave as if it isn't present
return false
}
return nnp.Ingress != nil &&
nnp.Ingress.Isolation != nil &&
*(nnp.Ingress.Isolation) == DefaultDeny
}
func hasIP(pod *coreapi.Pod) bool {
// Ensure pod isn't dead, has an IP address and isn't sharing the host network namespace
return pod.Status.Phase != "Succeeded" && pod.Status.Phase != "Failed" &&
@@ -532,10 +550,19 @@ func podComment(pod *coreapi.Pod) string {
return fmt.Sprintf("namespace: %s, pod: %s", pod.ObjectMeta.Namespace, pod.ObjectMeta.Name)
}
func (ns *ns) defaultAllowIPSetName(pt policyType) ipset.Name {
ipset := ns.ingressDefaultAllowIPSet
if pt == policyTypeEgress {
ipset = ns.egressDefaultAllowIPSet
}
return ipset
}
func (ns *ns) analyse(obj interface{}) (
uid types.UID,
rules map[string]*ruleSpec,
nsSelectors, podSelectors map[string]*selectorSpec,
ipBlocks map[string]*ipBlockSpec,
err error) {
switch p := obj.(type) {
@@ -550,16 +577,9 @@ func (ns *ns) analyse(obj interface{}) (
ns.policies[uid] = obj
// Analyse policy, determine which rules and ipsets are required
if ns.legacy {
rules, nsSelectors, podSelectors, err = ns.analysePolicyLegacy(obj.(*extnapi.NetworkPolicy))
if err != nil {
return
}
} else {
rules, nsSelectors, podSelectors, err = ns.analysePolicy(obj.(*networkingv1.NetworkPolicy))
if err != nil {
return
}
rules, nsSelectors, podSelectors, ipBlocks, err = ns.analysePolicy(obj.(*networkingv1.NetworkPolicy))
if err != nil {
return
}
return
npc/rule.go
+64 -23
View File
@@ -2,6 +2,7 @@ package npc
import (
"fmt"
"reflect"
"strings"
"k8s.io/apimachinery/pkg/types"
@@ -10,38 +11,71 @@ import (
"github.com/weaveworks/weave/npc/iptables"
)
type ruleSpec struct {
key string
args []string
type ruleHost interface {
// getRuleSpec returns source or destination specification and comment which
// are used in an iptables rule.
//
// If src=true then the rulespec is for source, otherwise - for destination.
getRuleSpec(src bool) ([]string, string)
}
func newRuleSpec(proto *string, srcHost *selectorSpec, dstHost *selectorSpec, dstPort *string) *ruleSpec {
type ruleSpec struct {
key string
args []string
policyType policyType
}
func newRuleSpec(policyType policyType, proto *string, srcHost, dstHost ruleHost, dstPort *string) *ruleSpec {
args := []string{}
if proto != nil {
args = append(args, "-p", *proto)
}
srcComment := "anywhere"
if srcHost != nil {
args = append(args, "-m", "set", "--match-set", string(srcHost.ipsetName), "src")
if srcHost.nsName != "" {
srcComment = fmt.Sprintf("pods: namespace: %s, selector: %s", srcHost.nsName, srcHost.key)
} else {
srcComment = fmt.Sprintf("namespaces: selector: %s", srcHost.key)
}
if srcHost != nil && !reflect.ValueOf(srcHost).IsNil() {
rule, comment := srcHost.getRuleSpec(true)
args = append(args, rule...)
srcComment = comment
}
dstComment := "anywhere"
if dstHost != nil {
args = append(args, "-m", "set", "--match-set", string(dstHost.ipsetName), "dst")
dstComment = fmt.Sprintf("pods: namespace: %s, selector: %s", dstHost.nsName, dstHost.key)
if dstHost != nil && !reflect.ValueOf(dstHost).IsNil() {
rule, comment := dstHost.getRuleSpec(false)
args = append(args, rule...)
dstComment = comment
}
if dstPort != nil {
args = append(args, "--dport", *dstPort)
}
args = append(args, "-j", "ACCEPT")
args = append(args, "-m", "comment", "--comment", fmt.Sprintf("%s -> %s", srcComment, dstComment))
// NOTE: if you remove the comment bellow, then embed `policyType` into `key`.
// Otherwise, the rule won't be provisioned if it exists for other policy type.
args = append(args, "-m", "comment", "--comment", fmt.Sprintf("%s -> %s (%s)", srcComment, dstComment, policyTypeStr(policyType)))
key := strings.Join(args, " ")
return &ruleSpec{key, args}
return &ruleSpec{key, args, policyType}
}
func (spec *ruleSpec) iptChain() string {
if spec.policyType == policyTypeEgress {
return EgressCustomChain
}
return IngressChain
}
func (spec *ruleSpec) iptRuleSpecs() [][]string {
if spec.policyType == policyTypeIngress {
rule := make([]string, len(spec.args))
copy(rule, spec.args)
rule = append(rule, "-j", "ACCEPT")
return [][]string{rule}
}
// policyTypeEgress
ruleMark := make([]string, len(spec.args))
copy(ruleMark, spec.args)
ruleMark = append(ruleMark, "-j", EgressMarkChain)
ruleReturn := make([]string, len(spec.args))
copy(ruleReturn, spec.args)
ruleReturn = append(ruleReturn, "-j", "RETURN")
return [][]string{ruleMark, ruleReturn}
}
type ruleSet struct {
@@ -58,10 +92,14 @@ func (rs *ruleSet) deprovision(user types.UID, current, desired map[string]*rule
if _, found := desired[key]; !found {
delete(rs.users[key], user)
if len(rs.users[key]) == 0 {
common.Log.Infof("deleting rule: %v", spec.args)
if err := rs.ipt.Delete(TableFilter, IngressChain, spec.args...); err != nil {
return err
chain := spec.iptChain()
for _, rule := range spec.iptRuleSpecs() {
common.Log.Infof("deleting rule %v from %q chain", rule, chain)
if err := rs.ipt.Delete(TableFilter, chain, rule...); err != nil {
return err
}
}
delete(rs.users, key)
}
}
@@ -74,9 +112,12 @@ func (rs *ruleSet) provision(user types.UID, current, desired map[string]*ruleSp
for key, spec := range desired {
if _, found := current[key]; !found {
if _, found := rs.users[key]; !found {
common.Log.Infof("adding rule: %v", spec.args)
if err := rs.ipt.Append(TableFilter, IngressChain, spec.args...); err != nil {
return err
chain := spec.iptChain()
for _, rule := range spec.iptRuleSpecs() {
common.Log.Infof("adding rule %v to %q chain", rule, chain)
if err := rs.ipt.Append(TableFilter, chain, rule...); err != nil {
return err
}
}
rs.users[key] = make(map[types.UID]struct{})
}
npc/rule_test.go
+1 -1
View File
@@ -9,7 +9,7 @@ func TestRegressionHandleNil3095(t *testing.T) {
// Test for handling nil values
// https://github.com/weaveworks/weave/issues/3095
rs := newRuleSpec(nil, nil, nil, nil)
rs := newRuleSpec(policyTypeIngress, nil, nil, nil, nil)
require.NotEqual(t, rs.key, "")
npc/selector.go
+65 -37
View File
@@ -1,6 +1,8 @@
package npc
import (
"fmt"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
@@ -10,25 +12,25 @@ import (
)
type selectorSpec struct {
key string // string representation (for hash keying/equality comparison)
selector labels.Selector // k8s Selector object (for matching)
dst bool // destination selector
key string // string representation (for hash keying/equality comparison)
selector labels.Selector // k8s Selector object (for matching)
policyTypes []policyType // If non-empty, then selectorSpec is a target selector for given policyTypes.
ipsetType ipset.Type // type of ipset to provision
ipsetName ipset.Name // generated ipset name
nsName string // Namespace name
}
func newSelectorSpec(json *metav1.LabelSelector, dst bool, nsName string, ipsetType ipset.Type) (*selectorSpec, error) {
func newSelectorSpec(json *metav1.LabelSelector, policyType []policyType, nsName string, ipsetType ipset.Type) (*selectorSpec, error) {
selector, err := metav1.LabelSelectorAsSelector(json)
if err != nil {
return nil, err
}
key := selector.String()
return &selectorSpec{
key: key,
selector: selector,
dst: dst,
key: key,
selector: selector,
policyTypes: policyType,
// We prefix the selector string with the namespace name when generating
// the shortname because you can specify the same selector in multiple
// namespaces - we need those to map to distinct ipsets
@@ -37,6 +39,23 @@ func newSelectorSpec(json *metav1.LabelSelector, dst bool, nsName string, ipsetT
nsName: nsName}, nil
}
func (spec *selectorSpec) getRuleSpec(src bool) ([]string, string) {
dir := "dst"
if src {
dir = "src"
}
rule := []string{"-m", "set", "--match-set", string(spec.ipsetName), dir}
comment := "anywhere"
if spec.nsName != "" {
comment = fmt.Sprintf("pods: namespace: %s, selector: %s", spec.nsName, spec.key)
} else {
comment = fmt.Sprintf("namespaces: selector: %s", spec.key)
}
return rule, comment
}
type selector struct {
ips ipset.Interface
spec *selectorSpec
@@ -55,49 +74,54 @@ func (s *selector) delEntry(user types.UID, entry string) error {
}
type selectorFn func(selector *selector) error
type selectorWithPolicyTypeFn func(selector *selector, policyType policyType) error
type selectorSet struct {
ips ipset.Interface
onNewSelector selectorFn
// invoked after dst selector has been provisioned for the first time
onNewDstSelector selectorFn
// invoked after the last instance of dst selector has been deprovisioned
onDestroyDstSelector selectorFn
// invoked after target selector has been provisioned for the first time
onNewTargetSelector selectorWithPolicyTypeFn
// invoked after the last instance of target selector has been deprovisioned
onDestroyTargetSelector selectorWithPolicyTypeFn
users map[string]map[types.UID]struct{} // list of users per selector
entries map[string]*selector
// We need to keep track of dst selector instances to be able to invoke
// onNewDstSelector and onDestroyDstSelector callbacks at the right time;
// selectorSpec.Key -> count
dstSelectorsCount map[string]int
// We need to keep track of target selector instances to be able to invoke
// onNewTargetSelector and onDestroyTargetSelector callbacks at the right time;
// selectorSpec.Key -> policyType -> count
targetSelectorsCount map[string]map[policyType]int
}
func newSelectorSet(ips ipset.Interface, onNewSelector, onNewDstSelector selectorFn, onDestroyDstSelector selectorFn) *selectorSet {
func newSelectorSet(ips ipset.Interface, onNewSelector selectorFn, onNewTargetSelector, onDestroyTargetSelector selectorWithPolicyTypeFn) *selectorSet {
return &selectorSet{
ips: ips,
onNewSelector: onNewSelector,
onNewDstSelector: onNewDstSelector,
onDestroyDstSelector: onDestroyDstSelector,
ips: ips,
onNewSelector: onNewSelector,
onNewTargetSelector: onNewTargetSelector,
onDestroyTargetSelector: onDestroyTargetSelector,
users: make(map[string]map[types.UID]struct{}),
entries: make(map[string]*selector),
dstSelectorsCount: make(map[string]int)}
targetSelectorsCount: make(map[string]map[policyType]int)}
}
func (ss *selectorSet) addToMatching(user types.UID, labelMap map[string]string, entry string, comment string) (bool, error) {
found := false
func (ss *selectorSet) addToMatching(user types.UID, labelMap map[string]string, entry string, comment string) (bool, bool, error) {
foundIngress := false
foundEgress := false
for _, s := range ss.entries {
if s.matches(labelMap) {
if ss.dstSelectorExist(s) {
found = true
if ss.targetSelectorExist(s, policyTypeIngress) {
foundIngress = true
}
if ss.targetSelectorExist(s, policyTypeEgress) {
foundEgress = true
}
if err := s.addEntry(user, entry, comment); err != nil {
return found, err
return foundIngress, foundEgress, err
}
}
}
return found, nil
return foundIngress, foundEgress, nil
}
func (ss *selectorSet) delFromMatching(user types.UID, labelMap map[string]string, entry string) error {
@@ -111,8 +135,8 @@ func (ss *selectorSet) delFromMatching(user types.UID, labelMap map[string]strin
return nil
}
func (ss *selectorSet) dstSelectorExist(s *selector) bool {
return ss.dstSelectorsCount[s.spec.key] > 0
func (ss *selectorSet) targetSelectorExist(s *selector, policyType policyType) bool {
return ss.targetSelectorsCount[s.spec.key][policyType] > 0
}
func (ss *selectorSet) deprovision(user types.UID, current, desired map[string]*selectorSpec) error {
@@ -129,13 +153,14 @@ func (ss *selectorSet) deprovision(user types.UID, current, desired map[string]*
delete(ss.users, key)
}
if spec.dst {
ss.dstSelectorsCount[key]--
if ss.dstSelectorsCount[key] == 0 {
if err := ss.onDestroyDstSelector(&selector{ss.ips, spec}); err != nil {
for _, policyType := range spec.policyTypes {
ss.targetSelectorsCount[key][policyType]--
if ss.targetSelectorsCount[key][policyType] == 0 {
if err := ss.onDestroyTargetSelector(&selector{ss.ips, spec}, policyType); err != nil {
return err
}
}
// TODO(brb) delete(...)
}
}
}
@@ -160,10 +185,13 @@ func (ss *selectorSet) provision(user types.UID, current, desired map[string]*se
}
ss.users[key][user] = struct{}{}
if spec.dst {
ss.dstSelectorsCount[key]++
if ss.dstSelectorsCount[key] == 1 {
if err := ss.onNewDstSelector(selector); err != nil {
for _, pt := range spec.policyTypes {
if _, found := ss.targetSelectorsCount[key]; !found {
ss.targetSelectorsCount[key] = make(map[policyType]int)
}
ss.targetSelectorsCount[key][pt]++
if ss.targetSelectorsCount[key][pt] == 1 {
if err := ss.onNewTargetSelector(selector, pt); err != nil {
return err
}
}
npc/types.go
+15
View File
@@ -0,0 +1,15 @@
package npc
type policyType byte
const (
policyTypeIngress policyType = iota
policyTypeEgress
)
func policyTypeStr(policyType policyType) string {
if policyType == policyTypeEgress {
return "egress"
}
return "ingress"
}
prog/weave-kube/weave-daemonset-k8s-1.7.yaml
-9
View File
@@ -25,14 +25,6 @@ items:
- get
- list
- watch
- apiGroups:
- extensions
resources:
- networkpolicies
verbs:
- get
- list
- watch
- apiGroups:
- 'networking.k8s.io'
resources:
@@ -163,7 +155,6 @@ items:
fieldPath: spec.nodeName
image: 'weaveworks/weave-npc:latest'
imagePullPolicy: Always
#npc-args
resources:
requests:
cpu: 10m
prog/weave-npc/main.go
+102 -17
View File
@@ -8,7 +8,6 @@ import (
"github.com/coreos/go-iptables/iptables"
"github.com/spf13/cobra"
coreapi "k8s.io/api/core/v1"
extnapi "k8s.io/api/extensions/v1beta1"
networkingv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
@@ -18,6 +17,7 @@ import (
"k8s.io/client-go/tools/cache"
"github.com/weaveworks/weave/common"
"github.com/weaveworks/weave/net"
"github.com/weaveworks/weave/net/ipset"
"github.com/weaveworks/weave/npc"
"github.com/weaveworks/weave/npc/metrics"
@@ -30,7 +30,6 @@ var (
logLevel string
allowMcast bool
nodeName string
legacy bool
maxList int
)
@@ -53,7 +52,27 @@ func resetIPTables(ipt *iptables.IPTables) error {
return err
}
return ipt.ClearChain(npc.TableFilter, npc.MainChain)
if err := ipt.ClearChain(npc.TableFilter, npc.MainChain); err != nil {
return err
}
if err := ipt.ClearChain(npc.TableFilter, npc.EgressMarkChain); err != nil {
return err
}
if err := ipt.ClearChain(npc.TableFilter, npc.EgressCustomChain); err != nil {
return err
}
if err := ipt.ClearChain(npc.TableFilter, npc.EgressDefaultChain); err != nil {
return err
}
// We do not clear npc.EgressChain here because otherwise, in the case of restarting
// weave-npc process, all egress traffic is allowed for a short period of time.
// The chain is created in createBaseRules.
return nil
}
func resetIPSets(ips ipset.Interface) error {
@@ -77,6 +96,11 @@ func resetIPSets(ips ipset.Interface) error {
}
for _, s := range sets {
// LocalIPset might be used by WEAVE-NPC-EGRESS chain which we do not
// flush, so we cannot destroy it.
if s == npc.LocalIpset {
continue
}
common.Log.Debugf("Destroying ipset '%s'", string(s))
if err := ips.Destroy(s); err != nil {
common.Log.Errorf("Failed to destroy ipset '%s'", string(s))
@@ -112,11 +136,81 @@ func createBaseRules(ipt *iptables.IPTables, ips ipset.Interface) error {
}
// If the destination address is not any of the local pods, let it through
if err := ips.Create(npc.LocalIpset, ipset.HashIP); err != nil {
found, err := ipsetExist(ips, npc.LocalIpset)
if err != nil {
return err
}
return ipt.Append(npc.TableFilter, npc.MainChain,
"-m", "set", "!", "--match-set", npc.LocalIpset, "dst", "-j", "ACCEPT")
if !found {
if err := ips.Create(npc.LocalIpset, ipset.HashIP); err != nil {
return err
}
}
if err := ipt.Append(npc.TableFilter, npc.MainChain,
"-m", "set", "!", "--match-set", npc.LocalIpset, "dst", "-j", "ACCEPT"); err != nil {
return err
}
if err := ipt.Append(npc.TableFilter, npc.EgressMarkChain,
"-j", "MARK", "--set-xmark", npc.EgressMark); err != nil {
return err
}
// Egress rules:
//
// -A WEAVE-NPC-EGRESS -m state --state RELATED,ESTABLISHED -j ACCEPT
// -A WEAVE-NPC-EGRESS -m set ! --match-set weave-local-pods src -j RETURN
// -A WEAVE-NPC-EGRESS -d 224.0.0.0/4 -j RETURN
// -A WEAVE-NPC-EGRESS -m state --state NEW -j WEAVE-NPC-EGRESS-DEFAULT
// -A WEAVE-NPC-EGRESS -m state --state NEW -m mark ! --mark 0x40000/0x40000 -j WEAVE-NPC-EGRESS-CUSTOM
// -A WEAVE-NPC-EGRESS -m state --state NEW -m mark ! --mark 0x40000/0x40000 -j NFLOG --nflog-group 86
// -A WEAVE-NPC-EGRESS -m mark ! --mark 0x40000/0x40000 -j DROP
//
// -A WEAVE-NPC-EGRESS-CUSTOM <rulespec> -j MARK --set-xmark 0x40000/0x40000
// -A WEAVE-NPC-EGRESS-CUSTOM <rulespec> -j RETURN
//
// -A WEAVE-NPC-EGRESS-DEFAULT <rulespec> -j MARK --set-xmark 0x40000/0x40000
// -A WEAVE-NPC-EGRESS-DEFAULT <rulespec> -j RETURN
//
// For each rule we create two (mark and return). We cannot just accept
// a packet if it matches any rule, as a packet might need to traverse
// the ingress npc as well which happens later in the chain (in some cases
// we cannot detect whether packet is ingress or egress, so we need to
// check both chains).
ruleSpecs := [][]string{
{"-m", "state", "--state", "RELATED,ESTABLISHED", "-j", "ACCEPT"},
{"-m", "state", "--state", "NEW", "-m", "set", "!", "--match-set", npc.LocalIpset, "src", "-j", "RETURN"},
}
if allowMcast {
ruleSpecs = append(ruleSpecs, []string{"-d", "224.0.0.0/4", "-j", "RETURN"})
}
ruleSpecs = append(ruleSpecs, [][]string{
{"-m", "state", "--state", "NEW", "-j", string(npc.EgressDefaultChain)},
{"-m", "state", "--state", "NEW", "-m", "mark", "!", "--mark", npc.EgressMark, "-j", string(npc.EgressCustomChain)},
{"-m", "state", "--state", "NEW", "-m", "mark", "!", "--mark", npc.EgressMark, "-j", "NFLOG", "--nflog-group", "86"},
{"-m", "mark", "!", "--mark", npc.EgressMark, "-j", "DROP"},
}...)
if err := net.AddChainWithRules(ipt, npc.TableFilter, npc.EgressChain, ruleSpecs); err != nil {
return err
}
return nil
}
// Dummy way to check whether a given ipset exists.
// TODO(brb) Use "ipset -exist create <..>" for our purpose instead (for some reasons
// creating an ipset with -exist fails).
func ipsetExist(ips ipset.Interface, name ipset.Name) (bool, error) {
sets, err := ips.List(string(name))
if err != nil {
return false, err
}
for _, s := range sets {
if s == name {
return true, nil
}
}
return false, nil
}
func root(cmd *cobra.Command, args []string) {
@@ -132,10 +226,6 @@ func root(cmd *cobra.Command, args []string) {
}
common.Log.Infof("Starting Weaveworks NPC %s; node name %q", version, nodeName)
if legacy {
common.Log.Info("Running in legacy mode (k8s pre-1.7 network policy semantics)")
}
if err := metrics.Start(metricsAddr); err != nil {
common.Log.Fatalf("Failed to start metrics: %v", err)
}
@@ -159,7 +249,7 @@ func root(cmd *cobra.Command, args []string) {
handleError(resetIPSets(ips))
handleError(createBaseRules(ipt, ips))
npc := npc.New(nodeName, legacy, ipt, ips)
npc := npc.New(nodeName, ipt, ips)
nsController := makeController(client.Core().RESTClient(), "namespaces", &coreapi.Namespace{},
cache.ResourceEventHandlerFuncs{
@@ -220,11 +310,7 @@ func root(cmd *cobra.Command, args []string) {
handleError(npc.UpdateNetworkPolicy(old, new))
},
}
if legacy {
npController = makeController(client.Extensions().RESTClient(), "networkpolicies", &extnapi.NetworkPolicy{}, npHandlers)
} else {
npController = makeController(client.NetworkingV1().RESTClient(), "networkpolicies", &networkingv1.NetworkPolicy{}, npHandlers)
}
npController = makeController(client.NetworkingV1().RESTClient(), "networkpolicies", &networkingv1.NetworkPolicy{}, npHandlers)
go nsController.Run(wait.NeverStop)
go podController.Run(wait.NeverStop)
@@ -245,7 +331,6 @@ func main() {
rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "debug", "logging level (debug, info, warning, error)")
rootCmd.PersistentFlags().BoolVar(&allowMcast, "allow-mcast", true, "allow all multicast traffic")
rootCmd.PersistentFlags().StringVar(&nodeName, "node-name", "", "only generate rules that apply to this node")
rootCmd.PersistentFlags().BoolVar(&legacy, "use-legacy-netpol", false, "use legacy network policies (pre k8s 1.7 vsn)")
rootCmd.PersistentFlags().IntVar(&maxList, "max-list-size", 1024, "maximum size of ipset list (for namespaces)")
handleError(rootCmd.Execute())
site/kubernetes/kube-addon.md
-1
View File
@@ -370,7 +370,6 @@ You can customise the YAML you get from `cloud.weave.works` by passing some of W
- `disable-npc`: boolean (`true|false`). Default: `false`.
- `env.NAME=VALUE`: add environment variable `NAME` and set it to `VALUE`.
- `seLinuxOptions.NAME=VALUE`: add SELinux option `NAME` and set it to `VALUE`, e.g. `seLinuxOptions.type=spc_t`
- `use-legacy-netpol`: use [legacy NetworkPolicy semantics](https://v1-6.docs.kubernetes.io/docs/api-reference/v1.6/#networkpolicy-v1beta1-extensions), boolean (`true|false`). Default: `true` for Kubernetes version <= 1.6, `false` for > 1.6.
The list of variables you can set is:
test/840_weave_kube_3_test.sh
+68 -16
View File
@@ -52,7 +52,6 @@ fi
# Ensure Kubernetes uses locally built container images and inject code coverage environment variable (or do nothing depending on $COVERAGE):
sed -e "s%imagePullPolicy: Always%imagePullPolicy: Never%" \
-e "s%env:%$WEAVE_ENV_VARS%" \
-e "s%#npc-args% args:\n - '--use-legacy-netpol'%" \
"$(dirname "$0")/../prog/weave-kube/weave-daemonset-k8s-1.8.yaml" | run_on "$HOST1" "$KUBECTL apply -n kube-system -f -"
sleep 5
@@ -76,10 +75,19 @@ assert "run_on $HOST1 ps aux | grep -c '[d]efunct'" "0"
assert "run_on $HOST2 ps aux | grep -c '[d]efunct'" "0"
assert "run_on $HOST3 ps aux | grep -c '[d]efunct'" "0"
# Set up a simple network policy so all our test pods can talk to each other
run_on $HOST1 "$KUBECTL annotate ns default net.beta.kubernetes.io/network-policy='{\"ingress\":{\"isolation\":\"DefaultDeny\"}}'"
# Set up simple network policies so all our test pods can talk to each other
run_on $HOST1 "$KUBECTL apply -f -" <<EOF
apiVersion: extensions/v1beta1
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: default-deny
spec:
podSelector: {}
policyTypes:
- Ingress
EOF
run_on $HOST1 "$KUBECTL apply -f -" <<EOF
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: test840
@@ -102,7 +110,7 @@ EOF
# Another policy, this time with no 'from' section, just to check that doesn't cause a crash
run_on $HOST1 "$KUBECTL apply -f -" <<EOF
apiVersion: extensions/v1beta1
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: test840f
@@ -179,17 +187,6 @@ run_on $HOST1 "$KUBECTL run nettest-deny --labels=\"access=deny,run=nettest-deny
denyPodName=$($SSH $HOST1 "$KUBECTL get pods -l run=nettest-deny -o go-template='{{(index .items 0).metadata.name}}'")
assert_raises "! $SSH $HOST1 $KUBECTL exec $denyPodName -- curl -s -S -f -m 2 http://$DOMAIN:8080/status >/dev/null"
# Restart weave-net with npc in non-legacy mode
$SSH $HOST1 "$KUBECTL delete ds weave-net -n=kube-system"
sed -e "s%imagePullPolicy: Always%imagePullPolicy: Never%" \
-e "s%env:%$WEAVE_ENV_VARS%" \
"$(dirname "$0")/../prog/weave-kube/weave-daemonset-k8s-1.8.yaml" | run_on "$HOST1" "$KUBECTL apply -n kube-system -f -"
assert_raises 'wait_for_x check_all_pods_communicate pods'
# nettest-deny should still not be able to reach nettest pods
assert_raises "! $SSH $HOST1 $KUBECTL exec $denyPodName -- curl -s -S -f -m 2 http://$DOMAIN:8080/status >/dev/null"
# check access via virtual IP
VIRTUAL_IP=$($SSH $HOST1 $KUBECTL get service netvirt -o template --template={{.spec.clusterIP}})
assert_raises "$SSH $HOST1 $KUBECTL exec $podName -- curl -s -S -f -m 2 http://$VIRTUAL_IP/status >/dev/null"
@@ -269,6 +266,61 @@ EOF
assert_raises "$SSH $HOST1 curl -s -S -f -m 2 http://$VIRTUAL_IP/status >/dev/null"
assert_raises "$SSH $HOST1 curl -s -S -f -m 2 http://$HOST2:31138/status >/dev/null"
# allow nettest-deny to access only to 8.8.8.0/24
run_on $HOST1 "$KUBECTL apply -f -" <<EOF
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: egress-nettest-deny-dns
namespace: default
spec:
podSelector:
matchLabels:
run: nettest-deny
policyTypes:
- Egress
egress:
- to:
- ipBlock:
cidr: 8.8.8.0/24
except:
- 8.8.8.4/32
EOF
assert_raises "! $SSH $HOST1 $KUBECTL exec $denyPodName -- curl -s -S -f -m 2 http://$DOMAIN:8080/status >/dev/null"
assert_raises "! $SSH $HOST1 $KUBECTL exec $denyPodName -- dig @8.8.8.4 google.com >/dev/null"
assert_raises "$SSH $HOST1 $KUBECTL exec $denyPodName -- dig @8.8.8.8 google.com >/dev/null"
# also, allow nettest-deny to access nettest and kube-system (for kube-dns)
run_on $HOST1 "$KUBECTL label namespace kube-system name=kube-system"
run_on $HOST1 "$KUBECTL replace -f -" <<EOF
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: egress-nettest-deny-dns
namespace: default
spec:
podSelector:
matchLabels:
run: nettest-deny
policyTypes:
- Egress
egress:
- to:
- ipBlock:
cidr: 8.8.8.0/24
except:
- 8.8.8.4/32
- podSelector:
matchLabels:
run: nettest
- namespaceSelector:
matchLabels:
name: kube-system
EOF
assert_raises "$SSH $HOST1 $KUBECTL exec $denyPodName -- curl -s -S -f -m 2 http://$DOMAIN:8080/status >/dev/null"
# Passing --no-masq-local and setting externalTrafficPolicy to Local must preserve
# the client source IP addr
test/images/network-tester/Dockerfile
+1 -1
View File
@@ -13,7 +13,7 @@
# limitations under the License.
FROM bash
RUN apk add --no-cache curl
RUN apk add --no-cache curl bind-tools
ADD webserver webserver
EXPOSE 8080
ENTRYPOINT ["/webserver"]
weave
+11 -1
View File
@@ -492,10 +492,20 @@ destroy_bridge() {
run_iptables -t filter -D FORWARD -o $BRIDGE -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || true
run_iptables -t filter -D FORWARD -i $BRIDGE -o $BRIDGE -j ACCEPT 2>/dev/null || true
run_iptables -F WEAVE-NPC >/dev/null 2>&1 || true
run_iptables -t filter -D FORWARD -o $BRIDGE -j WEAVE-NPC 2>/dev/null || true
run_iptables -t filter -D FORWARD -o $BRIDGE -m comment --comment "NOTE: this must go before '-j KUBE-FORWARD'" -j WEAVE-NPC 2>/dev/null || true
run_iptables -t filter -D FORWARD -o $BRIDGE -m state --state NEW -j NFLOG --nflog-group 86 2>/dev/null || true
run_iptables -t filter -D FORWARD -o $BRIDGE -j DROP 2>/dev/null || true
run_iptables -X WEAVE-NPC >/dev/null 2>&1 || true
run_iptables -F WEAVE-NPC-EGRESS >/dev/null 2>&1 || true
run_iptables -t filter -D FORWARD -i $BRIDGE -m comment --comment "NOTE: this must go before '-j KUBE-FORWARD'" -j WEAVE-NPC-EGRESS 2>/dev/null || true
run_iptables -t filter -D INPUT -i $BRIDGE -j WEAVE-NPC-EGRESS 2>/dev/null || true
run_iptables -X WEAVE-NPC-EGRESS >/dev/null 2>&1 || true
run_iptables -F WEAVE-NPC-EGRESS-DEFAULT >/dev/null 2>&1 || true
run_iptables -X WEAVE-NPC-EGRESS-DEFAULT >/dev/null 2>&1 || true
run_iptables -F WEAVE-NPC-EGRESS-CUSTOM >/dev/null 2>&1 || true
run_iptables -X WEAVE-NPC-EGRESS-CUSTOM >/dev/null 2>&1 || true
run_iptables -F WEAVE-NPC-EGRESS-ACCEPT >/dev/null 2>&1 || true
run_iptables -X WEAVE-NPC-EGRESS-ACCEPT >/dev/null 2>&1 || true
run_iptables -F WEAVE-EXPOSE >/dev/null 2>&1 || true
run_iptables -t filter -D FORWARD -o $BRIDGE -j WEAVE-EXPOSE 2>/dev/null || true