diff --git a/lb/datapath.go b/lb/datapath.go index 4c9a7982..91e19541 100644 --- a/lb/datapath.go +++ b/lb/datapath.go @@ -151,9 +151,12 @@ func (dp *Datapath) Apply(d Desired, e EgressDesired, self string, local Local) if err := dp.hooks(); err != nil { return err } - // the tunnel device must exist before rules naming it are restored (-o/-i on a missing device is fine for - // iptables, but the routes and tables need it) - if err := ensureTunnel(e.Tunnelled(self), local); err != nil { + // the tunnels must exist before the routes into them (iptables rules may name a device that does not exist yet) + var peers []Node + if e.Tunnelled(self) { + peers = e.Peers(self) + } + if err := ensureTunnels(peers, local); err != nil { return err } existing, err := dp.ipt.ListChains("nat") @@ -187,7 +190,7 @@ func (dp *Datapath) Apply(d Desired, e EgressDesired, self string, local Local) if err := dp.routes(d.Holders); err != nil { return err } - return egressNet(e, self, local) + return egressNet(e, self) } // hooks makes sure the built-in chains jump to ours: first (kube-proxy's and weave's rules must not see these @@ -287,14 +290,14 @@ func (dp *Datapath) routes(holders []Holder) error { return fmt.Errorf("removing stale rule to table %d: %v", r.Table, err) } } + existing, err := routesByTable(TableBase+1, TableBase+MaxMarks) + if err != nil { + return err + } for table := TableBase + 1; table <= TableBase+MaxMarks; table++ { h, ok := want[table] if !ok { - routes, err := netlink.RouteListFiltered(unix.AF_INET, &netlink.Route{Table: table}, netlink.RT_FILTER_TABLE) - if err != nil { - return err - } - for _, rt := range routes { + for _, rt := range existing[table] { if err := delRoute(rt); err != nil { return fmt.Errorf("removing stale route in table %d: %v", table, err) } diff --git a/lb/egress_datapath.go b/lb/egress_datapath.go index efdcfe70..d349271a 100644 --- a/lb/egress_datapath.go +++ b/lb/egress_datapath.go @@ -3,9 +3,11 @@ package lb import ( "bytes" "fmt" + "hash/fnv" "net" "os" "sort" + "strings" "github.com/vishvananda/netlink" "golang.org/x/sys/unix" @@ -17,15 +19,25 @@ const ( ClassChain = "REWEAVE-LB-CLASS" // FwdChain (filter, first in FORWARD) drops members' outside packets that would not leave as their address. FwdChain = "REWEAVE-LB-FWD" - // InChain (filter, first in INPUT) accepts the egress tunnel only from the cluster's nodes. + // InChain (filter, first in INPUT) accepts the egress tunnels only from the cluster's nodes. InChain = "REWEAVE-LB-IN" - // MSSChain (mangle, first in FORWARD) clamps TCP MSS into the tunnel. + // MSSChain (mangle, first in FORWARD) clamps TCP MSS into the tunnels. MSSChain = "REWEAVE-LB-MSS" - // Tunnel is the egress tunnel: ipip over weave between nodes' weave bridges, no fixed remote (each route names it). - Tunnel = "reweave-eg" + // TunnelPrefix names the egress tunnels: one ipip device per peer node, between the two nodes' weave bridges, + // so a holder knows which node a member's packet came from by the device it arrives on. + TunnelPrefix = "rweg-" + // tunnelGlob matches every egress tunnel in an iptables rule. + tunnelGlob = "rweg+" + // legacyTunnel is the single shared tunnel of earlier builds (removed when found). + legacyTunnel = "reweave-eg" + // EgressMark marks a member's packets to the outside (connmark on its connections, fwmark on each packet). EgressMark = uint32(0x100000) + // peerFlag, with a peer's index in the top byte, tags (on the holder) a connection that arrived from that peer's + // tunnel: replies and related ICMP errors carry it back there. + peerFlag = uint32(0x200000) + peerMask = markMask | peerFlag // HolderRulePriority: replies to a block go to its node; ahead of the local table, which moves to LocalRulePriority, // since the address is usually local on the holder. @@ -33,12 +45,28 @@ const ( LocalRulePriority = 100 // MemberRulePriority: a member's marked packets go to its holder (or nowhere). MemberRulePriority = 9100 - // EgressTableBase + peer index routes into the tunnel towards that peer; EgressTableBase itself is unreachable. + // EgressTableBase + peer index routes into that peer's tunnel; EgressTableBase itself is unreachable. EgressTableBase = 53000 ipipOverhead = 20 ) +// portProtocols carry ports: their replies are routed by port block alone (stateless), and members' SNAT keeps +// within the node's block. Everything else is routed back by the tag the holder put on the connection. +var portProtocols = []struct { + name string + num int +}{{"tcp", unix.IPPROTO_TCP}, {"udp", unix.IPPROTO_UDP}, {"sctp", unix.IPPROTO_SCTP}} + +// TunnelName is the egress tunnel towards a peer node (stable, within the 15 characters a device name may have). +func TunnelName(peer string) string { + h := fnv.New32a() + h.Write([]byte(peer)) + return fmt.Sprintf("%s%08x", TunnelPrefix, h.Sum32()) +} + +func peerTag(idx uint32) uint32 { return markValue(idx) | peerFlag } + // renderEgress appends the egress rules of each table to the bodies of S1's payloads. func renderEgress(e EgressDesired, self string, podNet *net.IPNet, nat, mangle, filter *bytes.Buffer) { members := e.Members @@ -66,80 +94,144 @@ func renderEgress(e EgressDesired, self string, podNet *net.IPNet, nat, mangle, fmt.Fprintf(filter, "-A %s -s %s/32 -m mark --mark 0x%x/0x%x -m comment --comment %q -j DROP\n", FwdChain, m.IP, EgressMark, EgressMark, m.Pod) continue } - out := "-o " + Tunnel + out := "-o " + TunnelName(m.Holder.Name) if m.Holder.Name == self { // emitted here, from the address: never into the pod network unSNATed out = "! -o " + bridge fmt.Fprintf(filter, "-A %s -s %s/32 -o %s -m mark --mark 0x%x/0x%x -j DROP\n", FwdChain, m.IP, bridge, EgressMark, EgressMark) } - for _, proto := range []string{"tcp", "udp"} { + // within this node's block for what has ports; the address alone for the rest (the holder tells those + // apart by the tunnel they arrive on) + for _, p := range portProtocols { fmt.Fprintf(nat, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -p %s -j SNAT --to-source %s:%d-%d\n", - PostChain, m.IP, out, EgressMark, EgressMark, proto, m.Address, m.From, m.To) - fmt.Fprintf(filter, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -p %s -j RETURN\n", FwdChain, m.IP, out, EgressMark, EgressMark, proto) + PostChain, m.IP, out, EgressMark, EgressMark, p.name, m.Address, m.From, m.To) } - // anything else a member sends outside (other protocols, other ways out) does not leave + fmt.Fprintf(nat, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -j SNAT --to-source %s\n", + PostChain, m.IP, out, EgressMark, EgressMark, m.Address) + fmt.Fprintf(filter, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -j RETURN\n", FwdChain, m.IP, out, EgressMark, EgressMark) + // a member's outside traffic leaves no other way fmt.Fprintf(filter, "-A %s -s %s/32 -m mark --mark 0x%x/0x%x -m comment --comment %q -j DROP\n", FwdChain, m.IP, EgressMark, EgressMark, m.Pod) } if !e.Tunnelled(self) { return } - // tunnel arrivals: replies to members here (established), members' packets from their blocks (as holder) + tags := peerIndex(e, self) + + // as holder: tag each connection arriving from a peer's tunnel with that peer, once; replies and related ICMP + // errors restore the tag, and the policy rule for it sends them back into that tunnel + byPeer := map[string][]BlockRoute{} + var peerNames []string + for _, r := range e.Routes { + if _, ok := byPeer[r.Peer.Name]; !ok { + peerNames = append(peerNames, r.Peer.Name) + } + byPeer[r.Peer.Name] = append(byPeer[r.Peer.Name], r) + } + sort.Strings(peerNames) + if len(peerNames) > 0 { + fmt.Fprintf(mangle, "-A %s -m conntrack --ctdir REPLY -m connmark --mark 0x%x/0x%x -j CONNMARK --restore-mark --nfmask 0x%x --ctmask 0x%x\n", + MarkChain, peerFlag, peerFlag, peerMask, peerMask) + } + for _, name := range peerNames { + dev, tag := TunnelName(name), peerTag(tags[name]) + fmt.Fprintf(mangle, "-A %s -i %s -m conntrack --ctstate NEW -m connmark --mark 0x0/0x%x -m comment --comment %q -j CONNMARK --set-xmark 0x%x/0x%x\n", + MarkChain, dev, peerMask, "from "+name, tag, peerMask) + // a connection another node already holds (a port-less protocol to the same remote): never cross-delivered + fmt.Fprintf(filter, "-A %s -i %s -m conntrack --ctdir ORIGINAL -m connmark ! --mark 0x%x/0x%x -j DROP\n", FwdChain, dev, tag, peerMask) + addrs := map[string]bool{} + for _, r := range byPeer[name] { + for _, p := range portProtocols { + fmt.Fprintf(filter, "-A %s -i %s -m conntrack --ctdir ORIGINAL -s %s/32 -p %s -m %s --sport %d:%d -m comment --comment %q -j RETURN\n", + FwdChain, dev, r.Address, p.name, p.name, r.From, r.To, "block of "+name) + } + addrs[r.Address.String()] = true + } + for _, a := range sortedSet(addrs) { + // outside its own block a node may not use the ports of others + for _, p := range portProtocols { + fmt.Fprintf(filter, "-A %s -i %s -m conntrack --ctdir ORIGINAL -s %s/32 -p %s -j DROP\n", FwdChain, dev, a, p.name) + } + fmt.Fprintf(filter, "-A %s -i %s -m conntrack --ctdir ORIGINAL -s %s/32 -j RETURN\n", FwdChain, dev, a) + } + } + // as a member's node: replies (and related errors) from the holder; nothing else comes back addrs := map[string]bool{} for _, m := range members { - if !m.Unreachable && m.Holder.Name != self && !addrs[m.Address.String()] { + if !m.Unreachable && m.Holder.Name != self { addrs[m.Address.String()] = true - // still addressed to the address after nat: no member connection matched it - fmt.Fprintf(filter, "-A %s -i %s -d %s/32 -j DROP\n", FwdChain, Tunnel, m.Address) } } - fmt.Fprintf(filter, "-A %s -i %s -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n", FwdChain, Tunnel) - for _, r := range e.Routes { - for _, proto := range []string{"tcp", "udp"} { - fmt.Fprintf(filter, "-A %s -i %s -s %s/32 -p %s -m %s --sport %d:%d -m comment --comment %q -j RETURN\n", - FwdChain, Tunnel, r.Address, proto, proto, r.From, r.To, "block of "+r.Peer.Name) - } + for _, a := range sortedSet(addrs) { + // still addressed to the address after nat: no member connection matched it + fmt.Fprintf(filter, "-A %s -i %s -d %s/32 -j DROP\n", FwdChain, tunnelGlob, a) } - fmt.Fprintf(filter, "-A %s -i %s -j DROP\n", FwdChain, Tunnel) - // the tunnel itself only from the cluster's nodes' bridges + fmt.Fprintf(filter, "-A %s -i %s -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n", FwdChain, tunnelGlob) + fmt.Fprintf(filter, "-A %s -i %s -j DROP\n", FwdChain, tunnelGlob) + // the tunnels themselves only from the cluster's nodes' bridges for _, p := range e.Peers(self) { fmt.Fprintf(filter, "-A %s -p 4 -s %s/32 -m mac --mac-source %s -j RETURN\n", InChain, p.WeaveIP, p.WeaveMAC) } fmt.Fprintf(filter, "-A %s -p 4 -j DROP\n", InChain) - fmt.Fprintf(mangle, "-A %s -o %s -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu\n", MSSChain, Tunnel) + fmt.Fprintf(mangle, "-A %s -o %s -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu\n", MSSChain, tunnelGlob) } -// peerTables gives each tunnel peer its routing table (EgressTableBase + a stable index). +func sortedSet(m map[string]bool) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// peerIndex gives each tunnel peer a stable index (1..MaxMarks): its tag and its routing table. +func peerIndex(e EgressDesired, self string) map[string]uint32 { + out := map[string]uint32{} + for _, h := range assignMarks(e.Peers(self)) { + out[h.Node.Name] = h.Mark + } + return out +} + +// peerTables gives each tunnel peer its routing table (EgressTableBase + its index). func peerTables(e EgressDesired, self string) map[string]int { out := map[string]int{} - for _, h := range assignMarks(e.Peers(self)) { - out[h.Node.Name] = EgressTableBase + int(h.Mark) + for name, idx := range peerIndex(e, self) { + out[name] = EgressTableBase + int(idx) } return out } // egressRule is a policy rule this package owns, in a comparable form. type egressRule struct { - priority int - src, dst string - ipproto int - from, to uint16 - mark uint32 - table int + priority int + src, dst string + ipproto int + from, to uint16 + mark, mask uint32 + table int } func (r egressRule) key() string { - return fmt.Sprintf("%d|%s|%s|%d|%d-%d|%x|%d", r.priority, r.src, r.dst, r.ipproto, r.from, r.to, r.mark, r.table) + return fmt.Sprintf("%d|%s|%s|%d|%d-%d|%x/%x|%d", r.priority, r.src, r.dst, r.ipproto, r.from, r.to, r.mark, r.mask, r.table) } // wantRules is every egress policy rule this node needs. func wantRules(e EgressDesired, self string) []egressRule { tables := peerTables(e, self) + tags := peerIndex(e, self) var out []egressRule + tagged := map[string]bool{} for _, r := range e.Routes { - for _, proto := range []int{unix.IPPROTO_TCP, unix.IPPROTO_UDP} { - out = append(out, egressRule{priority: HolderRulePriority, dst: r.Address.String() + "/32", ipproto: proto, + for _, p := range portProtocols { + out = append(out, egressRule{priority: HolderRulePriority, dst: r.Address.String() + "/32", ipproto: p.num, from: uint16(r.From), to: uint16(r.To), table: tables[r.Peer.Name]}) } + if !tagged[r.Peer.Name] { + // whatever the tag says came from that peer (replies without ports, ICMP errors) goes back to it + tagged[r.Peer.Name] = true + out = append(out, egressRule{priority: HolderRulePriority, mark: peerTag(tags[r.Peer.Name]), mask: peerMask, table: tables[r.Peer.Name]}) + } } for _, m := range e.Members { if !m.Unreachable && m.Holder.Name == self { @@ -149,7 +241,7 @@ func wantRules(e EgressDesired, self string) []egressRule { if !m.Unreachable { table = tables[m.Holder.Name] } - out = append(out, egressRule{priority: MemberRulePriority, src: m.IP.String() + "/32", mark: EgressMark, table: table}) + out = append(out, egressRule{priority: MemberRulePriority, src: m.IP.String() + "/32", mark: EgressMark, mask: EgressMark, table: table}) } sort.Slice(out, func(i, j int) bool { return out[i].key() < out[j].key() }) return out @@ -168,6 +260,7 @@ func ruleOf(r netlink.Rule) egressRule { } if r.Mark > 0 { er.mark = uint32(r.Mark) + er.mask = uint32(r.Mask) } if er.ipproto < 0 { er.ipproto = 0 @@ -175,53 +268,66 @@ func ruleOf(r netlink.Rule) egressRule { return er } -// egressNet makes the tunnel, the local rule's place, the policy rules and the tunnel tables exactly what e needs. -func egressNet(e EgressDesired, self string, local Local) error { - tunnelled := e.Tunnelled(self) // the device itself is made first, by Apply (ensureTunnel) +// routesByTable lists, in one dump, the IPv4 routes of tables lo..hi. +func routesByTable(lo, hi int) (map[int][]netlink.Route, error) { + all, err := netlink.RouteListFiltered(unix.AF_INET, &netlink.Route{Table: unix.RT_TABLE_UNSPEC}, netlink.RT_FILTER_TABLE) + if err != nil { + return nil, err + } + out := map[int][]netlink.Route{} + for _, rt := range all { + if rt.Table >= lo && rt.Table <= hi { + out[rt.Table] = append(out[rt.Table], rt) + } + } + return out, nil +} + +// egressNet makes the local rule's place, the policy rules and the tunnel tables exactly what e needs (the +// tunnels themselves are made first, by Apply). +func egressNet(e EgressDesired, self string) error { if len(e.Routes) > 0 { if err := moveLocalRule(); err != nil { return err } } tables := peerTables(e, self) - peers := map[int]Node{} + peerOf := map[int]Node{} for _, p := range e.Peers(self) { - peers[tables[p.Name]] = p + peerOf[tables[p.Name]] = p } unreachable := false for _, m := range e.Members { unreachable = unreachable || m.Unreachable } - var link netlink.Link - if tunnelled { - l, err := netlink.LinkByName(Tunnel) - if err != nil { - return fmt.Errorf("egress tunnel: %v", err) - } - link = l + existing, err := routesByTable(EgressTableBase, EgressTableBase+MaxMarks) + if err != nil { + return err } for table := EgressTableBase; table <= EgressTableBase+MaxMarks; table++ { var want *netlink.Route - if p, ok := peers[table]; ok && link != nil { + if p, ok := peerOf[table]; ok { + link, err := netlink.LinkByName(TunnelName(p.Name)) + if err != nil { + return fmt.Errorf("egress tunnel to %s: %v", p.Name, err) + } want = &netlink.Route{LinkIndex: link.Attrs().Index, Dst: &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)}, - Gw: p.WeaveIP, Table: table, Flags: int(netlink.FLAG_ONLINK)} + Scope: netlink.SCOPE_LINK, Table: table} } else if table == EgressTableBase && unreachable { want = &netlink.Route{Dst: &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)}, Table: table, Type: unix.RTN_UNREACHABLE} } - routes, err := netlink.RouteListFiltered(unix.AF_INET, &netlink.Route{Table: table}, netlink.RT_FILTER_TABLE) - if err != nil { - return err - } - for _, rt := range routes { + keep := false + for _, rt := range existing[table] { if want != nil && rt.Type == want.Type && - (want.Type == unix.RTN_UNREACHABLE || rt.LinkIndex == want.LinkIndex && rt.Gw.Equal(want.Gw)) { + (want.Type == unix.RTN_UNREACHABLE || rt.LinkIndex == want.LinkIndex && rt.Gw == nil) { + keep = true continue } if err := delRoute(rt); err != nil { return fmt.Errorf("removing stale route in table %d: %v", table, err) } } - if want != nil { + if want != nil && !keep { if err := netlink.RouteReplace(want); err != nil { return fmt.Errorf("route in table %d: %v", table, err) } @@ -269,7 +375,7 @@ func egressNet(e EgressDesired, self string, local Local) error { } if r.mark != 0 { rule.Mark = int(r.mark) - rule.Mask = int(r.mark) + rule.Mask = int(r.mask) } if err := netlink.RuleAdd(rule); err != nil { return fmt.Errorf("adding egress rule %s: %v", k, err) @@ -316,53 +422,85 @@ func moveLocalRule() error { return nil } -// ensureTunnel creates (or removes) the egress tunnel: ipip with this node's bridge address as its local end. -func ensureTunnel(want bool, local Local) error { - link, err := netlink.LinkByName(Tunnel) - exists := err == nil - if !want { - if exists { - return netlink.LinkDel(link) +// ensureTunnels makes one ipip tunnel per peer (this node's bridge address to the peer's), and removes every +// other egress tunnel (including the shared one of earlier builds). +func ensureTunnels(peers []Node, local Local) error { + links, err := netlink.LinkList() + if err != nil { + return err + } + want := map[string]Node{} + for _, p := range peers { + want[TunnelName(p.Name)] = p + } + have := map[string]netlink.Link{} + for _, l := range links { + name := l.Attrs().Name + if name == legacyTunnel || strings.HasPrefix(name, TunnelPrefix) { + if _, ok := want[name]; ok { + have[name] = l + continue + } + if err := netlink.LinkDel(l); err != nil { + return fmt.Errorf("removing egress tunnel %s: %v", name, err) + } } + } + if len(want) == 0 { return nil } if local.WeaveIP == nil { - return fmt.Errorf("egress tunnel needs the weave bridge's address") + return fmt.Errorf("egress tunnels need the weave bridge's address") } br, err := netlink.LinkByName(bridge) if err != nil { return fmt.Errorf("weave bridge: %v", err) } mtu := br.Attrs().MTU - ipipOverhead - if exists { - t, ok := link.(*netlink.Iptun) - if !ok || !t.Local.Equal(local.WeaveIP) || t.Remote != nil && !t.Remote.IsUnspecified() || t.Attrs().MTU != mtu { - if err := netlink.LinkDel(link); err != nil { + names := make([]string, 0, len(want)) + for n := range want { + names = append(names, n) + } + sort.Strings(names) + for _, name := range names { + peer := want[name] + link := have[name] + if link != nil { + t, ok := link.(*netlink.Iptun) + if !ok || !t.Local.Equal(local.WeaveIP) || !t.Remote.Equal(peer.WeaveIP) || t.Attrs().MTU != mtu { + if err := netlink.LinkDel(link); err != nil { + return err + } + link = nil + } + } + if link == nil { + t := &netlink.Iptun{LinkAttrs: netlink.LinkAttrs{Name: name, MTU: mtu}, Local: local.WeaveIP, Remote: peer.WeaveIP, PMtuDisc: 1} + if err := netlink.LinkAdd(t); err != nil { + return fmt.Errorf("creating egress tunnel %s to %s: %v", name, peer.Name, err) + } + if link, err = netlink.LinkByName(name); err != nil { return err } - exists = false + } + if err := tunnelSetup(link, local); err != nil { + return fmt.Errorf("%s: %v", name, err) } } - if !exists { - t := &netlink.Iptun{LinkAttrs: netlink.LinkAttrs{Name: Tunnel, MTU: mtu}, Local: local.WeaveIP, PMtuDisc: 1} - if err := netlink.LinkAdd(t); err != nil { - return fmt.Errorf("creating egress tunnel: %v", err) - } - if link, err = netlink.LinkByName(Tunnel); err != nil { - return err - } - } - // the holder receives members' packets from its own address; only on this device + return nil +} + +// tunnelSetup: the holder receives members' packets from its own address (accept_local); reverse-path filtering +// rejects what arrives on a device with no IPv4 address unless the reverse route leaves by that very device, so +// the tunnel carries this node's bridge address (never on the wire from here: the tunnel sets the outer source, +// and what crosses it is forwarded). +func tunnelSetup(link netlink.Link, local Local) error { + name := link.Attrs().Name for k, v := range map[string]string{"accept_local": "1", "rp_filter": "2"} { - if err := os.WriteFile("/proc/sys/net/ipv4/conf/"+Tunnel+"/"+k, []byte(v), 0o644); err != nil { - return fmt.Errorf("sysctl %s on %s: %v", k, Tunnel, err) + if err := os.WriteFile("/proc/sys/net/ipv4/conf/"+name+"/"+k, []byte(v), 0o644); err != nil { + return fmt.Errorf("sysctl %s: %v", k, err) } } - // reverse-path filtering rejects what arrives on a device with no IPv4 address unless the reverse route - // leaves by that very device (replies on a member's node, members' packets on the holder do not): the - // tunnel carries this node's bridge address (it never appears on the wire from here: the tunnel sets the - // outer source itself, and what crosses it is forwarded) - addr := &netlink.Addr{IPNet: &net.IPNet{IP: local.WeaveIP, Mask: net.CIDRMask(32, 32)}, Scope: unix.RT_SCOPE_LINK} addrs, err := netlink.AddrList(link, unix.AF_INET) if err != nil { return err @@ -375,12 +513,13 @@ func ensureTunnel(want bool, local Local) error { } a := a if err := netlink.AddrDel(link, &a); err != nil { - return fmt.Errorf("removing stale address %s from %s: %v", a.IPNet, Tunnel, err) + return fmt.Errorf("removing stale address %s: %v", a.IPNet, err) } } if !has { + addr := &netlink.Addr{IPNet: &net.IPNet{IP: local.WeaveIP, Mask: net.CIDRMask(32, 32)}, Scope: unix.RT_SCOPE_LINK} if err := netlink.AddrAdd(link, addr); err != nil { - return fmt.Errorf("address on %s: %v", Tunnel, err) + return fmt.Errorf("address: %v", err) } } return netlink.LinkSetUp(link) diff --git a/lb/egress_test.go b/lb/egress_test.go index 3343932b..d43e30f7 100644 --- a/lb/egress_test.go +++ b/lb/egress_test.go @@ -1,12 +1,12 @@ package lb import ( + "fmt" "net" "strings" "testing" "time" - "golang.org/x/sys/unix" coreapi "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -199,6 +199,7 @@ func TestRenderedEgressRules(t *testing.T) { _, podNet, _ := net.ParseCIDR("10.32.0.0/12") e := egressFor("mail", written()) svcs := []*coreapi.Service{mailSvc()} + toW1, toW2, toCP := TunnelName("w1"), TunnelName("w2"), TunnelName("cp") member := ComputeEgress("w2", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{pod("mail-0", "w2", "10.44.0.9", mailLabels)}, nil) r := Render(Desired{}, member, "w2", podNet, nil) for _, want := range []string{ @@ -206,19 +207,27 @@ func TestRenderedEgressRules(t *testing.T) { "-A REWEAVE-LB-CLASS -d 86.107.168.24/32 -j RETURN\n", // a node is not outside "-A REWEAVE-LB-CLASS -s 10.44.0.9/32 -m comment --comment \"gitea/mail-0\" -j CONNMARK --set-xmark 0x100000/0x100000\n", "-A REWEAVE-LB-CLASS -s 10.44.0.9/32 -j MARK --set-xmark 0x100000/0x100000\n", - "-A REWEAVE-LB-POST -s 10.44.0.9/32 -o reweave-eg -m mark --mark 0x100000/0x100000 -p tcp -j SNAT --to-source 86.107.168.15:62024-62535\n", - "-A REWEAVE-LB-POST -s 10.44.0.9/32 -o reweave-eg -m mark --mark 0x100000/0x100000 -p udp -j SNAT --to-source 86.107.168.15:62024-62535\n", + // port protocols within this node's block, into the holder's own tunnel + "-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -p tcp -j SNAT --to-source 86.107.168.15:62024-62535\n", + "-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -p udp -j SNAT --to-source 86.107.168.15:62024-62535\n", + "-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -p sctp -j SNAT --to-source 86.107.168.15:62024-62535\n", + // everything else (ping, GRE, ...) from the address alone + "-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -j SNAT --to-source 86.107.168.15\n", } { if !strings.Contains(r.Nat, want) { t.Fatalf("member nat lacks %q:\n%s", want, r.Nat) } } + // the catch-all SNAT comes after the port-ranged ones (or ports would not stay in the block) + if strings.Index(r.Nat, "-j SNAT --to-source 86.107.168.15\n") < strings.Index(r.Nat, "-p sctp -j SNAT") { + t.Fatalf("SNAT order:\n%s", r.Nat) + } for _, want := range []string{ - "-A REWEAVE-LB-FWD -s 10.44.0.9/32 -o reweave-eg -m mark --mark 0x100000/0x100000 -p tcp -j RETURN\n", + "-A REWEAVE-LB-FWD -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -j RETURN\n", "-A REWEAVE-LB-FWD -s 10.44.0.9/32 -m mark --mark 0x100000/0x100000 -m comment --comment \"gitea/mail-0\" -j DROP\n", - "-A REWEAVE-LB-FWD -i reweave-eg -d 86.107.168.15/32 -j DROP\n", - "-A REWEAVE-LB-FWD -i reweave-eg -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n", - "-A REWEAVE-LB-FWD -i reweave-eg -j DROP\n", + "-A REWEAVE-LB-FWD -i rweg+ -d 86.107.168.15/32 -j DROP\n", + "-A REWEAVE-LB-FWD -i rweg+ -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n", + "-A REWEAVE-LB-FWD -i rweg+ -j DROP\n", "-A REWEAVE-LB-IN -p 4 -s 10.40.0.0/32 -m mac --mac-source aa:aa:aa:aa:aa:02 -j RETURN\n", "-A REWEAVE-LB-IN -p 4 -j DROP\n", } { @@ -228,28 +237,52 @@ func TestRenderedEgressRules(t *testing.T) { } for _, want := range []string{ "-A REWEAVE-LB-MARK -m conntrack --ctdir ORIGINAL -m connmark --mark 0x100000/0x100000 -j MARK --set-xmark 0x100000/0x100000\n", - "-A REWEAVE-LB-MSS -o reweave-eg -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu\n", + "-A REWEAVE-LB-MSS -o rweg+ -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu\n", } { if !strings.Contains(r.Mangle, want) { t.Fatalf("member mangle lacks %q:\n%s", want, r.Mangle) } } - // the guard's DROP comes after the member's own RETURNs: nothing else of its leaves - if strings.Index(r.Filter, "-p tcp -j RETURN") > strings.Index(r.Filter, "\"gitea/mail-0\" -j DROP") { + if strings.Contains(r.Mangle, "--set-xmark 0x") && strings.Contains(r.Mangle, "-i "+toW1) { + t.Fatalf("a member's node does not tag what its holder sends back:\n%s", r.Mangle) + } + // the guard's DROP comes after the member's own RETURN: nothing else of its leaves + if strings.Index(r.Filter, toW1+" -m mark --mark 0x100000/0x100000 -j RETURN") > strings.Index(r.Filter, "\"gitea/mail-0\" -j DROP") { t.Fatalf("guard order:\n%s", r.Filter) } - // the holder: accepts each node's block from the tunnel, drops the rest; a member on it SNATs to its own block + + // the holder: tags each tunnel's connections with its node, accepts each node's block, nothing foreign h := ComputeEgress("w1", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{pod("mail-1", "w1", "10.40.0.4", mailLabels)}, nil) r = Render(Desired{}, h, "w1", podNet, nil) + idx := peerIndex(h, "w1") + tagW2 := fmt.Sprintf("0x%x", peerTag(idx["w2"])) for _, want := range []string{ - "-A REWEAVE-LB-FWD -i reweave-eg -s 86.107.168.15/32 -p tcp -m tcp --sport 62024:62535 -m comment --comment \"block of w2\" -j RETURN\n", - "-A REWEAVE-LB-FWD -i reweave-eg -s 86.107.168.15/32 -p udp -m udp --sport 61000:61511 -m comment --comment \"block of cp\" -j RETURN\n", + "-A REWEAVE-LB-MARK -m conntrack --ctdir REPLY -m connmark --mark 0x200000/0x200000 -j CONNMARK --restore-mark --nfmask 0xff200000 --ctmask 0xff200000\n", + "-A REWEAVE-LB-MARK -i " + toW2 + " -m conntrack --ctstate NEW -m connmark --mark 0x0/0xff200000 -m comment --comment \"from w2\" -j CONNMARK --set-xmark " + tagW2 + "/0xff200000\n", + } { + if !strings.Contains(r.Mangle, want) { + t.Fatalf("holder mangle lacks %q:\n%s", want, r.Mangle) + } + } + for _, want := range []string{ + "-A REWEAVE-LB-FWD -i " + toW2 + " -m conntrack --ctdir ORIGINAL -m connmark ! --mark " + tagW2 + "/0xff200000 -j DROP\n", + "-A REWEAVE-LB-FWD -i " + toW2 + " -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -p tcp -m tcp --sport 62024:62535 -m comment --comment \"block of w2\" -j RETURN\n", + "-A REWEAVE-LB-FWD -i " + toCP + " -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -p sctp -m sctp --sport 61000:61511 -m comment --comment \"block of cp\" -j RETURN\n", + "-A REWEAVE-LB-FWD -i " + toW2 + " -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -p udp -j DROP\n", + "-A REWEAVE-LB-FWD -i " + toW2 + " -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -j RETURN\n", "-A REWEAVE-LB-FWD -s 10.40.0.4/32 -o weave -m mark --mark 0x100000/0x100000 -j DROP\n", } { if !strings.Contains(r.Filter, want) { t.Fatalf("holder filter lacks %q:\n%s", want, r.Filter) } } + // foreign tags are refused before anything of that tunnel is accepted; a port outside the block before the + // catch-all accept + if strings.Index(r.Filter, "! --mark "+tagW2) > strings.Index(r.Filter, "\"block of w2\"") || + strings.Index(r.Filter, "-i "+toW2+" -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -p udp -j DROP") > + strings.Index(r.Filter, "-i "+toW2+" -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -j RETURN") { + t.Fatalf("holder filter order:\n%s", r.Filter) + } if !strings.Contains(r.Nat, "-A REWEAVE-LB-POST -s 10.40.0.4/32 ! -o weave -m mark --mark 0x100000/0x100000 -p tcp -j SNAT --to-source 86.107.168.15:61512-62023\n") { t.Fatalf("holder's own member:\n%s", r.Nat) } @@ -260,32 +293,56 @@ func TestRenderedEgressRules(t *testing.T) { } } +func TestTunnelNames(t *testing.T) { + a, b := TunnelName("reweave-worker-fe19b040"), TunnelName("reweave-cp-94a8db43") + if a == b || len(a) > 15 || !strings.HasPrefix(a, TunnelPrefix) || a != TunnelName("reweave-worker-fe19b040") { + t.Fatalf("tunnel names must be distinct, stable, short: %q %q", a, b) + } +} + func TestPolicyRules(t *testing.T) { e := egressFor("mail", written()) svcs := []*coreapi.Service{mailSvc()} h := ComputeEgress("w1", nodes, svcs, []*EgressService{e}, nil, nil) rules := wantRules(h, "w1") tables := peerTables(h, "w1") - if len(rules) != 4 { - t.Fatalf("2 peers x tcp,udp: %+v", rules) - } + idx := peerIndex(h, "w1") + ports, tags := 0, 0 for _, r := range rules { - if r.priority != HolderRulePriority || r.dst != "86.107.168.15/32" || r.src != "" || r.mark != 0 { + if r.priority != HolderRulePriority || r.src != "" { t.Fatalf("holder rule: %+v", r) } - if r.ipproto != unix.IPPROTO_TCP && r.ipproto != unix.IPPROTO_UDP { - t.Fatalf("protocol: %+v", r) - } - if (r.from == 62024) != (r.table == tables["w2"]) || (r.from == 61000) != (r.table == tables["cp"]) { - t.Fatalf("block routed to the wrong node: %+v %v", r, tables) + switch { + case r.ipproto != 0: + ports++ + if r.dst != "86.107.168.15/32" || r.mark != 0 { + t.Fatalf("port rule: %+v", r) + } + if (r.from == 62024) != (r.table == tables["w2"]) || (r.from == 61000) != (r.table == tables["cp"]) { + t.Fatalf("block routed to the wrong node: %+v %v", r, tables) + } + default: + tags++ + if r.mask != peerMask || r.dst != "" { + t.Fatalf("tag rule: %+v", r) + } + for name, i := range idx { + if r.mark == peerTag(i) && r.table != tables[name] { + t.Fatalf("tag of %s routed to table %d, want %d", name, r.table, tables[name]) + } + } } } + if ports != 2*len(portProtocols) || tags != 2 { + t.Fatalf("2 peers: %d port rules (tcp, udp, sctp each), %d tag rules: %+v", ports, tags, rules) + } if HolderRulePriority >= LocalRulePriority { t.Fatalf("holder rules must precede the local table") } m := ComputeEgress("w2", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{pod("mail-0", "w2", "10.44.0.9", mailLabels)}, nil) rules = wantRules(m, "w2") - if len(rules) != 1 || rules[0].src != "10.44.0.9/32" || rules[0].mark != EgressMark || rules[0].table != peerTables(m, "w2")["w1"] { + if len(rules) != 1 || rules[0].src != "10.44.0.9/32" || rules[0].mark != EgressMark || rules[0].mask != EgressMark || + rules[0].table != peerTables(m, "w2")["w1"] { t.Fatalf("member rule: %+v", rules) } u := ComputeEgress("w2", nodes, svcs, []*EgressService{egressFor("mail", EgressServiceStatus{})}, []*coreapi.Pod{pod("mail-0", "w2", "10.44.0.9", mailLabels)}, nil) diff --git a/reweave/test/egress/egress-test.yaml b/reweave/test/egress/egress-test.yaml index 096c83aa..36d99534 100644 --- a/reweave/test/egress/egress-test.yaml +++ b/reweave/test/egress/egress-test.yaml @@ -32,19 +32,25 @@ items: metadata: {name: member-remote, namespace: egtest, labels: {egress: member}} spec: nodeName: REMOTE - containers: [{name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"]}] + containers: + # NET_RAW: ping and traceroute (the runtime's default capabilities leave it out) + - {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"], securityContext: {capabilities: {add: [NET_RAW]}}} - apiVersion: v1 kind: Pod metadata: {name: member-other, namespace: egtest, labels: {egress: member}} spec: nodeName: OTHER - containers: [{name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"]}] + containers: + # NET_RAW: ping and traceroute (the runtime's default capabilities leave it out) + - {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"], securityContext: {capabilities: {add: [NET_RAW]}}} - apiVersion: v1 kind: Pod metadata: {name: member-holder, namespace: egtest, labels: {egress: member}} spec: nodeName: HOLDER - containers: [{name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"]}] + containers: + # NET_RAW: ping and traceroute (the runtime's default capabilities leave it out) + - {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"], securityContext: {capabilities: {add: [NET_RAW]}}} - apiVersion: v1 kind: Pod metadata: {name: member-dns, namespace: egtest, labels: {egress: member}} diff --git a/reweave/test/egress/run.sh b/reweave/test/egress/run.sh index 81c567e7..0ad3f6bb 100755 --- a/reweave/test/egress/run.sh +++ b/reweave/test/egress/run.sh @@ -10,10 +10,11 @@ # Asserts, in order: # 0. the holder writes the status: host, address, a port block for every node # 1. the first packet of a member started after the EgressService leaves as ADDR (CNI admission) -# A1/A2. members on every node leave as ADDR (TCP and UDP) and get their replies +# A1/A2. members on every node leave as ADDR (TCP and UDP) and get their replies; ping works from every node +# ICMP errors from routers on the way (traceroute's time-exceeded) reach a member on another node # A4. a non-member pod on the same node as a member leaves as its node (untouched), and no rule names it # A3. a member talking to a pod sees its own pod address there (east-west untouched) -# A5. naming an address nobody holds clears the status, and members' outside traffic is dropped, not leaked +# A5. naming an address nobody holds clears the status, and members' outside traffic (TCP, ping) is dropped # A6. deleting the EgressService removes its state: members leave as their node again, rules are gone # Leaves nothing behind (namespace egtest is deleted, also on failure). set -euo pipefail @@ -72,6 +73,19 @@ got=$(whoami_udp member-dns || true) [ "$got" = "$ADDR" ] || fail "A1: member-dns (UDP) left as '$got', want $ADDR" ok "A1/A2. members on $REMOTE, $OTHER and $HOLDER leave as $ADDR over TCP and UDP, replies come back" +for p in member-remote member-other member-holder; do + k -n egtest exec $p -- ping -c 2 -W 3 1.1.1.1 >/dev/null || fail "ping from $p got no reply" +done +ok "ping from members on all three nodes gets its replies" + +# hop 1 is the member's own node, hop 2 the holder (it answers its own address, the member cannot see it); every +# later hop is a router on the internet whose time-exceeded comes back to ADDR and must find the member +hops=$(k -n egtest exec member-remote -- traceroute -n -m 8 -q 1 -w 2 1.1.1.1 2>&1 || true) +echo "$hops" | awk '$1 >= 3 && $2 ~ /^[0-9]+\./ {found=1} END {exit !found}' \ + || fail "no router beyond the holder answered the member's traceroute: +$hops" +ok "ICMP errors from routers beyond the holder reach a member on $REMOTE (traceroute)" + got=$(whoami_tcp plain || true) [ "$got" = "$remote_ip" ] || fail "A4: non-member left as '$got', want its node $remote_ip" plain_ip=$(k -n egtest get pod plain -o "jsonpath='{.status.podIP}'") @@ -90,6 +104,9 @@ k -n egtest patch svc mail --type merge -p "'{\"spec\":{\"loadBalancerIP\":\"192 eventually 60 status_is " " || fail "A5: status not cleared: '$(status)'" got=$(whoami_tcp member-remote || true) [ -z "$got" ] || fail "A5: with no holder a member still got out, as '$got'" +if k -n egtest exec member-remote -- ping -c 1 -W 3 1.1.1.1 >/dev/null 2>&1; then + fail "A5: with no holder a member's ping still got out" +fi k -n egtest patch svc mail --type merge -p "'{\"spec\":{\"loadBalancerIP\":\"$ADDR\"}}'" >/dev/null eventually 60 status_is "$want" || fail "status not restored: '$(status)'" eventually 30 tcp_is member-remote "$ADDR" || fail "member did not get back out as $ADDR"