weave/lb/egress_datapath.go

package lb

import (
	"bytes"
	"fmt"
	"net"
	"os"
	"sort"

	"github.com/vishvananda/netlink"
	"golang.org/x/sys/unix"
)

const (
	// ClassChain (nat, appended to PREROUTING after kube-proxy's KUBE-SERVICES) marks members' new connections to
	// the outside. kube-proxy's DNAT ends the nat walk, so a connection to a Service never gets here.
	ClassChain = "REWEAVE-LB-CLASS"
	// FwdChain (filter, first in FORWARD) drops members' outside packets that would not leave as their address.
	FwdChain = "REWEAVE-LB-FWD"
	// InChain (filter, first in INPUT) accepts the egress tunnel only from the cluster's nodes.
	InChain = "REWEAVE-LB-IN"
	// MSSChain (mangle, first in FORWARD) clamps TCP MSS into the tunnel.
	MSSChain = "REWEAVE-LB-MSS"

	// Tunnel is the egress tunnel: ipip over weave between nodes' weave bridges, no fixed remote (each route names it).
	Tunnel = "reweave-eg"
	// EgressMark marks a member's packets to the outside (connmark on its connections, fwmark on each packet).
	EgressMark = uint32(0x100000)

	// HolderRulePriority: replies to a block go to its node; ahead of the local table, which moves to LocalRulePriority,
	// since the address is usually local on the holder.
	HolderRulePriority = 90
	LocalRulePriority  = 100
	// MemberRulePriority: a member's marked packets go to its holder (or nowhere).
	MemberRulePriority = 9100
	// EgressTableBase + peer index routes into the tunnel towards that peer; EgressTableBase itself is unreachable.
	EgressTableBase = 53000

	ipipOverhead = 20
)

// renderEgress appends the egress rules of each table to the bodies of S1's payloads.
func renderEgress(e EgressDesired, self string, podNet *net.IPNet, nat, mangle, filter *bytes.Buffer) {
	members := e.Members
	if len(members) > 0 {
		// what is not "outside": the pod network, multicast, the cluster's nodes
		fmt.Fprintf(nat, "-A %s -d %s -j RETURN\n", ClassChain, podNet)
		fmt.Fprintf(nat, "-A %s -d 224.0.0.0/4 -j RETURN\n", ClassChain)
		seen := map[string]bool{}
		for _, a := range e.NodeAddrs {
			if !seen[a.String()] {
				seen[a.String()] = true
				fmt.Fprintf(nat, "-A %s -d %s/32 -j RETURN\n", ClassChain, a)
			}
		}
		for _, m := range members {
			fmt.Fprintf(nat, "-A %s -s %s/32 -m comment --comment %q -j CONNMARK --set-xmark 0x%x/0x%x\n", ClassChain, m.IP, m.Pod, EgressMark, EgressMark)
			fmt.Fprintf(nat, "-A %s -s %s/32 -j MARK --set-xmark 0x%x/0x%x\n", ClassChain, m.IP, EgressMark, EgressMark)
		}
		// later packets of a member's outside connection carry the mark too (the nat table saw only the first)
		fmt.Fprintf(mangle, "-A %s -m conntrack --ctdir ORIGINAL -m connmark --mark 0x%x/0x%x -j MARK --set-xmark 0x%x/0x%x\n",
			MarkChain, EgressMark, EgressMark, EgressMark, EgressMark)
	}
	for _, m := range members {
		if m.Unreachable {
			fmt.Fprintf(filter, "-A %s -s %s/32 -m mark --mark 0x%x/0x%x -m comment --comment %q -j DROP\n", FwdChain, m.IP, EgressMark, EgressMark, m.Pod)
			continue
		}
		out := "-o " + Tunnel
		if m.Holder.Name == self {
			// emitted here, from the address: never into the pod network unSNATed
			out = "! -o " + bridge
			fmt.Fprintf(filter, "-A %s -s %s/32 -o %s -m mark --mark 0x%x/0x%x -j DROP\n", FwdChain, m.IP, bridge, EgressMark, EgressMark)
		}
		for _, proto := range []string{"tcp", "udp"} {
			fmt.Fprintf(nat, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -p %s -j SNAT --to-source %s:%d-%d\n",
				PostChain, m.IP, out, EgressMark, EgressMark, proto, m.Address, m.From, m.To)
			fmt.Fprintf(filter, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -p %s -j RETURN\n", FwdChain, m.IP, out, EgressMark, EgressMark, proto)
		}
		// anything else a member sends outside (other protocols, other ways out) does not leave
		fmt.Fprintf(filter, "-A %s -s %s/32 -m mark --mark 0x%x/0x%x -m comment --comment %q -j DROP\n", FwdChain, m.IP, EgressMark, EgressMark, m.Pod)
	}
	if !e.Tunnelled(self) {
		return
	}
	// tunnel arrivals: replies to members here (established), members' packets from their blocks (as holder)
	addrs := map[string]bool{}
	for _, m := range members {
		if !m.Unreachable && m.Holder.Name != self && !addrs[m.Address.String()] {
			addrs[m.Address.String()] = true
			// still addressed to the address after nat: no member connection matched it
			fmt.Fprintf(filter, "-A %s -i %s -d %s/32 -j DROP\n", FwdChain, Tunnel, m.Address)
		}
	}
	fmt.Fprintf(filter, "-A %s -i %s -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n", FwdChain, Tunnel)
	for _, r := range e.Routes {
		for _, proto := range []string{"tcp", "udp"} {
			fmt.Fprintf(filter, "-A %s -i %s -s %s/32 -p %s -m %s --sport %d:%d -m comment --comment %q -j RETURN\n",
				FwdChain, Tunnel, r.Address, proto, proto, r.From, r.To, "block of "+r.Peer.Name)
		}
	}
	fmt.Fprintf(filter, "-A %s -i %s -j DROP\n", FwdChain, Tunnel)
	// the tunnel itself only from the cluster's nodes' bridges
	for _, p := range e.Peers(self) {
		fmt.Fprintf(filter, "-A %s -p 4 -s %s/32 -m mac --mac-source %s -j RETURN\n", InChain, p.WeaveIP, p.WeaveMAC)
	}
	fmt.Fprintf(filter, "-A %s -p 4 -j DROP\n", InChain)
	fmt.Fprintf(mangle, "-A %s -o %s -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu\n", MSSChain, Tunnel)
}

// peerTables gives each tunnel peer its routing table (EgressTableBase + a stable index).
func peerTables(e EgressDesired, self string) map[string]int {
	out := map[string]int{}
	for _, h := range assignMarks(e.Peers(self)) {
		out[h.Node.Name] = EgressTableBase + int(h.Mark)
	}
	return out
}

// egressRule is a policy rule this package owns, in a comparable form.
type egressRule struct {
	priority int
	src, dst string
	ipproto  int
	from, to uint16
	mark     uint32
	table    int
}

func (r egressRule) key() string {
	return fmt.Sprintf("%d|%s|%s|%d|%d-%d|%x|%d", r.priority, r.src, r.dst, r.ipproto, r.from, r.to, r.mark, r.table)
}

// wantRules is every egress policy rule this node needs.
func wantRules(e EgressDesired, self string) []egressRule {
	tables := peerTables(e, self)
	var out []egressRule
	for _, r := range e.Routes {
		for _, proto := range []int{unix.IPPROTO_TCP, unix.IPPROTO_UDP} {
			out = append(out, egressRule{priority: HolderRulePriority, dst: r.Address.String() + "/32", ipproto: proto,
				from: uint16(r.From), to: uint16(r.To), table: tables[r.Peer.Name]})
		}
	}
	for _, m := range e.Members {
		if !m.Unreachable && m.Holder.Name == self {
			continue // emitted here: the main table already sends it out
		}
		table := EgressTableBase
		if !m.Unreachable {
			table = tables[m.Holder.Name]
		}
		out = append(out, egressRule{priority: MemberRulePriority, src: m.IP.String() + "/32", mark: EgressMark, table: table})
	}
	sort.Slice(out, func(i, j int) bool { return out[i].key() < out[j].key() })
	return out
}

func ruleOf(r netlink.Rule) egressRule {
	er := egressRule{priority: r.Priority, ipproto: r.IPProto, table: r.Table}
	if r.Src != nil {
		er.src = r.Src.String()
	}
	if r.Dst != nil {
		er.dst = r.Dst.String()
	}
	if r.Dport != nil {
		er.from, er.to = r.Dport.Start, r.Dport.End
	}
	if r.Mark > 0 {
		er.mark = uint32(r.Mark)
	}
	if er.ipproto < 0 {
		er.ipproto = 0
	}
	return er
}

// egressNet makes the tunnel, the local rule's place, the policy rules and the tunnel tables exactly what e needs.
func egressNet(e EgressDesired, self string, local Local) error {
	tunnelled := e.Tunnelled(self) // the device itself is made first, by Apply (ensureTunnel)
	if len(e.Routes) > 0 {
		if err := moveLocalRule(); err != nil {
			return err
		}
	}
	tables := peerTables(e, self)
	peers := map[int]Node{}
	for _, p := range e.Peers(self) {
		peers[tables[p.Name]] = p
	}
	unreachable := false
	for _, m := range e.Members {
		unreachable = unreachable || m.Unreachable
	}
	var link netlink.Link
	if tunnelled {
		l, err := netlink.LinkByName(Tunnel)
		if err != nil {
			return fmt.Errorf("egress tunnel: %v", err)
		}
		link = l
	}
	for table := EgressTableBase; table <= EgressTableBase+MaxMarks; table++ {
		var want *netlink.Route
		if p, ok := peers[table]; ok && link != nil {
			want = &netlink.Route{LinkIndex: link.Attrs().Index, Dst: &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)},
				Gw: p.WeaveIP, Table: table, Flags: int(netlink.FLAG_ONLINK)}
		} else if table == EgressTableBase && unreachable {
			want = &netlink.Route{Dst: &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)}, Table: table, Type: unix.RTN_UNREACHABLE}
		}
		routes, err := netlink.RouteListFiltered(unix.AF_INET, &netlink.Route{Table: table}, netlink.RT_FILTER_TABLE)
		if err != nil {
			return err
		}
		for _, rt := range routes {
			if want != nil && rt.Type == want.Type && rt.LinkIndex == want.LinkIndex && rt.Gw.Equal(want.Gw) {
				continue
			}
			rt := rt
			if err := netlink.RouteDel(&rt); err != nil {
				return fmt.Errorf("removing stale route in table %d: %v", table, err)
			}
		}
		if want != nil {
			if err := netlink.RouteReplace(want); err != nil {
				return fmt.Errorf("route in table %d: %v", table, err)
			}
		}
	}
	want := map[string]egressRule{}
	for _, r := range wantRules(e, self) {
		want[r.key()] = r
	}
	rules, err := netlink.RuleList(unix.AF_INET)
	if err != nil {
		return err
	}
	have := map[string]bool{}
	for _, r := range rules {
		if r.Priority != HolderRulePriority && r.Priority != MemberRulePriority {
			continue
		}
		k := ruleOf(r).key()
		if _, ok := want[k]; ok && !have[k] {
			have[k] = true
			continue
		}
		r := r
		if err := netlink.RuleDel(&r); err != nil {
			return fmt.Errorf("removing stale egress rule: %v", err)
		}
	}
	for k, r := range want {
		if have[k] {
			continue
		}
		rule := netlink.NewRule()
		rule.Priority = r.priority
		rule.Table = r.table
		if r.src != "" {
			_, rule.Src, _ = net.ParseCIDR(r.src)
		}
		if r.dst != "" {
			_, rule.Dst, _ = net.ParseCIDR(r.dst)
		}
		if r.ipproto != 0 {
			rule.IPProto = r.ipproto
			rule.Dport = netlink.NewRulePortRange(r.from, r.to)
		}
		if r.mark != 0 {
			rule.Mark = int(r.mark)
			rule.Mask = int(r.mark)
		}
		if err := netlink.RuleAdd(rule); err != nil {
			return fmt.Errorf("adding egress rule %s: %v", k, err)
		}
	}
	return nil
}

// moveLocalRule puts the local table's lookup after HolderRulePriority (it is at 0 by default), so the holder can
// route replies to its own address onwards to their node instead of delivering them to itself.
func moveLocalRule() error {
	rules, err := netlink.RuleList(unix.AF_INET)
	if err != nil {
		return err
	}
	var at0, atLocal bool
	for _, r := range rules {
		if r.Table == unix.RT_TABLE_LOCAL && r.Src == nil && r.Dst == nil && r.Mark <= 0 {
			switch r.Priority {
			case 0:
				at0 = true
			case LocalRulePriority:
				atLocal = true
			}
		}
	}
	if !atLocal {
		rule := netlink.NewRule()
		rule.Priority = LocalRulePriority
		rule.Table = unix.RT_TABLE_LOCAL
		if err := netlink.RuleAdd(rule); err != nil {
			return fmt.Errorf("adding local table rule at %d: %v", LocalRulePriority, err)
		}
	}
	if at0 {
		rule := netlink.NewRule()
		rule.Priority = 0
		rule.Table = unix.RT_TABLE_LOCAL
		if err := netlink.RuleDel(rule); err != nil {
			return fmt.Errorf("removing local table rule at 0: %v", err)
		}
	}
	return nil
}

// ensureTunnel creates (or removes) the egress tunnel: ipip with this node's bridge address as its local end.
func ensureTunnel(want bool, local Local) error {
	link, err := netlink.LinkByName(Tunnel)
	exists := err == nil
	if !want {
		if exists {
			return netlink.LinkDel(link)
		}
		return nil
	}
	if local.WeaveIP == nil {
		return fmt.Errorf("egress tunnel needs the weave bridge's address")
	}
	br, err := netlink.LinkByName(bridge)
	if err != nil {
		return fmt.Errorf("weave bridge: %v", err)
	}
	mtu := br.Attrs().MTU - ipipOverhead
	if exists {
		t, ok := link.(*netlink.Iptun)
		if !ok || !t.Local.Equal(local.WeaveIP) || t.Remote != nil && !t.Remote.IsUnspecified() || t.Attrs().MTU != mtu {
			if err := netlink.LinkDel(link); err != nil {
				return err
			}
			exists = false
		}
	}
	if !exists {
		t := &netlink.Iptun{LinkAttrs: netlink.LinkAttrs{Name: Tunnel, MTU: mtu}, Local: local.WeaveIP, PMtuDisc: 1}
		if err := netlink.LinkAdd(t); err != nil {
			return fmt.Errorf("creating egress tunnel: %v", err)
		}
		if link, err = netlink.LinkByName(Tunnel); err != nil {
			return err
		}
	}
	// the holder receives members' packets from its own address; only on this device
	for k, v := range map[string]string{"accept_local": "1", "rp_filter": "2"} {
		if err := os.WriteFile("/proc/sys/net/ipv4/conf/"+Tunnel+"/"+k, []byte(v), 0o644); err != nil {
			return fmt.Errorf("sysctl %s on %s: %v", k, Tunnel, err)
		}
	}
	return netlink.LinkSetUp(link)
}