package net import ( "encoding/binary" "errors" "fmt" "net" "strings" "github.com/vishvananda/netlink" "github.com/vishvananda/netlink/nl" "github.com/vishvananda/netns" "golang.org/x/sys/unix" ) // A pod's port is locked to the pod's own addresses: what the pod sends in through the host side of its veth has to // carry the pod's MAC as its Ethernet source and be either IPv4 from the pod's address or ARP speaking for exactly // that MAC and address (or probing, from 0.0.0.0). Everything else is dropped before the bridge sees it, so a pod can // claim neither another pod's identity nor a node's, on its own node or, through weave, on any other. // // The lock is a set of tc filters on the veth's ingress qdisc. They match the way iproute2 lays out "ether src" // (the Ethernet source at -8 and -4 from the network header) and "ip src" (at 12), and they go away with the veth. const ( portLockPrioIPv4 = 1 portLockPrioARP = 2 portLockPrioProbe = 3 portLockPrioDrop = 9 ) var ingressParent = netlink.MakeHandle(0xffff, 0) // PortLockFilters are the filters that lock the port with the given index to a pod's MAC and IPv4 address. func PortLockFilters(index int, mac net.HardwareAddr, ip net.IP) ([]netlink.Filter, error) { if len(mac) != 6 { return nil, fmt.Errorf("port lock needs an Ethernet MAC, got %q", mac) } ip4 := ip.To4() if ip4 == nil { return nil, fmt.Errorf("port lock needs an IPv4 address, got %q", ip) } src := etherSource(mac) arp := func(spa net.IP) []netlink.TcU32Key { keys := append([]netlink.TcU32Key{}, src...) return append(keys, // Ethernet/IPv4 ARP: hardware type 1, protocol 0x0800, address lengths 6 and 4 (any operation) netlink.TcU32Key{Mask: 0xffffffff, Val: 0x00010800, Off: 0}, netlink.TcU32Key{Mask: 0xffff0000, Val: 0x06040000, Off: 4}, // the sender hardware address (8..13) and the sender protocol address (14..17) netlink.TcU32Key{Mask: 0xffffffff, Val: binary.BigEndian.Uint32(mac[0:4]), Off: 8}, netlink.TcU32Key{Mask: 0xffffffff, Val: uint32(binary.BigEndian.Uint16(mac[4:6]))<<16 | uint32(binary.BigEndian.Uint16(spa[0:2])), Off: 12}, netlink.TcU32Key{Mask: 0xffff0000, Val: uint32(binary.BigEndian.Uint16(spa[2:4])) << 16, Off: 16}, ) } ipv4 := append(append([]netlink.TcU32Key{}, src...), netlink.TcU32Key{Mask: 0xffffffff, Val: binary.BigEndian.Uint32(ip4), Off: 12}) return []netlink.Filter{ portLockU32(index, portLockPrioIPv4, unix.ETH_P_IP, ipv4), portLockU32(index, portLockPrioARP, unix.ETH_P_ARP, arp(ip4)), portLockU32(index, portLockPrioProbe, unix.ETH_P_ARP, arp(net.IPv4zero.To4())), &netlink.MatchAll{ FilterAttrs: netlink.FilterAttrs{LinkIndex: index, Parent: ingressParent, Priority: portLockPrioDrop, Protocol: unix.ETH_P_ALL}, Actions: []netlink.Action{gact(netlink.TC_ACT_SHOT)}, }, }, nil } // etherSource matches the Ethernet source address, which sits 8 bytes before the network header. func etherSource(mac net.HardwareAddr) []netlink.TcU32Key { return []netlink.TcU32Key{ {Mask: 0xffffffff, Val: binary.BigEndian.Uint32(mac[0:4]), Off: -8}, {Mask: 0xffff0000, Val: uint32(binary.BigEndian.Uint16(mac[4:6])) << 16, Off: -4}, } } func portLockU32(index int, prio uint16, proto uint16, keys []netlink.TcU32Key) *netlink.U32 { // clipped: the vendored netlink serialises cap(Keys) keys, and spare capacity would go out as extra all-zero keys keys = keys[:len(keys):len(keys)] return &netlink.U32{ FilterAttrs: netlink.FilterAttrs{LinkIndex: index, Parent: ingressParent, Priority: prio, Protocol: proto}, Sel: &netlink.TcU32Sel{Flags: nl.TC_U32_TERMINAL, Keys: keys}, Actions: []netlink.Action{gact(netlink.TC_ACT_OK)}, } } func gact(action netlink.TcAct) netlink.Action { return &netlink.GenericAction{ActionAttrs: netlink.ActionAttrs{Action: action}} } func ingressQdisc(index int) *netlink.Ingress { return &netlink.Ingress{QdiscAttrs: netlink.QdiscAttrs{LinkIndex: index, Handle: ingressParent, Parent: netlink.HANDLE_INGRESS}} } // LockPort locks a pod's port (the host side of its veth) to the pod's MAC and IPv4 address. A port already locked // to them is left alone; anything else on its ingress is replaced, and the pod's own traffic is let through before // the rest is dropped, so a running pod never loses its own frames. func LockPort(link netlink.Link, mac net.HardwareAddr, ip net.IP) error { index := link.Attrs().Index want, err := PortLockFilters(index, mac, ip) if err != nil { return err } if locked, _ := portHasFilters(link, want); locked { return nil } // removing the ingress qdisc takes its filters with it if err := netlink.QdiscDel(ingressQdisc(index)); err != nil && !errors.Is(err, unix.ENOENT) && !errors.Is(err, unix.EINVAL) { return fmt.Errorf("clearing ingress of %s: %w", link.Attrs().Name, err) } if err := netlink.QdiscAdd(ingressQdisc(index)); err != nil { return fmt.Errorf("adding ingress qdisc to %s: %w", link.Attrs().Name, err) } for _, f := range want { if err := netlink.FilterAdd(f); err != nil { return fmt.Errorf("adding port lock filter (prio %d) to %s: %w", f.Attrs().Priority, link.Attrs().Name, err) } } return nil } // LockContainerPort locks the port of a pod attached by the CNI plugin: the host side of the veth whose container // end is ifName in ns, to that end's MAC and the pod's address. func LockContainerPort(ns netns.NsHandle, ifName string, ip net.IP) error { var mac net.HardwareAddr peer := 0 if err := WithNetNSLink(ns, ifName, func(link netlink.Link) error { // a veth's link attribute is its peer's index, in the peer's namespace: the host's mac, peer = link.Attrs().HardwareAddr, link.Attrs().ParentIndex return nil }); err != nil { return err } if peer == 0 { return fmt.Errorf("%s has no peer: not a veth", ifName) } host, err := netlink.LinkByIndex(peer) if err != nil { return fmt.Errorf("host side of %s: %w", ifName, err) } if host.Type() != "veth" || !strings.HasPrefix(host.Attrs().Name, vethPrefix+"pl") { return fmt.Errorf("host side of %s is %s (%s), not a weave pod veth", ifName, host.Attrs().Name, host.Type()) } return LockPort(host, mac, ip) } // PortLocked reports whether a port is locked to exactly this MAC and address. func PortLocked(link netlink.Link, mac net.HardwareAddr, ip net.IP) (bool, error) { want, err := PortLockFilters(link.Attrs().Index, mac, ip) if err != nil { return false, err } return portHasFilters(link, want) } func portHasFilters(link netlink.Link, want []netlink.Filter) (bool, error) { have, err := netlink.FilterList(link, ingressParent) if err != nil { return false, err } var got []netlink.Filter for _, f := range have { // u32 lists the hash table that holds a filter as a filter of its own, without a selector if u, ok := f.(*netlink.U32); ok && u.Sel == nil { continue } got = append(got, f) } if len(got) != len(want) { return false, nil } for _, w := range want { found := false for _, g := range got { if sameFilter(g, w) { found = true break } } if !found { return false, nil } } return true, nil } func sameFilter(a, b netlink.Filter) bool { if a.Attrs().Priority != b.Attrs().Priority || a.Attrs().Protocol != b.Attrs().Protocol || a.Type() != b.Type() { return false } switch fa := a.(type) { case *netlink.U32: fb := b.(*netlink.U32) if fa.Sel == nil || fb.Sel == nil || len(fa.Sel.Keys) != len(fb.Sel.Keys) || fa.Sel.Flags&nl.TC_U32_TERMINAL == 0 { return false } for i := range fa.Sel.Keys { ka, kb := fa.Sel.Keys[i], fb.Sel.Keys[i] if ka.Mask != kb.Mask || ka.Val != kb.Val || ka.Off != kb.Off { return false } } return sameActions(fa.Actions, fb.Actions) case *netlink.MatchAll: return sameActions(fa.Actions, b.(*netlink.MatchAll).Actions) } return false } func sameActions(a, b []netlink.Action) bool { if len(a) != len(b) { return false } for i := range a { ga, ok1 := a[i].(*netlink.GenericAction) gb, ok2 := b[i].(*netlink.GenericAction) if !ok1 || !ok2 || ga.Attrs().Action != gb.Attrs().Action { return false } } return true }