diff --git a/net/portlock.go b/net/portlock.go new file mode 100644 index 00000000..bee383b6 --- /dev/null +++ b/net/portlock.go @@ -0,0 +1,220 @@ +package net + +import ( + "encoding/binary" + "errors" + "fmt" + "net" + "strings" + + "github.com/vishvananda/netlink" + "github.com/vishvananda/netlink/nl" + "github.com/vishvananda/netns" + "golang.org/x/sys/unix" +) + +// A pod's port is locked to the pod's own addresses: what the pod sends in through the host side of its veth has to +// carry the pod's MAC as its Ethernet source and be either IPv4 from the pod's address or ARP speaking for exactly +// that MAC and address (or probing, from 0.0.0.0). Everything else is dropped before the bridge sees it, so a pod can +// claim neither another pod's identity nor a node's, on its own node or, through weave, on any other. +// +// The lock is a set of tc filters on the veth's ingress qdisc. They match the way iproute2 lays out "ether src" +// (the Ethernet source at -8 and -4 from the network header) and "ip src" (at 12), and they go away with the veth. + +const ( + portLockPrioIPv4 = 1 + portLockPrioARP = 2 + portLockPrioProbe = 3 + portLockPrioDrop = 9 +) + +var ingressParent = netlink.MakeHandle(0xffff, 0) + +// PortLockFilters are the filters that lock the port with the given index to a pod's MAC and IPv4 address. +func PortLockFilters(index int, mac net.HardwareAddr, ip net.IP) ([]netlink.Filter, error) { + if len(mac) != 6 { + return nil, fmt.Errorf("port lock needs an Ethernet MAC, got %q", mac) + } + ip4 := ip.To4() + if ip4 == nil { + return nil, fmt.Errorf("port lock needs an IPv4 address, got %q", ip) + } + src := etherSource(mac) + arp := func(spa net.IP) []netlink.TcU32Key { + keys := append([]netlink.TcU32Key{}, src...) + return append(keys, + // Ethernet/IPv4 ARP: hardware type 1, protocol 0x0800, address lengths 6 and 4 (any operation) + netlink.TcU32Key{Mask: 0xffffffff, Val: 0x00010800, Off: 0}, + netlink.TcU32Key{Mask: 0xffff0000, Val: 0x06040000, Off: 4}, + // the sender hardware address (8..13) and the sender protocol address (14..17) + netlink.TcU32Key{Mask: 0xffffffff, Val: binary.BigEndian.Uint32(mac[0:4]), Off: 8}, + netlink.TcU32Key{Mask: 0xffffffff, Val: uint32(binary.BigEndian.Uint16(mac[4:6]))<<16 | uint32(binary.BigEndian.Uint16(spa[0:2])), Off: 12}, + netlink.TcU32Key{Mask: 0xffff0000, Val: uint32(binary.BigEndian.Uint16(spa[2:4])) << 16, Off: 16}, + ) + } + ipv4 := append(append([]netlink.TcU32Key{}, src...), netlink.TcU32Key{Mask: 0xffffffff, Val: binary.BigEndian.Uint32(ip4), Off: 12}) + return []netlink.Filter{ + portLockU32(index, portLockPrioIPv4, unix.ETH_P_IP, ipv4), + portLockU32(index, portLockPrioARP, unix.ETH_P_ARP, arp(ip4)), + portLockU32(index, portLockPrioProbe, unix.ETH_P_ARP, arp(net.IPv4zero.To4())), + &netlink.MatchAll{ + FilterAttrs: netlink.FilterAttrs{LinkIndex: index, Parent: ingressParent, Priority: portLockPrioDrop, Protocol: unix.ETH_P_ALL}, + Actions: []netlink.Action{gact(netlink.TC_ACT_SHOT)}, + }, + }, nil +} + +// etherSource matches the Ethernet source address, which sits 8 bytes before the network header. +func etherSource(mac net.HardwareAddr) []netlink.TcU32Key { + return []netlink.TcU32Key{ + {Mask: 0xffffffff, Val: binary.BigEndian.Uint32(mac[0:4]), Off: -8}, + {Mask: 0xffff0000, Val: uint32(binary.BigEndian.Uint16(mac[4:6])) << 16, Off: -4}, + } +} + +func portLockU32(index int, prio uint16, proto uint16, keys []netlink.TcU32Key) *netlink.U32 { + // clipped: the vendored netlink serialises cap(Keys) keys, and spare capacity would go out as extra all-zero keys + keys = keys[:len(keys):len(keys)] + return &netlink.U32{ + FilterAttrs: netlink.FilterAttrs{LinkIndex: index, Parent: ingressParent, Priority: prio, Protocol: proto}, + Sel: &netlink.TcU32Sel{Flags: nl.TC_U32_TERMINAL, Keys: keys}, + Actions: []netlink.Action{gact(netlink.TC_ACT_OK)}, + } +} + +func gact(action netlink.TcAct) netlink.Action { + return &netlink.GenericAction{ActionAttrs: netlink.ActionAttrs{Action: action}} +} + +func ingressQdisc(index int) *netlink.Ingress { + return &netlink.Ingress{QdiscAttrs: netlink.QdiscAttrs{LinkIndex: index, Handle: ingressParent, Parent: netlink.HANDLE_INGRESS}} +} + +// LockPort locks a pod's port (the host side of its veth) to the pod's MAC and IPv4 address. A port already locked +// to them is left alone; anything else on its ingress is replaced, and the pod's own traffic is let through before +// the rest is dropped, so a running pod never loses its own frames. +func LockPort(link netlink.Link, mac net.HardwareAddr, ip net.IP) error { + index := link.Attrs().Index + want, err := PortLockFilters(index, mac, ip) + if err != nil { + return err + } + if locked, _ := portHasFilters(link, want); locked { + return nil + } + // removing the ingress qdisc takes its filters with it + if err := netlink.QdiscDel(ingressQdisc(index)); err != nil && !errors.Is(err, unix.ENOENT) && !errors.Is(err, unix.EINVAL) { + return fmt.Errorf("clearing ingress of %s: %w", link.Attrs().Name, err) + } + if err := netlink.QdiscAdd(ingressQdisc(index)); err != nil { + return fmt.Errorf("adding ingress qdisc to %s: %w", link.Attrs().Name, err) + } + for _, f := range want { + if err := netlink.FilterAdd(f); err != nil { + return fmt.Errorf("adding port lock filter (prio %d) to %s: %w", f.Attrs().Priority, link.Attrs().Name, err) + } + } + return nil +} + +// LockContainerPort locks the port of a pod attached by the CNI plugin: the host side of the veth whose container +// end is ifName in ns, to that end's MAC and the pod's address. +func LockContainerPort(ns netns.NsHandle, ifName string, ip net.IP) error { + var mac net.HardwareAddr + peer := 0 + if err := WithNetNSLink(ns, ifName, func(link netlink.Link) error { + // a veth's link attribute is its peer's index, in the peer's namespace: the host's + mac, peer = link.Attrs().HardwareAddr, link.Attrs().ParentIndex + return nil + }); err != nil { + return err + } + if peer == 0 { + return fmt.Errorf("%s has no peer: not a veth", ifName) + } + host, err := netlink.LinkByIndex(peer) + if err != nil { + return fmt.Errorf("host side of %s: %w", ifName, err) + } + if host.Type() != "veth" || !strings.HasPrefix(host.Attrs().Name, vethPrefix+"pl") { + return fmt.Errorf("host side of %s is %s (%s), not a weave pod veth", ifName, host.Attrs().Name, host.Type()) + } + return LockPort(host, mac, ip) +} + +// PortLocked reports whether a port is locked to exactly this MAC and address. +func PortLocked(link netlink.Link, mac net.HardwareAddr, ip net.IP) (bool, error) { + want, err := PortLockFilters(link.Attrs().Index, mac, ip) + if err != nil { + return false, err + } + return portHasFilters(link, want) +} + +func portHasFilters(link netlink.Link, want []netlink.Filter) (bool, error) { + have, err := netlink.FilterList(link, ingressParent) + if err != nil { + return false, err + } + var got []netlink.Filter + for _, f := range have { + // u32 lists the hash table that holds a filter as a filter of its own, without a selector + if u, ok := f.(*netlink.U32); ok && u.Sel == nil { + continue + } + got = append(got, f) + } + if len(got) != len(want) { + return false, nil + } + for _, w := range want { + found := false + for _, g := range got { + if sameFilter(g, w) { + found = true + break + } + } + if !found { + return false, nil + } + } + return true, nil +} + +func sameFilter(a, b netlink.Filter) bool { + if a.Attrs().Priority != b.Attrs().Priority || a.Attrs().Protocol != b.Attrs().Protocol || a.Type() != b.Type() { + return false + } + switch fa := a.(type) { + case *netlink.U32: + fb := b.(*netlink.U32) + if fa.Sel == nil || fb.Sel == nil || len(fa.Sel.Keys) != len(fb.Sel.Keys) || fa.Sel.Flags&nl.TC_U32_TERMINAL == 0 { + return false + } + for i := range fa.Sel.Keys { + ka, kb := fa.Sel.Keys[i], fb.Sel.Keys[i] + if ka.Mask != kb.Mask || ka.Val != kb.Val || ka.Off != kb.Off { + return false + } + } + return sameActions(fa.Actions, fb.Actions) + case *netlink.MatchAll: + return sameActions(fa.Actions, b.(*netlink.MatchAll).Actions) + } + return false +} + +func sameActions(a, b []netlink.Action) bool { + if len(a) != len(b) { + return false + } + for i := range a { + ga, ok1 := a[i].(*netlink.GenericAction) + gb, ok2 := b[i].(*netlink.GenericAction) + if !ok1 || !ok2 || ga.Attrs().Action != gb.Attrs().Action { + return false + } + } + return true +} diff --git a/net/portlock_kernel_test.go b/net/portlock_kernel_test.go new file mode 100644 index 00000000..8b6df396 --- /dev/null +++ b/net/portlock_kernel_test.go @@ -0,0 +1,228 @@ +package net + +import ( + "bytes" + "encoding/binary" + "net" + "os" + "runtime" + "testing" + "time" + + "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" + "golang.org/x/sys/unix" +) + +// TestPortLockInKernel builds a bridge with two pods in throwaway network namespaces, locks pod A's port and sends +// frames from A: what A may send reaches B, what claims anything but A's own addresses does not. It needs root. +func TestPortLockInKernel(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("needs root (creates network namespaces)") + } + runtime.LockOSThread() + defer runtime.UnlockOSThread() + orig, err := netns.Get() + if err != nil { + t.Fatal(err) + } + defer orig.Close() + defer netns.Set(orig) + + hub, err := netns.New() + if err != nil { + t.Fatal(err) + } + defer hub.Close() + podA, _ := netns.New() + defer podA.Close() + podB, _ := netns.New() + defer podB.Close() + must(t, netns.Set(hub)) + + br := &netlink.Bridge{LinkAttrs: netlink.LinkAttrs{Name: "br0"}} + must(t, netlink.LinkAdd(br)) + must(t, netlink.LinkSetUp(br)) + ipA, ipB := net.IPv4(10, 99, 0, 1).To4(), net.IPv4(10, 99, 0, 2).To4() + hostA, macA := attach(t, hub, podA, br, "vethwepltsta", ipA) + _, macB := attach(t, hub, podB, br, "vethwepltstb", ipB) + + // as the CNI plugin does it: from the pod's end of the veth, in the pod's namespace + must(t, LockContainerPort(podA, "eth0", ipA)) + if ok, err := PortLocked(hostA, macA, ipA); err != nil || !ok { + dumpIngress(t, hostA) + t.Fatalf("port not locked after LockContainerPort: %v %v", ok, err) + } + // locking again changes nothing (the filters are compared, not rebuilt) + must(t, LockPort(hostA, macA, ipA)) + if ok, _ := PortLocked(hostA, macA, ipA); !ok { + t.Fatal("port lost its lock on a repeated LockPort") + } + + send := socketIn(t, podA, "eth0") + recv := socketIn(t, podB, "eth0") + defer unix.Close(send) + defer unix.Close(recv) + other, _ := net.ParseMAC("02:00:00:00:00:99") + spoofIP := net.IPv4(10, 99, 0, 200).To4() + + cases := []struct { + name string + frame []byte + passes bool + }{ + {"ipv4 from its own addresses", ipv4Frame(macB, macA, ipA, ipB, "own-ip"), true}, + {"ipv4 from another MAC", ipv4Frame(macB, other, ipA, ipB, "mac-spoof"), false}, + {"ipv4 from another address", ipv4Frame(macB, macA, spoofIP, ipB, "ip-spoof"), false}, + {"arp for itself", arpFrame(macA, macA, ipA, ipB, "own-arp"), true}, + {"arp probe", arpFrame(macA, macA, net.IPv4zero.To4(), ipB, "probe"), true}, + {"arp claiming another address", arpFrame(macA, macA, spoofIP, ipB, "arp-ip-spoof"), false}, + {"arp claiming another MAC", arpFrame(macA, other, ipA, ipB, "arp-sha-spoof"), false}, + {"arp from another MAC", arpFrame(other, other, ipA, ipB, "arp-eth-spoof"), false}, + {"ipv6", etherFrame(macB, macA, 0x86dd, []byte("ipv6-frame")), false}, + {"other ethertype", etherFrame(macB, macA, 0x88b5, []byte("experimental")), false}, + } + for _, c := range cases { + drain(recv) + if _, err := unix.Write(send, c.frame); err != nil { + t.Fatalf("%s: send: %v", c.name, err) + } + got := sawPayload(recv, c.frame[len(c.frame)-12:], 300*time.Millisecond) + if got != c.passes { + t.Errorf("%s: reached B = %v, want %v", c.name, got, c.passes) + } + } +} + +func dumpIngress(t *testing.T, link netlink.Link) { + have, _ := netlink.FilterList(link, netlink.MakeHandle(0xffff, 0)) + for _, f := range have { + t.Logf("have %s %+v", f.Type(), f.Attrs()) + if u, ok := f.(*netlink.U32); ok && u.Sel != nil { + t.Logf(" sel flags %x keys %+v actions %+v", u.Sel.Flags, u.Sel.Keys, u.Actions) + } + if m, ok := f.(*netlink.MatchAll); ok { + t.Logf(" actions %+v", m.Actions) + } + } +} + +func must(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } +} + +// attach plugs a pod namespace into the bridge through a veth named host on the hub side, eth0 on the pod side. +func attach(t *testing.T, hub, pod netns.NsHandle, br netlink.Link, host string, ip net.IP) (netlink.Link, net.HardwareAddr) { + t.Helper() + must(t, netlink.LinkAdd(&netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: host}, PeerName: "tmp" + host[len(host)-2:]})) + h, err := netlink.LinkByName(host) + must(t, err) + must(t, netlink.LinkSetMaster(h, br)) + must(t, netlink.LinkSetUp(h)) + p, err := netlink.LinkByName("tmp" + host[len(host)-2:]) + must(t, err) + must(t, netlink.LinkSetNsFd(p, int(pod))) + must(t, netns.Set(pod)) + p, err = netlink.LinkByName("tmp" + host[len(host)-2:]) + must(t, err) + must(t, netlink.LinkSetName(p, "eth0")) + must(t, netlink.AddrAdd(p, &netlink.Addr{IPNet: &net.IPNet{IP: ip, Mask: net.CIDRMask(24, 32)}})) + must(t, netlink.LinkSetUp(p)) + mac := p.Attrs().HardwareAddr + must(t, netns.Set(hub)) + return h, mac +} + +// socketIn opens a raw packet socket on an interface of a namespace (the socket stays in that namespace). +func socketIn(t *testing.T, ns netns.NsHandle, ifName string) int { + t.Helper() + cur, _ := netns.Get() + defer cur.Close() + must(t, netns.Set(ns)) + defer netns.Set(cur) + link, err := netlink.LinkByName(ifName) + must(t, err) + fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(htons(unix.ETH_P_ALL))) + must(t, err) + must(t, unix.Bind(fd, &unix.SockaddrLinklayer{Protocol: htons(unix.ETH_P_ALL), Ifindex: link.Attrs().Index})) + return fd +} + +func htons(v uint16) uint16 { return v<<8 | v>>8 } + +func drain(fd int) { + buf := make([]byte, 2048) + for { + if _, _, err := unix.Recvfrom(fd, buf, unix.MSG_DONTWAIT); err != nil { + return + } + } +} + +func sawPayload(fd int, marker []byte, within time.Duration) bool { + buf := make([]byte, 2048) + deadline := time.Now().Add(within) + for time.Now().Before(deadline) { + n, _, err := unix.Recvfrom(fd, buf, unix.MSG_DONTWAIT) + if err != nil { + time.Sleep(5 * time.Millisecond) + continue + } + if bytes.Contains(buf[:n], marker) { + return true + } + } + return false +} + +func etherFrame(dst, src net.HardwareAddr, ethertype uint16, payload []byte) []byte { + f := make([]byte, 14, 14+len(payload)+12) + copy(f[0:6], dst) + copy(f[6:12], src) + binary.BigEndian.PutUint16(f[12:14], ethertype) + f = append(f, payload...) + // a fixed-size tail the receiver looks for, so each case is told apart + tail := make([]byte, 12) + copy(tail, payload) + return append(f, tail...) +} + +func ipv4Frame(dst, src net.HardwareAddr, from, to net.IP, tag string) []byte { + body := []byte(tag) + h := make([]byte, 20) + h[0] = 0x45 + binary.BigEndian.PutUint16(h[2:4], uint16(20+len(body)+12)) + h[8] = 64 + h[9] = 253 // experimental protocol: nothing answers it + copy(h[12:16], from) + copy(h[16:20], to) + binary.BigEndian.PutUint16(h[10:12], checksum(h)) + return etherFrame(dst, src, unix.ETH_P_IP, append(h, body...)) +} + +func arpFrame(ethSrc, sha net.HardwareAddr, spa, tpa net.IP, tag string) []byte { + a := make([]byte, 28) + binary.BigEndian.PutUint16(a[0:2], 1) + binary.BigEndian.PutUint16(a[2:4], unix.ETH_P_IP) + a[4], a[5] = 6, 4 + binary.BigEndian.PutUint16(a[6:8], 1) // request + copy(a[8:14], sha) + copy(a[14:18], spa) + copy(a[24:28], tpa) + bcast, _ := net.ParseMAC("ff:ff:ff:ff:ff:ff") + return etherFrame(bcast, ethSrc, unix.ETH_P_ARP, append(a, []byte(tag)...)) +} + +func checksum(b []byte) uint16 { + var s uint32 + for i := 0; i+1 < len(b); i += 2 { + s += uint32(binary.BigEndian.Uint16(b[i:])) + } + for s>>16 != 0 { + s = s&0xffff + s>>16 + } + return ^uint16(s) +} diff --git a/net/portlock_test.go b/net/portlock_test.go new file mode 100644 index 00000000..9a84de14 --- /dev/null +++ b/net/portlock_test.go @@ -0,0 +1,115 @@ +package net + +import ( + "net" + "testing" + + "github.com/vishvananda/netlink" + "github.com/vishvananda/netlink/nl" + "golang.org/x/sys/unix" +) + +// The keys iproute2 writes for "u32 match ether src 02:11:22:33:44:55 match ip src 10.32.0.7/32" (tc filter show, +// iproute2 6.x): the port lock has to match the same bytes. +func TestPortLockMatchesIproute2Layout(t *testing.T) { + mac, _ := net.ParseMAC("02:11:22:33:44:55") + fs, err := PortLockFilters(7, mac, net.ParseIP("10.32.0.7")) + if err != nil { + t.Fatal(err) + } + if len(fs) != 4 { + t.Fatalf("want 4 filters, got %d", len(fs)) + } + ipv4 := fs[0].(*netlink.U32) + want := []netlink.TcU32Key{ + {Val: 0x02112233, Mask: 0xffffffff, Off: -8}, + {Val: 0x44550000, Mask: 0xffff0000, Off: -4}, + {Val: 0x0a200007, Mask: 0xffffffff, Off: 12}, + } + if ipv4.Protocol != unix.ETH_P_IP || ipv4.Priority != portLockPrioIPv4 || ipv4.Sel.Flags&nl.TC_U32_TERMINAL == 0 { + t.Fatalf("ipv4 filter attrs: %+v flags %x", ipv4.FilterAttrs, ipv4.Sel.Flags) + } + assertKeys(t, "ipv4", ipv4.Sel.Keys, want) + for _, f := range fs { + if f.Attrs().LinkIndex != 7 || f.Attrs().Parent != netlink.MakeHandle(0xffff, 0) { + t.Fatalf("filter not on ingress of link 7: %+v", f.Attrs()) + } + } +} + +func TestPortLockARP(t *testing.T) { + mac, _ := net.ParseMAC("02:11:22:33:44:55") + fs, _ := PortLockFilters(7, mac, net.ParseIP("10.32.0.7")) + common := []netlink.TcU32Key{ + {Val: 0x02112233, Mask: 0xffffffff, Off: -8}, + {Val: 0x44550000, Mask: 0xffff0000, Off: -4}, + {Val: 0x00010800, Mask: 0xffffffff, Off: 0}, + {Val: 0x06040000, Mask: 0xffff0000, Off: 4}, + {Val: 0x02112233, Mask: 0xffffffff, Off: 8}, + } + own := append(append([]netlink.TcU32Key{}, common...), + netlink.TcU32Key{Val: 0x44550a20, Mask: 0xffffffff, Off: 12}, // sha 44:55, spa 10.32 + netlink.TcU32Key{Val: 0x00070000, Mask: 0xffff0000, Off: 16}, // spa .0.7 + ) + probe := append(append([]netlink.TcU32Key{}, common...), + netlink.TcU32Key{Val: 0x44550000, Mask: 0xffffffff, Off: 12}, + netlink.TcU32Key{Val: 0x00000000, Mask: 0xffff0000, Off: 16}, + ) + arp, pr := fs[1].(*netlink.U32), fs[2].(*netlink.U32) + if arp.Protocol != unix.ETH_P_ARP || pr.Protocol != unix.ETH_P_ARP { + t.Fatal("ARP filters must be protocol arp") + } + assertKeys(t, "arp", arp.Sel.Keys, own) + assertKeys(t, "probe", pr.Sel.Keys, probe) + drop := fs[3].(*netlink.MatchAll) + if drop.Protocol != unix.ETH_P_ALL || drop.Priority != portLockPrioDrop { + t.Fatalf("drop filter attrs: %+v", drop.FilterAttrs) + } + if a := drop.Actions[0].(*netlink.GenericAction); a.Attrs().Action != netlink.TC_ACT_SHOT { + t.Fatalf("last filter must drop, has action %v", a.Attrs().Action) + } + for _, f := range fs[:3] { + if a := f.(*netlink.U32).Actions[0].(*netlink.GenericAction); a.Attrs().Action != netlink.TC_ACT_OK { + t.Fatalf("prio %d must pass, has action %v", f.Attrs().Priority, a.Attrs().Action) + } + } +} + +func TestPortLockRefusesWhatItCannotPin(t *testing.T) { + mac, _ := net.ParseMAC("02:11:22:33:44:55") + if _, err := PortLockFilters(7, mac, net.ParseIP("fd00::1")); err == nil { + t.Fatal("an IPv6 address must be refused: the lock pins IPv4 and ARP") + } + long, _ := net.ParseMAC("02:11:22:33:44:55:66:77") + if _, err := PortLockFilters(7, long, net.ParseIP("10.32.0.7")); err == nil { + t.Fatal("a non-Ethernet hardware address must be refused") + } +} + +func TestSameFilterComparesKeysAndActions(t *testing.T) { + mac, _ := net.ParseMAC("02:11:22:33:44:55") + a, _ := PortLockFilters(7, mac, net.ParseIP("10.32.0.7")) + b, _ := PortLockFilters(7, mac, net.ParseIP("10.32.0.8")) + if !sameFilter(a[0], a[0]) || sameFilter(a[0], b[0]) { + t.Fatal("ipv4 filters for different addresses must differ") + } + if !sameFilter(a[3], b[3]) { + t.Fatal("the drop filters are the same for every pod") + } + if sameFilter(a[1], a[2]) { + t.Fatal("the ARP and probe filters differ in priority and keys") + } +} + +func assertKeys(t *testing.T, what string, got, want []netlink.TcU32Key) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("%s: want %d keys, got %d: %+v", what, len(want), len(got), got) + } + for i := range want { + if got[i].Val != want[i].Val || got[i].Mask != want[i].Mask || got[i].Off != want[i].Off { + t.Fatalf("%s key %d: want %08x/%08x at %d, got %08x/%08x at %d", what, i, + want[i].Val, want[i].Mask, want[i].Off, got[i].Val, got[i].Mask, got[i].Off) + } + } +} diff --git a/plugin/net/cni.go b/plugin/net/cni.go index 3f4bc3a7..5108bc05 100644 --- a/plugin/net/cni.go +++ b/plugin/net/cni.go @@ -161,6 +161,11 @@ func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error { }); err != nil { return fmt.Errorf("error setting up routes: %s", err) } + // the pod may only speak as itself: its port drops whatever claims another MAC or address. The pod has not + // started yet; if its port cannot be locked, it does not start. + if err := weavenet.LockContainerPort(ns, args.IfName, ip.Address.IP); err != nil { + return fmt.Errorf("error locking the pod's port: %s", err) + } result.DNS = conf.DNS return types.PrintResult(result, conf.CNIVersion) diff --git a/prog/weave-kube/init.sh b/prog/weave-kube/init.sh index eb93e88c..9782d04f 100755 --- a/prog/weave-kube/init.sh +++ b/prog/weave-kube/init.sh @@ -63,4 +63,9 @@ if ! mkdir -p $HOST_ROOT/opt/cni/bin ; then fi mkdir -p $HOST_ROOT/etc/cni/net.d export HOST_ROOT -/home/weave/weave --local setup-cni \ No newline at end of file +/home/weave/weave --local setup-cni + +# Lock the ports of pods attached before the CNI plugin locked ports itself (it locks every pod it attaches from +# now on), from the runtime's record of each attachment. A port that cannot be locked stays as it was and is +# reported; weave starts regardless, since holding back the whole node's network would not make that port safer. +/usr/bin/weaveutil lock-ports "$HOST_ROOT/var/lib/cni/results" weave || echo "Some pods' ports could not be locked (see above)" >&2 diff --git a/prog/weave-kube/weave-daemonset-k8s-1.11.yaml b/prog/weave-kube/weave-daemonset-k8s-1.11.yaml index 83b3f686..e1de7041 100644 --- a/prog/weave-kube/weave-daemonset-k8s-1.11.yaml +++ b/prog/weave-kube/weave-daemonset-k8s-1.11.yaml @@ -245,6 +245,15 @@ items: - name: xtables-lock mountPath: /run/xtables.lock readOnly: false + # the runtime's record of each pod attachment, and the pods' network namespaces it names: init + # locks the ports of pods attached before the CNI plugin locked ports itself + - name: cni-cache + mountPath: /host/var/lib/cni + readOnly: true + - name: netns + mountPath: /var/run/netns + mountPropagation: HostToContainer + readOnly: true containers: - name: weave command: @@ -346,6 +355,14 @@ items: hostPath: path: /run/weave-npc type: DirectoryOrCreate + - name: cni-cache + hostPath: + path: /var/lib/cni + type: DirectoryOrCreate + - name: netns + hostPath: + path: /var/run/netns + type: DirectoryOrCreate priorityClassName: system-node-critical updateStrategy: type: RollingUpdate diff --git a/prog/weaveutil/main.go b/prog/weaveutil/main.go index 8f86f257..a28f3bc3 100644 --- a/prog/weaveutil/main.go +++ b/prog/weaveutil/main.go @@ -42,6 +42,7 @@ func init() { "rewrite-etc-hosts": rewriteEtcHosts, "get-db-flag": getDBFlag, "set-db-flag": setDBFlag, + "lock-ports": lockPorts, } } diff --git a/prog/weaveutil/portlock.go b/prog/weaveutil/portlock.go new file mode 100644 index 00000000..df7176bf --- /dev/null +++ b/prog/weaveutil/portlock.go @@ -0,0 +1,114 @@ +package main + +import ( + "encoding/json" + "fmt" + "net" + "os" + "path/filepath" + "strings" + + "github.com/vishvananda/netns" + + weavenet "github.com/rajch/weave/net" +) + +// lockPorts locks the ports of pods that were attached before the CNI plugin locked ports itself (it locks every +// pod it attaches). It works from the runtime's own record of each attachment, libcni's result cache: the pod's +// network namespace, the interface the plugin set up in it and the address it gave it. Ports already locked are +// left alone; pods whose namespace is gone are skipped. +func lockPorts(args []string) error { + if len(args) != 2 { + cmdUsage("lock-ports", " ") + } + dir, network := args[0], args[1] + entries, err := os.ReadDir(dir) + if os.IsNotExist(err) { + return nil // nothing attached yet + } + if err != nil { + return err + } + var locked, gone, failed int + for _, e := range entries { + if e.IsDir() || !strings.HasPrefix(e.Name(), network+"-") { + continue + } + data, err := os.ReadFile(filepath.Join(dir, e.Name())) + if err != nil { + fmt.Fprintf(os.Stderr, "lock-ports: %s: %v\n", e.Name(), err) + failed++ + continue + } + a, err := parseAttachment(data) + if err != nil { + fmt.Fprintf(os.Stderr, "lock-ports: %s: %v\n", e.Name(), err) + failed++ + continue + } + if a.network != network { + continue + } + ns, err := netns.GetFromPath(a.netns) + if os.IsNotExist(err) { + gone++ // the pod is gone; the runtime has yet to clear its record + continue + } + if err != nil { + fmt.Fprintf(os.Stderr, "lock-ports: %s: namespace %s: %v\n", a.containerID, a.netns, err) + failed++ + continue + } + err = weavenet.LockContainerPort(ns, a.ifName, a.ip) + ns.Close() + if err != nil { + fmt.Fprintf(os.Stderr, "lock-ports: %s (%s): %v\n", a.containerID, a.ip, err) + failed++ + continue + } + locked++ + } + fmt.Printf("lock-ports: %d locked, %d gone, %d failed\n", locked, gone, failed) + if failed > 0 { + return fmt.Errorf("lock-ports: %d pods' ports not locked", failed) + } + return nil +} + +type attachment struct { + network, containerID, netns, ifName string + ip net.IP +} + +// parseAttachment reads one libcni cache entry ("cniCacheV1"): which network attached which container, in which +// namespace, through which interface, with which IPv4 address. +func parseAttachment(data []byte) (attachment, error) { + var c struct { + Kind string `json:"kind"` + ContainerID string `json:"containerId"` + IfName string `json:"ifName"` + NetworkName string `json:"networkName"` + NetNS string `json:"netns"` + Result struct { + IPs []struct { + Address string `json:"address"` + } `json:"ips"` + } `json:"result"` + } + if err := json.Unmarshal(data, &c); err != nil { + return attachment{}, err + } + if c.Kind != "cniCacheV1" { + return attachment{}, fmt.Errorf("unknown cache entry kind %q", c.Kind) + } + if c.NetNS == "" || c.IfName == "" { + return attachment{}, fmt.Errorf("cache entry for %s names no namespace or interface", c.ContainerID) + } + for _, a := range c.Result.IPs { + ip, _, err := net.ParseCIDR(a.Address) + if err == nil && ip.To4() != nil { + return attachment{network: c.NetworkName, containerID: c.ContainerID, netns: c.NetNS, ifName: c.IfName, ip: ip.To4()}, nil + } + } + return attachment{}, fmt.Errorf("cache entry for %s has no IPv4 address", c.ContainerID) +} diff --git a/prog/weaveutil/portlock_test.go b/prog/weaveutil/portlock_test.go new file mode 100644 index 00000000..6fd41c3e --- /dev/null +++ b/prog/weaveutil/portlock_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "net" + "testing" +) + +// shaped like CRI-O's libcni cache on the SG nodes (cniCacheV1, CNI 1.0.0 result); the config is elided +const cacheEntry = `{"kind":"cniCacheV1","containerId":"2b61982132d6039b210e2fa5896555c8a921a07c756bd8753963440e07dc5f98", +"config":"eyJ9","ifName":"eth0","networkName":"weave","netns":"/var/run/netns/8828bbba-9a68-460c-980b-266dc3c9342c", +"cniArgs":[["K8S_POD_NAMESPACE","default"]],"capabilityArgs":{}, +"result":{"cniVersion":"1.0.0","ips":[{"address":"10.44.0.12/12","gateway":"10.44.0.0"}]}}` + +func TestParseAttachment(t *testing.T) { + a, err := parseAttachment([]byte(cacheEntry)) + if err != nil { + t.Fatal(err) + } + if a.network != "weave" || a.ifName != "eth0" || a.netns != "/var/run/netns/8828bbba-9a68-460c-980b-266dc3c9342c" || + !a.ip.Equal(net.ParseIP("10.44.0.12")) || a.containerID[:7] != "2b61982" { + t.Fatalf("parsed %+v", a) + } +} + +func TestParseAttachmentRefusesWhatItCannotLock(t *testing.T) { + for name, entry := range map[string]string{ + "other kind": `{"kind":"cniCacheV2","netns":"/x","ifName":"eth0","result":{"ips":[{"address":"10.0.0.1/8"}]}}`, + "no netns": `{"kind":"cniCacheV1","ifName":"eth0","result":{"ips":[{"address":"10.0.0.1/8"}]}}`, + "no ipv4": `{"kind":"cniCacheV1","netns":"/x","ifName":"eth0","result":{"ips":[{"address":"fd00::1/64"}]}}`, + "not json": `{`, + "no addresses": `{"kind":"cniCacheV1","netns":"/x","ifName":"eth0","result":{}}`, + } { + if _, err := parseAttachment([]byte(entry)); err == nil { + t.Errorf("%s: want an error", name) + } + } +} diff --git a/reweave/test/portlock/netns.py b/reweave/test/portlock/netns.py new file mode 100644 index 00000000..c8853e95 --- /dev/null +++ b/reweave/test/portlock/netns.py @@ -0,0 +1,14 @@ +# netns.py : the network namespace the weave CNI plugin attached a pod in, from the runtime's +# record of the attachment (libcni's result cache). Runs on the node. +import glob +import json +import sys + +namespace, pod = sys.argv[1], sys.argv[2] +for path in glob.glob("/var/lib/cni/results/weave-*"): + with open(path) as f: + entry = json.load(f) + args = dict(entry.get("cniArgs") or []) + if args.get("K8S_POD_NAMESPACE") == namespace and args.get("K8S_POD_NAME") == pod: + print(entry["netns"]) + break diff --git a/reweave/test/portlock/portlock-test.yaml b/reweave/test/portlock/portlock-test.yaml new file mode 100644 index 00000000..1ee18579 --- /dev/null +++ b/reweave/test/portlock/portlock-test.yaml @@ -0,0 +1,30 @@ +# Pods for the port lock e2e (run.sh): a forger and a victim on node A, a victim on node B. None has any added +# capability; the forged frames are sent from the forger's network namespace by root on the node. +apiVersion: v1 +kind: Namespace +metadata: + name: pltest +--- +apiVersion: v1 +kind: Pod +metadata: {name: forger, namespace: pltest} +spec: + nodeName: NODE_A + containers: + - {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"]} +--- +apiVersion: v1 +kind: Pod +metadata: {name: victim-local, namespace: pltest} +spec: + nodeName: NODE_A + containers: + - {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"]} +--- +apiVersion: v1 +kind: Pod +metadata: {name: victim-remote, namespace: pltest} +spec: + nodeName: NODE_B + containers: + - {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"]} diff --git a/reweave/test/portlock/probe/main.go b/reweave/test/portlock/probe/main.go new file mode 100644 index 00000000..a33d64d3 --- /dev/null +++ b/reweave/test/portlock/probe/main.go @@ -0,0 +1,200 @@ +// Command probe sends hand-made Ethernet frames out of an interface, or waits for one carrying a tag. The port lock +// e2e runs it inside pods' network namespaces (nsenter, as root on the node): a pod forging another's addresses +// must not reach anyone. +// +// probe send ipv4|arp +// probe listen +package main + +import ( + "bytes" + "encoding/binary" + "fmt" + "net" + "os" + "strconv" + "time" + + "golang.org/x/sys/unix" +) + +func main() { + if len(os.Args) < 2 { + usage() + } + var err error + switch os.Args[1] { + case "send": + if len(os.Args) != 9 { + usage() + } + err = send(os.Args[2], os.Args[3], os.Args[4], os.Args[5], os.Args[6], os.Args[7], os.Args[8]) + case "listen": + if len(os.Args) != 5 { + usage() + } + secs, _ := strconv.Atoi(os.Args[4]) + var seen bool + seen, err = listen(os.Args[2], os.Args[3], time.Duration(secs)*time.Second) + if err == nil { + if seen { + fmt.Println("seen") + return + } + fmt.Println("not seen") + os.Exit(1) + } + default: + usage() + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } +} + +func usage() { + fmt.Fprintln(os.Stderr, "usage: probe send ipv4|arp ") + fmt.Fprintln(os.Stderr, " probe listen ") + os.Exit(2) +} + +func send(ifName, dstMAC, srcMAC, kind, srcIP, dstIP, tag string) error { + iface, err := net.InterfaceByName(ifName) + if err != nil { + return err + } + dst, err := net.ParseMAC(dstMAC) + if err != nil { + return err + } + src := iface.HardwareAddr + if srcMAC != "own" { + if src, err = net.ParseMAC(srcMAC); err != nil { + return err + } + } + var from net.IP + if srcIP == "own" { + addrs, err := iface.Addrs() + if err != nil { + return err + } + for _, a := range addrs { + if n, ok := a.(*net.IPNet); ok && n.IP.To4() != nil { + from = n.IP.To4() + } + } + if from == nil { + return fmt.Errorf("%s has no IPv4 address", ifName) + } + } else if from = net.ParseIP(srcIP).To4(); from == nil { + return fmt.Errorf("bad source address %q", srcIP) + } + to := net.ParseIP(dstIP).To4() + if to == nil { + return fmt.Errorf("bad destination address %q", dstIP) + } + var frame []byte + switch kind { + case "ipv4": + frame = ipv4Frame(dst, src, from, to, []byte(tag)) + case "arp": + frame = arpFrame(dst, src, from, to, []byte(tag)) + default: + return fmt.Errorf("unknown kind %q", kind) + } + fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(htons(unix.ETH_P_ALL))) + if err != nil { + return err + } + defer unix.Close(fd) + if err := unix.Bind(fd, &unix.SockaddrLinklayer{Protocol: htons(unix.ETH_P_ALL), Ifindex: iface.Index}); err != nil { + return err + } + // a few copies, spaced out: the bridge and weave may still be learning where the destination is + for i := 0; i < 3; i++ { + if _, err := unix.Write(fd, frame); err != nil { + return err + } + time.Sleep(200 * time.Millisecond) + } + return nil +} + +func listen(ifName, tag string, within time.Duration) (bool, error) { + iface, err := net.InterfaceByName(ifName) + if err != nil { + return false, err + } + fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(htons(unix.ETH_P_ALL))) + if err != nil { + return false, err + } + defer unix.Close(fd) + if err := unix.Bind(fd, &unix.SockaddrLinklayer{Protocol: htons(unix.ETH_P_ALL), Ifindex: iface.Index}); err != nil { + return false, err + } + tv := unix.NsecToTimeval((100 * time.Millisecond).Nanoseconds()) + if err := unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &tv); err != nil { + return false, err + } + buf := make([]byte, 65536) + deadline := time.Now().Add(within) + for time.Now().Before(deadline) { + n, _, err := unix.Recvfrom(fd, buf, 0) + if err != nil { + continue + } + if bytes.Contains(buf[:n], []byte(tag)) { + return true, nil + } + } + return false, nil +} + +func htons(v uint16) uint16 { return v<<8 | v>>8 } + +func ether(dst, src net.HardwareAddr, ethertype uint16, payload []byte) []byte { + f := make([]byte, 14, 14+len(payload)) + copy(f[0:6], dst) + copy(f[6:12], src) + binary.BigEndian.PutUint16(f[12:14], ethertype) + return append(f, payload...) +} + +func ipv4Frame(dst, src net.HardwareAddr, from, to net.IP, body []byte) []byte { + h := make([]byte, 20) + h[0] = 0x45 + binary.BigEndian.PutUint16(h[2:4], uint16(20+len(body))) + h[8] = 64 + h[9] = 253 // experimental protocol: nothing answers it, nothing on the way acts on it + copy(h[12:16], from) + copy(h[16:20], to) + binary.BigEndian.PutUint16(h[10:12], checksum(h)) + return ether(dst, src, unix.ETH_P_IP, append(h, body...)) +} + +// arpFrame is an ARP request speaking for (src, from), asking for to; the tag rides as Ethernet trailer. +func arpFrame(dst, src net.HardwareAddr, from, to net.IP, tag []byte) []byte { + a := make([]byte, 28) + binary.BigEndian.PutUint16(a[0:2], 1) + binary.BigEndian.PutUint16(a[2:4], unix.ETH_P_IP) + a[4], a[5] = 6, 4 + binary.BigEndian.PutUint16(a[6:8], 1) + copy(a[8:14], src) + copy(a[14:18], from) + copy(a[24:28], to) + return ether(dst, src, unix.ETH_P_ARP, append(a, tag...)) +} + +func checksum(b []byte) uint16 { + var s uint32 + for i := 0; i+1 < len(b); i += 2 { + s += uint32(binary.BigEndian.Uint16(b[i:])) + } + for s>>16 != 0 { + s = s&0xffff + s>>16 + } + return ^uint16(s) +} diff --git a/reweave/test/portlock/run.sh b/reweave/test/portlock/run.sh new file mode 100755 index 00000000..e944b183 --- /dev/null +++ b/reweave/test/portlock/run.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# End-to-end test of the port lock against a live cluster (assertion A7): a pod speaks only as itself. +# +# CP="ssh ... root@" A_SSH="ssh ... root@" B_SSH="ssh ... root@" A= B= \ +# reweave/test/portlock/run.sh +# +# Asserts, in order: +# 0. every pod port on the three nodes (the CP's, A's and B's) carries the lock +# 1. a pod's own frames reach a pod on its node and a pod on another node (the control) +# 2. frames forging another pod's address reach neither +# 3. frames forging a node's bridge (its weave MAC and address) reach neither +# 4. ARP claiming another pod's address reaches no one; ARP for itself does +# The forged frames are sent from the forging pod's network namespace by root on its node (nsenter), the most a +# pod could ever do; the pod itself has no added capability. +# Leaves nothing behind (namespace pltest is deleted, also on failure; the probe is removed from the nodes). +set -euo pipefail + +: "${CP:?}" "${A_SSH:?}" "${B_SSH:?}" "${A:?}" "${B:?}" +here=$(cd "$(dirname "$0")" && pwd) +k() { $CP "KUBECONFIG=/etc/kubernetes/admin.conf kubectl $*"; } +fail() { echo "FAIL: $*" >&2; exit 1; } +ok() { echo "ok: $*"; } +work=$(mktemp -d) +cleanup() { + k delete namespace pltest --ignore-not-found --wait=false >/dev/null 2>&1 || true + for s in "$A_SSH" "$B_SSH"; do $s 'rm -f /root/plprobe /root/plnetns.py' >/dev/null 2>&1 || true; done + rm -rf "$work" +} +trap cleanup EXIT + +(cd "$here/../../.." && CGO_ENABLED=0 go build -o "$work/probe" ./reweave/test/portlock/probe/) +for s in "$A_SSH" "$B_SSH"; do + $s 'cat > /root/plprobe && chmod +x /root/plprobe' < "$work/probe" + $s 'cat > /root/plnetns.py' < "$here/netns.py" +done + +# a port is locked when its ingress carries the three passes and the drop +unlocked() { + $1 'for p in $(ip -br link show master weave | sed "s/@.*//; s/ .*//" | grep ^vethwepl); do + f=$(tc filter show dev $p ingress) + for want in "protocol ip pref 1 u32" "protocol arp pref 2 u32" "protocol arp pref 3 u32" \ + "protocol all pref 9 matchall" "gact action drop"; do + echo "$f" | grep -q "$want" || { echo "$p"; break; } + done + done' +} +all_locked() { + for s in "$CP" "$A_SSH" "$B_SSH"; do + u=$(unlocked "$s") + [ -z "$u" ] || fail "0. unlocked pod ports ($1): $u" + done +} +all_locked "before the test pods" +ok "0. every pod port on the three nodes carries the lock" + +sed -e "s/\bNODE_A\b/$A/g; s/\bNODE_B\b/$B/g" "$here/portlock-test.yaml" | + $CP "KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f -" >/dev/null +k -n pltest wait --for=condition=Ready pod --all --timeout=300s >/dev/null || fail "test pods not ready" +all_locked "with the test pods" + +netns_of() { $1 "python3 /root/plnetns.py pltest $2"; } +mac_in() { $1 "nsenter --net=$2 ip -br link show eth0" | awk '{print $3}'; } +pod_ip() { k -n pltest get pod "$1" -o "jsonpath='{.status.podIP}'"; } +f_ns=$(netns_of "$A_SSH" forger) +vl_ns=$(netns_of "$A_SSH" victim-local) +vr_ns=$(netns_of "$B_SSH" victim-remote) +[ -n "$f_ns" ] && [ -n "$vl_ns" ] && [ -n "$vr_ns" ] || fail "test pods' namespaces not found" +vl_mac=$(mac_in "$A_SSH" "$vl_ns") +vr_mac=$(mac_in "$B_SSH" "$vr_ns") +vl_ip=$(pod_ip victim-local) +vr_ip=$(pod_ip victim-remote) +# the nodes' bridges, from the annotation the lb agent writes ("ip,mac") +bridge_of() { k get node "$1" -o "jsonpath='{.metadata.annotations.reweave\.levk\.dev/weave}'"; } +b_bridge=$(bridge_of "$B"); b_ip=${b_bridge%,*}; b_mac=${b_bridge#*,} +a_bridge=$(bridge_of "$A"); a_ip=${a_bridge%,*}; a_mac=${a_bridge#*,} +[ -n "$b_ip" ] && [ -n "$b_mac" ] && [ -n "$a_ip" ] && [ -n "$a_mac" ] || fail "no weave bridge annotation on $A or $B" + +# expect : the victim listens while the forger sends +expect() { + local want=$1 where=$2 tag=$3-$RANDOM + shift 3 + local vs vns + if [ "$where" = local ]; then vs=$A_SSH vns=$vl_ns; else vs=$B_SSH vns=$vr_ns; fi + $vs "nsenter --net=$vns /root/plprobe listen eth0 $tag 6" > "$work/listen" 2>&1 & + local listener=$! + sleep 1 + $A_SSH "nsenter --net=$f_ns /root/plprobe send eth0 $* $tag" || fail "probe could not send: $*" + wait $listener || true + local got + got=$(cat "$work/listen") + [ "$got" = "$want" ] || fail "$tag ($where): '$got', want '$want'" +} + +expect "seen" local own-local "$vl_mac own ipv4 own $vl_ip" +expect "seen" remote own-remote "$vr_mac own ipv4 own $vr_ip" +ok "1. the forger's own frames reach a pod on its node and one on $B" + +expect "not seen" remote pod-ip-remote "$vr_mac own ipv4 $vl_ip $vr_ip" +expect "not seen" local pod-ip-local "$vl_mac own ipv4 $vr_ip $vl_ip" +expect "not seen" remote pod-mac-remote "$vr_mac $vl_mac ipv4 $vl_ip $vr_ip" +ok "2. frames forging another pod's address reach no one" + +expect "not seen" local node-b-local "$vl_mac $b_mac ipv4 $b_ip $vl_ip" +expect "not seen" remote node-b-remote "$vr_mac $b_mac ipv4 $b_ip $vr_ip" +expect "not seen" remote node-a-remote "$vr_mac $a_mac ipv4 $a_ip $vr_ip" +ok "3. frames forging a node's bridge ($A's, $B's) reach no one" + +expect "not seen" local arp-claim-local "ff:ff:ff:ff:ff:ff own arp $vr_ip $vl_ip" +expect "not seen" remote arp-claim-remote "ff:ff:ff:ff:ff:ff own arp $vl_ip $vr_ip" +expect "seen" remote arp-own-remote "ff:ff:ff:ff:ff:ff own arp own $vr_ip" +ok "4. ARP claiming another pod's address reaches no one; the forger's ARP for itself does" +echo "PASS"