diff --git a/.gitmodules b/.gitmodules index d653c81a..8bdf7cbc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,7 +3,7 @@ url = https://github.com/weaveworks/build-tools [submodule "vendor/github.com/vishvananda/netlink"] path = vendor/github.com/vishvananda/netlink - url = https://github.com/vishvananda/netlink.git + url = https://github.com/brb/netlink.git [submodule "vendor/github.com/gorilla/mux"] path = vendor/github.com/gorilla/mux url = https://github.com/gorilla/mux.git diff --git a/docs/fastdp-crypto.md b/docs/fastdp-crypto.md new file mode 100644 index 00000000..fc381485 --- /dev/null +++ b/docs/fastdp-crypto.md @@ -0,0 +1,151 @@ +This document describes implementation details of the fast datapath encryption. + +# Overview + +At the high level, we use the ESP protocol ([RFC 2406][esp]) in the Transport +mode. Each packet is encrypted with AES in GCM mode ([RFC 4106][aesgcm]), with +32byte key and 4byte salt. This combo provides the following security +properties: + +* Data confidentiality. +* Data origin authentication. +* Integrity. +* Anti-replay service. +* Limited traffic flow confidentiality. + +## Notation + +* `SAin`: IPsec security association for inbound connections. +* `SAout`: IPsec security association for outbound connections. +* `SPout`: IPsec security policy for outbound connections. Used to match + outbound flows to SAout. + +## SA Key Derivation + +For each connection direction, a different AES-GCM key and salt is used. +The pairs are derived with HKDF ([RFC 5869][hkdf]): + +``` +SAin[KeyAndSalt] = HKDF(sha256, ikm=sessionKey, salt=nonceIn, info=localPeerName) +SAout[KeyAndSalt] = HKDF(sha256, ikm=sessionKey, salt=nonceOut, info=remotePeerName) +``` + +Where: + +* the mutual `sessionKey` is derived by the [Mesh][mesh] library during + the control plane connection establishment between the local peer and + the remote peer. +* `nonceIn` and `nonceOut` are randomly generated 32byte nonces which + are exchanged over the encrypted control plane channel. + +## SPI + +A directional secure connection between two peers is identified with SPI. + +The kernel requires the pair of SPI and dst IP to be unique among security +associations. Thus, to avoid collisions, we generate outbound SPI on a remote +peer. + +## Connection Establishment + +``` +Peer A Peer B +----------------------------------------------------------------------------------------------------------------- + +fastdp.fwd.Confirm(): + install iptables blocking rules, fastdp.fwd.Confirm(): + nonce_BA = rand(), install iptables blocking rules, + {key,salt}_BA = hkdf(sessionKey, nonce_BA, A), nonce_AB = rand(), + spi_BA = allocspi(), {key,salt}_AB = hkdf(sessionKey, nonce_AB, B), + create SA_BA(B<-A, spi_BA, key_BA, salt_BA), spi_AB = allocspi(), + send InitSARemote(spi_BA, nonce_BA). --> create SA_AB(A<-B, spi_AB, key_AB, salt_AB), + <-- send InitSARemote(spi_AB, nonce_AB). + + + --> recv InitSARemote(spi_BA, nonce_BA): +recv InitSARemote(spi_AB, nonce_AB): <-- {key,salt}_BA = hkdf(sessionKey, nonce_BA, A), + {key,salt}_AB = hkdf(sessionKey, nonce_AB, B), create SA_BA(B<-A, spi_BA, key_BA, salt_BA), + create SA_AB(A<-B, spi_AB, key_AB, salt_AB), create SP_BA(B<-A, spi_BA), + create SP_AB(A<-B, spi_AB), install marking rule. + install marking rule. +``` + +# Implementation Details + +## XFRM + +The implementation is based on the kernel IP packet transformation framework +called XFRM. Unfortunately, docs are barely existing and the complexity of +the framework is high. The best resource I found is Chapter 10 in +"Linux Kernel Networking: Implementation and Theory" by Rami Rosen. + +## iptables Rules + +The kernel VXLAN driver does not set a dst port of a tunnel in the ip flow +descriptor, thus XFRM policy lookup cannot match a policy (SPout) which includes +the port. This makes impossible to encrypt only tunneled traffic between +peers. To work around, we mark such outgoing packets with iptables and set +the same mark in the policy selector (funnily enough, `iptables_mangle` module +eventually sets the missing dst port in the flow descriptor). The challenge +here is to pick a mark that it would not interfere with other networking +applications before OUTPUT'ing a packet. For example, Kubernetes by default +uses 1<<14 and 1<<15 marks and we choose 1<<17 (0x20000). Additionally, +such workaround brings the requirement for at least the 4.2 kernel. + +The marking rules are the following: + +``` +iptables -t mangle -A OUTPUT -j WEAVE-IPSEC-OUT +iptables -t mangle -A WEAVE-IPSEC-OUT -s ${LOCAL_PEER_IP} -d ${REMOTE_PEER_IP} \ + -p udp --dport ${TUNNEL_PORT} -j WEAVE-IPSEC-OUT-MARK +iptables -t mangle -A WEAVE-IPSEC-OUT-MARK --set-xmark ${MARK} -j MARK +``` + +As Linux does not implement [IP Security Levels][ipseclevel], we install +additional iptables rules to prevent from accidentally sending unencrypted +traffic between peers which have previously established the secure connection. + +For inbound traffic, we mark each ESP packet with the mark and drop non-marked +tunnel traffic: + +``` +iptables -t mangle -A INPUT -j WEAVE-IPSEC-IN +iptables -t mangle -A WEAVE-IPSEC-IN -s ${REMOTE_PEER_IP} -d ${LOCAL_PEER_IP} \ + -m esp --espspi ${SPIin} -j WEAVE-IPSEC-IN-MARK +iptables -t mangle -A WEAVE-IPSEC-IN-MARK --set-xmark ${MARK} -j MARK + +iptables -t filter -A INPUT -j WEAVE-IPSEC-IN +iptables -t filter -A WEAVE-IPSEC-IN -s ${REMOTE_PEER_IP} -d ${LOCAL_PEER_IP} \ + -p udp --dport ${TUNNEL_PORT} -m mark ! --mark ${MARK} -j DROP +``` + +For outbound traffic, we drop marked traffic which does not match any SPout: + +``` +iptables -t filter -A OUTPUT ! -p esp -m policy --dir out --pol none \ + -m mark --mark ${MARK} -j DROP +``` + +# Misc + +## MTU + +In addition to the VXLAN overhead, the MTU calculation should take into account +the ESP overhead which is 34-37 bytes (encrypted Payload is 4 bytes aligned) and +consists of: + +* 4 bytes (SPI). +* 4 bytes (Sequence Number). +* 8 bytes (ESP IV). +* 1 byte (Pad Length). +* 1 byte (NextHeader). +* 16 bytes (ICV). +* 0-3 bytes (Padding). + + +[esp]: https://tools.ietf.org/html/rfc2406 +[aesgcm]: https://tools.ietf.org/html/rfc4106 +[hkdf]: https://tools.ietf.org/html/rfc5869 +[esn]: https://tools.ietf.org/html/rfc4304 +[ipseclevel]: https://tools.ietf.org/html/draft-mcdonald-simple-ipsec-api-01) +[mesh]: https://github.com/weaveworks/mesh diff --git a/net/ipsec/ipsec.go b/net/ipsec/ipsec.go new file mode 100644 index 00000000..97074b19 --- /dev/null +++ b/net/ipsec/ipsec.go @@ -0,0 +1,562 @@ +// package IPsec provides primitives for establishing IPsec in the fastdp mode. +package ipsec + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "fmt" + "io" + "net" + "strconv" + "sync" + "syscall" + + "github.com/Sirupsen/logrus" + "github.com/coreos/go-iptables/iptables" + "github.com/pkg/errors" + "github.com/vishvananda/netlink" + "golang.org/x/crypto/hkdf" + + "github.com/weaveworks/mesh" +) + +const ( + keySize = 36 // AES-GCM key 32 bytes + 4 bytes salt + nonceSize = 32 // HKDF nonce size + + mark = uint32(0x1) << 17 // iptables marks + markStr = "0x20000/0x20000" // update if the above mark changes + + tableMangle = "mangle" + tableFilter = "filter" + chainIn = "WEAVE-IPSEC-IN" + chainInMark = "WEAVE-IPSEC-IN-MARK" + chainOut = "WEAVE-IPSEC-OUT" + chainOutMark = "WEAVE-IPSEC-OUT-MARK" +) + +type SPI uint32 + +// Used to identify: +// - directional SPIs, +// - ipsec establishments. +type spiID [24]byte + +func getSPIId(srcPeer, dstPeer mesh.PeerName, connUID uint64) (id spiID) { + binary.BigEndian.PutUint64(id[:], uint64(srcPeer)) + binary.BigEndian.PutUint64(id[8:], uint64(dstPeer)) + binary.BigEndian.PutUint64(id[16:], connUID) + return +} + +type spiInfo struct { + spi SPI + isDirOut bool +} + +// IPSec + +type IPSec struct { + sync.RWMutex + ipt *iptables.IPTables + log *logrus.Logger + + spiInfo map[spiID]spiInfo + // A reference to spiInfo; spiInfo might be of an expired SPI. + spis map[SPI]*spiInfo +} + +func New(log *logrus.Logger) (*IPSec, error) { + ipt, err := iptables.New() + if err != nil { + return nil, errors.Wrap(err, "iptables new") + } + + ipsec := &IPSec{ + ipt: ipt, + log: log, + spiInfo: make(map[spiID]spiInfo), + spis: make(map[SPI]*spiInfo), + } + + return ipsec, nil +} + +// InitSALocal initializes inbound ipsec from remotePeer and triggers +// the initialization on remotePeer. +func (ipsec *IPSec) InitSALocal(localPeer, remotePeer mesh.PeerName, connUID uint64, localIP, remoteIP net.IP, udpPort int, sessionKey *[32]byte, initRemote func([]byte) error) error { + // ID of inbound SPI + spiID := getSPIId(remotePeer, localPeer, connUID) + + ipsec.Lock() + defer ipsec.Unlock() + + // Derive SA key + nonce, err := genNonce() + if err != nil { + return errors.Wrap(err, "generate nonce") + } + key, err := deriveKey(sessionKey[:], nonce, localPeer) + if err != nil { + return errors.Wrap(err, "derive key") + } + + // Allocate SA + sa, err := netlink.XfrmStateAllocSpi(xfrmAllocSpiState(remoteIP, localIP)) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("ip xfrm state allocspi (in, %s, %s)", remoteIP, localIP)) + } + + // Use SPI generated by the kernel. The kernel ensures {dstIP, spi} to + // be unique. + spi := SPI(sa.Spi) + + ipsec.log.Infof("ipsec: InitSALocal: %s -> %s :%d 0x%x", remoteIP, localIP, udpPort, spi) + + // Create SA + if sa, err := xfrmState(remoteIP, localIP, spi, false, key); err == nil { + if err := netlink.XfrmStateUpdate(sa); err != nil { + return errors.Wrap(err, fmt.Sprintf("xfrm state update (in, %s, %s, 0x%x)", sa.Src, sa.Dst, sa.Spi)) + } + } else { + return errors.Wrap(err, "new xfrm state (in)") + } + + // Install iptables rules + if err := ipsec.installDropNonEncrypted(localIP, remoteIP, udpPort, spi); err != nil { + return errors.Wrap(err, fmt.Sprintf("install protecting rules (%s, %s, %d, 0x%x)", localIP, remoteIP, udpPort, spi)) + } + + si := spiInfo{spi: spi, isDirOut: false} + ipsec.spiInfo[spiID] = si + ipsec.spis[spi] = &si + + // Trigger the initialization on the remote peer + msg := &msgInitSARemote{nonce, spi} + if err := initRemote(msg.serialize()); err != nil { + return errors.Wrap(err, "send InitSARemote") + } + + return nil +} + +// InitSARemote initializes outbound ipsec to remotePeer. +// Triggered by remotePeer. +func (ipsec *IPSec) InitSARemote(msgInitSARemote []byte, localPeer, remotePeer mesh.PeerName, connUID uint64, localIP, remoteIP net.IP, udpPort int, sessionKey *[32]byte) error { + // ID of outbound SPI + spiID := getSPIId(localPeer, remotePeer, connUID) + + msg, err := deserializeMsgInitSARemote(msgInitSARemote) + if err != nil { + return errors.Wrap(err, "deserialize InitSARemote") + } + spi := msg.spi + + ipsec.Lock() + defer ipsec.Unlock() + + ipsec.log.Infof("ipsec: InitSARemote: %s -> %s :%d 0x%x", localIP, remoteIP, udpPort, spi) + + // Derive SA key by using the received nonce + key, err := deriveKey(sessionKey[:], msg.nonce, remotePeer) + if err != nil { + return errors.Wrap(err, "derive key") + } + + // Create SA + if sa, err := xfrmState(localIP, remoteIP, spi, true, key); err == nil { + if err := netlink.XfrmStateAdd(sa); err != nil { + return errors.Wrap(err, fmt.Sprintf("xfrm state update (out, %s, %s, 0x%x)", sa.Src, sa.Dst, sa.Spi)) + } + } else { + return errors.Wrap(err, "new xfrm state (out)") + } + + // Create or update SP + sp := xfrmPolicy(localIP, remoteIP, spi) + if err := netlink.XfrmPolicyUpdate(sp); err != nil { + return errors.Wrap(err, fmt.Sprintf("xfrm policy update (%s, %s, 0x%x)", localIP, remoteIP, spi)) + } + + si := spiInfo{spi: spi, isDirOut: true} + ipsec.spiInfo[spiID] = si + ipsec.spis[spi] = &si + + return nil +} + +// Destroy destroys any (inbound / outbound) ipsec establishment between the peers. +func (ipsec *IPSec) Destroy(localPeer, remotePeer mesh.PeerName, connUID uint64, localIP, remoteIP net.IP, udpPort int) error { + outSPIID := getSPIId(localPeer, remotePeer, connUID) + inSPIID := getSPIId(remotePeer, localPeer, connUID) + + ipsec.Lock() + defer ipsec.Unlock() + + // Destroy inbound + ipsec.log.Infof(">>>>>>>>>>>>>>>>>> DESTROY <<<<<<<<<<<<<<<<<<<<<<") + + if inSPIInfo, ok := ipsec.spiInfo[inSPIID]; ok { + ipsec.log.Infof("ipsec: destroy: in %s -> %s 0x%x", remoteIP, localIP, inSPIInfo.spi) + + inSPI := inSPIInfo.spi + + inSA := &netlink.XfrmState{ + Src: remoteIP, + Dst: localIP, + Proto: netlink.XFRM_PROTO_ESP, + Spi: int(inSPI), + } + if err := netlink.XfrmStateDel(inSA); err != nil { + ipsec.log.Warnf("ipsec: xfrm state del (in, %s, %s, 0x%x) failed: %s", inSA.Src, inSA.Dst, inSA.Spi, err) + } + + if err := ipsec.removeDropNonEncrypted(localIP, remoteIP, udpPort, inSPI); err != nil { + ipsec.log.Warnf("ipsec: remove protecting rules (%s, %s, %d, 0x%x) failed: %s", localIP, remoteIP, udpPort, inSPI, err) + } + + delete(ipsec.spiInfo, inSPIID) + delete(ipsec.spis, inSPI) + } + + // Destroy outbound + + if outSPIInfo, ok := ipsec.spiInfo[outSPIID]; ok { + ipsec.log.Infof("ipsec: destroy: out %s -> %s 0x%x", localIP, remoteIP, outSPIInfo.spi) + + if err := netlink.XfrmPolicyDel(xfrmPolicy(localIP, remoteIP, outSPIInfo.spi)); err != nil { + ipsec.log.Warnf("ipsec: xfrm policy del (%s, %s, 0x%x) failed: %s", localIP, remoteIP, outSPIInfo.spi, err) + } + + outSA := &netlink.XfrmState{ + Src: localIP, + Dst: remoteIP, + Proto: netlink.XFRM_PROTO_ESP, + Spi: int(outSPIInfo.spi), + } + if err := netlink.XfrmStateDel(outSA); err != nil { + ipsec.log.Warnf("ipsec: xfrm state del (out, %s, %s, 0x%x) failed: %s", outSA.Src, outSA.Dst, outSA.Spi, err) + } + + delete(ipsec.spiInfo, outSPIID) + delete(ipsec.spis, outSPIInfo.spi) + } + + return nil +} + +// Flush removes all policies/SAs established by us. Also, it removes chains and +// rules of iptables. +// +// If destroy is true, the chains and the rules won't be re-created. +func (ipsec *IPSec) Flush(destroy bool) error { + ipsec.Lock() + defer ipsec.Unlock() + + policies, err := netlink.XfrmPolicyList(syscall.AF_INET) + if err != nil { + return errors.Wrap(err, "xfrm policy list") + } + for _, p := range policies { + if p.Mark != nil && p.Mark.Value == mark && len(p.Tmpls) != 0 { + spi := SPI(p.Tmpls[0].Spi) + if err := netlink.XfrmPolicyDel(&p); err != nil { + return errors.Wrap(err, fmt.Sprintf("xfrm policy del (%s, %s, 0x%x)", p.Src, p.Dst, spi)) + } + } + } + + states, err := netlink.XfrmStateList(syscall.AF_INET) + if err != nil { + return errors.Wrap(err, "xfrm state list") + } + for _, s := range states { + if _, ok := ipsec.spis[SPI(s.Spi)]; ok { + if err := netlink.XfrmStateDel(&s); err != nil { + return errors.Wrap(err, fmt.Sprintf("xfrm state list (%s, %s, 0x%x)", s.Src, s.Dst, s.Spi)) + } + } + } + + if err := ipsec.resetIPTables(destroy); err != nil { + return errors.Wrap(err, "reset ip tables") + } + + return nil +} + +// iptables + +type chain struct { + table string + chain string +} +type rule struct { + table string + chain string + rulespec []string + unique bool +} + +func (ipsec *IPSec) clearChains(chains []chain) error { + for _, c := range chains { + if err := ipsec.ipt.ClearChain(c.table, c.chain); err != nil { + return errors.Wrap(err, fmt.Sprintf("iptables clear chain (%s, %s)", c.table, c.chain)) + } + } + return nil +} + +func (ipsec *IPSec) deleteChains(chains []chain) error { + for _, c := range chains { + if err := ipsec.ipt.DeleteChain(c.table, c.chain); err != nil { + return errors.Wrap(err, fmt.Sprintf("iptables delete chain (%s, %s)", c.table, c.chain)) + } + } + return nil +} + +func (ipsec *IPSec) resetRules(rules []rule, destroy bool) error { + for _, r := range rules { + ok, err := ipsec.ipt.Exists(r.table, r.chain, r.rulespec...) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("iptables exists rule (%s, %s, %s)", r.table, r.chain, r.rulespec)) + } + switch { + case !destroy && !ok: + if err := ipsec.ipt.Append(r.table, r.chain, r.rulespec...); err != nil { + return errors.Wrap(err, fmt.Sprintf("iptables append rule (%s, %s, %s)", r.table, r.chain, r.rulespec)) + } + case destroy && ok: + if err := ipsec.ipt.Delete(r.table, r.chain, r.rulespec...); err != nil { + return errors.Wrap(err, fmt.Sprintf("iptables delete rule (%s, %s, %s)", r.table, r.chain, r.rulespec)) + } + } + } + return nil +} + +func (ipsec *IPSec) resetIPTables(destroy bool) error { + chains := []chain{ + {tableMangle, chainIn}, + {tableMangle, chainInMark}, + {tableFilter, chainIn}, + {tableMangle, chainOut}, + {tableMangle, chainOutMark}, + } + rules := []rule{ + {tableMangle, "INPUT", []string{"-j", chainIn}, true}, + {tableMangle, chainInMark, []string{"-j", "MARK", "--set-xmark", markStr}, true}, + {tableFilter, "INPUT", []string{"-j", chainIn}, true}, + {tableMangle, "OUTPUT", []string{"-j", chainOut}, true}, + {tableMangle, chainOutMark, []string{"-j", "MARK", "--set-xmark", markStr}, true}, + {tableFilter, "OUTPUT", + []string{ + "!", "-p", "esp", + "-m", "policy", "--dir", "out", "--pol", "none", + "-m", "mark", "--mark", markStr, + "-j", "DROP"}, true}, + } + + if err := ipsec.clearChains(chains); err != nil { + return err + } + + if err := ipsec.resetRules(rules, destroy); err != nil { + return err + } + + if destroy { + if err := ipsec.deleteChains(chains); err != nil { + return err + } + } + + return nil +} + +func ruleMarkInboundESP(srcIP, dstIP net.IP, inSPI SPI) rule { + return rule{tableMangle, chainIn, + []string{ + "-s", dstIP.String(), "-d", srcIP.String(), + "-p", "esp", + "-m", "esp", "--espspi", "0x" + strconv.FormatUint(uint64(inSPI), 16), + "-j", chainInMark, + }, true} +} + +func rulesDropNonEncrypted(srcIP, dstIP net.IP, udpPort int, inSPI SPI) []rule { + udpPortStr := strconv.FormatUint(uint64(udpPort), 10) + return []rule{ + ruleMarkInboundESP(srcIP, dstIP, inSPI), + {tableFilter, chainIn, + []string{ + "-s", dstIP.String(), "-d", srcIP.String(), + "-p", "udp", "--dport", udpPortStr, + "-m", "mark", "!", "--mark", markStr, + "-j", "DROP", + }, false}, + {tableMangle, chainOut, + []string{ + "-s", srcIP.String(), "-d", dstIP.String(), + "-p", "udp", "--dport", udpPortStr, + "-j", chainOutMark, + }, false}, + } +} + +func (ipsec *IPSec) installDropNonEncrypted(srcIP, dstIP net.IP, udpPort int, inSPI SPI) error { + rules := rulesDropNonEncrypted(srcIP, dstIP, udpPort, inSPI) + for _, r := range rules { + appendFunc := ipsec.ipt.Append + if r.unique { + appendFunc = ipsec.ipt.AppendUnique + } + if err := appendFunc(r.table, r.chain, r.rulespec...); err != nil { + return errors.Wrap(err, fmt.Sprintf("iptables append (%s, %s, %s)", r.table, r.chain, r.rulespec)) + } + } + return nil +} + +func (ipsec *IPSec) removeDropNonEncrypted(srcIP, dstIP net.IP, udpPort int, inSPI SPI) error { + rules := rulesDropNonEncrypted(srcIP, dstIP, udpPort, inSPI) + if err := ipsec.resetRules(rules, true); err != nil { + return err + } + return nil +} + +func (ipsec *IPSec) removeDropNonEncryptedInbound(srcIP, dstIP net.IP, inSPI SPI) error { + r := ruleMarkInboundESP(srcIP, dstIP, inSPI) + if err := ipsec.ipt.Delete(r.table, r.chain, r.rulespec...); err != nil { + return errors.Wrap(err, fmt.Sprintf("iptables delete unique (%s, %s, %s)", r.table, r.chain, r.rulespec)) + } + return nil +} + +// xfrm + +func xfrmAllocSpiState(srcIP, dstIP net.IP) *netlink.XfrmState { + return &netlink.XfrmState{ + Src: srcIP, + Dst: dstIP, + Proto: netlink.XFRM_PROTO_ESP, + Mode: netlink.XFRM_MODE_TRANSPORT, + ReplayWindow: 32, + } +} + +func xfrmState(srcIP, dstIP net.IP, spi SPI, isDirOut bool, key []byte) (*netlink.XfrmState, error) { + if len(key) != keySize { + return nil, fmt.Errorf("key should be %d bytes long", keySize) + } + + state := xfrmAllocSpiState(srcIP, dstIP) + + state.Spi = int(spi) + state.Aead = &netlink.XfrmStateAlgo{ + Name: "rfc4106(gcm(aes))", + Key: key, + ICVLen: 128, + } + + return state, nil +} + +func xfrmPolicy(srcIP, dstIP net.IP, spi SPI) *netlink.XfrmPolicy { + ipMask := []byte{0xff, 0xff, 0xff, 0xff} // /32 + + return &netlink.XfrmPolicy{ + Src: &net.IPNet{IP: srcIP, Mask: ipMask}, + Dst: &net.IPNet{IP: dstIP, Mask: ipMask}, + Proto: syscall.IPPROTO_UDP, + Dir: netlink.XFRM_DIR_OUT, + Mark: &netlink.XfrmMark{ + Value: mark, + Mask: mark, + }, + Tmpls: []netlink.XfrmPolicyTmpl{ + { + Src: srcIP, + Dst: dstIP, + Proto: netlink.XFRM_PROTO_ESP, + Mode: netlink.XFRM_MODE_TRANSPORT, + Spi: int(spi), + }, + }, + } +} + +// Key derivation + +func genNonce() ([]byte, error) { + buf := make([]byte, nonceSize) + n, err := rand.Read(buf) + if err != nil { + return nil, fmt.Errorf("crypto rand failed: %s", err) + } + if n != nonceSize { + return nil, fmt.Errorf("not enough of random data: %d", n) + } + return buf, nil +} + +func deriveKey(sessionKey []byte, nonce []byte, peerName mesh.PeerName) ([]byte, error) { + key := make([]byte, keySize) + + info := make([]byte, 8) + binary.BigEndian.PutUint64(info, uint64(peerName)) + + hkdf := hkdf.New(sha256.New, sessionKey, nonce, info) + + n, err := io.ReadFull(hkdf, key) + if err != nil { + return nil, err + } + if n != keySize { + return nil, fmt.Errorf("derived too short key: %d", n) + } + + return key, nil +} + +// Protocol Messages + +type msgInitSARemote struct { + nonce []byte + spi SPI +} + +func deserializeMsgInitSARemote(b []byte) (*msgInitSARemote, error) { + if len(b) == 0 { + return nil, fmt.Errorf("empty msg") + } + + msg := &msgInitSARemote{} + if len(b) != msg.size() { + return nil, fmt.Errorf("invalid payload size: %d", len(b)) + } + + msg.nonce = make([]byte, nonceSize) + copy(msg.nonce, b[:nonceSize]) + b = b[nonceSize:] + + msg.spi = SPI(binary.BigEndian.Uint32(b)) + + return msg, nil +} + +func (msg *msgInitSARemote) size() int { + return nonceSize + 32 // SPI +} + +func (msg *msgInitSARemote) serialize() []byte { + b := make([]byte, msg.size()) + + copy(b[:nonceSize], msg.nonce) + binary.BigEndian.PutUint32(b[nonceSize:], uint32(msg.spi)) + + return b +} diff --git a/prog/weaver/main.go b/prog/weaver/main.go index c4011d72..92609b79 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -224,7 +224,9 @@ func main() { networkConfig.PacketLogging = nopPacketLogging{} } - overlay, bridge := createOverlay(datapathName, ifaceName, isAWSVPC, config.Host, config.Port, bufSzMB) + config.Password = determinePassword(password) + + overlay, bridge := createOverlay(datapathName, ifaceName, isAWSVPC, config.Host, config.Port, bufSzMB, config.Password != nil) networkConfig.Bridge = bridge if bridge != nil { @@ -241,7 +243,6 @@ func main() { checkFatal(err) } - config.Password = determinePassword(password) config.TrustedSubnets = parseTrustedSubnets(trustedSubnetStr) config.PeerDiscovery = !noDiscovery @@ -405,7 +406,7 @@ func (nopPacketLogging) LogPacket(string, weave.PacketKey) { func (nopPacketLogging) LogForwardPacket(string, weave.ForwardPacketKey) { } -func createOverlay(datapathName string, ifaceName string, isAWSVPC bool, host string, port int, bufSzMB int) (weave.NetworkOverlay, weave.Bridge) { +func createOverlay(datapathName string, ifaceName string, isAWSVPC bool, host string, port int, bufSzMB int, enableEncryption bool) (weave.NetworkOverlay, weave.Bridge) { overlay := weave.NewOverlaySwitch() var bridge weave.Bridge var ignoreSleeve bool @@ -422,7 +423,7 @@ func createOverlay(datapathName string, ifaceName string, isAWSVPC bool, host st case datapathName != "": iface, err := weavenet.EnsureInterface(datapathName) checkFatal(err) - fastdp, err := weave.NewFastDatapath(iface, port) + fastdp, err := weave.NewFastDatapath(iface, port, enableEncryption) checkFatal(err) bridge = fastdp.Bridge() overlay.Add("fastdp", fastdp.Overlay()) diff --git a/router/awsvpc.go b/router/awsvpc.go index 84cd14a8..992aa101 100644 --- a/router/awsvpc.go +++ b/router/awsvpc.go @@ -64,6 +64,8 @@ func (vpc AWSVPC) Diagnostics() interface{} { return nil } +func (vpc AWSVPC) Stop() {} + // NetworkOverlay func (vpc AWSVPC) InvalidateRoutes() {} diff --git a/router/fastdp.go b/router/fastdp.go index 2c1df5ae..ffe28041 100644 --- a/router/fastdp.go +++ b/router/fastdp.go @@ -9,9 +9,12 @@ import ( "syscall" "time" + "github.com/pkg/errors" "github.com/vishvananda/netlink" "github.com/weaveworks/go-odp/odp" "github.com/weaveworks/mesh" + + "github.com/weaveworks/weave/net/ipsec" ) // The virtual bridge accepts packets from ODP vports and the router @@ -39,6 +42,7 @@ type FastDatapath struct { localPeer *mesh.Peer peers *mesh.Peers overlayConsumer OverlayConsumer + ipsec *ipsec.IPSec // Bridge state: How to send to the given bridge port sendToPort map[bridgePortID]bridgeSender @@ -53,6 +57,7 @@ type FastDatapath struct { vxlanUDPPorts map[int]odp.VportID vxlanVportIDs map[odp.VportID]struct{} mainVxlanVportID odp.VportID + mainVxlanUDPPort int // A singleton pool for the occasions when we need to decode // the packet. @@ -62,7 +67,9 @@ type FastDatapath struct { forwarders map[mesh.PeerName]*fastDatapathForwarder } -func NewFastDatapath(iface *net.Interface, port int) (*FastDatapath, error) { +func NewFastDatapath(iface *net.Interface, port int, encryptionEnabled bool) (*FastDatapath, error) { + var ipSec *ipsec.IPSec + dpif, err := odp.NewDpif() if err != nil { return nil, err @@ -80,11 +87,22 @@ func NewFastDatapath(iface *net.Interface, port int) (*FastDatapath, error) { return nil, err } + if encryptionEnabled { + var err error + if ipSec, err = ipsec.New(log); err != nil { + return nil, errors.Wrap(err, "ipsec new") + } + if err := ipSec.Flush(false); err != nil { + return nil, errors.Wrap(err, "ipsec flush") + } + } + fastdp := &FastDatapath{ iface: iface, dpif: dpif, dp: dp, missHandlers: make(map[odp.VportID]missHandler), + ipsec: ipSec, sendToPort: nil, sendToMAC: make(map[MAC]bridgeSender), seenMACs: make(map[MAC]struct{}), @@ -113,7 +131,8 @@ func NewFastDatapath(iface *net.Interface, port int) (*FastDatapath, error) { // numbers to be independent, but working out how to specify // them on the connecting side. So we can wait to find out if // anyone wants that. - fastdp.mainVxlanVportID, err = fastdp.getVxlanVportIDHarder(port+1, 5, time.Millisecond*10) + fastdp.mainVxlanUDPPort = port + 1 + fastdp.mainVxlanVportID, err = fastdp.getVxlanVportIDHarder(fastdp.mainVxlanUDPPort, 5, time.Millisecond*10) if err != nil { return nil, err } @@ -315,6 +334,14 @@ func (fastdp fastDatapathOverlay) InvalidateShortIDs() { checkWarn(fastdp.deleteFlows()) } +func (fastdp fastDatapathOverlay) Stop() { + if fastdp.ipsec != nil { + if err := fastdp.ipsec.Flush(true); err != nil { + log.Errorf("ipsec flush failed: %s", err) + } + } +} + func (fastDatapathOverlay) AddFeaturesTo(features map[string]string) { // Nothing needed. Fast datapath support is indicated through // OverlaySwitch. @@ -542,6 +569,10 @@ type fastDatapathForwarder struct { connUID uint64 vxlanVportID odp.VportID + sessionKey *[32]byte + isEncrypted bool + isOutboundIPSecEstablished bool + lock sync.RWMutex confirmed bool remoteAddr *net.UDPAddr @@ -557,27 +588,25 @@ type fastDatapathForwarder struct { } func (fastdp fastDatapathOverlay) PrepareConnection(params mesh.OverlayConnectionParams) (mesh.OverlayConnection, error) { - if params.SessionKey != nil { - // No encryption support in fastdp - return nil, fmt.Errorf("encryption not supported") - } - vxlanVportID := fastdp.mainVxlanVportID - var remoteAddr *net.UDPAddr + vxlanUDPPort := fastdp.mainVxlanUDPPort + remoteAddr := makeUDPAddr(params.RemoteAddr) if params.Outbound { - remoteAddr = makeUDPAddr(params.RemoteAddr) + var err error // The provided address contains the main weave port // number to connect to. We need to derive the vxlan // port number from that. vxlanRemoteAddr := *remoteAddr vxlanRemoteAddr.Port++ remoteAddr = &vxlanRemoteAddr - var err error - vxlanVportID, err = fastdp.getVxlanVportID(remoteAddr.Port) + vxlanUDPPort = remoteAddr.Port + vxlanVportID, err = fastdp.getVxlanVportID(vxlanUDPPort) if err != nil { return nil, err } + } else { + remoteAddr.Port = vxlanUDPPort } localIP, err := ipv4Bytes(params.LocalAddr.IP) @@ -592,6 +621,7 @@ func (fastdp fastDatapathOverlay) PrepareConnection(params mesh.OverlayConnectio sendControlMsg: params.SendControlMessage, connUID: params.ConnUID, vxlanVportID: vxlanVportID, + sessionKey: params.SessionKey, remoteAddr: remoteAddr, heartbeatInterval: FastHeartbeat, @@ -601,7 +631,7 @@ func (fastdp fastDatapathOverlay) PrepareConnection(params mesh.OverlayConnectio errorChan: make(chan error, 1), } - return fwd, err + return fwd, nil } func ipv4Bytes(ip net.IP) (res [4]byte, err error) { @@ -626,11 +656,30 @@ func (fwd *fastDatapathForwarder) Confirm() { log.Fatal(fwd.logPrefix(), "already confirmed") } + if fwd.fastdp.ipsec != nil && fwd.sessionKey != nil { + fwd.isEncrypted = true + log.Info("Setting up IPsec between ", fwd.fastdp.localPeer, " and ", fwd.remotePeer) + err := fwd.fastdp.ipsec.InitSALocal( + fwd.fastdp.localPeer.Name, fwd.remotePeer.Name, fwd.connUID, + net.IP(fwd.localIP[:]), fwd.remoteAddr.IP, + fwd.remoteAddr.Port, + fwd.sessionKey, + func(msg []byte) error { + return fwd.sendControlMsg(FastDatapathCryptoInitSARemote, msg) + }, + ) + if err != nil { + log.Error(fwd.logPrefix(), "ipsec init SA local failed: ", err) + fwd.handleError(err) + return + } + } + log.Debug(fwd.logPrefix(), "confirmed") fwd.fastdp.addForwarder(fwd.remotePeer.Name, fwd) fwd.confirmed = true - if fwd.remoteAddr != nil { + if fwd.remoteAddr != nil && (!fwd.isEncrypted || fwd.isOutboundIPSecEstablished) { // have the goroutine send a heartbeat straight away fwd.heartbeatTimer = time.NewTimer(0) } else { @@ -719,6 +768,7 @@ func (fwd *fastDatapathForwarder) sendHeartbeat() { const ( FastDatapathHeartbeatAck = iota + FastDatapathCryptoInitSARemote ) func (fwd *fastDatapathForwarder) handleVxlanSpecialPacket(frame []byte, sender *net.UDPAddr) { @@ -768,6 +818,8 @@ func (fwd *fastDatapathForwarder) ControlMessage(tag byte, msg []byte) { switch tag { case FastDatapathHeartbeatAck: fwd.handleHeartbeatAck() + case FastDatapathCryptoInitSARemote: + fwd.handleCryptoInitSARemote(msg) default: log.Info(fwd.logPrefix(), "Ignoring unknown control message: ", tag) @@ -790,6 +842,26 @@ func (fwd *fastDatapathForwarder) handleHeartbeatAck() { } } +func (fwd *fastDatapathForwarder) handleCryptoInitSARemote(msg []byte) { + log.Info(fwd.logPrefix(), "IPSec init SA remote") + err := fwd.fastdp.ipsec.InitSARemote( + msg, + fwd.fastdp.localPeer.Name, fwd.remotePeer.Name, fwd.connUID, + net.IP(fwd.localIP[:]), fwd.remoteAddr.IP, fwd.remoteAddr.Port, + fwd.sessionKey, + ) + if err != nil { + log.Warning(fwd.logPrefix(), "IPSec init SA remote failed: ", err) + fwd.handleError(err) + return + } + + if !fwd.isOutboundIPSecEstablished { + fwd.isOutboundIPSecEstablished = true + fwd.heartbeatTimer.Reset(0) + } +} + func (fwd *fastDatapathForwarder) Forward(key ForwardPacketKey) FlowOp { if !key.SrcPeer.HasShortID || !key.DstPeer.HasShortID { return nil @@ -839,6 +911,18 @@ func (fwd *fastDatapathForwarder) Stop() { defer fwd.lock.Unlock() fwd.sendControlMsg = func(byte, []byte) error { return nil } + if fwd.isEncrypted { + localIP := net.IP(fwd.localIP[:]) + log.Info("Destroying IPsec between ", fwd.fastdp.localPeer, " and ", fwd.remotePeer) + err := fwd.fastdp.ipsec.Destroy( + fwd.fastdp.localPeer.Name, fwd.remotePeer.Name, fwd.connUID, + localIP, fwd.remoteAddr.IP, fwd.remoteAddr.Port, + ) + if err != nil { + log.Errorf("ipsec destroy failed: %s", err) + } + } + // stop the heartbeat goroutine if !fwd.stopped { fwd.stopped = true diff --git a/router/overlay_switch.go b/router/overlay_switch.go index 8031ceb5..20f7dcbb 100644 --- a/router/overlay_switch.go +++ b/router/overlay_switch.go @@ -50,6 +50,12 @@ func (osw *OverlaySwitch) Diagnostics() interface{} { return diagnostics } +func (osw *OverlaySwitch) Stop() { + for _, overlay := range osw.overlays { + overlay.Stop() + } +} + func (osw *OverlaySwitch) InvalidateRoutes() { for _, overlay := range osw.overlays { overlay.InvalidateRoutes() diff --git a/router/sleeve.go b/router/sleeve.go index 4857ccf0..f4b01b27 100644 --- a/router/sleeve.go +++ b/router/sleeve.go @@ -141,6 +141,10 @@ func (*SleeveOverlay) Diagnostics() interface{} { return nil } +func (*SleeveOverlay) Stop() { + // do nothing +} + func (sleeve *SleeveOverlay) lookupForwarder(peer mesh.PeerName) *sleeveForwarder { sleeve.lock.Lock() defer sleeve.lock.Unlock() diff --git a/test/110_sleeve_encryption_2_test.sh b/test/110_sleeve_encryption_2_test.sh new file mode 100755 index 00000000..709548d9 --- /dev/null +++ b/test/110_sleeve_encryption_2_test.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +. ./config.sh + +C1=10.2.1.4 +C2=10.2.1.7 + +start_suite "Ping over encrypted cross-host weave network (sleeve)" + +WEAVE_NO_FASTDP=1 weave_on $HOST1 launch --password wfvAwt7sj +WEAVE_NO_FASTDP=1 weave_on $HOST2 launch --password wfvAwt7sj $HOST1 + +start_container $HOST1 $C1/24 --name=c1 +start_container $HOST2 $C2/24 --name=c2 +assert_raises "exec_on $HOST1 c1 $PING $C2" + +assert_raises "weave_on $HOST1 status connections | grep -P 'encrypted *sleeve'" +assert_raises "weave_on $HOST2 status connections | grep -P 'encrypted *sleeve'" + +end_suite diff --git a/test/111_fastdp_encryption_2_test.sh b/test/111_fastdp_encryption_2_test.sh new file mode 100755 index 00000000..0845282b --- /dev/null +++ b/test/111_fastdp_encryption_2_test.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +# TODO(mp) 1. test anti-replay 2. test sleeve <-> fastdp +# TODO(mp) test with --trusted-subnets + +. ./config.sh + +HOST1_IP=$($SSH $HOST1 getent ahosts $HOST1 | grep "RAW" | cut -d' ' -f1) +HOST2_IP=$($SSH $HOST2 getent ahosts $HOST2 | grep "RAW" | cut -d' ' -f1) + +start_tcpdump() { + HOST=$1 + PCAP=$2 + run_on $HOST "sudo nohup tcpdump -i any -w $PCAP >/dev/null 2>&1 & echo \$! > $PCAP.pid" +} + +stop_tcpdump() { + HOST=$1 + PCAP=$2 + $SSH $HOST "sudo kill \$(cat $PCAP.pid) && rm -f $PCAP.pid && sudo base64 $PCAP" \ + | base64 -d > $PCAP +} + +# Checks whether all vxlan-tunneled traffic is encrypted. +assert_pcap() { + PCAP=$1 + # UDP port is not exposed in ESP + assert "tcpdump -r $PCAP 'src $HOST1_IP && dst $HOST2_IP && dst port 6784'" "" + assert "tcpdump -r $PCAP 'src $HOST2_IP && dst $HOST1_IP && dst port 6784'" "" + # ip_proto(ESP) = 50 + assert_raises "[[ -n \"$(tcpdump -r $PCAP "src $HOST1_IP && dst $HOST2_IP && proto 50")\" ]]" + assert_raises "[[ -n \"$(tcpdump -r $PCAP "src $HOST2_IP && dst $HOST1_IP && proto 50")\" ]]" + + rm -f $PCAP +} + +C1=10.2.1.4 +C2=10.2.1.7 + +start_suite "Ping over encrypted cross-host weave network (fastdp)" + +WEAVE_MTU=1300 weave_on $HOST1 launch --password wfvAwt7sj +WEAVE_MTU=1300 weave_on $HOST2 launch --password wfvAwt7sj $HOST1 + +assert_raises "weave_on $HOST1 status connections | grep -P 'encrypted *fastdp'" +assert_raises "weave_on $HOST2 status connections | grep -P 'encrypted *fastdp'" + +PCAP1=$(mktemp) +PCAP2=$(mktemp) + +start_tcpdump $HOST1 $PCAP1 +start_tcpdump $HOST2 $PCAP2 + +start_container $HOST1 $C1/24 --name=c1 +start_container $HOST2 $C2/24 --name=c2 +assert_raises "exec_on $HOST1 c1 $PING $C2" + +stop_tcpdump $HOST1 $PCAP1 +stop_tcpdump $HOST2 $PCAP2 + +assert_pcap $PCAP1 +assert_pcap $PCAP2 + +weave_on $HOST2 reset +# policies/SAs and iptables rules should have been removed after terminating +# the connection. +assert "$SSH $HOST1 sudo ip xfrm state" "" +assert "$SSH $HOST1 sudo ip xfrm policy" "" +assert "$SSH $HOST1 sudo iptables -t mangle -S WEAVE-IPSEC | grep '\-A'" "" + +end_suite diff --git a/vendor/github.com/vishvananda/netlink b/vendor/github.com/vishvananda/netlink index 266f02d3..c540d7a6 160000 --- a/vendor/github.com/vishvananda/netlink +++ b/vendor/github.com/vishvananda/netlink @@ -1 +1 @@ -Subproject commit 266f02d3a80fb75c33302a2c91e7632ec33c1f32 +Subproject commit c540d7a60a21b09073f88af3af4549abdf331abb diff --git a/vendor/github.com/weaveworks/mesh b/vendor/github.com/weaveworks/mesh index 58098b96..5015f896 160000 --- a/vendor/github.com/weaveworks/mesh +++ b/vendor/github.com/weaveworks/mesh @@ -1 +1 @@ -Subproject commit 58098b961d033669ed507065ffce84c9412b57c7 +Subproject commit 5015f896ab62d3e9fe757456c757521ce0c3faff