Add fastdp crypto

.gitmodules

@@ -3,7 +3,7 @@
 	url = https://github.com/weaveworks/build-tools
 [submodule "vendor/github.com/vishvananda/netlink"]
 	path = vendor/github.com/vishvananda/netlink
-	url = https://github.com/vishvananda/netlink.git
+	url = https://github.com/brb/netlink.git
 [submodule "vendor/github.com/gorilla/mux"]
 	path = vendor/github.com/gorilla/mux
 	url = https://github.com/gorilla/mux.git

docs/fastdp-crypto.md

@@ -0,0 +1,151 @@
+This document describes implementation details of the fast datapath encryption.
+
+# Overview
+
+At the high level, we use the ESP protocol ([RFC 2406][esp]) in the Transport
+mode. Each packet is encrypted with AES in GCM mode ([RFC 4106][aesgcm]), with
+32byte key and 4byte salt. This combo provides the following security
+properties:
+
+* Data confidentiality.
+* Data origin authentication.
+* Integrity.
+* Anti-replay service.
+* Limited traffic flow confidentiality.
+
+## Notation
+
+* `SAin`:  IPsec security association for inbound connections.
+* `SAout`: IPsec security association for outbound connections.
+* `SPout`: IPsec security policy for outbound connections. Used to match
+           outbound flows to SAout.
+
+## SA Key Derivation
+
+For each connection direction, a different AES-GCM key and salt is used.
+The pairs are derived with HKDF ([RFC 5869][hkdf]):
+
+```
+SAin[KeyAndSalt] = HKDF(sha256, ikm=sessionKey, salt=nonceIn, info=localPeerName)
+SAout[KeyAndSalt] = HKDF(sha256, ikm=sessionKey, salt=nonceOut, info=remotePeerName)
+```
+
+Where:
+
+* the mutual `sessionKey` is derived by the [Mesh][mesh] library during
+  the control plane connection establishment between the local peer and
+  the remote peer.
+* `nonceIn` and `nonceOut` are randomly generated 32byte nonces which
+  are exchanged over the encrypted control plane channel.
+
+## SPI
+
+A directional secure connection between two peers is identified with SPI.
+
+The kernel requires the pair of SPI and dst IP to be unique among security
+associations. Thus, to avoid collisions, we generate outbound SPI on a remote
+peer.
+
+## Connection Establishment
+
+```
+Peer A                                                        Peer B
+-----------------------------------------------------------------------------------------------------------------
+
+fastdp.fwd.Confirm():
+    install iptables blocking rules,                          fastdp.fwd.Confirm():
+    nonce_BA = rand(),                                            install iptables blocking rules,
+    {key,salt}_BA = hkdf(sessionKey, nonce_BA, A),                nonce_AB = rand(),
+    spi_BA = allocspi(),                                          {key,salt}_AB = hkdf(sessionKey, nonce_AB, B),
+    create SA_BA(B<-A, spi_BA, key_BA, salt_BA),                  spi_AB = allocspi(),
+    send InitSARemote(spi_BA, nonce_BA). -->                      create SA_AB(A<-B, spi_AB, key_AB, salt_AB),
+                                                              <-- send InitSARemote(spi_AB, nonce_AB).
+
+
+                                                          --> recv InitSARemote(spi_BA, nonce_BA):
+recv InitSARemote(spi_AB, nonce_AB): <--                          {key,salt}_BA = hkdf(sessionKey, nonce_BA, A),
+    {key,salt}_AB = hkdf(sessionKey, nonce_AB, B),                create SA_BA(B<-A, spi_BA, key_BA, salt_BA),
+    create SA_AB(A<-B, spi_AB, key_AB, salt_AB),                  create SP_BA(B<-A, spi_BA),
+    create SP_AB(A<-B, spi_AB),                                   install marking rule.
+    install marking rule.
+```
+
+# Implementation Details
+
+## XFRM
+
+The implementation is based on the kernel IP packet transformation framework
+called XFRM. Unfortunately, docs are barely existing and the complexity of
+the framework is high. The best resource I found is Chapter 10 in
+"Linux Kernel Networking: Implementation and Theory" by Rami Rosen.
+
+## iptables Rules
+
+The kernel VXLAN driver does not set a dst port of a tunnel in the ip flow
+descriptor, thus XFRM policy lookup cannot match a policy (SPout) which includes
+the port. This makes impossible to encrypt only tunneled traffic between
+peers. To work around, we mark such outgoing packets with iptables and set
+the same mark in the policy selector (funnily enough, `iptables_mangle` module
+eventually sets the missing dst port in the flow descriptor). The challenge
+here is to pick a mark that it would not interfere with other networking
+applications before OUTPUT'ing a packet. For example, Kubernetes by default
+uses 1<<14 and 1<<15 marks and we choose 1<<17 (0x20000). Additionally,
+such workaround brings the requirement for at least the 4.2 kernel.
+
+The marking rules are the following:
+
+```
+iptables -t mangle -A OUTPUT -j WEAVE-IPSEC-OUT
+iptables -t mangle -A WEAVE-IPSEC-OUT -s ${LOCAL_PEER_IP} -d ${REMOTE_PEER_IP} \
+         -p udp --dport ${TUNNEL_PORT} -j WEAVE-IPSEC-OUT-MARK
+iptables -t mangle -A WEAVE-IPSEC-OUT-MARK --set-xmark ${MARK} -j MARK
+```
+
+As Linux does not implement [IP Security Levels][ipseclevel], we install
+additional iptables rules to prevent from accidentally sending unencrypted
+traffic between peers which have previously established the secure connection.
+
+For inbound traffic, we mark each ESP packet with the mark and drop non-marked
+tunnel traffic:
+
+```
+iptables -t mangle -A INPUT -j WEAVE-IPSEC-IN
+iptables -t mangle -A WEAVE-IPSEC-IN -s ${REMOTE_PEER_IP} -d ${LOCAL_PEER_IP} \
+         -m esp --espspi ${SPIin} -j WEAVE-IPSEC-IN-MARK
+iptables -t mangle -A WEAVE-IPSEC-IN-MARK --set-xmark ${MARK} -j MARK
+
+iptables -t filter -A INPUT -j WEAVE-IPSEC-IN
+iptables -t filter -A WEAVE-IPSEC-IN -s ${REMOTE_PEER_IP} -d ${LOCAL_PEER_IP} \
+         -p udp --dport ${TUNNEL_PORT} -m mark ! --mark ${MARK} -j DROP
+```
+
+For outbound traffic, we drop marked traffic which does not match any SPout:
+
+```
+iptables -t filter -A OUTPUT ! -p esp -m policy --dir out --pol none \
+         -m mark --mark ${MARK} -j DROP
+```
+
+# Misc
+
+## MTU
+
+In addition to the VXLAN overhead, the MTU calculation should take into account
+the ESP overhead which is 34-37 bytes (encrypted Payload is 4 bytes aligned) and
+consists of:
+
+* 4 bytes (SPI).
+* 4 bytes (Sequence Number).
+* 8 bytes (ESP IV).
+* 1 byte (Pad Length).
+* 1 byte (NextHeader).
+* 16 bytes (ICV).
+* 0-3 bytes (Padding).
+
+
+[esp]:              https://tools.ietf.org/html/rfc2406
+[aesgcm]:           https://tools.ietf.org/html/rfc4106
+[hkdf]:             https://tools.ietf.org/html/rfc5869
+[esn]:              https://tools.ietf.org/html/rfc4304
+[ipseclevel]:       https://tools.ietf.org/html/draft-mcdonald-simple-ipsec-api-01)
+[mesh]:             https://github.com/weaveworks/mesh

net/ipsec/ipsec.go

@@ -0,0 +1,562 @@
+// package IPsec provides primitives for establishing IPsec in the fastdp mode.
+package ipsec
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"net"
+	"strconv"
+	"sync"
+	"syscall"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/coreos/go-iptables/iptables"
+	"github.com/pkg/errors"
+	"github.com/vishvananda/netlink"
+	"golang.org/x/crypto/hkdf"
+
+	"github.com/weaveworks/mesh"
+)
+
+const (
+	keySize   = 36 // AES-GCM key 32 bytes + 4 bytes salt
+	nonceSize = 32 // HKDF nonce size
+
+	mark    = uint32(0x1) << 17 // iptables marks
+	markStr = "0x20000/0x20000" // update if the above mark changes
+
+	tableMangle  = "mangle"
+	tableFilter  = "filter"
+	chainIn      = "WEAVE-IPSEC-IN"
+	chainInMark  = "WEAVE-IPSEC-IN-MARK"
+	chainOut     = "WEAVE-IPSEC-OUT"
+	chainOutMark = "WEAVE-IPSEC-OUT-MARK"
+)
+
+type SPI uint32
+
+// Used to identify:
+// - directional SPIs,
+// - ipsec establishments.
+type spiID [24]byte
+
+func getSPIId(srcPeer, dstPeer mesh.PeerName, connUID uint64) (id spiID) {
+	binary.BigEndian.PutUint64(id[:], uint64(srcPeer))
+	binary.BigEndian.PutUint64(id[8:], uint64(dstPeer))
+	binary.BigEndian.PutUint64(id[16:], connUID)
+	return
+}
+
+type spiInfo struct {
+	spi      SPI
+	isDirOut bool
+}
+
+// IPSec
+
+type IPSec struct {
+	sync.RWMutex
+	ipt *iptables.IPTables
+	log *logrus.Logger
+
+	spiInfo map[spiID]spiInfo
+	// A reference to spiInfo; spiInfo might be of an expired SPI.
+	spis map[SPI]*spiInfo
+}
+
+func New(log *logrus.Logger) (*IPSec, error) {
+	ipt, err := iptables.New()
+	if err != nil {
+		return nil, errors.Wrap(err, "iptables new")
+	}
+
+	ipsec := &IPSec{
+		ipt:     ipt,
+		log:     log,
+		spiInfo: make(map[spiID]spiInfo),
+		spis:    make(map[SPI]*spiInfo),
+	}
+
+	return ipsec, nil
+}
+
+// InitSALocal initializes inbound ipsec from remotePeer and triggers
+// the initialization on remotePeer.
+func (ipsec *IPSec) InitSALocal(localPeer, remotePeer mesh.PeerName, connUID uint64, localIP, remoteIP net.IP, udpPort int, sessionKey *[32]byte, initRemote func([]byte) error) error {
+	// ID of inbound SPI
+	spiID := getSPIId(remotePeer, localPeer, connUID)
+
+	ipsec.Lock()
+	defer ipsec.Unlock()
+
+	// Derive SA key
+	nonce, err := genNonce()
+	if err != nil {
+		return errors.Wrap(err, "generate nonce")
+	}
+	key, err := deriveKey(sessionKey[:], nonce, localPeer)
+	if err != nil {
+		return errors.Wrap(err, "derive key")
+	}
+
+	// Allocate SA
+	sa, err := netlink.XfrmStateAllocSpi(xfrmAllocSpiState(remoteIP, localIP))
+	if err != nil {
+		return errors.Wrap(err, fmt.Sprintf("ip xfrm state allocspi (in, %s, %s)", remoteIP, localIP))
+	}
+
+	// Use SPI generated by the kernel. The kernel ensures {dstIP, spi} to
+	// be unique.
+	spi := SPI(sa.Spi)
+
+	ipsec.log.Infof("ipsec: InitSALocal: %s -> %s :%d 0x%x", remoteIP, localIP, udpPort, spi)
+
+	// Create SA
+	if sa, err := xfrmState(remoteIP, localIP, spi, false, key); err == nil {
+		if err := netlink.XfrmStateUpdate(sa); err != nil {
+			return errors.Wrap(err, fmt.Sprintf("xfrm state update (in, %s, %s, 0x%x)", sa.Src, sa.Dst, sa.Spi))
+		}
+	} else {
+		return errors.Wrap(err, "new xfrm state (in)")
+	}
+
+	// Install iptables rules
+	if err := ipsec.installDropNonEncrypted(localIP, remoteIP, udpPort, spi); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("install protecting rules (%s, %s, %d, 0x%x)", localIP, remoteIP, udpPort, spi))
+	}
+
+	si := spiInfo{spi: spi, isDirOut: false}
+	ipsec.spiInfo[spiID] = si
+	ipsec.spis[spi] = &si
+
+	// Trigger the initialization on the remote peer
+	msg := &msgInitSARemote{nonce, spi}
+	if err := initRemote(msg.serialize()); err != nil {
+		return errors.Wrap(err, "send InitSARemote")
+	}
+
+	return nil
+}
+
+// InitSARemote initializes outbound ipsec to remotePeer.
+// Triggered by remotePeer.
+func (ipsec *IPSec) InitSARemote(msgInitSARemote []byte, localPeer, remotePeer mesh.PeerName, connUID uint64, localIP, remoteIP net.IP, udpPort int, sessionKey *[32]byte) error {
+	// ID of outbound SPI
+	spiID := getSPIId(localPeer, remotePeer, connUID)
+
+	msg, err := deserializeMsgInitSARemote(msgInitSARemote)
+	if err != nil {
+		return errors.Wrap(err, "deserialize InitSARemote")
+	}
+	spi := msg.spi
+
+	ipsec.Lock()
+	defer ipsec.Unlock()
+
+	ipsec.log.Infof("ipsec: InitSARemote: %s -> %s :%d 0x%x", localIP, remoteIP, udpPort, spi)
+
+	// Derive SA key by using the received nonce
+	key, err := deriveKey(sessionKey[:], msg.nonce, remotePeer)
+	if err != nil {
+		return errors.Wrap(err, "derive key")
+	}
+
+	// Create SA
+	if sa, err := xfrmState(localIP, remoteIP, spi, true, key); err == nil {
+		if err := netlink.XfrmStateAdd(sa); err != nil {
+			return errors.Wrap(err, fmt.Sprintf("xfrm state update (out, %s, %s, 0x%x)", sa.Src, sa.Dst, sa.Spi))
+		}
+	} else {
+		return errors.Wrap(err, "new xfrm state (out)")
+	}
+
+	// Create or update SP
+	sp := xfrmPolicy(localIP, remoteIP, spi)
+	if err := netlink.XfrmPolicyUpdate(sp); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("xfrm policy update (%s, %s, 0x%x)", localIP, remoteIP, spi))
+	}
+
+	si := spiInfo{spi: spi, isDirOut: true}
+	ipsec.spiInfo[spiID] = si
+	ipsec.spis[spi] = &si
+
+	return nil
+}
+
+// Destroy destroys any (inbound / outbound) ipsec establishment between the peers.
+func (ipsec *IPSec) Destroy(localPeer, remotePeer mesh.PeerName, connUID uint64, localIP, remoteIP net.IP, udpPort int) error {
+	outSPIID := getSPIId(localPeer, remotePeer, connUID)
+	inSPIID := getSPIId(remotePeer, localPeer, connUID)
+
+	ipsec.Lock()
+	defer ipsec.Unlock()
+
+	// Destroy inbound
+	ipsec.log.Infof(">>>>>>>>>>>>>>>>>> DESTROY <<<<<<<<<<<<<<<<<<<<<<")
+
+	if inSPIInfo, ok := ipsec.spiInfo[inSPIID]; ok {
+		ipsec.log.Infof("ipsec: destroy: in %s -> %s 0x%x", remoteIP, localIP, inSPIInfo.spi)
+
+		inSPI := inSPIInfo.spi
+
+		inSA := &netlink.XfrmState{
+			Src:   remoteIP,
+			Dst:   localIP,
+			Proto: netlink.XFRM_PROTO_ESP,
+			Spi:   int(inSPI),
+		}
+		if err := netlink.XfrmStateDel(inSA); err != nil {
+			ipsec.log.Warnf("ipsec: xfrm state del (in, %s, %s, 0x%x) failed: %s", inSA.Src, inSA.Dst, inSA.Spi, err)
+		}
+
+		if err := ipsec.removeDropNonEncrypted(localIP, remoteIP, udpPort, inSPI); err != nil {
+			ipsec.log.Warnf("ipsec: remove protecting rules (%s, %s, %d, 0x%x) failed: %s", localIP, remoteIP, udpPort, inSPI, err)
+		}
+
+		delete(ipsec.spiInfo, inSPIID)
+		delete(ipsec.spis, inSPI)
+	}
+
+	// Destroy outbound
+
+	if outSPIInfo, ok := ipsec.spiInfo[outSPIID]; ok {
+		ipsec.log.Infof("ipsec: destroy: out %s -> %s 0x%x", localIP, remoteIP, outSPIInfo.spi)
+
+		if err := netlink.XfrmPolicyDel(xfrmPolicy(localIP, remoteIP, outSPIInfo.spi)); err != nil {
+			ipsec.log.Warnf("ipsec: xfrm policy del (%s, %s, 0x%x) failed: %s", localIP, remoteIP, outSPIInfo.spi, err)
+		}
+
+		outSA := &netlink.XfrmState{
+			Src:   localIP,
+			Dst:   remoteIP,
+			Proto: netlink.XFRM_PROTO_ESP,
+			Spi:   int(outSPIInfo.spi),
+		}
+		if err := netlink.XfrmStateDel(outSA); err != nil {
+			ipsec.log.Warnf("ipsec: xfrm state del (out, %s, %s, 0x%x) failed: %s", outSA.Src, outSA.Dst, outSA.Spi, err)
+		}
+
+		delete(ipsec.spiInfo, outSPIID)
+		delete(ipsec.spis, outSPIInfo.spi)
+	}
+
+	return nil
+}
+
+// Flush removes all policies/SAs established by us. Also, it removes chains and
+// rules of iptables.
+//
+// If destroy is true, the chains and the rules won't be re-created.
+func (ipsec *IPSec) Flush(destroy bool) error {
+	ipsec.Lock()
+	defer ipsec.Unlock()
+
+	policies, err := netlink.XfrmPolicyList(syscall.AF_INET)
+	if err != nil {
+		return errors.Wrap(err, "xfrm policy list")
+	}
+	for _, p := range policies {
+		if p.Mark != nil && p.Mark.Value == mark && len(p.Tmpls) != 0 {
+			spi := SPI(p.Tmpls[0].Spi)
+			if err := netlink.XfrmPolicyDel(&p); err != nil {
+				return errors.Wrap(err, fmt.Sprintf("xfrm policy del (%s, %s, 0x%x)", p.Src, p.Dst, spi))
+			}
+		}
+	}
+
+	states, err := netlink.XfrmStateList(syscall.AF_INET)
+	if err != nil {
+		return errors.Wrap(err, "xfrm state list")
+	}
+	for _, s := range states {
+		if _, ok := ipsec.spis[SPI(s.Spi)]; ok {
+			if err := netlink.XfrmStateDel(&s); err != nil {
+				return errors.Wrap(err, fmt.Sprintf("xfrm state list (%s, %s, 0x%x)", s.Src, s.Dst, s.Spi))
+			}
+		}
+	}
+
+	if err := ipsec.resetIPTables(destroy); err != nil {
+		return errors.Wrap(err, "reset ip tables")
+	}
+
+	return nil
+}
+
+// iptables
+
+type chain struct {
+	table string
+	chain string
+}
+type rule struct {
+	table    string
+	chain    string
+	rulespec []string
+	unique   bool
+}
+
+func (ipsec *IPSec) clearChains(chains []chain) error {
+	for _, c := range chains {
+		if err := ipsec.ipt.ClearChain(c.table, c.chain); err != nil {
+			return errors.Wrap(err, fmt.Sprintf("iptables clear chain (%s, %s)", c.table, c.chain))
+		}
+	}
+	return nil
+}
+
+func (ipsec *IPSec) deleteChains(chains []chain) error {
+	for _, c := range chains {
+		if err := ipsec.ipt.DeleteChain(c.table, c.chain); err != nil {
+			return errors.Wrap(err, fmt.Sprintf("iptables delete chain (%s, %s)", c.table, c.chain))
+		}
+	}
+	return nil
+}
+
+func (ipsec *IPSec) resetRules(rules []rule, destroy bool) error {
+	for _, r := range rules {
+		ok, err := ipsec.ipt.Exists(r.table, r.chain, r.rulespec...)
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("iptables exists rule (%s, %s, %s)", r.table, r.chain, r.rulespec))
+		}
+		switch {
+		case !destroy && !ok:
+			if err := ipsec.ipt.Append(r.table, r.chain, r.rulespec...); err != nil {
+				return errors.Wrap(err, fmt.Sprintf("iptables append rule (%s, %s, %s)", r.table, r.chain, r.rulespec))
+			}
+		case destroy && ok:
+			if err := ipsec.ipt.Delete(r.table, r.chain, r.rulespec...); err != nil {
+				return errors.Wrap(err, fmt.Sprintf("iptables delete rule (%s, %s, %s)", r.table, r.chain, r.rulespec))
+			}
+		}
+	}
+	return nil
+}
+
+func (ipsec *IPSec) resetIPTables(destroy bool) error {
+	chains := []chain{
+		{tableMangle, chainIn},
+		{tableMangle, chainInMark},
+		{tableFilter, chainIn},
+		{tableMangle, chainOut},
+		{tableMangle, chainOutMark},
+	}
+	rules := []rule{
+		{tableMangle, "INPUT", []string{"-j", chainIn}, true},
+		{tableMangle, chainInMark, []string{"-j", "MARK", "--set-xmark", markStr}, true},
+		{tableFilter, "INPUT", []string{"-j", chainIn}, true},
+		{tableMangle, "OUTPUT", []string{"-j", chainOut}, true},
+		{tableMangle, chainOutMark, []string{"-j", "MARK", "--set-xmark", markStr}, true},
+		{tableFilter, "OUTPUT",
+			[]string{
+				"!", "-p", "esp",
+				"-m", "policy", "--dir", "out", "--pol", "none",
+				"-m", "mark", "--mark", markStr,
+				"-j", "DROP"}, true},
+	}
+
+	if err := ipsec.clearChains(chains); err != nil {
+		return err
+	}
+
+	if err := ipsec.resetRules(rules, destroy); err != nil {
+		return err
+	}
+
+	if destroy {
+		if err := ipsec.deleteChains(chains); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func ruleMarkInboundESP(srcIP, dstIP net.IP, inSPI SPI) rule {
+	return rule{tableMangle, chainIn,
+		[]string{
+			"-s", dstIP.String(), "-d", srcIP.String(),
+			"-p", "esp",
+			"-m", "esp", "--espspi", "0x" + strconv.FormatUint(uint64(inSPI), 16),
+			"-j", chainInMark,
+		}, true}
+}
+
+func rulesDropNonEncrypted(srcIP, dstIP net.IP, udpPort int, inSPI SPI) []rule {
+	udpPortStr := strconv.FormatUint(uint64(udpPort), 10)
+	return []rule{
+		ruleMarkInboundESP(srcIP, dstIP, inSPI),
+		{tableFilter, chainIn,
+			[]string{
+				"-s", dstIP.String(), "-d", srcIP.String(),
+				"-p", "udp", "--dport", udpPortStr,
+				"-m", "mark", "!", "--mark", markStr,
+				"-j", "DROP",
+			}, false},
+		{tableMangle, chainOut,
+			[]string{
+				"-s", srcIP.String(), "-d", dstIP.String(),
+				"-p", "udp", "--dport", udpPortStr,
+				"-j", chainOutMark,
+			}, false},
+	}
+}
+
+func (ipsec *IPSec) installDropNonEncrypted(srcIP, dstIP net.IP, udpPort int, inSPI SPI) error {
+	rules := rulesDropNonEncrypted(srcIP, dstIP, udpPort, inSPI)
+	for _, r := range rules {
+		appendFunc := ipsec.ipt.Append
+		if r.unique {
+			appendFunc = ipsec.ipt.AppendUnique
+		}
+		if err := appendFunc(r.table, r.chain, r.rulespec...); err != nil {
+			return errors.Wrap(err, fmt.Sprintf("iptables append (%s, %s, %s)", r.table, r.chain, r.rulespec))
+		}
+	}
+	return nil
+}
+
+func (ipsec *IPSec) removeDropNonEncrypted(srcIP, dstIP net.IP, udpPort int, inSPI SPI) error {
+	rules := rulesDropNonEncrypted(srcIP, dstIP, udpPort, inSPI)
+	if err := ipsec.resetRules(rules, true); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ipsec *IPSec) removeDropNonEncryptedInbound(srcIP, dstIP net.IP, inSPI SPI) error {
+	r := ruleMarkInboundESP(srcIP, dstIP, inSPI)
+	if err := ipsec.ipt.Delete(r.table, r.chain, r.rulespec...); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("iptables delete unique (%s, %s, %s)", r.table, r.chain, r.rulespec))
+	}
+	return nil
+}
+
+// xfrm
+
+func xfrmAllocSpiState(srcIP, dstIP net.IP) *netlink.XfrmState {
+	return &netlink.XfrmState{
+		Src:          srcIP,
+		Dst:          dstIP,
+		Proto:        netlink.XFRM_PROTO_ESP,
+		Mode:         netlink.XFRM_MODE_TRANSPORT,
+		ReplayWindow: 32,
+	}
+}
+
+func xfrmState(srcIP, dstIP net.IP, spi SPI, isDirOut bool, key []byte) (*netlink.XfrmState, error) {
+	if len(key) != keySize {
+		return nil, fmt.Errorf("key should be %d bytes long", keySize)
+	}
+
+	state := xfrmAllocSpiState(srcIP, dstIP)
+
+	state.Spi = int(spi)
+	state.Aead = &netlink.XfrmStateAlgo{
+		Name:   "rfc4106(gcm(aes))",
+		Key:    key,
+		ICVLen: 128,
+	}
+
+	return state, nil
+}
+
+func xfrmPolicy(srcIP, dstIP net.IP, spi SPI) *netlink.XfrmPolicy {
+	ipMask := []byte{0xff, 0xff, 0xff, 0xff} // /32
+
+	return &netlink.XfrmPolicy{
+		Src:   &net.IPNet{IP: srcIP, Mask: ipMask},
+		Dst:   &net.IPNet{IP: dstIP, Mask: ipMask},
+		Proto: syscall.IPPROTO_UDP,
+		Dir:   netlink.XFRM_DIR_OUT,
+		Mark: &netlink.XfrmMark{
+			Value: mark,
+			Mask:  mark,
+		},
+		Tmpls: []netlink.XfrmPolicyTmpl{
+			{
+				Src:   srcIP,
+				Dst:   dstIP,
+				Proto: netlink.XFRM_PROTO_ESP,
+				Mode:  netlink.XFRM_MODE_TRANSPORT,
+				Spi:   int(spi),
+			},
+		},
+	}
+}
+
+// Key derivation
+
+func genNonce() ([]byte, error) {
+	buf := make([]byte, nonceSize)
+	n, err := rand.Read(buf)
+	if err != nil {
+		return nil, fmt.Errorf("crypto rand failed: %s", err)
+	}
+	if n != nonceSize {
+		return nil, fmt.Errorf("not enough of random data: %d", n)
+	}
+	return buf, nil
+}
+
+func deriveKey(sessionKey []byte, nonce []byte, peerName mesh.PeerName) ([]byte, error) {
+	key := make([]byte, keySize)
+
+	info := make([]byte, 8)
+	binary.BigEndian.PutUint64(info, uint64(peerName))
+
+	hkdf := hkdf.New(sha256.New, sessionKey, nonce, info)
+
+	n, err := io.ReadFull(hkdf, key)
+	if err != nil {
+		return nil, err
+	}
+	if n != keySize {
+		return nil, fmt.Errorf("derived too short key: %d", n)
+	}
+
+	return key, nil
+}
+
+// Protocol Messages
+
+type msgInitSARemote struct {
+	nonce []byte
+	spi   SPI
+}
+
+func deserializeMsgInitSARemote(b []byte) (*msgInitSARemote, error) {
+	if len(b) == 0 {
+		return nil, fmt.Errorf("empty msg")
+	}
+
+	msg := &msgInitSARemote{}
+	if len(b) != msg.size() {
+		return nil, fmt.Errorf("invalid payload size: %d", len(b))
+	}
+
+	msg.nonce = make([]byte, nonceSize)
+	copy(msg.nonce, b[:nonceSize])
+	b = b[nonceSize:]
+
+	msg.spi = SPI(binary.BigEndian.Uint32(b))
+
+	return msg, nil
+}
+
+func (msg *msgInitSARemote) size() int {
+	return nonceSize + 32 // SPI
+}
+
+func (msg *msgInitSARemote) serialize() []byte {
+	b := make([]byte, msg.size())
+
+	copy(b[:nonceSize], msg.nonce)
+	binary.BigEndian.PutUint32(b[nonceSize:], uint32(msg.spi))
+
+	return b
+}

prog/weaver/main.go

@@ -224,7 +224,9 @@ func main() {
 		networkConfig.PacketLogging = nopPacketLogging{}
 	}
 
-	overlay, bridge := createOverlay(datapathName, ifaceName, isAWSVPC, config.Host, config.Port, bufSzMB)
+	config.Password = determinePassword(password)
+
+	overlay, bridge := createOverlay(datapathName, ifaceName, isAWSVPC, config.Host, config.Port, bufSzMB, config.Password != nil)
 	networkConfig.Bridge = bridge
 
 	if bridge != nil {
@@ -241,7 +243,6 @@ func main() {
 		checkFatal(err)
 	}
 
-	config.Password = determinePassword(password)
 	config.TrustedSubnets = parseTrustedSubnets(trustedSubnetStr)
 	config.PeerDiscovery = !noDiscovery
 
@@ -405,7 +406,7 @@ func (nopPacketLogging) LogPacket(string, weave.PacketKey) {
 func (nopPacketLogging) LogForwardPacket(string, weave.ForwardPacketKey) {
 }
 
-func createOverlay(datapathName string, ifaceName string, isAWSVPC bool, host string, port int, bufSzMB int) (weave.NetworkOverlay, weave.Bridge) {
+func createOverlay(datapathName string, ifaceName string, isAWSVPC bool, host string, port int, bufSzMB int, enableEncryption bool) (weave.NetworkOverlay, weave.Bridge) {
 	overlay := weave.NewOverlaySwitch()
 	var bridge weave.Bridge
 	var ignoreSleeve bool
@@ -422,7 +423,7 @@ func createOverlay(datapathName string, ifaceName string, isAWSVPC bool, host st
 	case datapathName != "":
 		iface, err := weavenet.EnsureInterface(datapathName)
 		checkFatal(err)
-		fastdp, err := weave.NewFastDatapath(iface, port)
+		fastdp, err := weave.NewFastDatapath(iface, port, enableEncryption)
 		checkFatal(err)
 		bridge = fastdp.Bridge()
 		overlay.Add("fastdp", fastdp.Overlay())

router/awsvpc.go

@@ -64,6 +64,8 @@ func (vpc AWSVPC) Diagnostics() interface{} {
 	return nil
 }
 
+func (vpc AWSVPC) Stop() {}
+
 // NetworkOverlay
 
 func (vpc AWSVPC) InvalidateRoutes() {}

router/fastdp.go

@@ -9,9 +9,12 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/pkg/errors"
 	"github.com/vishvananda/netlink"
 	"github.com/weaveworks/go-odp/odp"
 	"github.com/weaveworks/mesh"
+
+	"github.com/weaveworks/weave/net/ipsec"
 )
 
 // The virtual bridge accepts packets from ODP vports and the router
@@ -39,6 +42,7 @@ type FastDatapath struct {
 	localPeer        *mesh.Peer
 	peers            *mesh.Peers
 	overlayConsumer  OverlayConsumer
+	ipsec            *ipsec.IPSec
 
 	// Bridge state: How to send to the given bridge port
 	sendToPort map[bridgePortID]bridgeSender
@@ -53,6 +57,7 @@ type FastDatapath struct {
 	vxlanUDPPorts    map[int]odp.VportID
 	vxlanVportIDs    map[odp.VportID]struct{}
 	mainVxlanVportID odp.VportID
+	mainVxlanUDPPort int
 
 	// A singleton pool for the occasions when we need to decode
 	// the packet.
@@ -62,7 +67,9 @@ type FastDatapath struct {
 	forwarders map[mesh.PeerName]*fastDatapathForwarder
 }
 
-func NewFastDatapath(iface *net.Interface, port int) (*FastDatapath, error) {
+func NewFastDatapath(iface *net.Interface, port int, encryptionEnabled bool) (*FastDatapath, error) {
+	var ipSec *ipsec.IPSec
+
 	dpif, err := odp.NewDpif()
 	if err != nil {
 		return nil, err
@@ -80,11 +87,22 @@ func NewFastDatapath(iface *net.Interface, port int) (*FastDatapath, error) {
 		return nil, err
 	}
 
+	if encryptionEnabled {
+		var err error
+		if ipSec, err = ipsec.New(log); err != nil {
+			return nil, errors.Wrap(err, "ipsec new")
+		}
+		if err := ipSec.Flush(false); err != nil {
+			return nil, errors.Wrap(err, "ipsec flush")
+		}
+	}
+
 	fastdp := &FastDatapath{
 		iface:         iface,
 		dpif:          dpif,
 		dp:            dp,
 		missHandlers:  make(map[odp.VportID]missHandler),
+		ipsec:         ipSec,
 		sendToPort:    nil,
 		sendToMAC:     make(map[MAC]bridgeSender),
 		seenMACs:      make(map[MAC]struct{}),
@@ -113,7 +131,8 @@ func NewFastDatapath(iface *net.Interface, port int) (*FastDatapath, error) {
 	// numbers to be independent, but working out how to specify
 	// them on the connecting side.  So we can wait to find out if
 	// anyone wants that.
-	fastdp.mainVxlanVportID, err = fastdp.getVxlanVportIDHarder(port+1, 5, time.Millisecond*10)
+	fastdp.mainVxlanUDPPort = port + 1
+	fastdp.mainVxlanVportID, err = fastdp.getVxlanVportIDHarder(fastdp.mainVxlanUDPPort, 5, time.Millisecond*10)
 	if err != nil {
 		return nil, err
 	}
@@ -315,6 +334,14 @@ func (fastdp fastDatapathOverlay) InvalidateShortIDs() {
 	checkWarn(fastdp.deleteFlows())
 }
 
+func (fastdp fastDatapathOverlay) Stop() {
+	if fastdp.ipsec != nil {
+		if err := fastdp.ipsec.Flush(true); err != nil {
+			log.Errorf("ipsec flush failed: %s", err)
+		}
+	}
+}
+
 func (fastDatapathOverlay) AddFeaturesTo(features map[string]string) {
 	// Nothing needed.  Fast datapath support is indicated through
 	// OverlaySwitch.
@@ -542,6 +569,10 @@ type fastDatapathForwarder struct {
 	connUID        uint64
 	vxlanVportID   odp.VportID
 
+	sessionKey                 *[32]byte
+	isEncrypted                bool
+	isOutboundIPSecEstablished bool
+
 	lock              sync.RWMutex
 	confirmed         bool
 	remoteAddr        *net.UDPAddr
@@ -557,27 +588,25 @@ type fastDatapathForwarder struct {
 }
 
 func (fastdp fastDatapathOverlay) PrepareConnection(params mesh.OverlayConnectionParams) (mesh.OverlayConnection, error) {
-	if params.SessionKey != nil {
-		// No encryption support in fastdp
-		return nil, fmt.Errorf("encryption not supported")
-	}
-
 	vxlanVportID := fastdp.mainVxlanVportID
-	var remoteAddr *net.UDPAddr
+	vxlanUDPPort := fastdp.mainVxlanUDPPort
 
+	remoteAddr := makeUDPAddr(params.RemoteAddr)
 	if params.Outbound {
-		remoteAddr = makeUDPAddr(params.RemoteAddr)
+		var err error
 		// The provided address contains the main weave port
 		// number to connect to.  We need to derive the vxlan
 		// port number from that.
 		vxlanRemoteAddr := *remoteAddr
 		vxlanRemoteAddr.Port++
 		remoteAddr = &vxlanRemoteAddr
-		var err error
-		vxlanVportID, err = fastdp.getVxlanVportID(remoteAddr.Port)
+		vxlanUDPPort = remoteAddr.Port
+		vxlanVportID, err = fastdp.getVxlanVportID(vxlanUDPPort)
 		if err != nil {
 			return nil, err
 		}
+	} else {
+		remoteAddr.Port = vxlanUDPPort
 	}
 
 	localIP, err := ipv4Bytes(params.LocalAddr.IP)
@@ -592,6 +621,7 @@ func (fastdp fastDatapathOverlay) PrepareConnection(params mesh.OverlayConnectio
 		sendControlMsg: params.SendControlMessage,
 		connUID:        params.ConnUID,
 		vxlanVportID:   vxlanVportID,
+		sessionKey:     params.SessionKey,
 
 		remoteAddr:        remoteAddr,
 		heartbeatInterval: FastHeartbeat,
@@ -601,7 +631,7 @@ func (fastdp fastDatapathOverlay) PrepareConnection(params mesh.OverlayConnectio
 		errorChan:       make(chan error, 1),
 	}
 
-	return fwd, err
+	return fwd, nil
 }
 
 func ipv4Bytes(ip net.IP) (res [4]byte, err error) {
@@ -626,11 +656,30 @@ func (fwd *fastDatapathForwarder) Confirm() {
 		log.Fatal(fwd.logPrefix(), "already confirmed")
 	}
 
+	if fwd.fastdp.ipsec != nil && fwd.sessionKey != nil {
+		fwd.isEncrypted = true
+		log.Info("Setting up IPsec between ", fwd.fastdp.localPeer, " and ", fwd.remotePeer)
+		err := fwd.fastdp.ipsec.InitSALocal(
+			fwd.fastdp.localPeer.Name, fwd.remotePeer.Name, fwd.connUID,
+			net.IP(fwd.localIP[:]), fwd.remoteAddr.IP,
+			fwd.remoteAddr.Port,
+			fwd.sessionKey,
+			func(msg []byte) error {
+				return fwd.sendControlMsg(FastDatapathCryptoInitSARemote, msg)
+			},
+		)
+		if err != nil {
+			log.Error(fwd.logPrefix(), "ipsec init SA local failed: ", err)
+			fwd.handleError(err)
+			return
+		}
+	}
+
 	log.Debug(fwd.logPrefix(), "confirmed")
 	fwd.fastdp.addForwarder(fwd.remotePeer.Name, fwd)
 	fwd.confirmed = true
 
-	if fwd.remoteAddr != nil {
+	if fwd.remoteAddr != nil && (!fwd.isEncrypted || fwd.isOutboundIPSecEstablished) {
 		// have the goroutine send a heartbeat straight away
 		fwd.heartbeatTimer = time.NewTimer(0)
 	} else {
@@ -719,6 +768,7 @@ func (fwd *fastDatapathForwarder) sendHeartbeat() {
 
 const (
 	FastDatapathHeartbeatAck = iota
+	FastDatapathCryptoInitSARemote
 )
 
 func (fwd *fastDatapathForwarder) handleVxlanSpecialPacket(frame []byte, sender *net.UDPAddr) {
@@ -768,6 +818,8 @@ func (fwd *fastDatapathForwarder) ControlMessage(tag byte, msg []byte) {
 	switch tag {
 	case FastDatapathHeartbeatAck:
 		fwd.handleHeartbeatAck()
+	case FastDatapathCryptoInitSARemote:
+		fwd.handleCryptoInitSARemote(msg)
 
 	default:
 		log.Info(fwd.logPrefix(), "Ignoring unknown control message: ", tag)
@@ -790,6 +842,26 @@ func (fwd *fastDatapathForwarder) handleHeartbeatAck() {
 	}
 }
 
+func (fwd *fastDatapathForwarder) handleCryptoInitSARemote(msg []byte) {
+	log.Info(fwd.logPrefix(), "IPSec init SA remote")
+	err := fwd.fastdp.ipsec.InitSARemote(
+		msg,
+		fwd.fastdp.localPeer.Name, fwd.remotePeer.Name, fwd.connUID,
+		net.IP(fwd.localIP[:]), fwd.remoteAddr.IP, fwd.remoteAddr.Port,
+		fwd.sessionKey,
+	)
+	if err != nil {
+		log.Warning(fwd.logPrefix(), "IPSec init SA remote failed: ", err)
+		fwd.handleError(err)
+		return
+	}
+
+	if !fwd.isOutboundIPSecEstablished {
+		fwd.isOutboundIPSecEstablished = true
+		fwd.heartbeatTimer.Reset(0)
+	}
+}
+
 func (fwd *fastDatapathForwarder) Forward(key ForwardPacketKey) FlowOp {
 	if !key.SrcPeer.HasShortID || !key.DstPeer.HasShortID {
 		return nil
@@ -839,6 +911,18 @@ func (fwd *fastDatapathForwarder) Stop() {
 	defer fwd.lock.Unlock()
 	fwd.sendControlMsg = func(byte, []byte) error { return nil }
 
+	if fwd.isEncrypted {
+		localIP := net.IP(fwd.localIP[:])
+		log.Info("Destroying IPsec between ", fwd.fastdp.localPeer, " and ", fwd.remotePeer)
+		err := fwd.fastdp.ipsec.Destroy(
+			fwd.fastdp.localPeer.Name, fwd.remotePeer.Name, fwd.connUID,
+			localIP, fwd.remoteAddr.IP, fwd.remoteAddr.Port,
+		)
+		if err != nil {
+			log.Errorf("ipsec destroy failed: %s", err)
+		}
+	}
+
 	// stop the heartbeat goroutine
 	if !fwd.stopped {
 		fwd.stopped = true

router/overlay_switch.go

@@ -50,6 +50,12 @@ func (osw *OverlaySwitch) Diagnostics() interface{} {
 	return diagnostics
 }
 
+func (osw *OverlaySwitch) Stop() {
+	for _, overlay := range osw.overlays {
+		overlay.Stop()
+	}
+}
+
 func (osw *OverlaySwitch) InvalidateRoutes() {
 	for _, overlay := range osw.overlays {
 		overlay.InvalidateRoutes()

router/sleeve.go

@@ -141,6 +141,10 @@ func (*SleeveOverlay) Diagnostics() interface{} {
 	return nil
 }
 
+func (*SleeveOverlay) Stop() {
+	// do nothing
+}
+
 func (sleeve *SleeveOverlay) lookupForwarder(peer mesh.PeerName) *sleeveForwarder {
 	sleeve.lock.Lock()
 	defer sleeve.lock.Unlock()

test/110_sleeve_encryption_2_test.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+. ./config.sh
+
+C1=10.2.1.4
+C2=10.2.1.7
+
+start_suite "Ping over encrypted cross-host weave network (sleeve)"
+
+WEAVE_NO_FASTDP=1 weave_on $HOST1 launch --password wfvAwt7sj
+WEAVE_NO_FASTDP=1 weave_on $HOST2 launch --password wfvAwt7sj $HOST1
+
+start_container $HOST1 $C1/24 --name=c1
+start_container $HOST2 $C2/24 --name=c2
+assert_raises "exec_on $HOST1 c1 $PING $C2"
+
+assert_raises "weave_on $HOST1 status connections | grep -P 'encrypted *sleeve'"
+assert_raises "weave_on $HOST2 status connections | grep -P 'encrypted *sleeve'"
+
+end_suite

test/111_fastdp_encryption_2_test.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+# TODO(mp) 1. test anti-replay 2. test sleeve <-> fastdp
+# TODO(mp) test with --trusted-subnets
+
+. ./config.sh
+
+HOST1_IP=$($SSH $HOST1 getent ahosts $HOST1 | grep "RAW" | cut -d' ' -f1)
+HOST2_IP=$($SSH $HOST2 getent ahosts $HOST2 | grep "RAW" | cut -d' ' -f1)
+
+start_tcpdump() {
+    HOST=$1
+    PCAP=$2
+    run_on $HOST "sudo nohup tcpdump -i any -w $PCAP >/dev/null 2>&1 & echo \$! > $PCAP.pid"
+}
+
+stop_tcpdump() {
+    HOST=$1
+    PCAP=$2
+    $SSH $HOST "sudo kill \$(cat $PCAP.pid) && rm -f $PCAP.pid && sudo base64 $PCAP" \
+        | base64 -d > $PCAP
+}
+
+# Checks whether all vxlan-tunneled traffic is encrypted.
+assert_pcap() {
+    PCAP=$1
+    # UDP port is not exposed in ESP
+    assert "tcpdump -r $PCAP 'src $HOST1_IP && dst $HOST2_IP && dst port 6784'" ""
+    assert "tcpdump -r $PCAP 'src $HOST2_IP && dst $HOST1_IP && dst port 6784'" ""
+    # ip_proto(ESP) = 50
+    assert_raises "[[ -n \"$(tcpdump -r $PCAP "src $HOST1_IP && dst $HOST2_IP && proto 50")\" ]]"
+    assert_raises "[[ -n \"$(tcpdump -r $PCAP "src $HOST2_IP && dst $HOST1_IP && proto 50")\" ]]"
+
+    rm -f $PCAP
+}
+
+C1=10.2.1.4
+C2=10.2.1.7
+
+start_suite "Ping over encrypted cross-host weave network (fastdp)"
+
+WEAVE_MTU=1300 weave_on $HOST1 launch --password wfvAwt7sj
+WEAVE_MTU=1300 weave_on $HOST2 launch --password wfvAwt7sj $HOST1
+
+assert_raises "weave_on $HOST1 status connections | grep -P 'encrypted *fastdp'"
+assert_raises "weave_on $HOST2 status connections | grep -P 'encrypted *fastdp'"
+
+PCAP1=$(mktemp)
+PCAP2=$(mktemp)
+
+start_tcpdump $HOST1 $PCAP1
+start_tcpdump $HOST2 $PCAP2
+
+start_container $HOST1 $C1/24 --name=c1
+start_container $HOST2 $C2/24 --name=c2
+assert_raises "exec_on $HOST1 c1 $PING $C2"
+
+stop_tcpdump $HOST1 $PCAP1
+stop_tcpdump $HOST2 $PCAP2
+
+assert_pcap $PCAP1
+assert_pcap $PCAP2
+
+weave_on $HOST2 reset
+# policies/SAs and iptables rules should have been removed after terminating
+# the connection.
+assert "$SSH $HOST1 sudo ip xfrm state" ""
+assert "$SSH $HOST1 sudo ip xfrm policy" ""
+assert "$SSH $HOST1 sudo iptables -t mangle -S WEAVE-IPSEC | grep '\-A'" ""
+
+end_suite