From d8cc45c6503d8b1d240ec8843d07dc921d57ca5b Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Fri, 29 Aug 2014 16:10:29 +0100
Subject: [PATCH 01/40] fix weave arg count check

---
 weaver/weave | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/weaver/weave b/weaver/weave
index 695f3a3b..e5a7a27c 100755
--- a/weaver/weave
+++ b/weaver/weave
@@ -1,6 +1,15 @@
 #!/bin/sh
 set -e
 
+usage() {
+    echo "Usage:"
+    echo "weave launch <ipaddr>/<subnet> [-passwd <passwd>] <peer_host> ..."
+    echo "weave run    <ipaddr>/<subnet> <docker run args> ..."
+    exit 1
+}
+
+[ $# -gt 1 ] || usage
+
 BRIDGE=weave
 CONTAINER_IFNAME=ethwe
 MTU=65535
@@ -9,14 +18,7 @@ SCRIPT_DIR=$(dirname $(readlink -f $0))
 COMMAND=$1
 IPADDR=$2
 
-usage() {
-    echo "Usage:"
-    echo "weave launch <ipaddr>/<subnet> [-passwd <passwd>] <peer_host> ..."
-    echo "weave run    <ipaddr>/<subnet> <docker run args> ..."
-    exit 1
-}
-
-shift 2 || usage
+shift 2
 
 [ `id -u` = 0 ] || {
     echo "weave must be run as 'root'"

From 180c142964465def77b8ddc9a12af46745ce243a Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Fri, 29 Aug 2014 16:12:44 +0100
Subject: [PATCH 02/40] move the most important check first

---
 weaver/weave | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/weaver/weave b/weaver/weave
index e5a7a27c..a06b3d08 100755
--- a/weaver/weave
+++ b/weaver/weave
@@ -8,6 +8,11 @@ usage() {
     exit 1
 }
 
+[ `id -u` = 0 ] || {
+    echo "weave must be run as 'root'"
+    exit 1
+}
+
 [ $# -gt 1 ] || usage
 
 BRIDGE=weave
@@ -20,11 +25,6 @@ IPADDR=$2
 
 shift 2
 
-[ `id -u` = 0 ] || {
-    echo "weave must be run as 'root'"
-    exit 1
-}
-
 # pre-create bridge so we can set the MTU
 [ ! -d /sys/class/net/$BRIDGE ] && {
     ip link add dev $BRIDGE type bridge

From 2c88710d627378041a8d943d884049cd8b3695e4 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Fri, 29 Aug 2014 16:16:16 +0100
Subject: [PATCH 03/40] we only need the bridge address when launching weave

---
 weaver/weave | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/weaver/weave b/weaver/weave
index a06b3d08..5766717c 100755
--- a/weaver/weave
+++ b/weaver/weave
@@ -39,15 +39,15 @@ shift 2
     ip link set dev $BRIDGE up
 }
 
-# We set the router name to the bridge mac since that is stable across
-# re-creations of the containers.
-#
-# TODO use the mac of one of the physical host interfaces (eth0,
-# wlan0, etc) so the name becomes stable across host restarts.
-MACADDR=`cat /sys/class/net/$BRIDGE/address`
-
 case "$COMMAND" in
     launch)
+        # We set the router name to the bridge mac since that is
+        # stable across re-creations of the containers.
+        #
+        # TODO use the mac of one of the physical host interfaces
+        # (eth0, wlan0, etc) so the name becomes stable across host
+        # restarts.
+        MACADDR=`cat /sys/class/net/$BRIDGE/address`
         CONTAINER=$(docker run --privileged -d --name=weave -p 6783:6783/tcp -p 6783:6783/udp $WEAVE_DOCKER_ARGS zettio/weave -name $MACADDR "$@")
         IFACE=eth0
         ;;

From 51a33929eae54b3098f7aeba38e54e14c3847a0e Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Fri, 29 Aug 2014 16:17:10 +0100
Subject: [PATCH 04/40] remove unused var

---
 weaver/weave | 1 -
 1 file changed, 1 deletion(-)

diff --git a/weaver/weave b/weaver/weave
index 5766717c..ec4bda30 100755
--- a/weaver/weave
+++ b/weaver/weave
@@ -19,7 +19,6 @@ BRIDGE=weave
 CONTAINER_IFNAME=ethwe
 MTU=65535
 
-SCRIPT_DIR=$(dirname $(readlink -f $0))
 COMMAND=$1
 IPADDR=$2
 

From d0d8949f7236616399c7929f4c3ae438151e6cc1 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Fri, 29 Aug 2014 16:20:41 +0100
Subject: [PATCH 05/40] make 'weave <unknown_command>' a no-op

---
 weaver/weave | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/weaver/weave b/weaver/weave
index ec4bda30..8b495ad8 100755
--- a/weaver/weave
+++ b/weaver/weave
@@ -24,22 +24,24 @@ IPADDR=$2
 
 shift 2
 
-# pre-create bridge so we can set the MTU
-[ ! -d /sys/class/net/$BRIDGE ] && {
-    ip link add dev $BRIDGE type bridge
-    ip link set dev $BRIDGE address 7a:$(od -txC -An -N5 /dev/random | tr \  : | tail -c+2)
-    # attempting to set the bridge MTU to a high value directly
-    # fails. Bridges take the lowest MTU of their interfaces. So
-    # instead we create a temporary interface with the desired MTU,
-    # attach that to the bridge, and then remove it again.
-    ip link add name v${CONTAINER_IFNAME}du mtu $MTU type dummy
-    ip link set dev v${CONTAINER_IFNAME}du master $BRIDGE
-    ip link del dev v${CONTAINER_IFNAME}du
-    ip link set dev $BRIDGE up
+create_bridge() {
+    [ ! -d /sys/class/net/$BRIDGE ] && {
+        ip link add dev $BRIDGE type bridge
+        ip link set dev $BRIDGE address 7a:$(od -txC -An -N5 /dev/random | tr \  : | tail -c+2)
+        # Attempting to set the bridge MTU to a high value directly
+        # fails. Bridges take the lowest MTU of their interfaces. So
+        # instead we create a temporary interface with the desired
+        # MTU, attach that to the bridge, and then remove it again.
+        ip link add name v${CONTAINER_IFNAME}du mtu $MTU type dummy
+        ip link set dev v${CONTAINER_IFNAME}du master $BRIDGE
+        ip link del dev v${CONTAINER_IFNAME}du
+        ip link set dev $BRIDGE up
+    }
 }
 
 case "$COMMAND" in
     launch)
+        create_bridge
         # We set the router name to the bridge mac since that is
         # stable across re-creations of the containers.
         #
@@ -51,6 +53,7 @@ case "$COMMAND" in
         IFACE=eth0
         ;;
     run)
+        create_bridge
         CONTAINER=$(docker run -d "$@")
         IFACE=$CONTAINER_IFNAME
         ;;

From 3107aba53f2b030454bf77feed9b0b1467ebe9d6 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Fri, 29 Aug 2014 17:40:38 +0100
Subject: [PATCH 06/40] unbreak script I have no idea why this matters

---
 weaver/weave | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/weaver/weave b/weaver/weave
index 8b495ad8..73da8bd0 100755
--- a/weaver/weave
+++ b/weaver/weave
@@ -35,8 +35,8 @@ create_bridge() {
         ip link add name v${CONTAINER_IFNAME}du mtu $MTU type dummy
         ip link set dev v${CONTAINER_IFNAME}du master $BRIDGE
         ip link del dev v${CONTAINER_IFNAME}du
-        ip link set dev $BRIDGE up
     }
+    ip link set dev $BRIDGE up
 }
 
 case "$COMMAND" in

From 5b6dc2d762c89ead3d8efd1ab937ad08db92f718 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Fri, 29 Aug 2014 19:15:18 +0100
Subject: [PATCH 07/40] (slightly) faster pmtu discovery

Issues with the previous scheme...

When we have just encountered an EMSGSIZE, the odds of encountering
another one straight away, when attemtping to send a verification
packet with the lower mtu, are slim, since for a new, lower mtu to be
set some packet must have made it further then the one that triggered
the icmp3.4 that gave rise to the EMSGSIZE, and we must have received
the resulting icmp3.4. All of which takes time.

Also, sending a bunch of udp packets in rapid succession is not a
great way to improve their chances of making it through to the other
peer, since they could easily all get dropped by a single congestion
event or other network glitch.

Now we send a single verification packet, wait for a short while, and
if we don't get a response then we send another one, wait for longer
period (2x), etc, until some upper bound is reached, at which point we
lower the pmtu and start the verification process again (same as
currently happens).

We are now also calling flush() to send the verification packet, just
as we do for ordinary packets. This makes for shorter code, consistent
error handling, and introduces a nice little recursion :)
---
 consts.go    | 33 +++++++++++++++++----------------
 forwarder.go | 45 +++++++++++++++++++++------------------------
 types.go     | 17 +++++++++--------
 3 files changed, 47 insertions(+), 48 deletions(-)

diff --git a/consts.go b/consts.go
index 106e1c8e..d682b0c7 100644
--- a/consts.go
+++ b/consts.go
@@ -5,22 +5,23 @@ import (
 )
 
 const (
-	Protocol          = "weave"
-	ProtocolVersion   = 8
-	EthernetOverhead  = 14
-	UDPOverhead       = 28 // 20 bytes for IPv4, 8 bytes for UDP
-	Port              = 6783
-	DefaultPMTU       = 65535
-	MaxUDPPacketSize  = 65536
-	ChannelSize       = 16
-	UDPNonceSendAt    = 8192
-	FragTestSize      = 60001
-	PMTUDiscoverySize = 60000
-	FastHeartbeat     = 500 * time.Millisecond
-	SlowHeartbeat     = 10 * time.Second
-	FetchAllInterval  = 30 * time.Second
-	FragTestInterval  = 5 * time.Minute
-	PMTUVerifyTimeout = 5 * time.Second
+	Protocol           = "weave"
+	ProtocolVersion    = 8
+	EthernetOverhead   = 14
+	UDPOverhead        = 28 // 20 bytes for IPv4, 8 bytes for UDP
+	Port               = 6783
+	DefaultPMTU        = 65535
+	MaxUDPPacketSize   = 65536
+	ChannelSize        = 16
+	UDPNonceSendAt     = 8192
+	FragTestSize       = 60001
+	PMTUDiscoverySize  = 60000
+	FastHeartbeat      = 500 * time.Millisecond
+	SlowHeartbeat      = 10 * time.Second
+	FetchAllInterval   = 30 * time.Second
+	FragTestInterval   = 5 * time.Minute
+	PMTUVerifyAttempts = 8
+	PMTUVerifyTimeout  = 10 * time.Millisecond // gets doubled with every attempt
 )
 
 const (
diff --git a/forwarder.go b/forwarder.go
index 7a15a25b..c48bcaf1 100644
--- a/forwarder.go
+++ b/forwarder.go
@@ -206,9 +206,15 @@ func (fwd *Forwarder) Run() {
 			// to other data.
 			fwd.pmtuVerifyTick = nil
 			if !fwd.conn.isEffectivePMTUVerfied() {
-				fwd.effectivePMTU -= 8
-				fwd.maxPayload -= 8
-				fwd.updateEffectivePMTU()
+				if fwd.pmtuVerifyCount == 0 {
+					fwd.pmtuVerifyCount = PMTUVerifyAttempts
+					fwd.effectivePMTU -= 8
+					fwd.maxPayload -= 8
+					fwd.conn.setEffectivePMTU(fwd.effectivePMTU)
+				} else {
+					fwd.pmtuVerifyCount--
+				}
+				fwd.verifyEffectivePMTU()
 			}
 		case frame = <-fwd.ch:
 			if !fwd.appendFrame(frame) {
@@ -242,27 +248,16 @@ func (fwd *Forwarder) setMaxPayloadAndEffectivePMTU(pmtu int) {
 	fwd.effectivePMTU = fwd.maxPayload - fwd.enc.PacketOverhead() - fwd.enc.FrameOverhead() - EthernetOverhead
 }
 
-func (fwd *Forwarder) updateEffectivePMTU() {
-	for gotNewPMTU := true; gotNewPMTU; {
-		fwd.conn.setEffectivePMTU(fwd.effectivePMTU)
-		pmtuVerifyFrame := &ForwardedFrame{
-			srcPeer: fwd.conn.local,
-			dstPeer: fwd.conn.remote,
-			frame:   make([]byte, fwd.effectivePMTU+EthernetOverhead)}
-		gotNewPMTU = false
-		for cnt := 0; cnt < 10; cnt += 1 {
-			fwd.enc.AppendFrame(pmtuVerifyFrame)
-			err := fwd.udpSender.Send(fwd.enc.Bytes())
-			if err != nil {
-				if mtbe, ok := err.(MsgTooBigError); ok {
-					fwd.setMaxPayloadAndEffectivePMTU(mtbe.PMTU)
-					gotNewPMTU = true
-					break
-				}
-			}
-		}
+func (fwd *Forwarder) verifyEffectivePMTU() {
+	pmtuVerifyFrame := &ForwardedFrame{
+		srcPeer: fwd.conn.local,
+		dstPeer: fwd.conn.remote,
+		frame:   make([]byte, fwd.effectivePMTU+EthernetOverhead)}
+	fwd.enc.AppendFrame(pmtuVerifyFrame)
+	fwd.flush()
+	if fwd.pmtuVerifyTick == nil {
+		fwd.pmtuVerifyTick = time.After(PMTUVerifyTimeout << (PMTUVerifyAttempts - fwd.pmtuVerifyCount))
 	}
-	fwd.pmtuVerifyTick = time.After(PMTUVerifyTimeout)
 }
 
 func (fwd *Forwarder) appendFrame(frame *ForwardedFrame) bool {
@@ -279,7 +274,9 @@ func (fwd *Forwarder) flush() {
 	if err != nil {
 		if mtbe, ok := err.(MsgTooBigError); ok {
 			fwd.setMaxPayloadAndEffectivePMTU(mtbe.PMTU)
-			fwd.updateEffectivePMTU()
+			fwd.conn.setEffectivePMTU(fwd.effectivePMTU)
+			fwd.pmtuVerifyCount = PMTUVerifyAttempts
+			fwd.verifyEffectivePMTU()
 		} else if PosixError(err) == syscall.ENOBUFS {
 			// TODO handle this better
 		} else {
diff --git a/types.go b/types.go
index cf675a21..861fa7f5 100644
--- a/types.go
+++ b/types.go
@@ -86,14 +86,15 @@ type ConnectionInteraction struct {
 }
 
 type Forwarder struct {
-	conn           *LocalConnection
-	ch             <-chan *ForwardedFrame
-	stop           <-chan interface{}
-	pmtuVerifyTick <-chan time.Time
-	enc            Encryptor
-	udpSender      UDPSender
-	maxPayload     int
-	effectivePMTU  int
+	conn            *LocalConnection
+	ch              <-chan *ForwardedFrame
+	stop            <-chan interface{}
+	pmtuVerifyTick  <-chan time.Time
+	pmtuVerifyCount uint
+	enc             Encryptor
+	udpSender       UDPSender
+	maxPayload      int
+	effectivePMTU   int
 }
 
 type UDPPacket struct {

From 98257d0df685a8ff6843e0a4b3c30e9bc8771450 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Sat, 30 Aug 2014 09:59:13 +0100
Subject: [PATCH 08/40] switch to ubuntu image for examples It comes with more
 stuff pre-installed, such as 'nc', so makes experimentation just a little
 more convenient.

---
 README.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 6211ee08..858477d9 100644
--- a/README.md
+++ b/README.md
@@ -40,7 +40,7 @@ two containers, one on each host.
 On $HOST1 run (as root)
 
     host1# WEAVE=$(weave launch 10.0.0.1/16)
-    host1# C=$(weave run 10.0.1.1/24 -t -i debian /bin/bash)
+    host1# C=$(weave run 10.0.1.1/24 -t -i ubuntu /bin/bash)
 
 The first line starts the weave router, in a container. This needs to
 be done once on each host. We tell weave that its IP address should
@@ -50,7 +50,7 @@ The second line starts our application container. We give it an IP
 address and network (a subnet of the weave network). `weave run`
 invokes `docker run -d` with all the parameter following the IP
 address and netmask. So we could be launching any container this way;
-here we just take a stock debian container and launch a shell in it.
+here we just take a stock ubuntu container and launch a shell in it.
 
 If our application consists of more than one container on this host we
 simply launch them with a variation on that 2nd line.
@@ -63,7 +63,7 @@ individual IP addresses must, of course, be unique.
 We repeat similar steps on $HOST2...
 
     host2# WEAVE=$(weave launch 10.0.0.2/16 $HOST1)
-    host2# C=$(weave run 10.0.1.2/24 -t -i debian /bin/bash)
+    host2# C=$(weave run 10.0.1.2/24 -t -i ubuntu /bin/bash)
 
 The only difference, apart from the IP addresses, is that we tell our
 weave that it should peer with the weave running on $HOST1. We could
@@ -135,8 +135,8 @@ To accomplish that, we assign each application a different subnet. So,
 in the above example, if we wanted to add another application similar
 to, but isolated from, our first, we'd launch the containers with...
 
-    host1# D=$(weave run 10.0.2.1/24 -t -i debian /bin/bash)
-    host2# D=$(weave run 10.0.2.2/24 -t -i debian /bin/bash)
+    host1# D=$(weave run 10.0.2.1/24 -t -i ubuntu /bin/bash)
+    host2# D=$(weave run 10.0.2.2/24 -t -i ubuntu /bin/bash)
 
 A quick 'ping' test in the containers confirms that they can talk to
 each other but not the containers of our first application...
@@ -263,7 +263,7 @@ example above, the 'launch' command would be run like this:
 
 and the 'run' command like this:
 
-    host1# C=$(boot2docker ssh "sudo ./weave run 10.0.1.1/24 -t -i debian /bin/bash")
+    host1# C=$(boot2docker ssh "sudo ./weave run 10.0.1.1/24 -t -i ubuntu /bin/bash")
 
 ## Building
 

From 2e73fbb797a6e056bd895ec43e6e7022e1185a41 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Sat, 30 Aug 2014 10:06:12 +0100
Subject: [PATCH 09/40] link to architecture documentation

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index 858477d9..3cc295c0 100644
--- a/README.md
+++ b/README.md
@@ -279,3 +279,6 @@ Then simply run
 
 This will build the weave router, produce a docker image
 `zettio/weave` and export that image to /tmp/weave.tar
+
+If you are interested in the inner workings of weave, read the
+[architecture documentation](docs/architecture.txt).

From 53f06c07a55c184d3e8a557830fa1810ec57461a Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Sat, 30 Aug 2014 11:44:03 +0100
Subject: [PATCH 10/40] document external service access

---
 README.md | 63 ++++++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 56 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 3cc295c0..2ef4069c 100644
--- a/README.md
+++ b/README.md
@@ -8,13 +8,15 @@ plugged into the same network switch, with no need to configure port
 mappings, links, etc. Weave can optionally encrypt traffic, allowing
 hosts to be connected across an untrusted network.
 
+Services provided by application containers on the weave network can
+be made accessible to the outside world, regardless of where those
+containers are running.
+
 With weave you can easily construct applications consisting of
 multiple containers, running anywhere.
 
-Weave complements Docker's existing networking capabilities - use
-weave to establish network connectivity *between* containers, and
-Docker port mappings etc to establish network connectivity *to*
-individual containers from the outside world.
+Weave works alongside Docker's existing (single host) networking
+capabilities, so these can continue to be used by containers.
 
 ## Installation
 
@@ -56,9 +58,10 @@ If our application consists of more than one container on this host we
 simply launch them with a variation on that 2nd line.
 
 The IP addresses and netmasks can be anything you like which doesn't
-conflict with any IP ranges of 'external' services your containers
-need to connect to. The same IP range must be used everywhere, and the
-individual IP addresses must, of course, be unique.
+conflict with any IP ranges of 'external' services the hosts or your
+containers need to connect to. The same IP range must be used
+everywhere, and the individual IP addresses must, of course, be
+unique.
 
 We repeat similar steps on $HOST2...
 
@@ -171,6 +174,52 @@ The same password must be specified for all weave peers; it is the a
 component in the creation of ephemeral session keys for connections
 between peers.
 
+### External access
+
+Services running in containers on a weave network can be made
+accessible to the outside world (or, more generally, other networks)
+from any weave host, regardless of where the service containers are
+located.
+
+Say we want to make our 'nc' "service" that is running in the
+container on $HOST1 accessible to the outside world via $HOST2. This
+requires the following steps on $HOST2...
+
+1. Add a NAT masquerading rule for the entire weave network. This
+   needs to be done just once.
+
+        host2# iptables -t nat -A POSTROUTING -d 10.0.0.2/16 ! -s 10.0.0.2/16 -j MASQUERADE
+
+   The IP address and netmask are the same as given in the `weave
+   launch` command.
+
+2. Add an IP address to the weave bridge. This needs to be done once
+   for every weave application subnet.
+
+        host2# ip addr add dev weave 10.0.1.102/24
+
+   The address must be on the destination container's weave subnet and
+   not already in use.
+
+3. Add a NAT rule to route from the outside world to the destination
+   container service. This needs to be done once for every service we
+   want to expose.
+
+        host2# iptables -t nat -A PREROUTING -p udp -i eth0 --dport 2211 -j DNAT --to-destination 10.0.1.1:4422
+
+    Here we are assuming that the "outside world" is connecting to
+    $HOST2 via 'eth0'. We want UDP traffic to port 2211 on the
+    external IPs to be routed to our 'nc' service, which is running on
+    port 4422 in the container with IP 10.0.1.1.
+
+With the above in place, we can connect to our 'nc' service from anywhere with
+
+    nc -u $HOST2 2211
+
+Similar NAT rules to the above can used to expose services provided by
+containers on the weave network not just to outside world but also
+other, internal, networks.
+
 ### Multi-cloud networking
 
 Weave can network containers hosted in different cloud providers /

From 86f993a1e8a15cfefd25359acbaa304211704d83 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Sat, 30 Aug 2014 11:45:29 +0100
Subject: [PATCH 11/40] cosmetic

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 2ef4069c..db782f08 100644
--- a/README.md
+++ b/README.md
@@ -205,7 +205,8 @@ requires the following steps on $HOST2...
    container service. This needs to be done once for every service we
    want to expose.
 
-        host2# iptables -t nat -A PREROUTING -p udp -i eth0 --dport 2211 -j DNAT --to-destination 10.0.1.1:4422
+        host2# iptables -t nat -A PREROUTING -p udp -i eth0 --dport 2211 -j DNAT \
+               --to-destination 10.0.1.1:4422
 
     Here we are assuming that the "outside world" is connecting to
     $HOST2 via 'eth0'. We want UDP traffic to port 2211 on the

From 038d5c4f80634998310d7bec33965968b81e6685 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Sat, 30 Aug 2014 11:46:03 +0100
Subject: [PATCH 12/40] cosmetic

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index db782f08..4b6d5602 100644
--- a/README.md
+++ b/README.md
@@ -213,7 +213,8 @@ requires the following steps on $HOST2...
     external IPs to be routed to our 'nc' service, which is running on
     port 4422 in the container with IP 10.0.1.1.
 
-With the above in place, we can connect to our 'nc' service from anywhere with
+With the above in place, we can connect to our 'nc' service from
+anywhere with
 
     nc -u $HOST2 2211
 

From b9be18c38696a2c57318580e8a6ad0ffa885b69c Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Sat, 30 Aug 2014 14:25:00 +0100
Subject: [PATCH 13/40] set up masquerading as part of 'weave launch' thus
 simplifying external access

---
 README.md    | 12 ++----------
 weaver/weave |  9 +++++++++
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 4b6d5602..42c6f3cf 100644
--- a/README.md
+++ b/README.md
@@ -185,15 +185,7 @@ Say we want to make our 'nc' "service" that is running in the
 container on $HOST1 accessible to the outside world via $HOST2. This
 requires the following steps on $HOST2...
 
-1. Add a NAT masquerading rule for the entire weave network. This
-   needs to be done just once.
-
-        host2# iptables -t nat -A POSTROUTING -d 10.0.0.2/16 ! -s 10.0.0.2/16 -j MASQUERADE
-
-   The IP address and netmask are the same as given in the `weave
-   launch` command.
-
-2. Add an IP address to the weave bridge. This needs to be done once
+1. Add an IP address to the weave bridge. This needs to be done once
    for every weave application subnet.
 
         host2# ip addr add dev weave 10.0.1.102/24
@@ -201,7 +193,7 @@ requires the following steps on $HOST2...
    The address must be on the destination container's weave subnet and
    not already in use.
 
-3. Add a NAT rule to route from the outside world to the destination
+2. Add a NAT rule to route from the outside world to the destination
    container service. This needs to be done once for every service we
    want to expose.
 
diff --git a/weaver/weave b/weaver/weave
index 73da8bd0..67b0efc1 100755
--- a/weaver/weave
+++ b/weaver/weave
@@ -39,9 +39,18 @@ create_bridge() {
     ip link set dev $BRIDGE up
 }
 
+setup_masquerading() {
+    IPTABLES_ARGS="POSTROUTING -d $IPADDR ! -s $IPADDR -j MASQUERADE"
+    if ! iptables -w -t nat -C $IPTABLES_ARGS >/dev/null 2>&1
+    then
+        iptables -w -t nat -A $IPTABLES_ARGS >/dev/null
+    fi
+}
+
 case "$COMMAND" in
     launch)
         create_bridge
+        setup_masquerading
         # We set the router name to the bridge mac since that is
         # stable across re-creations of the containers.
         #

From 731454a40617909f2e58fb680ed8fa73ff9faf00 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Sat, 30 Aug 2014 15:40:50 +0100
Subject: [PATCH 14/40] introduce 'weave expose' which simplifies access to
 containers from the host, as well as external access.

---
 README.md    | 59 ++++++++++++++++++++++++++++++++++++----------------
 weaver/weave | 12 +++++++++--
 2 files changed, 51 insertions(+), 20 deletions(-)

diff --git a/README.md b/README.md
index 42c6f3cf..ce7967f0 100644
--- a/README.md
+++ b/README.md
@@ -174,6 +174,31 @@ The same password must be specified for all weave peers; it is the a
 component in the creation of ephemeral session keys for connections
 between peers.
 
+### Host network integration
+
+Weave application networks can be integrated with a host's network,
+establishing connectivity between the host and application containers
+anywhere.
+
+Let's say we want $HOST2 to have access to the containers from our
+example. On $HOST2 we run
+
+    host2# weave expose 10.0.1.102/24
+
+choosing an unused IP address in the application subnet.
+
+Now
+
+    host2# ping 10.0.1.2
+
+will work. And, more interestingly,
+
+    host2# ping 10.0.1.1
+    host2# echo 'Hello, world.' | nc -u 10.0.1.1 1122
+
+will work too, which is interacting with a container that resides on
+$HOST1.
+
 ### External access
 
 Services running in containers on a weave network can be made
@@ -182,33 +207,31 @@ from any weave host, regardless of where the service containers are
 located.
 
 Say we want to make our 'nc' "service" that is running in the
-container on $HOST1 accessible to the outside world via $HOST2. This
-requires the following steps on $HOST2...
+container on $HOST1 accessible to the outside world via $HOST2.
 
-1. Add an IP address to the weave bridge. This needs to be done once
-   for every weave application subnet.
+First we need to expose the application network to the host, as
+explained in [Host network integration], i.e.
 
-        host2# ip addr add dev weave 10.0.1.102/24
+    host2# weave expose 10.0.1.102/24
 
-   The address must be on the destination container's weave subnet and
-   not already in use.
+Then we add a NAT rule to route from the outside world to the
+destination container service.
 
-2. Add a NAT rule to route from the outside world to the destination
-   container service. This needs to be done once for every service we
-   want to expose.
+    host2# iptables -t nat -A PREROUTING -p udp -i eth0 --dport 2211 -j DNAT \
+           --to-destination 10.0.1.1:4422
 
-        host2# iptables -t nat -A PREROUTING -p udp -i eth0 --dport 2211 -j DNAT \
-               --to-destination 10.0.1.1:4422
-
-    Here we are assuming that the "outside world" is connecting to
-    $HOST2 via 'eth0'. We want UDP traffic to port 2211 on the
-    external IPs to be routed to our 'nc' service, which is running on
-    port 4422 in the container with IP 10.0.1.1.
+Here we are assuming that the "outside world" is connecting to $HOST2
+via 'eth0'. We want UDP traffic to port 2211 on the external IPs to be
+routed to our 'nc' service, which is running on port 4422 in the
+container with IP 10.0.1.1.
 
 With the above in place, we can connect to our 'nc' service from
 anywhere with
 
-    nc -u $HOST2 2211
+    echo 'Hello, world.' | nc -u $HOST2 4422
+
+(NB: due to the way routing is handled in the Linux kernel, this won't
+work when run *on* $HOST2.)
 
 Similar NAT rules to the above can used to expose services provided by
 containers on the weave network not just to outside world but also
diff --git a/weaver/weave b/weaver/weave
index 67b0efc1..b03f195c 100755
--- a/weaver/weave
+++ b/weaver/weave
@@ -5,6 +5,7 @@ usage() {
     echo "Usage:"
     echo "weave launch <ipaddr>/<subnet> [-passwd <passwd>] <peer_host> ..."
     echo "weave run    <ipaddr>/<subnet> <docker run args> ..."
+    echo "weave expose <ipaddr>/<subnet>"
     exit 1
 }
 
@@ -59,13 +60,22 @@ case "$COMMAND" in
         # restarts.
         MACADDR=`cat /sys/class/net/$BRIDGE/address`
         CONTAINER=$(docker run --privileged -d --name=weave -p 6783:6783/tcp -p 6783:6783/udp $WEAVE_DOCKER_ARGS zettio/weave -name $MACADDR "$@")
+        echo $CONTAINER
         IFACE=eth0
         ;;
     run)
         create_bridge
         CONTAINER=$(docker run -d "$@")
+        echo $CONTAINER
         IFACE=$CONTAINER_IFNAME
         ;;
+    expose)
+        if ! ip addr list dev $BRIDGE | grep -qF $IPADDR
+        then
+            ip addr add dev $BRIDGE $IPADDR
+        fi
+        exit 0
+        ;;
     *)
         echo "Unknown weave command '$COMMAND'"
         usage
@@ -108,5 +118,3 @@ else
 fi
 
 [ -f /var/run/netns/$NSPID ] && rm -f /var/run/netns/$NSPID
-
-echo $CONTAINER

From f0a506c6d9e29c3eda6355557a539ba8c659bb63 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Sat, 30 Aug 2014 15:42:52 +0100
Subject: [PATCH 15/40] fix link

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ce7967f0..46c545a8 100644
--- a/README.md
+++ b/README.md
@@ -210,7 +210,7 @@ Say we want to make our 'nc' "service" that is running in the
 container on $HOST1 accessible to the outside world via $HOST2.
 
 First we need to expose the application network to the host, as
-explained in [Host network integration], i.e.
+explained [above](Host network integration), i.e.
 
     host2# weave expose 10.0.1.102/24
 

From 35c7457e54ff2a8ea2643158d3ad37e8d11aa90c Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Sat, 30 Aug 2014 15:46:45 +0100
Subject: [PATCH 16/40] more link fixing

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 46c545a8..7f44564a 100644
--- a/README.md
+++ b/README.md
@@ -210,7 +210,7 @@ Say we want to make our 'nc' "service" that is running in the
 container on $HOST1 accessible to the outside world via $HOST2.
 
 First we need to expose the application network to the host, as
-explained [above](Host network integration), i.e.
+explained [above](#Host network integration), i.e.
 
     host2# weave expose 10.0.1.102/24
 

From 1274dc6e017cdf4856aa1ba0d103cfe089771659 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Sat, 30 Aug 2014 15:47:55 +0100
Subject: [PATCH 17/40] yet more link fixing

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7f44564a..b3c888d5 100644
--- a/README.md
+++ b/README.md
@@ -210,7 +210,7 @@ Say we want to make our 'nc' "service" that is running in the
 container on $HOST1 accessible to the outside world via $HOST2.
 
 First we need to expose the application network to the host, as
-explained [above](#Host network integration), i.e.
+explained [above](#host-network-integration), i.e.
 
     host2# weave expose 10.0.1.102/24
 

From 309516ce78264501eea59834bf87436ce536d24c Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Sat, 30 Aug 2014 16:15:53 +0100
Subject: [PATCH 18/40] correct ports in example

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index b3c888d5..5c1c3023 100644
--- a/README.md
+++ b/README.md
@@ -194,7 +194,7 @@ Now
 will work. And, more interestingly,
 
     host2# ping 10.0.1.1
-    host2# echo 'Hello, world.' | nc -u 10.0.1.1 1122
+    host2# echo 'Hello, world.' | nc -u 10.0.1.1 4422
 
 will work too, which is interacting with a container that resides on
 $HOST1.
@@ -228,7 +228,7 @@ container with IP 10.0.1.1.
 With the above in place, we can connect to our 'nc' service from
 anywhere with
 
-    echo 'Hello, world.' | nc -u $HOST2 4422
+    echo 'Hello, world.' | nc -u $HOST2 2211
 
 (NB: due to the way routing is handled in the Linux kernel, this won't
 work when run *on* $HOST2.)

From aef5111af29a8cef993bb9e9e5037c7897006d57 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Sat, 30 Aug 2014 16:30:35 +0100
Subject: [PATCH 19/40] switch tcp nc service for bulk of examples since udp nc
 is full of weirdness

---
 README.md | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index 5c1c3023..3aa7ea63 100644
--- a/README.md
+++ b/README.md
@@ -111,15 +111,21 @@ To application containers, the network established by weave looks
 like a giant Ethernet switch to which all the containers are
 connected.
 
-That means *any* protocol is supported. Doesn't even have to be over
-TCP/IP, e.g. in the above example,
+Containers can easily access services from each other, e.g. in the
+container on $HOST1 we can start a netcat "service" with
 
-    root@28841bd02eff:/# nc -lu -p 4422
+    root@28841bd02eff:/# nc -lk -p 4422
 
-    root@f76829496120:/# echo 'Hello, world.' | nc -u 10.0.1.1 4422
+and then connect to it from the container on $HOST2 with
 
-sends some data from the 2nd container to the first over a UDP
-connection on port 4422.
+    root@f76829496120:/# echo 'Hello, world.' | nc 10.0.1.1 4422
+
+Note that *any* protocol is supported. Doesn't even have to be over
+TCP/IP, e.g. a netcat UDP service would be run with
+
+    root@28841bd02eff:/# nc -lu -p 5533
+
+    root@f76829496120:/# echo 'Hello, world.' | nc -u 10.0.1.1 5533
 
 We can deploy the entire arsenal of standard network tools and
 applications, developed over decades, to configure, secure, monitor,
@@ -194,10 +200,8 @@ Now
 will work. And, more interestingly,
 
     host2# ping 10.0.1.1
-    host2# echo 'Hello, world.' | nc -u 10.0.1.1 4422
 
-will work too, which is interacting with a container that resides on
-$HOST1.
+will work too, which is talking to a container that resides on $HOST1.
 
 ### External access
 
@@ -206,8 +210,8 @@ accessible to the outside world (or, more generally, other networks)
 from any weave host, regardless of where the service containers are
 located.
 
-Say we want to make our 'nc' "service" that is running in the
-container on $HOST1 accessible to the outside world via $HOST2.
+Say we want to make our example netcat "service", which is running in
+a container on $HOST1, accessible to the outside world via $HOST2.
 
 First we need to expose the application network to the host, as
 explained [above](#host-network-integration), i.e.
@@ -217,11 +221,11 @@ explained [above](#host-network-integration), i.e.
 Then we add a NAT rule to route from the outside world to the
 destination container service.
 
-    host2# iptables -t nat -A PREROUTING -p udp -i eth0 --dport 2211 -j DNAT \
+    host2# iptables -t nat -A PREROUTING -p tcp -i eth0 --dport 2211 -j DNAT \
            --to-destination 10.0.1.1:4422
 
 Here we are assuming that the "outside world" is connecting to $HOST2
-via 'eth0'. We want UDP traffic to port 2211 on the external IPs to be
+via 'eth0'. We want TCP traffic to port 2211 on the external IPs to be
 routed to our 'nc' service, which is running on port 4422 in the
 container with IP 10.0.1.1.
 

From 7dd0bcd1e083e58cb95fbad375812fb5cd96fce6 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Sat, 30 Aug 2014 23:18:44 +0100
Subject: [PATCH 20/40] deal with pmtu verification confirmation in forwarder
 goroutine ...rather than connection goroutine. This is a precurser to smarter
 pmtu verification, the logic for which is in the forwarder. issue #15

---
 forwarder.go | 29 ++++++++++++++++++-----------
 types.go     |  2 ++
 2 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/forwarder.go b/forwarder.go
index c48bcaf1..aecaa787 100644
--- a/forwarder.go
+++ b/forwarder.go
@@ -34,17 +34,21 @@ func (conn *LocalConnection) ensureForwarders() error {
 		forwardChanDF = make(chan *ForwardedFrame, ChannelSize)
 		stopForward   = make(chan interface{}, 0)
 		stopForwardDF = make(chan interface{}, 0)
+		verifyPMTU    = make(chan int, ChannelSize)
 	)
-	forwarder := NewForwarder(conn, forwardChan, stopForward, encryptor, udpSender, DefaultPMTU)
-	forwarderDF := NewForwarder(conn, forwardChanDF, stopForwardDF, encryptorDF, udpSenderDF, DefaultPMTU)
+	//NB: only forwarderDF can ever encounter EMSGSIZE errors, and
+	//thus perform PMTU verification
+	forwarder := NewForwarder(conn, forwardChan, stopForward, nil, encryptor, udpSender, DefaultPMTU)
+	forwarderDF := NewForwarder(conn, forwardChanDF, stopForwardDF, verifyPMTU, encryptorDF, udpSenderDF, DefaultPMTU)
 
-	// The forward chans and effectivePMTU in the conn struct are read
-	// by other processes, so we have to use locks.
+	// Various fields in the conn struct are read by other processes,
+	// so we have to use locks.
 	conn.Lock()
 	conn.forwardChan = forwardChan
 	conn.forwardChanDF = forwardChanDF
 	conn.stopForward = stopForward
 	conn.stopForwardDF = stopForwardDF
+	conn.verifyPMTU = verifyPMTU
 	conn.effectivePMTU = forwarder.effectivePMTU
 	conn.Unlock()
 
@@ -179,13 +183,14 @@ func fragment(eth layers.Ethernet, ip layers.IPv4, pmtu int, frame *ForwardedFra
 
 // Forwarder
 
-func NewForwarder(conn *LocalConnection, ch <-chan *ForwardedFrame, stop <-chan interface{}, enc Encryptor, udpSender UDPSender, pmtu int) *Forwarder {
-	forwarder := &Forwarder{
-		conn:      conn,
-		ch:        ch,
-		stop:      stop,
-		enc:       enc,
-		udpSender: udpSender}
+func NewForwarder(conn *LocalConnection, ch <-chan *ForwardedFrame, stop <-chan interface{}, pmtuVerified <-chan int, enc Encryptor, udpSender UDPSender, pmtu int) *Forwarder {
+ 	forwarder := &Forwarder{
+		conn:         conn,
+		ch:           ch,
+		stop:         stop,
+		pmtuVerified: pmtuVerified,
+		enc:          enc,
+		udpSender:    udpSender}
 	forwarder.setMaxPayloadAndEffectivePMTU(pmtu)
 	return forwarder
 }
@@ -216,6 +221,8 @@ func (fwd *Forwarder) Run() {
 				}
 				fwd.verifyEffectivePMTU()
 			}
+		case pmtu := <-fwd.pmtuVerified:
+			fwd.conn.effectivePMTUVerification(pmtu)
 		case frame = <-fwd.ch:
 			if !fwd.appendFrame(frame) {
 				fwd.logDrop(frame)
diff --git a/types.go b/types.go
index 861fa7f5..7826fb5f 100644
--- a/types.go
+++ b/types.go
@@ -73,6 +73,7 @@ type LocalConnection struct {
 	forwardChanDF chan<- *ForwardedFrame
 	stopForward   chan<- interface{}
 	stopForwardDF chan<- interface{}
+	verifyPMTU    chan<- int
 	Decryptor     Decryptor
 	Router        *Router
 	UID           uint64
@@ -90,6 +91,7 @@ type Forwarder struct {
 	ch              <-chan *ForwardedFrame
 	stop            <-chan interface{}
 	pmtuVerifyTick  <-chan time.Time
+	pmtuVerified    <-chan int
 	pmtuVerifyCount uint
 	enc             Encryptor
 	udpSender       UDPSender

From 848d852e0da53e3c2bad506b1733f745c33cf79c Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Sun, 31 Aug 2014 13:07:38 +0100
Subject: [PATCH 21/40] oops

---
 connection.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/connection.go b/connection.go
index caa85061..f569c2da 100644
--- a/connection.go
+++ b/connection.go
@@ -506,7 +506,7 @@ func (conn *LocalConnection) receiveTCP(decoder *gob.Decoder, usingPassword bool
 			}
 			conn.local.BroadcastTCP(Concat(ProtocolUpdateByte, newUpdate))
 		} else if msg[0] == ProtocolPMTUVerified {
-			conn.effectivePMTUVerification(int(binary.BigEndian.Uint16(msg[1:])))
+			conn.verifyPMTU <- int(binary.BigEndian.Uint16(msg[1:]))
 		} else {
 			conn.log("received unknown msg:\n", msg)
 		}

From 2649796369c0c7527ffbfae300550ff0b66f2579 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Sun, 31 Aug 2014 13:39:44 +0100
Subject: [PATCH 22/40] discover pmtu with binary search issue #15

---
 connection.go | 20 ++---------
 forwarder.go  | 98 ++++++++++++++++++++++++++++++++-------------------
 types.go      | 10 +++---
 3 files changed, 70 insertions(+), 58 deletions(-)

diff --git a/connection.go b/connection.go
index f569c2da..0eb1bd63 100644
--- a/connection.go
+++ b/connection.go
@@ -99,26 +99,12 @@ func (conn *LocalConnection) CheckFatal(err error) error {
 func (conn *LocalConnection) setEffectivePMTU(pmtu int) {
 	conn.Lock()
 	defer conn.Unlock()
-	conn.effectivePMTU = pmtu
-	conn.pmtuVerified = false
-	conn.log("Effective PMTU set to", pmtu)
-}
-
-func (conn *LocalConnection) effectivePMTUVerification(pmtu int) {
-	conn.Lock()
-	defer conn.Unlock()
-	if pmtu == conn.effectivePMTU && !conn.pmtuVerified {
-		conn.pmtuVerified = true
-		conn.log("Effective PMTU verified at", pmtu)
+	if (conn.effectivePMTU != pmtu) {
+		conn.effectivePMTU = pmtu
+		conn.log("Effective PMTU set to", pmtu)
 	}
 }
 
-func (conn *LocalConnection) isEffectivePMTUVerfied() bool {
-	conn.RLock()
-	defer conn.RUnlock()
-	return conn.pmtuVerified
-}
-
 // Called by the connection's actor process, by the connection's
 // heartbeat process and by the connection's TCP received
 // process. StackFrag is read in conn.Forward (called by router udp
diff --git a/forwarder.go b/forwarder.go
index aecaa787..74c168b8 100644
--- a/forwarder.go
+++ b/forwarder.go
@@ -49,7 +49,7 @@ func (conn *LocalConnection) ensureForwarders() error {
 	conn.stopForward = stopForward
 	conn.stopForwardDF = stopForwardDF
 	conn.verifyPMTU = verifyPMTU
-	conn.effectivePMTU = forwarder.effectivePMTU
+	conn.effectivePMTU = forwarder.unverifiedPMTU
 	conn.Unlock()
 
 	go forwarder.Run()
@@ -183,16 +183,17 @@ func fragment(eth layers.Ethernet, ip layers.IPv4, pmtu int, frame *ForwardedFra
 
 // Forwarder
 
-func NewForwarder(conn *LocalConnection, ch <-chan *ForwardedFrame, stop <-chan interface{}, pmtuVerified <-chan int, enc Encryptor, udpSender UDPSender, pmtu int) *Forwarder {
- 	forwarder := &Forwarder{
-		conn:         conn,
-		ch:           ch,
-		stop:         stop,
-		pmtuVerified: pmtuVerified,
-		enc:          enc,
-		udpSender:    udpSender}
-	forwarder.setMaxPayloadAndEffectivePMTU(pmtu)
-	return forwarder
+func NewForwarder(conn *LocalConnection, ch <-chan *ForwardedFrame, stop <-chan interface{}, verifyPMTU <-chan int, enc Encryptor, udpSender UDPSender, pmtu int) *Forwarder {
+	fwd := &Forwarder{
+		conn:       conn,
+		ch:         ch,
+		stop:       stop,
+		verifyPMTU: verifyPMTU,
+		enc:        enc,
+		udpSender:  udpSender}
+	fwd.unverifiedPMTU = pmtu - fwd.effectiveOverhead()
+	fwd.maxPayload = pmtu - UDPOverhead
+	return fwd
 }
 
 func (fwd *Forwarder) Run() {
@@ -205,24 +206,36 @@ func (fwd *Forwarder) Run() {
 		case <-fwd.stop:
 			fwd.drain()
 			return
-		case <-fwd.pmtuVerifyTick:
+		case <-fwd.verifyPMTUTick:
 			// We only do this case here when we know the buffers are
 			// all empty so that we don't risk appending verify-frames
 			// to other data.
-			fwd.pmtuVerifyTick = nil
-			if !fwd.conn.isEffectivePMTUVerfied() {
-				if fwd.pmtuVerifyCount == 0 {
-					fwd.pmtuVerifyCount = PMTUVerifyAttempts
-					fwd.effectivePMTU -= 8
-					fwd.maxPayload -= 8
-					fwd.conn.setEffectivePMTU(fwd.effectivePMTU)
-				} else {
-					fwd.pmtuVerifyCount--
-				}
-				fwd.verifyEffectivePMTU()
+			fwd.verifyPMTUTick = nil
+			if fwd.pmtuVerified {
+				continue
+			}
+			if fwd.pmtuVerifyCount > 0 {
+				fwd.pmtuVerifyCount--
+				fwd.attemptVerifyEffectivePMTU()
+			} else {
+				// we've exceeded the verification attempts of the
+				// unverifiedPMTU
+				fwd.lowestBadPMTU = fwd.unverifiedPMTU
+				fwd.verifyEffectivePMTU((fwd.highestGoodPMTU + fwd.lowestBadPMTU) / 2)
+			}
+		case epmtu := <-fwd.verifyPMTU:
+			if fwd.pmtuVerified || epmtu != fwd.unverifiedPMTU {
+				continue
+			}
+			if epmtu+1 < fwd.lowestBadPMTU {
+				fwd.highestGoodPMTU = fwd.unverifiedPMTU // = epmtu
+				fwd.verifyEffectivePMTU((fwd.highestGoodPMTU + fwd.lowestBadPMTU) / 2)
+			} else {
+				fwd.pmtuVerified = true
+				fwd.maxPayload = epmtu + fwd.effectiveOverhead() - UDPOverhead
+				fwd.conn.setEffectivePMTU(epmtu)
+				fwd.conn.log("Effective PMTU verified at", epmtu)
 			}
-		case pmtu := <-fwd.pmtuVerified:
-			fwd.conn.effectivePMTUVerification(pmtu)
 		case frame = <-fwd.ch:
 			if !fwd.appendFrame(frame) {
 				fwd.logDrop(frame)
@@ -250,20 +263,25 @@ func (fwd *Forwarder) Run() {
 	}
 }
 
-func (fwd *Forwarder) setMaxPayloadAndEffectivePMTU(pmtu int) {
-	fwd.maxPayload = pmtu - UDPOverhead
-	fwd.effectivePMTU = fwd.maxPayload - fwd.enc.PacketOverhead() - fwd.enc.FrameOverhead() - EthernetOverhead
+func (fwd *Forwarder) effectiveOverhead() int {
+	return UDPOverhead + fwd.enc.PacketOverhead() + fwd.enc.FrameOverhead() + EthernetOverhead
 }
 
-func (fwd *Forwarder) verifyEffectivePMTU() {
+func (fwd *Forwarder) verifyEffectivePMTU(newUnverifiedPMTU int) {
+	fwd.unverifiedPMTU = newUnverifiedPMTU
+	fwd.pmtuVerifyCount = PMTUVerifyAttempts
+	fwd.attemptVerifyEffectivePMTU()
+}
+
+func (fwd *Forwarder) attemptVerifyEffectivePMTU() {
 	pmtuVerifyFrame := &ForwardedFrame{
 		srcPeer: fwd.conn.local,
 		dstPeer: fwd.conn.remote,
-		frame:   make([]byte, fwd.effectivePMTU+EthernetOverhead)}
+		frame:   make([]byte, fwd.unverifiedPMTU+EthernetOverhead)}
 	fwd.enc.AppendFrame(pmtuVerifyFrame)
 	fwd.flush()
-	if fwd.pmtuVerifyTick == nil {
-		fwd.pmtuVerifyTick = time.After(PMTUVerifyTimeout << (PMTUVerifyAttempts - fwd.pmtuVerifyCount))
+	if fwd.verifyPMTUTick == nil {
+		fwd.verifyPMTUTick = time.After(PMTUVerifyTimeout << (PMTUVerifyAttempts - fwd.pmtuVerifyCount))
 	}
 }
 
@@ -280,10 +298,16 @@ func (fwd *Forwarder) flush() {
 	err := fwd.udpSender.Send(fwd.enc.Bytes())
 	if err != nil {
 		if mtbe, ok := err.(MsgTooBigError); ok {
-			fwd.setMaxPayloadAndEffectivePMTU(mtbe.PMTU)
-			fwd.conn.setEffectivePMTU(fwd.effectivePMTU)
-			fwd.pmtuVerifyCount = PMTUVerifyAttempts
-			fwd.verifyEffectivePMTU()
+			newUnverifiedPMTU := mtbe.PMTU - fwd.effectiveOverhead()
+			if newUnverifiedPMTU >= fwd.unverifiedPMTU {
+				return
+			}
+			fwd.pmtuVerified = false
+			fwd.maxPayload = mtbe.PMTU - UDPOverhead
+			fwd.highestGoodPMTU = 8
+			fwd.lowestBadPMTU = newUnverifiedPMTU + 1
+			fwd.conn.setEffectivePMTU(newUnverifiedPMTU)
+			fwd.verifyEffectivePMTU(newUnverifiedPMTU)
 		} else if PosixError(err) == syscall.ENOBUFS {
 			// TODO handle this better
 		} else {
@@ -306,7 +330,7 @@ func (fwd *Forwarder) drain() {
 }
 
 func (fwd *Forwarder) logDrop(frame *ForwardedFrame) {
-	fwd.conn.log("dropping too big frame during forwarding: frame len:", len(frame.frame), "; effective PMTU:", fwd.effectivePMTU)
+	fwd.conn.log("Dropping too big frame during forwarding: frame len:", len(frame.frame), "; effective PMTU:", fwd.maxPayload+UDPOverhead-fwd.effectiveOverhead())
 }
 
 // UDP Senders
diff --git a/types.go b/types.go
index 7826fb5f..020fba18 100644
--- a/types.go
+++ b/types.go
@@ -66,7 +66,6 @@ type LocalConnection struct {
 	established   bool
 	stackFrag     bool
 	effectivePMTU int
-	pmtuVerified  bool
 	SessionKey    *[32]byte
 	heartbeatStop chan<- interface{}
 	forwardChan   chan<- *ForwardedFrame
@@ -90,13 +89,16 @@ type Forwarder struct {
 	conn            *LocalConnection
 	ch              <-chan *ForwardedFrame
 	stop            <-chan interface{}
-	pmtuVerifyTick  <-chan time.Time
-	pmtuVerified    <-chan int
+	verifyPMTUTick  <-chan time.Time
+	verifyPMTU      <-chan int
 	pmtuVerifyCount uint
 	enc             Encryptor
 	udpSender       UDPSender
 	maxPayload      int
-	effectivePMTU   int
+	pmtuVerified    bool
+	highestGoodPMTU int
+	unverifiedPMTU  int
+	lowestBadPMTU   int
 }
 
 type UDPPacket struct {

From 971f6a63fd95dc48bc726cace19b242a7f5dcd42 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Sun, 31 Aug 2014 14:43:06 +0100
Subject: [PATCH 23/40] Turns out one OS thread isn't enough

---
 weaver/main.go | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/weaver/main.go b/weaver/main.go
index 174f9f93..7edc2268 100644
--- a/weaver/main.go
+++ b/weaver/main.go
@@ -46,7 +46,13 @@ func main() {
 	log.SetPrefix(weave.Protocol + " ")
 	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)
 
-	runtime.GOMAXPROCS(runtime.NumCPU())
+	procs := runtime.NumCPU()
+	// packet sniffing can block an OS thread, so we need one thread
+	// for that plus at least one more.
+	if procs < 2 {
+		procs = 2
+	}
+	runtime.GOMAXPROCS(procs)
 
 	var (
 		ifaceName  string

From 1676f104052896e29f126f1f995f93f9298830fb Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Sun, 31 Aug 2014 15:22:03 +0100
Subject: [PATCH 24/40] fix some typos

---
 README.md | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 3aa7ea63..16f05022 100644
--- a/README.md
+++ b/README.md
@@ -72,9 +72,8 @@ The only difference, apart from the IP addresses, is that we tell our
 weave that it should peer with the weave running on $HOST1. We could
 instead have told the weave on $HOST1 to connect to $HOST2, or told
 both about each other. Order doesn't matter here; weave automatically
-(re)connects to peers when they are become available. Also, we can
-tell weave to connect to multiple peers by supplying multiple
-addresses.
+(re)connects to peers when they become available. Also, we can tell
+weave to connect to multiple peers by supplying multiple addresses.
 
 Now that we've got everything set up, let's see whether our containers
 can talk to each other...
@@ -176,7 +175,7 @@ launching weave, e.g.
 
     host1# WEAVE=$(weave launch 10.0.0.1/16 -password wEaVe)
 
-The same password must be specified for all weave peers; it is the a
+The same password must be specified for all weave peers; it is a
 component in the creation of ephemeral session keys for connections
 between peers.
 
@@ -186,8 +185,8 @@ Weave application networks can be integrated with a host's network,
 establishing connectivity between the host and application containers
 anywhere.
 
-Let's say we want $HOST2 to have access to the containers from our
-example. On $HOST2 we run
+Let's say that in our example we we want $HOST2 to have access to the
+application containers. On $HOST2 we run
 
     host2# weave expose 10.0.1.102/24
 
@@ -206,7 +205,7 @@ will work too, which is talking to a container that resides on $HOST1.
 ### External access
 
 Services running in containers on a weave network can be made
-accessible to the outside world (or, more generally, other networks)
+accessible to the outside world (and, more generally, other networks)
 from any weave host, regardless of where the service containers are
 located.
 
@@ -237,9 +236,8 @@ anywhere with
 (NB: due to the way routing is handled in the Linux kernel, this won't
 work when run *on* $HOST2.)
 
-Similar NAT rules to the above can used to expose services provided by
-containers on the weave network not just to outside world but also
-other, internal, networks.
+Similar NAT rules to the above can used to expose services not just to
+outside world but also other, internal, networks.
 
 ### Multi-cloud networking
 

From eea1fbe536ff706cb0550cc1e9cba08980e09fce Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Sun, 31 Aug 2014 22:37:01 +0100
Subject: [PATCH 25/40] simplify instructions - no need to capture weave
 container id since it's a named container

---
 README.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 16f05022..55e00dff 100644
--- a/README.md
+++ b/README.md
@@ -41,7 +41,7 @@ two containers, one on each host.
 
 On $HOST1 run (as root)
 
-    host1# WEAVE=$(weave launch 10.0.0.1/16)
+    host1# weave launch 10.0.0.1/16
     host1# C=$(weave run 10.0.1.1/24 -t -i ubuntu /bin/bash)
 
 The first line starts the weave router, in a container. This needs to
@@ -65,7 +65,7 @@ unique.
 
 We repeat similar steps on $HOST2...
 
-    host2# WEAVE=$(weave launch 10.0.0.2/16 $HOST1)
+    host2# weave launch 10.0.0.2/16 $HOST1
     host2# C=$(weave run 10.0.1.2/24 -t -i ubuntu /bin/bash)
 
 The only difference, apart from the IP addresses, is that we tell our
@@ -173,7 +173,7 @@ In order to connect containers across untrusted networks, weave peers
 can be told to encrypt traffic by supplying a `-password` option when
 launching weave, e.g.
 
-    host1# WEAVE=$(weave launch 10.0.0.1/16 -password wEaVe)
+    host1# weave launch 10.0.0.1/16 -password wEaVe
 
 The same password must be specified for all weave peers; it is a
 component in the creation of ephemeral session keys for connections
@@ -296,7 +296,7 @@ enough.
 
 Check the weave container logs with
 
-    docker logs $WEAVE
+    docker logs weave
 
 A reasonable amount of information, and all errors, get logged there.
 
@@ -307,7 +307,7 @@ basis, so can produce a lot of output.
 One can ask a weave router to log a status report by sending it a USR1
 signal, e.g.
 
-    kill -USR1 `docker inspect --format='{{ .State.Pid }}' $WEAVE`
+    kill -USR1 `docker inspect --format='{{ .State.Pid }}' weave`
 
 and then inspect the logs.
 
@@ -327,7 +327,7 @@ make it executable like this:
 Then, if we were trying to create the same containers as in the first
 example above, the 'launch' command would be run like this:
 
-    host1$ WEAVE=$(boot2docker ssh "sudo ./weave launch 10.0.0.1/16")
+    host1$ boot2docker ssh "sudo ./weave launch 10.0.0.1/16"
 
 and the 'run' command like this:
 

From 37a9c13761a5109a1b32a43fdbabcc10cd4d65ad Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Mon, 1 Sep 2014 00:15:43 +0100
Subject: [PATCH 26/40] make a note of issue #18 in readme

---
 README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 55e00dff..7aacd3b4 100644
--- a/README.md
+++ b/README.md
@@ -287,10 +287,10 @@ the partition heals.
 The weave container is very light-weight - just over 8MB image size
 and a few 10s of MBs of runtime memory - and disposable. I.e. should
 weave ever run into difficulty, one can simply bounce the weave
-container. Application containers do *not* have to be restarted in
-that event, and indeed may not even experience a temporary
-connectivity failure if the weave container is restarted quickly
-enough.
+container (though note issue #18). Application containers do *not*
+have to be restarted in that event, and indeed may not even experience
+a temporary connectivity failure if the weave container is restarted
+quickly enough.
 
 ## Troubleshooting
 

From aed818fb228c50e5bda8ff515ec888d2f9704eca Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Mon, 1 Sep 2014 00:18:52 +0100
Subject: [PATCH 27/40] fix link

---
 README.md | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 7aacd3b4..a6fe370c 100644
--- a/README.md
+++ b/README.md
@@ -287,10 +287,11 @@ the partition heals.
 The weave container is very light-weight - just over 8MB image size
 and a few 10s of MBs of runtime memory - and disposable. I.e. should
 weave ever run into difficulty, one can simply bounce the weave
-container (though note issue #18). Application containers do *not*
-have to be restarted in that event, and indeed may not even experience
-a temporary connectivity failure if the weave container is restarted
-quickly enough.
+container (though note
+[issue #18](https://github.com/zettio/weave/issues/18)). Application
+containers do *not* have to be restarted in that event, and indeed may
+not even experience a temporary connectivity failure if the weave
+container is restarted quickly enough.
 
 ## Troubleshooting
 

From 0a7df2220cf047a7d7ef29c70d28c0212c0e2ae7 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Mon, 1 Sep 2014 16:08:47 +0100
Subject: [PATCH 28/40] introduce 'weave stop' which, as well as
 stopping&removing the weave container also clears the UDP connection tracking
 for the weave port, without which UDP connections to a quickly (<3min)
 re-started weave would fail.

Fixes #18
---
 README.md    | 22 ++++++++++++++--------
 weaver/weave | 28 ++++++++++++++++++++++++----
 2 files changed, 38 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index a6fe370c..4336d1f4 100644
--- a/README.md
+++ b/README.md
@@ -30,8 +30,15 @@ To run weave on a host, you need to install...
           https://raw.githubusercontent.com/zettio/weave/master/weaver/weave
         sudo chmod a+x /usr/local/bin/weave
 
-3. ethtool. On many systems that is installed already; if not then
-grab it via your favourite package manager.
+3. (recommended) ethtool. On many systems this is installed already;
+   if not then grab it via your favourite package manager. On some
+   systems, weave application container networking may not operate
+   correctly unless ethtool is available.
+
+4. (optional) conntrack. Install this via your favourite package
+   manager. Without conntrack, the weave network may not re-establish
+   itself fully when individual weave instances are stopped (with
+   `weave stop`) and restarted quickly (typically within ~3 minutes).
 
 ## Example
 
@@ -286,12 +293,11 @@ the partition heals.
 
 The weave container is very light-weight - just over 8MB image size
 and a few 10s of MBs of runtime memory - and disposable. I.e. should
-weave ever run into difficulty, one can simply bounce the weave
-container (though note
-[issue #18](https://github.com/zettio/weave/issues/18)). Application
-containers do *not* have to be restarted in that event, and indeed may
-not even experience a temporary connectivity failure if the weave
-container is restarted quickly enough.
+weave ever run into difficulty, one can simply stop it (with `weave
+stop`) and restart it. Application containers do *not* have to be
+restarted in that event, and indeed may not even experience a
+temporary connectivity failure if the weave container is restarted
+quickly enough.
 
 ## Troubleshooting
 
diff --git a/weaver/weave b/weaver/weave
index b03f195c..5d53930f 100755
--- a/weaver/weave
+++ b/weaver/weave
@@ -6,6 +6,7 @@ usage() {
     echo "weave launch <ipaddr>/<subnet> [-passwd <passwd>] <peer_host> ..."
     echo "weave run    <ipaddr>/<subnet> <docker run args> ..."
     echo "weave expose <ipaddr>/<subnet>"
+    echo "weave stop"
     exit 1
 }
 
@@ -14,16 +15,15 @@ usage() {
     exit 1
 }
 
-[ $# -gt 1 ] || usage
+[ $# -gt 0 ] || usage
 
 BRIDGE=weave
 CONTAINER_IFNAME=ethwe
 MTU=65535
 
 COMMAND=$1
-IPADDR=$2
 
-shift 2
+shift 1
 
 create_bridge() {
     [ ! -d /sys/class/net/$BRIDGE ] && {
@@ -50,6 +50,9 @@ setup_masquerading() {
 
 case "$COMMAND" in
     launch)
+        [ $# -gt 0 ] || usage
+        IPADDR=$1
+        shift 1
         create_bridge
         setup_masquerading
         # We set the router name to the bridge mac since that is
@@ -64,18 +67,35 @@ case "$COMMAND" in
         IFACE=eth0
         ;;
     run)
+        [ $# -gt 0 ] || usage
+        IPADDR=$1
+        shift 1
         create_bridge
         CONTAINER=$(docker run -d "$@")
         echo $CONTAINER
         IFACE=$CONTAINER_IFNAME
         ;;
     expose)
+        [ $# -gt 0 ] || usage
+        IPADDR=$1
+        shift 1
         if ! ip addr list dev $BRIDGE | grep -qF $IPADDR
         then
             ip addr add dev $BRIDGE $IPADDR
         fi
         exit 0
         ;;
+    stop)
+        docker kill weave > /dev/null
+        docker rm weave > /dev/null
+        if command -v conntrack >/dev/null 2>&1
+        then
+            conntrack -D -p udp --dport 6783 >/dev/null 2>&1
+        else
+            echo "Cannot find conntrack; please install it. Continuing without it." >&2
+        fi
+        exit 0
+        ;;
     *)
         echo "Unknown weave command '$COMMAND'"
         usage
@@ -114,7 +134,7 @@ if command -v ethtool >/dev/null 2>&1
 then
     ip netns exec $NSPID ethtool -K $IFACE tx off > /dev/null
 else
-    echo "Cannot find ethtool; continuing without it" >&2
+    echo "Cannot find ethtool; please install it. Continuing without it." >&2
 fi
 
 [ -f /var/run/netns/$NSPID ] && rm -f /var/run/netns/$NSPID

From f52fc71ef80bf9bfb394e30bfb83e177658b9e18 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Mon, 1 Sep 2014 17:21:06 +0100
Subject: [PATCH 29/40] add forward reference to interesting feature

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 4336d1f4..6507b207 100644
--- a/README.md
+++ b/README.md
@@ -80,7 +80,8 @@ weave that it should peer with the weave running on $HOST1. We could
 instead have told the weave on $HOST1 to connect to $HOST2, or told
 both about each other. Order doesn't matter here; weave automatically
 (re)connects to peers when they become available. Also, we can tell
-weave to connect to multiple peers by supplying multiple addresses.
+weave to connect to multiple peers by supplying multiple
+addresses. And we can [add peers dynamically](#dynamic-topologies).
 
 Now that we've got everything set up, let's see whether our containers
 can talk to each other...

From 1b40c842eb196cbda7a2149a0e883e04c80349d5 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Mon, 1 Sep 2014 22:34:15 +0100
Subject: [PATCH 30/40] add contact details

---
 README.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/README.md b/README.md
index 6507b207..726fe783 100644
--- a/README.md
+++ b/README.md
@@ -358,3 +358,9 @@ This will build the weave router, produce a docker image
 
 If you are interested in the inner workings of weave, read the
 [architecture documentation](docs/architecture.txt).
+
+## Contact Us
+
+Found a bug, want to suggest a feature, or have a question?
+[File an issue](https://github.com/zettio/weave/issues), or email us
+at weave@zett.io.

From 7cc2581a94a39a116c13cd4df0b16a8469a41cda Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Wed, 3 Sep 2014 09:04:46 +0100
Subject: [PATCH 31/40] fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 726fe783..661f10a0 100644
--- a/README.md
+++ b/README.md
@@ -239,7 +239,7 @@ container with IP 10.0.1.1.
 With the above in place, we can connect to our 'nc' service from
 anywhere with
 
-    echo 'Hello, world.' | nc -u $HOST2 2211
+    echo 'Hello, world.' | nc $HOST2 2211
 
 (NB: due to the way routing is handled in the Linux kernel, this won't
 work when run *on* $HOST2.)

From d0dc7b755640bb12c74e963214bfae8fb4b62261 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Wed, 3 Sep 2014 09:05:59 +0100
Subject: [PATCH 32/40] 'service import' feature

---
 README.md    | 46 +++++++++++++++++++++++++++++++++++++++++-----
 weaver/weave | 12 ++++++++----
 2 files changed, 49 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index 661f10a0..8bb25988 100644
--- a/README.md
+++ b/README.md
@@ -210,17 +210,17 @@ will work. And, more interestingly,
 
 will work too, which is talking to a container that resides on $HOST1.
 
-### External access
+### Service export
 
 Services running in containers on a weave network can be made
 accessible to the outside world (and, more generally, other networks)
-from any weave host, regardless of where the service containers are
+from any weave host, irrespective of where the service containers are
 located.
 
 Say we want to make our example netcat "service", which is running in
 a container on $HOST1, accessible to the outside world via $HOST2.
 
-First we need to expose the application network to the host, as
+First we need to expose the application network to $HOST2, as
 explained [above](#host-network-integration), i.e.
 
     host2# weave expose 10.0.1.102/24
@@ -228,8 +228,8 @@ explained [above](#host-network-integration), i.e.
 Then we add a NAT rule to route from the outside world to the
 destination container service.
 
-    host2# iptables -t nat -A PREROUTING -p tcp -i eth0 --dport 2211 -j DNAT \
-           --to-destination 10.0.1.1:4422
+    host2# iptables -t nat -A PREROUTING -p tcp -i eth0 --dport 2211 \
+           -j DNAT --to-destination 10.0.1.1:4422
 
 Here we are assuming that the "outside world" is connecting to $HOST2
 via 'eth0'. We want TCP traffic to port 2211 on the external IPs to be
@@ -247,6 +247,42 @@ work when run *on* $HOST2.)
 Similar NAT rules to the above can used to expose services not just to
 outside world but also other, internal, networks.
 
+### Service import
+
+Applications running in containers on a weave network can be given
+access to services which are only reachable from certain weave hosts,
+irrespective of where the application containers are located.
+
+Say that, as an extension of our example, we have a netcat service
+running on $HOST3, port 2211, and that $HOST3 is not part of the weave
+network and is only reachable from $HOST1, not $HOST2. Nevertheless we
+want to make the service accessible to an application running in a
+container on $HOST2.
+
+First we need to expose the application network to the host, as
+explained [above](#host-network-integration), this time on $HOST1,
+i.e.
+
+    host1# weave expose 10.0.1.101/24
+
+Then we add a NAT rule to route from the above IP to the destination
+service.
+
+    host1# iptables -t nat -A PREROUTING -p tcp -d 10.0.1.101 --dport 3322 \
+           -j DNAT --to-destination $HOST3:2211
+
+This allows any application container to reach the service by
+connecting to 10.0.0.1:3322. So if $HOST3 is indeed running a netcat
+service on port 2211, e.g.
+
+    host3# nc -kl 2211
+
+then we can connect to it from our application container on $HOST2 with
+
+    root@f76829496120:/# echo 'Hello, world.' | nc 10.0.1.101 3322
+
+The same command will work from any application container.
+
 ### Multi-cloud networking
 
 Weave can network containers hosted in different cloud providers /
diff --git a/weaver/weave b/weaver/weave
index 5d53930f..95e1b89f 100755
--- a/weaver/weave
+++ b/weaver/weave
@@ -40,14 +40,18 @@ create_bridge() {
     ip link set dev $BRIDGE up
 }
 
-setup_masquerading() {
-    IPTABLES_ARGS="POSTROUTING -d $IPADDR ! -s $IPADDR -j MASQUERADE"
-    if ! iptables -w -t nat -C $IPTABLES_ARGS >/dev/null 2>&1
+add_nat_rule() {
+    if ! iptables -w -t nat -C $1 >/dev/null 2>&1
     then
-        iptables -w -t nat -A $IPTABLES_ARGS >/dev/null
+        iptables -w -t nat -A $1 >/dev/null
     fi
 }
 
+setup_masquerading() {
+    add_nat_rule "POSTROUTING -d $IPADDR ! -s $IPADDR -j MASQUERADE"
+    add_nat_rule "POSTROUTING -s $IPADDR ! -d $IPADDR -j MASQUERADE"
+}
+
 case "$COMMAND" in
     launch)
         [ $# -gt 0 ] || usage

From 076796493480309060f62b75d311c92d2aed3d4f Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Wed, 3 Sep 2014 11:17:24 +0100
Subject: [PATCH 33/40] introduce 'weave hide'

---
 README.md    |  3 ++-
 weaver/weave | 11 +++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 8bb25988..abd2a532 100644
--- a/README.md
+++ b/README.md
@@ -198,7 +198,8 @@ application containers. On $HOST2 we run
 
     host2# weave expose 10.0.1.102/24
 
-choosing an unused IP address in the application subnet.
+choosing an unused IP address in the application subnet. (There is a
+corresponding 'hide' command to revert this step.)
 
 Now
 
diff --git a/weaver/weave b/weaver/weave
index 95e1b89f..7979e0de 100755
--- a/weaver/weave
+++ b/weaver/weave
@@ -6,6 +6,7 @@ usage() {
     echo "weave launch <ipaddr>/<subnet> [-passwd <passwd>] <peer_host> ..."
     echo "weave run    <ipaddr>/<subnet> <docker run args> ..."
     echo "weave expose <ipaddr>/<subnet>"
+    echo "weave hide   <ipaddr>/<subnet>"
     echo "weave stop"
     exit 1
 }
@@ -89,6 +90,16 @@ case "$COMMAND" in
         fi
         exit 0
         ;;
+    hide)
+        [ $# -gt 0 ] || usage
+        IPADDR=$1
+        shift 1
+        if ip addr list dev $BRIDGE | grep -qF $IPADDR
+        then
+            ip addr del dev $BRIDGE $IPADDR
+        fi
+        exit 0
+        ;;
     stop)
         docker kill weave > /dev/null
         docker rm weave > /dev/null

From 3fd3ff06b321fb2f6d4b70a4b8637314422bfe51 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Wed, 3 Sep 2014 15:22:50 +0100
Subject: [PATCH 34/40] update arch docs around pmtu discovery

---
 docs/architecture.txt | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/docs/architecture.txt b/docs/architecture.txt
index 5cfb4a73..79917b5f 100644
--- a/docs/architecture.txt
+++ b/docs/architecture.txt
@@ -203,7 +203,6 @@ dropped by firewalls. Whenever weave sends between peers with DF set
 and gets an error informing it of a new PMTU, it will attempt to
 verify that PMTU by sending packets of exactly that size to the remote
 peer. If it gets no indication from the remote weave that these
-packets have been received within a timeout period, it will
-artificially drop the PMTU by 8 and try again. Thus this will slowly
-reduce the PMTU until a working PMTU is found. This is slow, but it
-does work and works around ICMP packets being dropped.
+packets have been received within a timeout period, it will conduct a
+binary search - sending packets of different sizes - to determine the
+PMTU.

From a7e44b44dc5b1747186174bc1fc31aa194d8800d Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Wed, 3 Sep 2014 19:09:52 +0100
Subject: [PATCH 35/40] match on bridge name in iptable masquerade rules
 Instead of ip/netmask. This should be functionally equivalent (for now), and
 makes it easier to identify the rule as belonging to weave. Unfortunatley we
 must still match the source with ip/netmask, since we are in the POSTROUTING
 chain and source interface matches aren't valid there.

See also issue #20.
---
 weaver/weave | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/weaver/weave b/weaver/weave
index 7979e0de..202b650b 100755
--- a/weaver/weave
+++ b/weaver/weave
@@ -49,8 +49,8 @@ add_nat_rule() {
 }
 
 setup_masquerading() {
-    add_nat_rule "POSTROUTING -d $IPADDR ! -s $IPADDR -j MASQUERADE"
-    add_nat_rule "POSTROUTING -s $IPADDR ! -d $IPADDR -j MASQUERADE"
+    add_nat_rule "POSTROUTING -o $BRIDGE ! -s $IPADDR -j MASQUERADE"
+    add_nat_rule "POSTROUTING -s $IPADDR ! -o $BRIDGE -j MASQUERADE"
 }
 
 case "$COMMAND" in

From 9706934d897105cf504d1b831c834f38f42efa17 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Wed, 3 Sep 2014 23:43:40 +0100
Subject: [PATCH 36/40] log mac expiry for symmetry with mac discovery and w/o
 polluting the mac_cache code with logging

---
 mac_cache.go | 9 ++++++---
 router.go    | 5 ++++-
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/mac_cache.go b/mac_cache.go
index a387c136..c08e04c4 100644
--- a/mac_cache.go
+++ b/mac_cache.go
@@ -18,12 +18,14 @@ type MacCache struct {
 	table       map[uint64]*MacCacheEntry
 	maxAge      time.Duration
 	expiryTimer *time.Timer
+	onExpiry    func(net.HardwareAddr, *Peer)
 }
 
-func NewMacCache(maxAge time.Duration) *MacCache {
+func NewMacCache(maxAge time.Duration, onExpiry func(net.HardwareAddr, *Peer)) *MacCache {
 	res := &MacCache{
-		table:  make(map[uint64]*MacCacheEntry),
-		maxAge: maxAge}
+		table:    make(map[uint64]*MacCacheEntry),
+		maxAge:   maxAge,
+		onExpiry: onExpiry}
 	res.setExpiryTimer()
 	return res
 }
@@ -102,6 +104,7 @@ func (cache *MacCache) expire() {
 	for key, entry := range cache.table {
 		if now.After(entry.lastSeen.Add(cache.maxAge)) {
 			delete(cache.table, key)
+			cache.onExpiry(intmac(key), entry.peer)
 		}
 	}
 	cache.setExpiryTimer()
diff --git a/router.go b/router.go
index 17d395dc..649777fc 100644
--- a/router.go
+++ b/router.go
@@ -15,9 +15,12 @@ import (
 const macMaxAge = 10 * time.Minute
 
 func NewRouter(iface *net.Interface, name PeerName, password []byte, connLimit int, bufSz int, logFrame func(string, []byte, *layers.Ethernet)) *Router {
+	onMacExpiry := func(mac net.HardwareAddr, peer *Peer) {
+		log.Println("Expired MAC", mac, "at", peer.Name)
+	}
 	router := &Router{
 		Iface:     iface,
-		Macs:      NewMacCache(macMaxAge),
+		Macs:      NewMacCache(macMaxAge, onMacExpiry),
 		Peers:     NewPeerCache(),
 		ConnLimit: connLimit,
 		BufSz:     bufSz,

From b19abe4eab6c5c64bfa9383b0dd335718069c122 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Thu, 4 Sep 2014 00:02:08 +0100
Subject: [PATCH 37/40] move logging out of peer_cache

---
 peer_cache.go | 8 ++++----
 router.go     | 5 ++++-
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/peer_cache.go b/peer_cache.go
index a6c29f4e..fc0c2b2a 100644
--- a/peer_cache.go
+++ b/peer_cache.go
@@ -5,17 +5,17 @@ import (
 	"encoding/gob"
 	"fmt"
 	"io"
-	"log"
 	"sync"
 )
 
 type PeerCache struct {
 	sync.RWMutex
 	table map[PeerName]*Peer
+	onGC func(*Peer)
 }
 
-func NewPeerCache() *PeerCache {
-	return &PeerCache{table: make(map[PeerName]*Peer)}
+func NewPeerCache(onGC func(*Peer)) *PeerCache {
+	return &PeerCache{table: make(map[PeerName]*Peer), onGC: onGC}
 }
 
 func (cache *PeerCache) FetchWithDefault(peer *Peer) *Peer {
@@ -129,7 +129,7 @@ func (cache *PeerCache) garbageCollect(ourself *Peer) []*Peer {
 	for name, peer := range cache.table {
 		found, _ := ourself.HasPathTo(peer, false)
 		if !found && !peer.IsLocallyReferenced() {
-			log.Println("Removing unreachable", peer)
+			cache.onGC(peer)
 			delete(cache.table, name)
 			ourself.Router.Macs.Delete(peer)
 			removed = append(removed, peer)
diff --git a/router.go b/router.go
index 649777fc..ac8ae5fb 100644
--- a/router.go
+++ b/router.go
@@ -18,10 +18,13 @@ func NewRouter(iface *net.Interface, name PeerName, password []byte, connLimit i
 	onMacExpiry := func(mac net.HardwareAddr, peer *Peer) {
 		log.Println("Expired MAC", mac, "at", peer.Name)
 	}
+	onPeerGC := func(peer *Peer) {
+		log.Println("Removing unreachable", peer)
+	}
 	router := &Router{
 		Iface:     iface,
 		Macs:      NewMacCache(macMaxAge, onMacExpiry),
-		Peers:     NewPeerCache(),
+		Peers:     NewPeerCache(onPeerGC),
 		ConnLimit: connLimit,
 		BufSz:     bufSz,
 		LogFrame:  logFrame}

From e72803e564de3edd476d439557c2db3397ed9b31 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Thu, 4 Sep 2014 07:29:11 +0100
Subject: [PATCH 38/40] remove duplicate code

---
 router.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/router.go b/router.go
index ac8ae5fb..2ecd5c79 100644
--- a/router.go
+++ b/router.go
@@ -282,7 +282,6 @@ func (router *Router) handleUDPPacketFunc(dec *EthernetDecoder, po PacketSink) F
 		checkWarn(po.WritePacket(frame))
 		dstPeer, found = router.Macs.Lookup(dec.eth.DstMAC)
 		if !found || dec.BroadcastFrame() || dstPeer != router.Ourself {
-			df := decodedLen == 2 && (dec.ip.Flags&layers.IPv4DontFragment != 0)
 			checkFrameTooBig(router.Ourself.RelayBroadcast(srcPeer, df, frame, dec), srcPeer)
 		}
 		return nil

From 679e9b52b23b35d7faab78adba61f692870f76c3 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Thu, 4 Sep 2014 07:32:11 +0100
Subject: [PATCH 39/40] add some error logging that was missing

---
 router.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/router.go b/router.go
index 2ecd5c79..201e9872 100644
--- a/router.go
+++ b/router.go
@@ -282,7 +282,7 @@ func (router *Router) handleUDPPacketFunc(dec *EthernetDecoder, po PacketSink) F
 		checkWarn(po.WritePacket(frame))
 		dstPeer, found = router.Macs.Lookup(dec.eth.DstMAC)
 		if !found || dec.BroadcastFrame() || dstPeer != router.Ourself {
-			checkFrameTooBig(router.Ourself.RelayBroadcast(srcPeer, df, frame, dec), srcPeer)
+			return checkFrameTooBig(router.Ourself.RelayBroadcast(srcPeer, df, frame, dec), srcPeer)
 		}
 		return nil
 	}

From 8b8fc06fcf88c7ebb8c8e71d63f9b55015877284 Mon Sep 17 00:00:00 2001
From: Matthias Radestock <matthias.radestock@gmail.com>
Date: Thu, 4 Sep 2014 07:36:10 +0100
Subject: [PATCH 40/40] cosmetic

---
 router.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/router.go b/router.go
index 201e9872..39a7bac6 100644
--- a/router.go
+++ b/router.go
@@ -280,10 +280,12 @@ func (router *Router) handleUDPPacketFunc(dec *EthernetDecoder, po PacketSink) F
 		}
 		router.LogFrame("Injecting", frame, &dec.eth)
 		checkWarn(po.WritePacket(frame))
+
 		dstPeer, found = router.Macs.Lookup(dec.eth.DstMAC)
 		if !found || dec.BroadcastFrame() || dstPeer != router.Ourself {
 			return checkFrameTooBig(router.Ourself.RelayBroadcast(srcPeer, df, frame, dec), srcPeer)
 		}
+
 		return nil
 	}
 }