(slightly) faster pmtu discovery

Issues with the previous scheme...

When we have just encountered an EMSGSIZE, the odds of encountering
another one straight away, when attemtping to send a verification
packet with the lower mtu, are slim, since for a new, lower mtu to be
set some packet must have made it further then the one that triggered
the icmp3.4 that gave rise to the EMSGSIZE, and we must have received
the resulting icmp3.4. All of which takes time.

Also, sending a bunch of udp packets in rapid succession is not a
great way to improve their chances of making it through to the other
peer, since they could easily all get dropped by a single congestion
event or other network glitch.

Now we send a single verification packet, wait for a short while, and
if we don't get a response then we send another one, wait for longer
period (2x), etc, until some upper bound is reached, at which point we
lower the pmtu and start the verification process again (same as
currently happens).

We are now also calling flush() to send the verification packet, just
as we do for ordinary packets. This makes for shorter code, consistent
error handling, and introduces a nice little recursion :)

consts.go

@@ -5,22 +5,23 @@ import (
 )
 
 const (
-	Protocol          = "weave"
-	ProtocolVersion   = 8
-	EthernetOverhead  = 14
-	UDPOverhead       = 28 // 20 bytes for IPv4, 8 bytes for UDP
-	Port              = 6783
-	DefaultPMTU       = 65535
-	MaxUDPPacketSize  = 65536
-	ChannelSize       = 16
-	UDPNonceSendAt    = 8192
-	FragTestSize      = 60001
-	PMTUDiscoverySize = 60000
-	FastHeartbeat     = 500 * time.Millisecond
-	SlowHeartbeat     = 10 * time.Second
-	FetchAllInterval  = 30 * time.Second
-	FragTestInterval  = 5 * time.Minute
-	PMTUVerifyTimeout = 5 * time.Second
+	Protocol           = "weave"
+	ProtocolVersion    = 8
+	EthernetOverhead   = 14
+	UDPOverhead        = 28 // 20 bytes for IPv4, 8 bytes for UDP
+	Port               = 6783
+	DefaultPMTU        = 65535
+	MaxUDPPacketSize   = 65536
+	ChannelSize        = 16
+	UDPNonceSendAt     = 8192
+	FragTestSize       = 60001
+	PMTUDiscoverySize  = 60000
+	FastHeartbeat      = 500 * time.Millisecond
+	SlowHeartbeat      = 10 * time.Second
+	FetchAllInterval   = 30 * time.Second
+	FragTestInterval   = 5 * time.Minute
+	PMTUVerifyAttempts = 8
+	PMTUVerifyTimeout  = 10 * time.Millisecond // gets doubled with every attempt
 )
 
 const (

forwarder.go

@@ -206,9 +206,15 @@ func (fwd *Forwarder) Run() {
 			// to other data.
 			fwd.pmtuVerifyTick = nil
 			if !fwd.conn.isEffectivePMTUVerfied() {
-				fwd.effectivePMTU -= 8
-				fwd.maxPayload -= 8
-				fwd.updateEffectivePMTU()
+				if fwd.pmtuVerifyCount == 0 {
+					fwd.pmtuVerifyCount = PMTUVerifyAttempts
+					fwd.effectivePMTU -= 8
+					fwd.maxPayload -= 8
+					fwd.conn.setEffectivePMTU(fwd.effectivePMTU)
+				} else {
+					fwd.pmtuVerifyCount--
+				}
+				fwd.verifyEffectivePMTU()
 			}
 		case frame = <-fwd.ch:
 			if !fwd.appendFrame(frame) {
@@ -242,27 +248,16 @@ func (fwd *Forwarder) setMaxPayloadAndEffectivePMTU(pmtu int) {
 	fwd.effectivePMTU = fwd.maxPayload - fwd.enc.PacketOverhead() - fwd.enc.FrameOverhead() - EthernetOverhead
 }
 
-func (fwd *Forwarder) updateEffectivePMTU() {
-	for gotNewPMTU := true; gotNewPMTU; {
-		fwd.conn.setEffectivePMTU(fwd.effectivePMTU)
-		pmtuVerifyFrame := &ForwardedFrame{
-			srcPeer: fwd.conn.local,
-			dstPeer: fwd.conn.remote,
-			frame:   make([]byte, fwd.effectivePMTU+EthernetOverhead)}
-		gotNewPMTU = false
-		for cnt := 0; cnt < 10; cnt += 1 {
-			fwd.enc.AppendFrame(pmtuVerifyFrame)
-			err := fwd.udpSender.Send(fwd.enc.Bytes())
-			if err != nil {
-				if mtbe, ok := err.(MsgTooBigError); ok {
-					fwd.setMaxPayloadAndEffectivePMTU(mtbe.PMTU)
-					gotNewPMTU = true
-					break
-				}
-			}
-		}
+func (fwd *Forwarder) verifyEffectivePMTU() {
+	pmtuVerifyFrame := &ForwardedFrame{
+		srcPeer: fwd.conn.local,
+		dstPeer: fwd.conn.remote,
+		frame:   make([]byte, fwd.effectivePMTU+EthernetOverhead)}
+	fwd.enc.AppendFrame(pmtuVerifyFrame)
+	fwd.flush()
+	if fwd.pmtuVerifyTick == nil {
+		fwd.pmtuVerifyTick = time.After(PMTUVerifyTimeout << (PMTUVerifyAttempts - fwd.pmtuVerifyCount))
 	}
-	fwd.pmtuVerifyTick = time.After(PMTUVerifyTimeout)
 }
 
 func (fwd *Forwarder) appendFrame(frame *ForwardedFrame) bool {
@@ -279,7 +274,9 @@ func (fwd *Forwarder) flush() {
 	if err != nil {
 		if mtbe, ok := err.(MsgTooBigError); ok {
 			fwd.setMaxPayloadAndEffectivePMTU(mtbe.PMTU)
-			fwd.updateEffectivePMTU()
+			fwd.conn.setEffectivePMTU(fwd.effectivePMTU)
+			fwd.pmtuVerifyCount = PMTUVerifyAttempts
+			fwd.verifyEffectivePMTU()
 		} else if PosixError(err) == syscall.ENOBUFS {
 			// TODO handle this better
 		} else {

types.go

@@ -86,14 +86,15 @@ type ConnectionInteraction struct {
 }
 
 type Forwarder struct {
-	conn           *LocalConnection
-	ch             <-chan *ForwardedFrame
-	stop           <-chan interface{}
-	pmtuVerifyTick <-chan time.Time
-	enc            Encryptor
-	udpSender      UDPSender
-	maxPayload     int
-	effectivePMTU  int
+	conn            *LocalConnection
+	ch              <-chan *ForwardedFrame
+	stop            <-chan interface{}
+	pmtuVerifyTick  <-chan time.Time
+	pmtuVerifyCount uint
+	enc             Encryptor
+	udpSender       UDPSender
+	maxPayload      int
+	effectivePMTU   int
 }
 
 type UDPPacket struct {