From 5b6dc2d762c89ead3d8efd1ab937ad08db92f718 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Fri, 29 Aug 2014 19:15:18 +0100 Subject: [PATCH] (slightly) faster pmtu discovery Issues with the previous scheme... When we have just encountered an EMSGSIZE, the odds of encountering another one straight away, when attemtping to send a verification packet with the lower mtu, are slim, since for a new, lower mtu to be set some packet must have made it further then the one that triggered the icmp3.4 that gave rise to the EMSGSIZE, and we must have received the resulting icmp3.4. All of which takes time. Also, sending a bunch of udp packets in rapid succession is not a great way to improve their chances of making it through to the other peer, since they could easily all get dropped by a single congestion event or other network glitch. Now we send a single verification packet, wait for a short while, and if we don't get a response then we send another one, wait for longer period (2x), etc, until some upper bound is reached, at which point we lower the pmtu and start the verification process again (same as currently happens). We are now also calling flush() to send the verification packet, just as we do for ordinary packets. This makes for shorter code, consistent error handling, and introduces a nice little recursion :) --- consts.go | 33 +++++++++++++++++---------------- forwarder.go | 45 +++++++++++++++++++++------------------------ types.go | 17 +++++++++-------- 3 files changed, 47 insertions(+), 48 deletions(-) diff --git a/consts.go b/consts.go index 106e1c8e..d682b0c7 100644 --- a/consts.go +++ b/consts.go @@ -5,22 +5,23 @@ import ( ) const ( - Protocol = "weave" - ProtocolVersion = 8 - EthernetOverhead = 14 - UDPOverhead = 28 // 20 bytes for IPv4, 8 bytes for UDP - Port = 6783 - DefaultPMTU = 65535 - MaxUDPPacketSize = 65536 - ChannelSize = 16 - UDPNonceSendAt = 8192 - FragTestSize = 60001 - PMTUDiscoverySize = 60000 - FastHeartbeat = 500 * time.Millisecond - SlowHeartbeat = 10 * time.Second - FetchAllInterval = 30 * time.Second - FragTestInterval = 5 * time.Minute - PMTUVerifyTimeout = 5 * time.Second + Protocol = "weave" + ProtocolVersion = 8 + EthernetOverhead = 14 + UDPOverhead = 28 // 20 bytes for IPv4, 8 bytes for UDP + Port = 6783 + DefaultPMTU = 65535 + MaxUDPPacketSize = 65536 + ChannelSize = 16 + UDPNonceSendAt = 8192 + FragTestSize = 60001 + PMTUDiscoverySize = 60000 + FastHeartbeat = 500 * time.Millisecond + SlowHeartbeat = 10 * time.Second + FetchAllInterval = 30 * time.Second + FragTestInterval = 5 * time.Minute + PMTUVerifyAttempts = 8 + PMTUVerifyTimeout = 10 * time.Millisecond // gets doubled with every attempt ) const ( diff --git a/forwarder.go b/forwarder.go index 7a15a25b..c48bcaf1 100644 --- a/forwarder.go +++ b/forwarder.go @@ -206,9 +206,15 @@ func (fwd *Forwarder) Run() { // to other data. fwd.pmtuVerifyTick = nil if !fwd.conn.isEffectivePMTUVerfied() { - fwd.effectivePMTU -= 8 - fwd.maxPayload -= 8 - fwd.updateEffectivePMTU() + if fwd.pmtuVerifyCount == 0 { + fwd.pmtuVerifyCount = PMTUVerifyAttempts + fwd.effectivePMTU -= 8 + fwd.maxPayload -= 8 + fwd.conn.setEffectivePMTU(fwd.effectivePMTU) + } else { + fwd.pmtuVerifyCount-- + } + fwd.verifyEffectivePMTU() } case frame = <-fwd.ch: if !fwd.appendFrame(frame) { @@ -242,27 +248,16 @@ func (fwd *Forwarder) setMaxPayloadAndEffectivePMTU(pmtu int) { fwd.effectivePMTU = fwd.maxPayload - fwd.enc.PacketOverhead() - fwd.enc.FrameOverhead() - EthernetOverhead } -func (fwd *Forwarder) updateEffectivePMTU() { - for gotNewPMTU := true; gotNewPMTU; { - fwd.conn.setEffectivePMTU(fwd.effectivePMTU) - pmtuVerifyFrame := &ForwardedFrame{ - srcPeer: fwd.conn.local, - dstPeer: fwd.conn.remote, - frame: make([]byte, fwd.effectivePMTU+EthernetOverhead)} - gotNewPMTU = false - for cnt := 0; cnt < 10; cnt += 1 { - fwd.enc.AppendFrame(pmtuVerifyFrame) - err := fwd.udpSender.Send(fwd.enc.Bytes()) - if err != nil { - if mtbe, ok := err.(MsgTooBigError); ok { - fwd.setMaxPayloadAndEffectivePMTU(mtbe.PMTU) - gotNewPMTU = true - break - } - } - } +func (fwd *Forwarder) verifyEffectivePMTU() { + pmtuVerifyFrame := &ForwardedFrame{ + srcPeer: fwd.conn.local, + dstPeer: fwd.conn.remote, + frame: make([]byte, fwd.effectivePMTU+EthernetOverhead)} + fwd.enc.AppendFrame(pmtuVerifyFrame) + fwd.flush() + if fwd.pmtuVerifyTick == nil { + fwd.pmtuVerifyTick = time.After(PMTUVerifyTimeout << (PMTUVerifyAttempts - fwd.pmtuVerifyCount)) } - fwd.pmtuVerifyTick = time.After(PMTUVerifyTimeout) } func (fwd *Forwarder) appendFrame(frame *ForwardedFrame) bool { @@ -279,7 +274,9 @@ func (fwd *Forwarder) flush() { if err != nil { if mtbe, ok := err.(MsgTooBigError); ok { fwd.setMaxPayloadAndEffectivePMTU(mtbe.PMTU) - fwd.updateEffectivePMTU() + fwd.conn.setEffectivePMTU(fwd.effectivePMTU) + fwd.pmtuVerifyCount = PMTUVerifyAttempts + fwd.verifyEffectivePMTU() } else if PosixError(err) == syscall.ENOBUFS { // TODO handle this better } else { diff --git a/types.go b/types.go index cf675a21..861fa7f5 100644 --- a/types.go +++ b/types.go @@ -86,14 +86,15 @@ type ConnectionInteraction struct { } type Forwarder struct { - conn *LocalConnection - ch <-chan *ForwardedFrame - stop <-chan interface{} - pmtuVerifyTick <-chan time.Time - enc Encryptor - udpSender UDPSender - maxPayload int - effectivePMTU int + conn *LocalConnection + ch <-chan *ForwardedFrame + stop <-chan interface{} + pmtuVerifyTick <-chan time.Time + pmtuVerifyCount uint + enc Encryptor + udpSender UDPSender + maxPayload int + effectivePMTU int } type UDPPacket struct {