Files
weave/mesh/gossip.go
T
Matthias Radestock c6a73b8a75 don't stall broadcasting when there are blocked connections
Previously broadcasts were being handled by one GossipSender per
broadcast source (and channel), which sends the broadcast to all next
hops, and hence can stall when a single destination is stalled.

Here we get rid of these per broadcast source GossipSenders. Instead
broadcasts are sent to per-connection GossipSenders for all next hops.

For this we use the existing GossipSenders we have set up for ordinary
gossip. In order to deal with broadcasts they need one GossipData cell
per broadcast source, since only broadcasts from the same source can
be Merge()ed. So we add a PeerName->GossipData map of cells, in
addition to the existing cell for ordinary gossip.

The GossipSender goroutine picks one of the cells at a time, Encode()s
the contents and sends it. It prefers the ordinary gossip cell over
the broadcast cells since typically ordinary gossip is more important.

To reduce coupling, the GossipSenders don't actually know how to
construct protocol messages. They just invoke a couple of functions
for that - one for ordinary gossip and one for broadcast - which are
supplied by the GossipChannel.

There are two downsides to this change:

1. broadcasts get encoded per connection, rather than just once

2. we can potentially end up with O(n_peers^2) cells, each containing
accumulated (via GossipData.Merge()) broadcasts. For this to happen,
the peer must

- deal with gossip from most nodes. This doesn't happen in
  (near)complete connection topologies. Hypercube topologies are
  probably the worst case uniform topology. And a star topology is the
  worst case for a single (the centre) peer.

- have backlogged connections to most of its neighbours, without
  those connections being completely stalled (since that would cause
  heartbeat timeouts to terminate them).

Furthermore, for this to matter in practise, the accumulated broadcast
GossipData must be sizeable:

- For topology gossip, GossipData is just a set of PeerNames. Which
  takes very little space and is bounded, since there is a finite
  number of peers. And, if we could get rid of workaround for #1793
  (cbaa92dcb), then topology broadcasts would only ever contain
  information about the source peer, so the GossipData would contain
  just one PeerName.

- IPAM only employs broadcast during initialisation and shutdown.

- DNS broadcasts contain DNS entries for containers on the source
  peer. Each entry will typically be 100-200 bytes. The accumulated
  broadcast GossipData from a peer will contain entries for all the
  peer's DNS entries, worst case. This includes tombstones,
  i.e. entries for containers that have died. If there is churn,
  i.e. DNS entries being added and removed continuously, and the churn
  rate exceeds the rate at which we can forward those entries, then
  the accumulated broadcast GossipData can grow unbounded. Note that
  this is the case on master too; the difference here is that we can
  have up to n_peers copies of that GossipData.
2016-01-05 17:18:19 +00:00

241 lines
6.1 KiB
Go

package mesh
import (
"fmt"
"sync"
)
type GossipData interface {
Encode() [][]byte
Merge(GossipData) GossipData
}
type Gossip interface {
// specific message from one peer to another
// intermediate peers relay it using unicast topology.
GossipUnicast(dstPeerName PeerName, msg []byte) error
// send gossip to every peer, relayed using broadcast topology.
GossipBroadcast(update GossipData) error
}
type Gossiper interface {
OnGossipUnicast(sender PeerName, msg []byte) error
// merge received data into state and return a representation of
// the received data, for further propagation
OnGossipBroadcast(sender PeerName, update []byte) (GossipData, error)
// return state of everything we know; gets called periodically
Gossip() GossipData
// merge received data into state and return "everything new I've
// just learnt", or nil if nothing in the received data was new
OnGossip(update []byte) (GossipData, error)
}
// Accumulates GossipData that needs to be sent to one destination,
// and sends it when possible.
type GossipSender struct {
sync.Mutex
makeMsg func(msg []byte) ProtocolMsg
makeBroadcastMsg func(srcName PeerName, msg []byte) ProtocolMsg
sender ProtocolSender
gossip GossipData
broadcasts map[PeerName]GossipData
stop chan<- struct{}
more chan<- struct{}
flush chan<- chan<- bool // for testing
}
func NewGossipSender(makeMsg func(msg []byte) ProtocolMsg, makeBroadcastMsg func(srcName PeerName, msg []byte) ProtocolMsg, sender ProtocolSender) *GossipSender {
stop := make(chan struct{})
more := make(chan struct{}, 1)
flush := make(chan chan<- bool)
s := &GossipSender{
makeMsg: makeMsg,
makeBroadcastMsg: makeBroadcastMsg,
sender: sender,
broadcasts: make(map[PeerName]GossipData),
stop: stop,
more: more,
flush: flush}
go s.run(stop, more, flush)
return s
}
func (s *GossipSender) run(stop <-chan struct{}, more <-chan struct{}, flush <-chan chan<- bool) {
sent := false
for {
select {
case <-stop:
return
case <-more:
sent = s.deliver() || sent
case ch := <-flush: // for testing
// send anything pending, then reply back whether we sent
// anything since previous flush
select {
case <-more:
sent = s.deliver() || sent
default:
}
ch <- sent
sent = false
}
}
}
func (s *GossipSender) deliver() bool {
sent := false
// We must not hold our lock when sending, since that would block
// the callers of Send/Broadcast while we are stuck waiting for
// network congestion to clear. So we pick and send one piece of
// data at a time, only holding the lock during the picking.
for {
data, makeProtocolMsg := s.pick()
if data == nil {
return sent
}
for _, msg := range data.Encode() {
if s.sender.SendProtocolMsg(makeProtocolMsg(msg)) != nil {
break
}
}
sent = true
}
}
func (s *GossipSender) pick() (data GossipData, makeProtocolMsg func(msg []byte) ProtocolMsg) {
s.Lock()
defer s.Unlock()
switch {
case s.gossip != nil: // usually more important than broadcasts
data = s.gossip
makeProtocolMsg = s.makeMsg
s.gossip = nil
case len(s.broadcasts) > 0:
for srcName, d := range s.broadcasts {
data = d
makeProtocolMsg = func(msg []byte) ProtocolMsg { return s.makeBroadcastMsg(srcName, msg) }
delete(s.broadcasts, srcName)
break
}
}
return
}
func (s *GossipSender) Send(data GossipData) {
s.Lock()
defer s.Unlock()
if s.empty() {
defer s.prod()
}
if s.gossip == nil {
s.gossip = data
} else {
s.gossip = s.gossip.Merge(data)
}
}
func (s *GossipSender) Broadcast(srcName PeerName, data GossipData) {
s.Lock()
defer s.Unlock()
if s.empty() {
defer s.prod()
}
d, found := s.broadcasts[srcName]
if !found {
s.broadcasts[srcName] = data
} else {
s.broadcasts[srcName] = d.Merge(data)
}
}
func (s *GossipSender) empty() bool { return s.gossip == nil && len(s.broadcasts) == 0 }
func (s *GossipSender) prod() {
select {
case s.more <- void:
default:
}
}
// for testing
func (s *GossipSender) Flush() bool {
ch := make(chan bool)
s.flush <- ch
return <-ch
}
func (s *GossipSender) Stop() {
close(s.stop)
}
type GossipChannels map[string]*GossipChannel
func (router *Router) NewGossip(channelName string, g Gossiper) Gossip {
channel := NewGossipChannel(channelName, router.Ourself, router.Routes, g)
router.gossipLock.Lock()
defer router.gossipLock.Unlock()
if _, found := router.gossipChannels[channelName]; found {
checkFatal(fmt.Errorf("[gossip] duplicate channel %s", channelName))
}
router.gossipChannels[channelName] = channel
return channel
}
func (router *Router) gossipChannel(channelName string) *GossipChannel {
router.gossipLock.RLock()
channel, found := router.gossipChannels[channelName]
router.gossipLock.RUnlock()
if found {
return channel
}
router.gossipLock.Lock()
defer router.gossipLock.Unlock()
if channel, found = router.gossipChannels[channelName]; found {
return channel
}
channel = NewGossipChannel(channelName, router.Ourself, router.Routes, &surrogateGossiper)
channel.log("created surrogate channel")
router.gossipChannels[channelName] = channel
return channel
}
func (router *Router) gossipChannelSet() map[*GossipChannel]struct{} {
channels := make(map[*GossipChannel]struct{})
router.gossipLock.RLock()
defer router.gossipLock.RUnlock()
for _, channel := range router.gossipChannels {
channels[channel] = void
}
return channels
}
func (router *Router) SendAllGossip() {
for channel := range router.gossipChannelSet() {
if gossip := channel.gossiper.Gossip(); gossip != nil {
channel.Send(gossip)
}
}
}
func (router *Router) SendAllGossipDown(conn Connection) {
for channel := range router.gossipChannelSet() {
if gossip := channel.gossiper.Gossip(); gossip != nil {
channel.SendDown(conn, gossip)
}
}
}
// for testing
func (router *Router) sendPendingGossip() bool {
sentSomething := false
for channel := range router.gossipChannelSet() {
channel.Lock()
for _, sender := range channel.senders {
sentSomething = sender.Flush() || sentSomething
}
channel.Unlock()
}
return sentSomething
}