Previously broadcasts were being handled by one GossipSender per
broadcast source (and channel), which sends the broadcast to all next
hops, and hence can stall when a single destination is stalled.
Here we get rid of these per broadcast source GossipSenders. Instead
broadcasts are sent to per-connection GossipSenders for all next hops.
For this we use the existing GossipSenders we have set up for ordinary
gossip. In order to deal with broadcasts they need one GossipData cell
per broadcast source, since only broadcasts from the same source can
be Merge()ed. So we add a PeerName->GossipData map of cells, in
addition to the existing cell for ordinary gossip.
The GossipSender goroutine picks one of the cells at a time, Encode()s
the contents and sends it. It prefers the ordinary gossip cell over
the broadcast cells since typically ordinary gossip is more important.
To reduce coupling, the GossipSenders don't actually know how to
construct protocol messages. They just invoke a couple of functions
for that - one for ordinary gossip and one for broadcast - which are
supplied by the GossipChannel.
There are two downsides to this change:
1. broadcasts get encoded per connection, rather than just once
2. we can potentially end up with O(n_peers^2) cells, each containing
accumulated (via GossipData.Merge()) broadcasts. For this to happen,
the peer must
- deal with gossip from most nodes. This doesn't happen in
(near)complete connection topologies. Hypercube topologies are
probably the worst case uniform topology. And a star topology is the
worst case for a single (the centre) peer.
- have backlogged connections to most of its neighbours, without
those connections being completely stalled (since that would cause
heartbeat timeouts to terminate them).
Furthermore, for this to matter in practise, the accumulated broadcast
GossipData must be sizeable:
- For topology gossip, GossipData is just a set of PeerNames. Which
takes very little space and is bounded, since there is a finite
number of peers. And, if we could get rid of workaround for #1793
(cbaa92dcb), then topology broadcasts would only ever contain
information about the source peer, so the GossipData would contain
just one PeerName.
- IPAM only employs broadcast during initialisation and shutdown.
- DNS broadcasts contain DNS entries for containers on the source
peer. Each entry will typically be 100-200 bytes. The accumulated
broadcast GossipData from a peer will contain entries for all the
peer's DNS entries, worst case. This includes tombstones,
i.e. entries for containers that have died. If there is churn,
i.e. DNS entries being added and removed continuously, and the churn
rate exceeds the rate at which we can forward those entries, then
the accumulated broadcast GossipData can grow unbounded. Note that
this is the case on master too; the difference here is that we can
have up to n_peers copies of that GossipData.
241 lines
6.1 KiB
Go
241 lines
6.1 KiB
Go
package mesh
|
|
|
|
import (
|
|
"fmt"
|
|
"sync"
|
|
)
|
|
|
|
type GossipData interface {
|
|
Encode() [][]byte
|
|
Merge(GossipData) GossipData
|
|
}
|
|
|
|
type Gossip interface {
|
|
// specific message from one peer to another
|
|
// intermediate peers relay it using unicast topology.
|
|
GossipUnicast(dstPeerName PeerName, msg []byte) error
|
|
// send gossip to every peer, relayed using broadcast topology.
|
|
GossipBroadcast(update GossipData) error
|
|
}
|
|
|
|
type Gossiper interface {
|
|
OnGossipUnicast(sender PeerName, msg []byte) error
|
|
// merge received data into state and return a representation of
|
|
// the received data, for further propagation
|
|
OnGossipBroadcast(sender PeerName, update []byte) (GossipData, error)
|
|
// return state of everything we know; gets called periodically
|
|
Gossip() GossipData
|
|
// merge received data into state and return "everything new I've
|
|
// just learnt", or nil if nothing in the received data was new
|
|
OnGossip(update []byte) (GossipData, error)
|
|
}
|
|
|
|
// Accumulates GossipData that needs to be sent to one destination,
|
|
// and sends it when possible.
|
|
type GossipSender struct {
|
|
sync.Mutex
|
|
makeMsg func(msg []byte) ProtocolMsg
|
|
makeBroadcastMsg func(srcName PeerName, msg []byte) ProtocolMsg
|
|
sender ProtocolSender
|
|
gossip GossipData
|
|
broadcasts map[PeerName]GossipData
|
|
stop chan<- struct{}
|
|
more chan<- struct{}
|
|
flush chan<- chan<- bool // for testing
|
|
}
|
|
|
|
func NewGossipSender(makeMsg func(msg []byte) ProtocolMsg, makeBroadcastMsg func(srcName PeerName, msg []byte) ProtocolMsg, sender ProtocolSender) *GossipSender {
|
|
stop := make(chan struct{})
|
|
more := make(chan struct{}, 1)
|
|
flush := make(chan chan<- bool)
|
|
s := &GossipSender{
|
|
makeMsg: makeMsg,
|
|
makeBroadcastMsg: makeBroadcastMsg,
|
|
sender: sender,
|
|
broadcasts: make(map[PeerName]GossipData),
|
|
stop: stop,
|
|
more: more,
|
|
flush: flush}
|
|
go s.run(stop, more, flush)
|
|
return s
|
|
}
|
|
|
|
func (s *GossipSender) run(stop <-chan struct{}, more <-chan struct{}, flush <-chan chan<- bool) {
|
|
sent := false
|
|
for {
|
|
select {
|
|
case <-stop:
|
|
return
|
|
case <-more:
|
|
sent = s.deliver() || sent
|
|
case ch := <-flush: // for testing
|
|
// send anything pending, then reply back whether we sent
|
|
// anything since previous flush
|
|
select {
|
|
case <-more:
|
|
sent = s.deliver() || sent
|
|
default:
|
|
}
|
|
ch <- sent
|
|
sent = false
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *GossipSender) deliver() bool {
|
|
sent := false
|
|
// We must not hold our lock when sending, since that would block
|
|
// the callers of Send/Broadcast while we are stuck waiting for
|
|
// network congestion to clear. So we pick and send one piece of
|
|
// data at a time, only holding the lock during the picking.
|
|
for {
|
|
data, makeProtocolMsg := s.pick()
|
|
if data == nil {
|
|
return sent
|
|
}
|
|
for _, msg := range data.Encode() {
|
|
if s.sender.SendProtocolMsg(makeProtocolMsg(msg)) != nil {
|
|
break
|
|
}
|
|
}
|
|
sent = true
|
|
}
|
|
}
|
|
|
|
func (s *GossipSender) pick() (data GossipData, makeProtocolMsg func(msg []byte) ProtocolMsg) {
|
|
s.Lock()
|
|
defer s.Unlock()
|
|
switch {
|
|
case s.gossip != nil: // usually more important than broadcasts
|
|
data = s.gossip
|
|
makeProtocolMsg = s.makeMsg
|
|
s.gossip = nil
|
|
case len(s.broadcasts) > 0:
|
|
for srcName, d := range s.broadcasts {
|
|
data = d
|
|
makeProtocolMsg = func(msg []byte) ProtocolMsg { return s.makeBroadcastMsg(srcName, msg) }
|
|
delete(s.broadcasts, srcName)
|
|
break
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
func (s *GossipSender) Send(data GossipData) {
|
|
s.Lock()
|
|
defer s.Unlock()
|
|
if s.empty() {
|
|
defer s.prod()
|
|
}
|
|
if s.gossip == nil {
|
|
s.gossip = data
|
|
} else {
|
|
s.gossip = s.gossip.Merge(data)
|
|
}
|
|
}
|
|
|
|
func (s *GossipSender) Broadcast(srcName PeerName, data GossipData) {
|
|
s.Lock()
|
|
defer s.Unlock()
|
|
if s.empty() {
|
|
defer s.prod()
|
|
}
|
|
d, found := s.broadcasts[srcName]
|
|
if !found {
|
|
s.broadcasts[srcName] = data
|
|
} else {
|
|
s.broadcasts[srcName] = d.Merge(data)
|
|
}
|
|
}
|
|
|
|
func (s *GossipSender) empty() bool { return s.gossip == nil && len(s.broadcasts) == 0 }
|
|
|
|
func (s *GossipSender) prod() {
|
|
select {
|
|
case s.more <- void:
|
|
default:
|
|
}
|
|
}
|
|
|
|
// for testing
|
|
func (s *GossipSender) Flush() bool {
|
|
ch := make(chan bool)
|
|
s.flush <- ch
|
|
return <-ch
|
|
}
|
|
|
|
func (s *GossipSender) Stop() {
|
|
close(s.stop)
|
|
}
|
|
|
|
type GossipChannels map[string]*GossipChannel
|
|
|
|
func (router *Router) NewGossip(channelName string, g Gossiper) Gossip {
|
|
channel := NewGossipChannel(channelName, router.Ourself, router.Routes, g)
|
|
router.gossipLock.Lock()
|
|
defer router.gossipLock.Unlock()
|
|
if _, found := router.gossipChannels[channelName]; found {
|
|
checkFatal(fmt.Errorf("[gossip] duplicate channel %s", channelName))
|
|
}
|
|
router.gossipChannels[channelName] = channel
|
|
return channel
|
|
}
|
|
|
|
func (router *Router) gossipChannel(channelName string) *GossipChannel {
|
|
router.gossipLock.RLock()
|
|
channel, found := router.gossipChannels[channelName]
|
|
router.gossipLock.RUnlock()
|
|
if found {
|
|
return channel
|
|
}
|
|
router.gossipLock.Lock()
|
|
defer router.gossipLock.Unlock()
|
|
if channel, found = router.gossipChannels[channelName]; found {
|
|
return channel
|
|
}
|
|
channel = NewGossipChannel(channelName, router.Ourself, router.Routes, &surrogateGossiper)
|
|
channel.log("created surrogate channel")
|
|
router.gossipChannels[channelName] = channel
|
|
return channel
|
|
}
|
|
|
|
func (router *Router) gossipChannelSet() map[*GossipChannel]struct{} {
|
|
channels := make(map[*GossipChannel]struct{})
|
|
router.gossipLock.RLock()
|
|
defer router.gossipLock.RUnlock()
|
|
for _, channel := range router.gossipChannels {
|
|
channels[channel] = void
|
|
}
|
|
return channels
|
|
}
|
|
|
|
func (router *Router) SendAllGossip() {
|
|
for channel := range router.gossipChannelSet() {
|
|
if gossip := channel.gossiper.Gossip(); gossip != nil {
|
|
channel.Send(gossip)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (router *Router) SendAllGossipDown(conn Connection) {
|
|
for channel := range router.gossipChannelSet() {
|
|
if gossip := channel.gossiper.Gossip(); gossip != nil {
|
|
channel.SendDown(conn, gossip)
|
|
}
|
|
}
|
|
}
|
|
|
|
// for testing
|
|
|
|
func (router *Router) sendPendingGossip() bool {
|
|
sentSomething := false
|
|
for channel := range router.gossipChannelSet() {
|
|
channel.Lock()
|
|
for _, sender := range channel.senders {
|
|
sentSomething = sender.Flush() || sentSomething
|
|
}
|
|
channel.Unlock()
|
|
}
|
|
return sentSomething
|
|
}
|