Files
weave/router/peers.go
T
Matthias Radestock bfb050c9cc better convergence when peers restart
Previously when a peer restarted, information about the new
incarnation (i.e. with a different UID) was not accepted by other
peers (and connections would be dropped) unless all knowledge of the
previous incarnation had been purged. This could result in a lot of
connection churn and hence connectivity disruption, and, in some
pathological cases, very slow convergence and hence acceptance of the
new incarnation into the network.

We now no longer drop connections when encountering different
incarnations of a peer. There are two situations when that can happen:

1) on connection establishment

we simply proceed

2) on receipt of gossip

to ensure convergence we

a) treat the UID as an additional discriminator when deciding whether
we should update our information about a peer with that which was
gossiped. Specifically, we update the information we hold when a) the
gossiped version is greater, or b) is the same and the UID is
greater.

b) include the UID in the information we update

c) move our own version number beyond any we receive for ourselves, if
the received UID differs from ours.

With (a) we establishes a total order of peer information across
several incarnations of the same peer. i.e. we consider information to
be fresher if it has a higher version, or the same version and higher
UID. This may seem somewhat counter intutive, since it will generally
treat information about new incarnations as older than old
incarnations, since incarnations always start life with version 1. But
to do better we'd need to establish a total order of incarnations that
matches their temporal occurrence. Which requires some sort of durable
state.

So instead we have (c). Through that we learn the highest version
number of any old incarnation of ourselves that other peers still
hold, and then make sure that our version is greater than
that. Essentially we continue where the old incarnations left
off. It's as if instead of restarting we had simply changed UIDs. And
due to (a) and (b) the information about the new incarnation of
ourselves, now with a higher version, will supersede that of the old
incarnations.

Fixes #1554.
2015-11-09 14:58:33 +00:00

539 lines
13 KiB
Go

package router
import (
"bytes"
"encoding/gob"
"fmt"
"io"
"math/rand"
"sync"
)
var void = struct{}{}
type Peers struct {
sync.RWMutex
ourself *LocalPeer
byName map[PeerName]*Peer
byShortID map[PeerShortID]ShortIDPeers
onGC []func(*Peer)
// Called when the mapping from short ids to peers changes
onInvalidateShortIDs []func()
}
type ShortIDPeers struct {
// If we know about a single peer with the short id, this is
// that peer. If there is a collision, this is the peer with
// the lowest Name.
peer *Peer
// In case of a collision, this holds the other peers.
others []*Peer
}
type UnknownPeerError struct {
Name PeerName
}
func (upe UnknownPeerError) Error() string {
return fmt.Sprint("Reference to unknown peer ", upe.Name)
}
type PeerNameSet map[PeerName]struct{}
type ConnectionSummary struct {
NameByte []byte
RemoteTCPAddr string
Outbound bool
Established bool
}
// Pending notifications due to changes to Peers that need to be sent
// out once the Peers is unlocked.
type PeersPendingNotifications struct {
// Peers that have been GCed
removed []*Peer
// The mapping from shorts ids to peers changed
invalidateShortIDs bool
// The local short ID needs reassigning due to a collision
reassignLocalShortID bool
// The local peer was modified
localPeerModified bool
}
func NewPeers(ourself *LocalPeer) *Peers {
peers := &Peers{
ourself: ourself,
byName: make(map[PeerName]*Peer),
byShortID: make(map[PeerShortID]ShortIDPeers),
}
peers.FetchWithDefault(ourself.Peer)
return peers
}
func (peers *Peers) OnGC(callback func(*Peer)) {
peers.Lock()
defer peers.Unlock()
// Although the array underlying peers.onGC might be accessed
// without holding the lock in unlockAndNotify, we don't
// support removing callbacks, so a simple append here is
// safe.
peers.onGC = append(peers.onGC, callback)
}
func (peers *Peers) OnInvalidateShortIDs(callback func()) {
peers.Lock()
defer peers.Unlock()
// Safe, as in OnGC
peers.onInvalidateShortIDs = append(peers.onInvalidateShortIDs, callback)
}
func (peers *Peers) unlockAndNotify(pending *PeersPendingNotifications) {
broadcastLocalPeer := (pending.reassignLocalShortID && peers.reassignLocalShortID(pending)) ||
pending.localPeerModified
onGC := peers.onGC
onInvalidateShortIDs := peers.onInvalidateShortIDs
peers.Unlock()
if pending.removed != nil {
for _, callback := range onGC {
for _, peer := range pending.removed {
callback(peer)
}
}
}
if pending.invalidateShortIDs {
for _, callback := range onInvalidateShortIDs {
callback()
}
}
if broadcastLocalPeer {
peers.ourself.broadcastPeerUpdate()
}
}
func (peers *Peers) addByShortID(peer *Peer, pending *PeersPendingNotifications) {
if !peer.HasShortID {
return
}
entry, ok := peers.byShortID[peer.ShortID]
if !ok {
entry = ShortIDPeers{peer: peer}
} else if entry.peer == nil {
// This short ID is free, but was used in the past.
// Because we are reusing it, it's an invalidation
// event.
entry.peer = peer
pending.invalidateShortIDs = true
} else if peer.Name < entry.peer.Name {
// Short ID collision, this peer becomes the principal
// peer for the short ID, bumping the previous one
// into others.
if entry.peer == peers.ourself.Peer {
// The bumped peer is peers.ourself, so we
// need to look for a new short id
pending.reassignLocalShortID = true
}
entry.others = append(entry.others, entry.peer)
entry.peer = peer
pending.invalidateShortIDs = true
} else {
// Short ID collision, this peer is secondary
entry.others = append(entry.others, peer)
}
peers.byShortID[peer.ShortID] = entry
}
func (peers *Peers) deleteByShortID(peer *Peer, pending *PeersPendingNotifications) {
if !peer.HasShortID {
return
}
entry := peers.byShortID[peer.ShortID]
var otherIndex int
if peer != entry.peer {
// peer is secondary, find its index in others
otherIndex = -1
for i, other := range entry.others {
if peer == other {
otherIndex = i
break
}
}
if otherIndex < 0 {
return
}
} else if len(entry.others) != 0 {
// need to find the peer with the lowest name to
// become the new principal one
otherIndex = 0
minName := entry.others[0].Name
for i := 1; i < len(entry.others); i++ {
otherName := entry.others[i].Name
if otherName < minName {
minName = otherName
otherIndex = i
}
}
entry.peer = entry.others[otherIndex]
pending.invalidateShortIDs = true
} else {
// This is the last peer with the short id. We clear
// the entry, don't delete it, in order to detect when
// it gets re-used.
peers.byShortID[peer.ShortID] = ShortIDPeers{}
return
}
entry.others[otherIndex] = entry.others[len(entry.others)-1]
entry.others = entry.others[:len(entry.others)-1]
peers.byShortID[peer.ShortID] = entry
}
func (peers *Peers) reassignLocalShortID(pending *PeersPendingNotifications) bool {
newShortID, ok := peers.chooseShortID()
if ok {
peers.setLocalShortID(newShortID, pending)
return true
}
// Otherwise we'll try again later on in garbageColleect
return false
}
func (peers *Peers) setLocalShortID(newShortID PeerShortID, pending *PeersPendingNotifications) {
peers.deleteByShortID(peers.ourself.Peer, pending)
peers.ourself.setShortID(newShortID)
peers.addByShortID(peers.ourself.Peer, pending)
}
// Choose an available short id at random
func (peers *Peers) chooseShortID() (PeerShortID, bool) {
rng := rand.New(rand.NewSource(int64(randUint64())))
// First, just try picking some short ids at random, and
// seeing if they are available:
for i := 0; i < 10; i++ {
shortID := PeerShortID(rng.Intn(1 << PeerShortIDBits))
if peers.byShortID[shortID].peer == nil {
return shortID, true
}
}
// Looks like most short ids are used. So count the number of
// unused ones, and pick one at random.
available := int(1 << PeerShortIDBits)
for _, entry := range peers.byShortID {
if entry.peer != nil {
available--
}
}
if available == 0 {
// All short ids are used.
return 0, false
}
n := rng.Intn(available)
var i PeerShortID
for {
if peers.byShortID[i].peer == nil {
if n == 0 {
return PeerShortID(i), true
}
n--
}
i++
}
}
func (peers *Peers) FetchWithDefault(peer *Peer) *Peer {
peers.Lock()
var pending PeersPendingNotifications
defer peers.unlockAndNotify(&pending)
if existingPeer, found := peers.byName[peer.Name]; found {
existingPeer.localRefCount++
return existingPeer
}
peers.byName[peer.Name] = peer
peers.addByShortID(peer, &pending)
peer.localRefCount++
return peer
}
func (peers *Peers) Fetch(name PeerName) *Peer {
peers.RLock()
defer peers.RUnlock()
return peers.byName[name]
}
func (peers *Peers) FetchAndAddRef(name PeerName) *Peer {
peers.Lock()
defer peers.Unlock()
peer := peers.byName[name]
if peer != nil {
peer.localRefCount++
}
return peer
}
func (peers *Peers) FetchByShortID(shortID PeerShortID) *Peer {
peers.RLock()
defer peers.RUnlock()
return peers.byShortID[shortID].peer
}
func (peers *Peers) Dereference(peer *Peer) {
peers.Lock()
defer peers.Unlock()
peer.localRefCount--
}
func (peers *Peers) ForEach(fun func(*Peer)) {
peers.RLock()
defer peers.RUnlock()
for _, peer := range peers.byName {
fun(peer)
}
}
// Merge an incoming update with our own topology.
//
// We add peers hitherto unknown to us, and update peers for which the
// update contains a more recent version than known to us. The return
// value is a) a representation of the received update, and b) an
// "improved" update containing just these new/updated elements.
func (peers *Peers) ApplyUpdate(update []byte) (PeerNameSet, PeerNameSet, error) {
peers.Lock()
var pending PeersPendingNotifications
defer peers.unlockAndNotify(&pending)
newPeers, decodedUpdate, decodedConns, err := peers.decodeUpdate(update)
if err != nil {
return nil, nil, err
}
// By this point, we know the update doesn't refer to any peers we
// have no knowledge of. We can now apply the update. Start by
// adding in any new peers.
for name, newPeer := range newPeers {
peers.byName[name] = newPeer
peers.addByShortID(newPeer, &pending)
}
// Now apply the updates
newUpdate := peers.applyUpdate(decodedUpdate, decodedConns, &pending)
peers.garbageCollect(&pending)
for _, peerRemoved := range pending.removed {
delete(newUpdate, peerRemoved.Name)
}
updateNames := make(PeerNameSet)
for _, peer := range decodedUpdate {
updateNames[peer.Name] = void
}
return updateNames, newUpdate, nil
}
func (peers *Peers) Names() PeerNameSet {
peers.RLock()
defer peers.RUnlock()
names := make(PeerNameSet)
for name := range peers.byName {
names[name] = void
}
return names
}
func (peers *Peers) EncodePeers(names PeerNameSet) []byte {
buf := new(bytes.Buffer)
enc := gob.NewEncoder(buf)
peers.RLock()
defer peers.RUnlock()
for name := range names {
if peer, found := peers.byName[name]; found {
if peer == peers.ourself.Peer {
peers.ourself.Encode(enc)
} else {
peer.Encode(enc)
}
}
}
return buf.Bytes()
}
func (peers *Peers) GarbageCollect() {
peers.Lock()
var pending PeersPendingNotifications
defer peers.unlockAndNotify(&pending)
peers.garbageCollect(&pending)
}
func (peers *Peers) garbageCollect(pending *PeersPendingNotifications) {
peers.ourself.RLock()
_, reached := peers.ourself.Routes(nil, false)
peers.ourself.RUnlock()
for name, peer := range peers.byName {
if _, found := reached[peer.Name]; !found && peer.localRefCount == 0 {
delete(peers.byName, name)
peers.deleteByShortID(peer, pending)
pending.removed = append(pending.removed, peer)
}
}
if peers.byShortID[peers.ourself.ShortID].peer != peers.ourself.Peer {
// The local peer doesn't own its short id. Garbage
// collection might have freed some up, so try to
// reassign.
pending.reassignLocalShortID = true
}
}
func (peers *Peers) decodeUpdate(update []byte) (newPeers map[PeerName]*Peer, decodedUpdate []*Peer, decodedConns [][]ConnectionSummary, err error) {
newPeers = make(map[PeerName]*Peer)
decodedUpdate = []*Peer{}
decodedConns = [][]ConnectionSummary{}
decoder := gob.NewDecoder(bytes.NewReader(update))
for {
peerSummary, connSummaries, decErr := decodePeer(decoder)
if decErr == io.EOF {
break
} else if decErr != nil {
err = decErr
return
}
newPeer := NewPeerFromSummary(peerSummary)
decodedUpdate = append(decodedUpdate, newPeer)
decodedConns = append(decodedConns, connSummaries)
if _, found := peers.byName[newPeer.Name]; !found {
newPeers[newPeer.Name] = newPeer
}
}
for _, connSummaries := range decodedConns {
for _, connSummary := range connSummaries {
remoteName := PeerNameFromBin(connSummary.NameByte)
if _, found := newPeers[remoteName]; found {
continue
}
if _, found := peers.byName[remoteName]; found {
continue
}
// Update refers to a peer which we have no knowledge
// of. Thus we can't apply the update. Abort.
err = UnknownPeerError{remoteName}
return
}
}
return
}
func (peers *Peers) applyUpdate(decodedUpdate []*Peer, decodedConns [][]ConnectionSummary, pending *PeersPendingNotifications) PeerNameSet {
newUpdate := make(PeerNameSet)
for idx, newPeer := range decodedUpdate {
connSummaries := decodedConns[idx]
name := newPeer.Name
// guaranteed to find peer in the peers.byName
switch peer := peers.byName[name]; peer {
case peers.ourself.Peer:
if newPeer.UID != peer.UID {
// The update contains information about an old
// incarnation of ourselves. We increase our version
// number beyond that which we received, so our
// information supersedes the old one when it is
// received by other peers.
pending.localPeerModified = peers.ourself.setVersionBeyond(newPeer.Version)
}
case newPeer:
peer.connections = makeConnsMap(peer, connSummaries, peers.byName)
newUpdate[name] = void
default: // existing peer
if newPeer.Version < peer.Version ||
(newPeer.Version == peer.Version && newPeer.UID <= peer.UID) {
continue
}
peer.Version = newPeer.Version
peer.UID = newPeer.UID
peer.connections = makeConnsMap(peer, connSummaries, peers.byName)
if newPeer.ShortID != peer.ShortID {
peers.deleteByShortID(peer, pending)
peer.ShortID = newPeer.ShortID
peers.addByShortID(peer, pending)
}
newUpdate[name] = void
}
}
return newUpdate
}
func (peer *Peer) Encode(enc *gob.Encoder) {
checkFatal(enc.Encode(peer.PeerSummary))
connSummaries := []ConnectionSummary{}
for _, conn := range peer.connections {
connSummaries = append(connSummaries, ConnectionSummary{
conn.Remote().NameByte,
conn.RemoteTCPAddr(),
conn.Outbound(),
conn.Established(),
})
}
checkFatal(enc.Encode(connSummaries))
}
func (peer *LocalPeer) Encode(enc *gob.Encoder) {
peer.RLock()
defer peer.RUnlock()
peer.Peer.Encode(enc)
}
func decodePeer(dec *gob.Decoder) (peerSummary PeerSummary, connSummaries []ConnectionSummary, err error) {
if err = dec.Decode(&peerSummary); err != nil {
return
}
if err = dec.Decode(&connSummaries); err != nil {
return
}
return
}
func makeConnsMap(peer *Peer, connSummaries []ConnectionSummary, byName map[PeerName]*Peer) map[PeerName]Connection {
conns := make(map[PeerName]Connection)
for _, connSummary := range connSummaries {
name := PeerNameFromBin(connSummary.NameByte)
remotePeer := byName[name]
conn := NewRemoteConnection(peer, remotePeer, connSummary.RemoteTCPAddr, connSummary.Outbound, connSummary.Established)
conns[name] = conn
}
return conns
}