Cope with a Kubernetes node being deleted and coming back

Add features to weaver and kube-peers to find the peer name and check
if it is in the list, then in launch.sh delete persisted data if not.

prog/kube-peers/main.go

@@ -83,6 +83,18 @@ func addMyselfToPeerList(cml *configMapAnnotations, c *kubernetes.Clientset, pee
 	return list, err
 }
 
+func checkIamInPeerList(cml *configMapAnnotations, c *kubernetes.Clientset, peerName string) (bool, error) {
+	if err := cml.Init(); err != nil {
+		return false, err
+	}
+	list, err := cml.GetPeerList()
+	if err != nil {
+		return false, err
+	}
+	common.Log.Debugf("[kube-peers] Checking peer %q against list %v", peerName, list)
+	return list.contains(peerName), nil
+}
+
 // For each of those peers that is no longer listed as a node by
 // Kubernetes, remove it from Weave IPAM
 func reclaimRemovedPeers(weave *weaveapi.Client, cml *configMapAnnotations, nodes []nodeInfo, myPeerName string) error {
@@ -164,11 +176,13 @@ func reclaimRemovedPeers(weave *weaveapi.Client, cml *configMapAnnotations, node
 func main() {
 	var (
 		justReclaim bool
+		justCheck   bool
 		peerName    string
 		nodeName    string
 		logLevel    string
 	)
 	flag.BoolVar(&justReclaim, "reclaim", false, "reclaim IP space from dead peers")
+	flag.BoolVar(&justCheck, "check-peer-exists", false, "return success if peer name is stored in annotation")
 	flag.StringVar(&peerName, "peer-name", "unknown", "name of this Weave Net peer")
 	flag.StringVar(&nodeName, "node-name", "unknown", "name of this Kubernetes node")
 	flag.StringVar(&logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
@@ -183,6 +197,18 @@ func main() {
 	if err != nil {
 		common.Log.Fatalf("[kube-peers] Could not make Kubernetes connection: %v", err)
 	}
+	if justCheck {
+		cml := newConfigMapAnnotations(configMapNamespace, configMapName, c)
+		exists, err := checkIamInPeerList(cml, c, peerName)
+		if err != nil {
+			common.Log.Fatalf("[kube-peers] Could not check peer list: %v", err)
+		}
+		if exists {
+			os.Exit(0)
+		} else {
+			os.Exit(9)
+		}
+	}
 	peers, err := getKubePeers(c)
 	if err != nil {
 		common.Log.Fatalf("[kube-peers] Could not get peers: %v", err)

prog/weave-kube/launch.sh

@@ -30,6 +30,7 @@ HTTP_ADDR=${WEAVE_HTTP_ADDR:-127.0.0.1:6784}
 STATUS_ADDR=${WEAVE_STATUS_ADDR:-0.0.0.0:6782}
 HOST_ROOT=${HOST_ROOT:-/host}
 CONN_LIMIT=${CONN_LIMIT:-30}
+DB_PREFIX=${DB_PREFIX:-/weavedb/weave-net}
 
 # Check if the IP range overlaps anything existing on the host
 /usr/bin/weaveutil netcheck $IPALLOC_RANGE weave
@@ -87,6 +88,23 @@ if [ -z "$IPALLOC_INIT" ]; then
     IPALLOC_INIT="consensus=$(peer_count $KUBE_PEERS)"
 fi
 
+# Find out what peer name we will be using
+PEERNAME=$(/home/weave/weaver $EXTRA_ARGS --print-peer-name --host-root=$HOST_ROOT --db-prefix="$DB_PREFIX")
+if [ -z "$PEERNAME" ]; then
+    echo "Unable to get peer name" >&2
+    exit 2
+fi
+
+# If this peer name is not stored in the list, either we are a
+# brand-new peer or we were removed by another peer while temporarily
+# absent. In order to avoid a CRDT clash for the latter case, clean up
+if ! /home/weave/kube-peers -check-peer-exists -peer-name="$PEERNAME" ; then
+    if [ -f ${DB_PREFIX}data.db ]; then
+        echo "Peer not in list; removing persisted data" >&2
+        rm -f ${DB_PREFIX}data.db
+    fi
+fi
+
 post_start_actions() {
     # Wait for weave process to become responsive
     while true ; do
@@ -108,7 +126,7 @@ post_start_actions() {
     export HOST_ROOT
     /home/weave/weave --local setup-cni
 
-    /home/weave/kube-peers -reclaim -node-name="$HOSTNAME" -peer-name="$(cat /sys/class/net/weave/address)"
+    /home/weave/kube-peers -reclaim -node-name="$HOSTNAME" -peer-name="$PEERNAME"
 
     # Expose the weave network so host processes can communicate with pods
     /home/weave/weave --local expose $WEAVE_EXPOSE_IP
@@ -117,9 +135,10 @@ post_start_actions() {
 post_start_actions &
 
 /home/weave/weaver $EXTRA_ARGS --port=6783 $(router_bridge_opts) \
+     --name="$PEERNAME" \
      --host-root=$HOST_ROOT \
      --http-addr=$HTTP_ADDR --status-addr=$STATUS_ADDR --docker-api='' --no-dns \
-     --db-prefix="/weavedb/weave-net" \
+     --db-prefix="$DB_PREFIX" \
      --ipalloc-range=$IPALLOC_RANGE $NICKNAME_ARG \
      --ipalloc-init $IPALLOC_INIT \
      --conn-limit=$CONN_LIMIT \

prog/weaver/main.go

@@ -131,6 +131,7 @@ func main() {
 
 	var (
 		justVersion        bool
+		justPeerName       bool
 		config             mesh.Config
 		bridgeConfig       weavenet.BridgeConfig
 		networkConfig      weave.NetworkConfig
@@ -163,6 +164,7 @@ func main() {
 	)
 
 	mflag.BoolVar(&justVersion, []string{"-version"}, false, "print version and exit")
+	mflag.BoolVar(&justPeerName, []string{"-print-peer-name"}, false, "print peer name and exit")
 	mflag.StringVar(&config.Host, []string{"-host"}, "", "router host")
 	mflag.IntVar(&config.Port, []string{"-port"}, mesh.Port, "router port")
 	mflag.IntVar(&protocolMinVersion, []string{"-min-protocol-version"}, mesh.ProtocolMinVersion, "minimum weave protocol version")
@@ -230,6 +232,11 @@ func main() {
 		fmt.Printf("weave %s\n", version)
 		os.Exit(0)
 	}
+	name := peerName(routerName, bridgeConfig.WeaveBridgeName, dbPrefix, hostRoot)
+	if justPeerName {
+		fmt.Printf("%s\n", name)
+		os.Exit(0)
+	}
 
 	peers = mflag.Args()
 	if resume && len(peers) > 0 {
@@ -282,8 +289,6 @@ func main() {
 		}
 	}
 
-	name := peerName(routerName, bridgeConfig.WeaveBridgeName, dbPrefix, hostRoot)
-
 	bridgeConfig.Mac = name.String()
 	bridgeConfig.Port = config.Port
 	bridgeType, err := weavenet.EnsureBridge(procPath, &bridgeConfig, Log)