Merge pull request #3716 from weaveworks/870-flake

Randomise wait to reclaim IPs on node deletion, and fix flaky test 870

prog/kube-utils/main.go

@@ -261,6 +261,13 @@ func registerForNodeUpdates(client *kubernetes.Clientset, stopCh <-chan struct{}
 	common.Log.Debugln("registering for updates for node delete events")
 	nodeInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		DeleteFunc: func(obj interface{}) {
+			var name string
+			if apiObj, ok := obj.(api.Object); ok {
+				name = apiObj.GetName()
+			} else {
+				name = fmt.Sprintf("%#v", obj)
+			}
+			common.Log.Debugln("Delete event for", name)
 			// add random delay to avoid all nodes acting on node delete event at the same
 			// time leading to contention to use `weave-net` configmap
 			r := rand.Intn(5000)
@@ -343,6 +350,18 @@ func main() {
 		}
 		return
 	}
+	if runReclaimDaemon {
+		// Handle SIGINT and SIGTERM
+		ch := make(chan os.Signal)
+		signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
+		stopCh := make(chan struct{})
+		rand.Seed(time.Now().UnixNano())
+		registerForNodeUpdates(c, stopCh, nodeName, peerName)
+		<-ch
+		close(stopCh)
+		return
+	}
+
 	peers, err := getKubePeers(c, false)
 	if err != nil {
 		common.Log.Fatalf("[kube-peers] Could not get peers: %v", err)
@@ -350,14 +369,4 @@ func main() {
 	for _, node := range peers {
 		fmt.Println(node.addr)
 	}
-
-	if runReclaimDaemon {
-		// Handle SIGINT and SIGTERM
-		ch := make(chan os.Signal)
-		signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
-		stopCh := make(chan struct{})
-		registerForNodeUpdates(c, stopCh, nodeName, peerName)
-		<-ch
-		close(stopCh)
-	}
 }

prog/kube-utils/peerlist_test.go

@@ -5,6 +5,7 @@ import (
 	"math/rand"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 )
@@ -71,6 +72,7 @@ func doConcurrentIterations(iterations, concurrency int, f func(int)) {
 }
 
 func TestPeerListFuzz(t *testing.T) {
+	rand.Seed(time.Now().UnixNano())
 	const (
 		testIters   = 1000
 		maxNodes    = 100

test/870_weave_recovers_unreachable_ips_on_relaunch_3_test.sh

@@ -54,6 +54,8 @@ function setup_kubernetes_cluster {
 }
 
 function force_drop_node {
+    greyly echo "Shutting down Kubernetes on node $1"
+    run_on $1 "sudo kubeadm reset --force"
     greyly echo "Dropping node $1 with 'sudo kubectl delete node'"
     local target=$(echo "$1" | awk -F"." '{print $1}')
     run_on $HOST1 "$KUBECTL delete node $target"
@@ -84,7 +86,7 @@ function relaunch_weave_pod {
 # Suite
 #
 function main {
-    local IPAM_RECOVER_DELAY=90
+    local IPAM_RECOVER_DELAY=10
 
     start_suite "Test weave-net deallocates from IPAM on node failure";