diff --git a/prog/kube-utils/main.go b/prog/kube-utils/main.go index ed4fad20..9e9bf33b 100644 --- a/prog/kube-utils/main.go +++ b/prog/kube-utils/main.go @@ -261,6 +261,13 @@ func registerForNodeUpdates(client *kubernetes.Clientset, stopCh <-chan struct{} common.Log.Debugln("registering for updates for node delete events") nodeInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ DeleteFunc: func(obj interface{}) { + var name string + if apiObj, ok := obj.(api.Object); ok { + name = apiObj.GetName() + } else { + name = fmt.Sprintf("%#v", obj) + } + common.Log.Debugln("Delete event for", name) // add random delay to avoid all nodes acting on node delete event at the same // time leading to contention to use `weave-net` configmap r := rand.Intn(5000) @@ -343,6 +350,18 @@ func main() { } return } + if runReclaimDaemon { + // Handle SIGINT and SIGTERM + ch := make(chan os.Signal) + signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM) + stopCh := make(chan struct{}) + rand.Seed(time.Now().UnixNano()) + registerForNodeUpdates(c, stopCh, nodeName, peerName) + <-ch + close(stopCh) + return + } + peers, err := getKubePeers(c, false) if err != nil { common.Log.Fatalf("[kube-peers] Could not get peers: %v", err) @@ -350,14 +369,4 @@ func main() { for _, node := range peers { fmt.Println(node.addr) } - - if runReclaimDaemon { - // Handle SIGINT and SIGTERM - ch := make(chan os.Signal) - signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM) - stopCh := make(chan struct{}) - registerForNodeUpdates(c, stopCh, nodeName, peerName) - <-ch - close(stopCh) - } } diff --git a/prog/kube-utils/peerlist_test.go b/prog/kube-utils/peerlist_test.go index 093c575b..c5e47fee 100644 --- a/prog/kube-utils/peerlist_test.go +++ b/prog/kube-utils/peerlist_test.go @@ -5,6 +5,7 @@ import ( "math/rand" "sync" "testing" + "time" "github.com/stretchr/testify/require" ) @@ -71,6 +72,7 @@ func doConcurrentIterations(iterations, concurrency int, f func(int)) { } func TestPeerListFuzz(t *testing.T) { + rand.Seed(time.Now().UnixNano()) const ( testIters = 1000 maxNodes = 100 diff --git a/test/870_weave_recovers_unreachable_ips_on_relaunch_3_test.sh b/test/870_weave_recovers_unreachable_ips_on_relaunch_3_test.sh index e205381b..ea967251 100755 --- a/test/870_weave_recovers_unreachable_ips_on_relaunch_3_test.sh +++ b/test/870_weave_recovers_unreachable_ips_on_relaunch_3_test.sh @@ -54,6 +54,8 @@ function setup_kubernetes_cluster { } function force_drop_node { + greyly echo "Shutting down Kubernetes on node $1" + run_on $1 "sudo kubeadm reset --force" greyly echo "Dropping node $1 with 'sudo kubectl delete node'" local target=$(echo "$1" | awk -F"." '{print $1}') run_on $HOST1 "$KUBECTL delete node $target" @@ -84,7 +86,7 @@ function relaunch_weave_pod { # Suite # function main { - local IPAM_RECOVER_DELAY=90 + local IPAM_RECOVER_DELAY=10 start_suite "Test weave-net deallocates from IPAM on node failure";