From f5c2f519649e0f6fd34277001bcad42f7c2c4c72 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Fri, 11 Oct 2019 20:17:12 +0000 Subject: [PATCH 1/5] Use a different seed for random numbers each run Without this, each run gets the same random numbers so all nodes are racing to remove the deleted node. --- prog/kube-utils/main.go | 1 + prog/kube-utils/peerlist_test.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/prog/kube-utils/main.go b/prog/kube-utils/main.go index 0743d360..c81156e6 100644 --- a/prog/kube-utils/main.go +++ b/prog/kube-utils/main.go @@ -354,6 +354,7 @@ func main() { ch := make(chan os.Signal) signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM) stopCh := make(chan struct{}) + rand.Seed(time.Now().UnixNano()) registerForNodeUpdates(c, stopCh, nodeName, peerName) <-ch close(stopCh) diff --git a/prog/kube-utils/peerlist_test.go b/prog/kube-utils/peerlist_test.go index 093c575b..c5e47fee 100644 --- a/prog/kube-utils/peerlist_test.go +++ b/prog/kube-utils/peerlist_test.go @@ -5,6 +5,7 @@ import ( "math/rand" "sync" "testing" + "time" "github.com/stretchr/testify/require" ) @@ -71,6 +72,7 @@ func doConcurrentIterations(iterations, concurrency int, f func(int)) { } func TestPeerListFuzz(t *testing.T) { + rand.Seed(time.Now().UnixNano()) const ( testIters = 1000 maxNodes = 100 From 30ed34bd2c3a17688b0b18fa905d94751421650f Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Fri, 11 Oct 2019 16:47:53 +0000 Subject: [PATCH 2/5] Shut down Kubernetes in test of deleted node --- test/870_weave_recovers_unreachable_ips_on_relaunch_3_test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/870_weave_recovers_unreachable_ips_on_relaunch_3_test.sh b/test/870_weave_recovers_unreachable_ips_on_relaunch_3_test.sh index e205381b..3a9db15b 100755 --- a/test/870_weave_recovers_unreachable_ips_on_relaunch_3_test.sh +++ b/test/870_weave_recovers_unreachable_ips_on_relaunch_3_test.sh @@ -54,6 +54,8 @@ function setup_kubernetes_cluster { } function force_drop_node { + greyly echo "Shutting down Kubernetes on node $1" + run_on $1 "sudo kubeadm reset --force" greyly echo "Dropping node $1 with 'sudo kubectl delete node'" local target=$(echo "$1" | awk -F"." '{print $1}') run_on $HOST1 "$KUBECTL delete node $target" From 3ed7894e3968543fe9f0056b8f35d5193817e914 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Fri, 11 Oct 2019 17:07:07 +0000 Subject: [PATCH 3/5] Debug log the node-deletion event So we know when it came in, also which node. --- prog/kube-utils/main.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/prog/kube-utils/main.go b/prog/kube-utils/main.go index c81156e6..604c2903 100644 --- a/prog/kube-utils/main.go +++ b/prog/kube-utils/main.go @@ -259,6 +259,13 @@ func registerForNodeUpdates(client *kubernetes.Clientset, stopCh <-chan struct{} common.Log.Debugln("registering for updates for node delete events") nodeInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ DeleteFunc: func(obj interface{}) { + var name string + if apiObj, ok := obj.(api.Object); ok { + name = apiObj.GetName() + } else { + name = fmt.Sprintf("%#v", obj) + } + common.Log.Debugln("Delete event for", name) // add random delay to avoid all nodes acting on node delete event at the same // time leading to contention to use `weave-net` configmap r := rand.Intn(5000) From 821b78777457157f6d81d596e3352de8a3296b09 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Fri, 11 Oct 2019 20:18:12 +0000 Subject: [PATCH 4/5] Move printing of node addresses after reclaim loop Otherwise we see all the node addresses in the middle of the log. --- prog/kube-utils/main.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/prog/kube-utils/main.go b/prog/kube-utils/main.go index 604c2903..cc9bfd69 100644 --- a/prog/kube-utils/main.go +++ b/prog/kube-utils/main.go @@ -348,14 +348,6 @@ func main() { } return } - peers, err := getKubePeers(c, false) - if err != nil { - common.Log.Fatalf("[kube-peers] Could not get peers: %v", err) - } - for _, node := range peers { - fmt.Println(node.addr) - } - if runReclaimDaemon { // Handle SIGINT and SIGTERM ch := make(chan os.Signal) @@ -365,5 +357,14 @@ func main() { registerForNodeUpdates(c, stopCh, nodeName, peerName) <-ch close(stopCh) + return + } + + peers, err := getKubePeers(c, false) + if err != nil { + common.Log.Fatalf("[kube-peers] Could not get peers: %v", err) + } + for _, node := range peers { + fmt.Println(node.addr) } } From 79e032c984cbc0c63b2ee4f3ca6b4bfda832f36e Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 10 Oct 2019 13:30:16 +0000 Subject: [PATCH 5/5] Reduce time test script waits for IPs to be reclaimed Now we have a callback when a node is removed, there should be no reason to wait 90 seconds. The reclaim does wait up to 5 seconds before starting, so allowing 10 seconds in the script should be sufficient. --- test/870_weave_recovers_unreachable_ips_on_relaunch_3_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/870_weave_recovers_unreachable_ips_on_relaunch_3_test.sh b/test/870_weave_recovers_unreachable_ips_on_relaunch_3_test.sh index 3a9db15b..ea967251 100755 --- a/test/870_weave_recovers_unreachable_ips_on_relaunch_3_test.sh +++ b/test/870_weave_recovers_unreachable_ips_on_relaunch_3_test.sh @@ -86,7 +86,7 @@ function relaunch_weave_pod { # Suite # function main { - local IPAM_RECOVER_DELAY=90 + local IPAM_RECOVER_DELAY=10 start_suite "Test weave-net deallocates from IPAM on node failure";