From 269786abf51b43f1d935c21c08400eeeb03c2e3e Mon Sep 17 00:00:00 2001 From: naemono Date: Wed, 18 Mar 2020 15:05:53 -0500 Subject: [PATCH 1/3] Simply run os.Exit on panic of goroutine --- prog/weave-npc/main.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/prog/weave-npc/main.go b/prog/weave-npc/main.go index 7eb987fb..efd6af13 100644 --- a/prog/weave-npc/main.go +++ b/prog/weave-npc/main.go @@ -216,8 +216,17 @@ func hCtx() context.Context { return context.Background() } +func stopOnPanicRecover(stopChan chan struct{}) { + if r := recover(); r != nil { + os.Exit(1) + } +} + func root(cmd *cobra.Command, args []string) { - var npController cache.Controller + var ( + npController cache.Controller + stopChan chan struct{} + ) common.SetLogLevel(logLevel) if nodeName == "" { @@ -257,9 +266,11 @@ func root(cmd *cobra.Command, args []string) { nsController := makeController(client.CoreV1().RESTClient(), "namespaces", &coreapi.Namespace{}, cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { + defer stopOnPanicRecover(stopChan) handleError(npc.AddNamespace(hCtx(), obj.(*coreapi.Namespace))) }, DeleteFunc: func(obj interface{}) { + defer stopOnPanicRecover(stopChan) switch obj := obj.(type) { case *coreapi.Namespace: handleError(npc.DeleteNamespace(hCtx(), obj)) @@ -271,15 +282,18 @@ func root(cmd *cobra.Command, args []string) { } }, UpdateFunc: func(old, new interface{}) { + defer stopOnPanicRecover(stopChan) handleError(npc.UpdateNamespace(hCtx(), old.(*coreapi.Namespace), new.(*coreapi.Namespace))) }}) podController := makeController(client.CoreV1().RESTClient(), "pods", &coreapi.Pod{}, cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { + defer stopOnPanicRecover(stopChan) handleError(npc.AddPod(hCtx(), obj.(*coreapi.Pod))) }, DeleteFunc: func(obj interface{}) { + defer stopOnPanicRecover(stopChan) switch obj := obj.(type) { case *coreapi.Pod: handleError(npc.DeletePod(hCtx(), obj)) @@ -291,14 +305,17 @@ func root(cmd *cobra.Command, args []string) { } }, UpdateFunc: func(old, new interface{}) { + defer stopOnPanicRecover(stopChan) handleError(npc.UpdatePod(hCtx(), old.(*coreapi.Pod), new.(*coreapi.Pod))) }}) npHandlers := cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { + defer stopOnPanicRecover(stopChan) handleError(npc.AddNetworkPolicy(hCtx(), obj)) }, DeleteFunc: func(obj interface{}) { + defer stopOnPanicRecover(stopChan) switch obj := obj.(type) { case cache.DeletedFinalStateUnknown: // We know this object has gone away, but its final state is no longer @@ -310,6 +327,7 @@ func root(cmd *cobra.Command, args []string) { } }, UpdateFunc: func(old, new interface{}) { + defer stopOnPanicRecover(stopChan) handleError(npc.UpdateNetworkPolicy(hCtx(), old, new)) }, } From 23a827088b12e7639ae1db0c3b675554f91f9172 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 4 Aug 2020 14:31:28 +0000 Subject: [PATCH 2/3] Remove unused stopChan --- prog/weave-npc/main.go | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/prog/weave-npc/main.go b/prog/weave-npc/main.go index efd6af13..79909a24 100644 --- a/prog/weave-npc/main.go +++ b/prog/weave-npc/main.go @@ -216,17 +216,14 @@ func hCtx() context.Context { return context.Background() } -func stopOnPanicRecover(stopChan chan struct{}) { +func stopOnPanicRecover() { if r := recover(); r != nil { os.Exit(1) } } func root(cmd *cobra.Command, args []string) { - var ( - npController cache.Controller - stopChan chan struct{} - ) + var npController cache.Controller common.SetLogLevel(logLevel) if nodeName == "" { @@ -266,11 +263,11 @@ func root(cmd *cobra.Command, args []string) { nsController := makeController(client.CoreV1().RESTClient(), "namespaces", &coreapi.Namespace{}, cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { - defer stopOnPanicRecover(stopChan) + defer stopOnPanicRecover() handleError(npc.AddNamespace(hCtx(), obj.(*coreapi.Namespace))) }, DeleteFunc: func(obj interface{}) { - defer stopOnPanicRecover(stopChan) + defer stopOnPanicRecover() switch obj := obj.(type) { case *coreapi.Namespace: handleError(npc.DeleteNamespace(hCtx(), obj)) @@ -282,18 +279,18 @@ func root(cmd *cobra.Command, args []string) { } }, UpdateFunc: func(old, new interface{}) { - defer stopOnPanicRecover(stopChan) + defer stopOnPanicRecover() handleError(npc.UpdateNamespace(hCtx(), old.(*coreapi.Namespace), new.(*coreapi.Namespace))) }}) podController := makeController(client.CoreV1().RESTClient(), "pods", &coreapi.Pod{}, cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { - defer stopOnPanicRecover(stopChan) + defer stopOnPanicRecover() handleError(npc.AddPod(hCtx(), obj.(*coreapi.Pod))) }, DeleteFunc: func(obj interface{}) { - defer stopOnPanicRecover(stopChan) + defer stopOnPanicRecover() switch obj := obj.(type) { case *coreapi.Pod: handleError(npc.DeletePod(hCtx(), obj)) @@ -305,17 +302,17 @@ func root(cmd *cobra.Command, args []string) { } }, UpdateFunc: func(old, new interface{}) { - defer stopOnPanicRecover(stopChan) + defer stopOnPanicRecover() handleError(npc.UpdatePod(hCtx(), old.(*coreapi.Pod), new.(*coreapi.Pod))) }}) npHandlers := cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { - defer stopOnPanicRecover(stopChan) + defer stopOnPanicRecover() handleError(npc.AddNetworkPolicy(hCtx(), obj)) }, DeleteFunc: func(obj interface{}) { - defer stopOnPanicRecover(stopChan) + defer stopOnPanicRecover() switch obj := obj.(type) { case cache.DeletedFinalStateUnknown: // We know this object has gone away, but its final state is no longer @@ -327,7 +324,7 @@ func root(cmd *cobra.Command, args []string) { } }, UpdateFunc: func(old, new interface{}) { - defer stopOnPanicRecover(stopChan) + defer stopOnPanicRecover() handleError(npc.UpdateNetworkPolicy(hCtx(), old, new)) }, } From 4795ba58077c240bd3555ac73903420c36127880 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 4 Aug 2020 14:36:27 +0000 Subject: [PATCH 3/3] Show stack-trace from panic in weave-npc Rather than exiting silently, fetch the stack trace and print that for the user to understand what happened. Exit via existing fatal-error code. --- prog/weave-npc/main.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/prog/weave-npc/main.go b/prog/weave-npc/main.go index 79909a24..ba307bb5 100644 --- a/prog/weave-npc/main.go +++ b/prog/weave-npc/main.go @@ -2,8 +2,10 @@ package main import ( "context" + "fmt" "os" "os/signal" + goruntime "runtime" "syscall" "github.com/coreos/go-iptables/iptables" @@ -218,7 +220,9 @@ func hCtx() context.Context { func stopOnPanicRecover() { if r := recover(); r != nil { - os.Exit(1) + buf := make([]byte, 1<<20) + stacklen := goruntime.Stack(buf, false) + handleFatal(fmt.Errorf("panic: %v \n%s", r, buf[:stacklen])) } }