Merge pull request #1941 from weaveworks/issues/1873-check-running-master

Improve visibility of startup problems

Fixes #1873.

net/listen.go

@@ -0,0 +1,18 @@
+package net
+
+import (
+	"fmt"
+	"net"
+	"os"
+)
+
+func ListenUnixSocket(pathname string) (net.Listener, error) {
+	if err := os.Remove(pathname); err != nil && !os.IsNotExist(err) {
+		return nil, fmt.Errorf("Error removing existing unix socket: %s", err)
+	}
+	listener, err := net.Listen("unix", pathname)
+	if err != nil {
+		return nil, fmt.Errorf("ListenUnixSocket failed: %s", err)
+	}
+	return listener, nil
+}

prog/plugin/main.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"net"
+	"net/http"
 	"os"
 	"os/signal"
 	"syscall"
@@ -12,6 +13,7 @@ import (
 	weaveapi "github.com/weaveworks/weave/api"
 	. "github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/common/docker"
+	weavenet "github.com/weaveworks/weave/net"
 	ipamplugin "github.com/weaveworks/weave/plugin/ipam"
 	netplugin "github.com/weaveworks/weave/plugin/net"
 	"github.com/weaveworks/weave/plugin/skel"
@@ -79,6 +81,11 @@ func run(dockerClient *docker.Client, weave *weaveapi.Client, address, meshAddre
 		defer meshListener.Close()
 	}
 
+	statusListener, err := weavenet.ListenUnixSocket("/home/weave/status.sock")
+	if err != nil {
+		return err
+	}
+	go serveStatus(statusListener)
 	sigChan := make(chan os.Signal, 1)
 	signal.Notify(sigChan, os.Interrupt, os.Kill, syscall.SIGTERM)
 
@@ -102,13 +109,7 @@ func listenAndServe(dockerClient *docker.Client, weave *weaveapi.Client, address
 		i = ipamplugin.NewIpam(weave)
 	}
 
-	var listener net.Listener
-
-	// remove sockets from last invocation
-	if err := os.Remove(address); err != nil && !os.IsNotExist(err) {
-		return nil, err
-	}
-	listener, err = net.Listen("unix", address)
+	listener, err := weavenet.ListenUnixSocket(address)
 	if err != nil {
 		return nil, err
 	}
@@ -120,3 +121,12 @@ func listenAndServe(dockerClient *docker.Client, weave *weaveapi.Client, address
 
 	return listener, nil
 }
+
+func serveStatus(listener net.Listener) {
+	server := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintln(w, "ok")
+	})}
+	if err := server.Serve(listener); err != nil {
+		Log.Fatalf("ListenAndServeStatus failed: %s", err)
+	}
+}

proxy/proxy.go

@@ -19,6 +19,7 @@ import (
 	weaveapi "github.com/weaveworks/weave/api"
 	. "github.com/weaveworks/weave/common"
 	weavedocker "github.com/weaveworks/weave/common/docker"
+	weavenet "github.com/weaveworks/weave/net"
 )
 
 const (
@@ -297,10 +298,7 @@ func (proxy *Proxy) Serve(listeners []net.Listener) {
 }
 
 func (proxy *Proxy) ListenAndServeStatus(socket string) {
-	if err := os.Remove(socket); err != nil && !os.IsNotExist(err) {
-		Log.Fatalf("Error removing existing status socket: %s", err)
-	}
-	listener, err := net.Listen("unix", socket)
+	listener, err := weavenet.ListenUnixSocket(socket)
 	if err != nil {
 		Log.Fatalf("ListenAndServeStatus failed: %s", err)
 	}

weave

@@ -943,23 +943,14 @@ stop() {
 }
 
 # Given a container name or short ID in $1, ensure the specified
-# container is running and then print its full ID to stdout. If
-# it's not running or doesn't exist, print an error to stderr and
+# container exists and then print its full ID to stdout. If
+# it doesn't exist, print an error to stderr and
 # return with an indicative non-zero exit code.
 container_id() {
-    if ! status=$(docker inspect --format='{{.State.Running}} {{.Id}}' $1 2>/dev/null) ; then
+    if ! docker inspect --format='{{.Id}}' $1 2>/dev/null ; then
         echo "Error: No such container: $1" >&2
         return 1
     fi
-    case "$status" in
-        true*)
-            echo "${status#true }"
-            ;;
-        *)
-            echo "$1 container is not running." >&2
-            return 2
-            ;;
-    esac
 }
 
 http_call() {
@@ -1002,7 +993,8 @@ wait_for_status() {
     shift
     while true ; do
         "$@" GET /status >/dev/null 2>&1 && return 0
-        if ! container_id $container >/dev/null 2>&1 ; then
+        if ! check_running $container >/dev/null 2>&1 ; then
+            docker kill $container >/dev/null 2>&1 || true # stop it restarting
             echo $(death_msg $container) >&2
             return 1
         fi
@@ -1157,10 +1149,13 @@ rewrite_etc_hosts() {
 # Print an error to stderr and return with an indicative exit status
 # if the container $1 does not exist or isn't running.
 check_running() {
-    if ! STATUS=$(docker inspect --format='{{.State.Running}}' $1 2>/dev/null) ; then
+    if ! STATUS=$(docker inspect --format='{{.State.Running}} {{.State.Restarting}}' $1 2>/dev/null) ; then
         echo  "$1 container is not present. Have you launched it?" >&2
         return 1
-    elif [ "$STATUS" != "true" ] ; then
+    elif [ "$STATUS" == "true true" ] ; then
+        echo "$1 container is restarting." >&2
+        return 2
+    elif [ "$STATUS" != "true false" ] ; then
         echo "$1 container is not running." >&2
         return 2
     fi
@@ -1234,6 +1229,7 @@ collect_dns_add_remove_args() {
     [ $# -eq 0 -a \( -n "$C" -o \( $IP_COUNT -gt 0 -a -n "$FQDN" \) \) ] || usage
     check_running $CONTAINER_NAME
     if [ -n "$C" ] ; then
+        check_running $C
         CONTAINER=$(container_id $C)
         [ $IP_COUNT -gt 0 ] || IP_ARGS=$(with_container_addresses echo_ips $CONTAINER)
     fi
@@ -1665,6 +1661,7 @@ launch_plugin_if_not_running() {
         $PLUGIN_IMAGE "$@") ; then
         return 1
     fi
+    wait_for_status $PLUGIN_CONTAINER_NAME http_call_unix $PLUGIN_CONTAINER_NAME status.sock
     util_op create-plugin-network weave weavemesh
 }