Merge pull request #2565 from weaveworks/2537-fix-status-on-version-check-failures

Improve 'status' to handle cases when the checkpoint API is not available

prog/weaver/checkpoint.go

@@ -10,6 +10,7 @@ import (
 
 var checker *checkpoint.Checker
 var newVersion atomic.Value
+var success atomic.Value
 
 const (
 	updateCheckPeriod = 6 * time.Hour
@@ -17,9 +18,11 @@ const (
 
 func checkForUpdates(dockerVersion string, network string) {
 	newVersion.Store("")
+	success.Store(true)
 
 	handleResponse := func(r *checkpoint.CheckResponse, err error) {
 		if err != nil {
+			success.Store(false)
 			Log.Printf("Error checking version: %v", err)
 			return
 		}

prog/weaver/http.go

@@ -240,6 +240,7 @@ var ipamTemplate = defTemplate("ipamTemplate", `{{printIPAMRanges .Router .IPAM}
 
 type VersionCheck struct {
 	Enabled     bool
+	Success     bool
 	NewVersion  string
 	NextCheckAt time.Time
 }
@@ -251,6 +252,7 @@ func versionCheck() *VersionCheck {
 	}
 
 	v.Enabled = true
+	v.Success = success.Load().(bool)
 	v.NewVersion = newVersion.Load().(string)
 	v.NextCheckAt = checker.NextCheckAt()
 
@@ -261,12 +263,12 @@ func (v *VersionCheck) String() string {
 	switch {
 	case !v.Enabled:
 		return "version check update disabled"
+	case !v.Success:
+		return fmt.Sprintf("failed to check latest version - see logs; next check at %s", v.NextCheckAt.Format("2006/01/02 15:04:05"))
 	case v.NewVersion != "":
-		return fmt.Sprintf("version %s available - please upgrade!",
-			v.NewVersion)
+		return fmt.Sprintf("version %s available - please upgrade!", v.NewVersion)
 	default:
-		return fmt.Sprintf("up to date; next check at %s",
-			v.NextCheckAt.Format("2006/01/02 15:04:05"))
+		return fmt.Sprintf("up to date; next check at %s", v.NextCheckAt.Format("2006/01/02 15:04:05"))
 	}
 }
 

test/700_status_and_report_test.sh

@@ -3,9 +3,13 @@
 . ./config.sh
 
 UNIVERSE=10.2.0.0/16
+IPTABLES_BACKUP=/tmp/700_status_and_report_test.iptables
+CHECKPOINT_URL=https://checkpoint-api.weave.works/v1/check/weave-net
 
 start_suite "weave status/report"
 
+
+
 weave_on $HOST1 launch --ipalloc-range $UNIVERSE --name 8a:3e:3e:3e:3e:3e --nickname nicknamington
 
 check() {
@@ -29,10 +33,43 @@ assert "weave_on $HOST1 status connections | tr -s ' ' | cut -d ' ' -f 2" "10.2.
 assert "weave_on $HOST1 report -f '{{.VersionCheck.Enabled}}'" "false"
 assert_raises "weave_on $HOST1 status | grep 'version check update disabled'"
 
+
+
+# Block outgoing traffic to Weave's checkpoint API, to prevent retrieval of latest version:
+run_on $HOST1 "sudo iptables-save > $IPTABLES_BACKUP"
+[ -z "$DEBUG" ] || greyly echo "Backed iptables rules up:"
+[ -z "$DEBUG" ] || run_on $HOST1 cat $IPTABLES_BACKUP
+[ -z "$DEBUG" ] || greyly echo "Updating iptables rules."
+run_on $HOST1 sudo iptables -A OUTPUT -p tcp -d checkpoint-api.weave.works --dport 443 -j REJECT --reject-with tcp-reset
+[ -z "$DEBUG" ] || greyly echo "Updated iptables rules:"
+[ -z "$DEBUG" ] || run_on $HOST1 sudo iptables -L
+[ -z "$DEBUG" ] || greyly echo "Test access to checkpoint:"
+[ -z "$DEBUG" ] || run_on $HOST1 "curl -X GET $CHECKPOINT_URL || true"
+
+weave_on $HOST1 reset
+
+CHECKPOINT_DISABLE="" weave_on $HOST1 launch
+
+assert "weave_on $HOST1 report -f '{{.VersionCheck.Enabled}}'" "true"
+assert "weave_on $HOST1 report -f '{{.VersionCheck.Success}}'" "false"
+assert "weave_on $HOST1 report -f '{{.VersionCheck.NewVersion}}'" ""
+assert_raises "weave_on $HOST1 status | grep 'failed to check latest version - see logs; next check at'"
+
+
+[ -z "$DEBUG" ] || greyly echo "Reverting iptables rules."
+run_on $HOST1 "sudo iptables-restore < $IPTABLES_BACKUP"
+[ -z "$DEBUG" ] || greyly echo "Reverted iptables rules:"
+[ -z "$DEBUG" ] || run_on $HOST1 sudo iptables -L
+[ -z "$DEBUG" ] || greyly echo "Test access to checkpoint:"
+[ -z "$DEBUG" ] || run_on $HOST1 curl -X GET $CHECKPOINT_URL
+
+
+
 weave_on $HOST1 reset
 
 CHECKPOINT_DISABLE="" weave_on $HOST1 launch
 assert "weave_on $HOST1 report -f '{{.VersionCheck.Enabled}}'" "true"
+assert "weave_on $HOST1 report -f '{{.VersionCheck.Success}}'" "true"
 
 NEW_VSN=$(weave_on $HOST1 report  -f "{{.VersionCheck.NewVersion}}")
 if [ -z "$NEW_VSN" ]; then
@@ -41,4 +78,6 @@ else
     assert_raises "weave_on $HOST1 status | grep \"version $NEW_VSN available - please upgrade!\""
 fi
 
+
+
 end_suite