Merge pull request #2699 from weaveworks/2647-parallelise-sanity-checks

Parallelise sanity checks in order to speed them up.

bin/circle-test-smoke

@@ -9,5 +9,6 @@ if [ -n "$TEST_AND_PUBLISH" ] ; then
     cd $SRCDIR/test
     eval $(./gce.sh hosts)
     export COVERAGE=true
+    export WEAVE_NET_SANITY_CHECKS_FILES="$CIRCLE_ARTIFACTS/weave_net_sanity_check_*.log"
     ./run_all.sh
 fi

test/sanity_check.sh

@@ -4,24 +4,68 @@
 
 set -e
 
+begin=$(date +%s)
+sanity_checks_files=${WEAVE_NET_SANITY_CHECKS_FILES:-"/tmp/weave_net_sanity_check_*.log"}
+
 whitely echo Ping each host from the other
+
+# We wrap ping and echo in a function as we want the below parallel for loop
+# to exit early in case of a failed ping, and it cannot be done concisely,
+# e.g. using a one-liner, without losing the status code for ping.
+function check_ping() {
+    local output=$(run_on $1 $PING $2)
+    local status=$?
+    echo "$output" >> "${sanity_checks_files/\*/$1}"
+    return $status
+}
+
+pids=""
 for host in $HOSTS; do
+    cat >> "${sanity_checks_files/\*/$host}" <<EOF
+# =====================================
+# Host Ping Check: $host
+# =====================================
+EOF
     for other in $HOSTS; do
-        [ $host = $other ] || run_on $host $PING $other
+        if [ "$host" != "$other" ]; then
+            check_ping "$host" "$other" &
+            pids="$pids $!"
+        fi
     done
 done
+for pid in $pids; do wait $pid; done
+
 
 whitely echo Check we can reach docker
 
+function check_docker() {
+    docker_version=$(docker_on $1 version)
+    docker_info=$(docker_on $1 info)
+    docker_weave_version=$(docker_on $1 inspect -f {{.Created}} weaveworks/weave:${WEAVE_VERSION:-latest})
+    weave_version=$(weave_on $1 version)
+    cat >> "${sanity_checks_files/\*/$1}" << EOF
+# =====================================
+# Host Version Info: $1
+# =====================================
+# docker version
+$docker_version
+
+# docker info
+$docker_info
+
+# docker inspect -f {{.Created}} weaveworks/weave:<version>
+$docker_weave_version
+
+# weave version
+$weave_version
+EOF
+}
+
+pids=""
 for host in $HOSTS; do
-    echo
-    echo Host Version Info: $host
-    echo =====================================
-    echo "# docker version"
-    docker_on $host version
-    echo "# docker info"
-    docker_on $host info
-    echo "# weave version"
-    docker_on $host inspect -f {{.Created}} weaveworks/weave:${WEAVE_VERSION:-latest}
-    weave_on $host version
+    check_docker $host &
+    pids="$pids $!"
 done
+for pid in $pids; do wait $pid; done
+
+echo "Sanity checks completed successfully in $(date -u -d @$(($(date +%s)-$begin)) +"%T")."