Various improvements to unit test running

- units.sh should exit on error.
- Add race detector to unit test runner.
- Use local testing/cover/cover tool in gen_coverage_report.sh and units.sh
- Use the scheduler to parallelise the unit tests.
- Make test run id a string and differentiate between unit runs and integration runs in the scheduler.
- Pipe circle env vars through to build container so we can detect when to use the scheduler
- Add python requests to the weave build container.
- Quote test names in urls, make flask handle url segments with slashes in.

.gitignore

@@ -36,6 +36,7 @@ prog/weaveproxy/weaveproxy
 prog/sigproxy/sigproxy
 prog/weavewait/weavewait
 prog/netcheck/netcheck
+testing/cover/cover
 tools/bin
 tools/build
 releases

Makefile

@@ -15,8 +15,9 @@ WEAVEPROXY_EXE=prog/weaveproxy/weaveproxy
 SIGPROXY_EXE=prog/sigproxy/sigproxy
 WEAVEWAIT_EXE=prog/weavewait/weavewait
 NETCHECK_EXE=prog/netcheck/netcheck
+COVER_EXE=testing/cover/cover
 
-EXES=$(WEAVER_EXE) $(WEAVEDNS_EXE) $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(NETCHECK_EXE)
+EXES=$(WEAVER_EXE) $(WEAVEDNS_EXE) $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(NETCHECK_EXE) $(COVER_EXE)
 
 WEAVER_UPTODATE=.weaver.uptodate
 WEAVEDNS_UPTODATE=.weavedns.uptodate
@@ -75,11 +76,12 @@ $(NETCHECK_EXE): prog/netcheck/netcheck.go
 
 # Sigproxy and weavewait need separate rules as they fail the netgo check in
 # the main build stanza due to not importing net package
-
 $(SIGPROXY_EXE): prog/sigproxy/main.go
-	go build -o $@ ./$(@D)
-
 $(WEAVEWAIT_EXE): prog/weavewait/main.go
+$(COVER_EXE): testing/cover/cover.go
+
+$(WEAVEWAIT_EXE) $(SIGPROXY_EXE) $(COVER_EXE):
+	go get ./$(@D)
 	go build -o $@ ./$(@D)
 
 $(WEAVER_UPTODATE): prog/weaver/Dockerfile $(WEAVER_EXE)
@@ -106,7 +108,7 @@ $(WEAVE_EXPORT): $(IMAGES_UPTODATE)
 $(DOCKER_DISTRIB):
 	curl -o $(DOCKER_DISTRIB) $(DOCKER_DISTRIB_URL)
 
-tests:
+tests: $(COVER_EXE)
 	@test/units.sh
 
 $(PUBLISH): publish_%:

build/Dockerfile

@@ -3,7 +3,7 @@ FROM ubuntu
 RUN apt-get -y update && apt-get -y install --no-install-recommends ca-certificates apt-transport-https
 RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9
 RUN echo deb https://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list
-RUN apt-get -y update && apt-get -y install --no-install-recommends build-essential git lxc-docker-1.3.1 mercurial libpcap-dev curl make pkg-config gcc bison flex
+RUN apt-get -y update && apt-get -y install --no-install-recommends build-essential git lxc-docker-1.3.1 mercurial libpcap-dev curl make pkg-config gcc bison flex python-requests
 
 # When doing a build in a container, "apt-get update" happens twice,
 # which can be a very significant overhead for incremental builds.

circle.yml

@@ -35,7 +35,7 @@ dependencies:
 
 test:
   override:
-    - test "$CIRCLE_NODE_INDEX" != "0" || (docker run -v /var/run/docker.sock:/run/docker.sock -v /home/ubuntu:/home/go -e "COVERDIR=test/coverage" weaveworks/weave-build tests):
+    - docker run -v /var/run/docker.sock:/run/docker.sock -v /home/ubuntu:/home/go -e COVERDIR=test/coverage -e SLOW=true -e CIRCLECI -e CIRCLE_BUILD_NUM -e CIRCLE_NODE_TOTAL -e CIRCLE_NODE_INDEX weaveworks/weave-build tests:
         parallel: true
     - docker run -v /var/run/docker.sock:/run/docker.sock -v /home/ubuntu:/home/go -e COVERAGE=true weaveworks/weave-build:
         parallel: true
@@ -54,6 +54,7 @@ teardown:
   pre:
     - test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR/test; ./gen_coverage_reports.sh):
         parallel: true
-    - test "$CIRCLE_NODE_INDEX" != "0" || (go get github.com/mattn/goveralls && goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/test/profile.cov -service=circleci || true)
+    - test "$CIRCLE_NODE_INDEX" != "0" || (go get github.com/mattn/goveralls && goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/test/profile.cov -service=circleci || true):
+        parallel: true
     - test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR/test; cp coverage.* $CIRCLE_ARTIFACTS):
         parallel: true

test/gen_coverage_reports.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 set -ex
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 if [ -n "$CIRCLECI" ]; then
     for i in $(seq 1 $(($CIRCLE_NODE_TOTAL - 1))); do
@@ -8,7 +9,7 @@ if [ -n "$CIRCLECI" ]; then
     done
 fi
 
-go get github.com/weaveworks/weave/testing/cover
+$DIR/../testing/cover/cover
 cover ./coverage/* >profile.cov
 go tool cover -html=profile.cov -o coverage.html
 go tool cover -func=profile.cov -o coverage.txt

test/run_all.sh

@@ -29,7 +29,7 @@ TESTS="${@:-*_test.sh}"
 
 # If running on circle, use the scheduler to work out what tests to run
 if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" ]; then
-    TESTS=$(echo $TESTS | "$DIR/sched" sched $CIRCLE_BUILD_NUM $CIRCLE_NODE_TOTAL $CIRCLE_NODE_INDEX)
+    TESTS=$(echo $TESTS | "$DIR/sched" sched integration-$CIRCLE_BUILD_NUM $CIRCLE_NODE_TOTAL $CIRCLE_NODE_INDEX)
 fi
 
 echo Running $TESTS

test/sched

@@ -1,17 +1,17 @@
 #!/usr/bin/python
-import sys, string, json
+import sys, string, json, urllib
 import requests
 
 BASE_URL="http://positive-cocoa-90213.appspot.com"
 
 def test_time(test_name, runtime):
-  r = requests.post(BASE_URL + "/record/%s/%f" % (test_name, runtime))
+  r = requests.post(BASE_URL + "/record/%s/%f" % (urllib.quote(test_name, safe=""), runtime))
   print r.text
   assert r.status_code == 204
 
 def test_sched(test_run, shard_count, shard_id):
   tests = json.dumps({'tests': string.split(sys.stdin.read())})
-  r = requests.post(BASE_URL + "/schedule/%d/%d/%d" % (test_run, shard_count, shard_id), data=tests)
+  r = requests.post(BASE_URL + "/schedule/%s/%d/%d" % (test_run, shard_count, shard_id), data=tests)
   assert r.status_code == 200
   result = r.json()
   for test in sorted(result['tests']):
@@ -30,7 +30,7 @@ def main():
   if sys.argv[1] == "time":
     test_time(sys.argv[2], float(sys.argv[3]))
   elif sys.argv[1] == "sched":
-    test_sched(int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4]))
+    test_sched(sys.argv[2], int(sys.argv[3]), int(sys.argv[4]))
   else:
     usage()
 

test/scheduler/.gitignore

@@ -0,0 +1,7 @@
+*.egg-info
+*.dist-info
+flask
+jinja2
+markupsafe
+werkzeug
+itsdangerous.*

test/scheduler/README.md

@@ -0,0 +1,6 @@
+To upload newer version:
+
+```
+pip install -r requirements.txt -t .
+appcfg.py update .
+```

test/scheduler/main.py

@@ -19,7 +19,7 @@ class Test(ndb.Model):
 class Schedule(ndb.Model):
   shards = ndb.JsonProperty()
 
-@app.route('/record/<test_name>/<runtime>', methods=['POST'])
+@app.route('/record/<path:test_name>/<runtime>', methods=['POST'])
 @ndb.transactional
 def record(test_name, runtime):
   test = Test.get_by_id(test_name)
@@ -30,13 +30,13 @@ def record(test_name, runtime):
   test.put()
   return ('', 204)
 
-@app.route('/schedule/<int:test_run>/<int:shard_count>/<int:shard>', methods=['POST'])
+@app.route('/schedule/<test_run>/<int:shard_count>/<int:shard>', methods=['POST'])
 def schedule(test_run, shard_count, shard):
   # read tests from body
   test_names = flask.request.get_json(force=True)['tests']
 
   # first see if we have a scedule already
-  schedule_id = "%d-%d" % (test_run, shard_count)
+  schedule_id = "%s-%d" % (test_run, shard_count)
   schedule = Schedule.get_by_id(schedule_id)
   if schedule is not None:
     return flask.json.jsonify(tests=schedule.shards[str(shard)])

test/scheduler/requirements.txt

@@ -0,0 +1 @@
+flask

test/units.sh

@@ -1,6 +1,12 @@
 #!/bin/bash
 
-go get github.com/weaveworks/weave/testing/cover
+set -e
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SLOW=${SLOW-}
+GO_TEST_ARGS="-tags netgo -cpu 4"
+if [ -n "$SLOW" -o "$1" = "-slow" ]; then
+    GO_TEST_ARGS="$GO_TEST_ARGS -race -covermode=atomic"
 
     if [ -n "$COVERDIR" ] ; then
         coverdir="$COVERDIR"
@@ -9,20 +15,44 @@ else
     fi
 
     mkdir -p $coverdir
+fi
+
 fail=0
 
-for dir in $(find . -type f -name '*_test.go' | xargs -n1 dirname | grep -v prog | sort -u); do
+TESTDIRS=$(find . -type f -name '*_test.go' | xargs -n1 dirname | grep -v prog | sort -u)
+
+# If running on circle, use the scheduler to work out what tests to run on what shard
+if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" ]; then
+    TESTDIRS=$(echo $TESTDIRS | "$DIR/sched" sched units-$CIRCLE_BUILD_NUM $CIRCLE_NODE_TOTAL $CIRCLE_NODE_INDEX)
+    echo $TESTDIRS
+fi
+
+for dir in $TESTDIRS; do
+
+    GO_TEST_ARGS_RUN="$GO_TEST_ARGS"
+    if [ -n "$SLOW" ]; then
         go get -t -tags netgo $dir
         output=$(mktemp $coverdir/unit.XXXXXXXXXX)
-	if ! go test -cpu 4 -tags netgo -covermode=count -coverprofile=$output $dir ; then
+        GO_TEST_ARGS_RUN="$GO_TEST_ARGS -coverprofile=$output"
+    fi
+
+    START=$(date +%s)
+    if ! go test $GO_TEST_ARGS_RUN $dir ; then
         fail=1
     fi
+    RUNTIME=$(( $(date +%s) - $START ))
+
+    # Report test runtime when running on circle, to help scheduler
+    if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" ]; then
+        "$DIR/sched" time $dir $RUNTIME
+    fi
 done
 
-if [ -z "$COVERDIR" ] ; then
-	cover $coverdir/* >profile.cov
+if [ -n "$SLOW" -a -z "$COVERDIR" ] ; then
+    $DIR/../testing/cover/cover $coverdir/* >profile.cov
     rm -rf $coverdir
     go tool cover -html=profile.cov -o=coverage.html
+    go tool cover -func=profile.cov | tail -n1
 fi
 
 exit $fail