Merge pull request #1120 from weaveworks/998-race

Various improvements to unit test running

test/gen_coverage_reports.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 set -ex
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 if [ -n "$CIRCLECI" ]; then
     for i in $(seq 1 $(($CIRCLE_NODE_TOTAL - 1))); do
@@ -8,7 +9,7 @@ if [ -n "$CIRCLECI" ]; then
     done
 fi
 
-go get github.com/weaveworks/weave/testing/cover
+$DIR/../testing/cover/cover
 cover ./coverage/* >profile.cov
 go tool cover -html=profile.cov -o coverage.html
 go tool cover -func=profile.cov -o coverage.txt

test/run_all.sh

@@ -29,7 +29,7 @@ TESTS="${@:-*_test.sh}"
 
 # If running on circle, use the scheduler to work out what tests to run
 if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" ]; then
-    TESTS=$(echo $TESTS | "$DIR/sched" sched $CIRCLE_BUILD_NUM $CIRCLE_NODE_TOTAL $CIRCLE_NODE_INDEX)
+    TESTS=$(echo $TESTS | "$DIR/sched" sched integration-$CIRCLE_BUILD_NUM $CIRCLE_NODE_TOTAL $CIRCLE_NODE_INDEX)
 fi
 
 echo Running $TESTS

test/sched

@@ -1,17 +1,17 @@
 #!/usr/bin/python
-import sys, string, json
+import sys, string, json, urllib
 import requests
 
 BASE_URL="http://positive-cocoa-90213.appspot.com"
 
 def test_time(test_name, runtime):
-  r = requests.post(BASE_URL + "/record/%s/%f" % (test_name, runtime))
+  r = requests.post(BASE_URL + "/record/%s/%f" % (urllib.quote(test_name, safe=""), runtime))
   print r.text
   assert r.status_code == 204
 
 def test_sched(test_run, shard_count, shard_id):
   tests = json.dumps({'tests': string.split(sys.stdin.read())})
-  r = requests.post(BASE_URL + "/schedule/%d/%d/%d" % (test_run, shard_count, shard_id), data=tests)
+  r = requests.post(BASE_URL + "/schedule/%s/%d/%d" % (test_run, shard_count, shard_id), data=tests)
   assert r.status_code == 200
   result = r.json()
   for test in sorted(result['tests']):
@@ -30,7 +30,7 @@ def main():
   if sys.argv[1] == "time":
     test_time(sys.argv[2], float(sys.argv[3]))
   elif sys.argv[1] == "sched":
-    test_sched(int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4]))
+    test_sched(sys.argv[2], int(sys.argv[3]), int(sys.argv[4]))
   else:
     usage()
 

test/scheduler/.gitignore

@@ -0,0 +1,7 @@
+*.egg-info
+*.dist-info
+flask
+jinja2
+markupsafe
+werkzeug
+itsdangerous.*

test/scheduler/README.md

@@ -0,0 +1,6 @@
+To upload newer version:
+
+```
+pip install -r requirements.txt -t .
+appcfg.py update .
+```

test/scheduler/main.py

@@ -19,7 +19,7 @@ class Test(ndb.Model):
 class Schedule(ndb.Model):
   shards = ndb.JsonProperty()
 
-@app.route('/record/<test_name>/<runtime>', methods=['POST'])
+@app.route('/record/<path:test_name>/<runtime>', methods=['POST'])
 @ndb.transactional
 def record(test_name, runtime):
   test = Test.get_by_id(test_name)
@@ -30,13 +30,13 @@ def record(test_name, runtime):
   test.put()
   return ('', 204)
 
-@app.route('/schedule/<int:test_run>/<int:shard_count>/<int:shard>', methods=['POST'])
+@app.route('/schedule/<test_run>/<int:shard_count>/<int:shard>', methods=['POST'])
 def schedule(test_run, shard_count, shard):
   # read tests from body
   test_names = flask.request.get_json(force=True)['tests']
 
   # first see if we have a scedule already
-  schedule_id = "%d-%d" % (test_run, shard_count)
+  schedule_id = "%s-%d" % (test_run, shard_count)
   schedule = Schedule.get_by_id(schedule_id)
   if schedule is not None:
     return flask.json.jsonify(tests=schedule.shards[str(shard)])

test/scheduler/requirements.txt

@@ -0,0 +1 @@
+flask

test/units.sh

@@ -1,28 +1,58 @@
 #!/bin/bash
 
-go get github.com/weaveworks/weave/testing/cover
+set -e
 
-if [ -n "$COVERDIR" ] ; then
-	coverdir="$COVERDIR"
-else
-	coverdir=$(mktemp -d coverage.XXXXXXXXXX)
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SLOW=${SLOW-}
+GO_TEST_ARGS="-tags netgo -cpu 4"
+if [ -n "$SLOW" -o "$1" = "-slow" ]; then
+    GO_TEST_ARGS="$GO_TEST_ARGS -race -covermode=atomic"
+
+    if [ -n "$COVERDIR" ] ; then
+        coverdir="$COVERDIR"
+    else
+        coverdir=$(mktemp -d coverage.XXXXXXXXXX)
+    fi
+
+    mkdir -p $coverdir
 fi
 
-mkdir -p $coverdir
 fail=0
 
-for dir in $(find . -type f -name '*_test.go' | xargs -n1 dirname | grep -v prog | sort -u); do
-	go get -t -tags netgo $dir
-	output=$(mktemp $coverdir/unit.XXXXXXXXXX)
-	if ! go test -cpu 4 -tags netgo -covermode=count -coverprofile=$output $dir ; then
-		fail=1
-	fi
+TESTDIRS=$(find . -type f -name '*_test.go' | xargs -n1 dirname | grep -v prog | sort -u)
+
+# If running on circle, use the scheduler to work out what tests to run on what shard
+if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" ]; then
+    TESTDIRS=$(echo $TESTDIRS | "$DIR/sched" sched units-$CIRCLE_BUILD_NUM $CIRCLE_NODE_TOTAL $CIRCLE_NODE_INDEX)
+    echo $TESTDIRS
+fi
+
+for dir in $TESTDIRS; do
+
+    GO_TEST_ARGS_RUN="$GO_TEST_ARGS"
+    if [ -n "$SLOW" ]; then
+        go get -t -tags netgo $dir
+        output=$(mktemp $coverdir/unit.XXXXXXXXXX)
+        GO_TEST_ARGS_RUN="$GO_TEST_ARGS -coverprofile=$output"
+    fi
+
+    START=$(date +%s)
+    if ! go test $GO_TEST_ARGS_RUN $dir ; then
+        fail=1
+    fi
+    RUNTIME=$(( $(date +%s) - $START ))
+
+    # Report test runtime when running on circle, to help scheduler
+    if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" ]; then
+        "$DIR/sched" time $dir $RUNTIME
+    fi
 done
 
-if [ -z "$COVERDIR" ] ; then
-	cover $coverdir/* >profile.cov
-	rm -rf $coverdir
-	go tool cover -html=profile.cov -o=coverage.html
+if [ -n "$SLOW" -a -z "$COVERDIR" ] ; then
+    $DIR/../testing/cover/cover $coverdir/* >profile.cov
+    rm -rf $coverdir
+    go tool cover -html=profile.cov -o=coverage.html
+    go tool cover -func=profile.cov | tail -n1
 fi
 
 exit $fail