Various improvements to unit test running
- units.sh should exit on error. - Add race detector to unit test runner. - Use local testing/cover/cover tool in gen_coverage_report.sh and units.sh - Use the scheduler to parallelise the unit tests. - Make test run id a string and differentiate between unit runs and integration runs in the scheduler. - Pipe circle env vars through to build container so we can detect when to use the scheduler - Add python requests to the weave build container. - Quote test names in urls, make flask handle url segments with slashes in.
This commit is contained in:
Tom Wilkie committed
1 parent
7aab5cc5b4
commit
13f1239a3e
12 files changed
+82
-33
No files matched your search
@@ -36,6 +36,7 @@ prog/weaveproxy/weaveproxy
|
||||
prog/sigproxy/sigproxy
|
||||
prog/weavewait/weavewait
|
||||
prog/netcheck/netcheck
|
||||
testing/cover/cover
|
||||
tools/bin
|
||||
tools/build
|
||||
releases
|
||||
|
||||
@@ -15,8 +15,9 @@ WEAVEPROXY_EXE=prog/weaveproxy/weaveproxy
|
||||
SIGPROXY_EXE=prog/sigproxy/sigproxy
|
||||
WEAVEWAIT_EXE=prog/weavewait/weavewait
|
||||
NETCHECK_EXE=prog/netcheck/netcheck
|
||||
COVER_EXE=testing/cover/cover
|
||||
|
||||
EXES=$(WEAVER_EXE) $(WEAVEDNS_EXE) $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(NETCHECK_EXE)
|
||||
EXES=$(WEAVER_EXE) $(WEAVEDNS_EXE) $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(NETCHECK_EXE) $(COVER_EXE)
|
||||
|
||||
WEAVER_UPTODATE=.weaver.uptodate
|
||||
WEAVEDNS_UPTODATE=.weavedns.uptodate
|
||||
@@ -75,11 +76,12 @@ $(NETCHECK_EXE): prog/netcheck/netcheck.go
|
||||
|
||||
# Sigproxy and weavewait need separate rules as they fail the netgo check in
|
||||
# the main build stanza due to not importing net package
|
||||
|
||||
$(SIGPROXY_EXE): prog/sigproxy/main.go
|
||||
go build -o $@ ./$(@D)
|
||||
|
||||
$(WEAVEWAIT_EXE): prog/weavewait/main.go
|
||||
$(COVER_EXE): testing/cover/cover.go
|
||||
|
||||
$(WEAVEWAIT_EXE) $(SIGPROXY_EXE) $(COVER_EXE):
|
||||
go get ./$(@D)
|
||||
go build -o $@ ./$(@D)
|
||||
|
||||
$(WEAVER_UPTODATE): prog/weaver/Dockerfile $(WEAVER_EXE)
|
||||
@@ -106,7 +108,7 @@ $(WEAVE_EXPORT): $(IMAGES_UPTODATE)
|
||||
$(DOCKER_DISTRIB):
|
||||
curl -o $(DOCKER_DISTRIB) $(DOCKER_DISTRIB_URL)
|
||||
|
||||
tests:
|
||||
tests: $(COVER_EXE)
|
||||
@test/units.sh
|
||||
|
||||
$(PUBLISH): publish_%:
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@ FROM ubuntu
|
||||
RUN apt-get -y update && apt-get -y install --no-install-recommends ca-certificates apt-transport-https
|
||||
RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9
|
||||
RUN echo deb https://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list
|
||||
RUN apt-get -y update && apt-get -y install --no-install-recommends build-essential git lxc-docker-1.3.1 mercurial libpcap-dev curl make pkg-config gcc bison flex
|
||||
RUN apt-get -y update && apt-get -y install --no-install-recommends build-essential git lxc-docker-1.3.1 mercurial libpcap-dev curl make pkg-config gcc bison flex python-requests
|
||||
|
||||
# When doing a build in a container, "apt-get update" happens twice,
|
||||
# which can be a very significant overhead for incremental builds.
|
||||
|
||||
+3
-2
@@ -35,7 +35,7 @@ dependencies:
|
||||
|
||||
test:
|
||||
override:
|
||||
- test "$CIRCLE_NODE_INDEX" != "0" || (docker run -v /var/run/docker.sock:/run/docker.sock -v /home/ubuntu:/home/go -e "COVERDIR=test/coverage" weaveworks/weave-build tests):
|
||||
- docker run -v /var/run/docker.sock:/run/docker.sock -v /home/ubuntu:/home/go -e COVERDIR=test/coverage -e SLOW=true -e CIRCLECI -e CIRCLE_BUILD_NUM -e CIRCLE_NODE_TOTAL -e CIRCLE_NODE_INDEX weaveworks/weave-build tests:
|
||||
parallel: true
|
||||
- docker run -v /var/run/docker.sock:/run/docker.sock -v /home/ubuntu:/home/go -e COVERAGE=true weaveworks/weave-build:
|
||||
parallel: true
|
||||
@@ -54,6 +54,7 @@ teardown:
|
||||
pre:
|
||||
- test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR/test; ./gen_coverage_reports.sh):
|
||||
parallel: true
|
||||
- test "$CIRCLE_NODE_INDEX" != "0" || (go get github.com/mattn/goveralls && goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/test/profile.cov -service=circleci || true)
|
||||
- test "$CIRCLE_NODE_INDEX" != "0" || (go get github.com/mattn/goveralls && goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/test/profile.cov -service=circleci || true):
|
||||
parallel: true
|
||||
- test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR/test; cp coverage.* $CIRCLE_ARTIFACTS):
|
||||
parallel: true
|
||||
@@ -1,6 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
if [ -n "$CIRCLECI" ]; then
|
||||
for i in $(seq 1 $(($CIRCLE_NODE_TOTAL - 1))); do
|
||||
@@ -8,7 +9,7 @@ if [ -n "$CIRCLECI" ]; then
|
||||
done
|
||||
fi
|
||||
|
||||
go get github.com/weaveworks/weave/testing/cover
|
||||
$DIR/../testing/cover/cover
|
||||
cover ./coverage/* >profile.cov
|
||||
go tool cover -html=profile.cov -o coverage.html
|
||||
go tool cover -func=profile.cov -o coverage.txt
|
||||
|
||||
+1
-1
@@ -29,7 +29,7 @@ TESTS="${@:-*_test.sh}"
|
||||
|
||||
# If running on circle, use the scheduler to work out what tests to run
|
||||
if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" ]; then
|
||||
TESTS=$(echo $TESTS | "$DIR/sched" sched $CIRCLE_BUILD_NUM $CIRCLE_NODE_TOTAL $CIRCLE_NODE_INDEX)
|
||||
TESTS=$(echo $TESTS | "$DIR/sched" sched integration-$CIRCLE_BUILD_NUM $CIRCLE_NODE_TOTAL $CIRCLE_NODE_INDEX)
|
||||
fi
|
||||
|
||||
echo Running $TESTS
|
||||
|
||||
+4
-4
@@ -1,17 +1,17 @@
|
||||
#!/usr/bin/python
|
||||
import sys, string, json
|
||||
import sys, string, json, urllib
|
||||
import requests
|
||||
|
||||
BASE_URL="http://positive-cocoa-90213.appspot.com"
|
||||
|
||||
def test_time(test_name, runtime):
|
||||
r = requests.post(BASE_URL + "/record/%s/%f" % (test_name, runtime))
|
||||
r = requests.post(BASE_URL + "/record/%s/%f" % (urllib.quote(test_name, safe=""), runtime))
|
||||
print r.text
|
||||
assert r.status_code == 204
|
||||
|
||||
def test_sched(test_run, shard_count, shard_id):
|
||||
tests = json.dumps({'tests': string.split(sys.stdin.read())})
|
||||
r = requests.post(BASE_URL + "/schedule/%d/%d/%d" % (test_run, shard_count, shard_id), data=tests)
|
||||
r = requests.post(BASE_URL + "/schedule/%s/%d/%d" % (test_run, shard_count, shard_id), data=tests)
|
||||
assert r.status_code == 200
|
||||
result = r.json()
|
||||
for test in sorted(result['tests']):
|
||||
@@ -30,7 +30,7 @@ def main():
|
||||
if sys.argv[1] == "time":
|
||||
test_time(sys.argv[2], float(sys.argv[3]))
|
||||
elif sys.argv[1] == "sched":
|
||||
test_sched(int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4]))
|
||||
test_sched(sys.argv[2], int(sys.argv[3]), int(sys.argv[4]))
|
||||
else:
|
||||
usage()
|
||||
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
*.egg-info
|
||||
*.dist-info
|
||||
flask
|
||||
jinja2
|
||||
markupsafe
|
||||
werkzeug
|
||||
itsdangerous.*
|
||||
@@ -0,0 +1,6 @@
|
||||
To upload newer version:
|
||||
|
||||
```
|
||||
pip install -r requirements.txt -t .
|
||||
appcfg.py update .
|
||||
```
|
||||
@@ -19,7 +19,7 @@ class Test(ndb.Model):
|
||||
class Schedule(ndb.Model):
|
||||
shards = ndb.JsonProperty()
|
||||
|
||||
@app.route('/record/<test_name>/<runtime>', methods=['POST'])
|
||||
@app.route('/record/<path:test_name>/<runtime>', methods=['POST'])
|
||||
@ndb.transactional
|
||||
def record(test_name, runtime):
|
||||
test = Test.get_by_id(test_name)
|
||||
@@ -30,13 +30,13 @@ def record(test_name, runtime):
|
||||
test.put()
|
||||
return ('', 204)
|
||||
|
||||
@app.route('/schedule/<int:test_run>/<int:shard_count>/<int:shard>', methods=['POST'])
|
||||
@app.route('/schedule/<test_run>/<int:shard_count>/<int:shard>', methods=['POST'])
|
||||
def schedule(test_run, shard_count, shard):
|
||||
# read tests from body
|
||||
test_names = flask.request.get_json(force=True)['tests']
|
||||
|
||||
# first see if we have a scedule already
|
||||
schedule_id = "%d-%d" % (test_run, shard_count)
|
||||
schedule_id = "%s-%d" % (test_run, shard_count)
|
||||
schedule = Schedule.get_by_id(schedule_id)
|
||||
if schedule is not None:
|
||||
return flask.json.jsonify(tests=schedule.shards[str(shard)])
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
flask
|
||||
+46
-16
@@ -1,28 +1,58 @@
|
||||
#!/bin/bash
|
||||
|
||||
go get github.com/weaveworks/weave/testing/cover
|
||||
set -e
|
||||
|
||||
if [ -n "$COVERDIR" ] ; then
|
||||
coverdir="$COVERDIR"
|
||||
else
|
||||
coverdir=$(mktemp -d coverage.XXXXXXXXXX)
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SLOW=${SLOW-}
|
||||
GO_TEST_ARGS="-tags netgo -cpu 4"
|
||||
if [ -n "$SLOW" -o "$1" = "-slow" ]; then
|
||||
GO_TEST_ARGS="$GO_TEST_ARGS -race -covermode=atomic"
|
||||
|
||||
if [ -n "$COVERDIR" ] ; then
|
||||
coverdir="$COVERDIR"
|
||||
else
|
||||
coverdir=$(mktemp -d coverage.XXXXXXXXXX)
|
||||
fi
|
||||
|
||||
mkdir -p $coverdir
|
||||
fi
|
||||
|
||||
mkdir -p $coverdir
|
||||
fail=0
|
||||
|
||||
for dir in $(find . -type f -name '*_test.go' | xargs -n1 dirname | grep -v prog | sort -u); do
|
||||
go get -t -tags netgo $dir
|
||||
output=$(mktemp $coverdir/unit.XXXXXXXXXX)
|
||||
if ! go test -cpu 4 -tags netgo -covermode=count -coverprofile=$output $dir ; then
|
||||
fail=1
|
||||
fi
|
||||
TESTDIRS=$(find . -type f -name '*_test.go' | xargs -n1 dirname | grep -v prog | sort -u)
|
||||
|
||||
# If running on circle, use the scheduler to work out what tests to run on what shard
|
||||
if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" ]; then
|
||||
TESTDIRS=$(echo $TESTDIRS | "$DIR/sched" sched units-$CIRCLE_BUILD_NUM $CIRCLE_NODE_TOTAL $CIRCLE_NODE_INDEX)
|
||||
echo $TESTDIRS
|
||||
fi
|
||||
|
||||
for dir in $TESTDIRS; do
|
||||
|
||||
GO_TEST_ARGS_RUN="$GO_TEST_ARGS"
|
||||
if [ -n "$SLOW" ]; then
|
||||
go get -t -tags netgo $dir
|
||||
output=$(mktemp $coverdir/unit.XXXXXXXXXX)
|
||||
GO_TEST_ARGS_RUN="$GO_TEST_ARGS -coverprofile=$output"
|
||||
fi
|
||||
|
||||
START=$(date +%s)
|
||||
if ! go test $GO_TEST_ARGS_RUN $dir ; then
|
||||
fail=1
|
||||
fi
|
||||
RUNTIME=$(( $(date +%s) - $START ))
|
||||
|
||||
# Report test runtime when running on circle, to help scheduler
|
||||
if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" ]; then
|
||||
"$DIR/sched" time $dir $RUNTIME
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$COVERDIR" ] ; then
|
||||
cover $coverdir/* >profile.cov
|
||||
rm -rf $coverdir
|
||||
go tool cover -html=profile.cov -o=coverage.html
|
||||
if [ -n "$SLOW" -a -z "$COVERDIR" ] ; then
|
||||
$DIR/../testing/cover/cover $coverdir/* >profile.cov
|
||||
rm -rf $coverdir
|
||||
go tool cover -html=profile.cov -o=coverage.html
|
||||
go tool cover -func=profile.cov | tail -n1
|
||||
fi
|
||||
|
||||
exit $fail
|
||||
Reference in new issue
Block a user