diff --git a/circle.yml b/circle.yml index 21b34f5e..2a01b19f 100644 --- a/circle.yml +++ b/circle.yml @@ -21,11 +21,13 @@ dependencies: - if [[ -e "$WEAVE_BUILD" ]]; then docker load -i $WEAVE_BUILD; else - docker pull weaveworks/weave-build; + docker build -t weaveworks/weave-build build; mkdir -p $(dirname "$WEAVE_BUILD"); docker save weaveworks/weave-build >$WEAVE_BUILD; fi post: + - sudo apt-get install bc + - pip install requests - curl https://sdk.cloud.google.com | bash - bin/setup-circleci-secrets "$SECRET_PASSWORD" - mkdir -p $(dirname $SRCDIR) @@ -41,6 +43,7 @@ test: parallel: true - cd $SRCDIR/test; . ./gce.sh hosts; ./run_all.sh: parallel: true + timeout: 300 post: - cd $SRCDIR/test; ./gce.sh destroy: parallel: true diff --git a/test/.gitignore b/test/.gitignore index be2e5ea4..56dedd82 100644 --- a/test/.gitignore +++ b/test/.gitignore @@ -1,4 +1,3 @@ insecure_private_key -assert.sh .vagrant .ssh_known_hosts diff --git a/test/assert.sh b/test/assert.sh new file mode 100644 index 00000000..1a2f1f77 --- /dev/null +++ b/test/assert.sh @@ -0,0 +1,186 @@ +#!/bin/bash +# assert.sh 1.1 - bash unit testing framework +# Copyright (C) 2009-2015 Robert Lehmann +# +# http://github.com/lehmannro/assert.sh +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +export DISCOVERONLY=${DISCOVERONLY:-} +export DEBUG=${DEBUG:-} +export STOP=${STOP:-} +export INVARIANT=${INVARIANT:-} +export CONTINUE=${CONTINUE:-} + +args="$(getopt -n "$0" -l \ + verbose,help,stop,discover,invariant,continue vhxdic $*)" \ +|| exit -1 +for arg in $args; do + case "$arg" in + -h) + echo "$0 [-vxidc]" \ + "[--verbose] [--stop] [--invariant] [--discover] [--continue]" + echo "`sed 's/./ /g' <<< "$0"` [-h] [--help]" + exit 0;; + --help) + cat < [stdin] + (( tests_ran++ )) || : + [[ -z "$DISCOVERONLY" ]] || return + expected=$(echo -ne "${2:-}") + result="$(eval 2>/dev/null $1 <<< ${3:-})" || true + if [[ "$result" == "$expected" ]]; then + [[ -z "$DEBUG" ]] || echo -n . + return + fi + result="$(sed -e :a -e '$!N;s/\n/\\n/;ta' <<< "$result")" + [[ -z "$result" ]] && result="nothing" || result="\"$result\"" + [[ -z "$2" ]] && expected="nothing" || expected="\"$2\"" + _assert_fail "expected $expected${_indent}got $result" "$1" "$3" +} + +assert_raises() { + # assert_raises [stdin] + (( tests_ran++ )) || : + [[ -z "$DISCOVERONLY" ]] || return + status=0 + (eval $1 <<< ${3:-}) > /dev/null 2>&1 || status=$? + expected=${2:-0} + if [[ "$status" -eq "$expected" ]]; then + [[ -z "$DEBUG" ]] || echo -n . + return + fi + _assert_fail "program terminated with code $status instead of $expected" "$1" "$3" +} + +_assert_fail() { + # _assert_fail + [[ -n "$DEBUG" ]] && echo -n X + report="test #$tests_ran \"$2${3:+ <<< $3}\" failed:${_indent}$1" + if [[ -n "$STOP" ]]; then + [[ -n "$DEBUG" ]] && echo + echo "$report" + exit 1 + fi + tests_errors[$tests_failed]="$report" + (( tests_failed++ )) || : +} + +skip_if() { + # skip_if + (eval $@) > /dev/null 2>&1 && status=0 || status=$? + [[ "$status" -eq 0 ]] || return + skip +} + +skip() { + # skip (no arguments) + shopt -q extdebug && tests_extdebug=0 || tests_extdebug=1 + shopt -q -o errexit && tests_errexit=0 || tests_errexit=1 + # enable extdebug so returning 1 in a DEBUG trap handler skips next command + shopt -s extdebug + # disable errexit (set -e) so we can safely return 1 without causing exit + set +o errexit + tests_trapped=0 + trap _skip DEBUG +} +_skip() { + if [[ $tests_trapped -eq 0 ]]; then + # DEBUG trap for command we want to skip. Do not remove the handler + # yet because *after* the command we need to reset extdebug/errexit (in + # another DEBUG trap.) + tests_trapped=1 + [[ -z "$DEBUG" ]] || echo -n s + return 1 + else + trap - DEBUG + [[ $tests_extdebug -eq 0 ]] || shopt -u extdebug + [[ $tests_errexit -eq 1 ]] || set -o errexit + return 0 + fi +} + + +_assert_reset +: ${tests_suite_status:=0} # remember if any of the tests failed so far +_assert_cleanup() { + local status=$? + # modify exit code if it's not already non-zero + [[ $status -eq 0 && -z $CONTINUE ]] && exit $tests_suite_status +} +trap _assert_cleanup EXIT diff --git a/test/gce.sh b/test/gce.sh index bf632993..aa0a62df 100755 --- a/test/gce.sh +++ b/test/gce.sh @@ -33,7 +33,14 @@ function vm_names { # Delete all vms in this account function destroy { names="$(vm_names)" - gcloud compute instances delete --zone $ZONE -q $names || true + for i in {0..10}; do + # gcloud instances delete can sometimes hang. + timeout 60s /bin/bash -c "gcloud compute instances delete --zone $ZONE -q $names || true" + # 124 means it timed out + if [ $? -ne 124 ]; then + return + fi + done } function external_ip { diff --git a/test/run_all.sh b/test/run_all.sh index 7007925d..4884c81f 100755 --- a/test/run_all.sh +++ b/test/run_all.sh @@ -24,20 +24,22 @@ check_test_status() { # Overwrite assert.sh _assert_cleanup trap with our own trap check_test_status EXIT -# If running on circle, only run the tests for my "shard". -# CIRCLE_NODE_TOTAL is the total number of shards, and -# CIRCLE_NODE_INDEX is my index. -i=0 -TESTS= -for test in *_test.sh; do - if [ -z "$CIRCLECI" ] || [ $(($i % $CIRCLE_NODE_TOTAL)) -eq $CIRCLE_NODE_INDEX ]; then - TESTS="$TESTS $test" - fi - i=$(($i + 1)) -done +TESTS=*_test.sh + +# If running on circle, use the scheduler to work out what tests to run +if [ -n "$CIRCLECI" ]; then + TESTS=$(echo $TESTS | ./sched sched $CIRCLE_BUILD_NUM $CIRCLE_NODE_TOTAL $CIRCLE_NODE_INDEX) +fi + +echo Running $TESTS for t in $TESTS; do echo greyly echo "---= Running $t =---" . $t + + # Report test runtime when running on circle, to help scheduler + if [ -n "$CIRCLECI" ]; then + ./sched time $t $(bc -l <<< "$tests_time/1000000000") + fi done diff --git a/test/sched b/test/sched new file mode 100755 index 00000000..4a7c7021 --- /dev/null +++ b/test/sched @@ -0,0 +1,38 @@ +#!/usr/bin/python +import sys, string, json +import requests + +BASE_URL="http://positive-cocoa-90213.appspot.com" + +def test_time(test_name, runtime): + r = requests.post(BASE_URL + "/record/%s/%f" % (test_name, runtime)) + print r.text + assert r.status_code == 204 + +def test_sched(test_run, shard_count, shard_id): + tests = json.dumps({'tests': string.split(sys.stdin.read())}) + r = requests.post(BASE_URL + "/schedule/%d/%d/%d" % (test_run, shard_count, shard_id), data=tests) + assert r.status_code == 200 + result = r.json() + for test in sorted(result['tests']): + print test + +def usage(): + print "%s " % sys.argv[0] + print " time " + print " sched " + +def main(): + if len(sys.argv) < 4: + usage() + sys.exit(1) + + if sys.argv[1] == "time": + test_time(sys.argv[2], float(sys.argv[3])) + elif sys.argv[1] == "sched": + test_sched(int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4])) + else: + usage() + +if __name__ == '__main__': + main() diff --git a/test/scheduler/app.yaml b/test/scheduler/app.yaml new file mode 100644 index 00000000..09773cb8 --- /dev/null +++ b/test/scheduler/app.yaml @@ -0,0 +1,14 @@ +application: positive-cocoa-90213 +version: 1 +runtime: python27 +api_version: 1 +threadsafe: true + +handlers: +- url: .* + script: main.app + +libraries: +- name: webapp2 + version: latest + diff --git a/test/scheduler/main.py b/test/scheduler/main.py new file mode 100644 index 00000000..a2cf113b --- /dev/null +++ b/test/scheduler/main.py @@ -0,0 +1,66 @@ +import operator + +import flask + +from google.appengine.ext import ndb + +app = flask.Flask('scheduler') +app.debug = True + +# We use exponential moving average to record +# test run times. Higher alpha discounts historic +# observations faster. +alpha = 0.3 + +class Test(ndb.Model): + total_run_time = ndb.FloatProperty(default=0.) # Not total, but a EWMA + total_runs = ndb.IntegerProperty(default=0) + +class Schedule(ndb.Model): + shards = ndb.JsonProperty() + +@app.route('/record//', methods=['POST']) +@ndb.transactional +def record(test_name, runtime): + test = Test.get_by_id(test_name) + if test is None: + test = Test(id=test_name) + test.total_run_time = (test.total_run_time * (1-alpha)) + (float(runtime) * alpha) + test.total_runs += 1 + test.put() + return ('', 204) + +@app.route('/schedule///', methods=['POST']) +def schedule(test_run, shard_count, shard): + # read tests from body + test_names = flask.request.get_json(force=True)['tests'] + + # first see if we have a scedule already + schedule_id = "%d-%d" % (test_run, shard_count) + schedule = Schedule.get_by_id(schedule_id) + if schedule is not None: + return flask.json.jsonify(tests=schedule.shards[str(shard)]) + + # if not, do simple greedy algorithm + test_times = ndb.get_multi(ndb.Key(Test, test_name) for test_name in test_names) + def avg(test): + if test is not None: + return test.total_run_time + return 1 + test_times = [(test_name, avg(test)) for test_name, test in zip(test_names, test_times)] + test_times_dict = dict(test_times) + test_times.sort(key=operator.itemgetter(1)) + + shards = {i: [] for i in xrange(shard_count)} + while test_times: + test_name, time = test_times.pop() + + # find shortest shard and put it in that + s, _ = min(((i, sum(test_times_dict[t] for t in shards[i])) + for i in xrange(shard_count)), key=operator.itemgetter(1)) + + shards[s].append(test_name) + + # atomically insert or retrieve existing schedule + schedule = Schedule.get_or_insert(schedule_id, shards=shards) + return flask.json.jsonify(tests=schedule.shards[str(shard)]) diff --git a/test/setup.sh b/test/setup.sh index fd7d1ff9..9aae4c46 100755 --- a/test/setup.sh +++ b/test/setup.sh @@ -2,11 +2,6 @@ set -e -if ! [ -f "./assert.sh" ]; then - echo Fetching assert script - curl -sS https://raw.githubusercontent.com/lehmannro/assert.sh/master/assert.sh > ./assert.sh -fi - . ./config.sh echo Copying weave images and script to hosts