Merge pull request #797 from weaveworks/timeout

Various fixes to circleci integration tests

circle.yml

@@ -21,11 +21,13 @@ dependencies:
     - if [[ -e "$WEAVE_BUILD" ]]; then
         docker load -i $WEAVE_BUILD;
       else
-        docker pull weaveworks/weave-build;
+        docker build -t weaveworks/weave-build build;
         mkdir -p $(dirname "$WEAVE_BUILD");
         docker save weaveworks/weave-build >$WEAVE_BUILD;
       fi
   post:
+    - sudo apt-get install bc
+    - pip install requests
     - curl https://sdk.cloud.google.com | bash
     - bin/setup-circleci-secrets "$SECRET_PASSWORD"
     - mkdir -p $(dirname $SRCDIR)
@@ -41,6 +43,7 @@ test:
         parallel: true
     - cd $SRCDIR/test; . ./gce.sh hosts; ./run_all.sh:
         parallel: true
+        timeout: 300
   post:
     - cd $SRCDIR/test; ./gce.sh destroy:
         parallel: true

test/.gitignore

@@ -1,4 +1,3 @@
 insecure_private_key
-assert.sh
 .vagrant
 .ssh_known_hosts

test/assert.sh

@@ -0,0 +1,186 @@
+#!/bin/bash
+# assert.sh 1.1 - bash unit testing framework
+# Copyright (C) 2009-2015 Robert Lehmann
+#
+# http://github.com/lehmannro/assert.sh
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+export DISCOVERONLY=${DISCOVERONLY:-}
+export DEBUG=${DEBUG:-}
+export STOP=${STOP:-}
+export INVARIANT=${INVARIANT:-}
+export CONTINUE=${CONTINUE:-}
+
+args="$(getopt -n "$0" -l \
+    verbose,help,stop,discover,invariant,continue vhxdic $*)" \
+|| exit -1
+for arg in $args; do
+    case "$arg" in
+        -h)
+            echo "$0 [-vxidc]" \
+                "[--verbose] [--stop] [--invariant] [--discover] [--continue]"
+            echo "`sed 's/./ /g' <<< "$0"` [-h] [--help]"
+            exit 0;;
+        --help)
+            cat <<EOF
+Usage: $0 [options]
+Language-agnostic unit tests for subprocesses.
+
+Options:
+  -v, --verbose    generate output for every individual test case
+  -x, --stop       stop running tests after the first failure
+  -i, --invariant  do not measure timings to remain invariant between runs
+  -d, --discover   collect test suites only, do not run any tests
+  -c, --continue   do not modify exit code to test suite status
+  -h               show brief usage information and exit
+  --help           show this help message and exit
+EOF
+            exit 0;;
+        -v|--verbose)
+            DEBUG=1;;
+        -x|--stop)
+            STOP=1;;
+        -i|--invariant)
+            INVARIANT=1;;
+        -d|--discover)
+            DISCOVERONLY=1;;
+        -c|--continue)
+            CONTINUE=1;;
+    esac
+done
+
+_indent=$'\n\t' # local format helper
+
+_assert_reset() {
+    tests_ran=0
+    tests_failed=0
+    tests_errors=()
+    tests_starttime="$(date +%s%N)" # nanoseconds_since_epoch
+}
+
+assert_end() {
+    # assert_end [suite ..]
+    tests_endtime="$(date +%s%N)"
+    # required visible decimal place for seconds (leading zeros if needed)
+    tests_time="$( \
+        printf "%010d" "$(( ${tests_endtime/%N/000000000}
+                            - ${tests_starttime/%N/000000000} ))")"  # in ns
+    tests="$tests_ran ${*:+$* }tests"
+    [[ -n "$DISCOVERONLY" ]] && echo "collected $tests." && _assert_reset && return
+    [[ -n "$DEBUG" ]] && echo
+    # to get report_time split tests_time on 2 substrings:
+    #   ${tests_time:0:${#tests_time}-9} - seconds
+    #   ${tests_time:${#tests_time}-9:3} - milliseconds
+    [[ -z "$INVARIANT" ]] \
+        && report_time=" in ${tests_time:0:${#tests_time}-9}.${tests_time:${#tests_time}-9:3}s" \
+        || report_time=
+
+    if [[ "$tests_failed" -eq 0 ]]; then
+        echo "all $tests passed$report_time."
+    else
+        for error in "${tests_errors[@]}"; do echo "$error"; done
+        echo "$tests_failed of $tests failed$report_time."
+    fi
+    tests_failed_previous=$tests_failed
+    [[ $tests_failed -gt 0 ]] && tests_suite_status=1
+    _assert_reset
+}
+
+assert() {
+    # assert <command> <expected stdout> [stdin]
+    (( tests_ran++ )) || :
+    [[ -z "$DISCOVERONLY" ]] || return
+    expected=$(echo -ne "${2:-}")
+    result="$(eval 2>/dev/null $1 <<< ${3:-})" || true
+    if [[ "$result" == "$expected" ]]; then
+        [[ -z "$DEBUG" ]] || echo -n .
+        return
+    fi
+    result="$(sed -e :a -e '$!N;s/\n/\\n/;ta' <<< "$result")"
+    [[ -z "$result" ]] && result="nothing" || result="\"$result\""
+    [[ -z "$2" ]] && expected="nothing" || expected="\"$2\""
+    _assert_fail "expected $expected${_indent}got $result" "$1" "$3"
+}
+
+assert_raises() {
+    # assert_raises <command> <expected code> [stdin]
+    (( tests_ran++ )) || :
+    [[ -z "$DISCOVERONLY" ]] || return
+    status=0
+    (eval $1 <<< ${3:-}) > /dev/null 2>&1 || status=$?
+    expected=${2:-0}
+    if [[ "$status" -eq "$expected" ]]; then
+        [[ -z "$DEBUG" ]] || echo -n .
+        return
+    fi
+    _assert_fail "program terminated with code $status instead of $expected" "$1" "$3"
+}
+
+_assert_fail() {
+    # _assert_fail <failure> <command> <stdin>
+    [[ -n "$DEBUG" ]] && echo -n X
+    report="test #$tests_ran \"$2${3:+ <<< $3}\" failed:${_indent}$1"
+    if [[ -n "$STOP" ]]; then
+        [[ -n "$DEBUG" ]] && echo
+        echo "$report"
+        exit 1
+    fi
+    tests_errors[$tests_failed]="$report"
+    (( tests_failed++ )) || :
+}
+
+skip_if() {
+    # skip_if <command ..>
+    (eval $@) > /dev/null 2>&1 && status=0 || status=$?
+    [[ "$status" -eq 0 ]] || return
+    skip
+}
+
+skip() {
+    # skip  (no arguments)
+    shopt -q extdebug && tests_extdebug=0 || tests_extdebug=1
+    shopt -q -o errexit && tests_errexit=0 || tests_errexit=1
+    # enable extdebug so returning 1 in a DEBUG trap handler skips next command
+    shopt -s extdebug
+    # disable errexit (set -e) so we can safely return 1 without causing exit
+    set +o errexit
+    tests_trapped=0
+    trap _skip DEBUG
+}
+_skip() {
+    if [[ $tests_trapped -eq 0 ]]; then
+        # DEBUG trap for command we want to skip.  Do not remove the handler
+        # yet because *after* the command we need to reset extdebug/errexit (in
+        # another DEBUG trap.)
+        tests_trapped=1
+        [[ -z "$DEBUG" ]] || echo -n s
+        return 1
+    else
+        trap - DEBUG
+        [[ $tests_extdebug -eq 0 ]] || shopt -u extdebug
+        [[ $tests_errexit -eq 1 ]] || set -o errexit
+        return 0
+    fi
+}
+
+
+_assert_reset
+: ${tests_suite_status:=0}  # remember if any of the tests failed so far
+_assert_cleanup() {
+    local status=$?
+    # modify exit code if it's not already non-zero
+    [[ $status -eq 0 && -z $CONTINUE ]] && exit $tests_suite_status
+}
+trap _assert_cleanup EXIT

test/gce.sh

@@ -33,7 +33,14 @@ function vm_names {
 # Delete all vms in this account
 function destroy {
 	names="$(vm_names)"
-	gcloud compute instances delete --zone $ZONE -q $names || true
+	for i in {0..10}; do
+		# gcloud instances delete can sometimes hang.
+		timeout 60s /bin/bash -c "gcloud compute instances delete --zone $ZONE -q $names || true"
+		# 124 means it timed out
+		if [ $? -ne 124 ]; then
+			return
+		fi
+	done
 }
 
 function external_ip {

test/run_all.sh

@@ -24,20 +24,22 @@ check_test_status() {
 # Overwrite assert.sh _assert_cleanup trap with our own
 trap check_test_status EXIT
 
-# If running on circle, only run the tests for my "shard".
-# CIRCLE_NODE_TOTAL is the total number of shards, and
-# CIRCLE_NODE_INDEX is my index.
-i=0
-TESTS=
-for test in *_test.sh; do
-    if [ -z "$CIRCLECI" ] || [ $(($i % $CIRCLE_NODE_TOTAL)) -eq $CIRCLE_NODE_INDEX ]; then
-          TESTS="$TESTS $test"
-    fi
-    i=$(($i + 1))
-done
+TESTS=*_test.sh
+
+# If running on circle, use the scheduler to work out what tests to run
+if [ -n "$CIRCLECI" ]; then
+    TESTS=$(echo $TESTS | ./sched sched $CIRCLE_BUILD_NUM $CIRCLE_NODE_TOTAL $CIRCLE_NODE_INDEX)
+fi
+
+echo Running $TESTS
 
 for t in $TESTS; do
     echo
     greyly echo "---= Running $t =---"
     . $t
+
+    # Report test runtime when running on circle, to help scheduler
+    if [ -n "$CIRCLECI" ]; then
+        ./sched time $t $(bc -l <<< "$tests_time/1000000000")
+    fi
 done

test/sched

@@ -0,0 +1,38 @@
+#!/usr/bin/python
+import sys, string, json
+import requests
+
+BASE_URL="http://positive-cocoa-90213.appspot.com"
+
+def test_time(test_name, runtime):
+  r = requests.post(BASE_URL + "/record/%s/%f" % (test_name, runtime))
+  print r.text
+  assert r.status_code == 204
+
+def test_sched(test_run, shard_count, shard_id):
+  tests = json.dumps({'tests': string.split(sys.stdin.read())})
+  r = requests.post(BASE_URL + "/schedule/%d/%d/%d" % (test_run, shard_count, shard_id), data=tests)
+  assert r.status_code == 200
+  result = r.json()
+  for test in sorted(result['tests']):
+    print test
+
+def usage():
+  print "%s <cmd> <args..>" % sys.argv[0]
+  print "   time <test name> <run time>"
+  print "   sched <test run> <num shards> <shard id>"
+
+def main():
+  if len(sys.argv) < 4:
+    usage()
+    sys.exit(1)
+
+  if sys.argv[1] == "time":
+    test_time(sys.argv[2], float(sys.argv[3]))
+  elif sys.argv[1] == "sched":
+    test_sched(int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4]))
+  else:
+    usage()
+
+if __name__ == '__main__':
+  main()

test/scheduler/app.yaml

@@ -0,0 +1,14 @@
+application: positive-cocoa-90213
+version: 1
+runtime: python27
+api_version: 1
+threadsafe: true
+
+handlers:
+- url: .*
+  script: main.app
+
+libraries:
+- name: webapp2
+  version: latest
+

test/scheduler/main.py

@@ -0,0 +1,66 @@
+import operator
+
+import flask
+
+from google.appengine.ext import ndb
+
+app = flask.Flask('scheduler')
+app.debug = True
+
+# We use exponential moving average to record
+# test run times.  Higher alpha discounts historic
+# observations faster.
+alpha = 0.3
+
+class Test(ndb.Model):
+  total_run_time = ndb.FloatProperty(default=0.) # Not total, but a EWMA
+  total_runs = ndb.IntegerProperty(default=0)
+
+class Schedule(ndb.Model):
+  shards = ndb.JsonProperty()
+
+@app.route('/record/<test_name>/<runtime>', methods=['POST'])
+@ndb.transactional
+def record(test_name, runtime):
+  test = Test.get_by_id(test_name)
+  if test is None:
+    test = Test(id=test_name)
+  test.total_run_time = (test.total_run_time * (1-alpha)) + (float(runtime) * alpha)
+  test.total_runs += 1
+  test.put()
+  return ('', 204)
+
+@app.route('/schedule/<int:test_run>/<int:shard_count>/<int:shard>', methods=['POST'])
+def schedule(test_run, shard_count, shard):
+  # read tests from body
+  test_names = flask.request.get_json(force=True)['tests']
+
+  # first see if we have a scedule already
+  schedule_id = "%d-%d" % (test_run, shard_count)
+  schedule = Schedule.get_by_id(schedule_id)
+  if schedule is not None:
+    return flask.json.jsonify(tests=schedule.shards[str(shard)])
+
+  # if not, do simple greedy algorithm
+  test_times = ndb.get_multi(ndb.Key(Test, test_name) for test_name in test_names)
+  def avg(test):
+    if test is not None:
+      return test.total_run_time
+    return 1
+  test_times = [(test_name, avg(test)) for test_name, test in zip(test_names, test_times)]
+  test_times_dict = dict(test_times)
+  test_times.sort(key=operator.itemgetter(1))
+
+  shards = {i: [] for i in xrange(shard_count)}
+  while test_times:
+    test_name, time = test_times.pop()
+
+    # find shortest shard and put it in that
+    s, _ = min(((i, sum(test_times_dict[t] for t in shards[i]))
+      for i in xrange(shard_count)), key=operator.itemgetter(1))
+
+    shards[s].append(test_name)
+
+  # atomically insert or retrieve existing schedule
+  schedule = Schedule.get_or_insert(schedule_id, shards=shards)
+  return flask.json.jsonify(tests=schedule.shards[str(shard)])

test/setup.sh

@@ -2,11 +2,6 @@
 
 set -e
 
-if ! [ -f "./assert.sh" ]; then
-    echo Fetching assert script
-    curl -sS https://raw.githubusercontent.com/lehmannro/assert.sh/master/assert.sh > ./assert.sh
-fi
-
 . ./config.sh
 
 echo Copying weave images and script to hosts