Merge pull request #2694 from weaveworks/2647-testing-mvp

Testing MVP Fixes #2647
2017-02-08 16:25:56 +00:00
commit dff5b6d3f1
@@ -68,3 +68,5 @@ test/tls/*.pem
 test/coverage
 test/coverage.*
 *qemu-*-static
+terraform.tfstate
+terraform.tfstate.backup
@@ -1,8 +1,27 @@
 # IMPORTANT: Please only use GLOBs to ignore files and directories.

-# Do not lint any bash script in Weave Net, in order to 
+# Do not lint the below bash scripts in Weave Net, in order to 
 # avoid breaking the build on make lint:
-*.sh
-bin/*
+bin/circle-deploy-issues
+bin/circle-deploy-master
+bin/circle-teardown-pre
+bin/circle-test-teardown
+bin/circle-test-unit
+bin/install-wordepress
+bin/multiweave
+bin/publish-site
+bin/release
+build/build.sh
+prog/weave-kube/launch.sh
 prog/weaveexec/symlink
+test/*_test.sh
+test/600_proxy_docker_py.sh
+test/assert.sh
+test/config.sh
+test/gce.sh
+test/gen_coverage_reports.sh
+test/run_all.sh
+test/sanity_check.sh
+test/setup.sh
+vendor/*.sh
 weave
@@ -214,7 +214,7 @@ tests:
 	./tools/test -no-go-get -netgo -timeout 8m

 lint:
-	./tools/lint -nocomment -notestpackage .
+	./tools/lint -nocomment -notestpackage

 endif

@@ -359,3 +359,20 @@ build:

 run-smoketests: all testrunner
 	cd test && ./setup.sh && ./run_all.sh
+
+integration-tests: all testrunner
+# Usage:
+#   $ make \
+#     NAME="<prefix used to name VMs and other resources>" \
+#     PROVIDER="<provider among {vagrant|gcp|aws|do}>" \
+#     NUM_HOSTS="<# test machines>" \
+#     PLAYBOOK="<filename>" \
+#     RUNNER_ARGS="<...>" \
+#     TESTS="<...>" \  # Can be set to only run one or a few tests instead of the full test suite.
+#     DOCKER_VERSION=<...> \
+#     KUBERNETES_VERSION=<...> \
+#     KUBERNETES_CNI_VERSION=<...> \
+#     <...> # See also run-integration-test.sh for all variables and individual functions.
+#     integration-tests
+#
+	RUNNER_ARGS="-parallel" ./test/run-integration-tests.sh
@@ -1,45 +1,36 @@
-VAGRANTFILE_API_VERSION = "2"
-
-require './vagrant-common.rb'
+require File.expand_path(File.join(File.dirname(__FILE__), 'vagrant-common.rb'))

 vm_ip = "172.16.0.3" # arbitrary private IP

-pkgs = %w(
-  docker-engine=1.10.2-0~wily
-  aufs-tools
-  build-essential
-  ethtool
-  iputils-arping
-  libpcap-dev
-  git
-  mercurial
-  bc
-  jq
-)
-
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|

-  config.vm.box = "ubuntu/wily64"
+  config.vm.box = VAGRANT_IMAGE

  config.vm.network "private_network", ip: vm_ip
  config.vm.provider :virtualbox do |vb|
    vb.memory = 2048
-    vb.customize ["modifyvm", :id, "--natdnshostresolver1", "off"]
-    vb.customize ["modifyvm", :id, "--natdnsproxy1", "off"]
+    configure_nat_dns(vb)    
  end

+  # Disable default Vagrant shared folder, which we don't need:
  config.vm.synced_folder ".", "/vagrant", disabled: true
+  # Keep Weave Net sources' in sync:
  config.vm.synced_folder ".", "/home/vagrant/src/github.com/weaveworks/weave"
+  # Create a convenience symlink to $HOME/src/github.com/weaveworks/weave
+  config.vm.provision :shell, :inline => 'ln -sf ~vagrant/src/github.com/weaveworks/weave ~vagrant/'

  # Set SSH keys up to be able to run smoke tests straightaway:
  config.vm.provision "file", source: "~/.vagrant.d/insecure_private_key", destination: "/home/vagrant/src/github.com/weaveworks/weave/test/insecure_private_key"
-
-  install_build_deps config.vm, pkgs
-  install_go_toochain config.vm
-  tweak_user_env config.vm
-  tweak_docker_daemon config.vm
+  # Grant permissions on sources:
+  config.vm.provision :shell, :inline => 'sudo chown -R vagrant:vagrant ~vagrant/src', :privileged => false
  cleanup config.vm

+  config.vm.provision 'ansible' do |ansible|
+    ansible.playbook = 'tools/config_management/setup_weave-net_dev.yml'
+    ansible.extra_vars = {
+      go_version: GO_VERSION
+    }
+  end
 end

 begin
@@ -2,45 +2,16 @@

 set -e

-echo "TEST_AND_PUBLISH=1" > "$STATE"
-
-# Avoid doing a full build if we can.  Note if the PR doesn't exist
-# when the build is kicked off, we can't run this check so we do a
-# full build.  Subsequent pushes to the same branch will have the desired behaviour
-if [ -n "$CI_PULL_REQUEST" -a "$CI_PULL_REQUEST" = "$CI_PULL_REQUESTS" ] ; then
-    if PR=$(echo "$CI_PULL_REQUEST" | grep -oP '(?<=^https://github.com/weaveworks/weave/pull/)[0-9]+$') ; then
-        echo "GH_PR=$PR" >> "$STATE"
-        if FILES=$(curl -s https://api.github.com/repos/weaveworks/weave/pulls/$PR/files | jq -r '.[]|.filename') ; then
-            echo "PR touches [$FILES]"
-            if ! echo "$FILES" | grep -v ^site/ ; then
-                echo "PR contains site changes only - skipping tests"
-                echo "TEST_AND_PUBLISH=" >> "$STATE"
-            else
-                echo "PR contains non-site changes - running tests"
-            fi
-        else
-            echo "Unable to get files for PR $PR" >&2
-        fi
-    else
-        echo "Unable to extract PR number from $CI_PULL_REQUEST" >&2
-    fi
-fi
-
 source "$STATE"

-mkdir -p $(dirname $SRCDIR) && cp -r $(pwd)/ $SRCDIR
-
-if [ -n "$TEST_AND_PUBLISH" ] ; then
-    [ -z "$SECRET_PASSWORD" ] || bin/setup-circleci-secrets "$SECRET_PASSWORD"
-    cd $SRCDIR; git submodule update --init
-    if [ "$CIRCLE_NODE_INDEX" = "0" -a -n "$SECRET_PASSWORD" ] ; then
-        cd $SRCDIR/test; ./gce.sh make_template
-    fi
-    cd $SRCDIR/build
+if [ -n "$TEST_AND_PUBLISH" ]; then
+    cd "$SRCDIR"
+    git submodule update --init
+    cd "$SRCDIR/build"
    ../tools/rebuild-image weaveworks/weavebuild . Dockerfile build.sh
-    touch $SRCDIR/.build.uptodate
+    touch "$SRCDIR/.build.uptodate"
    sudo chown -R ubuntu /usr/local/go
-    cd $SRCDIR
+    cd "$SRCDIR"
    make testrunner
    make COVERAGE=true RM= exes all
 fi
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e
+
+echo "TEST_AND_PUBLISH=1" >"$STATE"
+
+# Avoid doing a full build if we can.  Note if the PR doesn't exist
+# when the build is kicked off, we can't run this check so we do a
+# full build.  Subsequent pushes to the same branch will have the desired behaviour
+if [ -n "$CI_PULL_REQUEST" -a "$CI_PULL_REQUEST" = "$CI_PULL_REQUESTS" ]; then
+    if PR=$(echo "$CI_PULL_REQUEST" | grep -oP '(?<=^https://github.com/weaveworks/weave/pull/)[0-9]+$'); then
+        echo "GH_PR=$PR" >>"$STATE"
+        if FILES=$(curl -s "https://api.github.com/repos/weaveworks/weave/pulls/$PR/files" | jq -r '.[]|.filename'); then
+            echo "PR touches [$FILES]"
+            if ! echo "$FILES" | grep -v ^site/; then
+                echo "PR contains site changes only - skipping tests"
+                echo "TEST_AND_PUBLISH=" >>"$STATE"
+            else
+                echo "PR contains non-site changes - running tests"
+            fi
+        else
+            echo "Unable to get files for PR $PR" >&2
+        fi
+    else
+        echo "Unable to extract PR number from $CI_PULL_REQUEST" >&2
+    fi
+fi
+
+source "$STATE"
+
+mkdir -p "$(dirname "$SRCDIR")" && cp -r "$(pwd)/" "$SRCDIR"
+
+if [ -n "$TEST_AND_PUBLISH" ]; then
+    cd "$SRCDIR"
+    git submodule update --init
+fi
@@ -2,9 +2,29 @@

 set -e

-source "$STATE"
+# Signal failures in lock file, in order to fail fast:
+function signal_failure() {
+    echo "KO" >"$TEST_VMS_READY_LOCK_FILE"
+    exit 1
+}
+trap signal_failure ERR

-if [ -n "$TEST_AND_PUBLISH" ] ; then
-    cd $SRCDIR/test
-    [ -z "$SECRET_PASSWORD" ] || (./gce.sh setup && eval $(./gce.sh hosts) && ./setup.sh)
+source "$STATE"
+source "$(dirname "$0")/wait_for.sh"
+
+if [ -n "$TEST_AND_PUBLISH" ]; then
+    [ -n "$SECRET_KEY" ] || {
+        echo "Cannot run smoke tests: no secret key"
+        exit 1
+    }
+
+    # Provisioning from image typically take 90 seconds, 
+    # but if we are creating a GCP image, it can take more than 10 minutes:
+    wait_for 900 "$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
+
+    # Upload built binaries to testing VMs:
+    cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests.
+    ./run-integration-tests.sh setup >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
+    echo "OK" >"$TEST_VMS_READY_LOCK_FILE"
+    echo "Test VMs now ready. $(date)." >>"$TEST_VMS_SETUP_OUTPUT_FILE"
 fi
@@ -3,12 +3,21 @@
 set -e

 source "$STATE"
+source "$(dirname "$0")/wait_for.sh"

-if [ -n "$TEST_AND_PUBLISH" ] ; then
-    [ -n "$SECRET_PASSWORD" ] || { echo "Cannot run smoke tests: no secret key"; exit 1; }
-    cd $SRCDIR/test
-    eval $(./gce.sh hosts)
+if [ -n "$TEST_AND_PUBLISH" ]; then
+    [ -n "$SECRET_KEY" ] || {
+        echo "Cannot run smoke tests: no secret key"
+        exit 1
+    }
+
+    # Provisioning from image typically take 90 seconds, 
+    # but if we are creating a GCP image, it can take more than 10 minutes:
+    wait_for 900 "$TEST_VMS_READY_LOCK_FILE"
+
+    # Run integration tests:
    export COVERAGE=true
    export WEAVE_NET_SANITY_CHECKS_FILES="$CIRCLE_ARTIFACTS/weave_net_sanity_check_*.log"
-    ./run_all.sh
+    cd "$SRCDIR/test" # Ensures we generate code coverage files in the right folder, and use the Terraform state of previously provisionned VMs.
+    ./run-integration-tests.sh test
 fi
@@ -6,6 +6,5 @@ source "$STATE"

 if [ -n "$TEST_AND_PUBLISH" ] ; then
    cd $SRCDIR/test
-    [ -z "$SECRET_PASSWORD" ] || ./gce.sh destroy
    test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR/test; ./gen_coverage_reports.sh)
 fi
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+set -e
+
+# Signal failures in lock file, in order to fail fast:
+function signal_failure() {
+    echo "KO" >"$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
+    exit 1
+}
+trap signal_failure ERR
+
+source "$STATE"
+
+function install_terraform() {
+    curl -fsS https://releases.hashicorp.com/terraform/0.8.5/terraform_0.8.5_linux_amd64.zip | gunzip >terraform && chmod +x terraform && sudo mv terraform /usr/bin
+}
+
+function install_ansible() {
+    sudo apt-get update || true
+    sudo apt-get install -qq -y python-pip python-dev libffi-dev libssl-dev && pip install --user -U cffi && pip install --user ansible
+    export PATH="$PATH:$HOME/.local/bin"
+}
+
+if [ -n "$TEST_AND_PUBLISH" ]; then
+    [ -n "$SECRET_KEY" ] || {
+        echo "Cannot run smoke tests: no secret key"
+        exit 1
+    }
+
+    install_terraform >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
+    install_ansible >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
+
+    # Only attempt to create GCP image in first container, wait for it to be created otherwise:
+    [ "$CIRCLE_NODE_INDEX" != "0" ] && export CREATE_IMAGE=0
+
+    # Provision and configure testing VMs:
+    cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests.
+    ./run-integration-tests.sh configure >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
+    echo "OK" >"$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
+    echo "Test VMs now provisioned and configured. $(date)." >>"$TEST_VMS_SETUP_OUTPUT_FILE"
+fi
@@ -1,115 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-# openssl enc -in do-setup-circleci-secrets.orig -out setup-circleci-secrets.orig -e -aes256 -pass stdin
-# openssl base64 < setup-circleci-secrets.orig
-
-openssl base64 -d << EOF \
-  | openssl enc \
-    -out bin/do-setup-circleci-secrets \
-		-d -aes256 -pass pass:$1
-U2FsdGVkX1/3QHnDgpZn0srp+GiEQBTlag0CBdsw6Bp6iaIKy6Jy8gFshrsmvl3+
-c+7hn1nqsBVpzVH0S5QA3Ex5nx9rl7/dcGoogsySFcwMmBFcauT0UTMK1bvX0Hqx
-QrshYSndUkww72KA7EsSnjR1BBJWm3AhCDOTIcnO6nQhig1Fc+SBEj5FGS5xTQ5Z
-/8hDsKf54Ms49vOO6QeE5w5ndZ+k7IvVJfq9Evhb7htD20RntDhR86foAwMnLV7R
-Wh9qAqI3gnRq0GmbeRKaHmB7pcLq9f0OdScqM/Tn1D+q0556fqJjBziXpZiGbxVs
-DxEp8zDJ8yNDAi0Wb2NXjoA79SXJ8Q7iTieaST4y2OmXQpY+Vw8RVDXxi6ERyVnH
-41OeMzyLm0qvCoMl32pe1sSG5DRCUiAm7wOtDTVvNDRqmGJYUgwwexyayecJI1wQ
-LlhYLJavUvw3UGDZOld8BaMQhfOKBVCdFrjmA4HuVxmOf3029qkgzOCSZiQVNqs+
-fvM6J05zBuk+GPy8YInvRkzfxum/x00vmUyGjMs/lHDFfrmYDFK3ux41EoVqvWLG
-S5uYxdZJFdR3Xsrs+sMr51XNBQGMmz6KoEEzFYyHs5fZgtdpRdPAuQS1m6FqgZyi
-wSz6+EKsALtmGorEp+CF8JN5YzKJCrKzNkg0fILBqsqmJZGP64lmX69950lRLNGM
-/l3KVR2zH31b4AnnzTcvldVSCKDXjaMw3JA2Ce/MK6XhOWOcpEv4oDt/62lbhUKP
-QUQTPbfpQRvVfBcoIrfAm3ghi/CZc2TDVEuD7DhasLTZD3r7LoK261BWC9O5Wubi
-vlXuFIF+uyrpTDf6EJKE/+XywtArppffMP6RkHu1Iqnd8JtsqPaIZsmMmZbjgb6j
-BX4ZLb5U9R7jYoSgIke4+u6geFcrMQ+Lny4r7MxXaS0Y1UJ6Jo1bwpYw1TBIobZc
-eQ30+xhbSeRC9vEw1NXI6qM2Ai2gIHtAZvEiBkBBXTG5Ld7N/mBAmDIJEzVRxh6F
-jRkB3dzESF6eeTBm61OQepwV3aW6lslMaf26drPGv2W2q+K/WOpQOejfggtHUgVs
-9SWqtn7hHYSpIASe1Ihi14HAWv3kPhQien0zY1DLKJWf4ilbRaICn1QSJjNuKwtl
-A2ZwEw7FFddFcsF4okHPkxEoCre3shq0DEw9UA1gnwEaEF1Q+i9cvLtQg56zK3+R
-MfTzFUv2K1z3j1s5tC6Z6LzL8j682EFOll7MkihqiIWh+po1zcNs6Ln0SkjB3v9T
-OFRYVJis9GqXYO9PUW1mI69udR4nXbKlBl5tWNyAxn9hufJHLSAmbGMwActKLRM5
-kROapKrN1Ra/sHUmhoSSDUK9AXeYKkqPXVMbhly9xC1Kya4hFaFokjRQIMLPmXdQ
-lP46RL3HOq0v1BkXg05poSwDm+BtitZkcIhOzcmeucUqn1whsz39y3VoOhJqQu2j
-0dUcj19Fue1CDZfwzACb9mLZr643h0yCIHQaVT7is/yiN2vEt0S0zynvOA1D+Xgy
-4bFdjqed9DgBShLwF/lh4/hur5iZPpVr9A8rt3Vq0k5A1tdjsZpXP3qv5jLc7lJv
-cAQxdVLEuv8WPwwTV/EQsoH7lpoEdjktw/gu0v5PkUgR/WBxEqJq4liTekViQ+8X
-BAmdm/FGWeIxSSEm0a8I+muiBpIuKjuQoJit1f0k+lLUohoVDqpDDfeYHh1TxXHE
-xSKAKNmFSc2XOr5Zg2mOInxeH7WoxH6iylXKADBgEgAQ4C55DoAPRjFNtGGz2iJl
-WkzhQOfgsDss1VxpPZ4SWDQO+xxuuEnYcCYdJ6Tyi5hAwZ0+RZmXx0g9BCK+5qzc
-Ffei066kCItBcayekwCs0i86r8JvDtsMUUkzEKDQTSWHpP8ypIZoz3P+Skdqcs6p
-IcZHy2fODIsClctRAsHOjZ9BnWSY2rqf1lysqvRfOkNTViTOUVx2hf+LuL+8rT6g
-K4GvS39Q+c9FYSMjCNxv1m37NfHJv2W7AlqVV95gMBEFB97M3388T7PfvmSLsyPv
-qAKDE7genTOycQvEX9oeEACxPo08ai2Mgs/YXCig0g0btZoScoKPelQKtD0HLpZ+
-j7ruaKFy4/vySDMostB3AKSkCE/SkGu/W9oI37n1guSYC/9yJjtTcwZULXAClfwt
-LyHvLAw4+7BdGJqDe9Tf8z4zqVLax+BcBHkW7pbP+pvxC6fhIA4PO8k5HwwAVlRw
-68lG6e3z9+Vst4zrFLewHJvkDRSOTSecs9h9SPOjReTK0ovlDzMHVakfUvqpk9mv
-nCj8SWaMPF70z138AuRuKN7X4exNXPBFnOcseKp/NpT1DyvgoHm/oIO65mq9vvo+
-42uSbyyegYiTdVK4SaXBc2hixkhaTteqdEy6GxfWqKCvHtYuRD2LyUSjZn/0Y6CG
-3wZapylKL1sxfH+b1gogv2wAA8KtZAfiTlMISuymwQr5RbOFma7UVMVecCa7XrNG
-fZwO8R+6++ru0efDzeC/gtLOItjn+yVfsbk2gfu0FikHIV1sogOjnsv3dl6IpH/s
-w6WbuK/yZhyrrmhIIueHF30ES512Peogf/Gxp8U/ZAmqBa7p4qsv93XWDR52IPYz
-yU1MtboLD02J39daBNrYZuO4Yfo/xAvU7Ijg2KIR9HS0bSWNuSFAZaZXlJ2bSag3
-8l0dZMx2fZa0FALxWvY2+nd1tWcH+Wo99RSxX+4TH3guCpe1x/58yKOKL4d1xRpu
-02nGaqerV9bHU6WjCUYTrrZurTp6yqivpSFCALDMhAU5Ihd/Qjz69zg4uZ7iqbNd
-WqIYJ3SAOei2tSfoOJbdh8mohotnMFFbupck3Fv+lXAXYDp0V/sSuK32MSeJxan1
-q3t0eenAh6zoBP0Tj53N8B9YmrgZ1i1AljQqYsAdb8N5Z3vecuzQa946U8H4axEc
-BL5fgmqSfsbj1EjJVUrw3cE9hgqeqAYBBvZsyW5zSAsLq/RJKYtBxDGADqr0TOOu
-TjVqfPObb1u9EAQj15ALBhhPORa2gBAK5gnhJWW5deNgA0UOYoWARFYfATuQvr1T
-9xTVbqzZ0VJR4n7X5JyxJo2tZ44QYVzA0fqX0lt+uRkarrkbZMgplKp4MJTFCp2N
-tLciWS0YSHPbKTm7svw5mROazUvEoj1N3SYrJT+Ai9UgSL1cA4FmQ5Jsi+GOeQbH
-7E4pzjeST4b1aATY9ZZSy4TzK+UNYBcCVqY1dN5Xnk0e5yPMnrkkcYI41s/6oOaW
-lQkp/yrHkvpzQWZa7vF2yWg54j3eX8YWx/TsLvmuJmSQqiIHfu9TXcAIPDispCpF
-gp4ZbYPZ8RDQLw3ROc+bZ9E0J8e+FHwgXEELsMLW9n4d3PTj9pLFej1vmBKa03o3
-g+JViNxb1gUPNmC6ADk4o7TQzBrjV2pmmuCza16kOmq6oAbJPOeCLtVWA5HJTX8d
-AuOzBLO37tWP457dNqoNDaCMik3LCy8cslZjAz02D6VG/QGOAXP9x8bJ3zg9N7vW
-EOn1BMtIeMI4yIxpC6HgotNWDNbUBWremAFWYQRgYQqo9L1SdW4hz6X8FqdlBFvK
-ml7sdUwjEy2tNW+aiZI4cgAlI/VvMWYr5kPfhvaUu1wGYeS1TD1B+DPcjcXpOGWo
-vB1f/uvGc61zZWD0ph9f3S9OnvEko5qV3VhxOnxYUZTrqlM7LVgy/sHrSaPdR7Eo
-nb4QGdi5wEGMxVgQlDGwOmEmS5nCaA0RcfHbkQV4SebRleocLB9qcSwj5JzPICAB
-sn1qeSv45LnGdTzUnCQErw1EzgrK1FXOknujjZ3GX5dM7a6Gag0E6cAxoQeC8Mh0
-aUWolyBo9aSXltn7Vnywoe4X3v96eVH6tUQtN3wt51LBlfSDTWOQDeXLFyW57oYL
-/ErbmJLaSHpANBcC9InI+rxNYePTTba7K8rdPDppmMEmdOXjaL+5aPay2Bay9K8T
-FUyZdoyrbzxFZYT9/YyVJV/ukVcT9ru3HpzLTg9Vgc/CyRfIvCZY/Gzh+pPUbRsR
-pHYDT09BFS+mjszTkcdbceweYXJvknXpajMiISrPUUkb9rwCTmLS4QR4y7I9zAgJ
-9U72mCZB9yhhzS17kyKAANihXvOFrMXCO7QTWZbtSkobMBS2SybDxYP3rM5b6824
-HQaY4MZW5qCndydfz4fa908XbPTDka9Nu3CZfI3fhGiiGDFWViotGEYE4i9BuGZJ
-vmtzII7zeIoHwM4t5fb5mYDd9H+Rkct1yATGEIFEx7cu1fj4Ht2177EzhdkF3Q6I
-Fccn43yGDW06nFvEjnjb+Lz4EvtGp6dDefUhNBohdzd4Rm2o+etKobm0lZ1xOX74
-2gOGBLfXS6PF+msNkWP76mqKE6hG0+R1iPrN06Arti+ZRBhNGJDp1pnOERl7IyB0
-hiLUqae7xVSMVRDeq4O04EUpMXh5ixs3RJXjthYILt7vX+6qskJC3rMuyThrsvMi
-Kk0wErDpzR5U7KFnM2fk4jY5by/0b/jHZ/PWFGnaFrXNOiwK/mV1RIJtg5FCrvMv
-xJN5Qo7o2ZYzUL+IBf5mP5O49FCnz3e++9r4ySY/K57ybhQft1l+cQPWS7xuvVAg
-wZGM9DPl9AhBus0I3V3XzfkcWWHkMynMRcfTnkBHrmKjMopIOggsq05JPalM9b0V
-NfwZngVvTFbQejbGjgQQQ+G7spav/icCIA8Iu8JPHaKZt2IX77L1xMcQOouqMrx8
-JCXnJ7zsuU+8W3GQGm70H2kcen4bf4beA8TSkFmbVsaqDNtGaG+ZopWO8ctZX3CP
-jFgIYgEGi137IWvs1KD5QNFHZI6l0JMzglmvcfnViRPDHLi+EQXAoL/98j/YzA9t
-a9L4xMO0oucovKkqYOS2cdFaQKVIUM9L5ibcbW7Tol4ugMo5ibIsTgjZoyFohG6t
-acg+h2wTTyxtrIapvOOBeAo9V/iQowTAPQf38LKQZ5tApksiCG1qvJerKALTMp1J
-yUHjTCPCPsaiXVVO0mfa9OPtzVvi/icLmlz7umk82mew13djedgMS3/ghq5aEHbg
-e7rtI5snFlm/GQnV6IaCDytb+jYxqQ0DZ9Mb0CHcMpnLyiBO6nzHfadpZYlrxrpx
-dPx9MVqfLavj1UVjKuVj4Mgec9J38IBBRGMJZ4MGhde2hrw/s2jVzG1BzaIzcvFa
-bWiJOkKKRC4BY9DBDYfae5c/obwzkxLmvkbWAVzoFXc+46K1IK6bbC2a4EerOAxc
-C7B3X8BucD8OtiPORwXgjctd9AJ7cXq0EEVGoZZ8jkILnklsK6RXsxrT4gSOGFpx
-cBU7TF4kHpH+o4KGQmjlFA2VYP1tXirjEep57LOHfL+zy+HiSXdsbexf6odkqLYA
-1RR0TzSkmqJYvP0GfHPMvp2xredx1kP934fhTIUcWvErrG3NPd/Dod4BzQye1nNK
-8Y8aFYF6fACcD4SFZ11ecK9DJmB+WXGlsxtO3w1LeEysny2lVkkH6HuMRIzPyXAM
-/sEpwh3hwPJosAh6vGM6JxuL7CXP8hC1oUvK91VR0hxeDcO/LgdcHtDnlNMqt1wC
-jvYt9Hg+3aY5GehBnQPRppnIMaGhiBrRC6HjBg8Umt6UTUr5PXNY2aia5wRcx4M6
-Tj0O0dAt8WWZ9ySgzOoq4SRoaAp6m4E7CpRNMgXH031OWEeHqtt8rsYqAcvcQLuW
-vIwts5hiXZOfgXyR6SBY3ANE1SjDY46na2G4oiG8FI7eAqYgehjHrXqCnGrzjefE
-V1eVcIr7DBYuM8wq7C9x+qdMir6RNV4MrVD+7rO/drivnd3MaqDHlG0LfMZo6GIK
-QZGIy0z1W5lOXm9x3RAz7/Ea11ynkZyPDDxBE5NRcV6JfukdjDrvsaQA/Qk2Wo7h
-PdUZjv17H4syMdap6tUDTc8Cohut6n1UxnDnhaAYEDGR3TgPqI5vEaJ+yoaM9itv
-b2I/K+0uK6GfWwca7PeTtYn92/5etO4czG99xEBFzw7tiEmBTA8KfseZIUe/i3fr
-9LO+rKEGalFB4mVTYyCKPYYx4hOIvjdCox/jEBTFyYUEhgGnZvArgw9XabvII7eM
-c1FmwnN2CM/MHUzM0Kw7cS/oaoRFcG/PqioO/krL2Bd+xThwSH4Zn+Y1UVO7aXNy
-90MOehHSc2nhN984Y2re3ffVsjPhC3KdWxr8jbho/Az65rUkYg64roiO2ZqdqRvf
-aYtX9fCe8N8SzgLTlWZt+l8sjY8HP2k9NoajphV1nG4iC/pX0ZJiVQ+VBRNaGhSl
-6IhM9gM1ZsI86cULJpOTAhNu1fZBv+Mq8IXYD8SqlNVwVtjvshtlzBN4InSGEKiO
-HM7coo7aDuu+Go5AEiIVMbR09jAa0VINsCVScfSbTFhlGRTrZr2DSO/s8WxbV+Mb
-EQAfZmyrLF71Z3MfIukg+ps5iuqFhZ+gJSXXSFwumpg=
-EOF
-
-exec sh bin/do-setup-circleci-secrets
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# shellcheck disable=SC2155
+wait_for() {
+    local timeout=$1
+    local lock_file=$2
+    for i in $(seq 1 "$timeout"); do
+        if [ -f "$lock_file" ]; then
+            local status="$(cat "$lock_file")"
+            if [ "$status" == "OK" ]; then
+                echo "[$i seconds]: $lock_file found and status: $status." && return
+            else
+                echo "[$i seconds]: $lock_file found and status: $status." && return 1
+            fi
+        fi
+        if ! ((i % 10)); then echo "[$i seconds]: Waiting for $lock_file to be created..."; fi
+        sleep 1
+    done
+    echo "Timed out waiting for test VMs to be ready. See details in: $TEST_VMS_SETUP_OUTPUT_FILE" >&2
+    return 1
+}
@@ -45,6 +45,8 @@ RUN apt-get update \
 RUN go get \
 	github.com/golang/lint/golint \
 	github.com/fzipp/gocyclo \
+	gopkg.in/mvdan/sh.v1/cmd/shfmt \
+	github.com/fatih/hclfmt \
 	github.com/client9/misspell/cmd/misspell

 # First clean, because we're gonna rebuild the std both with -race enabled and disabled
@@ -14,17 +14,32 @@ machine:
    PATH: $PATH:$HOME/.local/bin
    CLOUDSDK_CORE_DISABLE_PROMPTS: 1
    STATE: /home/ubuntu/state.env
+    NUM_HOSTS: 16
+    # Base name of VMs for integration tests:
+    NAME: test-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX
+    TEST_VMS_SETUP_OUTPUT_FILE: $CIRCLE_ARTIFACTS/test_vms_setup_output.txt
+    TEST_VMS_PROV_AND_CONF_LOCK_FILE: $CIRCLE_ARTIFACTS/test_vms_provisioned_and_configured.txt
+    TEST_VMS_READY_LOCK_FILE: $CIRCLE_ARTIFACTS/test_vms_ready.txt
+    # Google Cloud Platorm images for integration tests:
+    USE_IMAGE: 1
+    DISK_NAME_PREFIX: test-$CIRCLE_BUILD_NUM-0

 dependencies:
  cache_directories:
    - "~/docker"
  post:
+    - bin/circle-dependencies-post-sources
+    # Provision & configure testing VMs:
+    - "while [ ! -d $SRCDIR ]; do sleep 1; done; $SRCDIR/bin/provision_test_vms.sh":
+        parallel: true
+        background: true
    - bin/circle-dependencies-post

 test:
  pre:
    - "$SRCDIR/bin/circle-test-pre":
        parallel: true
+        background: true
  override:
    - "$SRCDIR/bin/circle-test-unit":
        parallel: true
@@ -37,6 +52,10 @@ test:

 teardown:
  pre:
+    # Destroy testing VMs:
+    - "cd $SRCDIR/test && ./run-integration-tests.sh destroy":
+        parallel: true
+        background: true
    - "$SRCDIR/bin/circle-teardown-pre":
        parallel: true

@@ -3,7 +3,7 @@
 # TODO(mp) 1. test anti-replay 2. test sleeve <-> fastdp
 # TODO(mp) test with --trusted-subnets

-. ./config.sh
+. "$(dirname "$0")/config.sh"

 HOST1_IP=$($SSH $HOST1 getent ahosts $HOST1 | grep "RAW" | cut -d' ' -f1)
 HOST2_IP=$($SSH $HOST2 getent ahosts $HOST2 | grep "RAW" | cut -d' ' -f1)
@@ -31,7 +31,7 @@ sed -e "s%imagePullPolicy: Always%imagePullPolicy: Never$COVERAGE_ARGS%" "$(dirn
 sleep 5

 wait_for_connections() {
-    for i in $(seq 1 30); do
+    for i in $(seq 1 45); do
        if run_on $HOST1 "curl -sS http://127.0.0.1:6784/status | grep \"$SUCCESS\"" ; then
            return
        fi
@@ -56,7 +56,7 @@ assert_raises "run_on $HOST3 $PING $HOST2EXPIP"
 run_on $HOST1 "kubectl run hello --image=weaveworks/hello-world --replicas=3"

 wait_for_pods() {
-    for i in $(seq 1 30); do
+    for i in $(seq 1 45); do
        if run_on $HOST1 "kubectl get pods | grep 'hello.*Running'" ; then
            return
        fi
@@ -2,40 +2,67 @@ This directory contains integration tests for weave.

 ## Requirements

-You need two VMs with docker >=1.6.0 installed and listening on TCP
-port 2375 (see below). You also need to be able to ssh to these VMs,
-preferably without having to input anything.
+You need three VMs with `docker` (`>=1.6.0`) installed and listening on TCP
+port `2375` (see below). You also need to be able to SSH to these VMs without 
+having to input anything.

-The `Vagrantfile` in this directory constructs two such VMs.
+The `Vagrantfile` in this directory constructs three such VMs.

-To create the VMs, open a shell and in this directory and type
+If you are [building Weave Net using Vagrant](https://www.weave.works/docs/net/latest/building/),
+it is recommended to run the tests from the build VM and not the host.

-    vagrant up
-
-To meet the aforementioned ssh requirement you may want to
-
-    cp ~/.vagrant.d/insecure_private_key .

 ## Running tests

-If you are [building weave using Vagrant](https://www.weave.works/docs/net/latest/building/),
-it is recommended to run the tests from the build VM and not the host.
+**TL;DR**: You can run the steps 2. to 7. below with one command: 

-    ./setup.sh
+    make PROVIDER=vagrant integration-tests

-uploads the weave images from where the Makefile puts them
-(`/var/tmp`) to the two docker hosts, and copies the weave script
-over.
+**Detailed steps**:

-Then you can use, e.g.,
+  1. Start the build virtual machine (see above article for more details):

-    ./200_dns_test.sh
+        vagrant up

-to run an individual test, or
+  2. Start the three testing VMs:

-    ./run_all.sh
+        cd test
+        vagrant up
+
+  3. SSH into the build VM and go to Weave Net's sources:
+
+        cd ..
+        vagrant ssh
+        # you are now on the build VM:
+        cd ~/weave 
+
+  4. Compile all code and dependencies:
+
+        make
+        make testrunner
+        cd test
+
+  5. Upload the weave images from where the `Makefile` puts them (`weave.tar.gz`) to 
+     the three docker hosts, `docker load` these, and copies the `weave` script over:
+
+        ./setup.sh
+
+  6. Run individual tests, e.g.:
+
+        ./200_dns_test.sh
+
+     or run all tests (everything named `*_test.sh`):
+
+        ./run_all.sh
+
+  7. Stop all VMs:
+
+        exit
+        # you are now on your host machine
+        vagrant destroy -f
+        cd test
+        vagrant destroy -f

-to run everything named `*_test.sh`.

 ## Using other VMs

@@ -60,20 +87,7 @@ to the file `/etc/default/docker`, then restart docker.

 ## Updating the GCE test image

-When a new version of Docker is released, you'll need to update the GCE test image.
-
-To do this, pick a fresh ```TEMPLATE_NAME``` and update any commands in
-```function make_template``` in gce.sh, then run:
-
-```
-./gce.sh make_template
-```
-
-For this you'll need the GCE credentials, which can be found in ```bin/setup-circleci-secrets```,
-which you'll need to decrypt and run (its echos the secrets into know locations):
-
-```
-./bin/setup-circleci-secrets "$SECRET_PASSWORD"
-```
-
-If you don't know the password, ask tom@weave.works.
+When a new version of Docker is released, you willneed to update the GCE test image.
+To do this, change the Docker version in `run-integration-tests.sh` and push the change.
+Next build in CircleCI will detect that there is no template for this version of Docker and will first create the template before running tests.
+Subsequent builds will then simply re-use the template.
@@ -1,59 +1,64 @@
-VAGRANTFILE_API_VERSION = "2"
-
 # Necessary because the switch disabling unwanted insecure key
 # replacement in Vagrant 1.7+ is not accepted by earlier versions
 Vagrant.require_version ">= 1.7.0"

 # these ought to match what is in config.sh
-n_machines = 3
-ip_prefix = "192.168.48"
-ip_suffix_base = 10
+N_MACHINES = 3
+IP_PREFIX = '192.168.48'
+IP_SUFFIX_BASE = 10

-require '../vagrant-common.rb'
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'vagrant-common.rb'))

-def configure_docker(host, hostname, ip)
-  pkgs = %w(docker-engine ethtool)
-
-  host.vm.box = "ubuntu/wily64"
-
-  host.vm.provision :shell, :inline => "hostnamectl set-hostname "+hostname
-  host.vm.network "private_network", ip: ip
-
-  host.vm.synced_folder ".", "/vagrant", disabled: true
-
-  host.vm.provision :shell, :inline => "sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D"
-  host.vm.provision :shell, :inline => "echo deb https://apt.dockerproject.org/repo ubuntu-wily main > /etc/apt/sources.list.d/docker.list"
-
-  install_packages host.vm, pkgs
-  tweak_docker_daemon host.vm
-  cleanup host.vm
+def ip_address(i)
+  "#{IP_PREFIX}.#{IP_SUFFIX_BASE + i}"
 end

-def configure_resolv_conf(host)
-  # Fix the resolution errors by using 8.8.8.8
-  host.vm.provision :shell, :inline => "echo 'nameserver 8.8.8.8' > /etc/resolv.conf"
+def hostname(i)
+  "host#{i}"
 end

+ETC_HOSTS_ENTRIES = (1..N_MACHINES).map { |i| "#{ip_address(i)} #{hostname(i)}\n"}.join('')

 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+  (1..N_MACHINES).each do |i|
+    hostname = hostname(i)
+    config.vm.define hostname do |host|
+      host.vm.provider :virtualbox do |vb|
+        vb.name = hostname
+        vb.memory = 384
+        configure_nat_dns(vb)
+      end

-  config.vm.provider :virtualbox do |vb|
-    vb.customize ["modifyvm", :id, "--natdnshostresolver1", "off"]
-    vb.customize ["modifyvm", :id, "--natdnsproxy1", "off"]
-    vb.memory = 384
-  end
+      # Disable Vagrant 1.7 per host insecure key replacement
+      host.ssh.insert_key = false

-  # Disable Vagrant 1.7 per host insecure key replacement
-  config.ssh.insert_key = false
+      host.vm.box = VAGRANT_IMAGE
+      host.vm.network "private_network", ip: ip_address(i)
+      host.vm.hostname = hostname

-  (1..n_machines).each do |i|
-    config.vm.define "host#{i}" do |host|
-      ip_suffix = ip_suffix_base + i
-      configure_resolv_conf(host)
-      configure_docker(host, "host#{i}", "#{ip_prefix}.#{ip_suffix}")
+      host.vm.provision :shell, :inline => <<END
+# Tests rely on hostnames, so we remove potentially incorrect entries (e.g. localhost), and add mappings of IP addresses to hostnames:
+sed -i '/#{hostname}/d' /etc/hosts
+echo "#{ETC_HOSTS_ENTRIES}" >> /etc/hosts
+
+# Fix the resolution errors by using 8.8.8.8
+echo 'nameserver 8.8.8.8' > /etc/resolv.conf
+END
+
+      # Disable default Vagrant shared folder, which we don't need:
+      host.vm.synced_folder ".", "/vagrant", disabled: true
+
+      cleanup host.vm
+
+      # Only execute once the Ansible provisioner, when all the machines are up and ready:
+      if i == N_MACHINES
+        host.vm.provision 'ansible' do |ansible|
+          ansible.limit = 'all'  # Disable default limit to connect to all the machines
+          ansible.playbook = '../tools/config_management/setup_weave-net_test.yml'
+        end
+      end
    end
  end
-
 end

 begin
@@ -30,9 +30,9 @@ HOST3=$(echo $HOSTS | cut -f 3 -d ' ')

 . "$DIR/assert.sh"

-
 SSH_DIR=${SSH_DIR:-$DIR}
-SSH=${SSH:-ssh -l vagrant -i "$SSH_DIR/insecure_private_key" -o "UserKnownHostsFile=$SSH_DIR/.ssh_known_hosts" -o CheckHostIP=no -o StrictHostKeyChecking=no}
+SSH_OPTS=${SSH_OPTS:-"-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no -o IdentitiesOnly=yes -o LogLevel=ERROR"}
+SSH=${SSH:-ssh -l vagrant -i "$SSH_DIR/insecure_private_key" $SSH_OPTS}

 SMALL_IMAGE="alpine"
 DNS_IMAGE="aanand/docker-dnsutils"
@@ -1,207 +0,0 @@
-#!/bin/bash
-# This script has a bunch of GCE-related functions:
-# ./gce.sh setup -  starts two VMs on GCE and configures them to run our integration tests
-# . ./gce.sh; ./run_all.sh - set a bunch of environment variables for the tests
-# ./gce.sh destroy - tear down the VMs
-# ./gce.sh make_template - make a fresh VM template; update TEMPLATE_NAME first!
-
-set -e
-
-: ${KEY_FILE:=/tmp/gce_private_key.json}
-: ${SSH_KEY_FILE:=$HOME/.ssh/gce_ssh_key}
-: ${PROJECT:=positive-cocoa-90213}
-: ${IMAGE_FAMILY:=ubuntu-1604-lts}
-: ${IMAGE_PROJECT:=ubuntu-os-cloud}
-: ${TEMPLATE_NAME:=test-template-12}
-: ${ZONE:=us-central1-a}
-: ${NUM_HOSTS:=5}
-SUFFIX=""
-if [ -n "$CIRCLECI" ]; then
-	SUFFIX="-${CIRCLE_BUILD_NUM}-$CIRCLE_NODE_INDEX"
-fi
-
-# Setup authentication
-gcloud auth activate-service-account --key-file $KEY_FILE 1>/dev/null
-gcloud config set project $PROJECT
-
-function vm_names {
-	local names=
-	for i in $(seq 1 $NUM_HOSTS); do
-		names="host$i$SUFFIX $names"
-	done
-	echo "$names"
-}
-
-# Delete all vms in this account
-function destroy {
-	if [ $(gcloud compute firewall-rules list test-allow-docker$SUFFIX 2>/dev/null | wc -l) -gt 0 ] ; then
-		gcloud compute firewall-rules delete test-allow-docker$SUFFIX
-	fi
-	names="$(vm_names)"
-	if [ $(gcloud compute instances list --zones $ZONE -q $names | wc -l) -le 1 ] ; then
-		return 0
-	fi
-	for i in {0..10}; do
-		# gcloud instances delete can sometimes hang.
-		case  $(set +e; timeout 60s /bin/bash -c "gcloud compute instances delete --zone $ZONE -q $names  >/dev/null 2>&1"; echo $?) in
-			0)
-				return 0
-				;;
-			124)
-				# 124 means it timed out
-				break
-				;;
-			*)
-				return 1
-		esac
-	done
-}
-
-function internal_ip {
-	jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].networkIP" $1
-}
-
-function external_ip {
-	jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].accessConfigs[0].natIP" $1
-}
-
-function try_connect {
-	for i in {0..10}; do
-		ssh -t $1 true && return
-		sleep 2
-	done
-}
-
-function install_docker_on {
-	name=$1
-	ssh -t $name sudo bash -x -s <<EOF
-curl -sSL https://get.docker.com/gpg | sudo apt-key add -
-curl -sSL https://get.docker.com/ | sed -e s/docker-engine/docker-engine=1.11.2-0~xenial/ | sh
-apt-get update -qq;
-apt-get install -q -y --force-yes --no-install-recommends ethtool;
-apt-get install -q -y bc jq;
-usermod -a -G docker vagrant;
-mkdir -p /etc/systemd/system/docker.service.d
-cat >/etc/systemd/system/docker.service.d/override.conf  <<OVERRIDE
-[Service]
-ExecStart=
-ExecStart=/usr/bin/docker daemon -H fd:// -H unix:///var/run/alt-docker.sock -H tcp://0.0.0.0:2375 -s overlay
-OVERRIDE
-systemctl daemon-reload
-systemctl restart docker
-# This installs nsenter.
-docker run --rm -v /usr/local/bin:/target jpetazzo/nsenter
-docker pull alpine
-docker pull aanand/docker-dnsutils
-docker pull weaveworks/hello-world
-EOF
-}
-
-function install_kubernetes_on {
-	name=$1
-	ssh -t $name sudo bash -x -s <<EOF
-curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
-echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list
-apt-get update -qq
-apt-get install -q -y kubelet kubeadm kubectl kubernetes-cni
-systemctl --now disable kubelet
-# Pre-pull images required for Kubernetes
-docker pull gcr.io/google_containers/etcd-amd64:2.2.5
-docker pull gcr.io/google_containers/kube-apiserver-amd64:v1.4.0
-docker pull gcr.io/google_containers/kube-controller-manager-amd64:v1.4.0
-docker pull gcr.io/google_containers/kube-proxy-amd64:v1.4.0
-docker pull gcr.io/google_containers/kube-scheduler-amd64:v1.4.0
-docker pull gcr.io/google_containers/kube-discovery-amd64:1.0
-docker pull gcr.io/google_containers/pause-amd64:3.0
-EOF
-}
-
-function copy_hosts {
-	hostname=$1
-	hosts=$2
-	cat $hosts | ssh -t "$hostname" "sudo -- sh -c \"cat >>/etc/hosts\""
-}
-
-# Create new set of VMs
-function setup {
-	destroy
-
-	names="$(vm_names)"
-	gcloud compute instances create $names --image $TEMPLATE_NAME --zone $ZONE --tags test$SUFFIX --network=test
-	my_ip="$(curl -s http://ipinfo.io/ip)"
-	gcloud compute firewall-rules create test-allow-docker$SUFFIX --network=test --allow tcp:2375,tcp:12375 --target-tags test$SUFFIX --source-ranges $my_ip
-	gcloud compute config-ssh --ssh-key-file $SSH_KEY_FILE
-	sed -i '/UserKnownHostsFile=\/dev\/null/d' ~/.ssh/config
-
-	# build an /etc/hosts file for these vms
-	hosts=$(mktemp hosts.XXXXXXXXXX)
-	json=$(mktemp json.XXXXXXXXXX)
-	gcloud compute instances list --format=json >$json
-	for name in $names; do
-		echo "$(internal_ip $json $name) $name.$ZONE.$PROJECT" >>$hosts
-	done
-
-	for name in $names; do
-		hostname="$name.$ZONE.$PROJECT"
-
-		# Add the remote ip to the local /etc/hosts
-		sudo sed -i "/$hostname/d" /etc/hosts
-		sudo sh -c "echo \"$(external_ip $json $name) $hostname\" >>/etc/hosts"
-		try_connect $hostname
-
-		copy_hosts $hostname $hosts &
-	done
-
-	wait
-
-	rm $hosts $json
-}
-
-function make_template {
-	gcloud compute instances create $TEMPLATE_NAME --image-family=$IMAGE_FAMILY --image-project=$IMAGE_PROJECT --zone $ZONE
-	gcloud compute config-ssh --ssh-key-file $SSH_KEY_FILE
-	name="$TEMPLATE_NAME.$ZONE.$PROJECT"
-	try_connect $name
-	install_docker_on $name
-	install_kubernetes_on $name
-	gcloud -q compute instances delete $TEMPLATE_NAME --keep-disks boot --zone $ZONE
-	gcloud compute images create $TEMPLATE_NAME --source-disk $TEMPLATE_NAME --source-disk-zone $ZONE
-}
-
-function hosts {
-	hosts=
-	json=$(mktemp json.XXXXXXXXXX)
-	gcloud compute instances list --format=json >$json
-	for name in $(vm_names); do
-		hostname="$name.$ZONE.$PROJECT"
-		hosts="$hostname $hosts"
-	done
-	echo export SSH=\"ssh -l vagrant\"
-	echo export HOSTS=\"$hosts\"
-	rm $json
-}
-
-case "$1" in
-setup)
-	setup
-	;;
-
-hosts)
-	hosts
-	;;
-
-destroy)
-	destroy
-	;;
-
-make_template)
-	# see if template exists
-	if ! gcloud compute images list | grep $PROJECT | grep $TEMPLATE_NAME; then
-		make_template
-	fi
-    ;;
-
-*)
-	echo "Unknown command:" $1 >&2
-    exit 1
-esac
@@ -0,0 +1,383 @@
+#!/bin/bash
+#
+# Description:
+#   This script runs all Weave Net's integration tests on the specified
+#   provider (default: Google Cloud Platform).
+#
+# Usage:
+#
+#   Run all integration tests on Google Cloud Platform:
+#   $ ./run-integration-tests.sh
+#
+#   Run all integration tests on Amazon Web Services:
+#   PROVIDER=aws ./run-integration-tests.sh
+#
+
+set -e
+DIR="$(dirname "$0")"
+. "$DIR/../tools/provisioning/setup.sh" # Import gcp_on, do_on, and aws_on.
+. "$DIR/config.sh"                      # Import greenly.
+
+# Variables:
+APP="weave-net"
+# shellcheck disable=SC2034
+PROJECT="weave-net-tests" # Only used when PROVIDER is gcp, by tools/provisioning/config.sh.
+NAME=${NAME:-"$(whoami | sed -e 's/[\.\_]*//g' | cut -c 1-4)"}
+PROVIDER=${PROVIDER:-gcp} # Provision using provided provider, or Google Cloud Platform by default.
+NUM_HOSTS=${NUM_HOSTS:-3}
+PLAYBOOK=${PLAYBOOK:-setup_weave-net_test.yml}
+TESTS=${TESTS:-}
+RUNNER_ARGS=${RUNNER_ARGS:-""}
+# Dependencies' versions:
+DOCKER_VERSION=${DOCKER_VERSION:-1.11.2}
+KUBERNETES_VERSION=${KUBERNETES_VERSION:-1.5.2}
+KUBERNETES_CNI_VERSION=${KUBERNETES_CNI_VERSION:-0.3.0.1}
+# Google Cloud Platform image's name & usage (only used when PROVIDER is gcp):
+IMAGE_NAME=${IMAGE_NAME:-"$(echo "$APP-docker$DOCKER_VERSION-k8s$KUBERNETES_VERSION-k8scni$KUBERNETES_CNI_VERSION" | sed -e 's/[\.\_]*//g')"}
+DISK_NAME_PREFIX=${DISK_NAME_PREFIX:-$NAME}
+USE_IMAGE=${USE_IMAGE:-1}
+CREATE_IMAGE=${CREATE_IMAGE:-1}
+CREATE_IMAGE_TIMEOUT_IN_SECS=${CREATE_IMAGE_TIMEOUT_IN_SECS:-600}
+# Lifecycle flags:
+SKIP_CONFIG=${SKIP_CONFIG:-}
+SKIP_DESTROY=${SKIP_DESTROY:-}
+
+function print_vars() {
+    echo "--- Variables: Main ---"
+    echo "PROVIDER=$PROVIDER"
+    echo "NUM_HOSTS=$NUM_HOSTS"
+    echo "PLAYBOOK=$PLAYBOOK"
+    echo "TESTS=$TESTS"
+    echo "SSH_OPTS=$SSH_OPTS"
+    echo "RUNNER_ARGS=$RUNNER_ARGS"
+    echo "--- Variables: Versions ---"
+    echo "DOCKER_VERSION=$DOCKER_VERSION"
+    echo "KUBERNETES_VERSION=$KUBERNETES_VERSION"
+    echo "KUBERNETES_CNI_VERSION=$KUBERNETES_CNI_VERSION"
+    echo "IMAGE_NAME=$IMAGE_NAME"
+    echo "DISK_NAME_PREFIX=$DISK_NAME_PREFIX"
+    echo "USE_IMAGE=$USE_IMAGE"
+    echo "CREATE_IMAGE=$CREATE_IMAGE"
+    echo "CREATE_IMAGE_TIMEOUT_IN_SECS=$CREATE_IMAGE_TIMEOUT_IN_SECS"
+    echo "--- Variables: Flags ---"
+    echo "SKIP_CONFIG=$SKIP_CONFIG"
+    echo "SKIP_DESTROY=$SKIP_DESTROY"
+}
+
+function verify_dependencies() {
+    local deps=(python terraform ansible-playbook)
+    for dep in "${deps[@]}"; do
+        if [ ! "$(which "$dep")" ]; then
+            echo >&2 "$dep is not installed or not in PATH."
+            exit 1
+        fi
+    done
+}
+
+# shellcheck disable=SC2155
+function provision_locally() {
+    export VAGRANT_CWD="$(dirname "${BASH_SOURCE[0]}")"
+    case "$1" in
+        on)
+            vagrant up
+            local status=$?
+
+            # Set up SSH connection details: 
+            local ssh_config=$(mktemp /tmp/vagrant_ssh_config_XXX)
+            vagrant ssh-config >"$ssh_config"
+            export SSH="ssh -F $ssh_config"
+            # Extract username, SSH private key, and VMs' IP addresses:
+            ssh_user="$(sed -ne 's/\ *User //p' "$ssh_config" | uniq)"
+            ssh_id_file="$(sed -ne 's/\ *IdentityFile //p' "$ssh_config" | uniq)"
+            ssh_hosts=$(sed -ne 's/Host //p' "$ssh_config")
+
+            # Set up /etc/hosts files on this ("local") machine and the ("remote") testing machines, to map hostnames and IP addresses, so that:
+            # - this machine communicates with the testing machines via their public IPs;
+            # - testing machines communicate between themselves via their private IPs;
+            # - we can simply use just the hostname in all scripts to refer to machines, and the difference between public and private IP becomes transparent.
+            # N.B.: if you decide to use public IPs everywhere, note that some tests may fail (e.g. test #115).
+            update_local_etc_hosts "$ssh_hosts" "$(for host in $ssh_hosts; do $SSH "$host" "cat /etc/hosts | grep $host"; done)"
+
+            SKIP_CONFIG=1 # Vagrant directly configures virtual machines using Ansible -- see also: Vagrantfile
+            return $status
+            ;;
+        off)
+            vagrant destroy -f
+            ;;
+        *)
+            echo >&2 "Unknown command $1. Usage: {on|off}."
+            exit 1
+            ;;
+    esac
+}
+
+function setup_gcloud() {
+    # Authenticate:
+    gcloud auth activate-service-account --key-file "$GOOGLE_CREDENTIALS_FILE" 1>/dev/null
+    # Set current project:
+    gcloud config set project $PROJECT
+}
+
+function image_exists() {
+    gcloud compute images list | grep "$PROJECT" | grep "$IMAGE_NAME"
+}
+
+function image_ready() {
+    # GCP images seem to be listed before they are actually ready for use, 
+    # typically failing the build with: "googleapi: Error 400: The resource is not ready".
+    # We therefore consider the image to be ready once the disk of its template instance has been deleted.
+    ! gcloud compute disks list | grep "$DISK_NAME_PREFIX"
+}
+
+function wait_for_image() {
+    greenly echo "> Waiting for GCP image $IMAGE_NAME to be created..."
+    for i in $(seq "$CREATE_IMAGE_TIMEOUT_IN_SECS"); do
+        image_exists && image_ready && return 0
+        if ! ((i % 60)); then echo "Waited for $i seconds and still waiting..."; fi
+        sleep 1
+    done
+    redly echo "> Waited $CREATE_IMAGE_TIMEOUT_IN_SECS seconds for GCP image $IMAGE_NAME to be created, but image could not be found."
+    exit 1
+}
+
+# shellcheck disable=SC2155
+function create_image() {
+    if [[ "$CREATE_IMAGE" == 1 ]]; then
+        greenly echo "> Creating GCP image $IMAGE_NAME..."
+        local begin_img=$(date +%s)
+        local num_hosts=1
+        terraform apply -input=false -var "app=$APP" -var "name=$NAME" -var "num_hosts=$num_hosts" "$DIR/../tools/provisioning/gcp"
+        configure_with_ansible "$(terraform output username)" "$(terraform output public_ips)," "$(terraform output private_key_path)" $num_hosts
+        local zone=$(terraform output zone)
+        local name=$(terraform output instances_names)
+        gcloud -q compute instances delete "$name" --keep-disks boot --zone "$zone"
+        gcloud compute images create "$IMAGE_NAME" --source-disk "$name" --source-disk-zone "$zone" \
+            --description "Testing image for Weave Net based on $(terraform output image), Docker $DOCKER_VERSION, Kubernetes $KUBERNETES_VERSION and Kubernetes CNI $KUBERNETES_CNI_VERSION."
+        gcloud compute disks delete "$name" --zone "$zone"
+        terraform destroy -force "$DIR/../tools/provisioning/gcp"
+        rm terraform.tfstate*
+        echo
+        greenly echo "> Created GCP image $IMAGE_NAME in $(date -u -d @$(($(date +%s) - begin_img)) +"%T")."
+    else
+        wait_for_image
+    fi
+}
+
+function use_or_create_image() {
+    setup_gcloud
+    image_exists || create_image
+    export TF_VAR_gcp_image="$IMAGE_NAME" # Override the default image name.
+    export SKIP_CONFIG=1                  # No need to configure the image, since already done when making the template
+}
+
+function update_local_etc_hosts() {
+    echo "> Updating local /etc/hosts..."
+    # Remove old entries (if present):
+    for host in $1; do sudo sed -i "/$host/d" /etc/hosts; done
+    # Add new entries:
+    sudo sh -c "echo \"$2\" >> /etc/hosts"
+}
+
+function upload_etc_hosts() {
+    # Remove old entries (if present):
+    # shellcheck disable=SC2016,SC2086
+    $SSH $3 'for host in '$1'; do sudo sed -i "/$host/d" /etc/hosts; done'
+    # Add new entries:
+    echo "$2" | $SSH "$3" "sudo -- sh -c \"cat >> /etc/hosts\""
+}
+
+function update_remote_etc_hosts() {
+    echo "> Updating remote /etc/hosts..."
+    local pids=""
+    for host in $1; do
+        upload_etc_hosts "$1" "$2" "$host" &
+        local pids="$pids $!"
+    done
+    for pid in $pids; do wait "$pid"; done
+}
+
+# shellcheck disable=SC2155
+function set_hosts() {
+    export HOSTS="$(echo "$ssh_hosts" | tr '\n' ' ')"
+}
+
+function provision_remotely() {
+    case "$1" in
+        on)
+            terraform apply -input=false -parallelism="$NUM_HOSTS" -var "app=$APP" -var "name=$NAME" -var "num_hosts=$NUM_HOSTS" "$DIR/../tools/provisioning/$2"
+            local status=$?
+            ssh_user=$(terraform output username)
+            ssh_id_file=$(terraform output private_key_path)
+            ssh_hosts=$(terraform output hostnames)
+            export SSH="ssh -l $ssh_user -i $ssh_id_file $SSH_OPTS"
+
+            # Set up /etc/hosts files on this ("local") machine and the ("remote") testing machines, to map hostnames and IP addresses, so that:
+            # - this machine communicates with the testing machines via their public IPs;
+            # - testing machines communicate between themselves via their private IPs;
+            # - we can simply use just the hostname in all scripts to refer to machines, and the difference between public and private IP becomes transparent.
+            # N.B.: if you decide to use public IPs everywhere, note that some tests may fail (e.g. test #115).
+            update_local_etc_hosts "$ssh_hosts" "$(terraform output public_etc_hosts)"
+            update_remote_etc_hosts "$ssh_hosts" "$(terraform output private_etc_hosts)"
+
+            return $status
+            ;;
+        off)
+            terraform destroy -force "$DIR/../tools/provisioning/$2"
+            ;;
+        *)
+            echo >&2 "Unknown command $1. Usage: {on|off}."
+            exit 1
+            ;;
+    esac
+}
+
+# shellcheck disable=SC2155
+function provision() {
+    local action=$([ "$1" == "on" ] && echo "Provisioning" || echo "Shutting down")
+    echo
+    greenly echo "> $action test host(s) on [$PROVIDER]..."
+    local begin_prov=$(date +%s)
+    case "$2" in
+        'aws')
+            aws_on
+            provision_remotely "$1" "$2"
+            ;;
+        'do')
+            do_on
+            provision_remotely "$1" "$2"
+            ;;
+        'gcp')
+            gcp_on
+            [[ "$1" == "on" ]] && [[ "$USE_IMAGE" == 1 ]] && use_or_create_image
+            provision_remotely "$1" "$2"
+            ;;
+        'vagrant')
+            provision_locally "$1"
+            ;;
+        *)
+            echo >&2 "Unknown provider $2. Usage: PROVIDER={gcp|aws|do|vagrant}."
+            exit 1
+            ;;
+    esac
+    [ "$1" == "on" ] && set_hosts
+    echo
+    greenly echo "> Provisioning took $(date -u -d @$(($(date +%s) - begin_prov)) +"%T")."
+}
+
+function configure_with_ansible() {
+    ansible-playbook -u "$1" -i "$2" --private-key="$3" --forks="${4:-$NUM_HOSTS}" \
+        --ssh-extra-args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+        --extra-vars "docker_version=$DOCKER_VERSION kubernetes_version=$KUBERNETES_VERSION kubernetes_cni_version=$KUBERNETES_CNI_VERSION" \
+        "$DIR/../tools/config_management/$PLAYBOOK"
+}
+
+# shellcheck disable=SC2155
+function configure() {
+    echo
+    if [ -n "$SKIP_CONFIG" ]; then
+        greenly echo "> Skipped configuration of test host(s)."
+    else
+        greenly echo "> Configuring test host(s)..."
+        local begin_conf=$(date +%s)
+        local inventory_file=$(mktemp /tmp/ansible_inventory_XXXXX)
+        echo "[all]" >"$inventory_file"
+        # shellcheck disable=SC2001
+        echo "$2" | sed "s/$/:$3/" >>"$inventory_file"
+
+        # Configure the provisioned machines using Ansible, allowing up to 3 retries upon failure (e.g. APT connectivity issues, etc.):
+        for i in $(seq 3); do
+            configure_with_ansible "$1" "$inventory_file" "$4" && break || echo >&2 "#$i: Ansible failed. Retrying now..."
+        done
+
+        echo
+        greenly echo "> Configuration took $(date -u -d @$(($(date +%s) - begin_conf)) +"%T")."
+    fi
+}
+
+# shellcheck disable=SC2155
+function run_tests() {
+    echo
+    greenly echo "> Running tests..."
+    local begin_tests=$(date +%s)
+    set +e # Do not fail this script upon test failure, since we need to shut down the test cluster regardless of success or failure.
+    "$DIR/run_all.sh" "$@"
+    local status=$?
+    echo
+    greenly echo "> Tests took $(date -u -d @$(($(date +%s) - begin_tests)) +"%T")."
+    return $status
+}
+
+function end() {
+    echo
+    echo "> Build took $(date -u -d @$(($(date +%s) - begin)) +"%T")."
+}
+
+function echo_export_hosts() {
+    exec 1>&111
+    # Print a command to set HOSTS in the calling script, so that subsequent calls to
+    # test scripts can point to the right testing machines while developing:
+    echo "export HOSTS=\"$HOSTS\""
+    exec 1>&2
+}
+
+function main() {
+    # Keep a reference to stdout in another file descriptor (FD #111), and then globally redirect all stdout to stderr.
+    # This is so that HOSTS can be eval'ed in the calling script using:
+    #   $ eval $(./run-integration-tests.sh [provision|configure|setup])
+    # and ultimately subsequent calls to test scripts can point to the right testing machines during development.
+    if [ "$1" == "provision" ] || [ "$1" == "configure" ] || [ "$1" == "setup" ]; then
+        exec 111>&1 # 111 ought to match the file descriptor used in echo_export_hosts.
+        exec 1>&2
+    fi
+
+    begin=$(date +%s)
+    trap end EXIT
+
+    print_vars
+    verify_dependencies
+
+    case "$1" in
+        "") # Provision, configure, run tests, and destroy test environment:
+            provision on "$PROVIDER"
+            configure "$ssh_user" "$ssh_hosts" "${ssh_port:-22}" "$ssh_id_file"
+            "$DIR/setup.sh"
+            run_tests "$TESTS"
+            status=$?
+            [ -z "$SKIP_DESTROY" ] && provision off "$PROVIDER"
+            exit $status
+            ;;
+
+        provision)
+            provision on "$PROVIDER"
+            echo_export_hosts
+            ;;
+
+        configure)
+            provision on "$PROVIDER" # Vagrant and Terraform do not provision twice if VMs are already provisioned, so we just set environment variables.
+            configure "$ssh_user" "$ssh_hosts" "${ssh_port:-22}" "$ssh_id_file"
+            echo_export_hosts
+            ;;
+
+        setup)
+            provision on "$PROVIDER" # Vagrant and Terraform do not provision twice if VMs are already provisioned, so we just set environment variables.
+            "$DIR/setup.sh"
+            echo_export_hosts
+            ;;
+
+        test)
+            provision on "$PROVIDER" # Vagrant and Terraform do not provision twice if VMs are already provisioned, so we just set environment variables.
+            run_tests "$TESTS"
+            ;;
+
+        destroy)
+            provision off "$PROVIDER"
+            ;;
+
+        *)
+            echo "Unknown command: $1" >&2
+            exit 1
+            ;;
+    esac
+}
+
+main "$@"
@@ -11,7 +11,7 @@ fi
 whitely echo ...ok

 TESTS="${@:-$(find "$DIR" -name '*_test.sh')}"
-RUNNER_ARGS=""
+RUNNER_ARGS=${RUNNER_ARGS:-""}

 # If running on circle, use the scheduler to work out what tests to run
 if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" ]; then
@@ -7,6 +7,7 @@ set -e
 begin=$(date +%s)
 sanity_checks_files=${WEAVE_NET_SANITY_CHECKS_FILES:-"/tmp/weave_net_sanity_check_*.log"}

+greenly echo "> Sanity-checking test machines: ping, check Docker, and check Weave..."
 whitely echo Ping each host from the other

 # We wrap ping and echo in a function as we want the below parallel for loop
@@ -68,4 +69,4 @@ for host in $HOSTS; do
 done
 for pid in $pids; do wait $pid; done

-echo "Sanity checks completed successfully in $(date -u -d @$(($(date +%s)-$begin)) +"%T")."
+greenly echo "> Sanity checks completed successfully in $(date -u -d @$(($(date +%s)-$begin)) +"%T")."
@@ -9,8 +9,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")"

 (cd ./tls && ./tls $HOSTS)

-echo "Copying weave images, scripts, and certificates to hosts, and"
-echo "  prefetch test images"
+greenly echo "> Setting up test machines: copying weave images, scripts and certificates to hosts, and prefetch test images..."

 exists_on() {
    docker_on $1 inspect --format=" " $2 >/dev/null 2>&1
@@ -60,4 +59,4 @@ for ppid in $ppids; do
    wait $ppid;
 done

-echo "Setup completed successfully in $(date -u -d @$(($(date +%s)-$begin)) +"%T")."
+greenly echo "> Setup completed successfully in $(date -u -d @$(($(date +%s)-$begin)) +"%T")."
@@ -1,73 +1,29 @@
-$go_regexp = /FROM golang:(\S*).*?/
-$dockerfile_path = File.expand_path(File.join(File.dirname(__FILE__), 'build', 'Dockerfile'))
-$go_version = File.readlines($dockerfile_path).first { |line| line.match($go_regexp) }.match($go_regexp).captures.first
-if $go_version.nil?
-  raise ArgumentError.new("Failed to read Go version from Dockerfile.")
+VAGRANT_IMAGE = 'bento/ubuntu-16.04'
+VAGRANTFILE_API_VERSION = '2'
+
+def get_go_version_from_build_dockerfile()
+  go_regexp = /FROM golang:(\S*).*?/
+  dockerfile_path = File.expand_path(File.join(File.dirname(__FILE__), 'build', 'Dockerfile'))
+  go_version = File.readlines(dockerfile_path).first { |line| line.match(go_regexp) }.match(go_regexp).captures.first
+  if go_version.nil?
+    raise ArgumentError.new("Failed to read Go version from Dockerfile.")
+  end
+  go_version
 end

-$go_path = "/usr/local/go/bin"
+GO_VERSION = get_go_version_from_build_dockerfile()

-def install_packages(vm, pkgs)
-  vm.provision :shell, :inline => <<SCRIPT
-apt-get update -qq
-apt-get install -qq -y --force-yes --no-install-recommends #{pkgs.join(' ')}
-SCRIPT
-end
-
-def install_build_deps(vm, pkgs)
-  vm.provision :shell, :inline => <<SCRIPT
-export DEBIAN_FRONTEND=noninteractive
-apt-key adv \
-  --keyserver hkp://p80.pool.sks-keyservers.net:80 \
-  --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
-echo 'deb https://apt.dockerproject.org/repo ubuntu-wily main' \
-  > /etc/apt/sources.list.d/docker.list
-SCRIPT
-  install_packages(vm, pkgs)
-end
-
-def install_go_toochain(vm)
-  vm.provision :shell, :inline => <<SCRIPT
-curl -s https://storage.googleapis.com/golang/go#{$go_version}.linux-amd64.tar.gz \
-  | tar xz -C /usr/local
-#{$go_path}/go clean -i net
-#{$go_path}/go install -tags netgo std
-SCRIPT
-end
-
-def tweak_user_env(vm)
-  script = <<SCRIPT
-echo 'export GOPATH="${HOME}"' \
-  >> ~vagrant/.profile
-echo 'export PATH="${HOME}/bin:#{$go_path}:${PATH}"' \
-  >> ~vagrant/.profile
-ln -sf ~vagrant/src/github.com/weaveworks/weave ~vagrant/
-sudo chown -R vagrant:vagrant ~vagrant/src
-SCRIPT
-  vm.provision :shell, :inline => script, :privileged => false
-end
-
-def tweak_docker_daemon(vm)
-  vm.provision :shell, :inline => <<SCRIPT
-usermod -a -G docker vagrant
-mkdir -p /etc/systemd/system/docker.service.d
-cat >/etc/systemd/system/docker.service.d/override.conf  <<EOF
-[Service]
-ExecStart=
-ExecStart=/usr/bin/docker daemon -H fd:// -H unix:///var/run/alt-docker.sock -H tcp://0.0.0.0:2375 -s overlay
-EOF
-systemctl daemon-reload
-systemctl restart docker
-systemctl enable docker
-SCRIPT
+def configure_nat_dns(vb)
+  vb.customize ["modifyvm", :id, "--natdnshostresolver1", "off"]
+  vb.customize ["modifyvm", :id, "--natdnsproxy1", "off"]
 end

 def cleanup(vm)
  vm.provision :shell, :inline => <<SCRIPT
 export DEBIAN_FRONTEND=noninteractive
 ## Who the hell thinks official images have to have both of these?
-/etc/init.d/chef-client stop
-/etc/init.d/puppet stop
+[ ! -f /etc/init.d/chef-client ] || /etc/init.d/chef-client stop
+[ ! -f /etc/init.d/puppet ]      || /etc/init.d/puppet stop
 apt-get -qq remove puppet chef
 apt-get -qq autoremove
 killall -9 chef-client 2>/dev/null || true