Provision and destroy test VMs in the background.

This should help speed-up the build by provisioning test VMs while we run unit tests, and destroying them while we generate coverage reports.

Makefile

@@ -369,14 +369,10 @@ integration-tests: all testrunner
 #     PLAYBOOK="<filename>" \
 #     RUNNER_ARGS="<...>" \
 #     TESTS="<...>" \  # Can be set to only run one or a few tests instead of the full test suite.
-#     SKIP_CREATE=1 \  # Skip the provisioning phase before running tests.
-#     SKIP_CONFIG=1 \  # Skip the configuration phase before running tests.
-#     SKIP_DESTROY=1 \ # Skip the destroy phase after running tests.
-#     ONLY_DESTROY=1 \ # Destroy the test cluster (and do not do anything else, like starting the test cluster, configuring it, or running tests).
 #     DOCKER_VERSION=<...> \
 #     KUBERNETES_VERSION=<...> \
 #     KUBERNETES_CNI_VERSION=<...> \
-#     <...> # See also run-integration-test.sh for all variables.
+#     <...> # See also run-integration-test.sh for all variables and individual functions.
 #     integration-tests
 #
 	RUNNER_ARGS="-parallel" ./test/run-integration-tests.sh

bin/circle-test-pre

@@ -3,7 +3,22 @@
 set -e
 
 source "$STATE"
+source "$(dirname "$0")/wait_for.sh"
 
-if [ -n "$TEST_AND_PUBLISH" ] ; then
-    cd $SRCDIR/test
+if [ -n "$TEST_AND_PUBLISH" ]; then
+    [ -n "$SECRET_KEY" ] || {
+        echo "Cannot run smoke tests: no secret key"
+        exit 1
+    }
+
+    # Provisioning from image typically take 90 seconds, 
+    # but if we are creating a GCP image, it can take more than 10 minutes:
+    wait_for 900 "$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
+
+    # Upload built binaries to testing VMs:
+    rm -f "$TEST_VMS_READY_LOCK_FILE"
+    cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests.
+    ./run-integration-tests.sh setup >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
+    touch "$TEST_VMS_READY_LOCK_FILE"
+    echo "Test VMs now ready. $(date)." >>"$TEST_VMS_SETUP_OUTPUT_FILE"
 fi

bin/circle-test-smoke

@@ -3,27 +3,21 @@
 set -e
 
 source "$STATE"
+source "$(dirname "$0")/wait_for.sh"
 
 if [ -n "$TEST_AND_PUBLISH" ]; then
     [ -n "$SECRET_KEY" ] || {
         echo "Cannot run smoke tests: no secret key"
         exit 1
     }
-    # Install Terraform and Ansible:
-    curl -fsS https://releases.hashicorp.com/terraform/0.8.5/terraform_0.8.5_linux_amd64.zip | gunzip >terraform && chmod +x terraform && sudo mv terraform /usr/bin
-    sudo apt-get update || true
-    sudo apt-get install -qq -y python-pip python-dev libffi-dev libssl-dev && pip install --user -U cffi && pip install --user ansible
-    export PATH="$PATH:$HOME/.local/bin"
+
+    # Provisioning from image typically take 90 seconds, 
+    # but if we are creating a GCP image, it can take more than 10 minutes:
+    wait_for 900 "$TEST_VMS_READY_LOCK_FILE"
 
     # Run integration tests:
     export COVERAGE=true
     export WEAVE_NET_SANITY_CHECKS_FILES="$CIRCLE_ARTIFACTS/weave_net_sanity_check_*.log"
-    export NAME="test-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX"
-    export DISK_NAME_PREFIX="test-$CIRCLE_BUILD_NUM-0"
-    export USE_IMAGE=1
-    # Only attempt to create GCP image in first container, wait for it to be created otherwise:
-    [ "$CIRCLE_NODE_INDEX" == "0" ] && export CREATE_IMAGE=1
-
-    cd "$SRCDIR/test" # Ensures we generate code coverage files in the right folder.
-    ./run-integration-tests.sh
+    cd "$SRCDIR/test" # Ensures we generate code coverage files in the right folder, and use the Terraform state of previously provisionned VMs.
+    ./run-integration-tests.sh test
 fi

bin/provision_test_vms.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -e
+
+source "$STATE"
+
+function install_terraform() {
+    curl -fsS https://releases.hashicorp.com/terraform/0.8.5/terraform_0.8.5_linux_amd64.zip | gunzip >terraform && chmod +x terraform && sudo mv terraform /usr/bin
+}
+
+function install_ansible() {
+    sudo apt-get update || true
+    sudo apt-get install -qq -y python-pip python-dev libffi-dev libssl-dev && pip install --user -U cffi && pip install --user ansible
+    export PATH="$PATH:$HOME/.local/bin"
+}
+
+if [ -n "$TEST_AND_PUBLISH" ]; then
+    [ -n "$SECRET_KEY" ] || {
+        echo "Cannot run smoke tests: no secret key"
+        exit 1
+    }
+
+    install_terraform >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
+    install_ansible >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
+
+    # Only attempt to create GCP image in first container, wait for it to be created otherwise:
+    [ "$CIRCLE_NODE_INDEX" == "0" ] && export CREATE_IMAGE=1
+
+    # Provision and configure testing VMs:
+    rm -f "$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
+    cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests.
+    ./run-integration-tests.sh configure >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
+    touch "$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
+    echo "Test VMs now provisioned and configured. $(date)." >>"$TEST_VMS_SETUP_OUTPUT_FILE"
+fi

bin/wait_for.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+wait_for() {
+    local timeout=$1
+    local lock_file=$2
+    for i in $(seq 1 "$timeout"); do
+        [ -f "$lock_file" ] && echo "[$i seconds]: $lock_file found." && return
+        if ! ((i % 10)); then echo "[$i seconds]: Waiting for $lock_file to be created..."; fi
+        sleep 1
+    done
+    echo "Timed out waiting for test VMs to be ready. See details in: $TEST_VMS_SETUP_OUTPUT_FILE" >&2
+    exit 1
+}

circle.yml

@@ -14,17 +14,30 @@ machine:
     PATH: $PATH:$HOME/.local/bin
     CLOUDSDK_CORE_DISABLE_PROMPTS: 1
     STATE: /home/ubuntu/state.env
+    # Base name of VMs for integration tests:
+    NAME: test-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX
+    TEST_VMS_SETUP_OUTPUT_FILE: $CIRCLE_ARTIFACTS/test_vms_setup_output.txt
+    TEST_VMS_PROV_AND_CONF_LOCK_FILE: $CIRCLE_ARTIFACTS/test_vms_provisioned_and_configured.txt
+    TEST_VMS_READY_LOCK_FILE: $CIRCLE_ARTIFACTS/test_vms_ready.txt
+    # Google Cloud Platorm images for integration tests:
+    USE_IMAGE: 1
+    DISK_NAME_PREFIX: test-$CIRCLE_BUILD_NUM-0
 
 dependencies:
   cache_directories:
     - "~/docker"
   post:
+    # Provision & configure testing VMs:
+    - "while [ ! -d $SRCDIR ]; do sleep 1; done; $SRCDIR/bin/provision_test_vms.sh":
+        parallel: true
+        background: true
     - bin/circle-dependencies-post
 
 test:
   pre:
     - "$SRCDIR/bin/circle-test-pre":
         parallel: true
+        background: true
   override:
     - "$SRCDIR/bin/circle-test-unit":
         parallel: true
@@ -37,6 +50,10 @@ test:
 
 teardown:
   pre:
+    # Destroy testing VMs:
+    - "cd $SRCDIR/test && ./run-integration-tests.sh destroy":
+        parallel: true
+        background: true
     - "$SRCDIR/bin/circle-teardown-pre":
         parallel: true
 

test/run-integration-tests.sh

@@ -194,6 +194,11 @@ function update_remote_etc_hosts() {
     for pid in $pids; do wait "$pid"; done
 }
 
+# shellcheck disable=SC2155
+function set_hosts() {
+    export HOSTS="$(echo "$ssh_hosts" | tr '\n' ' ')"
+}
+
 function provision_remotely() {
     case "$1" in
         on)
@@ -241,7 +246,7 @@ function provision() {
             ;;
         'gcp')
             gcp_on
-            [[ "$USE_IMAGE" == 1 ]] && use_or_create_image
+            [[ "$1" == "on" ]] && [[ "$USE_IMAGE" == 1 ]] && use_or_create_image
             provision_remotely "$1" "$2"
             ;;
         'vagrant')
@@ -252,7 +257,7 @@ function provision() {
             exit 1
             ;;
     esac
-
+    [ "$1" == "on" ] && set_hosts
     echo
     greenly echo "> Provisioning took $(date -u -d @$(($(date +%s) - begin_prov)) +"%T")."
 }
@@ -289,9 +294,6 @@ function configure() {
 
 # shellcheck disable=SC2155
 function run_tests() {
-    export HOSTS="$(echo "$3" | tr '\n' ' ')"
-    shift 3 # Drop the first 3 arguments, the remainder being, optionally, the list of tests to run.
-    "$DIR/setup.sh"
     echo
     greenly echo "> Running tests..."
     local begin_tests=$(date +%s)
@@ -303,36 +305,57 @@ function run_tests() {
     return $status
 }
 
+function end() {
+    echo
+    echo "> Build took $(date -u -d @$(($(date +%s) - begin)) +"%T")."
+}
+
 function main() {
     begin=$(date +%s)
+    trap end EXIT
+
     print_vars
     verify_dependencies
 
-    if [ -z "$ONLY_DESTROY" ]; then
-        provision on "$PROVIDER"
-        if [ $? -ne 0 ]; then
-            echo >&2 "> Failed to provision test host(s)."
-            exit 1
-        fi
-
-        if [ -z "$SKIP_CONFIG" ]; then
+    case "$1" in
+        "") # Provision, configure, run tests, and destroy test environment:
+            provision on "$PROVIDER"
             configure "$ssh_user" "$ssh_hosts" "${ssh_port:-22}" "$ssh_id_file"
-            if [ $? -ne 0 ]; then
-                echo >&2 "Failed to configure test host(s)."
-                exit 1
-            fi
-        fi
+            "$DIR/setup.sh"
+            run_tests "$TESTS"
+            status=$?
+            provision off "$PROVIDER"
+            exit $status
+            ;;
 
-        run_tests "$ssh_user" "$ssh_id_file" "$ssh_hosts" "$TESTS"
-        status=$?
-    fi
-    if [ -z "$SKIP_DESTROY" ]; then
-        provision off "$PROVIDER"
-    fi
+        provision)
+            provision on "$PROVIDER"
+            ;;
 
-    echo
-    greenly echo "> Build took $(date -u -d @$(($(date +%s) - begin)) +"%T")."
-    exit $status
+        configure)
+            provision on "$PROVIDER" # Vagrant and Terraform do not provision twice if VMs are already provisioned, so we just set environment variables.
+            configure "$ssh_user" "$ssh_hosts" "${ssh_port:-22}" "$ssh_id_file"
+            ;;
+
+        setup)
+            provision on "$PROVIDER" # Vagrant and Terraform do not provision twice if VMs are already provisioned, so we just set environment variables.
+            "$DIR/setup.sh"
+            ;;
+
+        test)
+            provision on "$PROVIDER" # Vagrant and Terraform do not provision twice if VMs are already provisioned, so we just set environment variables.
+            run_tests "$TESTS"
+            ;;
+
+        destroy)
+            provision off "$PROVIDER"
+            ;;
+
+        *)
+            echo "Unknown command: $1" >&2
+            exit 1
+            ;;
+    esac
 }
 
-main
+main "$@"