Signal provisioning failures to fail the build fast.

There may be situations when Terraform/Ansible/etc. fail to provision test machines.
In these cases, we want to fail fast instead of waiting until we time out in circle-test-pre and circle-test-smoke.

bin/circle-test-pre

@@ -2,6 +2,13 @@
 
 set -e
 
+# Signal failures in lock file, in order to fail fast:
+function signal_failure() {
+    echo "KO" >"$TEST_VMS_READY_LOCK_FILE"
+    exit 1
+}
+trap signal_failure ERR
+
 source "$STATE"
 source "$(dirname "$0")/wait_for.sh"
 
@@ -16,9 +23,8 @@ if [ -n "$TEST_AND_PUBLISH" ]; then
     wait_for 900 "$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
 
     # Upload built binaries to testing VMs:
-    rm -f "$TEST_VMS_READY_LOCK_FILE"
     cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests.
     ./run-integration-tests.sh setup >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
-    touch "$TEST_VMS_READY_LOCK_FILE"
+    echo "OK" >"$TEST_VMS_READY_LOCK_FILE"
     echo "Test VMs now ready. $(date)." >>"$TEST_VMS_SETUP_OUTPUT_FILE"
 fi

bin/provision_test_vms.sh

@@ -2,6 +2,13 @@
 
 set -e
 
+# Signal failures in lock file, in order to fail fast:
+function signal_failure() {
+    echo "KO" >"$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
+    exit 1
+}
+trap signal_failure ERR
+
 source "$STATE"
 
 function install_terraform() {
@@ -27,9 +34,8 @@ if [ -n "$TEST_AND_PUBLISH" ]; then
     [ "$CIRCLE_NODE_INDEX" == "0" ] && export CREATE_IMAGE=1
 
     # Provision and configure testing VMs:
-    rm -f "$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
     cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests.
     ./run-integration-tests.sh configure >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
-    touch "$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
+    echo "OK" >"$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
     echo "Test VMs now provisioned and configured. $(date)." >>"$TEST_VMS_SETUP_OUTPUT_FILE"
 fi

bin/wait_for.sh

@@ -1,13 +1,21 @@
 #!/bin/bash
 
+# shellcheck disable=SC2155
 wait_for() {
     local timeout=$1
     local lock_file=$2
     for i in $(seq 1 "$timeout"); do
-        [ -f "$lock_file" ] && echo "[$i seconds]: $lock_file found." && return
+        if [ -f "$lock_file" ]; then
+            local status="$(cat "$lock_file")"
+            if [ "$status" == "OK" ]; then
+                echo "[$i seconds]: $lock_file found and status: $status." && return
+            else
+                echo "[$i seconds]: $lock_file found and status: $status." && return 1
+            fi
+        fi
         if ! ((i % 10)); then echo "[$i seconds]: Waiting for $lock_file to be created..."; fi
         sleep 1
     done
     echo "Timed out waiting for test VMs to be ready. See details in: $TEST_VMS_SETUP_OUTPUT_FILE" >&2
-    exit 1
+    return 1
 }