diff --git a/Makefile b/Makefile index 99c694eb..4e6cb6ef 100644 --- a/Makefile +++ b/Makefile @@ -369,14 +369,10 @@ integration-tests: all testrunner # PLAYBOOK="" \ # RUNNER_ARGS="<...>" \ # TESTS="<...>" \ # Can be set to only run one or a few tests instead of the full test suite. -# SKIP_CREATE=1 \ # Skip the provisioning phase before running tests. -# SKIP_CONFIG=1 \ # Skip the configuration phase before running tests. -# SKIP_DESTROY=1 \ # Skip the destroy phase after running tests. -# ONLY_DESTROY=1 \ # Destroy the test cluster (and do not do anything else, like starting the test cluster, configuring it, or running tests). # DOCKER_VERSION=<...> \ # KUBERNETES_VERSION=<...> \ # KUBERNETES_CNI_VERSION=<...> \ -# <...> # See also run-integration-test.sh for all variables. +# <...> # See also run-integration-test.sh for all variables and individual functions. # integration-tests # RUNNER_ARGS="-parallel" ./test/run-integration-tests.sh diff --git a/bin/circle-test-pre b/bin/circle-test-pre index eafdd295..7f02223a 100755 --- a/bin/circle-test-pre +++ b/bin/circle-test-pre @@ -3,7 +3,22 @@ set -e source "$STATE" +source "$(dirname "$0")/wait_for.sh" -if [ -n "$TEST_AND_PUBLISH" ] ; then - cd $SRCDIR/test +if [ -n "$TEST_AND_PUBLISH" ]; then + [ -n "$SECRET_KEY" ] || { + echo "Cannot run smoke tests: no secret key" + exit 1 + } + + # Provisioning from image typically take 90 seconds, + # but if we are creating a GCP image, it can take more than 10 minutes: + wait_for 900 "$TEST_VMS_PROV_AND_CONF_LOCK_FILE" + + # Upload built binaries to testing VMs: + rm -f "$TEST_VMS_READY_LOCK_FILE" + cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests. + ./run-integration-tests.sh setup >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1 + touch "$TEST_VMS_READY_LOCK_FILE" + echo "Test VMs now ready. $(date)." >>"$TEST_VMS_SETUP_OUTPUT_FILE" fi diff --git a/bin/circle-test-smoke b/bin/circle-test-smoke index aaa50de1..250841eb 100755 --- a/bin/circle-test-smoke +++ b/bin/circle-test-smoke @@ -3,27 +3,21 @@ set -e source "$STATE" +source "$(dirname "$0")/wait_for.sh" if [ -n "$TEST_AND_PUBLISH" ]; then [ -n "$SECRET_KEY" ] || { echo "Cannot run smoke tests: no secret key" exit 1 } - # Install Terraform and Ansible: - curl -fsS https://releases.hashicorp.com/terraform/0.8.5/terraform_0.8.5_linux_amd64.zip | gunzip >terraform && chmod +x terraform && sudo mv terraform /usr/bin - sudo apt-get update || true - sudo apt-get install -qq -y python-pip python-dev libffi-dev libssl-dev && pip install --user -U cffi && pip install --user ansible - export PATH="$PATH:$HOME/.local/bin" + + # Provisioning from image typically take 90 seconds, + # but if we are creating a GCP image, it can take more than 10 minutes: + wait_for 900 "$TEST_VMS_READY_LOCK_FILE" # Run integration tests: export COVERAGE=true export WEAVE_NET_SANITY_CHECKS_FILES="$CIRCLE_ARTIFACTS/weave_net_sanity_check_*.log" - export NAME="test-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX" - export DISK_NAME_PREFIX="test-$CIRCLE_BUILD_NUM-0" - export USE_IMAGE=1 - # Only attempt to create GCP image in first container, wait for it to be created otherwise: - [ "$CIRCLE_NODE_INDEX" == "0" ] && export CREATE_IMAGE=1 - - cd "$SRCDIR/test" # Ensures we generate code coverage files in the right folder. - ./run-integration-tests.sh + cd "$SRCDIR/test" # Ensures we generate code coverage files in the right folder, and use the Terraform state of previously provisionned VMs. + ./run-integration-tests.sh test fi diff --git a/bin/provision_test_vms.sh b/bin/provision_test_vms.sh new file mode 100755 index 00000000..d3eb59aa --- /dev/null +++ b/bin/provision_test_vms.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -e + +source "$STATE" + +function install_terraform() { + curl -fsS https://releases.hashicorp.com/terraform/0.8.5/terraform_0.8.5_linux_amd64.zip | gunzip >terraform && chmod +x terraform && sudo mv terraform /usr/bin +} + +function install_ansible() { + sudo apt-get update || true + sudo apt-get install -qq -y python-pip python-dev libffi-dev libssl-dev && pip install --user -U cffi && pip install --user ansible + export PATH="$PATH:$HOME/.local/bin" +} + +if [ -n "$TEST_AND_PUBLISH" ]; then + [ -n "$SECRET_KEY" ] || { + echo "Cannot run smoke tests: no secret key" + exit 1 + } + + install_terraform >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1 + install_ansible >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1 + + # Only attempt to create GCP image in first container, wait for it to be created otherwise: + [ "$CIRCLE_NODE_INDEX" == "0" ] && export CREATE_IMAGE=1 + + # Provision and configure testing VMs: + rm -f "$TEST_VMS_PROV_AND_CONF_LOCK_FILE" + cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests. + ./run-integration-tests.sh configure >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1 + touch "$TEST_VMS_PROV_AND_CONF_LOCK_FILE" + echo "Test VMs now provisioned and configured. $(date)." >>"$TEST_VMS_SETUP_OUTPUT_FILE" +fi diff --git a/bin/wait_for.sh b/bin/wait_for.sh new file mode 100755 index 00000000..d3dff060 --- /dev/null +++ b/bin/wait_for.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +wait_for() { + local timeout=$1 + local lock_file=$2 + for i in $(seq 1 "$timeout"); do + [ -f "$lock_file" ] && echo "[$i seconds]: $lock_file found." && return + if ! ((i % 10)); then echo "[$i seconds]: Waiting for $lock_file to be created..."; fi + sleep 1 + done + echo "Timed out waiting for test VMs to be ready. See details in: $TEST_VMS_SETUP_OUTPUT_FILE" >&2 + exit 1 +} diff --git a/circle.yml b/circle.yml index 74639264..eb6fd1d5 100644 --- a/circle.yml +++ b/circle.yml @@ -14,17 +14,30 @@ machine: PATH: $PATH:$HOME/.local/bin CLOUDSDK_CORE_DISABLE_PROMPTS: 1 STATE: /home/ubuntu/state.env + # Base name of VMs for integration tests: + NAME: test-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX + TEST_VMS_SETUP_OUTPUT_FILE: $CIRCLE_ARTIFACTS/test_vms_setup_output.txt + TEST_VMS_PROV_AND_CONF_LOCK_FILE: $CIRCLE_ARTIFACTS/test_vms_provisioned_and_configured.txt + TEST_VMS_READY_LOCK_FILE: $CIRCLE_ARTIFACTS/test_vms_ready.txt + # Google Cloud Platorm images for integration tests: + USE_IMAGE: 1 + DISK_NAME_PREFIX: test-$CIRCLE_BUILD_NUM-0 dependencies: cache_directories: - "~/docker" post: + # Provision & configure testing VMs: + - "while [ ! -d $SRCDIR ]; do sleep 1; done; $SRCDIR/bin/provision_test_vms.sh": + parallel: true + background: true - bin/circle-dependencies-post test: pre: - "$SRCDIR/bin/circle-test-pre": parallel: true + background: true override: - "$SRCDIR/bin/circle-test-unit": parallel: true @@ -37,6 +50,10 @@ test: teardown: pre: + # Destroy testing VMs: + - "cd $SRCDIR/test && ./run-integration-tests.sh destroy": + parallel: true + background: true - "$SRCDIR/bin/circle-teardown-pre": parallel: true diff --git a/test/run-integration-tests.sh b/test/run-integration-tests.sh index 4cb1b1a9..5b0c40c5 100755 --- a/test/run-integration-tests.sh +++ b/test/run-integration-tests.sh @@ -194,6 +194,11 @@ function update_remote_etc_hosts() { for pid in $pids; do wait "$pid"; done } +# shellcheck disable=SC2155 +function set_hosts() { + export HOSTS="$(echo "$ssh_hosts" | tr '\n' ' ')" +} + function provision_remotely() { case "$1" in on) @@ -241,7 +246,7 @@ function provision() { ;; 'gcp') gcp_on - [[ "$USE_IMAGE" == 1 ]] && use_or_create_image + [[ "$1" == "on" ]] && [[ "$USE_IMAGE" == 1 ]] && use_or_create_image provision_remotely "$1" "$2" ;; 'vagrant') @@ -252,7 +257,7 @@ function provision() { exit 1 ;; esac - + [ "$1" == "on" ] && set_hosts echo greenly echo "> Provisioning took $(date -u -d @$(($(date +%s) - begin_prov)) +"%T")." } @@ -289,9 +294,6 @@ function configure() { # shellcheck disable=SC2155 function run_tests() { - export HOSTS="$(echo "$3" | tr '\n' ' ')" - shift 3 # Drop the first 3 arguments, the remainder being, optionally, the list of tests to run. - "$DIR/setup.sh" echo greenly echo "> Running tests..." local begin_tests=$(date +%s) @@ -303,36 +305,57 @@ function run_tests() { return $status } +function end() { + echo + echo "> Build took $(date -u -d @$(($(date +%s) - begin)) +"%T")." +} + function main() { begin=$(date +%s) + trap end EXIT + print_vars verify_dependencies - if [ -z "$ONLY_DESTROY" ]; then - provision on "$PROVIDER" - if [ $? -ne 0 ]; then - echo >&2 "> Failed to provision test host(s)." - exit 1 - fi - - if [ -z "$SKIP_CONFIG" ]; then + case "$1" in + "") # Provision, configure, run tests, and destroy test environment: + provision on "$PROVIDER" configure "$ssh_user" "$ssh_hosts" "${ssh_port:-22}" "$ssh_id_file" - if [ $? -ne 0 ]; then - echo >&2 "Failed to configure test host(s)." - exit 1 - fi - fi + "$DIR/setup.sh" + run_tests "$TESTS" + status=$? + provision off "$PROVIDER" + exit $status + ;; - run_tests "$ssh_user" "$ssh_id_file" "$ssh_hosts" "$TESTS" - status=$? - fi - if [ -z "$SKIP_DESTROY" ]; then - provision off "$PROVIDER" - fi + provision) + provision on "$PROVIDER" + ;; - echo - greenly echo "> Build took $(date -u -d @$(($(date +%s) - begin)) +"%T")." - exit $status + configure) + provision on "$PROVIDER" # Vagrant and Terraform do not provision twice if VMs are already provisioned, so we just set environment variables. + configure "$ssh_user" "$ssh_hosts" "${ssh_port:-22}" "$ssh_id_file" + ;; + + setup) + provision on "$PROVIDER" # Vagrant and Terraform do not provision twice if VMs are already provisioned, so we just set environment variables. + "$DIR/setup.sh" + ;; + + test) + provision on "$PROVIDER" # Vagrant and Terraform do not provision twice if VMs are already provisioned, so we just set environment variables. + run_tests "$TESTS" + ;; + + destroy) + provision off "$PROVIDER" + ;; + + *) + echo "Unknown command: $1" >&2 + exit 1 + ;; + esac } -main +main "$@"