From 3e5593dc4f4655ac13d4c02423e5af82756731b0 Mon Sep 17 00:00:00 2001 From: Marc CARRE Date: Mon, 16 Jan 2017 18:01:33 +0000 Subject: [PATCH] Add GCP image creation and usage. Provisioning and configuration being expensive time-wise, we create Google Cloud Platform images for each Docker/Kubernetes/Kubernetes CNI versions triple, and configure these once (instead of for each build), and then re-use these in subsequent builds, effectively implementing a 'cache' of images. --- bin/circle-test-smoke | 6 +++ test/README.md | 21 ++------- test/run-integration-tests.sh | 84 ++++++++++++++++++++++++++++++++--- 3 files changed, 89 insertions(+), 22 deletions(-) diff --git a/bin/circle-test-smoke b/bin/circle-test-smoke index ccca722e..aaa50de1 100755 --- a/bin/circle-test-smoke +++ b/bin/circle-test-smoke @@ -14,10 +14,16 @@ if [ -n "$TEST_AND_PUBLISH" ]; then sudo apt-get update || true sudo apt-get install -qq -y python-pip python-dev libffi-dev libssl-dev && pip install --user -U cffi && pip install --user ansible export PATH="$PATH:$HOME/.local/bin" + # Run integration tests: export COVERAGE=true export WEAVE_NET_SANITY_CHECKS_FILES="$CIRCLE_ARTIFACTS/weave_net_sanity_check_*.log" export NAME="test-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX" + export DISK_NAME_PREFIX="test-$CIRCLE_BUILD_NUM-0" + export USE_IMAGE=1 + # Only attempt to create GCP image in first container, wait for it to be created otherwise: + [ "$CIRCLE_NODE_INDEX" == "0" ] && export CREATE_IMAGE=1 + cd "$SRCDIR/test" # Ensures we generate code coverage files in the right folder. ./run-integration-tests.sh fi diff --git a/test/README.md b/test/README.md index 842579c3..b0f84238 100644 --- a/test/README.md +++ b/test/README.md @@ -60,20 +60,7 @@ to the file `/etc/default/docker`, then restart docker. ## Updating the GCE test image -When a new version of Docker is released, you'll need to update the GCE test image. - -To do this, pick a fresh ```TEMPLATE_NAME``` and update any commands in -```function make_template``` in gce.sh, then run: - -``` -./gce.sh make_template -``` - -For this you'll need the GCE credentials, which can be found in ```bin/setup-circleci-secrets```, -which you'll need to decrypt and run (its echos the secrets into know locations): - -``` -./bin/setup-circleci-secrets "$SECRET_PASSWORD" -``` - -If you don't know the password, ask tom@weave.works. +When a new version of Docker is released, you willneed to update the GCE test image. +To do this, change the Docker version in `run-integration-tests.sh` and push the change. +Next build in CircleCI will detect that there is no template for this version of Docker and will first create the template before running tests. +Subsequent builds will then simply re-use the template. diff --git a/test/run-integration-tests.sh b/test/run-integration-tests.sh index d422f589..4cb1b1a9 100755 --- a/test/run-integration-tests.sh +++ b/test/run-integration-tests.sh @@ -32,6 +32,12 @@ RUNNER_ARGS=${RUNNER_ARGS:-""} DOCKER_VERSION=${DOCKER_VERSION:-1.11.2} KUBERNETES_VERSION=${KUBERNETES_VERSION:-1.5.2} KUBERNETES_CNI_VERSION=${KUBERNETES_CNI_VERSION:-0.3.0.1} +# Google Cloud Platform image's name & usage (only used when PROVIDER is gcp): +IMAGE_NAME=${IMAGE_NAME:-"$(echo "$APP-docker$DOCKER_VERSION-k8s$KUBERNETES_VERSION-k8scni$KUBERNETES_CNI_VERSION" | sed -e 's/[\.\_]*//g')"} +DISK_NAME_PREFIX=${DISK_NAME_PREFIX:-$NAME} +USE_IMAGE=${USE_IMAGE:-1} +CREATE_IMAGE=${CREATE_IMAGE:-} +CREATE_IMAGE_TIMEOUT_IN_SECS=${CREATE_IMAGE_TIMEOUT_IN_SECS:-600} # Lifecycle flags: SKIP_CONFIG=${SKIP_CONFIG:-} @@ -47,6 +53,11 @@ function print_vars() { echo "DOCKER_VERSION=$DOCKER_VERSION" echo "KUBERNETES_VERSION=$KUBERNETES_VERSION" echo "KUBERNETES_CNI_VERSION=$KUBERNETES_CNI_VERSION" + echo "IMAGE_NAME=$IMAGE_NAME" + echo "DISK_NAME_PREFIX=$DISK_NAME_PREFIX" + echo "USE_IMAGE=$USE_IMAGE" + echo "CREATE_IMAGE=$CREATE_IMAGE" + echo "CREATE_IMAGE_TIMEOUT_IN_SECS=$CREATE_IMAGE_TIMEOUT_IN_SECS" echo "--- Variables: Flags ---" echo "SKIP_CONFIG=$SKIP_CONFIG" } @@ -98,6 +109,65 @@ function provision_locally() { esac } +function setup_gcloud() { + # Authenticate: + gcloud auth activate-service-account --key-file "$GOOGLE_CREDENTIALS_FILE" 1>/dev/null + # Set current project: + gcloud config set project $PROJECT +} + +function image_exists() { + gcloud compute images list | grep "$PROJECT" | grep "$IMAGE_NAME" +} + +function image_ready() { + # GCP images seem to be listed before they are actually ready for use, + # typically failing the build with: "googleapi: Error 400: The resource is not ready". + # We therefore consider the image to be ready once the disk of its template instance has been deleted. + ! gcloud compute disks list | grep "$DISK_NAME_PREFIX" +} + +function wait_for_image() { + greenly echo "> Waiting for GCP image $IMAGE_NAME to be created..." + for i in $(seq "$CREATE_IMAGE_TIMEOUT_IN_SECS"); do + image_exists && image_ready && return 0 + if ! ((i % 60)); then echo "Waited for $i seconds and still waiting..."; fi + sleep 1 + done + redly echo "> Waited $CREATE_IMAGE_TIMEOUT_IN_SECS seconds for GCP image $IMAGE_NAME to be created, but image could not be found." + exit 1 +} + +# shellcheck disable=SC2155 +function create_image() { + if [ -n "$CREATE_IMAGE" ]; then + greenly echo "> Creating GCP image $IMAGE_NAME..." + local begin_img=$(date +%s) + local num_hosts=1 + terraform apply -input=false -var "app=$APP" -var "name=$NAME" -var "num_hosts=$num_hosts" "$DIR/../tools/provisioning/gcp" + configure_with_ansible "$(terraform output username)" "$(terraform output public_ips)," "$(terraform output private_key_path)" $num_hosts + local zone=$(terraform output zone) + local name=$(terraform output instances_names) + gcloud -q compute instances delete "$name" --keep-disks boot --zone "$zone" + gcloud compute images create "$IMAGE_NAME" --source-disk "$name" --source-disk-zone "$zone" \ + --description "Testing image for Weave Net based on $(terraform output image), Docker $DOCKER_VERSION, Kubernetes $KUBERNETES_VERSION and Kubernetes CNI $KUBERNETES_CNI_VERSION." + gcloud compute disks delete "$name" --zone "$zone" + terraform destroy -force "$DIR/../tools/provisioning/gcp" + rm terraform.tfstate* + echo + greenly echo "> Created GCP image $IMAGE_NAME in $(date -u -d @$(($(date +%s) - begin_img)) +"%T")." + else + wait_for_image + fi +} + +function use_or_create_image() { + setup_gcloud + image_exists || create_image + export TF_VAR_gcp_image="$IMAGE_NAME" # Override the default image name. + export SKIP_CONFIG=1 # No need to configure the image, since already done when making the template +} + function update_local_etc_hosts() { echo "> Updating local /etc/hosts..." # Remove old entries (if present): @@ -171,6 +241,7 @@ function provision() { ;; 'gcp') gcp_on + [[ "$USE_IMAGE" == 1 ]] && use_or_create_image provision_remotely "$1" "$2" ;; 'vagrant') @@ -186,6 +257,13 @@ function provision() { greenly echo "> Provisioning took $(date -u -d @$(($(date +%s) - begin_prov)) +"%T")." } +function configure_with_ansible() { + ansible-playbook -u "$1" -i "$2" --private-key="$3" --forks="${4:-$NUM_HOSTS}" \ + --ssh-extra-args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \ + --extra-vars "docker_version=$DOCKER_VERSION kubernetes_version=$KUBERNETES_VERSION kubernetes_cni_version=$KUBERNETES_CNI_VERSION" \ + "$DIR/../tools/config_management/$PLAYBOOK" +} + # shellcheck disable=SC2155 function configure() { echo @@ -201,11 +279,7 @@ function configure() { # Configure the provisioned machines using Ansible, allowing up to 3 retries upon failure (e.g. APT connectivity issues, etc.): for i in $(seq 3); do - ansible-playbook -u "$1" -i "$inventory_file" --private-key="$4" --forks="$NUM_HOSTS" \ - --ssh-extra-args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \ - --extra-vars "docker_version=$DOCKER_VERSION kubernetes_version=$KUBERNETES_VERSION kubernetes_cni_version=$KUBERNETES_CNI_VERSION" \ - "$DIR/../tools/config_management/$PLAYBOOK" \ - && break || echo >&2 "#$i: Ansible failed. Retrying now..." + configure_with_ansible "$1" "$inventory_file" "$4" && break || echo >&2 "#$i: Ansible failed. Retrying now..." done echo