# NB only to be sourced set -e DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # Protect against being sourced multiple times to prevent # overwriting assert.sh global state if ! [ -z "$SOURCED_CONFIG_SH" ]; then return fi SOURCED_CONFIG_SH=true # these ought to match what is in Vagrantfile N_MACHINES=${N_MACHINES:-3} IP_PREFIX=${IP_PREFIX:-192.168.48} IP_SUFFIX_BASE=${IP_SUFFIX_BASE:-10} if [ -z "$HOSTS" ] ; then for i in $(seq 1 $N_MACHINES); do IP="${IP_PREFIX}.$((${IP_SUFFIX_BASE}+$i))" HOSTS="$HOSTS $IP" done fi # these are used by the tests HOST1=$(echo $HOSTS | cut -f 1 -d ' ') HOST2=$(echo $HOSTS | cut -f 2 -d ' ') HOST3=$(echo $HOSTS | cut -f 3 -d ' ') . "$DIR/assert.sh" SSH_DIR=${SSH_DIR:-$DIR} SSH_OPTS=${SSH_OPTS:-"-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no -o IdentitiesOnly=yes -o LogLevel=ERROR"} SSH=${SSH:-ssh -l vagrant -i "$SSH_DIR/insecure_private_key" $SSH_OPTS} SMALL_IMAGE="alpine" DNS_IMAGE="aanand/docker-dnsutils" TEST_IMAGES="$SMALL_IMAGE $DNS_IMAGE" PING="ping -nq -W 2 -c 1" CHECK_ETHWE_UP="grep ^1$ /sys/class/net/ethwe/carrier" CHECK_ETHWE_MISSING="test ! -d /sys/class/net/ethwe" DOCKER_PORT=2375 CHECKPOINT_DISABLE=true # The regexp here is far from precise, but good enough. (taken from weave script) IP_REGEXP="[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}" CIDR_REGEXP="(ip:|net:)?$IP_REGEXP/[0-9]{1,2}" is_cidr() { [ "$1" = "net:default" ] && return echo "$1" | grep -E "^$CIDR_REGEXP$" >/dev/null } upload_executable() { host=$1 file=$2 target=${3:-/usr/local/bin/$(basename "$file")} dir=$(dirname "$target") run_on $host "[ -e '$dir' ] || sudo mkdir -p '$dir'" [ -z "$DEBUG" ] || greyly echo "Uploading to $host: $file -> $target" >&2 <"$file" remote $host $SSH $host sh -c "cat | sudo tee $target >/dev/null" run_on $host "sudo chmod a+x $target" } remote() { rem=$1 shift 1 "$@" > >(while read line; do echo -e $'\e[0;34m'"$rem>"$'\e[0m'" $line"; done) } colourise() { [ -t 0 ] && echo -ne $'\e['$1'm' || true shift # It's important that we don't do this in a subshell, as some # commands we execute need to modify global state "$@" [ -t 0 ] && echo -ne $'\e[0m' || true } whitely() { colourise '1;37' "$@" } greyly () { colourise '0;37' "$@" } redly() { colourise '1;31' "$@" } greenly() { colourise '1;32' "$@" } run_on() { host=$1 shift 1 [ -z "$DEBUG" ] || greyly echo "Running on $host: $@" >&2 remote $host $SSH $host "$@" } get_command_output_on() { host=$1 shift 1 [ -z "$DEBUG" ] || greyly echo "Running on $host: $@" >&2 $SSH $host "$@" } docker_on() { host=$1 shift 1 [ -z "$DEBUG" ] || greyly echo "Docker on $host:$DOCKER_PORT: $@" >&2 docker -H tcp://$host:$DOCKER_PORT "$@" } docker_api_on() { host=$1 method=$2 url=$3 data=$4 shift 4 [ -z "$DEBUG" ] || greyly echo "Docker (API) on $host:$DOCKER_PORT: $method $url" >&2 echo -n "$data" | curl -s -f -X "$method" -H Content-Type:application/json "http://$host:$DOCKER_PORT/v1.15$url" -d @- } proxy() { DOCKER_PORT=12375 "$@" } weave_on() { host=$1 shift 1 [ -z "$DEBUG" ] || greyly echo "Weave on $host:$DOCKER_PORT: $@" >&2 CHECKPOINT_DISABLE="$CHECKPOINT_DISABLE" DOCKER_HOST=tcp://$host:$DOCKER_PORT $WEAVE "$@" } stop_weave_on() { host=$1 weave_on $host stop 1>/dev/null 2>&1 || true if [ -n "$COVERAGE" ]; then collect_coverage $host weave fi } exec_on() { host=$1 container=$2 shift 2 docker -H tcp://$host:$DOCKER_PORT exec $container "$@" } # Look through 'docker run' args and try to make the hostname match the name dns_args() { local host=$1 NAME_ARG="" HOSTNAME_SPECIFIED="" shift weave_on $host dns-args "$@" while [ $# -gt 0 ] ; do case "$1" in --name) NAME_ARG="$2" shift ;; --name=*) NAME_ARG="${1#*=}" ;; -h|--hostname|--hostname=*) HOSTNAME_SPECIFIED=1 ;; esac shift done if [ -n "$NAME_ARG" -a -z "$HOSTNAME_SPECIFIED" ] ; then echo " --hostname=$NAME_ARG.weave.local" fi } start_container_image() { local image=$1 host=$2 cidr="" weave_dns_args="" container shift 2 if [ "$1" = "--without-dns" ] ; then shift else weave_dns_args=$(dns_args $host "$@") fi is_cidr "$1" && { cidr=$1; shift; } container=$(docker_on $host run $weave_dns_args $name_args "$@" -dt $image /bin/sh) if ! weave_on $host attach $cidr $ATTACH_ARGS $container >/dev/null ; then docker_on $host rm -f $container return 1 fi echo $container } start_container() { start_container_image $SMALL_IMAGE "$@" } start_container_with_dns() { start_container_image $DNS_IMAGE "$@" } start_container_local_plugin() { host=$1 shift 1 # using ssh rather than docker -H because CircleCI docker client is older $SSH $host docker run "$@" -dt --net=weave $SMALL_IMAGE /bin/sh } proxy_start_container() { host=$1 shift 1 proxy docker_on $host run "$@" -dt $SMALL_IMAGE /bin/sh } proxy_start_container_with_dns() { host=$1 shift 1 proxy docker_on $host run "$@" -dt $DNS_IMAGE /bin/sh } # Execute command $1 repeatedly until it returns true; description in $2, optional repeat limit in $3 wait_for_x() { reps=${3:-45} for i in $(seq 1 $reps); do echo "Waiting for $2" if $1 ; then return fi sleep 1 done echo "Timed out waiting for $2" >&2 exit 1 } check_proxy() { proxy docker_on $1 info >/dev/null 2>&1 } wait_for_proxy() { wait_for_x check_proxy "proxy to start" 120 } rm_containers() { host=$1 shift [ $# -eq 0 ] || docker_on $host rm -f -v "$@" >/dev/null } container_ip() { weave_on $1 ps $2 | grep -o -E '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' } container_pid() { docker_on $1 inspect -f '{{.State.Pid}}' $2 } check_attached() { exec_on $1 $2 $CHECK_ETHWE_UP >/dev/null 2>&1 } wait_for_attached() { wait_for_x "check_attached $1 $2" "$2 on $1 to be attached" } connections_metric() { $SSH $1 curl -sS http://127.0.0.1:6784/metrics | awk -e "/^weave_connections.*$2/ { print \$2 }" } # assert_dns_record [ ...] assert_dns_record() { local host=$1 local container=$2 local name=$3 shift 3 exp_ips_regex=$(echo "$@" | sed -e 's/ /\\\|/g') [ -z "$DEBUG" ] || greyly echo "Checking whether $name exists at $host:$container" assert_raises "exec_on $host $container getent hosts $name | grep -q '$exp_ips_regex'" [ -z "$DEBUG" ] || greyly echo "Checking whether the IPs '$@' exists at $host:$container" for ip in "$@" ; do assert "exec_on $host $container getent hosts $ip | tr -s ' ' | tr '[:upper:]' '[:lower:]'" "$(echo $ip $name | tr '[:upper:]' '[:lower:]')" done } # assert_no_dns_record assert_no_dns_record() { host=$1 container=$2 name=$3 [ -z "$DEBUG" ] || greyly echo "Checking if '$name' does not exist at $host:$container" assert_raises "exec_on $host $container getent hosts $name" 2 } # assert_dns_a_record [] assert_dns_a_record() { exp_name=${5:-$3} assert "exec_on $1 $2 getent hosts $3 | tr -s ' ' | cut -d ' ' -f 1,2" "$4 $exp_name" } # assert_dns_ptr_record assert_dns_ptr_record() { assert "exec_on $1 $2 getent hosts $4 | tr -s ' '" "$4 $3" } # Kill a container process and make sure it's restarted by Docker check_restart() { OLD_PID=$(container_pid $1 $2) run_on $1 sudo kill $OLD_PID for i in $(seq 1 10); do NEW_PID=$(container_pid $1 $2) if [ $NEW_PID != 0 -a $NEW_PID != $OLD_PID ] ; then return 0 fi sleep 1 done return 1 } start_suite() { for host in $HOSTS; do [ -z "$DEBUG" ] || echo "Cleaning up on $host: removing all containers and resetting weave" rm_containers $host $(docker_on $host ps -aq 2>/dev/null) weave_on $host reset 2>/dev/null run_on $host sudo rm -f /opt/cni/bin/weave-plugin-latest /opt/cni/bin/weave-net /opt/cni/bin/weave-ipam /etc/cni/net.d/10-weave.conflist done whitely echo "$@" } # Common postconditions to assert on each host, after each test: assert_common_postconditions() { # Ensure we do not generate any defunct (a.k.a. zombie) process: assert "run_on $1 ps aux | grep '[d]efunct'" "" } end_suite() { for host in $HOSTS; do assert_common_postconditions "$host" done whitely assert_end for host in $HOSTS; do stop_weave_on $host done } collect_coverage() { host=$1 container=$2 mkdir -p ./coverage rm -f cover.prof docker_on $host cp $container:/home/weave/cover.prof . 2>/dev/null || return 0 # ideally we'd know the name of the test here, and put that in the filename mv cover.prof $(mktemp -u ./coverage/integration.XXXXXXXX) || true } WEAVE=$DIR/../weave DOCKER_NS=$DIR/../bin/docker-ns