Add release v1.9.3

Do not reset fastdp heartbeat timer for non-confirmed connections.  Fixes #2824

.gitignore

@@ -68,3 +68,5 @@ test/tls/*.pem
 test/coverage
 test/coverage.*
 *qemu-*-static
+terraform.tfstate
+terraform.tfstate.backup

.gitmodules

@@ -3,7 +3,7 @@
 	url = https://github.com/weaveworks/build-tools
 [submodule "vendor/github.com/vishvananda/netlink"]
 	path = vendor/github.com/vishvananda/netlink
-	url = https://github.com/weaveworks/netlink.git
+	url = https://github.com/vishvananda/netlink.git
 [submodule "vendor/github.com/gorilla/mux"]
 	path = vendor/github.com/gorilla/mux
 	url = https://github.com/gorilla/mux.git

.lintignore

@@ -1,8 +1,27 @@
 # IMPORTANT: Please only use GLOBs to ignore files and directories.
 
-# Do not lint any bash script in Weave Net, in order to 
+# Do not lint the below bash scripts in Weave Net, in order to 
 # avoid breaking the build on make lint:
-*.sh
-bin/*
+bin/circle-deploy-issues
+bin/circle-deploy-master
+bin/circle-teardown-pre
+bin/circle-test-teardown
+bin/circle-test-unit
+bin/install-wordepress
+bin/multiweave
+bin/publish-site
+bin/release
+build/build.sh
+prog/weave-kube/launch.sh
 prog/weaveexec/symlink
+test/*_test.sh
+test/600_proxy_docker_py.sh
+test/assert.sh
+test/config.sh
+test/gce.sh
+test/gen_coverage_reports.sh
+test/run_all.sh
+test/sanity_check.sh
+test/setup.sh
+vendor/*.sh
 weave

CHANGELOG.md

@@ -1,3 +1,44 @@
+## Release 1.9.3
+
+Bug fixes and minor improvements
+
+* Fixed a race condition in Fast Datapath encrypted connections which could lead Weave Net to crash: #2824, #2825.
+
+## Release 1.9.2
+
+Bug fixes and minor improvements
+
+* Fix a weave-kube bug when `br_netfilter` or `xt_set` module is compiled into
+  kernel #2820/#2821
+* Detect the absence of the required `xt_set` kernel module #2821
+
+## Release 1.9.1
+
+Bug fixes and minor improvements
+
+* Fix a race condition when the Weave Net container is restarted
+  which could allow a new container to be allocated the same IP
+  address as an existing one #2784,#2787
+* Handle the message type received when a pod has been deleted during
+  Kubernetes api-server fail-over #2772,#2773
+* Make weave-kube work with `dockerd --iptables=false` #2726
+* Ensure we have the right kernel modules loaded for Network Policy in weave-kube #2819
+* Reference-count addresses in Network Policy Controller, to avoid
+  errors when updates come in an unexpected order #2792,#2795
+* Allow the soft connection limit to be raised in weave-kube, so
+  larger clusters can be created #2781
+* WeaveDNS was incorrectly case-sensitive for reverse DNS lookups #2817,#2818
+
+Build and Testing
+
+* Scripts to create VMs to run automated tests were rewritten to use
+  Terraform and Ansible, to make it much easier to test with different
+  versions of components such as Docker and Kubernetes #2647,#2694,#2775,#2796
+* Upgrade to latest Weaveworks common build-tools #2780
+* Improve encryption tests #2793
+* Update vishvananda/netlink library to bring in changes we had previously forked #2790
+* Slight change to the build container to avoid permission errors and slow builds #2761,#2802
+
 ## Release 1.9.0
 
 Highlights:

DEPENDENCIES

@@ -0,0 +1,3 @@
+DOCKER_VERSION=1.11.2
+KUBERNETES_VERSION=1.5.2
+KUBERNETES_CNI_VERSION=0.3.0.1

Makefile

@@ -214,7 +214,7 @@ tests:
 	./tools/test -no-go-get -netgo -timeout 8m
 
 lint:
-	./tools/lint -nocomment -notestpackage .
+	./tools/lint -nocomment -notestpackage
 
 endif
 
@@ -359,3 +359,20 @@ build:
 
 run-smoketests: all testrunner
 	cd test && ./setup.sh && ./run_all.sh
+
+integration-tests: all testrunner
+# Usage:
+#   $ make \
+#     NAME="<prefix used to name VMs and other resources>" \
+#     PROVIDER="<provider among {vagrant|gcp|aws|do}>" \
+#     NUM_HOSTS="<# test machines>" \
+#     PLAYBOOK="<filename>" \
+#     RUNNER_ARGS="<...>" \
+#     TESTS="<...>" \  # Can be set to only run one or a few tests instead of the full test suite.
+#     DOCKER_VERSION=<...> \
+#     KUBERNETES_VERSION=<...> \
+#     KUBERNETES_CNI_VERSION=<...> \
+#     <...> # See also run-integration-test.sh for all variables and individual functions.
+#     integration-tests
+#
+	RUNNER_ARGS="-parallel" ./test/run-integration-tests.sh

README.md

@@ -23,5 +23,20 @@ Weave Net creates a virtual network that connects Docker containers across multi
 
 Services provided by application containers on the Weave network can be exposed to the outside world, regardless of where they are running. Similarly, existing internal systems can be opened to accept connections from application containers irrespective of their location.
 
-* [Install Weave Net](https://www.weave.works/docs/net/latest/installing-weave/)
-* [Find out more about Weave Net](https://www.weave.works/products/weave-net/)
+## <a name="help"></a>Getting help
+
+If you have any questions about, feedback for or a problem with Weave Net:
+
+- Read [the Weave Net docs](https://www.weave.works/docs/net/latest/introducing-weave/).
+- Invite yourself to the <a href="https://weaveworks.github.io/community-slack/" target="_blank"> #weave-community </a> slack channel.
+- Ask a question on the <a href="https://weave-community.slack.com/messages/general/"> #weave-community</a> slack channel.
+- Join the <a href="https://www.meetup.com/pro/Weave/"> Weave User Group </a> and get invited to online talks, hands-on training and meetups in your area.
+- Send an email to <a href="mailto:weave-users@weave.works">weave-users@weave.works</a>
+- <a href="https://github.com/weaveworks/weave/issues/new">File an issue.</a>
+
+Your feedback is always welcome!
+
+##Further Information
+
+* [Installing Weave Net](https://www.weave.works/docs/net/latest/installing-weave/)
+* [Weave Net Product Overview](https://www.weave.works/products/weave-net/)

Vagrantfile

@@ -1,45 +1,36 @@
-VAGRANTFILE_API_VERSION = "2"
-
-require './vagrant-common.rb'
+require File.expand_path(File.join(File.dirname(__FILE__), 'vagrant-common.rb'))
 
 vm_ip = "172.16.0.3" # arbitrary private IP
 
-pkgs = %w(
-  docker-engine=1.10.2-0~wily
-  aufs-tools
-  build-essential
-  ethtool
-  iputils-arping
-  libpcap-dev
-  git
-  mercurial
-  bc
-  jq
-)
-
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
-  config.vm.box = "ubuntu/wily64"
+  config.vm.box = VAGRANT_IMAGE
 
   config.vm.network "private_network", ip: vm_ip
   config.vm.provider :virtualbox do |vb|
     vb.memory = 2048
-    vb.customize ["modifyvm", :id, "--natdnshostresolver1", "off"]
-    vb.customize ["modifyvm", :id, "--natdnsproxy1", "off"]
+    configure_nat_dns(vb)
   end
 
+  # Disable default Vagrant shared folder, which we don't need:
   config.vm.synced_folder ".", "/vagrant", disabled: true
+  # Keep Weave Net sources' in sync:
   config.vm.synced_folder ".", "/home/vagrant/src/github.com/weaveworks/weave"
+  # Create a convenience symlink to $HOME/src/github.com/weaveworks/weave
+  config.vm.provision :shell, :inline => 'ln -sf ~vagrant/src/github.com/weaveworks/weave ~vagrant/'
 
   # Set SSH keys up to be able to run smoke tests straightaway:
   config.vm.provision "file", source: "~/.vagrant.d/insecure_private_key", destination: "/home/vagrant/src/github.com/weaveworks/weave/test/insecure_private_key"
-
-  install_build_deps config.vm, pkgs
-  install_go_toochain config.vm
-  tweak_user_env config.vm
-  tweak_docker_daemon config.vm
+  # Grant permissions on sources:
+  config.vm.provision :shell, :inline => 'sudo chown -R vagrant:vagrant ~vagrant/src', :privileged => false
   cleanup config.vm
 
+  config.vm.provision 'ansible' do |ansible|
+    ansible.playbook = 'tools/config_management/setup_weave-net_dev.yml'
+    ansible.extra_vars = {
+      go_version: GO_VERSION
+    }.merge(ansibleize(get_dependencies_version_from_file_and_env()))
+  end
 end
 
 begin

bin/circle-dependencies-post

@@ -2,45 +2,16 @@
 
 set -e
 
-echo "TEST_AND_PUBLISH=1" > "$STATE"
-
-# Avoid doing a full build if we can.  Note if the PR doesn't exist
-# when the build is kicked off, we can't run this check so we do a
-# full build.  Subsequent pushes to the same branch will have the desired behaviour
-if [ -n "$CI_PULL_REQUEST" -a "$CI_PULL_REQUEST" = "$CI_PULL_REQUESTS" ] ; then
-    if PR=$(echo "$CI_PULL_REQUEST" | grep -oP '(?<=^https://github.com/weaveworks/weave/pull/)[0-9]+$') ; then
-        echo "GH_PR=$PR" >> "$STATE"
-        if FILES=$(curl -s https://api.github.com/repos/weaveworks/weave/pulls/$PR/files | jq -r '.[]|.filename') ; then
-            echo "PR touches [$FILES]"
-            if ! echo "$FILES" | grep -v ^site/ ; then
-                echo "PR contains site changes only - skipping tests"
-                echo "TEST_AND_PUBLISH=" >> "$STATE"
-            else
-                echo "PR contains non-site changes - running tests"
-            fi
-        else
-            echo "Unable to get files for PR $PR" >&2
-        fi
-    else
-        echo "Unable to extract PR number from $CI_PULL_REQUEST" >&2
-    fi
-fi
-
 source "$STATE"
 
-mkdir -p $(dirname $SRCDIR) && cp -r $(pwd)/ $SRCDIR
-
-if [ -n "$TEST_AND_PUBLISH" ] ; then
-    [ -z "$SECRET_PASSWORD" ] || bin/setup-circleci-secrets "$SECRET_PASSWORD"
-    cd $SRCDIR; git submodule update --init
-    if [ "$CIRCLE_NODE_INDEX" = "0" -a -n "$SECRET_PASSWORD" ] ; then
-        cd $SRCDIR/test; ./gce.sh make_template
-    fi
-    cd $SRCDIR/build
+if [ -n "$TEST_AND_PUBLISH" ]; then
+    cd "$SRCDIR"
+    git submodule update --init
+    cd "$SRCDIR/build"
     ../tools/rebuild-image weaveworks/weavebuild . Dockerfile build.sh
-    touch $SRCDIR/.build.uptodate
+    touch "$SRCDIR/.build.uptodate"
     sudo chown -R ubuntu /usr/local/go
-    cd $SRCDIR
+    cd "$SRCDIR"
     make testrunner
     make COVERAGE=true RM= exes all
 fi

bin/circle-dependencies-post-sources

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e
+
+echo "TEST_AND_PUBLISH=1" >"$STATE"
+
+# Avoid doing a full build if we can.  Note if the PR doesn't exist
+# when the build is kicked off, we can't run this check so we do a
+# full build.  Subsequent pushes to the same branch will have the desired behaviour
+if [ -n "$CI_PULL_REQUEST" -a "$CI_PULL_REQUEST" = "$CI_PULL_REQUESTS" ]; then
+    if PR=$(echo "$CI_PULL_REQUEST" | grep -oP '(?<=^https://github.com/weaveworks/weave/pull/)[0-9]+$'); then
+        echo "GH_PR=$PR" >>"$STATE"
+        if FILES=$(curl -s "https://api.github.com/repos/weaveworks/weave/pulls/$PR/files" | jq -r '.[]|.filename'); then
+            echo "PR touches [$FILES]"
+            if ! echo "$FILES" | grep -v ^site/; then
+                echo "PR contains site changes only - skipping tests"
+                echo "TEST_AND_PUBLISH=" >>"$STATE"
+            else
+                echo "PR contains non-site changes - running tests"
+            fi
+        else
+            echo "Unable to get files for PR $PR" >&2
+        fi
+    else
+        echo "Unable to extract PR number from $CI_PULL_REQUEST" >&2
+    fi
+fi
+
+source "$STATE"
+
+mkdir -p "$(dirname "$SRCDIR")" && cp -r "$(pwd)/" "$SRCDIR"
+
+if [ -n "$TEST_AND_PUBLISH" ]; then
+    cd "$SRCDIR"
+    git submodule update --init
+fi

bin/circle-test-pre

@@ -2,9 +2,29 @@
 
 set -e
 
-source "$STATE"
+# Signal failures in lock file, in order to fail fast:
+function signal_failure() {
+    echo "KO" >"$TEST_VMS_READY_LOCK_FILE"
+    exit 1
+}
+trap signal_failure ERR
 
-if [ -n "$TEST_AND_PUBLISH" ] ; then
-    cd $SRCDIR/test
-    [ -z "$SECRET_PASSWORD" ] || (./gce.sh setup && eval $(./gce.sh hosts) && ./setup.sh)
+source "$STATE"
+source "$(dirname "$0")/wait_for.sh"
+
+if [ -n "$TEST_AND_PUBLISH" ]; then
+    [ -n "$SECRET_KEY" ] || {
+        echo "Cannot run smoke tests: no secret key"
+        exit 1
+    }
+
+    # Provisioning from image typically take 90 seconds, 
+    # but if we are creating a GCP image, it can take more than 10 minutes:
+    wait_for 900 "$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
+
+    # Upload built binaries to testing VMs:
+    cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests.
+    ./run-integration-tests.sh setup >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
+    echo "OK" >"$TEST_VMS_READY_LOCK_FILE"
+    echo "Test VMs now ready. $(date)." >>"$TEST_VMS_SETUP_OUTPUT_FILE"
 fi

bin/circle-test-smoke

@@ -3,12 +3,21 @@
 set -e
 
 source "$STATE"
+source "$(dirname "$0")/wait_for.sh"
 
-if [ -n "$TEST_AND_PUBLISH" ] ; then
-    [ -n "$SECRET_PASSWORD" ] || { echo "Cannot run smoke tests: no secret key"; exit 1; }
-    cd $SRCDIR/test
-    eval $(./gce.sh hosts)
+if [ -n "$TEST_AND_PUBLISH" ]; then
+    [ -n "$SECRET_KEY" ] || {
+        echo "Cannot run smoke tests: no secret key"
+        exit 1
+    }
+
+    # Provisioning from image typically take 90 seconds, 
+    # but if we are creating a GCP image, it can take more than 10 minutes:
+    wait_for 900 "$TEST_VMS_READY_LOCK_FILE"
+
+    # Run integration tests:
     export COVERAGE=true
     export WEAVE_NET_SANITY_CHECKS_FILES="$CIRCLE_ARTIFACTS/weave_net_sanity_check_*.log"
-    ./run_all.sh
+    cd "$SRCDIR/test" # Ensures we generate code coverage files in the right folder, and use the Terraform state of previously provisionned VMs.
+    ./run-integration-tests.sh test
 fi

bin/circle-test-teardown

@@ -6,6 +6,5 @@ source "$STATE"
 
 if [ -n "$TEST_AND_PUBLISH" ] ; then
     cd $SRCDIR/test
-    [ -z "$SECRET_PASSWORD" ] || ./gce.sh destroy
     test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR/test; ./gen_coverage_reports.sh)
 fi

bin/provision_test_vms.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+set -e
+
+# Signal failures in lock file, in order to fail fast:
+function signal_failure() {
+    echo "KO" >"$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
+    exit 1
+}
+trap signal_failure ERR
+
+source "$STATE"
+
+function install_terraform() {
+    curl -fsS https://releases.hashicorp.com/terraform/0.8.5/terraform_0.8.5_linux_amd64.zip | gunzip >terraform && chmod +x terraform && sudo mv terraform /usr/bin
+}
+
+function install_ansible() {
+    sudo apt-get update || true
+    sudo apt-get install -qq -y python-pip python-dev libffi-dev libssl-dev && pip install --user -U cffi && pip install --user ansible
+    export PATH="$PATH:$HOME/.local/bin"
+}
+
+if [ -n "$TEST_AND_PUBLISH" ]; then
+    [ -n "$SECRET_KEY" ] || {
+        echo "Cannot run smoke tests: no secret key"
+        exit 1
+    }
+
+    install_terraform >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
+    install_ansible >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
+
+    # Only attempt to create GCP image in first container, wait for it to be created otherwise:
+    [ "$CIRCLE_NODE_INDEX" != "0" ] && export CREATE_IMAGE=0
+
+    # Provision and configure testing VMs:
+    cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests.
+    ./run-integration-tests.sh configure >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
+    echo "OK" >"$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
+    echo "Test VMs now provisioned and configured. $(date)." >>"$TEST_VMS_SETUP_OUTPUT_FILE"
+fi

bin/release

@@ -89,6 +89,7 @@ build() {
     ## Inject the version numbers and build the distributables
     ## (library versions?)
     sed -i "/SCRIPT_VERSION=/ c\SCRIPT_VERSION=\"$VERSION\"" ./weave
+    sed -i -e "s/:latest/:$VERSION/" -e "/imagePullPolicy: Always/d" ./prog/weave-kube/weave-daemonset.yaml
     make SUDO=$SUDO WEAVE_VERSION=$VERSION DOCKERHUB_USER=$DOCKERHUB_USER
 
     if make tests; then
@@ -157,21 +158,6 @@ draft() {
         --name "weave" \
         --file "./weave"
 
-    github-release release $RELEASE_ARGS \
-        --user $GITHUB_USER \
-        --repo weave-kube \
-        --tag $LATEST_TAG \
-        --name "$RELEASE_NAME $VERSION" \
-        --description "$RELEASE_DESCRIPTION"
-
-    sed -i -e s/:latest/:$VERSION/ -e "/imagePullPolicy: Always/d" ./prog/weave-kube/weave-daemonset.yaml
-    github-release upload \
-        --user $GITHUB_USER \
-        --repo weave-kube \
-        --tag $LATEST_TAG \
-        --name "weave-daemonset.yaml" \
-        --file "./prog/weave-kube/weave-daemonset.yaml"
-
     github-release upload \
         --user $GITHUB_USER \
         --repo weave \
@@ -258,33 +244,6 @@ publish() {
             --name "weave" \
             --file "./weave"
 
-        github-release publish \
-            --user $GITHUB_USER \
-            --repo weave-kube \
-            --tag $LATEST_TAG
-
-        if github-release info --user $GITHUB_USER --repo weave-kube \
-            --tag latest_release >/dev/null 2>&1; then
-            github-release delete \
-                --user $GITHUB_USER \
-                --repo weave-kube \
-                --tag latest_release
-        fi
-
-        github-release release \
-            --user $GITHUB_USER \
-            --repo weave-kube \
-            --tag latest_release \
-            --name "$RELEASE_NAME latest ($VERSION)" \
-            --description "[Release Notes](https://github.com/$GITHUB_USER/weave/releases/$LATEST_TAG)"
-
-        github-release upload \
-            --user $GITHUB_USER \
-            --repo weave-kube \
-            --tag latest_release \
-            --name "weave-daemonset.yaml" \
-            --file "./prog/weave-kube/weave-daemonset.yaml"
-
         github-release upload \
             --user $GITHUB_USER \
             --repo weave \

bin/setup-circleci-secrets

@@ -1,115 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-# openssl enc -in do-setup-circleci-secrets.orig -out setup-circleci-secrets.orig -e -aes256 -pass stdin
-# openssl base64 < setup-circleci-secrets.orig
-
-openssl base64 -d << EOF \
-  | openssl enc \
-    -out bin/do-setup-circleci-secrets \
-		-d -aes256 -pass pass:$1
-U2FsdGVkX1/3QHnDgpZn0srp+GiEQBTlag0CBdsw6Bp6iaIKy6Jy8gFshrsmvl3+
-c+7hn1nqsBVpzVH0S5QA3Ex5nx9rl7/dcGoogsySFcwMmBFcauT0UTMK1bvX0Hqx
-QrshYSndUkww72KA7EsSnjR1BBJWm3AhCDOTIcnO6nQhig1Fc+SBEj5FGS5xTQ5Z
-/8hDsKf54Ms49vOO6QeE5w5ndZ+k7IvVJfq9Evhb7htD20RntDhR86foAwMnLV7R
-Wh9qAqI3gnRq0GmbeRKaHmB7pcLq9f0OdScqM/Tn1D+q0556fqJjBziXpZiGbxVs
-DxEp8zDJ8yNDAi0Wb2NXjoA79SXJ8Q7iTieaST4y2OmXQpY+Vw8RVDXxi6ERyVnH
-41OeMzyLm0qvCoMl32pe1sSG5DRCUiAm7wOtDTVvNDRqmGJYUgwwexyayecJI1wQ
-LlhYLJavUvw3UGDZOld8BaMQhfOKBVCdFrjmA4HuVxmOf3029qkgzOCSZiQVNqs+
-fvM6J05zBuk+GPy8YInvRkzfxum/x00vmUyGjMs/lHDFfrmYDFK3ux41EoVqvWLG
-S5uYxdZJFdR3Xsrs+sMr51XNBQGMmz6KoEEzFYyHs5fZgtdpRdPAuQS1m6FqgZyi
-wSz6+EKsALtmGorEp+CF8JN5YzKJCrKzNkg0fILBqsqmJZGP64lmX69950lRLNGM
-/l3KVR2zH31b4AnnzTcvldVSCKDXjaMw3JA2Ce/MK6XhOWOcpEv4oDt/62lbhUKP
-QUQTPbfpQRvVfBcoIrfAm3ghi/CZc2TDVEuD7DhasLTZD3r7LoK261BWC9O5Wubi
-vlXuFIF+uyrpTDf6EJKE/+XywtArppffMP6RkHu1Iqnd8JtsqPaIZsmMmZbjgb6j
-BX4ZLb5U9R7jYoSgIke4+u6geFcrMQ+Lny4r7MxXaS0Y1UJ6Jo1bwpYw1TBIobZc
-eQ30+xhbSeRC9vEw1NXI6qM2Ai2gIHtAZvEiBkBBXTG5Ld7N/mBAmDIJEzVRxh6F
-jRkB3dzESF6eeTBm61OQepwV3aW6lslMaf26drPGv2W2q+K/WOpQOejfggtHUgVs
-9SWqtn7hHYSpIASe1Ihi14HAWv3kPhQien0zY1DLKJWf4ilbRaICn1QSJjNuKwtl
-A2ZwEw7FFddFcsF4okHPkxEoCre3shq0DEw9UA1gnwEaEF1Q+i9cvLtQg56zK3+R
-MfTzFUv2K1z3j1s5tC6Z6LzL8j682EFOll7MkihqiIWh+po1zcNs6Ln0SkjB3v9T
-OFRYVJis9GqXYO9PUW1mI69udR4nXbKlBl5tWNyAxn9hufJHLSAmbGMwActKLRM5
-kROapKrN1Ra/sHUmhoSSDUK9AXeYKkqPXVMbhly9xC1Kya4hFaFokjRQIMLPmXdQ
-lP46RL3HOq0v1BkXg05poSwDm+BtitZkcIhOzcmeucUqn1whsz39y3VoOhJqQu2j
-0dUcj19Fue1CDZfwzACb9mLZr643h0yCIHQaVT7is/yiN2vEt0S0zynvOA1D+Xgy
-4bFdjqed9DgBShLwF/lh4/hur5iZPpVr9A8rt3Vq0k5A1tdjsZpXP3qv5jLc7lJv
-cAQxdVLEuv8WPwwTV/EQsoH7lpoEdjktw/gu0v5PkUgR/WBxEqJq4liTekViQ+8X
-BAmdm/FGWeIxSSEm0a8I+muiBpIuKjuQoJit1f0k+lLUohoVDqpDDfeYHh1TxXHE
-xSKAKNmFSc2XOr5Zg2mOInxeH7WoxH6iylXKADBgEgAQ4C55DoAPRjFNtGGz2iJl
-WkzhQOfgsDss1VxpPZ4SWDQO+xxuuEnYcCYdJ6Tyi5hAwZ0+RZmXx0g9BCK+5qzc
-Ffei066kCItBcayekwCs0i86r8JvDtsMUUkzEKDQTSWHpP8ypIZoz3P+Skdqcs6p
-IcZHy2fODIsClctRAsHOjZ9BnWSY2rqf1lysqvRfOkNTViTOUVx2hf+LuL+8rT6g
-K4GvS39Q+c9FYSMjCNxv1m37NfHJv2W7AlqVV95gMBEFB97M3388T7PfvmSLsyPv
-qAKDE7genTOycQvEX9oeEACxPo08ai2Mgs/YXCig0g0btZoScoKPelQKtD0HLpZ+
-j7ruaKFy4/vySDMostB3AKSkCE/SkGu/W9oI37n1guSYC/9yJjtTcwZULXAClfwt
-LyHvLAw4+7BdGJqDe9Tf8z4zqVLax+BcBHkW7pbP+pvxC6fhIA4PO8k5HwwAVlRw
-68lG6e3z9+Vst4zrFLewHJvkDRSOTSecs9h9SPOjReTK0ovlDzMHVakfUvqpk9mv
-nCj8SWaMPF70z138AuRuKN7X4exNXPBFnOcseKp/NpT1DyvgoHm/oIO65mq9vvo+
-42uSbyyegYiTdVK4SaXBc2hixkhaTteqdEy6GxfWqKCvHtYuRD2LyUSjZn/0Y6CG
-3wZapylKL1sxfH+b1gogv2wAA8KtZAfiTlMISuymwQr5RbOFma7UVMVecCa7XrNG
-fZwO8R+6++ru0efDzeC/gtLOItjn+yVfsbk2gfu0FikHIV1sogOjnsv3dl6IpH/s
-w6WbuK/yZhyrrmhIIueHF30ES512Peogf/Gxp8U/ZAmqBa7p4qsv93XWDR52IPYz
-yU1MtboLD02J39daBNrYZuO4Yfo/xAvU7Ijg2KIR9HS0bSWNuSFAZaZXlJ2bSag3
-8l0dZMx2fZa0FALxWvY2+nd1tWcH+Wo99RSxX+4TH3guCpe1x/58yKOKL4d1xRpu
-02nGaqerV9bHU6WjCUYTrrZurTp6yqivpSFCALDMhAU5Ihd/Qjz69zg4uZ7iqbNd
-WqIYJ3SAOei2tSfoOJbdh8mohotnMFFbupck3Fv+lXAXYDp0V/sSuK32MSeJxan1
-q3t0eenAh6zoBP0Tj53N8B9YmrgZ1i1AljQqYsAdb8N5Z3vecuzQa946U8H4axEc
-BL5fgmqSfsbj1EjJVUrw3cE9hgqeqAYBBvZsyW5zSAsLq/RJKYtBxDGADqr0TOOu
-TjVqfPObb1u9EAQj15ALBhhPORa2gBAK5gnhJWW5deNgA0UOYoWARFYfATuQvr1T
-9xTVbqzZ0VJR4n7X5JyxJo2tZ44QYVzA0fqX0lt+uRkarrkbZMgplKp4MJTFCp2N
-tLciWS0YSHPbKTm7svw5mROazUvEoj1N3SYrJT+Ai9UgSL1cA4FmQ5Jsi+GOeQbH
-7E4pzjeST4b1aATY9ZZSy4TzK+UNYBcCVqY1dN5Xnk0e5yPMnrkkcYI41s/6oOaW
-lQkp/yrHkvpzQWZa7vF2yWg54j3eX8YWx/TsLvmuJmSQqiIHfu9TXcAIPDispCpF
-gp4ZbYPZ8RDQLw3ROc+bZ9E0J8e+FHwgXEELsMLW9n4d3PTj9pLFej1vmBKa03o3
-g+JViNxb1gUPNmC6ADk4o7TQzBrjV2pmmuCza16kOmq6oAbJPOeCLtVWA5HJTX8d
-AuOzBLO37tWP457dNqoNDaCMik3LCy8cslZjAz02D6VG/QGOAXP9x8bJ3zg9N7vW
-EOn1BMtIeMI4yIxpC6HgotNWDNbUBWremAFWYQRgYQqo9L1SdW4hz6X8FqdlBFvK
-ml7sdUwjEy2tNW+aiZI4cgAlI/VvMWYr5kPfhvaUu1wGYeS1TD1B+DPcjcXpOGWo
-vB1f/uvGc61zZWD0ph9f3S9OnvEko5qV3VhxOnxYUZTrqlM7LVgy/sHrSaPdR7Eo
-nb4QGdi5wEGMxVgQlDGwOmEmS5nCaA0RcfHbkQV4SebRleocLB9qcSwj5JzPICAB
-sn1qeSv45LnGdTzUnCQErw1EzgrK1FXOknujjZ3GX5dM7a6Gag0E6cAxoQeC8Mh0
-aUWolyBo9aSXltn7Vnywoe4X3v96eVH6tUQtN3wt51LBlfSDTWOQDeXLFyW57oYL
-/ErbmJLaSHpANBcC9InI+rxNYePTTba7K8rdPDppmMEmdOXjaL+5aPay2Bay9K8T
-FUyZdoyrbzxFZYT9/YyVJV/ukVcT9ru3HpzLTg9Vgc/CyRfIvCZY/Gzh+pPUbRsR
-pHYDT09BFS+mjszTkcdbceweYXJvknXpajMiISrPUUkb9rwCTmLS4QR4y7I9zAgJ
-9U72mCZB9yhhzS17kyKAANihXvOFrMXCO7QTWZbtSkobMBS2SybDxYP3rM5b6824
-HQaY4MZW5qCndydfz4fa908XbPTDka9Nu3CZfI3fhGiiGDFWViotGEYE4i9BuGZJ
-vmtzII7zeIoHwM4t5fb5mYDd9H+Rkct1yATGEIFEx7cu1fj4Ht2177EzhdkF3Q6I
-Fccn43yGDW06nFvEjnjb+Lz4EvtGp6dDefUhNBohdzd4Rm2o+etKobm0lZ1xOX74
-2gOGBLfXS6PF+msNkWP76mqKE6hG0+R1iPrN06Arti+ZRBhNGJDp1pnOERl7IyB0
-hiLUqae7xVSMVRDeq4O04EUpMXh5ixs3RJXjthYILt7vX+6qskJC3rMuyThrsvMi
-Kk0wErDpzR5U7KFnM2fk4jY5by/0b/jHZ/PWFGnaFrXNOiwK/mV1RIJtg5FCrvMv
-xJN5Qo7o2ZYzUL+IBf5mP5O49FCnz3e++9r4ySY/K57ybhQft1l+cQPWS7xuvVAg
-wZGM9DPl9AhBus0I3V3XzfkcWWHkMynMRcfTnkBHrmKjMopIOggsq05JPalM9b0V
-NfwZngVvTFbQejbGjgQQQ+G7spav/icCIA8Iu8JPHaKZt2IX77L1xMcQOouqMrx8
-JCXnJ7zsuU+8W3GQGm70H2kcen4bf4beA8TSkFmbVsaqDNtGaG+ZopWO8ctZX3CP
-jFgIYgEGi137IWvs1KD5QNFHZI6l0JMzglmvcfnViRPDHLi+EQXAoL/98j/YzA9t
-a9L4xMO0oucovKkqYOS2cdFaQKVIUM9L5ibcbW7Tol4ugMo5ibIsTgjZoyFohG6t
-acg+h2wTTyxtrIapvOOBeAo9V/iQowTAPQf38LKQZ5tApksiCG1qvJerKALTMp1J
-yUHjTCPCPsaiXVVO0mfa9OPtzVvi/icLmlz7umk82mew13djedgMS3/ghq5aEHbg
-e7rtI5snFlm/GQnV6IaCDytb+jYxqQ0DZ9Mb0CHcMpnLyiBO6nzHfadpZYlrxrpx
-dPx9MVqfLavj1UVjKuVj4Mgec9J38IBBRGMJZ4MGhde2hrw/s2jVzG1BzaIzcvFa
-bWiJOkKKRC4BY9DBDYfae5c/obwzkxLmvkbWAVzoFXc+46K1IK6bbC2a4EerOAxc
-C7B3X8BucD8OtiPORwXgjctd9AJ7cXq0EEVGoZZ8jkILnklsK6RXsxrT4gSOGFpx
-cBU7TF4kHpH+o4KGQmjlFA2VYP1tXirjEep57LOHfL+zy+HiSXdsbexf6odkqLYA
-1RR0TzSkmqJYvP0GfHPMvp2xredx1kP934fhTIUcWvErrG3NPd/Dod4BzQye1nNK
-8Y8aFYF6fACcD4SFZ11ecK9DJmB+WXGlsxtO3w1LeEysny2lVkkH6HuMRIzPyXAM
-/sEpwh3hwPJosAh6vGM6JxuL7CXP8hC1oUvK91VR0hxeDcO/LgdcHtDnlNMqt1wC
-jvYt9Hg+3aY5GehBnQPRppnIMaGhiBrRC6HjBg8Umt6UTUr5PXNY2aia5wRcx4M6
-Tj0O0dAt8WWZ9ySgzOoq4SRoaAp6m4E7CpRNMgXH031OWEeHqtt8rsYqAcvcQLuW
-vIwts5hiXZOfgXyR6SBY3ANE1SjDY46na2G4oiG8FI7eAqYgehjHrXqCnGrzjefE
-V1eVcIr7DBYuM8wq7C9x+qdMir6RNV4MrVD+7rO/drivnd3MaqDHlG0LfMZo6GIK
-QZGIy0z1W5lOXm9x3RAz7/Ea11ynkZyPDDxBE5NRcV6JfukdjDrvsaQA/Qk2Wo7h
-PdUZjv17H4syMdap6tUDTc8Cohut6n1UxnDnhaAYEDGR3TgPqI5vEaJ+yoaM9itv
-b2I/K+0uK6GfWwca7PeTtYn92/5etO4czG99xEBFzw7tiEmBTA8KfseZIUe/i3fr
-9LO+rKEGalFB4mVTYyCKPYYx4hOIvjdCox/jEBTFyYUEhgGnZvArgw9XabvII7eM
-c1FmwnN2CM/MHUzM0Kw7cS/oaoRFcG/PqioO/krL2Bd+xThwSH4Zn+Y1UVO7aXNy
-90MOehHSc2nhN984Y2re3ffVsjPhC3KdWxr8jbho/Az65rUkYg64roiO2ZqdqRvf
-aYtX9fCe8N8SzgLTlWZt+l8sjY8HP2k9NoajphV1nG4iC/pX0ZJiVQ+VBRNaGhSl
-6IhM9gM1ZsI86cULJpOTAhNu1fZBv+Mq8IXYD8SqlNVwVtjvshtlzBN4InSGEKiO
-HM7coo7aDuu+Go5AEiIVMbR09jAa0VINsCVScfSbTFhlGRTrZr2DSO/s8WxbV+Mb
-EQAfZmyrLF71Z3MfIukg+ps5iuqFhZ+gJSXXSFwumpg=
-EOF
-
-exec sh bin/do-setup-circleci-secrets

bin/wait_for.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# shellcheck disable=SC2155
+wait_for() {
+    local timeout=$1
+    local lock_file=$2
+    for i in $(seq 1 "$timeout"); do
+        if [ -f "$lock_file" ]; then
+            local status="$(cat "$lock_file")"
+            if [ "$status" == "OK" ]; then
+                echo "[$i seconds]: $lock_file found and status: $status." && return
+            else
+                echo "[$i seconds]: $lock_file found and status: $status." && return 1
+            fi
+        fi
+        if ! ((i % 10)); then echo "[$i seconds]: Waiting for $lock_file to be created..."; fi
+        sleep 1
+    done
+    echo "Timed out waiting for test VMs to be ready. See details in: $TEST_VMS_SETUP_OUTPUT_FILE" >&2
+    return 1
+}

build/Dockerfile

@@ -45,6 +45,8 @@ RUN apt-get update \
 RUN go get \
 	github.com/golang/lint/golint \
 	github.com/fzipp/gocyclo \
+	gopkg.in/mvdan/sh.v1/cmd/shfmt \
+	github.com/fatih/hclfmt \
 	github.com/client9/misspell/cmd/misspell
 
 # First clean, because we're gonna rebuild the std both with -race enabled and disabled
@@ -60,6 +62,9 @@ RUN for platform in ${GO_CROSSPLATFORMS}; do \
 		GOOS=${platform%/*} GOARCH=${platform##*/} \
 			go install -tags netgo std; done
 
+# Allow full write access to the Go folders for anyone
+RUN chmod -R a+w /usr/local/go
+
 # The libpcap version from Debian packages is 1.6.2, matching that version here although newer versions of libpcap have been released
 # We have to cross-compile the libpcap library for the new architectures, that's what we're doing here
 ENV LIBPCAP_CROSS_VERSION=1.6.2

build/build.sh

@@ -25,7 +25,4 @@ echo "weave:x:$uid:$gid::$WEAVE_SRC:/bin/sh" >>/etc/passwd
 echo "weave:*:::::::" >>/etc/shadow
 echo "weave	ALL=(ALL)	NOPASSWD: ALL" >>/etc/sudoers
 
-# Allow full write access to the Go folders for the user Weave
-chown -R $uid:$gid /usr/local/go
-
 su weave -c "PATH=$PATH make -C $WEAVE_SRC BUILD_IN_CONTAINER=false $*"

circle.yml

@@ -14,17 +14,31 @@ machine:
     PATH: $PATH:$HOME/.local/bin
     CLOUDSDK_CORE_DISABLE_PROMPTS: 1
     STATE: /home/ubuntu/state.env
+    # Base name of VMs for integration tests:
+    NAME: test-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX
+    TEST_VMS_SETUP_OUTPUT_FILE: $CIRCLE_ARTIFACTS/test_vms_setup_output.txt
+    TEST_VMS_PROV_AND_CONF_LOCK_FILE: $CIRCLE_ARTIFACTS/test_vms_provisioned_and_configured.txt
+    TEST_VMS_READY_LOCK_FILE: $CIRCLE_ARTIFACTS/test_vms_ready.txt
+    # Google Cloud Platorm images for integration tests:
+    USE_IMAGE: 1
+    DISK_NAME_PREFIX: test-$CIRCLE_BUILD_NUM-0
 
 dependencies:
   cache_directories:
     - "~/docker"
   post:
+    - bin/circle-dependencies-post-sources
+    # Provision & configure testing VMs:
+    - "while [ ! -d $SRCDIR ]; do sleep 1; done; $SRCDIR/bin/provision_test_vms.sh":
+        parallel: true
+        background: true
     - bin/circle-dependencies-post
 
 test:
   pre:
     - "$SRCDIR/bin/circle-test-pre":
         parallel: true
+        background: true
   override:
     - "$SRCDIR/bin/circle-test-unit":
         parallel: true
@@ -37,6 +51,10 @@ test:
 
 teardown:
   pre:
+    # Destroy testing VMs:
+    - "cd $SRCDIR/test && ./run-integration-tests.sh destroy":
+        parallel: true
+        background: true
     - "$SRCDIR/bin/circle-teardown-pre":
         parallel: true
 

docs/fastdp-crypto.md

@@ -4,14 +4,15 @@ This document describes implementation details of the fast datapath encryption.
 
 At the high level, we use the ESP protocol ([RFC 2406][esp]) in the Transport
 mode. Each packet is encrypted with AES in GCM mode ([RFC 4106][aesgcm]), with
-32byte key and 4byte salt. This combo provides the following security
+32 byte key and 4 byte salt. This combo provides the following security
 properties:
 
 * Data confidentiality.
 * Data origin authentication.
 * Integrity.
-* Anti-replay service.
-* Limited traffic flow confidentiality.
+* Anti-replay.
+* Limited traffic flow confidentiality as fast datapath VXLAN packets are fully
+  encrypted.
 
 ## Notation
 

docs/fastdp.md

@@ -52,9 +52,9 @@ overlay, which has a more sophisticated dynamic mechanism for coping
 with low path MTUs.
 
 To avoid triggering this fallback in typical deployments, the datapath
-interface is statically configured with an MTU of 1410 bytes allowing
+interface is statically configured with an MTU of 1376 bytes allowing
 it to work with most underlay network provider MTUs, including GCE at
-1460 bytes (the fifty byte difference accommodates the vxlan overhead).
+1460 bytes (the eighty four byte difference accommodates the encrypted vxlan overhead).
 This value can be overridden by setting `WEAVE_MTU` at launch if
 necessary.
 

docs/ipam.md

@@ -65,14 +65,30 @@ peer for space:
   - it will continue to ask for space until it receives some, or its
     copy of the ring tells it all peers are full in that subnet.
 
+### Data persistence
+
+Key IPAM data is saved to disk, in a [BoltDB](https://github.com/boltdb/bolt)
+file, stored within a [data volume container](https://docs.docker.com/engine/userguide/containers/dockervolumes/#creating-and-mounting-a-data-volume-container)
+named `weavedb`.
+
+This file is used to persist data for various Weave Net components; for IPAM it contains:
+
+* the division of the IP allocation range amongst peers, and
+* allocation of addresses to containers on the local peer,
+
+so that it is immediately available when the peer restarts.
+
 ### Claiming an address
 
-If a Weave process is restarted, in most cases it will hear from
-another peer which ranges it used to own, but it needs to know which
-individual IP addresses are assigned to containers in order to avoid
-giving the same address out in subsequent allocation requests. The
-weave script invokes the `claim` command in `populate_ipam` to do
-this.
+If a Weave process is restarted,
+
+- if `weavedb` is present, then it loads persisted IPAM data from there, as
+described, in the previous section;
+- else, it learns from other peers which ranges it used to own.
+
+Weave Net then consults the allocator to know which individual IP addresses
+are assigned to containers, and therefore avoid giving the same address out in
+subsequent allocation requests.
 
 When the Allocator is told to claim an address, there are four
 possibilities:

docs/release-process.md

@@ -138,6 +138,8 @@ Finally, for **Mainline** releases only:
   published to https://www.weave.works
 * Add the new version of `weave-net` to the checkpoint system at
   https://checkpoint-api.weave.works/admin
+* File a PR to update the version of the daemonset at
+  https://github.com/kubernetes/kops/tree/master/upup/models/cloudup/resources/addons/networking.weave
 
 ## Troubleshooting
 

docs/weavenpc-design.md

@@ -19,6 +19,9 @@ NetworkPolicy object updates from the k8s API server:
   network policy, containing the IP addresses of all pods in the
   namespace whose labels match that selector
 
+IPsets are implemented by the kernel module `xt_set`, without which
+weave-npc will not work.
+
 ipset names are generated deterministically from a string
 representation of the corresponding label selector. Because ipset
 names are limited to 31 characters in length, this is done by taking a
@@ -94,6 +97,9 @@ The following traffic is NOT affected:
   (e.g. kubelet health checks)
 * Traffic routed from an application container to the internet
 
+The above mechanism relies on the kernel module `br_netfilter` being
+loaded and enabled via `/proc/sys/net/bridge/bridge-nf-call-iptables`.
+
 See these resources for helpful context:
 
 * http://ebtables.netfilter.org/br_fw_ia/br_fw_ia.html

ipam/allocate.go

@@ -14,9 +14,9 @@ type allocateResult struct {
 
 type allocate struct {
 	resultChan       chan<- allocateResult
-	ident            string
+	ident            string       // a container ID, something like "weave:expose", or api.NoContainerID
 	r                address.CIDR // Subnet we are trying to allocate within
-	isContainer      bool
+	isContainer      bool         // true if ident is a container ID
 	hasBeenCancelled func() bool
 }
 

ipam/allocator.go

@@ -76,6 +76,13 @@ type Allocator struct {
 	now               func() time.Time
 }
 
+// PreClaims are IP addresses discovered before we could initialize IPAM
+type PreClaim struct {
+	Ident       string // a container ID, something like "weave:expose", or api.NoContainerID
+	IsContainer bool   // true if Ident is a container ID
+	Cidr        address.CIDR
+}
+
 type Config struct {
 	OurName     mesh.PeerName
 	OurUID      mesh.PeerUID
@@ -83,6 +90,7 @@ type Config struct {
 	Seed        []mesh.PeerName
 	Universe    address.CIDR
 	IsObserver  bool
+	PreClaims   []PreClaim
 	Quorum      func() uint
 	Db          db.DB
 	IsKnownPeer func(name mesh.PeerName) bool
@@ -123,6 +131,12 @@ func NewAllocator(config Config) *Allocator {
 		dead:        make(map[string]time.Time),
 		now:         time.Now,
 	}
+
+	alloc.pendingClaims = make([]operation, len(config.PreClaims))
+	for i, c := range config.PreClaims {
+		alloc.pendingClaims[i] = &claim{ident: c.Ident, cidr: c.Cidr}
+	}
+
 	return alloc
 }
 
@@ -156,6 +170,9 @@ func (alloc *Allocator) Start() {
 	default:
 		alloc.infof("Initialising as observer - awaiting IPAM data from another peer")
 	}
+	if loadedPersistedData { // do any pre-claims right away
+		alloc.tryOps(&alloc.pendingClaims)
+	}
 	actionChan := make(chan func(), mesh.ChannelSize)
 	stopChan := make(chan struct{})
 	alloc.actionChan = actionChan

ipam/allocator_test.go

@@ -168,11 +168,13 @@ func TestAllocatorClaim(t *testing.T) {
 		container1 = "abcdef"
 		container3 = "b01df00d"
 		universe   = "10.0.3.0/24"
-		testAddr1  = "10.0.3.2/24"
+		testAddr1  = "10.0.3.5/24"
 		testAddr2  = "10.0.4.2/24"
+		testPre    = "10.0.3.1/24"
 	)
 
-	allocs, router, subnet := makeNetworkOfAllocators(2, universe)
+	preAddr, _ := address.ParseCIDR(testPre)
+	allocs, router, subnet := makeNetworkOfAllocators(2, universe, []PreClaim{{container1, true, preAddr}})
 	defer stopNetworkOfAllocators(allocs, router)
 	alloc := allocs[1]
 	addr1, _ := address.ParseCIDR(testAddr1)
@@ -184,6 +186,8 @@ func TestAllocatorClaim(t *testing.T) {
 	alloc.Prime()
 	// Do an allocate on the other peer, which we will try to claim later
 	addrx, err := allocs[0].Allocate(container1, subnet, true, returnFalse)
+	// Should not get the address we pre-claimed
+	require.NotEqual(t, addrx, preAddr)
 	router.Flush()
 
 	// Now try the claim again
@@ -201,6 +205,11 @@ func TestAllocatorClaim(t *testing.T) {
 	// claiming the address allocated on the other peer should fail
 	err = alloc.SimplyClaim(container1, address.MakeCIDR(subnet, addrx))
 	require.Error(t, err, "claiming address allocated on other peer should fail")
+	// claiming the pre-claimed address should fail on both peers
+	err = alloc.SimplyClaim(container3, preAddr)
+	require.Error(t, err, "claiming address allocated on other peer should fail")
+	err = allocs[0].SimplyClaim(container3, preAddr)
+	require.Error(t, err, "claiming address allocated on other peer should fail")
 	// Check an address outside of our universe
 	addr2, _ := address.ParseCIDR(testAddr2)
 	err = alloc.SimplyClaim(container1, addr2)

ipam/claim.go

@@ -12,10 +12,10 @@ import (
 
 type claim struct {
 	resultChan       chan<- error
-	ident            string
-	cidr             address.CIDR
-	isContainer      bool
-	noErrorOnUnknown bool
+	ident            string       // a container ID, something like "weave:expose", or api.NoContainerID
+	cidr             address.CIDR // single address being claimed
+	isContainer      bool         // true if ident is a container ID
+	noErrorOnUnknown bool         // if false, error or block if we don't know; if true return ok but keep trying
 	hasBeenCancelled func() bool
 }
 
@@ -35,7 +35,7 @@ func (c *claim) sendResult(result error) {
 
 // Try returns true for success (or failure), false if we need to try again later
 func (c *claim) Try(alloc *Allocator) bool {
-	if c.hasBeenCancelled() {
+	if c.hasBeenCancelled != nil && c.hasBeenCancelled() {
 		c.Cancel()
 		return true
 	}

ipam/testutils_test.go

@@ -128,7 +128,7 @@ type mockDB struct{}
 func (d *mockDB) Load(_ string, _ interface{}) (bool, error) { return false, nil }
 func (d *mockDB) Save(_ string, _ interface{}) error         { return nil }
 
-func makeAllocator(name string, cidrStr string, quorum uint) (*Allocator, address.CIDR) {
+func makeAllocator(name string, cidrStr string, quorum uint, preClaims ...PreClaim) (*Allocator, address.CIDR) {
 	peername, err := mesh.PeerNameFromString(name)
 	if err != nil {
 		panic(err)
@@ -145,6 +145,7 @@ func makeAllocator(name string, cidrStr string, quorum uint) (*Allocator, addres
 		OurNickname: "nick-" + name,
 		Universe:    cidr,
 		IsObserver:  quorum == 0,
+		PreClaims:   preClaims,
 		Quorum:      func() uint { return quorum },
 		Db:          new(mockDB),
 		IsKnownPeer: func(mesh.PeerName) bool { return true },
@@ -213,15 +214,19 @@ func AssertNothingSentErr(t *testing.T, ch <-chan error) {
 	}
 }
 
-func makeNetworkOfAllocators(size int, cidr string) ([]*Allocator, *gossip.TestRouter, address.CIDR) {
+func makeNetworkOfAllocators(size int, cidr string, preClaims ...[]PreClaim) ([]*Allocator, *gossip.TestRouter, address.CIDR) {
 	gossipRouter := gossip.NewTestRouter(0.0)
 	allocs := make([]*Allocator, size)
 	var subnet address.CIDR
 
 	for i := 0; i < size; i++ {
 		var alloc *Allocator
+		preClaim := []PreClaim{}
+		if i < len(preClaims) {
+			preClaim = preClaims[i]
+		}
 		alloc, subnet = makeAllocator(fmt.Sprintf("%02d:00:00:02:00:00", i),
-			cidr, uint(size/2+1))
+			cidr, uint(size/2+1), preClaim...)
 		alloc.SetInterfaces(gossipRouter.Connect(alloc.ourName, alloc))
 		alloc.Start()
 		allocs[i] = alloc

nameserver/dns.go

@@ -222,7 +222,7 @@ func (h *handler) handleReverse(w dns.ResponseWriter, req *dns.Msg) {
 		return
 	}
 
-	ipStr := strings.TrimSuffix(req.Question[0].Name, "."+reverseDNSdomain)
+	ipStr := strings.TrimSuffix(strings.ToLower(req.Question[0].Name), "."+reverseDNSdomain)
 	ip, err := address.ParseIP(ipStr)
 	if err != nil {
 		h.nameError(w, req)

npc/ipset/ipset.go

@@ -27,10 +27,11 @@ type Interface interface {
 }
 
 type ipset struct {
+	refCount
 }
 
 func New() Interface {
-	return &ipset{}
+	return &ipset{refCount: newRefCount()}
 }
 
 func (i *ipset) Create(ipsetName Name, ipsetType Type) error {
@@ -38,26 +39,36 @@ func (i *ipset) Create(ipsetName Name, ipsetType Type) error {
 }
 
 func (i *ipset) AddEntry(ipsetName Name, entry string) error {
+	if i.inc(ipsetName, entry) > 1 { // already in the set
+		return nil
+	}
 	return doExec("add", string(ipsetName), entry)
 }
 
 func (i *ipset) DelEntry(ipsetName Name, entry string) error {
+	if i.dec(ipsetName, entry) > 0 { // still needed
+		return nil
+	}
 	return doExec("del", string(ipsetName), entry)
 }
 
 func (i *ipset) Flush(ipsetName Name) error {
+	i.removeSet(ipsetName)
 	return doExec("flush", string(ipsetName))
 }
 
 func (i *ipset) FlushAll() error {
+	i.refCount = newRefCount()
 	return doExec("flush")
 }
 
 func (i *ipset) Destroy(ipsetName Name) error {
+	i.removeSet(ipsetName)
 	return doExec("destroy", string(ipsetName))
 }
 
 func (i *ipset) DestroyAll() error {
+	i.refCount = newRefCount()
 	return doExec("destroy")
 }
 
@@ -67,3 +78,38 @@ func doExec(args ...string) error {
 	}
 	return nil
 }
+
+// Reference-counting
+type key struct {
+	ipsetName Name
+	entry     string
+}
+
+// note no locking is required as all operations are serialised in the controller
+type refCount struct {
+	ref map[key]int
+}
+
+func newRefCount() refCount {
+	return refCount{ref: make(map[key]int)}
+}
+
+func (rc *refCount) inc(ipsetName Name, entry string) int {
+	k := key{ipsetName, entry}
+	rc.ref[k]++
+	return rc.ref[k]
+}
+
+func (rc *refCount) dec(ipsetName Name, entry string) int {
+	k := key{ipsetName, entry}
+	rc.ref[k]--
+	return rc.ref[k]
+}
+
+func (rc *refCount) removeSet(ipsetName Name) {
+	for k := range rc.ref {
+		if k.ipsetName == ipsetName {
+			delete(rc.ref, k)
+		}
+	}
+}

prog/weave-kube/launch.sh

@@ -2,11 +2,29 @@
 
 set -e
 
+modprobe_safe() {
+    modprobe $1 || echo "Ignore the error if \"$1\" is built-in in the kernel" >&2
+}
+
+# Check whether xt_set actually exists
+xt_set_exists() {
+    ipset create weave-kube-test hash:ip
+    iptables -t filter -N WEAVE-KUBE-TEST
+    if ! iptables -A WEAVE-KUBE-TEST -m set --match-set weave-kube-test src -j DROP; then
+        NOT_EXIST=1
+    fi
+    iptables -F WEAVE-KUBE-TEST
+    iptables -X WEAVE-KUBE-TEST
+    ipset destroy weave-kube-test
+    [ -z "$NOT_EXIST" ] || (echo "\"xt_set\" does not exist" >&2 && return 1)
+}
+
 # Default if not supplied - same as weave net default
 IPALLOC_RANGE=${IPALLOC_RANGE:-10.32.0.0/12}
 HTTP_ADDR=${WEAVE_HTTP_ADDR:-127.0.0.1:6784}
 STATUS_ADDR=${WEAVE_STATUS_ADDR:-0.0.0.0:6782}
 HOST_ROOT=${HOST_ROOT:-/host}
+CONN_LIMIT=${CONN_LIMIT:-30}
 
 # Check if the IP range overlaps anything existing on the host
 /usr/bin/weaveutil netcheck $IPALLOC_RANGE weave
@@ -14,6 +32,13 @@ HOST_ROOT=${HOST_ROOT:-/host}
 # Default for network policy
 EXPECT_NPC=${EXPECT_NPC:-1}
 
+# Ensure we have the required modules for NPC
+if [ "${EXPECT_NPC}" != "0" ]; then
+    modprobe_safe br_netfilter
+    modprobe_safe xt_set
+    xt_set_exists
+fi
+
 # kube-proxy requires that bridged traffic passes through netfilter
 if ! BRIDGE_NF_ENABLED=$(cat /proc/sys/net/bridge/bridge-nf-call-iptables); then
     echo "Cannot detect bridge-nf support - network policy and iptables mode kubeproxy may not work reliably" >&2
@@ -66,14 +91,6 @@ if [ -z "$IPALLOC_INIT" ]; then
     IPALLOC_INIT="consensus=$(peer_count $KUBE_PEERS)"
 fi
 
-reclaim_ips() {
-    ID=$1
-    shift
-    for CIDR in "$@" ; do
-        curl -s -S -X PUT "$HTTP_ADDR/ip/$ID/$CIDR" || true
-    done
-}
-
 post_start_actions() {
     # Wait for weave process to become responsive
     while true ; do
@@ -81,15 +98,6 @@ post_start_actions() {
         sleep 1
     done
 
-    # Tell the newly-started weave about existing weave bridge IPs
-    /usr/bin/weaveutil container-addrs weave weave:expose | while read ID IFACE MAC IPS; do
-        reclaim_ips "weave:expose" $IPS
-    done
-    # Tell weave about existing weave process IPs
-    /usr/bin/weaveutil process-addrs weave | while read ID IFACE MAC IPS; do
-        reclaim_ips "_" $IPS
-    done
-
     # Install CNI plugin binary to typical CNI bin location
     # with fall-back to CNI directory used by kube-up on GCI OS
     if ! mkdir -p $HOST_ROOT/opt/cni/bin ; then
@@ -113,5 +121,6 @@ exec /home/weave/weaver $EXTRA_ARGS --port=6783 $BRIDGE_OPTIONS \
      --http-addr=$HTTP_ADDR --status-addr=$STATUS_ADDR --docker-api='' --no-dns \
      --ipalloc-range=$IPALLOC_RANGE $NICKNAME_ARG \
      --ipalloc-init $IPALLOC_INIT \
+     --conn-limit=$CONN_LIMIT \
      "$@" \
      $KUBE_PEERS

prog/weave-kube/weave-daemonset.yaml

@@ -46,6 +46,8 @@ spec:
               mountPath: /host/etc
             - name: dbus
               mountPath: /host/var/lib/dbus
+            - name: lib-modules
+              mountPath: /lib/modules
           resources:
             requests:
               cpu: 10m
@@ -73,3 +75,6 @@ spec:
         - name: dbus
           hostPath:
             path: /var/lib/dbus
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules

prog/weave-npc/main.go

@@ -127,7 +127,15 @@ func root(cmd *cobra.Command, args []string) {
 				handleError(npc.AddNamespace(obj.(*coreapi.Namespace)))
 			},
 			DeleteFunc: func(obj interface{}) {
-				handleError(npc.DeleteNamespace(obj.(*coreapi.Namespace)))
+				switch obj := obj.(type) {
+				case *coreapi.Namespace:
+					handleError(npc.DeleteNamespace(obj))
+				case cache.DeletedFinalStateUnknown:
+					// We know this object has gone away, but its final state is no longer
+					// available from the API server. Instead we use the last copy of it
+					// that we have, which is good enough for our cleanup.
+					handleError(npc.DeleteNamespace(obj.Obj.(*coreapi.Namespace)))
+				}
 			},
 			UpdateFunc: func(old, new interface{}) {
 				handleError(npc.UpdateNamespace(old.(*coreapi.Namespace), new.(*coreapi.Namespace)))
@@ -139,7 +147,15 @@ func root(cmd *cobra.Command, args []string) {
 				handleError(npc.AddPod(obj.(*coreapi.Pod)))
 			},
 			DeleteFunc: func(obj interface{}) {
-				handleError(npc.DeletePod(obj.(*coreapi.Pod)))
+				switch obj := obj.(type) {
+				case *coreapi.Pod:
+					handleError(npc.DeletePod(obj))
+				case cache.DeletedFinalStateUnknown:
+					// We know this object has gone away, but its final state is no longer
+					// available from the API server. Instead we use the last copy of it
+					// that we have, which is good enough for our cleanup.
+					handleError(npc.DeletePod(obj.Obj.(*coreapi.Pod)))
+				}
 			},
 			UpdateFunc: func(old, new interface{}) {
 				handleError(npc.UpdatePod(old.(*coreapi.Pod), new.(*coreapi.Pod)))
@@ -151,7 +167,15 @@ func root(cmd *cobra.Command, args []string) {
 				handleError(npc.AddNetworkPolicy(obj.(*extnapi.NetworkPolicy)))
 			},
 			DeleteFunc: func(obj interface{}) {
-				handleError(npc.DeleteNetworkPolicy(obj.(*extnapi.NetworkPolicy)))
+				switch obj := obj.(type) {
+				case *extnapi.NetworkPolicy:
+					handleError(npc.DeleteNetworkPolicy(obj))
+				case cache.DeletedFinalStateUnknown:
+					// We know this object has gone away, but its final state is no longer
+					// available from the API server. Instead we use the last copy of it
+					// that we have, which is good enough for our cleanup.
+					handleError(npc.DeleteNetworkPolicy(obj.Obj.(*extnapi.NetworkPolicy)))
+				}
 			},
 			UpdateFunc: func(old, new interface{}) {
 				handleError(npc.UpdateNetworkPolicy(old.(*extnapi.NetworkPolicy), new.(*extnapi.NetworkPolicy)))

prog/weaveexec/Dockerfile.template

@@ -19,6 +19,7 @@ RUN apk add --update \
     curl \
     ethtool \
     iptables \
+    ipset \
     iproute2 \
     util-linux \
     conntrack-tools \

prog/weaver/main.go

@@ -306,13 +306,16 @@ func main() {
 			}
 			trackerName = "awsvpc"
 		}
-		allocator, defaultSubnet = createAllocator(router, ipamConfig, db, t, isKnownPeer)
-		observeContainers(allocator)
+		var allContainerIDs []string
 		if dockerCli != nil {
-			ids, err := dockerCli.AllContainerIDs()
+			allContainerIDs, err = dockerCli.AllContainerIDs()
 			checkFatal(err)
-			allocator.PruneOwned(ids)
 		}
+		preClaims, err := findExistingAddresses(dockerCli, allContainerIDs, weavenet.WeaveBridgeName)
+		checkFatal(err)
+		allocator, defaultSubnet = createAllocator(router, ipamConfig, preClaims, db, t, isKnownPeer)
+		observeContainers(allocator)
+		allocator.PruneOwned(allContainerIDs)
 	}
 
 	var (
@@ -445,7 +448,7 @@ func createOverlay(datapathName string, ifaceName string, isAWSVPC bool, host st
 	return overlay, bridge
 }
 
-func createAllocator(router *weave.NetworkRouter, config ipamConfig, db db.DB, track tracker.LocalRangeTracker, isKnownPeer func(mesh.PeerName) bool) (*ipam.Allocator, address.CIDR) {
+func createAllocator(router *weave.NetworkRouter, config ipamConfig, preClaims []ipam.PreClaim, db db.DB, track tracker.LocalRangeTracker, isKnownPeer func(mesh.PeerName) bool) (*ipam.Allocator, address.CIDR) {
 	ipRange, err := ipam.ParseCIDRSubnet(config.IPRangeCIDR)
 	checkFatal(err)
 	defaultSubnet := ipRange
@@ -464,6 +467,7 @@ func createAllocator(router *weave.NetworkRouter, config ipamConfig, db db.DB, t
 		Seed:        config.SeedPeerNames,
 		Universe:    ipRange,
 		IsObserver:  config.Observer,
+		PreClaims:   preClaims,
 		Quorum:      func() uint { return determineQuorum(config.PeerCount, router) },
 		Db:          db,
 		IsKnownPeer: isKnownPeer,

prog/weaver/metrics.go

@@ -82,6 +82,18 @@ var metrics = []metric{
 				ch <- intGauge(desc, metrics.Flows)
 			}
 		}},
+	{desc("weave_ipam_pending_allocates", "Number of pending allocates."),
+		func(s WeaveStatus, desc *prometheus.Desc, ch chan<- prometheus.Metric) {
+			if s.IPAM != nil {
+				ch <- intGauge(desc, len(s.IPAM.PendingAllocates))
+			}
+		}},
+	{desc("weave_ipam_pending_claims", "Number of pending claims."),
+		func(s WeaveStatus, desc *prometheus.Desc, ch chan<- prometheus.Metric) {
+			if s.IPAM != nil {
+				ch <- intGauge(desc, len(s.IPAM.PendingClaims))
+			}
+		}},
 }
 
 func fastDPMetrics(s WeaveStatus) *weave.FastDPMetrics {

prog/weaver/reclaim.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"net"
+
+	docker "github.com/fsouza/go-dockerclient"
+
+	"github.com/weaveworks/weave/api"
+	"github.com/weaveworks/weave/common"
+	weavedocker "github.com/weaveworks/weave/common/docker"
+	"github.com/weaveworks/weave/ipam"
+	weavenet "github.com/weaveworks/weave/net"
+	"github.com/weaveworks/weave/net/address"
+)
+
+func a(cidr *net.IPNet) address.CIDR {
+	prefixLength, _ := cidr.Mask.Size()
+	return address.CIDR{Addr: address.FromIP4(cidr.IP), PrefixLen: prefixLength}
+}
+
+// Get all the existing Weave IPs at startup, so we can stop IPAM
+// giving out any as duplicates
+func findExistingAddresses(dockerCli *weavedocker.Client, containerIDs []string, bridgeName string) (addrs []ipam.PreClaim, err error) {
+	Log.Infof("Checking for pre-existing addresses on %s bridge", bridgeName)
+	// First get the address for the bridge
+	bridgeNetDev, err := weavenet.GetBridgeNetDev(bridgeName)
+	if err != nil {
+		return nil, err
+	}
+	for _, cidr := range bridgeNetDev.CIDRs {
+		Log.Infof("%s bridge has address %v", bridgeName, cidr)
+		addrs = append(addrs, ipam.PreClaim{Ident: "weave:expose", Cidr: a(cidr)})
+	}
+
+	add := func(cid string, isContainer bool, netDevs []weavenet.Dev) {
+		for _, netDev := range netDevs {
+			for _, cidr := range netDev.CIDRs {
+				Log.Infof("Found address %v for ID %s", cidr, cid)
+				addrs = append(addrs, ipam.PreClaim{Ident: cid, IsContainer: isContainer, Cidr: a(cidr)})
+			}
+		}
+	}
+
+	// Then find all veths connected to the bridge
+	peerIDs, err := weavenet.ConnectedToBridgeVethPeerIds(bridgeName)
+	if err != nil {
+		return nil, err
+	}
+
+	// Now iterate over all containers to see if they have a network
+	// namespace with an attached interface
+	if dockerCli != nil {
+		for _, cid := range containerIDs {
+			container, err := dockerCli.InspectContainer(cid)
+			if err != nil {
+				if _, ok := err.(*docker.NoSuchContainer); ok {
+					continue
+				}
+				return nil, err
+			}
+			if container.State.Pid != 0 {
+				netDevs, err := weavenet.GetNetDevsByVethPeerIds(container.State.Pid, peerIDs)
+				if err != nil {
+					return nil, err
+				}
+				add(cid, true, netDevs)
+			}
+		}
+	} else {
+		// If we don't have a Docker connection, iterate over all processes
+		pids, err := common.AllPids("/proc")
+		if err != nil {
+			return nil, err
+		}
+		for _, pid := range pids {
+			netDevs, err := weavenet.GetNetDevsByVethPeerIds(pid, peerIDs)
+			if err != nil {
+				return nil, err
+			}
+			add(api.NoContainerID, false, netDevs)
+		}
+	}
+	return addrs, nil
+}

router/fastdp.go

@@ -577,8 +577,8 @@ type fastDatapathForwarder struct {
 	confirmed         bool
 	remoteAddr        *net.UDPAddr
 	heartbeatInterval time.Duration
-	heartbeatTimer    *time.Timer
-	heartbeatTimeout  *time.Timer
+	heartbeatTimer    *time.Timer // for sending
+	heartbeatTimeout  *time.Timer // for receiving
 	ackedHeartbeat    bool
 	stopChan          chan struct{}
 	stopped           bool
@@ -856,7 +856,9 @@ func (fwd *fastDatapathForwarder) handleCryptoInitSARemote(msg []byte) {
 		return
 	}
 
-	if !fwd.isOutboundIPSecEstablished {
+	// FastDatapathCryptoInitSARemote can be received before Confirm'ing
+	// connection, thus before InitSALocal.
+	if fwd.confirmed && !fwd.isOutboundIPSecEstablished {
 		fwd.isOutboundIPSecEstablished = true
 		fwd.heartbeatTimer.Reset(0)
 	}

site/faq.md

@@ -1,10 +1,10 @@
 ---
 title: FAQ
-menu_order: 100
+menu_order: 200
 ---
 
 
-
+<a name="container-ip"></a>
 **Q: How do I obtain the IP of a specific container when I'm using Weave?**
 
 You can use `weave ps <container>` to see the allocated address of a container on a Weave network.  
@@ -12,6 +12,7 @@ You can use `weave ps <container>` to see the allocated address of a container o
 See [Troubleshooting Weave - List attached containers](/site/troubleshooting.md#list-attached-containers).
 
 
+<a name="specific-ip"></a>
 **Q: My dockerized app needs to check the request of an application that uses a static IP. Is it possible to manually change the IP of a container?**
 
 
@@ -20,11 +21,13 @@ You can manually change the IP of a container using [Classless Inter-Domain Rout
 For more information, refer to [Manually Specifying the IP Address of a Container](/site/using-weave/manual-ip-address.md). 
 
 
+<a name="expose-container"></a>
 **Q: How do I expose one of my containers to the outside world?**
 
 Exposing a container to the outside world is described in [Exporting Services](/site/using-weave/service-management.md#exporting).
 
 
+<a name="legacy-network"></a>
 **Q: Can I connect my existing 'legacy' network with a Weave container network?**
 
 Yes you can. 
@@ -36,19 +39,15 @@ A simple way to accomplish this would be to run Weave on the host and then run,
 Yet another option is to expose a port from the container on host B and then connect to it. You can read about exposing ports in [Exporting Services](/site/using-weave/service-management.md#exporting).
 
 
+<a name="duplicate-ip"></a>
 **Q: Why am I seeing the same IP address assigned to two different containers on different hosts?**
 
-Under normal circumstances, this should never happen, but it can occur if  `weave forget` and `weave rmpeer` was run on more than one host. 
+Under normal circumstances, this should never happen, but it can occur if `weave rmpeer` was run on more than one host. 
 
-You cannot call `weave rmpeer` on more than one host. The address space, which was owned by the stale peer cannot be left dangling, and as a result it gets reassigned. In this instance, the address is reassigned to the peer on which `weave rmpeer` was run. Therefore, if you run `weave forget` and then `weave rmpeer` on more than one host at a time, it results in duplicate IPs on more than one host.
-
-Once the peers detect the inconsistency, they log the error and drop the connection that supplied the inconsistent data. The rest of the peers will carry on with their view of the world, but the network will not function correctly.
-
-Some peers may be able to communicate their claim to the others before they run `rmpeer` (i.e. it's a race), so what you can expect is a few cliques of peers that are still talking to each other, but repeatedly dropping attempted connections with peers in other cliques.
-
-For more information on see [Allocating IP Addresses](/site/ipam.md) and also, [Starting, Stopping and Removing Peers](/site/ipam/stop-remove-peers-ipam.md).
+For more information see [Starting, Stopping and Removing Peers](/site/ipam/stop-remove-peers-ipam.md).
 
 
+<a name="dead-node"></a>
 **Q: What is the best practice for resetting a node that goes out of service?**
 
 When a node goes out of service, the best option is to call `weave rmpeer` on one host and then `weave forget` on all the other hosts.
@@ -56,6 +55,7 @@ When a node goes out of service, the best option is to call `weave rmpeer` on on
 See [Starting, Stopping and Removing Peers](/site/ipam/stop-remove-peers-ipam.md) for an in-depth discussion.
 
 
+<a name="performance"></a>
 **Q: What about Weave's performance? Are software defined network overlays just as fast as native networking?**
 
 All virtualization techniques have some overhead, and Weave's overhead is typically around 2-3%. Unless your system is completely bottlenecked on the network, you won't notice this during normal operation. 
@@ -65,6 +65,7 @@ Weave Net also automatically uses the fastest datapath between two hosts. When W
 For more information about fast datapath see [How Fast Datapath Works](/site/how-it-works/fastdp-how-it-works.md).
 
 
+<a name="query-fastdp"></a>
 **Q: How can I tell if Weave is using fast datapath (fastdp) or not?**
 
 To view whether Weave is using fastdp or not, you can run, `weave status connections`
@@ -72,24 +73,36 @@ To view whether Weave is using fastdp or not, you can run, `weave status connect
 For more information on this command, see [Using Fast Datapath](/site/using-weave/fastdp.md).
 
 
+<a name="encrypted-fastdp"></a>
 **Q: Does encryption work with fastdp?**
 
-Encryption does not work with fast datapath. If you enable encryption using the `--password` option to launch Weave (or you use the `WEAVE_PASSWORD` environment variable), fast datapath will by default be disabled. 
-
-You can however have a mixture of fast datapath connections over trusted links, as well as, encrypted connections over untrusted links.
+Yes, 1.9 version of Weave Net added the encryption feature to fastdp.
 
 See [Using Fast Datapath](/site/using-weave/fastdp.md) for more information.
 
+<a name="app-isolation"></a>
 **Q: Can I create multiple networks where containers can communicate on one network, but are isolated from containers on other networks?**
 
 Yes, of course!  Weave allows you to run isolated networks and still allow open communications between individual containers from those isolated networks. You can find information on how to do this in [Application Isolation](/site/using-weave/application-isolation.md).
 
 
-**Q: Which ports does Weave Net use (e.g. if I am configuring a firewall) ?**
+**<a name=ports></a>Q: Which ports does Weave Net use (e.g. if I am configuring a firewall) ?**
 
 You must permit traffic to flow through TCP 6783 and UDP 6783/6784,
 which are Weave’s control and data ports.
 
+The daemon also uses TCP port 6782 for [metrics](/site/metrics.md), but
+you would only need to open up this port if you wish to collect metrics
+from another host.
+
+The Weave Net daemon listens on localhost (127.0.0.1) TCP port 6784
+for commands from other Weave Net components. This port should not be
+opened to other hosts.
+
+When using encrypted fast datapath, make sure that underlying
+network does not block ESP traffic (IP protocol 50). For instance
+on Google Cloud Platform a firewall rule for allowing ESP traffic has
+to be installed.
 
 **<a name=own-image></a>Q: Why do you use your own Docker image `weaveworks/ubuntu`?**
 

site/features.md

@@ -52,10 +52,6 @@ Weave Net automatically chooses the fastest available method to
 transport data between peers. The best performing of these 
 (the 'fast datapath') offers near-native throughput and latency.
 
-Fast datapath does not support encryption. For full details on configuring
-Weave when you have connections that traverse untrusted networks,
-see [Securing Connections Across Untrusted Networks](/site/using-weave/security-untrusted-networks.md) for more details.
-
 See [Using Fast Datapath](/site/using-weave/fastdp.md) and
 [How Fast Datapath Works](/site/how-it-works/fastdp-how-it-works.md).
 
@@ -177,7 +173,8 @@ mechanism which you can use in conjunction with or as an
 alternative to any other security technologies you have 
 running alongside Weave. 
 
-Weave Net implements encryption and security using the Go version of [Daniel J. Bernstein's NaCl library](http://nacl.cr.yp.to/index.html).
+Weave Net implements encryption and security using the Go version of [Daniel J.  Bernstein's NaCl library](http://nacl.cr.yp.to/index.html),
+and, additionally in the case of encrypted fast datapath using [the cryptography framework of the Linux kernel](https://en.wikipedia.org/wiki/Crypto_API_(Linux)).
 
 For information on how to secure your Docker network connections, see [Securing Connections Across Untrusted Networks](/site/using-weave/security-untrusted-networks.md) and for a more technical discussion on how Weave implements encryption see, [Weave Encryption](/site/how-it-works/encryption.md) and [How Weave Implements Encryption](/site/how-it-works/encryption-implementation.md).
 

site/how-it-works/encryption-implementation.md

@@ -107,6 +107,8 @@ attacks.
 
 ####<a name="udp"></a>Encrypting and Decrypting UDP Packets
 
+#####Sleeve
+
 UDP connections carry captured traffic between peers. For a UDP packet
 sent between peers that are using crypto, the encapsulation looks as
 follows:
@@ -179,7 +181,33 @@ contained in the set. The window spans at least 2^20 message sequence
 numbers, and hence any re-ordering between the most recent ~1 million
 messages is handled without dropping messages.
 
+#####Fast Datapath
+
+Encryption in fastdp uses [the ESP protocol of IPsec](https://tools.ietf.org/html/rfc2406)
+in the transport mode. Each VXLAN packet is encrypted with
+[AES in GCM mode](https://tools.ietf.org/html/rfc4106aesgcm), with 32 byte key and
+4 byte salt. This combo provides the following security properties:
+
+* Data confidentiality.
+* Data origin authentication.
+* Integrity.
+* Anti-replay.
+* Limited traffic flow confidentiality as VXLAN packets are fully encrypted.
+
+For each connection direction, a different AES-GCM key and salt is used.
+The pairs are derived with [HKDF](https://tools.ietf.org/html/rfc5869)
+to which we pass a randomly generated 32 byte salt transferred over the encrypted
+control plane channel between peers.
+
+To prevent from replay attacks, which are possible because of the size of
+sequence number field in ESP (4 bytes), we use extended sequence numbers
+implemented by [ESN](https://tools.ietf.org/html/rfc4304).
+
+Authentication of ESP packet integrity and origin is ensured by 16 byte
+Integrity Check Value of AES-GCM.
+
 **See Also**
 
  * [architecture documentation](https://github.com/weaveworks/weave/blob/master/docs/architecture.txt)
+ * [fastdp encryption](https://github.com/weaveworks/weave/blob/master/docs/fastdp-crypto.md)
  * [Securing Containers Across Untrusted Networks](/site/using-weave/security-untrusted-networks.md)

site/how-it-works/encryption.md

@@ -9,7 +9,8 @@ Weave Net peers
 communication
 [can be encrypted](/site/using-weave/security-untrusted-networks.md).
 
-Encryption is accomplished using the [NaCl](http://nacl.cr.yp.to/)
+Encryption of control plane traffic (TCP) and data plane traffic (UDP) of sleeve
+overlay is accomplished using the [NaCl](http://nacl.cr.yp.to/)
 crypto libraries, employing Curve25519, XSalsa20 and Poly1305 to
 encrypt and authenticate messages. Weave Net protects against
 injection and replay attacks for traffic forwarded between peers.
@@ -32,6 +33,11 @@ extensions to TLS such as [DTLS](https://tools.ietf.org/html/rfc4347)
 which can operate over UDP, these are not widely implemented and
 deployed.
 
+In the case of fast datapath, data plane traffic is encrypted by using
+[ESP of IPsec](https://tools.ietf.org/html/rfc2406).
+The process of encryption is handled by the Linux kernel and is controlled via
+the IP transformation framework (XFRM).
+
 **See Also**
 
  * [How Weave Implements Encryption](/site/how-it-works/encryption-implementation.md)

site/installing-weave.md

@@ -58,6 +58,9 @@ A [special no-overlay mode for EC2](/site/using-weave/awsvpc.md) can
 optionally be enabled, which allows containers to communicate at the
 full speed of the underlying network.
 
+To make encryption in fast datapath work on Google Cloud Platform, see
+[here](/site/faq.md#ports).
+
 **See Also** 
 
  * [Using Weave Net](/site/using-weave.md)

site/introducing-weave.md

@@ -42,7 +42,7 @@ See [How Fast Datapath Works](/site/using-weave/fastdp.md) for more information.
 
 Weave uses industry-standard VXLAN encapsulation between hosts. This means you can continue using your favorite packet analyzing tools, such as ‘Wireshark’ to inspect and troubleshoot protocols.
 
-###Weave Net is Secure With Built-in Encryption
+###Weave Net is Secure
 
 Weave Net traverses firewalls without requiring a TCP add-on. You can encrypt your traffic, which allows you to connect to apps on hosts even across an untrusted network.  
 

site/ipam/stop-remove-peers-ipam.md

@@ -32,8 +32,26 @@ caught fire, you can go to one of the other hosts and run:
 Weave Net takes all the IP address ranges owned by host3 and transfers
 them to be owned by host1. The name "host3" is resolved via the
 'nickname' feature of Weave Net, which defaults to the local host
-name. Alternatively, you can supply a peer name as shown in `weave
-status`.
+name. Alternatively, you can supply a peer name as shown in `weave status`.
+
+###<a name="caution-rmpeer"></a>Caution###
+
+You cannot call `weave rmpeer` on more than one host. The address
+space, which was owned by the stale peer cannot be left dangling, and
+as a result it gets reassigned. In this instance, the address is
+reassigned to the peer on which `weave rmpeer` was run. Therefore, if
+you run `weave forget` and then `weave rmpeer` on more than one host
+at a time, it results in duplicate IPs on more than one host.
+
+Once the peers detect the inconsistency, they log the error and drop
+the connection that supplied the inconsistent data. The rest of the
+peers will carry on with their view of the world, but the network will
+not function correctly.
+
+Some peers may be able to communicate their claim to the others before
+they run `rmpeer` (i.e. it's a race), so what you can expect is a few
+cliques of peers that are still talking to each other, but repeatedly
+dropping attempted connections with peers in other cliques.
 
 **See Also**
 

site/kube-addon.md

@@ -151,7 +151,7 @@ The list of variables you can set is:
 * WEAVE\_EXPOSE\_IP - set the IP address used as a gateway from the
   Weave network to the host network - this is useful if you are
   configuring the addon as a static pod.
-* WEAVE\_MTU - Weave Net defaults to 1410 bytes, but you can set a
+* WEAVE\_MTU - Weave Net defaults to 1376 bytes, but you can set a
   smaller size if your underlying network has a tighter limit, or set
   a larger size for better performance if your network supports jumbo
   frames - see [here](/site/using-weave/fastdp.md#mtu) for more

site/metrics.md

@@ -19,6 +19,8 @@ exposed:
 * `weave_max_ips` - Size of IP address space used by allocator.
 * `weave_dns_entries` - Number of DNS entries.
 * `weave_flows` - Number of FastDP flows.
+* `weave_ipam_pending_allocates` - Number of pending allocates.
+* `weave_ipam_pending_claims` - Number of pending claims.
 
 #### Publish Router Metrics Endpoint
 

site/operational-guide/tasks.md

@@ -7,7 +7,7 @@ The following administrative tasks are discussed:
 
 * [Configuring Weave Net to Start Automatically on Boot](#start-on-boot)
 * [Detecting and Reclaiming Lost IP Address Space](#detect-reclaim-ipam)
-    *[Manually Reclaiming Lost Address Space](#reclaim-address-space)
+* [Manually Reclaiming Lost Address Space](#manually-reclaim-address-space)
 * [Upgrading a Cluster](#cluster-upgrade)
 * [Resetting Persisted Data](#reset)
 

site/using-weave/application-isolation.md

@@ -32,7 +32,7 @@ Next, launch the two netcat containers onto the default subnet:
 And then to test the isolation, launch a few more containers onto a different subnet:
 
     host1$ docker run -e WEAVE_CIDR=net:10.2.2.0/24 --name b1 -ti weaveworks/ubuntu
-    host2$ docker run -e WEAVE_CIDR=net:10.2.2.0.24 --name b2 -ti weaveworks/ubuntu
+    host2$ docker run -e WEAVE_CIDR=net:10.2.2.0/24 --name b2 -ti weaveworks/ubuntu
 
 Ping each container to confirm that they can talk to each other, but not to the containers of our first subnet:
 

site/using-weave/awsvpc.md

@@ -89,7 +89,7 @@ your cluster.
 
 The Maximum Transmission Unit, or MTU, is the technical term for the
 limit on how big a single packet can be on the network. Weave Net
-defaults to 1410 bytes. This default works across almost all networks, but for better 
+defaults to 1376 bytes. This default works across almost all networks, but for better 
 performance you can set it to a larger MTU size. 
 
 The AWS network supports packets of up to 9000 bytes.  

site/using-weave/fastdp.md

@@ -18,9 +18,15 @@ You can disable fastdp by enabling the `WEAVE_NO_FASTDP` environment variable at
 
 ###Fast Datapath and Encryption
 
-Encryption does not work with fast datapath. If you enable encryption using the `--password` option to launch weave (or you use the `WEAVE_PASSWORD` environment variable), fast datapath will by default be disabled. 
+Fast datapath implements encryption using IPsec which is configured with IP
+transformation framework (XFRM) provided by the Linux kernel.
 
-When encryption is not in use there may be other conditions in which the fast datapath reverts to `sleeve mode`. Once these conditions pass, Weave Net reverts back to using fastdp. To view which mode Weave Net is using, run `weave status connections`.
+Each encrypted dataplane packet is encapsulated into [ESP](https://tools.ietf.org/html/rfc2406),
+thus in some networks a firewall rule for allowing ESP traffic needs to be installed. E.g. Google
+Cloud Platform denies ESP packets by default.
+
+See [How Weave Implements Encryption](/site/how-it-works/encryption-implementation.md)
+for more details for the fastdp encryption.
 
 ###Viewing Connection Mode Fastdp or Sleeve
 
@@ -46,12 +52,13 @@ Where fastdp indicates that fast datapath is being used on a connection. If fast
 
 The Maximum Transmission Unit, or MTU, is the technical term for the
 limit on how big a single packet can be on the network. Weave Net
-defaults to 1410 bytes, but you can set a smaller size if your
+defaults to 1376 bytes, but you can set a smaller size if your
 underlying network has a tighter limit, or set a larger size for
 better performance.
 
 The underlying network must be able to deliver packets of the size
-specified plus overheads of around 50 bytes, or else Weave Net will
+specified plus overheads of around 84-87 bytes (the final MTU should be
+divisible by four), or else Weave Net will
 fall back to Sleeve for that connection.  This requirement applies
 to _every path_ between peers. 
 
@@ -59,7 +66,7 @@ To specify a different MTU, before launching Weave Net set the
 environment variable `WEAVE_MTU`.  For example, for a typical "jumbo
 frame" configuration:
 
-    $ WEAVE_MTU=8950 weave launch host2 host3
+    $ WEAVE_MTU=8916 weave launch host2 host3
 
 **See Also**
 

site/using-weave/security-untrusted-networks.md

@@ -41,8 +41,8 @@ the `--trusted-subnets` argument with `weave launch`:
 
 If *both* peers at the end of a connection consider the other to be in
 a trusted subnet, Weave Net attempts to establish fast datapath
-connectivity, which is unencrypted. Otherwise the slower `sleeve` mode
-is used and communication is encrypted.
+connectivity, which is unencrypted. Otherwise communication is encrypted which
+imposes overheads.
 
 Configured trusted subnets are shown in [`weave status`](/site/troubleshooting.md#weave-status).
 

test/110_encryption_2_test.sh

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-. "$(dirname "$0")/config.sh"
-
-C1=10.2.1.4
-C2=10.2.1.7
-
-start_suite "Ping over encrypted cross-host weave network"
-
-weave_on $HOST1 launch --password wfvAwt7sj
-weave_on $HOST2 launch --password wfvAwt7sj $HOST1
-
-start_container $HOST1 $C1/24 --name=c1
-start_container $HOST2 $C2/24 --name=c2
-assert_raises "exec_on $HOST1 c1 $PING $C2"
-
-assert_raises "weave_on $HOST1 status connections | grep encrypted"
-assert_raises "weave_on $HOST2 status connections | grep encrypted"
-
-end_suite

test/110_sleeve_encryption_2_test.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-. ./config.sh
+. "$(dirname "$0")/config.sh"
 
 C1=10.2.1.4
 C2=10.2.1.7

test/111_fastdp_encryption_2_test.sh

@@ -1,30 +1,30 @@
 #!/bin/bash
 
-# TODO(mp) 1. test anti-replay 2. test sleeve <-> fastdp
-# TODO(mp) test with --trusted-subnets
-
-. ./config.sh
+. "$(dirname "$0")/config.sh"
 
 HOST1_IP=$($SSH $HOST1 getent ahosts $HOST1 | grep "RAW" | cut -d' ' -f1)
 HOST2_IP=$($SSH $HOST2 getent ahosts $HOST2 | grep "RAW" | cut -d' ' -f1)
 
+# Assume symmetry
+IFACE=$($SSH $HOST1 ip -o route get $HOST2_IP | grep -o 'dev [^ ]*' | cut -d' ' -f2)
+
 start_tcpdump() {
     HOST=$1
     PCAP=$2
-    run_on $HOST "sudo nohup tcpdump -i any -w $PCAP >/dev/null 2>&1 & echo \$! > $PCAP.pid"
+    run_on $HOST "sudo nohup tcpdump -i $IFACE -U -w $PCAP >/dev/null 2>&1 & echo \$! > $PCAP.pid"
 }
 
 stop_tcpdump() {
     HOST=$1
     PCAP=$2
-    $SSH $HOST "sudo kill \$(cat $PCAP.pid) && rm -f $PCAP.pid && sudo base64 $PCAP" \
+    $SSH $HOST "sudo kill -sSIGINT \$(cat $PCAP.pid) && rm -f $PCAP.pid && sudo base64 $PCAP" \
         | base64 -d > $PCAP
 }
 
-# Checks whether all vxlan-tunneled traffic is encrypted.
+# Checks whether all VXLAN-tunneled traffic is encrypted.
 assert_pcap() {
     PCAP=$1
-    # UDP port is not exposed in ESP
+    # VXLAN (UDP dst port 6784) packets should be encrypted
     assert "tcpdump -r $PCAP 'src $HOST1_IP && dst $HOST2_IP && dst port 6784'" ""
     assert "tcpdump -r $PCAP 'src $HOST2_IP && dst $HOST1_IP && dst port 6784'" ""
     # ip_proto(ESP) = 50
@@ -55,6 +55,9 @@ start_container $HOST1 $C1/24 --name=c1
 start_container $HOST2 $C2/24 --name=c2
 assert_raises "exec_on $HOST1 c1 $PING $C2"
 
+# Let tcpdump to flush captures...
+sleep 1
+
 stop_tcpdump $HOST1 $PCAP1
 stop_tcpdump $HOST2 $PCAP2
 
@@ -66,6 +69,7 @@ weave_on $HOST2 reset
 # the connection.
 assert "$SSH $HOST1 sudo ip xfrm state" ""
 assert "$SSH $HOST1 sudo ip xfrm policy" ""
-assert "$SSH $HOST1 sudo iptables -t mangle -S WEAVE-IPSEC | grep '\-A'" ""
+assert "$SSH $HOST1 sudo iptables -t mangle -S WEAVE-IPSEC-IN | grep '\-A'" ""
+assert "$SSH $HOST1 sudo iptables -t mangle -S WEAVE-IPSEC-OUT | grep '\-A'" ""
 
 end_suite

test/245_dns_case_insensitive_test.sh

@@ -6,6 +6,9 @@ C1=10.2.0.78
 C2=10.2.0.79
 C3=10.2.0.80
 
+REVERSE_C1_LOWER=78.0.2.10.in-addr.arpa
+REVERSE_C1_UPPER=78.0.2.10.IN-ADDR.ARPA
+
 start_suite "DNS lookup case (in)sensitivity"
 
 weave_on $HOST1 launch
@@ -17,6 +20,10 @@ assert_dns_record $HOST1 test seeone.weave.local $C1
 assert_dns_record $HOST1 test SeeOne.weave.local $C1
 assert_dns_record $HOST1 test SEEONE.weave.local $C1
 
+# Test reverse DNS using explicit in-addr.arpa format, lower and upper case
+assert "exec_on $HOST1 test dig +short -t PTR $REVERSE_C1_LOWER" seeone.weave.local.
+assert "exec_on $HOST1 test dig +short -t PTR $REVERSE_C1_UPPER" seeone.weave.local.
+
 start_container $HOST1 $C2/24 --name=SeEtWo
 assert_dns_record $HOST1 test seetwo.weave.local $C2
 assert_dns_record $HOST1 test SeeTwo.weave.local $C2

test/320_claim_3_test.sh

@@ -49,6 +49,7 @@ delete_persistence $HOST1 $HOST2
 # Now make host1 attempt to claim from host2, when host2 is stopped
 # the point being to check whether host1 will hang trying to talk to host2
 weave_on $HOST2 launch-router --ipalloc-range $UNIVERSE
+weave_on $HOST2 prime
 # Introduce host3 to remember the IPAM CRDT when we stop host2
 weave_on $HOST3 launch-router --ipalloc-range $UNIVERSE $HOST2
 weave_on $HOST3 prime

test/840_weave_kube_3_test.sh

@@ -31,7 +31,7 @@ sed -e "s%imagePullPolicy: Always%imagePullPolicy: Never$COVERAGE_ARGS%" "$(dirn
 sleep 5
 
 wait_for_connections() {
-    for i in $(seq 1 30); do
+    for i in $(seq 1 45); do
         if run_on $HOST1 "curl -sS http://127.0.0.1:6784/status | grep \"$SUCCESS\"" ; then
             return
         fi
@@ -56,7 +56,7 @@ assert_raises "run_on $HOST3 $PING $HOST2EXPIP"
 run_on $HOST1 "kubectl run hello --image=weaveworks/hello-world --replicas=3"
 
 wait_for_pods() {
-    for i in $(seq 1 30); do
+    for i in $(seq 1 45); do
         if run_on $HOST1 "kubectl get pods | grep 'hello.*Running'" ; then
             return
         fi

test/README.md

@@ -2,40 +2,67 @@ This directory contains integration tests for weave.
 
 ## Requirements
 
-You need two VMs with docker >=1.6.0 installed and listening on TCP
-port 2375 (see below). You also need to be able to ssh to these VMs,
-preferably without having to input anything.
+You need three VMs with `docker` (`>=1.6.0`) installed and listening on TCP
+port `2375` (see below). You also need to be able to SSH to these VMs without 
+having to input anything.
 
-The `Vagrantfile` in this directory constructs two such VMs.
+The `Vagrantfile` in this directory constructs three such VMs.
 
-To create the VMs, open a shell and in this directory and type
+If you are [building Weave Net using Vagrant](https://www.weave.works/docs/net/latest/building/),
+it is recommended to run the tests from the build VM and not the host.
 
-    vagrant up
-
-To meet the aforementioned ssh requirement you may want to
-
-    cp ~/.vagrant.d/insecure_private_key .
 
 ## Running tests
 
-If you are [building weave using Vagrant](https://www.weave.works/docs/net/latest/building/),
-it is recommended to run the tests from the build VM and not the host.
+**TL;DR**: You can run the steps 2. to 7. below with one command: 
 
-    ./setup.sh
+    make PROVIDER=vagrant integration-tests
 
-uploads the weave images from where the Makefile puts them
-(`/var/tmp`) to the two docker hosts, and copies the weave script
-over.
+**Detailed steps**:
 
-Then you can use, e.g.,
+  1. Start the build virtual machine (see above article for more details):
 
-    ./200_dns_test.sh
+        vagrant up
 
-to run an individual test, or
+  2. Start the three testing VMs:
 
-    ./run_all.sh
+        cd test
+        vagrant up
+
+  3. SSH into the build VM and go to Weave Net's sources:
+
+        cd ..
+        vagrant ssh
+        # you are now on the build VM:
+        cd ~/weave 
+
+  4. Compile all code and dependencies:
+
+        make
+        make testrunner
+        cd test
+
+  5. Upload the weave images from where the `Makefile` puts them (`weave.tar.gz`) to 
+     the three docker hosts, `docker load` these, and copies the `weave` script over:
+
+        ./setup.sh
+
+  6. Run individual tests, e.g.:
+
+        ./200_dns_test.sh
+
+     or run all tests (everything named `*_test.sh`):
+
+        ./run_all.sh
+
+  7. Stop all VMs:
+
+        exit
+        # you are now on your host machine
+        vagrant destroy -f
+        cd test
+        vagrant destroy -f
 
-to run everything named `*_test.sh`.
 
 ## Using other VMs
 
@@ -60,20 +87,7 @@ to the file `/etc/default/docker`, then restart docker.
 
 ## Updating the GCE test image
 
-When a new version of Docker is released, you'll need to update the GCE test image.
-
-To do this, pick a fresh ```TEMPLATE_NAME``` and update any commands in
-```function make_template``` in gce.sh, then run:
-
-```
-./gce.sh make_template
-```
-
-For this you'll need the GCE credentials, which can be found in ```bin/setup-circleci-secrets```,
-which you'll need to decrypt and run (its echos the secrets into know locations):
-
-```
-./bin/setup-circleci-secrets "$SECRET_PASSWORD"
-```
-
-If you don't know the password, ask tom@weave.works.
+When a new version of Docker is released, you willneed to update the GCE test image.
+To do this, change the Docker version in `run-integration-tests.sh` and push the change.
+Next build in CircleCI will detect that there is no template for this version of Docker and will first create the template before running tests.
+Subsequent builds will then simply re-use the template.

test/Vagrantfile

@@ -1,59 +1,65 @@
-VAGRANTFILE_API_VERSION = "2"
-
 # Necessary because the switch disabling unwanted insecure key
 # replacement in Vagrant 1.7+ is not accepted by earlier versions
 Vagrant.require_version ">= 1.7.0"
 
 # these ought to match what is in config.sh
-n_machines = 3
-ip_prefix = "192.168.48"
-ip_suffix_base = 10
+N_MACHINES = 3
+IP_PREFIX = '192.168.48'
+IP_SUFFIX_BASE = 10
 
-require '../vagrant-common.rb'
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'vagrant-common.rb'))
 
-def configure_docker(host, hostname, ip)
-  pkgs = %w(docker-engine ethtool)
-
-  host.vm.box = "ubuntu/wily64"
-
-  host.vm.provision :shell, :inline => "hostnamectl set-hostname "+hostname
-  host.vm.network "private_network", ip: ip
-
-  host.vm.synced_folder ".", "/vagrant", disabled: true
-
-  host.vm.provision :shell, :inline => "sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D"
-  host.vm.provision :shell, :inline => "echo deb https://apt.dockerproject.org/repo ubuntu-wily main > /etc/apt/sources.list.d/docker.list"
-
-  install_packages host.vm, pkgs
-  tweak_docker_daemon host.vm
-  cleanup host.vm
+def ip_address(i)
+  "#{IP_PREFIX}.#{IP_SUFFIX_BASE + i}"
 end
 
-def configure_resolv_conf(host)
-  # Fix the resolution errors by using 8.8.8.8
-  host.vm.provision :shell, :inline => "echo 'nameserver 8.8.8.8' > /etc/resolv.conf"
+def hostname(i)
+  "host#{i}"
 end
 
+ETC_HOSTS_ENTRIES = (1..N_MACHINES).map { |i| "#{ip_address(i)} #{hostname(i)}\n"}.join('')
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+  (1..N_MACHINES).each do |i|
+    hostname = hostname(i)
+    config.vm.define hostname do |host|
+      host.vm.provider :virtualbox do |vb|
+        vb.name = hostname
+        vb.memory = 384
+        configure_nat_dns(vb)
+      end
 
-  config.vm.provider :virtualbox do |vb|
-    vb.customize ["modifyvm", :id, "--natdnshostresolver1", "off"]
-    vb.customize ["modifyvm", :id, "--natdnsproxy1", "off"]
-    vb.memory = 384
-  end
+      # Disable Vagrant 1.7 per host insecure key replacement
+      host.ssh.insert_key = false
 
-  # Disable Vagrant 1.7 per host insecure key replacement
-  config.ssh.insert_key = false
+      host.vm.box = VAGRANT_IMAGE
+      host.vm.network "private_network", ip: ip_address(i)
+      host.vm.hostname = hostname
 
-  (1..n_machines).each do |i|
-    config.vm.define "host#{i}" do |host|
-      ip_suffix = ip_suffix_base + i
-      configure_resolv_conf(host)
-      configure_docker(host, "host#{i}", "#{ip_prefix}.#{ip_suffix}")
+      host.vm.provision :shell, :inline => <<END
+# Tests rely on hostnames, so we remove potentially incorrect entries (e.g. localhost), and add mappings of IP addresses to hostnames:
+sed -i '/#{hostname}/d' /etc/hosts
+echo "#{ETC_HOSTS_ENTRIES}" >> /etc/hosts
+
+# Fix the resolution errors by using 8.8.8.8
+echo 'nameserver 8.8.8.8' > /etc/resolv.conf
+END
+
+      # Disable default Vagrant shared folder, which we don't need:
+      host.vm.synced_folder ".", "/vagrant", disabled: true
+
+      cleanup host.vm
+
+      # Only execute once the Ansible provisioner, when all the machines are up and ready:
+      if i == N_MACHINES
+        host.vm.provision 'ansible' do |ansible|
+          ansible.limit = 'all'  # Disable default limit to connect to all the machines
+          ansible.playbook = '../tools/config_management/setup_weave-net_test.yml'
+          ansible.extra_vars = ansibleize(get_dependencies_version_from_file_and_env())
+        end
+      end
     end
   end
-
 end
 
 begin

test/config.sh

@@ -30,9 +30,9 @@ HOST3=$(echo $HOSTS | cut -f 3 -d ' ')
 
 . "$DIR/assert.sh"
 
-
 SSH_DIR=${SSH_DIR:-$DIR}
-SSH=${SSH:-ssh -l vagrant -i "$SSH_DIR/insecure_private_key" -o "UserKnownHostsFile=$SSH_DIR/.ssh_known_hosts" -o CheckHostIP=no -o StrictHostKeyChecking=no}
+SSH_OPTS=${SSH_OPTS:-"-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no -o IdentitiesOnly=yes -o LogLevel=ERROR"}
+SSH=${SSH:-ssh -l vagrant -i "$SSH_DIR/insecure_private_key" $SSH_OPTS}
 
 SMALL_IMAGE="alpine"
 DNS_IMAGE="aanand/docker-dnsutils"

test/gce.sh

@@ -1,207 +0,0 @@
-#!/bin/bash
-# This script has a bunch of GCE-related functions:
-# ./gce.sh setup -  starts two VMs on GCE and configures them to run our integration tests
-# . ./gce.sh; ./run_all.sh - set a bunch of environment variables for the tests
-# ./gce.sh destroy - tear down the VMs
-# ./gce.sh make_template - make a fresh VM template; update TEMPLATE_NAME first!
-
-set -e
-
-: ${KEY_FILE:=/tmp/gce_private_key.json}
-: ${SSH_KEY_FILE:=$HOME/.ssh/gce_ssh_key}
-: ${PROJECT:=positive-cocoa-90213}
-: ${IMAGE_FAMILY:=ubuntu-1604-lts}
-: ${IMAGE_PROJECT:=ubuntu-os-cloud}
-: ${TEMPLATE_NAME:=test-template-12}
-: ${ZONE:=us-central1-a}
-: ${NUM_HOSTS:=5}
-SUFFIX=""
-if [ -n "$CIRCLECI" ]; then
-	SUFFIX="-${CIRCLE_BUILD_NUM}-$CIRCLE_NODE_INDEX"
-fi
-
-# Setup authentication
-gcloud auth activate-service-account --key-file $KEY_FILE 1>/dev/null
-gcloud config set project $PROJECT
-
-function vm_names {
-	local names=
-	for i in $(seq 1 $NUM_HOSTS); do
-		names="host$i$SUFFIX $names"
-	done
-	echo "$names"
-}
-
-# Delete all vms in this account
-function destroy {
-	if [ $(gcloud compute firewall-rules list test-allow-docker$SUFFIX 2>/dev/null | wc -l) -gt 0 ] ; then
-		gcloud compute firewall-rules delete test-allow-docker$SUFFIX
-	fi
-	names="$(vm_names)"
-	if [ $(gcloud compute instances list --zones $ZONE -q $names | wc -l) -le 1 ] ; then
-		return 0
-	fi
-	for i in {0..10}; do
-		# gcloud instances delete can sometimes hang.
-		case  $(set +e; timeout 60s /bin/bash -c "gcloud compute instances delete --zone $ZONE -q $names  >/dev/null 2>&1"; echo $?) in
-			0)
-				return 0
-				;;
-			124)
-				# 124 means it timed out
-				break
-				;;
-			*)
-				return 1
-		esac
-	done
-}
-
-function internal_ip {
-	jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].networkIP" $1
-}
-
-function external_ip {
-	jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].accessConfigs[0].natIP" $1
-}
-
-function try_connect {
-	for i in {0..10}; do
-		ssh -t $1 true && return
-		sleep 2
-	done
-}
-
-function install_docker_on {
-	name=$1
-	ssh -t $name sudo bash -x -s <<EOF
-curl -sSL https://get.docker.com/gpg | sudo apt-key add -
-curl -sSL https://get.docker.com/ | sed -e s/docker-engine/docker-engine=1.11.2-0~xenial/ | sh
-apt-get update -qq;
-apt-get install -q -y --force-yes --no-install-recommends ethtool;
-apt-get install -q -y bc jq;
-usermod -a -G docker vagrant;
-mkdir -p /etc/systemd/system/docker.service.d
-cat >/etc/systemd/system/docker.service.d/override.conf  <<OVERRIDE
-[Service]
-ExecStart=
-ExecStart=/usr/bin/docker daemon -H fd:// -H unix:///var/run/alt-docker.sock -H tcp://0.0.0.0:2375 -s overlay
-OVERRIDE
-systemctl daemon-reload
-systemctl restart docker
-# This installs nsenter.
-docker run --rm -v /usr/local/bin:/target jpetazzo/nsenter
-docker pull alpine
-docker pull aanand/docker-dnsutils
-docker pull weaveworks/hello-world
-EOF
-}
-
-function install_kubernetes_on {
-	name=$1
-	ssh -t $name sudo bash -x -s <<EOF
-curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
-echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list
-apt-get update -qq
-apt-get install -q -y kubelet kubeadm kubectl kubernetes-cni
-systemctl --now disable kubelet
-# Pre-pull images required for Kubernetes
-docker pull gcr.io/google_containers/etcd-amd64:2.2.5
-docker pull gcr.io/google_containers/kube-apiserver-amd64:v1.4.0
-docker pull gcr.io/google_containers/kube-controller-manager-amd64:v1.4.0
-docker pull gcr.io/google_containers/kube-proxy-amd64:v1.4.0
-docker pull gcr.io/google_containers/kube-scheduler-amd64:v1.4.0
-docker pull gcr.io/google_containers/kube-discovery-amd64:1.0
-docker pull gcr.io/google_containers/pause-amd64:3.0
-EOF
-}
-
-function copy_hosts {
-	hostname=$1
-	hosts=$2
-	cat $hosts | ssh -t "$hostname" "sudo -- sh -c \"cat >>/etc/hosts\""
-}
-
-# Create new set of VMs
-function setup {
-	destroy
-
-	names="$(vm_names)"
-	gcloud compute instances create $names --image $TEMPLATE_NAME --zone $ZONE --tags test$SUFFIX --network=test
-	my_ip="$(curl -s http://ipinfo.io/ip)"
-	gcloud compute firewall-rules create test-allow-docker$SUFFIX --network=test --allow tcp:2375,tcp:12375 --target-tags test$SUFFIX --source-ranges $my_ip
-	gcloud compute config-ssh --ssh-key-file $SSH_KEY_FILE
-	sed -i '/UserKnownHostsFile=\/dev\/null/d' ~/.ssh/config
-
-	# build an /etc/hosts file for these vms
-	hosts=$(mktemp hosts.XXXXXXXXXX)
-	json=$(mktemp json.XXXXXXXXXX)
-	gcloud compute instances list --format=json >$json
-	for name in $names; do
-		echo "$(internal_ip $json $name) $name.$ZONE.$PROJECT" >>$hosts
-	done
-
-	for name in $names; do
-		hostname="$name.$ZONE.$PROJECT"
-
-		# Add the remote ip to the local /etc/hosts
-		sudo sed -i "/$hostname/d" /etc/hosts
-		sudo sh -c "echo \"$(external_ip $json $name) $hostname\" >>/etc/hosts"
-		try_connect $hostname
-
-		copy_hosts $hostname $hosts &
-	done
-
-	wait
-
-	rm $hosts $json
-}
-
-function make_template {
-	gcloud compute instances create $TEMPLATE_NAME --image-family=$IMAGE_FAMILY --image-project=$IMAGE_PROJECT --zone $ZONE
-	gcloud compute config-ssh --ssh-key-file $SSH_KEY_FILE
-	name="$TEMPLATE_NAME.$ZONE.$PROJECT"
-	try_connect $name
-	install_docker_on $name
-	install_kubernetes_on $name
-	gcloud -q compute instances delete $TEMPLATE_NAME --keep-disks boot --zone $ZONE
-	gcloud compute images create $TEMPLATE_NAME --source-disk $TEMPLATE_NAME --source-disk-zone $ZONE
-}
-
-function hosts {
-	hosts=
-	json=$(mktemp json.XXXXXXXXXX)
-	gcloud compute instances list --format=json >$json
-	for name in $(vm_names); do
-		hostname="$name.$ZONE.$PROJECT"
-		hosts="$hostname $hosts"
-	done
-	echo export SSH=\"ssh -l vagrant\"
-	echo export HOSTS=\"$hosts\"
-	rm $json
-}
-
-case "$1" in
-setup)
-	setup
-	;;
-
-hosts)
-	hosts
-	;;
-
-destroy)
-	destroy
-	;;
-
-make_template)
-	# see if template exists
-	if ! gcloud compute images list | grep $PROJECT | grep $TEMPLATE_NAME; then
-		make_template
-	fi
-    ;;
-
-*)
-	echo "Unknown command:" $1 >&2
-    exit 1
-esac

test/run-integration-tests.sh

@@ -0,0 +1,383 @@
+#!/bin/bash
+#
+# Description:
+#   This script runs all Weave Net's integration tests on the specified
+#   provider (default: Google Cloud Platform).
+#
+# Usage:
+#
+#   Run all integration tests on Google Cloud Platform:
+#   $ ./run-integration-tests.sh
+#
+#   Run all integration tests on Amazon Web Services:
+#   PROVIDER=aws ./run-integration-tests.sh
+#
+
+set -e
+DIR="$(dirname "$0")"
+. "$DIR/../tools/provisioning/setup.sh" # Import gcp_on, do_on, and aws_on.
+. "$DIR/config.sh"                      # Import greenly.
+
+# Variables:
+APP="weave-net"
+# shellcheck disable=SC2034
+PROJECT="weave-net-tests" # Only used when PROVIDER is gcp, by tools/provisioning/config.sh.
+NAME=${NAME:-"$(whoami | sed -e 's/[\.\_]*//g' | cut -c 1-4)"}
+PROVIDER=${PROVIDER:-gcp} # Provision using provided provider, or Google Cloud Platform by default.
+NUM_HOSTS=${NUM_HOSTS:-3}
+PLAYBOOK=${PLAYBOOK:-setup_weave-net_test.yml}
+TESTS=${TESTS:-}
+RUNNER_ARGS=${RUNNER_ARGS:-""}
+# Dependencies' versions:
+DOCKER_VERSION=${DOCKER_VERSION:-"$(grep -oP "(?<=DOCKER_VERSION=).*" "$DIR/../DEPENDENCIES")"}
+KUBERNETES_VERSION=${KUBERNETES_VERSION:-"$(grep -oP "(?<=KUBERNETES_VERSION=).*" "$DIR/../DEPENDENCIES")"}
+KUBERNETES_CNI_VERSION=${KUBERNETES_CNI_VERSION:-"$(grep -oP "(?<=KUBERNETES_CNI_VERSION=).*" "$DIR/../DEPENDENCIES")"}
+# Google Cloud Platform image's name & usage (only used when PROVIDER is gcp):
+IMAGE_NAME=${IMAGE_NAME:-"$(echo "$APP-docker$DOCKER_VERSION-k8s$KUBERNETES_VERSION-k8scni$KUBERNETES_CNI_VERSION" | sed -e 's/[\.\_]*//g')"}
+DISK_NAME_PREFIX=${DISK_NAME_PREFIX:-$NAME}
+USE_IMAGE=${USE_IMAGE:-1}
+CREATE_IMAGE=${CREATE_IMAGE:-1}
+CREATE_IMAGE_TIMEOUT_IN_SECS=${CREATE_IMAGE_TIMEOUT_IN_SECS:-600}
+# Lifecycle flags:
+SKIP_CONFIG=${SKIP_CONFIG:-}
+SKIP_DESTROY=${SKIP_DESTROY:-}
+
+function print_vars() {
+    echo "--- Variables: Main ---"
+    echo "PROVIDER=$PROVIDER"
+    echo "NUM_HOSTS=$NUM_HOSTS"
+    echo "PLAYBOOK=$PLAYBOOK"
+    echo "TESTS=$TESTS"
+    echo "SSH_OPTS=$SSH_OPTS"
+    echo "RUNNER_ARGS=$RUNNER_ARGS"
+    echo "--- Variables: Versions ---"
+    echo "DOCKER_VERSION=$DOCKER_VERSION"
+    echo "KUBERNETES_VERSION=$KUBERNETES_VERSION"
+    echo "KUBERNETES_CNI_VERSION=$KUBERNETES_CNI_VERSION"
+    echo "IMAGE_NAME=$IMAGE_NAME"
+    echo "DISK_NAME_PREFIX=$DISK_NAME_PREFIX"
+    echo "USE_IMAGE=$USE_IMAGE"
+    echo "CREATE_IMAGE=$CREATE_IMAGE"
+    echo "CREATE_IMAGE_TIMEOUT_IN_SECS=$CREATE_IMAGE_TIMEOUT_IN_SECS"
+    echo "--- Variables: Flags ---"
+    echo "SKIP_CONFIG=$SKIP_CONFIG"
+    echo "SKIP_DESTROY=$SKIP_DESTROY"
+}
+
+function verify_dependencies() {
+    local deps=(python terraform ansible-playbook)
+    for dep in "${deps[@]}"; do
+        if [ ! "$(which "$dep")" ]; then
+            echo >&2 "$dep is not installed or not in PATH."
+            exit 1
+        fi
+    done
+}
+
+# shellcheck disable=SC2155
+function provision_locally() {
+    export VAGRANT_CWD="$(dirname "${BASH_SOURCE[0]}")"
+    case "$1" in
+        on)
+            vagrant up
+            local status=$?
+
+            # Set up SSH connection details: 
+            local ssh_config=$(mktemp /tmp/vagrant_ssh_config_XXX)
+            vagrant ssh-config >"$ssh_config"
+            export SSH="ssh -F $ssh_config"
+            # Extract username, SSH private key, and VMs' IP addresses:
+            ssh_user="$(sed -ne 's/\ *User //p' "$ssh_config" | uniq)"
+            ssh_id_file="$(sed -ne 's/\ *IdentityFile //p' "$ssh_config" | uniq)"
+            ssh_hosts=$(sed -ne 's/Host //p' "$ssh_config")
+
+            # Set up /etc/hosts files on this ("local") machine and the ("remote") testing machines, to map hostnames and IP addresses, so that:
+            # - this machine communicates with the testing machines via their public IPs;
+            # - testing machines communicate between themselves via their private IPs;
+            # - we can simply use just the hostname in all scripts to refer to machines, and the difference between public and private IP becomes transparent.
+            # N.B.: if you decide to use public IPs everywhere, note that some tests may fail (e.g. test #115).
+            update_local_etc_hosts "$ssh_hosts" "$(for host in $ssh_hosts; do $SSH "$host" "cat /etc/hosts | grep $host"; done)"
+
+            SKIP_CONFIG=1 # Vagrant directly configures virtual machines using Ansible -- see also: Vagrantfile
+            return $status
+            ;;
+        off)
+            vagrant destroy -f
+            ;;
+        *)
+            echo >&2 "Unknown command $1. Usage: {on|off}."
+            exit 1
+            ;;
+    esac
+}
+
+function setup_gcloud() {
+    # Authenticate:
+    gcloud auth activate-service-account --key-file "$GOOGLE_CREDENTIALS_FILE" 1>/dev/null
+    # Set current project:
+    gcloud config set project $PROJECT
+}
+
+function image_exists() {
+    gcloud compute images list | grep "$PROJECT" | grep "$IMAGE_NAME"
+}
+
+function image_ready() {
+    # GCP images seem to be listed before they are actually ready for use, 
+    # typically failing the build with: "googleapi: Error 400: The resource is not ready".
+    # We therefore consider the image to be ready once the disk of its template instance has been deleted.
+    ! gcloud compute disks list | grep "$DISK_NAME_PREFIX"
+}
+
+function wait_for_image() {
+    greenly echo "> Waiting for GCP image $IMAGE_NAME to be created..."
+    for i in $(seq "$CREATE_IMAGE_TIMEOUT_IN_SECS"); do
+        image_exists && image_ready && return 0
+        if ! ((i % 60)); then echo "Waited for $i seconds and still waiting..."; fi
+        sleep 1
+    done
+    redly echo "> Waited $CREATE_IMAGE_TIMEOUT_IN_SECS seconds for GCP image $IMAGE_NAME to be created, but image could not be found."
+    exit 1
+}
+
+# shellcheck disable=SC2155
+function create_image() {
+    if [[ "$CREATE_IMAGE" == 1 ]]; then
+        greenly echo "> Creating GCP image $IMAGE_NAME..."
+        local begin_img=$(date +%s)
+        local num_hosts=1
+        terraform apply -input=false -var "app=$APP" -var "name=$NAME" -var "num_hosts=$num_hosts" "$DIR/../tools/provisioning/gcp"
+        configure_with_ansible "$(terraform output username)" "$(terraform output public_ips)," "$(terraform output private_key_path)" $num_hosts
+        local zone=$(terraform output zone)
+        local name=$(terraform output instances_names)
+        gcloud -q compute instances delete "$name" --keep-disks boot --zone "$zone"
+        gcloud compute images create "$IMAGE_NAME" --source-disk "$name" --source-disk-zone "$zone" \
+            --description "Testing image for Weave Net based on $(terraform output image), Docker $DOCKER_VERSION, Kubernetes $KUBERNETES_VERSION and Kubernetes CNI $KUBERNETES_CNI_VERSION."
+        gcloud compute disks delete "$name" --zone "$zone"
+        terraform destroy -force "$DIR/../tools/provisioning/gcp"
+        rm terraform.tfstate*
+        echo
+        greenly echo "> Created GCP image $IMAGE_NAME in $(date -u -d @$(($(date +%s) - begin_img)) +"%T")."
+    else
+        wait_for_image
+    fi
+}
+
+function use_or_create_image() {
+    setup_gcloud
+    image_exists || create_image
+    export TF_VAR_gcp_image="$IMAGE_NAME" # Override the default image name.
+    export SKIP_CONFIG=1                  # No need to configure the image, since already done when making the template
+}
+
+function update_local_etc_hosts() {
+    echo "> Updating local /etc/hosts..."
+    # Remove old entries (if present):
+    for host in $1; do sudo sed -i "/$host/d" /etc/hosts; done
+    # Add new entries:
+    sudo sh -c "echo \"$2\" >> /etc/hosts"
+}
+
+function upload_etc_hosts() {
+    # Remove old entries (if present):
+    # shellcheck disable=SC2016,SC2086
+    $SSH $3 'for host in '$1'; do sudo sed -i "/$host/d" /etc/hosts; done'
+    # Add new entries:
+    echo "$2" | $SSH "$3" "sudo -- sh -c \"cat >> /etc/hosts\""
+}
+
+function update_remote_etc_hosts() {
+    echo "> Updating remote /etc/hosts..."
+    local pids=""
+    for host in $1; do
+        upload_etc_hosts "$1" "$2" "$host" &
+        local pids="$pids $!"
+    done
+    for pid in $pids; do wait "$pid"; done
+}
+
+# shellcheck disable=SC2155
+function set_hosts() {
+    export HOSTS="$(echo "$ssh_hosts" | tr '\n' ' ')"
+}
+
+function provision_remotely() {
+    case "$1" in
+        on)
+            terraform apply -input=false -parallelism="$NUM_HOSTS" -var "app=$APP" -var "name=$NAME" -var "num_hosts=$NUM_HOSTS" "$DIR/../tools/provisioning/$2"
+            local status=$?
+            ssh_user=$(terraform output username)
+            ssh_id_file=$(terraform output private_key_path)
+            ssh_hosts=$(terraform output hostnames)
+            export SSH="ssh -l $ssh_user -i $ssh_id_file $SSH_OPTS"
+
+            # Set up /etc/hosts files on this ("local") machine and the ("remote") testing machines, to map hostnames and IP addresses, so that:
+            # - this machine communicates with the testing machines via their public IPs;
+            # - testing machines communicate between themselves via their private IPs;
+            # - we can simply use just the hostname in all scripts to refer to machines, and the difference between public and private IP becomes transparent.
+            # N.B.: if you decide to use public IPs everywhere, note that some tests may fail (e.g. test #115).
+            update_local_etc_hosts "$ssh_hosts" "$(terraform output public_etc_hosts)"
+            update_remote_etc_hosts "$ssh_hosts" "$(terraform output private_etc_hosts)"
+
+            return $status
+            ;;
+        off)
+            terraform destroy -force "$DIR/../tools/provisioning/$2"
+            ;;
+        *)
+            echo >&2 "Unknown command $1. Usage: {on|off}."
+            exit 1
+            ;;
+    esac
+}
+
+# shellcheck disable=SC2155
+function provision() {
+    local action=$([ "$1" == "on" ] && echo "Provisioning" || echo "Shutting down")
+    echo
+    greenly echo "> $action test host(s) on [$PROVIDER]..."
+    local begin_prov=$(date +%s)
+    case "$2" in
+        'aws')
+            aws_on
+            provision_remotely "$1" "$2"
+            ;;
+        'do')
+            do_on
+            provision_remotely "$1" "$2"
+            ;;
+        'gcp')
+            gcp_on
+            [[ "$1" == "on" ]] && [[ "$USE_IMAGE" == 1 ]] && use_or_create_image
+            provision_remotely "$1" "$2"
+            ;;
+        'vagrant')
+            provision_locally "$1"
+            ;;
+        *)
+            echo >&2 "Unknown provider $2. Usage: PROVIDER={gcp|aws|do|vagrant}."
+            exit 1
+            ;;
+    esac
+    [ "$1" == "on" ] && set_hosts
+    echo
+    greenly echo "> Provisioning took $(date -u -d @$(($(date +%s) - begin_prov)) +"%T")."
+}
+
+function configure_with_ansible() {
+    ansible-playbook -u "$1" -i "$2" --private-key="$3" --forks="${4:-$NUM_HOSTS}" \
+        --ssh-extra-args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+        --extra-vars "docker_version=$DOCKER_VERSION kubernetes_version=$KUBERNETES_VERSION kubernetes_cni_version=$KUBERNETES_CNI_VERSION" \
+        "$DIR/../tools/config_management/$PLAYBOOK"
+}
+
+# shellcheck disable=SC2155
+function configure() {
+    echo
+    if [ -n "$SKIP_CONFIG" ]; then
+        greenly echo "> Skipped configuration of test host(s)."
+    else
+        greenly echo "> Configuring test host(s)..."
+        local begin_conf=$(date +%s)
+        local inventory_file=$(mktemp /tmp/ansible_inventory_XXXXX)
+        echo "[all]" >"$inventory_file"
+        # shellcheck disable=SC2001
+        echo "$2" | sed "s/$/:$3/" >>"$inventory_file"
+
+        # Configure the provisioned machines using Ansible, allowing up to 3 retries upon failure (e.g. APT connectivity issues, etc.):
+        for i in $(seq 3); do
+            configure_with_ansible "$1" "$inventory_file" "$4" && break || echo >&2 "#$i: Ansible failed. Retrying now..."
+        done
+
+        echo
+        greenly echo "> Configuration took $(date -u -d @$(($(date +%s) - begin_conf)) +"%T")."
+    fi
+}
+
+# shellcheck disable=SC2155
+function run_tests() {
+    echo
+    greenly echo "> Running tests..."
+    local begin_tests=$(date +%s)
+    set +e # Do not fail this script upon test failure, since we need to shut down the test cluster regardless of success or failure.
+    "$DIR/run_all.sh" "$@"
+    local status=$?
+    echo
+    greenly echo "> Tests took $(date -u -d @$(($(date +%s) - begin_tests)) +"%T")."
+    return $status
+}
+
+function end() {
+    echo
+    echo "> Build took $(date -u -d @$(($(date +%s) - begin)) +"%T")."
+}
+
+function echo_export_hosts() {
+    exec 1>&111
+    # Print a command to set HOSTS in the calling script, so that subsequent calls to
+    # test scripts can point to the right testing machines while developing:
+    echo "export HOSTS=\"$HOSTS\""
+    exec 1>&2
+}
+
+function main() {
+    # Keep a reference to stdout in another file descriptor (FD #111), and then globally redirect all stdout to stderr.
+    # This is so that HOSTS can be eval'ed in the calling script using:
+    #   $ eval $(./run-integration-tests.sh [provision|configure|setup])
+    # and ultimately subsequent calls to test scripts can point to the right testing machines during development.
+    if [ "$1" == "provision" ] || [ "$1" == "configure" ] || [ "$1" == "setup" ]; then
+        exec 111>&1 # 111 ought to match the file descriptor used in echo_export_hosts.
+        exec 1>&2
+    fi
+
+    begin=$(date +%s)
+    trap end EXIT
+
+    print_vars
+    verify_dependencies
+
+    case "$1" in
+        "") # Provision, configure, run tests, and destroy test environment:
+            provision on "$PROVIDER"
+            configure "$ssh_user" "$ssh_hosts" "${ssh_port:-22}" "$ssh_id_file"
+            "$DIR/setup.sh"
+            run_tests "$TESTS"
+            status=$?
+            [ -z "$SKIP_DESTROY" ] && provision off "$PROVIDER"
+            exit $status
+            ;;
+
+        provision)
+            provision on "$PROVIDER"
+            echo_export_hosts
+            ;;
+
+        configure)
+            provision on "$PROVIDER" # Vagrant and Terraform do not provision twice if VMs are already provisioned, so we just set environment variables.
+            configure "$ssh_user" "$ssh_hosts" "${ssh_port:-22}" "$ssh_id_file"
+            echo_export_hosts
+            ;;
+
+        setup)
+            provision on "$PROVIDER" # Vagrant and Terraform do not provision twice if VMs are already provisioned, so we just set environment variables.
+            "$DIR/setup.sh"
+            echo_export_hosts
+            ;;
+
+        test)
+            provision on "$PROVIDER" # Vagrant and Terraform do not provision twice if VMs are already provisioned, so we just set environment variables.
+            run_tests "$TESTS"
+            ;;
+
+        destroy)
+            provision off "$PROVIDER"
+            ;;
+
+        *)
+            echo "Unknown command: $1" >&2
+            exit 1
+            ;;
+    esac
+}
+
+main "$@"

test/run_all.sh

@@ -11,7 +11,7 @@ fi
 whitely echo ...ok
 
 TESTS="${@:-$(find "$DIR" -name '*_test.sh')}"
-RUNNER_ARGS=""
+RUNNER_ARGS=${RUNNER_ARGS:-""}
 
 # If running on circle, use the scheduler to work out what tests to run
 if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" ]; then

test/sanity_check.sh

@@ -7,6 +7,7 @@ set -e
 begin=$(date +%s)
 sanity_checks_files=${WEAVE_NET_SANITY_CHECKS_FILES:-"/tmp/weave_net_sanity_check_*.log"}
 
+greenly echo "> Sanity-checking test machines: ping, check Docker, and check Weave..."
 whitely echo Ping each host from the other
 
 # We wrap ping and echo in a function as we want the below parallel for loop
@@ -68,4 +69,4 @@ for host in $HOSTS; do
 done
 for pid in $pids; do wait $pid; done
 
-echo "Sanity checks completed successfully in $(date -u -d @$(($(date +%s)-$begin)) +"%T")."
+greenly echo "> Sanity checks completed successfully in $(date -u -d @$(($(date +%s)-$begin)) +"%T")."

test/setup.sh

@@ -9,8 +9,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")"
 
 (cd ./tls && ./tls $HOSTS)
 
-echo "Copying weave images, scripts, and certificates to hosts, and"
-echo "  prefetch test images"
+greenly echo "> Setting up test machines: copying weave images, scripts and certificates to hosts, and prefetch test images..."
 
 exists_on() {
     docker_on $1 inspect --format=" " $2 >/dev/null 2>&1
@@ -60,4 +59,4 @@ for ppid in $ppids; do
     wait $ppid;
 done
 
-echo "Setup completed successfully in $(date -u -d @$(($(date +%s)-$begin)) +"%T")."
+greenly echo "> Setup completed successfully in $(date -u -d @$(($(date +%s)-$begin)) +"%T")."

vagrant-common.rb

@@ -1,73 +1,43 @@
-$go_regexp = /FROM golang:(\S*).*?/
-$dockerfile_path = File.expand_path(File.join(File.dirname(__FILE__), 'build', 'Dockerfile'))
-$go_version = File.readlines($dockerfile_path).first { |line| line.match($go_regexp) }.match($go_regexp).captures.first
-if $go_version.nil?
-  raise ArgumentError.new("Failed to read Go version from Dockerfile.")
+VAGRANT_IMAGE = 'bento/ubuntu-16.04'
+VAGRANTFILE_API_VERSION = '2'
+
+def get_dependencies_version_from_file_and_env()
+  dependencies_file = File.join(File.dirname(__FILE__), 'DEPENDENCIES')
+  # Read default version from the DEPENDENCIES file:
+  versions = File.readlines(dependencies_file).map{|line| line.strip.split('=')}.to_h
+  # Override with environment variables if defined:
+  versions.each do |k,v|
+    versions[k] = ENV.key?(k) ? ENV[k] : v
+  end
 end
 
-$go_path = "/usr/local/go/bin"
-
-def install_packages(vm, pkgs)
-  vm.provision :shell, :inline => <<SCRIPT
-apt-get update -qq
-apt-get install -qq -y --force-yes --no-install-recommends #{pkgs.join(' ')}
-SCRIPT
+def ansibleize(h)
+  h.map{|k,v| [k.downcase, v]}.to_h
 end
 
-def install_build_deps(vm, pkgs)
-  vm.provision :shell, :inline => <<SCRIPT
-export DEBIAN_FRONTEND=noninteractive
-apt-key adv \
-  --keyserver hkp://p80.pool.sks-keyservers.net:80 \
-  --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
-echo 'deb https://apt.dockerproject.org/repo ubuntu-wily main' \
-  > /etc/apt/sources.list.d/docker.list
-SCRIPT
-  install_packages(vm, pkgs)
+def get_go_version_from_build_dockerfile()
+  go_regexp = /FROM golang:(\S*).*?/
+  dockerfile_path = File.expand_path(File.join(File.dirname(__FILE__), 'build', 'Dockerfile'))
+  go_version = File.readlines(dockerfile_path).select { |line| line.match(go_regexp) }.first.match(go_regexp).captures.first
+  if go_version.nil?
+    raise ArgumentError.new("Failed to read Go version from Dockerfile.")
+  end
+  go_version
 end
 
-def install_go_toochain(vm)
-  vm.provision :shell, :inline => <<SCRIPT
-curl -s https://storage.googleapis.com/golang/go#{$go_version}.linux-amd64.tar.gz \
-  | tar xz -C /usr/local
-#{$go_path}/go clean -i net
-#{$go_path}/go install -tags netgo std
-SCRIPT
-end
+GO_VERSION = get_go_version_from_build_dockerfile()
 
-def tweak_user_env(vm)
-  script = <<SCRIPT
-echo 'export GOPATH="${HOME}"' \
-  >> ~vagrant/.profile
-echo 'export PATH="${HOME}/bin:#{$go_path}:${PATH}"' \
-  >> ~vagrant/.profile
-ln -sf ~vagrant/src/github.com/weaveworks/weave ~vagrant/
-sudo chown -R vagrant:vagrant ~vagrant/src
-SCRIPT
-  vm.provision :shell, :inline => script, :privileged => false
-end
-
-def tweak_docker_daemon(vm)
-  vm.provision :shell, :inline => <<SCRIPT
-usermod -a -G docker vagrant
-mkdir -p /etc/systemd/system/docker.service.d
-cat >/etc/systemd/system/docker.service.d/override.conf  <<EOF
-[Service]
-ExecStart=
-ExecStart=/usr/bin/docker daemon -H fd:// -H unix:///var/run/alt-docker.sock -H tcp://0.0.0.0:2375 -s overlay
-EOF
-systemctl daemon-reload
-systemctl restart docker
-systemctl enable docker
-SCRIPT
+def configure_nat_dns(vb)
+  vb.customize ["modifyvm", :id, "--natdnshostresolver1", "off"]
+  vb.customize ["modifyvm", :id, "--natdnsproxy1", "off"]
 end
 
 def cleanup(vm)
   vm.provision :shell, :inline => <<SCRIPT
 export DEBIAN_FRONTEND=noninteractive
 ## Who the hell thinks official images have to have both of these?
-/etc/init.d/chef-client stop
-/etc/init.d/puppet stop
+[ ! -f /etc/init.d/chef-client ] || /etc/init.d/chef-client stop
+[ ! -f /etc/init.d/puppet ]      || /etc/init.d/puppet stop
 apt-get -qq remove puppet chef
 apt-get -qq autoremove
 killall -9 chef-client 2>/dev/null || true

weave

@@ -1304,25 +1304,6 @@ check_overlap() {
     util_op netcheck $1 $BRIDGE
 }
 
-# Claim addresses for a container in IPAM.  Expects to be called from
-# with_container_addresses.
-ipam_reclaim() {
-    CONTAINER_ID="$1"
-    # The weave IP addresses of containers attached by the plugin are
-    # recorded specially in IPAM, since the container id is not know
-    # at the time.
-    [ "$2" = "$CONTAINER_IFNAME" ] || CONTAINER_ID="_"
-    for CIDR in $4 ; do
-        http_call $HTTP_ADDR PUT "/ip/$CONTAINER_ID/$CIDR?noErrorOnUnknown=true&?check-alive=true" || true
-    done
-}
-
-ipam_reclaim_no_check_alive() {
-    for CIDR in $4 ; do
-        http_call $HTTP_ADDR PUT /ip/$1/$CIDR?noErrorOnUnknown=true || true
-    done
-}
-
 detect_awsvpc() {
     # Ignoring errors here: if we cannot detect AWSVPC we will skip the relevant
     # steps, because "attach" should work without the weave router running.
@@ -1691,6 +1672,7 @@ launch_router() {
     ROUTER_CONTAINER=$(docker run -d --name=$CONTAINER_NAME \
         $(docker_run_options) \
         $RESTART_POLICY \
+        --pid=host \
         --volumes-from $DB_CONTAINER_NAME \
         -v $RESOLV_CONF_DIR:/var/run/weave/etc \
         -e WEAVE_PASSWORD \
@@ -1728,17 +1710,10 @@ setup_awsvpc() {
 # Recreate the parameter values that are set when the router is first launched
 fetch_router_args() {
     CONTAINER_ARGS=$(docker inspect -f '{{.Args}}' $CONTAINER_NAME) || return 1
-    IPRANGE=$(echo $CONTAINER_ARGS | grep -o -E -e '-ipalloc-range [0-9/.]+') || true
     NO_DNS_OPT=$(echo $CONTAINER_ARGS | grep -o -e '--no-dns') || true
 }
 
 populate_router() {
-    if [ -n "$IPRANGE" ] ; then
-        # Tell the newly-started weave IP allocator about existing weave IPs
-        # In the case of AWSVPC, we do expose before calling populate_router
-        [ -n "$AWSVPC" ] || with_container_addresses ipam_reclaim_no_check_alive weave:expose
-        with_container_addresses ipam_reclaim $(docker ps -q --no-trunc)
-    fi
     if [ -z "$NO_DNS_OPT" ] ; then
         # Tell the newly-started weaveDNS about existing weave IPs
         for CONTAINER in $(docker ps -q --no-trunc) ; do