Add release v2.6.4

Only drop traffic to the Weave Net port on 127.0.0.1

.circleci/config.yml

@@ -0,0 +1,123 @@
+version: 2
+defaults: &defaults
+  docker:
+    - image: weaveworks/weavebuild:go-1-13-1-5110a604
+  working_directory: /go/src/github.com/weaveworks/weave
+
+jobs:
+  lint:
+    <<: *defaults
+    steps:
+      - checkout
+      - run: git submodule update --init
+      - run: make DEBUG=1 BUILD_IN_CONTAINER=false lint
+
+  unit-test:
+    <<: *defaults
+    parallelism: 2
+    steps:
+      - checkout
+      - run: COVERDIR=test/coverage make BUILD_IN_CONTAINER=false tests
+      - persist_to_workspace:
+          root: .
+          paths:
+          - test/coverage
+
+  build:
+    <<: *defaults
+    steps:
+      - checkout
+      - setup_remote_docker
+      - run: git submodule update --init
+      - run: make COVERAGE=true BUILD_IN_CONTAINER=false SUDO= exes all
+      - persist_to_workspace:
+          root: .
+          paths:
+          - weave.tar.gz
+          - tools/runner/runner
+          - test/tls/tls
+
+  smoke-tests:
+    machine:
+      image: circleci/classic:201709-01
+    working_directory: /home/circleci/src/github.com/weaveworks/weave
+    environment:
+      SRCDIR: /home/circleci/src/github.com/weaveworks/weave
+      CIRCLE_ARTIFACTS: /tmp/artifacts
+    parallelism: 2
+    steps:
+      - checkout
+      - run: mkdir $CIRCLE_ARTIFACTS
+      - run: git submodule update --init
+      - attach_workspace:
+          at: .
+      # kick off creation of test VMs
+      - run: bin/provision_test_vms.sh
+      # when VMs are ready, copy built software to them
+      - run: bin/circle-test-pre
+      - run:
+          command: bin/circle-test-smoke
+          no_output_timeout: 360s
+      # Destroy testing VMs:
+      - run:
+          command: bin/circle-destroy-vms
+          background: true
+      - persist_to_workspace:
+          root: .
+          paths:
+          - test/coverage
+      - store_artifacts:
+          path: /tmp/artifacts
+
+  gen-coverage:
+    <<: *defaults
+    environment:
+      CIRCLE_ARTIFACTS: /tmp/artifacts
+    steps:
+      - checkout
+      - run: mkdir $CIRCLE_ARTIFACTS
+      - attach_workspace:
+          at: .
+      - run: cd test; ./gen_coverage_reports.sh
+      - run: goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=test/profile.cov -service=circleci
+      - run: cp test/coverage.* $CIRCLE_ARTIFACTS
+      - store_artifacts:
+          path: /tmp/artifacts
+
+  deploy:
+    <<: *defaults
+    steps:
+      - checkout
+      - setup_remote_docker
+      # Register qemu hooks, rebuild without coverage and push to dockerhub
+      - deploy:
+          command: |
+            docker login -u $DOCKER_USER -p $DOCKER_PASS
+            docker run --rm --privileged multiarch/qemu-user-static:register --reset
+            wget https://github.com/estesp/manifest-tool/releases/download/v0.8.0/manifest-tool-linux-amd64
+            chmod +x manifest-tool-linux-amd64
+            make clean-bin
+            make publish MANIFEST_TOOL_CMD=./manifest-tool-linux-amd64 UPDATE_LATEST=latest-only SUDO= BUILD_IN_CONTAINER=false
+
+workflows:
+  version: 2
+  build_test:
+    jobs:
+      - lint
+      - unit-test
+      - build
+      - smoke-tests:
+          requires:
+            - lint
+            - unit-test
+            - build
+      - gen-coverage:
+          requires:
+            - unit-test
+            - smoke-tests
+      - deploy:
+          requires:
+            - smoke-tests
+          filters:
+            branches:
+              only: master

.lintignore

@@ -4,7 +4,6 @@
 # avoid breaking the build on make lint:
 bin/circle-deploy-issues
 bin/circle-deploy-master
-bin/circle-teardown-pre
 bin/circle-test-teardown
 bin/circle-test-unit
 bin/install-wordepress

.ruby-version

@@ -1 +1 @@
-2.3.0
+2.6.2

CHANGELOG.md

@@ -1,3 +1,211 @@
+## Release 2.6.4
+
+Improves the iptables rule added in 2.6.3 to block just the Weave Net
+control port and avoid blocking other uses of 127.0.0.1.	#3811
+
+
+## Release 2.6.3
+
+This release has a couple of security improvements, and some other fixes.
+
+* Block non-local traffic to the Weave control port	#3805
+* Tell Linux not to accept router advisory messages	#3801
+* Network Policy Controller: add a metric to show errors while operating	#3804
+* Network Policy Controller: don't treat named port as a fatal error	#3790
+
+## Release 2.6.2
+
+This release fixes a regression found in 2.6.1 release and removes a possible case
+of CPU spinning when pcap is used in sleeve mode
+
+### Bug fixes
+
+* Weave Net can not be used in fastdp mode and always falls back to sleeve mode #3781, #3783
+* Restrict timeout value passed to pcap library to a value less than 2^31 microseconds to
+  prevent CPU spinning #3782
+
+## Release 2.6.1
+
+This release fixes a bug discovered since the release of Weave Net 2.6.0 and
+an improvement to support iptables 1.8
+
+### Bug fixes
+
+* Weave Net on node restart unable to form connections to peers and connections get stuck in
+  pending state and stops responding to API endpoint #3762 #3763
+
+### Other improvements
+
+* Weave Net Kubernetes images by default uses iptables 1.8 with legacy (netfilter) backend with
+  option to choose netfilter as backend #3465, #3747
+
+
+## Release 2.6.0
+
+This release reduces CPU and memory usage in larger clusters, by
+sending notifications to a smaller set of peers and coalescing updates
+to reduce topology recalculation. #3715, #3732
+
+The default soft limit on connections has been raised from 100 to 200.
+
+### Bug fixes
+
+* Fix a race condition in Kubernetes addon when reclaiming IP
+  addresses after node deletion #3724, #3716
+* Buffer events so Docker won't drop them, and Weave Net can clean up
+  after dead containers #3432, #3705
+* Weave reconnect occasionally fails after network interface
+  disconnect #3666, #3669, #3676
+* Ingress NetworkPolicy would accepts all traffic when specifying both
+  IPBlock and port #3653, #3654
+
+### Kubernetes improvements
+
+* Support both podSelector and namespaceSelector in NetworkPolicy #3312, #3647
+* Only add default-drop egress rule if network policies are in use #3639
+* Manifests use 'apps/v1' rather than deprecated 'apps/v1beta1' #3660
+* Avoid Weave Net pods being evicted by setting priorityClassName: system-node-critical #3697
+* Manifests use recommended DNS policy ClusterFirstWithHostNet #3692
+* Weave Net now tolerates 'NoExecute' taint #3655
+* Allow extra arguments to NetworkPolicy controller to be set in an environment variable #3683
+* Stop reporting a failure to connect to self #3454, #3585
+* Minor reduction in log noise when reclaiming IPs #3710
+
+### Other improvements
+
+* Avoid isolating nodes which have restarted by automatically
+  repairing inconsistencies in IP allocation data #1962, #3637, #3708
+* Build Weave Net for the s390x architecture #3685
+* When a IP address is requested that may be in use, make several
+  attempts to claim it before returning an error #3725
+* Improve logging for IP allocation updates #3627, #3630
+* Improve 'expecting PMTU update' log message on initial connection #3603
+
+### Build and test
+
+* Shut down Kubernetes on node when testing node deletion #3716
+* Update the base 'Alpine' container image to version 3.8 #3701
+* Update Go to version 1.13.3 #3712
+* Update gopacket library #3590
+* Pin busybox version to 1.28 to avoid CI failure in Python test #3689
+* Remove obsolete weave-daemonset.yaml file #3674
+
+Thanks to contributors @christian-2, @hpdvanwyk, @guirish, @kitt1987,
+@mmerrill3, @Pensu, @scritchley, @sidharthsurana, @tanishq-dubey
+
+[Full list of changes](https://github.com/weaveworks/weave/milestone/78?closed=1)
+
+
+## Release 2.5.2
+
+This release fixes several bugs causing inconsistencies in IPAM and fixes a 
+panic in daemon that reclaims and forgets deleted nodes in Kubernetes clusters.
+
+### Bug fixes
+
+* leaking IP addresses in IPAM when weave-net hangs or restarting #3587, #3638
+* Panic in kube-utils after config map update error resulting in reclaim daemon
+  not performing weave forget on deleted node #3613, #3623
+* preventing inconsistent IPAM entries and resolving the conflict in case of
+  conflict during IPAM ring merge #3629, #3635, #3632, #3444
+
+[Full list of changes](https://github.com/weaveworks/weave/milestone/79?closed=1)
+
+
+## Release 2.5.1
+
+This release fixes bugs reported for 2.5 release and small improvements.
+
+### Bug fixes
+
+* prevent warnings in kernal logs due to use of physdev model for
+  non-bridged traffic #3449, #3453
+* Check and report the error occurred while getting the list of Kubernetes 
+  peers #3581, #3582
+
+### Other improvements
+
+* Update Ruby version to match Weaveworks website #3458
+* Fix the link to "Troubleshooting the Kubernetes Addon" section in the docs #3461
+* Document concept of connection limit in Weave #3395, #3413
+
+[Full list of changes](https://github.com/weaveworks/weave/milestone/77?closed=1)
+
+
+## Release 2.5.0
+
+This release adds support for Kubernetes `hostPort` mapping (#3016,#3356)
+and the `ipBlock` NetworkPolicy feature (#3168,#3367)
+
+### Bug fixes
+
+* Fix a crash at start-up on Docker for Mac #3405, #3408
+* Network policy: block ingress traffic when no namespaceSelector or
+  podSelector is specified #3347
+* Reclaim IP addresses which are locked by a non-existent peer #3386, #3416
+* Fix a crash when blank IP data was loaded #3067, #3415
+
+### Other improvements
+
+* If a connection is downgraded to the slower "sleeve" mode, Weave Net
+  will now periodically try to upgrade it to "fast datapath" again. #1737, #3385
+* Reclaim removed Kubernetes nodes' IP space when they are deleted,
+  rather than on next restart #3372, #3399
+* Replace Kubernetes livenessProbe with readinessProbe, so the pod is
+  not killed if it runs slowly #3471, #3421
+* In Kubernetes NetworkPolicy controller, remove the need to maintain
+  a set of local pod IP addresses #3344, #3423
+* Don't crash on Kubernetes named port in NetworkPolicy, just report as unsupported #3375
+* Ensure the `weave` network bridge is accessible on Linux kernels
+  older than 3.14 #3442, #3297, #3239
+* Better reporting in the logs if the `weave` network device is in the Down state #3133, #3381
+* Change log-level to debug of calls through the Docker proxy, to reduce noise #3439
+* Add `--without-masquerade` option to `weave expose`, so external
+  services can see the original container IP address #3388
+* Include Kubernetes cluster information in checkpoint call #3324,#3431
+* Bump go-odp dependency, so that `fastdp` works on the 4.19 kernel #3430
+
+### Build and Testing
+
+* CI builds on master branch now publish images for all platforms
+* Fix golint path and use https for download of libpcap #3435
+* Update Kubernetes client-go to v8.0.0, removing code licenced under LGPL3 #3358,#3366
+* Migrate CircleCI to V2, which is much faster #3255,#3270
+
+### External Contributors
+
+Thanks to the following contributors:
+
+* @Ashiroq
+* @leprechau
+* @lkpdn
+
+[Full list of changes](https://github.com/weaveworks/weave/milestone/74?closed=1)
+
+
+## Release 2.4.1
+
+This release fixes several bugs causing inconsistencies in IPAM for
+Kubernetes users whose clusters scale up and down over time.
+
+### Bug fixes
+
+* Nodes unable to connect after Kubernetes addon erroneously reclaimed
+  node without any IP addresses #3392, #3393
+* Kubernetes addon could have run out of free IP addresses after nodes
+  are deleted #3384, #3400
+* Kubernetes addon had reduced free IP addresses due to not reclaiming
+  IP addresses when node name is re-used #3397
+
+### Other improvements
+
+* Support `--label` in `WEAVE_DOCKER_ARGS` when starting Weave #3370,#3371
+* Add missing `--token` argument in help for `weave launch` #3226, #3379
+* Print defunct processes after smoke-tests #3362
+
+[Full list of changes](https://github.com/weaveworks/weave/milestone/75?closed=1)
+
+
 ## Release 2.4.0
 
 This release introduces a support for Kubernetes Egress Network Policy (#2624, #3313)

CONTRIBUTORS

@@ -1,5 +0,0 @@
-This file lists all individuals having contributed content to the
-repository.
-
-Matthew Sackman <matthew@wellquite.org>
-Matthias Radestock <matthias.radestock@gmail.com>

COPYING.LGPL-3

@@ -1,23 +1,14 @@
-package cmd
+./test/assert.sh is a copy of
 
-func initLgpl() {
-	Licenses["lgpl"] = License{
-		Name:            "GNU Lesser General Public License",
-		PossibleMatches: []string{"lgpl", "lesser gpl", "gnu lgpl"},
-		Header: `
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Lesser General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
+	https://github.com/lehmannro/assert.sh/blob/master/assert.sh
 
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Lesser General Public License for more details.
+Since it was imported from its original source, it has only received
+cosmetic modifications. As it is licensed under the LGPL-3, here's the
+license text in its entirety:
 
-You should have received a copy of the GNU Lesser General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.`,
-		Text: `                   GNU LESSER GENERAL PUBLIC LICENSE
+
+
+                   GNU LESSER GENERAL PUBLIC LICENSE
                        Version 3, 29 June 2007
 
  Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
@@ -181,6 +172,4 @@ General Public License ever published by the Free Software Foundation.
 whether future versions of the GNU Lesser General Public License shall
 apply, that proxy's public statement of acceptance of any version is
 permanent authorization for you to choose that version for the
-Library.`,
-	}
-}
+Library.

Gopkg.toml

@@ -24,6 +24,11 @@
   branch = "master"
   name = "github.com/docker/distribution"
 
+# v8.0.0 removes juju/ratelimit licensed under LGPL3
+[[constraint]]
+  name = "k8s.io/client-go"
+  revision = "v8.0.0"
+
 [prune]
   go-tests = true
   unused-packages = true

Makefile

@@ -17,15 +17,15 @@ QEMU_VERSION=v2.7.0
 
 # A list of all supported architectures here. Should be named as Go is naming platforms
 # All supported architectures must have an "ifeq" block below that customizes the parameters
-ALL_ARCHITECTURES=amd64 arm arm64 ppc64le
-ML_PLATFORMS=linux/amd64,linux/arm,linux/arm64,linux/ppc64le
+ALL_ARCHITECTURES=amd64 arm arm64 ppc64le s390x
+ML_PLATFORMS=linux/amd64,linux/arm,linux/arm64,linux/ppc64le,linux/s390x
 
 ifeq ($(ARCH),amd64)
 # The architecture to use when downloading the docker binary
 	WEAVEEXEC_DOCKER_ARCH?=x86_64
 
 # The name of the alpine baseimage to use as the base for weave images
-	ALPINE_BASEIMAGE?=alpine:3.6
+	ALPINE_BASEIMAGE?=alpine:3.10
 
 # The extension for the made images
 # Specifying none means for example weaveworks/weave:latest
@@ -42,7 +42,7 @@ ifeq ($(ARCH),arm)
 	WEAVEEXEC_DOCKER_ARCH?=armel
 
 # Using the (semi-)official alpine image
-	ALPINE_BASEIMAGE?=arm32v6/alpine:3.6
+	ALPINE_BASEIMAGE?=arm32v6/alpine:3.8
 
 # arm images have the -arm suffix, for instance weaveworks/weave-arm:latest
 	ARCH_EXT?=-arm
@@ -62,7 +62,7 @@ ifeq ($(ARCH),arm64)
 	WEAVEEXEC_DOCKER_ARCH?=armel
 
 # Using the (semi-)official alpine image
-	ALPINE_BASEIMAGE?=arm64v8/alpine:3.6
+	ALPINE_BASEIMAGE?=arm64v8/alpine:3.8
 
 # arm64 images have the -arm64 suffix, for instance weaveworks/weave-arm64:latest
 	ARCH_EXT?=-arm64
@@ -82,7 +82,7 @@ ifeq ($(ARCH),ppc64le)
 	WEAVEEXEC_DOCKER_ARCH?=ppc64le
 
 # Using the (semi-)official alpine image
-	ALPINE_BASEIMAGE?=ppc64le/alpine:3.6
+	ALPINE_BASEIMAGE?=ppc64le/alpine:3.8
 
 # ppc64le images have the -ppc64le suffix, for instance weaveworks/weave-ppc64le:latest
 	ARCH_EXT?=-ppc64le
@@ -97,6 +97,26 @@ ifeq ($(ARCH),ppc64le)
 # Tell the gcc linker to search for libpcap here
 	CGO_LDFLAGS="-L/usr/local/lib/$(CC)"
 endif
+ifeq ($(ARCH),s390x)
+# The architecture to use when downloading the docker binary
+	WEAVEEXEC_DOCKER_ARCH?=s390x
+
+# Using the (semi-)official alpine image
+	ALPINE_BASEIMAGE?=s390x/alpine:3.6
+
+# s390x images have the -s390x suffix, for instance weaveworks/weave-s390x:latest
+	ARCH_EXT?=-s390x
+
+# The name of the gcc binary
+	CC=s390x-linux-gnu-gcc
+
+# The architecture name to use when downloading a prebuilt QEMU binary
+	QEMUARCH=s390x
+
+# In the weaveworks/build image; libpcap libraries for s390x are placed here
+# Tell the gcc linker to search for libpcap here
+	CGO_LDFLAGS="-L/usr/local/lib/$(CC)"
+endif
 
 # The name of the user that this Makefile should produce image artifacts for. Can/should be overridden
 DOCKERHUB_USER?=weaveworks
@@ -116,8 +136,8 @@ WEAVEWAIT_NOOP_EXE=prog/weavewait/weavewait_noop
 WEAVEWAIT_NOMCAST_EXE=prog/weavewait/weavewait_nomcast
 WEAVEUTIL_EXE=prog/weaveutil/weaveutil
 RUNNER_EXE=tools/runner/runner
-# manifest-tool needs registry credentials; we assume here that the active user has logged in
-MANIFEST_TOOL_CMD=docker run --rm -v $(HOME)/.docker:/.docker weshigbee/manifest-tool --docker-cfg=/.docker
+# Get this from https://github.com/estesp/manifest-tool/releases
+MANIFEST_TOOL_CMD=manifest-tool
 TEST_TLS_EXE=test/tls/tls
 NETWORKTESTER_EXE=test/images/network-tester/webserver
 
@@ -156,18 +176,11 @@ PUSH_ML=push_ml_weave push_ml_weaveexec push_ml_weave-kube push_ml_weave-npc
 
 WEAVE_EXPORT=weave$(ARCH_EXT).tar.gz
 
-NETGO_CHECK=@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
-	rm $@; \
-	echo "\nYour go standard library was built without the 'netgo' build tag."; \
-	echo "To fix that, run"; \
-	echo "    sudo go clean -i net"; \
-	echo "    sudo go install -tags netgo std"; \
-	false; \
-}
 # The flags we are passing to go build. -extldflags -static for making a static binary, 
-# -linkmode external for linking external C libraries into the binary, -X main.version for telling the
-# Go binary which version it is, -tags netgo for enforcing the native Go DNS resolver
-BUILD_FLAGS=-i -ldflags "-linkmode external -extldflags -static -X main.version=$(WEAVE_VERSION)" -tags netgo
+# -X main.version for telling the Go binary which version it is,
+# -s -w drop symbol tables used by debuggers (not Go's internal symbol info)
+# -tags "osusergo netgo" to use native Go UID and DNS implementations
+BUILD_FLAGS=-ldflags "-extldflags -static -X main.version=$(WEAVE_VERSION) -s -w" -tags "osusergo netgo"
 
 PACKAGE_BASE=$(shell go list -e ./)
 
@@ -196,10 +209,8 @@ ifeq ($(BUILD_IN_CONTAINER),true)
 # This make target compiles all binaries inside of the weaveworks/build container
 # It bind-mounts the source into the container and passes all important variables
 exes $(EXES) tests lint: $(BUILD_UPTODATE)
-	@mkdir -p $(shell pwd)/.pkg
 	$(SUDO) docker run $(RM) $(RUN_FLAGS) \
 	    -v $(shell pwd):/go/src/github.com/weaveworks/weave \
-		-v $(shell pwd)/.pkg:/go/pkg \
 		-e GOARCH=$(ARCH) -e CGO_ENABLED=1 -e GOOS=linux -e CIRCLECI -e CIRCLE_BUILD_NUM -e CIRCLE_NODE_TOTAL -e CIRCLE_NODE_INDEX -e COVERDIR -e SLOW -e DEBUG \
 		$(BUILD_IMAGE) COVERAGE=$(COVERAGE) WEAVE_VERSION=$(WEAVE_VERSION) CC=$(CC) QEMUARCH=$(QEMUARCH) CGO_LDFLAGS=$(CGO_LDFLAGS) $@
 	touch $@
@@ -215,19 +226,15 @@ ifeq ($(COVERAGE),true)
 else
 	go build $(BUILD_FLAGS) -o $@ ./$(@D)
 endif
-	$(NETGO_CHECK)
 
 $(WEAVEUTIL_EXE) $(KUBEUTILS_EXE) $(WEAVENPC_EXE) $(NETWORKTESTER_EXE):
 	go build $(BUILD_FLAGS) -o $@ ./$(@D)
-	$(NETGO_CHECK)
 
 $(WEAVEWAIT_EXE):
-	go build $(BUILD_FLAGS) -tags "netgo iface mcast" -o $@ ./$(@D)
-	$(NETGO_CHECK)
+	go build $(BUILD_FLAGS) -tags "osusergo netgo iface mcast" -o $@ ./$(@D)
 
 $(WEAVEWAIT_NOMCAST_EXE):
-	go build $(BUILD_FLAGS) -tags "netgo iface" -o $@ ./$(@D)
-	$(NETGO_CHECK)
+	go build $(BUILD_FLAGS) -tags "osusergo netgo iface" -o $@ ./$(@D)
 
 # These programs need a separate rule as they fail the netgo check in
 # the main build stanza due to not importing net package
@@ -244,8 +251,9 @@ endif
 
 # This rule makes sure the build image is up-to-date.
 # It also makes sure the multiarch hooks are reqistered in the kernel so the QEMU emulation works
-$(BUILD_UPTODATE): build/*
+$(BUILD_UPTODATE): build/* tools/.git
 	$(SUDO) docker build -t $(BUILD_IMAGE) build/
+	$(SUDO) docker tag $(BUILD_IMAGE) $(BUILD_IMAGE):$(shell tools/image-tag)
 	$(SUDO) docker run --rm --privileged multiarch/qemu-user-static:register --reset
 	touch $@
 
@@ -391,15 +399,14 @@ clean-bin:
 	-$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker rmi $(IMAGES)
 	find prog -type f -name "Dockerfile.*" -not -name "Dockerfile.template" -print | xargs rm -f
 	find prog -type f -name "*qemu-*" -print | xargs rm -f
-	rm -rf $(EXES) $(IMAGES_UPTODATE) $(WEAVEDB_UPTODATE) weave*.tar.gz .pkg
+	go clean ./...
+	rm -rf $(EXES) $(IMAGES_UPTODATE) $(WEAVEDB_UPTODATE) weave*.tar.gz .cache
 
 clean: clean-bin clean-work-dir
 	-$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker rmi $(BUILD_IMAGE)
 	rm -rf test/tls/*.pem test/coverage.* test/coverage $(BUILD_UPTODATE)
 
 build:
-	$(SUDO) go clean -i net
-	$(SUDO) go install -tags netgo std
 	$(MAKE)
 
 run-smoketests: all testrunner

README.md

@@ -7,12 +7,11 @@
 
 # About Weaveworks
 
-[Weaveworks](https://www.weave.works) is the company that delivers the most productive way for developers to connect, observe and control
-Docker containers.
+[Weaveworks](https://www.weave.works) is the company that delivers the most productive way for developers to connect, observe and control Docker containers.
 
-This repository contains [Weave Net](https://www.weave.works/products/weave-net/), the first product developed by Weaveworks, and with over 8 million downloads to date, enables you to get started with Docker clusters and portable apps in a fraction of the time compared with other solutions.
+This repository contains [Weave Net](https://www.weave.works/products/weave-net/), the first product developed by Weaveworks, with over 8 million downloads to date. Weave Net enables you to get started with Docker clusters and portable apps in a fraction of the time required by other solutions.
 
-## Weave Cloud: 
+## Weave Cloud:
 
 [Weave Cloud](https://www.weave.works/product/cloud/) is software-as-a-service that simplifies deployment, monitoring and management for containers and microservices. It extends and complements popular orchestrators, and enables developers and DevOps to realize faster deployments, insightful monitoring, visualization and networking.
 
@@ -29,11 +28,10 @@ Services provided by application containers on the Weave network can be exposed
 If you have any questions about, feedback for or a problem with Weave Net:
 
 - Read [the Weave Net docs](https://www.weave.works/docs/net/latest/overview).
-- Invite yourself to the <a href="https://weaveworks.github.io/community-slack/" target="_blank"> #weave-community </a> slack channel.
-- Ask a question on the <a href="https://weave-community.slack.com/messages/general/"> #weave-community</a> slack channel.
+- Invite yourself to the <a href="https://slack.weave.works/" target="_blank">Weave Users Slack</a>.
+- Ask a question on the [#net-dev](https://weave-community.slack.com/messages/net-dev/) slack channel.
 - Join the <a href="https://www.meetup.com/pro/Weave/"> Weave User Group </a> and get invited to online talks, hands-on training and meetups in your area.
-- Send an email to <a href="mailto:weave-users@weave.works">weave-users@weave.works</a>
-- <a href="https://github.com/weaveworks/weave/issues/new">File an issue.</a>
+- [File an issue](https://github.com/weaveworks/weave/issues/new).
 
 Your feedback is always welcome!
 

VENDORED_CODE.md

@@ -0,0 +1,17 @@
+# Use of vendored code in Weave Net
+
+Weave Net is licensed under the [Apache 2.0 license](LICENSE).
+
+Some vendored code is under different licenses though, all of them ship the
+entire license text they are under.
+
+- https://github.com/weaveworks/go-checkpoint
+  can be found in the ./vendor/ directory, is under MPL-2.0.
+
+- Pulled in by dependencies are  
+  https://github.com/hashicorp/golang-lru (MPL-2.0)  
+  https://github.com/hashicorp/go-cleanhttp (MPL-2.0)  
+  https://github.com/opencontainers/go-digest (docs are under CC by-sa 4.0)
+
+[One file used in tests](COPYING.LGPL-3) is under LGPL-3, that's why we ship
+the license text in this repository.

api/router.go

@@ -3,6 +3,7 @@ package api
 import (
 	"fmt"
 	"net"
+	"net/url"
 )
 
 // Expose calls the router to assign the given IP addr to the weave bridge.
@@ -10,3 +11,9 @@ func (client *Client) Expose(ipAddr *net.IPNet) error {
 	_, err := client.httpVerb("POST", fmt.Sprintf("/expose/%s", ipAddr), nil)
 	return err
 }
+
+// ReplacePeers replace the current set of peers
+func (client *Client) ReplacePeers(peers []string) error {
+	_, err := client.httpVerb("POST", "/connect", url.Values{"replace": {"true"}, "peer": peers})
+	return err
+}

bin/circle-dependencies-post

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-set -e
-
-source "$STATE"
-
-if [ -n "$TEST_AND_PUBLISH" ]; then
-    cd "$SRCDIR"
-    git submodule update --init
-    cd "$SRCDIR/build"
-    ../tools/rebuild-image weaveworks/weavebuild . Dockerfile build.sh
-    touch "$SRCDIR/.build.uptodate"
-    sudo chown -R ubuntu /usr/local/go
-    cd "$SRCDIR"
-    make testrunner
-    make COVERAGE=true RM= exes all
-fi

bin/circle-dependencies-post-sources

@@ -1,36 +0,0 @@
-#!/bin/bash
-
-set -e
-
-echo "TEST_AND_PUBLISH=1" >"$STATE"
-
-# Avoid doing a full build if we can.  Note if the PR doesn't exist
-# when the build is kicked off, we can't run this check so we do a
-# full build.  Subsequent pushes to the same branch will have the desired behaviour
-if [ -n "$CI_PULL_REQUEST" -a "$CI_PULL_REQUEST" = "$CI_PULL_REQUESTS" ]; then
-    if PR=$(echo "$CI_PULL_REQUEST" | grep -oP '(?<=^https://github.com/weaveworks/weave/pull/)[0-9]+$'); then
-        echo "GH_PR=$PR" >>"$STATE"
-        if FILES=$(curl -s "https://api.github.com/repos/weaveworks/weave/pulls/$PR/files" | jq -r '.[]|.filename'); then
-            echo "PR touches [$FILES]"
-            if ! echo "$FILES" | grep -v ^site/; then
-                echo "PR contains site changes only - skipping tests"
-                echo "TEST_AND_PUBLISH=" >>"$STATE"
-            else
-                echo "PR contains non-site changes - running tests"
-            fi
-        else
-            echo "Unable to get files for PR $PR" >&2
-        fi
-    else
-        echo "Unable to extract PR number from $CI_PULL_REQUEST" >&2
-    fi
-fi
-
-source "$STATE"
-
-mkdir -p "$(dirname "$SRCDIR")" && cp -r "$(pwd)/" "$SRCDIR"
-
-if [ -n "$TEST_AND_PUBLISH" ]; then
-    cd "$SRCDIR"
-    git submodule update --init
-fi

bin/circle-deploy-master

@@ -2,12 +2,7 @@
 
 set -e
 
-source "$STATE"
-
-if [ -n "$TEST_AND_PUBLISH" ] ; then
-    docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
-    cd $SRCDIR
-    make clean-bin
-    # CircleCI doesn't let us install the qemu module to do ARM docker builds
-    make publish ALL_ARCHITECTURES=amd64 UPDATE_LATEST=latest-only
-fi
+docker login -u $DOCKER_USER -p $DOCKER_PASS
+make clean-bin
+# CircleCI doesn't let us install the qemu module to do ARM docker builds
+make publish ALL_ARCHITECTURES=amd64 UPDATE_LATEST=latest-only SUDO= BUILD_IN_CONTAINER=false

bin/circle-destroy-vms

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e
+
+source "$SRCDIR/bin/circle-env"
+cd "$SRCDIR/test"
+./run-integration-tests.sh destroy

bin/circle-env

@@ -0,0 +1,11 @@
+# Set environment variables here because CircleCI can't expand $FOO in 'env' section
+
+export PATH="$PATH:$HOME/.local/bin:/opt/google-cloud-sdk/bin"
+export CLOUDSDK_CORE_DISABLE_PROMPTS=1
+# Base name of VMs for integration tests:
+export NAME=test-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX
+export TEST_VMS_PROV_AND_CONF_LOCK_FILE=$CIRCLE_ARTIFACTS/test_vms_provisioned_and_configured.txt
+export TEST_VMS_READY_LOCK_FILE=$CIRCLE_ARTIFACTS/test_vms_ready.txt
+# Google Cloud Platorm images for integration tests:
+export USE_IMAGE=1
+export DISK_NAME_PREFIX=test-$CIRCLE_BUILD_NUM-0

bin/circle-teardown-pre

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-set -e
-
-source "$STATE"
-
-if [ -n "$TEST_AND_PUBLISH" ] ; then
-    if [ "$CIRCLE_NODE_INDEX" = "0" ] ; then
-        go get github.com/mattn/goveralls
-        goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/test/profile.cov -service=circleci
-        cd $SRCDIR/test; cp coverage.* $CIRCLE_ARTIFACTS
-    fi
-fi

bin/circle-test-pre

@@ -9,22 +9,21 @@ function signal_failure() {
 }
 trap signal_failure ERR
 
-source "$STATE"
 source "$(dirname "$0")/wait_for.sh"
 
-if [ -n "$TEST_AND_PUBLISH" ]; then
-    [ -n "$SECRET_KEY" ] || {
-        echo "Cannot run smoke tests: no secret key"
-        exit 1
-    }
+[ -n "$SECRET_KEY" ] || {
+    echo "Cannot run smoke tests: no secret key"
+    exit 1
+}
 
-    # Provisioning from image typically take 90 seconds, 
-    # but if we are creating a GCP image, it can take more than 10 minutes:
-    wait_for 900 "$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
+source "$SRCDIR/bin/circle-env"
 
-    # Upload built binaries to testing VMs:
-    cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests.
-    ./run-integration-tests.sh setup >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
-    echo "OK" >"$TEST_VMS_READY_LOCK_FILE"
-    echo "Test VMs now ready. $(date)." >>"$TEST_VMS_SETUP_OUTPUT_FILE"
-fi
+# Provisioning from image typically take 90 seconds, 
+# but if we are creating a GCP image, it can take more than 10 minutes:
+wait_for 900 "$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
+
+# Upload built binaries to testing VMs:
+cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests.
+./run-integration-tests.sh setup
+echo "OK" >"$TEST_VMS_READY_LOCK_FILE"
+echo "Test VMs now ready. $(date)."

bin/circle-test-smoke

@@ -2,22 +2,21 @@
 
 set -e
 
-source "$STATE"
 source "$(dirname "$0")/wait_for.sh"
 
-if [ -n "$TEST_AND_PUBLISH" ]; then
-    [ -n "$SECRET_KEY" ] || {
-        echo "Cannot run smoke tests: no secret key"
-        exit 1
-    }
+[ -n "$SECRET_KEY" ] || {
+    echo "Cannot run smoke tests: no secret key"
+    exit 1
+}
 
-    # Provisioning from image typically take 90 seconds, 
-    # but if we are creating a GCP image, it can take more than 10 minutes:
-    wait_for 900 "$TEST_VMS_READY_LOCK_FILE"
+source "$SRCDIR/bin/circle-env"
 
-    # Run integration tests:
-    export COVERAGE=true
-    export WEAVE_NET_SANITY_CHECKS_FILES="$CIRCLE_ARTIFACTS/weave_net_sanity_check_*.log"
-    cd "$SRCDIR/test" # Ensures we generate code coverage files in the right folder, and use the Terraform state of previously provisionned VMs.
-    ./run-integration-tests.sh test
-fi
+# Provisioning from image typically take 90 seconds, 
+# but if we are creating a GCP image, it can take more than 10 minutes:
+wait_for 900 "$TEST_VMS_READY_LOCK_FILE"
+
+# Run integration tests:
+export COVERAGE=true
+export WEAVE_NET_SANITY_CHECKS_FILES="$CIRCLE_ARTIFACTS/weave_net_sanity_check_*.log"
+cd "$SRCDIR/test" # Ensures we generate code coverage files in the right folder, and use the Terraform state of previously provisionned VMs.
+./run-integration-tests.sh test

bin/circle-test-teardown

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-set -e
-
-source "$STATE"
-
-if [ -n "$TEST_AND_PUBLISH" ] ; then
-    cd $SRCDIR/test
-    test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR/test; ./gen_coverage_reports.sh)
-fi

bin/circle-test-unit

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -e
-
-source "$STATE"
-
-if [ -n "$TEST_AND_PUBLISH" ] ; then
-    cd $SRCDIR
-    make DEBUG=1 lint
-    COVERDIR=test/coverage make RM= tests
-fi

bin/provision_test_vms.sh

@@ -9,35 +9,24 @@ function signal_failure() {
 }
 trap signal_failure ERR
 
-source "$STATE"
-
 function install_terraform() {
-    curl -fsS https://releases.hashicorp.com/terraform/0.8.5/terraform_0.8.5_linux_amd64.zip | gunzip >terraform && chmod +x terraform && sudo mv terraform /usr/bin
+    curl -fsS https://releases.hashicorp.com/terraform/0.11.14/terraform_0.11.14_linux_amd64.zip | gunzip >terraform && chmod +x terraform && sudo mv terraform /usr/bin
 }
 
-function install_ansible() {
-    sudo apt-get update || true
-    sudo apt-get install -qq -y python-pip python-dev libffi-dev libssl-dev \
-        && pip install --user -U setuptools cffi \
-        && pip install --user ansible
-    export PATH="$PATH:$HOME/.local/bin"
+[ -n "$SECRET_KEY" ] || {
+    echo "Cannot run smoke tests: no secret key"
+    exit 1
 }
 
-if [ -n "$TEST_AND_PUBLISH" ]; then
-    [ -n "$SECRET_KEY" ] || {
-        echo "Cannot run smoke tests: no secret key"
-        exit 1
-    }
+source "$SRCDIR/bin/circle-env"
 
-    install_terraform >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
-    install_ansible >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
+install_terraform
 
-    # Only attempt to create GCP image in first container, wait for it to be created otherwise:
-    [ "$CIRCLE_NODE_INDEX" != "0" ] && export CREATE_IMAGE=0
+# Only attempt to create GCP image in first container, wait for it to be created otherwise:
+[ "$CIRCLE_NODE_INDEX" != "0" ] && export CREATE_IMAGE=0
 
-    # Provision and configure testing VMs:
-    cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests.
-    ./run-integration-tests.sh configure >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
-    echo "OK" >"$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
-    echo "Test VMs now provisioned and configured. $(date)." >>"$TEST_VMS_SETUP_OUTPUT_FILE"
-fi
+# Provision and configure testing VMs:
+cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests.
+./run-integration-tests.sh configure
+echo "OK" >"$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
+echo "Test VMs now provisioned and configured. $(date)."

bin/release

@@ -89,10 +89,11 @@ build() {
     ## Inject the version numbers and build the distributables
     ## (library versions?)
     sed -i.bak "/SCRIPT_VERSION=/ c\SCRIPT_VERSION=\"$VERSION\"" ./weave
-    sed -i.bak -e "s/:latest/:$VERSION/" -e "/imagePullPolicy: Always/d" ./prog/weave-kube/weave-daemonset.yaml
     sed -i.bak -e "s/:latest/:$VERSION/" -e "/imagePullPolicy: Always/d" ./prog/weave-kube/weave-daemonset-k8s-1.6.yaml
     sed -i.bak -e "s/:latest/:$VERSION/" -e "/imagePullPolicy: Always/d" ./prog/weave-kube/weave-daemonset-k8s-1.7.yaml
     sed -i.bak -e "s/:latest/:$VERSION/" -e "/imagePullPolicy: Always/d" ./prog/weave-kube/weave-daemonset-k8s-1.8.yaml
+    sed -i.bak -e "s/:latest/:$VERSION/" -e "/imagePullPolicy: Always/d" ./prog/weave-kube/weave-daemonset-k8s-1.9.yaml
+    sed -i.bak -e "s/:latest/:$VERSION/" -e "/imagePullPolicy: Always/d" ./prog/weave-kube/weave-daemonset-k8s-1.11.yaml
     make SUDO=$SUDO WEAVE_VERSION=$VERSION DOCKERHUB_USER=$DOCKERHUB_USER
 
     if make tests; then
@@ -161,13 +162,6 @@ draft() {
         --name "weave" \
         --file "./weave"
 
-    github-release upload \
-        --user $GITHUB_USER \
-        --repo weave \
-        --tag $LATEST_TAG \
-        --name "weave-daemonset.yaml" \
-        --file "./prog/weave-kube/weave-daemonset.yaml"
-
     github-release upload \
         --user $GITHUB_USER \
         --repo weave \
@@ -188,7 +182,21 @@ draft() {
         --tag $LATEST_TAG \
         --name "weave-daemonset-k8s-1.8.yaml" \
         --file "./prog/weave-kube/weave-daemonset-k8s-1.8.yaml"
-    
+
+    github-release upload \
+        --user $GITHUB_USER \
+        --repo weave \
+        --tag $LATEST_TAG \
+        --name "weave-daemonset-k8s-1.9.yaml" \
+        --file "./prog/weave-kube/weave-daemonset-k8s-1.9.yaml"
+ 
+    github-release upload \
+        --user $GITHUB_USER \
+        --repo weave \
+        --tag $LATEST_TAG \
+        --name "weave-daemonset-k8s-1.11.yaml" \
+        --file "./prog/weave-kube/weave-daemonset-k8s-1.11.yaml"
+
     echo "** Draft $TYPE $RELEASE_NAME $VERSION created at"
     echo -e "\thttps://github.com/$GITHUB_USER/weave/releases/$LATEST_TAG"
 }
@@ -276,13 +284,6 @@ publish() {
             --name "weave" \
             --file "./weave"
 
-        github-release upload \
-            --user $GITHUB_USER \
-            --repo weave \
-            --tag latest_release \
-            --name "weave-daemonset.yaml" \
-            --file "./prog/weave-kube/weave-daemonset.yaml"
-
         github-release upload \
             --user $GITHUB_USER \
             --repo weave \
@@ -304,6 +305,20 @@ publish() {
             --name "weave-daemonset-k8s-1.8.yaml" \
             --file "./prog/weave-kube/weave-daemonset-k8s-1.8.yaml"
 
+         github-release upload \
+            --user $GITHUB_USER \
+            --repo weave \
+            --tag latest_release \
+            --name "weave-daemonset-k8s-1.9.yaml" \
+            --file "./prog/weave-kube/weave-daemonset-k8s-1.9.yaml"
+
+         github-release upload \
+            --user $GITHUB_USER \
+            --repo weave \
+            --tag latest_release \
+            --name "weave-daemonset-k8s-1.11.yaml" \
+            --file "./prog/weave-kube/weave-daemonset-k8s-1.11.yaml"
+
         echo "** Release $RELEASE_NAME $VERSION published at"
         echo -e "\thttps://github.com/$GITHUB_USER/weave/releases/$LATEST_TAG"
         echo -e "\thttps://github.com/$GITHUB_USER/weave/releases/latest_release"

bin/wait_for.sh

@@ -16,6 +16,6 @@ wait_for() {
         if ! ((i % 10)); then echo "[$i seconds]: Waiting for $lock_file to be created..."; fi
         sleep 1
     done
-    echo "Timed out waiting for test VMs to be ready. See details in: $TEST_VMS_SETUP_OUTPUT_FILE" >&2
+    echo "Timed out waiting for test VMs to be ready." >&2
     return 1
 }

build/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.10.1-stretch
+FROM golang:1.13.3-buster
 
 # Support Raspberry Pi 2 and newer
 ENV GOARM 7
@@ -7,25 +7,25 @@ ENV GOARM 7
 ENV DEB_CROSSPLATFORMS \
 	armhf \
 	arm64 \
-	ppc64el
+	ppc64el \
+	s390x
 
 # The go platforms we are supporting to cross-compile
 ENV GO_CROSSPLATFORMS \
 	linux/arm \
 	linux/arm64 \
-	linux/ppc64le
+	linux/ppc64le \
+	linux/s390x
 
 # The names of the gcc cross-compilers we have installed
 ENV GCC_CROSSCOMPILERS \
 	arm-linux-gnueabihf \
 	aarch64-linux-gnu \
-	powerpc64le-linux-gnu
+	powerpc64le-linux-gnu \
+	s390x-linux-gnu
 
-# Temporarily add the Debian repositories, because we're going to install gcc cross-compilers from there
 # Install the build-essential and crossbuild-essential-ARCH packages
-RUN echo "deb http://emdebian.org/tools/debian/ jessie main" > /etc/apt/sources.list.d/cgocrosscompiling.list \
-  && curl http://emdebian.org/tools/debian/emdebian-toolchain-archive.key | apt-key add - \
-  && for platform in ${DEB_CROSSPLATFORMS}; do dpkg --add-architecture $platform; done \
+RUN for platform in ${DEB_CROSSPLATFORMS}; do dpkg --add-architecture $platform; done \
   && apt-get update \
   && apt-get install -y build-essential \
   && for platform in ${DEB_CROSSPLATFORMS}; do apt-get install -y crossbuild-essential-${platform}; done \
@@ -50,31 +50,20 @@ RUN curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt
 
 # Install common Go tools
 RUN go get \
-	github.com/golang/lint/golint \
+    github.com/weaveworks/build-tools/cover \
+    github.com/mattn/goveralls \
+	golang.org/x/lint/golint \
 	github.com/fzipp/gocyclo \
 	github.com/fatih/hclfmt \
 	github.com/client9/misspell/cmd/misspell
 
-# First clean, because we're gonna rebuild the std both with -race enabled and disabled
-# Running go install std twice is intentional and required, as content is written in two different directories
-# (/usr/local/go/pkg/linux_amd64 vs /usr/local/go/pkg/linux_amd64_race)
-# The other architectures do not have support for -race
-RUN go clean -i net \
-	&& go install -tags netgo std \
-	&& go install -race -tags netgo std
-
-# Prebuild std for the other architectures
-RUN for platform in ${GO_CROSSPLATFORMS}; do \
-		GOOS=${platform%/*} GOARCH=${platform##*/} \
-			go install -tags netgo std; done
-
 # Allow full write access to the Go folders for anyone
 RUN chmod -R a+w /usr/local/go
 
 # The libpcap version from Debian packages is 1.6.2, matching that version here although newer versions of libpcap have been released
 # We have to cross-compile the libpcap library for the new architectures, that's what we're doing here
 ENV LIBPCAP_CROSS_VERSION=1.6.2
-RUN curl -sSL http://www.tcpdump.org/release/libpcap-${LIBPCAP_CROSS_VERSION}.tar.gz | tar -xz \
+RUN curl -sSL https://www.tcpdump.org/release/libpcap-${LIBPCAP_CROSS_VERSION}.tar.gz | tar -xz \
 	&& cd libpcap-${LIBPCAP_CROSS_VERSION} \
 	&& for crosscompiler in ${GCC_CROSSCOMPILERS}; do \
 		CC=${crosscompiler}-gcc ac_cv_linux_vers=2 ./configure --host=${crosscompiler} --with-pcap=linux \
@@ -86,5 +75,8 @@ RUN curl -sSL http://www.tcpdump.org/release/libpcap-${LIBPCAP_CROSS_VERSION}.ta
 		&& ln -sf libpcap.so.${MAJOR_VER} /usr/local/lib/libpcap.so \
 		&& make clean; done
 
+# Install Docker
+RUN curl -fsSL https://get.docker.com | VERSION=18.06.0-ce /bin/sh
+
 COPY build.sh /
 ENTRYPOINT ["sh", "/build.sh"]

build/shasums/qemu-s390x-static.sha256sum

@@ -0,0 +1 @@
+bab3b4d04263479b3d9f85cf59bd0cad1f2b434f29856b2204f8288f2f3598e4  qemu-s390x-static

circle.yml

@@ -1,66 +0,0 @@
-general:
-  branches:
-    ignore:
-      - gh\-pages
-
-machine:
-  pre:
-    - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0
-  services:
-    - docker
-  environment:
-    GOPATH: /home/ubuntu:$GOPATH
-    SRCDIR: /home/ubuntu/src/github.com/weaveworks/weave
-    PATH: $PATH:$HOME/.local/bin:/opt/google-cloud-sdk/bin
-    CLOUDSDK_CORE_DISABLE_PROMPTS: 1
-    STATE: /home/ubuntu/state.env
-    # Base name of VMs for integration tests:
-    NAME: test-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX
-    TEST_VMS_SETUP_OUTPUT_FILE: $CIRCLE_ARTIFACTS/test_vms_setup_output.txt
-    TEST_VMS_PROV_AND_CONF_LOCK_FILE: $CIRCLE_ARTIFACTS/test_vms_provisioned_and_configured.txt
-    TEST_VMS_READY_LOCK_FILE: $CIRCLE_ARTIFACTS/test_vms_ready.txt
-    # Google Cloud Platorm images for integration tests:
-    USE_IMAGE: 1
-    DISK_NAME_PREFIX: test-$CIRCLE_BUILD_NUM-0
-
-dependencies:
-  cache_directories:
-    - "~/docker"
-  post:
-    - bin/circle-dependencies-post-sources
-    # Provision & configure testing VMs:
-    - "while [ ! -d $SRCDIR ]; do sleep 1; done; $SRCDIR/bin/provision_test_vms.sh":
-        parallel: true
-        background: true
-    - bin/circle-dependencies-post
-
-test:
-  pre:
-    - "$SRCDIR/bin/circle-test-pre":
-        parallel: true
-        background: true
-  override:
-    - "$SRCDIR/bin/circle-test-unit":
-        parallel: true
-    - "$SRCDIR/bin/circle-test-smoke":
-        parallel: true
-        timeout: 360
-  post:
-    - "$SRCDIR/bin/circle-test-teardown":
-        parallel: true
-
-teardown:
-  pre:
-    # Destroy testing VMs:
-    - "cd $SRCDIR/test && ./run-integration-tests.sh destroy":
-        parallel: true
-        background: true
-    - "$SRCDIR/bin/circle-teardown-pre":
-        parallel: true
-
-deployment:
-  master:
-    branch: master
-    owner: weaveworks
-    commands:
-      - "$SRCDIR/bin/circle-deploy-master"

common/docker/client.go

@@ -99,7 +99,9 @@ func (c *Client) AddObserver(ob ContainerObserver) error {
 		pending := make(pendingStarts)
 		retryInterval := InitialInterval
 		for {
-			events := make(chan *docker.APIEvents)
+			// Use a buffered chan so the client library can run ahead of the listener
+			// - Docker will drop an event if it is not collected quickly enough.
+			events := make(chan *docker.APIEvents, 1024)
 			if err := c.AddEventListener(events); err != nil {
 				c.errorf("Unable to add listener to Docker API: %s - retrying in %ds", err, retryInterval/time.Second)
 			} else {

common/logging.go

@@ -54,6 +54,12 @@ func SetLogLevel(levelname string) {
 	Log.Level = level
 }
 
+func CheckError(e error) {
+	if e != nil {
+		Log.Error(e)
+	}
+}
+
 func CheckFatal(e error) {
 	if e != nil {
 		Log.Fatal(e)

docs/entrypoints.md

@@ -35,21 +35,21 @@ weave-kube is an image with the weave binaries and a wrapper script installed. I
 
 1. Setting up the weave network
 2. Connecting to peers
-3. Copy the weave CNI-compatible binary plugin `weaveutil` to `/opt/cni/bin/weave-net` and weave config in `/etc/cni/net.d/10-weave.conf`
+3. Copy the weave CNI-compatible binary plugin `weaveutil` to `/opt/cni/bin/weave-net` and weave config in `/etc/cni/net.d/10-weave.conflist`
 4. Running the weave network policy controller (weave-npc) to implement kubernetes' `NetworkPolicy`
 
 Once installation is complete, each network-relevant change in a container leads `kubelet` to:
 
 1. Set certain environment variables to configure the CNI plugin, mostly related to the container ID
 2. Launch `/opt/cni/bin/weave-net`
-3. Pass the CNI config - the contents of `/etc/cni/net.d/10-weave.conf` to `weave-net`
+3. Pass the CNI config - the contents of `/etc/cni/net.d/10-weave.conflist` to `weave-net`
 
 Thus, when each container is changed, and `kubelet` calls weave as a CNI plugin, it really just is launching `weaveutil` as `weave-net`.
 
 The installation and setup of all of the above - and therefore the entrypoint to the weave-kube image - is the script `prog/weave-kube/launch.sh`. `launch.sh` does the following:
 
 1. Read configuration variables from the environment. When the documentation for `weave-kube` describes configuring the weave network by changing the environment variables in the daemonset in the `.yml` file, `launch.sh` reads these environment variables.
-2. Set up the config file at `/etc/cni/net.d/10-weave.conf`
+2. Set up the config file at `/etc/cni/net.d/10-weave.conflist`
 3. Run the `weave` initialization script
 4. Run `weaver` with the correct configuration passed as command-line options
 
@@ -64,7 +64,7 @@ To add new options to how weave should run with each invocation, you would do th
 5. Add a command-line option `--option` to `weaveutil` or `weaver` as appropriate.
 6. If the option can be configured for CNI:
     * have `prog/weave-kube/launch.sh` read it as an environment variable and set inside
-    * set a default for the environment variable in `prog/weave-kube/weave-daemonset-k8s-1.6.yaml` and `weave-daemonset.yaml`
+    * set a default for the environment variable in `prog/weave-kube/weave-daemonset-k8s-N.N.yaml`
     * if it should be set via `weaver` globally on its one-time initialization invocation, pass it on in `launch.sh`
-    * if it needs to be set via `weaveutil` on each invocation of `weave-net`, have `launch.sh` save it as an option in the CNI config file `/etc/cni/net.d/10-weave.conf` and then have the CNI code in `plugin/net/cni.go` in `weaveutil` read it and use where appropriate
+    * if it needs to be set via `weaveutil` on each invocation of `weave-net`, have `launch.sh` save it as an option in the CNI config file `/etc/cni/net.d/10-weave.conflist` and then have the CNI code in `plugin/net/cni.go` in `weaveutil` read it and use where appropriate
 7. Document it!

docs/k8s-src-ip.md

@@ -4,7 +4,7 @@
 
 * Weave Net: v2.2.1.
 * Kubernetes: v1.10.0.
-* kube-proxy: iptables mode (default kubeadm options).
+* kube-proxy: iptables/ipvs mode with `--masquerade-all=false` and `--cluster-cidr` unspecified(default kubeadm options).
 * Hosts: `h1` and `h2`.
 
 ## Source IP
@@ -14,7 +14,7 @@
 1  | Pod_h1      | ip(Pod_h2)        | Pod_h2 | ip(Pod_h1)       | OK
 2  | h1          | ClusterIP         | Pod_h1 | ip(weave_h1)     | **NOK**
 3  | h1          | ClusterIP         | Pod_h2 | ip(weave_h1)     | OK
-4  | Pod_h1      | ClusterIP         | Pod_h1 | ip(weave_h1)     | **NOK**
+4  | Pod_h1      | ClusterIP         | Pod_h1 | ip(Pod_h1)       | OK
 5  | Pod_h1      | ClusterIP         | Pod_h2 | ip(weave_h1)     | OK
 6  | h1          | ip(h1):NodePort   | Pod_h1 | ip(weave_h1)     | **NOK**
 7  | h1          | ip(h1):NodePort   | Pod_h2 | ip(weave_h1)     | OK

docs/release-process.md

@@ -3,6 +3,7 @@
 
 * Up-to-date versions of `git`, `go` etc
 * Install GitHub release tool `go get github.com/weaveworks/github-release`
+* Install manifest tool `go get github.com/estesp/manifest-tool`
 * Create a [github token for
   github-release](https://help.github.com/articles/creating-an-access-token-for-command-line-use/); select the `repo` OAuth scope; set and export `$GITHUB_TOKEN` with this value
 
@@ -135,7 +136,7 @@ Now, you can validate whether images were published for all platforms:
     # Published platforms per image:
     for img in $(grep '^PUBLISH=' Makefile); do
         img="weaveworks/$(echo $img | cut -d_ -f2):${TAG#v}"
-        platforms=$(vendor/github.com/estesp/manifest-tool/manifest-tool \
+        platforms=$(manifest-tool \
                 inspect --raw "$img" | \
                 jq '.[].Platform | .os + "/" +.architecture' | \
                 tr '\n' ' ')
@@ -154,6 +155,7 @@ Weave Net is [available in Docker Store](https://store.docker.com/plugins/weave-
 * Go to "_Plans & Pricing_" > "_Free Tier_".
 * Under "_Source Repositories & Tags_", click on "_Add Source_", select "_net-plugin_" under "_Repository_" and the version to release under "_Tag_".
 * Under "_Resources_" > "_Installation Instructions_", update the lines containing `export NET_VERSION=<version>`.
+* Click "_Save_"
 * Click "_Submit For Review_". You should see a message like: "_Your product has been submitted for approval. [...] We'll be in touch with next steps soon!_".
 * You should receive an email saying:
 
@@ -177,6 +179,8 @@ Weave Net is [available in Docker Store](https://store.docker.com/plugins/weave-
   https://checkpoint-api.weave.works/admin
 * File a PR to update the version of the daemonset at
   https://github.com/kubernetes/kops/tree/master/upup/models/cloudup/resources/addons/networking.weave
+  and kops/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go
+  and kops/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml
 
 ## Troubleshooting
 

docs/weavenpc-design.md

@@ -9,7 +9,9 @@ destroyed automatically in response to Pod, Namespace and
 NetworkPolicy object updates from the k8s API server:
 
 * A `hash:ip` set per namespace, containing the IP addresses of all
-  pods in that namespace
+  pods in that namespace for which default ingress is allowed
+* A `hash:ip` set per namespace, containing the IP addresses of all
+  pods in that namespace for which default egress is allowed  
 * A `list:set` per distinct (across all network policies in all
   namespaces) namespace selector mentioned in a network policy,
   containing the names of any of the above hash:ip sets whose
@@ -18,6 +20,9 @@ NetworkPolicy object updates from the k8s API server:
   containing network policy's namespace) pod selector mentioned in a
   network policy, containing the IP addresses of all pods in the
   namespace whose labels match that selector
+* A `hash:net` set for each distinct (within the scope of the
+  containing network policy's namespace) `except` list of CIDR's mentioned in
+  the network policies
 
 IPsets are implemented by the kernel module `xt_set`, without which
 weave-npc will not work.
@@ -38,11 +43,21 @@ hashing to avoid clashes.
 
 # iptables chains
 
-The policy controller maintains two iptables chains in response to
-changes to pods, namespaces and network policies. One chain contains
-the ingress rules that implement the network policy specifications,
-and the other is used to bypass the ingress rules for namespaces which
-have an ingress isolation policy of `DefaultAllow`.
+The policy controller maintains several iptables chains in response to
+changes to pods, namespaces and network policies. 
+
+## Static `WEAVE-NPC` chain
+
+`WEAVE-NPC` chain contains static rules to ACCEPT traffic that is `RELATED,ESTABLISHED`
+and run `NEW` traffic through `WEAVE-NPC-DEFAULT` followed by `WEAVE-NPC-INGRESS` chains 
+
+Static configuration:
+
+```
+iptables -A WEAVE-NPC -m state --state RELATED,ESTABLISHED -j ACCEPT
+iptables -A WEAVE-NPC -m state --state NEW -j WEAVE-NPC-DEFAULT
+iptables -A WEAVE-NPC -m state --state NEW -j WEAVE-NPC-INGRESS
+```
 
 ## Dynamically maintained `WEAVE-NPC-DEFAULT` chain
 
@@ -63,23 +78,58 @@ For each namespace network policy ingress rule peer/port combination:
 iptables -A WEAVE-NPC-INGRESS -p $PROTO [-m set --match-set $SRCSET] -m set --match-set $DSTSET --dport $DPORT -j ACCEPT
 ```
 
-## Static `WEAVE-NPC` chain
+## Static `WEAVE-NPC-EGRESS` chain
+
+`WEAVE-NPC-EGRESS` chain contains static rules to ACCEPT traffic that is `RELATED,ESTABLISHED`
+and run `NEW` traffic through `WEAVE-NPC-EGRESS-DEFAULT` followed by `WEAVE-NPC-EGRESS-CUSTOM` chains 
 
 Static configuration:
 
 ```
-iptables -A WEAVE-NPC -m state --state RELATED,ESTABLISHED -j ACCEPT
-iptables -A WEAVE-NPC -m state --state NEW -j WEAVE-NPC-DEFAULT
-iptables -A WEAVE-NPC -m state --state NEW -j WEAVE-NPC-INGRESS
+iptables -A WEAVE-NPC-EGRESS -m state --state RELATED,ESTABLISHED -j ACCEPT
+iptables -A WEAVE-NPC-EGRESS -m state --state NEW -j WEAVE-NPC-EGRESS-DEFAULT
+iptables -A WEAVE-NPC-EGRESS -m state --state NEW -m mark ! --mark 0x40000/0x40000 -j WEAVE-NPC-EGRESS-CUSTOM
+iptables -A WEAVE-NPC-EGRESS -m mark ! --mark 0x40000/0x40000 -j DROP
 ```
 
+## Dynamically maintained `WEAVE-NPC-EGRESS-DEFAULT` chain
+
+The policy controller maintains a rule in this chain for every
+namespace whose egress isolation policy is `DefaultAllow`. The
+purpose of this rule is simply to ACCEPT any traffic originating from such namespace before it reaches the egress chain.
+
+```
+iptables -A WEAVE-NPC-EGRESS-DEFAULT -m set --match-set $NSIPSET src -j WEAVE-NPC-EGRESS-ACCEPT
+iptables -A WEAVE-NPC-EGRESS-DEFAULT -m set --match-set $NSIPSET src -j RETURN
+```
+
+## Static `WEAVE-NPC-EGRESS-ACCEPT` chain
+
+`WEAVE-NPC-EGRESS-ACCEPTS` chain contains static rules to mark traffic
+
+Static configuration:
+
+```
+iptables -A WEAVE-NPC-EGRESS-ACCEPT -j MARK --set-xmark 0x40000/0x40000
+```
+
+## Dynamically maintained `WEAVE-NPC-EGRESS-CUSTOM` chain
+
+For each namespace network policy egress rule peer/port combination:
+
+```
+iptables -A WEAVE-NPC-EGRESS-CUSTOM -p $PROTO [-m set --match-set $SRCSET] -m set --match-set $DSTSET --dport $DPORT -j ACCEPT
+```
+
+
 # Steering traffic into the policy engine
 
 To direct traffic into the policy engine:
 
 ```
-iptables -A FORWARD -o weave -m physdev ! --physdev-out vethwe-bridge -j WEAVE-NPC
-iptables -A FORWARD -o weave -m physdev ! --physdev-out vethwe-bridge -j DROP
+iptables -A INPUT -i weave -j WEAVE-NPC-EGRESS
+iptables -A FORWARD -i weave -j WEAVE-NPC-EGRESS
+iptables -A FORWARD -o weave -j WEAVE-NPC
 ```
 
 Note this only affects traffic which egresses the bridge on a physical

ipam/allocator.go

@@ -275,7 +275,7 @@ func (alloc *Allocator) spaceRequestDenied(sender mesh.PeerName, r address.Range
 	for i := 0; i < len(alloc.pendingClaims); {
 		claim := alloc.pendingClaims[i].(*claim)
 		if r.Contains(claim.cidr.Addr) {
-			claim.deniedBy(alloc, sender)
+			claim.deniedBy(alloc, sender, nil)
 			alloc.pendingClaims = append(alloc.pendingClaims[:i], alloc.pendingClaims[i+1:]...)
 			continue
 		}
@@ -814,6 +814,16 @@ func (alloc *Allocator) sendRingUpdate(dest mesh.PeerName) {
 	alloc.gossip.GossipUnicast(dest, msg)
 }
 
+func (alloc *Allocator) hasAllocations(rs []address.Range) bool {
+	for _, r := range rs {
+		alloc.debugf("hasAllocations: %v %v vs %v", r, alloc.space.NumFreeAddressesInRange(r), r.Size())
+		if alloc.space.NumFreeAddressesInRange(r) != r.Size() {
+			return true
+		}
+	}
+	return false
+}
+
 func (alloc *Allocator) update(sender mesh.PeerName, msg []byte) error {
 	reader := bytes.NewReader(msg)
 	decoder := gob.NewDecoder(reader)
@@ -832,7 +842,7 @@ func (alloc *Allocator) update(sender mesh.PeerName, msg []byte) error {
 	// If someone sent us a ring, merge it into ours. Note this will move us
 	// out of the awaiting-consensus state if we didn't have a ring already.
 	case data.Ring != nil:
-		updated, err := alloc.ring.Merge(*data.Ring)
+		updated, err := alloc.ring.Merge(*data.Ring, alloc.hasAllocations)
 		switch err {
 		case nil:
 			if updated {
@@ -910,6 +920,7 @@ func (alloc *Allocator) donateSpace(r address.Range, to mesh.PeerName) {
 	alloc.debugln("Giving range", chunk, "to", to)
 	alloc.ring.GrantRangeToHost(chunk.Start, chunk.End, to)
 	alloc.persistRing()
+	alloc.gossip.GossipBroadcast(alloc.Gossip())
 }
 
 func (alloc *Allocator) assertInvariants() {
@@ -985,7 +996,7 @@ func (alloc *Allocator) loadPersistedData() bool {
 		alloc.persistOwned()
 	}
 
-	if !nameFound || !ringFound {
+	if !nameFound || !ringFound || persistedRing == nil || persistedRing.Empty() {
 		overwritePersisted("No valid persisted data")
 		return false
 	}

ipam/claim.go

@@ -12,6 +12,7 @@ import (
 
 type claim struct {
 	resultChan       chan<- error
+	tryCount         int
 	ident            string       // a container ID, something like "weave:expose", or api.NoContainerID
 	cidr             address.CIDR // single address being claimed
 	isContainer      bool         // true if ident is a container ID
@@ -19,6 +20,8 @@ type claim struct {
 	hasBeenCancelled func() bool
 }
 
+const maxTryCount = 5
+
 // Send an error (or nil for success) back to caller listening on resultChan
 func (c *claim) sendResult(result error) {
 	// Make sure we only send a result once, since listener stops listening after that
@@ -40,6 +43,8 @@ func (c *claim) Try(alloc *Allocator) bool {
 		return true
 	}
 
+	c.tryCount++
+
 	addOwned := func() {
 		if c.ident == api.NoContainerID {
 			alloc.addOwned(c.cidr.Addr.String(), c.cidr, c.isContainer)
@@ -90,8 +95,10 @@ func (c *claim) Try(alloc *Allocator) bool {
 			if c.noErrorOnUnknown {
 				alloc.infof("Claim %s for %s: %s; will try later.", c.cidr, c.ident, err)
 				c.sendResult(nil)
-			} else { // just tell the user they can't do this.
-				c.deniedBy(alloc, owner)
+			} else if c.tryCount > maxTryCount {
+				// give up, tell the user they can't do this
+				c.deniedBy(alloc, owner, err)
+				return true
 			}
 		}
 		return false
@@ -128,12 +135,16 @@ func (c *claim) Try(alloc *Allocator) bool {
 	return true
 }
 
-func (c *claim) deniedBy(alloc *Allocator, owner mesh.PeerName) {
+func (c *claim) deniedBy(alloc *Allocator, owner mesh.PeerName, err error) {
 	name, found := alloc.nicknames[owner]
 	if found {
 		name = " (" + name + ")"
 	}
-	c.sendResult(fmt.Errorf("address %s is owned by other peer %s%s", c.cidr.String(), owner, name))
+	reason := ""
+	if err != nil {
+		reason = fmt.Sprintf(" - %s", err)
+	}
+	c.sendResult(fmt.Errorf("address %s is owned by other peer %s%s%s", c.cidr.String(), owner, name, reason))
 }
 
 func (c *claim) Cancel() {

ipam/http.go

@@ -199,6 +199,10 @@ func (alloc *Allocator) HandleHTTP(router *mux.Router, defaultSubnet address.CID
 	})
 
 	router.Methods("GET").Path("/ipinfo/tracker").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintf(w, alloc.tracker.String())
+		tracker := ""
+		if alloc.tracker != nil {
+			tracker = alloc.tracker.String()
+		}
+		fmt.Fprintf(w, tracker)
 	})
 }

ipam/ring/entry.go

@@ -1,6 +1,7 @@
 package ring
 
 import (
+	"fmt"
 	"sort"
 
 	"github.com/weaveworks/mesh"
@@ -25,6 +26,7 @@ func (e *entry) Equal(e2 *entry) bool {
 func (e *entry) update(peername mesh.PeerName, free address.Count) {
 	e.Peer = peername
 	e.Version++
+	common.Log.Debugln(fmt.Sprintf("Updating ring token=%s current peer=%s updated peer=%s new version=%s", e.Token, e.Peer, peername, fmt.Sprint(e.Version)))
 	e.Free = free
 }
 

ipam/ring/ring.go

@@ -213,7 +213,7 @@ func (r *Ring) GrantRangeToHost(start, end address.Address, peer mesh.PeerName)
 
 // Merge the given ring into this ring and indicate whether this ring
 // got updated as a result.
-func (r *Ring) Merge(gossip Ring) (bool, error) {
+func (r *Ring) Merge(gossip Ring, hasAllocations func(r []address.Range) bool) (bool, error) {
 	r.assertInvariants()
 	defer r.trackUpdates()()
 
@@ -238,12 +238,22 @@ func (r *Ring) Merge(gossip Ring) (bool, error) {
 		return false, ErrDifferentRange
 	}
 
-	result, updated, err := r.Entries.merge(gossip.Entries, r.Peer)
+	result, updated, err := r.Entries.merge(gossip.Entries, r.Peer, r, hasAllocations)
 
 	if err != nil {
 		return false, err
 	}
 
+	// reset the free space for entries if there is invalid free space
+	// due to accepting an unexpected update from the peers
+	for i := 0; i < len(result); i++ {
+		distance := r.distance(result.entry(i).Token, result.entry(i+1).Token)
+		if result.entry(i).Peer == r.Peer && result.entry(i).Free > distance {
+			// case that can arise when a range that we own that got split and had no allocations
+			result.entry(i).Free = distance
+		}
+	}
+
 	if err := r.checkEntries(result); err != nil {
 		return false, fmt.Errorf("Merge of incoming data causes: %s", err)
 	}
@@ -260,7 +270,7 @@ func (r *Ring) Merge(gossip Ring) (bool, error) {
 // entries belonging to ourPeer. Returns the merged entries and an
 // indication whether the merge resulted in any changes, i.e. the
 // result differs from the original.
-func (es entries) merge(other entries, ourPeer mesh.PeerName) (result entries, updated bool, err error) {
+func (es entries) merge(other entries, ourPeer mesh.PeerName, r *Ring, hasAllocations func(r []address.Range) bool) (result entries, updated bool, err error) {
 	var mine, theirs *entry
 	var previousOwner *mesh.PeerName
 	addToResult := func(e entry) { result = append(result, &e) }
@@ -274,6 +284,7 @@ func (es entries) merge(other entries, ourPeer mesh.PeerName) (result entries, u
 	var i, j int
 	for i < len(es) && j < len(other) {
 		mine, theirs = es[i], other[j]
+		common.Log.Debugf("[ring %s]: Merge mine.Token=%s theirs.Token=%s mine.Peer=%s theirs.Peer=%s mine.Version=%v theirs.Version=%v", ourPeer, mine.Token, theirs.Token, mine.Peer, theirs.Peer, mine.Version, theirs.Version)
 		switch {
 		case mine.Token < theirs.Token:
 			addToResult(*mine)
@@ -282,8 +293,11 @@ func (es entries) merge(other entries, ourPeer mesh.PeerName) (result entries, u
 		case mine.Token > theirs.Token:
 			// insert, checking that a range owned by us hasn't been split
 			if previousOwner != nil && *previousOwner == ourPeer && theirs.Peer != ourPeer {
-				err = errEntryInMyRange(theirs)
-				return
+				// check we have no allocations in the range that got split
+				if hasAllocations(r.makeRanges(theirs.Token, mine.Token)) {
+					err = errEntryInMyRange(theirs)
+					return
+				}
 			}
 			addTheirs(*theirs)
 			j++
@@ -292,17 +306,44 @@ func (es entries) merge(other entries, ourPeer mesh.PeerName) (result entries, u
 			switch {
 			case mine.Version >= theirs.Version:
 				if mine.Version == theirs.Version && !mine.Equal(theirs) {
-					err = errInconsistentEntry(mine, theirs)
-					return
+					if mine.Peer == ourPeer {
+						// if we own the entry and has allocations
+						if hasAllocations(r.makeRanges(theirs.Token, es.entry(i+1).Token)) {
+							err = errInconsistentEntry(mine, theirs)
+							return
+						}
+					}
+					// tie-break here, pick the entry with the highest free count
+					if mine.Free >= theirs.Free {
+						addToResult(*mine)
+						previousOwner = &mine.Peer
+					} else {
+						addTheirs(*theirs)
+					}
+				} else {
+					addToResult(*mine)
+					previousOwner = &mine.Peer
 				}
-				addToResult(*mine)
-				previousOwner = &mine.Peer
 			case mine.Version < theirs.Version:
-				if mine.Peer == ourPeer { // We shouldn't receive updates to our own tokens
-					err = errNewerVersion(mine, theirs)
-					return
+				if mine.Peer == ourPeer {
+					// We received update to our own tokens accept the received entry
+					// if either it belongs to a different peer and we do not have allocations
+					// in the range effectively given away, or it belongs to our own peer, in
+					// which case we should set our version to the one received plus one,
+					// effectively imposing our existing entry.
+					if theirs.Peer != ourPeer && !hasAllocations(r.makeRanges(mine.Token, es.entry(i+1).Token)) {
+						common.Log.Infof("ring: repair inconsistent data by accepting token %v from %v - no allocations up to %v", theirs.Token, theirs.Peer, es.entry(i+1).Token)
+						addTheirs(*theirs)
+					} else if theirs.Peer == ourPeer {
+						mine.Version = theirs.Version + 1
+						addToResult(*mine)
+					} else {
+						err = errNewerVersion(mine, theirs)
+						return
+					}
+				} else {
+					addTheirs(*theirs)
 				}
-				addTheirs(*theirs)
 			}
 			i++
 			j++
@@ -334,6 +375,11 @@ func (r *Ring) Empty() bool {
 	return len(r.Entries) == 0
 }
 
+// convenience function to normalise a single range that may span zero
+func (r *Ring) makeRanges(start, end address.Address) []address.Range {
+	return r.splitRangesOverZero([]address.Range{{Start: start, End: end}})
+}
+
 // Given a slice of ranges which are all in the right order except
 // possibly the last one spans zero, fix that up and return the slice
 func (r *Ring) splitRangesOverZero(ranges []address.Range) []address.Range {
@@ -384,10 +430,7 @@ type RangeInfo struct {
 
 func (r *Ring) AllRangeInfo() (result []RangeInfo) {
 	for i, entry := range r.Entries {
-		nextEntry := r.Entries.entry(i + 1)
-		ranges := []address.Range{{Start: entry.Token, End: nextEntry.Token}}
-		ranges = r.splitRangesOverZero(ranges)
-		for _, r := range ranges {
+		for _, r := range r.makeRanges(entry.Token, r.Entries.entry(i+1).Token) {
 			result = append(result, RangeInfo{entry.Peer, r, entry.Version})
 		}
 	}
@@ -485,6 +528,7 @@ func (r *Ring) ReportFree(freespace map[address.Address]address.Count) (updated
 
 		entries[i].Free = free
 		entries[i].Version++
+		common.Log.Debugf("[ring %s]: ReportFree token=%s peer=%s version=%v", r.Peer, entries[i].Token, entries[i].Peer, entries[i].Version)
 		updated = true
 	}
 	return
@@ -565,7 +609,10 @@ func (r *Ring) Transfer(from, to mesh.PeerName) []address.Range {
 	for i, entry := range r.Entries {
 		if entry.Peer == from {
 			entry.Peer = to
-			entry.Version++
+			// bump version by large value so that transfer of range takes precedence (if there is any conflict in merging entry)
+			// over other opertaions on the entry that increments the version
+			entry.Version = entry.Version + 100
+			common.Log.Debugf("[ring %s]: Transfer token=%s from=%s to=%s version=%v", r.Peer, entry.Token, from, to, entry.Version)
 			newRanges = append(newRanges, address.Range{Start: entry.Token, End: r.Entries.entry(i + 1).Token})
 		}
 	}

ipam/ring/ring_test.go

@@ -32,7 +32,27 @@ func ParseIP(s string) address.Address {
 }
 
 func merge(r1, r2 *Ring) error {
-	_, err := r1.Merge(*r2)
+	distance := func(start, end address.Address) address.Count {
+		if end > start {
+			return address.Count(end - start)
+		}
+		return address.Count((r1.End - start) + (end - r1.Start))
+	}
+	check := func(rs []address.Range) bool {
+		for _, r := range rs {
+			size := distance(r.Start, r.End)
+			entry, found := r1.Entries.get(r.Start)
+			if !found {
+				continue
+			}
+			if entry.Free < size {
+				return true
+			}
+		}
+		return false
+	}
+
+	_, err := r1.Merge(*r2, check)
 	return err
 }
 
@@ -219,6 +239,61 @@ func TestMergeSimple(t *testing.T) {
 	require.Equal(t, ring2.Entries, ring1.Entries)
 }
 
+func TestMergeWithConflicts(t *testing.T) {
+
+	// received update to entry with a token and version identical to an entry we have
+	// but but holding different content, if there are no allocations, resolve the
+	// conflict by picking entry one with high free count
+	ring1 := NewRing(start, end, peer1name)
+	ring2 := NewRing(start, end, peer2name)
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 256, Version: 1}}
+	ring2.Entries = []*entry{{Token: start, Peer: peer2name, Free: 256, Version: 1}}
+	require.NoError(t, merge(ring1, ring2))
+
+	// received update to entry with a token and version identical to an entry we have
+	// but but holding different content, if there are allocations, then reject the received
+	// entry
+	ring1 = NewRing(start, end, peer1name)
+	ring2 = NewRing(start, end, peer2name)
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 128, Version: 1}}
+	ring2.Entries = []*entry{{Token: start, Peer: peer2name, Free: 256, Version: 1}}
+	require.Error(t, merge(ring1, ring2), "Expected error")
+
+	// received an entry with update to one of our own tokens and with new version,
+	// accept the received entry if its still going to belong to us
+	ring1 = NewRing(start, end, peer1name)
+	ring2 = NewRing(start, end, peer2name)
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name}}
+	ring2.Entries = []*entry{{Token: start, Peer: peer1name, Version: 1}}
+	require.NoError(t, merge(ring1, ring2))
+
+	// received an entry with update to one of our own tokens and with new version,
+	// but belongs to a different peer, accept the received entry if we do not
+	// have allocations in the range
+	ring1 = NewRing(start, end, peer1name)
+	ring2 = NewRing(start, end, peer2name)
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 256}}
+	ring2.Entries = []*entry{{Token: start, Peer: peer2name, Free: 256, Version: 1}}
+	require.NoError(t, merge(ring1, ring2))
+
+	// received an entry with update to one of our own tokens and with new version,
+	// reject received entry if we have allocations in the range
+	ring1 = NewRing(start, end, peer1name)
+	ring2 = NewRing(start, end, peer2name)
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 128}}
+	ring2.Entries = []*entry{{Token: start, Peer: peer2name, Version: 1}}
+	require.Error(t, merge(ring1, ring2), "Expected error")
+
+	// we receive an entry that splits one of our ranges, giving some of it
+	// away to another peer. accept the entry provided that we have no allocations
+	// in the range that got given away.
+	ring1 = NewRing(start, end, peer1name)
+	ring2 = NewRing(start, end, peer2name)
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 256, Version: 1}}
+	ring2.Entries = []*entry{{Token: start, Peer: peer1name, Free: 128, Version: 2}, {Token: middle, Peer: peer2name, Free: 128}}
+	require.NoError(t, merge(ring1, ring2))
+}
+
 func TestMergeErrors(t *testing.T) {
 	// Cannot Merge in an invalid ring
 	ring1 := NewRing(start, end, peer1name)
@@ -231,13 +306,6 @@ func TestMergeErrors(t *testing.T) {
 	ring2.Entries = []*entry{}
 	require.True(t, merge(ring1, ring2) == ErrDifferentRange, "Expected ErrDifferentRange")
 
-	// Cannot Merge newer version of entry I own
-	ring2 = NewRing(start, end, peer2name)
-	ring1.Entries = []*entry{{Token: start, Peer: peer1name}}
-	ring2.Entries = []*entry{{Token: start, Peer: peer1name, Version: 1}}
-	fmt.Println(merge(ring1, ring2))
-	require.Error(t, merge(ring1, ring2), "Expected error")
-
 	// Cannot Merge two entries with same version but different hosts
 	ring1.Entries = []*entry{{Token: start, Peer: peer1name}}
 	ring2.Entries = []*entry{{Token: start, Peer: peer2name}}

ipam/testutils_test.go

@@ -65,6 +65,9 @@ func (m *mockGossipComms) GossipBroadcast(update mesh.GossipData) {
 	}
 }
 
+func (m *mockGossipComms) GossipNeighbourSubset(update mesh.GossipData) {
+	m.GossipBroadcast(update)
+}
 func equalByteBuffer(a, b []byte) bool {
 	if len(a) != len(b) {
 		return false

net/bridge.go

@@ -5,6 +5,7 @@ import (
 	"io/ioutil"
 	"net"
 	"path/filepath"
+	"syscall"
 
 	"github.com/coreos/go-iptables/iptables"
 	"github.com/pkg/errors"
@@ -62,10 +63,10 @@ const (
 )
 
 type Bridge interface {
-	init(config *BridgeConfig) error // create and initialise bridge device(s)
-	attach(veth *netlink.Veth) error // attach veth to bridge
-	IsFastdp() bool                  // does this bridge use fastdp?
-	String() string                  // human-readable type string
+	init(procPath string, config *BridgeConfig) error // create and initialise bridge device(s)
+	attach(veth *netlink.Veth) error                  // attach veth to bridge
+	IsFastdp() bool                                   // does this bridge use fastdp?
+	String() string                                   // human-readable type string
 }
 
 // Used to indicate a fallback to the Bridge type
@@ -220,6 +221,7 @@ type BridgeConfig struct {
 	MTU              int
 	Mac              string
 	Port             int
+	ControlPort      string
 	NoMasqLocal      bool
 }
 
@@ -249,7 +251,7 @@ func EnsureBridge(procPath string, config *BridgeConfig, log *logrus.Logger, ips
 	}
 
 	for {
-		if err := bridgeType.init(config); err != nil {
+		if err := bridgeType.init(procPath, config); err != nil {
 			if errors.Cause(err) == errBridgeNotSupported {
 				log.Warnf("Skipping bridge creation of %q due to: %s", bridgeType, err)
 				bridgeType = bridgeImpl{}
@@ -278,11 +280,19 @@ func EnsureBridge(procPath string, config *BridgeConfig, log *logrus.Logger, ips
 			return bridgeType, errors.Wrap(err, "setting proxy_arp")
 		}
 	}
+	// No ipv6 router advertisments please
+	if err := sysctl(procPath, "net/ipv6/conf/"+config.WeaveBridgeName+"/accept_ra", "0"); err != nil {
+		return bridgeType, errors.Wrap(err, "setting accept_ra to 0")
+	}
 
 	if err := linkSetUpByName(config.WeaveBridgeName); err != nil {
 		return bridgeType, err
 	}
 
+	if err := monitorInterface(config.WeaveBridgeName, log); err != nil {
+		return bridgeType, err
+	}
+
 	if err := configureARPCache(procPath, config.WeaveBridgeName); err != nil {
 		return bridgeType, errors.Wrapf(err, "configuring ARP cache on bridge %q", config.WeaveBridgeName)
 	}
@@ -304,7 +314,10 @@ func (b bridgeImpl) initPrep(config *BridgeConfig) error {
 
 	linkAttrs := netlink.NewLinkAttrs()
 	linkAttrs.Name = config.WeaveBridgeName
-	linkAttrs.HardwareAddr = mac
+	// NB: Do not set MAC addr when creating the bridge, set it manually
+	// afterwards instead. Otherwise, on an older than 3.14 kernel FDB
+	// entry won't be created which results in containers not being able to
+	// reach the bridge w/o promiscuous mode.
 	if config.MTU == 0 {
 		config.MTU = 65535
 	}
@@ -337,11 +350,11 @@ func (b bridgeImpl) initPrep(config *BridgeConfig) error {
 	return nil
 }
 
-func (b bridgeImpl) init(config *BridgeConfig) error {
+func (b bridgeImpl) init(procPath string, config *BridgeConfig) error {
 	if err := b.initPrep(config); err != nil {
 		return err
 	}
-	if _, err := CreateAndAttachVeth(BridgeIfName, PcapIfName, config.WeaveBridgeName, config.MTU, true, false, func(veth netlink.Link) error {
+	if _, err := CreateAndAttachVeth(procPath, BridgeIfName, PcapIfName, config.WeaveBridgeName, config.MTU, true, false, func(veth netlink.Link) error {
 		return netlink.LinkSetUp(veth)
 	}); err != nil {
 		return errors.Wrap(err, "creating pcap veth pair")
@@ -353,7 +366,7 @@ func (b bridgeImpl) init(config *BridgeConfig) error {
 	return nil
 }
 
-func (f fastdpImpl) init(config *BridgeConfig) error {
+func (f fastdpImpl) init(procPath string, config *BridgeConfig) error {
 	odpSupported, err := odp.CreateDatapath(f.datapathName)
 	if !odpSupported {
 		msg := ""
@@ -384,14 +397,14 @@ func (f fastdpImpl) init(config *BridgeConfig) error {
 	return nil
 }
 
-func (bf bridgedFastdpImpl) init(config *BridgeConfig) error {
-	if err := bf.fastdpImpl.init(config); err != nil {
+func (bf bridgedFastdpImpl) init(procPath string, config *BridgeConfig) error {
+	if err := bf.fastdpImpl.init(procPath, config); err != nil {
 		return err
 	}
 	if err := bf.bridgeImpl.initPrep(config); err != nil {
 		return err
 	}
-	if _, err := CreateAndAttachVeth(BridgeIfName, DatapathIfName, config.WeaveBridgeName, config.MTU, true, false, func(veth netlink.Link) error {
+	if _, err := CreateAndAttachVeth(procPath, BridgeIfName, DatapathIfName, config.WeaveBridgeName, config.MTU, true, false, func(veth netlink.Link) error {
 		if err := netlink.LinkSetUp(veth); err != nil {
 			return errors.Wrapf(err, "setting link up on %q", veth.Attrs().Name)
 		}
@@ -457,6 +470,18 @@ func configureIPTables(config *BridgeConfig, ips ipset.Interface) error {
 		}
 	}
 
+	if config.ControlPort != "" {
+		if err = ipt.AppendUnique("filter", "INPUT", "-p", "tcp", "--dst", "127.0.0.1", "--dport", config.ControlPort,
+			"-m", "addrtype", "!", "--src-type", "LOCAL",
+			"-m", "conntrack", "!", "--ctstate", "RELATED,ESTABLISHED",
+			"-m", "comment", "--comment", "Block non-local access to Weave Net control port",
+			"-j", "DROP"); err != nil {
+			return err
+		}
+	}
+	// Remove the rule from Weave Net 2.6.3 which dropped too much.
+	_ = ipt.Delete("filter", "INPUT", "-p", "tcp", "--dst", "127.0.0.1", "-m", "addrtype", "!", "--src-type", "LOCAL", "-m", "conntrack", "!", "--ctstate", "RELATED,ESTABLISHED", "-j", "DROP")
+
 	if config.NPC {
 		// Steer traffic via the NPC.
 
@@ -602,10 +627,31 @@ func reexpose(config *BridgeConfig, log *logrus.Logger) error {
 
 	for _, addr := range addrs {
 		log.Infof("Re-exposing %s on bridge %q", addr.IPNet, config.WeaveBridgeName)
-		if err := Expose(config.WeaveBridgeName, addr.IPNet, config.AWSVPC, config.NPC); err != nil {
+		if err := Expose(config.WeaveBridgeName, addr.IPNet, config.AWSVPC, config.NPC, false); err != nil {
 			return errors.Wrapf(err, "unable to re-expose %s on bridge: %q", addr.IPNet, config.WeaveBridgeName)
 		}
 	}
 
 	return nil
 }
+
+func monitorInterface(ifaceName string, log *logrus.Logger) error {
+	_, err := netlink.LinkByName(ifaceName)
+	if err != nil {
+		return fmt.Errorf("Unable to find link %q: %s", ifaceName, err)
+	}
+
+	updatesChannel := make(chan netlink.LinkUpdate)
+	if err := netlink.LinkSubscribe(updatesChannel, nil); err != nil {
+		return errors.Wrapf(err, "error monitoring link %q for UP/DOWN notifications", ifaceName)
+	}
+
+	go func() {
+		for update := range updatesChannel {
+			if update.Link.Attrs().Name == ifaceName && update.IfInfomsg.Flags&syscall.IFF_UP == 0 {
+				log.Errorf("Interface %q which needs to be in UP state for Weave functioning is found to be in DOWN state", ifaceName)
+			}
+		}
+	}()
+	return nil
+}

net/expose.go

@@ -18,7 +18,8 @@ import (
 // * "ipAddr" - IP addr to be assigned to the bridge.
 // * "removeDefaultRoute" - whether to remove a default route installed by the kernel (used only in the AWSVPC mode).
 // * "npc" - whether is Weave NPC running.
-func Expose(bridgeName string, ipAddr *net.IPNet, removeDefaultRoute, npc bool) error {
+// * "skipNAT" - whether to skip adding iptables NAT rules
+func Expose(bridgeName string, ipAddr *net.IPNet, removeDefaultRoute, npc bool, skipNAT bool) error {
 	ipt, err := iptables.New()
 	if err != nil {
 		return errors.Wrap(err, "iptables.New")
@@ -29,8 +30,10 @@ func Expose(bridgeName string, ipAddr *net.IPNet, removeDefaultRoute, npc bool)
 		return errors.Wrap(err, "addBridgeIPAddr")
 	}
 
-	if err := exposeNAT(ipt, cidr); err != nil {
-		return errors.Wrap(err, "exposeNAT")
+	if !skipNAT {
+		if err := exposeNAT(ipt, cidr); err != nil {
+			return errors.Wrap(err, "exposeNAT")
+		}
 	}
 
 	if !npc {

net/ipsec/ipsec.go

@@ -85,7 +85,7 @@ func New(log *logrus.Logger) (*IPSec, error) {
 
 // InitSALocal initializes inbound ipsec from remotePeer and triggers
 // the initialization on remotePeer.
-func (ipsec *IPSec) InitSALocal(localPeer, remotePeer mesh.PeerName, connUID uint64, localIP, remoteIP net.IP, udpPort int, sessionKey *[32]byte, initRemote func([]byte) error) error {
+func (ipsec *IPSec) InitSALocal(localPeer, remotePeer mesh.PeerName, connUID uint64, localIP, remoteIP net.IP, udpPort int, sessionKey *[32]byte) ([]byte, error) {
 	// ID of inbound SPI
 	spiID := getSPIId(remotePeer, localPeer, connUID)
 
@@ -95,17 +95,17 @@ func (ipsec *IPSec) InitSALocal(localPeer, remotePeer mesh.PeerName, connUID uin
 	// Derive SA key
 	nonce, err := genNonce()
 	if err != nil {
-		return errors.Wrap(err, "generate nonce")
+		return nil, errors.Wrap(err, "generate nonce")
 	}
 	key, err := deriveKey(sessionKey[:], nonce, localPeer)
 	if err != nil {
-		return errors.Wrap(err, "derive key")
+		return nil, errors.Wrap(err, "derive key")
 	}
 
 	// Allocate SA
 	sa, err := netlink.XfrmStateAllocSpi(xfrmAllocSpiState(remoteIP, localIP))
 	if err != nil {
-		return errors.Wrap(err, fmt.Sprintf("ip xfrm state allocspi (in, %s, %s)", remoteIP, localIP))
+		return nil, errors.Wrap(err, fmt.Sprintf("ip xfrm state allocspi (in, %s, %s)", remoteIP, localIP))
 	}
 
 	// Use SPI generated by the kernel. The kernel ensures {dstIP, spi} to
@@ -117,28 +117,24 @@ func (ipsec *IPSec) InitSALocal(localPeer, remotePeer mesh.PeerName, connUID uin
 	// Create SA
 	if sa, err := xfrmState(remoteIP, localIP, spi, false, key); err == nil {
 		if err := netlink.XfrmStateUpdate(sa); err != nil {
-			return errors.Wrap(err, fmt.Sprintf("xfrm state update (in, %s, %s, 0x%x)", sa.Src, sa.Dst, sa.Spi))
+			return nil, errors.Wrap(err, fmt.Sprintf("xfrm state update (in, %s, %s, 0x%x)", sa.Src, sa.Dst, sa.Spi))
 		}
 	} else {
-		return errors.Wrap(err, "new xfrm state (in)")
+		return nil, errors.Wrap(err, "new xfrm state (in)")
 	}
 
 	// Install iptables rules
 	if err := ipsec.installDropNonEncrypted(localIP, remoteIP, udpPort, spi); err != nil {
-		return errors.Wrap(err, fmt.Sprintf("install protecting rules (%s, %s, %d, 0x%x)", localIP, remoteIP, udpPort, spi))
+		return nil, errors.Wrap(err, fmt.Sprintf("install protecting rules (%s, %s, %d, 0x%x)", localIP, remoteIP, udpPort, spi))
 	}
 
 	si := spiInfo{spi: spi, isDirOut: false}
 	ipsec.spiInfo[spiID] = si
 	ipsec.spis[spi] = &si
 
-	// Trigger the initialization on the remote peer
+	// Generate the message to trigger initialization on the remote peer
 	msg := &msgInitSARemote{nonce, spi}
-	if err := initRemote(msg.serialize()); err != nil {
-		return errors.Wrap(err, "send InitSARemote")
-	}
-
-	return nil
+	return msg.serialize(), nil
 }
 
 // InitSARemote initializes outbound ipsec to remotePeer.
@@ -223,9 +219,19 @@ func (ipsec *IPSec) Destroy(localPeer, remotePeer mesh.PeerName, connUID uint64,
 
 	if outSPIInfo, ok := ipsec.spiInfo[outSPIID]; ok {
 		ipsec.log.Infof("ipsec: destroy: out %s -> %s 0x%x", localIP, remoteIP, outSPIInfo.spi)
-
-		if err := netlink.XfrmPolicyDel(xfrmPolicy(localIP, remoteIP, outSPIInfo.spi)); err != nil {
-			ipsec.log.Warnf("ipsec: xfrm policy del (%s, %s, 0x%x) failed: %s", localIP, remoteIP, outSPIInfo.spi, err)
+		policy, err := netlink.XfrmPolicyGet(xfrmPolicy(localIP, remoteIP, outSPIInfo.spi))
+		if err != nil {
+			ipsec.log.Warnf("ipsec: xfrm policy get (%s, %s, 0x%x) failed: %s", localIP, remoteIP, outSPIInfo.spi, err)
+		} else {
+			if len(policy.Tmpls) == 1 {
+				if policy.Tmpls[0].Spi == int(outSPIInfo.spi) {
+					if err := netlink.XfrmPolicyDel(xfrmPolicy(localIP, remoteIP, outSPIInfo.spi)); err != nil {
+						ipsec.log.Warnf("ipsec: xfrm policy del (%s, %s, 0x%x) failed: %s", localIP, remoteIP, outSPIInfo.spi, err)
+					}
+				} else {
+					ipsec.log.Debugf("ipsec: xfrm not my policy (%s, %s, 0x%x) got 0x%x ", localIP, remoteIP, outSPIInfo.spi, policy.Tmpls[0].Spi)
+				}
+			}
 		}
 
 		outSA := &netlink.XfrmState{

net/netns.go

@@ -55,9 +55,9 @@ func WithNetNSByPath(path string, work func() error) error {
 }
 
 func NSPathByPid(pid int) string {
-	return NSPathByPidWithRoot("/", pid)
+	return NSPathByPidWithProc("/proc", pid)
 }
 
-func NSPathByPidWithRoot(root string, pid int) string {
-	return filepath.Join(root, fmt.Sprintf("/proc/%d/ns/net", pid))
+func NSPathByPidWithProc(procPath string, pid int) string {
+	return filepath.Join(procPath, fmt.Sprint(pid), "/ns/net")
 }

net/veth.go

@@ -13,7 +13,7 @@ import (
 )
 
 // create and attach a veth to the Weave bridge
-func CreateAndAttachVeth(name, peerName, bridgeName string, mtu int, keepTXOn bool, errIfLinkExist bool, init func(peer netlink.Link) error) (*netlink.Veth, error) {
+func CreateAndAttachVeth(procPath, name, peerName, bridgeName string, mtu int, keepTXOn bool, errIfLinkExist bool, init func(peer netlink.Link) error) (*netlink.Veth, error) {
 	bridge, err := netlink.LinkByName(bridgeName)
 	if err != nil {
 		return nil, fmt.Errorf(`bridge "%s" not present; did you launch weave?`, bridgeName)
@@ -49,6 +49,13 @@ func CreateAndAttachVeth(name, peerName, bridgeName string, mtu int, keepTXOn bo
 	if err := bridgeType.attach(veth); err != nil {
 		return cleanup("attaching veth %q to %q: %s", name, bridgeName, err)
 	}
+	// No ipv6 router advertisments please
+	if err := sysctl(procPath, "net/ipv6/conf/"+name+"/accept_ra", "0"); err != nil {
+		return cleanup("setting accept_ra to 0: %s", err)
+	}
+	if err := sysctl(procPath, "net/ipv6/conf/"+peerName+"/accept_ra", "0"); err != nil {
+		return cleanup("setting accept_ra to 0: %s", err)
+	}
 	if !bridgeType.IsFastdp() && !keepTXOn {
 		if err := EthtoolTXOff(veth.PeerName); err != nil {
 			return cleanup(`unable to set tx off on %q: %s`, peerName, err)
@@ -112,6 +119,9 @@ func interfaceExistsInNamespace(netNSPath string, ifName string) bool {
 }
 
 func AttachContainer(netNSPath, id, ifName, bridgeName string, mtu int, withMulticastRoute bool, cidrs []*net.IPNet, keepTXOn bool, hairpinMode bool) error {
+	// AttachContainer expects to be called in host pid namespace
+	const procPath = "/proc"
+
 	ns, err := netns.GetFromPath(netNSPath)
 	if err != nil {
 		return err
@@ -124,12 +134,12 @@ func AttachContainer(netNSPath, id, ifName, bridgeName string, mtu int, withMult
 			id = id[:maxIDLen] // trim passed ID if too long
 		}
 		name, peerName := vethPrefix+"pl"+id, vethPrefix+"pg"+id
-		veth, err := CreateAndAttachVeth(name, peerName, bridgeName, mtu, keepTXOn, true, func(veth netlink.Link) error {
+		veth, err := CreateAndAttachVeth(procPath, name, peerName, bridgeName, mtu, keepTXOn, true, func(veth netlink.Link) error {
 			if err := netlink.LinkSetNsFd(veth, int(ns)); err != nil {
 				return fmt.Errorf("failed to move veth to container netns: %s", err)
 			}
 			if err := WithNetNS(ns, func() error {
-				return setupIface(peerName, ifName)
+				return setupIface(procPath, peerName, ifName)
 			}); err != nil {
 				return fmt.Errorf("error setting up interface: %s", err)
 			}
@@ -206,7 +216,7 @@ func setupIfaceAddrs(veth netlink.Link, withMulticastRoute bool, cidrs []*net.IP
 }
 
 // setupIface expects to be called in the container's netns
-func setupIface(ifaceName, newIfName string) error {
+func setupIface(procPath, ifaceName, newIfName string) error {
 	ipt, err := iptables.New()
 	if err != nil {
 		return err
@@ -219,15 +229,14 @@ func setupIface(ifaceName, newIfName string) error {
 	if err := netlink.LinkSetName(link, newIfName); err != nil {
 		return err
 	}
-	// This is only called by AttachContainer which is only called in host pid namespace
-	if err := configureARPCache("/proc", newIfName); err != nil {
+	if err := configureARPCache(procPath, newIfName); err != nil {
 		return err
 	}
 	return ipt.Append("filter", "INPUT", "-i", newIfName, "-d", "224.0.0.0/4", "-j", "DROP")
 }
 
 // configureARP is a helper for the Docker plugin which doesn't set the addresses itself
-func ConfigureARP(prefix, rootPath string) error {
+func ConfigureARP(prefix, procPath string) error {
 	links, err := netlink.LinkList()
 	if err != nil {
 		return err
@@ -235,7 +244,7 @@ func ConfigureARP(prefix, rootPath string) error {
 	for _, link := range links {
 		ifName := link.Attrs().Name
 		if strings.HasPrefix(ifName, prefix) {
-			configureARPCache(rootPath+"/proc", ifName)
+			configureARPCache(procPath, ifName)
 			if addrs, err := netlink.AddrList(link, netlink.FAMILY_V4); err == nil {
 				for _, addr := range addrs {
 					arping.GratuitousArpOverIfaceByName(addr.IPNet.IP, ifName)

npc/analyser.go

@@ -12,12 +12,14 @@ import (
 
 func (ns *ns) analysePolicy(policy *networkingv1.NetworkPolicy) (
 	rules map[string]*ruleSpec,
-	nsSelectors, podSelectors map[string]*selectorSpec,
+	nsSelectors, podSelectors, namespacedPodSelectors map[string]*selectorSpec,
 	ipBlocks map[string]*ipBlockSpec,
 	err error) {
 
 	nsSelectors = make(map[string]*selectorSpec)
 	podSelectors = make(map[string]*selectorSpec)
+	namespacedPodSelectors = make(map[string]*selectorSpec)
+
 	ipBlocks = make(map[string]*ipBlockSpec)
 	rules = make(map[string]*ruleSpec)
 	policyTypes := make([]policyType, 0)
@@ -31,9 +33,9 @@ func (ns *ns) analysePolicy(policy *networkingv1.NetworkPolicy) (
 		}
 	}
 	// If empty, matches all pods in a namespace
-	targetSelector, err := newSelectorSpec(&policy.Spec.PodSelector, policyTypes, ns.name, ipset.HashIP)
+	targetSelector, err := newSelectorSpec(&policy.Spec.PodSelector, nil, policyTypes, ns.name, ipset.HashIP)
 	if err != nil {
-		return nil, nil, nil, nil, err
+		return nil, nil, nil, nil, nil, err
 	}
 
 	// To prevent targetSelector being overwritten by a subsequent selector with
@@ -50,6 +52,12 @@ func (ns *ns) analysePolicy(policy *networkingv1.NetworkPolicy) (
 			// If From is empty or missing, this rule matches all sources
 			allSources := ingressRule.From == nil || len(ingressRule.From) == 0
 
+			if !allPorts {
+				if err = checkForNamedPorts(ingressRule.Ports); err != nil {
+					return nil, nil, nil, nil, nil, fmt.Errorf("named ports in network policies is not supported yet. "+
+						"Rejecting network policy: %s from further processing. "+err.Error(), policy.Name)
+				}
+			}
 			if allSources {
 				if allPorts {
 					rule := newRuleSpec(policyTypeIngress, nil, nil, targetSelector, nil)
@@ -63,30 +71,42 @@ func (ns *ns) analysePolicy(policy *networkingv1.NetworkPolicy) (
 			} else {
 				for _, peer := range ingressRule.From {
 					var srcSelector *selectorSpec
+					var srcRuleHost ruleHost
 
 					// NetworkPolicyPeer describes a peer to allow traffic from.
-					// Exactly one of its fields must be specified.
-					if peer.PodSelector != nil {
-						srcSelector, err = newSelectorSpec(peer.PodSelector, nil, ns.name, ipset.HashIP)
+					if peer.PodSelector != nil && peer.NamespaceSelector != nil {
+						srcSelector, err = newSelectorSpec(peer.PodSelector, peer.NamespaceSelector, nil, "", ipset.HashIP)
 						if err != nil {
-							return nil, nil, nil, nil, err
+							return nil, nil, nil, nil, nil, err
+						}
+						addIfNotExist(srcSelector, namespacedPodSelectors)
+						srcRuleHost = srcSelector
+					} else if peer.PodSelector != nil {
+						srcSelector, err = newSelectorSpec(peer.PodSelector, nil, nil, ns.name, ipset.HashIP)
+						if err != nil {
+							return nil, nil, nil, nil, nil, err
 						}
 						addIfNotExist(srcSelector, podSelectors)
-
+						srcRuleHost = srcSelector
 					} else if peer.NamespaceSelector != nil {
-						srcSelector, err = newSelectorSpec(peer.NamespaceSelector, nil, "", ipset.ListSet)
+						srcSelector, err = newSelectorSpec(nil, peer.NamespaceSelector, nil, "", ipset.ListSet)
 						if err != nil {
-							return nil, nil, nil, nil, err
+							return nil, nil, nil, nil, nil, err
 						}
 						nsSelectors[srcSelector.key] = srcSelector
+						srcRuleHost = srcSelector
+					} else if peer.IPBlock != nil {
+						ipBlock := newIPBlockSpec(peer.IPBlock, ns.name)
+						ipBlocks[ipBlock.key] = ipBlock
+						srcRuleHost = ipBlock
 					}
 
 					if allPorts {
-						rule := newRuleSpec(policyTypeIngress, nil, srcSelector, targetSelector, nil)
+						rule := newRuleSpec(policyTypeIngress, nil, srcRuleHost, targetSelector, nil)
 						rules[rule.key] = rule
 					} else {
 						withNormalisedProtoAndPort(ingressRule.Ports, func(proto, port string) {
-							rule := newRuleSpec(policyTypeIngress, &proto, srcSelector, targetSelector, &port)
+							rule := newRuleSpec(policyTypeIngress, &proto, srcRuleHost, targetSelector, &port)
 							rules[rule.key] = rule
 						})
 					}
@@ -103,6 +123,12 @@ func (ns *ns) analysePolicy(policy *networkingv1.NetworkPolicy) (
 			// If To is empty or missing, this rule matches all destinations
 			allDestinations := egressRule.To == nil || len(egressRule.To) == 0
 
+			if !allPorts {
+				if err = checkForNamedPorts(egressRule.Ports); err != nil {
+					return nil, nil, nil, nil, nil, fmt.Errorf("named ports in network policies is not supported yet. "+
+						"Rejecting network policy: %s from further processing. "+err.Error(), policy.Name)
+				}
+			}
 			if allDestinations {
 				if allPorts {
 					rule := newRuleSpec(policyTypeEgress, nil, targetSelector, nil, nil)
@@ -119,19 +145,25 @@ func (ns *ns) analysePolicy(policy *networkingv1.NetworkPolicy) (
 					var dstRuleHost ruleHost
 
 					// NetworkPolicyPeer describes a peer to allow traffic to.
-					// Exactly one of its fields must be specified.
-					if peer.PodSelector != nil {
-						dstSelector, err = newSelectorSpec(peer.PodSelector, nil, ns.name, ipset.HashIP)
+					if peer.PodSelector != nil && peer.NamespaceSelector != nil {
+						dstSelector, err = newSelectorSpec(peer.PodSelector, peer.NamespaceSelector, nil, "", ipset.HashIP)
 						if err != nil {
-							return nil, nil, nil, nil, err
+							return nil, nil, nil, nil, nil, err
+						}
+						addIfNotExist(dstSelector, namespacedPodSelectors)
+						dstRuleHost = dstSelector
+					} else if peer.PodSelector != nil {
+						dstSelector, err = newSelectorSpec(peer.PodSelector, nil, nil, ns.name, ipset.HashIP)
+						if err != nil {
+							return nil, nil, nil, nil, nil, err
 						}
 						addIfNotExist(dstSelector, podSelectors)
 						dstRuleHost = dstSelector
 
 					} else if peer.NamespaceSelector != nil {
-						dstSelector, err = newSelectorSpec(peer.NamespaceSelector, nil, "", ipset.ListSet)
+						dstSelector, err = newSelectorSpec(nil, peer.NamespaceSelector, nil, "", ipset.ListSet)
 						if err != nil {
-							return nil, nil, nil, nil, err
+							return nil, nil, nil, nil, nil, err
 						}
 						nsSelectors[dstSelector.key] = dstSelector
 						dstRuleHost = dstSelector
@@ -155,7 +187,7 @@ func (ns *ns) analysePolicy(policy *networkingv1.NetworkPolicy) (
 		}
 	}
 
-	return rules, nsSelectors, podSelectors, ipBlocks, nil
+	return rules, nsSelectors, podSelectors, namespacedPodSelectors, ipBlocks, nil
 }
 
 func addIfNotExist(s *selectorSpec, ss map[string]*selectorSpec) {
@@ -195,3 +227,12 @@ func port(p *intstr.IntOrString) string {
 
 	return port
 }
+
+func checkForNamedPorts(ports []networkingv1.NetworkPolicyPort) error {
+	for _, npProtocolPort := range ports {
+		if npProtocolPort.Port != nil && npProtocolPort.Port.Type == intstr.String {
+			return fmt.Errorf("named port %s in network policy", port(npProtocolPort.Port))
+		}
+	}
+	return nil
+}

npc/controller.go

@@ -1,12 +1,15 @@
 package npc
 
 import (
+	"fmt"
 	"sync"
 
 	"github.com/pkg/errors"
 	coreapi "k8s.io/api/core/v1"
 	extnapi "k8s.io/api/extensions/v1beta1"
 	networkingv1 "k8s.io/api/networking/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
 
 	"github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/net/ipset"
@@ -32,30 +35,33 @@ type controller struct {
 
 	nodeName string // my node name
 
-	ipt iptables.Interface
-	ips ipset.Interface
-
-	nss         map[string]*ns // ns name -> ns struct
-	nsSelectors *selectorSet   // selector string -> nsSelector
+	ipt                    iptables.Interface
+	ips                    ipset.Interface
+	clientset              kubernetes.Interface
+	nss                    map[string]*ns // ns name -> ns struct
+	nsSelectors            *selectorSet   // selector string -> nsSelector
+	namespacedPodSelectors *selectorSet
+	defaultEgressDrop      bool // flag to track if base iptable rule to drop egress traffic is added or not
 }
 
-func New(nodeName string, ipt iptables.Interface, ips ipset.Interface) NetworkPolicyController {
+func New(nodeName string, ipt iptables.Interface, ips ipset.Interface, clientset kubernetes.Interface) NetworkPolicyController {
 	c := &controller{
-		nodeName: nodeName,
-		ipt:      ipt,
-		ips:      ips,
-		nss:      make(map[string]*ns)}
+		nodeName:  nodeName,
+		ipt:       ipt,
+		ips:       ips,
+		clientset: clientset,
+		nss:       make(map[string]*ns)}
 
 	doNothing := func(*selector, policyType) error { return nil }
 	c.nsSelectors = newSelectorSet(ips, c.onNewNsSelector, doNothing, doNothing)
-
+	c.namespacedPodSelectors = newSelectorSet(ips, c.onNewNamespacePodsSelector, doNothing, doNothing)
 	return c
 }
 
 func (npc *controller) onNewNsSelector(selector *selector) error {
 	for _, ns := range npc.nss {
 		if ns.namespace != nil {
-			if selector.matches(ns.namespace.ObjectMeta.Labels) {
+			if selector.matchesNamespaceSelector(ns.namespace.ObjectMeta.Labels) {
 				if err := selector.addEntry(ns.namespace.ObjectMeta.UID, string(ns.allPods.ipsetName), namespaceComment(ns)); err != nil {
 					return err
 				}
@@ -65,10 +71,32 @@ func (npc *controller) onNewNsSelector(selector *selector) error {
 	return nil
 }
 
+func (npc *controller) onNewNamespacePodsSelector(selector *selector) error {
+	for _, ns := range npc.nss {
+		if ns.namespace != nil && len(ns.pods) > 0 {
+			for _, pod := range ns.pods {
+				if hasIP(pod) {
+					if selector.matchesNamespacedPodSelector(pod.ObjectMeta.Labels, ns.namespace.ObjectMeta.Labels) {
+						if err := selector.addEntry(pod.ObjectMeta.UID, pod.Status.PodIP, podComment(pod)); err != nil {
+							return err
+						}
+
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
 func (npc *controller) withNS(name string, f func(ns *ns) error) error {
 	ns, found := npc.nss[name]
 	if !found {
-		newNs, err := newNS(name, npc.nodeName, npc.ipt, npc.ips, npc.nsSelectors)
+		namespace, err := npc.clientset.CoreV1().Namespaces().Get(name, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+		newNs, err := newNS(name, npc.nodeName, npc.ipt, npc.ips, npc.nsSelectors, npc.namespacedPodSelectors, namespace)
 		if err != nil {
 			return err
 		}
@@ -121,11 +149,26 @@ func (npc *controller) AddNetworkPolicy(obj interface{}) error {
 	npc.Lock()
 	defer npc.Unlock()
 
+	// lazily add default rule to drop egress traffic only when network policies are applied
+	if !npc.defaultEgressDrop {
+		egressNetworkPolicy, err := isEgressNetworkPolicy(obj)
+		if err != nil {
+			return err
+		}
+		if egressNetworkPolicy {
+			npc.defaultEgressDrop = true
+			if err := npc.ipt.Append(TableFilter, EgressChain,
+				"-m", "mark", "!", "--mark", EgressMark, "-j", "DROP"); err != nil {
+				npc.defaultEgressDrop = false
+				return fmt.Errorf("Failed to add iptable rule to drop egress traffic from the pods by default due to %s", err.Error())
+			}
+		}
+	}
+
 	nsName, err := nsName(obj)
 	if err != nil {
 		return err
 	}
-
 	common.Log.Infof("EVENT AddNetworkPolicy %s", js(obj))
 	return npc.withNS(nsName, func(ns *ns) error {
 		return errors.Wrap(ns.addNetworkPolicy(obj), "add network policy")
@@ -202,3 +245,20 @@ func nsName(obj interface{}) (string, error) {
 
 	return "", errInvalidNetworkPolicyObjType
 }
+
+func isEgressNetworkPolicy(obj interface{}) (bool, error) {
+	if policy, ok := obj.(*networkingv1.NetworkPolicy); ok {
+		if len(policy.Spec.PolicyTypes) > 0 {
+			for _, policyType := range policy.Spec.PolicyTypes {
+				if policyType == networkingv1.PolicyTypeEgress {
+					return true, nil
+				}
+			}
+		}
+		if policy.Spec.Egress != nil {
+			return true, nil
+		}
+		return false, nil
+	}
+	return false, errInvalidNetworkPolicyObjType
+}

npc/controller_test.go

@@ -14,6 +14,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/client-go/kubernetes/fake"
 )
 
 type mockSet struct {
@@ -198,7 +199,10 @@ func TestRegressionPolicyNamespaceOrdering3059(t *testing.T) {
 
 	// Namespaces first
 	m := newMockIPSet()
-	controller := New("foo", newMockIPTables(), &m)
+	client := fake.NewSimpleClientset()
+	controller := New("foo", newMockIPTables(), &m, client)
+	client.CoreV1().Namespaces().Create(sourceNamespace)
+	client.CoreV1().Namespaces().Create(destinationNamespace)
 
 	const (
 		selectorIPSetName = "weave-I239Zp%sCvoVt*D6u=A!2]YEk"
@@ -214,7 +218,7 @@ func TestRegressionPolicyNamespaceOrdering3059(t *testing.T) {
 
 	// NetworkPolicy first
 	m = newMockIPSet()
-	controller = New("foo", newMockIPTables(), &m)
+	controller = New("foo", newMockIPTables(), &m, &fake.Clientset{})
 
 	controller.AddNetworkPolicy(networkPolicy)
 
@@ -235,7 +239,7 @@ func TestDefaultAllow(t *testing.T) {
 	)
 
 	m := newMockIPSet()
-	controller := New("bar", newMockIPTables(), &m)
+	controller := New("bar", newMockIPTables(), &m, &fake.Clientset{})
 
 	defaultNamespace := &coreapi.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
@@ -363,13 +367,15 @@ func TestOutOfOrderPodEvents(t *testing.T) {
 	)
 
 	m := newMockIPSet()
-	controller := New("qux", newMockIPTables(), &m)
+	client := fake.NewSimpleClientset()
+	controller := New("qux", newMockIPTables(), &m, client)
 
 	defaultNamespace := &coreapi.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "default",
 		},
 	}
+	client.CoreV1().Namespaces().Create(defaultNamespace)
 	controller.AddNamespace(defaultNamespace)
 
 	podFoo := &coreapi.Pod{
@@ -445,13 +451,15 @@ func TestNewTargetSelector(t *testing.T) {
 	)
 
 	m := newMockIPSet()
-	controller := New("baz", newMockIPTables(), &m)
+	client := fake.NewSimpleClientset()
+	controller := New("baz", newMockIPTables(), &m, client)
 
 	defaultNamespace := &coreapi.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "default",
 		},
 	}
+	client.CoreV1().Namespaces().Create(defaultNamespace)
 	controller.AddNamespace(defaultNamespace)
 
 	podFoo := &coreapi.Pod{
@@ -521,7 +529,21 @@ func TestEgressPolicyWithIPBlock(t *testing.T) {
 
 	m := newMockIPSet()
 	ipt := newMockIPTables()
-	controller := New("foo", ipt, &m)
+	client := fake.NewSimpleClientset()
+	controller := New("foo", ipt, &m, client)
+
+	defaultNamespace := &coreapi.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "default",
+		},
+	}
+	nonDefaultNamespace := &coreapi.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "non-default",
+		},
+	}
+	client.CoreV1().Namespaces().Create(defaultNamespace)
+	client.CoreV1().Namespaces().Create(nonDefaultNamespace)
 
 	netpolFoo := &networkingv1.NetworkPolicy{
 		ObjectMeta: metav1.ObjectMeta{
@@ -592,3 +614,79 @@ func TestEgressPolicyWithIPBlock(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, 2, len(m.sets[exceptIPSetNameInNonDefault].subSets))
 }
+
+// Test case for https://github.com/weaveworks/weave/issues/3653
+func TestIngressPolicyWithIPBlockAndPortSpecified(t *testing.T) {
+	const (
+		barPodIP        = "10.32.0.11"
+		runBarIPSetName = "weave-bZ~x=yBgzH)Ht()K*Uv3z{M]Y"
+	)
+
+	m := newMockIPSet()
+	ipt := newMockIPTables()
+	client := fake.NewSimpleClientset()
+	controller := New("any", ipt, &m, client)
+
+	defaultNamespace := &coreapi.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "default",
+		},
+	}
+
+	client.CoreV1().Namespaces().Create(defaultNamespace)
+
+	podBar := &coreapi.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			UID:       "bar",
+			Namespace: "default",
+			Name:      "bar",
+			Labels:    map[string]string{"run": "bar"}},
+		Status: coreapi.PodStatus{PodIP: barPodIP}}
+	controller.AddPod(podBar)
+	defer controller.DeletePod(podBar)
+
+	portProtocol := coreapi.ProtocolTCP
+	port := intstr.FromInt(80)
+	netpolicty := &networkingv1.NetworkPolicy{
+		ObjectMeta: metav1.ObjectMeta{
+			UID:       "ipblock-bar",
+			Name:      "allow-ipblock-to-bar",
+			Namespace: "default",
+		},
+		Spec: networkingv1.NetworkPolicySpec{
+			PolicyTypes: []networkingv1.PolicyType{
+				networkingv1.PolicyTypeIngress,
+			},
+			PodSelector: metav1.LabelSelector{MatchLabels: podBar.Labels},
+			Ingress: []networkingv1.NetworkPolicyIngressRule{
+				{
+					Ports: []networkingv1.NetworkPolicyPort{
+						{
+							Protocol: &portProtocol,
+							Port:     &port,
+						},
+					},
+					From: []networkingv1.NetworkPolicyPeer{
+						{
+							IPBlock: &networkingv1.IPBlock{
+								CIDR: "192.168.48.4/32",
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	err := controller.AddNetworkPolicy(netpolicty)
+	require.NoError(t, err)
+	defer controller.DeleteNetworkPolicy(netpolicty)
+
+	require.Equal(t, 1, len(m.sets[runBarIPSetName].subSets))
+	require.True(t, m.entryExists(runBarIPSetName, barPodIP))
+
+	require.Equal(t, 1, len(ipt.rules[IngressChain]))
+	for rule := range ipt.rules[IngressChain] {
+		require.Contains(t, rule, "-s 192.168.48.4/32 -m set --match-set "+runBarIPSetName+" dst --dport 80")
+	}
+}

npc/metrics/metrics.go

@@ -22,6 +22,12 @@ var (
 		},
 		[]string{"protocol", "dport"},
 	)
+	PolicyEnforcementErrors = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "weavenpc_policy_enforcement_errors",
+			Help: "Errors occured in enforcing the network policies.",
+		},
+	)
 )
 
 func gatherMetrics() {
@@ -95,6 +101,9 @@ func Start(addr string) error {
 	if err := prometheus.Register(blockedConnections); err != nil {
 		return err
 	}
+	if err := prometheus.Register(PolicyEnforcementErrors); err != nil {
+		return err
+	}
 
 	http.Handle("/metrics", promhttp.Handler())
 

npc/namespace.go

@@ -37,30 +37,33 @@ type ns struct {
 	ingressDefaultAllowIPSet ipset.Name
 	egressDefaultAllowIPSet  ipset.Name
 
-	nsSelectors  *selectorSet
-	podSelectors *selectorSet
-	ipBlocks     *ipBlockSet
-	rules        *ruleSet
+	nsSelectors             *selectorSet // reference to global selectorSet that is shared across the `ns`. Used to represent all pods in the matching namespaces
+	podSelectors            *selectorSet // used to represent the matching pods in namespace respresented by this `ns`
+	namespacedPodsSelectors *selectorSet // reference to global selectorSet that is shared across the `ns`. Used to represent matching pods in matching namespace
+	ipBlocks                *ipBlockSet
+	rules                   *ruleSet
 }
 
-func newNS(name, nodeName string, ipt iptables.Interface, ips ipset.Interface, nsSelectors *selectorSet) (*ns, error) {
-	allPods, err := newSelectorSpec(&metav1.LabelSelector{}, nil, name, ipset.HashIP)
+func newNS(name, nodeName string, ipt iptables.Interface, ips ipset.Interface, nsSelectors *selectorSet, namespacedPodsSelectors *selectorSet, namespaceObj *coreapi.Namespace) (*ns, error) {
+	allPods, err := newSelectorSpec(&metav1.LabelSelector{}, nil, nil, name, ipset.HashIP)
 	if err != nil {
 		return nil, err
 	}
 
 	ns := &ns{
-		ipt:         ipt,
-		ips:         ips,
-		name:        name,
-		nodeName:    nodeName,
-		pods:        make(map[types.UID]*coreapi.Pod),
-		policies:    make(map[types.UID]interface{}),
-		uid:         uuid.NewUUID(),
-		allPods:     allPods,
-		nsSelectors: nsSelectors,
-		ipBlocks:    newIPBlockSet(ips),
-		rules:       newRuleSet(ipt),
+		ipt:                     ipt,
+		ips:                     ips,
+		name:                    name,
+		namespace:               namespaceObj,
+		nodeName:                nodeName,
+		pods:                    make(map[types.UID]*coreapi.Pod),
+		policies:                make(map[types.UID]interface{}),
+		uid:                     uuid.NewUUID(),
+		allPods:                 allPods,
+		nsSelectors:             nsSelectors,
+		namespacedPodsSelectors: namespacedPodsSelectors,
+		ipBlocks:                newIPBlockSet(ips),
+		rules:                   newRuleSet(ipt),
 	}
 
 	ns.podSelectors = newSelectorSet(ips, ns.onNewPodSelector, ns.onNewTargetPodSelector, ns.onDestroyTargetPodSelector)
@@ -95,7 +98,7 @@ func (ns *ns) destroy() error {
 func (ns *ns) onNewPodSelector(selector *selector) error {
 	for _, pod := range ns.pods {
 		if hasIP(pod) {
-			if selector.matches(pod.ObjectMeta.Labels) {
+			if selector.matchesPodSelector(pod.ObjectMeta.Labels) {
 				if err := selector.addEntry(pod.ObjectMeta.UID, pod.Status.PodIP, podComment(pod)); err != nil {
 					return err
 				}
@@ -111,7 +114,7 @@ func (ns *ns) onNewTargetPodSelector(selector *selector, policyType policyType)
 		if hasIP(pod) {
 			// Remove the pod from default-allow if dst podselector matches the pod
 			ipset := ns.defaultAllowIPSetName(policyType)
-			if selector.matches(pod.ObjectMeta.Labels) {
+			if selector.matchesPodSelector(pod.ObjectMeta.Labels) {
 				if err := ns.ips.DelEntry(pod.ObjectMeta.UID, ipset, pod.Status.PodIP); err != nil {
 					return err
 				}
@@ -125,7 +128,7 @@ func (ns *ns) onNewTargetPodSelector(selector *selector, policyType policyType)
 func (ns *ns) onDestroyTargetPodSelector(selector *selector, policyType policyType) error {
 	for _, pod := range ns.pods {
 		if hasIP(pod) {
-			if selector.matches(pod.ObjectMeta.Labels) {
+			if selector.matchesPodSelector(pod.ObjectMeta.Labels) {
 				if err := ns.addToDefaultAllowIfNoMatching(pod, policyType); err != nil {
 					return err
 				}
@@ -141,7 +144,7 @@ func (ns *ns) addToDefaultAllowIfNoMatching(pod *coreapi.Pod, policyType policyT
 	found := false
 	// TODO(mp) optimize (avoid iterating over selectors) by ref counting IP addrs.
 	for _, s := range ns.podSelectors.entries {
-		if ns.podSelectors.targetSelectorExist(s, policyType) && s.matches(pod.ObjectMeta.Labels) {
+		if ns.podSelectors.targetSelectorExist(s, policyType) && s.matchesPodSelector(pod.ObjectMeta.Labels) {
 			found = true
 			break
 		}
@@ -169,11 +172,7 @@ func (ns *ns) addPod(obj *coreapi.Pod) error {
 		return nil
 	}
 
-	if ns.checkLocalPod(obj) {
-		ns.ips.AddEntry(obj.ObjectMeta.UID, LocalIpset, obj.Status.PodIP, podComment(obj))
-	}
-
-	foundIngress, foundEgress, err := ns.podSelectors.addToMatching(obj.ObjectMeta.UID, obj.ObjectMeta.Labels, obj.Status.PodIP, podComment(obj))
+	foundIngress, foundEgress, err := ns.podSelectors.addToMatchingPodSelector(obj.ObjectMeta.UID, obj.ObjectMeta.Labels, obj.Status.PodIP, podComment(obj))
 	if err != nil {
 		return err
 	}
@@ -189,6 +188,11 @@ func (ns *ns) addPod(obj *coreapi.Pod) error {
 		}
 	}
 
+	err = ns.namespacedPodsSelectors.addToMatchingNamespacedPodSelector(obj.ObjectMeta.UID, obj.ObjectMeta.Labels, ns.namespace.ObjectMeta.Labels, obj.Status.PodIP, podComment(obj))
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -201,25 +205,21 @@ func (ns *ns) updatePod(oldObj, newObj *coreapi.Pod) error {
 	}
 
 	if hasIP(oldObj) && !hasIP(newObj) {
-		if ns.checkLocalPod(oldObj) {
-			ns.ips.DelEntry(oldObj.ObjectMeta.UID, LocalIpset, oldObj.Status.PodIP)
-		}
-
 		if err := ns.ips.DelEntry(oldObj.ObjectMeta.UID, ns.ingressDefaultAllowIPSet, oldObj.Status.PodIP); err != nil {
 			return err
 		}
 		if err := ns.ips.DelEntry(oldObj.ObjectMeta.UID, ns.egressDefaultAllowIPSet, oldObj.Status.PodIP); err != nil {
 			return err
 		}
+		if err := ns.namespacedPodsSelectors.delFromMatchingNamespacedPodSelector(oldObj.ObjectMeta.UID, oldObj.ObjectMeta.Labels, ns.namespace.ObjectMeta.Labels, oldObj.Status.PodIP); err != nil {
+			return err
+		}
 
-		return ns.podSelectors.delFromMatching(oldObj.ObjectMeta.UID, oldObj.ObjectMeta.Labels, oldObj.Status.PodIP)
+		return ns.podSelectors.delFromMatchingPodSelector(oldObj.ObjectMeta.UID, oldObj.ObjectMeta.Labels, oldObj.Status.PodIP)
 	}
 
 	if !hasIP(oldObj) && hasIP(newObj) {
-		if ns.checkLocalPod(newObj) {
-			ns.ips.AddEntry(newObj.ObjectMeta.UID, LocalIpset, newObj.Status.PodIP, podComment(newObj))
-		}
-		foundIngress, foundEgress, err := ns.podSelectors.addToMatching(newObj.ObjectMeta.UID, newObj.ObjectMeta.Labels, newObj.Status.PodIP, podComment(newObj))
+		foundIngress, foundEgress, err := ns.podSelectors.addToMatchingPodSelector(newObj.ObjectMeta.UID, newObj.ObjectMeta.Labels, newObj.Status.PodIP, podComment(newObj))
 		if err != nil {
 			return err
 		}
@@ -234,6 +234,10 @@ func (ns *ns) updatePod(oldObj, newObj *coreapi.Pod) error {
 				return err
 			}
 		}
+		err = ns.namespacedPodsSelectors.addToMatchingNamespacedPodSelector(newObj.ObjectMeta.UID, newObj.ObjectMeta.Labels, ns.namespace.ObjectMeta.Labels, newObj.Status.PodIP, podComment(newObj))
+		if err != nil {
+			return err
+		}
 
 		return nil
 	}
@@ -251,8 +255,8 @@ func (ns *ns) updatePod(oldObj, newObj *coreapi.Pod) error {
 		oldObj.Status.PodIP != newObj.Status.PodIP {
 
 		for _, ps := range ns.podSelectors.entries {
-			oldMatch := ps.matches(oldObj.ObjectMeta.Labels)
-			newMatch := ps.matches(newObj.ObjectMeta.Labels)
+			oldMatch := ps.matchesPodSelector(oldObj.ObjectMeta.Labels)
+			newMatch := ps.matchesPodSelector(newObj.ObjectMeta.Labels)
 			if oldMatch == newMatch && oldObj.Status.PodIP == newObj.Status.PodIP {
 				continue
 			}
@@ -274,6 +278,23 @@ func (ns *ns) updatePod(oldObj, newObj *coreapi.Pod) error {
 				return err
 			}
 		}
+		for _, ps := range ns.namespacedPodsSelectors.entries {
+			oldMatch := ps.matchesNamespacedPodSelector(oldObj.ObjectMeta.Labels, ns.namespace.ObjectMeta.Labels)
+			newMatch := ps.matchesNamespacedPodSelector(newObj.ObjectMeta.Labels, ns.namespace.ObjectMeta.Labels)
+			if oldMatch == newMatch && oldObj.Status.PodIP == newObj.Status.PodIP {
+				continue
+			}
+			if oldMatch {
+				if err := ps.delEntry(oldObj.ObjectMeta.UID, oldObj.Status.PodIP); err != nil {
+					return err
+				}
+			}
+			if newMatch {
+				if err := ps.addEntry(newObj.ObjectMeta.UID, newObj.Status.PodIP, podComment(newObj)); err != nil {
+					return err
+				}
+			}
+		}
 	}
 
 	return nil
@@ -303,24 +324,25 @@ func (ns *ns) deletePod(obj *coreapi.Pod) error {
 		return nil
 	}
 
-	if ns.checkLocalPod(obj) {
-		ns.ips.DelEntry(obj.ObjectMeta.UID, LocalIpset, obj.Status.PodIP)
-	}
-
 	if err := ns.ips.DelEntry(obj.ObjectMeta.UID, ns.ingressDefaultAllowIPSet, obj.Status.PodIP); err != nil {
 		return err
 	}
 	if err := ns.ips.DelEntry(obj.ObjectMeta.UID, ns.egressDefaultAllowIPSet, obj.Status.PodIP); err != nil {
 		return err
 	}
-
-	return ns.podSelectors.delFromMatching(obj.ObjectMeta.UID, obj.ObjectMeta.Labels, obj.Status.PodIP)
+	if err := ns.podSelectors.delFromMatchingPodSelector(obj.ObjectMeta.UID, obj.ObjectMeta.Labels, obj.Status.PodIP); err != nil {
+		return err
+	}
+	if err := ns.namespacedPodsSelectors.delFromMatchingNamespacedPodSelector(obj.ObjectMeta.UID, obj.ObjectMeta.Labels, ns.namespace.ObjectMeta.Labels, obj.Status.PodIP); err != nil {
+		return err
+	}
+	return nil
 }
 
 func (ns *ns) addNetworkPolicy(obj interface{}) error {
 	// Analyse policy, determine which rules and ipsets are required
 
-	uid, rules, nsSelectors, podSelectors, ipBlocks, err := ns.analyse(obj)
+	uid, rules, nsSelectors, podSelectors, namespacedPodsSelectors, ipBlocks, err := ns.analyse(obj)
 	if err != nil {
 		return err
 	}
@@ -332,6 +354,9 @@ func (ns *ns) addNetworkPolicy(obj interface{}) error {
 	if err := ns.podSelectors.provision(uid, nil, podSelectors); err != nil {
 		return err
 	}
+	if err := ns.namespacedPodsSelectors.provision(uid, nil, namespacedPodsSelectors); err != nil {
+		return err
+	}
 	if err := ns.ipBlocks.provision(uid, nil, ipBlocks); err != nil {
 		return err
 	}
@@ -340,11 +365,11 @@ func (ns *ns) addNetworkPolicy(obj interface{}) error {
 
 func (ns *ns) updateNetworkPolicy(oldObj, newObj interface{}) error {
 	// Analyse the old and the new policy so we can determine differences
-	oldUID, oldRules, oldNsSelectors, oldPodSelectors, oldIPBlocks, err := ns.analyse(oldObj)
+	oldUID, oldRules, oldNsSelectors, oldPodSelectors, oldNamespacedPodsSelectors, oldIPBlocks, err := ns.analyse(oldObj)
 	if err != nil {
 		return err
 	}
-	newUID, newRules, newNsSelectors, newPodSelectors, newIPBlocks, err := ns.analyse(newObj)
+	newUID, newRules, newNsSelectors, newPodSelectors, newNamespacedPodsSelectors, newIPBlocks, err := ns.analyse(newObj)
 	if err != nil {
 		return err
 	}
@@ -362,6 +387,9 @@ func (ns *ns) updateNetworkPolicy(oldObj, newObj interface{}) error {
 	if err := ns.podSelectors.deprovision(oldUID, oldPodSelectors, newPodSelectors); err != nil {
 		return err
 	}
+	if err := ns.namespacedPodsSelectors.deprovision(oldUID, oldNamespacedPodsSelectors, newNamespacedPodsSelectors); err != nil {
+		return err
+	}
 	if err := ns.ipBlocks.deprovision(oldUID, oldIPBlocks, newIPBlocks); err != nil {
 		return err
 	}
@@ -371,6 +399,9 @@ func (ns *ns) updateNetworkPolicy(oldObj, newObj interface{}) error {
 	if err := ns.podSelectors.provision(oldUID, oldPodSelectors, newPodSelectors); err != nil {
 		return err
 	}
+	if err := ns.namespacedPodsSelectors.provision(oldUID, oldNamespacedPodsSelectors, newNamespacedPodsSelectors); err != nil {
+		return err
+	}
 	if err := ns.ipBlocks.provision(oldUID, oldIPBlocks, newIPBlocks); err != nil {
 		return err
 	}
@@ -379,7 +410,7 @@ func (ns *ns) updateNetworkPolicy(oldObj, newObj interface{}) error {
 
 func (ns *ns) deleteNetworkPolicy(obj interface{}) error {
 	// Analyse network policy to free resources
-	uid, rules, nsSelectors, podSelectors, ipBlocks, err := ns.analyse(obj)
+	uid, rules, nsSelectors, podSelectors, namespacedPodsSelectors, ipBlocks, err := ns.analyse(obj)
 	if err != nil {
 		return err
 	}
@@ -396,6 +427,9 @@ func (ns *ns) deleteNetworkPolicy(obj interface{}) error {
 	if err := ns.podSelectors.deprovision(uid, podSelectors, nil); err != nil {
 		return err
 	}
+	if err := ns.namespacedPodsSelectors.deprovision(uid, namespacedPodsSelectors, nil); err != nil {
+		return err
+	}
 	if err := ns.ipBlocks.deprovision(uid, ipBlocks, nil); err != nil {
 		return err
 	}
@@ -469,7 +503,7 @@ func (ns *ns) addNamespace(obj *coreapi.Namespace) error {
 	}
 
 	// Add namespace ipset to matching namespace selectors
-	_, _, err := ns.nsSelectors.addToMatching(obj.ObjectMeta.UID, obj.ObjectMeta.Labels, string(ns.allPods.ipsetName), namespaceComment(ns))
+	err := ns.nsSelectors.addToMatchingNamespaceSelector(obj.ObjectMeta.UID, obj.ObjectMeta.Labels, string(ns.allPods.ipsetName), namespaceComment(ns))
 	return err
 }
 
@@ -479,8 +513,8 @@ func (ns *ns) updateNamespace(oldObj, newObj *coreapi.Namespace) error {
 	// Re-evaluate namespace selector membership if labels have changed
 	if !equals(oldObj.ObjectMeta.Labels, newObj.ObjectMeta.Labels) {
 		for _, selector := range ns.nsSelectors.entries {
-			oldMatch := selector.matches(oldObj.ObjectMeta.Labels)
-			newMatch := selector.matches(newObj.ObjectMeta.Labels)
+			oldMatch := selector.matchesNamespaceSelector(oldObj.ObjectMeta.Labels)
+			newMatch := selector.matchesNamespaceSelector(newObj.ObjectMeta.Labels)
 			if oldMatch == newMatch {
 				continue
 			}
@@ -509,7 +543,7 @@ func (ns *ns) deleteNamespace(obj *coreapi.Namespace) error {
 	}
 
 	// Remove namespace ipset from any matching namespace selectors
-	err := ns.nsSelectors.delFromMatching(obj.ObjectMeta.UID, obj.ObjectMeta.Labels, string(ns.allPods.ipsetName))
+	err := ns.nsSelectors.delFromMatchingNamespaceSelector(obj.ObjectMeta.UID, obj.ObjectMeta.Labels, string(ns.allPods.ipsetName))
 	if err != nil {
 		return err
 	}
@@ -561,7 +595,7 @@ func (ns *ns) defaultAllowIPSetName(pt policyType) ipset.Name {
 func (ns *ns) analyse(obj interface{}) (
 	uid types.UID,
 	rules map[string]*ruleSpec,
-	nsSelectors, podSelectors map[string]*selectorSpec,
+	nsSelectors, podSelectors, namespacedPodsSelectors map[string]*selectorSpec,
 	ipBlocks map[string]*ipBlockSpec,
 	err error) {
 
@@ -577,7 +611,7 @@ func (ns *ns) analyse(obj interface{}) (
 	ns.policies[uid] = obj
 
 	// Analyse policy, determine which rules and ipsets are required
-	rules, nsSelectors, podSelectors, ipBlocks, err = ns.analysePolicy(obj.(*networkingv1.NetworkPolicy))
+	rules, nsSelectors, podSelectors, namespacedPodsSelectors, ipBlocks, err = ns.analysePolicy(obj.(*networkingv1.NetworkPolicy))
 	if err != nil {
 		return
 	}

npc/selector.go

@@ -12,25 +12,39 @@ import (
 )
 
 type selectorSpec struct {
-	key         string          // string representation (for hash keying/equality comparison)
-	selector    labels.Selector // k8s Selector object (for matching)
-	policyTypes []policyType    // If non-empty, then selectorSpec is a target selector for given policyTypes.
+	key               string          // string representation (for hash keying/equality comparison)
+	podSelector       labels.Selector // k8s Selector object (for matching pods)
+	namespaceSelector labels.Selector // k8s Selector object (for matching namespaces)
+	policyTypes       []policyType    // If non-empty, then selectorSpec is a target selector for given policyTypes.
 
 	ipsetType ipset.Type // type of ipset to provision
 	ipsetName ipset.Name // generated ipset name
 	nsName    string     // Namespace name
 }
 
-func newSelectorSpec(json *metav1.LabelSelector, policyType []policyType, nsName string, ipsetType ipset.Type) (*selectorSpec, error) {
-	selector, err := metav1.LabelSelectorAsSelector(json)
-	if err != nil {
-		return nil, err
+func newSelectorSpec(podJSON *metav1.LabelSelector, namespaceJSON *metav1.LabelSelector, policyType []policyType, nsName string, ipsetType ipset.Type) (*selectorSpec, error) {
+	var podSelector, namespaceSelector labels.Selector
+	var err error
+	var key string
+	if namespaceJSON != nil {
+		namespaceSelector, err = metav1.LabelSelectorAsSelector(namespaceJSON)
+		if err != nil {
+			return nil, err
+		}
+		key = namespaceSelector.String()
+	}
+	if podJSON != nil {
+		podSelector, err = metav1.LabelSelectorAsSelector(podJSON)
+		if err != nil {
+			return nil, err
+		}
+		key = key + podSelector.String()
 	}
-	key := selector.String()
 	return &selectorSpec{
-		key:         key,
-		selector:    selector,
-		policyTypes: policyType,
+		key:               key,
+		podSelector:       podSelector,
+		namespaceSelector: namespaceSelector,
+		policyTypes:       policyType,
 		// We prefix the selector string with the namespace name when generating
 		// the shortname because you can specify the same selector in multiple
 		// namespaces - we need those to map to distinct ipsets
@@ -61,8 +75,16 @@ type selector struct {
 	spec *selectorSpec
 }
 
-func (s *selector) matches(labelMap map[string]string) bool {
-	return s.spec.selector.Matches(labels.Set(labelMap))
+func (s *selector) matchesPodSelector(podsLabelMap map[string]string) bool {
+	return s.spec.podSelector.Matches(labels.Set(podsLabelMap))
+}
+
+func (s *selector) matchesNamespaceSelector(namespaceLabelMap map[string]string) bool {
+	return s.spec.namespaceSelector.Matches(labels.Set(namespaceLabelMap))
+}
+
+func (s *selector) matchesNamespacedPodSelector(podsLabelMap, namespaceLabelMap map[string]string) bool {
+	return s.spec.podSelector.Matches(labels.Set(podsLabelMap)) && s.spec.namespaceSelector.Matches(labels.Set(namespaceLabelMap))
 }
 
 func (s *selector) addEntry(user types.UID, entry string, comment string) error {
@@ -105,11 +127,11 @@ func newSelectorSet(ips ipset.Interface, onNewSelector selectorFn, onNewTargetSe
 		targetSelectorsCount: make(map[string]map[policyType]int)}
 }
 
-func (ss *selectorSet) addToMatching(user types.UID, labelMap map[string]string, entry string, comment string) (bool, bool, error) {
+func (ss *selectorSet) addToMatchingPodSelector(user types.UID, podLabelsMap map[string]string, entry string, comment string) (bool, bool, error) {
 	foundIngress := false
 	foundEgress := false
 	for _, s := range ss.entries {
-		if s.matches(labelMap) {
+		if s.matchesPodSelector(podLabelsMap) {
 			if ss.targetSelectorExist(s, policyTypeIngress) {
 				foundIngress = true
 			}
@@ -124,9 +146,53 @@ func (ss *selectorSet) addToMatching(user types.UID, labelMap map[string]string,
 	return foundIngress, foundEgress, nil
 }
 
-func (ss *selectorSet) delFromMatching(user types.UID, labelMap map[string]string, entry string) error {
+func (ss *selectorSet) addToMatchingNamespaceSelector(user types.UID, namespaceLabelsMap map[string]string, entry string, comment string) error {
 	for _, s := range ss.entries {
-		if s.matches(labelMap) {
+		if s.matchesNamespaceSelector(namespaceLabelsMap) {
+			if err := s.addEntry(user, entry, comment); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (ss *selectorSet) addToMatchingNamespacedPodSelector(user types.UID, podLabelsMap map[string]string, namespaceLabelsMap map[string]string, entry string, comment string) error {
+	for _, s := range ss.entries {
+		if s.matchesNamespacedPodSelector(podLabelsMap, namespaceLabelsMap) {
+			if err := s.addEntry(user, entry, comment); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (ss *selectorSet) delFromMatchingPodSelector(user types.UID, podLabelsMap map[string]string, entry string) error {
+	for _, s := range ss.entries {
+		if s.matchesPodSelector(podLabelsMap) {
+			if err := s.delEntry(user, entry); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (ss *selectorSet) delFromMatchingNamespaceSelector(user types.UID, namespaceLabelsMap map[string]string, entry string) error {
+	for _, s := range ss.entries {
+		if s.matchesNamespaceSelector(namespaceLabelsMap) {
+			if err := s.delEntry(user, entry); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (ss *selectorSet) delFromMatchingNamespacedPodSelector(user types.UID, podLabelsMap map[string]string, namespaceLabelsMap map[string]string, entry string) error {
+	for _, s := range ss.entries {
+		if s.matchesNamespacedPodSelector(podLabelsMap, namespaceLabelsMap) {
 			if err := s.delEntry(user, entry); err != nil {
 				return err
 			}

plugin/net/cni.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"strings"
 
 	"github.com/containernetworking/cni/pkg/ipam"
 	"github.com/containernetworking/cni/pkg/skel"
@@ -206,7 +207,13 @@ func (c *CNIPlugin) CmdDel(args *skel.CmdArgs) error {
 		err = ipam.ExecDel(conf.IPAM.Type, args.StdinData)
 	}
 	if err != nil {
+		if strings.Contains(err.Error(), fmt.Sprintf("Delete: no addresses for %s", args.ContainerID)) {
+			// ignore the error in the case where there are no IP addresses associated with container
+			fmt.Fprintln(os.Stderr, "weave-cni:", fmt.Sprintf("Delete: no addresses for %s", args.ContainerID))
+			return nil
+		}
 		logOnStderr(fmt.Errorf("unable to release IP address: %s", err))
+		return err
 	}
 	return nil
 }

plugin/net/driver.go

@@ -39,9 +39,10 @@ type driver struct {
 	// used only by plugin-v2
 	forceMulticast bool
 	networks       map[string]network
+	procPath       string
 }
 
-func New(client *docker.Client, weave *weaveapi.Client, name, scope string, dns, isPluginV2, forceMulticast bool) (skel.Driver, error) {
+func New(client *docker.Client, weave *weaveapi.Client, name, scope string, dns, isPluginV2, forceMulticast bool, procPath string) (skel.Driver, error) {
 	driver := &driver{
 		name:       name,
 		scope:      scope,
@@ -51,6 +52,7 @@ func New(client *docker.Client, weave *weaveapi.Client, name, scope string, dns,
 		// make sure that it's used only by plugin-v2
 		forceMulticast: isPluginV2 && forceMulticast,
 		networks:       make(map[string]network),
+		procPath:       procPath,
 	}
 
 	// Do not start watcher in the case of plugin v2, which prevents us from
@@ -134,7 +136,7 @@ func (driver *driver) CreateEndpoint(create *api.CreateEndpointRequest) (*api.Cr
 
 	// create veths. note we assume endpoint IDs are unique in the first 9 chars
 	name, peerName := vethPair(create.EndpointID)
-	if _, err := weavenet.CreateAndAttachVeth(name, peerName, weavenet.WeaveBridgeName, 0, false, true, nil); err != nil {
+	if _, err := weavenet.CreateAndAttachVeth(driver.procPath, name, peerName, weavenet.WeaveBridgeName, 0, false, true, nil); err != nil {
 		return nil, driver.error("JoinEndpoint", "%s", err)
 	}
 

plugin/net/watcher.go

@@ -43,14 +43,9 @@ func (w *watcher) ContainerStarted(id string) {
 					w.driver.warn("ContainerStarted", "unable to register %s with weaveDNS: %s", id, err)
 				}
 			}
-			rootDir := "/"
-			if w.driver.isPluginV2 {
-				// We bind mount host's /proc to /host/proc for plugin-v2
-				rootDir = "/host"
-			}
-			netNSPath := weavenet.NSPathByPidWithRoot(rootDir, info.State.Pid)
+			netNSPath := weavenet.NSPathByPidWithProc(w.driver.procPath, info.State.Pid)
 			if err := weavenet.WithNetNSByPath(netNSPath, func() error {
-				return weavenet.ConfigureARP(weavenet.VethName, rootDir)
+				return weavenet.ConfigureARP(weavenet.VethName, w.driver.procPath)
 			}); err != nil {
 				w.driver.warn("ContainerStarted", "unable to configure interfaces: %s", err)
 			}

plugin/plugin.go

@@ -32,6 +32,7 @@ type Config struct {
 	EnableV2Multicast bool
 	DNS               bool
 	DefaultSubnet     string
+	ProcPath          string // path to reach host /proc filesystem
 }
 
 type Plugin struct {
@@ -62,7 +63,7 @@ func (plugin *Plugin) run(dockerClient *docker.Client, weave *weaveapi.Client, r
 	endChan := make(chan error, 1)
 
 	if plugin.Socket != "" {
-		globalListener, err := listenAndServe(dockerClient, weave, plugin.Socket, endChan, "global", false, plugin.DNS, plugin.EnableV2, plugin.EnableV2Multicast)
+		globalListener, err := listenAndServe(dockerClient, weave, plugin.Socket, endChan, "global", false, plugin.DNS, plugin.EnableV2, plugin.EnableV2Multicast, plugin.ProcPath)
 		if err != nil {
 			return err
 		}
@@ -70,7 +71,7 @@ func (plugin *Plugin) run(dockerClient *docker.Client, weave *weaveapi.Client, r
 		defer globalListener.Close()
 	}
 	if plugin.MeshSocket != "" {
-		meshListener, err := listenAndServe(dockerClient, weave, plugin.MeshSocket, endChan, "local", true, plugin.DNS, plugin.EnableV2, plugin.EnableV2Multicast)
+		meshListener, err := listenAndServe(dockerClient, weave, plugin.MeshSocket, endChan, "local", true, plugin.DNS, plugin.EnableV2, plugin.EnableV2Multicast, plugin.ProcPath)
 		if err != nil {
 			return err
 		}
@@ -87,7 +88,7 @@ func (plugin *Plugin) run(dockerClient *docker.Client, weave *weaveapi.Client, r
 	return <-endChan
 }
 
-func listenAndServe(dockerClient *docker.Client, weave *weaveapi.Client, address string, endChan chan<- error, scope string, withIpam, dns bool, isPluginV2, forceMulticast bool) (net.Listener, error) {
+func listenAndServe(dockerClient *docker.Client, weave *weaveapi.Client, address string, endChan chan<- error, scope string, withIpam, dns bool, isPluginV2, forceMulticast bool, procPath string) (net.Listener, error) {
 	var name string
 	if isPluginV2 {
 		name = pluginV2Name
@@ -95,7 +96,7 @@ func listenAndServe(dockerClient *docker.Client, weave *weaveapi.Client, address
 		name = pluginNameFromAddress(address)
 	}
 
-	d, err := netplugin.New(dockerClient, weave, name, scope, dns, isPluginV2, forceMulticast)
+	d, err := netplugin.New(dockerClient, weave, name, scope, dns, isPluginV2, forceMulticast, procPath)
 	if err != nil {
 		return nil, err
 	}

prog/kube-utils/annotations.go

@@ -2,13 +2,10 @@
 In order to keep track of active weave peers, we use annotations on the Kubernetes cluster.
 
 Kubernetes uses etcd to distribute and synchronise these annotations so we don't have to.
-
-This module deals with operations on the peerlist backed by Kubernetes' annotation mechanism.
 */
 package main
 
 import (
-	"encoding/json"
 	"log"
 	"time"
 
@@ -29,43 +26,11 @@ type configMapAnnotations struct {
 	cm            *v1.ConfigMap
 }
 
-func newConfigMapAnnotations(ns string, configMapName string, clientset *kubernetes.Clientset) *configMapAnnotations {
+func newConfigMapAnnotations(ns string, configMapName string, c kubernetes.Interface) *configMapAnnotations {
 	return &configMapAnnotations{
 		Namespace:     ns,
 		ConfigMapName: configMapName,
-		Client:        clientset.CoreV1(),
-	}
-}
-
-type peerList struct {
-	Peers []peerInfo
-}
-
-type peerInfo struct {
-	PeerName string // Weave internal unique ID
-	NodeName string // Kubernetes node name
-}
-
-func (pl peerList) contains(peerName string) bool {
-	for _, peer := range pl.Peers {
-		if peer.PeerName == peerName {
-			return true
-		}
-	}
-	return false
-}
-
-func (pl *peerList) add(peerName string, name string) {
-	pl.Peers = append(pl.Peers, peerInfo{PeerName: peerName, NodeName: name})
-}
-
-func (pl *peerList) remove(peerNameToRemove string) {
-	for i := 0; i < len(pl.Peers); {
-		if pl.Peers[i].PeerName == peerNameToRemove {
-			pl.Peers = append(pl.Peers[:i], pl.Peers[i+1:]...)
-		} else {
-			i++
-		}
+		Client:        c.CoreV1(),
 	}
 }
 
@@ -111,27 +76,6 @@ func (cml *configMapAnnotations) Init() error {
 	return nil
 }
 
-func (cml *configMapAnnotations) GetPeerList() (*peerList, error) {
-	var record peerList
-	if cml.cm == nil {
-		return nil, errors.New("endpoint not initialized, call Init first")
-	}
-	if recordBytes, found := cml.cm.Annotations[KubePeersAnnotationKey]; found {
-		if err := json.Unmarshal([]byte(recordBytes), &record); err != nil {
-			return nil, err
-		}
-	}
-	return &record, nil
-}
-
-func (cml *configMapAnnotations) UpdatePeerList(list peerList) error {
-	recordBytes, err := json.Marshal(list)
-	if err != nil {
-		return err
-	}
-	return cml.UpdateAnnotation(KubePeersAnnotationKey, string(recordBytes))
-}
-
 // Clean up a string so it meets the Kubernetes requiremements for Annotation keys:
 // name part must consist of alphanumeric characters, '-', '_' or '.', and must
 // start and end with an alphanumeric character (e.g. 'MyName', or 'my.name', or '123-abc')
@@ -152,33 +96,33 @@ func (cml *configMapAnnotations) GetAnnotation(key string) (string, bool) {
 }
 
 func (cml *configMapAnnotations) UpdateAnnotation(key, value string) (err error) {
-	if cml.cm == nil {
+	if cml.cm == nil || cml.cm.Annotations == nil {
 		return errors.New("endpoint not initialized, call Init first")
 	}
 	// speculatively change the state, then replace with whatever comes back
-	// from Update(), which will be the latest on the server whatever happened
+	// from Update(), which will be the latest on the server, or nil if error
 	cml.cm.Annotations[cleanKey(key)] = value
 	cml.cm, err = cml.Client.ConfigMaps(cml.Namespace).Update(cml.cm)
 	return err
 }
 
 func (cml *configMapAnnotations) RemoveAnnotation(key string) (err error) {
-	if cml.cm == nil {
+	if cml.cm == nil || cml.cm.Annotations == nil {
 		return errors.New("endpoint not initialized, call Init first")
 	}
 	// speculatively change the state, then replace with whatever comes back
-	// from Update(), which will be the latest on the server whatever happened
+	// from Update(), which will be the latest on the server, or nil if error
 	delete(cml.cm.Annotations, cleanKey(key))
 	cml.cm, err = cml.Client.ConfigMaps(cml.Namespace).Update(cml.cm)
 	return err
 }
 
 func (cml *configMapAnnotations) RemoveAnnotationsWithValue(valueToRemove string) (err error) {
-	if cml.cm == nil {
+	if cml.cm == nil || cml.cm.Annotations == nil {
 		return errors.New("endpoint not initialized, call Init first")
 	}
 	// speculatively change the state, then replace with whatever comes back
-	// from Update(), which will be the latest on the server whatever happened
+	// from Update(), which will be the latest on the server, or nil if error
 	for key, value := range cml.cm.Annotations {
 		if value == valueToRemove {
 			delete(cml.cm.Annotations, key) // don't need to clean this key as it came from the map

prog/kube-utils/kube_test.go

@@ -0,0 +1,30 @@
+// Adapted from k8s.io/client-go/kubernetes/fake to use a modified ObjectTracker
+package main
+
+import (
+	"k8s.io/apimachinery/pkg/runtime"
+	serializer "k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes/fake"
+	"k8s.io/client-go/testing"
+
+	weavetracker "github.com/weaveworks/weave/testing/kubernetes/testing"
+)
+
+var scheme = runtime.NewScheme()
+var codecs = serializer.NewCodecFactory(scheme)
+
+func NewSimpleClientset(objects ...runtime.Object) *fake.Clientset {
+	o := weavetracker.NewObjectTracker(scheme, codecs.UniversalDecoder())
+	for _, obj := range objects {
+		if err := o.Add(obj); err != nil {
+			panic(err)
+		}
+	}
+
+	ret := fake.Clientset{}
+	ret.AddReactor("*", "*", weavetracker.ObjectReaction(o))
+	ret.AddWatchReactor("*", testing.DefaultWatchReactor(watch.NewFake(), nil))
+
+	return &ret
+}

prog/kube-utils/main.go

@@ -8,12 +8,20 @@ package main
 import (
 	"flag"
 	"fmt"
+	"math/rand"
 	"net"
 	"os"
+	"os/signal"
+	"syscall"
+	"time"
 
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
 	api "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
 
 	weaveapi "github.com/weaveworks/weave/api"
 	"github.com/weaveworks/weave/common"
@@ -25,7 +33,7 @@ type nodeInfo struct {
 }
 
 // return the IP addresses of all nodes in the cluster
-func getKubePeers(c *kubernetes.Clientset) ([]nodeInfo, error) {
+func getKubePeers(c kubernetes.Interface, includeWithNoIPAddr bool) ([]nodeInfo, error) {
 	nodeList, err := c.CoreV1().Nodes().List(api.ListOptions{})
 	if err != nil {
 		return nil, err
@@ -48,71 +56,74 @@ func getKubePeers(c *kubernetes.Clientset) ([]nodeInfo, error) {
 
 		// Fallback for cases where a Node has an ExternalIP but no InternalIP
 		if internalIP != "" {
+			// exclude self from the list of peers this node will peer with
+			if isLocalNodeIP(internalIP) {
+				continue
+			}
 			addresses = append(addresses, nodeInfo{name: peer.Name, addr: internalIP})
 		} else if externalIP != "" {
 			addresses = append(addresses, nodeInfo{name: peer.Name, addr: externalIP})
+		} else if includeWithNoIPAddr {
+			addresses = append(addresses, nodeInfo{name: peer.Name, addr: ""})
 		}
 	}
 	return addresses, nil
 }
 
-const (
-	configMapName      = "weave-net"
-	configMapNamespace = "kube-system"
-)
-
-// update the list of all peers that have gone through this code path
-func addMyselfToPeerList(cml *configMapAnnotations, c *kubernetes.Clientset, peerName, name string) (*peerList, error) {
-	var list *peerList
-	err := cml.LoopUpdate(func() error {
-		var err error
-		list, err = cml.GetPeerList()
-		if err != nil {
-			return err
+// returns true if given IP matches with one of the local IP's
+func isLocalNodeIP(ip string) bool {
+	addrs, err := netlink.AddrList(nil, unix.AF_INET)
+	if err != nil {
+		return false
+	}
+	for _, addr := range addrs {
+		if addr.Peer.IP.String() == ip {
+			return true
 		}
-		if !list.contains(peerName) {
-			list.add(peerName, name)
-			err = cml.UpdatePeerList(*list)
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	})
-	return list, err
+	}
+	return false
 }
 
-func checkIamInPeerList(cml *configMapAnnotations, c *kubernetes.Clientset, peerName string) (bool, error) {
-	if err := cml.Init(); err != nil {
-		return false, err
-	}
-	list, err := cml.GetPeerList()
-	if err != nil {
-		return false, err
-	}
-	common.Log.Debugf("[kube-peers] Checking peer %q against list %v", peerName, list)
-	return list.contains(peerName), nil
+// (minimal, incomplete) interface so weaver can be mocked for testing.
+type weaveClient interface {
+	RmPeer(peerName string) (string, error)
 }
 
 // For each of those peers that is no longer listed as a node by
 // Kubernetes, remove it from Weave IPAM
-func reclaimRemovedPeers(weave *weaveapi.Client, cml *configMapAnnotations, nodes []nodeInfo, myPeerName string) error {
+func reclaimRemovedPeers(kube kubernetes.Interface, cml *configMapAnnotations, myPeerName, myNodeName string) error {
+	weave := weaveapi.NewClient(os.Getenv("WEAVE_HTTP_ADDR"), common.Log)
 	for loopsWhenNothingChanged := 0; loopsWhenNothingChanged < 3; loopsWhenNothingChanged++ {
-		if err := cml.Init(); err != nil {
-			return err
-		}
 		// 1. Compare peers stored in the peerList against all peers reported by k8s now.
-		storedPeerList, err := cml.GetPeerList()
+		storedPeerList, err := getRefreshedPeerList(cml)
 		if err != nil {
 			return err
 		}
+		nodes, err := getKubePeers(kube, true)
+		if err != nil {
+			return err
+		}
+		nodeSet := make(map[string]struct{}, len(nodes))
+		for _, node := range nodes {
+			nodeSet[node.name] = struct{}{}
+		}
 		peerMap := make(map[string]peerInfo, len(storedPeerList.Peers))
 		for _, peer := range storedPeerList.Peers {
-			peerMap[peer.NodeName] = peer
+			if peer.PeerName != myPeerName {
+				peerMap[peer.PeerName] = peer
+			}
 		}
-		for _, node := range nodes {
-			delete(peerMap, node.name)
+		// remove entries from the peer map that are current nodes
+		for key, peer := range peerMap {
+			if _, found := nodeSet[peer.NodeName]; found {
+				// unless they have a duplicate of my NodeName but are not me
+				if peer.NodeName == myNodeName && peer.PeerName != myPeerName {
+					continue
+				}
+				delete(peerMap, key)
+			}
 		}
+		// so the remainder is everything we want to clean up
 		common.Log.Debugln("[kube-peers] Nodes that have disappeared:", peerMap)
 		if len(peerMap) == 0 {
 			break
@@ -123,63 +134,178 @@ func reclaimRemovedPeers(weave *weaveapi.Client, cml *configMapAnnotations, node
 				common.Log.Warnln("[kube-peers] not removing myself", peer)
 				continue
 			}
-			common.Log.Debugln("[kube-peers] Preparing to remove disappeared peer", peer)
-			okToRemove := false
-			// 3. Check if there is an existing annotation with key X
-			if existingAnnotation, found := cml.GetAnnotation(KubePeersPrefix + peer.PeerName); found {
-				common.Log.Debugln("[kube-peers] Existing annotation", existingAnnotation)
-				// 4.   If annotation already contains my identity, ok;
-				if existingAnnotation == myPeerName {
-					okToRemove = true
-				}
-			} else {
-				// 5.   If non-existent, write an annotation with key X and contents "my identity"
-				common.Log.Debugln("[kube-peers] Noting I plan to remove ", peer.PeerName)
-				if err := cml.UpdateAnnotation(KubePeersPrefix+peer.PeerName, myPeerName); err == nil {
-					okToRemove = true
-				} else {
-					common.Log.Debugln("[kube-peers] error from UpdateAnnotation: ", err)
-				}
+			changed, err := reclaimPeer(weave, cml, peer.PeerName, myPeerName)
+			if err != nil {
+				return err
 			}
-			if okToRemove {
-				// 6.   If step 4 or 5 succeeded, rmpeer X
-				result, err := weave.RmPeer(peer.PeerName)
-				common.Log.Infof("[kube-peers] rmpeer of %s: %s", peer.PeerName, result)
-				if err != nil {
-					return err
-				}
+			if changed {
 				loopsWhenNothingChanged = 0
-				cml.LoopUpdate(func() error {
-					// 7aa.   Remove any annotations Z* that have contents X
-					if err := cml.RemoveAnnotationsWithValue(peer.PeerName); err != nil {
-						return err
-					}
-					// 7a.    Remove X from peerList
-					storedPeerList.remove(peer.PeerName)
-					if err := cml.UpdatePeerList(*storedPeerList); err != nil {
-						return err
-					}
-					// 7b.    Remove annotation with key X
-					return cml.RemoveAnnotation(KubePeersPrefix + peer.PeerName)
-				})
 			}
-			// 8.   If step 5 failed due to optimistic lock conflict, stop: someone else is handling X
-
-			// Step 3-5 is to protect against two simultaneous rmpeers of X
-			// Step 4 is to pick up again after a restart between step 5 and step 7b
-			// If the peer doing the reclaim disappears between steps 5 and 7a, then someone will clean it up in step 7aa
-			// If peer doing the reclaim disappears forever between 7a and 7b then we get a dangling annotation
-			// This should be sufficiently rare that we don't care.
 		}
 
 		// 9. Go back to step 1 until there is no difference between the two sets
 		// (or we hit the counter that says we've been round the loop 3 times and nothing happened)
 	}
+	return nil
+}
+
+// Attempt to reclaim the IP addresses owned by peerName, using the
+// Kubernetes api-server as a point of consensus so that only one peer
+// actions the reclaim.
+// Return a bool to show whether we attempted to change anything,
+// and an error if something went wrong.
+func reclaimPeer(weave weaveClient, cml *configMapAnnotations, peerName string, myPeerName string) (changed bool, err error) {
+	common.Log.Debugln("[kube-peers] Preparing to remove disappeared peer", peerName)
+	okToRemove := false
+	nonExistentPeer := false
+
+	// Re-read status from Kubernetes; speculative updates may leave it un-set
+	if err := cml.Init(); err != nil {
+		return false, err
+	}
+
+	// 3. Check if there is an existing annotation with key X
+	existingAnnotation, found := cml.GetAnnotation(KubePeersPrefix + peerName)
+	if found {
+		common.Log.Debugln("[kube-peers] Existing annotation", existingAnnotation)
+		// 4.   If annotation already contains my identity, ok;
+		if existingAnnotation == myPeerName {
+			okToRemove = true
+		} else {
+			storedPeerList, err := cml.GetPeerList()
+			if err != nil {
+				return false, err
+			}
+			// handle an edge case where peer claimed to own the action to reclaim but no longer
+			// exists hence lock persists foever
+			if !storedPeerList.contains(existingAnnotation) {
+				nonExistentPeer = true
+				common.Log.Debugln("[kube-peers] Existing annotation", existingAnnotation, " has a non-existent peer so owning the reclaim action")
+			}
+		}
+	}
+	if !found || nonExistentPeer {
+		// 5.   If non-existent, write an annotation with key X and contents "my identity"
+		common.Log.Debugln("[kube-peers] Noting I plan to remove ", peerName)
+		if err := cml.UpdateAnnotation(KubePeersPrefix+peerName, myPeerName); err == nil {
+			okToRemove = true
+		} else {
+			common.Log.Errorln("[kube-peers] error from UpdateAnnotation: ", err)
+		}
+	}
+	if !okToRemove {
+		return false, nil
+	}
+	// 6.   If step 4 or 5 succeeded, rmpeer X
+	// we will try once more to see if the peerName has been removed from the annotation
+	// by another weave pod
+	storedPeerList, err := getRefreshedPeerList(cml)
+	if storedPeerList.contains(peerName) {
+		result, err := weave.RmPeer(peerName)
+		common.Log.Infof("[kube-peers] rmpeer of %s: %s", peerName, result)
+		if err != nil {
+			return false, err
+		}
+	}
+	err = cml.LoopUpdate(func() error {
+		// 7aa.   Remove any annotations Z* that have contents X
+		if err := cml.RemoveAnnotationsWithValue(peerName); err != nil {
+			return err
+		}
+		// 7a.    Remove X from peerList
+		storedPeerList, err := cml.GetPeerList()
+		if err != nil {
+			return err
+		}
+		if storedPeerList.contains(peerName) {
+			storedPeerList.remove(peerName)
+			if err := cml.UpdatePeerList(*storedPeerList); err != nil {
+				return err
+			}
+		} else {
+			common.Log.Infof("[kube-peers] annotation was removed already for %s", peerName)
+		}
+		// 7b.    Remove annotation with key X
+		return cml.RemoveAnnotation(KubePeersPrefix + peerName)
+	})
+	// 8.   If step 5 failed due to optimistic lock conflict, stop: someone else is handling X
+
+	// Step 3-5 is to protect against two simultaneous rmpeers of X
+	// Step 4 is to pick up again after a restart between step 5 and step 7b
+	// If the peer doing the reclaim disappears between steps 5 and 7a, then someone will clean it up in step 7aa
+	// If peer doing the reclaim disappears forever between 7a and 7b then we get a dangling annotation
+	// This should be sufficiently rare that we don't care.
+
 	// Question: Should we narrow step 2 by checking against Weave Net IPAM?
 	// i.e. If peer X owns any address space and is marked unreachable, we want to rmpeer X
+	return true, err
+}
+
+// resetPeers replaces the peers list with current set of peers
+func resetPeers(kube kubernetes.Interface) error {
+	nodes, err := getKubePeers(kube, false)
+	if err != nil {
+		return err
+	}
+	peerList := make([]string, 0)
+	for _, node := range nodes {
+		peerList = append(peerList, node.addr)
+	}
+	weave := weaveapi.NewClient(os.Getenv("WEAVE_HTTP_ADDR"), common.Log)
+	err = weave.ReplacePeers(peerList)
+	if err != nil {
+		return err
+	}
 	return nil
 }
 
+// regiesters with Kubernetes API server for node delete events. Node delete event handler
+// invokes reclaimRemovedPeers to remove it from IPAM so that IP space is reclaimed
+func registerForNodeUpdates(client *kubernetes.Clientset, stopCh <-chan struct{}, nodeName, peerName string) {
+	informerFactory := informers.NewSharedInformerFactory(client, 0)
+	nodeInformer := informerFactory.Core().V1().Nodes().Informer()
+	common.Log.Debugln("registering for updates for node delete events")
+	nodeInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+		DeleteFunc: func(obj interface{}) {
+			var name string
+			if apiObj, ok := obj.(api.Object); ok {
+				name = apiObj.GetName()
+			} else {
+				name = fmt.Sprintf("%#v", obj)
+			}
+			common.Log.Debugln("Delete event for", name)
+			// add random delay to avoid all nodes acting on node delete event at the same
+			// time leading to contention to use `weave-net` configmap
+			r := rand.Intn(5000)
+			time.Sleep(time.Duration(r) * time.Millisecond)
+
+			cml := newConfigMapAnnotations(configMapNamespace, configMapName, client)
+			err := reclaimRemovedPeers(client, cml, peerName, nodeName)
+			if err != nil {
+				common.Log.Fatalf("[kube-peers] Error while reclaiming space: %v", err)
+			}
+			err = resetPeers(client)
+			if err != nil {
+				common.Log.Fatalf("[kube-peers] Error resetting peer list: %v", err)
+			}
+		},
+	})
+	informerFactory.WaitForCacheSync(stopCh)
+	informerFactory.Start(stopCh)
+}
+
+// getRefreshedPeerList will fetch the latest peer list from the config map annotation
+func getRefreshedPeerList(cml *configMapAnnotations) (*peerList, error) {
+	if err := cml.Init(); err != nil {
+		return nil, err
+	}
+	storedPeerList, err := cml.GetPeerList()
+	if err != nil {
+		return nil, err
+	}
+	return storedPeerList, nil
+}
+
 func main() {
 	var (
 		justReclaim       bool
@@ -188,8 +314,10 @@ func main() {
 		peerName          string
 		nodeName          string
 		logLevel          string
+		runReclaimDaemon  bool
 	)
 	flag.BoolVar(&justReclaim, "reclaim", false, "reclaim IP space from dead peers")
+	flag.BoolVar(&runReclaimDaemon, "run-reclaim-daemon", false, "run background process that reclaim IP space from dead peers ")
 	flag.BoolVar(&justCheck, "check-peer-new", false, "return success if peer name is not stored in annotation")
 	flag.BoolVar(&justSetNodeStatus, "set-node-status", false, "set NodeNetworkUnavailable to false")
 	flag.StringVar(&peerName, "peer-name", "unknown", "name of this Weave Net peer")
@@ -225,10 +353,6 @@ func main() {
 		}
 		return
 	}
-	peers, err := getKubePeers(c)
-	if err != nil {
-		common.Log.Fatalf("[kube-peers] Could not get peers: %v", err)
-	}
 	if justReclaim {
 		cml := newConfigMapAnnotations(configMapNamespace, configMapName, c)
 
@@ -238,13 +362,28 @@ func main() {
 		}
 		common.Log.Infoln("[kube-peers] Added myself to peer list", list)
 
-		weave := weaveapi.NewClient(os.Getenv("WEAVE_HTTP_ADDR"), common.Log)
-		err = reclaimRemovedPeers(weave, cml, peers, peerName)
+		err = reclaimRemovedPeers(c, cml, peerName, nodeName)
 		if err != nil {
 			common.Log.Fatalf("[kube-peers] Error while reclaiming space: %v", err)
 		}
 		return
 	}
+	if runReclaimDaemon {
+		// Handle SIGINT and SIGTERM
+		ch := make(chan os.Signal)
+		signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
+		stopCh := make(chan struct{})
+		rand.Seed(time.Now().UnixNano())
+		registerForNodeUpdates(c, stopCh, nodeName, peerName)
+		<-ch
+		close(stopCh)
+		return
+	}
+
+	peers, err := getKubePeers(c, false)
+	if err != nil {
+		common.Log.Fatalf("[kube-peers] Could not get peers: %v", err)
+	}
 	for _, node := range peers {
 		fmt.Println(node.addr)
 	}

prog/kube-utils/nodestatus.go

@@ -10,7 +10,7 @@ import (
 )
 
 // based on k8s.io/kubernetes/pkg/controller/nodeipam/ipam/adapter.go
-func setNodeNetworkUnavailableFalse(c *kubernetes.Clientset, nodeName string) error {
+func setNodeNetworkUnavailableFalse(c kubernetes.Interface, nodeName string) error {
 	condition := v1.NodeCondition{
 		Type:               v1.NodeNetworkUnavailable,
 		Status:             v1.ConditionFalse,

prog/kube-utils/peerlist.go

@@ -0,0 +1,104 @@
+/*
+This module deals with operations on the peerlist backed by Kubernetes' annotation mechanism.
+*/
+package main
+
+import (
+	"encoding/json"
+
+	"github.com/pkg/errors"
+	"k8s.io/client-go/kubernetes"
+
+	"github.com/weaveworks/weave/common"
+)
+
+const (
+	configMapName      = "weave-net"
+	configMapNamespace = "kube-system"
+)
+
+type peerList struct {
+	Peers []peerInfo
+}
+
+type peerInfo struct {
+	PeerName string // Weave internal unique ID
+	NodeName string // Kubernetes node name
+}
+
+func (pl *peerList) contains(peerName string) bool {
+	for _, peer := range pl.Peers {
+		if peer.PeerName == peerName {
+			return true
+		}
+	}
+	return false
+}
+
+func (pl *peerList) add(peerName string, name string) {
+	pl.Peers = append(pl.Peers, peerInfo{PeerName: peerName, NodeName: name})
+}
+
+func (pl *peerList) remove(peerNameToRemove string) {
+	for i := 0; i < len(pl.Peers); {
+		if pl.Peers[i].PeerName == peerNameToRemove {
+			pl.Peers = append(pl.Peers[:i], pl.Peers[i+1:]...)
+		} else {
+			i++
+		}
+	}
+}
+
+func (cml *configMapAnnotations) GetPeerList() (*peerList, error) {
+	var record peerList
+	if cml.cm == nil {
+		return nil, errors.New("endpoint not initialized, call Init first")
+	}
+	if recordBytes, found := cml.cm.Annotations[KubePeersAnnotationKey]; found {
+		if err := json.Unmarshal([]byte(recordBytes), &record); err != nil {
+			return nil, err
+		}
+	}
+	return &record, nil
+}
+
+func (cml *configMapAnnotations) UpdatePeerList(list peerList) error {
+	recordBytes, err := json.Marshal(list)
+	if err != nil {
+		return err
+	}
+	return cml.UpdateAnnotation(KubePeersAnnotationKey, string(recordBytes))
+}
+
+// update the list of all peers that have gone through this code path
+func addMyselfToPeerList(cml *configMapAnnotations, c kubernetes.Interface, peerName, name string) (*peerList, error) {
+	var list *peerList
+	err := cml.LoopUpdate(func() error {
+		var err error
+		list, err = cml.GetPeerList()
+		if err != nil {
+			return err
+		}
+		if !list.contains(peerName) {
+			list.add(peerName, name)
+			err = cml.UpdatePeerList(*list)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	return list, err
+}
+
+func checkIamInPeerList(cml *configMapAnnotations, c kubernetes.Interface, peerName string) (bool, error) {
+	if err := cml.Init(); err != nil {
+		return false, err
+	}
+	list, err := cml.GetPeerList()
+	if err != nil {
+		return false, err
+	}
+	common.Log.Debugf("[kube-peers] Checking peer %q against list %v", peerName, list)
+	return list.contains(peerName), nil
+}

prog/kube-utils/peerlist_test.go

@@ -0,0 +1,184 @@
+package main
+
+import (
+	"fmt"
+	"math/rand"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	peerName1 = "01:00:00:00:00:00"
+	peerName2 = "02:00:00:00:00:00"
+	peerName3 = "03:00:00:00:00:00"
+	nodeName1 = "fake-node-1"
+	nodeName2 = "fake-node-2"
+
+	testIters = 1000
+)
+
+func TestPeerListBasic(t *testing.T) {
+	c := NewSimpleClientset()
+
+	cml := newConfigMapAnnotations(configMapNamespace, configMapName, c)
+	err := cml.Init()
+	require.NoError(t, err)
+
+	list1, err := addMyselfToPeerList(cml, c, peerName1, nodeName1)
+	require.NoError(t, err)
+	require.Equal(t, 1, len(list1.Peers))
+	list2, err := addMyselfToPeerList(cml, c, peerName2, nodeName2)
+	require.NoError(t, err)
+	require.Equal(t, 2, len(list2.Peers))
+
+	check1, err := checkIamInPeerList(cml, c, peerName1)
+	require.NoError(t, err)
+	require.Equal(t, true, check1)
+	check3, err := checkIamInPeerList(cml, c, peerName3)
+	require.NoError(t, err)
+	require.Equal(t, false, check3)
+
+	storedPeerList, err := cml.GetPeerList()
+	require.NoError(t, err)
+
+	storedPeerList.remove(peerName1)
+	require.Equal(t, 1, len(storedPeerList.Peers))
+	err = cml.UpdatePeerList(*storedPeerList)
+	require.NoError(t, err)
+	check1, err = checkIamInPeerList(cml, c, peerName1)
+	require.NoError(t, err)
+	require.Equal(t, false, check1)
+}
+
+// Run function _f_ _iterations_ times, in _concurrency_
+// number of goroutines
+func doConcurrentIterations(iterations, concurrency int, f func(int)) {
+	iterationsPerThread := iterations / concurrency
+
+	wg := sync.WaitGroup{}
+	for i := 0; i < concurrency; i++ {
+		wg.Add(1)
+		go func(j int) {
+			defer wg.Done()
+			for k := 0; k < iterationsPerThread; k++ {
+				f((j * iterationsPerThread) + k)
+			}
+		}(i)
+	}
+	wg.Wait()
+}
+
+func TestPeerListFuzz(t *testing.T) {
+	rand.Seed(time.Now().UnixNano())
+	const (
+		testIters   = 1000
+		maxNodes    = 100
+		concurrency = 5
+	)
+	peerName := func(i int) string { return fmt.Sprintf("%02d:00:00:02:00:00", i) }
+	nodeName := func(i int) string { return fmt.Sprintf("node%d", i) }
+
+	stateLock := sync.Mutex{}
+	nodes := []int{}
+	nodeMap := map[int]bool{} // true if an operation is pending
+
+	c := NewSimpleClientset()
+
+	addNode := func() {
+		i := rand.Intn(maxNodes)
+		// add to test state first, then to Kubernetes
+		stateLock.Lock()
+		pending, found := nodeMap[i]
+		if !found {
+			nodes = append(nodes, i)
+		}
+		nodeMap[i] = true
+		stateLock.Unlock()
+		if pending { // someone else is adding or deleting this node
+			return
+		}
+
+		cml := newConfigMapAnnotations(configMapNamespace, configMapName, c)
+		err := cml.Init()
+		require.NoError(t, err)
+		_, err = addMyselfToPeerList(cml, c, peerName(i), nodeName(i))
+		require.NoError(t, err)
+
+		stateLock.Lock()
+		nodeMap[i] = false
+		stateLock.Unlock()
+	}
+
+	removeNode := func() {
+		stateLock.Lock()
+		if len(nodes) == 0 { // can't remove if there aren't any nodes
+			stateLock.Unlock()
+			return
+		}
+		pos := rand.Intn(len(nodes))
+		i := nodes[pos]
+		if pending, found := nodeMap[i]; found {
+			if pending {
+				stateLock.Unlock()
+				return // someone else is adding or deleting this node
+			}
+			nodeMap[i] = true
+		} else {
+			t.Errorf("missing entry from node map: %d", i)
+		}
+		stateLock.Unlock()
+
+		// remove from Kubernetes first, then from test state
+		cml := newConfigMapAnnotations(configMapNamespace, configMapName, c)
+		require.NoError(t, cml.Init())
+		storedPeerList, err := cml.GetPeerList()
+		require.NoError(t, err)
+		found := storedPeerList.contains(peerName(i))
+		require.True(t, found, "peer %d not found in stored list", i)
+
+		_, err = reclaimPeer(mockWeave{}, cml, peerName(i), fmt.Sprintf("deleter-%d", i))
+		require.NoError(t, err)
+
+		require.NoError(t, cml.Init())
+		storedPeerList, err = cml.GetPeerList()
+		require.NoError(t, err)
+		stateLock.Lock()
+		delete(nodeMap, i)
+		for pos = 0; pos < len(nodes); pos++ {
+			if nodes[pos] == i {
+				// Remove item from list by swapping it with last and reducing slice length by 1
+				nodes[pos] = nodes[len(nodes)-1]
+				nodes = nodes[:len(nodes)-1]
+				break
+			}
+		}
+		stateLock.Unlock()
+	}
+
+	doConcurrentIterations(testIters, concurrency, func(iteration int) {
+		r := rand.Float32()
+		switch {
+		case 0.0 <= r && r < 0.4:
+			addNode()
+
+		case (0.4 <= r && r < 0.8):
+			removeNode()
+		}
+	})
+
+	cml := newConfigMapAnnotations(configMapNamespace, configMapName, c)
+	err := cml.Init()
+	require.NoError(t, err)
+	storedPeerList, err := cml.GetPeerList()
+	require.NoError(t, err)
+	require.Equal(t, len(storedPeerList.Peers), len(nodes))
+}
+
+type mockWeave struct{}
+
+func (mockWeave) RmPeer(peerName string) (string, error) {
+	return "", nil
+}

prog/weave-kube/launch.sh

@@ -6,6 +6,34 @@ modprobe_safe() {
     modprobe $1 || echo "Ignore the error if \"$1\" is built-in in the kernel" >&2
 }
 
+# Setup iptables backend to be legacy or nftable
+setup_iptables_backend() {
+    if [ -n "${IPTABLES_BACKEND}" ]; then
+      mode=$IPTABLES_BACKEND
+    else
+      # auto-detect if iptables backend mode to use if not specified explicitly
+      num_legacy_lines=$( (iptables-legacy-save || true) 2>/dev/null | grep '^-' | wc -l)
+      num_nft_lines=$( (iptables-nft-save || true) 2>/dev/null | grep '^-' | wc -l)
+      if [ "${num_legacy_lines}" -ge 10 ]; then
+        mode="legacy"
+      else
+        if [ "${num_legacy_lines}" -ge "${num_nft_lines}" ]; then
+          mode="legacy"
+        else
+          mode="nft"
+        fi
+      fi
+    fi
+    if [ $mode = "nft" ]; then
+      rm /sbin/iptables
+      rm /sbin/iptables-save
+      rm /sbin/iptables-restore
+      ln -s /sbin/iptables-nft /sbin/iptables
+      ln -s /sbin/iptables-nft-save /sbin/iptables-save
+      ln -s /sbin/iptables-nft-restore /sbin/iptables-restore
+    fi
+}
+
 # Check whether xt_set actually exists
 xt_set_exists() {
     # Clean everything up in advance, in case there's leftovers
@@ -24,12 +52,14 @@ xt_set_exists() {
     [ -z "$NOT_EXIST" ] || (echo "\"xt_set\" does not exist" >&2 && return 1)
 }
 
+setup_iptables_backend
+
 # Default if not supplied - same as weave net default
 IPALLOC_RANGE=${IPALLOC_RANGE:-10.32.0.0/12}
 HTTP_ADDR=${WEAVE_HTTP_ADDR:-127.0.0.1:6784}
 METRICS_ADDR=${WEAVE_METRICS_ADDR:-0.0.0.0:6782}
 HOST_ROOT=${HOST_ROOT:-/host}
-CONN_LIMIT=${CONN_LIMIT:-100}
+CONN_LIMIT=${CONN_LIMIT:-200}
 DB_PREFIX=${DB_PREFIX:-/weavedb/weave-net}
 
 # Check if the IP range overlaps anything existing on the host
@@ -85,7 +115,7 @@ router_bridge_opts() {
 }
 
 if [ -z "$KUBE_PEERS" ]; then
-    if ! KUBE_PEERS=$(/home/weave/kube-utils) || [ -z "$KUBE_PEERS" ]; then
+    if ! KUBE_PEERS=$(/home/weave/kube-utils); then
         echo Failed to get peers >&2
         exit 1
     fi
@@ -151,6 +181,8 @@ post_start_actions() {
 
     # Mark network as up
     /home/weave/kube-utils -set-node-status -node-name="$HOSTNAME"
+
+    /home/weave/kube-utils -run-reclaim-daemon -node-name="$HOSTNAME" -peer-name="$PEERNAME" -log-level=debug&
 }
 
 post_start_actions &

prog/weave-kube/weave-daemonset-k8s-1.11.yaml

@@ -0,0 +1,215 @@
+apiVersion: v1
+kind: List
+items:
+  - apiVersion: v1
+    kind: ServiceAccount
+    metadata:
+      name: weave-net
+      labels:
+        name: weave-net
+      namespace: kube-system
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: ClusterRole
+    metadata:
+      name: weave-net
+      labels:
+        name: weave-net
+    rules:
+      - apiGroups:
+          - ''
+        resources:
+          - pods
+          - namespaces
+          - nodes
+        verbs:
+          - get
+          - list
+          - watch
+      - apiGroups:
+          - extensions
+        resources:
+          - networkpolicies
+        verbs:
+          - get
+          - list
+          - watch
+      - apiGroups:
+          - 'networking.k8s.io'
+        resources:
+          - networkpolicies
+        verbs:
+          - get
+          - list
+          - watch
+      - apiGroups:
+        - ''
+        resources:
+        - nodes/status
+        verbs:
+        - patch
+        - update
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: ClusterRoleBinding
+    metadata:
+      name: weave-net
+      labels:
+        name: weave-net
+    roleRef:
+      kind: ClusterRole
+      name: weave-net
+      apiGroup: rbac.authorization.k8s.io
+    subjects:
+      - kind: ServiceAccount
+        name: weave-net
+        namespace: kube-system
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: Role
+    metadata:
+      name: weave-net
+      namespace: kube-system
+      labels:
+        name: weave-net
+    rules:
+      - apiGroups:
+          - ''
+        resources:
+          - configmaps
+        resourceNames:
+          - weave-net
+        verbs:
+          - get
+          - update
+      - apiGroups:
+          - ''
+        resources:
+          - configmaps
+        verbs:
+          - create
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata:
+      name: weave-net
+      namespace: kube-system
+      labels:
+        name: weave-net
+    roleRef:
+      kind: Role
+      name: weave-net
+      apiGroup: rbac.authorization.k8s.io
+    subjects:
+      - kind: ServiceAccount
+        name: weave-net
+        namespace: kube-system
+  - apiVersion: apps/v1
+    kind: DaemonSet
+    metadata:
+      name: weave-net
+      labels:
+        name: weave-net
+      namespace: kube-system
+    spec:
+      # Wait 5 seconds to let pod connect before rolling next pod
+      selector:
+        matchLabels:
+          name: weave-net
+      minReadySeconds: 5
+      template:
+        metadata:
+          labels:
+            name: weave-net
+        spec:
+          containers:
+            - name: weave
+              command:
+                - /home/weave/launch.sh
+              env:
+                - name: HOSTNAME
+                  valueFrom:
+                    fieldRef:
+                      apiVersion: v1
+                      fieldPath: spec.nodeName
+              image: 'weaveworks/weave-kube:latest'
+              imagePullPolicy: Always
+              readinessProbe:
+                httpGet:
+                  host: 127.0.0.1
+                  path: /status
+                  port: 6784
+              resources:
+                requests:
+                  cpu: 50m
+              securityContext:
+                privileged: true
+              volumeMounts:
+                - name: weavedb
+                  mountPath: /weavedb
+                - name: cni-bin
+                  mountPath: /host/opt
+                - name: cni-bin2
+                  mountPath: /host/home
+                - name: cni-conf
+                  mountPath: /host/etc
+                - name: dbus
+                  mountPath: /host/var/lib/dbus
+                - name: lib-modules
+                  mountPath: /lib/modules
+                - name: xtables-lock
+                  mountPath: /run/xtables.lock
+                  readOnly: false
+            - name: weave-npc
+              env:
+                - name: HOSTNAME
+                  valueFrom:
+                    fieldRef:
+                      apiVersion: v1
+                      fieldPath: spec.nodeName
+              image: 'weaveworks/weave-npc:latest'
+              imagePullPolicy: Always
+#npc-args
+              resources:
+                requests:
+                  cpu: 50m
+              securityContext:
+                privileged: true
+              volumeMounts:
+                - name: xtables-lock
+                  mountPath: /run/xtables.lock
+                  readOnly: false
+          hostNetwork: true
+          dnsPolicy: ClusterFirstWithHostNet
+          hostPID: true
+          restartPolicy: Always
+          securityContext:
+            seLinuxOptions: {}
+          serviceAccountName: weave-net
+          tolerations:
+            - effect: NoSchedule
+              operator: Exists
+            - effect: NoExecute
+              operator: Exists
+          volumes:
+            - name: weavedb
+              hostPath:
+                path: /var/lib/weave
+            - name: cni-bin
+              hostPath:
+                path: /opt
+            - name: cni-bin2
+              hostPath:
+                path: /home
+            - name: cni-conf
+              hostPath:
+                path: /etc
+            - name: dbus
+              hostPath:
+                path: /var/lib/dbus
+            - name: lib-modules
+              hostPath:
+                path: /lib/modules
+            - name: xtables-lock
+              hostPath:
+                path: /run/xtables.lock
+                type: FileOrCreate
+          priorityClassName: system-node-critical
+      updateStrategy:
+        type: RollingUpdate

prog/weave-kube/weave-daemonset-k8s-1.6.yaml

@@ -110,15 +110,14 @@ items:
                       fieldPath: spec.nodeName
               image: 'weaveworks/weave-kube:latest'
               imagePullPolicy: Always
-              livenessProbe:
+              readinessProbe:
                 httpGet:
                   host: 127.0.0.1
                   path: /status
                   port: 6784
-                initialDelaySeconds: 30
               resources:
                 requests:
-                  cpu: 10m
+                  cpu: 50m
               securityContext:
                 privileged: true
               volumeMounts:
@@ -145,7 +144,7 @@ items:
               imagePullPolicy: Always
               resources:
                 requests:
-                  cpu: 10m
+                  cpu: 50m
               securityContext:
                 privileged: true
           hostNetwork: true
@@ -157,6 +156,8 @@ items:
           tolerations:
             - effect: NoSchedule
               operator: Exists
+            - effect: NoExecute
+              operator: Exists
           volumes:
             - name: weavedb
               hostPath:

prog/weave-kube/weave-daemonset-k8s-1.7.yaml

@@ -119,15 +119,14 @@ items:
                       fieldPath: spec.nodeName
               image: 'weaveworks/weave-kube:latest'
               imagePullPolicy: Always
-              livenessProbe:
+              readinessProbe:
                 httpGet:
                   host: 127.0.0.1
                   path: /status
                   port: 6784
-                initialDelaySeconds: 30
               resources:
                 requests:
-                  cpu: 10m
+                  cpu: 50m
               securityContext:
                 privileged: true
               volumeMounts:
@@ -157,7 +156,7 @@ items:
               imagePullPolicy: Always
               resources:
                 requests:
-                  cpu: 10m
+                  cpu: 50m
               securityContext:
                 privileged: true
               volumeMounts:
@@ -173,6 +172,8 @@ items:
           tolerations:
             - effect: NoSchedule
               operator: Exists
+            - effect: NoExecute
+              operator: Exists
           volumes:
             - name: weavedb
               hostPath:

prog/weave-kube/weave-daemonset-k8s-1.8.yaml

@@ -8,7 +8,7 @@ items:
       labels:
         name: weave-net
       namespace: kube-system
-  - apiVersion: rbac.authorization.k8s.io/v1beta1
+  - apiVersion: rbac.authorization.k8s.io/v1
     kind: ClusterRole
     metadata:
       name: weave-net
@@ -48,7 +48,7 @@ items:
         verbs:
         - patch
         - update
-  - apiVersion: rbac.authorization.k8s.io/v1beta1
+  - apiVersion: rbac.authorization.k8s.io/v1
     kind: ClusterRoleBinding
     metadata:
       name: weave-net
@@ -62,7 +62,7 @@ items:
       - kind: ServiceAccount
         name: weave-net
         namespace: kube-system
-  - apiVersion: rbac.authorization.k8s.io/v1beta1
+  - apiVersion: rbac.authorization.k8s.io/v1
     kind: Role
     metadata:
       name: weave-net
@@ -85,7 +85,7 @@ items:
           - configmaps
         verbs:
           - create
-  - apiVersion: rbac.authorization.k8s.io/v1beta1
+  - apiVersion: rbac.authorization.k8s.io/v1
     kind: RoleBinding
     metadata:
       name: weave-net
@@ -127,15 +127,14 @@ items:
                       fieldPath: spec.nodeName
               image: 'weaveworks/weave-kube:latest'
               imagePullPolicy: Always
-              livenessProbe:
+              readinessProbe:
                 httpGet:
                   host: 127.0.0.1
                   path: /status
                   port: 6784
-                initialDelaySeconds: 30
               resources:
                 requests:
-                  cpu: 10m
+                  cpu: 50m
               securityContext:
                 privileged: true
               volumeMounts:
@@ -166,7 +165,7 @@ items:
 #npc-args
               resources:
                 requests:
-                  cpu: 10m
+                  cpu: 50m
               securityContext:
                 privileged: true
               volumeMounts:
@@ -182,6 +181,8 @@ items:
           tolerations:
             - effect: NoSchedule
               operator: Exists
+            - effect: NoExecute
+              operator: Exists
           volumes:
             - name: weavedb
               hostPath:

prog/weave-kube/weave-daemonset-k8s-1.9.yaml

@@ -0,0 +1,214 @@
+apiVersion: v1
+kind: List
+items:
+  - apiVersion: v1
+    kind: ServiceAccount
+    metadata:
+      name: weave-net
+      labels:
+        name: weave-net
+      namespace: kube-system
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: ClusterRole
+    metadata:
+      name: weave-net
+      labels:
+        name: weave-net
+    rules:
+      - apiGroups:
+          - ''
+        resources:
+          - pods
+          - namespaces
+          - nodes
+        verbs:
+          - get
+          - list
+          - watch
+      - apiGroups:
+          - extensions
+        resources:
+          - networkpolicies
+        verbs:
+          - get
+          - list
+          - watch
+      - apiGroups:
+          - 'networking.k8s.io'
+        resources:
+          - networkpolicies
+        verbs:
+          - get
+          - list
+          - watch
+      - apiGroups:
+        - ''
+        resources:
+        - nodes/status
+        verbs:
+        - patch
+        - update
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: ClusterRoleBinding
+    metadata:
+      name: weave-net
+      labels:
+        name: weave-net
+    roleRef:
+      kind: ClusterRole
+      name: weave-net
+      apiGroup: rbac.authorization.k8s.io
+    subjects:
+      - kind: ServiceAccount
+        name: weave-net
+        namespace: kube-system
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: Role
+    metadata:
+      name: weave-net
+      namespace: kube-system
+      labels:
+        name: weave-net
+    rules:
+      - apiGroups:
+          - ''
+        resources:
+          - configmaps
+        resourceNames:
+          - weave-net
+        verbs:
+          - get
+          - update
+      - apiGroups:
+          - ''
+        resources:
+          - configmaps
+        verbs:
+          - create
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata:
+      name: weave-net
+      namespace: kube-system
+      labels:
+        name: weave-net
+    roleRef:
+      kind: Role
+      name: weave-net
+      apiGroup: rbac.authorization.k8s.io
+    subjects:
+      - kind: ServiceAccount
+        name: weave-net
+        namespace: kube-system
+  - apiVersion: apps/v1
+    kind: DaemonSet
+    metadata:
+      name: weave-net
+      labels:
+        name: weave-net
+      namespace: kube-system
+    spec:
+      # Wait 5 seconds to let pod connect before rolling next pod
+      selector:
+        matchLabels:
+          name: weave-net
+      minReadySeconds: 5
+      template:
+        metadata:
+          labels:
+            name: weave-net
+        spec:
+          containers:
+            - name: weave
+              command:
+                - /home/weave/launch.sh
+              env:
+                - name: HOSTNAME
+                  valueFrom:
+                    fieldRef:
+                      apiVersion: v1
+                      fieldPath: spec.nodeName
+              image: 'weaveworks/weave-kube:latest'
+              imagePullPolicy: Always
+              readinessProbe:
+                httpGet:
+                  host: 127.0.0.1
+                  path: /status
+                  port: 6784
+              resources:
+                requests:
+                  cpu: 50m
+              securityContext:
+                privileged: true
+              volumeMounts:
+                - name: weavedb
+                  mountPath: /weavedb
+                - name: cni-bin
+                  mountPath: /host/opt
+                - name: cni-bin2
+                  mountPath: /host/home
+                - name: cni-conf
+                  mountPath: /host/etc
+                - name: dbus
+                  mountPath: /host/var/lib/dbus
+                - name: lib-modules
+                  mountPath: /lib/modules
+                - name: xtables-lock
+                  mountPath: /run/xtables.lock
+                  readOnly: false
+            - name: weave-npc
+              env:
+                - name: HOSTNAME
+                  valueFrom:
+                    fieldRef:
+                      apiVersion: v1
+                      fieldPath: spec.nodeName
+              image: 'weaveworks/weave-npc:latest'
+              imagePullPolicy: Always
+#npc-args
+              resources:
+                requests:
+                  cpu: 50m
+              securityContext:
+                privileged: true
+              volumeMounts:
+                - name: xtables-lock
+                  mountPath: /run/xtables.lock
+                  readOnly: false
+          hostNetwork: true
+          dnsPolicy: ClusterFirstWithHostNet
+          hostPID: true
+          restartPolicy: Always
+          securityContext:
+            seLinuxOptions: {}
+          serviceAccountName: weave-net
+          tolerations:
+            - effect: NoSchedule
+              operator: Exists
+            - effect: NoExecute
+              operator: Exists
+          volumes:
+            - name: weavedb
+              hostPath:
+                path: /var/lib/weave
+            - name: cni-bin
+              hostPath:
+                path: /opt
+            - name: cni-bin2
+              hostPath:
+                path: /home
+            - name: cni-conf
+              hostPath:
+                path: /etc
+            - name: dbus
+              hostPath:
+                path: /var/lib/dbus
+            - name: lib-modules
+              hostPath:
+                path: /lib/modules
+            - name: xtables-lock
+              hostPath:
+                path: /run/xtables.lock
+                type: FileOrCreate
+      updateStrategy:
+        type: RollingUpdate

prog/weave-kube/weave-daemonset.yaml

@@ -1,101 +0,0 @@
-apiVersion: v1
-kind: List
-items:
-  - apiVersion: v1
-    kind: ServiceAccount
-    metadata:
-      name: weave-net
-      labels:
-        name: weave-net
-      namespace: kube-system
-  - apiVersion: extensions/v1beta1
-    kind: DaemonSet
-    metadata:
-      name: weave-net
-      labels:
-        name: weave-net
-      namespace: kube-system
-    spec:
-      template:
-        metadata:
-          annotations:
-            scheduler.alpha.kubernetes.io/tolerations: >-
-              [{"key":"dedicated","operator":"Equal","value":"master","effect":"NoSchedule"}]
-          labels:
-            name: weave-net
-        spec:
-          containers:
-            - name: weave
-              command:
-                - /home/weave/launch.sh
-              env:
-                - name: HOSTNAME
-                  valueFrom:
-                    fieldRef:
-                      apiVersion: v1
-                      fieldPath: spec.nodeName
-              image: 'weaveworks/weave-kube:latest'
-              imagePullPolicy: Always
-              livenessProbe:
-                httpGet:
-                  host: 127.0.0.1
-                  path: /status
-                  port: 6784
-                initialDelaySeconds: 30
-              resources:
-                requests:
-                  cpu: 10m
-              securityContext:
-                privileged: true
-              volumeMounts:
-                - name: weavedb
-                  mountPath: /weavedb
-                - name: cni-bin
-                  mountPath: /host/opt
-                - name: cni-bin2
-                  mountPath: /host/home
-                - name: cni-conf
-                  mountPath: /host/etc
-                - name: dbus
-                  mountPath: /host/var/lib/dbus
-                - name: lib-modules
-                  mountPath: /lib/modules
-            - name: weave-npc
-              env:
-                - name: HOSTNAME
-                  valueFrom:
-                    fieldRef:
-                      apiVersion: v1
-                      fieldPath: spec.nodeName
-              image: 'weaveworks/weave-npc:latest'
-              imagePullPolicy: Always
-              resources:
-                requests:
-                  cpu: 10m
-              securityContext:
-                privileged: true
-          hostNetwork: true
-          hostPID: true
-          restartPolicy: Always
-          securityContext:
-            seLinuxOptions: {}
-          serviceAccountName: weave-net
-          volumes:
-            - name: weavedb
-              hostPath:
-                path: /var/lib/weave
-            - name: cni-bin
-              hostPath:
-                path: /opt
-            - name: cni-bin2
-              hostPath:
-                path: /home
-            - name: cni-conf
-              hostPath:
-                path: /etc
-            - name: dbus
-              hostPath:
-                path: /var/lib/dbus
-            - name: lib-modules
-              hostPath:
-                path: /lib/modules

prog/weave-npc/Dockerfile.template

@@ -27,7 +27,8 @@ RUN apk add --update \
   && mknod /var/log/ulogd.pcap p
 COPY ./weave-npc /usr/bin/weave-npc
 COPY ./ulogd.conf /etc/ulogd.conf
-ENTRYPOINT ["/usr/bin/weave-npc"]
+COPY ./launch.sh /usr/bin/
+ENTRYPOINT ["/usr/bin/launch.sh"]
 
 # These labels will change for every build, and should therefore be the last layer of the image:
 ARG revision

prog/weave-npc/launch.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+set -e
+
+# Setup iptables backend to be legacy or nftable
+setup_iptables_backend() {
+    if [ -n "${IPTABLES_BACKEND}" ]; then
+      mode=$IPTABLES_BACKEND
+    else
+      # auto-detect if iptables backend mode to use if not specified explicitly
+      num_legacy_lines=$( (iptables-legacy-save || true) 2>/dev/null | grep '^-' | wc -l)
+      num_nft_lines=$( (iptables-nft-save || true) 2>/dev/null | grep '^-' | wc -l)
+      if [ "${num_legacy_lines}" -ge 10 ]; then
+        mode="legacy"
+      else
+        if [ "${num_legacy_lines}" -ge "${num_nft_lines}" ]; then
+          mode="legacy"
+        else
+          mode="nft"
+        fi
+      fi
+    fi
+    if [ $mode = "nft" ]; then
+      rm /sbin/iptables
+      rm /sbin/iptables-save
+      rm /sbin/iptables-restore
+      ln -s /sbin/iptables-nft /sbin/iptables
+      ln -s /sbin/iptables-nft-save /sbin/iptables-save
+      ln -s /sbin/iptables-nft-restore /sbin/iptables-restore
+    fi
+}
+
+setup_iptables_backend
+
+# Start weave-npc with any flags specified in $EXTRA_ARGS as well as any flags passed to this container (for backwards compatibility)
+exec /usr/bin/weave-npc $EXTRA_ARGS $@

prog/weave-npc/main.go

@@ -25,15 +25,23 @@ import (
 )
 
 var (
-	version     = "unreleased"
-	metricsAddr string
-	logLevel    string
-	allowMcast  bool
-	nodeName    string
-	maxList     int
+	version        = "unreleased"
+	metricsAddr    string
+	logLevel       string
+	allowMcast     bool
+	nodeName       string
+	maxList        int
+	bridgePortName string
 )
 
-func handleError(err error) { common.CheckFatal(err) }
+func handleError(err error) {
+	if err != nil {
+		metrics.PolicyEnforcementErrors.Inc()
+		common.Log.Error(err)
+	}
+}
+
+func handleFatal(err error) { common.CheckFatal(err) }
 
 func makeController(getter cache.Getter, resource string,
 	objType runtime.Object, handlers cache.ResourceEventHandlerFuncs) cache.Controller {
@@ -125,6 +133,12 @@ func createBaseRules(ipt *iptables.IPTables, ips ipset.Interface) error {
 		}
 	}
 
+	// If the destination address is not any of the local pods, let it through
+	if err := ipt.Append(npc.TableFilter, npc.MainChain,
+		"-m", "physdev", "--physdev-is-bridged", "--physdev-out="+bridgePortName, "-j", "ACCEPT"); err != nil {
+		return err
+	}
+
 	if err := ipt.Append(npc.TableFilter, npc.MainChain,
 		"-m", "state", "--state", "NEW", "-j", string(npc.DefaultChain)); err != nil {
 		return err
@@ -135,21 +149,6 @@ func createBaseRules(ipt *iptables.IPTables, ips ipset.Interface) error {
 		return err
 	}
 
-	// If the destination address is not any of the local pods, let it through
-	found, err := ipsetExist(ips, npc.LocalIpset)
-	if err != nil {
-		return err
-	}
-	if !found {
-		if err := ips.Create(npc.LocalIpset, ipset.HashIP); err != nil {
-			return err
-		}
-	}
-	if err := ipt.Append(npc.TableFilter, npc.MainChain,
-		"-m", "set", "!", "--match-set", npc.LocalIpset, "dst", "-j", "ACCEPT"); err != nil {
-		return err
-	}
-
 	if err := ipt.Append(npc.TableFilter, npc.EgressMarkChain,
 		"-j", "MARK", "--set-xmark", npc.EgressMark); err != nil {
 		return err
@@ -158,8 +157,8 @@ func createBaseRules(ipt *iptables.IPTables, ips ipset.Interface) error {
 	// Egress rules:
 	//
 	// -A WEAVE-NPC-EGRESS -m state --state RELATED,ESTABLISHED -j ACCEPT
-	// -A WEAVE-NPC-EGRESS -m set ! --match-set weave-local-pods src -j RETURN
-	// -A WEAVE-NPC-EGRESS -d 224.0.0.0/4 -j RETURN
+	// -A WEAVE-NPC-EGRESS -m physdev --physdev-in vethwe-bridge --physdev-is-bridged -j RETURN
+	// -A WEAVE-NPC-EGRESS -m addrtype --dst-type LOCAL -j RETURN
 	// -A WEAVE-NPC-EGRESS -m state --state NEW -j WEAVE-NPC-EGRESS-DEFAULT
 	// -A WEAVE-NPC-EGRESS -m state --state NEW -m mark ! --mark 0x40000/0x40000 -j WEAVE-NPC-EGRESS-CUSTOM
 	// -A WEAVE-NPC-EGRESS -m state --state NEW -m mark ! --mark 0x40000/0x40000 -j NFLOG --nflog-group 86
@@ -179,7 +178,10 @@ func createBaseRules(ipt *iptables.IPTables, ips ipset.Interface) error {
 
 	ruleSpecs := [][]string{
 		{"-m", "state", "--state", "RELATED,ESTABLISHED", "-j", "ACCEPT"},
-		{"-m", "state", "--state", "NEW", "-m", "set", "!", "--match-set", npc.LocalIpset, "src", "-j", "RETURN"},
+		// skip running through egress network policies for the traffic not coming from local pods
+		{"-m", "physdev", "--physdev-is-bridged", "--physdev-in=" + bridgePortName, "-j", "RETURN"},
+		// skip running through egress network policies for the traffic bound for IP address assigned for the bridge
+		{"-m", "addrtype", "--dst-type", "LOCAL", "-j", "RETURN"},
 	}
 	if allowMcast {
 		ruleSpecs = append(ruleSpecs, []string{"-d", "224.0.0.0/4", "-j", "RETURN"})
@@ -188,12 +190,22 @@ func createBaseRules(ipt *iptables.IPTables, ips ipset.Interface) error {
 		{"-m", "state", "--state", "NEW", "-j", string(npc.EgressDefaultChain)},
 		{"-m", "state", "--state", "NEW", "-m", "mark", "!", "--mark", npc.EgressMark, "-j", string(npc.EgressCustomChain)},
 		{"-m", "state", "--state", "NEW", "-m", "mark", "!", "--mark", npc.EgressMark, "-j", "NFLOG", "--nflog-group", "86"},
-		{"-m", "mark", "!", "--mark", npc.EgressMark, "-j", "DROP"},
 	}...)
 	if err := net.AddChainWithRules(ipt, npc.TableFilter, npc.EgressChain, ruleSpecs); err != nil {
 		return err
 	}
 
+	// delete `weave-local-pods` ipset which is no longer used by weave-npc
+	weaveLocalPodExist, err := ipsetExist(ips, npc.LocalIpset)
+	if err != nil {
+		common.Log.Errorf("Failed to look if ipset '%s' exists", npc.LocalIpset)
+	} else if weaveLocalPodExist {
+		common.Log.Debugf("Destroying ipset '%s'", npc.LocalIpset)
+		if err := ips.Destroy(npc.LocalIpset); err != nil {
+			common.Log.Errorf("Failed to destroy ipset '%s'", npc.LocalIpset)
+		}
+	}
+
 	return nil
 }
 
@@ -235,21 +247,21 @@ func root(cmd *cobra.Command, args []string) {
 	}
 
 	config, err := rest.InClusterConfig()
-	handleError(err)
+	handleFatal(err)
 
 	client, err := kubernetes.NewForConfig(config)
-	handleError(err)
+	handleFatal(err)
 
 	ipt, err := iptables.New()
-	handleError(err)
+	handleFatal(err)
 
 	ips := ipset.New(common.LogLogger(), maxList)
 
-	handleError(resetIPTables(ipt))
-	handleError(resetIPSets(ips))
-	handleError(createBaseRules(ipt, ips))
+	handleFatal(resetIPTables(ipt))
+	handleFatal(resetIPSets(ips))
+	handleFatal(createBaseRules(ipt, ips))
 
-	npc := npc.New(nodeName, ipt, ips)
+	npc := npc.New(nodeName, ipt, ips, client)
 
 	nsController := makeController(client.Core().RESTClient(), "namespaces", &coreapi.Namespace{},
 		cache.ResourceEventHandlerFuncs{
@@ -332,6 +344,7 @@ func main() {
 	rootCmd.PersistentFlags().BoolVar(&allowMcast, "allow-mcast", true, "allow all multicast traffic")
 	rootCmd.PersistentFlags().StringVar(&nodeName, "node-name", "", "only generate rules that apply to this node")
 	rootCmd.PersistentFlags().IntVar(&maxList, "max-list-size", 1024, "maximum size of ipset list (for namespaces)")
+	rootCmd.PersistentFlags().StringVar(&bridgePortName, "bridge-port-name", "vethwe-bridge", "name of the brige port on which packets are received and sent")
 
-	handleError(rootCmd.Execute())
+	handleFatal(rootCmd.Execute())
 }

prog/weaver/checkpoint.go

@@ -1,12 +1,17 @@
 package main
 
 import (
+	"fmt"
+	"os"
 	"sync/atomic"
 	"syscall"
 	"time"
 
-	"github.com/weaveworks/go-checkpoint"
+	checkpoint "github.com/weaveworks/go-checkpoint"
 	weave "github.com/weaveworks/weave/router"
+	api "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 )
 
 var checker *checkpoint.Checker
@@ -14,10 +19,12 @@ var newVersion atomic.Value
 var success atomic.Value
 
 const (
-	updateCheckPeriod = 6 * time.Hour
+	updateCheckPeriod  = 6 * time.Hour
+	configMapName      = "weave-net"
+	configMapNamespace = "kube-system"
 )
 
-func checkForUpdates(dockerVersion string, router *weave.NetworkRouter) {
+func checkForUpdates(dockerVersion string, router *weave.NetworkRouter, clusterSize uint) {
 	newVersion.Store("")
 	success.Store(true)
 
@@ -53,6 +60,8 @@ func checkForUpdates(dockerVersion string, router *weave.NetworkRouter) {
 		"kernel-version": kernelVersion,
 	}
 
+	checkpointKubernetes(flags, clusterSize)
+
 	// Start background version checking
 	params := checkpoint.CheckParams{
 		Product:       "weave-net",
@@ -77,3 +86,37 @@ func checkpointFlags(router *weave.NetworkRouter) []checkpoint.Flag {
 	}
 	return flags
 }
+
+// checkpoint Kubernetes specific details
+func checkpointKubernetes(flags map[string]string, clusterSize uint) {
+	// checks if weaver is running in Kubernetes
+	host := os.Getenv("KUBERNETES_SERVICE_HOST")
+	if len(host) == 0 {
+		return
+	}
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		Log.Printf("Could not get Kubernetes in-cluster config: %v", err)
+		return
+	}
+	c, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		Log.Printf("Could not make Kubernetes client: %v", err)
+		return
+	}
+	k8sVersion, err := c.Discovery().ServerVersion()
+	if err != nil {
+		Log.Printf("Could not get Kubernetes version: %v", err)
+		return
+	}
+	flags["kubernetes-version"] = k8sVersion.String()
+
+	// use UID of `weave-net` configmap as unique ID of the Kubenerets cluster
+	cm, err := c.CoreV1().ConfigMaps(configMapNamespace).Get(configMapName, api.GetOptions{})
+	if err != nil {
+		Log.Printf("Unable to fetch ConfigMap %s/%s to infer unique cluster ID", configMapNamespace, configMapName)
+		return
+	}
+	flags["kubernetes-cluster-uid"] = string(cm.ObjectMeta.UID)
+	flags["kubernetes-cluster-size"] = fmt.Sprint(clusterSize)
+}

prog/weaver/main.go

@@ -188,7 +188,7 @@ func main() {
 	mflag.StringVar(&logLevel, []string{"-log-level"}, "info", "logging level (debug, info, warning, error)")
 	mflag.BoolVar(&pktdebug, []string{"-pkt-debug"}, false, "enable per-packet debug logging")
 	mflag.StringVar(&prof, []string{"-profile"}, "", "enable profiling and write profiles to given path")
-	mflag.IntVar(&config.ConnLimit, []string{"-conn-limit"}, 100, "connection limit (0 for unlimited)")
+	mflag.IntVar(&config.ConnLimit, []string{"-conn-limit"}, 200, "connection limit (0 for unlimited)")
 	mflag.BoolVar(&noDiscovery, []string{"-no-discovery"}, false, "disable peer discovery")
 	mflag.IntVar(&bufSzMB, []string{"-bufsz"}, 8, "capture buffer size in MB")
 	mflag.IntVar(&bridgeConfig.MTU, []string{"-mtu"}, 0, "MTU size")
@@ -307,6 +307,11 @@ func main() {
 
 	bridgeConfig.Mac = name.String()
 	bridgeConfig.Port = config.Port
+	if httpAddr != "" {
+		if _, port, err := net.SplitHostPort(httpAddr); err == nil {
+			bridgeConfig.ControlPort = port
+		}
+	}
 	ips := ipset.New(common.LogLogger(), 0)
 	bridgeType, err := weavenet.EnsureBridge(procPath, &bridgeConfig, Log, ips)
 	checkFatal(err)
@@ -385,7 +390,7 @@ func main() {
 		dockerVersion = dockerCli.DockerVersion()
 	}
 
-	checkForUpdates(dockerVersion, router)
+	checkForUpdates(dockerVersion, router, uint(len(peers)))
 
 	observeContainers := func(o docker.ContainerObserver) {
 		if dockerCli != nil {
@@ -453,6 +458,7 @@ func main() {
 
 	pluginConfig.DNS = !noDNS
 	pluginConfig.DefaultSubnet = defaultSubnet.String()
+	pluginConfig.ProcPath = procPath
 	plugin := plugin.NewPlugin(pluginConfig)
 
 	// The weave script always waits for a status call to succeed,
@@ -513,7 +519,7 @@ func exposeForAWSVPC(alloc *ipam.Allocator, subnet address.CIDR, bridgeName stri
 	addr, err := alloc.Allocate("weave:expose", subnet, false, func() bool { return false })
 	checkFatal(err)
 	cidr := address.MakeCIDR(subnet, addr)
-	err = weavenet.Expose(bridgeName, cidr.IPNet(), true, false)
+	err = weavenet.Expose(bridgeName, cidr.IPNet(), true, false, false)
 	checkFatal(err)
 	Log.Printf("Bridge %q exposed on address %v", bridgeName, cidr)
 	ready()

prog/weaveutil/container.go

@@ -89,8 +89,9 @@ func containerFQDN(args []string) error {
 	return nil
 }
 
-func runContainer(args []string) error {
+func parseContainerArgs(args []string) docker.CreateContainerOptions {
 	env := []string{}
+	labels := map[string]string{}
 	name := ""
 	net := ""
 	pid := ""
@@ -109,6 +110,10 @@ func runContainer(args []string) error {
 				env = append(env, args[i+1])
 			}
 			args = append(args[:i], args[i+2:]...)
+		case "-l", "--label":
+			key, value := parseLabel(args[i+1])
+			labels[key] = value
+			args = append(args[:i], args[i+2:]...)
 		case "--name":
 			name = args[i+1]
 			args = append(args[:i], args[i+2:]...)
@@ -161,19 +166,25 @@ func runContainer(args []string) error {
 	image := args[0]
 	cmds := args[1:]
 
+	config := docker.Config{Image: image, Env: env, Cmd: cmds, Labels: labels}
+	hostConfig := docker.HostConfig{NetworkMode: net, PidMode: pid, Privileged: privileged, RestartPolicy: restart, Binds: volumes, VolumesFrom: volumesFrom}
+	return docker.CreateContainerOptions{Name: name, Config: &config, HostConfig: &hostConfig}
+}
+
+func runContainer(args []string) error {
+	containerOptions := parseContainerArgs(args)
+
 	c, err := docker.NewVersionedClientFromEnv("1.18")
 	if err != nil {
 		return fmt.Errorf("unable to connect to docker: %s", err)
 	}
 
-	config := docker.Config{Image: image, Env: env, Cmd: cmds}
-	hostConfig := docker.HostConfig{NetworkMode: net, PidMode: pid, Privileged: privileged, RestartPolicy: restart, Binds: volumes, VolumesFrom: volumesFrom}
-	container, err := c.CreateContainer(docker.CreateContainerOptions{Name: name, Config: &config, HostConfig: &hostConfig})
+	container, err := c.CreateContainer(containerOptions)
 	if err != nil {
 		return fmt.Errorf("unable to create container: %s", err)
 	}
 
-	err = c.StartContainer(container.ID, &hostConfig)
+	err = c.StartContainer(container.ID, containerOptions.HostConfig)
 	if err != nil {
 		return fmt.Errorf("unable to start container: %s", err)
 	}
@@ -182,6 +193,14 @@ func runContainer(args []string) error {
 	return nil
 }
 
+func parseLabel(s string) (key, value string) {
+	pos := strings.Index(s, "=")
+	if pos == -1 { // no value - set it to blank
+		return s, ""
+	}
+	return s[:pos], s[pos+1:]
+}
+
 func listContainers(args []string) error {
 	if len(args) > 1 {
 		cmdUsage("list-containers", "[<label>]")

prog/weaveutil/container_test.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"testing"
+
+	docker "github.com/fsouza/go-dockerclient"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseContainerArgs(t *testing.T) {
+	args := []string{"--name", "weave", "--privileged", "--net", "host",
+		"-l", "foo=bar", "-l", "foobar",
+		"-v", "/var/run/docker.sock:/var/run/docker.sock", "-v", "/etc:/host/etc",
+		"--restart", "always", "--pid", "host", "--volumes-from", "weavedb",
+		"-e", "WEAVE_DEBUG", "-e", "EXEC_IMAGE=weaveworks/weaveexec:latest",
+		"weaveworks/weave:latest", "cmd", "arg1", "arg2"}
+	expected := docker.CreateContainerOptions{
+		Name: "weave",
+		Config: &docker.Config{
+			Image:  "weaveworks/weave:latest",
+			Env:    []string{"WEAVE_DEBUG", "EXEC_IMAGE=weaveworks/weaveexec:latest"},
+			Cmd:    []string{"cmd", "arg1", "arg2"},
+			Labels: map[string]string{"foo": "bar", "foobar": ""},
+		},
+		HostConfig: &docker.HostConfig{
+			NetworkMode:   "host",
+			PidMode:       "host",
+			Privileged:    true,
+			RestartPolicy: docker.RestartPolicy{Name: "always"},
+			Binds:         []string{"/var/run/docker.sock:/var/run/docker.sock", "/etc:/host/etc"},
+			VolumesFrom:   []string{"weavedb"},
+		},
+	}
+	have := parseContainerArgs(args)
+	require.Equal(t, expected, have)
+}

proxy/proxy.go

@@ -225,7 +225,7 @@ func (proxy *Proxy) findVolume(v string) (string, error) {
 }
 
 func (proxy *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	Log.Infof("%s %s", r.Method, r.URL)
+	Log.Debugf("%s %s", r.Method, r.URL)
 	path := r.URL.Path
 	var i interceptor
 	switch {

router/fastdp.go

@@ -482,7 +482,7 @@ func (fastdp *FastDatapath) getVxlanVportID(udpPort int) (odp.VportID, error) {
 			// The netdev interface is down, so most likely bringing it up
 			// has failed due to the UDP port being in use.
 			if err := fastdp.dp.DeleteVport(vxlanVportID); err != nil {
-				log.Warning("Unable to remove vxlan vport %d: %s", vxlanVportID, err)
+				log.Warningf("Unable to remove vxlan vport %d: %s", vxlanVportID, err)
 			}
 			return 0, odp.NetlinkError(syscall.EADDRINUSE)
 		}
@@ -582,9 +582,11 @@ type fastDatapathForwarder struct {
 	ackedHeartbeat    bool
 	stopChan          chan struct{}
 	stopped           bool
-
-	establishedChan chan struct{}
-	errorChan       chan error
+	healthy           bool
+	established       bool
+	establishedChan   chan struct{}
+	errorChan         chan error
+	healthChan        chan bool
 }
 
 func (fastdp fastDatapathOverlay) PrepareConnection(params mesh.OverlayConnectionParams) (mesh.OverlayConnection, error) {
@@ -622,6 +624,7 @@ func (fastdp fastDatapathOverlay) PrepareConnection(params mesh.OverlayConnectio
 		connUID:        params.ConnUID,
 		vxlanVportID:   vxlanVportID,
 		sessionKey:     params.SessionKey,
+		healthy:        true,
 
 		remoteAddr:        remoteAddr,
 		heartbeatInterval: FastHeartbeat,
@@ -629,6 +632,7 @@ func (fastdp fastDatapathOverlay) PrepareConnection(params mesh.OverlayConnectio
 
 		establishedChan: make(chan struct{}),
 		errorChan:       make(chan error, 1),
+		healthChan:      make(chan bool),
 	}
 
 	return fwd, nil
@@ -650,27 +654,26 @@ func (fwd *fastDatapathForwarder) logPrefix() string {
 
 func (fwd *fastDatapathForwarder) Confirm() {
 	fwd.lock.Lock()
-	defer fwd.lock.Unlock()
 
 	if fwd.confirmed {
 		log.Fatal(fwd.logPrefix(), "already confirmed")
 	}
 
+	var controlMsg []byte
 	if fwd.fastdp.ipsec != nil && fwd.sessionKey != nil {
 		fwd.isEncrypted = true
 		log.Info("Setting up IPsec between ", fwd.fastdp.localPeer, " and ", fwd.remotePeer)
-		err := fwd.fastdp.ipsec.InitSALocal(
+		var err error
+		controlMsg, err = fwd.fastdp.ipsec.InitSALocal(
 			fwd.fastdp.localPeer.Name, fwd.remotePeer.Name, fwd.connUID,
 			net.IP(fwd.localIP[:]), fwd.remoteAddr.IP,
 			fwd.remoteAddr.Port,
 			fwd.sessionKey,
-			func(msg []byte) error {
-				return fwd.sendControlMsg(FastDatapathCryptoInitSARemote, msg)
-			},
 		)
 		if err != nil {
 			log.Error(fwd.logPrefix(), "ipsec init SA local failed: ", err)
 			fwd.handleError(err)
+			fwd.lock.Unlock()
 			return
 		}
 	}
@@ -688,6 +691,17 @@ func (fwd *fastDatapathForwarder) Confirm() {
 	}
 
 	fwd.heartbeatTimeout = time.NewTimer(HeartbeatTimeout)
+
+	fwd.lock.Unlock() // unlock before calling send() which may block
+
+	if len(controlMsg) > 0 {
+		if err := fwd.sendControlMsg(FastDatapathCryptoInitSARemote, controlMsg); err != nil {
+			log.Error(fwd.logPrefix(), "ipsec send InitSARemote failed: ", err)
+			fwd.handleError(err)
+			return
+		}
+	}
+
 	go fwd.doHeartbeats()
 }
 
@@ -699,19 +713,40 @@ func (fwd *fastDatapathForwarder) ErrorChannel() <-chan error {
 	return fwd.errorChan
 }
 
+func (fwd *fastDatapathForwarder) HealthChannel() <-chan bool {
+	return fwd.healthChan
+}
+
 func (fwd *fastDatapathForwarder) doHeartbeats() {
 	var err error
-
 	for err == nil {
 		select {
 		case <-fwd.heartbeatTimer.C:
 			if fwd.confirmed {
+				log.Debug(fwd.logPrefix(), "sending Heartbeat to peer")
 				fwd.sendHeartbeat()
 			}
 			fwd.heartbeatTimer.Reset(fwd.heartbeatInterval)
 
 		case <-fwd.heartbeatTimeout.C:
-			err = fmt.Errorf("timed out waiting for vxlan heartbeat")
+			log.Debug(fwd.logPrefix(), "missed Heartbeat from peer, marking fastdp forwarder as un-healthy")
+
+			// treat missed heartbeat as transient error. Indicate to overlay forwarder
+			// that Forwarder is un-healthy so it can pick next best forwarder
+			fwd.healthChan <- false
+
+			// switch from fast-heartbeat to slow-heartbeat to avoid aggressive heartbeats
+			// when there is no estbalished session with peer yet
+			if fwd.heartbeatInterval != SlowHeartbeat {
+				fwd.heartbeatInterval = SlowHeartbeat
+				if fwd.heartbeatTimer != nil {
+					fwd.heartbeatTimer.Reset(fwd.heartbeatInterval)
+				}
+			}
+
+			if fwd.healthy {
+				fwd.healthy = false
+			}
 
 		case <-fwd.stopChan:
 			return
@@ -744,7 +779,6 @@ func (fwd *fastDatapathForwarder) handleError(err error) {
 
 func (fwd *fastDatapathForwarder) sendHeartbeat() {
 	fwd.lock.RLock()
-	log.Debug(fwd.logPrefix(), "sendHeartbeat")
 
 	// the heartbeat payload consists of the 64-bit connection uid
 	// followed by the 16-bit packet size.
@@ -801,13 +835,23 @@ func (fwd *fastDatapathForwarder) handleVxlanSpecialPacket(frame []byte, sender
 
 	if !fwd.ackedHeartbeat {
 		fwd.ackedHeartbeat = true
+		log.Debug(fwd.logPrefix(), "Ack Heartbeat from peer")
 		fwd.handleError(fwd.sendControlMsg(FastDatapathHeartbeatAck, nil))
+	} else {
+		log.Debug(fwd.logPrefix(), "Got Heartbeat Ack from peer")
 	}
 
 	// we can receive a heartbeat before Confirm() has set up
 	// heartbeatTimeout
 	if fwd.heartbeatTimeout != nil {
 		fwd.heartbeatTimeout.Reset(HeartbeatTimeout)
+		if !fwd.healthy {
+			// If fastdp was marked as unhealthy earlier due to missed heartbeat then indicate to
+			// overlay forwarder that Forwarder is healthy now so it can pick next best forwarder
+			log.Debug(fwd.logPrefix(), "got Heartbeat from peer, marking fastdp forwarder as healthy")
+			fwd.healthy = true
+			fwd.healthChan <- true
+		}
 	}
 }
 
@@ -833,8 +877,12 @@ func (fwd *fastDatapathForwarder) Attrs() map[string]interface{} {
 func (fwd *fastDatapathForwarder) handleHeartbeatAck() {
 	log.Debug(fwd.logPrefix(), "handleHeartbeatAck")
 
-	if fwd.heartbeatInterval != SlowHeartbeat {
+	if !fwd.established {
 		close(fwd.establishedChan)
+		fwd.established = true
+	}
+
+	if fwd.heartbeatInterval != SlowHeartbeat {
 		fwd.heartbeatInterval = SlowHeartbeat
 		if fwd.heartbeatTimer != nil {
 			fwd.heartbeatTimer.Reset(fwd.heartbeatInterval)
@@ -843,6 +891,11 @@ func (fwd *fastDatapathForwarder) handleHeartbeatAck() {
 }
 
 func (fwd *fastDatapathForwarder) handleCryptoInitSARemote(msg []byte) {
+	if fwd.stopped {
+		log.Info(fwd.logPrefix(), "IPSec init SA remote failed: forwarder has already been stopped")
+		return
+	}
+
 	log.Info(fwd.logPrefix(), "IPSec init SA remote")
 	err := fwd.fastdp.ipsec.InitSARemote(
 		msg,

router/http.go

@@ -3,6 +3,7 @@ package router
 import (
 	"fmt"
 	"net/http"
+	"strconv"
 
 	"github.com/gorilla/mux"
 
@@ -37,7 +38,14 @@ func (router *NetworkRouter) HandleHTTP(muxRouter *mux.Router) {
 			return
 		}
 
-		if err = net.Expose(router.BridgeConfig.WeaveBridgeName, cidr.IPNet(), router.BridgeConfig.AWSVPC, router.BridgeConfig.NPC); err != nil {
+		var skipNAT bool
+		if r.FormValue("skipNAT") != "" {
+			if skipNAT, err = strconv.ParseBool(r.FormValue("skipNAT")); err != nil {
+				http.Error(w, fmt.Sprint("unable to parse skipNAT option: ", err.Error()), http.StatusBadRequest)
+			}
+		}
+
+		if err = net.Expose(router.BridgeConfig.WeaveBridgeName, cidr.IPNet(), router.BridgeConfig.AWSVPC, router.BridgeConfig.NPC, skipNAT); err != nil {
 			http.Error(w, fmt.Sprint("unable to expose: ", err.Error()), http.StatusInternalServerError)
 			return
 		}

router/network_overlay.go

@@ -31,6 +31,8 @@ type OverlayForwarder interface {
 	// Confirm().  The return value nil means the key could not be
 	// handled by this forwarder.
 	Forward(ForwardPacketKey) FlowOp
+
+	HealthChannel() <-chan bool
 }
 
 type NullNetworkOverlay struct{ mesh.NullOverlay }

router/overlay_switch.go

@@ -142,6 +142,7 @@ type overlaySwitchForwarder struct {
 	alreadyEstablished bool
 	establishedChan    chan struct{}
 	errorChan          chan error
+	healthChan         chan bool
 }
 
 // A subsidiary forwarder
@@ -154,6 +155,9 @@ type subForwarder struct {
 
 	// closed to tell the forwarder monitor goroutine to stop
 	stopChan chan<- struct{}
+
+	// set to true to skip this forwarder in chooseBest()
+	onHold bool
 }
 
 // An event from a subsidiary forwarder
@@ -166,6 +170,9 @@ type subForwarderEvent struct {
 
 	// is this an error event?
 	err error
+
+	// event to indicate if forwarder is in healthy state to be considered for best forwarded
+	healthy *bool
 }
 
 func (osw *OverlaySwitch) PrepareConnection(params mesh.OverlayConnectionParams) (mesh.OverlayConnection, error) {
@@ -248,6 +255,9 @@ loop:
 			e.established = true
 			establishedChan = nil
 
+		case healthy := <-fwd.HealthChannel():
+			e.healthy = &healthy
+
 		case err := <-fwd.ErrorChannel():
 			e.err = err
 
@@ -282,6 +292,8 @@ loop:
 				fwd.established(e.index)
 			case e.err != nil:
 				fwd.error(e.index, e.err)
+			case e.healthy != nil:
+				fwd.healthCheck(e.index, *e.healthy)
 			}
 		}
 	}
@@ -318,6 +330,23 @@ func (fwd *overlaySwitchForwarder) error(index int, err error) {
 	fwd.chooseBest()
 }
 
+func (fwd *overlaySwitchForwarder) healthCheck(index int, healthy bool) {
+	fwd.lock.Lock()
+	defer fwd.lock.Unlock()
+
+	if healthy == true && fwd.forwarders[index].onHold {
+		log.Debug(fwd.logPrefix(), "Adding "+fwd.forwarders[index].overlayName+" to the list of forwarders")
+		fwd.forwarders[index].onHold = false
+		fwd.chooseBest()
+	}
+
+	if healthy == false && !fwd.forwarders[index].onHold {
+		log.Debug(fwd.logPrefix(), "Removing "+fwd.forwarders[index].overlayName+" from the list of forwarders")
+		fwd.forwarders[index].onHold = true
+		fwd.chooseBest()
+	}
+}
+
 func (fwd *overlaySwitchForwarder) stopFrom(index int) {
 	for index < len(fwd.forwarders) {
 		subFwd := &fwd.forwarders[index]
@@ -337,7 +366,7 @@ func (fwd *overlaySwitchForwarder) chooseBest() {
 
 	for i := range fwd.forwarders {
 		subFwd := &fwd.forwarders[i]
-		if subFwd.fwd == nil {
+		if subFwd.fwd == nil || subFwd.onHold {
 			continue
 		}
 
@@ -415,6 +444,10 @@ func (fwd *overlaySwitchForwarder) ErrorChannel() <-chan error {
 	return fwd.errorChan
 }
 
+func (fwd *overlaySwitchForwarder) HealthChannel() <-chan bool {
+	return fwd.healthChan
+}
+
 func (fwd *overlaySwitchForwarder) Stop() {
 	fwd.lock.Lock()
 	defer fwd.lock.Unlock()

router/pcap.go

@@ -4,10 +4,14 @@ import (
 	"fmt"
 	"net"
 	"sync"
+	"time"
 
 	"github.com/google/gopacket/pcap"
 )
 
+// Must be less than 2^31 usec - see https://github.com/google/gopacket/issues/708
+const longTimeout = time.Duration(30 * time.Minute)
+
 type Pcap struct {
 	NonDiscardingFlowOp
 
@@ -77,7 +81,7 @@ func newPcapHandle(ifName string, promisc bool, snaplen int, bufSz int) (handle
 	if err = inactive.SetSnapLen(snaplen); err != nil {
 		return
 	}
-	if err = inactive.SetTimeout(MaxDuration); err != nil {
+	if err = inactive.SetTimeout(longTimeout); err != nil {
 		return
 	}
 	if err = inactive.SetImmediateMode(true); err != nil {

router/sleeve.go

@@ -326,6 +326,7 @@ type sleeveForwarder struct {
 	// listener channels
 	establishedChan chan struct{}
 	errorChan       chan error
+	healthChan      chan bool
 
 	// Explicitly locked state
 	lock       sync.RWMutex
@@ -445,6 +446,10 @@ func (fwd *sleeveForwarder) ErrorChannel() <-chan error {
 	return fwd.errorChan
 }
 
+func (fwd *sleeveForwarder) HealthChannel() <-chan bool {
+	return fwd.healthChan
+}
+
 type curriedForward struct {
 	NonDiscardingFlowOp
 	fwd *sleeveForwarder
@@ -1087,7 +1092,7 @@ func (sender *udpSenderDF) send(msg []byte, raddr *net.UDPAddr) error {
 	}
 	defer f.Close()
 
-	log.Print("EMSGSIZE on send, expecting PMTU update (IP packet was ", len(packet), " bytes, payload was ", len(msg), " bytes)")
+	log.Debug("sleeve ->[", sender, "] expecting PMTU update (IP packet was ", len(packet), " bytes, payload was ", len(msg), " bytes)")
 	pmtu, err := syscall.GetsockoptInt(int(f.Fd()), syscall.IPPROTO_IP, syscall.IP_MTU)
 	if err != nil {
 		return err

site/concepts/network-topology.md

@@ -142,6 +142,15 @@ is not reachable, then the packet is dropped.  For most protocols
 (for example, TCP), the transmission will be retried a short time later, by
 which time the topology should have updated.
 
+#### <a name="connection-limit"></a>Connection Limit
+
+In a fully connected mesh of N peers, this would mean N^2 connections. To constrain the resource consumption and 
+communication between the peers and complexity of route calculation there is a configurable upper limit on the 
+maximum number of connections that a peer can make to the remote peers. There is safe default value (100) which works
+for most deployment. However if you would like to increase the number of peers, you should be able to explicitly override the
+default value by passing the value through `connlimit` flag for e.g.`weave launch --connlimit=100`. If you are using Weave with
+Kubernetes using [Kubernetes Addon](/site/kubernetes/kube-addon.md) then you can set CONN_LIMIT environment variable to set
+the connection limit.
 
 **See Also**
 

site/install.md

@@ -31,7 +31,6 @@ Install Weave Net from the command line on its own or if you are using Docker, K
 * [Integrating Kubernetes and Mesos via the CNI Plugin](https://www.weave.works/docs/net/latest/kubernetes/)
 * [Integrating Kubernetes via the Addon](https://www.weave.works/docs/net/latest/kubernetes/kube-addon/)
      * [Installation](https://www.weave.works/docs/net/latest/kubernetes/kube-addon/#install)
-     * [Upgrading Kubernetes to version 1.6](https://www.weave.works/docs/net/latest/kubernetes/kube-addon/#kube-1.6-upgrade)
      * [Upgrading the Daemon Sets](https://www.weave.works/docs/net/latest/kubernetes/kube-addon/#daemon-sets)
      * [CPU and Memory Requirements](https://www.weave.works/docs/net/latest/kubernetes/kube-addon/#resources)
      * [Pod Eviction](https://www.weave.works/docs/net/latest/kubernetes/kube-addon/#eviction)

site/install/installing-weave.md

@@ -48,10 +48,8 @@ To disable this check, run the following before launching Weave Net:
 
 ### Guides for Specific Platforms
 
-CoreOS users see [here](/guides/networking-docker-containers-with-weave-on-coreos/) for an example of installing Weave using cloud-config.
-
-Amazon ECS users see [here](https://github.com/weaveworks/integrations/blob/master/aws/ecs/README.md)
-for the latest Weave AMIs and [here](https://www.weave.works/docs/tutorials/old-guides/ecs/) to get started with Weave Net on ECS.
+Amazon ECS users see [here](https://www.weave.works/docs/scope/latest/ami/)
+for the latest Weave AMIs.
 
 If you're on Amazon EC2, the standard installation instructions at the
 top of this page, provide the simplest setup and the most flexibility.

site/kubernetes.md

@@ -30,6 +30,8 @@ Then run:
 
     weave setup
 
+Weave net depends on the *portmap* [standard](https://github.com/containernetworking/plugins) CNI plugin to support *hostport* functionality. Please ensure that *portmap* CNI plugin is installed in `/opt/cni/bin` directory.
+
 #### Launching Weave Net
 
 To create a network that spans multiple hosts, the Weave peers must be connected to each other.  
@@ -45,7 +47,7 @@ for a discussion on peer connections.
 
 All CNI plugins are configured by a JSON file in the directory
 `/etc/cni/net.d/`.  `weave setup` installs a minimal configuration
-file named `10-weave.conf`, which you can alter to suit your needs.
+file named `10-weave.conflist`, which you can alter to suit your needs.
 
 See the [CNI Spec](https://github.com/appc/cni/blob/master/SPEC.md#network-configuration)
 for details on the format and contents of this file.

site/kubernetes/kube-addon.md

@@ -7,7 +7,8 @@ search_type: Documentation
 The following topics are discussed:
 
 * [Installation](#install)
-   * [Upgrading Kubernetes to version 1.6](#kube-1.6-upgrade)
+   * [Installing on GKE](#gke)
+   * [Installing on EKS](#eks)
    * [Upgrading the Daemon Sets](#daemon-sets)
    * [CPU and Memory Requirements](#resources)
    * [Pod Eviction](#eviction)
@@ -19,6 +20,7 @@ The following topics are discussed:
    * [Troubleshooting Blocked Connections](#blocked-connections)
    * [Things to watch out for](#key-points)
 * [Changing Configuration Options](#configuration-options)
+* [Securing The Setup](#securing-the-setup)
 
 
 ## <a name="install"></a> Installation
@@ -34,6 +36,8 @@ with a single command:
 $ kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')"
 ```
 
+**Important:** this configuration won't enable encryption by default. If your data plane traffic isn't secured that could allow malicious actors to access your pod network. Read on to see the alternatives.
+
 After a few seconds, a Weave Net pod should be running on each
 Node and any further pods you create will be automatically attached to the Weave
 network.
@@ -50,6 +54,11 @@ recommend your master node has at least two CPU cores.
 > [kubeadm](http://kubernetes.io/docs/getting-started-guides/kubeadm/).
 >
 > Alternatively, you can [configure CNI yourself](http://kubernetes.io/docs/admin/network-plugins/#cni)
+>
+> Weave net depends on the *portmap* [standard](https://github.com/containernetworking/plugins) CNI plugin
+> to support *hostport* functionality. Please ensure that *portmap* CNI plugin is installed 
+> (either by cluster installers like kubeadm or manually if you have configured CNI yourself) in `/opt/cni/bin` directory.
+
 
 **Note:** If using the [Weave CNI
 Plugin](/site/kubernetes.md) from a prior full install of Weave Net with your
@@ -64,38 +73,33 @@ Shut down Kubernetes, and _on all nodes_ perform the following:
 Then relaunch Kubernetes and install the addon as described
 above.
 
-## <a name="kube-1.6-upgrade"></a> Upgrading Kubernetes to version 1.6
+### <a name="gke"></a> Installing on GKE
+Please note that you must grant the user the ability to create roles in Kubernetes before launching Weave Net.
+This is a prerequisite to use use role-based access control on GKE. Please see the GKE [instructions](https://cloud.google.com/kubernetes-engine/docs/how-to/role-based-access-control).
 
-In version 1.6, Kubernetes has increased security, so we need to
-create a special service account to run Weave Net. This is done in
-the file `weave-daemonset-k8s-1.6.yaml` attached to the [Weave Net
-release](https://github.com/weaveworks/weave/releases/latest).
+### <a name="gke"></a> Installing on EKS
 
-Also, the
-[toleration](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/taint-toleration-dedicated.md)
-required to let Weave Net run on master nodes has moved from an
-annotation to a field on the DaemonSet spec object.
+EKS by default installs `amazon-vpc-cni-k8s` CNI. Please follow below steps to use Weave-net as CNI
+
+- create EKS cluster in any of [prescribed](https://docs.aws.amazon.com/eks/latest/userguide/create-cluster.html) way
+- delete `amazon-vpc-cni-k8s` deamonset by running `kubectl delete ds aws-node -n kube-system` command
+- delete `/etc/cni/net.d/10-aws.conflist` on each of the node
+- edit instance security group to allow TCP 6783 and UDP 6783/6784 ports
+- flush iptables nat, mangle, filter tables to clear any iptables configurations done by `amazon-vpc-cni-k8s`
+- restart kube-proxy pods to reconfigure iptables
+- apply weave-net daemoset by following above installation steps
+- delete existing pods so they get recreated in Weave pod CIDR's address-space. 
+
+Please note that while pods can connect to the Kubernetes API server for your cluser, API server will not be able to connect to the pods as API server nodes are not connected to Weave Net (they run on network managed by EKS).
 
-If you have edited the Weave Net DaemonSet from a previous release,
-you will need to re-make your changes against the new version.
 
 ### <a name="daemon-sets"></a> Upgrading the Daemon Sets
 
-For Kubernetes 1.6 and above the DaemonSet definition specifies
-[Rolling Updates](https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/),
+The DaemonSet definition specifies [Rolling
+Updates](https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/),
 so when you apply a new version Kubernetes will automatically restart
 the Weave Net pods one by one.
 
-Kubernetes v1.5 and below does not support rolling upgrades of daemon sets,
-and so you will need to perform the procedure manually:
-
-* Apply the updated addon manifest `kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')"`
-* Kill each Weave Net pod with `kubectl delete` and then wait for it to reboot before moving on to the next pod.
-
-**Note:** In versions prior to Weave Net 2.0, deleting all Weave Net pods at the same time
-  will result in them losing track of IP address range ownership, possibly leading to
-  duplicate IP addresses if you then start a new copy of Weave Net.
-
 ## <a name="resources"></a>CPU and Memory Requirements
 
 Kubernetes manages
@@ -103,23 +107,18 @@ Kubernetes manages
 on each node, and only schedules pods to run on nodes that have enough
 free resources.
 
-The components of a typical Kubernetes installation (with the master
-node running etcd, scheduler, api-server, etc.) take up about 95% of a
-CPU, which leaves little room to run anything else. For all of Weave
-Net's features to work, it must run on every node, including the
-master.
+In the example manifests we request 10% of a CPU, which will be enough
+for small installations, but you should monitor how much it uses in
+your production clusters and adjust the requests to suit.
 
-The best way to resolve this issue is to use machines with at least
-two CPU cores. However if you are installing Kubernetes and Weave Net
-for the first time, you may not be aware of this requirement. For this
-reason, Weave Net launches as a DaemonSet with a specification that
-reserves at least 1% CPU for each container. This enables Weave Net to
-start up seamlessly on a single-CPU node.
+We do not recommend that you set a CPU or memory _limit_ unless you
+are very experienced in such matters, because the implementation of
+limits in the Linux kernel sometimes behaves in a surprising way.
 
-Depending on the workload, Weave Net may need more than 1% of the
-CPU. The percentage set in the DaemonSet is the minimum and not a
-limit. This minimum setting allows Weave Net to take advantage of
-available CPU and "burst" above that limit if it needs to.
+On a 1-node single-CPU cluster you may find Weave Net does not install
+at all, because other Kubernetes components already take 95% of the
+CPU. The best way to resolve this issue is to use machines with at
+least two CPU cores.
 
 ## <a name="eviction"></a>Pod Eviction
 
@@ -180,7 +179,7 @@ labels. For more information on configuring network policies in
 Kubernetes see the
 [walkthrough](https://kubernetes.io/docs/tasks/administer-cluster/declare-network-policy/)
 and the [NetworkPolicy API object
-definition](https://v1-7.docs.kubernetes.io/docs/api-reference/v1.7/#networkpolicy-v1-networking)
+definition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#networkpolicy-v1-networking-k8s-io)
 
 **Note:** as of version 1.9 of Weave Net, the Network Policy
   Controller allows all multicast traffic. Since a single multicast
@@ -189,6 +188,11 @@ definition](https://v1-7.docs.kubernetes.io/docs/api-reference/v1.7/#networkpoli
   all multicast traffic) by adding `--allow-mcast=false` as an
   argument to `weave-npc` in the YAML configuration.
 
+**Note:** Since ingress traffic is masqueraded, it makes sense to use
+`ipBlock` selector in an ingress rule only when limiting access to a Service
+annotated with `externalTrafficPolicy=Local` or between Pods when `podIP` is used
+to access a Pod.
+
 ## <a name="troubleshooting"></a> Troubleshooting
 
 The first thing to check is whether Weave Net is up and
@@ -221,12 +225,26 @@ ask for the logs like this:
 $ kubectl logs -n kube-system weave-net-1jkl6 weave
 ```
 
-For easier viewing, pipe the output into a file , especially if
+For easier viewing, pipe the output into a file, especially if
 it is long.
 
+By default log level of `weave` container is set to `info` level. If you wish to see more detailed logs you can set the desired log level for the `--log-level` flag through the `EXTRA_ARGS` environment variable for the `weave` container in the weave-net daemon set. Add environment variable as below.
+
+```yaml
+      containers:
+      - command:
+        - /home/weave/launch.sh
+        name: weave
+        env:
+        - name: EXTRA_ARGS
+          value: --log-level=debug
+```
+
+You may also set the `--log-level` flag to `warning` or `error` if you
+prefer to only log exceptional conditions.
+
 Many Kubernetes network issues occur at a higher level than Weave Net.
-The [Kubernetes Service Debugging Guide]
-(https://kubernetes.io/docs/tasks/debug-application-cluster/debug-service/)
+The [Kubernetes Service Debugging Guide](https://kubernetes.io/docs/tasks/debug-application-cluster/debug-service/)
 has a detailed step-by-step guide.
 
 Once it is up and running, the status of Weave Net can be checked by
@@ -336,6 +354,8 @@ UDP connection from 10.32.0.7:56648 to 10.32.0.11:80 blocked by Weave NPC.
 
 ### <a name="key-points"></a> Things to watch out for
 
+- Weave Net does not work on hosts running iptables 1.8 or above, only with 1.6.
+  Track this via issue [#3465](https://github.com/weaveworks/weave/issues/3465)
 - Don't turn on `--masquerade-all` on kube-proxy: this will change the
   source address of every pod-to-pod conversation which will make it
   impossible to correctly enforce network policies that restrict which
@@ -376,7 +396,7 @@ The list of variables you can set is:
 * `CHECKPOINT_DISABLE` - if set to 1, disable checking for new Weave Net
   versions (default is blank, i.e. check is enabled)
 * `CONN_LIMIT` - soft limit on the number of connections between
-  peers. Defaults to 30.
+  peers. Defaults to 200.
 * `HAIRPIN_MODE` - Weave Net defaults to enabling hairpin on the bridge side of
   the `veth` pair for containers attached. If you need to disable hairpin, e.g. your
   kernel is one of those that can panic if hairpin is enabled, then you can disable it
@@ -404,6 +424,7 @@ The list of variables you can set is:
 * `NO_MASQ_LOCAL` - set to 1 to preserve the client source IP address when
   accessing Service annotated with `service.spec.externalTrafficPolicy=Local`.
   The feature works only with Weave IPAM (default).
+* `IPTABLES_BACKEND` - set to `nft` to use `nftables` backend for `iptables` (default is `iptables`) 
 
 Example:
 ```
@@ -442,3 +463,19 @@ For example,
             - name: IPALLOC_RANGE
               value: 10.0.0.0/16
 ```
+
+## <a name="securing-the-setup"></a> Securing the Setup
+
+You must pass the `password-secret` option as noted in the previous section to enable the data plane encryption; this is a recommended option in case you cannot be sure about the security of the fabric between your nodes.
+
+A different option is to use `trusted-subnets` and whitelist only the subnets that host your k8s nodes. Mind that depending on your circumstances that might allow a malicious container running in your cluster to access the weave dataplane, still.
+
+Read on the [Securing Connections Across Untrusted Networks](/site/tasks/manage/security-untrusted-networks/) document to see the alternatives.
+
+To improve security drop `CAP_NET_RAW` from pod capabilities: by default pods can forge packets from anywhere on the network, which enables attacks such as DNS spoofing.
+
+```
+     securityContext:
+       capabilities:
+         drop: ["NET_RAW"]
+```

site/overview.md

@@ -9,6 +9,9 @@ Weave Net creates a virtual network that connects Docker containers across multi
 
 Services provided by application containers on the weave network can be exposed to the outside world, regardless of where they are running. Similarly, existing internal systems can be opened to accept connections from application containers irrespective of their location.
 
+### How does it work?
+
+Weave Net creates a new Layer 2 network using Linux kernel features; one daemon sets up that network and manages routing between machines and there are various ways to attach to that network (e.g. a CNI plugin which runs as a separate process). Also one more daemon if you are using Kubernetes Network Policy.
 
 ## Why Weave?
 

site/tasks/ipam/manual-ip-address.md

@@ -13,7 +13,22 @@ Weave Net detects when a container has exited and releases its allocated address
 
 See the [Automatic IP Address Management](/site/tasks/ipam/ipam.md) and also an explanation of [the basics of IP addressing](/site/concepts/ip-addresses.md) for further details.
 
-Instead of allowing Weave Net to allocate IP addresses automatically (using IPAM), there may be instances where you need to control a particular container or a cluster by setting an IP address for it.  
+Instead of allowing Weave Net to allocate IP addresses automatically (using IPAM), there may be instances where you need to control a particular container or a cluster by setting an IP address for it.  The way to do this varies according to how containers are attached to Weave Net.
+
+## Manually specifying an IP address when using CNI
+
+The software making a call to CNI has to specify an allocator that allows static IPs.
+For instance the [static allocator](https://github.com/containernetworking/plugins/tree/master/plugins/ipam/static)
+
+(At the time of writing there is no straightforward way to ask Kubernetes to do this.)
+
+## Manually specifying an IP address when using the Weave Net Docker Plugin
+
+Docker has a flag `--ip` to specify an address, e.g.:
+
+    $ docker run --network=mynetwork --ip=172.18.0.22 ...
+
+## Manually specifying an IP address when using the Weave Net Docker Proxy
 
 You can specify an IP address and a network explicitly, using Classless Inter-Domain Routing or [CIDR notation](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing).
 

site/troubleshooting.md

@@ -332,6 +332,6 @@ and the container image versions as git hashes.
 
 **See Also**
 
- * [Troubleshooting the Kubernetes Addon](/site/kubernetes/kube-addon/#troubleshooting)
+ * [Troubleshooting the Kubernetes Addon](/site/kubernetes/kube-addon.md#troubleshooting)
  * [Troubleshooting IPAM](/site/tasks/ipam/troubleshooting-ipam.md)
  * [Troubleshooting the Proxy](/site/tasks/weave-docker-api/using-proxy.md)

test/150_connect_forget_2_test.sh

@@ -26,10 +26,15 @@ start_container $HOST2 $C2/24 --name=c2
 assert_raises "weave_on $HOST2 connect 10.2.2.1 10.2.2.2"
 assert_peers $HOST2 "10.2.2.1\n10.2.2.2"
 
+sleep 2  # Allow topology to propagate
+
 # Replace bogus hosts with real hosts
 assert_raises "exec_on $HOST1 c1 sh -c '! $PING $C2'"
 assert_raises "weave_on $HOST2 connect --replace $HOST1 $HOST2"
 assert_peers $HOST2 "$HOST1\n$HOST2"
+
+sleep 2  # Allow topology to propagate
+
 assert_raises "exec_on $HOST1 c1 $PING $C2"
 
 # Forget everyone and disconnect
@@ -37,6 +42,9 @@ assert_raises "weave_on $HOST2 forget $HOST1 $HOST2"
 assert_raises "stop_weave_on $HOST1"
 assert_raises "weave_on $HOST1 launch"
 assert_peers $HOST2 ""
+
+sleep 2  # Allow topology to propagate
+
 assert_raises "exec_on $HOST1 c1 sh -c '! $PING $C2'"
 
 end_suite

test/191_create_bridge_2_test.sh

@@ -9,7 +9,7 @@ kill_weaver() {
     run_on $HOST1 sudo ip link set weave down
     WEAVER_PID=$(container_pid $HOST1 weave)
     run_on $HOST1 sudo kill -9 $WEAVER_PID
-    sleep 3
+    sleep 5
 }
 
 start_suite "Re-create bridge after restart"
@@ -20,6 +20,7 @@ WEAVE_NO_FASTDP=1 weave_on $HOST2 launch $HOST1
 
 start_container $HOST1 $C1/24 --name=c1
 start_container $HOST2 $C2/24 --name=c2
+sleep 2 # give topology gossip some time to propagate
 assert_raises "exec_on $HOST1 c1 $PING $C2"
 
 kill_weaver # should re-create the bridge
@@ -30,6 +31,7 @@ assert_raises "exec_on $HOST1 c1 $PING $C2"
 weave_on $HOST1 reset
 weave_on $HOST1 launch $HOST2
 weave_on $HOST1 attach $C1/24 c1
+sleep 2 # give topology gossip some time to propagate
 assert_raises "exec_on $HOST1 c1 $PING $C2"
 
 kill_weaver # should re-create the bridge

test/210_dns_cross_hosts_3_test.sh

@@ -16,6 +16,8 @@ for host in $HOSTS; do
     weave_on $host launch --ipalloc-range $UNIVERSE $HOSTS
 done
 
+sleep 2  # Allow topology updates to propagate
+
 # Basic test
 start_container          $HOST2 $C2/24 --name=c2 -h $NAME2
 start_container_with_dns $HOST1 $C1/24 --name=c1

test/215_stale_dns_3_test.sh

@@ -8,6 +8,8 @@ weave_on $HOST1 launch --no-discovery
 weave_on $HOST2 launch --no-discovery $HOST1
 weave_on $HOST3 launch --no-discovery $HOST2
 
+sleep 2  # Allow topology updates to propagate
+
 start_container $HOST1 --name=c1
 C1=$(container_ip $HOST1 c1)
 start_container_with_dns $HOST3 --name=c3
@@ -25,7 +27,7 @@ weave_on $HOST1 launch --no-discovery $HOST3
 # Now re-enable gossip from host2 to host3
 run_on $HOST3 "sudo iptables -D INPUT -s $HOST2 -j DROP"
 
-sleep 1
+sleep 2 # allow some time for connections to reform and updates to propagate
 
 assert_no_dns_record $HOST3 c3 c1.weave.local
 

test/320_claim_3_test.sh

@@ -66,7 +66,7 @@ weave_on $HOST1 launch --ipalloc-range $UNIVERSE $HOST2
 # Start another container on host1. The starting should block, because host1 is
 # not able to establish the ring due to host2 being offline.
 CMD="proxy_start_container $HOST1 --name c4 -e WEAVE_CIDR=$C4/12"
-assert_raises "timeout 5 cat <( $CMD )" 124
+assert_raises "timeout --signal=9 5 cat <( $CMD )" 137
 
 # However, allocation for an external subnet should not block.
 assert_raises "proxy_start_container $HOST1 -e WEAVE_CIDR=10.3.0.1/12"

test/380_ipam_seed_2_test.sh

@@ -15,6 +15,9 @@ assert_raises "timeout 10 cat <( start_container $HOST2 --name c2)"
 # Connect routers
 weave_on $HOST2 connect $HOST1
 
+# Allow topology to propagate
+sleep 2
+
 # Check connectivity
 assert_raises "exec_on $HOST1 c1 $PING c2"
 assert_raises "exec_on $HOST2 c2 $PING c1"

test/600_proxy_docker_py.sh

@@ -13,6 +13,9 @@ docker_py_test() {
 
     # Work round https://github.com/docker/docker-py/issues/852
     docker_on $HOST1 pull busybox:buildroot-2014.02 >/dev/null
+    # Work round https://github.com/weaveworks/weave/issues/3680 - pin busybox version to 1.28
+    docker_on $HOST1 pull busybox:1.28 >/dev/null
+    docker_on $HOST1 tag busybox:1.28 busybox:latest >/dev/null
     # Get a list of the tests for use to shard
     docker_on $HOST1 pull $IMAGE >/dev/null
     CANDIDATES=$(docker_on $HOST1 run \

test/800_plugin_cross_hosts_2_test.sh

@@ -15,6 +15,8 @@ weave_on $HOST2 launch $HOST1
 start_container_local_plugin $HOST1 --name=c1 --hostname=$C1_NAME --dns=$DNS_IP
 start_container_local_plugin $HOST2 --name=c2 --hostname=$C2_NAME --dns=$DNS_IP
 
+sleep 2  # Allow topology to propagate
+
 assert_raises "exec_on $HOST1 c1 $PING $C2_NAME"
 assert_raises "exec_on $HOST2 c2 $PING $C1_NAME"
 

test/840_weave_kube_3_test.sh

@@ -52,7 +52,7 @@ fi
 # Ensure Kubernetes uses locally built container images and inject code coverage environment variable (or do nothing depending on $COVERAGE):
 sed -e "s%imagePullPolicy: Always%imagePullPolicy: Never%" \
     -e "s%env:%$WEAVE_ENV_VARS%" \
-    "$(dirname "$0")/../prog/weave-kube/weave-daemonset-k8s-1.8.yaml" | run_on "$HOST1" "$KUBECTL apply -n kube-system -f -"
+    "$(dirname "$0")/../prog/weave-kube/weave-daemonset-k8s-1.9.yaml" | run_on "$HOST1" "$KUBECTL apply -n kube-system -f -"
 
 sleep 5
 
@@ -214,11 +214,17 @@ spec:
               access: deny
 EOF
 
+# Allow some time for the policy change to take effect
+sleep 2
+
 assert_raises "$SSH $HOST1 $KUBECTL exec $denyPodName -- curl -s -S -f -m 2 http://$DOMAIN:8080/status >/dev/null"
 
 # remove the access for nettest-deny
 run_on $HOST1 "$KUBECTL delete netpol allow-nettest-deny"
 
+# Allow some time for the policy change to take effect
+sleep 2
+
 # nettest-deny should still not be able to reach nettest pods
 assert_raises "! $SSH $HOST1 $KUBECTL exec $denyPodName -- curl -s -S -f -m 2 http://$DOMAIN:8080/status >/dev/null"
 
@@ -249,6 +255,27 @@ assert_raises "$SSH $HOST1 $KUBECTL exec $denyPodName -- curl -s -S -f -m 2 http
 assert_raises "! $SSH $HOST1 curl -s -S -f -m 2 http://$VIRTUAL_IP/status >/dev/null"
 assert_raises "! $SSH $HOST1 curl -s -S -f -m 2 http://$HOST2:31138/status >/dev/null"
 
+# allow access only from weave bridge IP of host1 to test ipBlock
+run_on $HOST1 "$KUBECTL apply -f -" <<EOF
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-via-weave-bridge-of-host1
+  namespace: default
+spec:
+  podSelector: {}
+  ingress:
+    - from:
+      - ipBlock:
+          cidr: $HOST1EXPIP/32
+EOF
+
+sleep 2
+
+assert_raises "$SSH $HOST1 curl -s -S -f -m 2 http://$VIRTUAL_IP/status >/dev/null"
+
+run_on $HOST1 "$KUBECTL delete netpol allow-via-weave-bridge-of-host1"
+
 # allow access from anywhere
 run_on $HOST1 "$KUBECTL apply -f -" <<EOF
 apiVersion: networking.k8s.io/v1
@@ -330,7 +357,7 @@ WEAVE_ENV_VARS="${WEAVE_ENV_VARS}\\n                - name: NO_MASQ_LOCAL\\n
 $SSH $HOST1 "$KUBECTL delete ds weave-net -n=kube-system"
 sed -e "s%imagePullPolicy: Always%imagePullPolicy: Never%" \
     -e "s%env:%$WEAVE_ENV_VARS%" \
-    "$(dirname "$0")/../prog/weave-kube/weave-daemonset-k8s-1.8.yaml" | run_on "$HOST1" "$KUBECTL apply -n kube-system -f -"
+    "$(dirname "$0")/../prog/weave-kube/weave-daemonset-k8s-1.9.yaml" | run_on "$HOST1" "$KUBECTL apply -n kube-system -f -"
 
 sleep 5