From dd34099e06428496375f5d402265f94861c729f5 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Thu, 1 Jan 2015 00:54:23 +0000 Subject: [PATCH 1/7] machinery to build ethtool and conntrack --- tools/Dockerfile | 3 +++ tools/build.sh | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 tools/Dockerfile create mode 100644 tools/build.sh diff --git a/tools/Dockerfile b/tools/Dockerfile new file mode 100644 index 00000000..f244fee7 --- /dev/null +++ b/tools/Dockerfile @@ -0,0 +1,3 @@ +FROM scratch +MAINTAINER Weaveworks Inc +ADD ./bin /bin diff --git a/tools/build.sh b/tools/build.sh new file mode 100644 index 00000000..e4f66961 --- /dev/null +++ b/tools/build.sh @@ -0,0 +1,40 @@ +#!/bin/sh +set -e + +apt-get -y update +apt-get -y install curl make pkg-config gcc bison flex + +BASEDIR=$(dirname $0) +mkdir -p "$BASEDIR"/bin + +# ethtool + +ETHTOOL=ethtool-3.16 + +rm -rf $ETHTOOL + +curl -s -S https://www.kernel.org/pub/software/network/ethtool/$ETHTOOL.tar.gz | tar xvz +(cd $ETHTOOL; ./configure LDFLAGS=-static; make) +cp $ETHTOOL/ethtool "$BASEDIR"/bin/ + +# conntrack + +PACKAGES="libmnl-1.0.3 libnfnetlink-1.0.1 libnetfilter_cttimeout-1.0.0 libnetfilter_cthelper-1.0.0 libnetfilter_queue-1.0.2 libnetfilter_conntrack-1.0.4" +CONNTRACK=conntrack-tools-1.4.2 + +rm -rf $PACKAGES $CONNTRACK + +fetch() { + curl -s -S http://www.netfilter.org/projects/${1%-*}/files/$1.tar.bz2 | tar xj +} + +for PACKAGE in $PACKAGES; do + fetch $PACKAGE + (cd $PACKAGE; ./configure && make LDFLAGS=-static install) +done + +fetch $CONNTRACK +(cd $CONNTRACK; ./configure && make LDFLAGS=-static && rm -f src/conntrack && make LDFLAGS=-all-static) +cp $CONNTRACK/src/conntrack "$BASEDIR"/bin/ + +touch "$BASEDIR"/bin From 7b6ea37fea79bebd1c6ce02c087b0745fcb82131 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Thu, 1 Jan 2015 00:57:02 +0000 Subject: [PATCH 2/7] wire in weavetools image build We use a plain ubuntu container as a build environment for ethtool & conntrack, so a) we don't require onerous tooling on the build host, and b) don't pollute the build host with build artefacts. The latter arise because I can only get conntrack to build by installing its dependencies 'globally'. Th dir containing the build script is volume-mounted into the container, and that is where we also expect to see the result, namely a 'bin' sub-dir containing two executables. The weavetools image is then constructed from the 'scratch' image and contains just that dir. --- Makefile | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 10109783..9ceecace 100644 --- a/Makefile +++ b/Makefile @@ -7,12 +7,15 @@ SUDO=sudo WEAVE_VERSION=git-$(shell git rev-parse --short=12 HEAD) WEAVER_EXE=weaver/weaver WEAVEDNS_EXE=weavedns/weavedns +WEAVETOOLS_EXES=tools/bin WEAVER_IMAGE=zettio/weave WEAVEDNS_IMAGE=zettio/weavedns +WEAVETOOLS_IMAGE=zettio/weavetools WEAVER_EXPORT=/var/tmp/weave.tar WEAVEDNS_EXPORT=/var/tmp/weavedns.tar +WEAVETOOLS_EXPORT=/var/tmp/weavetools.tar -all: $(WEAVER_EXPORT) $(WEAVEDNS_EXPORT) +all: $(WEAVER_EXPORT) $(WEAVEDNS_EXPORT) $(WEAVETOOLS_EXPORT) $(WEAVER_EXE) $(WEAVEDNS_EXE): go get -tags netgo ./$(shell dirname $@) @@ -29,6 +32,9 @@ $(WEAVER_EXE) $(WEAVEDNS_EXE): $(WEAVER_EXE): router/*.go weaver/main.go $(WEAVEDNS_EXE): nameserver/*.go weavedns/main.go +$(WEAVETOOLS_EXES): tools/build.sh + $(SUDO) docker run --rm -v $(realpath $( $@ @@ -37,10 +43,15 @@ $(WEAVEDNS_EXPORT): weavedns/Dockerfile $(WEAVEDNS_EXE) $(SUDO) docker build -t $(WEAVEDNS_IMAGE) weavedns $(SUDO) docker save $(WEAVEDNS_IMAGE):latest > $@ +$(WEAVETOOLS_EXPORT): tools/Dockerfile $(WEAVETOOLS_EXES) + $(SUDO) docker build -t $(WEAVETOOLS_IMAGE) tools + $(SUDO) docker save $(WEAVETOOLS_IMAGE):latest > $@ + # Add more directories in here as more tests are created tests: cd nameserver; go test -tags netgo clean: - -$(SUDO) docker rmi $(WEAVER_IMAGE) $(WEAVEDNS_IMAGE) - rm -f $(WEAVER_EXE) $(WEAVEDNS_EXE) $(WEAVER_EXPORT) $(WEAVEDNS_EXPORT) + -$(SUDO) docker rmi $(WEAVER_IMAGE) $(WEAVEDNS_IMAGE) $(WEAVETOOLS_IMAGE) + rm -f $(WEAVER_EXE) $(WEAVEDNS_EXE) $(WEAVER_EXPORT) $(WEAVEDNS_EXPORT) $(WEAVETOOLS_EXPORT) + $(SUDO) rm -rf $(WEAVETOOLS_EXES) From eb628ec84a68eb3810c3bbe9c2a342daaad5867b Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Thu, 1 Jan 2015 01:05:27 +0000 Subject: [PATCH 3/7] invoke ethtool and conntrack from the weavetools image --- weave | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/weave b/weave index 8179f930..a0abe201 100755 --- a/weave +++ b/weave @@ -43,8 +43,10 @@ IMAGE_VERSION=${VERSION:-$IMAGE_VERSION} BASE_IMAGE=zettio/weave BASE_DNS_IMAGE=zettio/weavedns +BASE_TOOLS_IMAGE=zettio/weavetools IMAGE=$BASE_IMAGE:$IMAGE_VERSION DNS_IMAGE=$BASE_DNS_IMAGE:$IMAGE_VERSION +TOOLS_IMAGE=$BASE_TOOLS_IMAGE:$IMAGE_VERSION CONTAINER_NAME=weave DNS_CONTAINER_NAME=weavedns BRIDGE=weave @@ -65,11 +67,11 @@ command_exists () { command -v $1 >/dev/null 2>&1 } -command_exists_or_warn () { - if ! command_exists $1 ; then - echo "Cannot find $1; please install it. Continuing without it." >&2 - return 1 - fi +run_tool() { + TOOL_NET=$1 + TOOL_COMMAND=$2 + shift 2 + docker run --rm --privileged --net=$TOOL_NET $TOOLS_IMAGE /bin/$TOOL_COMMAND "$@" } is_cidr() { @@ -129,7 +131,7 @@ create_bridge() { ip link set dev v${CONTAINER_IFNAME}du master $BRIDGE ip link del dev v${CONTAINER_IFNAME}du # disable offloading - if command_exists_or_warn ethtool; then ethtool -K $BRIDGE tx off >/dev/null; fi + run_tool host ethtool -K $BRIDGE tx off >/dev/null # Work around the situation where there are no rules allowing traffic # across our bridge. E.g. ufw add_iptables_rule filter FORWARD -i $BRIDGE -o $BRIDGE -j ACCEPT @@ -215,7 +217,7 @@ connect_container_to_bridge() { return 1 fi - if { if command_exists_or_warn ethtool; then ethtool -K $GUEST_IFNAME tx off >/dev/null; fi } && + if run_tool host ethtool -K $GUEST_IFNAME tx off >/dev/null && ip link set $LOCAL_IFNAME master $BRIDGE && ip link set $LOCAL_IFNAME up && ip link set $GUEST_IFNAME netns $NETNS ; then @@ -234,7 +236,7 @@ launch() { return 1 fi connect_container_to_bridge && - { if command_exists_or_warn ethtool ; then ip netns exec $NETNS ethtool -K eth0 tx off >/dev/null; fi } && + run_tool container:$CONTAINER ethtool -K eth0 tx off >/dev/null && ip netns exec $NETNS ip link set $CONTAINER_IFNAME up } @@ -598,9 +600,7 @@ case "$COMMAND" in echo "Weave is not running." >&2 fi docker rm -f $CONTAINER_NAME >/dev/null 2>&1 || true - if command_exists_or_warn conntrack; then - conntrack -D -p udp --dport $PORT >/dev/null 2>&1 || true - fi + run_tool host conntrack -D -p udp --dport $PORT >/dev/null 2>&1 || true ;; stop-dns) [ $# -eq 0 ] || usage @@ -615,9 +615,7 @@ case "$COMMAND" in docker kill $DNS_CONTAINER_NAME >/dev/null 2>&1 || true docker rm -f $CONTAINER_NAME >/dev/null 2>&1 || true docker rm -f $DNS_CONTAINER_NAME >/dev/null 2>&1 || true - if command_exists_or_warn conntrack; then - conntrack -D -p udp --dport $PORT >/dev/null 2>&1 || true - fi + run_tool host conntrack -D -p udp --dport $PORT >/dev/null 2>&1 || true destroy_bridge for LOCAL_IFNAME in $(ip link show | grep v${CONTAINER_IFNAME}pl | cut -d ' ' -f 2 | tr -d ':') ; do ip link del $LOCAL_IFNAME From 57e86a87966d16fb168316fb9bed66a54891a49b Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Thu, 1 Jan 2015 11:50:51 +0000 Subject: [PATCH 4/7] update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9250c674..46a83f2c 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,5 @@ _testmain.go # Project specific weaver/weaver weavedns/weavedns +tools/bin releases From e33885eccaa691113f6364c4c6744e538ef0e17c Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Thu, 1 Jan 2015 11:51:15 +0000 Subject: [PATCH 5/7] wire weavetools into release --- bin/release | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/bin/release b/bin/release index 464e61f1..708de7e3 100755 --- a/bin/release +++ b/bin/release @@ -121,16 +121,13 @@ publish() { echo '** Sanity checks OK for publishing tag' $LATEST_TAG as $DOCKERHUB_USER/weave:$VERSION - echo == Tagging $DOCKERHUB_USER/weave and $DOCKERHUB_USER/weavedns as $VERSION - docker tag $DOCKERHUB_USER/weave $DOCKERHUB_USER/weave:$VERSION - docker tag $DOCKERHUB_USER/weavedns $DOCKERHUB_USER/weavedns:$VERSION - - docker push $DOCKERHUB_USER/weave:$VERSION - docker push $DOCKERHUB_USER/weave:latest - echo "** Pushed $DOCKERHUB_USER/weave:$VERSION to docker hub" - docker push $DOCKERHUB_USER/weavedns:$VERSION - docker push $DOCKERHUB_USER/weavedns:latest - echo "** Pushed $DOCKERHUB_USER/weavedns:$VERSION to docker hub" + for IMAGE in weave weavedns weavetools ; do + echo == Tagging $DOCKERHUB_USER/$IMAGE as $VERSION + docker tag $DOCKERHUB_USER/$IMAGE $DOCKERHUB_USER/$IMAGE:$VERSION + docker push $DOCKERHUB_USER/$IMAGE:$VERSION + docker push $DOCKERHUB_USER/$IMAGE:latest + echo "** Pushed $DOCKERHUB_USER/$IMAGE:$VERSION to docker hub" + done echo "== Creating GitHub release $RELEASE_NAME $VERSION" # This cannot be done as a draft because of a bug in From 1fc37bf8dfee1b2546347ea7815b67bfde33b58b Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Thu, 1 Jan 2015 12:14:11 +0000 Subject: [PATCH 6/7] update README --- README.md | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index c02e16b6..a633cee9 100644 --- a/README.md +++ b/README.md @@ -27,26 +27,12 @@ capabilities, so these can continue to be used by containers. ## Installation -To run weave on a host, you need to install... +Ensure you are running Linux (kernel 3.5 or later) and have Docker +(version 0.9.1 or later) installed. Then install weave with -1. Linux and Docker. We've tested with Docker versions 0.9.1 through - 1.3.1, but other versions should work too. Linux kernels after 3.5 - are known to work; the newer the better. -2. weave. Install this with - - sudo wget -O /usr/local/bin/weave \ - https://github.com/zettio/weave/releases/download/latest_release/weave - sudo chmod a+x /usr/local/bin/weave - -3. (recommended) ethtool. On many systems this is installed already; - if not then grab it via your favourite package manager. On some - systems, weave application container networking may not operate - correctly unless ethtool is available. - -4. (optional) conntrack. Install this via your favourite package - manager. Without conntrack, the weave network may not re-establish - itself fully when individual weave instances are stopped (with - `weave stop`) and restarted quickly (typically within ~3 minutes). + sudo wget -O /usr/local/bin/weave \ + https://github.com/zettio/weave/releases/download/latest_release/weave + sudo chmod a+x /usr/local/bin/weave ## Quick Start Screencast From 2d43b9257268ddf64e8f83f6886810c708df4d42 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Fri, 2 Jan 2015 18:29:52 +0000 Subject: [PATCH 7/7] display weavetools version --- weave | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/weave b/weave index a0abe201..8b11b9d4 100755 --- a/weave +++ b/weave @@ -517,6 +517,16 @@ case "$COMMAND" in echo weave script $SCRIPT_VERSION ask_version $CONTAINER_NAME $IMAGE ask_version $DNS_CONTAINER_NAME $DNS_IMAGE + if ! TOOLS_IMAGE_ID=$(docker inspect --format='{{ .Id }}' $TOOLS_IMAGE 2>/dev/null) ; then + echo "Unable to find $TOOLS_IMAGE image." >&2 + else + TOOLS_VERSION=$(docker images --no-trunc | grep $TOOLS_IMAGE_ID | grep -v latest | tr -s ' ' | cut -d ' ' -f 2 | tr "\n" ' ') + if [ -n "$TOOLS_VERSION" ] ; then + echo "weave tools $TOOLS_VERSION" + else + echo "weave tools (unreleased version)" + fi + fi ;; run) [ $# -gt 0 ] || usage