Merge pull request #309 from rade/101_eliminate_dependencies

eliminate dependency on ethtool and conntrack on host

Closes #101.

.gitignore

@@ -29,4 +29,5 @@ _testmain.go
 # Project specific
 weaver/weaver
 weavedns/weavedns
+tools/bin
 releases

Makefile

@@ -7,12 +7,15 @@ SUDO=sudo
 WEAVE_VERSION=git-$(shell git rev-parse --short=12 HEAD)
 WEAVER_EXE=weaver/weaver
 WEAVEDNS_EXE=weavedns/weavedns
+WEAVETOOLS_EXES=tools/bin
 WEAVER_IMAGE=zettio/weave
 WEAVEDNS_IMAGE=zettio/weavedns
+WEAVETOOLS_IMAGE=zettio/weavetools
 WEAVER_EXPORT=/var/tmp/weave.tar
 WEAVEDNS_EXPORT=/var/tmp/weavedns.tar
+WEAVETOOLS_EXPORT=/var/tmp/weavetools.tar
 
-all: $(WEAVER_EXPORT) $(WEAVEDNS_EXPORT)
+all: $(WEAVER_EXPORT) $(WEAVEDNS_EXPORT) $(WEAVETOOLS_EXPORT)
 
 update:
 	go get -u -f -v -tags -netgo ./$(dir $(WEAVER_EXE)) ./$(dir $(WEAVEDNS_EXE))
@@ -32,6 +35,9 @@ $(WEAVER_EXE) $(WEAVEDNS_EXE):
 $(WEAVER_EXE): router/*.go weaver/main.go
 $(WEAVEDNS_EXE): nameserver/*.go weavedns/main.go
 
+$(WEAVETOOLS_EXES): tools/build.sh
+	$(SUDO) docker run --rm -v $(realpath $(<D)):/home/weave ubuntu sh /home/weave/build.sh
+
 $(WEAVER_EXPORT): weaver/Dockerfile $(WEAVER_EXE)
 	$(SUDO) docker build -t $(WEAVER_IMAGE) weaver
 	$(SUDO) docker save $(WEAVER_IMAGE):latest > $@
@@ -40,10 +46,15 @@ $(WEAVEDNS_EXPORT): weavedns/Dockerfile $(WEAVEDNS_EXE)
 	$(SUDO) docker build -t $(WEAVEDNS_IMAGE) weavedns
 	$(SUDO) docker save $(WEAVEDNS_IMAGE):latest > $@
 
+$(WEAVETOOLS_EXPORT): tools/Dockerfile $(WEAVETOOLS_EXES)
+	$(SUDO) docker build -t $(WEAVETOOLS_IMAGE) tools
+	$(SUDO) docker save $(WEAVETOOLS_IMAGE):latest > $@
+
 # Add more directories in here as more tests are created
 tests:
 	cd nameserver; go test -tags netgo
 
 clean:
-	-$(SUDO) docker rmi $(WEAVER_IMAGE) $(WEAVEDNS_IMAGE)
-	rm -f $(WEAVER_EXE) $(WEAVEDNS_EXE) $(WEAVER_EXPORT) $(WEAVEDNS_EXPORT)
+	-$(SUDO) docker rmi $(WEAVER_IMAGE) $(WEAVEDNS_IMAGE) $(WEAVETOOLS_IMAGE)
+	rm -f $(WEAVER_EXE) $(WEAVEDNS_EXE) $(WEAVER_EXPORT) $(WEAVEDNS_EXPORT) $(WEAVETOOLS_EXPORT)
+	$(SUDO) rm -rf $(WEAVETOOLS_EXES)

README.md

@@ -27,26 +27,12 @@ capabilities, so these can continue to be used by containers.
 
 ## Installation
 
-To run weave on a host, you need to install...
+Ensure you are running Linux (kernel 3.5 or later) and have Docker
+(version 0.9.1 or later) installed. Then install weave with
 
-1. Linux and Docker. We've tested with Docker versions 0.9.1 through
-   1.3.1, but other versions should work too. Linux kernels after 3.5
-   are known to work; the newer the better.
-2. weave. Install this with
-
-        sudo wget -O /usr/local/bin/weave \
-          https://github.com/zettio/weave/releases/download/latest_release/weave
-        sudo chmod a+x /usr/local/bin/weave
-
-3. (recommended) ethtool. On many systems this is installed already;
-   if not then grab it via your favourite package manager. On some
-   systems, weave application container networking may not operate
-   correctly unless ethtool is available.
-
-4. (optional) conntrack. Install this via your favourite package
-   manager. Without conntrack, the weave network may not re-establish
-   itself fully when individual weave instances are stopped (with
-   `weave stop`) and restarted quickly (typically within ~3 minutes).
+    sudo wget -O /usr/local/bin/weave \
+      https://github.com/zettio/weave/releases/download/latest_release/weave
+    sudo chmod a+x /usr/local/bin/weave
 
 ## Quick Start Screencast
 

bin/release

@@ -121,16 +121,13 @@ publish() {
 
     echo '** Sanity checks OK for publishing tag' $LATEST_TAG as $DOCKERHUB_USER/weave:$VERSION
 
-    echo == Tagging $DOCKERHUB_USER/weave and $DOCKERHUB_USER/weavedns as $VERSION
-    docker tag $DOCKERHUB_USER/weave $DOCKERHUB_USER/weave:$VERSION
-    docker tag $DOCKERHUB_USER/weavedns $DOCKERHUB_USER/weavedns:$VERSION
-
-    docker push $DOCKERHUB_USER/weave:$VERSION
-    docker push $DOCKERHUB_USER/weave:latest
-    echo "** Pushed $DOCKERHUB_USER/weave:$VERSION to docker hub"
-    docker push $DOCKERHUB_USER/weavedns:$VERSION
-    docker push $DOCKERHUB_USER/weavedns:latest
-    echo "** Pushed $DOCKERHUB_USER/weavedns:$VERSION to docker hub"
+    for IMAGE in weave weavedns weavetools ; do
+        echo == Tagging $DOCKERHUB_USER/$IMAGE as $VERSION
+        docker tag  $DOCKERHUB_USER/$IMAGE $DOCKERHUB_USER/$IMAGE:$VERSION
+        docker push $DOCKERHUB_USER/$IMAGE:$VERSION
+        docker push $DOCKERHUB_USER/$IMAGE:latest
+        echo "** Pushed $DOCKERHUB_USER/$IMAGE:$VERSION to docker hub"
+    done
 
     echo "== Creating GitHub release $RELEASE_NAME $VERSION"
     # This cannot be done as a draft because of a bug in

tools/Dockerfile

@@ -0,0 +1,3 @@
+FROM scratch
+MAINTAINER Weaveworks Inc <help@weave.works>
+ADD ./bin /bin

tools/build.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+set -e
+
+apt-get -y update
+apt-get -y install curl make pkg-config gcc bison flex
+
+BASEDIR=$(dirname $0)
+mkdir -p "$BASEDIR"/bin
+
+# ethtool
+
+ETHTOOL=ethtool-3.16
+
+rm -rf $ETHTOOL
+
+curl -s -S https://www.kernel.org/pub/software/network/ethtool/$ETHTOOL.tar.gz | tar xvz
+(cd $ETHTOOL; ./configure LDFLAGS=-static; make)
+cp $ETHTOOL/ethtool "$BASEDIR"/bin/
+
+# conntrack
+
+PACKAGES="libmnl-1.0.3 libnfnetlink-1.0.1 libnetfilter_cttimeout-1.0.0 libnetfilter_cthelper-1.0.0 libnetfilter_queue-1.0.2 libnetfilter_conntrack-1.0.4"
+CONNTRACK=conntrack-tools-1.4.2
+
+rm -rf $PACKAGES $CONNTRACK
+
+fetch() {
+    curl -s -S http://www.netfilter.org/projects/${1%-*}/files/$1.tar.bz2 | tar xj
+}
+
+for PACKAGE in $PACKAGES; do
+    fetch $PACKAGE
+    (cd $PACKAGE; ./configure && make LDFLAGS=-static install)
+done
+
+fetch $CONNTRACK
+(cd $CONNTRACK; ./configure && make LDFLAGS=-static && rm -f src/conntrack && make LDFLAGS=-all-static)
+cp $CONNTRACK/src/conntrack "$BASEDIR"/bin/
+
+touch "$BASEDIR"/bin

weave

@@ -43,8 +43,10 @@ IMAGE_VERSION=${VERSION:-$IMAGE_VERSION}
 
 BASE_IMAGE=zettio/weave
 BASE_DNS_IMAGE=zettio/weavedns
+BASE_TOOLS_IMAGE=zettio/weavetools
 IMAGE=$BASE_IMAGE:$IMAGE_VERSION
 DNS_IMAGE=$BASE_DNS_IMAGE:$IMAGE_VERSION
+TOOLS_IMAGE=$BASE_TOOLS_IMAGE:$IMAGE_VERSION
 CONTAINER_NAME=weave
 DNS_CONTAINER_NAME=weavedns
 BRIDGE=weave
@@ -65,11 +67,11 @@ command_exists () {
     command -v $1 >/dev/null 2>&1
 }
 
-command_exists_or_warn () {
-    if ! command_exists $1 ; then
-        echo "Cannot find $1; please install it. Continuing without it." >&2
-        return 1
-    fi
+run_tool() {
+    TOOL_NET=$1
+    TOOL_COMMAND=$2
+    shift 2
+    docker run --rm --privileged --net=$TOOL_NET $TOOLS_IMAGE /bin/$TOOL_COMMAND "$@"
 }
 
 is_cidr() {
@@ -129,7 +131,7 @@ create_bridge() {
         ip link set dev v${CONTAINER_IFNAME}du master $BRIDGE
         ip link del dev v${CONTAINER_IFNAME}du
         # disable offloading
-        if command_exists_or_warn ethtool; then ethtool -K $BRIDGE tx off >/dev/null; fi
+        run_tool host ethtool -K $BRIDGE tx off >/dev/null
         # Work around the situation where there are no rules allowing traffic
         # across our bridge. E.g. ufw
         add_iptables_rule filter FORWARD -i $BRIDGE -o $BRIDGE -j ACCEPT
@@ -215,7 +217,7 @@ connect_container_to_bridge() {
         return 1
     fi
 
-    if { if command_exists_or_warn ethtool; then ethtool -K $GUEST_IFNAME tx off >/dev/null; fi } &&
+    if run_tool host ethtool -K $GUEST_IFNAME tx off >/dev/null &&
         ip link set $LOCAL_IFNAME master $BRIDGE &&
         ip link set $LOCAL_IFNAME up &&
         ip link set $GUEST_IFNAME netns $NETNS ; then
@@ -234,7 +236,7 @@ launch() {
         return 1
     fi
     connect_container_to_bridge &&
-        { if command_exists_or_warn ethtool ; then ip netns exec $NETNS ethtool -K eth0 tx off >/dev/null; fi } &&
+        run_tool container:$CONTAINER ethtool -K eth0 tx off >/dev/null &&
         ip netns exec $NETNS ip link set $CONTAINER_IFNAME up
 }
 
@@ -515,6 +517,16 @@ case "$COMMAND" in
         echo weave script $SCRIPT_VERSION
         ask_version $CONTAINER_NAME $IMAGE
         ask_version $DNS_CONTAINER_NAME $DNS_IMAGE
+        if ! TOOLS_IMAGE_ID=$(docker inspect --format='{{ .Id }}' $TOOLS_IMAGE 2>/dev/null) ; then
+            echo "Unable to find $TOOLS_IMAGE image." >&2
+        else
+            TOOLS_VERSION=$(docker images --no-trunc | grep $TOOLS_IMAGE_ID | grep -v latest | tr -s ' ' | cut -d ' ' -f 2 | tr "\n" ' ')
+            if [ -n "$TOOLS_VERSION" ] ; then
+                echo "weave tools $TOOLS_VERSION"
+            else
+                echo "weave tools (unreleased version)"
+            fi
+        fi
         ;;
     run)
         [ $# -gt 0 ] || usage
@@ -598,9 +610,7 @@ case "$COMMAND" in
             echo "Weave is not running." >&2
         fi
         docker rm -f $CONTAINER_NAME >/dev/null 2>&1 || true
-        if command_exists_or_warn conntrack; then
-            conntrack -D -p udp --dport $PORT >/dev/null 2>&1 || true
-        fi
+        run_tool host conntrack -D -p udp --dport $PORT >/dev/null 2>&1 || true
         ;;
     stop-dns)
         [ $# -eq 0 ] || usage
@@ -615,9 +625,7 @@ case "$COMMAND" in
         docker kill  $DNS_CONTAINER_NAME >/dev/null 2>&1 || true
         docker rm -f $CONTAINER_NAME     >/dev/null 2>&1 || true
         docker rm -f $DNS_CONTAINER_NAME >/dev/null 2>&1 || true
-        if command_exists_or_warn conntrack; then
-            conntrack -D -p udp --dport $PORT >/dev/null 2>&1 || true
-        fi
+        run_tool host conntrack -D -p udp --dport $PORT >/dev/null 2>&1 || true
         destroy_bridge
         for LOCAL_IFNAME in $(ip link show | grep v${CONTAINER_IFNAME}pl | cut -d ' ' -f 2 | tr -d ':') ; do
             ip link del $LOCAL_IFNAME