Merge pull request #1861 from weaveworks/1657-1656-vendor-and-build-in-container

Build in a container by default, using go 1.5 and vendored dependencies.

.gitignore

@@ -25,6 +25,7 @@ _testmain.go
 *.prof
 profile.cov
 coverage.html
+.pkg
 
 # Emacs backup files
 *~

.gitmodules

@@ -1,3 +1,51 @@
 [submodule "tools"]
 	path = tools
 	url = https://github.com/weaveworks/build-tools
+[submodule "vendor/github.com/vishvananda/netlink"]
+	path = vendor/github.com/vishvananda/netlink
+	url = https://github.com/vishvananda/netlink.git
+[submodule "vendor/github.com/davecheney/profile"]
+	path = vendor/github.com/davecheney/profile
+	url = https://github.com/davecheney/profile.git
+[submodule "vendor/github.com/gorilla/mux"]
+	path = vendor/github.com/gorilla/mux
+	url = https://github.com/gorilla/mux.git
+[submodule "vendor/github.com/gorilla/context"]
+	path = vendor/github.com/gorilla/context
+	url = https://github.com/gorilla/context.git
+[submodule "vendor/github.com/google/gopacket"]
+	path = vendor/github.com/google/gopacket
+	url = https://github.com/google/gopacket.git
+[submodule "vendor/github.com/weaveworks/go-odp"]
+	path = vendor/github.com/weaveworks/go-odp
+	url = https://github.com/weaveworks/go-odp.git
+[submodule "vendor/github.com/opencontainers/runc"]
+	path = vendor/github.com/opencontainers/runc
+	url = https://github.com/opencontainers/runc.git
+[submodule "vendor/github.com/fsouza/go-dockerclient"]
+	path = vendor/github.com/fsouza/go-dockerclient
+	url = https://github.com/fsouza/go-dockerclient.git
+[submodule "vendor/github.com/docker/libnetwork"]
+	path = vendor/github.com/docker/libnetwork
+	url = https://github.com/docker/libnetwork.git
+[submodule "vendor/github.com/docker/docker"]
+	path = vendor/github.com/docker/docker
+	url = https://github.com/docker/docker.git
+[submodule "vendor/github.com/Sirupsen/logrus"]
+	path = vendor/github.com/Sirupsen/logrus
+	url = https://github.com/Sirupsen/logrus.git
+[submodule "vendor/github.com/andybalholm/go-bit"]
+	path = vendor/github.com/andybalholm/go-bit
+	url = https://github.com/andybalholm/go-bit.git
+[submodule "vendor/github.com/miekg/dns"]
+	path = vendor/github.com/miekg/dns
+	url = https://github.com/miekg/dns.git
+[submodule "vendor/github.com/stretchr/testify"]
+	path = vendor/github.com/stretchr/testify
+	url = https://github.com/stretchr/testify.git
+[submodule "vendor/golang.org/x/crypto"]
+	path = vendor/golang.org/x/crypto
+	url = https://github.com/golang/crypto.git
+[submodule "vendor/github.com/docker/machine"]
+	path = vendor/github.com/docker/machine
+	url = https://github.com/docker/machine.git

Makefile

@@ -4,7 +4,11 @@ PUBLISH=publish_weave publish_weaveexec publish_plugin
 .PHONY: all exes update tests lint publish $(PUBLISH) clean clean-bin prerequisites build run-smoketests
 
 # If you can use docker without being root, you can do "make SUDO="
-SUDO=sudo
+SUDO=sudo -E
+BUILD_IN_CONTAINER=true
+RM=--rm
+RUN_FLAGS=-ti
+COVERAGE=
 
 DOCKERHUB_USER=weaveworks
 WEAVE_VERSION=git-$(shell git rev-parse --short=12 HEAD)
@@ -22,15 +26,17 @@ TEST_TLS_EXE=test/tls/tls
 
 EXES=$(WEAVER_EXE) $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(WEAVEWAIT_NOMCAST_EXE) $(WEAVEUTIL_EXE) $(DOCKERPLUGIN_EXE) $(TEST_TLS_EXE)
 
+BUILD_UPTODATE=.build.uptodate
 WEAVER_UPTODATE=.weaver.uptodate
 WEAVEEXEC_UPTODATE=.weaveexec.uptodate
 DOCKERPLUGIN_UPTODATE=.dockerplugin.uptodate
 
-IMAGES_UPTODATE=$(WEAVER_UPTODATE) $(WEAVEEXEC_UPTODATE) $(DOCKERPLUGIN_UPTODATE)
+IMAGES_UPTODATE=$(WEAVER_UPTODATE) $(WEAVEEXEC_UPTODATE) $(DOCKERPLUGIN_UPTODATE) $(BUILD_UPTODATE)
 
 WEAVER_IMAGE=$(DOCKERHUB_USER)/weave
 WEAVEEXEC_IMAGE=$(DOCKERHUB_USER)/weaveexec
 DOCKERPLUGIN_IMAGE=$(DOCKERHUB_USER)/plugin
+BUILD_IMAGE=$(DOCKERHUB_USER)/weavebuild
 
 IMAGES=$(WEAVER_IMAGE) $(WEAVEEXEC_IMAGE) $(DOCKERPLUGIN_IMAGE)
 
@@ -47,17 +53,15 @@ NETGO_CHECK=@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
 	echo "    sudo go install -tags netgo std"; \
 	false; \
 }
-BUILD_FLAGS=-ldflags "-extldflags \"-static\" -X main.version $(WEAVE_VERSION)" -tags netgo
+BUILD_FLAGS=-i -ldflags "-extldflags \"-static\" -X main.version=$(WEAVE_VERSION)" -tags netgo
 
 PACKAGE_BASE=$(shell go list -e ./)
 
 all: $(WEAVE_EXPORT) $(RUNNER_EXE) $(TEST_TLS_EXE)
-
 exes: $(EXES)
 
-update:
-	go get -u -f -v -tags netgo $(addprefix ./,$(dir $(EXES)))
 
+$(EXES): $(BUILD_UPTODATE)
 $(WEAVER_EXE) $(WEAVEPROXY_EXE) $(WEAVEUTIL_EXE): common/*.go common/*/*.go net/*.go net/*/*.go
 $(WEAVER_EXE): router/*.go mesh/*.go ipam/*.go ipam/*/*.go nameserver/*.go prog/weaver/*.go
 $(WEAVEPROXY_EXE): proxy/*.go prog/weaveproxy/*.go
@@ -68,39 +72,60 @@ $(TEST_TLS_EXE): test/tls/*.go
 $(WEAVEWAIT_NOOP_EXE): prog/weavewait/*.go
 $(WEAVEWAIT_EXE): prog/weavewait/*.go net/*.go
 $(WEAVEWAIT_NOMCAST_EXE): prog/weavewait/*.go net/*.go
+tests: $(BUILD_UPTODATE) tools/.git
+lint: $(BUILD_UPTODATE) tools/.git
+
+ifeq ($(BUILD_IN_CONTAINER),true)
+
+$(EXES) tests lint:
+	git submodule update
+	@mkdir -p $(shell pwd)/.pkg
+	$(SUDO) docker run $(RM) $(RUN_FLAGS) \
+	    -v $(shell pwd):/go/src/github.com/weaveworks/weave \
+		-v $(shell pwd)/.pkg:/go/pkg \
+		-e GOARCH -e GOOS -e CIRCLECI -e CIRCLE_BUILD_NUM -e CIRCLE_NODE_TOTAL -e CIRCLE_NODE_INDEX -e COVERDIR -e SLOW \
+		$(BUILD_IMAGE) COVERAGE=$(COVERAGE) WEAVE_VERSION=$(WEAVE_VERSION) $@
+
+else
 
 $(WEAVER_EXE) $(WEAVEPROXY_EXE):
 ifeq ($(COVERAGE),true)
 	$(eval COVERAGE_MODULES := $(shell (go list ./$(@D); go list -f '{{join .Deps "\n"}}' ./$(@D) | grep "^$(PACKAGE_BASE)/") | paste -s -d,))
-	go get -t -tags netgo ./$(@D)
 	go test -c -o ./$@ $(BUILD_FLAGS) -v -covermode=atomic -coverpkg $(COVERAGE_MODULES) ./$(@D)/
 else
-	go get -tags netgo ./$(@D)
 	go build $(BUILD_FLAGS) -o $@ ./$(@D)
 endif
 	$(NETGO_CHECK)
 
 $(WEAVEUTIL_EXE):
-	go get -tags netgo ./$(@D)
 	go build $(BUILD_FLAGS) -o $@ ./$(@D)
 	$(NETGO_CHECK)
 
 $(WEAVEWAIT_EXE):
-	go get -tags netgo ./$(@D)
 	go build $(BUILD_FLAGS) -tags "netgo iface mcast" -o $@ ./$(@D)
 	$(NETGO_CHECK)
 
 $(WEAVEWAIT_NOMCAST_EXE):
-	go get -tags netgo ./$(@D)
 	go build $(BUILD_FLAGS) -tags "netgo iface" -o $@ ./$(@D)
 	$(NETGO_CHECK)
 
 # These programs need a separate rule as they fail the netgo check in
 # the main build stanza due to not importing net package
 $(SIGPROXY_EXE) $(DOCKERPLUGIN_EXE) $(TEST_TLS_EXE) $(WEAVEWAIT_NOOP_EXE):
-	go get -tags netgo ./$(@D)
 	go build $(BUILD_FLAGS) -o $@ ./$(@D)
 
+tests:
+	./tools/test -no-go-get
+
+lint:
+	./tools/lint -nocomment -notestpackage .
+
+endif
+
+$(BUILD_UPTODATE): build/*
+	$(SUDO) docker build -t $(BUILD_IMAGE) build/
+	touch $@
+
 $(WEAVER_UPTODATE): prog/weaver/Dockerfile $(WEAVER_EXE)
 	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker build -t $(WEAVER_IMAGE) prog/weaver
 	touch $@
@@ -127,12 +152,6 @@ $(WEAVE_EXPORT): $(IMAGES_UPTODATE)
 $(DOCKER_DISTRIB):
 	curl -o $(DOCKER_DISTRIB) $(DOCKER_DISTRIB_URL)
 
-tests: tools/.git
-	tools/test
-
-lint: tools/.git
-	tools/lint -nocomment -notestpackage .
-
 tools/.git:
 	git submodule update --init
 
@@ -149,9 +168,8 @@ endif
 publish: $(PUBLISH)
 
 clean-bin:
-	-$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker rmi $(IMAGES)
-	go clean -r $(addprefix ./,$(dir $(EXES)))
-	rm -f $(EXES) $(IMAGES_UPTODATE) $(WEAVE_EXPORT)
+	-$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker rmi $(IMAGES) $(BUILD_IMAGE)
+	rm -rf $(EXES) $(IMAGES_UPTODATE) $(WEAVE_EXPORT) .pkg
 
 clean: clean-bin
 	rm -rf test/tls/*.pem test/coverage.* test/coverage

build/Dockerfile

@@ -1,24 +1,8 @@
-FROM ubuntu
-
-RUN apt-get -y update && apt-get -y install --no-install-recommends ca-certificates apt-transport-https
-RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9
-RUN echo deb https://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list
-RUN apt-get -y update && apt-get -y install --no-install-recommends build-essential git lxc-docker-1.6.2 mercurial libpcap-dev curl make pkg-config gcc bison flex python-requests
-
-# When doing a build in a container, "apt-get update" happens twice,
-# which can be a very significant overhead for incremental builds.
-# And it's unnecessary, because if any of the prerequisites do change,
-# the right thing to do is to rebuild the container image.  Rather
-# than suppressing the "apt-get update" commands explicitly, we clear
-# out sources.list so that 'apt-get update' doesn't do anything.
-RUN echo >/etc/apt/sources.list
-
-ENV GO_VERSION 1.4.2
-RUN curl -sSL https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz | tar -C /usr/local -xz
-ENV PATH /usr/local/go/bin:$PATH
-
-# So that sudo does not trample on PATH
-RUN sed -ie '/secure_path/d' /etc/sudoers
-
+FROM golang:1.5.2
+ENV GO15VENDOREXPERIMENT 1
+RUN apt-get update && apt-get install -y libpcap-dev python-requests time
+RUN go get github.com/golang/lint/golint
+RUN go clean -i net && go install -tags netgo std
+RUN go install -race -tags netgo std
 COPY build.sh /
 ENTRYPOINT ["sh", "/build.sh"]

build/build.sh

@@ -1,45 +1,28 @@
 #!/bin/sh
-set -e
 
-GOPATH=/home/go
-export GOPATH
+set -eu
 
 WEAVE_SRC=$GOPATH/src/github.com/weaveworks/weave
 
-if [ $# -eq 0 -o "$1" = "tests" ] ; then
-    # No arguments.  Expect that the weave repo will be bind-mounted
-    # into $GOPATH
-    if ! [ -e $WEAVE_SRC ] ; then
-        cat 2>&1 <<EOF
-No container arguments supplied, and nothing at ${WEAVE_SRC}.  Please
-either bind-mount the golang workspace containing weave with the
+if ! [ -e $WEAVE_SRC ] ; then
+    cat 2>&1 <<EOF
+Nothing at ${WEAVE_SRC}.  Please bind-mount the weave repo with the
 docker run -v option, e.g.:
 
-    $ docker run -v <host gopath>:${GOPATH} \\
-          -v /var/run/docker.sock:/var/run/docker.sock weaveworks/weave-build
-
-Or supply git clone arguments to retrieve it, e.g.:
-
-    $ docker run -v /var/run/docker.sock:/var/run/docker.sock \\
-          weaveworks/weave-build https://github.com/weaveworks/weave.git
+    $ cd weave
+	$ docker run -v $(pwd):/go/src/github.com/weaveworks/weave \\
+		weaveworks/weave-build
 EOF
-        exit 1
-    fi
-
-    # If we run make directly, any files created on the bind mount
-    # will have awkward ownership.  So we switch to a user with the
-    # same user and group IDs as source directory.  We have to set a
-    # few things up so that sudo works without complaining later on.
-    uid=$(stat --format="%u" $WEAVE_SRC)
-    gid=$(stat --format="%g" $WEAVE_SRC)
-    echo "weave:x:$uid:$gid::$WEAVE_SRC:/bin/sh" >>/etc/passwd
-    echo "weave:*:::::::" >>/etc/shadow
-    echo "weave	ALL=(ALL)	NOPASSWD: ALL" >>/etc/sudoers
-
-    su weave -c "PATH=$PATH make -C $WEAVE_SRC build"
-else
-    # There are arguments to pass to git-clone
-    mkdir -p ${WEAVE_SRC%/*}
-    git clone "$@" $WEAVE_SRC
-    make -C $WEAVE_SRC build
 fi
+
+# If we run make directly, any files created on the bind mount
+# will have awkward ownership.  So we switch to a user with the
+# same user and group IDs as source directory.  We have to set a
+# few things up so that sudo works without complaining later on.
+uid=$(stat --format="%u" $WEAVE_SRC)
+gid=$(stat --format="%g" $WEAVE_SRC)
+echo "weave:x:$uid:$gid::$WEAVE_SRC:/bin/sh" >>/etc/passwd
+echo "weave:*:::::::" >>/etc/shadow
+echo "weave	ALL=(ALL)	NOPASSWD: ALL" >>/etc/sudoers
+
+su weave -c "PATH=$PATH make -C $WEAVE_SRC BUILD_IN_CONTAINER=false $*"

circle.yml

@@ -12,35 +12,25 @@ machine:
     SRCDIR: /home/ubuntu/src/github.com/weaveworks/weave
     PATH: $PATH:$HOME/.local/bin:$HOME/google-cloud-sdk/bin/
     CLOUDSDK_CORE_DISABLE_PROMPTS: 1
-    WEAVE_TEST: $HOME/docker/weave-test.tar
 
 dependencies:
-  cache_directories:
-    - "~/docker"
-  override:
-    - if [[ -e "$WEAVE_TEST" ]]; then
-        docker load -i $WEAVE_TEST;
-      else
-        docker build -t weaveworks/weave-build build;
-        docker build -t weaveworks/weave-test test/image;
-        mkdir -p $(dirname "$WEAVE_TEST");
-        docker save weaveworks/weave-test >$WEAVE_TEST;
-      fi
   post:
-    - sudo apt-get update && sudo apt-get install bc jq && pip install requests
+    - "sudo apt-get update && sudo apt-get install bc jq"
     - curl https://sdk.cloud.google.com | bash
     - bin/setup-circleci-secrets "$SECRET_PASSWORD"
-    - mkdir -p $(dirname $SRCDIR) && cp -r $(pwd)/ $SRCDIR
+    - "mkdir -p $(dirname $SRCDIR) && cp -r $(pwd)/ $SRCDIR"
+    - "cd $SRCDIR; git submodule update --init"
     - cd $SRCDIR/test; ./gce.sh make_template:
         parallel: false
+    - "cd $SRCDIR/build; ../tools/rebuild-image weaveworks/weavebuild . Dockerfile build.sh && touch $SRCDIR/.build.uptodate"
 
 test:
   override:
     - cd $SRCDIR; make lint:
         parallel: true
-    - docker run -v /home/ubuntu:/home/go -e COVERDIR=test/coverage -e SLOW=true -e CIRCLECI -e CIRCLE_BUILD_NUM -e CIRCLE_NODE_TOTAL -e CIRCLE_NODE_INDEX weaveworks/weave-test:
+    - cd $SRCDIR; COVERDIR=test/coverage make RM= tests:
         parallel: true
-    - docker run -v /var/run/docker.sock:/run/docker.sock -v /home/ubuntu:/home/go -e COVERAGE=true weaveworks/weave-build:
+    - cd $SRCDIR; make COVERAGE=true RM=:
         parallel: true
     - cd $SRCDIR/test; ./gce.sh setup && eval $(./gce.sh hosts) && ./setup.sh:
         parallel: true
@@ -49,7 +39,7 @@ test:
         timeout: 300
     - cd $SRCDIR; make clean-bin:
         parallel: true
-    - docker run -v /var/run/docker.sock:/run/docker.sock -v /home/ubuntu:/home/go weaveworks/weave-build:
+    - cd $SRCDIR; make RM=:
         parallel: true
   post:
     - cd $SRCDIR/test; ./gce.sh destroy:

site/building.md

@@ -12,20 +12,14 @@ Apart from the `weave` shell script, weave is delivered as a set of
 container images.  There is no distribution-specific packaging, so in
 principle it shouldn't matter which Linux distribution you build
 under.  But naturally, Docker is a prerequisite (version 1.6.0 or
-later).  And it is difficult to build under Fedora because [Fedora
-does not include static
-libraries](http://fedoraproject.org/wiki/Packaging:Guidelines#Packaging_Static_Libraries).
-So we recommend building under Ubuntu.
+later).
 
-You can also build in a container under any system that supports
-Docker.  And you can run Ubuntu in a VM and build there.  These
-options are described below.
+The only way to build is by using the the build container; the
+Makefile is setup to make this transparent.  This method is
+documented below, and can be run directly on your machine or using
+a Vagrant VM.
 
- * [Building directly on Ubuntu](#ubuntu)
- * [Building in a Docker container](#docker)
- * [Building using Vagrant](#vagrant)
-
-## <a name="ubuntu"></a>Building directly on Ubuntu
+## <a name="ubuntu"></a>Building directly on your machine
 
 The weave git repository should be cloned into
 `$GOPATH/src/github.com/weaveworks/weave`, in accordance with [the go
@@ -37,18 +31,8 @@ $ git clone https://$WEAVE $GOPATH/src/$WEAVE
 $ cd $GOPATH/src/$WEAVE
 ```
 
-Some prerequisites are needed to build weave:
-
-* Install Docker if you haven't already, by following the instructions
-  [on the Docker
-  site](https://docs.docker.com/installation/ubuntulinux/).
-* Building weave requires Go 1.4.2, and the `golang` package in Ubuntu
-  14.04 LTS is too old.  So you may need to [install the tarball from
-  golang.org](http://golang.org/doc/install).
-* A few other packages are also needed:
-```bash
-$ sudo apt-get install build-essential git mercurial libpcap-dev
-```
+Next install Docker if you haven't already, by following the instructions
+[on the Docker site](https://docs.docker.com/installation/ubuntulinux/).
 
 Then to actually build, simply do:
 
@@ -56,59 +40,16 @@ Then to actually build, simply do:
 $ make
 ```
 
-This will build the weave components and package them into three
-Docker images (`weaveworks/weave`, `weaveworks/weaveexec`, and
-`weaveworks/plugin`).  These are then exported as `weave.tar.gz`.
+On a fresh repository, the Makefile will do the following:
+- assemble the build container
+- download specific versions of all the dependencies
+- build the weave components in the build container
+- package them into three Docker images (`weaveworks/weave`,
+`weaveworks/weaveexec`, and `weaveworks/plugin`)
+- Exported these images as `weave.tar.gz`
 
-## <a name="docker"></a>Building in a Docker container
-
-As a preliminary step, we create a container image based on Ubuntu
-that has all the prerequisites.  This avoids the need to download and
-install them for each build.  In the `weave` directory, do:
-
-```bash
-$ sudo docker build -t weaveworks/weave-build build
-```
-
-Next we run a container based on that image. That container requires
-access to the Docker daemon on the host, via
-`/var/run/docker.sock`. If you are building under a Fedora or RHEL
-Docker host (or another distribution that uses SELinux), and you have
-SELinux set to enforcing mode, it will block attempts to access
-`/var/run/docker.sock` inside the container.  See
-[dpw/selinux-dockersock](https://github.com/dpw/selinux-dockersock)
-for a way to work around this problem.
-
-To perform a build, run:
-
-```bash
-$ sudo docker run -v /var/run/docker.sock:/var/run/docker.sock weaveworks/weave-build https://github.com/weaveworks/weave.git
-```
-
-This will clone the weave git repository, then do the build.
-
-When the build completes, the resulting images are stored in docker on
-the host, as when building directly under Ubuntu.
-
-The container arguments are passed to `git clone`, so for example, you
-can build from a forked repository and a specific branch with:
-
-```bash
-$ sudo docker run -v /var/run/docker.sock:/var/run/docker.sock weaveworks/weave-build -b <branch name> <repo URI>
-```
-
-Alternatively, you might want to build from a weave source tree
-already present on the host.  You can do this by using the `-v` option
-to bind the bind your go workspace containing the weave repository to
-`/home/go` inside the container.  No container arguments should be
-passed in this case:
-
-```bash
-$ sudo docker run -v /var/run/docker.sock:/var/run/docker.sock -v <host gopath>:/home/go weaveworks/weave-build
-```
-
-This will leave the intermediate build artifacts on the host, so that
-you can modify the weave source code and rebuild quickly.
+The first two steps may take a while - don't worry, they are
+are cached and should not need to be redone very often.
 
 ## <a name="vagrant"></a>Building using Vagrant
 

test/image/Dockerfile

@@ -1,3 +0,0 @@
-FROM weaveworks/weave-build
-COPY test.sh /
-ENTRYPOINT ["sh", "/test.sh"]

test/image/test.sh

@@ -1,20 +0,0 @@
-#!/bin/sh
-set -e
-
-GOPATH=/home/go
-export GOPATH
-
-WEAVE_SRC=$GOPATH/src/github.com/weaveworks/weave
-
-# If we run make directly, any files created on the bind mount
-# will have awkward ownership.  So we switch to a user with the
-# same user and group IDs as source directory.  We have to set a
-# few things up so that sudo works without complaining later on.
-uid=$(stat --format="%u" $WEAVE_SRC)
-gid=$(stat --format="%g" $WEAVE_SRC)
-echo "weave:x:$uid:$gid::$WEAVE_SRC:/bin/sh" >>/etc/passwd
-echo "weave:*:::::::" >>/etc/shadow
-echo "weave	ALL=(ALL)	NOPASSWD: ALL" >>/etc/sudoers
-
-su weave -c "PATH=$PATH make -C $WEAVE_SRC tests"
-