diff --git a/.gitignore b/.gitignore index aef563bc..5d9b497c 100644 --- a/.gitignore +++ b/.gitignore @@ -52,7 +52,7 @@ testing/cover/cover testing/runner/runner releases .*.uptodate -weave.tar.gz +weave*.tar.gz prog/weaveexec/weave prog/weaveexec/weavehosts prog/weaveexec/weaveproxy @@ -61,9 +61,10 @@ prog/weaveexec/weavewait_noop prog/weaveexec/weavewait_nomcast prog/weaveexec/sigproxy prog/weaveexec/weaveutil -prog/weaveexec/docker*.tgz +prog/weaveexec/docker* Vagrantfile.local test/tls/tls test/tls/*.pem test/coverage test/coverage.* +*qemu-*-static diff --git a/.gitmodules b/.gitmodules index c1047860..1e3b0441 100644 --- a/.gitmodules +++ b/.gitmodules @@ -121,3 +121,6 @@ [submodule "vendor/github.com/mgutz/ansi"] path = vendor/github.com/mgutz/ansi url = https://github.com/mgutz/ansi +[submodule "vendor/github.com/estesp/manifest-tool"] + path = vendor/github.com/estesp/manifest-tool + url = https://github.com/estesp/manifest-tool diff --git a/Makefile b/Makefile index 34d7132f..1c97ea6f 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,5 @@ -PUBLISH=publish_weave publish_weaveexec publish_plugin publish_weave-kube publish_weave-npc - .DEFAULT: all -.PHONY: all exes testrunner update tests lint publish $(PUBLISH) clean clean-bin prerequisites build run-smoketests +.PHONY: all exes testrunner update tests lint publish-one-arch $(PUBLISH) clean clean-bin prerequisites build run-smoketests # If you can use docker without being root, you can do "make SUDO=" SUDO=$(shell docker info >/dev/null 2>&1 || echo "sudo -E") @@ -10,9 +8,82 @@ RM=--rm RUN_FLAGS=-ti COVERAGE= -DOCKERHUB_USER=weaveworks -WEAVE_VERSION=git-$(shell git rev-parse --short=12 HEAD) +# This specifies the architecture we're building for +ARCH?=amd64 +# We're using QEMU to be able to build Docker images for other architectures +# from amd64 machines. +QEMU_VERSION=v2.7.0 + +# A list of all supported architectures here. Should be named as Go is naming platforms +# All supported architectures must have an "ifeq" block below that customizes the parameters +ALL_ARCHITECTURES=amd64 arm arm64 +ML_PLATFORMS=linux/amd64,linux/arm,linux/arm64 + +ifeq ($(ARCH),amd64) +# The architecture to use when downloading the docker binary + WEAVEEXEC_DOCKER_ARCH?=x86_64 + +# The name of the alpine baseimage to use as the base for weave images + ALPINE_BASEIMAGE?=alpine + +# The extension for the made images +# Specifying none means for example weaveworks/weave:latest + ARCH_EXT?= + +# The name of the gcc we're using for compiling C code + CC=gcc + +# Optional parameters that can be passed when linking C code into the Go binary + CGO_LDFLAGS= +endif +ifeq ($(ARCH),arm) +# The architecture to use when downloading the docker binary + WEAVEEXEC_DOCKER_ARCH?=armel + +# Using the (semi-)official alpine image + ALPINE_BASEIMAGE?=armhf/alpine:3.4 + +# arm images have the -arm suffix, for instance weaveworks/weave-arm:latest + ARCH_EXT?=-arm + +# The name of the gcc binary + CC=arm-linux-gnueabihf-gcc + +# The architecture name to use when downloading a prebuilt QEMU binary + QEMUARCH=arm + +# In the weaveworks/build image; libpcap libraries for arm are placed here +# Tell the gcc linker to search for libpcap here + CGO_LDFLAGS="-L/usr/local/lib/$(CC)" +endif +ifeq ($(ARCH),arm64) +# Use the arm 32-bit docker client variant + WEAVEEXEC_DOCKER_ARCH?=armel + +# Using the (semi-)official alpine image + ALPINE_BASEIMAGE?=aarch64/alpine:3.5 + +# arm64 images have the -arm64 suffix, for instance weaveworks/weave-arm64:latest + ARCH_EXT?=-arm64 + +# The name of the gcc binary + CC=aarch64-linux-gnu-gcc + +# The architecture name to use when downloading a prebuilt QEMU binary + QEMUARCH=aarch64 + +# In the weaveworks/build image; libpcap libraries for arm64 are placed here +# Tell the gcc linker to search for libpcap here + CGO_LDFLAGS="-L/usr/local/lib/$(CC)" +endif + +# The name of the user that this Makefile should produce image artifacts for. Can/should be overridden +DOCKERHUB_USER?=weaveworks +# The default version that's chosen when pushing the images. Can/should be overridden +WEAVE_VERSION?=git-$(shell git rev-parse --short=12 HEAD) + +# Paths to all relevant binaries that should be compiled WEAVER_EXE=prog/weaver/weaver WEAVEPROXY_EXE=prog/weaveproxy/weaveproxy SIGPROXY_EXE=prog/sigproxy/sigproxy @@ -24,35 +95,43 @@ WEAVEWAIT_NOMCAST_EXE=prog/weavewait/weavewait_nomcast WEAVEUTIL_EXE=prog/weaveutil/weaveutil PLUGIN_EXE=prog/plugin/plugin RUNNER_EXE=tools/runner/runner +MANIFEST_TOOL_DIR=vendor/github.com/estesp/manifest-tool +MANIFEST_TOOL_EXE=$(MANIFEST_TOOL_DIR)/manifest-tool TEST_TLS_EXE=test/tls/tls +# All binaries together in a list EXES=$(WEAVER_EXE) $(SIGPROXY_EXE) $(KUBEPEERS_EXE) $(WEAVENPC_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(WEAVEWAIT_NOMCAST_EXE) $(WEAVEUTIL_EXE) $(PLUGIN_EXE) $(RUNNER_EXE) $(TEST_TLS_EXE) +# These stamp files are used to mark the current state of the build; whether an image has been built or not BUILD_UPTODATE=.build.uptodate -WEAVER_UPTODATE=.weaver.uptodate -WEAVEEXEC_UPTODATE=.weaveexec.uptodate -PLUGIN_UPTODATE=.plugin.uptodate -WEAVEKUBE_UPTODATE=.weavekube.uptodate -WEAVENPC_UPTODATE=.weavenpc.uptodate +WEAVER_UPTODATE=.weaver$(ARCH_EXT).uptodate +WEAVEEXEC_UPTODATE=.weaveexec$(ARCH_EXT).uptodate +PLUGIN_UPTODATE=.plugin$(ARCH_EXT).uptodate +WEAVEKUBE_UPTODATE=.weavekube$(ARCH_EXT).uptodate +WEAVENPC_UPTODATE=.weavenpc$(ARCH_EXT).uptodate WEAVEDB_UPTODATE=.weavedb.uptodate -IMAGES_UPTODATE=$(WEAVER_UPTODATE) $(WEAVEEXEC_UPTODATE) $(PLUGIN_UPTODATE) $(WEAVEKUBE_UPTODATE) $(WEAVENPC_UPTODATE) $(WEAVEDB_UPTODATE) +IMAGES_UPTODATE=$(WEAVER_UPTODATE) $(WEAVEEXEC_UPTODATE) $(PLUGIN_UPTODATE) $(WEAVEKUBE_UPTODATE) $(WEAVENPC_UPTODATE) -WEAVER_IMAGE=$(DOCKERHUB_USER)/weave -WEAVEEXEC_IMAGE=$(DOCKERHUB_USER)/weaveexec -PLUGIN_IMAGE=$(DOCKERHUB_USER)/plugin -WEAVEKUBE_IMAGE=$(DOCKERHUB_USER)/weave-kube -WEAVENPC_IMAGE=$(DOCKERHUB_USER)/weave-npc +# The names of the images. Note that the images for other architectures than amd64 have a suffix in the image name. +WEAVER_IMAGE=$(DOCKERHUB_USER)/weave$(ARCH_EXT) +WEAVEEXEC_IMAGE=$(DOCKERHUB_USER)/weaveexec$(ARCH_EXT) +PLUGIN_IMAGE=$(DOCKERHUB_USER)/plugin$(ARCH_EXT) +WEAVEKUBE_IMAGE=$(DOCKERHUB_USER)/weave-kube$(ARCH_EXT) +WEAVENPC_IMAGE=$(DOCKERHUB_USER)/weave-npc$(ARCH_EXT) BUILD_IMAGE=weaveworks/weavebuild WEAVEDB_IMAGE=$(DOCKERHUB_USER)/weavedb IMAGES=$(WEAVER_IMAGE) $(WEAVEEXEC_IMAGE) $(PLUGIN_IMAGE) $(WEAVEKUBE_IMAGE) $(WEAVENPC_IMAGE) $(WEAVEDB_IMAGE) -WEAVE_EXPORT=weave.tar.gz +PUBLISH=publish_weave publish_weaveexec publish_plugin publish_weave-kube publish_weave-npc +PUSH_ML=push_ml_weave push_ml_weaveexec push_ml_plugin push_ml_weave-kube push_ml_weave-npc -WEAVEEXEC_DOCKER_VERSION=1.6.2 -DOCKER_DISTRIB=prog/weaveexec/docker-$(WEAVEEXEC_DOCKER_VERSION).tgz -DOCKER_DISTRIB_URL=https://get.docker.com/builds/Linux/x86_64/docker-$(WEAVEEXEC_DOCKER_VERSION).tgz +WEAVE_EXPORT=weave$(ARCH_EXT).tar.gz + +DOCKER_VERSION=1.10.3 +DOCKER_DISTRIB=prog/weaveexec/docker-$(DOCKER_VERSION).tgz +DOCKER_DISTRIB_URL=https://get.docker.com/builds/Linux/$(WEAVEEXEC_DOCKER_ARCH)/docker-$(DOCKER_VERSION).tgz NETGO_CHECK=@strings $@ | grep cgo_stub\\\.go >/dev/null || { \ rm $@; \ echo "\nYour go standard library was built without the 'netgo' build tag."; \ @@ -61,6 +140,9 @@ NETGO_CHECK=@strings $@ | grep cgo_stub\\\.go >/dev/null || { \ echo " sudo go install -tags netgo std"; \ false; \ } +# The flags we are passing to go build. -extldflags -static for making a static binary, +# -linkmode external for linking external C libraries into the binary, -X main.version for telling the +# Go binary which version it is, -tags netgo for enforcing the native Go DNS resolver BUILD_FLAGS=-i -ldflags "-linkmode external -extldflags -static -X main.version=$(WEAVE_VERSION)" -tags netgo PACKAGE_BASE=$(shell go list -e ./) @@ -86,14 +168,16 @@ lint: tools/.git ifeq ($(BUILD_IN_CONTAINER),true) +# This make target compiles all binaries inside of the weaveworks/build container +# It bind-mounts the source into the container and passes all important variables exes $(EXES) tests lint: $(BUILD_UPTODATE) git submodule update --init @mkdir -p $(shell pwd)/.pkg $(SUDO) docker run $(RM) $(RUN_FLAGS) \ -v $(shell pwd):/go/src/github.com/weaveworks/weave \ -v $(shell pwd)/.pkg:/go/pkg \ - -e GOARCH -e GOOS -e CIRCLECI -e CIRCLE_BUILD_NUM -e CIRCLE_NODE_TOTAL -e CIRCLE_NODE_INDEX -e COVERDIR -e SLOW \ - $(BUILD_IMAGE) COVERAGE=$(COVERAGE) WEAVE_VERSION=$(WEAVE_VERSION) $@ + -e GOARCH=$(ARCH) -e CGO_ENABLED=1 -e GOOS=linux -e CIRCLECI -e CIRCLE_BUILD_NUM -e CIRCLE_NODE_TOTAL -e CIRCLE_NODE_INDEX -e COVERDIR -e SLOW \ + $(BUILD_IMAGE) COVERAGE=$(COVERAGE) WEAVE_VERSION=$(WEAVE_VERSION) CC=$(CC) QEMUARCH=$(QEMUARCH) CGO_LDFLAGS=$(CGO_LDFLAGS) $@ else @@ -133,18 +217,37 @@ lint: endif +# This rule makes sure the build image is up-to-date. +# It also makes sure the multiarch hooks are reqistered in the kernel so the QEMU emulation works $(BUILD_UPTODATE): build/* $(SUDO) docker build -t $(BUILD_IMAGE) build/ + $(SUDO) docker run --rm --privileged multiarch/qemu-user-static:register --reset touch $@ +# Creates the Dockerfile.your-user-here file from the template +# Also replaces all placeholders with real values +# If the architecture is amd64, it deletes all CROSS_BUILD lines +# but otherwise, it only removes the "CROSS_BUILD_" placeholder and downloads QEMU %/Dockerfile.$(DOCKERHUB_USER): %/Dockerfile.template - sed -e "s/DOCKERHUB_USER/$(DOCKERHUB_USER)/" $^ > $@ + echo "DOCKERHUB_USER|$(DOCKERHUB_USER)|g;s|ARCH_EXT|$(ARCH_EXT)|g;s|ALPINE_BASEIMAGE|$(ALPINE_BASEIMAGE)|g;s|QEMUARCH|$(QEMUARCH)" + sed -e "s|DOCKERHUB_USER|$(DOCKERHUB_USER)|g;s|ARCH_EXT|$(ARCH_EXT)|g;s|ALPINE_BASEIMAGE|$(ALPINE_BASEIMAGE)|g;s|QEMUARCH|$(QEMUARCH)|g" $^ > $@ +ifeq ($(ARCH),amd64) + # When building "normally" for amd64, remove the whole line, it has no part in the amd64 image + sed -i "/CROSS_BUILD_/d" $@ +else + # When cross-building, only the placeholder "CROSS_BUILD_" should be removed + # Register /usr/bin/qemu-ARCH-static as the handler for ARM binaries in the kernel + curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/$(QEMU_VERSION)/x86_64_qemu-$(QEMUARCH)-static.tar.gz | tar -xz -C $(shell dirname $@) + cd $(shell dirname $@) && sha256sum -c $(shell pwd)/build/shasums/qemu-$(QEMUARCH)-static.sha256sum + sed -i "s/CROSS_BUILD_//g" $@ +endif +# The targets below builds the weave images $(WEAVER_UPTODATE): prog/weaver/Dockerfile.$(DOCKERHUB_USER) $(WEAVER_EXE) $(WEAVEEXEC_UPTODATE) $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker build -f prog/weaver/Dockerfile.$(DOCKERHUB_USER) -t $(WEAVER_IMAGE) prog/weaver touch $@ -$(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile prog/weaveexec/symlink $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(WEAVEWAIT_NOMCAST_EXE) $(WEAVEUTIL_EXE) +$(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile.$(DOCKERHUB_USER) prog/weaveexec/symlink $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(WEAVEWAIT_NOMCAST_EXE) $(WEAVEUTIL_EXE) cp weave prog/weaveexec/weave cp $(SIGPROXY_EXE) prog/weaveexec/sigproxy cp $(WEAVEPROXY_EXE) prog/weaveexec/weaveproxy @@ -152,8 +255,9 @@ $(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile prog/weaveexec/symlink $(DOCKER cp $(WEAVEWAIT_NOOP_EXE) prog/weaveexec/weavewait_noop cp $(WEAVEWAIT_NOMCAST_EXE) prog/weaveexec/weavewait_nomcast cp $(WEAVEUTIL_EXE) prog/weaveexec/weaveutil - cp $(DOCKER_DISTRIB) prog/weaveexec/docker.tgz - $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker build -t $(WEAVEEXEC_IMAGE) prog/weaveexec + tar -xf $(DOCKER_DISTRIB) usr/local/bin/docker -O > prog/weaveexec/docker + chmod +x prog/weaveexec/docker + $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker build -f prog/weaveexec/Dockerfile.$(DOCKERHUB_USER) -t $(WEAVEEXEC_IMAGE) prog/weaveexec touch $@ $(PLUGIN_UPTODATE): prog/plugin/Dockerfile.$(DOCKERHUB_USER) $(PLUGIN_EXE) $(WEAVER_UPTODATE) @@ -165,42 +269,89 @@ $(WEAVEKUBE_UPTODATE): prog/weave-kube/Dockerfile.$(DOCKERHUB_USER) prog/weave-k $(SUDO) docker build -f prog/weave-kube/Dockerfile.$(DOCKERHUB_USER) -t $(WEAVEKUBE_IMAGE) prog/weave-kube touch $@ -$(WEAVENPC_UPTODATE): prog/weave-npc/Dockerfile $(WEAVENPC_EXE) prog/weave-npc/ulogd.conf - $(SUDO) docker build -f prog/weave-npc/Dockerfile -t $(WEAVENPC_IMAGE) prog/weave-npc +$(WEAVENPC_UPTODATE): prog/weave-npc/Dockerfile.$(DOCKERHUB_USER) $(WEAVENPC_EXE) prog/weave-npc/ulogd.conf + $(SUDO) docker build -f prog/weave-npc/Dockerfile.$(DOCKERHUB_USER) -t $(WEAVENPC_IMAGE) prog/weave-npc touch $@ $(WEAVEDB_UPTODATE): prog/weavedb/Dockerfile $(SUDO) docker build -t $(WEAVEDB_IMAGE) prog/weavedb touch $@ -$(WEAVE_EXPORT): $(IMAGES_UPTODATE) +$(WEAVE_EXPORT): $(IMAGES_UPTODATE) $(WEAVEDB_UPTODATE) $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker save $(addsuffix :latest,$(IMAGES)) | gzip > $@ $(DOCKER_DISTRIB): curl -o $(DOCKER_DISTRIB) $(DOCKER_DISTRIB_URL) + cd $(shell dirname $@) && sha256sum -c $(shell pwd)/build/shasums/docker-tgz-$(WEAVEEXEC_DOCKER_ARCH).sha256sum -tools/.git: +tools/.git $(MANIFEST_TOOL_DIR)/.git: git submodule update --init +$(MANIFEST_TOOL_EXE): $(MANIFEST_TOOL_DIR)/.git + docker run -u $(shell id -u):$(shell id -g) -v $(shell pwd)/$(MANIFEST_TOOL_DIR):/go/src/github.com/estesp/manifest-tool -w /go/src/github.com/estesp/manifest-tool \ + golang:1.7 /bin/bash -c "go build -o manifest-tool github.com/estesp/manifest-tool" + +# CODE FOR PUBLISHING THE IMAGES + +# This target first runs "make publish" for each architecture +# Then it pushes the manifest lists +publish: $(MANIFEST_TOOL_EXE) $(addprefix sub-publish-,$(ALL_ARCHITECTURES)) $(WEAVEDB_UPTODATE) + $(MAKE) DOCKER_HOST=$(DOCKER_HOST) DOCKERHUB_USER=$(DOCKERHUB_USER) WEAVE_VERSION=$(WEAVE_VERSION) UPDATE_LATEST=$(UPDATE_LATEST) $(PUSH_ML) +ifeq ($(PUBLISH_WEAVEDB),true) + $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker push $(DOCKERHUB_USER)/weavedb:latest +endif + +sub-publish-%: + $(MAKE) ARCH=$* DOCKER_HOST=$(DOCKER_HOST) clean-bin + $(MAKE) ARCH=$* DOCKER_HOST=$(DOCKER_HOST) DOCKERHUB_USER=$(DOCKERHUB_USER) WEAVE_VERSION=$(WEAVE_VERSION) UPDATE_LATEST=$(UPDATE_LATEST) $(IMAGES_UPTODATE) + $(MAKE) ARCH=$* DOCKER_HOST=$(DOCKER_HOST) DOCKERHUB_USER=$(DOCKERHUB_USER) WEAVE_VERSION=$(WEAVE_VERSION) UPDATE_LATEST=$(UPDATE_LATEST) publish-one-arch + +publish-one-arch: $(PUBLISH) + +# This rule handles the pushing of the built images $(PUBLISH): publish_%: $(IMAGES_UPTODATE) - $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker tag $(DOCKERHUB_USER)/$* $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION) + # Tag :latest with the real version + $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker tag $(DOCKERHUB_USER)/$*$(ARCH_EXT) $(DOCKERHUB_USER)/$*$(ARCH_EXT):$(WEAVE_VERSION) + # Push the image with the arch suffix for arm and arm64, and without suffix for amd64 + $(MAKE) DOCKER_HOST=$(DOCKER_HOST) DOCKERHUB_USER=$(DOCKERHUB_USER) WEAVE_VERSION=$(WEAVE_VERSION) UPDATE_LATEST=$(UPDATE_LATEST) push_$*$(ARCH_EXT) + +ifeq ($(ARCH),amd64) + # If the architecture is amd64, add the -amd64 suffix. + $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker tag $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION) $(DOCKERHUB_USER)/$*-amd64:$(WEAVE_VERSION) + # If the version is latest, tag the -amd64 with latest as well +ifneq ($(UPDATE_LATEST),false) + $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker tag $(DOCKERHUB_USER)/$*-amd64:$(WEAVE_VERSION) $(DOCKERHUB_USER)/$*-amd64:latest +endif + # Push the image with the -amd64-suffix so BINARY-ARCH-named images exist for all arches, so manifest lists may be made and ARCH is replaceable in scripts + $(MAKE) DOCKER_HOST=$(DOCKER_HOST) DOCKERHUB_USER=$(DOCKERHUB_USER) WEAVE_VERSION=$(WEAVE_VERSION) UPDATE_LATEST=$(UPDATE_LATEST) push_$*-amd64 +endif + +# This target pushes an image, and if UPDATE_LATEST is anything but false, also updates the latest tag +# It takes one parameter, the image name +push_%: $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker push $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION) ifneq ($(UPDATE_LATEST),false) $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker push $(DOCKERHUB_USER)/$*:latest endif -publish: $(PUBLISH) -ifeq ($(PUBLISH_WEAVEDB),true) - $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker push $(DOCKERHUB_USER)/weavedb:latest +# This target pushes a manifest list, and if UPDATE_LATEST is anything but false, also updates the latest tag +# It takes one parameter, the image name +$(PUSH_ML): push_ml_%: + $(MANIFEST_TOOL_EXE) push from-args --platforms $(ML_PLATFORMS) --template $(DOCKERHUB_USER)/$*-ARCH:$(WEAVE_VERSION) --target $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION) +ifneq ($(UPDATE_LATEST),false) + # Push the manifest list to :latest as well + $(MANIFEST_TOOL_EXE) push from-args --platforms $(ML_PLATFORMS) --template $(DOCKERHUB_USER)/$*-ARCH:latest --target $(DOCKERHUB_USER)/$*:latest endif clean-bin: -$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker rmi $(IMAGES) - rm -rf $(EXES) $(IMAGES_UPTODATE) $(WEAVE_EXPORT) .pkg + find prog -type f -name "Dockerfile.*" -not -name "Dockerfile.template" -print | xargs rm -f + find prog -type f -name "*qemu-*" -print | xargs rm -f + rm -rf $(EXES) $(IMAGES_UPTODATE) weave*.tar.gz $(DOCKER_DISTRIB) prog/weaveexec/docker .pkg clean: clean-bin -$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker rmi $(BUILD_IMAGE) - rm -rf test/tls/*.pem test/coverage.* test/coverage $(BUILD_UPTODATE) + rm -rf test/tls/*.pem test/coverage.* test/coverage $(BUILD_UPTODATE) $(MANIFEST_TOOL_EXE) build: $(SUDO) go clean -i net diff --git a/build/Dockerfile b/build/Dockerfile index 713c4d63..aa8a16f9 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,7 +1,79 @@ -FROM golang:1.7.3 -RUN apt-get update && apt-get install -y libpcap-dev python-requests time file -RUN go get github.com/golang/lint/golint github.com/fzipp/gocyclo github.com/client9/misspell/cmd/misspell -RUN go clean -i net && go install -tags netgo std -RUN go install -race -tags netgo std +FROM golang:1.7.4 + +# Support Raspberry Pi 2 and newer +ENV GOARM 7 + +# The names of the architectures that we're fetching cross-compilers for from the Ubuntu packages +ENV DEB_CROSSPLATFORMS \ + armhf \ + arm64 + +# The go platforms we are supporting to cross-compile +ENV GO_CROSSPLATFORMS \ + linux/arm \ + linux/arm64 + +# The names of the gcc cross-compilers we have installed +ENV GCC_CROSSCOMPILERS \ + arm-linux-gnueabihf \ + aarch64-linux-gnu + +# Temporarily add the Ubuntu repositories, because we're going to install gcc cross-compilers from there +# Install the build-essential and crossbuild-essential-ARCH packages +RUN echo "deb http://archive.ubuntu.com/ubuntu xenial main universe" > /etc/apt/sources.list.d/cgocrosscompiling.list \ + && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5 3B4FE6ACC0B21F32 \ + && apt-get update \ + && apt-get install -y build-essential \ + && for platform in ${DEB_CROSSPLATFORMS}; do apt-get install -y crossbuild-essential-${platform}; done \ + && rm /etc/apt/sources.list.d/cgocrosscompiling.list \ + && apt-get update \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +# Install some required packages +# libpcap is required because we're linking against its C libraries from the prog/weaver binary +# flex and bison are required packages for compiling libpcap manually later +RUN apt-get update \ + && apt-get install -y \ + libpcap-dev \ + python-requests \ + time \ + file \ + flex \ + bison + +# Install common Go tools +RUN go get \ + github.com/golang/lint/golint \ + github.com/fzipp/gocyclo \ + github.com/client9/misspell/cmd/misspell + +# First clean, because we're gonna rebuild the std both with -race enabled and disabled +# Running go install std twice is intentional and required, as content is written in two different directories +# (/usr/local/go/pkg/linux_amd64 vs /usr/local/go/pkg/linux_amd64_race) +# The other architectures do not have support for -race +RUN go clean -i net \ + && go install -tags netgo std \ + && go install -race -tags netgo std + +# Prebuild std for the other architectures +RUN for platform in ${GO_CROSSPLATFORMS}; do \ + GOOS=${platform%/*} GOARCH=${platform##*/} \ + go install -tags netgo std; done + +# The libpcap version from Debian packages is 1.6.2, matching that version here although newer versions of libpcap have been released +# We have to cross-compile the libpcap library for the new architectures, that's what we're doing here +ENV LIBPCAP_CROSS_VERSION=1.6.2 +RUN curl -sSL http://www.tcpdump.org/release/libpcap-${LIBPCAP_CROSS_VERSION}.tar.gz | tar -xz \ + && cd libpcap-${LIBPCAP_CROSS_VERSION} \ + && for crosscompiler in ${GCC_CROSSCOMPILERS}; do \ + CC=${crosscompiler}-gcc ac_cv_linux_vers=2 ./configure --host=${crosscompiler} --with-pcap=linux \ + && make \ + && export LIB_DIR="/usr/local/lib/${crosscompiler}-gcc" VER="$(cat ./VERSION)" MAJOR_VER="$(sed 's/\([0-9][0-9]*\)\..*/\1/' ./VERSION)" \ + && mkdir -p ${LIB_DIR} \ + && cp -f libpcap.a libpcap.so.${VER} ${LIB_DIR} \ + && ln -sf libpcap.so.${VER} /usr/local/lib/libpcap.so.${MAJOR_VER} \ + && ln -sf libpcap.so.${MAJOR_VER} /usr/local/lib/libpcap.so \ + && make clean; done + COPY build.sh / ENTRYPOINT ["sh", "/build.sh"] diff --git a/build/build.sh b/build/build.sh index 57099760..6f139812 100644 --- a/build/build.sh +++ b/build/build.sh @@ -25,4 +25,7 @@ echo "weave:x:$uid:$gid::$WEAVE_SRC:/bin/sh" >>/etc/passwd echo "weave:*:::::::" >>/etc/shadow echo "weave ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers +# Allow full write access to the Go folders for the user Weave +chown -R $uid:$gid /usr/local/go + su weave -c "PATH=$PATH make -C $WEAVE_SRC BUILD_IN_CONTAINER=false $*" diff --git a/build/shasums/docker-tgz-armel.sha256sum b/build/shasums/docker-tgz-armel.sha256sum new file mode 100644 index 00000000..49fdca83 --- /dev/null +++ b/build/shasums/docker-tgz-armel.sha256sum @@ -0,0 +1 @@ +8f313a9df867c207ae9ceac6e0935d1d2a0360aafce780dd5d1e0687ee98e0c3 docker-1.10.3.tgz diff --git a/build/shasums/docker-tgz-x86_64.sha256sum b/build/shasums/docker-tgz-x86_64.sha256sum new file mode 100644 index 00000000..330e1425 --- /dev/null +++ b/build/shasums/docker-tgz-x86_64.sha256sum @@ -0,0 +1 @@ +7541b5490cee1183d911f909c0e91d3d9d57b2b775e40f14a1a7a6de216ca4a8 docker-1.10.3.tgz diff --git a/build/shasums/qemu-aarch64-static.sha256sum b/build/shasums/qemu-aarch64-static.sha256sum new file mode 100644 index 00000000..d0940ab9 --- /dev/null +++ b/build/shasums/qemu-aarch64-static.sha256sum @@ -0,0 +1 @@ +c02d861383d035ba020b361ef93bf14beb3964a42a6ae645b505b6148493e7ec qemu-aarch64-static diff --git a/build/shasums/qemu-arm-static.sha256sum b/build/shasums/qemu-arm-static.sha256sum new file mode 100644 index 00000000..bd560eab --- /dev/null +++ b/build/shasums/qemu-arm-static.sha256sum @@ -0,0 +1 @@ +0ff63682b8868620d771aa812d3053dc3c579dc8633ce62d3d7ae7eee83beb06 qemu-arm-static diff --git a/circle.yml b/circle.yml index e994239a..74639264 100644 --- a/circle.yml +++ b/circle.yml @@ -4,6 +4,8 @@ general: - gh\-pages machine: + pre: + - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0 services: - docker environment: diff --git a/prog/plugin/Dockerfile.template b/prog/plugin/Dockerfile.template index c516233f..4de1c580 100644 --- a/prog/plugin/Dockerfile.template +++ b/prog/plugin/Dockerfile.template @@ -1,3 +1,3 @@ -FROM DOCKERHUB_USER/weave +FROM DOCKERHUB_USER/weaveARCH_EXT ADD ./plugin /home/weave/ ENTRYPOINT ["/home/weave/plugin"] diff --git a/prog/weave-kube/Dockerfile.template b/prog/weave-kube/Dockerfile.template index 0831985f..8ceaae36 100644 --- a/prog/weave-kube/Dockerfile.template +++ b/prog/weave-kube/Dockerfile.template @@ -1,3 +1,3 @@ -FROM DOCKERHUB_USER/weave +FROM DOCKERHUB_USER/weaveARCH_EXT ADD ./launch.sh ./kube-peers /home/weave/ ENTRYPOINT ["/home/weave/launch.sh"] diff --git a/prog/weave-npc/Dockerfile b/prog/weave-npc/Dockerfile.template similarity index 71% rename from prog/weave-npc/Dockerfile rename to prog/weave-npc/Dockerfile.template index 711c7eca..529cf015 100644 --- a/prog/weave-npc/Dockerfile +++ b/prog/weave-npc/Dockerfile.template @@ -1,4 +1,8 @@ -FROM alpine:3.4 +FROM ALPINE_BASEIMAGE + +# If we're building for another architecture than amd64, the CROSS_BUILD_ placeholder is removed so e.g. CROSS_BUILD_COPY turns into COPY +# If we're building normally, for amd64, CROSS_BUILD lines are removed +CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ LABEL maintainer "Weaveworks Inc " LABEL works.weave.role=system \ diff --git a/prog/weaveexec/Dockerfile b/prog/weaveexec/Dockerfile.template similarity index 74% rename from prog/weaveexec/Dockerfile rename to prog/weaveexec/Dockerfile.template index abb1d29f..90c87de8 100644 --- a/prog/weaveexec/Dockerfile +++ b/prog/weaveexec/Dockerfile.template @@ -1,4 +1,8 @@ -FROM alpine:3.4 +FROM ALPINE_BASEIMAGE + +# If we're building for another architecture than amd64, the CROSS_BUILD_ placeholder is removed so e.g. CROSS_BUILD_COPY turns into COPY +# If we're building normally, for amd64, CROSS_BUILD lines are removed +CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ LABEL maintainer "Weaveworks Inc " LABEL works.weave.role=system \ @@ -26,4 +30,4 @@ ADD ./weaveutil /usr/bin/ ADD ./weavewait /w/w ADD ./weavewait_noop /w-noop/w ADD ./weavewait_nomcast /w-nomcast/w -ADD ./docker.tgz / +ADD ./docker /usr/bin/ diff --git a/prog/weaver/Dockerfile.template b/prog/weaver/Dockerfile.template index e83538be..8d60a095 100644 --- a/prog/weaver/Dockerfile.template +++ b/prog/weaver/Dockerfile.template @@ -1,4 +1,4 @@ -FROM DOCKERHUB_USER/weaveexec +FROM DOCKERHUB_USER/weaveexecARCH_EXT WORKDIR /home/weave ADD ./weaver /home/weave/ ADD weavedata.db /weavedb/ diff --git a/vendor/github.com/estesp/manifest-tool b/vendor/github.com/estesp/manifest-tool new file mode 160000 index 00000000..2db1e2f7 --- /dev/null +++ b/vendor/github.com/estesp/manifest-tool @@ -0,0 +1 @@ +Subproject commit 2db1e2f73b11b24699394e7b15f2fd0241cc4cb7 diff --git a/weave b/weave index 8be24ee5..3493ca4a 100755 --- a/weave +++ b/weave @@ -8,14 +8,13 @@ IMAGE_VERSION=latest [ "$SCRIPT_VERSION" = "unreleased" ] || IMAGE_VERSION=$SCRIPT_VERSION IMAGE_VERSION=${WEAVE_VERSION:-$IMAGE_VERSION} -# - Docker versions prior to 1.6 cannot pull images from DockerHub -# - The weavexec image embeds a Docker 1.6.2 client. Docker will give +# - The weavexec image embeds a Docker 1.10.3 client. Docker will give # a "client is newer than server error" if the daemon has an older # API version, which could be confusing if the user's Docker client # is correctly matched with that older version. # -# We therefore check that the user's Docker *client* is >= 1.6 -MIN_DOCKER_VERSION=1.6.0 +# We therefore check that the user's Docker *client* is >= 1.10.3 +MIN_DOCKER_VERSION=1.10.0 # These are needed for remote execs, hence we introduce them here DOCKERHUB_USER=${DOCKERHUB_USER:-weaveworks} @@ -718,11 +717,10 @@ init_bridge_prep() { ip link add name $BRIDGE type bridge - # for Docker 1.12 this needs to be docker inspect -f '{{with index .Mounts 0}}{{.Source}}{{end}}' - # so running this from 'weave --local' may fail. - # also if weaveexec is not on the path it will be run as a Docker container - # that does not have access to /etc/machine-id - WEAVEDB_PATH=$(docker inspect -f '{{index .Volumes "/weavedb"}}' $DB_CONTAINER_NAME 2>/dev/null) + # Running this from 'weave --local' when weaveexec is not on the + # path will run it as a Docker container that does not have access + # to /etc/machine-id, so will not give the full range of persistent IDs + WEAVEDB_PATH=$(docker inspect -f '{{with index .Mounts 0}}{{.Source}}{{end}}' $DB_CONTAINER_NAME 2>/dev/null) MAC=$(util_op unique-id "$HOST_ROOT/$WEAVEDB_PATH/weave" "$HOST_ROOT") ip link set dev $BRIDGE address $MAC