From a311623bb9ac4984a0c60d17e7289970491c9a79 Mon Sep 17 00:00:00 2001 From: David Wragg Date: Tue, 19 May 2015 11:00:57 +0100 Subject: [PATCH 01/82] Eliminate the Start method on LocalConnection --- router/connection.go | 21 +++++++++------------ router/local_peer.go | 3 +-- router/router.go | 3 +-- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/router/connection.go b/router/connection.go index adf9e04c..3d15c269 100644 --- a/router/connection.go +++ b/router/connection.go @@ -98,26 +98,23 @@ func (conn *RemoteConnection) String() string { return fmt.Sprint("Connection ", from, "->", to) } -func NewLocalConnection(connRemote *RemoteConnection, tcpConn *net.TCPConn, udpAddr *net.UDPAddr, router *Router) *LocalConnection { +// Does not return anything. If the connection is successful, it will +// end up in the local peer's connections map. +func StartLocalConnection(connRemote *RemoteConnection, tcpConn *net.TCPConn, udpAddr *net.UDPAddr, router *Router, acceptNewPeer bool) { if connRemote.local != router.Ourself.Peer { log.Fatal("Attempt to create local connection from a peer which is not ourself") } // NB, we're taking a copy of connRemote here. - return &LocalConnection{ + actionChan := make(chan ConnectionAction, ChannelSize) + finished := make(chan struct{}) + conn := &LocalConnection{ RemoteConnection: *connRemote, Router: router, TCPConn: tcpConn, remoteUDPAddr: udpAddr, - effectivePMTU: DefaultPMTU} -} - -// Async. Does not return anything. If the connection is successful, -// it will end up in the local peer's connections map. -func (conn *LocalConnection) Start(acceptNewPeer bool) { - actionChan := make(chan ConnectionAction, ChannelSize) - conn.actionChan = actionChan - finished := make(chan struct{}) - conn.finished = finished + effectivePMTU: DefaultPMTU, + actionChan: actionChan, + finished: finished} go conn.run(actionChan, finished, acceptNewPeer) } diff --git a/router/local_peer.go b/router/local_peer.go index 09498629..0a15076d 100644 --- a/router/local_peer.go +++ b/router/local_peer.go @@ -132,8 +132,7 @@ func (peer *LocalPeer) CreateConnection(peerAddr string, acceptNewPeer bool) err return err } connRemote := NewRemoteConnection(peer.Peer, nil, tcpConn.RemoteAddr().String(), true, false) - connLocal := NewLocalConnection(connRemote, tcpConn, udpAddr, peer.router) - connLocal.Start(acceptNewPeer) + StartLocalConnection(connRemote, tcpConn, udpAddr, peer.router, acceptNewPeer) return nil } diff --git a/router/router.go b/router/router.go index ab671a0d..520ca080 100644 --- a/router/router.go +++ b/router/router.go @@ -224,8 +224,7 @@ func (router *Router) acceptTCP(tcpConn *net.TCPConn) { remoteAddrStr := tcpConn.RemoteAddr().String() log.Printf("->[%s] connection accepted\n", remoteAddrStr) connRemote := NewRemoteConnection(router.Ourself.Peer, nil, remoteAddrStr, false, false) - connLocal := NewLocalConnection(connRemote, tcpConn, nil, router) - connLocal.Start(true) + StartLocalConnection(connRemote, tcpConn, nil, router, true) } func (router *Router) listenUDP(localPort int, po PacketSink) *net.UDPConn { From d6bceb139ee12eaaa1f5bf00e65c0128227939d8 Mon Sep 17 00:00:00 2001 From: David Wragg Date: Tue, 19 May 2015 11:12:41 +0100 Subject: [PATCH 02/82] Eliminate MacCache Start method --- router/mac_cache.go | 6 ++---- router/router.go | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/router/mac_cache.go b/router/mac_cache.go index d251d194..5eb03f41 100644 --- a/router/mac_cache.go +++ b/router/mac_cache.go @@ -22,14 +22,12 @@ type MacCache struct { } func NewMacCache(maxAge time.Duration, onExpiry func(net.HardwareAddr, *Peer)) *MacCache { - return &MacCache{ + cache := &MacCache{ table: make(map[uint64]*MacCacheEntry), maxAge: maxAge, onExpiry: onExpiry} -} - -func (cache *MacCache) Start() { cache.setExpiryTimer() + return cache } func (cache *MacCache) Enter(mac net.HardwareAddr, peer *Peer) bool { diff --git a/router/router.go b/router/router.go index 520ca080..b5f40071 100644 --- a/router/router.go +++ b/router/router.go @@ -96,7 +96,6 @@ func (router *Router) Start() { checkFatal(err) } router.Ourself.Start() - router.Macs.Start() router.Routes.Start() router.ConnectionMaker.Start() router.UDPListener = router.listenUDP(router.Port, po) From 72039bbe5a308a5dd6a93ed3950fb1b1186fb645 Mon Sep 17 00:00:00 2001 From: David Wragg Date: Tue, 19 May 2015 11:14:45 +0100 Subject: [PATCH 03/82] Eliminate the LocalPeer Start method --- router/local_peer.go | 11 ++++++----- router/router.go | 1 - 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/router/local_peer.go b/router/local_peer.go index 0a15076d..b9ea939b 100644 --- a/router/local_peer.go +++ b/router/local_peer.go @@ -18,13 +18,14 @@ type LocalPeer struct { type LocalPeerAction func() func NewLocalPeer(name PeerName, nickName string, router *Router) *LocalPeer { - return &LocalPeer{Peer: NewPeer(name, nickName, 0, 0), router: router} -} - -func (peer *LocalPeer) Start() { actionChan := make(chan LocalPeerAction, ChannelSize) - peer.actionChan = actionChan + peer := &LocalPeer{ + Peer: NewPeer(name, nickName, 0, 0), + router: router, + actionChan: actionChan, + } go peer.actorLoop(actionChan) + return peer } func (peer *LocalPeer) Forward(dstPeer *Peer, df bool, frame []byte, dec *EthernetDecoder) error { diff --git a/router/router.go b/router/router.go index b5f40071..c3cc1464 100644 --- a/router/router.go +++ b/router/router.go @@ -95,7 +95,6 @@ func (router *Router) Start() { po, err = NewPcapO(router.Iface.Name) checkFatal(err) } - router.Ourself.Start() router.Routes.Start() router.ConnectionMaker.Start() router.UDPListener = router.listenUDP(router.Port, po) From ae1805de76e8bc76de986955897d5ad1f82c5061 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Tue, 23 Jun 2015 19:33:01 +0100 Subject: [PATCH 04/82] separate the image build and export phases That way the export can be just a single tar, which is more convenient. And without introducing unnecessary image builds. --- .gitignore | 5 +++-- Makefile | 31 ++++++++++++++++++------------- site/building.md | 11 ++++------- test/setup.sh | 2 -- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index 6c9406eb..609b440f 100644 --- a/.gitignore +++ b/.gitignore @@ -39,9 +39,10 @@ prog/netcheck/netcheck tools/bin tools/build releases +.weaver.uptodate +.weavedns.uptodate +.weaveexec.uptodate weave.tar -weavedns.tar -weaveexec.tar prog/weaveexec/weave prog/weaveexec/weaveproxy prog/weaveexec/weavewait diff --git a/Makefile b/Makefile index bfd649ee..5031dede 100644 --- a/Makefile +++ b/Makefile @@ -18,23 +18,25 @@ NETCHECK_EXE=prog/netcheck/netcheck EXES=$(WEAVER_EXE) $(WEAVEDNS_EXE) $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(NETCHECK_EXE) +WEAVER_UPTODATE=.weaver.uptodate +WEAVEDNS_UPTODATE=.weavedns.uptodate +WEAVEEXEC_UPTODATE=.weaveexec.uptodate + +IMAGES_UPTODATE=$(WEAVER_UPTODATE) $(WEAVEDNS_UPTODATE) $(WEAVEEXEC_UPTODATE) + WEAVER_IMAGE=$(DOCKERHUB_USER)/weave WEAVEDNS_IMAGE=$(DOCKERHUB_USER)/weavedns WEAVEEXEC_IMAGE=$(DOCKERHUB_USER)/weaveexec IMAGES=$(WEAVER_IMAGE) $(WEAVEDNS_IMAGE) $(WEAVEEXEC_IMAGE) -WEAVER_EXPORT=weave.tar -WEAVEDNS_EXPORT=weavedns.tar -WEAVEEXEC_EXPORT=weaveexec.tar - -EXPORTS=$(WEAVER_EXPORT) $(WEAVEDNS_EXPORT) $(WEAVEEXEC_EXPORT) +WEAVE_EXPORT=weave.tar WEAVEEXEC_DOCKER_VERSION=1.3.1 DOCKER_DISTRIB=prog/weaveexec/docker-$(WEAVEEXEC_DOCKER_VERSION).tgz DOCKER_DISTRIB_URL=https://get.docker.com/builds/Linux/x86_64/docker-$(WEAVEEXEC_DOCKER_VERSION).tgz -all: $(EXPORTS) +all: $(WEAVE_EXPORT) travis: $(EXES) @@ -66,15 +68,15 @@ $(SIGPROXY_EXE): prog/sigproxy/main.go $(WEAVEWAIT_EXE): prog/weavewait/main.go go build -o $@ ./$(@D) -$(WEAVER_EXPORT): prog/weaver/Dockerfile $(WEAVER_EXE) +$(WEAVER_UPTODATE): prog/weaver/Dockerfile $(WEAVER_EXE) $(SUDO) docker build -t $(WEAVER_IMAGE) prog/weaver - $(SUDO) docker save $(WEAVER_IMAGE):latest > $@ + touch $@ -$(WEAVEDNS_EXPORT): prog/weavedns/Dockerfile $(WEAVEDNS_EXE) +$(WEAVEDNS_UPTODATE): prog/weavedns/Dockerfile $(WEAVEDNS_EXE) $(SUDO) docker build -t $(WEAVEDNS_IMAGE) prog/weavedns - $(SUDO) docker save $(WEAVEDNS_IMAGE):latest > $@ + touch $@ -$(WEAVEEXEC_EXPORT): prog/weaveexec/Dockerfile $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(NETCHECK_EXE) +$(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(NETCHECK_EXE) cp weave prog/weaveexec/weave cp $(SIGPROXY_EXE) prog/weaveexec/sigproxy cp $(WEAVEPROXY_EXE) prog/weaveexec/weaveproxy @@ -82,7 +84,10 @@ $(WEAVEEXEC_EXPORT): prog/weaveexec/Dockerfile $(DOCKER_DISTRIB) weave $(SIGPROX cp $(NETCHECK_EXE) prog/weaveexec/netcheck cp $(DOCKER_DISTRIB) prog/weaveexec/docker.tgz $(SUDO) docker build -t $(WEAVEEXEC_IMAGE) prog/weaveexec - $(SUDO) docker save $(WEAVEEXEC_IMAGE):latest > $@ + touch $@ + +$(WEAVE_EXPORT): $(IMAGES_UPTODATE) + $(SUDO) docker save $(addsuffix :latest,$(IMAGES)) > $@ $(DOCKER_DISTRIB): curl -o $(DOCKER_DISTRIB) $(DOCKER_DISTRIB_URL) @@ -112,7 +117,7 @@ publish: $(PUBLISH) clean: -$(SUDO) docker rmi $(IMAGES) - rm -f $(EXES) $(EXPORTS) + rm -f $(EXES) $(IMAGES_UPTODATE) $(WEAVE_EXPORT) rm -f test/tls/*.pem build: diff --git a/site/building.md b/site/building.md index 04b998ab..4b91d59f 100644 --- a/site/building.md +++ b/site/building.md @@ -57,8 +57,7 @@ $ make This will build the weave components and package them into three Docker images (`weaveworks/weave`, `weaveworks/weavedns`, and -`weaveworks/weaveexec`). These are then exported (as `weave.tar`, -`weavedns.tar` and `weaveexec.tar`). +`weaveworks/weaveexec`). These are then exported as `weave.tar`. ## Building in a Docker container @@ -171,15 +170,13 @@ you can do so with $ vagrant ssh -c 'make -C src/github.com/weaveworks/weave' ``` -you should then find container snapshot tarballs in the top-level -directory. You can use these snapshots with `docker load` against a -different host, e.g. +you should then find a `weave.tar` container snapshot tarball in the +top-level directory. You can use that snapshot with `docker load` +against a different host, e.g. ```bash $ export DOCKER_HOST=tcp:// $ docker load < weave.tar -$ docker load < weavedns.tar -$ docker load < weaveexec.tar ``` You can provide extra Vagrant configuration by putting a file diff --git a/test/setup.sh b/test/setup.sh index d2b49f01..7235b4a4 100755 --- a/test/setup.sh +++ b/test/setup.sh @@ -10,8 +10,6 @@ echo "Copying weave images, scripts, and certificates to hosts, and" echo " prefetch test images" for HOST in $HOSTS; do docker_on $HOST load -i ../weave.tar - docker_on $HOST load -i ../weavedns.tar - docker_on $HOST load -i ../weaveexec.tar run_on $HOST mkdir -p bin cat ../bin/docker-ns | run_on $HOST sh -c "cat > $DOCKER_NS" cat ../weave | run_on $HOST sh -c "cat > ./weave" From bdda9d03ae05ba52c03a5e8713d3d85b97ed0dca Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Wed, 24 Jun 2015 02:59:24 +0100 Subject: [PATCH 05/82] bump docker version in weave-build docker image to 1.3.1, the same version we use everywhere else, e.g. in weavexec. --- build/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build/Dockerfile b/build/Dockerfile index 562673c8..23861bdf 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,6 +1,9 @@ FROM ubuntu -RUN apt-get -y update && apt-get -y install --no-install-recommends build-essential git ca-certificates docker.io mercurial libpcap-dev curl make pkg-config gcc bison flex +RUN apt-get -y update && apt-get -y install --no-install-recommends ca-certificates apt-transport-https +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9 +RUN echo deb https://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list +RUN apt-get -y update && apt-get -y install --no-install-recommends build-essential git lxc-docker-1.3.1 mercurial libpcap-dev curl make pkg-config gcc bison flex # When doing a build in a container, "apt-get update" happens twice, # which can be a very significant overhead for incremental builds. From e3574f243f2e5331e93266debf9c8ddfee32c858 Mon Sep 17 00:00:00 2001 From: Fintan Ryan Date: Wed, 24 Jun 2015 10:35:12 +0100 Subject: [PATCH 06/82] Remove Link to Dockercon from README --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 37392793..792c298f 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,6 @@ # About Weaveworks -[Come meet Weaveworks at Dockercon](http://weave.works/dockercon) - [Weaveworks](http://weave.works) is the company that develops Weave - the most productive way for developers to connect, observe and control Docker containers. To learn about our products, including getting started tutorials, visit our [website](http://weave.works), read about the [1.0 release of Weave](http://blog.weave.works/2015/06/18/weave-1-0-is-out-and-it-is-good/), visit our [documentation](http://docs.weave.works) or continue to read about some of the more technical aspects of Weave in this readme. # Weave From 4b11874390f20ab1bb8d5d6797463992738ce080 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Tue, 23 Jun 2015 17:12:51 +0100 Subject: [PATCH 07/82] sleep for 0.1s instead of 1s in the wait_for_status loop This reduces `weave launch|launch-dns` execution time. Closes #992. --- weave | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/weave b/weave index 9c6533d2..7b58efb1 100755 --- a/weave +++ b/weave @@ -183,6 +183,21 @@ command_exists() { command -v $1 >/dev/null 2>&1 } +fractional_sleep() { + case $1 in + *.*) + if [ -z "$NO_FRACTIONAL_SLEEP" ] ; then + sleep $1 >/dev/null 2>&1 && return 0 + NO_FRACTIONAL_SLEEP=1 + fi + sleep $((${1%.*} + 1)) + ;; + *) + sleep $1 + ;; + esac +} + run_iptables() { # -w is recent addition to iptables if [ -z "$CHECKED_IPTABLES_W" ] ; then @@ -585,7 +600,7 @@ wait_for_status() { echo "$err_msg" >&2 return 1 fi - sleep 1 + fractional_sleep 0.1 done } From 409961fff3188ddecae9c948f29218e5f88812ca Mon Sep 17 00:00:00 2001 From: Adam Harrison Date: Wed, 24 Jun 2015 10:56:34 +0100 Subject: [PATCH 08/82] Rename launch to launch-router --- weave | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/weave b/weave index 9c6533d2..ad52497b 100755 --- a/weave +++ b/weave @@ -26,22 +26,22 @@ CIDR_REGEXP="$IP_REGEXP/[0-9]{1,2}" usage() { echo "Usage:" echo "weave setup" - echo "weave launch [-password ] [-nickname ] [-nodiscovery]" - echo " [-iprange [-ipsubnet ]] ..." - echo "weave launch-dns []" - echo "weave launch-proxy [-H ] [--with-dns | --without-dns]" - echo " [--no-default-ipam]" - echo "weave connect [--replace] [ ...]" - echo "weave forget ..." - echo "weave run [--with-dns | --without-dns] [ ...] ..." - echo "weave start [ ...] " - echo "weave attach [ ...] " - echo "weave detach [ ...] " - echo "weave dns-add [ ...] [-h ]" - echo "weave dns-remove [ ...] [-h ]" - echo "weave expose [ ...] [-h ]" - echo "weave hide [ ...]" - echo "weave ps [ ...]" + echo "weave launch-router [-password ] [-nickname ] [-nodiscovery]" + echo " [-iprange [-ipsubnet ]] ..." + echo "weave launch-dns []" + echo "weave launch-proxy [-H ] [--with-dns | --without-dns]" + echo " [--no-default-ipam]" + echo "weave connect [--replace] [ ...]" + echo "weave forget ..." + echo "weave run [--with-dns | --without-dns] [ ...] ..." + echo "weave start [ ...] " + echo "weave attach [ ...] " + echo "weave detach [ ...] " + echo "weave dns-add [ ...] [-h ]" + echo "weave dns-remove [ ...] [-h ]" + echo "weave expose [ ...] [-h ]" + echo "weave hide [ ...]" + echo "weave ps [ ...]" echo "weave status" echo "weave proxy-env" echo "weave proxy-config" @@ -50,7 +50,7 @@ usage() { echo "weave stop-dns" echo "weave stop-proxy" echo "weave reset" - echo "weave rmpeer " + echo "weave rmpeer " echo echo "where = [:]" echo " = /" @@ -977,7 +977,7 @@ case "$COMMAND" in create-bridge) create_bridge --without-ethtool ;; - launch) + launch-router) check_not_running $CONTAINER_NAME $BASE_IMAGE create_bridge # We set the router name to the bridge mac since that is From 8250e056189d33e4723c21974bab4a5e8deb8a13 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Wed, 24 Jun 2015 14:37:41 +0200 Subject: [PATCH 09/82] include stylesheet in repo, update footer need because website's custom.css is fingerprinted --- site/_includes/footer.html | 27 +- site/_includes/header-navbaronly.html | 14 +- site/_layouts/default.html | 6 +- site/assets/custom.css | 5880 +++++++++++++++++++++++++ 4 files changed, 5910 insertions(+), 17 deletions(-) create mode 100644 site/assets/custom.css diff --git a/site/_includes/footer.html b/site/_includes/footer.html index e569c16e..1d114958 100644 --- a/site/_includes/footer.html +++ b/site/_includes/footer.html @@ -2,38 +2,49 @@