diff --git a/.gitignore b/.gitignore index 4506fcc5..4d5f7002 100644 --- a/.gitignore +++ b/.gitignore @@ -40,25 +40,30 @@ prog/weaveproxy/weaveproxy prog/sigproxy/sigproxy prog/weavewait/weavewait prog/weavewait/weavewait_noop +prog/weavewait/weavewait_nomcast prog/netcheck/netcheck prog/docker_tls_args/docker_tls_args +prog/plugin/plugin testing/cover/cover testing/runner/runner releases .weaver.uptodate .weavedns.uptodate .weaveexec.uptodate +.dockerplugin.uptodate weave.tar.gz prog/weaveexec/weave prog/weaveexec/weavehosts prog/weaveexec/weaveproxy prog/weaveexec/weavewait prog/weaveexec/weavewait_noop +prog/weaveexec/weavewait_nomcast prog/weaveexec/sigproxy prog/weaveexec/netcheck prog/weaveexec/docker_tls_args prog/weaveexec/docker*.tgz Vagrantfile.local +test/tls/tls test/tls/*.pem test/coverage test/coverage.* diff --git a/CHANGELOG.md b/CHANGELOG.md index 1136390a..c1c3a8a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,30 @@ +## Release 1.4.0 + +**Highlights** + +* The Docker Network plugin can now operate without a cluster store, so it is now run by default. +* You can now use the fast datapath over trusted links and Weave encryption over untrusted links, in the same network. + +Plus many bug fixes and minor enhancements. More details below and in +the [change log](https://github.com/weaveworks/weave/issues?q=milestone%3A1.4.0). + +## Release 1.3.1 + +**Highlights** + +* The minimum Docker version has been increased to 1.6 due to the + [upcoming deprecation of Dockerhub access](https://blog.docker.com/2015/10/docker-hub-deprecation-1-5/) + for old clients. From December 7th onwards previous versions of the + `weave` script will fail to pull down images from the hub; if you + are unable to upgrade to 1.3.1 immediately you can work around this + by running `weave --local setup` in conjunction with a compatible + Docker client installation +* Docker networking plugin now works with older kernels and allows you + to configure the MTU + +More details in the +[change log](https://github.com/weaveworks/weave/issues?q=milestone%3A1.3.1). + ## Release 1.3.0 **Highlights** diff --git a/Makefile b/Makefile index b13b3454..48b2da7f 100644 --- a/Makefile +++ b/Makefile @@ -14,25 +14,30 @@ WEAVEPROXY_EXE=prog/weaveproxy/weaveproxy SIGPROXY_EXE=prog/sigproxy/sigproxy WEAVEWAIT_EXE=prog/weavewait/weavewait WEAVEWAIT_NOOP_EXE=prog/weavewait/weavewait_noop +WEAVEWAIT_NOMCAST_EXE=prog/weavewait/weavewait_nomcast NETCHECK_EXE=prog/netcheck/netcheck DOCKERTLSARGS_EXE=prog/docker_tls_args/docker_tls_args +DOCKERPLUGIN_EXE=prog/plugin/plugin RUNNER_EXE=tools/runner/runner +TEST_TLS_EXE=test/tls/tls -EXES=$(WEAVER_EXE) $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(NETCHECK_EXE) $(DOCKERTLSARGS_EXE) +EXES=$(WEAVER_EXE) $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(WEAVEWAIT_NOMCAST_EXE) $(NETCHECK_EXE) $(DOCKERTLSARGS_EXE) $(DOCKERPLUGIN_EXE) $(TEST_TLS_EXE) WEAVER_UPTODATE=.weaver.uptodate WEAVEEXEC_UPTODATE=.weaveexec.uptodate +DOCKERPLUGIN_UPTODATE=.dockerplugin.uptodate -IMAGES_UPTODATE=$(WEAVER_UPTODATE) $(WEAVEEXEC_UPTODATE) +IMAGES_UPTODATE=$(WEAVER_UPTODATE) $(WEAVEEXEC_UPTODATE) $(DOCKERPLUGIN_UPTODATE) WEAVER_IMAGE=$(DOCKERHUB_USER)/weave WEAVEEXEC_IMAGE=$(DOCKERHUB_USER)/weaveexec +DOCKERPLUGIN_IMAGE=$(DOCKERHUB_USER)/plugin -IMAGES=$(WEAVER_IMAGE) $(WEAVEEXEC_IMAGE) +IMAGES=$(WEAVER_IMAGE) $(WEAVEEXEC_IMAGE) $(DOCKERPLUGIN_IMAGE) WEAVE_EXPORT=weave.tar.gz -WEAVEEXEC_DOCKER_VERSION=1.3.1 +WEAVEEXEC_DOCKER_VERSION=1.6.2 DOCKER_DISTRIB=prog/weaveexec/docker-$(WEAVEEXEC_DOCKER_VERSION).tgz DOCKER_DISTRIB_URL=https://get.docker.com/builds/Linux/x86_64/docker-$(WEAVEEXEC_DOCKER_VERSION).tgz NETGO_CHECK=@strings $@ | grep cgo_stub\\\.go >/dev/null || { \ @@ -47,7 +52,7 @@ BUILD_FLAGS=-ldflags "-extldflags \"-static\" -X main.version $(WEAVE_VERSION)" PACKAGE_BASE=$(shell go list -e ./) -all: $(WEAVE_EXPORT) $(RUNNER_EXE) +all: $(WEAVE_EXPORT) $(RUNNER_EXE) $(TEST_TLS_EXE) travis: $(EXES) @@ -74,38 +79,52 @@ $(WEAVER_EXE): router/*.go mesh/*.go ipam/*.go ipam/*/*.go nameserver/*.go prog/ $(WEAVEPROXY_EXE): proxy/*.go prog/weaveproxy/main.go $(NETCHECK_EXE): prog/netcheck/netcheck.go -# Sigproxy and weavewait need separate rules as they fail the netgo check in +# These next programs need separate rules as they fail the netgo check in # the main build stanza due to not importing net package $(SIGPROXY_EXE): prog/sigproxy/main.go -$(WEAVEWAIT_EXE): prog/weavewait/*.go net/*.go $(DOCKERTLSARGS_EXE): prog/docker_tls_args/*.go +$(DOCKERPLUGIN_EXE): prog/plugin/*.go plugin/net/*.go plugin/ipam/*.go plugin/skel/*.go api/*.go common/docker/*.go +$(TEST_TLS_EXE): test/tls/*.go -$(WEAVEWAIT_EXE) $(SIGPROXY_EXE) $(DOCKERTLSARGS_EXE): +$(SIGPROXY_EXE) $(DOCKERTLSARGS_EXE) $(DOCKERPLUGIN_EXE) $(TEST_TLS_EXE): go get -tags netgo ./$(@D) go build $(BUILD_FLAGS) -o $@ ./$(@D) +$(WEAVEWAIT_EXE): prog/weavewait/*.go net/*.go + go get -tags netgo ./$(@D) + go build $(BUILD_FLAGS) -tags "netgo iface mcast" -o $@ ./$(@D) + +$(WEAVEWAIT_NOMCAST_EXE): prog/weavewait/*.go net/*.go + go get -tags netgo ./$(@D) + go build $(BUILD_FLAGS) -tags "netgo iface" -o $@ ./$(@D) + $(WEAVEWAIT_NOOP_EXE): prog/weavewait/*.go go get -tags netgo ./$(@D) - go build $(BUILD_FLAGS) -tags noop -o $@ ./$(@D) + go build $(BUILD_FLAGS) -o $@ ./$(@D) $(WEAVER_UPTODATE): prog/weaver/Dockerfile $(WEAVER_EXE) - $(SUDO) docker build -t $(WEAVER_IMAGE) prog/weaver + $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker build -t $(WEAVER_IMAGE) prog/weaver touch $@ -$(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(NETCHECK_EXE) $(DOCKERTLSARGS_EXE) +$(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile prog/weaveexec/symlink $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(WEAVEWAIT_NOMCAST_EXE) $(NETCHECK_EXE) $(DOCKERTLSARGS_EXE) cp weave prog/weaveexec/weave cp $(SIGPROXY_EXE) prog/weaveexec/sigproxy cp $(WEAVEPROXY_EXE) prog/weaveexec/weaveproxy cp $(WEAVEWAIT_EXE) prog/weaveexec/weavewait cp $(WEAVEWAIT_NOOP_EXE) prog/weaveexec/weavewait_noop + cp $(WEAVEWAIT_NOMCAST_EXE) prog/weaveexec/weavewait_nomcast cp $(NETCHECK_EXE) prog/weaveexec/netcheck cp $(DOCKERTLSARGS_EXE) prog/weaveexec/docker_tls_args cp $(DOCKER_DISTRIB) prog/weaveexec/docker.tgz - $(SUDO) docker build -t $(WEAVEEXEC_IMAGE) prog/weaveexec + $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker build -t $(WEAVEEXEC_IMAGE) prog/weaveexec + touch $@ + +$(DOCKERPLUGIN_UPTODATE): prog/plugin/Dockerfile $(DOCKERPLUGIN_EXE) + $(SUDO) docker build -t $(DOCKERPLUGIN_IMAGE) prog/plugin touch $@ $(WEAVE_EXPORT): $(IMAGES_UPTODATE) - $(SUDO) docker save $(addsuffix :latest,$(IMAGES)) | gzip > $@ + $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker save $(addsuffix :latest,$(IMAGES)) | gzip > $@ $(DOCKER_DISTRIB): curl -o $(DOCKER_DISTRIB) $(DOCKER_DISTRIB_URL) @@ -123,16 +142,16 @@ $(RUNNER_EXE): tools/.git make -C tools/runner $(PUBLISH): publish_%: $(IMAGES_UPTODATE) - $(SUDO) docker tag -f $(DOCKERHUB_USER)/$* $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION) - $(SUDO) docker push $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION) + $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker tag -f $(DOCKERHUB_USER)/$* $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION) + $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker push $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION) ifneq ($(UPDATE_LATEST),false) - $(SUDO) docker push $(DOCKERHUB_USER)/$*:latest + $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker push $(DOCKERHUB_USER)/$*:latest endif publish: $(PUBLISH) clean-bin: - -$(SUDO) docker rmi $(IMAGES) + -$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker rmi $(IMAGES) go clean -r $(addprefix ./,$(dir $(EXES))) rm -f $(EXES) $(IMAGES_UPTODATE) $(WEAVE_EXPORT) diff --git a/README.md b/README.md index 0f024ee3..93d5fd66 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ capabilities, so these can continue to be used by containers. ## Installation Ensure you are running Linux (kernel 3.8 or later) and have Docker -(version 1.3.1 or later) installed. Then install weave with +(version 1.6.0 or later) installed. Then install weave with sudo curl -L git.io/weave -o /usr/local/bin/weave sudo chmod a+x /usr/local/bin/weave diff --git a/api/dns.go b/api/dns.go new file mode 100644 index 00000000..36ec8630 --- /dev/null +++ b/api/dns.go @@ -0,0 +1,18 @@ +package api + +import ( + "fmt" + "net/url" +) + +func (client *Client) RegisterWithDNS(ID string, fqdn string, ip string) error { + data := url.Values{} + data.Add("fqdn", fqdn) + _, err := client.httpVerb("PUT", fmt.Sprintf("/name/%s/%s", ID, ip), data) + return err +} + +func (client *Client) DeregisterWithDNS(ID string, ip string) error { + _, err := client.httpVerb("DELETE", fmt.Sprintf("/name/%s/%s", ID, ip), nil) + return err +} diff --git a/api/ipam.go b/api/ipam.go new file mode 100644 index 00000000..8be57b27 --- /dev/null +++ b/api/ipam.go @@ -0,0 +1,49 @@ +package api + +import ( + "fmt" + "net" +) + +func (client *Client) ipamOp(ID string, op string) (*net.IPNet, error) { + ip, err := client.httpVerb(op, fmt.Sprintf("/ip/%s", ID), nil) + if err != nil { + return nil, err + } + return parseIP(ip) +} + +// returns an IP for the ID given, allocating a fresh one if necessary +func (client *Client) AllocateIP(ID string) (*net.IPNet, error) { + return client.ipamOp(ID, "POST") +} + +// returns an IP for the ID given, or nil if one has not been +// allocated +func (client *Client) LookupIP(ID string) (*net.IPNet, error) { + return client.ipamOp(ID, "GET") +} + +// release an IP which is no longer needed +func (client *Client) ReleaseIP(ID string) error { + _, err := client.ipamOp(ID, "DELETE") + return err +} + +func (client *Client) DefaultSubnet() (*net.IPNet, error) { + cidr, err := client.httpVerb("GET", fmt.Sprintf("/ipinfo/defaultsubnet"), nil) + if err != nil { + return nil, err + } + _, ipnet, err := net.ParseCIDR(cidr) + return ipnet, err +} + +func parseIP(body string) (*net.IPNet, error) { + ip, ipnet, err := net.ParseCIDR(string(body)) + if err != nil { + return nil, err + } + ipnet.IP = ip + return ipnet, nil +} diff --git a/api/weaveapi.go b/api/weaveapi.go new file mode 100644 index 00000000..8be6b334 --- /dev/null +++ b/api/weaveapi.go @@ -0,0 +1,73 @@ +package api + +import ( + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "strings" + + . "github.com/weaveworks/weave/common" +) + +const ( + WeaveHttpPort = 6784 +) + +type Client struct { + baseUrl string + resolve func() (string, error) +} + +func (client *Client) httpVerb(verb string, url string, values url.Values) (string, error) { + baseUrl := client.baseUrl + if client.resolve != nil { + addr, err := client.resolve() + if err != nil { + return "", err + } + baseUrl = fmt.Sprintf("http://%s:%d", addr, WeaveHttpPort) + } + url = baseUrl + url + Log.Debugf("weave %s to %s with %v", verb, url, values) + var body io.Reader + if values != nil { + body = strings.NewReader(values.Encode()) + } + req, err := http.NewRequest(verb, url, body) + if err != nil { + return "", err + } + if values != nil { + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + } + if resp, err := http.DefaultClient.Do(req); err != nil { + return "", err + } else { + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNoContent { + return string(body), nil + } else { + return "", errors.New(resp.Status + ": " + string(body)) + } + } +} + +func NewClient(addr string) *Client { + return &Client{baseUrl: fmt.Sprintf("http://%s:%d", addr, WeaveHttpPort)} +} + +func NewClientWithResolver(resolver func() (string, error)) *Client { + return &Client{resolve: resolver} +} + +func (client *Client) Connect(remote string) error { + _, err := client.httpVerb("POST", "/connect", url.Values{"peer": {remote}}) + return err +} diff --git a/bin/multiweave b/bin/multiweave index ad4441ac..560b6dd8 100755 --- a/bin/multiweave +++ b/bin/multiweave @@ -29,6 +29,7 @@ random_mac() { case "$1" in launch) shift 1 + echo $(date -R) launching for i in $(seq $START $FINISH); do WEAVE_CONTAINER_NAME=weave$i WEAVE_PORT=$((5000+i)) $weavedir/weave launch-router --iface '' --name $(random_mac) --nickname weave$i --port $PORT --no-dns "$@" done @@ -36,6 +37,7 @@ case "$1" in status=$(docker inspect --format='{{.State.Running}} {{.NetworkSettings.IPAddress}}' weave$((START-1))) PREV="${status#true }" fi + echo $(date -R) connecting for i in $(seq $START $FINISH); do if [ -n "$PREV" ] ; then echo connecting weave$i to $PREV @@ -44,6 +46,7 @@ case "$1" in status=$(docker inspect --format='{{.State.Running}} {{.NetworkSettings.IPAddress}}' weave$i) PREV="${status#true }" done + echo $(date -R) done ;; stop) for i in $(seq $START $FINISH); do diff --git a/build/Dockerfile b/build/Dockerfile index 08df1d9d..aae48d06 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -3,7 +3,7 @@ FROM ubuntu RUN apt-get -y update && apt-get -y install --no-install-recommends ca-certificates apt-transport-https RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9 RUN echo deb https://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list -RUN apt-get -y update && apt-get -y install --no-install-recommends build-essential git lxc-docker-1.3.1 mercurial libpcap-dev curl make pkg-config gcc bison flex python-requests +RUN apt-get -y update && apt-get -y install --no-install-recommends build-essential git lxc-docker-1.6.2 mercurial libpcap-dev curl make pkg-config gcc bison flex python-requests # When doing a build in a container, "apt-get update" happens twice, # which can be a very significant overhead for incremental builds. diff --git a/common/docker/client.go b/common/docker/client.go index 6a4eaff1..86cb9dd7 100644 --- a/common/docker/client.go +++ b/common/docker/client.go @@ -1,7 +1,12 @@ package docker import ( - "github.com/fsouza/go-dockerclient" + "errors" + "fmt" + "strings" + + docker "github.com/fsouza/go-dockerclient" + . "github.com/weaveworks/weave/common" ) @@ -17,32 +22,52 @@ type Client struct { // NewClient creates a new Docker client and checks we can talk to Docker func NewClient(apiPath string) (*Client, error) { + if apiPath != "" && !strings.Contains(apiPath, "://") { + apiPath = "tcp://" + apiPath + } dc, err := docker.NewClient(apiPath) if err != nil { return nil, err } client := &Client{dc} - return client, client.checkWorking(apiPath) + return client, client.checkWorking() } func NewVersionedClient(apiPath string, apiVersionString string) (*Client, error) { + if !strings.Contains(apiPath, "://") { + apiPath = "tcp://" + apiPath + } dc, err := docker.NewVersionedClient(apiPath, apiVersionString) if err != nil { return nil, err } client := &Client{dc} - return client, client.checkWorking(apiPath) + return client, client.checkWorking() } -func (c *Client) checkWorking(apiPath string) error { - env, err := c.Version() +func NewVersionedClientFromEnv(apiVersionString string) (*Client, error) { + dc, err := docker.NewVersionedClientFromEnv(apiVersionString) if err != nil { - return err + return nil, err + } + client := &Client{dc} + + return client, client.checkWorking() +} + +func (c *Client) checkWorking() error { + _, err := c.Version() + return err +} + +func (c *Client) Info() string { + if env, err := c.Version(); err != nil { + return fmt.Sprintf("Docker API error: %s", err) + } else { + return fmt.Sprintf("Docker API on %s: %v", c.Endpoint(), env) } - Log.Infof("[docker] Using Docker API on %s: %v", apiPath, env) - return nil } // AddObserver adds an observer for docker events @@ -80,3 +105,28 @@ func (c *Client) IsContainerNotRunning(idStr string) bool { Log.Errorf("[docker] Could not check container status: %s", err) return false } + +// This is intended to find an IP address that we can reach the container on; +// if it is on the Docker bridge network then that address; if on the host network +// then localhost +func (c *Client) GetContainerIP(nameOrID string) (string, error) { + Log.Debugf("Getting IP for container %s", nameOrID) + info, err := c.InspectContainer(nameOrID) + if err != nil { + return "", err + } + if info.NetworkSettings.Networks != nil { + Log.Debugln("Networks: ", info.NetworkSettings.Networks) + if bridgeNetwork, ok := info.NetworkSettings.Networks["bridge"]; ok { + return bridgeNetwork.IPAddress, nil + } else if _, ok := info.NetworkSettings.Networks["host"]; ok { + return "127.0.0.1", nil + } + } else if info.HostConfig.NetworkMode == "host" { + return "127.0.0.1", nil + } + if info.NetworkSettings.IPAddress == "" { + return "", errors.New("No IP address found for container " + nameOrID) + } + return info.NetworkSettings.IPAddress, nil +} diff --git a/docs/fastdp.md b/docs/fastdp.md new file mode 100644 index 00000000..51aa211e --- /dev/null +++ b/docs/fastdp.md @@ -0,0 +1,170 @@ +# Overlay Method Selection + +Weave automatically selects the best working overlay implementation +when connecting to a new peer. It does this by initiating both +forwarders in parallel, eventually settling on the most preferred +established forwarder at which point the other forwarder is shut down, +assuming it had not failed already. In the event that the remaining +forwarder fails at a later time (e.g. because of a heartbeat timeout) +the control plane TCP connection will recycle allowing the selection +algorithm to run again. By default fast datapath is preferred over +Sleeve. + +# Local Bridging + +Unlike the weave bridge netdev used by Sleeve, the OVS datapath has no +inherent bridging capability. Consequently fastdp implements an +ethernet bridge in addition to the vxlan overlay, maintaining its own +forwarding database by learning MAC addresses and dispatching +broadcast, unicast & multicast traffic accordingly. + +# Short Peer IDs + +The router relies on having access to the source and destination peer +when deciding how to forward packets. When the Sleeve overlay is in +use this information is conveyed directly within the encapsulation by +including the names of the peers in question, a solution not +accomodated directly by the vxlan wire format. Fortunately vxlan does +have a twenty four bit segment ID field in the header we can use to +encode this data - the challenge is to identify peers uniquely with a +twelve bit identifier instead of the seventeen bytes used by Sleeve. +In practice this is achieved by peers adopting a random 'short ID' and +resolving ownership collisions of same via the existing gossip +mechanism. + +# Heartbeats via vxlan + +Heartbeats are implemented by sending an encapsulated ethernet frame +with source and destination MAC addresses of `00:00:00:00:00:00` and a +payload consisting of the connection UID and total ethernet frame +length. The vxlan vport miss handler on the receiving side detects the +all-zero addresses and acknowledges the heartbeat via the TCP control +channel after validating the connection UID and frame length against +their expected values. + +# PMTU Discovery + +The length of the heartbeat ethernet frame is set to the MTU of the +datapath interface. In the event the vxlan packet is dropped or +truncated, the heartbeat will not be acknowledged; this lack of +acknowledgement will cause the peers to fall back to the Sleeve +overlay, which has a more sophisticated dynamic mechanism for coping +with low path MTUs. + +To avoid triggering this fallback in typical deployments, the datapath +interface is statically configured with an MTU of 1410 bytes allowing +it to work with most underlay network provider MTUs, including GCE at +1460 bytes (the fifty byte difference accomodates the vxlan overhead). +This value can be overriden by setting `WEAVE_MTU` at launch if +necessary. + +# Virtual Ports + +There are three kinds of virtual port associated with the weave +datapath: + +* `internal` - one of these is created automatically, named after the + datapath. It corresponds to the network interface of the same name + that appears on the host when the datapath is created; it is the + ingress/egress port for `weave expose` +* `netdev` - one for each application container, corresponding to the + host end of the veth pair +* `vxlan` - one for each UDP port on which we are listening for vxlan + encapsulated packets. Typically there is only one of these, but + there may be more in a network in which peers do not all use the same + port configuration + +# Misses and Flow Creation + +As mentioned above, the datapath has no inherent behaviour - any +ingressing packet which does not match a flow is passed to a userspace +miss handler. The miss handler is then responsible for a) instructing +the datapath to take actions for the packet that caused the miss (for +example by copying it to one or more ports) and optionally b) +installing flow rules which will allow the datapath to act on similar +packets in future without invoking the miss handler. + +At a high level the fast datapath implementation can be viewed as a +set of miss handlers that determine what actions the OVS datapath +should take based on router topology information, together with some +additional machinery that manages the expiry of resulting flows when +that state changes. + +When a miss handler is invoked, it has two pieces of context: a byte +array containing the packet that triggered the miss, and a set of +'flow keys' that have been extracted from the packet by the kernel +module. The following flow keys are of interest to weave: + +* `InPort` - the identifier of the ingress virtual port +* `Ethernet` - the source and destination MAC addresses of the + ethernet frame +* `Tunnel` - tunnel identifier (see section on short peer IDs above), + and source/dest IPv4 addresses. Only present for ingress via a vxlan + vport + +Crucially, the router must use this information alone to determine +which actions to take - this allows the specification of a flow which +matches these keys and instructs the kernel to take action +automatically in future without further userspace involvement. + +Two actions are of interest: + +* `Output` - output the packet to the specified vport +* `SetTunnel` - update the effective vxlan tunnel parameters. Only + required prior to output from a vxlan vport + +It is possible to have multiple actions in a flow, so the router can +for example create a single rule that copies broadcast traffic to all +'local' vports (save the ingress vport obviously) as well as to vxlan +vports for onward peers as dictated by the routing topology. + +Every flow specified by the router has the following characteristics: + +* An `InPort` key matching the ingress vport +* A `Tunnel` key if the ingress vport is of type vxlan +* An `Ethernet` key matching the source and destination MAC +* A list of `Output` (for internal and netdev vports) and + `SetTunnel`+`Output` (for vxlan vports) actions + +Under certain circumstances the miss handler will instruct the +datapath to execute a set of actions for a packet without creating a +corresponding flow: + +* When broadcasting packets to discover the path to unknown + destination MACs. This condition is transient in the overwhelming + majority of cases and so the benefit of creating a flow is + outweighed by the need to invalidate it once the MAC is learned +* When the packet also needs to be forwarded to one or more peers via + Sleeve. In this case the router needs to handle all subsequent + matching packets as there is no OVS flow action to do it without + userspace involvement + +# Flow Invalidation + +Once a flow has been created for a particular combination of keys, the +miss handler will never again be invoked for matching packets. It is +therefore extremely important that we detect events which invalidate +existing flow actions: + +* Addition of netdev vports to the datapath +* Route invalidation (topology change) +* Short peer ID collision + +The response in each case is the same - delete all flows from the +datapath, allowing them to be recreated taking into account the +updated state. + +In addition to these event based invalidations there is an expiry +process that executes every five minutes. This process enumerates all +flows in the datapath, removing any which have not been used since the +last check; this cleans up: + +* Flows referring to netdev vports which have been removed +* Flows created by forwarders which have been stopped +* Flows related to MAC addresses which have not communicated recently + +Flows keyed on tunnel IPv4 address do not need to be cleared when a +peer appears to change IP address due to NAT; this will cause a miss +resulting in a new flow, and the old flow will expire naturally via +the timer mechanism. + diff --git a/ipam/allocate.go b/ipam/allocate.go index e8950e37..d6dd2f10 100644 --- a/ipam/allocate.go +++ b/ipam/allocate.go @@ -38,6 +38,11 @@ func (g *allocate) Try(alloc *Allocator) bool { alloc.establishRing() if ok, addr := alloc.space.Allocate(g.r); ok { + // If caller hasn't supplied a unique ID, file it under the IP address + // which lets the caller then release the address using DELETE /ip/address + if g.ident == "_" { + g.ident = addr.String() + } alloc.debugln("Allocated", addr, "for", g.ident, "in", g.r) alloc.addOwned(g.ident, addr) g.resultChan <- allocateResult{addr, nil} diff --git a/ipam/allocator.go b/ipam/allocator.go index 888b9433..a98697c8 100644 --- a/ipam/allocator.go +++ b/ipam/allocator.go @@ -478,8 +478,8 @@ type ipamGossipData struct { alloc *Allocator } -func (d *ipamGossipData) Merge(other mesh.GossipData) { - // no-op +func (d *ipamGossipData) Merge(other mesh.GossipData) mesh.GossipData { + return d // no-op } func (d *ipamGossipData) Encode() [][]byte { diff --git a/ipam/http.go b/ipam/http.go index a86fbc50..eb51b87a 100644 --- a/ipam/http.go +++ b/ipam/http.go @@ -57,6 +57,10 @@ func (alloc *Allocator) handleHTTPAllocate(dockerCli *docker.Client, w http.Resp // HandleHTTP wires up ipams HTTP endpoints to the provided mux. func (alloc *Allocator) HandleHTTP(router *mux.Router, defaultSubnet address.CIDR, dockerCli *docker.Client) { + router.Methods("GET").Path("/ipinfo/defaultsubnet").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "%s", defaultSubnet) + }) + router.Methods("PUT").Path("/ip/{id}/{ip}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) ident := vars["id"] diff --git a/ipam/http_test.go b/ipam/http_test.go index 36183b56..77777b1b 100644 --- a/ipam/http_test.go +++ b/ipam/http_test.go @@ -14,7 +14,6 @@ import ( "github.com/stretchr/testify/require" "github.com/weaveworks/weave/common" "github.com/weaveworks/weave/net/address" - wt "github.com/weaveworks/weave/testing" ) func HTTPPost(t *testing.T, url string) string { @@ -145,12 +144,6 @@ func TestBadHttp(t *testing.T) { } func TestHTTPCancel(t *testing.T) { - wt.RunWithTimeout(t, 2*time.Second, func() { - impTestHTTPCancel(t) - }) -} - -func impTestHTTPCancel(t *testing.T) { var ( containerID = "deadbeef" testCIDR1 = "10.0.3.0/29" diff --git a/ipam/ring/ring.go b/ipam/ring/ring.go index 30f32b8d..2e384eca 100644 --- a/ipam/ring/ring.go +++ b/ipam/ring/ring.go @@ -316,6 +316,25 @@ func (r *Ring) OwnedRanges() (result []address.Range) { return r.splitRangesOverZero(result) } +// For printing status +type RangeInfo struct { + Peer mesh.PeerName + address.Range + Version uint32 +} + +func (r *Ring) AllRangeInfo() (result []RangeInfo) { + for i, entry := range r.Entries { + nextEntry := r.Entries.entry(i + 1) + ranges := []address.Range{{Start: entry.Token, End: nextEntry.Token}} + ranges = r.splitRangesOverZero(ranges) + for _, r := range ranges { + result = append(result, RangeInfo{entry.Peer, r, entry.Version}) + } + } + return +} + // ClaimForPeers claims the entire ring for the array of peers passed // in. Only works for empty rings. func (r *Ring) ClaimForPeers(peers []mesh.PeerName) { diff --git a/ipam/status.go b/ipam/status.go index 72df8d3e..e7a252ba 100644 --- a/ipam/status.go +++ b/ipam/status.go @@ -1,6 +1,7 @@ package ipam import ( + "fmt" "github.com/weaveworks/weave/ipam/paxos" "github.com/weaveworks/weave/net/address" ) @@ -8,6 +9,7 @@ import ( type Status struct { Paxos *paxos.Status Range string + RangeNumIPs int DefaultSubnet string Entries []EntryStatus PendingClaims []ClaimStatus @@ -15,9 +17,12 @@ type Status struct { } type EntryStatus struct { - Token string - Peer string - Version uint32 + Token string + Size uint32 + Peer string + Nickname string + IsKnownPeer bool + Version uint32 } type ClaimStatus struct { @@ -40,6 +45,7 @@ func NewStatus(allocator *Allocator, defaultSubnet address.CIDR) *Status { resultChan <- &Status{ paxosStatus, allocator.universe.String(), + int(allocator.universe.Size()), defaultSubnet.String(), newEntryStatusSlice(allocator), newClaimStatusSlice(allocator), @@ -56,8 +62,15 @@ func newEntryStatusSlice(allocator *Allocator) []EntryStatus { return slice } - for _, entry := range allocator.ring.Entries { - slice = append(slice, EntryStatus{entry.Token.String(), entry.Peer.String(), entry.Version}) + for _, r := range allocator.ring.AllRangeInfo() { + slice = append(slice, EntryStatus{ + Token: r.Start.String(), + Size: uint32(r.Size()), + Peer: r.Peer.String(), + Nickname: allocator.nicknames[r.Peer], + IsKnownPeer: allocator.isKnownPeer(r.Peer), + Version: r.Version, + }) } return slice @@ -76,7 +89,7 @@ func newAllocateIdentSlice(allocator *Allocator) []string { var slice []string for _, op := range allocator.pendingAllocates { allocate := op.(*allocate) - slice = append(slice, allocate.ident) + slice = append(slice, fmt.Sprintf("%s %s", allocate.ident, allocate.r.String())) } return slice } diff --git a/mesh/connection.go b/mesh/connection.go index b19d2c2c..ae03017d 100644 --- a/mesh/connection.go +++ b/mesh/connection.go @@ -40,16 +40,19 @@ type RemoteConnection struct { type LocalConnection struct { sync.RWMutex RemoteConnection - TCPConn *net.TCPConn - version byte - tcpSender TCPSender - SessionKey *[32]byte - heartbeatTCP *time.Ticker - Router *Router - uid uint64 - actionChan chan<- ConnectionAction - finished <-chan struct{} // closed to signal that actorLoop has finished - OverlayConn OverlayConnection + TCPConn *net.TCPConn + TrustRemote bool // is remote on a trusted subnet? + TrustedByRemote bool // does remote trust us? + version byte + tcpSender TCPSender + SessionKey *[32]byte + heartbeatTCP *time.Ticker + Router *Router + uid uint64 + actionChan chan<- ConnectionAction + errorChan chan<- error + finished <-chan struct{} // closed to signal that actorLoop has finished + OverlayConn OverlayConnection } type ConnectionAction func() error @@ -86,17 +89,19 @@ func StartLocalConnection(connRemote *RemoteConnection, tcpConn *net.TCPConn, ro if connRemote.local != router.Ourself.Peer { log.Fatal("Attempt to create local connection from a peer which is not ourself") } - // NB, we're taking a copy of connRemote here. actionChan := make(chan ConnectionAction, ChannelSize) + errorChan := make(chan error, 1) finished := make(chan struct{}) conn := &LocalConnection{ - RemoteConnection: *connRemote, + RemoteConnection: *connRemote, // NB, we're taking a copy of connRemote here. Router: router, TCPConn: tcpConn, + TrustRemote: router.Trusts(connRemote), uid: randUint64(), actionChan: actionChan, + errorChan: errorChan, finished: finished} - go conn.run(actionChan, finished, acceptNewPeer) + go conn.run(actionChan, errorChan, finished, acceptNewPeer) } func (conn *LocalConnection) BreakTie(dupConn Connection) ConnectionTieBreak { @@ -116,11 +121,6 @@ func (conn *LocalConnection) Established() bool { return conn.established } -// Send directly, not via the Actor. If it goes via the Actor we can -// get a deadlock where LocalConnection is blocked talking to -// LocalPeer and LocalPeer is blocked trying send a ProtocolMsg via -// LocalConnection, and the channels are full in both directions so -// nothing can proceed. func (conn *LocalConnection) SendProtocolMsg(m ProtocolMsg) { if err := conn.sendProtocolMsg(m); err != nil { conn.Shutdown(err) @@ -134,15 +134,17 @@ func (conn *LocalConnection) SendProtocolMsg(m ProtocolMsg) { // do not need locks for reading, and only need write locks for fields // read by other processes. -// Async +// Non-blocking. func (conn *LocalConnection) Shutdown(err error) { // err should always be a real error, even if only io.EOF if err == nil { panic("nil error") } - // Run on its own goroutine in case the channel is backed up - go func() { conn.sendAction(func() error { return err }) }() + select { + case conn.errorChan <- err: + default: + } } // Send an actor request to the actorLoop, but don't block if @@ -157,7 +159,7 @@ func (conn *LocalConnection) sendAction(action ConnectionAction) { // ACTOR server -func (conn *LocalConnection) run(actionChan <-chan ConnectionAction, finished chan<- struct{}, acceptNewPeer bool) { +func (conn *LocalConnection) run(actionChan <-chan ConnectionAction, errorChan <-chan error, finished chan<- struct{}, acceptNewPeer bool) { var err error // important to use this var and not create another one with 'err :=' defer func() { conn.shutdown(err) }() defer close(finished) @@ -190,13 +192,19 @@ func (conn *LocalConnection) run(actionChan <-chan ConnectionAction, finished ch conn.Log("connection ready; using protocol version", conn.version) + // only use negotiated session key for untrusted connections + var sessionKey *[32]byte + if conn.Untrusted() { + sessionKey = conn.SessionKey + } + params := OverlayConnectionParams{ RemotePeer: conn.remote, LocalAddr: conn.TCPConn.LocalAddr().(*net.TCPAddr), RemoteAddr: conn.TCPConn.RemoteAddr().(*net.TCPAddr), Outbound: conn.outbound, ConnUID: conn.uid, - SessionKey: conn.SessionKey, + SessionKey: sessionKey, SendControlMessage: conn.sendOverlayControlMessage, Features: intro.Features, } @@ -243,7 +251,7 @@ func (conn *LocalConnection) run(actionChan <-chan ConnectionAction, finished ch // running AddConnection in a separate goroutine, at least not // without some synchronisation. Which in turn requires the // launching of the receiveTCP goroutine to precede actorLoop. - err = conn.actorLoop(actionChan) + err = conn.actorLoop(actionChan, errorChan) } func (conn *LocalConnection) makeFeatures() map[string]string { @@ -254,6 +262,7 @@ func (conn *LocalConnection) makeFeatures() map[string]string { "ShortID": fmt.Sprint(conn.local.ShortID), "UID": fmt.Sprint(conn.local.UID), "ConnID": fmt.Sprint(conn.uid), + "Trusted": fmt.Sprint(conn.TrustRemote), } conn.Router.Overlay.AddFeaturesTo(features) return features @@ -295,13 +304,21 @@ func (conn *LocalConnection) parseFeatures(features features) (*Peer, error) { var hasShortID bool if shortIDStr, present := features["ShortID"]; present { hasShortID = true - shortID, err = strconv.ParseUint(shortIDStr, 10, - PeerShortIDBits) + shortID, err = strconv.ParseUint(shortIDStr, 10, PeerShortIDBits) if err != nil { return nil, err } } + var trusted bool + if trustedStr, present := features["Trusted"]; present { + trusted, err = strconv.ParseBool(trustedStr) + if err != nil { + return nil, err + } + } + conn.TrustedByRemote = trusted + uid, err := ParsePeerUID(features.Get("UID")) if err != nil { return nil, err @@ -335,7 +352,7 @@ func (conn *LocalConnection) registerRemote(remote *Peer, acceptNewPeer bool) er return nil } -func (conn *LocalConnection) actorLoop(actionChan <-chan ConnectionAction) (err error) { +func (conn *LocalConnection) actorLoop(actionChan <-chan ConnectionAction, errorChan <-chan error) (err error) { fwdErrorChan := conn.OverlayConn.ErrorChannel() fwdEstablishedChan := conn.OverlayConn.EstablishedChannel() @@ -344,6 +361,8 @@ func (conn *LocalConnection) actorLoop(actionChan <-chan ConnectionAction) (err case action := <-actionChan: err = action() + case err = <-errorChan: + case <-conn.heartbeatTCP.C: err = conn.sendSimpleProtocolMsg(ProtocolHeartbeat) @@ -435,3 +454,7 @@ func (conn *LocalConnection) handleProtocolMsg(tag ProtocolTag, payload []byte) func (conn *LocalConnection) extendReadDeadline() { conn.TCPConn.SetReadDeadline(time.Now().Add(TCPHeartbeat * 2)) } + +func (conn *LocalConnection) Untrusted() bool { + return !conn.TrustRemote || !conn.TrustedByRemote +} diff --git a/mesh/connection_maker.go b/mesh/connection_maker.go index 78edaf58..d38f94c4 100644 --- a/mesh/connection_maker.go +++ b/mesh/connection_maker.go @@ -81,7 +81,7 @@ func (cm *ConnectionMaker) InitiateConnections(peers []string, replace bool) []e for peer, addr := range addrs { cm.directPeers[peer] = addr // curtail any existing reconnect interval - if target, found := cm.targets[addr.String()]; found { + if target, found := cm.targets[cm.completeAddr(*addr)]; found { target.nextTryNow() } } @@ -159,6 +159,13 @@ func (cm *ConnectionMaker) queryLoop(actionChan <-chan ConnectionMakerAction) { } } +func (cm *ConnectionMaker) completeAddr(addr net.TCPAddr) string { + if addr.Port == 0 { + addr.Port = cm.port + } + return addr.String() +} + func (cm *ConnectionMaker) checkStateAndAttemptConnections() time.Duration { var ( validTarget = make(map[string]struct{}) @@ -181,18 +188,16 @@ func (cm *ConnectionMaker) checkStateAndAttemptConnections() time.Duration { // Add direct targets that are not connected for _, addr := range cm.directPeers { - completeAddr := *addr attempt := true - if completeAddr.Port == 0 { - completeAddr.Port = cm.port + if addr.Port == 0 { // If a peer was specified w/o a port, then we do not // attempt to connect to it if we have any inbound // connections from that IP. - if _, connected := ourInboundIPs[completeAddr.IP.String()]; connected { + if _, connected := ourInboundIPs[addr.IP.String()]; connected { attempt = false } } - address := completeAddr.String() + address := cm.completeAddr(*addr) directTarget[address] = void if attempt { addTarget(address) diff --git a/mesh/gossip.go b/mesh/gossip.go index abfab8fa..2b3a4807 100644 --- a/mesh/gossip.go +++ b/mesh/gossip.go @@ -6,7 +6,7 @@ import ( type GossipData interface { Encode() [][]byte - Merge(GossipData) + Merge(GossipData) GossipData } type Gossip interface { @@ -79,8 +79,7 @@ func (sender *GossipSender) Send(data GossipData) { // NB: this must not be invoked concurrently select { case pending := <-sender.cell: - pending.Merge(data) - sender.cell <- pending + sender.cell <- pending.Merge(data) default: sender.cell <- data } @@ -142,7 +141,7 @@ func (router *Router) SendAllGossip() { func (router *Router) SendAllGossipDown(conn Connection) { for channel := range router.gossipChannelSet() { if gossip := channel.gossiper.Gossip(); gossip != nil { - channel.SendDown(conn, channel.gossiper.Gossip()) + channel.SendDown(conn, gossip) } } } diff --git a/mesh/gossip_channel.go b/mesh/gossip_channel.go index 098553e7..569ca48b 100644 --- a/mesh/gossip_channel.go +++ b/mesh/gossip_channel.go @@ -97,78 +97,6 @@ func (c *GossipChannel) deliver(srcName PeerName, _ []byte, dec *gob.Decoder) er return nil } -func (c *GossipChannel) Send(srcName PeerName, data GossipData) { - // do this outside the lock below so we avoid lock nesting - c.routes.EnsureRecalculated() - selectedConnections := make(ConnectionSet) - for name := range c.routes.RandomNeighbours(srcName) { - if conn, found := c.ourself.ConnectionTo(name); found { - selectedConnections[conn] = void - } - } - if len(selectedConnections) == 0 { - return - } - connections := c.ourself.Connections() - c.Lock() - defer c.Unlock() - c.gcSenders(connections) - for conn := range selectedConnections { - c.sendDown(conn, data) - } -} - -func (c *GossipChannel) SendDown(conn Connection, data GossipData) { - connections := c.ourself.Connections() - c.Lock() - defer c.Unlock() - c.gcSenders(connections) - c.sendDown(conn, data) -} - -// GC - randomly (courtesy of go's map iterator) pick some existing -// senders and stop&remove them if the associated connection is no -// longer active. We stop as soon as we encounter a valid entry; the -// idea being that when there is little or no garbage then this -// executes close to O(1)[1], whereas when there is lots of garbage we -// remove it quickly. -// -// [1] TODO Unfortunately, due to the desire to avoid nested locks, -// instead of simply invoking LocalPeer.ConnectionTo(name), we pass in -// the result of LocalPeer.Connections(). That is O(n_our_connections) -// at best. -func (c *GossipChannel) gcSenders(connections ConnectionSet) { - for conn, sender := range c.senders { - if _, found := connections[conn]; !found { - delete(c.senders, conn) - sender.Stop() - } else { - break - } - } -} - -// We have seen a couple of failures which suggest a >128GB slice was encountered. -// 100MB should be enough for anyone. -const maxFeasibleMessageLen = 100 * 1024 * 1024 - -func (c *GossipChannel) sendDown(conn Connection, data GossipData) { - sender, found := c.senders[conn] - if !found { - sender = NewGossipSender(func(pending GossipData) { - for _, msg := range pending.Encode() { - if len(msg) > maxFeasibleMessageLen { - panic(fmt.Sprintf("Gossip message too large: len=%d bytes; on channel '%s' from %+v", len(msg), c.name, pending)) - } - protocolMsg := ProtocolMsg{ProtocolGossip, GobEncode(c.name, c.ourself.Name, msg)} - conn.(ProtocolSender).SendProtocolMsg(protocolMsg) - } - }) - c.senders[conn] = sender - } - sender.Send(data) -} - func (c *GossipChannel) GossipUnicast(dstPeerName PeerName, msg []byte) error { return c.relayUnicast(dstPeerName, GobEncode(c.name, c.ourself.Name, dstPeerName, msg)) } @@ -236,6 +164,72 @@ func (c *GossipChannel) sendBroadcast(srcName PeerName, update GossipData) { } } +func (c *GossipChannel) Send(srcName PeerName, data GossipData) { + c.routes.EnsureRecalculated() + destinations := c.routes.RandomNeighbours(srcName) + if len(destinations) == 0 { + return + } + c.sendDown(c.ourself.ConnectionsTo(destinations), data) +} + +func (c *GossipChannel) SendDown(conn Connection, data GossipData) { + c.sendDown([]Connection{conn}, data) +} + +func (c *GossipChannel) sendDown(selectedConnections []Connection, data GossipData) { + if len(selectedConnections) == 0 { + return + } + connections := c.ourself.Connections() + c.Lock() + defer c.Unlock() + // GC - randomly (courtesy of go's map iterator) pick some + // existing senders and stop&remove them if the associated + // connection is no longer active. We stop as soon as we + // encounter a valid entry; the idea being that when there is + // little or no garbage then this executes close to O(1)[1], + // whereas when there is lots of garbage we remove it quickly. + // + // [1] TODO Unfortunately, due to the desire to avoid nested + // locks, instead of simply invoking LocalPeer.ConnectionTo(name), + // we operate on LocalPeer.Connections(). That is + // O(n_our_connections) at best. + for conn, sender := range c.senders { + if _, found := connections[conn]; !found { + delete(c.senders, conn) + sender.Stop() + } else { + break + } + } + // start senders, if necessary, and send. + for _, conn := range selectedConnections { + sender, found := c.senders[conn] + if !found { + sender = c.makeSender(conn) + c.senders[conn] = sender + } + sender.Send(data) + } +} + +// We have seen a couple of failures which suggest a >128GB slice was encountered. +// 100MB should be enough for anyone. +const maxFeasibleMessageLen = 100 * 1024 * 1024 + +func (c *GossipChannel) makeSender(conn Connection) *GossipSender { + return NewGossipSender(func(pending GossipData) { + for _, msg := range pending.Encode() { + if len(msg) > maxFeasibleMessageLen { + panic(fmt.Sprintf("Gossip message too large: len=%d bytes; on channel '%s' from %+v", len(msg), c.name, pending)) + } + protocolMsg := ProtocolMsg{ProtocolGossip, GobEncode(c.name, c.ourself.Name, msg)} + conn.(ProtocolSender).SendProtocolMsg(protocolMsg) + } + }) +} + func (c *GossipChannel) log(args ...interface{}) { log.Println(append(append([]interface{}{}, "[gossip "+c.name+"]:"), args...)...) } diff --git a/mesh/gossip_test.go b/mesh/gossip_test.go index 5f622826..d5e15afc 100644 --- a/mesh/gossip_test.go +++ b/mesh/gossip_test.go @@ -4,18 +4,17 @@ import ( "fmt" "sync" "testing" - "time" "github.com/stretchr/testify/require" - wt "github.com/weaveworks/weave/testing" ) // TODO test gossip unicast; atm we only test topology gossip and // surrogates, neither of which employ unicast. -type mockChannelConnection struct { +type mockGossipConnection struct { RemoteConnection - dest *Router + dest *Router + start chan struct{} } func NewTestRouter(name string) *Router { @@ -25,12 +24,17 @@ func NewTestRouter(name string) *Router { return router } -func (conn *mockChannelConnection) SendProtocolMsg(protocolMsg ProtocolMsg) { +func (conn *mockGossipConnection) SendProtocolMsg(protocolMsg ProtocolMsg) { + <-conn.start if err := conn.dest.handleGossip(protocolMsg.tag, protocolMsg.msg); err != nil { panic(err) } } +func (conn *mockGossipConnection) Start() { + close(conn.start) +} + func sendPendingGossip(routers ...*Router) { // Loop until all routers report they didn't send anything for sentSomething := true; sentSomething; { @@ -41,38 +45,32 @@ func sendPendingGossip(routers ...*Router) { } } -func (router *Router) AddTestChannelConnection(r *Router) { - fromPeer := NewPeerFrom(router.Ourself.Peer) - toPeer := NewPeerFrom(r.Ourself.Peer) +func AddTestGossipConnection(r1, r2 *Router) { + c1 := r1.NewTestGossipConnection(r2) + c2 := r2.NewTestGossipConnection(r1) + c1.Start() + c2.Start() +} - r.Peers.FetchWithDefault(fromPeer) // Has side-effect of incrementing refcount - router.Peers.FetchWithDefault(toPeer) // +func (router *Router) NewTestGossipConnection(r *Router) *mockGossipConnection { + to := r.Ourself.Peer + toPeer := NewPeer(to.Name, to.NickName, to.UID, 0, to.ShortID) + toPeer = router.Peers.FetchWithDefault(toPeer) // Has side-effect of incrementing refcount - conn := &mockChannelConnection{RemoteConnection{router.Ourself.Peer, toPeer, "", false, true}, r} + conn := &mockGossipConnection{ + RemoteConnection{router.Ourself.Peer, toPeer, "", false, true}, r, make(chan struct{})} router.Ourself.handleAddConnection(conn) router.Ourself.handleConnectionEstablished(conn) + return conn } -func (router *Router) DeleteTestChannelConnection(r *Router) { - fromName := router.Ourself.Peer.Name +func (router *Router) DeleteTestGossipConnection(r *Router) { toName := r.Ourself.Peer.Name - - fromPeer := r.Peers.Fetch(fromName) - toPeer := router.Peers.Fetch(toName) - - r.Peers.Dereference(fromPeer) - router.Peers.Dereference(toPeer) - conn, _ := router.Ourself.ConnectionTo(toName) + router.Peers.Dereference(conn.Remote()) router.Ourself.handleDeleteConnection(conn) } -func TestGossipTopology(t *testing.T) { - wt.RunWithTimeout(t, 5*time.Second, func() { - implTestGossipTopology(t) - }) -} - // Create a Peer representing the receiver router, with connections to // the routers supplied as arguments, carrying across all UID and // version information. @@ -95,69 +93,46 @@ func checkTopology(t *testing.T, router *Router, wantedPeers ...*Peer) { router.Peers.RUnlock() } -func implTestGossipTopology(t *testing.T) { +func flushAndCheckTopology(t *testing.T, routers []*Router, wantedPeers ...*Peer) { + sendPendingGossip(routers...) + for _, r := range routers { + checkTopology(t, r, wantedPeers...) + } +} + +func TestGossipTopology(t *testing.T) { // Create some peers that will talk to each other r1 := NewTestRouter("01:00:00:01:00:00") r2 := NewTestRouter("02:00:00:02:00:00") r3 := NewTestRouter("03:00:00:03:00:00") - + routers := []*Router{r1, r2, r3} // Check state when they have no connections checkTopology(t, r1, r1.tp()) checkTopology(t, r2, r2.tp()) // Now try adding some connections - r1.AddTestChannelConnection(r2) - sendPendingGossip(r1, r2) - checkTopology(t, r1, r1.tp(r2), r2.tp()) - checkTopology(t, r2, r1.tp(r2), r2.tp()) - r2.AddTestChannelConnection(r1) + AddTestGossipConnection(r1, r2) sendPendingGossip(r1, r2) checkTopology(t, r1, r1.tp(r2), r2.tp(r1)) checkTopology(t, r2, r1.tp(r2), r2.tp(r1)) - // Currently, the connection from 2 to 3 is one-way only - r2.AddTestChannelConnection(r3) - sendPendingGossip(r1, r2, r3) - checkTopology(t, r1, r1.tp(r2), r2.tp(r1, r3), r3.tp()) - checkTopology(t, r2, r1.tp(r2), r2.tp(r1, r3), r3.tp()) - // When r2 gossiped to r3, 1 was unreachable from r3 so it got removed from the - // list of peers, but remains referenced in the connection from 1 to 3. - checkTopology(t, r3, r2.tp(r1, r3), r3.tp()) + AddTestGossipConnection(r2, r3) + flushAndCheckTopology(t, routers, r1.tp(r2), r2.tp(r1, r3), r3.tp(r2)) - // Add a connection from 3 to 1 and now r1 is reachable. - r3.AddTestChannelConnection(r1) - sendPendingGossip(r1, r2, r3) - checkTopology(t, r1, r1.tp(r2), r2.tp(r1, r3), r3.tp(r1)) - checkTopology(t, r2, r1.tp(r2), r2.tp(r1, r3), r3.tp(r1)) - checkTopology(t, r3, r1.tp(), r2.tp(r1, r3), r3.tp(r1)) - - r1.AddTestChannelConnection(r3) - sendPendingGossip(r1, r2, r3) - checkTopology(t, r1, r1.tp(r2, r3), r2.tp(r1, r3), r3.tp(r1)) - checkTopology(t, r2, r1.tp(r2, r3), r2.tp(r1, r3), r3.tp(r1)) - checkTopology(t, r3, r1.tp(r2, r3), r2.tp(r1, r3), r3.tp(r1)) + AddTestGossipConnection(r3, r1) + flushAndCheckTopology(t, routers, r1.tp(r2, r3), r2.tp(r1, r3), r3.tp(r1, r2)) // Drop the connection from 2 to 3 - r2.DeleteTestChannelConnection(r3) - sendPendingGossip(r1, r2, r3) - checkTopology(t, r1, r1.tp(r2, r3), r2.tp(r1), r3.tp(r1)) - checkTopology(t, r2, r1.tp(r2, r3), r2.tp(r1)) - checkTopology(t, r3, r1.tp(r2, r3), r2.tp(r1), r3.tp(r1)) + r2.DeleteTestGossipConnection(r3) + flushAndCheckTopology(t, routers, r1.tp(r2, r3), r2.tp(r1), r3.tp(r1, r2)) // Drop the connection from 1 to 3 - r1.DeleteTestChannelConnection(r3) + r1.DeleteTestGossipConnection(r3) sendPendingGossip(r1, r2, r3) - checkTopology(t, r1, r1.tp(r2), r2.tp(r1), r3.tp(r1)) - - checkTopology(t, r1, r1.tp(r2), r2.tp(r1), r3.tp(r1)) + checkTopology(t, r1, r1.tp(r2), r2.tp(r1)) checkTopology(t, r2, r1.tp(r2), r2.tp(r1)) // r3 still thinks r1 has a connection to it - checkTopology(t, r3, r1.tp(r2, r3), r2.tp(r1), r3.tp(r1)) - - // On a timer, r3 will gossip to r1 - r3.SendAllGossip() - sendPendingGossip(r1, r2, r3) - checkTopology(t, r1, r1.tp(r2), r2.tp(r1), r3.tp(r1)) + checkTopology(t, r3, r1.tp(r2, r3), r2.tp(r1), r3.tp(r1, r2)) } func TestGossipSurrogate(t *testing.T) { @@ -165,14 +140,10 @@ func TestGossipSurrogate(t *testing.T) { r1 := NewTestRouter("01:00:00:01:00:00") r2 := NewTestRouter("02:00:00:02:00:00") r3 := NewTestRouter("03:00:00:03:00:00") - r1.AddTestChannelConnection(r2) - r2.AddTestChannelConnection(r1) - r3.AddTestChannelConnection(r2) - r2.AddTestChannelConnection(r3) - sendPendingGossip(r1, r2, r3) - checkTopology(t, r1, r1.tp(r2), r2.tp(r1, r3), r3.tp(r2)) - checkTopology(t, r2, r1.tp(r2), r2.tp(r1, r3), r3.tp(r2)) - checkTopology(t, r3, r1.tp(r2), r2.tp(r1, r3), r3.tp(r2)) + routers := []*Router{r1, r2, r3} + AddTestGossipConnection(r1, r2) + AddTestGossipConnection(r3, r2) + flushAndCheckTopology(t, routers, r1.tp(r2), r2.tp(r1, r3), r3.tp(r2)) // create a gossiper at either end, but not the middle g1 := newTestGossiper() @@ -238,10 +209,10 @@ func (g *testGossiper) OnGossip(update []byte) (GossipData, error) { g.state[v] = void } } - if len(delta) > 0 { - return NewSurrogateGossipData(delta), nil + if len(delta) == 0 { + return nil, nil } - return nil, nil + return NewSurrogateGossipData(delta), nil } func (g *testGossiper) checkHas(t *testing.T, vs ...byte) { diff --git a/mesh/mocks_test.go b/mesh/mocks_test.go index 08ae90aa..12ae0d74 100644 --- a/mesh/mocks_test.go +++ b/mesh/mocks_test.go @@ -17,7 +17,7 @@ func (peers *Peers) AddTestConnection(p *Peer) { summary := p.PeerSummary summary.Version = 0 toPeer := NewPeerFromSummary(summary) - peers.FetchWithDefault(toPeer) // Has side-effect of incrementing refcount + toPeer = peers.FetchWithDefault(toPeer) // Has side-effect of incrementing refcount conn := newMockConnection(peers.ourself.Peer, toPeer) peers.ourself.addConnection(conn) peers.ourself.connectionEstablished(conn) diff --git a/mesh/peer.go b/mesh/peer.go index 1846f352..44d14893 100644 --- a/mesh/peer.go +++ b/mesh/peer.go @@ -26,7 +26,12 @@ func randUint16() (r uint16) { type PeerUID uint64 func randomPeerUID() PeerUID { - return PeerUID(randUint64()) + for { + uid := randUint64() + if uid != 0 { // uid 0 is reserved for peer placeholder + return PeerUID(uid) + } + } } func ParsePeerUID(s string) (PeerUID, error) { @@ -95,6 +100,10 @@ func NewPeer(name PeerName, nickName string, uid PeerUID, version uint64, shortI }) } +func NewPeerPlaceholder(name PeerName) *Peer { + return NewPeerFromSummary(PeerSummary{NameByte: name.Bin()}) +} + func NewPeerFrom(peer *Peer) *Peer { return NewPeerFromSummary(peer.PeerSummary) } diff --git a/mesh/peers.go b/mesh/peers.go index 70ec4c8d..8fc0761b 100644 --- a/mesh/peers.go +++ b/mesh/peers.go @@ -3,7 +3,6 @@ package mesh import ( "bytes" "encoding/gob" - "fmt" "io" "math/rand" "sync" @@ -32,14 +31,6 @@ type ShortIDPeers struct { others []*Peer } -type UnknownPeerError struct { - Name PeerName -} - -func (upe UnknownPeerError) Error() string { - return fmt.Sprint("Reference to unknown peer ", upe.Name) -} - type PeerNameSet map[PeerName]struct{} type ConnectionSummary struct { @@ -334,9 +325,7 @@ func (peers *Peers) ApplyUpdate(update []byte) (PeerNameSet, PeerNameSet, error) return nil, nil, err } - // By this point, we know the update doesn't refer to any peers we - // have no knowledge of. We can now apply the update. Start by - // adding in any new peers. + // Add new peers for name, newPeer := range newPeers { peers.byName[name] = newPeer peers.addByShortID(newPeer, &pending) @@ -446,10 +435,8 @@ func (peers *Peers) decodeUpdate(update []byte) (newPeers map[PeerName]*Peer, de if _, found := peers.byName[remoteName]; found { continue } - // Update refers to a peer which we have no knowledge - // of. Thus we can't apply the update. Abort. - err = UnknownPeerError{remoteName} - return + // Update refers to a peer which we have no knowledge of. + newPeers[remoteName] = NewPeerPlaceholder(remoteName) } } return @@ -476,16 +463,21 @@ func (peers *Peers) applyUpdate(decodedUpdate []*Peer, decodedConns [][]Connecti newUpdate[name] = void default: // existing peer if newPeer.Version < peer.Version || - (newPeer.Version == peer.Version && newPeer.UID <= peer.UID) { + (newPeer.Version == peer.Version && + (newPeer.UID < peer.UID || + (newPeer.UID == peer.UID && + (!newPeer.HasShortID || peer.HasShortID)))) { continue } peer.Version = newPeer.Version peer.UID = newPeer.UID + peer.NickName = newPeer.NickName peer.connections = makeConnsMap(peer, connSummaries, peers.byName) - if newPeer.ShortID != peer.ShortID { + if newPeer.ShortID != peer.ShortID || newPeer.HasShortID != peer.HasShortID { peers.deleteByShortID(peer, pending) peer.ShortID = newPeer.ShortID + peer.HasShortID = newPeer.HasShortID peers.addByShortID(peer, pending) } newUpdate[name] = void diff --git a/mesh/router.go b/mesh/router.go index 916ea5b7..598e485d 100644 --- a/mesh/router.go +++ b/mesh/router.go @@ -37,6 +37,7 @@ type Config struct { Password []byte ConnLimit int PeerDiscovery bool + TrustedSubnets []*net.IPNet } type Router struct { @@ -122,26 +123,28 @@ type TopologyGossipData struct { update PeerNameSet } +func (d *TopologyGossipData) Merge(other GossipData) GossipData { + names := make(PeerNameSet) + for name := range d.update { + names[name] = void + } + for name := range other.(*TopologyGossipData).update { + names[name] = void + } + return &TopologyGossipData{peers: d.peers, update: names} +} + +func (d *TopologyGossipData) Encode() [][]byte { + return [][]byte{d.peers.EncodePeers(d.update)} +} + func (router *Router) BroadcastTopologyUpdate(update []*Peer) { names := make(PeerNameSet) for _, p := range update { names[p.Name] = void } - - router.TopologyGossip.GossipBroadcast(&TopologyGossipData{ - peers: router.Peers, - update: names, - }) -} - -func (d *TopologyGossipData) Merge(other GossipData) { - for name := range other.(*TopologyGossipData).update { - d.update[name] = void - } -} - -func (d *TopologyGossipData) Encode() [][]byte { - return [][]byte{d.peers.EncodePeers(d.update)} + router.TopologyGossip.GossipBroadcast( + &TopologyGossipData{peers: router.Peers, update: names}) } func (router *Router) OnGossipUnicast(sender PeerName, msg []byte) error { @@ -170,14 +173,6 @@ func (router *Router) OnGossip(update []byte) (GossipData, error) { func (router *Router) applyTopologyUpdate(update []byte) (PeerNameSet, PeerNameSet, error) { origUpdate, newUpdate, err := router.Peers.ApplyUpdate(update) - if _, ok := err.(UnknownPeerError); err != nil && ok { - // That update contained a reference to a peer which wasn't - // itself included in the update, and we didn't know about - // already. We ignore this; eventually we should receive an - // update containing a complete topology. - log.Println("Topology gossip:", err) - return nil, nil, nil - } if err != nil { return nil, nil, err } @@ -187,3 +182,17 @@ func (router *Router) applyTopologyUpdate(update []byte) (PeerNameSet, PeerNameS } return origUpdate, newUpdate, nil } + +func (router *Router) Trusts(remote *RemoteConnection) bool { + if tcpAddr, err := net.ResolveTCPAddr("tcp4", remote.remoteTCPAddr); err == nil { + for _, trustedSubnet := range router.TrustedSubnets { + if trustedSubnet.Contains(tcpAddr.IP) { + return true + } + } + } else { + // Should not happen as remoteTCPAddr was obtained from TCPConn + log.Errorf("Unable to parse remote TCP addr: %s", err) + } + return false +} diff --git a/mesh/routes.go b/mesh/routes.go index d7787af9..675f1550 100644 --- a/mesh/routes.go +++ b/mesh/routes.go @@ -92,20 +92,24 @@ func (routes *Routes) BroadcastAll(name PeerName) []PeerName { // returns a higher proportion of neighbours than elsewhere. In // extremis, on peers with fewer than log2(n_peers) neighbours, all // neighbours are returned. -func (routes *Routes) RandomNeighbours(except PeerName) PeerNameSet { - res := make(PeerNameSet) +func (routes *Routes) RandomNeighbours(except PeerName) []PeerName { + destinations := make(PeerNameSet) routes.RLock() defer routes.RUnlock() count := int(math.Log2(float64(len(routes.unicastAll)))) // depends on go's random map iteration for _, dst := range routes.unicastAll { if dst != UnknownPeerName && dst != except { - res[dst] = void - if len(res) >= count { + destinations[dst] = void + if len(destinations) >= count { break } } } + res := make([]PeerName, 0, len(destinations)) + for dst := range destinations { + res = append(res, dst) + } return res } diff --git a/mesh/status.go b/mesh/status.go index 0dce08bb..559dae66 100644 --- a/mesh/status.go +++ b/mesh/status.go @@ -2,6 +2,7 @@ package mesh import ( "fmt" + "net" ) type Status struct { @@ -19,6 +20,7 @@ type Status struct { Connections []LocalConnectionStatus Targets []string OverlayDiagnostics interface{} + TrustedSubnets []string } type PeerStatus struct { @@ -69,7 +71,8 @@ func NewStatus(router *Router) *Status { NewBroadcastRouteStatusSlice(router.Routes), NewLocalConnectionStatusSlice(router.ConnectionMaker), NewTargetSlice(router.ConnectionMaker), - router.Overlay.Diagnostics()} + router.Overlay.Diagnostics(), + NewTrustedSubnetsSlice(router.TrustedSubnets)} } func NewPeerStatusSlice(peers *Peers) []PeerStatus { @@ -147,6 +150,13 @@ func NewLocalConnectionStatusSlice(cm *ConnectionMaker) []LocalConnectionStatus } lc, _ := conn.(*LocalConnection) info := fmt.Sprintf("%-6v %v", lc.OverlayConn.DisplayName(), conn.Remote()) + if lc.Router.UsingPassword() { + if lc.Untrusted() { + info = fmt.Sprintf("%-11v %v", "encrypted", info) + } else { + info = fmt.Sprintf("%-11v %v", "unencrypted", info) + } + } slice = append(slice, LocalConnectionStatus{conn.RemoteTCPAddr(), conn.Outbound(), state, info}) } for address, target := range cm.targets { @@ -191,3 +201,11 @@ func NewTargetSlice(cm *ConnectionMaker) []string { } return <-resultChan } + +func NewTrustedSubnetsSlice(trustedSubnets []*net.IPNet) []string { + trustedSubnetStrs := []string{} + for _, trustedSubnet := range trustedSubnets { + trustedSubnetStrs = append(trustedSubnetStrs, trustedSubnet.String()) + } + return trustedSubnetStrs +} diff --git a/mesh/surrogate_gossiper.go b/mesh/surrogate_gossiper.go index aef844a8..7204d4aa 100644 --- a/mesh/surrogate_gossiper.go +++ b/mesh/surrogate_gossiper.go @@ -12,8 +12,12 @@ func (d *SurrogateGossipData) Encode() [][]byte { return d.messages } -func (d *SurrogateGossipData) Merge(other GossipData) { - d.messages = append(d.messages, other.(*SurrogateGossipData).messages...) +func (d *SurrogateGossipData) Merge(other GossipData) GossipData { + o := other.(*SurrogateGossipData) + messages := make([][]byte, 0, len(d.messages)+len(o.messages)) + messages = append(messages, d.messages...) + messages = append(messages, o.messages...) + return &SurrogateGossipData{messages: messages} } // SurrogateGossiper ignores unicasts and relays broadcasts and gossips. diff --git a/nameserver/entry.go b/nameserver/entry.go index 524845a3..cb0830fa 100644 --- a/nameserver/entry.go +++ b/nameserver/entry.go @@ -245,12 +245,14 @@ type GossipData struct { Entries } -func (g *GossipData) Merge(o mesh.GossipData) { +func (g *GossipData) Merge(o mesh.GossipData) mesh.GossipData { other := o.(*GossipData) - g.Entries.merge(other.Entries) - if g.Timestamp < other.Timestamp { - g.Timestamp = other.Timestamp + gossip := g.copy() + gossip.Entries.merge(other.Entries) + if gossip.Timestamp < other.Timestamp { + gossip.Timestamp = other.Timestamp } + return gossip } func (g *GossipData) Decode(msg []byte) error { @@ -264,9 +266,7 @@ func (g *GossipData) Decode(msg []byte) error { } func (g *GossipData) Encode() [][]byte { - // Make a copy so we can sort: all outgoing data is sent in case-sensitive order - g2 := GossipData{Timestamp: g.Timestamp, Entries: make(Entries, len(g.Entries))} - copy(g2.Entries, g.Entries) + g2 := g.copy() sort.Sort(CaseSensitive(g2.Entries)) buf := &bytes.Buffer{} if err := gob.NewEncoder(buf).Encode(g2); err != nil { @@ -274,3 +274,9 @@ func (g *GossipData) Encode() [][]byte { } return [][]byte{buf.Bytes()} } + +func (g *GossipData) copy() *GossipData { + g2 := &GossipData{Timestamp: g.Timestamp, Entries: make(Entries, len(g.Entries))} + copy(g2.Entries, g.Entries) + return g2 +} diff --git a/nameserver/entry_test.go b/nameserver/entry_test.go index 1e794660..57dfde02 100644 --- a/nameserver/entry_test.go +++ b/nameserver/entry_test.go @@ -117,7 +117,7 @@ func TestGossipDataMerge(t *testing.T) { g1 := GossipData{Entries: makeEntries("AcDf")} g2 := GossipData{Entries: makeEntries("BEf")} - g1.Merge(&g2) + g3 := g1.Merge(&g2).(*GossipData) - require.Equal(t, GossipData{Entries: makeEntries("ABcDEf")}, g1) + require.Equal(t, GossipData{Entries: makeEntries("ABcDEf")}, *g3) } diff --git a/nameserver/nameserver.go b/nameserver/nameserver.go index 0d33d184..201bda3b 100644 --- a/nameserver/nameserver.go +++ b/nameserver/nameserver.go @@ -74,13 +74,13 @@ func (n *Nameserver) Stop() { } func (n *Nameserver) broadcastEntries(es ...Entry) error { - if n.gossip != nil { - return n.gossip.GossipBroadcast(&GossipData{ - Entries: Entries(es), - Timestamp: now(), - }) + if n.gossip == nil || len(es) == 0 { + return nil } - return nil + return n.gossip.GossipBroadcast(&GossipData{ + Entries: Entries(es), + Timestamp: now(), + }) } func (n *Nameserver) AddEntry(hostname, containerid string, origin mesh.PeerName, addr address.Address) error { @@ -133,10 +133,8 @@ func (n *Nameserver) ContainerDied(ident string) { return false }) n.Unlock() - if len(entries) > 0 { - if err := n.broadcastEntries(entries...); err != nil { - n.errorf("failed to broadcast container %s death: %v", ident, err) - } + if err := n.broadcastEntries(entries...); err != nil { + n.errorf("failed to broadcast container %s death: %v", ident, err) } } diff --git a/nameserver/nameserver_test.go b/nameserver/nameserver_test.go index f7b603b5..f30e53a4 100644 --- a/nameserver/nameserver_test.go +++ b/nameserver/nameserver_test.go @@ -13,7 +13,6 @@ import ( "github.com/weaveworks/weave/mesh" "github.com/weaveworks/weave/net/address" - wt "github.com/weaveworks/weave/testing" "github.com/weaveworks/weave/testing/gossip" ) @@ -76,12 +75,6 @@ func (m mapping) Addrs() []address.Address { } func TestNameservers(t *testing.T) { - wt.RunWithTimeout(t, 2*time.Minute, func() { - testNameservers(t) - }) -} - -func testNameservers(t *testing.T) { //common.SetLogLevel("debug") lookupTimeout := 10 // ms diff --git a/nameserver/status.go b/nameserver/status.go index 6124f08d..2a280c57 100644 --- a/nameserver/status.go +++ b/nameserver/status.go @@ -1,10 +1,11 @@ package nameserver type Status struct { - Domain string - Address string - TTL uint32 - Entries []EntryStatus + Domain string + Upstream []string + Address string + TTL uint32 + Entries []EntryStatus } type EntryStatus struct { @@ -37,6 +38,7 @@ func NewStatus(ns *Nameserver, dnsServer *DNSServer) *Status { return &Status{ dnsServer.domain, + dnsServer.upstream.Servers, dnsServer.address, dnsServer.ttl, entryStatusSlice} diff --git a/net/if.go b/net/if.go index b99993ec..780f4290 100644 --- a/net/if.go +++ b/net/if.go @@ -5,28 +5,33 @@ import ( "net" "syscall" - "github.com/vishvananda/netlink/nl" + "github.com/vishvananda/netlink" ) // Wait for an interface to come up. func EnsureInterface(ifaceName string) (*net.Interface, error) { - s, err := nl.Subscribe(syscall.NETLINK_ROUTE, syscall.RTNLGRP_LINK) - if err != nil { - return nil, err - } - defer s.Close() - iface, err := ensureInterface(s, ifaceName) + iface, err := ensureInterface(ifaceName) if err != nil { return nil, err } return iface, err } -func ensureInterface(s *nl.NetlinkSocket, ifaceName string) (*net.Interface, error) { +func ensureInterface(ifaceName string) (*net.Interface, error) { if iface, err := findInterface(ifaceName); err == nil { return iface, nil } - netlinkReceiveUntil(s, matchIfName(ifaceName)) + ch := make(chan netlink.LinkUpdate) + done := make(chan struct{}) + defer close(done) + if err := netlink.LinkSubscribe(ch, done); err != nil { + return nil, err + } + for update := range ch { + if ifaceName == update.Link.Attrs().Name && update.IfInfomsg.Flags&syscall.IFF_UP != 0 { + break + } + } iface, err := findInterface(ifaceName) return iface, err } @@ -35,12 +40,7 @@ func ensureInterface(s *nl.NetlinkSocket, ifaceName string) (*net.Interface, err // This matches the behaviour in 'weave attach', which is the only context in which // we expect this to be called. If you change one, change the other to match. func EnsureInterfaceAndMcastRoute(ifaceName string) (*net.Interface, error) { - s, err := nl.Subscribe(syscall.NETLINK_ROUTE, syscall.RTNLGRP_LINK, syscall.RTNLGRP_IPV4_ROUTE) - if err != nil { - return nil, err - } - defer s.Close() - iface, err := ensureInterface(s, ifaceName) + iface, err := ensureInterface(ifaceName) if err != nil { return nil, err } @@ -48,8 +48,19 @@ func EnsureInterfaceAndMcastRoute(ifaceName string) (*net.Interface, error) { if CheckRouteExists(ifaceName, dest) { return iface, err } - netlinkReceiveUntil(s, matchRoute(ifaceName, dest)) - return iface, err + ch := make(chan netlink.RouteUpdate) + done := make(chan struct{}) + defer close(done) + if err := netlink.RouteSubscribe(ch, done); err != nil { + return nil, err + } + for update := range ch { + link, _ := netlink.LinkByIndex(update.Route.LinkIndex) + if link.Attrs().Name == ifaceName && update.Route.Dst.IP.Equal(dest) { + break + } + } + return iface, nil } func findInterface(ifaceName string) (iface *net.Interface, err error) { @@ -61,40 +72,3 @@ func findInterface(ifaceName string) (iface *net.Interface, err error) { } return } - -func netlinkReceiveUntil(s *nl.NetlinkSocket, f func(syscall.NetlinkMessage) (bool, error)) error { - for { - msgs, err := s.Receive() - if err != nil { - return err - } - for _, m := range msgs { - if done, err := f(m); done { - return err - } - } - } -} - -func matchIfName(ifaceName string) func(m syscall.NetlinkMessage) (bool, error) { - return func(m syscall.NetlinkMessage) (bool, error) { - switch m.Header.Type { - case syscall.RTM_NEWLINK: // receive this type for link 'up' - ifmsg := nl.DeserializeIfInfomsg(m.Data) - attrs, err := syscall.ParseNetlinkRouteAttr(&m) - if err != nil { - return true, err - } - name := "" - for _, attr := range attrs { - if attr.Attr.Type == syscall.IFA_LABEL { - name = string(attr.Value[:len(attr.Value)-1]) - } - } - if ifaceName == name && ifmsg.Flags&syscall.IFF_UP != 0 { - return true, nil - } - } - return false, nil - } -} diff --git a/net/route.go b/net/route.go index 560ae22e..3424826c 100644 --- a/net/route.go +++ b/net/route.go @@ -3,10 +3,8 @@ package net import ( "fmt" "net" - "syscall" "github.com/vishvananda/netlink" - "github.com/vishvananda/netlink/nl" ) // A network is considered free if it does not overlap any existing @@ -65,51 +63,3 @@ func CheckRouteExists(ifaceName string, dest net.IP) bool { }) return found } - -func matchRoute(ifaceName string, dest net.IP) func(m syscall.NetlinkMessage) (bool, error) { - return func(m syscall.NetlinkMessage) (bool, error) { - switch m.Header.Type { - case syscall.RTM_NEWROUTE: - route, err := deserializeRoute(m.Data) - if err != nil { - return true, err - } - link, _ := netlink.LinkByIndex(route.LinkIndex) - if link.Attrs().Name == ifaceName && route.Dst.IP.Equal(dest) { - return true, nil - } - } - return false, nil - } -} - -// This code proposed for addition to the vishvananda/netlink library -// deserializeRoute decodes a binary netlink message into a Route struct -func deserializeRoute(m []byte) (netlink.Route, error) { - route := netlink.Route{} - msg := nl.DeserializeRtMsg(m) - attrs, err := nl.ParseRouteAttr(m[msg.Len():]) - if err != nil { - return route, err - } - route.Scope = netlink.Scope(msg.Scope) - - native := nl.NativeEndian() - for _, attr := range attrs { - switch attr.Attr.Type { - case syscall.RTA_GATEWAY: - route.Gw = net.IP(attr.Value) - case syscall.RTA_PREFSRC: - route.Src = net.IP(attr.Value) - case syscall.RTA_DST: - route.Dst = &net.IPNet{ - IP: attr.Value, - Mask: net.CIDRMask(int(msg.Dst_len), 8*len(attr.Value)), - } - case syscall.RTA_OIF: - routeIndex := int(native.Uint32(attr.Value[0:4])) - route.LinkIndex = routeIndex - } - } - return route, nil -} diff --git a/plugin/README.md b/plugin/README.md new file mode 100644 index 00000000..1e246c1c --- /dev/null +++ b/plugin/README.md @@ -0,0 +1,9 @@ +# Weave network driver extension for Docker + +A [plugin](http://docs.docker.com/engine/extend/plugin_api/) to +integrate [weave Net](http://weave.works/net/) with Docker. + +The plugin runs automatically when you `weave launch`, provided your +Docker daemon is version 1.9 or newer. + +More detail [here](http://docs.weave.works/weave/latest_release/plugin.html). diff --git a/plugin/ipam/driver.go b/plugin/ipam/driver.go new file mode 100644 index 00000000..766aa9a1 --- /dev/null +++ b/plugin/ipam/driver.go @@ -0,0 +1,58 @@ +package ipamplugin + +import ( + "net" + + "github.com/docker/libnetwork/ipamapi" + "github.com/docker/libnetwork/netlabel" + "github.com/weaveworks/weave/api" + . "github.com/weaveworks/weave/common" + "github.com/weaveworks/weave/common/docker" +) + +const ( + WeaveContainer = "weave" +) + +type ipam struct { + client *docker.Client + weave *api.Client +} + +func NewIpam(client *docker.Client, version string) (ipamapi.Ipam, error) { + resolver := func() (string, error) { return client.GetContainerIP(WeaveContainer) } + return &ipam{client: client, weave: api.NewClientWithResolver(resolver)}, nil +} + +func (i *ipam) GetDefaultAddressSpaces() (string, string, error) { + Log.Debugln("GetDefaultAddressSpaces") + return "weavelocal", "weaveglobal", nil +} + +func (i *ipam) RequestPool(addressSpace, pool, subPool string, options map[string]string, v6 bool) (string, *net.IPNet, map[string]string, error) { + Log.Debugln("RequestPool", addressSpace, pool, subPool, options) + cidr, err := i.weave.DefaultSubnet() + Log.Debugln("RequestPool returning ", cidr, err) + // Pass back a fake "gateway address"; we don't actually use it, + // so just give the network address. + data := map[string]string{netlabel.Gateway: cidr.String()} + return "weavepool", cidr, data, err +} + +func (i *ipam) ReleasePool(poolID string) error { + Log.Debugln("ReleasePool", poolID) + return nil +} + +func (i *ipam) RequestAddress(poolID string, address net.IP, options map[string]string) (*net.IPNet, map[string]string, error) { + Log.Debugln("RequestAddress", poolID, address, options) + // Pass magic string to weave IPAM, which then stores the address under its own string + ip, err := i.weave.AllocateIP("_") + Log.Debugln("allocateIP returned", ip, err) + return ip, nil, err +} + +func (i *ipam) ReleaseAddress(poolID string, address net.IP) error { + Log.Debugln("ReleaseAddress", poolID, address) + return i.weave.ReleaseIP(address.String()) +} diff --git a/plugin/net/driver.go b/plugin/net/driver.go new file mode 100644 index 00000000..12b4a62f --- /dev/null +++ b/plugin/net/driver.go @@ -0,0 +1,206 @@ +package plugin + +import ( + "fmt" + "sync" + + "github.com/docker/libnetwork/drivers/remote/api" + "github.com/docker/libnetwork/types" + + . "github.com/weaveworks/weave/common" + "github.com/weaveworks/weave/common/docker" + "github.com/weaveworks/weave/common/odp" + "github.com/weaveworks/weave/plugin/skel" + + "github.com/vishvananda/netlink" +) + +const ( + WeaveContainer = "weave" + WeaveBridge = "weave" +) + +type driver struct { + version string + nameserver string + scope string + noMulticastRoute bool + sync.RWMutex + endpoints map[string]struct{} +} + +func New(client *docker.Client, version string, nameserver string, scope string, noMulticastRoute bool) (skel.Driver, error) { + driver := &driver{ + nameserver: nameserver, + noMulticastRoute: noMulticastRoute, + version: version, + scope: scope, + endpoints: make(map[string]struct{}), + } + + _, err := NewWatcher(client, driver) + if err != nil { + return nil, err + } + return driver, nil +} + +func errorf(format string, a ...interface{}) error { + Log.Errorf(format, a...) + return fmt.Errorf(format, a...) +} + +// === protocol handlers + +func (driver *driver) GetCapabilities() (*api.GetCapabilityResponse, error) { + var caps = &api.GetCapabilityResponse{ + Scope: driver.scope, + } + Log.Debugf("Get capabilities: responded with %+v", caps) + return caps, nil +} + +func (driver *driver) CreateNetwork(create *api.CreateNetworkRequest) error { + Log.Debugf("Create network request %+v", create) + Log.Infof("Create network %s", create.NetworkID) + return nil +} + +func (driver *driver) DeleteNetwork(delete *api.DeleteNetworkRequest) error { + Log.Debugf("Delete network request: %+v", delete) + Log.Infof("Destroy network %s", delete.NetworkID) + return nil +} + +func (driver *driver) CreateEndpoint(create *api.CreateEndpointRequest) (*api.CreateEndpointResponse, error) { + Log.Debugf("Create endpoint request %+v", create) + endID := create.EndpointID + + if create.Interface == nil { + return nil, fmt.Errorf("Not supported: creating an interface from within CreateEndpoint") + } + driver.Lock() + driver.endpoints[endID] = struct{}{} + driver.Unlock() + resp := &api.CreateEndpointResponse{} + + Log.Infof("Create endpoint %s %+v", endID, resp) + return resp, nil +} + +func (driver *driver) DeleteEndpoint(deleteReq *api.DeleteEndpointRequest) error { + Log.Debugf("Delete endpoint request: %+v", deleteReq) + Log.Infof("Delete endpoint %s", deleteReq.EndpointID) + driver.Lock() + delete(driver.endpoints, deleteReq.EndpointID) + driver.Unlock() + return nil +} + +func (driver *driver) HasEndpoint(endpointID string) bool { + driver.Lock() + _, found := driver.endpoints[endpointID] + driver.Unlock() + return found +} + +func (driver *driver) EndpointInfo(req *api.EndpointInfoRequest) (*api.EndpointInfoResponse, error) { + Log.Debugf("Endpoint info request: %+v", req) + Log.Infof("Endpoint info %s", req.EndpointID) + return &api.EndpointInfoResponse{Value: map[string]interface{}{}}, nil +} + +func (driver *driver) JoinEndpoint(j *api.JoinRequest) (*api.JoinResponse, error) { + endID := j.EndpointID + + maybeBridge, err := netlink.LinkByName(WeaveBridge) + if err != nil { + return nil, errorf(`bridge "%s" not present; did you launch weave?`, WeaveBridge) + } + + // create and attach local name to the bridge + local := vethPair(endID[:5]) + local.Attrs().MTU = maybeBridge.Attrs().MTU + if err := netlink.LinkAdd(local); err != nil { + return nil, errorf("could not create veth pair: %s", err) + } + + switch maybeBridge.(type) { + case *netlink.Bridge: + if err := netlink.LinkSetMasterByIndex(local, maybeBridge.Attrs().Index); err != nil { + return nil, errorf(`unable to set master: %s`, err) + } + case *netlink.GenericLink: + if maybeBridge.Type() != "openvswitch" { + Log.Errorf("device %s is %+v", WeaveBridge, maybeBridge) + return nil, errorf(`device "%s" is of type "%s"`, WeaveBridge, maybeBridge.Type()) + } + odp.AddDatapathInterface(WeaveBridge, local.Name) + case *netlink.Device: + Log.Warnf("kernel does not report what kind of device %s is, just %+v", WeaveBridge, maybeBridge) + // Assume it's our openvswitch device, and the kernel has not been updated to report the kind. + odp.AddDatapathInterface(WeaveBridge, local.Name) + default: + Log.Errorf("device %s is %+v", WeaveBridge, maybeBridge) + return nil, errorf(`device "%s" not a bridge`, WeaveBridge) + } + if err := netlink.LinkSetUp(local); err != nil { + return nil, errorf(`unable to bring veth up: %s`, err) + } + + ifname := &api.InterfaceName{ + SrcName: local.PeerName, + DstPrefix: "ethwe", + } + + response := &api.JoinResponse{ + InterfaceName: ifname, + } + if driver.nameserver != "" { + routeToDNS := api.StaticRoute{ + Destination: driver.nameserver + "/32", + RouteType: types.CONNECTED, + NextHop: "", + } + response.StaticRoutes = []api.StaticRoute{routeToDNS} + } + if !driver.noMulticastRoute { + multicastRoute := api.StaticRoute{ + Destination: "224.0.0.0/4", + RouteType: types.CONNECTED, + } + response.StaticRoutes = append(response.StaticRoutes, multicastRoute) + } + Log.Infof("Join endpoint %s:%s to %s", j.NetworkID, j.EndpointID, j.SandboxKey) + return response, nil +} + +func (driver *driver) LeaveEndpoint(leave *api.LeaveRequest) error { + Log.Debugf("Leave request: %+v", leave) + + local := vethPair(leave.EndpointID[:5]) + if err := netlink.LinkDel(local); err != nil { + Log.Warningf("unable to delete veth on leave: %s", err) + } + Log.Infof("Leave %s:%s", leave.NetworkID, leave.EndpointID) + return nil +} + +func (driver *driver) DiscoverNew(disco *api.DiscoveryNotification) error { + Log.Debugf("Dicovery new notification: %+v", disco) + return nil +} + +func (driver *driver) DiscoverDelete(disco *api.DiscoveryNotification) error { + Log.Debugf("Dicovery delete notification: %+v", disco) + return nil +} + +// === + +func vethPair(suffix string) *netlink.Veth { + return &netlink.Veth{ + LinkAttrs: netlink.LinkAttrs{Name: "vethwl" + suffix}, + PeerName: "vethwg" + suffix, + } +} diff --git a/plugin/net/watcher.go b/plugin/net/watcher.go new file mode 100644 index 00000000..05241d4f --- /dev/null +++ b/plugin/net/watcher.go @@ -0,0 +1,57 @@ +package plugin + +import ( + "fmt" + + weaveapi "github.com/weaveworks/weave/api" + . "github.com/weaveworks/weave/common" + "github.com/weaveworks/weave/common/docker" +) + +const ( + WeaveDomain = "weave.local" +) + +type watcher struct { + client *docker.Client + weave *weaveapi.Client + driver *driver +} + +type Watcher interface { +} + +func NewWatcher(client *docker.Client, driver *driver) (Watcher, error) { + resolver := func() (string, error) { return client.GetContainerIP(WeaveContainer) } + w := &watcher{client: client, driver: driver, + weave: weaveapi.NewClientWithResolver(resolver)} + err := client.AddObserver(w) + if err != nil { + return nil, err + } + + return w, nil +} + +func (w *watcher) ContainerStarted(id string) { + Log.Debugf("Container started %s", id) + info, err := w.client.InspectContainer(id) + if err != nil { + Log.Warningf("error inspecting container: %s", err) + return + } + // check that it's on our network, via the endpointID + for _, net := range info.NetworkSettings.Networks { + if w.driver.HasEndpoint(net.EndpointID) { + fqdn := fmt.Sprintf("%s.%s", info.Config.Hostname, info.Config.Domainname) + if err := w.weave.RegisterWithDNS(id, fqdn, net.IPAddress); err != nil { + Log.Warningf("unable to register with weaveDNS: %s", err) + } + } + } +} + +func (w *watcher) ContainerDied(id string) { + // don't need to do this as WeaveDNS removes names on container died anyway + // (note by the time we get this event we can't see the EndpointID) +} diff --git a/plugin/skel/listener.go b/plugin/skel/listener.go new file mode 100644 index 00000000..2b4deb71 --- /dev/null +++ b/plugin/skel/listener.go @@ -0,0 +1,298 @@ +package skel + +import ( + "encoding/json" + "fmt" + "net" + "net/http" + + "github.com/gorilla/mux" + + "github.com/docker/libnetwork/drivers/remote/api" + "github.com/docker/libnetwork/ipamapi" + iapi "github.com/docker/libnetwork/ipams/remote/api" +) + +const ( + networkReceiver = "NetworkDriver" + ipamReceiver = ipamapi.PluginEndpointType +) + +type Driver interface { + GetCapabilities() (*api.GetCapabilityResponse, error) + CreateNetwork(create *api.CreateNetworkRequest) error + DeleteNetwork(delete *api.DeleteNetworkRequest) error + CreateEndpoint(create *api.CreateEndpointRequest) (*api.CreateEndpointResponse, error) + DeleteEndpoint(delete *api.DeleteEndpointRequest) error + EndpointInfo(req *api.EndpointInfoRequest) (*api.EndpointInfoResponse, error) + JoinEndpoint(j *api.JoinRequest) (response *api.JoinResponse, error error) + LeaveEndpoint(leave *api.LeaveRequest) error + DiscoverNew(discover *api.DiscoveryNotification) error + DiscoverDelete(delete *api.DiscoveryNotification) error +} + +type listener struct { + d Driver + i ipamapi.Ipam +} + +func Listen(socket net.Listener, driver Driver, ipamDriver ipamapi.Ipam) error { + router := mux.NewRouter() + router.NotFoundHandler = http.HandlerFunc(notFound) + + listener := &listener{driver, ipamDriver} + + router.Methods("POST").Path("/Plugin.Activate").HandlerFunc(listener.handshake) + + handleMethod := func(receiver, method string, h http.HandlerFunc) { + router.Methods("POST").Path(fmt.Sprintf("/%s.%s", receiver, method)).HandlerFunc(h) + } + + handleMethod(networkReceiver, "GetCapabilities", listener.getCapabilities) + + if driver != nil { + handleMethod(networkReceiver, "CreateNetwork", listener.createNetwork) + handleMethod(networkReceiver, "DeleteNetwork", listener.deleteNetwork) + handleMethod(networkReceiver, "CreateEndpoint", listener.createEndpoint) + handleMethod(networkReceiver, "DeleteEndpoint", listener.deleteEndpoint) + handleMethod(networkReceiver, "EndpointOperInfo", listener.infoEndpoint) + handleMethod(networkReceiver, "Join", listener.joinEndpoint) + handleMethod(networkReceiver, "Leave", listener.leaveEndpoint) + } + + if ipamDriver != nil { + handleMethod(ipamReceiver, "GetDefaultAddressSpaces", listener.getDefaultAddressSpaces) + handleMethod(ipamReceiver, "RequestPool", listener.requestPool) + handleMethod(ipamReceiver, "ReleasePool", listener.releasePool) + handleMethod(ipamReceiver, "RequestAddress", listener.requestAddress) + handleMethod(ipamReceiver, "ReleaseAddress", listener.releaseAddress) + } + + return http.Serve(socket, router) +} + +func decode(w http.ResponseWriter, r *http.Request, v interface{}) error { + err := json.NewDecoder(r.Body).Decode(v) + if err != nil { + sendError(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest) + } + return err +} + +// === protocol handlers + +type handshakeResp struct { + Implements []string +} + +func (listener *listener) handshake(w http.ResponseWriter, r *http.Request) { + var resp handshakeResp + if listener.d != nil { + resp.Implements = append(resp.Implements, "NetworkDriver") + } + if listener.i != nil { + resp.Implements = append(resp.Implements, "IpamDriver") + } + err := json.NewEncoder(w).Encode(&resp) + if err != nil { + sendError(w, "encode error", http.StatusInternalServerError) + return + } +} + +func (listener *listener) getCapabilities(w http.ResponseWriter, r *http.Request) { + caps, err := listener.d.GetCapabilities() + objectOrErrorResponse(w, caps, err) +} + +func (listener *listener) createNetwork(w http.ResponseWriter, r *http.Request) { + var create api.CreateNetworkRequest + err := json.NewDecoder(r.Body).Decode(&create) + if err != nil { + sendError(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest) + return + } + emptyOrErrorResponse(w, listener.d.CreateNetwork(&create)) +} + +func (listener *listener) deleteNetwork(w http.ResponseWriter, r *http.Request) { + var delete api.DeleteNetworkRequest + if err := json.NewDecoder(r.Body).Decode(&delete); err != nil { + sendError(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest) + return + } + emptyOrErrorResponse(w, listener.d.DeleteNetwork(&delete)) +} + +func (listener *listener) createEndpoint(w http.ResponseWriter, r *http.Request) { + var create api.CreateEndpointRequest + if err := json.NewDecoder(r.Body).Decode(&create); err != nil { + sendError(w, "unable to decode JSON payload: "+err.Error(), http.StatusBadRequest) + return + } + res, err := listener.d.CreateEndpoint(&create) + objectOrErrorResponse(w, res, err) +} + +func (listener *listener) deleteEndpoint(w http.ResponseWriter, r *http.Request) { + var delete api.DeleteEndpointRequest + if err := json.NewDecoder(r.Body).Decode(&delete); err != nil { + sendError(w, "Could not decode JSON encode payload", http.StatusBadRequest) + return + } + emptyOrErrorResponse(w, listener.d.DeleteEndpoint(&delete)) +} + +func (listener *listener) infoEndpoint(w http.ResponseWriter, r *http.Request) { + var req api.EndpointInfoRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + sendError(w, "Could not decode JSON encode payload", http.StatusBadRequest) + return + } + info, err := listener.d.EndpointInfo(&req) + objectOrErrorResponse(w, info, err) +} + +func (listener *listener) joinEndpoint(w http.ResponseWriter, r *http.Request) { + var join api.JoinRequest + if err := json.NewDecoder(r.Body).Decode(&join); err != nil { + sendError(w, "Could not decode JSON encode payload", http.StatusBadRequest) + return + } + res, err := listener.d.JoinEndpoint(&join) + objectOrErrorResponse(w, res, err) +} + +func (listener *listener) leaveEndpoint(w http.ResponseWriter, r *http.Request) { + var l api.LeaveRequest + if err := json.NewDecoder(r.Body).Decode(&l); err != nil { + sendError(w, "Could not decode JSON encode payload", http.StatusBadRequest) + return + } + emptyOrErrorResponse(w, listener.d.LeaveEndpoint(&l)) +} + +func (listener *listener) discoverNew(w http.ResponseWriter, r *http.Request) { + var disco api.DiscoveryNotification + if err := json.NewDecoder(r.Body).Decode(&disco); err != nil { + sendError(w, "Could not decode JSON encode payload", http.StatusBadRequest) + return + } + emptyOrErrorResponse(w, listener.d.DiscoverNew(&disco)) +} + +func (listener *listener) discoverDelete(w http.ResponseWriter, r *http.Request) { + var disco api.DiscoveryNotification + if err := json.NewDecoder(r.Body).Decode(&disco); err != nil { + sendError(w, "Could not decode JSON encode payload", http.StatusBadRequest) + return + } + emptyOrErrorResponse(w, listener.d.DiscoverDelete(&disco)) +} + +// === + +func (listener *listener) getDefaultAddressSpaces(w http.ResponseWriter, r *http.Request) { + local, global, err := listener.i.GetDefaultAddressSpaces() + response := &iapi.GetAddressSpacesResponse{ + LocalDefaultAddressSpace: local, + GlobalDefaultAddressSpace: global, + } + objectOrErrorResponse(w, response, err) +} + +func (listener *listener) requestPool(w http.ResponseWriter, r *http.Request) { + var rq iapi.RequestPoolRequest + if err := decode(w, r, &rq); err != nil { + return + } + poolID, pool, data, err := listener.i.RequestPool(rq.AddressSpace, rq.Pool, rq.SubPool, rq.Options, rq.V6) + if err != nil { + errorResponse(w, err.Error()) + return + } + response := &iapi.RequestPoolResponse{ + PoolID: poolID, + Pool: pool.String(), + Data: data, + } + objectResponse(w, response) +} + +func (listener *listener) releasePool(w http.ResponseWriter, r *http.Request) { + var rq iapi.ReleasePoolRequest + if err := decode(w, r, &rq); err != nil { + return + } + err := listener.i.ReleasePool(rq.PoolID) + emptyOrErrorResponse(w, err) +} + +func (listener *listener) requestAddress(w http.ResponseWriter, r *http.Request) { + var rq iapi.RequestAddressRequest + if err := decode(w, r, &rq); err != nil { + return + } + address, data, err := listener.i.RequestAddress(rq.PoolID, net.ParseIP(rq.Address), rq.Options) + if err != nil { + errorResponse(w, err.Error()) + return + } + response := &iapi.RequestAddressResponse{ + Address: address.String(), + Data: data, + } + objectResponse(w, response) +} + +func (listener *listener) releaseAddress(w http.ResponseWriter, r *http.Request) { + var rq iapi.ReleaseAddressRequest + if err := decode(w, r, &rq); err != nil { + return + } + err := listener.i.ReleaseAddress(rq.PoolID, net.ParseIP(rq.Address)) + emptyOrErrorResponse(w, err) +} + +// === + +func notFound(w http.ResponseWriter, r *http.Request) { + http.NotFound(w, r) +} + +func sendError(w http.ResponseWriter, msg string, code int) { + http.Error(w, msg, code) +} + +func errorResponse(w http.ResponseWriter, fmtString string, item ...interface{}) { + json.NewEncoder(w).Encode(map[string]string{ + "Err": fmt.Sprintf(fmtString, item...), + }) +} + +func objectResponse(w http.ResponseWriter, obj interface{}) { + if err := json.NewEncoder(w).Encode(obj); err != nil { + sendError(w, "Could not JSON encode response", http.StatusInternalServerError) + return + } +} + +func emptyResponse(w http.ResponseWriter) { + json.NewEncoder(w).Encode(map[string]string{}) +} + +func objectOrErrorResponse(w http.ResponseWriter, obj interface{}, err error) { + if err != nil { + errorResponse(w, err.Error()) + return + } + objectResponse(w, obj) +} + +func emptyOrErrorResponse(w http.ResponseWriter, err error) { + if err != nil { + errorResponse(w, err.Error()) + return + } + emptyResponse(w) +} diff --git a/prog/plugin/Dockerfile b/prog/plugin/Dockerfile new file mode 100644 index 00000000..946f0b28 --- /dev/null +++ b/prog/plugin/Dockerfile @@ -0,0 +1,3 @@ +FROM weaveworks/weaveexec +ADD ./plugin /home/weave/ +ENTRYPOINT ["/home/weave/plugin"] diff --git a/prog/plugin/main.go b/prog/plugin/main.go new file mode 100644 index 00000000..01bad7f6 --- /dev/null +++ b/prog/plugin/main.go @@ -0,0 +1,163 @@ +package main + +import ( + "flag" + "fmt" + "net" + "os" + "os/signal" + "strings" + "syscall" + + "github.com/docker/libnetwork/ipamapi" + go_docker "github.com/fsouza/go-dockerclient" + . "github.com/weaveworks/weave/common" + "github.com/weaveworks/weave/common/docker" + ipamplugin "github.com/weaveworks/weave/plugin/ipam" + netplugin "github.com/weaveworks/weave/plugin/net" + "github.com/weaveworks/weave/plugin/skel" +) + +var version = "(unreleased version)" + +func main() { + var ( + justVersion bool + address string + nameserver string + meshAddress string + logLevel string + meshNetworkName string + noMulticastRoute bool + removeNetwork bool + ) + + flag.BoolVar(&justVersion, "version", false, "print version and exit") + flag.StringVar(&logLevel, "log-level", "info", "logging level (debug, info, warning, error)") + flag.StringVar(&address, "socket", "/run/docker/plugins/weave.sock", "socket on which to listen") + flag.StringVar(&nameserver, "nameserver", "", "nameserver to provide to containers") + flag.StringVar(&meshAddress, "meshsocket", "/run/docker/plugins/weavemesh.sock", "socket on which to listen in mesh mode") + flag.StringVar(&meshNetworkName, "mesh-network-name", "weave", "network name to create in mesh mode") + flag.BoolVar(&noMulticastRoute, "no-multicast-route", false, "do not add a multicast route to network endpoints") + flag.BoolVar(&removeNetwork, "remove-network", false, "remove mesh network and exit") + + flag.Parse() + + if justVersion { + fmt.Printf("weave plugin %s\n", version) + os.Exit(0) + } + + SetLogLevel(logLevel) + + // API 1.21 is the first version that supports docker network commands + dockerClient, err := docker.NewVersionedClientFromEnv("1.21") + if err != nil { + Log.Fatalf("unable to connect to docker: %s", err) + } + + if removeNetwork { + if _, err = dockerClient.Client.NetworkInfo(meshNetworkName); err == nil { + err = dockerClient.Client.RemoveNetwork(meshNetworkName) + if err != nil { + Log.Fatalf("unable to remove network: %s", err) + } + } + os.Exit(0) + } + + Log.Println("Weave plugin", version, "Command line options:", os.Args[1:]) + Log.Info(dockerClient.Info()) + + var globalListener, meshListener net.Listener + endChan := make(chan error, 1) + if address != "" { + globalListener, err := listenAndServe(dockerClient, address, nameserver, noMulticastRoute, endChan, "global", false) + if err != nil { + Log.Fatalf("unable to create driver: %s", err) + } + defer globalListener.Close() + } + if meshAddress != "" { + meshListener, err := listenAndServe(dockerClient, meshAddress, nameserver, noMulticastRoute, endChan, "local", true) + if err != nil { + Log.Fatalf("unable to create driver: %s", err) + } + defer meshListener.Close() + } + + if meshNetworkName != "" { + createNetwork(dockerClient, meshNetworkName, meshAddress) + } + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, os.Kill, syscall.SIGTERM) + + select { + case sig := <-sigChan: + Log.Debugf("Caught signal %s; shutting down", sig) + if err := dockerClient.Client.RemoveNetwork(meshNetworkName); err != nil { + Log.Fatal(err) + } + case err := <-endChan: + if err != nil { + Log.Errorf("Error from listener: %s", err) + globalListener.Close() + meshListener.Close() + os.Exit(1) + } + } +} + +func listenAndServe(dockerClient *docker.Client, address, nameserver string, noMulticastRoute bool, endChan chan<- error, scope string, withIpam bool) (net.Listener, error) { + d, err := netplugin.New(dockerClient, version, nameserver, scope, noMulticastRoute) + if err != nil { + return nil, err + } + + var i ipamapi.Ipam + if withIpam { + if i, err = ipamplugin.NewIpam(dockerClient, version); err != nil { + return nil, err + } + } + + var listener net.Listener + + // remove sockets from last invocation + if err := os.Remove(address); err != nil && !os.IsNotExist(err) { + return nil, err + } + listener, err = net.Listen("unix", address) + if err != nil { + return nil, err + } + Log.Printf("Listening on %s for %s scope", address, scope) + + go func() { + endChan <- skel.Listen(listener, d, i) + }() + + return listener, nil +} + +func createNetwork(dockerClient *docker.Client, networkName, address string) { + if _, err := dockerClient.Client.NetworkInfo(networkName); err == nil { + Log.Printf("Docker network '%s' already exists", networkName) + } else if _, ok := err.(*go_docker.NoSuchNetwork); ok { + driverName := strings.TrimSuffix(address, ".sock") + if i := strings.LastIndex(driverName, "/"); i >= 0 { + driverName = driverName[i+1:] + } + options := go_docker.CreateNetworkOptions{ + Name: networkName, + CheckDuplicate: true, + Driver: driverName, + IPAM: go_docker.IPAMOptions{Driver: driverName}, + } + _, err := dockerClient.Client.CreateNetwork(options) + if err != nil { + Log.Fatalf("Error creating network: %s", err) + } + } +} diff --git a/prog/weaveexec/Dockerfile b/prog/weaveexec/Dockerfile index 3003d03e..f700f20a 100644 --- a/prog/weaveexec/Dockerfile +++ b/prog/weaveexec/Dockerfile @@ -6,6 +6,7 @@ LABEL works.weave.role=system WORKDIR /home/weave VOLUME /w VOLUME /w-noop +VOLUME /w-nomcast ENTRYPOINT ["/home/weave/sigproxy", "/home/weave/weave"] RUN apk add --update \ @@ -18,8 +19,9 @@ RUN apk add --update \ bind-tools \ && rm -rf /var/cache/apk/* -ADD ./weave ./sigproxy ./weaveproxy /home/weave/ +ADD ./weave ./sigproxy ./weaveproxy ./symlink /home/weave/ ADD ./netcheck ./docker_tls_args /usr/bin/ ADD ./weavewait /w/w ADD ./weavewait_noop /w-noop/w +ADD ./weavewait_nomcast /w-nomcast/w ADD ./docker.tgz / diff --git a/prog/weaveexec/symlink b/prog/weaveexec/symlink new file mode 100755 index 00000000..2c06e901 --- /dev/null +++ b/prog/weaveexec/symlink @@ -0,0 +1,18 @@ +#!/bin/sh +# This lets us force-symlink a bunch of stuff all at once, so we can minimize +# running containers. + +set -e + +[ -z "$WEAVE_DEBUG" ] || set -x + +target="$1" +user="$(stat -c '%u:%g' "$target")" +mode="$(stat -c '%a' "$target")" +shift 1 + +for link in "$@" ; do + ln -sfT "$target" "$link" +done +chown -h "$user" "$@" +chmod "$mode" "$@" diff --git a/prog/weaveproxy/main.go b/prog/weaveproxy/main.go index a0a247df..d634e22a 100644 --- a/prog/weaveproxy/main.go +++ b/prog/weaveproxy/main.go @@ -12,14 +12,15 @@ import ( ) var ( - version = "(unreleased version)" + version = "(unreleased version)" + defaultDockerHost = "unix:///var/run/docker.sock" ) func main() { var ( justVersion bool logLevel = "info" - c = proxy.Config{ListenAddrs: []string{}} + c = proxy.Config{Image: "weaveworks/weaveexec"} ) c.Version = version @@ -35,11 +36,11 @@ func main() { mflag.BoolVar(&c.NoRewriteHosts, []string{"-no-rewrite-hosts"}, false, "do not automatically rewrite /etc/hosts. Use if you need the docker IP to remain in /etc/hosts") mflag.StringVar(&c.TLSConfig.CACert, []string{"#tlscacert", "-tlscacert"}, "", "Trust certs signed only by this CA") mflag.StringVar(&c.TLSConfig.Cert, []string{"#tlscert", "-tlscert"}, "", "Path to TLS certificate file") - mflag.BoolVar(&c.TLSConfig.Enabled, []string{"#tls", "-tls"}, false, "Use TLS; implied by --tls-verify") + mflag.BoolVar(&c.TLSConfig.Enabled, []string{"#tls", "-tls"}, false, "Use TLS; implied by --tlsverify") mflag.StringVar(&c.TLSConfig.Key, []string{"#tlskey", "-tlskey"}, "", "Path to TLS key file") mflag.BoolVar(&c.TLSConfig.Verify, []string{"#tlsverify", "-tlsverify"}, false, "Use TLS and verify the remote") - mflag.BoolVar(&c.WithDNS, []string{"-with-dns", "w"}, false, "instruct created containers to always use weaveDNS as their nameserver") mflag.BoolVar(&c.WithoutDNS, []string{"-without-dns"}, false, "instruct created containers to never use weaveDNS as their nameserver") + mflag.BoolVar(&c.NoMulticastRoute, []string{"-no-multicast-route"}, false, "do not add a multicast route via the weave interface when attaching containers") mflag.Parse() if justVersion { @@ -47,15 +48,20 @@ func main() { os.Exit(0) } - if c.WithDNS && c.WithoutDNS { - Log.Fatalf("Cannot use both '--with-dns' and '--without-dns' flags") - } - SetLogLevel(logLevel) + if image := os.Getenv("EXEC_IMAGE"); image != "" { + c.Image = image + } + Log.Infoln("weave proxy", version) Log.Infoln("Command line arguments:", strings.Join(os.Args[1:], " ")) + c.DockerHost = defaultDockerHost + if dockerHost := os.Getenv("DOCKER_HOST"); dockerHost != "" { + c.DockerHost = dockerHost + } + p, err := proxy.NewProxy(c) if err != nil { Log.Fatalf("Could not start proxy: %s", err) diff --git a/prog/weaver/http.go b/prog/weaver/http.go index cb828f47..c02b2bdb 100644 --- a/prog/weaver/http.go +++ b/prog/weaver/http.go @@ -1,8 +1,13 @@ package main import ( + "bytes" "encoding/json" "fmt" + "net/http" + "strings" + "text/template" + "github.com/gorilla/mux" . "github.com/weaveworks/weave/common" "github.com/weaveworks/weave/ipam" @@ -10,9 +15,6 @@ import ( "github.com/weaveworks/weave/nameserver" "github.com/weaveworks/weave/net/address" weave "github.com/weaveworks/weave/router" - "net/http" - "strings" - "text/template" ) var rootTemplate = template.New("root").Funcs(map[string]interface{}{ @@ -25,6 +27,66 @@ var rootTemplate = template.New("root").Funcs(map[string]interface{}{ } return count }, + "printList": func(list []string) string { + if len(list) == 0 { + return "none" + } + return strings.Join(list, ", ") + }, + "printIPAMRanges": func(router weave.NetworkRouterStatus, status ipam.Status) string { + var buffer bytes.Buffer + + type stats struct { + ips uint32 + nickname string + reachable bool + } + + peerStats := make(map[string]*stats) + + for _, entry := range status.Entries { + s, found := peerStats[entry.Peer] + if !found { + s = &stats{nickname: entry.Nickname, reachable: entry.IsKnownPeer} + peerStats[entry.Peer] = s + } + s.ips += entry.Size + } + + printOwned := func(name string, nickName string, reachable bool, ips uint32) { + reachableStr := "" + if !reachable { + reachableStr = "- unreachable!" + } + percentageRanges := float32(ips) * 100.0 / float32(status.RangeNumIPs) + + displayName := name + "(" + nickName + ")" + fmt.Fprintf(&buffer, "%-37v %8d IPs (%04.1f%% of total) %s\n", + displayName, ips, percentageRanges, reachableStr) + } + + // print the local info first + if ourStats := peerStats[router.Name]; ourStats != nil { + printOwned(router.Name, ourStats.nickname, true, ourStats.ips) + } + + // and then the rest + for peer, stats := range peerStats { + if peer != router.Name { + printOwned(peer, stats.nickname, stats.reachable, stats.ips) + } + } + + return buffer.String() + }, + "allIPAMOwnersUnreachable": func(status ipam.Status) bool { + for _, entry := range status.Entries { + if entry.Size > 0 && entry.IsKnownPeer { + return false + } + } + return true + }, "printConnectionCounts": func(conns []mesh.LocalConnectionStatus) string { counts := make(map[string]int) for _, conn := range conns { @@ -76,40 +138,48 @@ func defTemplate(name string, text string) *template.Template { } var statusTemplate = defTemplate("status", `\ - Version: {{.Version}} + Version: {{.Version}} - Service: router - Protocol: {{.Router.Protocol}} \ + Service: router + Protocol: {{.Router.Protocol}} \ {{if eq .Router.ProtocolMinVersion .Router.ProtocolMaxVersion}}\ {{.Router.ProtocolMaxVersion}}\ {{else}}\ {{.Router.ProtocolMinVersion}}..{{.Router.ProtocolMaxVersion}}\ {{end}} - Name: {{.Router.Name}}({{.Router.NickName}}) - Encryption: {{printState .Router.Encryption}} - PeerDiscovery: {{printState .Router.PeerDiscovery}} - Targets: {{len .Router.Targets}} - Connections: {{len .Router.Connections}}{{with printConnectionCounts .Router.Connections}} ({{.}}){{end}} - Peers: {{len .Router.Peers}}{{with printPeerConnectionCounts .Router.Peers}} (with {{.}} connections){{end}} + Name: {{.Router.Name}}({{.Router.NickName}}) + Encryption: {{printState .Router.Encryption}} + PeerDiscovery: {{printState .Router.PeerDiscovery}} + Targets: {{len .Router.Targets}} + Connections: {{len .Router.Connections}}{{with printConnectionCounts .Router.Connections}} ({{.}}){{end}} + Peers: {{len .Router.Peers}}{{with printPeerConnectionCounts .Router.Peers}} (with {{.}} connections){{end}} + TrustedSubnets: {{printList .Router.TrustedSubnets}} {{if .IPAM}}\ - Service: ipam + Service: ipam {{if .IPAM.Entries}}\ - Consensus: achieved -{{else if .IPAM.Paxos}}\ - Consensus: waiting (quorum: {{.IPAM.Paxos.Quorum}}, known: {{.IPAM.Paxos.KnownNodes}}) +{{if allIPAMOwnersUnreachable .IPAM}}\ + Status: all IP ranges owned by unreachable peers - use 'rmpeer' if they are dead +{{else if len .IPAM.PendingAllocates}}\ + Status: waiting for IP range grant from peers {{else}}\ - Consensus: deferred + Status: ready {{end}}\ - Range: {{.IPAM.Range}} - DefaultSubnet: {{.IPAM.DefaultSubnet}} +{{else if .IPAM.Paxos}}\ + Status: awaiting consensus (quorum: {{.IPAM.Paxos.Quorum}}, known: {{.IPAM.Paxos.KnownNodes}}) +{{else}}\ + Status: idle +{{end}}\ + Range: {{.IPAM.Range}} + DefaultSubnet: {{.IPAM.DefaultSubnet}} {{end}}\ {{if .DNS}}\ - Service: dns - Domain: {{.DNS.Domain}} - TTL: {{.DNS.TTL}} - Entries: {{countDNSEntries .DNS.Entries}} + Service: dns + Domain: {{.DNS.Domain}} + Upstream: {{printList .DNS.Upstream}} + TTL: {{.DNS.TTL}} + Entries: {{countDNSEntries .DNS.Entries}} {{end}}\ `) @@ -129,7 +199,7 @@ var peersTemplate = defTemplate("peers", `\ {{.Name}}({{.NickName}}) {{range .Connections}}\ {{if .Outbound}}->{{else}}<-{{end}} {{printf "%-21v" .Address}} \ -{{$nameNickName := printf "%v(%v)" .Name .NickName}}{{printf "%-32v" $nameNickName}} \ +{{$nameNickName := printf "%v(%v)" .Name .NickName}}{{printf "%-37v" $nameNickName}} \ {{if .Established}}established{{else}}pending{{end}} {{end}}\ {{end}}\ @@ -145,6 +215,8 @@ var dnsEntriesTemplate = defTemplate("dnsEntries", `\ {{end}}\ `) +var ipamTemplate = defTemplate("ipamTemplate", `{{printIPAMRanges .Router .IPAM}}`) + type WeaveStatus struct { Version string Router *weave.NetworkRouterStatus `json:"Router,omitempty"` @@ -174,7 +246,13 @@ func HandleHTTP(muxRouter *mux.Router, version string, router *weave.NetworkRout muxRouter.Methods("GET").Path("/report").Queries("format", "{format}").HandlerFunc( func(w http.ResponseWriter, r *http.Request) { - formatTemplate, err := template.New("format").Parse(mux.Vars(r)["format"]) + funcs := template.FuncMap{ + "json": func(v interface{}) string { + a, _ := json.Marshal(v) + return string(a) + }, + } + formatTemplate, err := template.New("format").Funcs(funcs).Parse(mux.Vars(r)["format"]) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -200,5 +278,6 @@ func HandleHTTP(muxRouter *mux.Router, version string, router *weave.NetworkRout defHandler("/status/connections", connectionsTemplate) defHandler("/status/peers", peersTemplate) defHandler("/status/dns", dnsEntriesTemplate) + defHandler("/status/ipam", ipamTemplate) } diff --git a/prog/weaver/main.go b/prog/weaver/main.go index d5fb2553..ba09998a 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -69,8 +69,15 @@ func main() { dnsEffectiveListenAddress string iface *net.Interface datapathName string + trustedSubnetStr string + + defaultDockerHost = "unix:///var/run/docker.sock" ) + if val := os.Getenv("DOCKER_HOST"); val != "" { + defaultDockerHost = val + } + mflag.BoolVar(&justVersion, []string{"#version", "-version"}, false, "print version and exit") mflag.BoolVar(&createDatapath, []string{"-create-datapath"}, false, "create ODP datapath and exit") mflag.BoolVar(&deleteDatapath, []string{"-delete-datapath"}, false, "delete ODP datapath and exit") @@ -92,7 +99,7 @@ func main() { mflag.StringVar(&iprangeCIDR, []string{"#iprange", "#-iprange", "-ipalloc-range"}, "", "IP address range reserved for automatic allocation, in CIDR notation") mflag.StringVar(&ipsubnetCIDR, []string{"#ipsubnet", "#-ipsubnet", "-ipalloc-default-subnet"}, "", "subnet to allocate within by default, in CIDR notation") mflag.IntVar(&peerCount, []string{"#initpeercount", "#-initpeercount", "-init-peer-count"}, 0, "number of peers in network (for IP address allocation)") - mflag.StringVar(&dockerAPI, []string{"#api", "#-api", "-docker-api"}, "", "Docker API endpoint, e.g. unix:///var/run/docker.sock") + mflag.StringVar(&dockerAPI, []string{"#api", "#-api", "-docker-api"}, defaultDockerHost, "Docker API endpoint") mflag.BoolVar(&noDNS, []string{"-no-dns"}, false, "disable DNS server") mflag.StringVar(&dnsDomain, []string{"-dns-domain"}, nameserver.DefaultDomain, "local domain to server requests for") mflag.StringVar(&dnsListenAddress, []string{"-dns-listen-address"}, nameserver.DefaultListenAddress, "address to listen on for DNS requests") @@ -101,6 +108,8 @@ func main() { mflag.StringVar(&dnsEffectiveListenAddress, []string{"-dns-effective-listen-address"}, "", "address DNS will actually be listening, after Docker port mapping") mflag.StringVar(&datapathName, []string{"-datapath"}, "", "ODP datapath name") + mflag.StringVar(&trustedSubnetStr, []string{"-trusted-subnets"}, "", "Command separated list of trusted subnets in CIDR notation") + // crude way of detecting that we probably have been started in a // container, with `weave launch` --> suppress misleading paths in // mflags error messages. @@ -193,10 +202,7 @@ func main() { Log.Println("Communication between peers is unencrypted.") } else { config.Password = []byte(password) - Log.Println("Communication between peers is encrypted.") - - // fastdp doesn't support encryption - fastDPOverlay = nil + Log.Println("Communication between peers via untrusted networks is encrypted.") } overlays := weave.NewOverlaySwitch() @@ -237,6 +243,10 @@ func main() { networkConfig.PacketLogging = nopPacketLogging{} } + if config.TrustedSubnets, err = parseTrustedSubnets(trustedSubnetStr); err != nil { + Log.Fatal("Unable to parse trusted subnets: ", err) + } + router := weave.NewNetworkRouter(config, networkConfig, name, nickName, overlays) Log.Println("Our name is", router.Ourself) @@ -245,6 +255,8 @@ func main() { dc, err := docker.NewClient(dockerAPI) if err != nil { Log.Fatal("Unable to start docker client: ", err) + } else { + Log.Info(dc.Info()) } dockerCli = dc } @@ -405,6 +417,24 @@ func determineQuorum(initPeerCountFlag int, peers []string) uint { return quorum } +func parseTrustedSubnets(trustedSubnetStr string) ([]*net.IPNet, error) { + trustedSubnets := []*net.IPNet{} + + if trustedSubnetStr == "" { + return trustedSubnets, nil + } + + for _, subnetStr := range strings.Split(trustedSubnetStr, ",") { + _, subnet, err := net.ParseCIDR(subnetStr) + if err != nil { + return nil, err + } + trustedSubnets = append(trustedSubnets, subnet) + } + + return trustedSubnets, nil +} + func listenAndServeHTTP(httpAddr string, muxRouter *mux.Router) { protocol := "tcp" if strings.HasPrefix(httpAddr, "/") { diff --git a/prog/weavewait/check_network.go b/prog/weavewait/check_network.go index 9c17bf8e..53b5385d 100644 --- a/prog/weavewait/check_network.go +++ b/prog/weavewait/check_network.go @@ -1,12 +1,7 @@ -// +build !noop +// +build !iface,!mcast package main -import ( - weavenet "github.com/weaveworks/weave/net" -) - func checkNetwork() error { - _, err := weavenet.EnsureInterfaceAndMcastRoute("ethwe") - return err + return nil } diff --git a/prog/weavewait/check_network_iface.go b/prog/weavewait/check_network_iface.go new file mode 100644 index 00000000..9a627af8 --- /dev/null +++ b/prog/weavewait/check_network_iface.go @@ -0,0 +1,12 @@ +// +build iface,!mcast + +package main + +import ( + weavenet "github.com/weaveworks/weave/net" +) + +func checkNetwork() error { + _, err := weavenet.EnsureInterface("ethwe") + return err +} diff --git a/prog/weavewait/check_network_iface_mcast.go b/prog/weavewait/check_network_iface_mcast.go new file mode 100644 index 00000000..898c3b3d --- /dev/null +++ b/prog/weavewait/check_network_iface_mcast.go @@ -0,0 +1,12 @@ +// +build iface,mcast + +package main + +import ( + weavenet "github.com/weaveworks/weave/net" +) + +func checkNetwork() error { + _, err := weavenet.EnsureInterfaceAndMcastRoute("ethwe") + return err +} diff --git a/prog/weavewait/noop.go b/prog/weavewait/noop.go deleted file mode 100644 index 2342311d..00000000 --- a/prog/weavewait/noop.go +++ /dev/null @@ -1,7 +0,0 @@ -// +build noop - -package main - -func checkNetwork() error { - return nil -} diff --git a/proxy/common.go b/proxy/common.go index 9a927cca..a4810c66 100644 --- a/proxy/common.go +++ b/proxy/common.go @@ -39,6 +39,7 @@ func callWeave(args ...string) ([]byte, []byte, error) { } propagateEnv("DOCKER_BRIDGE") + propagateEnv("DOCKER_HOST") // Propogage WEAVE_DEBUG, to make debugging easier. propagateEnv("WEAVE_DEBUG") diff --git a/proxy/create_container_interceptor.go b/proxy/create_container_interceptor.go index 7f4bc84e..b13d16f3 100644 --- a/proxy/create_container_interceptor.go +++ b/proxy/create_container_interceptor.go @@ -53,8 +53,14 @@ func (i *createContainerInterceptor) InterceptRequest(r *http.Request) error { Log.Infof("Leaving container alone because %s", err) } else { Log.Infof("Creating container with WEAVE_CIDR \"%s\"", strings.Join(cidrs, " ")) - if err := addVolume(hostConfig, i.proxy.weaveWaitVolume, "/w", "ro"); err != nil { - return err + if i.proxy.NoMulticastRoute { + if err := addVolume(hostConfig, i.proxy.weaveWaitNomcastVolume, "/w", "ro"); err != nil { + return err + } + } else { + if err := addVolume(hostConfig, i.proxy.weaveWaitVolume, "/w", "ro"); err != nil { + return err + } } if err := i.setWeaveWaitEntrypoint(container); err != nil { return err diff --git a/proxy/proxy.go b/proxy/proxy.go index 778742d4..9dfb7972 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -1,6 +1,7 @@ package proxy import ( + "bytes" "crypto/tls" "errors" "fmt" @@ -8,6 +9,7 @@ import ( "net" "net/http" "os" + "path/filepath" "regexp" "strings" "sync" @@ -16,15 +18,15 @@ import ( docker "github.com/fsouza/go-dockerclient" . "github.com/weaveworks/weave/common" weavedocker "github.com/weaveworks/weave/common/docker" - "github.com/weaveworks/weave/nameserver" ) const ( defaultCaFile = "ca.pem" defaultKeyFile = "key.pem" defaultCertFile = "cert.pem" - dockerSock = "/var/run/docker.sock" - dockerSockUnix = "unix://" + dockerSock + + weaveSock = "/var/run/weave/weave.sock" + weaveSockUnix = "unix://" + weaveSock ) var ( @@ -46,14 +48,16 @@ type Config struct { HostnameFromLabel string HostnameMatch string HostnameReplacement string + Image string ListenAddrs []string RewriteInspect bool NoDefaultIPAM bool NoRewriteHosts bool TLSConfig TLSConfig Version string - WithDNS bool WithoutDNS bool + NoMulticastRoute bool + DockerHost string } type wait struct { @@ -65,13 +69,14 @@ type wait struct { type Proxy struct { sync.Mutex Config - client *docker.Client - dockerBridgeIP string - hostnameMatchRegexp *regexp.Regexp - weaveWaitVolume string - weaveWaitNoopVolume string - normalisedAddrs []string - waiters map[*http.Request]*wait + client *docker.Client + dockerBridgeIP string + hostnameMatchRegexp *regexp.Regexp + weaveWaitVolume string + weaveWaitNoopVolume string + weaveWaitNomcastVolume string + normalisedAddrs []string + waiters map[*http.Request]*wait } func NewProxy(c Config) (*Proxy, error) { @@ -81,14 +86,16 @@ func NewProxy(c Config) (*Proxy, error) { Log.Fatalf("Could not configure tls for proxy: %s", err) } - // We pin the protocol version to 1.15 (which corresponds to - // Docker 1.3.x; the earliest version supported by weave) in order + // We pin the protocol version to 1.18 (which corresponds to + // Docker 1.6.x; the earliest version supported by weave) in order // to insulate ourselves from breaking changes to the API, as // happened in 1.20 (Docker 1.8.0) when the presentation of // volumes changed in `inspect`. - client, err := weavedocker.NewVersionedClient(dockerSockUnix, "1.15") + client, err := weavedocker.NewVersionedClient(c.DockerHost, "1.18") if err != nil { return nil, err + } else { + Log.Info(client.Info()) } p.client = client.Client @@ -132,7 +139,16 @@ func (proxy *Proxy) AttachExistingContainers() { } func (proxy *Proxy) Dial() (net.Conn, error) { - return net.Dial("unix", dockerSock) + proto := "tcp" + addr := proxy.Config.DockerHost + switch { + case strings.HasPrefix(addr, "unix://"): + proto = "unix" + addr = strings.TrimPrefix(addr, "unix://") + case strings.HasPrefix(addr, "tcp://"): + addr = strings.TrimPrefix(addr, "tcp://") + } + return net.Dial(proto, addr) } func (proxy *Proxy) findWeaveWaitVolumes() error { @@ -140,7 +156,10 @@ func (proxy *Proxy) findWeaveWaitVolumes() error { if proxy.weaveWaitVolume, err = proxy.findVolume("/w"); err != nil { return err } - proxy.weaveWaitNoopVolume, err = proxy.findVolume("/w-noop") + if proxy.weaveWaitNoopVolume, err = proxy.findVolume("/w-noop"); err != nil { + return err + } + proxy.weaveWaitNomcastVolume, err = proxy.findVolume("/w-nomcast") return err } @@ -186,7 +205,12 @@ func (proxy *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (proxy *Proxy) Listen() []net.Listener { listeners := []net.Listener{} proxy.normalisedAddrs = []string{} + unixAddrs := []string{} for _, addr := range proxy.ListenAddrs { + if strings.HasPrefix(addr, "unix://") || strings.HasPrefix(addr, "/") { + unixAddrs = append(unixAddrs, addr) + continue + } listener, normalisedAddr, err := proxy.listen(addr) if err != nil { Log.Fatalf("Cannot listen on %s: %s", addr, err) @@ -195,6 +219,20 @@ func (proxy *Proxy) Listen() []net.Listener { proxy.normalisedAddrs = append(proxy.normalisedAddrs, normalisedAddr) } + if len(unixAddrs) > 0 { + listener, _, err := proxy.listen(weaveSockUnix) + if err != nil { + Log.Fatalf("Cannot listen on %s: %s", weaveSockUnix, err) + } + listeners = append(listeners, listener) + + if err := proxy.symlink(unixAddrs); err != nil { + Log.Fatalf("Cannot listen on unix sockets: %s", err) + } + + proxy.normalisedAddrs = append(proxy.normalisedAddrs, weaveSockUnix) + } + for _, addr := range proxy.normalisedAddrs { Log.Infoln("proxy listening on", addr) } @@ -291,8 +329,10 @@ func (proxy *Proxy) listen(protoAndAddr string) (net.Listener, string, error) { if err != nil { return nil, "", err } - if err = copyOwnerAndPermissions(dockerSock, addr); err != nil { - return nil, "", err + if strings.HasPrefix(proxy.Config.DockerHost, "unix://") { + if err = copyOwnerAndPermissions(strings.TrimPrefix(proxy.Config.DockerHost, "unix://"), addr); err != nil { + return nil, "", err + } } default: @@ -405,6 +445,9 @@ func (proxy *Proxy) attach(container *docker.Container, orDie bool) error { } } } + if proxy.NoMulticastRoute { + args = append(args, "--no-multicast-route") + } if orDie { args = append(args, "--or-die") } @@ -476,10 +519,6 @@ func (proxy *Proxy) getDNSDomain() (domain string) { return "" } - if proxy.WithDNS { - domain = nameserver.DefaultDomain - } - weaveContainer, err := proxy.client.InspectContainer("weave") var weaveIP string if err == nil && weaveContainer.NetworkSettings != nil { @@ -527,3 +566,76 @@ func (proxy *Proxy) updateContainerNetworkSettings(container jsonObject) error { networkSettings["IPPrefixLen"], _ = nets[0].Mask.Size() return nil } + +func (proxy *Proxy) symlink(unixAddrs []string) (err error) { + var container *docker.Container + binds := []string{"/var/run/weave:/var/run/weave"} + froms := []string{} + for _, addr := range unixAddrs { + if addr == weaveSockUnix { + continue + } + from := strings.TrimPrefix(addr, "unix://") + dir := filepath.Dir(from) + binds = append(binds, dir+":"+filepath.Join("/host", dir)) + froms = append(froms, filepath.Join("/host", from)) + proxy.normalisedAddrs = append(proxy.normalisedAddrs, addr) + } + if len(froms) == 0 { + return + } + + env := []string{ + "PATH=/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + } + + if val := os.Getenv("WEAVE_DEBUG"); val != "" { + env = append(env, fmt.Sprintf("%s=%s", "WEAVE_DEBUG", val)) + } + + container, err = proxy.client.CreateContainer(docker.CreateContainerOptions{ + Config: &docker.Config{ + Image: proxy.Image, + Entrypoint: []string{"/home/weave/symlink", weaveSock}, + Cmd: froms, + Env: env, + }, + HostConfig: &docker.HostConfig{Binds: binds}, + }) + if err != nil { + return + } + + defer func() { + err2 := proxy.client.RemoveContainer(docker.RemoveContainerOptions{ID: container.ID}) + if err == nil { + err = err2 + } + }() + + err = proxy.client.StartContainer(container.ID, nil) + if err != nil { + return + } + + var buf bytes.Buffer + err = proxy.client.AttachToContainer(docker.AttachToContainerOptions{ + Container: container.ID, + ErrorStream: &buf, + Logs: true, + Stderr: true, + }) + if err != nil { + return + } + + var rc int + rc, err = proxy.client.WaitContainer(container.ID) + if err != nil { + return + } + if rc != 0 { + err = errors.New(buf.String()) + } + return +} diff --git a/proxy/start_container_interceptor.go b/proxy/start_container_interceptor.go index 6e68059e..678fbacf 100644 --- a/proxy/start_container_interceptor.go +++ b/proxy/start_container_interceptor.go @@ -41,8 +41,14 @@ func (i *startContainerInterceptor) InterceptRequest(r *http.Request) error { return err } } else { - if err := addVolume(hostConfig, i.proxy.weaveWaitVolume, "/w", "ro"); err != nil { - return err + if i.proxy.NoMulticastRoute { + if err := addVolume(hostConfig, i.proxy.weaveWaitNomcastVolume, "/w", "ro"); err != nil { + return err + } + } else { + if err := addVolume(hostConfig, i.proxy.weaveWaitVolume, "/w", "ro"); err != nil { + return err + } } if dnsDomain := i.proxy.getDNSDomain(); dnsDomain != "" { if err := i.proxy.setWeaveDNS(hostConfig, container.Config.Hostname, dnsDomain); err != nil { diff --git a/router/fastdp.go b/router/fastdp.go index b4e7eeff..c23f1a38 100644 --- a/router/fastdp.go +++ b/router/fastdp.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "sync" + "syscall" "time" "github.com/weaveworks/go-odp/odp" @@ -40,6 +41,7 @@ type FastDatapath struct { dpif *odp.Dpif dp odp.DatapathHandle deleteFlowsCount uint64 + missCount uint64 missHandlers map[odp.VportID]missHandler localPeer *mesh.Peer peers *mesh.Peers @@ -109,6 +111,10 @@ func NewFastDatapath(config FastDatapathConfig) (*FastDatapath, error) { forwarders: make(map[mesh.PeerName]*fastDatapathForwarder), } + // This delete happens asynchronously in the kernel, meaning that + // we can sometimes fail to recreate the vxlan vport with EADDRINUSE - + // consequently we retry a small number of times in + // getVxlanVportIDHarder() to compensate. if err := fastdp.deleteVxlanVports(); err != nil { return nil, err } @@ -125,7 +131,7 @@ func NewFastDatapath(config FastDatapathConfig) (*FastDatapath, error) { // numbers to be independent, but working out how to specify // them on the connecting side. So we can wait to find out if // anyone wants that. - fastdp.mainVxlanVportID, err = fastdp.getVxlanVportID(config.Port + 1) + fastdp.mainVxlanVportID, err = fastdp.getVxlanVportIDHarder(config.Port+1, 5, time.Millisecond*10) if err != nil { return nil, err } @@ -216,8 +222,13 @@ func (fastdp fastDatapathBridge) String() string { return fmt.Sprint(fastdp.dpname, " (via ODP)") } -func (fastDatapathBridge) Stats() map[string]int { - return nil +func (fastdp fastDatapathBridge) Stats() map[string]int { + lock := fastdp.startLock() + defer lock.unlock() + + return map[string]int{ + "FlowMisses": int(fastdp.missCount), + } } var routerBridgePortID = bridgePortID{router: true} @@ -400,6 +411,20 @@ func (fastdp fastDatapathOverlay) StartConsumingPackets(localPeer *mesh.Peer, pe return nil } +func (fastdp *FastDatapath) getVxlanVportIDHarder(udpPort int, retries int, duration time.Duration) (odp.VportID, error) { + var vxlanVportID odp.VportID + var err error + for try := 0; try < retries; try++ { + vxlanVportID, err = fastdp.getVxlanVportID(udpPort) + if err == nil || err != odp.NetlinkError(syscall.EADDRINUSE) { + return vxlanVportID, err + } + log.Warning("Address already in use creating vxlan vport ", udpPort, " - retrying") + time.Sleep(duration) + } + return 0, err +} + func (fastdp *FastDatapath) getVxlanVportID(udpPort int) (odp.VportID, error) { fastdp.lock.Lock() defer fastdp.lock.Unlock() @@ -509,9 +534,8 @@ type fastDatapathForwarder struct { func (fastdp fastDatapathOverlay) PrepareConnection(params mesh.OverlayConnectionParams) (mesh.OverlayConnection, error) { if params.SessionKey != nil { - // No encryption suport in fastdp. The weaver main.go - // is responsible for ensuring this doesn't happen. - log.Fatal("Attempt to use FastDatapath with encryption") + // No encryption support in fastdp + return nil, fmt.Errorf("encryption not supported") } vxlanVportID := fastdp.mainVxlanVportID @@ -937,6 +961,8 @@ func (fastdp *FastDatapath) Miss(packet []byte, fks odp.FlowKeys) error { lock := fastdp.startLock() defer lock.unlock() + fastdp.missCount++ + handler := fastdp.getMissHandler(ingress) if handler == nil { return nil diff --git a/router/overlay_switch.go b/router/overlay_switch.go index 9e6efd00..da4bab78 100644 --- a/router/overlay_switch.go +++ b/router/overlay_switch.go @@ -204,8 +204,16 @@ func (osw *OverlaySwitch) PrepareConnection(params mesh.OverlayConnectionParams) subConn, err := overlay.PrepareConnection(params) if err != nil { - fwd.stopFrom(0) - return nil, err + log.Infof("Unable to use %s for connection to %s(%s): %s", + overlay.name, + params.RemotePeer.Name, + params.RemotePeer.NickName, + err) + // failed to start subforwarder - record overlay name and continue + fwd.forwarders[i] = subForwarder{ + overlayName: overlay.name, + } + continue } subFwd := subConn.(OverlayForwarder) diff --git a/site/building.md b/site/building.md index 41b0bfe1..a94ace3b 100644 --- a/site/building.md +++ b/site/building.md @@ -11,7 +11,7 @@ You only need to build weave if you want to work on the weave codebase Apart from the `weave` shell script, weave is delivered as a set of container images. There is no distribution-specific packaging, so in principle it shouldn't matter which Linux distribution you build -under. But naturally, Docker is a prerequisite (version 1.3.1 or +under. But naturally, Docker is a prerequisite (version 1.6.0 or later). And it is difficult to build under Fedora because [Fedora does not include static libraries](http://fedoraproject.org/wiki/Packaging:Guidelines#Packaging_Static_Libraries). diff --git a/site/features.md b/site/features.md index 4dbbb7ab..f69393b7 100644 --- a/site/features.md +++ b/site/features.md @@ -63,7 +63,9 @@ Weave automatically chooses the fastest available method to transport data between peers. The most performant of these ('fastdp') offers near-native throughput and latency but does not support encryption; consequently supplying a password will cause the router to fall back -to a slower mode ('sleeve') that does. +to a slower mode ('sleeve') that does, for connections that traverse +untrusted networks (see the [security](#security) section for more +details). Even when encryption is not in use, certain adverse network conditions will cause this fallback to occur dynamically; in these circumstances, @@ -91,21 +93,12 @@ re-attached to the weave network by the weave Docker API proxy. ### Docker network plugin -Alternatively, you can use weave as a Docker plugin. First you must -configure the Docker daemon to use a cluster store as documented -[here](https://docs.docker.com/engine/userguide/networking/dockernetworks/#an-overlay-network); -you can then launch weave and create a network: - - $ weave launch - $ weave launch-plugin - $ docker network create --driver=weave weave - -and then start a container: +Alternatively, you can use weave as a Docker plugin. A Docker network +named `weave` is created by `weave launch`, which you can use like this: $ docker run --net=weave -ti ubuntu -For more details see the -[plugin README](https://github.com/weaveworks/docker-plugin). +For more details see the [plugin documentation](plugin.html). ### Address allocation @@ -321,10 +314,16 @@ way to generate a random password which satsifies this requirement is < /dev/urandom tr -dc A-Za-z0-9 | head -c9 ; echo -The same password must be specified for all weave peers. Note that -supplying a password will [cause weave to fall back to a slower -method](#fast-data-path) for transporting data between -peers. +The same password must be specified for all weave peers; by default +both control and data plane traffic will then use authenticated +encryption. If some of your peers are colocated in a trusted network +(for example within the boundary of your own datacentre) you can use +the `--trusted-subnets` argument to `weave launch` to selectively +disable data plane encryption as an optimisation. Both peers must +consider the other to be in a trusted subnet for this to take place - +if they do not, weave will [fall back to a slower +method](#fast-data-path) for transporting data between peers as fast +datapath does not support encryption. Be aware that: diff --git a/site/plugin.md b/site/plugin.md new file mode 100644 index 00000000..21e18821 --- /dev/null +++ b/site/plugin.md @@ -0,0 +1,92 @@ +--- +title: Using Weave Net via Docker Networking +layout: default +--- + +New in Docker version 1.9 is a plugin mechanism to add different +network providers. + +To install, download the `weave` script as per the [readme][]. + +The Weave Net plugin runs automatically when you `weave launch`. You +must still tell the weave peers to connect to one another, either via +`weave launch` or `weave connect`. + +Note that the plugin container (`weaveplugin`) is run with +`--restart=always`, so that it is there after a restart or reboot. If +you remove this container (e.g. using `weave reset`) before removing +all endpoints created using `--net=weave`, Docker can +[hang](https://github.com/docker/libnetwork/issues/813). + +## Starting a container: + + $ docker run --net=weave -ti ubuntu + +## Using WeaveDNS for Service Discovery: + +You need to pass the additional arguments `--dns` and `-dns-search`, +for which we provide a helper in the weave script: + + $ docker run --net=weave -h foo.weave.local $(weave dns-args) -tdi ubuntu + $ docker run --net=weave -h bar.weave.local $(weave dns-args) -ti ubuntu + # ping foo + +For more details see the [Weave Service Discovery documentation][service-discovery]. + +## Under the hood + +The Weave Net plugin actually provides *two* network drivers to Docker +- one named `weavemesh` that can operate without a [cluster +store](#cluster-store) (like Docker's overlay driver) and one named +`weave` that can only work with one. + +### `weavemesh` driver + +* Weave handles all co-ordination between hosts (referred to by Docker as a "local scope" driver) +* Supports only a single network (we create one named `weave` for you automatically) +* Uses Weave's partition tolerant IPAM + +### `weave` driver + +* This runs in what Docker call "global scope"; requires a cluster store +* Supports multiple networks (these must be created with `docker network create --driver weave ...`) +* Used with Docker's cluster store based IPAM + +There's no specific documentation from Docker on using a cluster +store, but the first part of [Getting Started with Docker Multi-host Networking][docker-net] +should point the way. + +Note that in the case of multiple docker networks, all containers are +on the same virtual network but Docker allocates their addresses on +different subnets so they cannot talk to each other directly. + +## Plugin command-line arguments + +If you want to give some arguments to the plugin independently, don't +use `weave launch`; instead run: + + $ weave launch-router [other peers] + $ weave launch-plugin [plugin arguments] + +The plugin command-line arguments are: + + * `--log-level=debug|info|warning|error`, which tells the plugin + how much information to emit for debugging. + * `--mesh-network-name=`: set it to blank to disable creation + of a default network, or a name of your own choice. + * `--no-multicast-route`: stop weave adding a static IP route for + multicast traffic on its interface + +## Restarting + +We start the plugin with a restart policy, because Docker attempts to +recreate networks on startup. If it isn't started automatically by +Docker with a restart policy, it's not possible to start it manually +because Docker is blocked. + +Specifically, we use the policy `on-failure:3`, so Docker will give up +after three attempts if something just isn't working. + +[readme]: https://github.com/weaveworks/weave/blob/master/README.md#installation +[service-discovery]: weavedns.html +[docker-net]: https://github.com/docker/docker/blob/master/docs/userguide/networking/get-started-overlay.md diff --git a/site/proxy.md b/site/proxy.md index c34fbe30..8f4285b4 100644 --- a/site/proxy.md +++ b/site/proxy.md @@ -231,11 +231,11 @@ be able to launch a TLS-enabled proxy with: host1$ weave launch-proxy -You can also manually configure the proxy's TLS. This is accomplished -by launching the proxy with the same TLS-related command-line flags as -supplied to the docker daemon. For example, if you have generated your -certificates and keys into the docker host's `/tls` directory, we can -launch the proxy with: +To disable auto-detection of TLS configuration, you can either pass +the `--no-detect-tls` flag, or manually configure the proxy's TLS with +the same TLS-related command-line flags as supplied to the docker +daemon. For example, if you have generated your certificates and keys +into the docker host's `/tls` directory, we can launch the proxy with: host1$ weave launch-proxy --tlsverify --tlscacert=/tls/ca.pem \ --tlscert=/tls/server-cert.pem --tlskey=/tls/server-key.pem diff --git a/site/troubleshooting.md b/site/troubleshooting.md index bc5bbb2b..e6272849 100644 --- a/site/troubleshooting.md +++ b/site/troubleshooting.md @@ -68,29 +68,33 @@ A status summary can be obtained with `weave status`: ```` $ weave status - Version: 1.1.0 + Version: 1.1.0 - Service: router - Protocol: weave 1..2 - Name: 4a:0f:f6:ec:1c:93(host1) - Encryption: disabled - PeerDiscovery: enabled - Targets: [192.168.48.14 192.168.48.15] - Connections: 5 (1 established, 1 pending, 1 retrying, 1 failed, 1 connecting) - Peers: 3 (with 5 established, 1 pending connections) + Service: router + Protocol: weave 1..2 + Name: 4a:0f:f6:ec:1c:93(host1) + Encryption: disabled + PeerDiscovery: enabled + Targets: [192.168.48.14 192.168.48.15] + Connections: 5 (1 established, 1 pending, 1 retrying, 1 failed, 1 connecting) + Peers: 3 (with 5 established, 1 pending connections) + TrustedSubnets: none - Service: ipam - Consensus: achieved - Range: 10.32.0.0-10.47.255.255 - DefaultSubnet: 10.32.0.0/12 + Service: ipam + Consensus: achieved + Range: 10.32.0.0-10.47.255.255 + DefaultSubnet: 10.32.0.0/12 - Service: dns - Domain: weave.local. - TTL: 1 - Entries: 9 + Service: dns + Domain: weave.local. + TTL: 1 + Entries: 9 - Service: proxy - Address: tcp://127.0.0.1:12375 + Service: proxy + Address: tcp://127.0.0.1:12375 + + Service: plugin + DriverName: weave ```` @@ -129,6 +133,9 @@ state. Further details are available with number of connections peers have to other peers. Further details are available with [`weave status peers`](#weave-status-peers). +'TrustedSubnets' shows subnets which the router trusts as specified by +the `--trusted-subnets` option to `weave launch`. + There are further sections for the [IP address allocator](ipam.html#troubleshooting), [weaveDNS](weavedns.html#troubleshooting), and [Weave Docker API @@ -141,8 +148,8 @@ obtained with `weave status connections`: ```` $ weave status connections -<- 192.168.48.12:33866 established fastdp 7e:21:4a:70:2f:45(host2) -<- 192.168.48.13:60773 pending fastdp 7e:ae:cd:d5:23:8d(host3) +<- 192.168.48.12:33866 established unencrypted fastdp 7e:21:4a:70:2f:45(host2) +<- 192.168.48.13:60773 pending encrypted fastdp 7e:ae:cd:d5:23:8d(host3) -> 192.168.48.14:6783 retrying dial tcp4 192.168.48.14:6783: no route to host -> 192.168.48.15:6783 failed dial tcp4 192.168.48.15:6783: no route to host, retry: 2015-08-06 18:55:38.246910357 +0000 UTC -> 192.168.48.16:6783 connecting @@ -162,8 +169,8 @@ The columns are as follows: heartbeat * `established` - TCP connection and corresponding UDP path are up * Info - the failure reason for failed and retrying connections, or - the data transport method, remote peer name and nickname for - pending and established connections + the encryption mode, data transport method, remote peer name and + nickname for pending and established connections ### List peers @@ -228,6 +235,12 @@ inspect`: $ weave report -f {% raw %}'{{.DNS.Domain}}'{% endraw %} weave.local. +Weave adds a template function, `json`, which can be applied to get +results in JSON format. + + $ weave report -f {% raw %}'{{json .DNS}}'{% endraw %} + {% raw %}{"Domain":"weave.local.","Upstream":["8.8.8.8","8.8.4.4"],"Address":"172.17.0.1:53","TTL":1,"Entries":null}{% endraw %} + ### List attached containers weave ps diff --git a/site/weavedns.md b/site/weavedns.md index 5b9458f5..8246212f 100644 --- a/site/weavedns.md +++ b/site/weavedns.md @@ -259,6 +259,7 @@ DNS: Service: dns Domain: weave.local. + Upstream: 8.8.8.8, 8.8.4.4 TTL: 1 Entries: 9 @@ -271,6 +272,7 @@ guide](troubleshooting.html#weave-status) for more detail. The 'Service: dns' section is pertinent to weaveDNS, and includes: * The local domain suffix which is being served +* The list of upstream servers used for resolving names not in the local domain * The response ttl * The total number of entries diff --git a/test/110_encryption_2_test.sh b/test/110_encryption_2_test.sh index 5f46a463..205a9928 100755 --- a/test/110_encryption_2_test.sh +++ b/test/110_encryption_2_test.sh @@ -14,4 +14,7 @@ start_container $HOST1 $C1/24 --name=c1 start_container $HOST2 $C2/24 --name=c2 assert_raises "exec_on $HOST1 c1 $PING $C2" +assert_raises "weave_on $HOST1 status connections | grep encrypted" +assert_raises "weave_on $HOST2 status connections | grep encrypted" + end_suite diff --git a/test/115_optional_encryption_2_test.sh b/test/115_optional_encryption_2_test.sh new file mode 100755 index 00000000..a84f2e65 --- /dev/null +++ b/test/115_optional_encryption_2_test.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +. ./config.sh + +start_suite "Optional encryption via trusted subnets" + +# Determine subnet for hosts given either an IP or name. We need to resolve +# these entries on the remote hosts to make sure we get the private +# IP addresses in the Circle/GCE context +HOST1_IP=$($SSH $HOST1 "getent hosts $HOST1" | grep $HOST1 | cut -d ' ' -f 1) +HOST2_IP=$($SSH $HOST2 "getent hosts $HOST2" | grep $HOST2 | cut -d ' ' -f 1) +HOST1_CIDR=$($SSH $HOST1 "ip addr show" | grep -oP $HOST1_IP/[0-9]+) +HOST2_CIDR=$($SSH $HOST2 "ip addr show" | grep -oP $HOST2_IP/[0-9]+) + +# Check asymmetric trust - connections should be encrypted +weave_on $HOST1 launch --password wfvAwt7sj --trusted-subnets $HOST2_CIDR +weave_on $HOST2 launch --password wfvAwt7sj $HOST1 +assert_raises "weave_on $HOST1 status connections | grep encrypted" +assert_raises "weave_on $HOST2 status connections | grep encrypted" + +weave_on $HOST1 stop +weave_on $HOST2 stop + +# Check symmetric trust - overlay in plaintext +weave_on $HOST1 launch --password wfvAwt7sj --trusted-subnets $HOST2_CIDR +weave_on $HOST2 launch --password wfvAwt7sj --trusted-subnets $HOST1_CIDR $HOST1 +assert_raises "weave_on $HOST1 status connections | grep unencrypted" +assert_raises "weave_on $HOST2 status connections | grep unencrypted" + +end_suite diff --git a/test/195_no_multicast_route_1_test.sh b/test/195_no_multicast_route_1_test.sh new file mode 100755 index 00000000..0b5cd457 --- /dev/null +++ b/test/195_no_multicast_route_1_test.sh @@ -0,0 +1,33 @@ +#! /bin/bash + +. ./config.sh + +show_multicast_route_on() { + exec_on $1 $2 ip route show 224.0.0.0/4 +} + +start_suite "--no-multicast-route operation" + +# Ensure containers run either way have no multicast route +weave_on $HOST1 launch-router +weave_on $HOST1 launch-proxy --no-multicast-route +weave_on $HOST1 launch-plugin --no-multicast-route + +start_container $HOST1 --no-multicast-route --name c1 +proxy_start_container $HOST1 --name c2 +start_container_local_plugin $HOST1 --name=c3 + +assert "show_multicast_route_on $HOST1 c1" +assert "show_multicast_route_on $HOST1 c2" +assert "show_multicast_route_on $HOST1 c3" + +# Ensure current proxy options are obeyed on container start +docker_on $HOST1 stop c2 +weave_on $HOST1 stop-proxy +weave_on $HOST1 launch-proxy + +proxy docker_on $HOST1 start c2 + +assert "show_multicast_route_on $HOST1 c2" "224.0.0.0/4 dev ethwe " + +end_suite diff --git a/test/280_with_or_without_dns_test.sh b/test/280_with_or_without_dns_test.sh index 21051f08..b75f14ce 100755 --- a/test/280_with_or_without_dns_test.sh +++ b/test/280_with_or_without_dns_test.sh @@ -27,6 +27,5 @@ start_container $HOST1 $TARGET_IP/24 --name c2 -h $TARGET check_dns assert_no_dns_record --without-dns check_dns assert_dns_record -check_dns assert_dns_record --with-dns end_suite diff --git a/test/600_proxy_docker_py.sh b/test/600_proxy_docker_py.sh index ce9a0250..724fbf4a 100755 --- a/test/600_proxy_docker_py.sh +++ b/test/600_proxy_docker_py.sh @@ -9,18 +9,20 @@ docker_py_test() { start_suite "Run docker-py test suite against the proxy" + # Work round https://github.com/docker/docker-py/issues/852 + docker_on $HOST1 pull busybox:buildroot-2014.02 >/dev/null # Get a list of the tests for use to shard docker_on $HOST1 pull joffrey/docker-py >/dev/null CANDIDATES=$(docker_on $HOST1 run \ joffrey/docker-py \ - py.test --collect-only tests/integration_test.py \ - | sed -En "s/\s*/\1/p") + py.test --collect-only tests/integration/ \ + | sed -En "s/\s*/\1/p") i=0 TESTS= for test in $CANDIDATES; do if [ $(($i % $TOTAL_SHARDS)) -eq $SHARD ]; then - TESTS="$TESTS tests/integration_test.py::$test" + TESTS="$TESTS $test" fi i=$(($i + 1)) done diff --git a/test/630_proxy_dns_test.sh b/test/630_proxy_dns_test.sh index 63b9f694..0c6d6e43 100755 --- a/test/630_proxy_dns_test.sh +++ b/test/630_proxy_dns_test.sh @@ -23,11 +23,6 @@ start_suite "Proxy registers containers with dns" bridge_ip=$(weave_on $HOST1 docker-bridge-ip) -# Assert behaviour without weaveDNS running, but dns forced -weave_on $HOST1 launch-proxy --with-dns -do_assert_resolution assert_no_dns_record -weave_on $HOST1 stop-proxy - # Assert behaviour without weaveDNS weave_on $HOST1 launch-proxy do_assert_resolution assert_no_dns_record diff --git a/test/650_proxy_env_test.sh b/test/650_proxy_env_test.sh index 593f3d6d..bfcbbe46 100755 --- a/test/650_proxy_env_test.sh +++ b/test/650_proxy_env_test.sh @@ -28,4 +28,10 @@ weave_on $HOST1 stop weave_on $HOST1 launch-proxy -H tcp://0.0.0.0:12375 -H unix:///var/run/weave/weave.sock check +# Check we can use weave env/config with unix -Hs specified +weave_on $HOST1 stop +run_on $HOST1 "COVERAGE=$COVERAGE weave launch-proxy -H unix:///var/run/weave/weave.sock" +assert_raises "run_on $HOST1 'eval \$(weave env) ; docker $CMD'" +assert_raises "run_on $HOST1 'docker \$(weave config) $CMD'" + end_suite diff --git a/test/690_proxy_autoconfig_test.sh b/test/690_proxy_config_test.sh similarity index 63% rename from test/690_proxy_autoconfig_test.sh rename to test/690_proxy_config_test.sh index ce6e18b2..11ddcbda 100755 --- a/test/690_proxy_autoconfig_test.sh +++ b/test/690_proxy_config_test.sh @@ -2,10 +2,10 @@ . ./config.sh -start_suite "Boot the proxy should only listen on client's interface" +start_suite "Various launch-proxy configurations" # Booting it over unix socket listens on unix socket -run_on $HOST1 COVERAGE=$COVERAGE sudo -E weave launch-proxy +run_on $HOST1 COVERAGE=$COVERAGE weave launch-proxy assert_raises "run_on $HOST1 sudo docker -H unix:///var/run/weave/weave.sock ps" assert_raises "proxy docker_on $HOST1 ps" 1 weave_on $HOST1 stop-proxy @@ -17,11 +17,21 @@ assert_raises "proxy docker_on $HOST1 ps" weave_on $HOST1 stop-proxy # Booting it over tcp (no prefix) listens on tcp -DOCKER_CLIENT_ARGS="-H $HOST1:$DOCKER_PORT" $WEAVE launch-proxy +DOCKER_HOST=tcp://$HOST1:$DOCKER_PORT $WEAVE launch-proxy assert_raises "run_on $HOST1 sudo docker -H unix:///var/run/weave/weave.sock ps" 1 assert_raises "proxy docker_on $HOST1 ps" weave_on $HOST1 stop-proxy +# Booting it with -H outside /var/run/weave, still works +socket="$($SSH $HOST1 mktemp -d)/weave.sock" +weave_on $HOST1 launch-proxy -H unix://$socket +assert_raises "run_on $HOST1 sudo docker -H unix:///$socket ps" 0 +weave_on $HOST1 stop-proxy + +# Booting it against non-standard docker unix sock +run_on $HOST1 "DOCKER_HOST=unix:///var/run/alt-docker.sock COVERAGE=$COVERAGE weave launch-proxy -H tcp://0.0.0.0:12375" +assert_raises "proxy docker_on $HOST1 ps" +weave_on $HOST1 stop-proxy # Booting it over tls errors assert_raises "DOCKER_CLIENT_ARGS='--tls' weave_on $HOST1 launch-proxy" 1 diff --git a/test/800_plugin_cross_hosts_2_test.sh b/test/800_plugin_cross_hosts_2_test.sh new file mode 100755 index 00000000..47f82d50 --- /dev/null +++ b/test/800_plugin_cross_hosts_2_test.sh @@ -0,0 +1,21 @@ +#! /bin/bash + +. ./config.sh + +C1_NAME=c1.weave.local +C2_NAME=seetwo.weave.local + +start_suite "Ping and DNS over cross-host weave network (via 'local' plugin)" + +DNS_IP=$(weave_on $host docker-bridge-ip) + +weave_on $HOST1 launch +weave_on $HOST2 launch $HOST1 + +start_container_local_plugin $HOST1 --name=c1 --hostname=$C1_NAME --dns=$DNS_IP +start_container_local_plugin $HOST2 --name=c2 --hostname=$C2_NAME --dns=$DNS_IP + +assert_raises "exec_on $HOST1 c1 $PING $C2_NAME" +assert_raises "exec_on $HOST2 c2 $PING $C1_NAME" + +end_suite diff --git a/test/README.md b/test/README.md index d7358b86..804c226e 100644 --- a/test/README.md +++ b/test/README.md @@ -2,7 +2,7 @@ This directory contains integration tests for weave. ## Requirements -You need two VMs with docker >=1.3.1 installed and listening on TCP +You need two VMs with docker >=1.6.0 installed and listening on TCP port 2375 (see below). You also need to be able to ssh to these VMs, preferably without having to input anything. diff --git a/test/config.sh b/test/config.sh index f50e03b5..a5651428 100644 --- a/test/config.sh +++ b/test/config.sh @@ -151,7 +151,14 @@ start_container() { start_container_with_dns() { host=$1 shift 1 - weave_on $host run --with-dns "$@" -t $DNS_IMAGE /bin/sh + weave_on $host run "$@" -t $DNS_IMAGE /bin/sh +} + +start_container_local_plugin() { + host=$1 + shift 1 + # using ssh rather than docker -H because CircleCI docker client is older + $SSH $host docker run "$@" -dt --net=weave $SMALL_IMAGE /bin/sh } proxy_start_container() { @@ -217,8 +224,12 @@ assert_dns_ptr_record() { start_suite() { for host in $HOSTS; do [ -z "$DEBUG" ] || echo "Cleaning up on $host: removing all containers and resetting weave" + PLUGIN_ID=$(docker_on $host ps -aq --filter=name=weaveplugin) + PLUGIN_FILTER="cat" + [ -n "$PLUGIN_ID" ] && PLUGIN_FILTER="grep -v $PLUGIN_ID" + rm_containers $host $(docker_on $host ps -aq 2>/dev/null | $PLUGIN_FILTER) + run_on $host "docker network ls | grep -q ' weave ' && docker network rm weave" || true weave_on $host reset 2>/dev/null - rm_containers $host $(docker_on $host ps -aq 2>/dev/null) done whitely echo "$@" } diff --git a/test/gce.sh b/test/gce.sh index b1cc7cca..e42b9209 100755 --- a/test/gce.sh +++ b/test/gce.sh @@ -11,7 +11,7 @@ set -e : ${SSH_KEY_FILE:=$HOME/.ssh/gce_ssh_key} : ${PROJECT:=positive-cocoa-90213} : ${IMAGE:=ubuntu-14-04} -: ${TEMPLATE_NAME:=test-template-8} +: ${TEMPLATE_NAME:=test-template-9} : ${ZONE:=us-central1-a} : ${NUM_HOSTS:=5} SUFFIX="" @@ -73,7 +73,7 @@ curl -sSL https://get.docker.com/ | sh apt-get update -qq; apt-get install -q -y --force-yes --no-install-recommends ethtool; usermod -a -G docker vagrant; -echo 'DOCKER_OPTS="-H unix:///var/run/docker.sock -H tcp://0.0.0.0:2375 -s overlay"' >> /etc/default/docker; +echo 'DOCKER_OPTS="-H unix:///var/run/docker.sock -H unix:///var/run/alt-docker.sock -H tcp://0.0.0.0:2375 -s overlay"' >> /etc/default/docker; service docker restart EOF # It seems we need a short delay for docker to start up, so I put this in @@ -140,7 +140,7 @@ function hosts { hosts="$hostname $hosts" args="--add-host=$hostname:$(internal_ip $json $name) $args" done - echo export SSH=ssh + echo export SSH=\"ssh -l vagrant\" echo export HOSTS=\"$hosts\" echo export ADD_HOST_ARGS=\"$args\" rm $json diff --git a/test/setup.sh b/test/setup.sh index 9312fc20..1e8ce207 100755 --- a/test/setup.sh +++ b/test/setup.sh @@ -6,7 +6,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")" . ./config.sh -(cd ./tls && go get -tags netgo ./... && go run generate_certs.go $HOSTS) +(cd ./tls && ./tls $HOSTS) echo "Copying weave images, scripts, and certificates to hosts, and" echo " prefetch test images" @@ -19,7 +19,7 @@ setup_host() { run_on $HOST mkdir -p bin upload_executable $HOST ../bin/docker-ns upload_executable $HOST ../weave - rsync -az -e "$SSH" ./tls/ $HOST:~/tls + rsync -az -e "$SSH" --exclude=tls ./tls/ $HOST:~/tls for IMG in $TEST_IMAGES ; do docker_on $HOST inspect --format=" " $IMG >/dev/null 2>&1 || docker_on $HOST pull $IMG done diff --git a/testing/util.go b/testing/util.go index 42f39cee..aed7ae31 100644 --- a/testing/util.go +++ b/testing/util.go @@ -6,7 +6,6 @@ import ( "reflect" "runtime" "testing" - "time" "github.com/stretchr/testify/require" ) @@ -36,22 +35,6 @@ func stackTrace(all bool) string { return string(buf[:stacklen]) } -// Borrowed from net/http tests: -// goTimeout runs f, failing t if f takes more than d to complete. -func RunWithTimeout(t *testing.T, d time.Duration, f func()) { - ch := make(chan bool, 2) - timer := time.AfterFunc(d, func() { - t.Errorf("Timeout expired after %v: stacks:\n%s", d, stackTrace(true)) - ch <- true - }) - defer timer.Stop() - go func() { - defer func() { ch <- true }() - f() - }() - <-ch -} - // TrimTestArgs finds the first -- in os.Args and trim all args before that func TrimTestArgs() { i, l := 0, len(os.Args) diff --git a/vagrant-common.rb b/vagrant-common.rb index 15efa7ac..1cfeb599 100644 --- a/vagrant-common.rb +++ b/vagrant-common.rb @@ -45,7 +45,7 @@ end def tweak_docker_daemon(vm) vm.provision :shell, :inline => <