Add release v1.7.0

Implement `--expect-npc` flag to `create-bridge`

.gitmodules

@@ -76,3 +76,9 @@
 [submodule "vendor/github.com/aws/aws-sdk-go"]
 	path = vendor/github.com/aws/aws-sdk-go
 	url = https://github.com/aws/aws-sdk-go
+[submodule "vendor/golang.org/x/sys"]
+	path = vendor/golang.org/x/sys
+	url = https://go.googlesource.com/sys
+[submodule "vendor/github.com/weaveworks/docker"]
+	path = vendor/github.com/weaveworks/docker
+	url = https://github.com/weaveworks/docker

CHANGELOG.md

@@ -1,3 +1,12 @@
+## Release 1.7.0
+
+Highlights:
+
+* weave-kube - Deploy Weave Net to Kubernetes with a single command
+* weave-npc - Kubernetes network policy controller implementation
+
+More details in the [change log](https://github.com/weaveworks/weave/issues?q=milestone%3A1.7.0).
+
 ## Release 1.6.2
 
 Bug fixes and minor improvements

common/mflagext/listvar.go

@@ -3,7 +3,7 @@ package mflagext
 import (
 	"fmt"
 
-	"github.com/docker/docker/pkg/mflag"
+	"github.com/weaveworks/docker/pkg/mflag"
 )
 
 type listOpts struct {

nameserver/dns.go

@@ -28,7 +28,7 @@ const (
 )
 
 type Upstream interface {
-	Config() *dns.ClientConfig
+	Config() (*dns.ClientConfig, error)
 }
 
 func NewUpstream(resolvConf, filterAddress string) Upstream {
@@ -48,27 +48,31 @@ type upstream struct {
 	lastModified  time.Time
 }
 
-func (u *upstream) Config() *dns.ClientConfig {
+func (u *upstream) Config() (*dns.ClientConfig, error) {
 	u.Lock()
 	defer u.Unlock()
 
 	// If we checked less than five seconds ago return what we already have
 	now := time.Now()
 	if now.Sub(u.lastStat) < 5*time.Second {
-		return u.cachedConfig
+		return u.cachedConfig, nil
 	}
 	u.lastStat = now
 
 	// Reread file if it has changed
-	if fi, err := os.Stat(u.resolvConf); err == nil && fi.ModTime() != u.lastModified {
+	if fi, err := os.Stat(u.resolvConf); err != nil {
+		return u.cachedConfig, err
+	} else if fi.ModTime() != u.lastModified {
 		if config, err := dns.ClientConfigFromFile(u.resolvConf); err == nil {
 			config.Servers = filter(config.Servers, u.filterAddress)
 			u.lastModified = fi.ModTime()
 			u.cachedConfig = config
+		} else {
+			return u.cachedConfig, err
 		}
 	}
 
-	return u.cachedConfig
+	return u.cachedConfig, nil
 }
 
 func filter(ss []string, s string) []string {
@@ -257,7 +261,10 @@ func (h *handler) handleRecursive(w dns.ResponseWriter, req *dns.Msg) {
 		}
 	}
 
-	upstreamConfig := h.upstream.Config()
+	upstreamConfig, err := h.upstream.Config()
+	if err != nil {
+		h.ns.errorf("unable to read upstream config: %s", err)
+	}
 	for _, server := range upstreamConfig.Servers {
 		reqCopy := req.Copy()
 		reqCopy.Id = dns.Id()

nameserver/dns_test.go

@@ -19,8 +19,8 @@ type mockUpstream struct {
 	config *dns.ClientConfig
 }
 
-func (mu *mockUpstream) Config() *dns.ClientConfig {
-	return mu.config
+func (mu *mockUpstream) Config() (*dns.ClientConfig, error) {
+	return mu.config, nil
 }
 
 func startServer(t *testing.T, upstream *dns.ClientConfig) (*DNSServer, *Nameserver, int, int) {

nameserver/status.go

@@ -36,9 +36,10 @@ func NewStatus(ns *Nameserver, dnsServer *DNSServer) *Status {
 			entry.Tombstone})
 	}
 
+	upstreamConfig, _ := dnsServer.upstream.Config()
 	return &Status{
 		dnsServer.domain,
-		dnsServer.upstream.Config().Servers,
+		upstreamConfig.Servers,
 		dnsServer.address,
 		dnsServer.ttl,
 		entryStatusSlice}

net/veth.go

@@ -3,8 +3,10 @@ package net
 import (
 	"fmt"
 	"net"
+	"strings"
 	"time"
 
+	"github.com/coreos/go-iptables/iptables"
 	"github.com/j-keck/arping"
 	"github.com/vishvananda/netlink"
 	"github.com/vishvananda/netns"
@@ -112,6 +114,11 @@ func interfaceExistsInNamespace(ns netns.NsHandle, ifName string) bool {
 }
 
 func AttachContainer(ns netns.NsHandle, id, ifName, bridgeName string, mtu int, withMulticastRoute bool, cidrs []*net.IPNet, keepTXOn bool) error {
+	ipt, err := iptables.New()
+	if err != nil {
+		return err
+	}
+
 	if !interfaceExistsInNamespace(ns, ifName) {
 		maxIDLen := IFNAMSIZ - 1 - len(vethPrefix+"pl")
 		if len(id) > maxIDLen {
@@ -129,6 +136,9 @@ func AttachContainer(ns netns.NsHandle, id, ifName, bridgeName string, mtu int,
 				if err := ConfigureARPCache(ifName); err != nil {
 					return err
 				}
+				if err := ipt.Append("filter", "INPUT", "-i", ifName, "-d", "224.0.0.0/4", "-j", "DROP"); err != nil {
+					return err
+				}
 				return nil
 			}); err != nil {
 				return fmt.Errorf("error setting up interface: %s", err)
@@ -145,6 +155,21 @@ func AttachContainer(ns netns.NsHandle, id, ifName, bridgeName string, mtu int,
 		if err != nil {
 			return err
 		}
+
+		// Add multicast ACCEPT rules for new subnets
+		for _, ipnet := range newAddresses {
+			acceptRule := []string{"-i", ifName, "-s", subnet(ipnet), "-d", "224.0.0.0/4", "-j", "ACCEPT"}
+			exists, err := ipt.Exists("filter", "INPUT", acceptRule...)
+			if err != nil {
+				return err
+			}
+			if !exists {
+				if err := ipt.Insert("filter", "INPUT", 1, acceptRule...); err != nil {
+					return err
+				}
+			}
+		}
+
 		if err := netlink.LinkSetUp(veth); err != nil {
 			return err
 		}
@@ -177,6 +202,11 @@ func AttachContainer(ns netns.NsHandle, id, ifName, bridgeName string, mtu int,
 }
 
 func DetachContainer(ns netns.NsHandle, id, ifName string, cidrs []*net.IPNet) error {
+	ipt, err := iptables.New()
+	if err != nil {
+		return err
+	}
+
 	return WithNetNSLinkUnsafe(ns, ifName, func(veth netlink.Link) error {
 		existingAddrs, err := netlink.AddrList(veth, netlink.FAMILY_V4)
 		if err != nil {
@@ -194,7 +224,31 @@ func DetachContainer(ns netns.NsHandle, id, ifName string, cidrs []*net.IPNet) e
 		if err != nil {
 			return fmt.Errorf("failed to get IP address for %q: %v", veth.Attrs().Name, err)
 		}
+
+		// Remove multicast ACCEPT rules for subnets we no longer have addresses in
+		subnets := subnets(addrs)
+		rules, err := ipt.List("filter", "INPUT")
+		if err != nil {
+			return err
+		}
+		for _, rule := range rules {
+			ps := strings.Split(rule, " ")
+			if len(ps) == 10 &&
+				ps[0] == "-A" && ps[2] == "-s" && ps[4] == "-d" && ps[5] == "224.0.0.0/4" &&
+				ps[6] == "-i" && ps[7] == ifName && ps[8] == "-j" && ps[9] == "ACCEPT" {
+
+				if _, found := subnets[ps[3]]; !found {
+					if err := ipt.Delete("filter", "INPUT", ps[2:]...); err != nil {
+						return err
+					}
+				}
+			}
+		}
+
 		if len(addrs) == 0 { // all addresses gone: remove the interface
+			if err := ipt.Delete("filter", "INPUT", "-i", ifName, "-d", "224.0.0.0/4", "-j", "DROP"); err != nil {
+				return err
+			}
 			if err := netlink.LinkDel(veth); err != nil {
 				return err
 			}
@@ -202,3 +256,16 @@ func DetachContainer(ns netns.NsHandle, id, ifName string, cidrs []*net.IPNet) e
 		return nil
 	})
 }
+
+func subnet(ipn *net.IPNet) string {
+	ones, _ := ipn.Mask.Size()
+	return fmt.Sprintf("%s/%d", ipn.IP.Mask(ipn.Mask).String(), ones)
+}
+
+func subnets(addrs []netlink.Addr) map[string]struct{} {
+	subnets := make(map[string]struct{})
+	for _, addr := range addrs {
+		subnets[subnet(addr.IPNet)] = struct{}{}
+	}
+	return subnets
+}

plugin/net/driver.go

@@ -8,6 +8,7 @@ import (
 	"github.com/docker/libnetwork/drivers/remote/api"
 	"github.com/docker/libnetwork/netlabel"
 	"github.com/docker/libnetwork/types"
+	"golang.org/x/sys/unix"
 
 	"github.com/vishvananda/netlink"
 	weaveapi "github.com/weaveworks/weave/api"
@@ -94,11 +95,21 @@ func (driver *driver) DeleteNetwork(delreq *api.DeleteNetworkRequest) error {
 
 func (driver *driver) CreateEndpoint(create *api.CreateEndpointRequest) (*api.CreateEndpointResponse, error) {
 	driver.logReq("CreateEndpoint", create, create.EndpointID)
+	common.Log.Debugf("interface %+v", create.Interface)
 
 	if create.Interface == nil {
 		return nil, driver.error("CreateEndpoint", "Not supported: creating an interface from within CreateEndpoint")
 	}
-	resp := &api.CreateEndpointResponse{}
+
+	// create veths. note we assume endpoint IDs are unique in the first 9 chars
+	name, peerName := vethPair(create.EndpointID)
+	if _, err := weavenet.CreateAndAttachVeth(name, peerName, weavenet.WeaveBridgeName, 0, false, nil); err != nil {
+		return nil, driver.error("JoinEndpoint", "%s", err)
+	}
+
+	// Send back the MAC address
+	link, _ := netlink.LinkByName(peerName)
+	resp := &api.CreateEndpointResponse{Interface: &api.EndpointInterface{MacAddress: link.Attrs().HardwareAddr.String()}}
 
 	driver.logRes("CreateEndpoint", resp)
 	return resp, nil
@@ -106,6 +117,17 @@ func (driver *driver) CreateEndpoint(create *api.CreateEndpointRequest) (*api.Cr
 
 func (driver *driver) DeleteEndpoint(deleteReq *api.DeleteEndpointRequest) error {
 	driver.logReq("DeleteEndpoint", deleteReq, deleteReq.EndpointID)
+	name, _ := vethPair(deleteReq.EndpointID)
+	veth := &netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: name}}
+	if err := netlink.LinkDel(veth); err != nil {
+		// Try again using the name construction from earlier plugin version,
+		// in case user has upgraded with endpoints still extant
+		veth.Name = "vethwl" + deleteReq.EndpointID[:5]
+		if err2 := netlink.LinkDel(veth); err2 != nil {
+			// Note we report the first error
+			driver.warn("LeaveEndpoint", "unable to delete veth %q: %s", name, err)
+		}
+	}
 	return nil
 }
 
@@ -122,11 +144,7 @@ func (driver *driver) JoinEndpoint(j *api.JoinRequest) (*api.JoinResponse, error
 		return nil, driver.error("JoinEndpoint", "unable to find network info: %s", err)
 	}
 
-	name, peerName := vethPair(j.EndpointID)
-	if _, err := weavenet.CreateAndAttachVeth(name, peerName, weavenet.WeaveBridgeName, 0, false, nil); err != nil {
-		return nil, driver.error("JoinEndpoint", "%s", err)
-	}
-
+	_, peerName := vethPair(j.EndpointID)
 	response := &api.JoinResponse{
 		InterfaceName: &api.InterfaceName{
 			SrcName:   peerName,
@@ -186,12 +204,6 @@ func (driver *driver) setupNetworkInfo(id string, isOurs bool, options map[strin
 
 func (driver *driver) LeaveEndpoint(leave *api.LeaveRequest) error {
 	driver.logReq("LeaveEndpoint", leave, fmt.Sprintf("%s:%s", leave.NetworkID, leave.EndpointID))
-
-	name, _ := vethPair(leave.EndpointID)
-	veth := &netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: name}}
-	if err := netlink.LinkDel(veth); err != nil {
-		driver.warn("LeaveEndpoint", "unable to delete veth: %s", err)
-	}
 	return nil
 }
 
@@ -206,7 +218,8 @@ func (driver *driver) DiscoverDelete(disco *api.DiscoveryNotification) error {
 }
 
 func vethPair(id string) (string, string) {
-	return "vethwl" + id[:5], "vethwg" + id[:5]
+	// IFNAMSIZ is buffer length; subtract 6 for "vethwl" and 1 for terminating nul
+	return "vethwl" + id[:unix.IFNAMSIZ-7], "vethwg" + id[:unix.IFNAMSIZ-7]
 }
 
 // logging

prog/weaveproxy/main.go

@@ -5,7 +5,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/docker/docker/pkg/mflag"
+	"github.com/weaveworks/docker/pkg/mflag"
 	"github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/common/mflagext"
 	"github.com/weaveworks/weave/proxy"

prog/weaver/main.go

@@ -11,9 +11,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/docker/docker/pkg/mflag"
 	"github.com/gorilla/mux"
 	"github.com/pkg/profile"
+	"github.com/weaveworks/docker/pkg/mflag"
 	"github.com/weaveworks/mesh"
 
 	"github.com/weaveworks/weave/common"
@@ -322,6 +322,7 @@ func main() {
 	if errors := router.InitiateConnections(peers, false); len(errors) > 0 {
 		Log.Fatal(common.ErrorMessages(errors))
 	}
+	checkFatal(router.CreateRestartSentinel())
 
 	// The weave script always waits for a status call to succeed,
 	// so there is no point in doing "weave launch --http-addr ''".

router/network_router.go

@@ -239,12 +239,16 @@ func (router *NetworkRouter) InitialPeers(resume bool, peers []string) ([]string
 		return storedPeers, nil
 	}
 
-	sentinel, err := os.Create("restart.sentinel")
-	if err != nil {
-		return nil, fmt.Errorf("error creating sentinel: %v", err)
-	}
-	sentinel.Close()
-
 	log.Println("Launch detected - using supplied peer list:", peers)
 	return peers, nil
 }
+
+func (router *NetworkRouter) CreateRestartSentinel() error {
+	sentinel, err := os.Create("restart.sentinel")
+	if err != nil {
+		return fmt.Errorf("error creating sentinel: %v", err)
+	}
+	sentinel.Close()
+
+	return nil
+}

site/features.md

@@ -73,7 +73,7 @@ Containers started in this way that subsequently restart, either
 by an explicit `docker restart` command or by Docker restart 
 policy, are re-attached to the Weave network by the `Weave Docker API Proxy`.
 
-See [Using the Weave Docker API Proxy](/site/weave-docker-api/using-proxy.md).
+See [Integrating Docker via the API Proxy](/site/weave-docker-api.md).
 
 
 ###<a name="plugin"></a>Weave Network Docker Plugin
@@ -88,14 +88,14 @@ Using the Weave plugin enables you to take advantage of [Docker's network functi
 Also, Weave’s Docker Network plugin doesn't require an external cluster store and you can start and stop containers even 
 when there are network connectivity problems.
 
-See [Using the Weave Docker Network Plugin](/site/plugin.md) for more details.
+See [Integrating Docker via the Network Plugin](/site/plugin.md) for more details.
 
 
 ###<a name="cniplugin"></a>Weave Network CNI Plugin
 
-Weave can be used as a plugin to systems that support the [Container Network Interface](https://github.com/appc/cni), such as Kubernetes.
+Weave can be used as a plugin to systems that support the [Container Network Interface](https://github.com/appc/cni), such as Kubernetes and Mesosphere.
 
-See [Using the Weave CNI Plugin](/site/cni-plugin.md) for more details.
+See [Integrating Kubernetes and Mesos via the CNI Plugin](/site/cni-plugin.md) for more details.
 
 
 ###<a name="addressing"></a>IP Address Management (IPAM)
@@ -120,7 +120,7 @@ and are discoverable by using standard, simple name lookups:
 
 WeaveDNS also supports [load balancing](/site/weavedns/load-balance-fault-weavedns.md), [fault resilience](/site/weavedns/load-balance-fault-weavedns.md) and [hot swapping](/site/weavedns/managing-entries-weavedns.md). 
 
-See [Naming and discovery with WeaveDNS](/site/weavedns/how-works-weavedns.md).
+See [Discovering Containers with WeaveDNS](/site/weavedns.md).
  
 ###<a name="application-isolation"></a>Application Isolation
 

site/plugin/plugin-how-it-works.md

@@ -39,4 +39,4 @@ The plugin accepts the following options via `docker network create ... --opt`:
 **See Also**
 
  * [Integrating Docker via the Network Plugin](/site/plugin.md)
- 
+

site/weave-docker-api.md

@@ -3,15 +3,38 @@ title: Integrating Docker via the API Proxy
 menu_order: 50
 ---
 
-
 The Docker API proxy automatically attaches containers to the Weave
 network when they are started using the ordinary Docker
 [command-line interface](https://docs.docker.com/reference/commandline/cli/)
 or the [remote API](https://docs.docker.com/reference/api/docker_remote_api/),
 instead of `weave run`.
 
+###<a name="attaching-containers"></a>Attaching Containers to a Weave Network
 
-###Setting Up The Weave Net Docker API Proxy
+There are three ways to attach containers to a Weave network (which method to use is 
+entirely up to you):
+
+**1.** The Weave Net Docker API Proxy. See [Setting Up the Weave Net Docker API Proxy](#weave-api-proxy).  
+
+**2.**  The Docker Network Plugin framework. The Docker Network Plugin is used when 
+Docker containers are started with the --net flag, for example: 
+
+`docker run --net <docker-run-options>`
+
+**Where,** 
+
+ * `<docker-run-options>` are the [docker run options](https://docs.docker.com/engine/reference/run/) 
+ you give to your container on start 
+
+Note that if a Docker container is started with the --net flag, then the Weave Docker API Proxy
+is automatically disabled and is not used to attach containers. 
+See [Integrating Docker via the Network Plugin](plugin.md).
+
+**3.** Containers can also be attached to the Weave network with `weave run` commands. This method also
+does not use the Weave Docker API Proxy. 
+See [Launching Containers With Weave Run (without the Proxy)](/site/weave-docker-api/launching-without-proxy.md). 
+
+###<a name="weave-api-proxy"></a>Setting Up The Weave Net Docker API Proxy
 
 The proxy sits between the Docker client (command line or API) and the
 Docker daemon, and intercepts the communication between the two. You can

site/weavedns.md

@@ -44,5 +44,5 @@ disable weaveDNS itself, launch weave with the `--no-dns` option.
 
 **See Also**
 
- * [How Weave Finds Containers](/site/weave-docker-api.md)
+ * [Integrating Docker via the API Proxy](/site/weave-docker-api.md)
  * [Load Balancing and Fault Resilience with WeaveDNS](/site/weavedns/load-balance-fault-weavedns.md)

weave

@@ -597,9 +597,17 @@ create_bridge() {
         add_iptables_rule filter INPUT -i $DOCKER_BRIDGE -p udp --dport 53  -j ACCEPT
         add_iptables_rule filter INPUT -i $DOCKER_BRIDGE -p tcp --dport 53  -j ACCEPT
 
-        # Work around the situation where there are no rules allowing traffic
-        # across our bridge. E.g. ufw
-        add_iptables_rule filter FORWARD -i $BRIDGE -o $BRIDGE -j ACCEPT
+        if [ "$2" = "--expect-npc" ] ; then # matches usage in weave-kube launch.sh
+            # Steer traffic via the NPC
+            run_iptables -N WEAVE-NPC >/dev/null 2>&1 || true
+            add_iptables_rule filter FORWARD -o $BRIDGE -j WEAVE-NPC
+            add_iptables_rule filter FORWARD -o $BRIDGE -j LOG --log-prefix=WEAVE-NPC:
+            add_iptables_rule filter FORWARD -o $BRIDGE -j DROP
+        else
+            # Work around the situation where there are no rules allowing traffic
+            # across our bridge. E.g. ufw
+            add_iptables_rule filter FORWARD -i $BRIDGE -o $BRIDGE -j ACCEPT
+        fi
 
         # create a chain for masquerading
         run_iptables -t nat -N WEAVE >/dev/null 2>&1 || true
@@ -807,11 +815,6 @@ do_or_die() {
     fi
 }
 
-# Execute arguments as a command within the $CONTAINER_NETNS network namespace
-netnsenter() {
-    nsenter --net=$CONTAINER_NETNS "$@"
-}
-
 add_iface_fastdp() {
     util_op add-datapath-interface $DATAPATH $1
 }
@@ -858,10 +861,6 @@ ask_version() {
     [ -n "$DOCKERIMAGE" ] && docker run --rm --net=none -e WEAVE_CIDR=none $3 $DOCKERIMAGE $COVERAGE_ARGS --version
 }
 
-container_in_host_ns() {
-    [ -h "$CONTAINER_NETNS" -a -h "/proc/self/ns/net" -a "$(readlink $CONTAINER_NETNS)" = "$(readlink /proc/self/ns/net)" ]
-}
-
 attach() {
     ATTACH_ARGS=""
     [ -n "$NO_MULTICAST_ROUTE" ] && ATTACH_ARGS="--no-multicast-route"
@@ -1948,8 +1947,10 @@ really want to do this please set the WEAVE_NO_FASTDP environment variable.
 EOF
                 exit 1
             fi
+        else
+            shift 1
         fi
-        create_bridge --without-ethtool
+        create_bridge --without-ethtool "$@"
         ;;
     attach-bridge)
         if detect_bridge_type ; then