From 986a3f8079a9aea15c0230b05f11796e8e8e91e4 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 9 Apr 2017 19:00:42 +0000 Subject: [PATCH 01/36] Pass bridge name into awsvpc --- ipam/tracker/awsvpc.go | 7 +++---- prog/weaver/main.go | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ipam/tracker/awsvpc.go b/ipam/tracker/awsvpc.go index 51424379..d4836953 100644 --- a/ipam/tracker/awsvpc.go +++ b/ipam/tracker/awsvpc.go @@ -30,7 +30,6 @@ import ( "github.com/vishvananda/netlink" "github.com/weaveworks/weave/common" - wnet "github.com/weaveworks/weave/net" "github.com/weaveworks/weave/net/address" ) @@ -42,7 +41,7 @@ type AWSVPCTracker struct { } // NewAWSVPCTracker creates and initialises AWS VPC based tracker. -func NewAWSVPCTracker() (*AWSVPCTracker, error) { +func NewAWSVPCTracker(bridgeName string) (*AWSVPCTracker, error) { var ( err error session = session.New() @@ -69,9 +68,9 @@ func NewAWSVPCTracker() (*AWSVPCTracker, error) { t.routeTableID = *routeTableID // Detect Weave bridge link index - link, err := netlink.LinkByName(wnet.WeaveBridgeName) + link, err := netlink.LinkByName(bridgeName) if err != nil { - return nil, fmt.Errorf("cannot find \"%s\" interface: %s", wnet.WeaveBridgeName, err) + return nil, fmt.Errorf("cannot find \"%s\" interface: %s", bridgeName, err) } t.linkIndex = link.Attrs().Index diff --git a/prog/weaver/main.go b/prog/weaver/main.go index 7982a2fb..cfd68331 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -337,7 +337,7 @@ func main() { var t tracker.LocalRangeTracker if isAWSVPC { Log.Infoln("Creating AWSVPC LocalRangeTracker") - t, err = tracker.NewAWSVPCTracker() + t, err = tracker.NewAWSVPCTracker(weavenet.WeaveBridgeName) if err != nil { Log.Fatalf("Cannot create AWSVPC LocalRangeTracker: %s", err) } From f5d0a277fb21b0175057474c2429525d768425e0 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 13 Apr 2017 18:37:56 +0000 Subject: [PATCH 02/36] Move canonicalisation and checking of DNS entries into nameserver --- nameserver/http.go | 11 +++-------- nameserver/nameserver.go | 9 +++++++++ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/nameserver/http.go b/nameserver/http.go index ece2b6c2..ac421c0e 100644 --- a/nameserver/http.go +++ b/nameserver/http.go @@ -27,7 +27,7 @@ func (n *Nameserver) HandleHTTP(router *mux.Router, dockerCli *docker.Client) { vars = mux.Vars(r) container = vars["container"] ipStr = vars["ip"] - hostname = dns.Fqdn(r.FormValue("fqdn")) + fqdn = r.FormValue("fqdn") ip, err = address.ParseIP(ipStr) ) if err != nil { @@ -35,16 +35,11 @@ func (n *Nameserver) HandleHTTP(router *mux.Router, dockerCli *docker.Client) { return } - if !dns.IsSubDomain(n.domain, hostname) { - n.infof("Ignoring registration %s %s %s (not a subdomain of %s)", hostname, ipStr, container, n.domain) - return - } - - n.AddEntry(hostname, container, n.ourName, ip) + n.AddEntryFQDN(fqdn, container, n.ourName, ip) if r.FormValue("check-alive") == "true" && dockerCli != nil && dockerCli.IsContainerNotRunning(container) { n.infof("container '%s' is not running: removing", container) - n.Delete(hostname, container, ipStr, ip) + n.Delete(dns.Fqdn(fqdn), container, ipStr, ip) } w.WriteHeader(204) diff --git a/nameserver/nameserver.go b/nameserver/nameserver.go index 6fcf1e84..75879276 100644 --- a/nameserver/nameserver.go +++ b/nameserver/nameserver.go @@ -91,6 +91,15 @@ func (n *Nameserver) AddEntry(hostname, containerid string, origin mesh.PeerName n.broadcastEntries(entry) } +func (n *Nameserver) AddEntryFQDN(fqdn, containerid string, origin mesh.PeerName, addr address.Address) { + hostname := dns.Fqdn(fqdn) + if !dns.IsSubDomain(n.domain, hostname) { + n.infof("Ignoring registration %s %s %s (not a subdomain of %s)", hostname, addr.String(), containerid, n.domain) + return + } + n.AddEntry(hostname, containerid, origin, addr) +} + func (n *Nameserver) Lookup(hostname string) []address.Address { n.RLock() defer n.RUnlock() From 36521b04a2d476b5bb17ac7b740eeb5c61f77551 Mon Sep 17 00:00:00 2001 From: Adam Harrison Date: Wed, 10 Feb 2016 11:52:40 +0000 Subject: [PATCH 03/36] Utility function to enforce docker bridge addr_assign_type --- net/bridge.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/net/bridge.go b/net/bridge.go index 6845c0bf..83ad0b40 100644 --- a/net/bridge.go +++ b/net/bridge.go @@ -1,7 +1,9 @@ package net import ( + "errors" "fmt" + "io/ioutil" "net" "github.com/Sirupsen/logrus" @@ -39,6 +41,40 @@ func DetectBridgeType(weaveBridgeName, datapathName string) BridgeType { } } +var ErrSetBridgeMac = errors.New("Setting $DOCKER_BRIDGE MAC (mitigate https://github.com/docker/docker/issues/14908)") + +func EnforceAddrAssignType(bridgeName string) error { + addrAssignType, err := ioutil.ReadFile(fmt.Sprintf("/sys/class/net/%s/addr_assign_type", bridgeName)) + if err != nil { + return err + } + + // From include/uapi/linux/netdevice.h + // #define NET_ADDR_PERM 0 /* address is permanent (default) */ + // #define NET_ADDR_RANDOM 1 /* address is generated randomly */ + // #define NET_ADDR_STOLEN 2 /* address is stolen from other device */ + // #define NET_ADDR_SET 3 /* address is set using dev_set_mac_address() */ + // Note the file typically has a newline at the end, so we just look at the first char + if addrAssignType[0] != '3' { + link, err := netlink.LinkByName(bridgeName) + if err != nil { + return err + } + + mac, err := RandomMAC() + if err != nil { + return err + } + + if err := netlink.LinkSetHardwareAddr(link, mac); err != nil { + return err + } + return ErrSetBridgeMac + } + + return nil +} + func isBridge(link netlink.Link) bool { _, isBridge := link.(*netlink.Bridge) return isBridge From 7dd37d2a1332caa842f491e034359946f33a00e1 Mon Sep 17 00:00:00 2001 From: Adam Harrison Date: Wed, 10 Feb 2016 16:02:50 +0000 Subject: [PATCH 04/36] Add functions to create bridge --- net/bridge.go | 132 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/net/bridge.go b/net/bridge.go index 83ad0b40..3b684c36 100644 --- a/net/bridge.go +++ b/net/bridge.go @@ -8,6 +8,8 @@ import ( "github.com/Sirupsen/logrus" "github.com/vishvananda/netlink" + + "github.com/weaveworks/weave/common/odp" ) type BridgeType int @@ -144,3 +146,133 @@ func FindBridgeIP(bridgeName string, subnet *net.IPNet) (net.IP, error) { // No subnet, or none in the required subnet; just return the first one return netdev.CIDRs[0].IP, nil } + +type BridgeConfig struct { + DBPrefix string + HostRoot string + DockerBridgeName string + WeaveBridgeName string + DatapathName string + NoFastdp bool + NoBridgedFastdp bool + MTU int + Mac string +} + +func CreateBridge(config *BridgeConfig) (BridgeType, error) { + bridgeType := DetectBridgeType(config.WeaveBridgeName, config.DatapathName) + + if bridgeType == None { + bridgeType = Bridge + if !config.NoFastdp { + bridgeType = BridgedFastdp + if config.NoBridgedFastdp { + bridgeType = Fastdp + config.DatapathName = config.WeaveBridgeName + } + odpSupported, err := odp.CreateDatapath(config.DatapathName) + if err != nil { + return None, err + } + if !odpSupported { + bridgeType = Bridge + } + } + + var err error + switch bridgeType { + case Bridge: + err = initBridge(config) + case Fastdp: + err = initFastdp(config) + case BridgedFastdp: + err = initBridgedFastdp(config) + default: + err = fmt.Errorf("Cannot initialise bridge type %v", bridgeType) + } + if err != nil { + return None, err + } + + configureIPTables(config) + } + + if bridgeType == Bridge { + if err := EthtoolTXOff(config.WeaveBridgeName); err != nil { + return bridgeType, err + } + } + + if err := linkSetUpByName(config.WeaveBridgeName); err != nil { + return bridgeType, err + } + + if err := ConfigureARPCache(config.WeaveBridgeName); err != nil { + return bridgeType, err + } + + return bridgeType, nil +} + +func initBridge(config *BridgeConfig) error { + mac, err := net.ParseMAC(config.Mac) + if err != nil { + return err + } + + linkAttrs := netlink.NewLinkAttrs() + linkAttrs.Name = config.WeaveBridgeName + linkAttrs.HardwareAddr = mac + linkAttrs.MTU = config.MTU // TODO this probably doesn't work - see weave script + if err = netlink.LinkAdd(&netlink.Bridge{LinkAttrs: linkAttrs}); err != nil { + return err + } + + return nil +} + +func initFastdp(config *BridgeConfig) error { + datapath, err := netlink.LinkByName(config.DatapathName) + if err != nil { + return err + } + return netlink.LinkSetMTU(datapath, config.MTU) +} + +func initBridgedFastdp(config *BridgeConfig) error { + if err := initFastdp(config); err != nil { + return err + } + if err := initBridge(config); err != nil { + return err + } + if _, err := CreateAndAttachVeth("vethwe-bridge", "vethwe-datapath", config.WeaveBridgeName, config.MTU, true, func(veth netlink.Link) error { + if err := netlink.LinkSetUp(veth); err != nil { + return err + } + if err := odp.AddDatapathInterface(config.DatapathName, veth.Attrs().Name); err != nil { + return err + } + return nil + }); err != nil { + return err + } + + if err := linkSetUpByName(config.DatapathName); err != nil { + return err + } + + return nil +} + +func configureIPTables(config *BridgeConfig) error { + return fmt.Errorf("Not implemented") +} + +func linkSetUpByName(linkName string) error { + link, err := netlink.LinkByName(linkName) + if err != nil { + return err + } + return netlink.LinkSetUp(link) +} From 8e5384ee0160e324120695f32e4833bef7b17bad Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 11 Apr 2017 14:31:08 +0000 Subject: [PATCH 05/36] Create pcap veth for no-fastdp operation --- net/bridge.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/net/bridge.go b/net/bridge.go index 3b684c36..46841aab 100644 --- a/net/bridge.go +++ b/net/bridge.go @@ -17,6 +17,8 @@ type BridgeType int const ( WeaveBridgeName = "weave" DatapathName = "datapath" + BridgeIfName = "vethwe-bridge" + PcapIfaceName = "vethwe-pcap" None BridgeType = iota Bridge @@ -214,7 +216,7 @@ func CreateBridge(config *BridgeConfig) (BridgeType, error) { return bridgeType, nil } -func initBridge(config *BridgeConfig) error { +func initBridgePrep(config *BridgeConfig) error { mac, err := net.ParseMAC(config.Mac) if err != nil { return err @@ -231,6 +233,19 @@ func initBridge(config *BridgeConfig) error { return nil } +func initBridge(config *BridgeConfig) error { + if err := initBridgePrep(config); err != nil { + return err + } + if _, err := CreateAndAttachVeth(BridgeIfName, PcapIfaceName, config.WeaveBridgeName, config.MTU, true, func(veth netlink.Link) error { + return netlink.LinkSetUp(veth) + }); err != nil { + return err + } + + return nil +} + func initFastdp(config *BridgeConfig) error { datapath, err := netlink.LinkByName(config.DatapathName) if err != nil { @@ -243,7 +258,7 @@ func initBridgedFastdp(config *BridgeConfig) error { if err := initFastdp(config); err != nil { return err } - if err := initBridge(config); err != nil { + if err := initBridgePrep(config); err != nil { return err } if _, err := CreateAndAttachVeth("vethwe-bridge", "vethwe-datapath", config.WeaveBridgeName, config.MTU, true, func(veth netlink.Link) error { From 2d037d874c8d3fa4a7b61145dec0194624119bf4 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 16 May 2016 15:55:44 +0100 Subject: [PATCH 06/36] MTU selection closer to weave script Note that if initFastdp sets the MTU for the datapath then the same value is used for the bridge also. --- net/bridge.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/net/bridge.go b/net/bridge.go index 46841aab..5224a8f8 100644 --- a/net/bridge.go +++ b/net/bridge.go @@ -225,6 +225,9 @@ func initBridgePrep(config *BridgeConfig) error { linkAttrs := netlink.NewLinkAttrs() linkAttrs.Name = config.WeaveBridgeName linkAttrs.HardwareAddr = mac + if config.MTU == 0 { + config.MTU = 65535 + } linkAttrs.MTU = config.MTU // TODO this probably doesn't work - see weave script if err = netlink.LinkAdd(&netlink.Bridge{LinkAttrs: linkAttrs}); err != nil { return err @@ -251,6 +254,15 @@ func initFastdp(config *BridgeConfig) error { if err != nil { return err } + if config.MTU == 0 { + /* GCE has the lowest underlay network MTU we're likely to encounter on + a local network, at 1460 bytes. To get the overlay MTU from that we + subtract 20 bytes for the outer IPv4 header, 8 bytes for the outer + UDP header, 8 bytes for the vxlan header, and 14 bytes for the inner + ethernet header. In addition, we subtract 34 bytes for the ESP overhead + which is needed for the vxlan encryption. */ + config.MTU = 1376 + } return netlink.LinkSetMTU(datapath, config.MTU) } From 6fd9f471ff4f68cccfaa8935ba2b8000ae3f76ab Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 16 May 2016 16:57:03 +0100 Subject: [PATCH 07/36] Bridge setup iptables rules --- net/bridge.go | 57 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/net/bridge.go b/net/bridge.go index 5224a8f8..f5e89587 100644 --- a/net/bridge.go +++ b/net/bridge.go @@ -7,6 +7,7 @@ import ( "net" "github.com/Sirupsen/logrus" + "github.com/coreos/go-iptables/iptables" "github.com/vishvananda/netlink" "github.com/weaveworks/weave/common/odp" @@ -159,6 +160,7 @@ type BridgeConfig struct { NoBridgedFastdp bool MTU int Mac string + Port int } func CreateBridge(config *BridgeConfig) (BridgeType, error) { @@ -196,7 +198,9 @@ func CreateBridge(config *BridgeConfig) (BridgeType, error) { return None, err } - configureIPTables(config) + if err = configureIPTables(config); err != nil { + return bridgeType, err + } } if bridgeType == Bridge { @@ -293,7 +297,56 @@ func initBridgedFastdp(config *BridgeConfig) error { } func configureIPTables(config *BridgeConfig) error { - return fmt.Errorf("Not implemented") + ipt, err := iptables.New() + if err != nil { + return err + } + if config.DockerBridgeName != "" { + if config.WeaveBridgeName != config.DockerBridgeName { + err := ipt.Insert("filter", "FORWARD", 1, "-i", config.DockerBridgeName, "-o", config.WeaveBridgeName, "-j", "DROP") + if err != nil { + return err + } + } + + dockerBridgeIP, err := FindBridgeIP(config.DockerBridgeName, nil) + if err != nil { + return err + } + + // forbid traffic to the Weave port from other containers + if err = ipt.AppendUnique("filter", "INPUT", "-i", config.DockerBridgeName, "-p", "tcp", "--dst", dockerBridgeIP.String(), "--dport", fmt.Sprint(config.Port), "-j", "DROP"); err != nil { + return err + } + if err = ipt.AppendUnique("filter", "INPUT", "-i", config.DockerBridgeName, "-p", "udp", "--dst", dockerBridgeIP.String(), "--dport", fmt.Sprint(config.Port), "-j", "DROP"); err != nil { + return err + } + if err = ipt.AppendUnique("filter", "INPUT", "-i", config.DockerBridgeName, "-p", "udp", "--dst", dockerBridgeIP.String(), "--dport", fmt.Sprint(config.Port+1), "-j", "DROP"); err != nil { + return err + } + + // let DNS traffic to weaveDNS, since otherwise it might get blocked by the likes of UFW + if err = ipt.AppendUnique("filter", "INPUT", "-i", config.DockerBridgeName, "-p", "udp", "--dport", "53", "-j", "ACCEPT"); err != nil { + return err + } + if err = ipt.AppendUnique("filter", "INPUT", "-i", config.DockerBridgeName, "-p", "tcp", "--dport", "53", "-j", "ACCEPT"); err != nil { + return err + } + } + + // Work around the situation where there are no rules allowing traffic + // across our bridge. E.g. ufw + if err = ipt.AppendUnique("filter", "FORWARD", "-i", config.WeaveBridgeName, "-o", config.WeaveBridgeName, "-j", "ACCEPT"); err != nil { + return err + } + + // create a chain for masquerading + ipt.NewChain("nat", "WEAVE") + if err = ipt.AppendUnique("nat", "POSTROUTING", "-j", "WEAVE"); err != nil { + return err + } + + return nil } func linkSetUpByName(linkName string) error { From 8051f51d46abe84c308d24c90ef6a3c7ee352ca2 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 16 May 2016 17:41:43 +0100 Subject: [PATCH 08/36] Move bridge-creation code from weave script into Go code via weaveutil create-bridge function --- net/bridge.go | 19 ++- prog/weaveutil/bridge.go | 45 ++++++++ prog/weaveutil/datapath.go | 13 --- prog/weaveutil/main.go | 3 +- weave | 230 ++++++++----------------------------- 5 files changed, 115 insertions(+), 195 deletions(-) create mode 100644 prog/weaveutil/bridge.go diff --git a/net/bridge.go b/net/bridge.go index f5e89587..a88a7334 100644 --- a/net/bridge.go +++ b/net/bridge.go @@ -28,6 +28,23 @@ const ( Inconsistent ) +// Returns a string that is consistent with the weave script +func (t BridgeType) String() string { + switch t { + case None: + return "none" + case Bridge: + return "bridge" + case Fastdp: + return "fastdp" + case BridgedFastdp: + return "bridged_fastdp" + case Inconsistent: + return "inconsistent" + } + return "unknown" +} + func DetectBridgeType(weaveBridgeName, datapathName string) BridgeType { bridge, _ := netlink.LinkByName(weaveBridgeName) datapath, _ := netlink.LinkByName(datapathName) @@ -151,8 +168,6 @@ func FindBridgeIP(bridgeName string, subnet *net.IPNet) (net.IP, error) { } type BridgeConfig struct { - DBPrefix string - HostRoot string DockerBridgeName string WeaveBridgeName string DatapathName string diff --git a/prog/weaveutil/bridge.go b/prog/weaveutil/bridge.go new file mode 100644 index 00000000..b82f8fdb --- /dev/null +++ b/prog/weaveutil/bridge.go @@ -0,0 +1,45 @@ +package main + +import ( + "fmt" + "strconv" + + weavenet "github.com/weaveworks/weave/net" +) + +func detectBridgeType(args []string) error { + if len(args) != 2 { + cmdUsage("detect-bridge-type", " ") + } + bridgeType := weavenet.DetectBridgeType(args[0], args[1]) + fmt.Println(bridgeType.String()) + return nil +} + +func createBridge(args []string) error { + if len(args) != 8 { + cmdUsage("create-bridge", " ") + } + + mtu, err := strconv.Atoi(args[3]) + if err != nil && args[3] != "" { + return err + } + port, err := strconv.Atoi(args[4]) + if err != nil { + return err + } + config := weavenet.BridgeConfig{ + DockerBridgeName: args[0], + WeaveBridgeName: args[1], + DatapathName: args[2], + MTU: mtu, + Port: port, + Mac: args[5], + NoFastdp: args[6] != "", + NoBridgedFastdp: args[7] != "", + } + bridgeType, err := weavenet.CreateBridge(&config) + fmt.Println(bridgeType.String()) + return err +} diff --git a/prog/weaveutil/datapath.go b/prog/weaveutil/datapath.go index 52e320f8..f4494967 100644 --- a/prog/weaveutil/datapath.go +++ b/prog/weaveutil/datapath.go @@ -31,19 +31,6 @@ func deleteDatapath(args []string) error { return odp.DeleteDatapath(args[0]) } -// Checks whether a datapath can be created by actually creating and destroying it -func checkDatapath(args []string) error { - if len(args) != 1 { - cmdUsage("check-datapath", "") - } - - if err := createDatapath(args); err != nil { - return err - } - - return odp.DeleteDatapath(args[0]) -} - func addDatapathInterface(args []string) error { if len(args) != 2 { cmdUsage("add-datapath-interface", " ") diff --git a/prog/weaveutil/main.go b/prog/weaveutil/main.go index bc954761..46ffa719 100644 --- a/prog/weaveutil/main.go +++ b/prog/weaveutil/main.go @@ -17,9 +17,10 @@ func init() { "help": help, "netcheck": netcheck, "docker-tls-args": dockerTLSArgs, + "create-bridge": createBridge, + "detect-bridge-type": detectBridgeType, "create-datapath": createDatapath, "delete-datapath": deleteDatapath, - "check-datapath": checkDatapath, "add-datapath-interface": addDatapathInterface, "create-plugin-network": createPluginNetwork, "remove-plugin-network": removePluginNetwork, diff --git a/weave b/weave index 7349989a..0f053d3f 100755 --- a/weave +++ b/weave @@ -365,15 +365,6 @@ delete_iptables_rule() { fi } -# Configure the ARP cache parameters for the given interface. This -# makes containers react more quickly to a change in the MAC address -# associated with an IP address. -configure_arp_cache() { - $2 sh -c "echo 5 >$PROC_PATH/sys/net/ipv4/neigh/$1/base_reachable_time && - echo 2 >$PROC_PATH/sys/net/ipv4/neigh/$1/delay_first_probe_time && - echo 1 >$PROC_PATH/sys/net/ipv4/neigh/$1/ucast_solicit" -} - # Send out an ARP announcement # (https://tools.ietf.org/html/rfc5227#page-15) to update ARP cache # entries across the weave network. We do this in addition to @@ -449,29 +440,21 @@ enforce_docker_bridge_addr_assign_type() { # bridge/datapath devices have yet to be configured. If netdevs do exist # but are in an inconsistent state the script aborts with an error. detect_bridge_type() { - BRIDGE_TYPE= - if [ -d /sys/class/net/$DATAPATH ] ; then - # Unfortunately there's no simple way to positively check whether - # $DATAPATH is an ODP netdev so we have to make sure it isn't - # a bridge instead (and that $BRIDGE is). - if [ ! -d /sys/class/net/$DATAPATH/bridge -a -d /sys/class/net/$BRIDGE/bridge ] ; then - BRIDGE_TYPE=bridged_fastdp - else + BRIDGE_TYPE=$(util_op detect-bridge-type "$BRIDGE" "$DATAPATH") + case "$BRIDGE_TYPE" in + bridge|bridged_fastdp) + ;; + fastdp) + DATAPATH="$BRIDGE" + ;; + inconsistent) echo "Inconsistent bridge state detected. Please do 'weave reset' and try again." >&2 exit 1 - fi - elif [ -d /sys/class/net/$BRIDGE ] ; then - if [ -d /sys/class/net/$BRIDGE/bridge ] ; then - BRIDGE_TYPE=bridge - else - BRIDGE_TYPE=fastdp - # The datapath is the bridge when there is no intermediary - DATAPATH="$BRIDGE" - fi - else - # No bridge/datapath devices configured - return 1 - fi + ;; + *) + return 1 + ;; + esac # WEAVE_MTU may have been specified when the bridge was # created (perhaps implicitly with WEAVE_NO_FASTDP). So take @@ -481,46 +464,19 @@ detect_bridge_type() { } try_create_bridge() { - if ! detect_bridge_type ; then - BRIDGE_TYPE=bridge - if [ -z "$WEAVE_NO_FASTDP" ] ; then - BRIDGE_TYPE=bridged_fastdp - if [ -n "$WEAVE_NO_BRIDGED_FASTDP" ] ; then - BRIDGE_TYPE=fastdp - # The datapath is the bridge when there is no intermediary - DATAPATH="$BRIDGE" - fi - if util_op create-datapath $DATAPATH ; then - : # ODP datapath created successfully - elif [ $? = 17 ] ; then - # Exit status of 17 means the kernel doesn't have ODP - BRIDGE_TYPE=bridge - else - return 1 - fi - fi + # Running this from 'weave --local' when weaveexec is not on the + # path will run it as a Docker container that does not have access + # to /etc/machine-id, so will not give the full range of persistent IDs + if [ -z "$WEAVEDB_DIR_PATH" ]; then + WEAVEDB_DIR_PATH="$HOST_ROOT/$(docker inspect -f '{{with index .Mounts 0}}{{.Source}}{{end}}' $DB_CONTAINER_NAME 2>/dev/null)" + fi + MAC=$(util_op unique-id "$WEAVEDB_DIR_PATH/weave" "$HOST_ROOT") + BRIDGE_TYPE=$(util_op create-bridge "$DOCKER_BRIDGE" "$BRIDGE" "$DATAPATH" "$WEAVE_MTU" "$PORT" "$MAC" "$WEAVE_NO_FASTDP" "$WEAVE_NO_BRIDGED_FASTDP") + # Set some variables that are expected by code that comes later + [ "$BRIDGE_TYPE" != "fastdp" ] || DATAPATH="$BRIDGE" + MTU=${WEAVE_MTU:-$(cat /sys/class/net/$BRIDGE/mtu)} - init_$BRIDGE_TYPE - - # Drop traffic from Docker bridge to Weave; it can break - # subnet isolation - if [ "$DOCKER_BRIDGE" != "$BRIDGE" ] ; then - # Note using -I to insert ahead of Docker's bridge rules - run_iptables -t filter -I FORWARD -i $DOCKER_BRIDGE -o $BRIDGE -j DROP - fi - - [ -n "$DOCKER_BRIDGE_IP" ] || DOCKER_BRIDGE_IP=$(util_op bridge-ip $DOCKER_BRIDGE) - - # forbid traffic to the Weave port from other containers - add_iptables_rule filter INPUT -i $DOCKER_BRIDGE -p tcp --dst $DOCKER_BRIDGE_IP --dport $PORT -j DROP - add_iptables_rule filter INPUT -i $DOCKER_BRIDGE -p udp --dst $DOCKER_BRIDGE_IP --dport $PORT -j DROP - add_iptables_rule filter INPUT -i $DOCKER_BRIDGE -p udp --dst $DOCKER_BRIDGE_IP --dport $(($PORT + 1)) -j DROP - - # let DNS traffic to weaveDNS, since otherwise it might get blocked by the likes of UFW - add_iptables_rule filter INPUT -i $DOCKER_BRIDGE -p udp --dport 53 -j ACCEPT - add_iptables_rule filter INPUT -i $DOCKER_BRIDGE -p tcp --dport 53 -j ACCEPT - - if [ "$2" = "--expect-npc" ] ; then # matches usage in weave-kube launch.sh + if [ "$1" = "--expect-npc" ] ; then # matches usage in weave-kube launch.sh # Steer traffic via the NPC run_iptables -N WEAVE-NPC >/dev/null 2>&1 || true add_iptables_rule filter FORWARD -o $BRIDGE -j WEAVE-NPC @@ -535,41 +491,10 @@ try_create_bridge() { add_iptables_rule filter FORWARD -i $BRIDGE ! -o $BRIDGE -j ACCEPT # and allow replies back add_iptables_rule filter FORWARD -o $BRIDGE -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - - # create a chain for masquerading - run_iptables -t nat -N WEAVE >/dev/null 2>&1 || true - add_iptables_rule nat POSTROUTING -j WEAVE - else - if [ -n "$LAUNCHING_ROUTER" ] ; then - if [ "$BRIDGE_TYPE" = bridge -a -z "$WEAVE_NO_FASTDP" ] && - util_op check-datapath $DATAPATH 2>/dev/null ; then - cat <&2 -WEAVE_NO_FASTDP is not set, but there is already a bridge present of -the wrong type for fast datapath. Please do 'weave reset' to remove -the bridge first. -EOF - return 1 - fi - if [ "$BRIDGE_TYPE" != bridge -a -n "$WEAVE_NO_FASTDP" ] ; then - cat <&2 -WEAVE_NO_FASTDP is set, but there is already a weave fast datapath -bridge present. Please do 'weave reset' to remove the bridge first. -EOF - return 1 - fi - fi - fi - - [ "$1" = "--without-ethtool" -o -n "$AWSVPC" ] || ethtool_tx_off_$BRIDGE_TYPE $BRIDGE - - ip link set dev $BRIDGE up - - # Configure the ARP cache parameters on the bridge interface for - # the sake of 'weave expose' - configure_arp_cache $BRIDGE } create_bridge() { + validate_bridge_type if ! try_create_bridge "$@" ; then echo "Creating bridge '$BRIDGE' failed" >&2 # reset to original value so we destroy both kinds @@ -579,6 +504,28 @@ create_bridge() { fi } +validate_bridge_type() { + detect_bridge_type || return 0 + + if [ -n "$LAUNCHING_ROUTER" ] ; then + if [ "$BRIDGE_TYPE" = bridge -a -z "$WEAVE_NO_FASTDP" ] ; then + cat <&2 +WEAVE_NO_FASTDP is not set, but there is already a bridge present of +the wrong type for fast datapath. Please do 'weave reset' to remove +the bridge first. +EOF + return 1 + fi + if [ "$BRIDGE_TYPE" != bridge -a -n "$WEAVE_NO_FASTDP" ] ; then + cat <&2 +WEAVE_NO_FASTDP is set, but there is already a weave fast datapath +bridge present. Please do 'weave reset' to remove the bridge first. +EOF + return 1 + fi + fi +} + expose_ip() { ipam_cidrs allocate_no_check_alive weave:expose $CIDR_ARGS for CIDR in $ALL_CIDRS ; do @@ -614,80 +561,6 @@ create_veth() { fi } -init_fastdp() { - # GCE has the lowest underlay network MTU we're likely to encounter on - # a local network, at 1460 bytes. To get the overlay MTU from that we - # subtract 20 bytes for the outer IPv4 header, 8 bytes for the outer - # UDP header, 8 bytes for the vxlan header, and 14 bytes for the inner - # ethernet header. In addition, we subtract 34 bytes for the ESP overhead - # which is needed for the vxlan encryption. - MTU=${WEAVE_MTU:-1376} - - # create_bridge already created the datapath netdev - ip link set dev $DATAPATH mtu $MTU -} - -init_bridge_prep() { - # Observe any MTU that is already set - [ -n "$MTU" ] || MTU=${WEAVE_MTU:-65535} - - ip link add name $BRIDGE type bridge - - # Running this from 'weave --local' when weaveexec is not on the - # path will run it as a Docker container that does not have access - # to /etc/machine-id, so will not give the full range of persistent IDs - if [ -z "$WEAVEDB_DIR_PATH" ]; then - WEAVEDB_DIR_PATH="$HOST_ROOT/$(docker inspect -f '{{with index .Mounts 0}}{{.Source}}{{end}}' $DB_CONTAINER_NAME 2>/dev/null)" - fi - MAC=$(util_op unique-id "$WEAVEDB_DIR_PATH/weave" "$HOST_ROOT") - - ip link set dev $BRIDGE address $MAC - - # Attempting to set the bridge MTU to a high value directly - # fails. Bridges take the lowest MTU of their interfaces. So - # instead we create a temporary interface with the desired - # MTU, attach that to the bridge, and then remove it again. - ip link add name v${CONTAINER_IFNAME}du mtu $MTU type dummy - ip link set dev v${CONTAINER_IFNAME}du master $BRIDGE - ip link del dev v${CONTAINER_IFNAME}du -} - -init_bridge() { - init_bridge_prep - create_veth $BRIDGE_IFNAME $PCAP_IFNAME add_iface_bridge $BRIDGE_IFNAME -} - -init_bridged_fastdp() { - # Initialise the datapath as normal. NB sets MTU for use below - init_fastdp - - # Initialise the bridge using fast datapath MTU - init_bridge_prep - - # Create linking veth pair - create_veth $BRIDGE_IFNAME $DATAPATH_IFNAME configure_veth_bridged_fastdp - - # Finally, bring the datapath up - ip link set dev $DATAPATH up -} - -configure_veth_bridged_fastdp() { - add_iface_fastdp $DATAPATH_IFNAME || return 1 - add_iface_bridge $BRIDGE_IFNAME || return 1 -} - -ethtool_tx_off_fastdp() { - true -} - -ethtool_tx_off_bridge() { - ethtool -K $1 tx off >/dev/null -} - -ethtool_tx_off_bridged_fastdp() { - true -} - destroy_bridge() { # It's important that detect_bridge_type has not been called so # we have distinct values for $BRIDGE and $DATAPATH. Make best efforts @@ -1854,8 +1727,7 @@ EOF else shift 1 fi - - create_bridge --without-ethtool "$@" + create_bridge "$@" ;; attach-bridge) if detect_bridge_type ; then @@ -2062,7 +1934,7 @@ EOF [ $# -eq 2 -a "$1" = "-h" ] || usage FQDN="$2" fi - create_bridge --without-ethtool + create_bridge expose_ip util_op expose-nat $ALL_CIDRS show_addrs $ALL_CIDRS @@ -2071,7 +1943,7 @@ EOF collect_cidr_args "$@" shift $CIDR_ARG_COUNT ipam_cidrs lookup weave:expose $CIDR_ARGS - create_bridge --without-ethtool + create_bridge for CIDR in $ALL_CIDRS ; do if ip addr show dev $BRIDGE | grep -qF $CIDR ; then ip addr del dev $BRIDGE $CIDR From a8eb7966b6f6abb07d13e5e0b3900712aa90fd40 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 17 May 2016 13:32:30 +0100 Subject: [PATCH 09/36] Wire up enforce_docker_bridge_addr_assign_type --- prog/weaveutil/bridge.go | 7 +++++++ prog/weaveutil/main.go | 1 + weave | 28 +--------------------------- 3 files changed, 9 insertions(+), 27 deletions(-) diff --git a/prog/weaveutil/bridge.go b/prog/weaveutil/bridge.go index b82f8fdb..4916899a 100644 --- a/prog/weaveutil/bridge.go +++ b/prog/weaveutil/bridge.go @@ -43,3 +43,10 @@ func createBridge(args []string) error { fmt.Println(bridgeType.String()) return err } + +func enforceAddrAssign(args []string) error { + if len(args) != 1 { + cmdUsage("enforce-bridge-addr-assign-type", "") + } + return weavenet.EnforceAddrAssignType(args[0]) +} diff --git a/prog/weaveutil/main.go b/prog/weaveutil/main.go index 46ffa719..b0f7a33e 100644 --- a/prog/weaveutil/main.go +++ b/prog/weaveutil/main.go @@ -19,6 +19,7 @@ func init() { "docker-tls-args": dockerTLSArgs, "create-bridge": createBridge, "detect-bridge-type": detectBridgeType, + "enforce-addr-assign": enforceAddrAssign, "create-datapath": createDatapath, "delete-datapath": deleteDatapath, "add-datapath-interface": addDatapathInterface, diff --git a/weave b/weave index 0f053d3f..6391a62d 100755 --- a/weave +++ b/weave @@ -379,20 +379,6 @@ arp_update() { ! command_exists arping || $3 arping -U -q -I $1 -c 1 ${2%/*} } -# Generate a MAC value from a stdin containing six space separated -# 2-digit hexadecimal numbers. -mac_from_hex() { - # In the first byte of the MAC, the 'multicast' bit should be - # clear and 'locally administered' bit should be set. All other - # bits should be random. - read a b c d e f && printf "%02x:$b:$c:$d:$e:$f" $((0x$a & ~1 | 2)) -} - -# Generate a random MAC value -random_mac() { - od -txC -An -N6 /dev/urandom | mac_from_hex -} - ###################################################################### # weave and docker specific helpers ###################################################################### @@ -419,19 +405,7 @@ EOF } enforce_docker_bridge_addr_assign_type() { - if ! ADDR_ASSIGN_TYPE=$(cat /sys/class/net/$DOCKER_BRIDGE/addr_assign_type 2>/dev/null) ; then - echo "Could not determine address assignment type of $DOCKER_BRIDGE" >&2 - return - fi - # From include/uapi/linux/netdevice.h - # #define NET_ADDR_PERM 0 /* address is permanent (default) */ - # #define NET_ADDR_RANDOM 1 /* address is generated randomly */ - # #define NET_ADDR_STOLEN 2 /* address is stolen from other device */ - # #define NET_ADDR_SET 3 /* address is set using dev_set_mac_address() */ - if [ $ADDR_ASSIGN_TYPE != 3 ] ; then - echo "Setting $DOCKER_BRIDGE MAC (mitigate https://github.com/docker/docker/issues/14908)" >&2 - ip link set dev $DOCKER_BRIDGE address $(random_mac) || true - fi + util_op enforce-addr-assign "$DOCKER_BRIDGE" || true } # Detect the current bridge/datapath state. When invoked, the values of From dd232775e68a18122560af20637d96626e991d94 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Fri, 25 Nov 2016 18:15:40 +0000 Subject: [PATCH 10/36] Move expect-npc setup from weave script to Go code --- net/bridge.go | 33 ++++++++++++++++++++++++++++----- prog/weaveutil/bridge.go | 5 +++-- weave | 33 +-------------------------------- 3 files changed, 32 insertions(+), 39 deletions(-) diff --git a/net/bridge.go b/net/bridge.go index a88a7334..533942d7 100644 --- a/net/bridge.go +++ b/net/bridge.go @@ -173,6 +173,7 @@ type BridgeConfig struct { DatapathName string NoFastdp bool NoBridgedFastdp bool + ExpectNPC bool MTU int Mac string Port int @@ -349,14 +350,36 @@ func configureIPTables(config *BridgeConfig) error { } } - // Work around the situation where there are no rules allowing traffic - // across our bridge. E.g. ufw - if err = ipt.AppendUnique("filter", "FORWARD", "-i", config.WeaveBridgeName, "-o", config.WeaveBridgeName, "-j", "ACCEPT"); err != nil { - return err + if config.ExpectNPC { + // Steer traffic via the NPC + ipt.NewChain("filter", "WEAVE-NPC") // ignoring errors such as 'chain already exists' + if err = ipt.AppendUnique("filter", "FORWARD", "-o", config.WeaveBridgeName, "-j", "WEAVE-NPC"); err != nil { + return err + } + if err = ipt.AppendUnique("filter", "FORWARD", "-o", config.WeaveBridgeName, "-m", "state", "--state", "NEW", "-j", "NFLOG", "--nflog-group", "86"); err != nil { + return err + } + if err = ipt.AppendUnique("filter", "FORWARD", "-o", config.WeaveBridgeName, "-j", "DROP"); err != nil { + return err + } + } else { + // Work around the situation where there are no rules allowing traffic + // across our bridge. E.g. ufw + if err = ipt.AppendUnique("filter", "FORWARD", "-i", config.WeaveBridgeName, "-o", config.WeaveBridgeName, "-j", "ACCEPT"); err != nil { + return err + } + // Forward from weave to the rest of the world + if err = ipt.AppendUnique("filter", "FORWARD", "-i", config.WeaveBridgeName, "!", "-o", config.WeaveBridgeName, "-j", "ACCEPT"); err != nil { + return err + } + // and allow replies back + if err = ipt.AppendUnique("filter", "FORWARD", "-o", config.WeaveBridgeName, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"); err != nil { + return err + } } // create a chain for masquerading - ipt.NewChain("nat", "WEAVE") + ipt.NewChain("nat", "WEAVE") // ignoring errors such as 'chain already exists' if err = ipt.AppendUnique("nat", "POSTROUTING", "-j", "WEAVE"); err != nil { return err } diff --git a/prog/weaveutil/bridge.go b/prog/weaveutil/bridge.go index 4916899a..0b63b336 100644 --- a/prog/weaveutil/bridge.go +++ b/prog/weaveutil/bridge.go @@ -17,8 +17,8 @@ func detectBridgeType(args []string) error { } func createBridge(args []string) error { - if len(args) != 8 { - cmdUsage("create-bridge", " ") + if len(args) != 9 { + cmdUsage("create-bridge", " ") } mtu, err := strconv.Atoi(args[3]) @@ -38,6 +38,7 @@ func createBridge(args []string) error { Mac: args[5], NoFastdp: args[6] != "", NoBridgedFastdp: args[7] != "", + ExpectNPC: args[8] == "--expect-npc", } bridgeType, err := weavenet.CreateBridge(&config) fmt.Println(bridgeType.String()) diff --git a/weave b/weave index 6391a62d..88d9e494 100755 --- a/weave +++ b/weave @@ -326,21 +326,6 @@ run_iptables() { iptables $IPTABLES_W "$@" } -# Add a rule to iptables, if it doesn't exist already -add_iptables_rule() { - IPTABLES_TABLE="$1" - shift 1 - if ! run_iptables -t $IPTABLES_TABLE -C "$@" >/dev/null 2>&1 ; then - ## Loop until we get an exit code other than "temporarily unavailable" - while true ; do - run_iptables -t $IPTABLES_TABLE -A "$@" >/dev/null && return 0 - if [ $? != 4 ] ; then - return 1 - fi - done - fi -} - # Insert a rule in iptables, if it doesn't exist already insert_iptables_rule() { IPTABLES_TABLE="$1" @@ -445,26 +430,10 @@ try_create_bridge() { WEAVEDB_DIR_PATH="$HOST_ROOT/$(docker inspect -f '{{with index .Mounts 0}}{{.Source}}{{end}}' $DB_CONTAINER_NAME 2>/dev/null)" fi MAC=$(util_op unique-id "$WEAVEDB_DIR_PATH/weave" "$HOST_ROOT") - BRIDGE_TYPE=$(util_op create-bridge "$DOCKER_BRIDGE" "$BRIDGE" "$DATAPATH" "$WEAVE_MTU" "$PORT" "$MAC" "$WEAVE_NO_FASTDP" "$WEAVE_NO_BRIDGED_FASTDP") + BRIDGE_TYPE=$(util_op create-bridge "$DOCKER_BRIDGE" "$BRIDGE" "$DATAPATH" "$WEAVE_MTU" "$PORT" "$MAC" "$WEAVE_NO_FASTDP" "$WEAVE_NO_BRIDGED_FASTDP" "$1") # Set some variables that are expected by code that comes later [ "$BRIDGE_TYPE" != "fastdp" ] || DATAPATH="$BRIDGE" MTU=${WEAVE_MTU:-$(cat /sys/class/net/$BRIDGE/mtu)} - - if [ "$1" = "--expect-npc" ] ; then # matches usage in weave-kube launch.sh - # Steer traffic via the NPC - run_iptables -N WEAVE-NPC >/dev/null 2>&1 || true - add_iptables_rule filter FORWARD -o $BRIDGE -j WEAVE-NPC - add_iptables_rule filter FORWARD -o $BRIDGE -m state --state NEW -j NFLOG --nflog-group 86 - add_iptables_rule filter FORWARD -o $BRIDGE -j DROP - else - # Work around the situation where there are no rules allowing traffic - # across our bridge. E.g. ufw - add_iptables_rule filter FORWARD -i $BRIDGE -o $BRIDGE -j ACCEPT - fi - # Forward from weave to the rest of the world - add_iptables_rule filter FORWARD -i $BRIDGE ! -o $BRIDGE -j ACCEPT - # and allow replies back - add_iptables_rule filter FORWARD -o $BRIDGE -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT } create_bridge() { From e7e3608642b939c107d6159a58eda697c06a5f32 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 17 May 2016 18:20:12 +0100 Subject: [PATCH 11/36] Move creation of bridge inside router --- prog/weaver/main.go | 14 ++++++++++++++ weave | 27 ++++++--------------------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/prog/weaver/main.go b/prog/weaver/main.go index cfd68331..65245579 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -125,6 +125,7 @@ func main() { var ( justVersion bool config mesh.Config + bridgeConfig weavenet.BridgeConfig networkConfig weave.NetworkConfig protocolMinVersion int resume bool @@ -169,6 +170,8 @@ func main() { mflag.IntVar(&protocolMinVersion, []string{"-min-protocol-version"}, mesh.ProtocolMinVersion, "minimum weave protocol version") mflag.BoolVar(&resume, []string{"-resume"}, false, "resume connections to previous peers") mflag.StringVar(&ifaceName, []string{"#iface", "-iface"}, "", "name of interface to capture/inject from (disabled if blank)") + mflag.StringVar(&bridgeConfig.WeaveBridgeName, []string{"-weave-bridge"}, "weave", "name of weave bridge") + mflag.StringVar(&bridgeConfig.DockerBridgeName, []string{"-docker-bridge"}, "", "name of Docker bridge") mflag.StringVar(&routerName, []string{"#name", "-name"}, "", "name of router (defaults to MAC of interface)") mflag.StringVar(&nickName, []string{"#nickname", "-nickname"}, "", "nickname of peer (defaults to hostname)") mflag.StringVar(&password, []string{"#password", "-password"}, "", "network password") @@ -178,6 +181,7 @@ func main() { mflag.IntVar(&config.ConnLimit, []string{"#connlimit", "#-connlimit", "-conn-limit"}, 30, "connection limit (0 for unlimited)") mflag.BoolVar(&noDiscovery, []string{"#nodiscovery", "#-nodiscovery", "-no-discovery"}, false, "disable peer discovery") mflag.IntVar(&bufSzMB, []string{"#bufsz", "-bufsz"}, 8, "capture buffer size in MB") + mflag.IntVar(&bridgeConfig.MTU, []string{"-mtu"}, 0, "MTU size") mflag.StringVar(&httpAddr, []string{"#httpaddr", "#-httpaddr", "-http-addr"}, "", "address to bind HTTP interface to (disabled if blank, absolute path indicates unix domain socket)") mflag.StringVar(&statusAddr, []string{"-status-addr"}, "", "address to bind status+metrics interface to (disabled if blank, absolute path indicates unix domain socket)") mflag.StringVar(&ipamConfig.Mode, []string{"-ipalloc-init"}, "", "allocator initialisation strategy (consensus, seed or observer)") @@ -244,6 +248,16 @@ func main() { networkConfig.PacketLogging = nopPacketLogging{} } + bridgeConfig.DatapathName = datapathName + if bridgeConfig.DockerBridgeName != "" { + if err := weavenet.EnforceAddrAssignType(bridgeConfig.DockerBridgeName); err != nil { + Log.Errorf("While checking address assignment type of %s: %s", bridgeConfig.DockerBridgeName, err) + } + } + bridgeType, err := weavenet.CreateBridge(&bridgeConfig) + checkFatal(err) + Log.Println("Bridge type is", bridgeType) + config.Password = determinePassword(password) overlay, bridge := createOverlay(datapathName, ifaceName, isAWSVPC, config.Host, config.Port, bufSzMB, config.Password != nil) diff --git a/weave b/weave index 88d9e494..adbed5ba 100755 --- a/weave +++ b/weave @@ -281,7 +281,6 @@ DATAPATH=datapath CONTAINER_IFNAME=ethwe BRIDGE_IFNAME=v${CONTAINER_IFNAME}-bridge DATAPATH_IFNAME=v${CONTAINER_IFNAME}-datapath -PCAP_IFNAME=v${CONTAINER_IFNAME}-pcap PORT=${WEAVE_PORT:-6783} HTTP_ADDR=${WEAVE_HTTP_ADDR:-127.0.0.1:6784} STATUS_ADDR=${WEAVE_STATUS_ADDR:-127.0.0.1:6782} @@ -576,16 +575,9 @@ configure_veth_attached_bridge() { ip link set $GUEST_IFNAME master $bridge } -router_opts_fastdp() { - echo "--datapath $DATAPATH" -} - -router_opts_bridge() { - echo "--iface $PCAP_IFNAME" -} - -router_opts_bridged_fastdp() { - router_opts_fastdp "$@" +router_bridge_opts() { + echo --docker-bridge "$DOCKER_BRIDGE" --weave-bridge "$BRIDGE" --datapath "$DATAPATH" + [ -z "$WEAVE_MTU" ] || echo --mtu "$WEAVE_MTU" } ask_version() { @@ -1287,8 +1279,6 @@ common_launch_args() { launch_router() { LAUNCHING_ROUTER=1 check_forwarding_rules - enforce_docker_bridge_addr_assign_type - # backward compatibility... if is_cidr "$1" ; then echo "WARNING: $1 parameter ignored; 'weave launch' no longer takes a CIDR as the first parameter" >&2 @@ -1358,11 +1348,7 @@ launch_router() { eval "set -- $ARGS" setup_cni - create_bridge - # We set the router name to the bridge MAC, which in turn is - # derived from the system UUID (if available), and thus stable - # across reboots. - PEERNAME=$(cat /sys/class/net/$BRIDGE/address) + validate_bridge_type if [ -z "$IPRANGE_SPECIFIED" ] ; then IPRANGE="10.32.0.0/12" @@ -1415,8 +1401,8 @@ launch_router() { -e WEAVE_PASSWORD \ -e CHECKPOINT_DISABLE \ $WEAVE_DOCKER_ARGS $IMAGE $COVERAGE_ARGS \ - --port $CONTAINER_PORT --name "$PEERNAME" --nickname "$(hostname)" \ - $(router_opts_$BRIDGE_TYPE) \ + --port $CONTAINER_PORT --nickname "$(hostname)" \ + $(router_bridge_opts) \ --ipalloc-range "$IPRANGE" \ --dns-effective-listen-address $DOCKER_BRIDGE_IP \ $DNS_ROUTER_OPTS $NO_DNS_OPT \ @@ -1705,7 +1691,6 @@ EOF enforce_docker_bridge_addr_assign_type # We cannot use detect_awsvpc here, because HTTP server might not be started ! docker inspect -f '{{.Config.Cmd}}' $CONTAINER_NAME | grep -q -- "--awsvpc" || AWSVPC=1 - create_bridge fetch_router_args wait_for_status $CONTAINER_NAME http_call $HTTP_ADDR setup_awsvpc From 833eb6166f1aab40c9e695056871d5586b817f1d Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Fri, 14 Apr 2017 16:43:13 +0000 Subject: [PATCH 12/36] Add AWSVPC bridge type --- net/bridge.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/net/bridge.go b/net/bridge.go index 533942d7..d81f1823 100644 --- a/net/bridge.go +++ b/net/bridge.go @@ -25,6 +25,7 @@ const ( Bridge Fastdp BridgedFastdp + AWSVPC Inconsistent ) @@ -39,6 +40,8 @@ func (t BridgeType) String() string { return "fastdp" case BridgedFastdp: return "bridged_fastdp" + case AWSVPC: + return "AWSVPC" case Inconsistent: return "inconsistent" } @@ -58,6 +61,7 @@ func DetectBridgeType(weaveBridgeName, datapathName string) BridgeType { return Fastdp case isDatapath(datapath) && isBridge(bridge): return BridgedFastdp + // TODO: AWSVPC default: return Inconsistent } From abadbf883a62678a4d8ec3e5d62f99d685fae9ab Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 11 Apr 2017 17:01:30 +0000 Subject: [PATCH 13/36] AWSVPC mode now part of BridgeConfig --- net/bridge.go | 8 +++++++- prog/weaver/main.go | 15 +++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/net/bridge.go b/net/bridge.go index d81f1823..f883cc19 100644 --- a/net/bridge.go +++ b/net/bridge.go @@ -61,7 +61,8 @@ func DetectBridgeType(weaveBridgeName, datapathName string) BridgeType { return Fastdp case isDatapath(datapath) && isBridge(bridge): return BridgedFastdp - // TODO: AWSVPC + // We cannot detect AWSVPC here; it looks just like Bridge. + // Anyone who cares will have to know some other way. default: return Inconsistent } @@ -177,6 +178,7 @@ type BridgeConfig struct { DatapathName string NoFastdp bool NoBridgedFastdp bool + IsAWSVPC bool ExpectNPC bool MTU int Mac string @@ -223,6 +225,10 @@ func CreateBridge(config *BridgeConfig) (BridgeType, error) { } } + if config.IsAWSVPC { + bridgeType = AWSVPC + } + if bridgeType == Bridge { if err := EthtoolTXOff(config.WeaveBridgeName); err != nil { return bridgeType, err diff --git a/prog/weaver/main.go b/prog/weaver/main.go index 65245579..daccdd6f 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -148,7 +148,6 @@ func main() { datapathName string trustedSubnetStr string dbPrefix string - isAWSVPC bool discoveryEndpoint string token string advertiseAddress string @@ -199,7 +198,7 @@ func main() { mflag.StringVar(&datapathName, []string{"-datapath"}, "", "ODP datapath name") mflag.StringVar(&trustedSubnetStr, []string{"-trusted-subnets"}, "", "comma-separated list of trusted subnets in CIDR notation") mflag.StringVar(&dbPrefix, []string{"-db-prefix"}, "/weavedb/weave", "pathname/prefix of filename to store data") - mflag.BoolVar(&isAWSVPC, []string{"-awsvpc"}, false, "use AWS VPC for routing") + mflag.BoolVar(&bridgeConfig.IsAWSVPC, []string{"-awsvpc"}, false, "use AWS VPC for routing") mflag.StringVar(&discoveryEndpoint, []string{"-peer-discovery-url"}, "https://cloud.weave.works/api/net", "url for peer discovery") mflag.StringVar(&token, []string{"-token"}, "", "token for peer discovery") mflag.StringVar(&advertiseAddress, []string{"-advertise-address"}, "", "address to advertise for peer discovery") @@ -260,7 +259,7 @@ func main() { config.Password = determinePassword(password) - overlay, bridge := createOverlay(datapathName, ifaceName, isAWSVPC, config.Host, config.Port, bufSzMB, config.Password != nil) + overlay, bridge := createOverlay(datapathName, ifaceName, bridgeConfig, config.Host, config.Port, bufSzMB, config.Password != nil) networkConfig.Bridge = bridge if bridge != nil { @@ -280,7 +279,7 @@ func main() { config.TrustedSubnets = parseTrustedSubnets(trustedSubnetStr) config.PeerDiscovery = !noDiscovery - if isAWSVPC && len(config.Password) > 0 { + if bridgeConfig.IsAWSVPC && len(config.Password) > 0 { Log.Fatalf("--awsvpc mode is not compatible with the --password option") } @@ -326,7 +325,7 @@ func main() { } network := "" - if isAWSVPC { + if bridgeConfig.IsAWSVPC { network = "awsvpc" } checkForUpdates(dockerVersion, network) @@ -349,7 +348,7 @@ func main() { ) if ipamConfig.Enabled() { var t tracker.LocalRangeTracker - if isAWSVPC { + if bridgeConfig.IsAWSVPC { Log.Infoln("Creating AWSVPC LocalRangeTracker") t, err = tracker.NewAWSVPCTracker(weavenet.WeaveBridgeName) if err != nil { @@ -467,13 +466,13 @@ func (nopPacketLogging) LogPacket(string, weave.PacketKey) { func (nopPacketLogging) LogForwardPacket(string, weave.ForwardPacketKey) { } -func createOverlay(datapathName string, ifaceName string, isAWSVPC bool, host string, port int, bufSzMB int, enableEncryption bool) (weave.NetworkOverlay, weave.Bridge) { +func createOverlay(datapathName string, ifaceName string, config weavenet.BridgeConfig, host string, port int, bufSzMB int, enableEncryption bool) (weave.NetworkOverlay, weave.Bridge) { overlay := weave.NewOverlaySwitch() var bridge weave.Bridge var ignoreSleeve bool switch { - case isAWSVPC: + case config.IsAWSVPC: vpc := weave.NewAWSVPC() overlay.Add("awsvpc", vpc) bridge = weave.NullBridge{} From a26fc67ded1dce7193e314e7bbe5f814dde4d60b Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 18 May 2016 13:15:06 +0100 Subject: [PATCH 14/36] New way of initializing; remove --iface option --- prog/weaver/main.go | 25 ++++++++++--------------- weave | 1 + 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/prog/weaver/main.go b/prog/weaver/main.go index daccdd6f..fe222889 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -129,7 +129,6 @@ func main() { networkConfig weave.NetworkConfig protocolMinVersion int resume bool - ifaceName string routerName string nickName string password string @@ -145,7 +144,6 @@ func main() { peers []string noDNS bool dnsConfig dnsConfig - datapathName string trustedSubnetStr string dbPrefix string discoveryEndpoint string @@ -168,7 +166,6 @@ func main() { mflag.IntVar(&config.Port, []string{"#port", "-port"}, mesh.Port, "router port") mflag.IntVar(&protocolMinVersion, []string{"-min-protocol-version"}, mesh.ProtocolMinVersion, "minimum weave protocol version") mflag.BoolVar(&resume, []string{"-resume"}, false, "resume connections to previous peers") - mflag.StringVar(&ifaceName, []string{"#iface", "-iface"}, "", "name of interface to capture/inject from (disabled if blank)") mflag.StringVar(&bridgeConfig.WeaveBridgeName, []string{"-weave-bridge"}, "weave", "name of weave bridge") mflag.StringVar(&bridgeConfig.DockerBridgeName, []string{"-docker-bridge"}, "", "name of Docker bridge") mflag.StringVar(&routerName, []string{"#name", "-name"}, "", "name of router (defaults to MAC of interface)") @@ -195,7 +192,8 @@ func main() { mflag.DurationVar(&dnsConfig.ClientTimeout, []string{"-dns-fallback-timeout"}, nameserver.DefaultClientTimeout, "timeout for fallback DNS requests") mflag.StringVar(&dnsConfig.EffectiveListenAddress, []string{"-dns-effective-listen-address"}, "", "address DNS will actually be listening, after Docker port mapping") mflag.StringVar(&dnsConfig.ResolvConf, []string{"-resolv-conf"}, "", "path to resolver configuration for fallback DNS lookups") - mflag.StringVar(&datapathName, []string{"-datapath"}, "", "ODP datapath name") + mflag.StringVar(&bridgeConfig.DatapathName, []string{"-datapath"}, "", "ODP datapath name") + mflag.BoolVar(&bridgeConfig.NoFastdp, []string{"-no-fastdp"}, false, "Disable Fast Datapath") mflag.StringVar(&trustedSubnetStr, []string{"-trusted-subnets"}, "", "comma-separated list of trusted subnets in CIDR notation") mflag.StringVar(&dbPrefix, []string{"-db-prefix"}, "/weavedb/weave", "pathname/prefix of filename to store data") mflag.BoolVar(&bridgeConfig.IsAWSVPC, []string{"-awsvpc"}, false, "use AWS VPC for routing") @@ -247,7 +245,6 @@ func main() { networkConfig.PacketLogging = nopPacketLogging{} } - bridgeConfig.DatapathName = datapathName if bridgeConfig.DockerBridgeName != "" { if err := weavenet.EnforceAddrAssignType(bridgeConfig.DockerBridgeName); err != nil { Log.Errorf("While checking address assignment type of %s: %s", bridgeConfig.DockerBridgeName, err) @@ -259,7 +256,7 @@ func main() { config.Password = determinePassword(password) - overlay, bridge := createOverlay(datapathName, ifaceName, bridgeConfig, config.Host, config.Port, bufSzMB, config.Password != nil) + overlay, bridge := createOverlay(bridgeType, bridgeConfig, config.Host, config.Port, bufSzMB, config.Password != nil) networkConfig.Bridge = bridge if bridge != nil { @@ -466,29 +463,27 @@ func (nopPacketLogging) LogPacket(string, weave.PacketKey) { func (nopPacketLogging) LogForwardPacket(string, weave.ForwardPacketKey) { } -func createOverlay(datapathName string, ifaceName string, config weavenet.BridgeConfig, host string, port int, bufSzMB int, enableEncryption bool) (weave.NetworkOverlay, weave.Bridge) { +func createOverlay(bridgeType weavenet.BridgeType, config weavenet.BridgeConfig, host string, port int, bufSzMB int, enableEncryption bool) (weave.NetworkOverlay, weave.Bridge) { overlay := weave.NewOverlaySwitch() var bridge weave.Bridge var ignoreSleeve bool - switch { - case config.IsAWSVPC: + switch bridgeType { + case weavenet.AWSVPC: vpc := weave.NewAWSVPC() overlay.Add("awsvpc", vpc) bridge = weave.NullBridge{} // Currently, we do not support any overlay with AWSVPC ignoreSleeve = true - case datapathName != "" && ifaceName != "": - Log.Fatal("At most one of --datapath and --iface must be specified.") - case datapathName != "": - iface, err := weavenet.EnsureInterface(datapathName) + case weavenet.Fastdp, weavenet.BridgedFastdp: + iface, err := weavenet.EnsureInterface(config.DatapathName) checkFatal(err) fastdp, err := weave.NewFastDatapath(iface, port, enableEncryption) checkFatal(err) bridge = fastdp.Bridge() overlay.Add("fastdp", fastdp.Overlay()) - case ifaceName != "": - iface, err := weavenet.EnsureInterface(ifaceName) + case weavenet.Bridge: + iface, err := weavenet.EnsureInterface(weavenet.PcapIfaceName) checkFatal(err) bridge, err = weave.NewPcap(iface, bufSzMB*1024*1024) // bufsz flag is in MB checkFatal(err) diff --git a/weave b/weave index adbed5ba..95156f63 100755 --- a/weave +++ b/weave @@ -578,6 +578,7 @@ configure_veth_attached_bridge() { router_bridge_opts() { echo --docker-bridge "$DOCKER_BRIDGE" --weave-bridge "$BRIDGE" --datapath "$DATAPATH" [ -z "$WEAVE_MTU" ] || echo --mtu "$WEAVE_MTU" + [ -z $WEAVE_NO_FASTDP ] || echo --no-fastdp } ask_version() { From 8297652204deafe3dd10aaa6d514e652778db64a Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 7 Mar 2017 23:33:27 +0000 Subject: [PATCH 15/36] Read our peerName from persistence file if necessary Mount directories for machine ID files into weave container --- prog/weaver/main.go | 18 ++++++++++++------ weave | 2 ++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/prog/weaver/main.go b/prog/weaver/main.go index fe222889..b6b304f0 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -146,6 +146,7 @@ func main() { dnsConfig dnsConfig trustedSubnetStr string dbPrefix string + hostRoot string discoveryEndpoint string token string advertiseAddress string @@ -197,6 +198,7 @@ func main() { mflag.StringVar(&trustedSubnetStr, []string{"-trusted-subnets"}, "", "comma-separated list of trusted subnets in CIDR notation") mflag.StringVar(&dbPrefix, []string{"-db-prefix"}, "/weavedb/weave", "pathname/prefix of filename to store data") mflag.BoolVar(&bridgeConfig.IsAWSVPC, []string{"-awsvpc"}, false, "use AWS VPC for routing") + mflag.StringVar(&hostRoot, []string{"-host-root"}, "", "path to reach host filesystem") mflag.StringVar(&discoveryEndpoint, []string{"-peer-discovery-url"}, "https://cloud.weave.works/api/net", "url for peer discovery") mflag.StringVar(&token, []string{"-token"}, "", "token for peer discovery") mflag.StringVar(&advertiseAddress, []string{"-advertise-address"}, "", "address to advertise for peer discovery") @@ -250,6 +252,10 @@ func main() { Log.Errorf("While checking address assignment type of %s: %s", bridgeConfig.DockerBridgeName, err) } } + + name := peerName(routerName, dbPrefix, hostRoot) + + bridgeConfig.Mac = name.String() bridgeType, err := weavenet.CreateBridge(&bridgeConfig) checkFatal(err) Log.Println("Bridge type is", bridgeType) @@ -265,8 +271,6 @@ func main() { } } - name := peerName(routerName) - if nickName == "" { var err error nickName, err = os.Hostname() @@ -586,13 +590,15 @@ func determinePassword(password string) []byte { return []byte(password) } -func peerName(routerName string) mesh.PeerName { +func peerName(routerName, dbPrefix, hostRoot string) mesh.PeerName { if routerName == "" { iface, err := net.InterfaceByName(weavenet.WeaveBridgeName) - if err != nil { - Log.Fatalf("Unable to find bridge %q", weavenet.WeaveBridgeName) + if err == nil { + routerName = iface.HardwareAddr.String() + } else { + routerName, err = weavenet.GetSystemPeerName(dbPrefix, hostRoot) + checkFatal(err) } - routerName = iface.HardwareAddr.String() } name, err := mesh.PeerNameFromUserInput(routerName) checkFatal(err) diff --git a/weave b/weave index 95156f63..5fbfeb0c 100755 --- a/weave +++ b/weave @@ -1399,10 +1399,12 @@ launch_router() { --volumes-from $DB_CONTAINER_NAME \ -v $RESOLV_CONF_DIR:/var/run/weave/etc \ -v /run/docker/plugins:/run/docker/plugins \ + -v /etc:/host/etc -v /var/lib/dbus:/host/var/lib/dbus \ -e WEAVE_PASSWORD \ -e CHECKPOINT_DISABLE \ $WEAVE_DOCKER_ARGS $IMAGE $COVERAGE_ARGS \ --port $CONTAINER_PORT --nickname "$(hostname)" \ + --host-root=/host \ $(router_bridge_opts) \ --ipalloc-range "$IPRANGE" \ --dns-effective-listen-address $DOCKER_BRIDGE_IP \ From c3a41286fd9b0297e7169bb690fbe0acabfd770c Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 9 Apr 2017 19:00:16 +0000 Subject: [PATCH 16/36] Use the bridge name passed in on the command-line more --- prog/weaver/main.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/prog/weaver/main.go b/prog/weaver/main.go index b6b304f0..773f6939 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -253,7 +253,7 @@ func main() { } } - name := peerName(routerName, dbPrefix, hostRoot) + name := peerName(routerName, bridgeConfig.WeaveBridgeName, dbPrefix, hostRoot) bridgeConfig.Mac = name.String() bridgeType, err := weavenet.CreateBridge(&bridgeConfig) @@ -351,14 +351,14 @@ func main() { var t tracker.LocalRangeTracker if bridgeConfig.IsAWSVPC { Log.Infoln("Creating AWSVPC LocalRangeTracker") - t, err = tracker.NewAWSVPCTracker(weavenet.WeaveBridgeName) + t, err = tracker.NewAWSVPCTracker(bridgeConfig.WeaveBridgeName) if err != nil { Log.Fatalf("Cannot create AWSVPC LocalRangeTracker: %s", err) } trackerName = "awsvpc" } - preClaims, err := findExistingAddresses(dockerCli, weavenet.WeaveBridgeName) + preClaims, err := findExistingAddresses(dockerCli, bridgeConfig.WeaveBridgeName) checkFatal(err) allocator, defaultSubnet = createAllocator(router, ipamConfig, preClaims, db, t, isKnownPeer) @@ -590,9 +590,9 @@ func determinePassword(password string) []byte { return []byte(password) } -func peerName(routerName, dbPrefix, hostRoot string) mesh.PeerName { +func peerName(routerName, bridgeName, dbPrefix, hostRoot string) mesh.PeerName { if routerName == "" { - iface, err := net.InterfaceByName(weavenet.WeaveBridgeName) + iface, err := net.InterfaceByName(bridgeName) if err == nil { routerName = iface.HardwareAddr.String() } else { From 9494f0655d83e3b5466d34b846bb0f3128e4c260 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 11 Apr 2017 13:44:01 +0000 Subject: [PATCH 17/36] More meaningful errors --- net/bridge.go | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/net/bridge.go b/net/bridge.go index f883cc19..59077be8 100644 --- a/net/bridge.go +++ b/net/bridge.go @@ -1,13 +1,13 @@ package net import ( - "errors" "fmt" "io/ioutil" "net" "github.com/Sirupsen/logrus" "github.com/coreos/go-iptables/iptables" + "github.com/pkg/errors" "github.com/vishvananda/netlink" "github.com/weaveworks/weave/common/odp" @@ -71,9 +71,10 @@ func DetectBridgeType(weaveBridgeName, datapathName string) BridgeType { var ErrSetBridgeMac = errors.New("Setting $DOCKER_BRIDGE MAC (mitigate https://github.com/docker/docker/issues/14908)") func EnforceAddrAssignType(bridgeName string) error { - addrAssignType, err := ioutil.ReadFile(fmt.Sprintf("/sys/class/net/%s/addr_assign_type", bridgeName)) + sysctlFilename := fmt.Sprintf("/sys/class/net/%s/addr_assign_type", bridgeName) + addrAssignType, err := ioutil.ReadFile(sysctlFilename) if err != nil { - return err + return errors.Wrapf(err, "reading %q", sysctlFilename) } // From include/uapi/linux/netdevice.h @@ -85,16 +86,16 @@ func EnforceAddrAssignType(bridgeName string) error { if addrAssignType[0] != '3' { link, err := netlink.LinkByName(bridgeName) if err != nil { - return err + return errors.Wrapf(err, "EnforceAddrAssignType finding bridge %s", bridgeName) } mac, err := RandomMAC() if err != nil { - return err + return errors.Wrap(err, "creating random MAC") } if err := netlink.LinkSetHardwareAddr(link, mac); err != nil { - return err + return errors.Wrapf(err, "setting bridge %s address to %v", bridgeName, mac) } return ErrSetBridgeMac } @@ -198,7 +199,7 @@ func CreateBridge(config *BridgeConfig) (BridgeType, error) { } odpSupported, err := odp.CreateDatapath(config.DatapathName) if err != nil { - return None, err + return None, errors.Wrapf(err, "creating datapath %q", config.DatapathName) } if !odpSupported { bridgeType = Bridge @@ -221,7 +222,7 @@ func CreateBridge(config *BridgeConfig) (BridgeType, error) { } if err = configureIPTables(config); err != nil { - return bridgeType, err + return bridgeType, errors.Wrap(err, "configuring iptables") } } @@ -231,7 +232,7 @@ func CreateBridge(config *BridgeConfig) (BridgeType, error) { if bridgeType == Bridge { if err := EthtoolTXOff(config.WeaveBridgeName); err != nil { - return bridgeType, err + return bridgeType, errors.Wrap(err, "setting tx off") } } @@ -240,7 +241,7 @@ func CreateBridge(config *BridgeConfig) (BridgeType, error) { } if err := ConfigureARPCache(config.WeaveBridgeName); err != nil { - return bridgeType, err + return bridgeType, errors.Wrapf(err, "configuring ARP cache on bridge %q", config.WeaveBridgeName) } return bridgeType, nil @@ -249,7 +250,7 @@ func CreateBridge(config *BridgeConfig) (BridgeType, error) { func initBridgePrep(config *BridgeConfig) error { mac, err := net.ParseMAC(config.Mac) if err != nil { - return err + return errors.Wrapf(err, "parsing bridge MAC %q", config.Mac) } linkAttrs := netlink.NewLinkAttrs() @@ -260,7 +261,7 @@ func initBridgePrep(config *BridgeConfig) error { } linkAttrs.MTU = config.MTU // TODO this probably doesn't work - see weave script if err = netlink.LinkAdd(&netlink.Bridge{LinkAttrs: linkAttrs}); err != nil { - return err + return errors.Wrapf(err, "creating bridge %q", config.WeaveBridgeName) } return nil @@ -273,7 +274,7 @@ func initBridge(config *BridgeConfig) error { if _, err := CreateAndAttachVeth(BridgeIfName, PcapIfaceName, config.WeaveBridgeName, config.MTU, true, func(veth netlink.Link) error { return netlink.LinkSetUp(veth) }); err != nil { - return err + return errors.Wrap(err, "creating pcap veth pair") } return nil @@ -282,7 +283,7 @@ func initBridge(config *BridgeConfig) error { func initFastdp(config *BridgeConfig) error { datapath, err := netlink.LinkByName(config.DatapathName) if err != nil { - return err + return errors.Wrapf(err, "finding datapath %q", config.DatapathName) } if config.MTU == 0 { /* GCE has the lowest underlay network MTU we're likely to encounter on @@ -293,7 +294,10 @@ func initFastdp(config *BridgeConfig) error { which is needed for the vxlan encryption. */ config.MTU = 1376 } - return netlink.LinkSetMTU(datapath, config.MTU) + if err := netlink.LinkSetMTU(datapath, config.MTU); err != nil { + return errors.Wrapf(err, "setting datapath %q mtu %d", config.DatapathName, config.MTU) + } + return nil } func initBridgedFastdp(config *BridgeConfig) error { @@ -305,14 +309,14 @@ func initBridgedFastdp(config *BridgeConfig) error { } if _, err := CreateAndAttachVeth("vethwe-bridge", "vethwe-datapath", config.WeaveBridgeName, config.MTU, true, func(veth netlink.Link) error { if err := netlink.LinkSetUp(veth); err != nil { - return err + return errors.Wrapf(err, "setting link up on %q", veth.Attrs().Name) } if err := odp.AddDatapathInterface(config.DatapathName, veth.Attrs().Name); err != nil { - return err + return errors.Wrapf(err, "adding interface %q to datapath %q", veth.Attrs().Name, config.DatapathName) } return nil }); err != nil { - return err + return errors.Wrap(err, "creating bridged fastdp veth pair") } if err := linkSetUpByName(config.DatapathName); err != nil { @@ -325,7 +329,7 @@ func initBridgedFastdp(config *BridgeConfig) error { func configureIPTables(config *BridgeConfig) error { ipt, err := iptables.New() if err != nil { - return err + return errors.Wrap(err, "creating iptables object") } if config.DockerBridgeName != "" { if config.WeaveBridgeName != config.DockerBridgeName { @@ -400,7 +404,7 @@ func configureIPTables(config *BridgeConfig) error { func linkSetUpByName(linkName string) error { link, err := netlink.LinkByName(linkName) if err != nil { - return err + return errors.Wrapf(err, "setting link up on %q", linkName) } return netlink.LinkSetUp(link) } From ab380ed2c1212b13c0d441653c1f161bb43879ee Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 9 Apr 2017 22:50:45 +0000 Subject: [PATCH 18/36] Set weave bridge address and mtu including the dance with the dummy interface to get MTU set high --- net/bridge.go | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/net/bridge.go b/net/bridge.go index 59077be8..7ca50a51 100644 --- a/net/bridge.go +++ b/net/bridge.go @@ -259,10 +259,31 @@ func initBridgePrep(config *BridgeConfig) error { if config.MTU == 0 { config.MTU = 65535 } - linkAttrs.MTU = config.MTU // TODO this probably doesn't work - see weave script - if err = netlink.LinkAdd(&netlink.Bridge{LinkAttrs: linkAttrs}); err != nil { + link := &netlink.Bridge{LinkAttrs: linkAttrs} + if err = netlink.LinkAdd(link); err != nil { return errors.Wrapf(err, "creating bridge %q", config.WeaveBridgeName) } + if err := netlink.LinkSetHardwareAddr(link, mac); err != nil { + return errors.Wrapf(err, "setting bridge %q mac %v", config.WeaveBridgeName, mac) + } + // Attempting to set the bridge MTU to a high value directly + // fails. Bridges take the lowest MTU of their interfaces. So + // instead we create a temporary interface with the desired MTU, + // attach that to the bridge, and then remove it again. + dummy := &netlink.Dummy{LinkAttrs: netlink.NewLinkAttrs()} + dummy.LinkAttrs.Name = "vethwedu" + if err = netlink.LinkAdd(dummy); err != nil { + return errors.Wrap(err, "creating dummy interface") + } + if err := netlink.LinkSetMTU(dummy, config.MTU); err != nil { + return errors.Wrapf(err, "setting dummy interface mtu to %d", config.MTU) + } + if err := netlink.LinkSetMaster(dummy, link); err != nil { + return errors.Wrap(err, "setting dummy interface master") + } + if err := netlink.LinkDel(dummy); err != nil { + return errors.Wrap(err, "deleting dummy interface") + } return nil } From c8e0c12d34a690460acd493561f0f76feeda8dfd Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sat, 15 Apr 2017 10:29:15 +0000 Subject: [PATCH 19/36] Move AWSVPC sysctls into Go code --- net/bridge.go | 12 ++++++++++++ weave | 8 -------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/net/bridge.go b/net/bridge.go index 7ca50a51..c0514e50 100644 --- a/net/bridge.go +++ b/net/bridge.go @@ -228,6 +228,18 @@ func CreateBridge(config *BridgeConfig) (BridgeType, error) { if config.IsAWSVPC { bridgeType = AWSVPC + // Set proxy_arp on the bridge, so that it could accept packets destined + // to containers within the same subnet but running on remote hosts. + // Without it, exact routes on each container are required. + if err := sysctl("net/ipv4/conf/"+config.WeaveBridgeName+"/proxy_arp", "1"); err != nil { + return bridgeType, errors.Wrap(err, "setting proxy_arp") + } + // Avoid delaying the first ARP request. Also, setting it to 0 avoids + // placing the request into a bounded queue as it can be seen: + // https://git.kernel.org/cgit/linux/kernel/git/stable/linux-stable.git/tree/net/ipv4/arp.c?id=refs/tags/v4.6.1#n819 + if err := sysctl("net/ipv4/neigh/"+config.WeaveBridgeName+"/proxy_delay", "0"); err != nil { + return bridgeType, errors.Wrap(err, "setting proxy_arp") + } } if bridgeType == Bridge { diff --git a/weave b/weave index 5fbfeb0c..b79fe4e1 100755 --- a/weave +++ b/weave @@ -1427,14 +1427,6 @@ launch_router() { setup_awsvpc() { if [ -n "$AWSVPC" ]; then - # Set proxy_arp on the bridge, so that it could accept packets destined - # to containers within the same subnet but running on remote hosts. - # Without it, exact routes on each container are required. - echo 1 >$PROC_PATH/sys/net/ipv4/conf/$BRIDGE/proxy_arp - # Avoid delaying the first ARP request. Also, setting it to 0 avoids - # placing the request into a bounded queue as it can be seen: - # https://git.kernel.org/cgit/linux/kernel/git/stable/linux-stable.git/tree/net/ipv4/arp.c?id=refs/tags/v4.6.1#n819 - echo 0 >$PROC_PATH/sys/net/ipv4/neigh/$BRIDGE/proxy_delay expose_ip fi } From f05bb6c4e01674524c52fb4c5c74a124ab18b256 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 12 Apr 2017 17:10:06 +0000 Subject: [PATCH 20/36] Don't assume we always run in in host pid space Pass the path to the proc filesystem down --- net/arp.go | 13 +++++++------ net/bridge.go | 11 ++++++----- plugin/net/watcher.go | 2 +- prog/weaver/main.go | 4 +++- prog/weaveutil/bridge.go | 9 +++++---- prog/weaveutil/netdev.go | 10 ++++++---- weave | 2 +- 7 files changed, 29 insertions(+), 22 deletions(-) diff --git a/net/arp.go b/net/arp.go index 8315db94..39322b9a 100644 --- a/net/arp.go +++ b/net/arp.go @@ -1,5 +1,6 @@ package net +import "path/filepath" import "fmt" import "io" import "os" @@ -7,21 +8,21 @@ import "os" // Configure the ARP cache parameters for the given interface. This // makes containers react more quickly to a change in the MAC address // associated with an IP address. -func ConfigureARPCache(name string) error { - if err := sysctl(fmt.Sprintf("net/ipv4/neigh/%s/base_reachable_time", name), "5"); err != nil { +func ConfigureARPCache(procPath, name string) error { + if err := sysctl(procPath, fmt.Sprintf("net/ipv4/neigh/%s/base_reachable_time", name), "5"); err != nil { return err } - if err := sysctl(fmt.Sprintf("net/ipv4/neigh/%s/delay_first_probe_time", name), "2"); err != nil { + if err := sysctl(procPath, fmt.Sprintf("net/ipv4/neigh/%s/delay_first_probe_time", name), "2"); err != nil { return err } - if err := sysctl(fmt.Sprintf("net/ipv4/neigh/%s/ucast_solicit", name), "1"); err != nil { + if err := sysctl(procPath, fmt.Sprintf("net/ipv4/neigh/%s/ucast_solicit", name), "1"); err != nil { return err } return nil } -func sysctl(variable, value string) error { - f, err := os.OpenFile(fmt.Sprintf("/proc/sys/%s", variable), os.O_WRONLY, 0) +func sysctl(procPath, variable, value string) error { + f, err := os.OpenFile(filepath.Join(procPath, "/sys/", variable), os.O_WRONLY, 0) if err != nil { return err } diff --git a/net/bridge.go b/net/bridge.go index c0514e50..a5949b63 100644 --- a/net/bridge.go +++ b/net/bridge.go @@ -4,6 +4,7 @@ import ( "fmt" "io/ioutil" "net" + "path/filepath" "github.com/Sirupsen/logrus" "github.com/coreos/go-iptables/iptables" @@ -71,7 +72,7 @@ func DetectBridgeType(weaveBridgeName, datapathName string) BridgeType { var ErrSetBridgeMac = errors.New("Setting $DOCKER_BRIDGE MAC (mitigate https://github.com/docker/docker/issues/14908)") func EnforceAddrAssignType(bridgeName string) error { - sysctlFilename := fmt.Sprintf("/sys/class/net/%s/addr_assign_type", bridgeName) + sysctlFilename := filepath.Join("/sys/class/net/", bridgeName, "/addr_assign_type") addrAssignType, err := ioutil.ReadFile(sysctlFilename) if err != nil { return errors.Wrapf(err, "reading %q", sysctlFilename) @@ -186,7 +187,7 @@ type BridgeConfig struct { Port int } -func CreateBridge(config *BridgeConfig) (BridgeType, error) { +func CreateBridge(procPath string, config *BridgeConfig) (BridgeType, error) { bridgeType := DetectBridgeType(config.WeaveBridgeName, config.DatapathName) if bridgeType == None { @@ -231,13 +232,13 @@ func CreateBridge(config *BridgeConfig) (BridgeType, error) { // Set proxy_arp on the bridge, so that it could accept packets destined // to containers within the same subnet but running on remote hosts. // Without it, exact routes on each container are required. - if err := sysctl("net/ipv4/conf/"+config.WeaveBridgeName+"/proxy_arp", "1"); err != nil { + if err := sysctl(procPath, "net/ipv4/conf/"+config.WeaveBridgeName+"/proxy_arp", "1"); err != nil { return bridgeType, errors.Wrap(err, "setting proxy_arp") } // Avoid delaying the first ARP request. Also, setting it to 0 avoids // placing the request into a bounded queue as it can be seen: // https://git.kernel.org/cgit/linux/kernel/git/stable/linux-stable.git/tree/net/ipv4/arp.c?id=refs/tags/v4.6.1#n819 - if err := sysctl("net/ipv4/neigh/"+config.WeaveBridgeName+"/proxy_delay", "0"); err != nil { + if err := sysctl(procPath, "net/ipv4/neigh/"+config.WeaveBridgeName+"/proxy_delay", "0"); err != nil { return bridgeType, errors.Wrap(err, "setting proxy_arp") } } @@ -252,7 +253,7 @@ func CreateBridge(config *BridgeConfig) (BridgeType, error) { return bridgeType, err } - if err := ConfigureARPCache(config.WeaveBridgeName); err != nil { + if err := ConfigureARPCache(procPath, config.WeaveBridgeName); err != nil { return bridgeType, errors.Wrapf(err, "configuring ARP cache on bridge %q", config.WeaveBridgeName) } diff --git a/plugin/net/watcher.go b/plugin/net/watcher.go index 736751d7..9a795611 100644 --- a/plugin/net/watcher.go +++ b/plugin/net/watcher.go @@ -53,7 +53,7 @@ func (w *watcher) ContainerStarted(id string) { rootDir = "/host" } netNSPath := weavenet.NSPathByPidWithRoot(rootDir, info.State.Pid) - if _, err := weavenet.WithNetNS(netNSPath, "configure-arp", weavenet.VethName); err != nil { + if _, err := weavenet.WithNetNS(netNSPath, "configure-arp", weavenet.VethName, rootDir); err != nil { w.driver.warn("ContainerStarted", "unable to configure interfaces: %s", err) } } diff --git a/prog/weaver/main.go b/prog/weaver/main.go index 773f6939..3da99359 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -147,6 +147,7 @@ func main() { trustedSubnetStr string dbPrefix string hostRoot string + procPath string discoveryEndpoint string token string advertiseAddress string @@ -197,6 +198,7 @@ func main() { mflag.BoolVar(&bridgeConfig.NoFastdp, []string{"-no-fastdp"}, false, "Disable Fast Datapath") mflag.StringVar(&trustedSubnetStr, []string{"-trusted-subnets"}, "", "comma-separated list of trusted subnets in CIDR notation") mflag.StringVar(&dbPrefix, []string{"-db-prefix"}, "/weavedb/weave", "pathname/prefix of filename to store data") + mflag.StringVar(&procPath, []string{"-proc-path"}, "/proc", "path to reach host /proc filesystem") mflag.BoolVar(&bridgeConfig.IsAWSVPC, []string{"-awsvpc"}, false, "use AWS VPC for routing") mflag.StringVar(&hostRoot, []string{"-host-root"}, "", "path to reach host filesystem") mflag.StringVar(&discoveryEndpoint, []string{"-peer-discovery-url"}, "https://cloud.weave.works/api/net", "url for peer discovery") @@ -256,7 +258,7 @@ func main() { name := peerName(routerName, bridgeConfig.WeaveBridgeName, dbPrefix, hostRoot) bridgeConfig.Mac = name.String() - bridgeType, err := weavenet.CreateBridge(&bridgeConfig) + bridgeType, err := weavenet.CreateBridge(procPath, &bridgeConfig) checkFatal(err) Log.Println("Bridge type is", bridgeType) diff --git a/prog/weaveutil/bridge.go b/prog/weaveutil/bridge.go index 0b63b336..9c38c815 100644 --- a/prog/weaveutil/bridge.go +++ b/prog/weaveutil/bridge.go @@ -17,8 +17,8 @@ func detectBridgeType(args []string) error { } func createBridge(args []string) error { - if len(args) != 9 { - cmdUsage("create-bridge", " ") + if len(args) != 10 { + cmdUsage("create-bridge", " ") } mtu, err := strconv.Atoi(args[3]) @@ -38,9 +38,10 @@ func createBridge(args []string) error { Mac: args[5], NoFastdp: args[6] != "", NoBridgedFastdp: args[7] != "", - ExpectNPC: args[8] == "--expect-npc", + ExpectNPC: args[9] == "--expect-npc", } - bridgeType, err := weavenet.CreateBridge(&config) + procPath := args[8] + bridgeType, err := weavenet.CreateBridge(procPath, &config) fmt.Println(bridgeType.String()) return err } diff --git a/prog/weaveutil/netdev.go b/prog/weaveutil/netdev.go index d35e395b..457e30e3 100644 --- a/prog/weaveutil/netdev.go +++ b/prog/weaveutil/netdev.go @@ -60,7 +60,8 @@ func setupIface(args []string) error { if err := netlink.LinkSetName(link, newIfName); err != nil { return err } - if err := weavenet.ConfigureARPCache(newIfName); err != nil { + // This is only called by AttachContainer which is only called in host pid namespace + if err := weavenet.ConfigureARPCache("/proc", newIfName); err != nil { return err } if err := ipt.Append("filter", "INPUT", "-i", newIfName, "-d", "224.0.0.0/4", "-j", "DROP"); err != nil { @@ -71,10 +72,11 @@ func setupIface(args []string) error { } func configureARP(args []string) error { - if len(args) != 1 { - cmdUsage("configure-arp", "") + if len(args) != 2 { + cmdUsage("configure-arp", " ") } prefix := args[0] + rootPath := args[1] links, err := netlink.LinkList() if err != nil { @@ -83,7 +85,7 @@ func configureARP(args []string) error { for _, link := range links { ifName := link.Attrs().Name if strings.HasPrefix(ifName, prefix) { - weavenet.ConfigureARPCache(ifName) + weavenet.ConfigureARPCache(rootPath+"/proc", ifName) if addrs, err := netlink.AddrList(link, netlink.FAMILY_V4); err == nil { for _, addr := range addrs { arping.GratuitousArpOverIfaceByName(addr.IPNet.IP, ifName) diff --git a/weave b/weave index b79fe4e1..95d7673b 100755 --- a/weave +++ b/weave @@ -429,7 +429,7 @@ try_create_bridge() { WEAVEDB_DIR_PATH="$HOST_ROOT/$(docker inspect -f '{{with index .Mounts 0}}{{.Source}}{{end}}' $DB_CONTAINER_NAME 2>/dev/null)" fi MAC=$(util_op unique-id "$WEAVEDB_DIR_PATH/weave" "$HOST_ROOT") - BRIDGE_TYPE=$(util_op create-bridge "$DOCKER_BRIDGE" "$BRIDGE" "$DATAPATH" "$WEAVE_MTU" "$PORT" "$MAC" "$WEAVE_NO_FASTDP" "$WEAVE_NO_BRIDGED_FASTDP" "$1") + BRIDGE_TYPE=$(util_op create-bridge "$DOCKER_BRIDGE" "$BRIDGE" "$DATAPATH" "$WEAVE_MTU" "$PORT" "$MAC" "$WEAVE_NO_FASTDP" "$WEAVE_NO_BRIDGED_FASTDP" "$PROC_PATH" "$1") # Set some variables that are expected by code that comes later [ "$BRIDGE_TYPE" != "fastdp" ] || DATAPATH="$BRIDGE" MTU=${WEAVE_MTU:-$(cat /sys/class/net/$BRIDGE/mtu)} From 3b54372eabe66e8ce7b923811acafd0acc3c8a07 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Fri, 14 Apr 2017 16:28:38 +0000 Subject: [PATCH 21/36] Let weaver create the bridge in net-plugin launch --- prog/net-plugin/launch.sh | 25 ++++++++----------------- weave | 16 ---------------- 2 files changed, 8 insertions(+), 33 deletions(-) diff --git a/prog/net-plugin/launch.sh b/prog/net-plugin/launch.sh index a372be13..493a74c5 100755 --- a/prog/net-plugin/launch.sh +++ b/prog/net-plugin/launch.sh @@ -29,22 +29,11 @@ SWARM_MANAGER_PEERS=$(/usr/bin/weaveutil swarm-manager-peers) # Prevent from restoring from a persisted peers list rm -f "/restart.sentinel" -/home/weave/weave --local create-bridge \ - --proc-path=/host/proc \ - --weavedb-dir-path=$WEAVE_DIR \ - --force - -BRIDGE_OPTIONS="--datapath=datapath" -if [ "$(/home/weave/weave --local bridge-type)" = "bridge" ]; then - # TODO: Call into weave script to do this - if ! ip link show vethwe-pcap >/dev/null 2>&1; then - ip link add name vethwe-bridge type veth peer name vethwe-pcap - ip link set vethwe-bridge up - ip link set vethwe-pcap up - ip link set vethwe-bridge master weave - fi - BRIDGE_OPTIONS="--iface=vethwe-pcap" -fi +router_bridge_opts() { + echo --datapath=datapath + [ -z "$WEAVE_MTU" ] || echo --mtu "$WEAVE_MTU" + [ -z "$WEAVE_NO_FASTDP" ] || echo --no-fastdp +} if [ -z "$IPALLOC_INIT" ]; then IPALLOC_INIT="observer" @@ -53,7 +42,9 @@ if [ -z "$IPALLOC_INIT" ]; then fi fi -exec /home/weave/weaver $EXTRA_ARGS --port=6783 $BRIDGE_OPTIONS \ +exec /home/weave/weaver $EXTRA_ARGS --port=6783 $(router_bridge_opts) \ + --host-root=/host \ + --proc-path=/host/proc \ --http-addr=$HTTP_ADDR --status-addr=$STATUS_ADDR \ --no-dns \ --ipalloc-range=$IPALLOC_RANGE \ diff --git a/weave b/weave index 95d7673b..2d01d577 100755 --- a/weave +++ b/weave @@ -1625,22 +1625,6 @@ case "$COMMAND" in # intentionally undocumented since it assumes knowledge of weave # internals create-bridge) - while [ $# -gt 0 ]; do - case "$1" in - --proc-path=*) - PROC_PATH="${1#*=}" - shift - ;; - --weavedb-dir-path=*) - WEAVEDB_DIR_PATH="${1#*=}" - shift - ;; - *) - break - ;; - esac - done - if [ "$1" != "--force" ] ; then cat 1>&2 < Date: Thu, 13 Apr 2017 10:11:46 +0000 Subject: [PATCH 22/36] Let weaver create the bridge in weave-kube --- prog/weave-kube/launch.sh | 16 +++------------- prog/weaver/main.go | 1 + 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/prog/weave-kube/launch.sh b/prog/weave-kube/launch.sh index 1b851d49..35557fb4 100755 --- a/prog/weave-kube/launch.sh +++ b/prog/weave-kube/launch.sh @@ -54,9 +54,6 @@ if [ "${EXPECT_NPC}" = "0" ]; then WEAVE_NPC_OPTS="" fi -export HOST_ROOT=/host -/home/weave/weave --local create-bridge --force $WEAVE_NPC_OPTS - # Kubernetes sets HOSTNAME to the host's hostname # when running a pod in host namespace. NICKNAME_ARG="" @@ -65,16 +62,6 @@ if [ -n "$HOSTNAME" ] ; then fi BRIDGE_OPTIONS="--datapath=datapath" -if [ "$(/home/weave/weave --local bridge-type)" = "bridge" ] ; then - # TODO: Call into weave script to do this - if ! ip link show vethwe-pcap >/dev/null 2>&1 ; then - ip link add name vethwe-bridge type veth peer name vethwe-pcap - ip link set vethwe-bridge up - ip link set vethwe-pcap up - ip link set vethwe-bridge master weave - fi - BRIDGE_OPTIONS="--iface=vethwe-pcap" -fi if [ -z "$KUBE_PEERS" ]; then if ! KUBE_PEERS=$(/home/weave/kube-peers) || [ -z "$KUBE_PEERS" ]; then @@ -109,6 +96,7 @@ post_start_actions() { fi fi mkdir -p $HOST_ROOT/etc/cni/net.d + export HOST_ROOT /home/weave/weave --local setup-cni # Expose the weave network so host processes can communicate with pods @@ -118,9 +106,11 @@ post_start_actions() { post_start_actions & /home/weave/weaver $EXTRA_ARGS --port=6783 $BRIDGE_OPTIONS \ + --host-root=$HOST_ROOT \ --http-addr=$HTTP_ADDR --status-addr=$STATUS_ADDR --docker-api='' --no-dns \ --ipalloc-range=$IPALLOC_RANGE $NICKNAME_ARG \ --ipalloc-init $IPALLOC_INIT \ --conn-limit=$CONN_LIMIT \ + $WEAVE_NPC_OPTS \ "$@" \ $KUBE_PEERS diff --git a/prog/weaver/main.go b/prog/weaver/main.go index 3da99359..830cb630 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -170,6 +170,7 @@ func main() { mflag.BoolVar(&resume, []string{"-resume"}, false, "resume connections to previous peers") mflag.StringVar(&bridgeConfig.WeaveBridgeName, []string{"-weave-bridge"}, "weave", "name of weave bridge") mflag.StringVar(&bridgeConfig.DockerBridgeName, []string{"-docker-bridge"}, "", "name of Docker bridge") + mflag.BoolVar(&bridgeConfig.ExpectNPC, []string{"-expect-npc"}, false, "set up iptables rules for npc") mflag.StringVar(&routerName, []string{"#name", "-name"}, "", "name of router (defaults to MAC of interface)") mflag.StringVar(&nickName, []string{"#nickname", "-nickname"}, "", "nickname of peer (defaults to hostname)") mflag.StringVar(&password, []string{"#password", "-password"}, "", "network password") From 4ad1c22ef6c1832786773e134eb4dbbba105b78a Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 13 Apr 2017 11:54:15 +0000 Subject: [PATCH 23/36] Move initial DNS population into Go code --- prog/weaver/main.go | 3 +++ prog/weaver/reclaim.go | 49 ++++++++++++++++++++++++++++++++++++++++++ weave | 13 ----------- 3 files changed, 52 insertions(+), 13 deletions(-) diff --git a/prog/weaver/main.go b/prog/weaver/main.go index 830cb630..ec4d04c3 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -384,6 +384,9 @@ func main() { ns.Start() defer ns.Stop() dnsserver.ActivateAndServe() + if dockerCli != nil { + populateDNS(ns, dockerCli, name, bridgeConfig.WeaveBridgeName) + } defer dnsserver.Stop() } diff --git a/prog/weaver/reclaim.go b/prog/weaver/reclaim.go index a90edfe9..88461818 100644 --- a/prog/weaver/reclaim.go +++ b/prog/weaver/reclaim.go @@ -2,13 +2,16 @@ package main import ( "net" + "strings" docker "github.com/fsouza/go-dockerclient" + "github.com/weaveworks/mesh" "github.com/weaveworks/weave/api" "github.com/weaveworks/weave/common" weavedocker "github.com/weaveworks/weave/common/docker" "github.com/weaveworks/weave/ipam" + "github.com/weaveworks/weave/nameserver" weavenet "github.com/weaveworks/weave/net" "github.com/weaveworks/weave/net/address" ) @@ -87,3 +90,49 @@ func findExistingAddresses(dockerCli *weavedocker.Client, bridgeName string) (ad } return addrs, nil } + +func populateDNS(ns *nameserver.Nameserver, dockerCli *weavedocker.Client, ourName mesh.PeerName, bridgeName string) (addrs []ipam.PreClaim, err error) { + // Find all veths connected to the bridge + peerIDs, err := weavenet.ConnectedToBridgeVethPeerIds(bridgeName) + if err != nil { + return nil, err + } + + // Now iterate over all containers to see if they have a network + // namespace with an attached interface + if dockerCli != nil { + containerIDs, err := dockerCli.RunningContainerIDs() + if err != nil { + return nil, err + } + + for _, cid := range containerIDs { + container, err := dockerCli.InspectContainer(cid) + if err != nil { + if _, ok := err.(*docker.NoSuchContainer); ok { + continue + } + return nil, err + } + if container.State.Pid != 0 { + // Docker allows containers to have fqdns in two wauys: + // where the container explicitly has a domain name, and when its + // hostname contains the domain name, recognized by it having dots. + if container.Config.Domainname == "" && !strings.Contains(container.Config.Hostname, ".") { + continue + } + netDevs, err := weavenet.GetNetDevsByVethPeerIds(container.State.Pid, peerIDs) + if err != nil { + return nil, err + } + fqdn := container.Config.Hostname + "." + container.Config.Domainname + for _, netDev := range netDevs { + for _, cidr := range netDev.CIDRs { + ns.AddEntryFQDN(fqdn, cid, ourName, address.FromIP4(cidr.IP)) + } + } + } + } + } + return addrs, nil +} diff --git a/weave b/weave index 2d01d577..402b4db6 100755 --- a/weave +++ b/weave @@ -1417,7 +1417,6 @@ launch_router() { "$@") wait_for_status $CONTAINER_NAME http_call $HTTP_ADDR setup_awsvpc - populate_router if [ -n "$ENABLE_PLUGIN" ]; then wait_for_status $CONTAINER_NAME http_call_unix $CONTAINER_NAME plugin-status.sock @@ -1437,17 +1436,6 @@ fetch_router_args() { NO_DNS_OPT=$(echo $CONTAINER_ARGS | grep -o -e '--no-dns') || true } -populate_dns_for_one_container() { - [ -z "$4" ] || with_container_fqdn $1 put_dns_fqdn $4 -} - -populate_router() { - if [ -z "$NO_DNS_OPT" ] ; then - # Tell the newly-started weaveDNS about existing weave IPs - with_container_addresses populate_dns_for_one_container weave:allids - fi -} - stop_router() { ! is_plugin_v1_enabled || util_op remove-plugin-network weave || true stop $CONTAINER_NAME "Weave" @@ -1673,7 +1661,6 @@ EOF fetch_router_args wait_for_status $CONTAINER_NAME http_call $HTTP_ADDR setup_awsvpc - populate_router ;; launch-proxy) deprecation_warnings "$@" From 41a261bdcd87b308fcbaae29c6be6b8da0cb937b Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 16 Apr 2017 14:04:42 +0000 Subject: [PATCH 24/36] Add conversion from CIDR to IPNet --- net/address/address.go | 5 +++++ net/address/address_test.go | 13 +++++++++++++ 2 files changed, 18 insertions(+) diff --git a/net/address/address.go b/net/address/address.go index d5fb8e00..89ddf415 100644 --- a/net/address/address.go +++ b/net/address/address.go @@ -160,6 +160,11 @@ func (cidr CIDR) String() string { return fmt.Sprintf("%s/%d", cidr.Addr.String(), cidr.PrefixLen) } +func (cidr CIDR) IPNet() (r *net.IPNet) { + mask := net.CIDRMask(cidr.PrefixLen, 8*net.IPv4len) + return &net.IPNet{IP: cidr.Addr.IP4(), Mask: mask} +} + // FromIP4 converts an ipv4 address to our integer address type func FromIP4(ip4 net.IP) (r Address) { for _, b := range ip4.To4() { diff --git a/net/address/address_test.go b/net/address/address_test.go index 8718fdb6..6b7b3033 100644 --- a/net/address/address_test.go +++ b/net/address/address_test.go @@ -1,6 +1,7 @@ package address import ( + "net" "testing" "testing/quick" @@ -67,3 +68,15 @@ func TestCIDRStartAndEnd(t *testing.T) { require.Equal(t, ip("10.0.0.0"), cidr.Start(), "") require.Equal(t, ip("10.0.1.0"), cidr.End(), "") } + +func TestConversions(t *testing.T) { + addrStr := "10.2.3.0" + cidrStr := addrStr + "/24" + addr := ip(addrStr) + require.Equal(t, addr.String(), addrStr) + require.Equal(t, addr.IP4(), net.ParseIP(addrStr).To4()) + cidr, _ := ParseCIDR(cidrStr) + require.Equal(t, cidr.String(), cidrStr) + _, ipnet, _ := net.ParseCIDR(cidrStr) + require.Equal(t, cidr.IPNet(), ipnet) +} From ae625a18c8c3647b16e6f7964770bbdf9c16a95d Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 17 Apr 2017 10:31:27 +0000 Subject: [PATCH 25/36] Move awsvpc bridge expose from weave script to Go code --- net/bridge.go | 33 +++++++++++++++++++++++++++++++++ prog/weaver/main.go | 16 ++++++++++++++++ weave | 14 -------------- 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/net/bridge.go b/net/bridge.go index a5949b63..19931c82 100644 --- a/net/bridge.go +++ b/net/bridge.go @@ -8,6 +8,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/coreos/go-iptables/iptables" + "github.com/j-keck/arping" "github.com/pkg/errors" "github.com/vishvananda/netlink" @@ -442,3 +443,35 @@ func linkSetUpByName(linkName string) error { } return netlink.LinkSetUp(link) } + +func AddBridgeAddr(bridgeName string, addr *net.IPNet, removeDefaultRoute bool) error { + link, err := netlink.LinkByName(bridgeName) + if err != nil { + return errors.Wrapf(err, "AddBridgeAddr finding bridge %q", bridgeName) + } + newAddresses, err := AddAddresses(link, []*net.IPNet{addr}) + if err != nil { + return errors.Wrapf(err, "adding address %v to %q", addr, bridgeName) + } + for _, ipnet := range newAddresses { + arping.GratuitousArpOverIfaceByName(ipnet.IP, bridgeName) + } + if removeDefaultRoute { + routeFilter := &netlink.Route{ + LinkIndex: link.Attrs().Index, + Dst: &net.IPNet{IP: addr.IP.Mask(addr.Mask), Mask: addr.Mask}, + Protocol: 2, // RTPROT_KERNEL + } + filterMask := netlink.RT_FILTER_OIF | netlink.RT_FILTER_DST | netlink.RT_FILTER_PROTOCOL + routes, err := netlink.RouteListFiltered(netlink.FAMILY_V4, routeFilter, filterMask) + if err != nil { + return errors.Wrapf(err, "failed to get route list for bridge %q", bridgeName) + } + for _, r := range routes { + if err = netlink.RouteDel(&r); err != nil { + return errors.Wrapf(err, "failed to delete default route %+v", r) + } + } + } + return nil +} diff --git a/prog/weaver/main.go b/prog/weaver/main.go index ec4d04c3..5c2d8cb5 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -430,9 +430,25 @@ func main() { go plugin.Start(httpAddr, dockerCli, pluginSocket, pluginMeshSocket, !noDNS, enablePluginV2) } + if bridgeConfig.IsAWSVPC { + // Run this on its own goroutine because the allocator can block + // We remove the default route installed by the kernel, + // because awsvpc has installed it as well + go expose(allocator, defaultSubnet, bridgeConfig.WeaveBridgeName, bridgeConfig.IsAWSVPC) + } + signals.SignalHandlerLoop(common.Log, router) } +func expose(alloc *ipam.Allocator, subnet address.CIDR, bridgeName string, removeDefaultRoute bool) { + addr, err := alloc.Allocate("weave:expose", subnet, false, func() bool { return false }) + checkFatal(err) + cidr := address.MakeCIDR(subnet, addr) + err = weavenet.AddBridgeAddr(bridgeName, cidr.IPNet(), removeDefaultRoute) + checkFatal(err) + Log.Printf("Bridge %q exposed on address %v", bridgeName, cidr) +} + func options() map[string]string { options := make(map[string]string) mflag.Visit(func(f *mflag.Flag) { diff --git a/weave b/weave index 402b4db6..f483188d 100755 --- a/weave +++ b/weave @@ -474,12 +474,6 @@ expose_ip() { if ! ip addr show dev $BRIDGE | grep -qF $CIDR ; then ip addr add dev $BRIDGE $CIDR arp_update $BRIDGE $CIDR || true - # Remove a default route installed by the kernel, because awsvpc - # has installed it as well - if [ -n "$AWSVPC" ]; then - RCIDR=$(ip route list exact $CIDR proto kernel | head -n1 | cut -d' ' -f1) - [ -n "$RCIDR" ] && ip route del dev $BRIDGE proto kernel $RCIDR - fi fi [ -z "$FQDN" ] || when_weave_running put_dns_fqdn_no_check_alive weave:expose $FQDN $CIDR done @@ -1416,7 +1410,6 @@ launch_router() { --resolv-conf "/var/run/weave/etc/$RESOLV_CONF_BASE" \ "$@") wait_for_status $CONTAINER_NAME http_call $HTTP_ADDR - setup_awsvpc if [ -n "$ENABLE_PLUGIN" ]; then wait_for_status $CONTAINER_NAME http_call_unix $CONTAINER_NAME plugin-status.sock @@ -1424,12 +1417,6 @@ launch_router() { fi } -setup_awsvpc() { - if [ -n "$AWSVPC" ]; then - expose_ip - fi -} - # Recreate the parameter values that are set when the router is first launched fetch_router_args() { CONTAINER_ARGS=$(docker inspect -f '{{.Args}}' $CONTAINER_NAME) || return 1 @@ -1660,7 +1647,6 @@ EOF ! docker inspect -f '{{.Config.Cmd}}' $CONTAINER_NAME | grep -q -- "--awsvpc" || AWSVPC=1 fetch_router_args wait_for_status $CONTAINER_NAME http_call $HTTP_ADDR - setup_awsvpc ;; launch-proxy) deprecation_warnings "$@" From f479e578705a760e6a78a1db40f9dbb2ae4fd9dd Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 17 Apr 2017 10:32:06 +0000 Subject: [PATCH 26/36] Remove attach-router since it no longer does anything --- proxy/proxy.go | 9 --------- weave | 16 +--------------- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/proxy/proxy.go b/proxy/proxy.go index e753ae26..40e00968 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -408,11 +408,6 @@ func containerShouldAttach(container *docker.Container) bool { return len(container.Config.Entrypoint) > 0 && container.Config.Entrypoint[0] == weaveWaitEntrypoint[0] } -func containerIsWeaveRouter(container *docker.Container) bool { - return container.Name == weaveContainerName && - len(container.Config.Entrypoint) > 0 && container.Config.Entrypoint[0] == weaveEntrypoint -} - func (proxy *Proxy) createWait(r *http.Request, ident string) { proxy.Lock() proxy.waiters[r] = &wait{ident: ident, ch: make(chan error, 1)} @@ -489,10 +484,6 @@ func (proxy *Proxy) attach(containerID string) error { } return nil } - if containerIsWeaveRouter(container) { - Log.Infof("Attaching weave router container: %s", container.ID) - return callWeaveAttach(container, []string{"attach-router"}) - } if !containerShouldAttach(container) || !container.State.Running { return nil } diff --git a/weave b/weave index f483188d..87086842 100755 --- a/weave +++ b/weave @@ -136,7 +136,7 @@ exec_options() { ;; # All the other commands that may create the bridge and need machine id files. # We don't mount '/' to avoid recursive mounts of '/var' - create-bridge|attach-router|attach|expose|hide) + create-bridge|attach|expose|hide) echo -v /etc:/host/etc -v /var/lib/dbus:/host/var/lib/dbus -e HOST_ROOT=/host ;; esac @@ -1417,12 +1417,6 @@ launch_router() { fi } -# Recreate the parameter values that are set when the router is first launched -fetch_router_args() { - CONTAINER_ARGS=$(docker inspect -f '{{.Args}}' $CONTAINER_NAME) || return 1 - NO_DNS_OPT=$(echo $CONTAINER_ARGS | grep -o -e '--no-dns') || true -} - stop_router() { ! is_plugin_v1_enabled || util_op remove-plugin-network weave || true stop $CONTAINER_NAME "Weave" @@ -1640,14 +1634,6 @@ EOF launch_router "$@" echo $ROUTER_CONTAINER ;; - attach-router) - check_running $CONTAINER_NAME - enforce_docker_bridge_addr_assign_type - # We cannot use detect_awsvpc here, because HTTP server might not be started - ! docker inspect -f '{{.Config.Cmd}}' $CONTAINER_NAME | grep -q -- "--awsvpc" || AWSVPC=1 - fetch_router_args - wait_for_status $CONTAINER_NAME http_call $HTTP_ADDR - ;; launch-proxy) deprecation_warnings "$@" check_not_running proxy $PROXY_CONTAINER_NAME $BASE_EXEC_IMAGE From c929b2d5f9dd1362e75474ea60005c895a14fd72 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 17 Apr 2017 19:21:44 +0000 Subject: [PATCH 27/36] ip netns and nsenter no longer called by weave script --- weave | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/weave b/weave index 87086842..93d018b0 100755 --- a/weave +++ b/weave @@ -1559,17 +1559,6 @@ if ! command_exists ip ; then exit 1 fi -if ! ip netns list >/dev/null 2>&1 ; then - echo "ERROR: $(ip -V) does not support network namespaces." >&2 - echo " Please install iproute2-ss111010 or later." >&2 - exit 1 -fi - -if ! command_exists nsenter ; then - echo "ERROR: nsenter utility missing. Please install it." >&2 - exit 1 -fi - [ $# -gt 0 ] || usage COMMAND=$1 shift 1 From 3574da267b049961ea7d5b45d7623db713bd76c6 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 17 Apr 2017 19:33:17 +0000 Subject: [PATCH 28/36] Remove -enforce_docker_bridge_addr_assign_type no longer used --- prog/weaveutil/bridge.go | 7 ------- prog/weaveutil/main.go | 1 - weave | 4 ---- 3 files changed, 12 deletions(-) diff --git a/prog/weaveutil/bridge.go b/prog/weaveutil/bridge.go index 9c38c815..625e279f 100644 --- a/prog/weaveutil/bridge.go +++ b/prog/weaveutil/bridge.go @@ -45,10 +45,3 @@ func createBridge(args []string) error { fmt.Println(bridgeType.String()) return err } - -func enforceAddrAssign(args []string) error { - if len(args) != 1 { - cmdUsage("enforce-bridge-addr-assign-type", "") - } - return weavenet.EnforceAddrAssignType(args[0]) -} diff --git a/prog/weaveutil/main.go b/prog/weaveutil/main.go index b0f7a33e..46ffa719 100644 --- a/prog/weaveutil/main.go +++ b/prog/weaveutil/main.go @@ -19,7 +19,6 @@ func init() { "docker-tls-args": dockerTLSArgs, "create-bridge": createBridge, "detect-bridge-type": detectBridgeType, - "enforce-addr-assign": enforceAddrAssign, "create-datapath": createDatapath, "delete-datapath": deleteDatapath, "add-datapath-interface": addDatapathInterface, diff --git a/weave b/weave index 93d018b0..f5e4e6fb 100755 --- a/weave +++ b/weave @@ -388,10 +388,6 @@ EOF fi } -enforce_docker_bridge_addr_assign_type() { - util_op enforce-addr-assign "$DOCKER_BRIDGE" || true -} - # Detect the current bridge/datapath state. When invoked, the values of # $BRIDGE and $DATAPATH are expected to be distinct. $BRIDGE_TYPE and # $DATAPATH are set correctly on success; failure indicates that the From 8f02c81f167527933cad9a0d4c219e1a6b79889e Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 18 Apr 2017 10:45:05 +0000 Subject: [PATCH 29/36] Update docs for change where proxy no longer calls attach-router --- site/troubleshooting.md | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/site/troubleshooting.md b/site/troubleshooting.md index b2630525..9d593eb6 100644 --- a/site/troubleshooting.md +++ b/site/troubleshooting.md @@ -283,14 +283,11 @@ on reboot. This can be disabled via: weave launch --no-restart -Note that the -[Weave Net Docker API Proxy](/site/weave-docker-api.md) -is responsible for reconfiguring the Weave router and re-attaching -application containers to the Weave network at startup, so if you -choose not to run it you must make arrangements for this -reconfiguration to take place. In this scenario, set up your favourite -process manager to run `weave launch-router` every time the machine -reboots. +Note that the Weave Net router will create the `weave` network bridge +if necessary when it restarts, and the [Weave Net Docker API +Proxy](/site/weave-docker-api.md) will re-attach any application +containers that it originally attached to the Weave network when they +restart. ## Snapshot Releases From fddb3ee06eca04068f9ac520a30fde6cf7e2c063 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 19 Apr 2017 10:09:48 +0000 Subject: [PATCH 30/36] Review feedback --- net/arp.go | 10 +++--- net/bridge.go | 71 ++++++++++++++++++++++++++++++---------- prog/weaver/main.go | 22 +++++++------ prog/weaveutil/bridge.go | 7 ++-- 4 files changed, 76 insertions(+), 34 deletions(-) diff --git a/net/arp.go b/net/arp.go index 39322b9a..4a4e0d78 100644 --- a/net/arp.go +++ b/net/arp.go @@ -1,9 +1,11 @@ package net -import "path/filepath" -import "fmt" -import "io" -import "os" +import ( + "fmt" + "io" + "os" + "path/filepath" +) // Configure the ARP cache parameters for the given interface. This // makes containers react more quickly to a change in the MAC address diff --git a/net/bridge.go b/net/bridge.go index 19931c82..9e6b06e9 100644 --- a/net/bridge.go +++ b/net/bridge.go @@ -17,11 +17,44 @@ import ( type BridgeType int +/* This code implements three possible configurations to connect + containers to the Weave Net overlay: + +1. Bridge + +-------+ +(container-veth)-+ weave +-(vethwe-bridge)--(vethwe-pcap) + +-------+ + +"weave" is a Linux bridge. "vethwe-pcap" (end of veth pair) is used +to capture and inject packets, by router/pcap.go. + +2. BridgedFastdp + + +-------+ /----------\ +(container-veth)-+ weave +-(vethwe-bridge)--(vethwe-datapath)-+ datapath + + +-------+ \----------/ + +"weave" is a Linux bridge and "datapath" is an Open vSwitch datapath; +they are connected via a veth pair. Packet capture and injection use +the "datapath" device, via "router/fastdp.go:fastDatapathBridge" + +3. Fastdp + + /-------\ +(container-veth)-+ weave + + \-------/ + +"weave" is an Open vSwitch datapath, and capture/injection are as in +BridgedFastdp. Not used by default due to missing conntrack support in +datapath of old kernel versions (https://github.com/weaveworks/weave/issues/1577). +*/ + const ( WeaveBridgeName = "weave" DatapathName = "datapath" + DatapathIfName = "vethwe-datapath" BridgeIfName = "vethwe-bridge" - PcapIfaceName = "vethwe-pcap" + PcapIfName = "vethwe-pcap" None BridgeType = iota Bridge @@ -70,13 +103,11 @@ func DetectBridgeType(weaveBridgeName, datapathName string) BridgeType { } } -var ErrSetBridgeMac = errors.New("Setting $DOCKER_BRIDGE MAC (mitigate https://github.com/docker/docker/issues/14908)") - -func EnforceAddrAssignType(bridgeName string) error { +func EnforceAddrAssignType(bridgeName string) (setAddr bool, err error) { sysctlFilename := filepath.Join("/sys/class/net/", bridgeName, "/addr_assign_type") addrAssignType, err := ioutil.ReadFile(sysctlFilename) if err != nil { - return errors.Wrapf(err, "reading %q", sysctlFilename) + return false, errors.Wrapf(err, "reading %q", sysctlFilename) } // From include/uapi/linux/netdevice.h @@ -88,21 +119,21 @@ func EnforceAddrAssignType(bridgeName string) error { if addrAssignType[0] != '3' { link, err := netlink.LinkByName(bridgeName) if err != nil { - return errors.Wrapf(err, "EnforceAddrAssignType finding bridge %s", bridgeName) + return false, errors.Wrapf(err, "EnforceAddrAssignType finding bridge %s", bridgeName) } mac, err := RandomMAC() if err != nil { - return errors.Wrap(err, "creating random MAC") + return false, errors.Wrap(err, "creating random MAC") } if err := netlink.LinkSetHardwareAddr(link, mac); err != nil { - return errors.Wrapf(err, "setting bridge %s address to %v", bridgeName, mac) + return false, errors.Wrapf(err, "setting bridge %s address to %v", bridgeName, mac) } - return ErrSetBridgeMac + return true, nil } - return nil + return false, nil } func isBridge(link netlink.Link) bool { @@ -181,8 +212,8 @@ type BridgeConfig struct { DatapathName string NoFastdp bool NoBridgedFastdp bool - IsAWSVPC bool - ExpectNPC bool + AWSVPC bool + NPC bool MTU int Mac string Port int @@ -228,7 +259,7 @@ func CreateBridge(procPath string, config *BridgeConfig) (BridgeType, error) { } } - if config.IsAWSVPC { + if config.AWSVPC { bridgeType = AWSVPC // Set proxy_arp on the bridge, so that it could accept packets destined // to containers within the same subnet but running on remote hosts. @@ -306,7 +337,7 @@ func initBridge(config *BridgeConfig) error { if err := initBridgePrep(config); err != nil { return err } - if _, err := CreateAndAttachVeth(BridgeIfName, PcapIfaceName, config.WeaveBridgeName, config.MTU, true, func(veth netlink.Link) error { + if _, err := CreateAndAttachVeth(BridgeIfName, PcapIfName, config.WeaveBridgeName, config.MTU, true, func(veth netlink.Link) error { return netlink.LinkSetUp(veth) }); err != nil { return errors.Wrap(err, "creating pcap veth pair") @@ -342,7 +373,7 @@ func initBridgedFastdp(config *BridgeConfig) error { if err := initBridgePrep(config); err != nil { return err } - if _, err := CreateAndAttachVeth("vethwe-bridge", "vethwe-datapath", config.WeaveBridgeName, config.MTU, true, func(veth netlink.Link) error { + if _, err := CreateAndAttachVeth(BridgeIfName, DatapathIfName, config.WeaveBridgeName, config.MTU, true, func(veth netlink.Link) error { if err := netlink.LinkSetUp(veth); err != nil { return errors.Wrapf(err, "setting link up on %q", veth.Attrs().Name) } @@ -399,9 +430,11 @@ func configureIPTables(config *BridgeConfig) error { } } - if config.ExpectNPC { + if config.NPC { // Steer traffic via the NPC - ipt.NewChain("filter", "WEAVE-NPC") // ignoring errors such as 'chain already exists' + if err = ipt.ClearChain("nat", "WEAVE-NPC"); err != nil { + return errors.Wrap(err, "clearing WEAVE-NPC chain") + } if err = ipt.AppendUnique("filter", "FORWARD", "-o", config.WeaveBridgeName, "-j", "WEAVE-NPC"); err != nil { return err } @@ -428,7 +461,9 @@ func configureIPTables(config *BridgeConfig) error { } // create a chain for masquerading - ipt.NewChain("nat", "WEAVE") // ignoring errors such as 'chain already exists' + if err = ipt.ClearChain("nat", "WEAVE"); err != nil { + return errors.Wrap(err, "clearing WEAVE chain") + } if err = ipt.AppendUnique("nat", "POSTROUTING", "-j", "WEAVE"); err != nil { return err } diff --git a/prog/weaver/main.go b/prog/weaver/main.go index 5c2d8cb5..08d2a90d 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -170,7 +170,7 @@ func main() { mflag.BoolVar(&resume, []string{"-resume"}, false, "resume connections to previous peers") mflag.StringVar(&bridgeConfig.WeaveBridgeName, []string{"-weave-bridge"}, "weave", "name of weave bridge") mflag.StringVar(&bridgeConfig.DockerBridgeName, []string{"-docker-bridge"}, "", "name of Docker bridge") - mflag.BoolVar(&bridgeConfig.ExpectNPC, []string{"-expect-npc"}, false, "set up iptables rules for npc") + mflag.BoolVar(&bridgeConfig.NPC, []string{"-expect-npc"}, false, "set up iptables rules for npc") mflag.StringVar(&routerName, []string{"#name", "-name"}, "", "name of router (defaults to MAC of interface)") mflag.StringVar(&nickName, []string{"#nickname", "-nickname"}, "", "nickname of peer (defaults to hostname)") mflag.StringVar(&password, []string{"#password", "-password"}, "", "network password") @@ -200,7 +200,7 @@ func main() { mflag.StringVar(&trustedSubnetStr, []string{"-trusted-subnets"}, "", "comma-separated list of trusted subnets in CIDR notation") mflag.StringVar(&dbPrefix, []string{"-db-prefix"}, "/weavedb/weave", "pathname/prefix of filename to store data") mflag.StringVar(&procPath, []string{"-proc-path"}, "/proc", "path to reach host /proc filesystem") - mflag.BoolVar(&bridgeConfig.IsAWSVPC, []string{"-awsvpc"}, false, "use AWS VPC for routing") + mflag.BoolVar(&bridgeConfig.AWSVPC, []string{"-awsvpc"}, false, "use AWS VPC for routing") mflag.StringVar(&hostRoot, []string{"-host-root"}, "", "path to reach host filesystem") mflag.StringVar(&discoveryEndpoint, []string{"-peer-discovery-url"}, "https://cloud.weave.works/api/net", "url for peer discovery") mflag.StringVar(&token, []string{"-token"}, "", "token for peer discovery") @@ -251,8 +251,10 @@ func main() { } if bridgeConfig.DockerBridgeName != "" { - if err := weavenet.EnforceAddrAssignType(bridgeConfig.DockerBridgeName); err != nil { + if setAddr, err := weavenet.EnforceAddrAssignType(bridgeConfig.DockerBridgeName); err != nil { Log.Errorf("While checking address assignment type of %s: %s", bridgeConfig.DockerBridgeName, err) + } else if setAddr { + Log.Warningf("Setting %s MAC (mitigate https://github.com/docker/docker/issues/14908)", bridgeConfig.DockerBridgeName) } } @@ -269,7 +271,7 @@ func main() { networkConfig.Bridge = bridge if bridge != nil { - if err := weavenet.DetectHairpin("vethwe-bridge", Log); err != nil { + if err := weavenet.DetectHairpin(weavenet.BridgeIfName, Log); err != nil { Log.Errorf("DetectHairpin failed: %s", err) } } @@ -283,7 +285,7 @@ func main() { config.TrustedSubnets = parseTrustedSubnets(trustedSubnetStr) config.PeerDiscovery = !noDiscovery - if bridgeConfig.IsAWSVPC && len(config.Password) > 0 { + if bridgeConfig.AWSVPC && len(config.Password) > 0 { Log.Fatalf("--awsvpc mode is not compatible with the --password option") } @@ -329,7 +331,7 @@ func main() { } network := "" - if bridgeConfig.IsAWSVPC { + if bridgeConfig.AWSVPC { network = "awsvpc" } checkForUpdates(dockerVersion, network) @@ -352,7 +354,7 @@ func main() { ) if ipamConfig.Enabled() { var t tracker.LocalRangeTracker - if bridgeConfig.IsAWSVPC { + if bridgeConfig.AWSVPC { Log.Infoln("Creating AWSVPC LocalRangeTracker") t, err = tracker.NewAWSVPCTracker(bridgeConfig.WeaveBridgeName) if err != nil { @@ -430,11 +432,11 @@ func main() { go plugin.Start(httpAddr, dockerCli, pluginSocket, pluginMeshSocket, !noDNS, enablePluginV2) } - if bridgeConfig.IsAWSVPC { + if bridgeConfig.AWSVPC { // Run this on its own goroutine because the allocator can block // We remove the default route installed by the kernel, // because awsvpc has installed it as well - go expose(allocator, defaultSubnet, bridgeConfig.WeaveBridgeName, bridgeConfig.IsAWSVPC) + go expose(allocator, defaultSubnet, bridgeConfig.WeaveBridgeName, bridgeConfig.AWSVPC) } signals.SignalHandlerLoop(common.Log, router) @@ -509,7 +511,7 @@ func createOverlay(bridgeType weavenet.BridgeType, config weavenet.BridgeConfig, bridge = fastdp.Bridge() overlay.Add("fastdp", fastdp.Overlay()) case weavenet.Bridge: - iface, err := weavenet.EnsureInterface(weavenet.PcapIfaceName) + iface, err := weavenet.EnsureInterface(weavenet.PcapIfName) checkFatal(err) bridge, err = weave.NewPcap(iface, bufSzMB*1024*1024) // bufsz flag is in MB checkFatal(err) diff --git a/prog/weaveutil/bridge.go b/prog/weaveutil/bridge.go index 625e279f..de300fdb 100644 --- a/prog/weaveutil/bridge.go +++ b/prog/weaveutil/bridge.go @@ -38,10 +38,13 @@ func createBridge(args []string) error { Mac: args[5], NoFastdp: args[6] != "", NoBridgedFastdp: args[7] != "", - ExpectNPC: args[9] == "--expect-npc", + NPC: args[9] == "--expect-npc", } procPath := args[8] bridgeType, err := weavenet.CreateBridge(procPath, &config) + if err != nil { + return err + } fmt.Println(bridgeType.String()) - return err + return nil } From a7feb41e62bc7a96f850be2fa06f10fe51767905 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 19 Apr 2017 12:00:45 +0000 Subject: [PATCH 31/36] Replace switch statements with Bridge polymorphism --- net/bridge.go | 145 +++++++++++++++++++-------------------- net/veth.go | 27 +++----- prog/weaver/main.go | 14 ++-- prog/weaveutil/bridge.go | 10 ++- weave | 4 -- 5 files changed, 94 insertions(+), 106 deletions(-) diff --git a/net/bridge.go b/net/bridge.go index 9e6b06e9..3f8cdfa3 100644 --- a/net/bridge.go +++ b/net/bridge.go @@ -15,8 +15,6 @@ import ( "github.com/weaveworks/weave/common/odp" ) -type BridgeType int - /* This code implements three possible configurations to connect containers to the Weave Net overlay: @@ -55,51 +53,47 @@ const ( DatapathIfName = "vethwe-datapath" BridgeIfName = "vethwe-bridge" PcapIfName = "vethwe-pcap" - - None BridgeType = iota - Bridge - Fastdp - BridgedFastdp - AWSVPC - Inconsistent ) -// Returns a string that is consistent with the weave script -func (t BridgeType) String() string { - switch t { - case None: - return "none" - case Bridge: - return "bridge" - case Fastdp: - return "fastdp" - case BridgedFastdp: - return "bridged_fastdp" - case AWSVPC: - return "AWSVPC" - case Inconsistent: - return "inconsistent" - } - return "unknown" +type Bridge interface { + init(config *BridgeConfig) error // create and initialise bridge device(s) + attach(veth *netlink.Veth) error // attach veth to bridge + IsFastdp() bool // does this bridge use fastdp? + String() string // human-readable type string } -func DetectBridgeType(weaveBridgeName, datapathName string) BridgeType { +type bridgeImpl struct{ bridge netlink.Link } +type fastdpImpl struct{ datapathName string } +type bridgedFastdpImpl struct { + bridgeImpl + fastdpImpl +} + +// Returns a string that is consistent with the weave script +func (bridgeImpl) String() string { return "bridge" } +func (fastdpImpl) String() string { return "fastdp" } +func (bridgedFastdpImpl) String() string { return "bridged_fastdp" } + +// Used to decide whether to manage ODP tunnels +func (bridgeImpl) IsFastdp() bool { return false } +func (fastdpImpl) IsFastdp() bool { return true } +func (bridgedFastdpImpl) IsFastdp() bool { return true } + +func DetectBridgeType(weaveBridgeName, datapathName string) (Bridge, error) { bridge, _ := netlink.LinkByName(weaveBridgeName) datapath, _ := netlink.LinkByName(datapathName) switch { case bridge == nil && datapath == nil: - return None + return nil, nil case isBridge(bridge) && datapath == nil: - return Bridge + return bridgeImpl{bridge: bridge}, nil case isDatapath(bridge) && datapath == nil: - return Fastdp + return fastdpImpl{datapathName: datapathName}, nil case isDatapath(datapath) && isBridge(bridge): - return BridgedFastdp - // We cannot detect AWSVPC here; it looks just like Bridge. - // Anyone who cares will have to know some other way. + return bridgedFastdpImpl{bridgeImpl{bridge: bridge}, fastdpImpl{datapathName: datapathName}}, nil default: - return Inconsistent + return nil, errors.New("Inconsistent bridge state detected. Please do 'weave reset' and try again") } } @@ -219,48 +213,37 @@ type BridgeConfig struct { Port int } -func CreateBridge(procPath string, config *BridgeConfig) (BridgeType, error) { - bridgeType := DetectBridgeType(config.WeaveBridgeName, config.DatapathName) +func CreateBridge(procPath string, config *BridgeConfig) (Bridge, error) { + bridgeType, err := DetectBridgeType(config.WeaveBridgeName, config.DatapathName) + if err != nil { + return nil, err + } - if bridgeType == None { - bridgeType = Bridge + if bridgeType == nil { + bridgeType = bridgeImpl{} if !config.NoFastdp { - bridgeType = BridgedFastdp + bridgeType = bridgedFastdpImpl{} if config.NoBridgedFastdp { - bridgeType = Fastdp + bridgeType = fastdpImpl{} config.DatapathName = config.WeaveBridgeName } odpSupported, err := odp.CreateDatapath(config.DatapathName) if err != nil { - return None, errors.Wrapf(err, "creating datapath %q", config.DatapathName) + return nil, errors.Wrapf(err, "creating datapath %q", config.DatapathName) } if !odpSupported { - bridgeType = Bridge + bridgeType = bridgeImpl{} } } - - var err error - switch bridgeType { - case Bridge: - err = initBridge(config) - case Fastdp: - err = initFastdp(config) - case BridgedFastdp: - err = initBridgedFastdp(config) - default: - err = fmt.Errorf("Cannot initialise bridge type %v", bridgeType) + if err := bridgeType.init(config); err != nil { + return nil, err } - if err != nil { - return None, err - } - - if err = configureIPTables(config); err != nil { + if err := configureIPTables(config); err != nil { return bridgeType, errors.Wrap(err, "configuring iptables") } } if config.AWSVPC { - bridgeType = AWSVPC // Set proxy_arp on the bridge, so that it could accept packets destined // to containers within the same subnet but running on remote hosts. // Without it, exact routes on each container are required. @@ -275,12 +258,6 @@ func CreateBridge(procPath string, config *BridgeConfig) (BridgeType, error) { } } - if bridgeType == Bridge { - if err := EthtoolTXOff(config.WeaveBridgeName); err != nil { - return bridgeType, errors.Wrap(err, "setting tx off") - } - } - if err := linkSetUpByName(config.WeaveBridgeName); err != nil { return bridgeType, err } @@ -292,7 +269,7 @@ func CreateBridge(procPath string, config *BridgeConfig) (BridgeType, error) { return bridgeType, nil } -func initBridgePrep(config *BridgeConfig) error { +func (b bridgeImpl) initPrep(config *BridgeConfig) error { mac, err := net.ParseMAC(config.Mac) if err != nil { return errors.Wrapf(err, "parsing bridge MAC %q", config.Mac) @@ -304,11 +281,11 @@ func initBridgePrep(config *BridgeConfig) error { if config.MTU == 0 { config.MTU = 65535 } - link := &netlink.Bridge{LinkAttrs: linkAttrs} - if err = netlink.LinkAdd(link); err != nil { + b.bridge = &netlink.Bridge{LinkAttrs: linkAttrs} + if err = netlink.LinkAdd(b.bridge); err != nil { return errors.Wrapf(err, "creating bridge %q", config.WeaveBridgeName) } - if err := netlink.LinkSetHardwareAddr(link, mac); err != nil { + if err := netlink.LinkSetHardwareAddr(b.bridge, mac); err != nil { return errors.Wrapf(err, "setting bridge %q mac %v", config.WeaveBridgeName, mac) } // Attempting to set the bridge MTU to a high value directly @@ -323,7 +300,7 @@ func initBridgePrep(config *BridgeConfig) error { if err := netlink.LinkSetMTU(dummy, config.MTU); err != nil { return errors.Wrapf(err, "setting dummy interface mtu to %d", config.MTU) } - if err := netlink.LinkSetMaster(dummy, link); err != nil { + if err := netlink.LinkSetMasterByIndex(dummy, b.bridge.Attrs().Index); err != nil { return errors.Wrap(err, "setting dummy interface master") } if err := netlink.LinkDel(dummy); err != nil { @@ -333,8 +310,8 @@ func initBridgePrep(config *BridgeConfig) error { return nil } -func initBridge(config *BridgeConfig) error { - if err := initBridgePrep(config); err != nil { +func (b bridgeImpl) init(config *BridgeConfig) error { + if err := b.initPrep(config); err != nil { return err } if _, err := CreateAndAttachVeth(BridgeIfName, PcapIfName, config.WeaveBridgeName, config.MTU, true, func(veth netlink.Link) error { @@ -342,15 +319,19 @@ func initBridge(config *BridgeConfig) error { }); err != nil { return errors.Wrap(err, "creating pcap veth pair") } + if err := EthtoolTXOff(config.WeaveBridgeName); err != nil { + return errors.Wrap(err, "setting tx off") + } return nil } -func initFastdp(config *BridgeConfig) error { +func (f fastdpImpl) init(config *BridgeConfig) error { datapath, err := netlink.LinkByName(config.DatapathName) if err != nil { return errors.Wrapf(err, "finding datapath %q", config.DatapathName) } + f.datapathName = config.DatapathName if config.MTU == 0 { /* GCE has the lowest underlay network MTU we're likely to encounter on a local network, at 1460 bytes. To get the overlay MTU from that we @@ -366,11 +347,11 @@ func initFastdp(config *BridgeConfig) error { return nil } -func initBridgedFastdp(config *BridgeConfig) error { - if err := initFastdp(config); err != nil { +func (bf bridgedFastdpImpl) init(config *BridgeConfig) error { + if err := bf.fastdpImpl.init(config); err != nil { return err } - if err := initBridgePrep(config); err != nil { + if err := bf.bridgeImpl.initPrep(config); err != nil { return err } if _, err := CreateAndAttachVeth(BridgeIfName, DatapathIfName, config.WeaveBridgeName, config.MTU, true, func(veth netlink.Link) error { @@ -392,6 +373,18 @@ func initBridgedFastdp(config *BridgeConfig) error { return nil } +func (b bridgeImpl) attach(veth *netlink.Veth) error { + return netlink.LinkSetMasterByIndex(veth, b.bridge.Attrs().Index) +} + +func (bf bridgedFastdpImpl) attach(veth *netlink.Veth) error { + return bf.bridgeImpl.attach(veth) +} + +func (f fastdpImpl) attach(veth *netlink.Veth) error { + return odp.AddDatapathInterface(f.datapathName, veth.Attrs().Name) +} + func configureIPTables(config *BridgeConfig) error { ipt, err := iptables.New() if err != nil { diff --git a/net/veth.go b/net/veth.go index a95553e8..8637ac9f 100644 --- a/net/veth.go +++ b/net/veth.go @@ -11,8 +11,6 @@ import ( "github.com/j-keck/arping" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" - - "github.com/weaveworks/weave/common/odp" ) // create and attach a veth to the Weave bridge @@ -40,22 +38,17 @@ func CreateAndAttachVeth(name, peerName, bridgeName string, mtu int, keepTXOn bo return nil, fmt.Errorf(format, a...) } - switch bridgeType := DetectBridgeType(bridgeName, DatapathName); bridgeType { - case Bridge, BridgedFastdp: - if err := netlink.LinkSetMasterByIndex(veth, bridge.Attrs().Index); err != nil { - return cleanup(`unable to set master of %s: %s`, name, err) + bridgeType, err := DetectBridgeType(bridgeName, DatapathName) + if err != nil { + return cleanup("detect bridge type: %s", err) + } + if err := bridgeType.attach(veth); err != nil { + return cleanup("attaching veth %q to %q: %s", name, bridgeName, err) + } + if !bridgeType.IsFastdp() && !keepTXOn { + if err := EthtoolTXOff(veth.PeerName); err != nil { + return cleanup(`unable to set tx off on %q: %s`, peerName, err) } - if bridgeType == Bridge && !keepTXOn { - if err := EthtoolTXOff(peerName); err != nil { - return cleanup(`unable to set tx off on %q: %s`, peerName, err) - } - } - case Fastdp: - if err := odp.AddDatapathInterface(bridgeName, name); err != nil { - return cleanup(`failed to attach %s to device "%s": %s`, name, bridgeName, err) - } - default: - return cleanup(`invalid bridge configuration`) } if init != nil { diff --git a/prog/weaver/main.go b/prog/weaver/main.go index 08d2a90d..93ec293b 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -491,32 +491,32 @@ func (nopPacketLogging) LogPacket(string, weave.PacketKey) { func (nopPacketLogging) LogForwardPacket(string, weave.ForwardPacketKey) { } -func createOverlay(bridgeType weavenet.BridgeType, config weavenet.BridgeConfig, host string, port int, bufSzMB int, enableEncryption bool) (weave.NetworkOverlay, weave.Bridge) { +func createOverlay(bridgeType weavenet.Bridge, config weavenet.BridgeConfig, host string, port int, bufSzMB int, enableEncryption bool) (weave.NetworkOverlay, weave.Bridge) { overlay := weave.NewOverlaySwitch() var bridge weave.Bridge var ignoreSleeve bool - switch bridgeType { - case weavenet.AWSVPC: + switch { + case config.AWSVPC: vpc := weave.NewAWSVPC() overlay.Add("awsvpc", vpc) bridge = weave.NullBridge{} // Currently, we do not support any overlay with AWSVPC ignoreSleeve = true - case weavenet.Fastdp, weavenet.BridgedFastdp: + case bridgeType == nil: + bridge = weave.NullBridge{} + case bridgeType.IsFastdp(): iface, err := weavenet.EnsureInterface(config.DatapathName) checkFatal(err) fastdp, err := weave.NewFastDatapath(iface, port, enableEncryption) checkFatal(err) bridge = fastdp.Bridge() overlay.Add("fastdp", fastdp.Overlay()) - case weavenet.Bridge: + case !bridgeType.IsFastdp(): iface, err := weavenet.EnsureInterface(weavenet.PcapIfName) checkFatal(err) bridge, err = weave.NewPcap(iface, bufSzMB*1024*1024) // bufsz flag is in MB checkFatal(err) - default: - bridge = weave.NullBridge{} } if !ignoreSleeve { diff --git a/prog/weaveutil/bridge.go b/prog/weaveutil/bridge.go index de300fdb..cf87e3ab 100644 --- a/prog/weaveutil/bridge.go +++ b/prog/weaveutil/bridge.go @@ -11,8 +11,14 @@ func detectBridgeType(args []string) error { if len(args) != 2 { cmdUsage("detect-bridge-type", " ") } - bridgeType := weavenet.DetectBridgeType(args[0], args[1]) - fmt.Println(bridgeType.String()) + bridgeType, err := weavenet.DetectBridgeType(args[0], args[1]) + if err != nil { + return err + } else if bridgeType == nil { + fmt.Println("none") + } else { + fmt.Println(bridgeType.String()) + } return nil } diff --git a/weave b/weave index f5e4e6fb..a4b7547f 100755 --- a/weave +++ b/weave @@ -401,10 +401,6 @@ detect_bridge_type() { fastdp) DATAPATH="$BRIDGE" ;; - inconsistent) - echo "Inconsistent bridge state detected. Please do 'weave reset' and try again." >&2 - exit 1 - ;; *) return 1 ;; From 108f4784a655402aa85a95f70e0ef6a34f5147ee Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 19 Apr 2017 16:59:21 +0000 Subject: [PATCH 32/36] Add back check for datapath supported --- prog/weaveutil/datapath.go | 13 +++++++++++++ prog/weaveutil/main.go | 1 + weave | 3 ++- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/prog/weaveutil/datapath.go b/prog/weaveutil/datapath.go index f4494967..52e320f8 100644 --- a/prog/weaveutil/datapath.go +++ b/prog/weaveutil/datapath.go @@ -31,6 +31,19 @@ func deleteDatapath(args []string) error { return odp.DeleteDatapath(args[0]) } +// Checks whether a datapath can be created by actually creating and destroying it +func checkDatapath(args []string) error { + if len(args) != 1 { + cmdUsage("check-datapath", "") + } + + if err := createDatapath(args); err != nil { + return err + } + + return odp.DeleteDatapath(args[0]) +} + func addDatapathInterface(args []string) error { if len(args) != 2 { cmdUsage("add-datapath-interface", " ") diff --git a/prog/weaveutil/main.go b/prog/weaveutil/main.go index 46ffa719..b24a6fe5 100644 --- a/prog/weaveutil/main.go +++ b/prog/weaveutil/main.go @@ -21,6 +21,7 @@ func init() { "detect-bridge-type": detectBridgeType, "create-datapath": createDatapath, "delete-datapath": deleteDatapath, + "check-datapath": checkDatapath, "add-datapath-interface": addDatapathInterface, "create-plugin-network": createPluginNetwork, "remove-plugin-network": removePluginNetwork, diff --git a/weave b/weave index a4b7547f..cd3f90c8 100755 --- a/weave +++ b/weave @@ -442,7 +442,8 @@ validate_bridge_type() { detect_bridge_type || return 0 if [ -n "$LAUNCHING_ROUTER" ] ; then - if [ "$BRIDGE_TYPE" = bridge -a -z "$WEAVE_NO_FASTDP" ] ; then + if [ "$BRIDGE_TYPE" = bridge -a -z "$WEAVE_NO_FASTDP" ] && + util_op check-datapath 2>/dev/null ; then cat <&2 WEAVE_NO_FASTDP is not set, but there is already a bridge present of the wrong type for fast datapath. Please do 'weave reset' to remove From 9fd1a43769ff47c83630989e24016c0b02d5bc6f Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 24 Apr 2017 09:43:10 +0000 Subject: [PATCH 33/36] Put back some weave bridge options ability to set MTU was lost in an earlier change --- prog/weave-kube/launch.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/prog/weave-kube/launch.sh b/prog/weave-kube/launch.sh index 35557fb4..60899189 100755 --- a/prog/weave-kube/launch.sh +++ b/prog/weave-kube/launch.sh @@ -61,7 +61,11 @@ if [ -n "$HOSTNAME" ] ; then NICKNAME_ARG="--nickname=$HOSTNAME" fi -BRIDGE_OPTIONS="--datapath=datapath" +router_bridge_opts() { + echo --datapath=datapath + [ -z "$WEAVE_MTU" ] || echo --mtu "$WEAVE_MTU" + [ -z "$WEAVE_NO_FASTDP" ] || echo --no-fastdp +} if [ -z "$KUBE_PEERS" ]; then if ! KUBE_PEERS=$(/home/weave/kube-peers) || [ -z "$KUBE_PEERS" ]; then @@ -105,7 +109,7 @@ post_start_actions() { post_start_actions & -/home/weave/weaver $EXTRA_ARGS --port=6783 $BRIDGE_OPTIONS \ +/home/weave/weaver $EXTRA_ARGS --port=6783 $(router_bridge_opts) \ --host-root=$HOST_ROOT \ --http-addr=$HTTP_ADDR --status-addr=$STATUS_ADDR --docker-api='' --no-dns \ --ipalloc-range=$IPALLOC_RANGE $NICKNAME_ARG \ From 44beed4027ad3e1caad1ca8a08828a70f43087f1 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 24 Apr 2017 16:26:16 +0000 Subject: [PATCH 34/36] Refactor decision logic for initial bridge type --- net/bridge.go | 58 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/net/bridge.go b/net/bridge.go index 3f8cdfa3..fa1c1700 100644 --- a/net/bridge.go +++ b/net/bridge.go @@ -62,6 +62,8 @@ type Bridge interface { String() string // human-readable type string } +var errBridgeFallback = errors.New("special error value used to trigger a fallback to bridge type") + type bridgeImpl struct{ bridge netlink.Link } type fastdpImpl struct{ datapathName string } type bridgedFastdpImpl struct { @@ -213,6 +215,17 @@ type BridgeConfig struct { Port int } +func (config *BridgeConfig) configuredBridgeType() Bridge { + switch { + case config.NoFastdp: + return bridgeImpl{} + case config.NoBridgedFastdp: + return fastdpImpl{datapathName: config.WeaveBridgeName} + default: + return bridgedFastdpImpl{fastdpImpl: fastdpImpl{datapathName: config.DatapathName}} + } +} + func CreateBridge(procPath string, config *BridgeConfig) (Bridge, error) { bridgeType, err := DetectBridgeType(config.WeaveBridgeName, config.DatapathName) if err != nil { @@ -220,23 +233,16 @@ func CreateBridge(procPath string, config *BridgeConfig) (Bridge, error) { } if bridgeType == nil { - bridgeType = bridgeImpl{} - if !config.NoFastdp { - bridgeType = bridgedFastdpImpl{} - if config.NoBridgedFastdp { - bridgeType = fastdpImpl{} - config.DatapathName = config.WeaveBridgeName + bridgeType = config.configuredBridgeType() + for { + if err := bridgeType.init(config); err != nil { + if err == errBridgeFallback { + bridgeType = bridgeImpl{} + continue + } + return nil, err } - odpSupported, err := odp.CreateDatapath(config.DatapathName) - if err != nil { - return nil, errors.Wrapf(err, "creating datapath %q", config.DatapathName) - } - if !odpSupported { - bridgeType = bridgeImpl{} - } - } - if err := bridgeType.init(config); err != nil { - return nil, err + break } if err := configureIPTables(config); err != nil { return bridgeType, errors.Wrap(err, "configuring iptables") @@ -327,11 +333,17 @@ func (b bridgeImpl) init(config *BridgeConfig) error { } func (f fastdpImpl) init(config *BridgeConfig) error { - datapath, err := netlink.LinkByName(config.DatapathName) + odpSupported, err := odp.CreateDatapath(f.datapathName) if err != nil { - return errors.Wrapf(err, "finding datapath %q", config.DatapathName) + return errors.Wrapf(err, "creating datapath %q", f.datapathName) + } + if !odpSupported { + return errBridgeFallback + } + datapath, err := netlink.LinkByName(f.datapathName) + if err != nil { + return errors.Wrapf(err, "finding datapath %q", f.datapathName) } - f.datapathName = config.DatapathName if config.MTU == 0 { /* GCE has the lowest underlay network MTU we're likely to encounter on a local network, at 1460 bytes. To get the overlay MTU from that we @@ -342,7 +354,7 @@ func (f fastdpImpl) init(config *BridgeConfig) error { config.MTU = 1376 } if err := netlink.LinkSetMTU(datapath, config.MTU); err != nil { - return errors.Wrapf(err, "setting datapath %q mtu %d", config.DatapathName, config.MTU) + return errors.Wrapf(err, "setting datapath %q mtu %d", f.datapathName, config.MTU) } return nil } @@ -358,15 +370,15 @@ func (bf bridgedFastdpImpl) init(config *BridgeConfig) error { if err := netlink.LinkSetUp(veth); err != nil { return errors.Wrapf(err, "setting link up on %q", veth.Attrs().Name) } - if err := odp.AddDatapathInterface(config.DatapathName, veth.Attrs().Name); err != nil { - return errors.Wrapf(err, "adding interface %q to datapath %q", veth.Attrs().Name, config.DatapathName) + if err := odp.AddDatapathInterface(bf.datapathName, veth.Attrs().Name); err != nil { + return errors.Wrapf(err, "adding interface %q to datapath %q", veth.Attrs().Name, bf.datapathName) } return nil }); err != nil { return errors.Wrap(err, "creating bridged fastdp veth pair") } - if err := linkSetUpByName(config.DatapathName); err != nil { + if err := linkSetUpByName(bf.datapathName); err != nil { return err } From a731d47d9763f5935a9f2c5d20e26142a1eaf99e Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 24 Apr 2017 16:42:05 +0000 Subject: [PATCH 35/36] Rename bridge detection/creation functions --- net/bridge.go | 6 +++--- net/veth.go | 2 +- prog/weaver/main.go | 2 +- prog/weaveutil/bridge.go | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/net/bridge.go b/net/bridge.go index fa1c1700..55a67ac3 100644 --- a/net/bridge.go +++ b/net/bridge.go @@ -81,7 +81,7 @@ func (bridgeImpl) IsFastdp() bool { return false } func (fastdpImpl) IsFastdp() bool { return true } func (bridgedFastdpImpl) IsFastdp() bool { return true } -func DetectBridgeType(weaveBridgeName, datapathName string) (Bridge, error) { +func ExistingBridgeType(weaveBridgeName, datapathName string) (Bridge, error) { bridge, _ := netlink.LinkByName(weaveBridgeName) datapath, _ := netlink.LinkByName(datapathName) @@ -226,8 +226,8 @@ func (config *BridgeConfig) configuredBridgeType() Bridge { } } -func CreateBridge(procPath string, config *BridgeConfig) (Bridge, error) { - bridgeType, err := DetectBridgeType(config.WeaveBridgeName, config.DatapathName) +func EnsureBridge(procPath string, config *BridgeConfig) (Bridge, error) { + bridgeType, err := ExistingBridgeType(config.WeaveBridgeName, config.DatapathName) if err != nil { return nil, err } diff --git a/net/veth.go b/net/veth.go index 8637ac9f..b7708ccc 100644 --- a/net/veth.go +++ b/net/veth.go @@ -38,7 +38,7 @@ func CreateAndAttachVeth(name, peerName, bridgeName string, mtu int, keepTXOn bo return nil, fmt.Errorf(format, a...) } - bridgeType, err := DetectBridgeType(bridgeName, DatapathName) + bridgeType, err := ExistingBridgeType(bridgeName, DatapathName) if err != nil { return cleanup("detect bridge type: %s", err) } diff --git a/prog/weaver/main.go b/prog/weaver/main.go index 93ec293b..5337f014 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -261,7 +261,7 @@ func main() { name := peerName(routerName, bridgeConfig.WeaveBridgeName, dbPrefix, hostRoot) bridgeConfig.Mac = name.String() - bridgeType, err := weavenet.CreateBridge(procPath, &bridgeConfig) + bridgeType, err := weavenet.EnsureBridge(procPath, &bridgeConfig) checkFatal(err) Log.Println("Bridge type is", bridgeType) diff --git a/prog/weaveutil/bridge.go b/prog/weaveutil/bridge.go index cf87e3ab..a102ea21 100644 --- a/prog/weaveutil/bridge.go +++ b/prog/weaveutil/bridge.go @@ -11,7 +11,7 @@ func detectBridgeType(args []string) error { if len(args) != 2 { cmdUsage("detect-bridge-type", " ") } - bridgeType, err := weavenet.DetectBridgeType(args[0], args[1]) + bridgeType, err := weavenet.ExistingBridgeType(args[0], args[1]) if err != nil { return err } else if bridgeType == nil { @@ -47,7 +47,7 @@ func createBridge(args []string) error { NPC: args[9] == "--expect-npc", } procPath := args[8] - bridgeType, err := weavenet.CreateBridge(procPath, &config) + bridgeType, err := weavenet.EnsureBridge(procPath, &config) if err != nil { return err } From 4f7d47690718696f8cfe55bf36d34647af2c141a Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 24 Apr 2017 16:48:19 +0000 Subject: [PATCH 36/36] Only do set-up on bridge/datapath devices once at creation time This is a change to the previous behaviour, but makes the code easier to read and seems more sensible. --- net/bridge.go | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/net/bridge.go b/net/bridge.go index 55a67ac3..79a6995c 100644 --- a/net/bridge.go +++ b/net/bridge.go @@ -228,25 +228,24 @@ func (config *BridgeConfig) configuredBridgeType() Bridge { func EnsureBridge(procPath string, config *BridgeConfig) (Bridge, error) { bridgeType, err := ExistingBridgeType(config.WeaveBridgeName, config.DatapathName) - if err != nil { - return nil, err + if bridgeType != nil || err != nil { + return bridgeType, err } - if bridgeType == nil { - bridgeType = config.configuredBridgeType() - for { - if err := bridgeType.init(config); err != nil { - if err == errBridgeFallback { - bridgeType = bridgeImpl{} - continue - } - return nil, err + bridgeType = config.configuredBridgeType() + for { + if err := bridgeType.init(config); err != nil { + if err == errBridgeFallback { + bridgeType = bridgeImpl{} + continue } - break - } - if err := configureIPTables(config); err != nil { - return bridgeType, errors.Wrap(err, "configuring iptables") + return nil, err } + break + } + + if err := configureIPTables(config); err != nil { + return bridgeType, errors.Wrap(err, "configuring iptables") } if config.AWSVPC {