diff --git a/ipam/tracker/awsvpc.go b/ipam/tracker/awsvpc.go index 51424379..d4836953 100644 --- a/ipam/tracker/awsvpc.go +++ b/ipam/tracker/awsvpc.go @@ -30,7 +30,6 @@ import ( "github.com/vishvananda/netlink" "github.com/weaveworks/weave/common" - wnet "github.com/weaveworks/weave/net" "github.com/weaveworks/weave/net/address" ) @@ -42,7 +41,7 @@ type AWSVPCTracker struct { } // NewAWSVPCTracker creates and initialises AWS VPC based tracker. -func NewAWSVPCTracker() (*AWSVPCTracker, error) { +func NewAWSVPCTracker(bridgeName string) (*AWSVPCTracker, error) { var ( err error session = session.New() @@ -69,9 +68,9 @@ func NewAWSVPCTracker() (*AWSVPCTracker, error) { t.routeTableID = *routeTableID // Detect Weave bridge link index - link, err := netlink.LinkByName(wnet.WeaveBridgeName) + link, err := netlink.LinkByName(bridgeName) if err != nil { - return nil, fmt.Errorf("cannot find \"%s\" interface: %s", wnet.WeaveBridgeName, err) + return nil, fmt.Errorf("cannot find \"%s\" interface: %s", bridgeName, err) } t.linkIndex = link.Attrs().Index diff --git a/nameserver/http.go b/nameserver/http.go index ece2b6c2..ac421c0e 100644 --- a/nameserver/http.go +++ b/nameserver/http.go @@ -27,7 +27,7 @@ func (n *Nameserver) HandleHTTP(router *mux.Router, dockerCli *docker.Client) { vars = mux.Vars(r) container = vars["container"] ipStr = vars["ip"] - hostname = dns.Fqdn(r.FormValue("fqdn")) + fqdn = r.FormValue("fqdn") ip, err = address.ParseIP(ipStr) ) if err != nil { @@ -35,16 +35,11 @@ func (n *Nameserver) HandleHTTP(router *mux.Router, dockerCli *docker.Client) { return } - if !dns.IsSubDomain(n.domain, hostname) { - n.infof("Ignoring registration %s %s %s (not a subdomain of %s)", hostname, ipStr, container, n.domain) - return - } - - n.AddEntry(hostname, container, n.ourName, ip) + n.AddEntryFQDN(fqdn, container, n.ourName, ip) if r.FormValue("check-alive") == "true" && dockerCli != nil && dockerCli.IsContainerNotRunning(container) { n.infof("container '%s' is not running: removing", container) - n.Delete(hostname, container, ipStr, ip) + n.Delete(dns.Fqdn(fqdn), container, ipStr, ip) } w.WriteHeader(204) diff --git a/nameserver/nameserver.go b/nameserver/nameserver.go index 6fcf1e84..75879276 100644 --- a/nameserver/nameserver.go +++ b/nameserver/nameserver.go @@ -91,6 +91,15 @@ func (n *Nameserver) AddEntry(hostname, containerid string, origin mesh.PeerName n.broadcastEntries(entry) } +func (n *Nameserver) AddEntryFQDN(fqdn, containerid string, origin mesh.PeerName, addr address.Address) { + hostname := dns.Fqdn(fqdn) + if !dns.IsSubDomain(n.domain, hostname) { + n.infof("Ignoring registration %s %s %s (not a subdomain of %s)", hostname, addr.String(), containerid, n.domain) + return + } + n.AddEntry(hostname, containerid, origin, addr) +} + func (n *Nameserver) Lookup(hostname string) []address.Address { n.RLock() defer n.RUnlock() diff --git a/net/address/address.go b/net/address/address.go index d5fb8e00..89ddf415 100644 --- a/net/address/address.go +++ b/net/address/address.go @@ -160,6 +160,11 @@ func (cidr CIDR) String() string { return fmt.Sprintf("%s/%d", cidr.Addr.String(), cidr.PrefixLen) } +func (cidr CIDR) IPNet() (r *net.IPNet) { + mask := net.CIDRMask(cidr.PrefixLen, 8*net.IPv4len) + return &net.IPNet{IP: cidr.Addr.IP4(), Mask: mask} +} + // FromIP4 converts an ipv4 address to our integer address type func FromIP4(ip4 net.IP) (r Address) { for _, b := range ip4.To4() { diff --git a/net/address/address_test.go b/net/address/address_test.go index 8718fdb6..6b7b3033 100644 --- a/net/address/address_test.go +++ b/net/address/address_test.go @@ -1,6 +1,7 @@ package address import ( + "net" "testing" "testing/quick" @@ -67,3 +68,15 @@ func TestCIDRStartAndEnd(t *testing.T) { require.Equal(t, ip("10.0.0.0"), cidr.Start(), "") require.Equal(t, ip("10.0.1.0"), cidr.End(), "") } + +func TestConversions(t *testing.T) { + addrStr := "10.2.3.0" + cidrStr := addrStr + "/24" + addr := ip(addrStr) + require.Equal(t, addr.String(), addrStr) + require.Equal(t, addr.IP4(), net.ParseIP(addrStr).To4()) + cidr, _ := ParseCIDR(cidrStr) + require.Equal(t, cidr.String(), cidrStr) + _, ipnet, _ := net.ParseCIDR(cidrStr) + require.Equal(t, cidr.IPNet(), ipnet) +} diff --git a/net/arp.go b/net/arp.go index 8315db94..4a4e0d78 100644 --- a/net/arp.go +++ b/net/arp.go @@ -1,27 +1,30 @@ package net -import "fmt" -import "io" -import "os" +import ( + "fmt" + "io" + "os" + "path/filepath" +) // Configure the ARP cache parameters for the given interface. This // makes containers react more quickly to a change in the MAC address // associated with an IP address. -func ConfigureARPCache(name string) error { - if err := sysctl(fmt.Sprintf("net/ipv4/neigh/%s/base_reachable_time", name), "5"); err != nil { +func ConfigureARPCache(procPath, name string) error { + if err := sysctl(procPath, fmt.Sprintf("net/ipv4/neigh/%s/base_reachable_time", name), "5"); err != nil { return err } - if err := sysctl(fmt.Sprintf("net/ipv4/neigh/%s/delay_first_probe_time", name), "2"); err != nil { + if err := sysctl(procPath, fmt.Sprintf("net/ipv4/neigh/%s/delay_first_probe_time", name), "2"); err != nil { return err } - if err := sysctl(fmt.Sprintf("net/ipv4/neigh/%s/ucast_solicit", name), "1"); err != nil { + if err := sysctl(procPath, fmt.Sprintf("net/ipv4/neigh/%s/ucast_solicit", name), "1"); err != nil { return err } return nil } -func sysctl(variable, value string) error { - f, err := os.OpenFile(fmt.Sprintf("/proc/sys/%s", variable), os.O_WRONLY, 0) +func sysctl(procPath, variable, value string) error { + f, err := os.OpenFile(filepath.Join(procPath, "/sys/", variable), os.O_WRONLY, 0) if err != nil { return err } diff --git a/net/bridge.go b/net/bridge.go index 6845c0bf..79a6995c 100644 --- a/net/bridge.go +++ b/net/bridge.go @@ -2,43 +2,136 @@ package net import ( "fmt" + "io/ioutil" "net" + "path/filepath" "github.com/Sirupsen/logrus" + "github.com/coreos/go-iptables/iptables" + "github.com/j-keck/arping" + "github.com/pkg/errors" "github.com/vishvananda/netlink" + + "github.com/weaveworks/weave/common/odp" ) -type BridgeType int +/* This code implements three possible configurations to connect + containers to the Weave Net overlay: + +1. Bridge + +-------+ +(container-veth)-+ weave +-(vethwe-bridge)--(vethwe-pcap) + +-------+ + +"weave" is a Linux bridge. "vethwe-pcap" (end of veth pair) is used +to capture and inject packets, by router/pcap.go. + +2. BridgedFastdp + + +-------+ /----------\ +(container-veth)-+ weave +-(vethwe-bridge)--(vethwe-datapath)-+ datapath + + +-------+ \----------/ + +"weave" is a Linux bridge and "datapath" is an Open vSwitch datapath; +they are connected via a veth pair. Packet capture and injection use +the "datapath" device, via "router/fastdp.go:fastDatapathBridge" + +3. Fastdp + + /-------\ +(container-veth)-+ weave + + \-------/ + +"weave" is an Open vSwitch datapath, and capture/injection are as in +BridgedFastdp. Not used by default due to missing conntrack support in +datapath of old kernel versions (https://github.com/weaveworks/weave/issues/1577). +*/ const ( WeaveBridgeName = "weave" DatapathName = "datapath" - - None BridgeType = iota - Bridge - Fastdp - BridgedFastdp - Inconsistent + DatapathIfName = "vethwe-datapath" + BridgeIfName = "vethwe-bridge" + PcapIfName = "vethwe-pcap" ) -func DetectBridgeType(weaveBridgeName, datapathName string) BridgeType { +type Bridge interface { + init(config *BridgeConfig) error // create and initialise bridge device(s) + attach(veth *netlink.Veth) error // attach veth to bridge + IsFastdp() bool // does this bridge use fastdp? + String() string // human-readable type string +} + +var errBridgeFallback = errors.New("special error value used to trigger a fallback to bridge type") + +type bridgeImpl struct{ bridge netlink.Link } +type fastdpImpl struct{ datapathName string } +type bridgedFastdpImpl struct { + bridgeImpl + fastdpImpl +} + +// Returns a string that is consistent with the weave script +func (bridgeImpl) String() string { return "bridge" } +func (fastdpImpl) String() string { return "fastdp" } +func (bridgedFastdpImpl) String() string { return "bridged_fastdp" } + +// Used to decide whether to manage ODP tunnels +func (bridgeImpl) IsFastdp() bool { return false } +func (fastdpImpl) IsFastdp() bool { return true } +func (bridgedFastdpImpl) IsFastdp() bool { return true } + +func ExistingBridgeType(weaveBridgeName, datapathName string) (Bridge, error) { bridge, _ := netlink.LinkByName(weaveBridgeName) datapath, _ := netlink.LinkByName(datapathName) switch { case bridge == nil && datapath == nil: - return None + return nil, nil case isBridge(bridge) && datapath == nil: - return Bridge + return bridgeImpl{bridge: bridge}, nil case isDatapath(bridge) && datapath == nil: - return Fastdp + return fastdpImpl{datapathName: datapathName}, nil case isDatapath(datapath) && isBridge(bridge): - return BridgedFastdp + return bridgedFastdpImpl{bridgeImpl{bridge: bridge}, fastdpImpl{datapathName: datapathName}}, nil default: - return Inconsistent + return nil, errors.New("Inconsistent bridge state detected. Please do 'weave reset' and try again") } } +func EnforceAddrAssignType(bridgeName string) (setAddr bool, err error) { + sysctlFilename := filepath.Join("/sys/class/net/", bridgeName, "/addr_assign_type") + addrAssignType, err := ioutil.ReadFile(sysctlFilename) + if err != nil { + return false, errors.Wrapf(err, "reading %q", sysctlFilename) + } + + // From include/uapi/linux/netdevice.h + // #define NET_ADDR_PERM 0 /* address is permanent (default) */ + // #define NET_ADDR_RANDOM 1 /* address is generated randomly */ + // #define NET_ADDR_STOLEN 2 /* address is stolen from other device */ + // #define NET_ADDR_SET 3 /* address is set using dev_set_mac_address() */ + // Note the file typically has a newline at the end, so we just look at the first char + if addrAssignType[0] != '3' { + link, err := netlink.LinkByName(bridgeName) + if err != nil { + return false, errors.Wrapf(err, "EnforceAddrAssignType finding bridge %s", bridgeName) + } + + mac, err := RandomMAC() + if err != nil { + return false, errors.Wrap(err, "creating random MAC") + } + + if err := netlink.LinkSetHardwareAddr(link, mac); err != nil { + return false, errors.Wrapf(err, "setting bridge %s address to %v", bridgeName, mac) + } + return true, nil + } + + return false, nil +} + func isBridge(link netlink.Link) bool { _, isBridge := link.(*netlink.Bridge) return isBridge @@ -108,3 +201,316 @@ func FindBridgeIP(bridgeName string, subnet *net.IPNet) (net.IP, error) { // No subnet, or none in the required subnet; just return the first one return netdev.CIDRs[0].IP, nil } + +type BridgeConfig struct { + DockerBridgeName string + WeaveBridgeName string + DatapathName string + NoFastdp bool + NoBridgedFastdp bool + AWSVPC bool + NPC bool + MTU int + Mac string + Port int +} + +func (config *BridgeConfig) configuredBridgeType() Bridge { + switch { + case config.NoFastdp: + return bridgeImpl{} + case config.NoBridgedFastdp: + return fastdpImpl{datapathName: config.WeaveBridgeName} + default: + return bridgedFastdpImpl{fastdpImpl: fastdpImpl{datapathName: config.DatapathName}} + } +} + +func EnsureBridge(procPath string, config *BridgeConfig) (Bridge, error) { + bridgeType, err := ExistingBridgeType(config.WeaveBridgeName, config.DatapathName) + if bridgeType != nil || err != nil { + return bridgeType, err + } + + bridgeType = config.configuredBridgeType() + for { + if err := bridgeType.init(config); err != nil { + if err == errBridgeFallback { + bridgeType = bridgeImpl{} + continue + } + return nil, err + } + break + } + + if err := configureIPTables(config); err != nil { + return bridgeType, errors.Wrap(err, "configuring iptables") + } + + if config.AWSVPC { + // Set proxy_arp on the bridge, so that it could accept packets destined + // to containers within the same subnet but running on remote hosts. + // Without it, exact routes on each container are required. + if err := sysctl(procPath, "net/ipv4/conf/"+config.WeaveBridgeName+"/proxy_arp", "1"); err != nil { + return bridgeType, errors.Wrap(err, "setting proxy_arp") + } + // Avoid delaying the first ARP request. Also, setting it to 0 avoids + // placing the request into a bounded queue as it can be seen: + // https://git.kernel.org/cgit/linux/kernel/git/stable/linux-stable.git/tree/net/ipv4/arp.c?id=refs/tags/v4.6.1#n819 + if err := sysctl(procPath, "net/ipv4/neigh/"+config.WeaveBridgeName+"/proxy_delay", "0"); err != nil { + return bridgeType, errors.Wrap(err, "setting proxy_arp") + } + } + + if err := linkSetUpByName(config.WeaveBridgeName); err != nil { + return bridgeType, err + } + + if err := ConfigureARPCache(procPath, config.WeaveBridgeName); err != nil { + return bridgeType, errors.Wrapf(err, "configuring ARP cache on bridge %q", config.WeaveBridgeName) + } + + return bridgeType, nil +} + +func (b bridgeImpl) initPrep(config *BridgeConfig) error { + mac, err := net.ParseMAC(config.Mac) + if err != nil { + return errors.Wrapf(err, "parsing bridge MAC %q", config.Mac) + } + + linkAttrs := netlink.NewLinkAttrs() + linkAttrs.Name = config.WeaveBridgeName + linkAttrs.HardwareAddr = mac + if config.MTU == 0 { + config.MTU = 65535 + } + b.bridge = &netlink.Bridge{LinkAttrs: linkAttrs} + if err = netlink.LinkAdd(b.bridge); err != nil { + return errors.Wrapf(err, "creating bridge %q", config.WeaveBridgeName) + } + if err := netlink.LinkSetHardwareAddr(b.bridge, mac); err != nil { + return errors.Wrapf(err, "setting bridge %q mac %v", config.WeaveBridgeName, mac) + } + // Attempting to set the bridge MTU to a high value directly + // fails. Bridges take the lowest MTU of their interfaces. So + // instead we create a temporary interface with the desired MTU, + // attach that to the bridge, and then remove it again. + dummy := &netlink.Dummy{LinkAttrs: netlink.NewLinkAttrs()} + dummy.LinkAttrs.Name = "vethwedu" + if err = netlink.LinkAdd(dummy); err != nil { + return errors.Wrap(err, "creating dummy interface") + } + if err := netlink.LinkSetMTU(dummy, config.MTU); err != nil { + return errors.Wrapf(err, "setting dummy interface mtu to %d", config.MTU) + } + if err := netlink.LinkSetMasterByIndex(dummy, b.bridge.Attrs().Index); err != nil { + return errors.Wrap(err, "setting dummy interface master") + } + if err := netlink.LinkDel(dummy); err != nil { + return errors.Wrap(err, "deleting dummy interface") + } + + return nil +} + +func (b bridgeImpl) init(config *BridgeConfig) error { + if err := b.initPrep(config); err != nil { + return err + } + if _, err := CreateAndAttachVeth(BridgeIfName, PcapIfName, config.WeaveBridgeName, config.MTU, true, func(veth netlink.Link) error { + return netlink.LinkSetUp(veth) + }); err != nil { + return errors.Wrap(err, "creating pcap veth pair") + } + if err := EthtoolTXOff(config.WeaveBridgeName); err != nil { + return errors.Wrap(err, "setting tx off") + } + + return nil +} + +func (f fastdpImpl) init(config *BridgeConfig) error { + odpSupported, err := odp.CreateDatapath(f.datapathName) + if err != nil { + return errors.Wrapf(err, "creating datapath %q", f.datapathName) + } + if !odpSupported { + return errBridgeFallback + } + datapath, err := netlink.LinkByName(f.datapathName) + if err != nil { + return errors.Wrapf(err, "finding datapath %q", f.datapathName) + } + if config.MTU == 0 { + /* GCE has the lowest underlay network MTU we're likely to encounter on + a local network, at 1460 bytes. To get the overlay MTU from that we + subtract 20 bytes for the outer IPv4 header, 8 bytes for the outer + UDP header, 8 bytes for the vxlan header, and 14 bytes for the inner + ethernet header. In addition, we subtract 34 bytes for the ESP overhead + which is needed for the vxlan encryption. */ + config.MTU = 1376 + } + if err := netlink.LinkSetMTU(datapath, config.MTU); err != nil { + return errors.Wrapf(err, "setting datapath %q mtu %d", f.datapathName, config.MTU) + } + return nil +} + +func (bf bridgedFastdpImpl) init(config *BridgeConfig) error { + if err := bf.fastdpImpl.init(config); err != nil { + return err + } + if err := bf.bridgeImpl.initPrep(config); err != nil { + return err + } + if _, err := CreateAndAttachVeth(BridgeIfName, DatapathIfName, config.WeaveBridgeName, config.MTU, true, func(veth netlink.Link) error { + if err := netlink.LinkSetUp(veth); err != nil { + return errors.Wrapf(err, "setting link up on %q", veth.Attrs().Name) + } + if err := odp.AddDatapathInterface(bf.datapathName, veth.Attrs().Name); err != nil { + return errors.Wrapf(err, "adding interface %q to datapath %q", veth.Attrs().Name, bf.datapathName) + } + return nil + }); err != nil { + return errors.Wrap(err, "creating bridged fastdp veth pair") + } + + if err := linkSetUpByName(bf.datapathName); err != nil { + return err + } + + return nil +} + +func (b bridgeImpl) attach(veth *netlink.Veth) error { + return netlink.LinkSetMasterByIndex(veth, b.bridge.Attrs().Index) +} + +func (bf bridgedFastdpImpl) attach(veth *netlink.Veth) error { + return bf.bridgeImpl.attach(veth) +} + +func (f fastdpImpl) attach(veth *netlink.Veth) error { + return odp.AddDatapathInterface(f.datapathName, veth.Attrs().Name) +} + +func configureIPTables(config *BridgeConfig) error { + ipt, err := iptables.New() + if err != nil { + return errors.Wrap(err, "creating iptables object") + } + if config.DockerBridgeName != "" { + if config.WeaveBridgeName != config.DockerBridgeName { + err := ipt.Insert("filter", "FORWARD", 1, "-i", config.DockerBridgeName, "-o", config.WeaveBridgeName, "-j", "DROP") + if err != nil { + return err + } + } + + dockerBridgeIP, err := FindBridgeIP(config.DockerBridgeName, nil) + if err != nil { + return err + } + + // forbid traffic to the Weave port from other containers + if err = ipt.AppendUnique("filter", "INPUT", "-i", config.DockerBridgeName, "-p", "tcp", "--dst", dockerBridgeIP.String(), "--dport", fmt.Sprint(config.Port), "-j", "DROP"); err != nil { + return err + } + if err = ipt.AppendUnique("filter", "INPUT", "-i", config.DockerBridgeName, "-p", "udp", "--dst", dockerBridgeIP.String(), "--dport", fmt.Sprint(config.Port), "-j", "DROP"); err != nil { + return err + } + if err = ipt.AppendUnique("filter", "INPUT", "-i", config.DockerBridgeName, "-p", "udp", "--dst", dockerBridgeIP.String(), "--dport", fmt.Sprint(config.Port+1), "-j", "DROP"); err != nil { + return err + } + + // let DNS traffic to weaveDNS, since otherwise it might get blocked by the likes of UFW + if err = ipt.AppendUnique("filter", "INPUT", "-i", config.DockerBridgeName, "-p", "udp", "--dport", "53", "-j", "ACCEPT"); err != nil { + return err + } + if err = ipt.AppendUnique("filter", "INPUT", "-i", config.DockerBridgeName, "-p", "tcp", "--dport", "53", "-j", "ACCEPT"); err != nil { + return err + } + } + + if config.NPC { + // Steer traffic via the NPC + if err = ipt.ClearChain("nat", "WEAVE-NPC"); err != nil { + return errors.Wrap(err, "clearing WEAVE-NPC chain") + } + if err = ipt.AppendUnique("filter", "FORWARD", "-o", config.WeaveBridgeName, "-j", "WEAVE-NPC"); err != nil { + return err + } + if err = ipt.AppendUnique("filter", "FORWARD", "-o", config.WeaveBridgeName, "-m", "state", "--state", "NEW", "-j", "NFLOG", "--nflog-group", "86"); err != nil { + return err + } + if err = ipt.AppendUnique("filter", "FORWARD", "-o", config.WeaveBridgeName, "-j", "DROP"); err != nil { + return err + } + } else { + // Work around the situation where there are no rules allowing traffic + // across our bridge. E.g. ufw + if err = ipt.AppendUnique("filter", "FORWARD", "-i", config.WeaveBridgeName, "-o", config.WeaveBridgeName, "-j", "ACCEPT"); err != nil { + return err + } + // Forward from weave to the rest of the world + if err = ipt.AppendUnique("filter", "FORWARD", "-i", config.WeaveBridgeName, "!", "-o", config.WeaveBridgeName, "-j", "ACCEPT"); err != nil { + return err + } + // and allow replies back + if err = ipt.AppendUnique("filter", "FORWARD", "-o", config.WeaveBridgeName, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"); err != nil { + return err + } + } + + // create a chain for masquerading + if err = ipt.ClearChain("nat", "WEAVE"); err != nil { + return errors.Wrap(err, "clearing WEAVE chain") + } + if err = ipt.AppendUnique("nat", "POSTROUTING", "-j", "WEAVE"); err != nil { + return err + } + + return nil +} + +func linkSetUpByName(linkName string) error { + link, err := netlink.LinkByName(linkName) + if err != nil { + return errors.Wrapf(err, "setting link up on %q", linkName) + } + return netlink.LinkSetUp(link) +} + +func AddBridgeAddr(bridgeName string, addr *net.IPNet, removeDefaultRoute bool) error { + link, err := netlink.LinkByName(bridgeName) + if err != nil { + return errors.Wrapf(err, "AddBridgeAddr finding bridge %q", bridgeName) + } + newAddresses, err := AddAddresses(link, []*net.IPNet{addr}) + if err != nil { + return errors.Wrapf(err, "adding address %v to %q", addr, bridgeName) + } + for _, ipnet := range newAddresses { + arping.GratuitousArpOverIfaceByName(ipnet.IP, bridgeName) + } + if removeDefaultRoute { + routeFilter := &netlink.Route{ + LinkIndex: link.Attrs().Index, + Dst: &net.IPNet{IP: addr.IP.Mask(addr.Mask), Mask: addr.Mask}, + Protocol: 2, // RTPROT_KERNEL + } + filterMask := netlink.RT_FILTER_OIF | netlink.RT_FILTER_DST | netlink.RT_FILTER_PROTOCOL + routes, err := netlink.RouteListFiltered(netlink.FAMILY_V4, routeFilter, filterMask) + if err != nil { + return errors.Wrapf(err, "failed to get route list for bridge %q", bridgeName) + } + for _, r := range routes { + if err = netlink.RouteDel(&r); err != nil { + return errors.Wrapf(err, "failed to delete default route %+v", r) + } + } + } + return nil +} diff --git a/net/veth.go b/net/veth.go index a95553e8..b7708ccc 100644 --- a/net/veth.go +++ b/net/veth.go @@ -11,8 +11,6 @@ import ( "github.com/j-keck/arping" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" - - "github.com/weaveworks/weave/common/odp" ) // create and attach a veth to the Weave bridge @@ -40,22 +38,17 @@ func CreateAndAttachVeth(name, peerName, bridgeName string, mtu int, keepTXOn bo return nil, fmt.Errorf(format, a...) } - switch bridgeType := DetectBridgeType(bridgeName, DatapathName); bridgeType { - case Bridge, BridgedFastdp: - if err := netlink.LinkSetMasterByIndex(veth, bridge.Attrs().Index); err != nil { - return cleanup(`unable to set master of %s: %s`, name, err) + bridgeType, err := ExistingBridgeType(bridgeName, DatapathName) + if err != nil { + return cleanup("detect bridge type: %s", err) + } + if err := bridgeType.attach(veth); err != nil { + return cleanup("attaching veth %q to %q: %s", name, bridgeName, err) + } + if !bridgeType.IsFastdp() && !keepTXOn { + if err := EthtoolTXOff(veth.PeerName); err != nil { + return cleanup(`unable to set tx off on %q: %s`, peerName, err) } - if bridgeType == Bridge && !keepTXOn { - if err := EthtoolTXOff(peerName); err != nil { - return cleanup(`unable to set tx off on %q: %s`, peerName, err) - } - } - case Fastdp: - if err := odp.AddDatapathInterface(bridgeName, name); err != nil { - return cleanup(`failed to attach %s to device "%s": %s`, name, bridgeName, err) - } - default: - return cleanup(`invalid bridge configuration`) } if init != nil { diff --git a/plugin/net/watcher.go b/plugin/net/watcher.go index 736751d7..9a795611 100644 --- a/plugin/net/watcher.go +++ b/plugin/net/watcher.go @@ -53,7 +53,7 @@ func (w *watcher) ContainerStarted(id string) { rootDir = "/host" } netNSPath := weavenet.NSPathByPidWithRoot(rootDir, info.State.Pid) - if _, err := weavenet.WithNetNS(netNSPath, "configure-arp", weavenet.VethName); err != nil { + if _, err := weavenet.WithNetNS(netNSPath, "configure-arp", weavenet.VethName, rootDir); err != nil { w.driver.warn("ContainerStarted", "unable to configure interfaces: %s", err) } } diff --git a/prog/net-plugin/launch.sh b/prog/net-plugin/launch.sh index a372be13..493a74c5 100755 --- a/prog/net-plugin/launch.sh +++ b/prog/net-plugin/launch.sh @@ -29,22 +29,11 @@ SWARM_MANAGER_PEERS=$(/usr/bin/weaveutil swarm-manager-peers) # Prevent from restoring from a persisted peers list rm -f "/restart.sentinel" -/home/weave/weave --local create-bridge \ - --proc-path=/host/proc \ - --weavedb-dir-path=$WEAVE_DIR \ - --force - -BRIDGE_OPTIONS="--datapath=datapath" -if [ "$(/home/weave/weave --local bridge-type)" = "bridge" ]; then - # TODO: Call into weave script to do this - if ! ip link show vethwe-pcap >/dev/null 2>&1; then - ip link add name vethwe-bridge type veth peer name vethwe-pcap - ip link set vethwe-bridge up - ip link set vethwe-pcap up - ip link set vethwe-bridge master weave - fi - BRIDGE_OPTIONS="--iface=vethwe-pcap" -fi +router_bridge_opts() { + echo --datapath=datapath + [ -z "$WEAVE_MTU" ] || echo --mtu "$WEAVE_MTU" + [ -z "$WEAVE_NO_FASTDP" ] || echo --no-fastdp +} if [ -z "$IPALLOC_INIT" ]; then IPALLOC_INIT="observer" @@ -53,7 +42,9 @@ if [ -z "$IPALLOC_INIT" ]; then fi fi -exec /home/weave/weaver $EXTRA_ARGS --port=6783 $BRIDGE_OPTIONS \ +exec /home/weave/weaver $EXTRA_ARGS --port=6783 $(router_bridge_opts) \ + --host-root=/host \ + --proc-path=/host/proc \ --http-addr=$HTTP_ADDR --status-addr=$STATUS_ADDR \ --no-dns \ --ipalloc-range=$IPALLOC_RANGE \ diff --git a/prog/weave-kube/launch.sh b/prog/weave-kube/launch.sh index 1b851d49..60899189 100755 --- a/prog/weave-kube/launch.sh +++ b/prog/weave-kube/launch.sh @@ -54,9 +54,6 @@ if [ "${EXPECT_NPC}" = "0" ]; then WEAVE_NPC_OPTS="" fi -export HOST_ROOT=/host -/home/weave/weave --local create-bridge --force $WEAVE_NPC_OPTS - # Kubernetes sets HOSTNAME to the host's hostname # when running a pod in host namespace. NICKNAME_ARG="" @@ -64,17 +61,11 @@ if [ -n "$HOSTNAME" ] ; then NICKNAME_ARG="--nickname=$HOSTNAME" fi -BRIDGE_OPTIONS="--datapath=datapath" -if [ "$(/home/weave/weave --local bridge-type)" = "bridge" ] ; then - # TODO: Call into weave script to do this - if ! ip link show vethwe-pcap >/dev/null 2>&1 ; then - ip link add name vethwe-bridge type veth peer name vethwe-pcap - ip link set vethwe-bridge up - ip link set vethwe-pcap up - ip link set vethwe-bridge master weave - fi - BRIDGE_OPTIONS="--iface=vethwe-pcap" -fi +router_bridge_opts() { + echo --datapath=datapath + [ -z "$WEAVE_MTU" ] || echo --mtu "$WEAVE_MTU" + [ -z "$WEAVE_NO_FASTDP" ] || echo --no-fastdp +} if [ -z "$KUBE_PEERS" ]; then if ! KUBE_PEERS=$(/home/weave/kube-peers) || [ -z "$KUBE_PEERS" ]; then @@ -109,6 +100,7 @@ post_start_actions() { fi fi mkdir -p $HOST_ROOT/etc/cni/net.d + export HOST_ROOT /home/weave/weave --local setup-cni # Expose the weave network so host processes can communicate with pods @@ -117,10 +109,12 @@ post_start_actions() { post_start_actions & -/home/weave/weaver $EXTRA_ARGS --port=6783 $BRIDGE_OPTIONS \ +/home/weave/weaver $EXTRA_ARGS --port=6783 $(router_bridge_opts) \ + --host-root=$HOST_ROOT \ --http-addr=$HTTP_ADDR --status-addr=$STATUS_ADDR --docker-api='' --no-dns \ --ipalloc-range=$IPALLOC_RANGE $NICKNAME_ARG \ --ipalloc-init $IPALLOC_INIT \ --conn-limit=$CONN_LIMIT \ + $WEAVE_NPC_OPTS \ "$@" \ $KUBE_PEERS diff --git a/prog/weaver/main.go b/prog/weaver/main.go index 7982a2fb..5337f014 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -125,10 +125,10 @@ func main() { var ( justVersion bool config mesh.Config + bridgeConfig weavenet.BridgeConfig networkConfig weave.NetworkConfig protocolMinVersion int resume bool - ifaceName string routerName string nickName string password string @@ -144,10 +144,10 @@ func main() { peers []string noDNS bool dnsConfig dnsConfig - datapathName string trustedSubnetStr string dbPrefix string - isAWSVPC bool + hostRoot string + procPath string discoveryEndpoint string token string advertiseAddress string @@ -168,7 +168,9 @@ func main() { mflag.IntVar(&config.Port, []string{"#port", "-port"}, mesh.Port, "router port") mflag.IntVar(&protocolMinVersion, []string{"-min-protocol-version"}, mesh.ProtocolMinVersion, "minimum weave protocol version") mflag.BoolVar(&resume, []string{"-resume"}, false, "resume connections to previous peers") - mflag.StringVar(&ifaceName, []string{"#iface", "-iface"}, "", "name of interface to capture/inject from (disabled if blank)") + mflag.StringVar(&bridgeConfig.WeaveBridgeName, []string{"-weave-bridge"}, "weave", "name of weave bridge") + mflag.StringVar(&bridgeConfig.DockerBridgeName, []string{"-docker-bridge"}, "", "name of Docker bridge") + mflag.BoolVar(&bridgeConfig.NPC, []string{"-expect-npc"}, false, "set up iptables rules for npc") mflag.StringVar(&routerName, []string{"#name", "-name"}, "", "name of router (defaults to MAC of interface)") mflag.StringVar(&nickName, []string{"#nickname", "-nickname"}, "", "nickname of peer (defaults to hostname)") mflag.StringVar(&password, []string{"#password", "-password"}, "", "network password") @@ -178,6 +180,7 @@ func main() { mflag.IntVar(&config.ConnLimit, []string{"#connlimit", "#-connlimit", "-conn-limit"}, 30, "connection limit (0 for unlimited)") mflag.BoolVar(&noDiscovery, []string{"#nodiscovery", "#-nodiscovery", "-no-discovery"}, false, "disable peer discovery") mflag.IntVar(&bufSzMB, []string{"#bufsz", "-bufsz"}, 8, "capture buffer size in MB") + mflag.IntVar(&bridgeConfig.MTU, []string{"-mtu"}, 0, "MTU size") mflag.StringVar(&httpAddr, []string{"#httpaddr", "#-httpaddr", "-http-addr"}, "", "address to bind HTTP interface to (disabled if blank, absolute path indicates unix domain socket)") mflag.StringVar(&statusAddr, []string{"-status-addr"}, "", "address to bind status+metrics interface to (disabled if blank, absolute path indicates unix domain socket)") mflag.StringVar(&ipamConfig.Mode, []string{"-ipalloc-init"}, "", "allocator initialisation strategy (consensus, seed or observer)") @@ -192,10 +195,13 @@ func main() { mflag.DurationVar(&dnsConfig.ClientTimeout, []string{"-dns-fallback-timeout"}, nameserver.DefaultClientTimeout, "timeout for fallback DNS requests") mflag.StringVar(&dnsConfig.EffectiveListenAddress, []string{"-dns-effective-listen-address"}, "", "address DNS will actually be listening, after Docker port mapping") mflag.StringVar(&dnsConfig.ResolvConf, []string{"-resolv-conf"}, "", "path to resolver configuration for fallback DNS lookups") - mflag.StringVar(&datapathName, []string{"-datapath"}, "", "ODP datapath name") + mflag.StringVar(&bridgeConfig.DatapathName, []string{"-datapath"}, "", "ODP datapath name") + mflag.BoolVar(&bridgeConfig.NoFastdp, []string{"-no-fastdp"}, false, "Disable Fast Datapath") mflag.StringVar(&trustedSubnetStr, []string{"-trusted-subnets"}, "", "comma-separated list of trusted subnets in CIDR notation") mflag.StringVar(&dbPrefix, []string{"-db-prefix"}, "/weavedb/weave", "pathname/prefix of filename to store data") - mflag.BoolVar(&isAWSVPC, []string{"-awsvpc"}, false, "use AWS VPC for routing") + mflag.StringVar(&procPath, []string{"-proc-path"}, "/proc", "path to reach host /proc filesystem") + mflag.BoolVar(&bridgeConfig.AWSVPC, []string{"-awsvpc"}, false, "use AWS VPC for routing") + mflag.StringVar(&hostRoot, []string{"-host-root"}, "", "path to reach host filesystem") mflag.StringVar(&discoveryEndpoint, []string{"-peer-discovery-url"}, "https://cloud.weave.works/api/net", "url for peer discovery") mflag.StringVar(&token, []string{"-token"}, "", "token for peer discovery") mflag.StringVar(&advertiseAddress, []string{"-advertise-address"}, "", "address to advertise for peer discovery") @@ -244,18 +250,31 @@ func main() { networkConfig.PacketLogging = nopPacketLogging{} } - config.Password = determinePassword(password) - - overlay, bridge := createOverlay(datapathName, ifaceName, isAWSVPC, config.Host, config.Port, bufSzMB, config.Password != nil) - networkConfig.Bridge = bridge - - if bridge != nil { - if err := weavenet.DetectHairpin("vethwe-bridge", Log); err != nil { - Log.Errorf("DetectHairpin failed: %s", err) + if bridgeConfig.DockerBridgeName != "" { + if setAddr, err := weavenet.EnforceAddrAssignType(bridgeConfig.DockerBridgeName); err != nil { + Log.Errorf("While checking address assignment type of %s: %s", bridgeConfig.DockerBridgeName, err) + } else if setAddr { + Log.Warningf("Setting %s MAC (mitigate https://github.com/docker/docker/issues/14908)", bridgeConfig.DockerBridgeName) } } - name := peerName(routerName) + name := peerName(routerName, bridgeConfig.WeaveBridgeName, dbPrefix, hostRoot) + + bridgeConfig.Mac = name.String() + bridgeType, err := weavenet.EnsureBridge(procPath, &bridgeConfig) + checkFatal(err) + Log.Println("Bridge type is", bridgeType) + + config.Password = determinePassword(password) + + overlay, bridge := createOverlay(bridgeType, bridgeConfig, config.Host, config.Port, bufSzMB, config.Password != nil) + networkConfig.Bridge = bridge + + if bridge != nil { + if err := weavenet.DetectHairpin(weavenet.BridgeIfName, Log); err != nil { + Log.Errorf("DetectHairpin failed: %s", err) + } + } if nickName == "" { var err error @@ -266,7 +285,7 @@ func main() { config.TrustedSubnets = parseTrustedSubnets(trustedSubnetStr) config.PeerDiscovery = !noDiscovery - if isAWSVPC && len(config.Password) > 0 { + if bridgeConfig.AWSVPC && len(config.Password) > 0 { Log.Fatalf("--awsvpc mode is not compatible with the --password option") } @@ -312,7 +331,7 @@ func main() { } network := "" - if isAWSVPC { + if bridgeConfig.AWSVPC { network = "awsvpc" } checkForUpdates(dockerVersion, network) @@ -335,16 +354,16 @@ func main() { ) if ipamConfig.Enabled() { var t tracker.LocalRangeTracker - if isAWSVPC { + if bridgeConfig.AWSVPC { Log.Infoln("Creating AWSVPC LocalRangeTracker") - t, err = tracker.NewAWSVPCTracker() + t, err = tracker.NewAWSVPCTracker(bridgeConfig.WeaveBridgeName) if err != nil { Log.Fatalf("Cannot create AWSVPC LocalRangeTracker: %s", err) } trackerName = "awsvpc" } - preClaims, err := findExistingAddresses(dockerCli, weavenet.WeaveBridgeName) + preClaims, err := findExistingAddresses(dockerCli, bridgeConfig.WeaveBridgeName) checkFatal(err) allocator, defaultSubnet = createAllocator(router, ipamConfig, preClaims, db, t, isKnownPeer) @@ -367,6 +386,9 @@ func main() { ns.Start() defer ns.Stop() dnsserver.ActivateAndServe() + if dockerCli != nil { + populateDNS(ns, dockerCli, name, bridgeConfig.WeaveBridgeName) + } defer dnsserver.Stop() } @@ -410,9 +432,25 @@ func main() { go plugin.Start(httpAddr, dockerCli, pluginSocket, pluginMeshSocket, !noDNS, enablePluginV2) } + if bridgeConfig.AWSVPC { + // Run this on its own goroutine because the allocator can block + // We remove the default route installed by the kernel, + // because awsvpc has installed it as well + go expose(allocator, defaultSubnet, bridgeConfig.WeaveBridgeName, bridgeConfig.AWSVPC) + } + signals.SignalHandlerLoop(common.Log, router) } +func expose(alloc *ipam.Allocator, subnet address.CIDR, bridgeName string, removeDefaultRoute bool) { + addr, err := alloc.Allocate("weave:expose", subnet, false, func() bool { return false }) + checkFatal(err) + cidr := address.MakeCIDR(subnet, addr) + err = weavenet.AddBridgeAddr(bridgeName, cidr.IPNet(), removeDefaultRoute) + checkFatal(err) + Log.Printf("Bridge %q exposed on address %v", bridgeName, cidr) +} + func options() map[string]string { options := make(map[string]string) mflag.Visit(func(f *mflag.Flag) { @@ -453,34 +491,32 @@ func (nopPacketLogging) LogPacket(string, weave.PacketKey) { func (nopPacketLogging) LogForwardPacket(string, weave.ForwardPacketKey) { } -func createOverlay(datapathName string, ifaceName string, isAWSVPC bool, host string, port int, bufSzMB int, enableEncryption bool) (weave.NetworkOverlay, weave.Bridge) { +func createOverlay(bridgeType weavenet.Bridge, config weavenet.BridgeConfig, host string, port int, bufSzMB int, enableEncryption bool) (weave.NetworkOverlay, weave.Bridge) { overlay := weave.NewOverlaySwitch() var bridge weave.Bridge var ignoreSleeve bool switch { - case isAWSVPC: + case config.AWSVPC: vpc := weave.NewAWSVPC() overlay.Add("awsvpc", vpc) bridge = weave.NullBridge{} // Currently, we do not support any overlay with AWSVPC ignoreSleeve = true - case datapathName != "" && ifaceName != "": - Log.Fatal("At most one of --datapath and --iface must be specified.") - case datapathName != "": - iface, err := weavenet.EnsureInterface(datapathName) + case bridgeType == nil: + bridge = weave.NullBridge{} + case bridgeType.IsFastdp(): + iface, err := weavenet.EnsureInterface(config.DatapathName) checkFatal(err) fastdp, err := weave.NewFastDatapath(iface, port, enableEncryption) checkFatal(err) bridge = fastdp.Bridge() overlay.Add("fastdp", fastdp.Overlay()) - case ifaceName != "": - iface, err := weavenet.EnsureInterface(ifaceName) + case !bridgeType.IsFastdp(): + iface, err := weavenet.EnsureInterface(weavenet.PcapIfName) checkFatal(err) bridge, err = weave.NewPcap(iface, bufSzMB*1024*1024) // bufsz flag is in MB checkFatal(err) - default: - bridge = weave.NullBridge{} } if !ignoreSleeve { @@ -578,13 +614,15 @@ func determinePassword(password string) []byte { return []byte(password) } -func peerName(routerName string) mesh.PeerName { +func peerName(routerName, bridgeName, dbPrefix, hostRoot string) mesh.PeerName { if routerName == "" { - iface, err := net.InterfaceByName(weavenet.WeaveBridgeName) - if err != nil { - Log.Fatalf("Unable to find bridge %q", weavenet.WeaveBridgeName) + iface, err := net.InterfaceByName(bridgeName) + if err == nil { + routerName = iface.HardwareAddr.String() + } else { + routerName, err = weavenet.GetSystemPeerName(dbPrefix, hostRoot) + checkFatal(err) } - routerName = iface.HardwareAddr.String() } name, err := mesh.PeerNameFromUserInput(routerName) checkFatal(err) diff --git a/prog/weaver/reclaim.go b/prog/weaver/reclaim.go index a90edfe9..88461818 100644 --- a/prog/weaver/reclaim.go +++ b/prog/weaver/reclaim.go @@ -2,13 +2,16 @@ package main import ( "net" + "strings" docker "github.com/fsouza/go-dockerclient" + "github.com/weaveworks/mesh" "github.com/weaveworks/weave/api" "github.com/weaveworks/weave/common" weavedocker "github.com/weaveworks/weave/common/docker" "github.com/weaveworks/weave/ipam" + "github.com/weaveworks/weave/nameserver" weavenet "github.com/weaveworks/weave/net" "github.com/weaveworks/weave/net/address" ) @@ -87,3 +90,49 @@ func findExistingAddresses(dockerCli *weavedocker.Client, bridgeName string) (ad } return addrs, nil } + +func populateDNS(ns *nameserver.Nameserver, dockerCli *weavedocker.Client, ourName mesh.PeerName, bridgeName string) (addrs []ipam.PreClaim, err error) { + // Find all veths connected to the bridge + peerIDs, err := weavenet.ConnectedToBridgeVethPeerIds(bridgeName) + if err != nil { + return nil, err + } + + // Now iterate over all containers to see if they have a network + // namespace with an attached interface + if dockerCli != nil { + containerIDs, err := dockerCli.RunningContainerIDs() + if err != nil { + return nil, err + } + + for _, cid := range containerIDs { + container, err := dockerCli.InspectContainer(cid) + if err != nil { + if _, ok := err.(*docker.NoSuchContainer); ok { + continue + } + return nil, err + } + if container.State.Pid != 0 { + // Docker allows containers to have fqdns in two wauys: + // where the container explicitly has a domain name, and when its + // hostname contains the domain name, recognized by it having dots. + if container.Config.Domainname == "" && !strings.Contains(container.Config.Hostname, ".") { + continue + } + netDevs, err := weavenet.GetNetDevsByVethPeerIds(container.State.Pid, peerIDs) + if err != nil { + return nil, err + } + fqdn := container.Config.Hostname + "." + container.Config.Domainname + for _, netDev := range netDevs { + for _, cidr := range netDev.CIDRs { + ns.AddEntryFQDN(fqdn, cid, ourName, address.FromIP4(cidr.IP)) + } + } + } + } + } + return addrs, nil +} diff --git a/prog/weaveutil/bridge.go b/prog/weaveutil/bridge.go new file mode 100644 index 00000000..a102ea21 --- /dev/null +++ b/prog/weaveutil/bridge.go @@ -0,0 +1,56 @@ +package main + +import ( + "fmt" + "strconv" + + weavenet "github.com/weaveworks/weave/net" +) + +func detectBridgeType(args []string) error { + if len(args) != 2 { + cmdUsage("detect-bridge-type", " ") + } + bridgeType, err := weavenet.ExistingBridgeType(args[0], args[1]) + if err != nil { + return err + } else if bridgeType == nil { + fmt.Println("none") + } else { + fmt.Println(bridgeType.String()) + } + return nil +} + +func createBridge(args []string) error { + if len(args) != 10 { + cmdUsage("create-bridge", " ") + } + + mtu, err := strconv.Atoi(args[3]) + if err != nil && args[3] != "" { + return err + } + port, err := strconv.Atoi(args[4]) + if err != nil { + return err + } + config := weavenet.BridgeConfig{ + DockerBridgeName: args[0], + WeaveBridgeName: args[1], + DatapathName: args[2], + MTU: mtu, + Port: port, + Mac: args[5], + NoFastdp: args[6] != "", + NoBridgedFastdp: args[7] != "", + NPC: args[9] == "--expect-npc", + } + procPath := args[8] + bridgeType, err := weavenet.EnsureBridge(procPath, &config) + if err != nil { + return err + } + fmt.Println(bridgeType.String()) + return nil +} diff --git a/prog/weaveutil/main.go b/prog/weaveutil/main.go index bc954761..b24a6fe5 100644 --- a/prog/weaveutil/main.go +++ b/prog/weaveutil/main.go @@ -17,6 +17,8 @@ func init() { "help": help, "netcheck": netcheck, "docker-tls-args": dockerTLSArgs, + "create-bridge": createBridge, + "detect-bridge-type": detectBridgeType, "create-datapath": createDatapath, "delete-datapath": deleteDatapath, "check-datapath": checkDatapath, diff --git a/prog/weaveutil/netdev.go b/prog/weaveutil/netdev.go index d35e395b..457e30e3 100644 --- a/prog/weaveutil/netdev.go +++ b/prog/weaveutil/netdev.go @@ -60,7 +60,8 @@ func setupIface(args []string) error { if err := netlink.LinkSetName(link, newIfName); err != nil { return err } - if err := weavenet.ConfigureARPCache(newIfName); err != nil { + // This is only called by AttachContainer which is only called in host pid namespace + if err := weavenet.ConfigureARPCache("/proc", newIfName); err != nil { return err } if err := ipt.Append("filter", "INPUT", "-i", newIfName, "-d", "224.0.0.0/4", "-j", "DROP"); err != nil { @@ -71,10 +72,11 @@ func setupIface(args []string) error { } func configureARP(args []string) error { - if len(args) != 1 { - cmdUsage("configure-arp", "") + if len(args) != 2 { + cmdUsage("configure-arp", " ") } prefix := args[0] + rootPath := args[1] links, err := netlink.LinkList() if err != nil { @@ -83,7 +85,7 @@ func configureARP(args []string) error { for _, link := range links { ifName := link.Attrs().Name if strings.HasPrefix(ifName, prefix) { - weavenet.ConfigureARPCache(ifName) + weavenet.ConfigureARPCache(rootPath+"/proc", ifName) if addrs, err := netlink.AddrList(link, netlink.FAMILY_V4); err == nil { for _, addr := range addrs { arping.GratuitousArpOverIfaceByName(addr.IPNet.IP, ifName) diff --git a/proxy/proxy.go b/proxy/proxy.go index e753ae26..40e00968 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -408,11 +408,6 @@ func containerShouldAttach(container *docker.Container) bool { return len(container.Config.Entrypoint) > 0 && container.Config.Entrypoint[0] == weaveWaitEntrypoint[0] } -func containerIsWeaveRouter(container *docker.Container) bool { - return container.Name == weaveContainerName && - len(container.Config.Entrypoint) > 0 && container.Config.Entrypoint[0] == weaveEntrypoint -} - func (proxy *Proxy) createWait(r *http.Request, ident string) { proxy.Lock() proxy.waiters[r] = &wait{ident: ident, ch: make(chan error, 1)} @@ -489,10 +484,6 @@ func (proxy *Proxy) attach(containerID string) error { } return nil } - if containerIsWeaveRouter(container) { - Log.Infof("Attaching weave router container: %s", container.ID) - return callWeaveAttach(container, []string{"attach-router"}) - } if !containerShouldAttach(container) || !container.State.Running { return nil } diff --git a/site/troubleshooting.md b/site/troubleshooting.md index b2630525..9d593eb6 100644 --- a/site/troubleshooting.md +++ b/site/troubleshooting.md @@ -283,14 +283,11 @@ on reboot. This can be disabled via: weave launch --no-restart -Note that the -[Weave Net Docker API Proxy](/site/weave-docker-api.md) -is responsible for reconfiguring the Weave router and re-attaching -application containers to the Weave network at startup, so if you -choose not to run it you must make arrangements for this -reconfiguration to take place. In this scenario, set up your favourite -process manager to run `weave launch-router` every time the machine -reboots. +Note that the Weave Net router will create the `weave` network bridge +if necessary when it restarts, and the [Weave Net Docker API +Proxy](/site/weave-docker-api.md) will re-attach any application +containers that it originally attached to the Weave network when they +restart. ## Snapshot Releases diff --git a/weave b/weave index d11a45c6..be5779cb 100755 --- a/weave +++ b/weave @@ -136,7 +136,7 @@ exec_options() { ;; # All the other commands that may create the bridge and need machine id files. # We don't mount '/' to avoid recursive mounts of '/var' - create-bridge|attach-router|attach|expose|hide) + create-bridge|attach|expose|hide) echo -v /etc:/host/etc -v /var/lib/dbus:/host/var/lib/dbus -e HOST_ROOT=/host ;; esac @@ -281,7 +281,6 @@ DATAPATH=datapath CONTAINER_IFNAME=ethwe BRIDGE_IFNAME=v${CONTAINER_IFNAME}-bridge DATAPATH_IFNAME=v${CONTAINER_IFNAME}-datapath -PCAP_IFNAME=v${CONTAINER_IFNAME}-pcap PORT=${WEAVE_PORT:-6783} HTTP_ADDR=${WEAVE_HTTP_ADDR:-127.0.0.1:6784} STATUS_ADDR=${WEAVE_STATUS_ADDR:-127.0.0.1:6782} @@ -326,21 +325,6 @@ run_iptables() { iptables $IPTABLES_W "$@" } -# Add a rule to iptables, if it doesn't exist already -add_iptables_rule() { - IPTABLES_TABLE="$1" - shift 1 - if ! run_iptables -t $IPTABLES_TABLE -C "$@" >/dev/null 2>&1 ; then - ## Loop until we get an exit code other than "temporarily unavailable" - while true ; do - run_iptables -t $IPTABLES_TABLE -A "$@" >/dev/null && return 0 - if [ $? != 4 ] ; then - return 1 - fi - done - fi -} - # Insert a rule in iptables, if it doesn't exist already insert_iptables_rule() { IPTABLES_TABLE="$1" @@ -365,15 +349,6 @@ delete_iptables_rule() { fi } -# Configure the ARP cache parameters for the given interface. This -# makes containers react more quickly to a change in the MAC address -# associated with an IP address. -configure_arp_cache() { - $2 sh -c "echo 5 >$PROC_PATH/sys/net/ipv4/neigh/$1/base_reachable_time && - echo 2 >$PROC_PATH/sys/net/ipv4/neigh/$1/delay_first_probe_time && - echo 1 >$PROC_PATH/sys/net/ipv4/neigh/$1/ucast_solicit" -} - # Send out an ARP announcement # (https://tools.ietf.org/html/rfc5227#page-15) to update ARP cache # entries across the weave network. We do this in addition to @@ -388,20 +363,6 @@ arp_update() { ! command_exists arping || $3 arping -U -q -I $1 -c 1 ${2%/*} } -# Generate a MAC value from a stdin containing six space separated -# 2-digit hexadecimal numbers. -mac_from_hex() { - # In the first byte of the MAC, the 'multicast' bit should be - # clear and 'locally administered' bit should be set. All other - # bits should be random. - read a b c d e f && printf "%02x:$b:$c:$d:$e:$f" $((0x$a & ~1 | 2)) -} - -# Generate a random MAC value -random_mac() { - od -txC -An -N6 /dev/urandom | mac_from_hex -} - ###################################################################### # weave and docker specific helpers ###################################################################### @@ -427,51 +388,23 @@ EOF fi } -enforce_docker_bridge_addr_assign_type() { - if ! ADDR_ASSIGN_TYPE=$(cat /sys/class/net/$DOCKER_BRIDGE/addr_assign_type 2>/dev/null) ; then - echo "Could not determine address assignment type of $DOCKER_BRIDGE" >&2 - return - fi - # From include/uapi/linux/netdevice.h - # #define NET_ADDR_PERM 0 /* address is permanent (default) */ - # #define NET_ADDR_RANDOM 1 /* address is generated randomly */ - # #define NET_ADDR_STOLEN 2 /* address is stolen from other device */ - # #define NET_ADDR_SET 3 /* address is set using dev_set_mac_address() */ - if [ $ADDR_ASSIGN_TYPE != 3 ] ; then - echo "Setting $DOCKER_BRIDGE MAC (mitigate https://github.com/docker/docker/issues/14908)" >&2 - ip link set dev $DOCKER_BRIDGE address $(random_mac) || true - fi -} - # Detect the current bridge/datapath state. When invoked, the values of # $BRIDGE and $DATAPATH are expected to be distinct. $BRIDGE_TYPE and # $DATAPATH are set correctly on success; failure indicates that the # bridge/datapath devices have yet to be configured. If netdevs do exist # but are in an inconsistent state the script aborts with an error. detect_bridge_type() { - BRIDGE_TYPE= - if [ -d /sys/class/net/$DATAPATH ] ; then - # Unfortunately there's no simple way to positively check whether - # $DATAPATH is an ODP netdev so we have to make sure it isn't - # a bridge instead (and that $BRIDGE is). - if [ ! -d /sys/class/net/$DATAPATH/bridge -a -d /sys/class/net/$BRIDGE/bridge ] ; then - BRIDGE_TYPE=bridged_fastdp - else - echo "Inconsistent bridge state detected. Please do 'weave reset' and try again." >&2 - exit 1 - fi - elif [ -d /sys/class/net/$BRIDGE ] ; then - if [ -d /sys/class/net/$BRIDGE/bridge ] ; then - BRIDGE_TYPE=bridge - else - BRIDGE_TYPE=fastdp - # The datapath is the bridge when there is no intermediary + BRIDGE_TYPE=$(util_op detect-bridge-type "$BRIDGE" "$DATAPATH") + case "$BRIDGE_TYPE" in + bridge|bridged_fastdp) + ;; + fastdp) DATAPATH="$BRIDGE" - fi - else - # No bridge/datapath devices configured - return 1 - fi + ;; + *) + return 1 + ;; + esac # WEAVE_MTU may have been specified when the bridge was # created (perhaps implicitly with WEAVE_NO_FASTDP). So take @@ -481,95 +414,21 @@ detect_bridge_type() { } try_create_bridge() { - if ! detect_bridge_type ; then - BRIDGE_TYPE=bridge - if [ -z "$WEAVE_NO_FASTDP" ] ; then - BRIDGE_TYPE=bridged_fastdp - if [ -n "$WEAVE_NO_BRIDGED_FASTDP" ] ; then - BRIDGE_TYPE=fastdp - # The datapath is the bridge when there is no intermediary - DATAPATH="$BRIDGE" - fi - if util_op create-datapath $DATAPATH ; then - : # ODP datapath created successfully - elif [ $? = 17 ] ; then - # Exit status of 17 means the kernel doesn't have ODP - BRIDGE_TYPE=bridge - else - return 1 - fi - fi - - init_$BRIDGE_TYPE - - # Drop traffic from Docker bridge to Weave; it can break - # subnet isolation - if [ "$DOCKER_BRIDGE" != "$BRIDGE" ] ; then - # Note using -I to insert ahead of Docker's bridge rules - run_iptables -t filter -I FORWARD -i $DOCKER_BRIDGE -o $BRIDGE -j DROP - fi - - [ -n "$DOCKER_BRIDGE_IP" ] || DOCKER_BRIDGE_IP=$(util_op bridge-ip $DOCKER_BRIDGE) - - # forbid traffic to the Weave port from other containers - add_iptables_rule filter INPUT -i $DOCKER_BRIDGE -p tcp --dst $DOCKER_BRIDGE_IP --dport $PORT -j DROP - add_iptables_rule filter INPUT -i $DOCKER_BRIDGE -p udp --dst $DOCKER_BRIDGE_IP --dport $PORT -j DROP - add_iptables_rule filter INPUT -i $DOCKER_BRIDGE -p udp --dst $DOCKER_BRIDGE_IP --dport $(($PORT + 1)) -j DROP - - # let DNS traffic to weaveDNS, since otherwise it might get blocked by the likes of UFW - add_iptables_rule filter INPUT -i $DOCKER_BRIDGE -p udp --dport 53 -j ACCEPT - add_iptables_rule filter INPUT -i $DOCKER_BRIDGE -p tcp --dport 53 -j ACCEPT - - if [ "$2" = "--expect-npc" ] ; then # matches usage in weave-kube launch.sh - # Steer traffic via the NPC - run_iptables -N WEAVE-NPC >/dev/null 2>&1 || true - add_iptables_rule filter FORWARD -o $BRIDGE -j WEAVE-NPC - add_iptables_rule filter FORWARD -o $BRIDGE -m state --state NEW -j NFLOG --nflog-group 86 - add_iptables_rule filter FORWARD -o $BRIDGE -j DROP - else - # Work around the situation where there are no rules allowing traffic - # across our bridge. E.g. ufw - add_iptables_rule filter FORWARD -i $BRIDGE -o $BRIDGE -j ACCEPT - fi - # Forward from weave to the rest of the world - add_iptables_rule filter FORWARD -i $BRIDGE ! -o $BRIDGE -j ACCEPT - # and allow replies back - add_iptables_rule filter FORWARD -o $BRIDGE -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - - # create a chain for masquerading - run_iptables -t nat -N WEAVE >/dev/null 2>&1 || true - add_iptables_rule nat POSTROUTING -j WEAVE - else - if [ -n "$LAUNCHING_ROUTER" ] ; then - if [ "$BRIDGE_TYPE" = bridge -a -z "$WEAVE_NO_FASTDP" ] && - util_op check-datapath $DATAPATH 2>/dev/null ; then - cat <&2 -WEAVE_NO_FASTDP is not set, but there is already a bridge present of -the wrong type for fast datapath. Please do 'weave reset' to remove -the bridge first. -EOF - return 1 - fi - if [ "$BRIDGE_TYPE" != bridge -a -n "$WEAVE_NO_FASTDP" ] ; then - cat <&2 -WEAVE_NO_FASTDP is set, but there is already a weave fast datapath -bridge present. Please do 'weave reset' to remove the bridge first. -EOF - return 1 - fi - fi + # Running this from 'weave --local' when weaveexec is not on the + # path will run it as a Docker container that does not have access + # to /etc/machine-id, so will not give the full range of persistent IDs + if [ -z "$WEAVEDB_DIR_PATH" ]; then + WEAVEDB_DIR_PATH="$HOST_ROOT/$(docker inspect -f '{{with index .Mounts 0}}{{.Source}}{{end}}' $DB_CONTAINER_NAME 2>/dev/null)" fi - - [ "$1" = "--without-ethtool" -o -n "$AWSVPC" ] || ethtool_tx_off_$BRIDGE_TYPE $BRIDGE - - ip link set dev $BRIDGE up - - # Configure the ARP cache parameters on the bridge interface for - # the sake of 'weave expose' - configure_arp_cache $BRIDGE + MAC=$(util_op unique-id "$WEAVEDB_DIR_PATH/weave" "$HOST_ROOT") + BRIDGE_TYPE=$(util_op create-bridge "$DOCKER_BRIDGE" "$BRIDGE" "$DATAPATH" "$WEAVE_MTU" "$PORT" "$MAC" "$WEAVE_NO_FASTDP" "$WEAVE_NO_BRIDGED_FASTDP" "$PROC_PATH" "$1") + # Set some variables that are expected by code that comes later + [ "$BRIDGE_TYPE" != "fastdp" ] || DATAPATH="$BRIDGE" + MTU=${WEAVE_MTU:-$(cat /sys/class/net/$BRIDGE/mtu)} } create_bridge() { + validate_bridge_type if ! try_create_bridge "$@" ; then echo "Creating bridge '$BRIDGE' failed" >&2 # reset to original value so we destroy both kinds @@ -579,18 +438,35 @@ create_bridge() { fi } +validate_bridge_type() { + detect_bridge_type || return 0 + + if [ -n "$LAUNCHING_ROUTER" ] ; then + if [ "$BRIDGE_TYPE" = bridge -a -z "$WEAVE_NO_FASTDP" ] && + util_op check-datapath 2>/dev/null ; then + cat <&2 +WEAVE_NO_FASTDP is not set, but there is already a bridge present of +the wrong type for fast datapath. Please do 'weave reset' to remove +the bridge first. +EOF + return 1 + fi + if [ "$BRIDGE_TYPE" != bridge -a -n "$WEAVE_NO_FASTDP" ] ; then + cat <&2 +WEAVE_NO_FASTDP is set, but there is already a weave fast datapath +bridge present. Please do 'weave reset' to remove the bridge first. +EOF + return 1 + fi + fi +} + expose_ip() { ipam_cidrs allocate_no_check_alive weave:expose $CIDR_ARGS for CIDR in $ALL_CIDRS ; do if ! ip addr show dev $BRIDGE | grep -qF $CIDR ; then ip addr add dev $BRIDGE $CIDR arp_update $BRIDGE $CIDR || true - # Remove a default route installed by the kernel, because awsvpc - # has installed it as well - if [ -n "$AWSVPC" ]; then - RCIDR=$(ip route list exact $CIDR proto kernel | head -n1 | cut -d' ' -f1) - [ -n "$RCIDR" ] && ip route del dev $BRIDGE proto kernel $RCIDR - fi fi [ -z "$FQDN" ] || when_weave_running put_dns_fqdn_no_check_alive weave:expose $FQDN $CIDR done @@ -614,80 +490,6 @@ create_veth() { fi } -init_fastdp() { - # GCE has the lowest underlay network MTU we're likely to encounter on - # a local network, at 1460 bytes. To get the overlay MTU from that we - # subtract 20 bytes for the outer IPv4 header, 8 bytes for the outer - # UDP header, 8 bytes for the vxlan header, and 14 bytes for the inner - # ethernet header. In addition, we subtract 34 bytes for the ESP overhead - # which is needed for the vxlan encryption. - MTU=${WEAVE_MTU:-1376} - - # create_bridge already created the datapath netdev - ip link set dev $DATAPATH mtu $MTU -} - -init_bridge_prep() { - # Observe any MTU that is already set - [ -n "$MTU" ] || MTU=${WEAVE_MTU:-65535} - - ip link add name $BRIDGE type bridge - - # Running this from 'weave --local' when weaveexec is not on the - # path will run it as a Docker container that does not have access - # to /etc/machine-id, so will not give the full range of persistent IDs - if [ -z "$WEAVEDB_DIR_PATH" ]; then - WEAVEDB_DIR_PATH="$HOST_ROOT/$(docker inspect -f '{{with index .Mounts 0}}{{.Source}}{{end}}' $DB_CONTAINER_NAME 2>/dev/null)" - fi - MAC=$(util_op unique-id "$WEAVEDB_DIR_PATH/weave" "$HOST_ROOT") - - ip link set dev $BRIDGE address $MAC - - # Attempting to set the bridge MTU to a high value directly - # fails. Bridges take the lowest MTU of their interfaces. So - # instead we create a temporary interface with the desired - # MTU, attach that to the bridge, and then remove it again. - ip link add name v${CONTAINER_IFNAME}du mtu $MTU type dummy - ip link set dev v${CONTAINER_IFNAME}du master $BRIDGE - ip link del dev v${CONTAINER_IFNAME}du -} - -init_bridge() { - init_bridge_prep - create_veth $BRIDGE_IFNAME $PCAP_IFNAME add_iface_bridge $BRIDGE_IFNAME -} - -init_bridged_fastdp() { - # Initialise the datapath as normal. NB sets MTU for use below - init_fastdp - - # Initialise the bridge using fast datapath MTU - init_bridge_prep - - # Create linking veth pair - create_veth $BRIDGE_IFNAME $DATAPATH_IFNAME configure_veth_bridged_fastdp - - # Finally, bring the datapath up - ip link set dev $DATAPATH up -} - -configure_veth_bridged_fastdp() { - add_iface_fastdp $DATAPATH_IFNAME || return 1 - add_iface_bridge $BRIDGE_IFNAME || return 1 -} - -ethtool_tx_off_fastdp() { - true -} - -ethtool_tx_off_bridge() { - ethtool -K $1 tx off >/dev/null -} - -ethtool_tx_off_bridged_fastdp() { - true -} - destroy_bridge() { # It's important that detect_bridge_type has not been called so # we have distinct values for $BRIDGE and $DATAPATH. Make best efforts @@ -760,16 +562,10 @@ configure_veth_attached_bridge() { ip link set $GUEST_IFNAME master $bridge } -router_opts_fastdp() { - echo "--datapath $DATAPATH" -} - -router_opts_bridge() { - echo "--iface $PCAP_IFNAME" -} - -router_opts_bridged_fastdp() { - router_opts_fastdp "$@" +router_bridge_opts() { + echo --docker-bridge "$DOCKER_BRIDGE" --weave-bridge "$BRIDGE" --datapath "$DATAPATH" + [ -z "$WEAVE_MTU" ] || echo --mtu "$WEAVE_MTU" + [ -z $WEAVE_NO_FASTDP ] || echo --no-fastdp } ask_version() { @@ -1468,8 +1264,6 @@ common_launch_args() { launch_router() { LAUNCHING_ROUTER=1 check_forwarding_rules - enforce_docker_bridge_addr_assign_type - # backward compatibility... if is_cidr "$1" ; then echo "WARNING: $1 parameter ignored; 'weave launch' no longer takes a CIDR as the first parameter" >&2 @@ -1539,11 +1333,7 @@ launch_router() { eval "set -- $ARGS" setup_cni - create_bridge - # We set the router name to the bridge MAC, which in turn is - # derived from the system UUID (if available), and thus stable - # across reboots. - PEERNAME=$(cat /sys/class/net/$BRIDGE/address) + validate_bridge_type if [ -z "$IPRANGE_SPECIFIED" ] ; then IPRANGE="10.32.0.0/12" @@ -1593,11 +1383,13 @@ launch_router() { --volumes-from $DB_CONTAINER_NAME \ -v $RESOLV_CONF_DIR:/var/run/weave/etc \ -v /run/docker/plugins:/run/docker/plugins \ + -v /etc:/host/etc -v /var/lib/dbus:/host/var/lib/dbus \ -e WEAVE_PASSWORD \ -e CHECKPOINT_DISABLE \ $WEAVE_DOCKER_ARGS $IMAGE $COVERAGE_ARGS \ - --port $CONTAINER_PORT --name "$PEERNAME" --nickname "$(hostname)" \ - $(router_opts_$BRIDGE_TYPE) \ + --port $CONTAINER_PORT --nickname "$(hostname)" \ + --host-root=/host \ + $(router_bridge_opts) \ --ipalloc-range "$IPRANGE" \ --dns-effective-listen-address $DOCKER_BRIDGE_IP \ $DNS_ROUTER_OPTS $NO_DNS_OPT \ @@ -1608,8 +1400,6 @@ launch_router() { --resolv-conf "/var/run/weave/etc/$RESOLV_CONF_BASE" \ "$@") wait_for_status $CONTAINER_NAME http_call $HTTP_ADDR - setup_awsvpc - populate_router if [ -n "$ENABLE_PLUGIN" ]; then wait_for_status $CONTAINER_NAME http_call_unix $CONTAINER_NAME plugin-status.sock @@ -1617,37 +1407,6 @@ launch_router() { fi } -setup_awsvpc() { - if [ -n "$AWSVPC" ]; then - # Set proxy_arp on the bridge, so that it could accept packets destined - # to containers within the same subnet but running on remote hosts. - # Without it, exact routes on each container are required. - echo 1 >$PROC_PATH/sys/net/ipv4/conf/$BRIDGE/proxy_arp - # Avoid delaying the first ARP request. Also, setting it to 0 avoids - # placing the request into a bounded queue as it can be seen: - # https://git.kernel.org/cgit/linux/kernel/git/stable/linux-stable.git/tree/net/ipv4/arp.c?id=refs/tags/v4.6.1#n819 - echo 0 >$PROC_PATH/sys/net/ipv4/neigh/$BRIDGE/proxy_delay - expose_ip - fi -} - -# Recreate the parameter values that are set when the router is first launched -fetch_router_args() { - CONTAINER_ARGS=$(docker inspect -f '{{.Args}}' $CONTAINER_NAME) || return 1 - NO_DNS_OPT=$(echo $CONTAINER_ARGS | grep -o -e '--no-dns') || true -} - -populate_dns_for_one_container() { - [ -z "$4" ] || with_container_fqdn $1 put_dns_fqdn $4 -} - -populate_router() { - if [ -z "$NO_DNS_OPT" ] ; then - # Tell the newly-started weaveDNS about existing weave IPs - with_container_addresses populate_dns_for_one_container weave:allids - fi -} - stop_router() { ! is_plugin_v1_enabled || util_op remove-plugin-network weave || true stop $CONTAINER_NAME "Weave" @@ -1753,17 +1512,6 @@ if ! command_exists ip ; then exit 1 fi -if ! ip netns list >/dev/null 2>&1 ; then - echo "ERROR: $(ip -V) does not support network namespaces." >&2 - echo " Please install iproute2-ss111010 or later." >&2 - exit 1 -fi - -if ! command_exists nsenter ; then - echo "ERROR: nsenter utility missing. Please install it." >&2 - exit 1 -fi - [ $# -gt 0 ] || usage COMMAND=$1 shift 1 @@ -1788,22 +1536,6 @@ case "$COMMAND" in # intentionally undocumented since it assumes knowledge of weave # internals create-bridge) - while [ $# -gt 0 ]; do - case "$1" in - --proc-path=*) - PROC_PATH="${1#*=}" - shift - ;; - --weavedb-dir-path=*) - WEAVEDB_DIR_PATH="${1#*=}" - shift - ;; - *) - break - ;; - esac - done - if [ "$1" != "--force" ] ; then cat 1>&2 <