From ade67387896c07ceddad50d331fed1d1f70bfb47 Mon Sep 17 00:00:00 2001 From: Martynas Pumputis Date: Fri, 18 Aug 2017 13:54:57 +0200 Subject: [PATCH] Handle "weave expose" in weaver The change makes "weave expose" to call the "/expose" HTTP endpoint implemented in router which exposes the bridge. Such architectural change required as we need to know whether NPC is enabled when exposing the bridge and weaver knows this fact the best. Unfortunately, this change makes impossible to "weave expose" when weaver is not running. --- api/router.go | 12 +++++ net/bridge.go | 33 ------------ net/expose.go | 106 +++++++++++++++++++++++++++++++++++++++ net/veth.go | 34 ------------- plugin/net/cni.go | 26 ++-------- prog/weaver/main.go | 8 +-- prog/weaveutil/expose.go | 23 --------- prog/weaveutil/main.go | 1 - router/http.go | 19 +++++++ router/network_router.go | 6 ++- test/130_expose_test.sh | 7 --- weave | 7 +-- 12 files changed, 151 insertions(+), 131 deletions(-) create mode 100644 api/router.go create mode 100644 net/expose.go delete mode 100644 prog/weaveutil/expose.go diff --git a/api/router.go b/api/router.go new file mode 100644 index 00000000..555ee5f8 --- /dev/null +++ b/api/router.go @@ -0,0 +1,12 @@ +package api + +import ( + "fmt" + "net" +) + +// Expose calls the router to assign the given IP addr to the weave bridge. +func (client *Client) Expose(ipAddr *net.IPNet) error { + _, err := client.httpVerb("POST", fmt.Sprintf("/expose/%s", ipAddr), nil) + return err +} diff --git a/net/bridge.go b/net/bridge.go index 54fd70bd..713ecf6b 100644 --- a/net/bridge.go +++ b/net/bridge.go @@ -8,7 +8,6 @@ import ( "github.com/Sirupsen/logrus" "github.com/coreos/go-iptables/iptables" - "github.com/j-keck/arping" "github.com/pkg/errors" "github.com/vishvananda/netlink" @@ -479,35 +478,3 @@ func linkSetUpByName(linkName string) error { } return netlink.LinkSetUp(link) } - -func AddBridgeAddr(bridgeName string, addr *net.IPNet, removeDefaultRoute bool) error { - link, err := netlink.LinkByName(bridgeName) - if err != nil { - return errors.Wrapf(err, "AddBridgeAddr finding bridge %q", bridgeName) - } - newAddresses, err := AddAddresses(link, []*net.IPNet{addr}) - if err != nil { - return errors.Wrapf(err, "adding address %v to %q", addr, bridgeName) - } - for _, ipnet := range newAddresses { - arping.GratuitousArpOverIfaceByName(ipnet.IP, bridgeName) - } - if removeDefaultRoute { - routeFilter := &netlink.Route{ - LinkIndex: link.Attrs().Index, - Dst: &net.IPNet{IP: addr.IP.Mask(addr.Mask), Mask: addr.Mask}, - Protocol: 2, // RTPROT_KERNEL - } - filterMask := netlink.RT_FILTER_OIF | netlink.RT_FILTER_DST | netlink.RT_FILTER_PROTOCOL - routes, err := netlink.RouteListFiltered(netlink.FAMILY_V4, routeFilter, filterMask) - if err != nil { - return errors.Wrapf(err, "failed to get route list for bridge %q", bridgeName) - } - for _, r := range routes { - if err = netlink.RouteDel(&r); err != nil { - return errors.Wrapf(err, "failed to delete default route %+v", r) - } - } - } - return nil -} diff --git a/net/expose.go b/net/expose.go new file mode 100644 index 00000000..7b7454f9 --- /dev/null +++ b/net/expose.go @@ -0,0 +1,106 @@ +package net + +import ( + "net" + "syscall" + + "github.com/coreos/go-iptables/iptables" + "github.com/j-keck/arping" + "github.com/pkg/errors" + "github.com/vishvananda/netlink" +) + +// Expose makes the network accessible from a host by assigning a given IP address +// to the weave bridge. +// +// List of params: +// * "bridgeName" - a name of the weave bridge. +// * "ipAddr" - IP addr to be assigned to the bridge. +// * "removeDefaultRoute" - whether to remove a default route installed by the kernel (used only in the AWSVPC mode). +// * "npc" - whether is Weave NPC running. +func Expose(bridgeName string, ipAddr *net.IPNet, removeDefaultRoute, npc bool) error { + if err := addBridgeIPAddr(bridgeName, ipAddr, removeDefaultRoute); err != nil { + return errors.Wrap(err, "addBridgeIPAddr") + } + + if err := exposeNAT(ipAddr); err != nil { + return errors.Wrap(err, "exposeNAT") + } + + if !npc { + // TODO comment why not in npc mode && add filter rules and docs + } + + return nil +} + +func addBridgeIPAddr(bridgeName string, addr *net.IPNet, removeDefaultRoute bool) error { + link, err := netlink.LinkByName(bridgeName) + if err != nil { + return errors.Wrapf(err, "addBridgeIPAddr finding bridge %q", bridgeName) + } + err = netlink.AddrAdd(link, &netlink.Addr{IPNet: addr}) + // The IP addr might have been already set by a concurrent request, just ignore then + if err != nil && err != syscall.Errno(syscall.EEXIST) { + return errors.Wrapf(err, "adding address %v to %q", addr, bridgeName) + } + + // Sending multiple ARP REQUESTs in the case of EEXIST above does not hurt + arping.GratuitousArpOverIfaceByName(addr.IP, bridgeName) + + // Remove a default route installed by the kernel. Required by the AWSVPC mode. + if removeDefaultRoute { + routeFilter := &netlink.Route{ + LinkIndex: link.Attrs().Index, + Dst: &net.IPNet{IP: addr.IP.Mask(addr.Mask), Mask: addr.Mask}, + Protocol: 2, // RTPROT_KERNEL + } + filterMask := netlink.RT_FILTER_OIF | netlink.RT_FILTER_DST | netlink.RT_FILTER_PROTOCOL + routes, err := netlink.RouteListFiltered(netlink.FAMILY_V4, routeFilter, filterMask) + if err != nil { + return errors.Wrapf(err, "failed to get route list for bridge %q", bridgeName) + } + for _, r := range routes { + err = netlink.RouteDel(&r) + // Again, there might be a concurrent request for removing routes + if err != nil && err != syscall.Errno(syscall.ESRCH) { + return errors.Wrapf(err, "failed to delete default route %+v", r) + } + } + } + return nil +} + +func exposeNAT(ipnet *net.IPNet) error { + ipt, err := iptables.New() + if err != nil { + return err + } + cidr := ipnet.String() + + if err := addNatRule(ipt, "-s", cidr, "-d", "224.0.0.0/4", "-j", "RETURN"); err != nil { + return err + } + if err := addNatRule(ipt, "-d", cidr, "!", "-s", cidr, "-j", "MASQUERADE"); err != nil { + return err + } + return addNatRule(ipt, "-s", cidr, "!", "-d", cidr, "-j", "MASQUERADE") +} + +func addNatRule(ipt *iptables.IPTables, rulespec ...string) error { + // Loop until we get an exit code other than "temporarily unavailable" + for { + if err := ipt.AppendUnique("nat", "WEAVE", rulespec...); err != nil { + if ierr, ok := err.(*iptables.Error); ok { + if status, ok := ierr.ExitError.Sys().(syscall.WaitStatus); ok { + // (magic exit code 4 found in iptables source code; undocumented) + if status.ExitStatus() == 4 { + continue + } + } + } + return err + } + return nil + } +} diff --git a/net/veth.go b/net/veth.go index 9a336ab5..f7f44c2a 100644 --- a/net/veth.go +++ b/net/veth.go @@ -4,7 +4,6 @@ import ( "fmt" "net" "strings" - "syscall" "time" "github.com/coreos/go-iptables/iptables" @@ -297,36 +296,3 @@ func subnets(addrs []netlink.Addr) map[string]struct{} { } return subnets } - -func addNatRule(ipt *iptables.IPTables, rulespec ...string) error { - // Loop until we get an exit code other than "temporarily unavailable" - for { - if err := ipt.AppendUnique("nat", "WEAVE", rulespec...); err != nil { - if ierr, ok := err.(*iptables.Error); ok { - if status, ok := ierr.ExitError.Sys().(syscall.WaitStatus); ok { - // (magic exit code 4 found in iptables source code; undocumented) - if status.ExitStatus() == 4 { - continue - } - } - } - return err - } - return nil - } -} - -func ExposeNAT(ipnet net.IPNet) error { - ipt, err := iptables.New() - if err != nil { - return err - } - cidr := ipnet.String() - if err := addNatRule(ipt, "-s", cidr, "-d", "224.0.0.0/4", "-j", "RETURN"); err != nil { - return err - } - if err := addNatRule(ipt, "-d", cidr, "!", "-s", cidr, "-j", "MASQUERADE"); err != nil { - return err - } - return addNatRule(ipt, "-s", cidr, "!", "-d", cidr, "-j", "MASQUERADE") -} diff --git a/plugin/net/cni.go b/plugin/net/cni.go index bfc4f45e..06df7e3a 100644 --- a/plugin/net/cni.go +++ b/plugin/net/cni.go @@ -6,7 +6,6 @@ import ( "fmt" "net" "os" - "syscall" "github.com/containernetworking/cni/pkg/ipam" "github.com/containernetworking/cni/pkg/skel" @@ -100,12 +99,11 @@ func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error { return fmt.Errorf("unable to allocate IP address for bridge: %s", err) } bridgeCIDR := bridgeIPResult.IPs[0].Address - if err := assignBridgeIP(conf.BrName, bridgeCIDR); err != nil { - return fmt.Errorf("unable to assign IP address to bridge: %s", err) - } - if err := weavenet.ExposeNAT(bridgeCIDR); err != nil { - return fmt.Errorf("unable to create NAT rules: %s", err) + + if err := c.weave.Expose(&bridgeCIDR); err != nil { + return fmt.Errorf("unable to expose bridge %q: %s", conf.BrName, err) } + bridgeIP = bridgeCIDR.IP } else if err != nil { return err @@ -167,22 +165,6 @@ func setupRoutes(link netlink.Link, name string, ipnet net.IPNet, gw net.IP, rou return nil } -func assignBridgeIP(bridgeName string, ipnet net.IPNet) error { - link, err := netlink.LinkByName(bridgeName) - if err != nil { - return err - } - if err := netlink.AddrAdd(link, &netlink.Addr{IPNet: &ipnet}); err != nil { - // Treat as non-error if this address is already there - // - maybe another copy of this program just added it - if err == syscall.Errno(syscall.EEXIST) { - return nil - } - return fmt.Errorf("failed to add IP address to %q: %v", bridgeName, err) - } - return nil -} - // As of CNI 0.5 spec: // "Plugins should generally complete a DEL action without error even if some resources are missing" // this method should therefore return nil in most, if not all, cases. diff --git a/prog/weaver/main.go b/prog/weaver/main.go index 0bdb6c3a..321917fb 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -322,7 +322,7 @@ func main() { checkFatal(err) defer db.Close() - router, err := weave.NewNetworkRouter(config, networkConfig, name, nickName, overlay, db) + router, err := weave.NewNetworkRouter(config, networkConfig, bridgeConfig, name, nickName, overlay, db) checkFatal(err) Log.Println("Our name is", router.Ourself) @@ -469,17 +469,17 @@ func main() { // Run this on its own goroutine because the allocator can block // We remove the default route installed by the kernel, // because awsvpc has installed it as well - go expose(allocator, defaultSubnet, bridgeConfig.WeaveBridgeName, bridgeConfig.AWSVPC, waitReady.Add()) + go exposeForAWSVPC(allocator, defaultSubnet, bridgeConfig.WeaveBridgeName, waitReady.Add()) } signals.SignalHandlerLoop(common.Log, router) } -func expose(alloc *ipam.Allocator, subnet address.CIDR, bridgeName string, removeDefaultRoute bool, ready func()) { +func exposeForAWSVPC(alloc *ipam.Allocator, subnet address.CIDR, bridgeName string, ready func()) { addr, err := alloc.Allocate("weave:expose", subnet, false, func() bool { return false }) checkFatal(err) cidr := address.MakeCIDR(subnet, addr) - err = weavenet.AddBridgeAddr(bridgeName, cidr.IPNet(), removeDefaultRoute) + err = weavenet.Expose(bridgeName, cidr.IPNet(), true, false) checkFatal(err) Log.Printf("Bridge %q exposed on address %v", bridgeName, cidr) ready() diff --git a/prog/weaveutil/expose.go b/prog/weaveutil/expose.go deleted file mode 100644 index 758285a7..00000000 --- a/prog/weaveutil/expose.go +++ /dev/null @@ -1,23 +0,0 @@ -package main - -import ( - weavenet "github.com/weaveworks/weave/net" -) - -func exposeNAT(args []string) error { - if len(args) < 1 { - cmdUsage("expose-nat", "...") - } - - cidrs, err := parseCIDRs(args) - if err != nil { - return err - } - - for _, cidr := range cidrs { - if err := weavenet.ExposeNAT(*cidr); err != nil { - return err - } - } - return nil -} diff --git a/prog/weaveutil/main.go b/prog/weaveutil/main.go index 4efabb39..065208b1 100644 --- a/prog/weaveutil/main.go +++ b/prog/weaveutil/main.go @@ -45,7 +45,6 @@ func init() { "list-netdevs": listNetDevs, "cni-net": cniNet, "cni-ipam": cniIPAM, - "expose-nat": exposeNAT, "bridge-ip": bridgeIP, "unique-id": uniqueID, "swarm-manager-peers": swarmManagerPeers, diff --git a/router/http.go b/router/http.go index 90079ba7..88e66ca2 100644 --- a/router/http.go +++ b/router/http.go @@ -5,7 +5,10 @@ import ( "net/http" "github.com/gorilla/mux" + "github.com/weaveworks/weave/common" + "github.com/weaveworks/weave/net" + "github.com/weaveworks/weave/net/address" ) func (router *NetworkRouter) HandleHTTP(muxRouter *mux.Router) { @@ -26,4 +29,20 @@ func (router *NetworkRouter) HandleHTTP(muxRouter *mux.Router) { router.ForgetConnections(r.Form["peer"]) }) + muxRouter.Methods("POST").Path("/expose/{ip}/{prefixlen}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + cidr, err := address.ParseCIDR(vars["ip"] + "/" + vars["prefixlen"]) + if err != nil { + http.Error(w, fmt.Sprint("unable to parse ip addr: ", err.Error()), http.StatusBadRequest) + return + } + + if err = net.Expose(router.BridgeConfig.WeaveBridgeName, cidr.IPNet(), router.BridgeConfig.AWSVPC, router.BridgeConfig.NPC); err != nil { + http.Error(w, fmt.Sprint("unable to expose: ", err.Error()), http.StatusInternalServerError) + return + } + + w.WriteHeader(204) + }) + } diff --git a/router/network_router.go b/router/network_router.go index 6825e584..cfb3b948 100644 --- a/router/network_router.go +++ b/router/network_router.go @@ -11,6 +11,7 @@ import ( "github.com/weaveworks/weave/common" "github.com/weaveworks/weave/db" + weavenet "github.com/weaveworks/weave/net" ) const ( @@ -47,11 +48,12 @@ type PacketLogging interface { type NetworkRouter struct { *mesh.Router NetworkConfig + weavenet.BridgeConfig Macs *MacCache db db.DB } -func NewNetworkRouter(config mesh.Config, networkConfig NetworkConfig, name mesh.PeerName, nickName string, overlay NetworkOverlay, db db.DB) (*NetworkRouter, error) { +func NewNetworkRouter(config mesh.Config, networkConfig NetworkConfig, bridgeConfig weavenet.BridgeConfig, name mesh.PeerName, nickName string, overlay NetworkOverlay, db db.DB) (*NetworkRouter, error) { if overlay == nil { overlay = NullNetworkOverlay{} } @@ -63,7 +65,7 @@ func NewNetworkRouter(config mesh.Config, networkConfig NetworkConfig, name mesh if err != nil { return nil, err } - router := &NetworkRouter{Router: meshRouter, NetworkConfig: networkConfig, db: db} + router := &NetworkRouter{Router: meshRouter, NetworkConfig: networkConfig, BridgeConfig: bridgeConfig, db: db} router.Peers.OnInvalidateShortIDs(overlay.InvalidateShortIDs) router.Routes.OnChange(overlay.InvalidateRoutes) router.Macs = NewMacCache(macMaxAge, diff --git a/test/130_expose_test.sh b/test/130_expose_test.sh index ccb00791..b2d1c6fe 100755 --- a/test/130_expose_test.sh +++ b/test/130_expose_test.sh @@ -33,13 +33,6 @@ check_container_connectivity() { start_suite "exposing weave network to host" -# expose/hide with a CIDR work prior to launching weave -run_on1 "! $PING $EXP" -weave_on1 "expose $EXP/24" -run_on1 " $PING $EXP" -weave_on1 "hide $EXP/24" -run_on1 "! $PING $EXP" - weave_on $HOST1 launch --ipalloc-range $UNIVERSE start_container $HOST1 $C1/24 --name=c1 diff --git a/weave b/weave index 5a75c4fc..238668de 100755 --- a/weave +++ b/weave @@ -443,10 +443,8 @@ EOF expose_ip() { ipam_cidrs allocate_no_check_alive weave:expose $CIDR_ARGS for CIDR in $ALL_CIDRS ; do - if ! ip addr show dev $BRIDGE | grep -qF $CIDR ; then - ip addr add dev $BRIDGE $CIDR - arp_update $BRIDGE $CIDR || true - fi + call_weave "POST" "/expose/$CIDR" + [ -z "$FQDN" ] || when_weave_running put_dns_fqdn_no_check_alive weave:expose $FQDN $CIDR done } @@ -1484,7 +1482,6 @@ case "$COMMAND" in fi create_bridge expose_ip - util_op expose-nat $ALL_CIDRS show_addrs $ALL_CIDRS ;; hide)