From 6e89b036e2f4b99e64f34c49a4f62ccb1c124beb Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 23 Jul 2015 10:58:42 +0100 Subject: [PATCH 1/6] Wait for ethwe via netlink, not via polling or signals ('docker kill' breaks restart policy) --- Makefile | 3 +- net/if.go | 60 ++++++++++++++++++++++---------- prog/weaver/main.go | 4 +-- prog/weavewait/main.go | 17 ++------- proxy/create_exec_interceptor.go | 2 +- 5 files changed, 47 insertions(+), 39 deletions(-) diff --git a/Makefile b/Makefile index bd856c57..4950d11c 100644 --- a/Makefile +++ b/Makefile @@ -74,7 +74,8 @@ $(NETCHECK_EXE): prog/netcheck/netcheck.go # Sigproxy and weavewait need separate rules as they fail the netgo check in # the main build stanza due to not importing net package $(SIGPROXY_EXE): prog/sigproxy/main.go -$(WEAVEWAIT_EXE): prog/weavewait/main.go +$(WEAVEWAIT_EXE): prog/weavewait/main.go net/*.go +$(WEAVEHOSTS_EXE): prog/weavehosts/weavehosts.go $(COVER_EXE): testing/cover/cover.go $(RUNNER_EXE): testing/runner/runner.go diff --git a/net/if.go b/net/if.go index 3cf3baab..1319dea7 100644 --- a/net/if.go +++ b/net/if.go @@ -2,30 +2,24 @@ package net import ( "fmt" - "math" "net" - "time" + "syscall" + + "github.com/vishvananda/netlink/nl" ) -const ( - sleepTime = 100 * time.Millisecond -) - -// Wait `wait` seconds for an interface to come up. Pass zero to check once -// and return immediately, or a negative value to wait indefinitely. -func EnsureInterface(ifaceName string, wait int) (iface *net.Interface, err error) { - if iface, err = findInterface(ifaceName); err == nil || wait == 0 { - return +// Wait for an interface to come up. +func EnsureInterface(ifaceName string) (*net.Interface, error) { + s, err := nl.Subscribe(syscall.NETLINK_ROUTE, syscall.RTNLGRP_LINK) + if err != nil { + return nil, err } - i := int64(math.MaxInt64) - if wait > 0 { - i = int64(time.Duration(wait) * time.Second / sleepTime) + defer s.Close() + if iface, err := findInterface(ifaceName); err == nil { + return iface, nil } - for ; err != nil && i > 0; i-- { - time.Sleep(sleepTime) - iface, err = findInterface(ifaceName) - } - return + waitForIfUp(s, ifaceName) + return findInterface(ifaceName) } func findInterface(ifaceName string) (iface *net.Interface, err error) { @@ -37,3 +31,31 @@ func findInterface(ifaceName string) (iface *net.Interface, err error) { } return } + +func waitForIfUp(s *nl.NetlinkSocket, ifaceName string) error { + for { + msgs, err := s.Receive() + if err != nil { + return err + } + for _, m := range msgs { + switch m.Header.Type { + case syscall.RTM_NEWLINK: // receive this type for link 'up' + ifmsg := nl.DeserializeIfInfomsg(m.Data) + attrs, err := syscall.ParseNetlinkRouteAttr(&m) + if err != nil { + return err + } + name := "" + for _, attr := range attrs { + if attr.Attr.Type == syscall.IFA_LABEL { + name = string(attr.Value[:len(attr.Value)-1]) + } + } + if ifaceName == name && ifmsg.Flags&syscall.IFF_UP != 0 { + return nil + } + } + } + } +} diff --git a/prog/weaver/main.go b/prog/weaver/main.go index 7c6c6f28..1ac9c315 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -43,7 +43,6 @@ func main() { routerName string nickName string password string - wait int pktdebug bool logLevel string prof string @@ -71,7 +70,6 @@ func main() { mflag.StringVar(&routerName, []string{"#name", "-name"}, "", "name of router (defaults to MAC of interface)") mflag.StringVar(&nickName, []string{"#nickname", "-nickname"}, "", "nickname of peer (defaults to hostname)") mflag.StringVar(&password, []string{"#password", "-password"}, "", "network password") - mflag.IntVar(&wait, []string{"#wait", "-wait"}, -1, "number of seconds to wait for interface to come up (0=don't wait, -1=wait forever)") mflag.StringVar(&logLevel, []string{"-log-level"}, "info", "logging level (debug, info, warning, error)") mflag.BoolVar(&pktdebug, []string{"#pktdebug", "#-pktdebug", "-pkt-debug"}, false, "enable per-packet debug logging") mflag.StringVar(&prof, []string{"#profile", "-profile"}, "", "enable profiling and write profiles to given path") @@ -118,7 +116,7 @@ func main() { var err error if ifaceName != "" { - config.Iface, err = weavenet.EnsureInterface(ifaceName, wait) + config.Iface, err = weavenet.EnsureInterface(ifaceName) if err != nil { Log.Fatal(err) } diff --git a/prog/weavewait/main.go b/prog/weavewait/main.go index c3adfafc..69315bb1 100644 --- a/prog/weavewait/main.go +++ b/prog/weavewait/main.go @@ -5,7 +5,6 @@ import ( "fmt" "os" "os/exec" - "os/signal" "syscall" weavenet "github.com/weaveworks/weave/net" @@ -17,22 +16,10 @@ var ( func main() { var ( - args = os.Args[1:] - notInExec = true + args = os.Args[1:] ) - if len(args) > 0 && args[0] == "-s" { - notInExec = false - args = args[1:] - } - - if notInExec { - usr2 := make(chan os.Signal) - signal.Notify(usr2, syscall.SIGUSR2) - <-usr2 - } - - _, err := weavenet.EnsureInterface("ethwe", -1) + _, err := weavenet.EnsureInterface("ethwe") checkErr(err) if len(args) == 0 { diff --git a/proxy/create_exec_interceptor.go b/proxy/create_exec_interceptor.go index 9e7ea475..99bacafc 100644 --- a/proxy/create_exec_interceptor.go +++ b/proxy/create_exec_interceptor.go @@ -35,7 +35,7 @@ func (i *createExecInterceptor) InterceptRequest(r *http.Request) error { Log.Infof("Leaving container %s alone because %s", container.ID, err) } else if hasWeaveWait { Log.Infof("Exec in container %s with WEAVE_CIDR \"%s\"", container.ID, strings.Join(cidrs, " ")) - cmd := append(weaveWaitEntrypoint, "-s") + cmd := weaveWaitEntrypoint options.Cmd = append(cmd, options.Cmd...) } From 3d25620d13cfe4d0137f5b19f6c09043e5b18334 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Fri, 14 Aug 2015 14:59:54 +0100 Subject: [PATCH 2/6] In proxy, rewrite /etc/hosts from 'weave attach', via new utility weavehosts --- .gitignore | 2 + Makefile | 8 +- prog/weaveexec/Dockerfile | 2 +- prog/weavehosts/weavehosts.go | 120 +++++++++++++++++++++++ proxy/create_exec_interceptor.go | 3 +- proxy/start_container_interceptor.go | 136 +-------------------------- weave | 22 ++++- 7 files changed, 151 insertions(+), 142 deletions(-) create mode 100644 prog/weavehosts/weavehosts.go diff --git a/.gitignore b/.gitignore index 3889f49b..95bb841f 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ coverage.html # Project specific prog/weaver/weaver prog/weavedns/weavedns +prog/weavehosts/weavehosts prog/weaveproxy/weaveproxy prog/sigproxy/sigproxy prog/weavewait/weavewait @@ -49,6 +50,7 @@ releases .weaveexec.uptodate weave.tar prog/weaveexec/weave +prog/weaveexec/weavehosts prog/weaveexec/weaveproxy prog/weaveexec/weavewait prog/weaveexec/sigproxy diff --git a/Makefile b/Makefile index 4950d11c..3d4ca10c 100644 --- a/Makefile +++ b/Makefile @@ -13,11 +13,12 @@ WEAVER_EXE=prog/weaver/weaver WEAVEPROXY_EXE=prog/weaveproxy/weaveproxy SIGPROXY_EXE=prog/sigproxy/sigproxy WEAVEWAIT_EXE=prog/weavewait/weavewait +WEAVEHOSTS_EXE=prog/weavehosts/weavehosts NETCHECK_EXE=prog/netcheck/netcheck COVER_EXE=testing/cover/cover RUNNER_EXE=testing/runner/runner -EXES=$(WEAVER_EXE) $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(NETCHECK_EXE) $(COVER_EXE) $(RUNNER_EXE) +EXES=$(WEAVER_EXE) $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEHOSTS_EXE) $(NETCHECK_EXE) $(COVER_EXE) $(RUNNER_EXE) WEAVER_UPTODATE=.weaver.uptodate WEAVEEXEC_UPTODATE=.weaveexec.uptodate @@ -79,7 +80,7 @@ $(WEAVEHOSTS_EXE): prog/weavehosts/weavehosts.go $(COVER_EXE): testing/cover/cover.go $(RUNNER_EXE): testing/runner/runner.go -$(WEAVEWAIT_EXE) $(SIGPROXY_EXE) $(COVER_EXE) $(RUNNER_EXE): +$(WEAVEWAIT_EXE) $(SIGPROXY_EXE) $(WEAVEHOSTS_EXE) $(COVER_EXE) $(RUNNER_EXE): go get ./$(@D) go build -o $@ ./$(@D) @@ -87,11 +88,12 @@ $(WEAVER_UPTODATE): prog/weaver/Dockerfile $(WEAVER_EXE) $(SUDO) docker build -t $(WEAVER_IMAGE) prog/weaver touch $@ -$(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(NETCHECK_EXE) +$(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEHOSTS_EXE) $(NETCHECK_EXE) cp weave prog/weaveexec/weave cp $(SIGPROXY_EXE) prog/weaveexec/sigproxy cp $(WEAVEPROXY_EXE) prog/weaveexec/weaveproxy cp $(WEAVEWAIT_EXE) prog/weaveexec/weavewait + cp $(WEAVEHOSTS_EXE) prog/weaveexec/weavehosts cp $(NETCHECK_EXE) prog/weaveexec/netcheck cp $(DOCKER_DISTRIB) prog/weaveexec/docker.tgz $(SUDO) docker build -t $(WEAVEEXEC_IMAGE) prog/weaveexec diff --git a/prog/weaveexec/Dockerfile b/prog/weaveexec/Dockerfile index 9d101bd7..e17bfc25 100644 --- a/prog/weaveexec/Dockerfile +++ b/prog/weaveexec/Dockerfile @@ -17,6 +17,6 @@ RUN apk add --update \ && rm -rf /var/cache/apk/* ADD ./weave ./sigproxy ./weaveproxy /home/weave/ -ADD ./netcheck /usr/bin/ +ADD ./netcheck ./weavehosts /usr/bin/ ADD ./weavewait /w/w ADD ./docker.tgz / diff --git a/prog/weavehosts/weavehosts.go b/prog/weavehosts/weavehosts.go new file mode 100644 index 00000000..b1694365 --- /dev/null +++ b/prog/weavehosts/weavehosts.go @@ -0,0 +1,120 @@ +/* weavehosts: rewrite /etc/hosts as necessary to put the weave address(es) at the top */ +package main + +import ( + "bufio" + "bytes" + "fmt" + "io/ioutil" + "net" + "os" + "sort" + "strings" +) + +func main() { + args := os.Args + if len(args) < 4 { + fmt.Fprintln(os.Stderr, "usage: weavehosts hostpath hostname cidr [cidr ...]") + os.Exit(1) + } + + checkErr(updateHosts(args[1], args[2], args[3:])) + + os.Exit(0) +} + +func checkErr(err error) { + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func updateHosts(path, hostname string, cidrs []string) error { + hosts, err := parseHosts(path) + if err != nil { + return err + } + + // Remove existing ips pointing to our hostname + toRemove := []string{} + for ip, addrs := range hosts { + for _, addr := range addrs { + if addr == hostname { + toRemove = append(toRemove, ip) + break + } + } + } + for _, ip := range toRemove { + delete(hosts, ip) + } + + // Add the weave ip(s) + for _, cidr := range cidrs { + ip, _, err := net.ParseCIDR(cidr) + checkErr(err) + ipStr := ip.String() + hosts[ipStr] = append(hosts[ipStr], hostname) + } + + return writeHosts(path, hosts) +} + +func parseHosts(path string) (map[string][]string, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + ips := map[string][]string{} + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + + // Remove any comments + if i := strings.IndexByte(line, '#'); i != -1 { + line = line[:i] + } + + fields := strings.Fields(line) + if len(fields) > 0 { + ips[fields[0]] = append(ips[fields[0]], fields[1:]...) + } + } + if scanner.Err() != nil { + return nil, scanner.Err() + } + return ips, nil +} + +func writeHosts(path string, contents map[string][]string) error { + ips := []string{} + for ip := range contents { + ips = append(ips, ip) + } + sort.Strings(ips) + + buf := &bytes.Buffer{} + fmt.Fprintln(buf, "# modified by weave") + for _, ip := range ips { + if addrs := contents[ip]; len(addrs) > 0 { + fmt.Fprintf(buf, "%s\t%s\n", ip, strings.Join(uniqueStrs(addrs), " ")) + } + } + return ioutil.WriteFile(path, buf.Bytes(), 644) +} + +func uniqueStrs(s []string) []string { + m := map[string]struct{}{} + result := []string{} + for _, str := range s { + if _, ok := m[str]; !ok { + m[str] = struct{}{} + result = append(result, str) + } + } + sort.Strings(result) + return result +} diff --git a/proxy/create_exec_interceptor.go b/proxy/create_exec_interceptor.go index 99bacafc..80168c95 100644 --- a/proxy/create_exec_interceptor.go +++ b/proxy/create_exec_interceptor.go @@ -35,8 +35,7 @@ func (i *createExecInterceptor) InterceptRequest(r *http.Request) error { Log.Infof("Leaving container %s alone because %s", container.ID, err) } else if hasWeaveWait { Log.Infof("Exec in container %s with WEAVE_CIDR \"%s\"", container.ID, strings.Join(cidrs, " ")) - cmd := weaveWaitEntrypoint - options.Cmd = append(cmd, options.Cmd...) + options.Cmd = append(weaveWaitEntrypoint, options.Cmd...) } if err := marshalRequestBody(r, options); err != nil { diff --git a/proxy/start_container_interceptor.go b/proxy/start_container_interceptor.go index e17e3647..267f68c4 100644 --- a/proxy/start_container_interceptor.go +++ b/proxy/start_container_interceptor.go @@ -1,18 +1,10 @@ package proxy import ( - "bufio" - "bytes" "errors" - "fmt" - "io/ioutil" - "net" "net/http" - "os" - "sort" "strings" - "github.com/fsouza/go-dockerclient" . "github.com/weaveworks/weave/common" ) @@ -40,6 +32,9 @@ func (i *startContainerInterceptor) InterceptResponse(r *http.Response) error { Log.Infof("Attaching container %s with WEAVE_CIDR \"%s\" to weave network", container.ID, strings.Join(cidrs, " ")) args := []string{"attach"} args = append(args, cidrs...) + if !i.proxy.NoRewriteHosts { + args = append(args, "--rewrite-hosts") + } args = append(args, "--or-die", container.ID) if _, stderr, err := callWeave(args...); err != nil { Log.Warningf("Attaching container %s to weave network failed: %s", container.ID, string(stderr)) @@ -48,128 +43,5 @@ func (i *startContainerInterceptor) InterceptResponse(r *http.Response) error { Log.Warningf("Attaching container %s to weave network: %s", container.ID, string(stderr)) } - if !i.proxy.NoRewriteHosts { - ips, err := weaveContainerIPs(container) - if err != nil { - return err - } - if len(ips) > 0 { - if err := updateHosts(container.HostsPath, container.Config.Hostname, ips); err != nil { - return err - } - } - } - - return i.proxy.client.KillContainer(docker.KillContainerOptions{ID: container.ID, Signal: docker.SIGUSR2}) -} - -func weaveContainerIPs(container *docker.Container) ([]net.IP, error) { - stdout, stderr, err := callWeave("ps", container.ID) - if err != nil || len(stderr) > 0 { - return nil, errors.New(string(stderr)) - } - if len(stdout) <= 0 { - return nil, nil - } - - fields := strings.Fields(string(stdout)) - if len(fields) <= 2 { - return nil, nil - } - - var ips []net.IP - for _, cidr := range fields[2:] { - ip, _, err := net.ParseCIDR(cidr) - if err != nil { - return nil, err - } - ips = append(ips, ip) - } - return ips, nil -} - -func updateHosts(path, hostname string, ips []net.IP) error { - hosts, err := parseHosts(path) - if err != nil { - return err - } - - // Remove existing ips pointing to our hostname - toRemove := []string{} - for ip, addrs := range hosts { - for _, addr := range addrs { - if addr == hostname { - toRemove = append(toRemove, ip) - break - } - } - } - for _, ip := range toRemove { - delete(hosts, ip) - } - - // Add the weave ip(s) - for _, ip := range ips { - ipStr := ip.String() - hosts[ipStr] = append(hosts[ipStr], hostname) - } - - return writeHosts(path, hosts) -} - -func parseHosts(path string) (map[string][]string, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() - ips := map[string][]string{} - scanner := bufio.NewScanner(f) - for scanner.Scan() { - line := scanner.Text() - - // Remove any comments - if i := strings.IndexByte(line, '#'); i != -1 { - line = line[:i] - } - - fields := strings.Fields(line) - if len(fields) > 0 { - ips[fields[0]] = append(ips[fields[0]], fields[1:]...) - } - } - if scanner.Err() != nil { - return nil, scanner.Err() - } - return ips, nil -} - -func writeHosts(path string, contents map[string][]string) error { - ips := []string{} - for ip := range contents { - ips = append(ips, ip) - } - sort.Strings(ips) - - buf := &bytes.Buffer{} - fmt.Fprintln(buf, "# modified by weave") - for _, ip := range ips { - if addrs := contents[ip]; len(addrs) > 0 { - fmt.Fprintf(buf, "%s\t%s\n", ip, strings.Join(uniqueStrs(addrs), " ")) - } - } - return ioutil.WriteFile(path, buf.Bytes(), 644) -} - -func uniqueStrs(s []string) []string { - m := map[string]struct{}{} - result := []string{} - for _, str := range s { - if _, ok := m[str]; !ok { - m[str] = struct{}{} - result = append(result, str) - } - } - sort.Strings(result) - return result + return nil } diff --git a/weave b/weave index cc4a74af..9a8b9855 100755 --- a/weave +++ b/weave @@ -1446,14 +1446,28 @@ case "$COMMAND" in attach) collect_cidr_args "$@" shift $CIDR_ARG_COUNT - if [ "$1" = "--or-die" ] ; then - ATTACH_TYPE="_or_die" - shift - fi + while [ $# -gt 0 ]; do + case "$1" in + --or-die) + ATTACH_TYPE="_or_die" + ;; + --rewrite-hosts) + REWRITE_HOSTS=1 + ;; + *) + break + ;; + esac + shift + done [ $# -eq 1 ] || usage CONTAINER=$(container_id $1) create_bridge ipam_cidrs$ATTACH_TYPE $CONTAINER $CIDR_ARGS + if [ -n "$REWRITE_HOSTS" ] && command_exists weavehosts ; then + PATH_AND_NAME=$(docker inspect -f '{{.HostsPath}} {{.Config.Hostname}}' $CONTAINER) + weavehosts $PATH_AND_NAME $ALL_CIDRS + fi with_container_netns$ATTACH_TYPE $CONTAINER attach $ALL_CIDRS >/dev/null when_weave_running with_container_fqdn $CONTAINER put_dns_fqdn $ALL_CIDRS show_addrs $ALL_CIDRS From 2eed101a938617dd787f3a820de9a4e8d474af84 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 18 Aug 2015 14:46:34 +0100 Subject: [PATCH 3/6] In weavewait, wait for the multicast route to be added before calling container entrypoint. We have moved the 'route add' command to the end of 'attach', so this indicates that all network interface set-up is complete. --- net/if.go | 32 +++++++++++++++++++++++- net/route.go | 57 ++++++++++++++++++++++++++++++++++++++---- prog/weavewait/main.go | 2 +- weave | 11 ++++---- 4 files changed, 90 insertions(+), 12 deletions(-) diff --git a/net/if.go b/net/if.go index 1319dea7..8270bcec 100644 --- a/net/if.go +++ b/net/if.go @@ -15,11 +15,41 @@ func EnsureInterface(ifaceName string) (*net.Interface, error) { return nil, err } defer s.Close() + iface, err := ensureInterface(s, ifaceName) + if err != nil { + return nil, err + } + return iface, err +} + +func ensureInterface(s *nl.NetlinkSocket, ifaceName string) (*net.Interface, error) { if iface, err := findInterface(ifaceName); err == nil { return iface, nil } waitForIfUp(s, ifaceName) - return findInterface(ifaceName) + iface, err := findInterface(ifaceName) + return iface, err +} + +// Wait for an interface to come up and have a route added to the multicast subnet. +// This matches the behaviour in 'weave attach', which is the only context in which +// we expect this to be called. If you change one, change the other to match. +func EnsureInterfaceAndMcastRoute(ifaceName string) (*net.Interface, error) { + s, err := nl.Subscribe(syscall.NETLINK_ROUTE, syscall.RTNLGRP_LINK, syscall.RTNLGRP_IPV4_ROUTE) + if err != nil { + return nil, err + } + defer s.Close() + iface, err := ensureInterface(s, ifaceName) + if err != nil { + return nil, err + } + dest := net.IPv4(224, 0, 0, 0) + if CheckRouteExists(ifaceName, dest) { + return iface, err + } + waitForRoute(s, ifaceName, dest) + return iface, err } func findInterface(ifaceName string) (iface *net.Interface, err error) { diff --git a/net/route.go b/net/route.go index 463f843b..990458b3 100644 --- a/net/route.go +++ b/net/route.go @@ -3,14 +3,16 @@ package net import ( "fmt" "net" + "syscall" "github.com/vishvananda/netlink" + "github.com/vishvananda/netlink/nl" ) // A network is considered free if it does not overlap any existing // routes on this host. This is the same approach taken by Docker. func CheckNetworkFree(subnet *net.IPNet, ignoreIfaceNames map[string]struct{}) error { - return forEachRoute(ignoreIfaceNames, func(route netlink.Route) error { + return forEachRoute(ignoreIfaceNames, func(name string, route netlink.Route) error { if route.Dst != nil && overlaps(route.Dst, subnet) { return fmt.Errorf("Network %s overlaps with existing route %s on host.", subnet, route.Dst) } @@ -26,7 +28,7 @@ func overlaps(n1, n2 *net.IPNet) bool { // For a specific address, we only care if it is actually *inside* an // existing route, because weave-local traffic never hits IP routing. func CheckAddressOverlap(addr net.IP, ignoreIfaceNames map[string]struct{}) error { - return forEachRoute(ignoreIfaceNames, func(route netlink.Route) error { + return forEachRoute(ignoreIfaceNames, func(name string, route netlink.Route) error { if route.Dst != nil && route.Dst.Contains(addr) { return fmt.Errorf("Address %s overlaps with existing route %s on host.", addr, route.Dst) } @@ -34,20 +36,65 @@ func CheckAddressOverlap(addr net.IP, ignoreIfaceNames map[string]struct{}) erro }) } -func forEachRoute(ignoreIfaceNames map[string]struct{}, check func(netlink.Route) error) error { +func forEachRoute(ignoreIfaceNames map[string]struct{}, check func(name string, r netlink.Route) error) error { routes, err := netlink.RouteList(nil, netlink.FAMILY_V4) if err != nil { return err } for _, route := range routes { - if link, err := netlink.LinkByIndex(route.LinkIndex); err == nil { + link, err := netlink.LinkByIndex(route.LinkIndex) + if err == nil { if _, found := ignoreIfaceNames[link.Attrs().Name]; found { continue } } - if err := check(route); err != nil { + if err := check(link.Attrs().Name, route); err != nil { return err } } return nil } + +func CheckRouteExists(ifaceName string, dest net.IP) bool { + found := false + forEachRoute(map[string]struct{}{}, func(name string, route netlink.Route) error { + if name == ifaceName && route.Dst.IP.Equal(dest) { + found = true + } + return nil + }) + return found +} + +func waitForRoute(s *nl.NetlinkSocket, ifaceName string, dest net.IP) error { + native := nl.NativeEndian() + for { + msgs, err := s.Receive() + if err != nil { + return err + } + for _, m := range msgs { + switch m.Header.Type { + case syscall.RTM_NEWROUTE: + attrs, err := syscall.ParseNetlinkRouteAttr(&m) + if err != nil { + return err + } + var ip net.IP + var link netlink.Link + for _, attr := range attrs { + switch attr.Attr.Type { + case syscall.RTA_DST: + ip = attr.Value + case syscall.RTA_OIF: + index := int(native.Uint32(attr.Value[0:4])) + link, _ = netlink.LinkByIndex(index) + } + } + if link.Attrs().Name == ifaceName && ip.Equal(dest) { + return nil + } + } + } + } +} diff --git a/prog/weavewait/main.go b/prog/weavewait/main.go index 69315bb1..43158e93 100644 --- a/prog/weavewait/main.go +++ b/prog/weavewait/main.go @@ -19,7 +19,7 @@ func main() { args = os.Args[1:] ) - _, err := weavenet.EnsureInterface("ethwe") + _, err := weavenet.EnsureInterfaceAndMcastRoute("ethwe") checkErr(err) if len(args) == 0 { diff --git a/weave b/weave index 9a8b9855..fcea80cd 100755 --- a/weave +++ b/weave @@ -528,14 +528,15 @@ attach() { netnsenter ip link set $CONTAINER_IFNAME up || return 1 - # Route multicast packets across the weave network. - if ! netnsenter ip route show | grep '^224\.0\.0\.0/4' >/dev/null ; then - netnsenter ip route add 224.0.0.0/4 dev $CONTAINER_IFNAME - fi - for ADDR in "$@" ; do arp_update $CONTAINER_IFNAME $ADDR "netnsenter" done + + # Route multicast packets across the weave network. + # This must come last in 'attach'. If you change this, change weavewait to match. + if ! netnsenter ip route show | grep '^224\.0\.0\.0/4' >/dev/null ; then + netnsenter ip route add 224.0.0.0/4 dev $CONTAINER_IFNAME + fi } detach() { From 7801dcd8497d26aa5b83315550fc34ef3f220a57 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 27 Aug 2015 11:58:48 +0100 Subject: [PATCH 4/6] Factor out common loop receiving from netlink --- net/if.go | 47 ++++++++++++++++++++++++++++------------------- net/route.go | 49 ++++++++++++++++++++++--------------------------- 2 files changed, 50 insertions(+), 46 deletions(-) diff --git a/net/if.go b/net/if.go index 8270bcec..b99993ec 100644 --- a/net/if.go +++ b/net/if.go @@ -26,7 +26,7 @@ func ensureInterface(s *nl.NetlinkSocket, ifaceName string) (*net.Interface, err if iface, err := findInterface(ifaceName); err == nil { return iface, nil } - waitForIfUp(s, ifaceName) + netlinkReceiveUntil(s, matchIfName(ifaceName)) iface, err := findInterface(ifaceName) return iface, err } @@ -48,7 +48,7 @@ func EnsureInterfaceAndMcastRoute(ifaceName string) (*net.Interface, error) { if CheckRouteExists(ifaceName, dest) { return iface, err } - waitForRoute(s, ifaceName, dest) + netlinkReceiveUntil(s, matchRoute(ifaceName, dest)) return iface, err } @@ -62,30 +62,39 @@ func findInterface(ifaceName string) (iface *net.Interface, err error) { return } -func waitForIfUp(s *nl.NetlinkSocket, ifaceName string) error { +func netlinkReceiveUntil(s *nl.NetlinkSocket, f func(syscall.NetlinkMessage) (bool, error)) error { for { msgs, err := s.Receive() if err != nil { return err } for _, m := range msgs { - switch m.Header.Type { - case syscall.RTM_NEWLINK: // receive this type for link 'up' - ifmsg := nl.DeserializeIfInfomsg(m.Data) - attrs, err := syscall.ParseNetlinkRouteAttr(&m) - if err != nil { - return err - } - name := "" - for _, attr := range attrs { - if attr.Attr.Type == syscall.IFA_LABEL { - name = string(attr.Value[:len(attr.Value)-1]) - } - } - if ifaceName == name && ifmsg.Flags&syscall.IFF_UP != 0 { - return nil - } + if done, err := f(m); done { + return err } } } } + +func matchIfName(ifaceName string) func(m syscall.NetlinkMessage) (bool, error) { + return func(m syscall.NetlinkMessage) (bool, error) { + switch m.Header.Type { + case syscall.RTM_NEWLINK: // receive this type for link 'up' + ifmsg := nl.DeserializeIfInfomsg(m.Data) + attrs, err := syscall.ParseNetlinkRouteAttr(&m) + if err != nil { + return true, err + } + name := "" + for _, attr := range attrs { + if attr.Attr.Type == syscall.IFA_LABEL { + name = string(attr.Value[:len(attr.Value)-1]) + } + } + if ifaceName == name && ifmsg.Flags&syscall.IFF_UP != 0 { + return true, nil + } + } + return false, nil + } +} diff --git a/net/route.go b/net/route.go index 990458b3..65dcfc89 100644 --- a/net/route.go +++ b/net/route.go @@ -66,35 +66,30 @@ func CheckRouteExists(ifaceName string, dest net.IP) bool { return found } -func waitForRoute(s *nl.NetlinkSocket, ifaceName string, dest net.IP) error { - native := nl.NativeEndian() - for { - msgs, err := s.Receive() - if err != nil { - return err - } - for _, m := range msgs { - switch m.Header.Type { - case syscall.RTM_NEWROUTE: - attrs, err := syscall.ParseNetlinkRouteAttr(&m) - if err != nil { - return err - } - var ip net.IP - var link netlink.Link - for _, attr := range attrs { - switch attr.Attr.Type { - case syscall.RTA_DST: - ip = attr.Value - case syscall.RTA_OIF: - index := int(native.Uint32(attr.Value[0:4])) - link, _ = netlink.LinkByIndex(index) - } - } - if link.Attrs().Name == ifaceName && ip.Equal(dest) { - return nil +func matchRoute(ifaceName string, dest net.IP) func(m syscall.NetlinkMessage) (bool, error) { + return func(m syscall.NetlinkMessage) (bool, error) { + switch m.Header.Type { + case syscall.RTM_NEWROUTE: + attrs, err := syscall.ParseNetlinkRouteAttr(&m) + if err != nil { + return true, err + } + var ip net.IP + var link netlink.Link + for _, attr := range attrs { + switch attr.Attr.Type { + case syscall.RTA_DST: + ip = attr.Value + case syscall.RTA_OIF: + native := nl.NativeEndian() + index := int(native.Uint32(attr.Value[0:4])) + link, _ = netlink.LinkByIndex(index) } } + if link.Attrs().Name == ifaceName && ip.Equal(dest) { + return true, nil + } } + return false, nil } } From 6fce79816484cd66878003dd161ebba3cdfc91e2 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 27 Aug 2015 12:20:01 +0100 Subject: [PATCH 5/6] Factor out deserializeRoute() func --- net/route.go | 48 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/net/route.go b/net/route.go index 65dcfc89..560ae22e 100644 --- a/net/route.go +++ b/net/route.go @@ -70,26 +70,46 @@ func matchRoute(ifaceName string, dest net.IP) func(m syscall.NetlinkMessage) (b return func(m syscall.NetlinkMessage) (bool, error) { switch m.Header.Type { case syscall.RTM_NEWROUTE: - attrs, err := syscall.ParseNetlinkRouteAttr(&m) + route, err := deserializeRoute(m.Data) if err != nil { return true, err } - var ip net.IP - var link netlink.Link - for _, attr := range attrs { - switch attr.Attr.Type { - case syscall.RTA_DST: - ip = attr.Value - case syscall.RTA_OIF: - native := nl.NativeEndian() - index := int(native.Uint32(attr.Value[0:4])) - link, _ = netlink.LinkByIndex(index) - } - } - if link.Attrs().Name == ifaceName && ip.Equal(dest) { + link, _ := netlink.LinkByIndex(route.LinkIndex) + if link.Attrs().Name == ifaceName && route.Dst.IP.Equal(dest) { return true, nil } } return false, nil } } + +// This code proposed for addition to the vishvananda/netlink library +// deserializeRoute decodes a binary netlink message into a Route struct +func deserializeRoute(m []byte) (netlink.Route, error) { + route := netlink.Route{} + msg := nl.DeserializeRtMsg(m) + attrs, err := nl.ParseRouteAttr(m[msg.Len():]) + if err != nil { + return route, err + } + route.Scope = netlink.Scope(msg.Scope) + + native := nl.NativeEndian() + for _, attr := range attrs { + switch attr.Attr.Type { + case syscall.RTA_GATEWAY: + route.Gw = net.IP(attr.Value) + case syscall.RTA_PREFSRC: + route.Src = net.IP(attr.Value) + case syscall.RTA_DST: + route.Dst = &net.IPNet{ + IP: attr.Value, + Mask: net.CIDRMask(int(msg.Dst_len), 8*len(attr.Value)), + } + case syscall.RTA_OIF: + routeIndex := int(native.Uint32(attr.Value[0:4])) + route.LinkIndex = routeIndex + } + } + return route, nil +} From a45e5ac1525065a3a480a59e8044c04fc8c53e8a Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Fri, 28 Aug 2015 16:20:31 +0100 Subject: [PATCH 6/6] Test calling 'start' with a HOST_CONFIG not supplied to 'create' --- test/666_the_beast.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100755 test/666_the_beast.sh diff --git a/test/666_the_beast.sh b/test/666_the_beast.sh new file mode 100755 index 00000000..a0073532 --- /dev/null +++ b/test/666_the_beast.sh @@ -0,0 +1,15 @@ +#! /bin/bash + +. ./config.sh + +start_suite "Abuse of 'start' operation" + +weave_on $HOST1 launch + +proxy_start_container $HOST1 --name=c1 +proxy docker_on $HOST1 create --name=c2 $SMALL_IMAGE grep "^1$" /sys/class/net/ethwe/carrier +# Now start c2 with a sneaky HostConfig +curl -X POST -H Content-Type:application/json -d '{"HostConfig": {"NetworkMode": "container:c1"}}' http://$HOST1:12375/containers/c2/start +assert "docker_on $HOST1 inspect -f '{{.State.Running}} {{.State.ExitCode}}' c2" "false 0" + +end_suite