Merge pull request #2947 from weaveworks/issues/1940-remove-proxy-attach-dep

Attach containers from Proxy using Go code

api/ipam.go

@@ -3,6 +3,7 @@ package api
 import (
 	"fmt"
 	"net"
+	"net/url"
 )
 
 // Special token used in place of a container identifier when:
@@ -14,21 +15,29 @@ import (
 // which should be kept in mind if the entry needs to be removed at any point.
 const NoContainerID string = "_"
 
-func (client *Client) ipamOp(ID string, op string) (*net.IPNet, error) {
-	ip, err := client.httpVerb(op, fmt.Sprintf("/ip/%s", ID), nil)
+func (client *Client) ipamOp(ID string, op string, values url.Values) (*net.IPNet, error) {
+	ip, err := client.httpVerb(op, fmt.Sprintf("/ip/%s", ID), values)
 	if err != nil {
 		return nil, err
 	}
 	return parseIP(ip)
 }
 
-// returns an IP for the ID given, allocating a fresh one if necessary
-func (client *Client) AllocateIP(ID string) (*net.IPNet, error) {
-	return client.ipamOp(ID, "POST")
+func ipamValues(checkAlive bool) url.Values {
+	values := make(url.Values)
+	if checkAlive {
+		values.Set("check-alive", "true")
+	}
+	return values
 }
 
-func (client *Client) AllocateIPInSubnet(ID string, subnet *net.IPNet) (*net.IPNet, error) {
-	ip, err := client.httpVerb("POST", fmt.Sprintf("/ip/%s/%s", ID, subnet), nil)
+// returns an IP for the ID given, allocating a fresh one if necessary
+func (client *Client) AllocateIP(ID string, checkAlive bool) (*net.IPNet, error) {
+	return client.ipamOp(ID, "POST", ipamValues(checkAlive))
+}
+
+func (client *Client) AllocateIPInSubnet(ID string, subnet *net.IPNet, checkAlive bool) (*net.IPNet, error) {
+	ip, err := client.httpVerb("POST", fmt.Sprintf("/ip/%s/%s", ID, subnet), ipamValues(checkAlive))
 	if err != nil {
 		return nil, err
 	}
@@ -38,7 +47,7 @@ func (client *Client) AllocateIPInSubnet(ID string, subnet *net.IPNet) (*net.IPN
 // returns an IP for the ID given, or nil if one has not been
 // allocated
 func (client *Client) LookupIP(ID string) (*net.IPNet, error) {
-	return client.ipamOp(ID, "GET")
+	return client.ipamOp(ID, "GET", nil)
 }
 
 // Claim a specific IP on behalf of the ID

plugin/ipam/cni.go

@@ -38,14 +38,14 @@ func (i *Ipam) Allocate(args *skel.CmdArgs) (types.Result, error) {
 	var ipnet *net.IPNet
 
 	if conf.Subnet == "" {
-		ipnet, err = i.weave.AllocateIP(containerID)
+		ipnet, err = i.weave.AllocateIP(containerID, false)
 	} else {
 		var subnet *net.IPNet
 		subnet, err = types.ParseCIDR(conf.Subnet)
 		if err != nil {
 			return nil, fmt.Errorf("subnet given in config, but not parseable: %s", err)
 		}
-		ipnet, err = i.weave.AllocateIPInSubnet(containerID, subnet)
+		ipnet, err = i.weave.AllocateIPInSubnet(containerID, subnet, false)
 	}
 
 	if err != nil {

plugin/ipam/driver.go

@@ -78,7 +78,7 @@ func (i *Ipam) RequestAddress(poolID string, address net.IP, options map[string]
 	i.logReq("RequestAddress", poolID, address, options)
 	defer func() { i.logRes("RequestAddress", err, ip) }()
 	if poolID == "weavepool" { // old-style
-		ip, err = i.weave.AllocateIP(api.NoContainerID)
+		ip, err = i.weave.AllocateIP(api.NoContainerID, false)
 		return
 	}
 	subnet, iprange, err := splitPoolID(poolID)
@@ -92,7 +92,7 @@ func (i *Ipam) RequestAddress(poolID string, address net.IP, options map[string]
 		}
 	} else {
 		// We are lying slightly to IPAM here: the range is not a subnet
-		if ip, err = i.weave.AllocateIPInSubnet(api.NoContainerID, iprange); err != nil {
+		if ip, err = i.weave.AllocateIPInSubnet(api.NoContainerID, iprange, false); err != nil {
 			return
 		}
 		ip.Mask = subnet.Mask // fix up the subnet we lied about

prog/weaver/main.go

@@ -220,6 +220,7 @@ func main() {
 	proxyConfig := configureProxy(version, defaultDockerHost)
 	if bridgeConfig.AWSVPC {
 		proxyConfig.NoMulticastRoute = true
+		proxyConfig.KeepTXOn = true
 	}
 
 	// crude way of detecting that we probably have been started in a

prog/weaveutil/attach.go

@@ -3,13 +3,16 @@ package main
 import (
 	"fmt"
 	"net"
+	"os"
 	"strconv"
+	"strings"
 	"syscall"
 
 	docker "github.com/fsouza/go-dockerclient"
 	"github.com/vishvananda/netns"
 
 	weavenet "github.com/weaveworks/weave/net"
+	"github.com/weaveworks/weave/proxy"
 )
 
 func attach(args []string) error {
@@ -110,3 +113,30 @@ func detach(args []string) error {
 	}
 	return weavenet.DetachContainer(weavenet.NSPathByPid(pid), args[0], weavenet.VethName, cidrs)
 }
+
+func rewriteEtcHosts(args []string) error {
+	if len(args) < 4 {
+		cmdUsage("rewrite-etc-hosts", "<hosts-path> <fqdn> <image> <cidr>... [name:addr...]")
+	}
+	hostsPath := args[0]
+	fqdn := args[1]
+	image := args[2]
+	var ips []*net.IPNet
+	for _, cidr := range strings.Fields(args[3]) {
+		_, ipnet, err := net.ParseCIDR(cidr)
+		if err != nil {
+			return err
+		}
+		ips = append(ips, ipnet)
+	}
+	extraHosts := args[4:]
+	docker := os.Getenv("DOCKER_HOST")
+	if docker == "" {
+		docker = "unix:///var/run/docker.sock"
+	}
+	p, err := proxy.StubProxy(proxy.Config{DockerHost: docker, Image: image})
+	if err != nil {
+		return err
+	}
+	return p.RewriteEtcHosts(hostsPath, fqdn, ips, extraHosts)
+}

prog/weaveutil/main.go

@@ -40,6 +40,7 @@ func init() {
 		"unique-id":                uniqueID,
 		"swarm-manager-peers":      swarmManagerPeers,
 		"is-docker-plugin-enabled": isDockerPluginEnabled,
+		"rewrite-etc-hosts":        rewriteEtcHosts,
 	}
 }
 

proxy/common.go

@@ -6,13 +6,12 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
-	"os"
-	"os/exec"
 	"regexp"
 	"strings"
 
 	"github.com/fsouza/go-dockerclient"
 	"github.com/weaveworks/weave/common"
+	weavedocker "github.com/weaveworks/weave/common/docker"
 )
 
 var (
@@ -24,42 +23,6 @@ var (
 	Log = common.Log
 )
 
-func callWeave(args ...string) ([]byte, []byte, error) {
-	args = append([]string{"--local"}, args...)
-	cmd := exec.Command("./weave", args...)
-	cmd.Env = []string{
-		"PATH=/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
-	}
-
-	propagateEnv := func(key string) {
-		if val := os.Getenv(key); val != "" {
-			cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", key, val))
-		}
-	}
-
-	propagateEnv("DOCKER_HOST")
-
-	// Propagate WEAVE_DEBUG, to make debugging easier.
-	propagateEnv("WEAVE_DEBUG")
-
-	// This prevents the code coverage contortions in our
-	// integration test suite breaking things.
-	propagateEnv("COVERAGE")
-
-	// In case the router control endpoint address is non-standard.
-	propagateEnv("WEAVE_HTTP_ADDR")
-
-	Log.Debug("Calling weave args: ", args, "env: ", cmd.Env)
-	var stdout, stderr bytes.Buffer
-	cmd.Stdout = &stdout
-	cmd.Stderr = &stderr
-	err := cmd.Run()
-	if err != nil {
-		Log.Debug("weave returned error: ", err)
-	}
-	return stdout.Bytes(), stderr.Bytes(), err
-}
-
 func unmarshalRequestBody(r *http.Request, target interface{}) error {
 	body, err := ioutil.ReadAll(r.Body)
 	if err != nil {
@@ -116,7 +79,7 @@ func marshalResponseBody(r *http.Response, body interface{}) error {
 	return nil
 }
 
-func inspectContainerInPath(client *docker.Client, path string) (*docker.Container, error) {
+func inspectContainerInPath(client *weavedocker.Client, path string) (*docker.Container, error) {
 	subs := containerIDRegexp.FindStringSubmatch(path)
 	if subs == nil {
 		err := fmt.Errorf("No container id found in request with path %s", path)

proxy/hosts.go

@@ -0,0 +1,65 @@
+package proxy
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net"
+	"path"
+	"strings"
+)
+
+// rewrite /etc/hosts, unlinking the file (so Docker does not modify it again) but
+// leaving it with valid contents...
+func (proxy *Proxy) RewriteEtcHosts(hostsPath, fqdn string, ips []*net.IPNet, extraHosts []string) error {
+	hostsPathDir := path.Dir(hostsPath)
+	mnt := "/container"
+	mntHosts := path.Join(mnt, path.Base(hostsPath))
+	var buf bytes.Buffer
+	writeEtcHostsContents(&buf, fqdn, ips, extraHosts)
+	contents := buf.String()
+	cmdLine := fmt.Sprintf("echo '%s' > %s && rm -f %s && echo '%s' > %s", contents, mntHosts, mntHosts, contents, mntHosts)
+	mounts := []string{hostsPathDir + ":" + mnt}
+	proxy.runTransientContainer([]string{"sh"}, []string{"-c", cmdLine}, mounts)
+	return nil
+}
+
+// we assume (for compatibility with the weave script) that fqdn has a dot
+// between hostname and domain. domain may be blank.
+func writeEtcHostsContents(w io.Writer, fqdn string, cidrs []*net.IPNet, extraHosts []string) {
+	index := strings.Index(fqdn, ".")
+	name := fqdn[:index]
+	var hostnames string
+	if index == len(fqdn)-1 { // dot comes at the end - no domain
+		hostnames = name
+	} else {
+		if fqdn[len(fqdn)-1] == '.' { // strip a final trailing dot
+			fqdn = fqdn[:len(fqdn)-1]
+		}
+		hostnames = fqdn + " " + name
+	}
+
+	fmt.Fprintln(w, "# created by Weave - BEGIN")
+	fmt.Fprintln(w, "# container hostname")
+	for _, cidr := range cidrs {
+		fmt.Fprintf(w, "%-15s %s\n", cidr.IP, hostnames)
+	}
+	fmt.Fprintln(w)
+	fmt.Fprintln(w, "# static names added with --add-host")
+	for _, eh := range extraHosts {
+		parts := strings.SplitN(eh, ":", 2)
+		if len(parts) != 2 {
+			continue
+		}
+		fmt.Fprintf(w, "%-15s %s\n", parts[1], parts[0])
+	}
+	fmt.Fprintln(w)
+	fmt.Fprintln(w, "# default localhost entries")
+	fmt.Fprintln(w, "127.0.0.1       localhost")
+	fmt.Fprintln(w, "::1             ip6-localhost ip6-loopback")
+	fmt.Fprintln(w, "fe00::0         ip6-localnet")
+	fmt.Fprintln(w, "ff00::0         ip6-mcastprefix")
+	fmt.Fprintln(w, "ff02::1         ip6-allnodes")
+	fmt.Fprintln(w, "ff02::2         ip6-allrouters")
+	fmt.Fprintln(w, "# created by Weave - END")
+}

proxy/proxy.go

@@ -3,7 +3,6 @@ package proxy
 import (
 	"bytes"
 	"crypto/tls"
-	"errors"
 	"fmt"
 	"net"
 	"net/http"
@@ -16,10 +15,11 @@ import (
 	"time"
 
 	docker "github.com/fsouza/go-dockerclient"
+	"github.com/pkg/errors"
+
 	weaveapi "github.com/weaveworks/weave/api"
 	weavedocker "github.com/weaveworks/weave/common/docker"
 	weavenet "github.com/weaveworks/weave/net"
-	"github.com/weaveworks/weave/net/address"
 )
 
 const (
@@ -63,6 +63,7 @@ type Config struct {
 	Version             string
 	WithoutDNS          bool
 	NoMulticastRoute    bool
+	KeepTXOn            bool
 	DockerBridge        string
 	DockerHost          string
 }
@@ -76,7 +77,8 @@ type wait struct {
 type Proxy struct {
 	sync.Mutex
 	Config
-	client                 *docker.Client
+	client                 *weavedocker.Client
+	weave                  *weaveapi.Client
 	dockerBridgeIP         string
 	hostnameMatchRegexp    *regexp.Regexp
 	weaveWaitVolume        string
@@ -123,16 +125,13 @@ func (j attachJob) Stop() {
 	j.timer.Stop()
 }
 
-func NewProxy(c Config) (*Proxy, error) {
+func StubProxy(c Config) (*Proxy, error) {
 	p := &Proxy{
 		Config:     c,
 		waiters:    make(map[*http.Request]*wait),
 		attachJobs: make(map[string]*attachJob),
 		quit:       make(chan struct{}),
-	}
-
-	if err := p.TLSConfig.LoadCerts(); err != nil {
-		Log.Fatalf("Could not configure tls for proxy: %s", err)
+		weave:      weaveapi.NewClient(os.Getenv("WEAVE_HTTP_ADDR"), Log),
 	}
 
 	// We pin the protocol version to 1.18 (which corresponds to
@@ -144,9 +143,21 @@ func NewProxy(c Config) (*Proxy, error) {
 	if err != nil {
 		return nil, err
 	}
-	Log.Info(client.Info())
+	p.client = client
+	return p, nil
+}
 
-	p.client = client.Client
+func NewProxy(c Config) (*Proxy, error) {
+	p, err := StubProxy(c)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := p.TLSConfig.LoadCerts(); err != nil {
+		Log.Fatalf("Could not configure tls for proxy: %s", err)
+	}
+
+	Log.Info(p.client.Info())
 
 	if !p.WithoutDNS {
 		ip, err := weavenet.FindBridgeIP(c.DockerBridge, nil)
@@ -167,7 +178,7 @@ func NewProxy(c Config) (*Proxy, error) {
 		return nil, err
 	}
 
-	client.AddObserver(p)
+	p.client.AddObserver(p)
 
 	return p, nil
 }
@@ -499,53 +510,77 @@ func (proxy *Proxy) attach(containerID string) error {
 		return nil
 	}
 	Log.Infof("Attaching container %s with WEAVE_CIDR \"%s\" to weave network", container.ID, strings.Join(cidrs, " "))
-	if err := validateCIDRs(cidrs); err != nil {
+	ips, err := proxy.allocateCIDRs(container.ID, cidrs)
+	if err != nil {
 		return err
 	}
 
-	args := []string{"attach"}
-	args = append(args, cidrs...)
+	fqdn := container.Config.Hostname + "." + container.Config.Domainname
 	if !proxy.NoRewriteHosts {
-		args = append(args, "--rewrite-hosts")
-
+		var extraHosts []string
 		if container.HostConfig != nil {
-			for _, eh := range container.HostConfig.ExtraHosts {
-				args = append(args, fmt.Sprintf("--add-host=%s", eh))
+			extraHosts = container.HostConfig.ExtraHosts
+		}
+		proxy.RewriteEtcHosts(container.HostsPath, fqdn, ips, extraHosts)
+	}
+
+	pid := container.State.Pid
+	// Passing 0 for mtu means it will be taken from the bridge
+	err = weavenet.AttachContainer(weavenet.NSPathByPid(pid), fmt.Sprint(pid), weavenet.VethName, weavenet.WeaveBridgeName, 0, !proxy.NoMulticastRoute, ips, proxy.KeepTXOn)
+	if err != nil {
+		return err
+	}
+
+	if !proxy.WithoutDNS {
+		for _, ip := range ips {
+			if err := proxy.weave.RegisterWithDNS(container.ID, fqdn, ip.IP.String()); err != nil {
+				return errors.Wrapf(err, "unable to register %s with weaveDNS: %s", container.ID, err)
 			}
 		}
 	}
-	if proxy.NoMulticastRoute {
-		args = append(args, "--no-multicast-route")
-	}
-	args = append(args, container.ID)
-	return callWeaveAttach(container, args)
+
+	return err
 }
 
-func callWeaveAttach(container *docker.Container, args []string) error {
-	if _, stderr, err := callWeave(args...); err != nil {
-		Log.Warningf("Attaching container %s to weave network failed: %s", container.ID, string(stderr))
-		return errors.New(string(stderr))
-	} else if len(stderr) > 0 {
-		Log.Warningf("Attaching container %s to weave network: %s", container.ID, string(stderr))
+func (proxy *Proxy) allocateCIDRs(containerID string, cidrs []string) ([]*net.IPNet, error) {
+	if len(cidrs) == 0 {
+		cidrs = []string{"net:default"}
 	}
-	return nil
-}
-
-func validateCIDRs(cidrs []string) error {
+	var ipnet *net.IPNet
+	var err error
+	var ipnets []*net.IPNet
 	for _, cidr := range cidrs {
-		if cidr == "net:default" {
-			continue
-		}
-		for _, prefix := range []string{"ip:", "net:", ""} {
-			if strings.HasPrefix(cidr, prefix) {
-				if _, err := address.ParseCIDR(strings.TrimPrefix(cidr, prefix)); err == nil {
-					break
-				}
-				return fmt.Errorf("invalid WEAVE_CIDR: %s", cidr)
+		switch {
+		case cidr == "net:default":
+			ipnet, err = proxy.weave.AllocateIP(containerID, true)
+		case strings.HasPrefix(cidr, "net:"):
+			var subnet *net.IPNet
+			_, subnet, err = net.ParseCIDR(strings.TrimPrefix(cidr, "net:"))
+			if err != nil {
+				break
 			}
+			ipnet, err = proxy.weave.AllocateIPInSubnet(containerID, subnet, true)
+		case strings.HasPrefix(cidr, "ip:"):
+			ipnet, err = proxy.claimCIDR(containerID, strings.TrimPrefix(cidr, "ip:"))
+		default:
+			ipnet, err = proxy.claimCIDR(containerID, cidr)
 		}
+		if err != nil {
+			return nil, errors.Wrapf(err, "for %q", cidr)
+		}
+		ipnets = append(ipnets, ipnet)
 	}
-	return nil
+	return ipnets, nil
+}
+
+func (proxy *Proxy) claimCIDR(containerID, cidr string) (*net.IPNet, error) {
+	ip, ipnet, err := net.ParseCIDR(cidr)
+	if err != nil {
+		return nil, err
+	}
+	ipnet.IP = ip // we want the specific IP plus the mask
+	err = proxy.weave.ClaimIP(containerID, ipnet)
+	return ipnet, err
 }
 
 func (proxy *Proxy) weaveCIDRs(networkMode string, env []string) ([]string, error) {
@@ -594,8 +629,7 @@ func (proxy *Proxy) getDNSDomain() string {
 	if proxy.WithoutDNS {
 		return ""
 	}
-	weave := weaveapi.NewClient(os.Getenv("WEAVE_HTTP_ADDR"), Log)
-	domain, _ := weave.DNSDomain()
+	domain, _ := proxy.weave.DNSDomain()
 	return domain
 }
 
@@ -634,7 +668,6 @@ func (proxy *Proxy) updateContainerNetworkSettings(container jsonObject) error {
 }
 
 func (proxy *Proxy) symlink(unixAddrs []string) (err error) {
-	var container *docker.Container
 	binds := []string{"/var/run/weave:/var/run/weave"}
 	froms := []string{}
 	for _, addr := range unixAddrs {
@@ -650,7 +683,10 @@ func (proxy *Proxy) symlink(unixAddrs []string) (err error) {
 	if len(froms) == 0 {
 		return
 	}
+	return proxy.runTransientContainer([]string{"/home/weave/symlink", weaveSock}, froms, binds)
+}
 
+func (proxy *Proxy) runTransientContainer(entrypoint, cmd, binds []string) (err error) {
 	env := []string{
 		"PATH=/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
 	}
@@ -659,14 +695,15 @@ func (proxy *Proxy) symlink(unixAddrs []string) (err error) {
 		env = append(env, fmt.Sprintf("%s=%s", "WEAVE_DEBUG", val))
 	}
 
+	var container *docker.Container
 	container, err = proxy.client.CreateContainer(docker.CreateContainerOptions{
 		Config: &docker.Config{
 			Image:      proxy.Image,
-			Entrypoint: []string{"/home/weave/symlink", weaveSock},
-			Cmd:        froms,
+			Entrypoint: entrypoint,
+			Cmd:        cmd,
 			Env:        env,
 		},
-		HostConfig: &docker.HostConfig{Binds: binds},
+		HostConfig: &docker.HostConfig{Binds: binds, NetworkMode: "none"},
 	})
 	if err != nil {
 		return

weave

@@ -851,52 +851,13 @@ dns_args() {
     fi
 }
 
-etc_hosts_contents() {
-    FQDN=$1
-    shift
-    NAME=${FQDN%%.*}
-    HOSTNAMES="$NAME"
-    [ "$NAME" = "$FQDN" -o "$NAME." = "$FQDN" ] || HOSTNAMES="${FQDN%.} $HOSTNAMES"
-
-    echo "# created by Weave - BEGIN"
-    echo "# container hostname"
-    for CIDR in $ALL_CIDRS ; do
-        echo "${CIDR%/*}    $HOSTNAMES"
-    done
-    echo
-    echo "# static names added with --add-host"
-    for EXTRA_HOST in "$@" ; do
-        echo "${EXTRA_HOST#*:}     ${EXTRA_HOST%:*}"
-    done
-
-    cat <<-EOF
-
-# default localhost entries
-127.0.0.1       localhost
-::1             ip6-localhost ip6-loopback
-fe00::0         ip6-localnet
-ff00::0         ip6-mcastprefix
-ff02::1         ip6-allnodes
-ff02::2         ip6-allrouters
-EOF
-
-    echo "# created by Weave - END"
-}
-
 rewrite_etc_hosts() {
     HOSTS_PATH_AND_FQDN=$(docker inspect -f '{{.HostsPath}} {{.Config.Hostname}}.{{.Config.Domainname}}' $CONTAINER) || return 1
     HOSTS=${HOSTS_PATH_AND_FQDN% *}
     FQDN=${HOSTS_PATH_AND_FQDN#* }
-    CONTAINERS_PATH=$(dirname $HOSTS)
-    MNT=/container
-    MNT_HOSTS=$MNT/$(basename $HOSTS)
-    CONTENTS="$(etc_hosts_contents $FQDN "$@")"
     # rewrite /etc/hosts, unlinking the file (so Docker does not modify it again) but
     # leaving it with valid contents...
-    docker run --rm --net=none -e WEAVE_CIDR=none \
-        -v $CONTAINERS_PATH:$MNT \
-        --entrypoint=sh \
-        $EXEC_IMAGE -c "echo '$CONTENTS' > $MNT_HOSTS && rm -f $MNT_HOSTS && echo '$CONTENTS' > $MNT_HOSTS"
+    util_op rewrite-etc-hosts "$HOSTS" "$FQDN" "$EXEC_IMAGE" "$ALL_CIDRS" "$@"
 }
 
 # Print an error to stderr and return with an indicative exit status
@@ -1361,9 +1322,7 @@ launch() {
         -v /etc:/host/etc -v /var/lib/dbus:/host/var/lib/dbus \
         -e DOCKER_BRIDGE \
         -e WEAVE_DEBUG \
-        -e COVERAGE \
         -e WEAVE_HTTP_ADDR \
-        -e EXEC_IMAGE=$EXEC_IMAGE \
         -e WEAVE_PASSWORD \
         -e CHECKPOINT_DISABLE \
         $WEAVE_DOCKER_ARGS \