From bd2b8df7c56ea963672b1fba0e44849b9cb198c1 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 26 Nov 2015 15:17:50 +0000 Subject: [PATCH 01/10] Refactor to use Weave's docker client and watcher --- common/docker/client.go | 13 +++++++++++++ plugin/dns.go | 4 ++-- plugin/docker.go | 16 +--------------- plugin/driver.go | 2 +- plugin/ipam.go | 4 ++-- plugin/watcher.go | 28 ++++++---------------------- 6 files changed, 25 insertions(+), 42 deletions(-) diff --git a/common/docker/client.go b/common/docker/client.go index 6a4eaff1..ded610b3 100644 --- a/common/docker/client.go +++ b/common/docker/client.go @@ -80,3 +80,16 @@ func (c *Client) IsContainerNotRunning(idStr string) bool { Log.Errorf("[docker] Could not check container status: %s", err) return false } + +func (c *Client) GetContainerBridgeIP(nameOrID string) (string, error) { + Log.Debugf("Getting IP for container %s", nameOrID) + info, err := c.InspectContainer(nameOrID) + if err != nil { + return "", err + } + return info.NetworkSettings.IPAddress, nil +} + +func (c *Client) InspectContainer(nameOrId string) (*docker.Container, error) { + return c.Client.InspectContainer(nameOrId) +} diff --git a/plugin/dns.go b/plugin/dns.go index 27652cfe..526d46bd 100644 --- a/plugin/dns.go +++ b/plugin/dns.go @@ -8,7 +8,7 @@ import ( ) func (d *dockerer) registerWithDNS(ID string, fqdn string, ip string) error { - dnsip, err := d.getContainerBridgeIP(WeaveDNSContainer) + dnsip, err := d.client.GetContainerBridgeIP(WeaveContainer) if err != nil { return fmt.Errorf("nameserver not available: %s", err) } @@ -33,7 +33,7 @@ func (d *dockerer) registerWithDNS(ID string, fqdn string, ip string) error { } func (d *dockerer) deregisterWithDNS(ID string, ip string) error { - dnsip, err := d.getContainerBridgeIP(WeaveDNSContainer) + dnsip, err := d.client.GetContainerBridgeIP(WeaveContainer) if err != nil { return fmt.Errorf("nameserver not available: %s", err) } diff --git a/plugin/docker.go b/plugin/docker.go index d7b4b337..4934d737 100644 --- a/plugin/docker.go +++ b/plugin/docker.go @@ -1,23 +1,9 @@ package plugin import ( - "github.com/fsouza/go-dockerclient" - . "github.com/weaveworks/weave/common" + "github.com/weaveworks/weave/common/docker" ) type dockerer struct { client *docker.Client } - -func (d *dockerer) getContainerBridgeIP(nameOrID string) (string, error) { - Log.Debugf("Getting IP for container %s", nameOrID) - info, err := d.InspectContainer(nameOrID) - if err != nil { - return "", err - } - return info.NetworkSettings.IPAddress, nil -} - -func (d *dockerer) InspectContainer(nameOrId string) (*docker.Container, error) { - return d.client.InspectContainer(nameOrId) -} diff --git a/plugin/driver.go b/plugin/driver.go index 8467c58f..85cded7c 100644 --- a/plugin/driver.go +++ b/plugin/driver.go @@ -7,10 +7,10 @@ import ( "github.com/docker/libnetwork/types" . "github.com/weaveworks/weave/common" + "github.com/weaveworks/weave/common/docker" "github.com/weaveworks/weave/common/odp" "github.com/weaveworks/weave/plugin/skel" - docker "github.com/fsouza/go-dockerclient" "github.com/vishvananda/netlink" ) diff --git a/plugin/ipam.go b/plugin/ipam.go index 927f7fff..bbc28aaf 100644 --- a/plugin/ipam.go +++ b/plugin/ipam.go @@ -10,7 +10,7 @@ import ( ) func (d *dockerer) ipamOp(ID string, op string) (*net.IPNet, error) { - weaveip, err := d.getContainerBridgeIP(WeaveContainer) + weaveip, err := d.client.GetContainerBridgeIP(WeaveContainer) Log.Debugf("IPAM operation %s for %s", op, ID) if err != nil { return nil, err @@ -52,7 +52,7 @@ func (d *dockerer) lookupIP(ID string) (*net.IPNet, error) { // release an IP which is no longer needed func (d *dockerer) releaseIP(ID string) error { - weaveip, err := d.getContainerBridgeIP(WeaveContainer) + weaveip, err := d.client.GetContainerBridgeIP(WeaveContainer) if err != nil { return err } diff --git a/plugin/watcher.go b/plugin/watcher.go index 50f87dd5..1bfafd03 100644 --- a/plugin/watcher.go +++ b/plugin/watcher.go @@ -4,19 +4,16 @@ import ( "fmt" "strings" - "github.com/fsouza/go-dockerclient" . "github.com/weaveworks/weave/common" + "github.com/weaveworks/weave/common/docker" ) const ( - WeaveDNSContainer = "weavedns" - WeaveDomain = "weave.local" + WeaveDomain = "weave.local" ) type watcher struct { dockerer - networks map[string]bool - events chan *docker.APIEvents } type Watcher interface { @@ -27,31 +24,18 @@ func NewWatcher(client *docker.Client) (Watcher, error) { dockerer: dockerer{ client: client, }, - networks: make(map[string]bool), - events: make(chan *docker.APIEvents), } - err := client.AddEventListener(w.events) + err := client.AddObserver(w) if err != nil { return nil, err } - go func() { - for event := range w.events { - switch event.Status { - case "start": - w.ContainerStart(event.ID) - case "die": - w.ContainerDied(event.ID) - } - } - }() - return w, nil } -func (w *watcher) ContainerStart(id string) { +func (w *watcher) ContainerStarted(id string) { Log.Debugf("Container started %s", id) - info, err := w.InspectContainer(id) + info, err := w.client.InspectContainer(id) if err != nil { Log.Warningf("error inspecting container: %s", err) return @@ -69,7 +53,7 @@ func (w *watcher) ContainerStart(id string) { func (w *watcher) ContainerDied(id string) { Log.Debugf("Container died %s", id) - info, err := w.InspectContainer(id) + info, err := w.client.InspectContainer(id) if err != nil { Log.Warningf("error inspecting container: %s", err) return From e101dde62e1373f523b1194f8f6a812669170be7 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 26 Nov 2015 16:36:19 +0000 Subject: [PATCH 02/10] Extract 'api' package --- Makefile | 2 +- api/dns.go | 18 ++++++++++ api/ipam.go | 43 +++++++++++++++++++++++ api/weaveapi.go | 59 +++++++++++++++++++++++++++++++ plugin/dns.go | 55 ----------------------------- plugin/ipam.go | 89 ----------------------------------------------- plugin/watcher.go | 22 +++++++++--- 7 files changed, 139 insertions(+), 149 deletions(-) create mode 100644 api/dns.go create mode 100644 api/ipam.go create mode 100644 api/weaveapi.go delete mode 100644 plugin/dns.go delete mode 100644 plugin/ipam.go diff --git a/Makefile b/Makefile index d802efc0..d2adf329 100644 --- a/Makefile +++ b/Makefile @@ -82,7 +82,7 @@ $(NETCHECK_EXE): prog/netcheck/netcheck.go $(SIGPROXY_EXE): prog/sigproxy/main.go $(WEAVEWAIT_EXE): prog/weavewait/*.go net/*.go $(DOCKERTLSARGS_EXE): prog/docker_tls_args/*.go -$(DOCKERPLUGIN_EXE): prog/plugin/*.go plugin/*.go plugin/skel/*.go +$(DOCKERPLUGIN_EXE): prog/plugin/*.go plugin/*.go plugin/skel/*.go api/*.go common/docker/*.go $(WEAVEWAIT_EXE) $(SIGPROXY_EXE) $(DOCKERTLSARGS_EXE) $(DOCKERPLUGIN_EXE): go get -tags netgo ./$(@D) diff --git a/api/dns.go b/api/dns.go new file mode 100644 index 00000000..750c34f2 --- /dev/null +++ b/api/dns.go @@ -0,0 +1,18 @@ +package api + +import ( + "fmt" + "net/url" +) + +func (client *Client) RegisterWithDNS(ID string, fqdn string, ip string) error { + data := url.Values{} + data.Add("fqdn", fqdn) + _, err := httpVerb("PUT", fmt.Sprintf("%s/name/%s/%s", client.baseUrl, ID, ip), data) + return err +} + +func (client *Client) DeregisterWithDNS(ID string, ip string) error { + _, err := httpVerb("DELETE", fmt.Sprintf("%s/name/%s/%s", client.baseUrl, ID, ip), nil) + return err +} diff --git a/api/ipam.go b/api/ipam.go new file mode 100644 index 00000000..91a3053a --- /dev/null +++ b/api/ipam.go @@ -0,0 +1,43 @@ +package api + +import ( + "fmt" + "net" + + . "github.com/weaveworks/weave/common" +) + +func (client *Client) ipamOp(ID string, op string) (*net.IPNet, error) { + Log.Debugf("IPAM operation %s for %s", op, ID) + ip, err := httpVerb(op, fmt.Sprintf("%s/ip/%s", client.baseUrl, ID), nil) + if err != nil { + return nil, err + } + return parseIP(ip) +} + +// returns an IP for the ID given, allocating a fresh one if necessary +func (client *Client) AllocateIP(ID string) (*net.IPNet, error) { + return client.ipamOp(ID, "POST") +} + +// returns an IP for the ID given, or nil if one has not been +// allocated +func (client *Client) LookupIP(ID string) (*net.IPNet, error) { + return client.ipamOp(ID, "GET") +} + +// release an IP which is no longer needed +func (client *Client) ReleaseIP(ID string) error { + _, err := client.ipamOp(ID, "DELETE") + return err +} + +func parseIP(body string) (*net.IPNet, error) { + ip, ipnet, err := net.ParseCIDR(string(body)) + if err != nil { + return nil, err + } + ipnet.IP = ip + return ipnet, nil +} diff --git a/api/weaveapi.go b/api/weaveapi.go new file mode 100644 index 00000000..d75394bd --- /dev/null +++ b/api/weaveapi.go @@ -0,0 +1,59 @@ +package api + +import ( + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "strings" + + . "github.com/weaveworks/weave/common" +) + +const ( + WeaveHttpPort = 6784 +) + +type Client struct { + baseUrl string +} + +func httpVerb(verb string, url string, values url.Values) (string, error) { + Log.Debugf("weave %s to %s with %v", verb, url, values) + var body io.Reader + if values != nil { + body = strings.NewReader(values.Encode()) + } + req, err := http.NewRequest(verb, url, body) + if err != nil { + return "", err + } + if values != nil { + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + } + if resp, err := http.DefaultClient.Do(req); err != nil { + return "", err + } else { + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNoContent { + return string(body), nil + } else { + return "", errors.New(resp.Status + ": " + string(body)) + } + } +} + +func NewClient(addr string) *Client { + return &Client{baseUrl: fmt.Sprintf("http://%s:%d", addr, WeaveHttpPort)} +} + +func (client *Client) Connect(remote string) error { + _, err := httpVerb("POST", client.baseUrl+"/connect", url.Values{"peer": {remote}}) + return err +} diff --git a/plugin/dns.go b/plugin/dns.go deleted file mode 100644 index 526d46bd..00000000 --- a/plugin/dns.go +++ /dev/null @@ -1,55 +0,0 @@ -package plugin - -import ( - "fmt" - "net/http" - "net/url" - "strings" -) - -func (d *dockerer) registerWithDNS(ID string, fqdn string, ip string) error { - dnsip, err := d.client.GetContainerBridgeIP(WeaveContainer) - if err != nil { - return fmt.Errorf("nameserver not available: %s", err) - } - data := url.Values{} - data.Add("fqdn", fqdn) - - req, err := http.NewRequest("PUT", fmt.Sprintf("http://%s:6785/name/%s/%s", dnsip, ID, ip), strings.NewReader(data.Encode())) - if err != nil { - return err - } - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - - cl := &http.Client{} - res, err := cl.Do(req) - if err != nil { - return err - } - if res.StatusCode != http.StatusOK { - return fmt.Errorf("non-OK status from nameserver: %d", res.StatusCode) - } - return nil -} - -func (d *dockerer) deregisterWithDNS(ID string, ip string) error { - dnsip, err := d.client.GetContainerBridgeIP(WeaveContainer) - if err != nil { - return fmt.Errorf("nameserver not available: %s", err) - } - - req, err := http.NewRequest("DELETE", fmt.Sprintf("http://%s:6785/name/%s/%s", dnsip, ID, ip), nil) - if err != nil { - return err - } - - cl := &http.Client{} - res, err := cl.Do(req) - if err != nil { - return err - } - if res.StatusCode != http.StatusOK { - return fmt.Errorf("non-OK status from nameserver: %d", res.StatusCode) - } - return nil -} diff --git a/plugin/ipam.go b/plugin/ipam.go deleted file mode 100644 index bbc28aaf..00000000 --- a/plugin/ipam.go +++ /dev/null @@ -1,89 +0,0 @@ -package plugin - -import ( - "fmt" - "io/ioutil" - "net" - "net/http" - - . "github.com/weaveworks/weave/common" -) - -func (d *dockerer) ipamOp(ID string, op string) (*net.IPNet, error) { - weaveip, err := d.client.GetContainerBridgeIP(WeaveContainer) - Log.Debugf("IPAM operation %s for %s", op, ID) - if err != nil { - return nil, err - } - - var res *http.Response - - url := fmt.Sprintf("http://%s:6784/ip/%s", weaveip, ipamID(ID)) - Log.Debugf("Attempting to %s to %s", op, url) - if op == "POST" { - res, err = http.Post(url, "", nil) - } else if op == "GET" { - res, err = http.Get(url) - } - - if err != nil { - return nil, err - } - if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("received status %d from IPAM", res.StatusCode) - } - body, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - return parseIP(string(body)) -} - -// returns an IP for the ID given, allocating a fresh one if necessary -func (d *dockerer) allocateIP(ID string) (*net.IPNet, error) { - return d.ipamOp(ID, "POST") -} - -// returns an IP for the ID given, or nil if one has not been -// allocated -func (d *dockerer) lookupIP(ID string) (*net.IPNet, error) { - return d.ipamOp(ID, "GET") -} - -// release an IP which is no longer needed -func (d *dockerer) releaseIP(ID string) error { - weaveip, err := d.client.GetContainerBridgeIP(WeaveContainer) - if err != nil { - return err - } - req, err := http.NewRequest("DELETE", fmt.Sprintf("http://%s:6784/ip/%s", weaveip, ipamID(ID)), nil) - if err != nil { - return err - } - cl := &http.Client{} - res, err := cl.Do(req) - if err != nil { - return err - } - if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusNoContent { - return fmt.Errorf("unexpected HTTP status code from IP release: %d", res.StatusCode) - } - return nil -} - -func parseIP(body string) (*net.IPNet, error) { - ip, ipnet, err := net.ParseCIDR(string(body)) - if err != nil { - return nil, err - } - ipnet.IP = ip - return ipnet, nil -} - -// If something looking like a container ID is supplied to IPAM, it -// will try to check that it's running, and return nothing (and empty -// string) if it's not. So I make sure it doesn't look like a -// container ID by including non-hex characters. -func ipamID(ID string) string { - return "endpoint:" + ID -} diff --git a/plugin/watcher.go b/plugin/watcher.go index 1bfafd03..3dbbe504 100644 --- a/plugin/watcher.go +++ b/plugin/watcher.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" + weaveapi "github.com/weaveworks/weave/api" . "github.com/weaveworks/weave/common" "github.com/weaveworks/weave/common/docker" ) @@ -14,6 +15,7 @@ const ( type watcher struct { dockerer + weave *weaveapi.Client } type Watcher interface { @@ -33,6 +35,18 @@ func NewWatcher(client *docker.Client) (Watcher, error) { return w, nil } +func (w *watcher) haveWeaveClient() bool { + if w.weave == nil { + dnsip, err := w.client.GetContainerBridgeIP(WeaveContainer) + if err != nil { + Log.Warningf("nameserver not available: %s", err) + return false + } + w.weave = weaveapi.NewClient(dnsip) + } + return true +} + func (w *watcher) ContainerStarted(id string) { Log.Debugf("Container started %s", id) info, err := w.client.InspectContainer(id) @@ -41,11 +55,11 @@ func (w *watcher) ContainerStarted(id string) { return } // FIXME: check that it's on our network; but, the docker client lib doesn't know about .NetworkID - if isSubdomain(info.Config.Domainname, WeaveDomain) { + if isSubdomain(info.Config.Domainname, WeaveDomain) && w.haveWeaveClient() { // one of ours ip := info.NetworkSettings.IPAddress fqdn := fmt.Sprintf("%s.%s", info.Config.Hostname, info.Config.Domainname) - if err := w.registerWithDNS(id, fqdn, ip); err != nil { + if err := w.weave.RegisterWithDNS(id, fqdn, ip); err != nil { Log.Warningf("unable to register with weaveDNS: %s", err) } } @@ -58,9 +72,9 @@ func (w *watcher) ContainerDied(id string) { Log.Warningf("error inspecting container: %s", err) return } - if isSubdomain(info.Config.Domainname, WeaveDomain) { + if isSubdomain(info.Config.Domainname, WeaveDomain) && w.haveWeaveClient() { ip := info.NetworkSettings.IPAddress - if err := w.deregisterWithDNS(id, ip); err != nil { + if err := w.weave.DeregisterWithDNS(id, ip); err != nil { Log.Warningf("unable to deregister with weaveDNS: %s", err) } } From 85e465cfa41d055665b1daf52025836875eea65b Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 26 Nov 2015 17:16:43 +0000 Subject: [PATCH 03/10] Eliminate 'dockerer' --- plugin/docker.go | 9 --------- plugin/watcher.go | 10 +++------- 2 files changed, 3 insertions(+), 16 deletions(-) delete mode 100644 plugin/docker.go diff --git a/plugin/docker.go b/plugin/docker.go deleted file mode 100644 index 4934d737..00000000 --- a/plugin/docker.go +++ /dev/null @@ -1,9 +0,0 @@ -package plugin - -import ( - "github.com/weaveworks/weave/common/docker" -) - -type dockerer struct { - client *docker.Client -} diff --git a/plugin/watcher.go b/plugin/watcher.go index 3dbbe504..701c53b9 100644 --- a/plugin/watcher.go +++ b/plugin/watcher.go @@ -14,19 +14,15 @@ const ( ) type watcher struct { - dockerer - weave *weaveapi.Client + client *docker.Client + weave *weaveapi.Client } type Watcher interface { } func NewWatcher(client *docker.Client) (Watcher, error) { - w := &watcher{ - dockerer: dockerer{ - client: client, - }, - } + w := &watcher{client: client} err := client.AddObserver(w) if err != nil { return nil, err From 5eb99fe5efd90530a9bfd3e4606332bb83a6cfd2 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 26 Nov 2015 18:11:37 +0000 Subject: [PATCH 04/10] Get container IP for various networks --- common/docker/client.go | 15 ++++++++++++++- plugin/watcher.go | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/common/docker/client.go b/common/docker/client.go index ded610b3..f6de8981 100644 --- a/common/docker/client.go +++ b/common/docker/client.go @@ -81,12 +81,25 @@ func (c *Client) IsContainerNotRunning(idStr string) bool { return false } -func (c *Client) GetContainerBridgeIP(nameOrID string) (string, error) { +// This is intended to find an IP address that we can reach the container on; +// if it is on the Docker bridge network then that address; if on the host network +// then localhost +func (c *Client) GetContainerIP(nameOrID string) (string, error) { Log.Debugf("Getting IP for container %s", nameOrID) info, err := c.InspectContainer(nameOrID) if err != nil { return "", err } + if info.NetworkSettings.Networks != nil { + Log.Debugln("Networks: ", info.NetworkSettings.Networks) + if bridgeNetwork, ok := info.NetworkSettings.Networks["bridge"]; ok { + return bridgeNetwork.IPAddress, nil + } else if _, ok := info.NetworkSettings.Networks["host"]; ok { + return "127.0.0.1", nil + } + } else if info.HostConfig.NetworkMode == "host" { + return "127.0.0.1", nil + } return info.NetworkSettings.IPAddress, nil } diff --git a/plugin/watcher.go b/plugin/watcher.go index 701c53b9..c37b41e0 100644 --- a/plugin/watcher.go +++ b/plugin/watcher.go @@ -33,7 +33,7 @@ func NewWatcher(client *docker.Client) (Watcher, error) { func (w *watcher) haveWeaveClient() bool { if w.weave == nil { - dnsip, err := w.client.GetContainerBridgeIP(WeaveContainer) + dnsip, err := w.client.GetContainerIP(WeaveContainer) if err != nil { Log.Warningf("nameserver not available: %s", err) return false From a22b88ed89110125893ddcb76fa3d0eb437b8f9e Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Fri, 27 Nov 2015 16:14:23 +0000 Subject: [PATCH 05/10] Track endpoints so we can register the right address with DNS --- plugin/driver.go | 36 +++++++++++++++++++++++++++--------- plugin/watcher.go | 43 +++++++++++++++---------------------------- 2 files changed, 42 insertions(+), 37 deletions(-) diff --git a/plugin/driver.go b/plugin/driver.go index 85cded7c..e5b1e413 100644 --- a/plugin/driver.go +++ b/plugin/driver.go @@ -2,6 +2,7 @@ package plugin import ( "fmt" + "sync" "github.com/docker/libnetwork/drivers/remote/api" "github.com/docker/libnetwork/types" @@ -22,6 +23,8 @@ const ( type driver struct { version string nameserver string + sync.RWMutex + endpoints map[string]struct{} } func New(version string, nameserver string) (skel.Driver, error) { @@ -30,15 +33,17 @@ func New(version string, nameserver string) (skel.Driver, error) { return nil, errorf("could not connect to docker: %s", err) } - _, err = NewWatcher(client) + driver := &driver{ + nameserver: nameserver, + version: version, + endpoints: make(map[string]struct{}), + } + + _, err = NewWatcher(client, driver) if err != nil { return nil, err } - - return &driver{ - nameserver: nameserver, - version: version, - }, nil + return driver, nil } func errorf(format string, a ...interface{}) error { @@ -76,18 +81,31 @@ func (driver *driver) CreateEndpoint(create *api.CreateEndpointRequest) (*api.Cr if create.Interface == nil { return nil, fmt.Errorf("Not supported: creating an interface from within CreateEndpoint") } + driver.Lock() + driver.endpoints[endID] = struct{}{} + driver.Unlock() resp := &api.CreateEndpointResponse{} Log.Infof("Create endpoint %s %+v", endID, resp) return resp, nil } -func (driver *driver) DeleteEndpoint(delete *api.DeleteEndpointRequest) error { - Log.Debugf("Delete endpoint request: %+v", delete) - Log.Infof("Delete endpoint %s", delete.EndpointID) +func (driver *driver) DeleteEndpoint(deleteReq *api.DeleteEndpointRequest) error { + Log.Debugf("Delete endpoint request: %+v", deleteReq) + Log.Infof("Delete endpoint %s", deleteReq.EndpointID) + driver.Lock() + delete(driver.endpoints, deleteReq.EndpointID) + driver.Unlock() return nil } +func (driver *driver) HasEndpoint(endpointID string) bool { + driver.Lock() + _, found := driver.endpoints[endpointID] + driver.Unlock() + return found +} + func (driver *driver) EndpointInfo(req *api.EndpointInfoRequest) (*api.EndpointInfoResponse, error) { Log.Debugf("Endpoint info request: %+v", req) Log.Infof("Endpoint info %s", req.EndpointID) diff --git a/plugin/watcher.go b/plugin/watcher.go index c37b41e0..fbf43014 100644 --- a/plugin/watcher.go +++ b/plugin/watcher.go @@ -2,7 +2,6 @@ package plugin import ( "fmt" - "strings" weaveapi "github.com/weaveworks/weave/api" . "github.com/weaveworks/weave/common" @@ -16,13 +15,14 @@ const ( type watcher struct { client *docker.Client weave *weaveapi.Client + driver *driver } type Watcher interface { } -func NewWatcher(client *docker.Client) (Watcher, error) { - w := &watcher{client: client} +func NewWatcher(client *docker.Client, driver *driver) (Watcher, error) { + w := &watcher{client: client, driver: driver} err := client.AddObserver(w) if err != nil { return nil, err @@ -50,34 +50,21 @@ func (w *watcher) ContainerStarted(id string) { Log.Warningf("error inspecting container: %s", err) return } - // FIXME: check that it's on our network; but, the docker client lib doesn't know about .NetworkID - if isSubdomain(info.Config.Domainname, WeaveDomain) && w.haveWeaveClient() { - // one of ours - ip := info.NetworkSettings.IPAddress - fqdn := fmt.Sprintf("%s.%s", info.Config.Hostname, info.Config.Domainname) - if err := w.weave.RegisterWithDNS(id, fqdn, ip); err != nil { - Log.Warningf("unable to register with weaveDNS: %s", err) + if !w.haveWeaveClient() { + return + } + // check that it's on our network, via the endpointID + for _, net := range info.NetworkSettings.Networks { + if w.driver.HasEndpoint(net.EndpointID) { + fqdn := fmt.Sprintf("%s.%s", info.Config.Hostname, info.Config.Domainname) + if err := w.weave.RegisterWithDNS(id, fqdn, net.IPAddress); err != nil { + Log.Warningf("unable to register with weaveDNS: %s", err) + } } } } func (w *watcher) ContainerDied(id string) { - Log.Debugf("Container died %s", id) - info, err := w.client.InspectContainer(id) - if err != nil { - Log.Warningf("error inspecting container: %s", err) - return - } - if isSubdomain(info.Config.Domainname, WeaveDomain) && w.haveWeaveClient() { - ip := info.NetworkSettings.IPAddress - if err := w.weave.DeregisterWithDNS(id, ip); err != nil { - Log.Warningf("unable to deregister with weaveDNS: %s", err) - } - } -} - -// Cheap and cheerful way to check x is, or is a subdomain, of -// y. Neither are expected to start with a '.'. -func isSubdomain(x string, y string) bool { - return x == y || strings.HasSuffix(x, "."+y) + // don't need to do this as WeaveDNS removes names on container died anyway + // (note by the time we get this event we can't see the EndpointID) } From cd7c88e5a51696022e096bc1afe52fdf354609d9 Mon Sep 17 00:00:00 2001 From: Paul Bellamy Date: Fri, 20 Nov 2015 16:55:17 +0000 Subject: [PATCH 06/10] [proxy] Symlink sockets to /var/run/weave/weave.sock from inside the proxy We copy docker.sock owner to symlinks. Permissions are not copied, because the permissions will inherit from the target. use versioned exec image to create symlinks --- Makefile | 2 +- prog/weaveexec/Dockerfile | 2 +- prog/weaveexec/symlink | 18 ++++ prog/weaveproxy/main.go | 6 +- proxy/proxy.go | 96 +++++++++++++++++++ test/650_proxy_env_test.sh | 6 ++ ...onfig_test.sh => 690_proxy_config_test.sh} | 7 +- weave | 24 ++++- 8 files changed, 152 insertions(+), 9 deletions(-) create mode 100755 prog/weaveexec/symlink rename test/{690_proxy_autoconfig_test.sh => 690_proxy_config_test.sh} (81%) diff --git a/Makefile b/Makefile index d802efc0..c27dbc3d 100644 --- a/Makefile +++ b/Makefile @@ -96,7 +96,7 @@ $(WEAVER_UPTODATE): prog/weaver/Dockerfile $(WEAVER_EXE) $(SUDO) docker build -t $(WEAVER_IMAGE) prog/weaver touch $@ -$(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(NETCHECK_EXE) $(DOCKERTLSARGS_EXE) +$(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile prog/weaveexec/symlink $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(NETCHECK_EXE) $(DOCKERTLSARGS_EXE) cp weave prog/weaveexec/weave cp $(SIGPROXY_EXE) prog/weaveexec/sigproxy cp $(WEAVEPROXY_EXE) prog/weaveexec/weaveproxy diff --git a/prog/weaveexec/Dockerfile b/prog/weaveexec/Dockerfile index 3003d03e..2b44f344 100644 --- a/prog/weaveexec/Dockerfile +++ b/prog/weaveexec/Dockerfile @@ -18,7 +18,7 @@ RUN apk add --update \ bind-tools \ && rm -rf /var/cache/apk/* -ADD ./weave ./sigproxy ./weaveproxy /home/weave/ +ADD ./weave ./sigproxy ./weaveproxy ./symlink /home/weave/ ADD ./netcheck ./docker_tls_args /usr/bin/ ADD ./weavewait /w/w ADD ./weavewait_noop /w-noop/w diff --git a/prog/weaveexec/symlink b/prog/weaveexec/symlink new file mode 100755 index 00000000..56078149 --- /dev/null +++ b/prog/weaveexec/symlink @@ -0,0 +1,18 @@ +#!/bin/sh +# This lets us force-symlink a bunch of stuff all at once, so we can minimize +# running containers. + +set -e + +[ -z "$DEBUG" ] || set -x + +target="$1" +user="$(stat -c '%u:%g' "$target")" +mode="$(stat -c '%a' "$target")" +shift 1 + +for link in "$@" ; do + ln -sfT "$target" "$link" +done +chown -h "$user" "$@" +chmod "$mode" "$@" diff --git a/prog/weaveproxy/main.go b/prog/weaveproxy/main.go index b166d5b3..9db07cd5 100644 --- a/prog/weaveproxy/main.go +++ b/prog/weaveproxy/main.go @@ -19,7 +19,7 @@ func main() { var ( justVersion bool logLevel = "info" - c = proxy.Config{ListenAddrs: []string{}} + c = proxy.Config{Image: "weaveworks/weaveexec"} ) c.Version = version @@ -53,6 +53,10 @@ func main() { SetLogLevel(logLevel) + if image := os.Getenv("EXEC_IMAGE"); image != "" { + c.Image = image + } + Log.Infoln("weave proxy", version) Log.Infoln("Command line arguments:", strings.Join(os.Args[1:], " ")) diff --git a/proxy/proxy.go b/proxy/proxy.go index d6fdaa13..b43a288f 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -1,6 +1,7 @@ package proxy import ( + "bytes" "crypto/tls" "errors" "fmt" @@ -8,6 +9,7 @@ import ( "net" "net/http" "os" + "path/filepath" "regexp" "strings" "sync" @@ -25,6 +27,9 @@ const ( defaultCertFile = "cert.pem" dockerSock = "/var/run/docker.sock" dockerSockUnix = "unix://" + dockerSock + + weaveSock = "/var/run/weave/weave.sock" + weaveSockUnix = "unix://" + weaveSock ) var ( @@ -46,6 +51,7 @@ type Config struct { HostnameFromLabel string HostnameMatch string HostnameReplacement string + Image string ListenAddrs []string RewriteInspect bool NoDefaultIPAM bool @@ -186,7 +192,12 @@ func (proxy *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (proxy *Proxy) Listen() []net.Listener { listeners := []net.Listener{} proxy.normalisedAddrs = []string{} + unixAddrs := []string{} for _, addr := range proxy.ListenAddrs { + if strings.HasPrefix(addr, "unix://") { + unixAddrs = append(unixAddrs, addr) + continue + } listener, normalisedAddr, err := proxy.listen(addr) if err != nil { Log.Fatalf("Cannot listen on %s: %s", addr, err) @@ -195,6 +206,20 @@ func (proxy *Proxy) Listen() []net.Listener { proxy.normalisedAddrs = append(proxy.normalisedAddrs, normalisedAddr) } + if len(unixAddrs) > 0 { + listener, _, err := proxy.listen(weaveSockUnix) + if err != nil { + Log.Fatalf("Cannot listen on %s: %s", weaveSockUnix, err) + } + listeners = append(listeners, listener) + + if err := proxy.symlink(unixAddrs); err != nil { + Log.Fatalf("Cannot listen on unix sockets: %s", err) + } + + proxy.normalisedAddrs = append(proxy.normalisedAddrs, weaveSockUnix) + } + for _, addr := range proxy.normalisedAddrs { Log.Infoln("proxy listening on", addr) } @@ -527,3 +552,74 @@ func (proxy *Proxy) updateContainerNetworkSettings(container jsonObject) error { networkSettings["IPPrefixLen"], _ = nets[0].Mask.Size() return nil } + +func (proxy *Proxy) symlink(unixAddrs []string) (err error) { + var container *docker.Container + binds := []string{"/var/run/weave:/var/run/weave"} + froms := []string{} + for _, addr := range unixAddrs { + if addr == weaveSockUnix { + continue + } + from := strings.TrimPrefix(addr, "unix://") + dir := filepath.Dir(from) + binds = append(binds, dir+":"+filepath.Join("/host", dir)) + froms = append(froms, filepath.Join("/host", from)) + proxy.normalisedAddrs = append(proxy.normalisedAddrs, addr) + } + if len(froms) == 0 { + return nil + } + + env := []string{ + "PATH=/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + } + + if val := os.Getenv("DEBUG"); val != "" { + env = append(env, fmt.Sprintf("%s=%s", "DEBUG", val)) + } + + container, err = proxy.client.CreateContainer(docker.CreateContainerOptions{ + Config: &docker.Config{ + Image: proxy.Image, + Entrypoint: []string{"/home/weave/symlink", weaveSock}, + Cmd: froms, + Env: env, + }, + HostConfig: &docker.HostConfig{Binds: binds}, + }) + if err != nil { + return err + } + + defer func() { + err2 := proxy.client.RemoveContainer(docker.RemoveContainerOptions{ID: container.ID}) + if err == nil { + err = err2 + } + }() + + if err := proxy.client.StartContainer(container.ID, nil); err != nil { + return err + } + + var buf bytes.Buffer + err = proxy.client.AttachToContainer(docker.AttachToContainerOptions{ + Container: container.ID, + ErrorStream: &buf, + Logs: true, + Stderr: true, + }) + if err != nil { + return err + } + + rc, err := proxy.client.WaitContainer(container.ID) + if err != nil { + return err + } + if rc != 0 { + return errors.New(buf.String()) + } + return nil +} diff --git a/test/650_proxy_env_test.sh b/test/650_proxy_env_test.sh index 593f3d6d..345f661e 100755 --- a/test/650_proxy_env_test.sh +++ b/test/650_proxy_env_test.sh @@ -28,4 +28,10 @@ weave_on $HOST1 stop weave_on $HOST1 launch-proxy -H tcp://0.0.0.0:12375 -H unix:///var/run/weave/weave.sock check +# Check we can use weave env/config with unix -Hs specified +weave_on $HOST1 stop +weave_on $HOST1 launch-proxy -H unix:///var/run/weave/weave.sock +assert_raises "run_on $HOST1 'eval \$(weave env) ; docker $CMD'" +assert_raises "run_on $HOST1 'docker \$(weave config) $CMD'" + end_suite diff --git a/test/690_proxy_autoconfig_test.sh b/test/690_proxy_config_test.sh similarity index 81% rename from test/690_proxy_autoconfig_test.sh rename to test/690_proxy_config_test.sh index ce6e18b2..6293a3f5 100755 --- a/test/690_proxy_autoconfig_test.sh +++ b/test/690_proxy_config_test.sh @@ -2,7 +2,7 @@ . ./config.sh -start_suite "Boot the proxy should only listen on client's interface" +start_suite "Various launch-proxy configurations" # Booting it over unix socket listens on unix socket run_on $HOST1 COVERAGE=$COVERAGE sudo -E weave launch-proxy @@ -22,6 +22,11 @@ assert_raises "run_on $HOST1 sudo docker -H unix:///var/run/weave/weave.sock ps" assert_raises "proxy docker_on $HOST1 ps" weave_on $HOST1 stop-proxy +# Booting it with -H outside /var/run/weave, still works +socket="$(mktemp -d)/weave.sock" +weave_on $HOST1 launch-proxy -H unix://$socket +assert_raises "run_on $HOST1 sudo docker -H unix:///$socket ps" 0 +weave_on $HOST1 stop-proxy # Booting it over tls errors assert_raises "DOCKER_CLIENT_ARGS='--tls' weave_on $HOST1 launch-proxy" 1 diff --git a/weave b/weave index 30acc00e..55300e55 100755 --- a/weave +++ b/weave @@ -1275,17 +1275,30 @@ tls_arg() { PROXY_ARGS="$PROXY_ARGS $1 /home/weave/tls/$3.pem" } +# TODO: Handle relative paths for args +# TODO: Handle args with spaces +host_arg() { + PROXY_HOST="$1" + if [ "$PROXY_HOST" != "${PROXY_HOST#unix://}" ]; then + host=$(dirname ${PROXY_HOST#unix://}) + if [ "$host" = "${host#/}" ]; then + echo "When launching the proxy, unix sockets must be specified as an absolute path." >&2 + exit 1 + fi + PROXY_VOLUMES="$PROXY_VOLUMES -v /var/run/weave:/var/run/weave" + fi + PROXY_ARGS="$PROXY_ARGS -H $1" +} + proxy_parse_args() { while [ $# -gt 0 ]; do case "$1" in -H) - PROXY_HOST="$2" - PROXY_ARGS="$PROXY_ARGS $1 $2" + host_arg "$2" shift ;; -H=*) - PROXY_HOST="${1#*=}" - PROXY_ARGS="$PROXY_ARGS $1" + host_arg "${1#*=}" ;; -no-detect-tls|--no-detect-tls) PROXY_TLS_DETECTION_DISABLED=1 @@ -1346,7 +1359,7 @@ proxy_args() { PROXY_HOST="tcp://0.0.0.0:12375" ;; esac - PROXY_ARGS="$PROXY_ARGS -H $PROXY_HOST" + host_arg "$PROXY_HOST" fi } @@ -1559,6 +1572,7 @@ launch_proxy() { -e DOCKER_BRIDGE \ -e WEAVE_DEBUG \ -e COVERAGE \ + -e EXEC_IMAGE=$EXEC_IMAGE \ --entrypoint=/home/weave/weaveproxy \ $WEAVEPROXY_DOCKER_ARGS $EXEC_IMAGE $COVERAGE_ARGS $PROXY_ARGS) wait_for_status $PROXY_CONTAINER_NAME http_call_unix $PROXY_CONTAINER_NAME status.sock From 295a2ced320849dc0fbf62af6927f454c63938d1 Mon Sep 17 00:00:00 2001 From: Paul Bellamy Date: Tue, 1 Dec 2015 15:45:32 +0000 Subject: [PATCH 07/10] [proxy] treat addrs prefix with / as unix addrs --- proxy/proxy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/proxy.go b/proxy/proxy.go index b43a288f..b23aebd1 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -194,7 +194,7 @@ func (proxy *Proxy) Listen() []net.Listener { proxy.normalisedAddrs = []string{} unixAddrs := []string{} for _, addr := range proxy.ListenAddrs { - if strings.HasPrefix(addr, "unix://") { + if strings.HasPrefix(addr, "unix://") || strings.HasPrefix(addr, "/") { unixAddrs = append(unixAddrs, addr) continue } From 6f25d2170d150eb353080b05450a93b8d78a3981 Mon Sep 17 00:00:00 2001 From: Paul Bellamy Date: Tue, 1 Dec 2015 15:51:16 +0000 Subject: [PATCH 08/10] [proxy] have symlinking respond to WEAVE_DEBUG not DEBUG --- prog/weaveexec/symlink | 2 +- proxy/proxy.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/prog/weaveexec/symlink b/prog/weaveexec/symlink index 56078149..2c06e901 100755 --- a/prog/weaveexec/symlink +++ b/prog/weaveexec/symlink @@ -4,7 +4,7 @@ set -e -[ -z "$DEBUG" ] || set -x +[ -z "$WEAVE_DEBUG" ] || set -x target="$1" user="$(stat -c '%u:%g' "$target")" diff --git a/proxy/proxy.go b/proxy/proxy.go index b23aebd1..d267a610 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -575,8 +575,8 @@ func (proxy *Proxy) symlink(unixAddrs []string) (err error) { "PATH=/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", } - if val := os.Getenv("DEBUG"); val != "" { - env = append(env, fmt.Sprintf("%s=%s", "DEBUG", val)) + if val := os.Getenv("WEAVE_DEBUG"); val != "" { + env = append(env, fmt.Sprintf("%s=%s", "WEAVE_DEBUG", val)) } container, err = proxy.client.CreateContainer(docker.CreateContainerOptions{ From ac1cf633c72d14a319b1ff707ba5665661287667 Mon Sep 17 00:00:00 2001 From: Paul Bellamy Date: Tue, 1 Dec 2015 16:09:41 +0000 Subject: [PATCH 09/10] clarify some variable shadowing --- proxy/proxy.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/proxy/proxy.go b/proxy/proxy.go index d267a610..bf14cb76 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -568,7 +568,7 @@ func (proxy *Proxy) symlink(unixAddrs []string) (err error) { proxy.normalisedAddrs = append(proxy.normalisedAddrs, addr) } if len(froms) == 0 { - return nil + return } env := []string{ @@ -589,7 +589,7 @@ func (proxy *Proxy) symlink(unixAddrs []string) (err error) { HostConfig: &docker.HostConfig{Binds: binds}, }) if err != nil { - return err + return } defer func() { @@ -599,8 +599,9 @@ func (proxy *Proxy) symlink(unixAddrs []string) (err error) { } }() - if err := proxy.client.StartContainer(container.ID, nil); err != nil { - return err + err = proxy.client.StartContainer(container.ID, nil) + if err != nil { + return } var buf bytes.Buffer @@ -611,15 +612,16 @@ func (proxy *Proxy) symlink(unixAddrs []string) (err error) { Stderr: true, }) if err != nil { - return err + return } - rc, err := proxy.client.WaitContainer(container.ID) + var rc int + rc, err = proxy.client.WaitContainer(container.ID) if err != nil { - return err + return } if rc != 0 { - return errors.New(buf.String()) + err = errors.New(buf.String()) } - return nil + return } From ec420c66d487e4136e2cfcdcfff1c8a7d811fd75 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 1 Dec 2015 16:40:19 +0000 Subject: [PATCH 10/10] Review feedback --- common/docker/client.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/common/docker/client.go b/common/docker/client.go index f6de8981..bd4b78f6 100644 --- a/common/docker/client.go +++ b/common/docker/client.go @@ -1,7 +1,9 @@ package docker import ( + "errors" "github.com/fsouza/go-dockerclient" + . "github.com/weaveworks/weave/common" ) @@ -100,9 +102,8 @@ func (c *Client) GetContainerIP(nameOrID string) (string, error) { } else if info.HostConfig.NetworkMode == "host" { return "127.0.0.1", nil } + if info.NetworkSettings.IPAddress == "" { + return "", errors.New("No IP address found for container " + nameOrID) + } return info.NetworkSettings.IPAddress, nil } - -func (c *Client) InspectContainer(nameOrId string) (*docker.Container, error) { - return c.Client.InspectContainer(nameOrId) -}