From 8b061fd555d3381298b02151da210d82040c827b Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Mon, 29 Jun 2015 15:37:15 +0000 Subject: [PATCH] Gossip-based DNS embedded in the router. - Nameserver holds a sorted list of (hostname, peer, container id, ip) tuples. - Entries are gossiped using topology gossip / broadcast - only new/updated entries are propogated as expected. - Entries for a given peer are removed when the connection to the peer is lost. - Entries are tombstoned when containers die. Tombstones are kept for 10mins, as they do not need to survive partition, only gossip propagation delay. --- Makefile | 2 +- nameserver/dns.go | 275 ++++++++++++++++++++++ nameserver/dns_test.go | 95 ++++++++ nameserver/entry.go | 184 +++++++++++++++ nameserver/entry_test.go | 113 +++++++++ nameserver/http.go | 85 +++++++ nameserver/nameserver.go | 251 ++++++++++++++++++++ nameserver/nameserver_test.go | 248 +++++++++++++++++++ net/address/address.go | 7 + prog/weaver/main.go | 46 +++- proxy/create_container_interceptor.go | 12 +- site/ipam.md | 2 +- site/proxy.md | 2 +- site/weavedns-design.md | 102 +++----- site/weavedns.md | 145 ++++-------- test/200_dns_test.sh | 2 +- test/210_dns_cross_hosts_test.sh | 35 +++ test/210_dns_multicast_test.sh | 61 ----- test/220_dns_custom_test.sh | 15 +- test/230_dns_unqualified_test.sh | 2 +- test/240_dns_add_name_test.sh | 2 +- test/250_dns_restart_test.sh | 25 ++ test/260_dns_removal_test.sh | 2 +- test/280_with_or_without_dns_test.sh | 11 +- test/290_dns_fallback_test.sh | 2 +- test/295_dns_large_response.sh | 23 ++ test/500_weave_multi_cidr_test.sh | 1 - test/630_proxy_dns_test.sh | 2 +- test/635_proxy_dns_unqualified_test.sh | 3 +- test/640_proxy_restart_reattaches_test.sh | 3 +- test/config.sh | 6 - weave | 93 +++----- 32 files changed, 1528 insertions(+), 329 deletions(-) create mode 100644 nameserver/dns.go create mode 100644 nameserver/dns_test.go create mode 100644 nameserver/entry.go create mode 100644 nameserver/entry_test.go create mode 100644 nameserver/http.go create mode 100644 nameserver/nameserver.go create mode 100644 nameserver/nameserver_test.go create mode 100755 test/210_dns_cross_hosts_test.sh delete mode 100755 test/210_dns_multicast_test.sh create mode 100755 test/250_dns_restart_test.sh create mode 100755 test/295_dns_large_response.sh diff --git a/Makefile b/Makefile index dca84405..3f93170b 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,7 @@ $(WEAVEPROXY_EXE) $(NETCHECK_EXE): common/*.go common/*/*.go net/*.go go build -ldflags "-extldflags \"-static\" -X main.version $(WEAVE_VERSION)" -tags netgo -o $@ ./$(@D) $(NETGO_CHECK) -$(WEAVER_EXE): router/*.go ipam/*.go ipam/*/*.go prog/weaver/main.go +$(WEAVER_EXE): router/*.go ipam/*.go ipam/*/*.go nameserver/*.go prog/weaver/main.go $(WEAVEPROXY_EXE): proxy/*.go prog/weaveproxy/main.go $(NETCHECK_EXE): prog/netcheck/netcheck.go diff --git a/nameserver/dns.go b/nameserver/dns.go new file mode 100644 index 00000000..af437897 --- /dev/null +++ b/nameserver/dns.go @@ -0,0 +1,275 @@ +package nameserver + +import ( + "bytes" + "fmt" + "math/rand" + "net" + "sort" + "strings" + "time" + + "github.com/miekg/dns" + + "github.com/weaveworks/weave/net/address" +) + +const ( + topDomain = "." + reverseDNSdomain = "in-addr.arpa." + etcResolvConf = "/etc/resolv.conf" + udpBuffSize = uint16(4096) + minUDPSize = 512 + + DefaultPort = 53 + DefaultTTL = 1 + DefaultClientTimeout = 5 * time.Second +) + +type DNSServer struct { + ns *Nameserver + domain string + ttl uint32 + port int + + servers []*dns.Server + upstream *dns.ClientConfig + tcpClient *dns.Client + udpClient *dns.Client +} + +func NewDNSServer(ns *Nameserver, domain string, port int, ttl uint32, + clientTimeout time.Duration) (*DNSServer, error) { + + s := &DNSServer{ + ns: ns, + domain: dns.Fqdn(domain), + ttl: ttl, + port: port, + tcpClient: &dns.Client{Net: "tcp", ReadTimeout: clientTimeout}, + udpClient: &dns.Client{Net: "udp", ReadTimeout: clientTimeout, UDPSize: udpBuffSize}, + } + var err error + if s.upstream, err = dns.ClientConfigFromFile(etcResolvConf); err != nil { + return nil, err + } + + err = s.listen(port) + return s, err +} + +func (d *DNSServer) String() string { + var buf bytes.Buffer + fmt.Fprintf(&buf, "WeaveDNS (%s)\n", d.ns.ourName) + fmt.Fprintf(&buf, " listening on port %d, for domain %s\n", d.port, d.domain) + fmt.Fprintf(&buf, " response ttl %d\n", d.ttl) + return buf.String() +} + +func (d *DNSServer) listen(port int) error { + udpListener, err := net.ListenPacket("udp", fmt.Sprintf(":%d", port)) + if err != nil { + return err + } + udpServer := &dns.Server{PacketConn: udpListener, Handler: d.createMux(d.udpClient, minUDPSize)} + + tcpListener, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + if err != nil { + udpServer.Shutdown() + return err + } + tcpServer := &dns.Server{Listener: tcpListener, Handler: d.createMux(d.tcpClient, -1)} + + d.servers = []*dns.Server{udpServer, tcpServer} + return nil +} + +func (d *DNSServer) ActivateAndServe() { + for _, server := range d.servers { + go func(server *dns.Server) { + server.ActivateAndServe() + }(server) + } +} + +func (d *DNSServer) Stop() error { + for _, server := range d.servers { + if err := server.Shutdown(); err != nil { + return err + } + } + return nil +} + +func (d *DNSServer) errorResponse(r *dns.Msg, code int, w dns.ResponseWriter) { + m := dns.Msg{} + m.SetReply(r) + m.RecursionAvailable = true + m.Rcode = code + + d.ns.debugf("error response: %+v", m) + if err := w.WriteMsg(&m); err != nil { + d.ns.infof("error responding: %v", err) + } +} + +func (d *DNSServer) createMux(client *dns.Client, defaultMaxResponseSize int) *dns.ServeMux { + m := dns.NewServeMux() + m.HandleFunc(d.domain, d.handleLocal(defaultMaxResponseSize)) + m.HandleFunc(reverseDNSdomain, d.handleReverse(client, defaultMaxResponseSize)) + m.HandleFunc(topDomain, d.handleRecursive(client, defaultMaxResponseSize)) + return m +} + +func (d *DNSServer) handleLocal(defaultMaxResponseSize int) func(dns.ResponseWriter, *dns.Msg) { + return func(w dns.ResponseWriter, req *dns.Msg) { + d.ns.debugf("local request: %+v", *req) + if len(req.Question) != 1 || req.Question[0].Qtype != dns.TypeA { + d.errorResponse(req, dns.RcodeNameError, w) + return + } + + hostname := req.Question[0].Name + addrs := d.ns.Lookup(hostname) + + response := dns.Msg{} + response.RecursionAvailable = true + response.Authoritative = true + response.SetReply(req) + response.Answer = make([]dns.RR, len(addrs)) + + header := dns.RR_Header{ + Name: hostname, + Rrtype: dns.TypeA, + Class: dns.ClassINET, + Ttl: d.ttl, + } + + for i, addr := range addrs { + ip := addr.IP4() + response.Answer[i] = &dns.A{Hdr: header, A: ip} + } + + shuffleAnswers(&response.Answer) + maxResponseSize := getMaxResponseSize(req, defaultMaxResponseSize) + truncateResponse(&response, maxResponseSize) + + d.ns.debugf("response: %+v", response) + if err := w.WriteMsg(&response); err != nil { + d.ns.infof("error responding: %v", err) + } + } +} + +func (d *DNSServer) handleReverse(client *dns.Client, defaultMaxResponseSize int) func(dns.ResponseWriter, *dns.Msg) { + return func(w dns.ResponseWriter, req *dns.Msg) { + d.ns.debugf("reverse request: %+v", *req) + if len(req.Question) != 1 || req.Question[0].Qtype != dns.TypePTR { + d.errorResponse(req, dns.RcodeNameError, w) + return + } + + ipStr := strings.TrimSuffix(req.Question[0].Name, "."+reverseDNSdomain) + ip, err := address.ParseIP(ipStr) + if err != nil { + d.errorResponse(req, dns.RcodeNameError, w) + return + } + + hostname, err := d.ns.ReverseLookup(ip.Reverse()) + if err != nil { + d.handleRecursive(client, defaultMaxResponseSize)(w, req) + return + } + + response := dns.Msg{} + response.RecursionAvailable = true + response.Authoritative = true + response.SetReply(req) + + header := dns.RR_Header{ + Name: req.Question[0].Name, + Rrtype: dns.TypePTR, + Class: dns.ClassINET, + Ttl: d.ttl, + } + + response.Answer = []dns.RR{&dns.PTR{ + Hdr: header, + Ptr: hostname, + }} + + maxResponseSize := getMaxResponseSize(req, defaultMaxResponseSize) + truncateResponse(&response, maxResponseSize) + + d.ns.debugf("response: %+v", response) + if err := w.WriteMsg(&response); err != nil { + d.ns.infof("error responding: %v", err) + } + } +} + +func (d *DNSServer) handleRecursive(client *dns.Client, defaultMaxResponseSize int) func(dns.ResponseWriter, *dns.Msg) { + return func(w dns.ResponseWriter, req *dns.Msg) { + d.ns.debugf("recursive cdrequest: %+v", *req) + for _, server := range d.upstream.Servers { + response, _, err := client.Exchange(req, fmt.Sprintf("%s:%s", server, d.upstream.Port)) + if err != nil || response == nil { + d.ns.debugf("network error trying %s (%s)", server, err) + continue + } + if response.Rcode != dns.RcodeSuccess && !response.Authoritative { + d.ns.debugf("network error trying %s (%s)", server, err) + continue + } + d.ns.debugf("response: %+v", response) + if err := w.WriteMsg(response); err != nil { + d.ns.infof("error responding: %v", err) + } + return + } + + d.errorResponse(req, dns.RcodeServerFailure, w) + } +} + +func shuffleAnswers(answers *[]dns.RR) { + if len(*answers) <= 1 { + return + } + + for i := range *answers { + j := rand.Intn(i + 1) + (*answers)[i], (*answers)[j] = (*answers)[j], (*answers)[i] + } +} + +func truncateResponse(response *dns.Msg, maxSize int) { + if len(response.Answer) <= 1 || maxSize <= 0 { + return + } + + // take a copy of answers, as we're going to mutate response + answers := response.Answer + + // search for smallest i that is too big + i := sort.Search(len(response.Answer), func(i int) bool { + // return true if too big + response.Answer = answers[:i+1] + return response.Len() > maxSize + }) + if i == len(answers) { + response.Answer = answers + return + } + + response.Answer = answers[:i] + response.Truncated = true +} + +func getMaxResponseSize(req *dns.Msg, defaultMaxResponseSize int) int { + if opt := req.IsEdns0(); opt != nil { + return int(opt.UDPSize()) + } + return defaultMaxResponseSize +} diff --git a/nameserver/dns_test.go b/nameserver/dns_test.go new file mode 100644 index 00000000..b44ce94b --- /dev/null +++ b/nameserver/dns_test.go @@ -0,0 +1,95 @@ +package nameserver + +import ( + "fmt" + "math/rand" + "net" + "testing" + "time" + + "github.com/miekg/dns" + "github.com/stretchr/testify/require" + + "github.com/weaveworks/weave/net/address" + "github.com/weaveworks/weave/router" +) + +func TestTruncation(t *testing.T) { + //common.SetLogLevel("debug") + + peername, err := router.PeerNameFromString("00:00:00:02:00:00") + require.Nil(t, err) + nameserver := New(peername, nil, nil, "") + dnsserver, err := NewDNSServer(nameserver, "weave.local.", 0, 30, 5*time.Second) + require.Nil(t, err) + udpPort := dnsserver.servers[0].PacketConn.LocalAddr().(*net.UDPAddr).Port + tcpPort := dnsserver.servers[1].Listener.Addr().(*net.TCPAddr).Port + go dnsserver.ActivateAndServe() + defer dnsserver.Stop() + + // Add 100 mappings to nameserver + addrs := []address.Address{} + for i := address.Address(0); i < 100; i++ { + addrs = append(addrs, i) + nameserver.AddEntry("foo.weave.local.", "", peername, i) + } + + doRequest := func(client *dns.Client, request *dns.Msg, port int) *dns.Msg { + request.SetQuestion("foo.weave.local.", dns.TypeA) + response, _, err := client.Exchange(request, fmt.Sprintf("127.0.0.1:%d", port)) + require.Nil(t, err) + return response + } + + // do a udp query, ensure we get a truncated response + { + udpClient := dns.Client{Net: "udp", UDPSize: minUDPSize} + response := doRequest(&udpClient, &dns.Msg{}, udpPort) + require.Nil(t, err) + require.True(t, response.MsgHdr.Truncated) + require.True(t, len(response.Answer) < 100) + } + + // do a udp query with big size, ensure we don't get a truncated response + { + udpClient := dns.Client{Net: "udp", UDPSize: 65535} + request := &dns.Msg{} + request.SetEdns0(65535, false) + response := doRequest(&udpClient, request, udpPort) + require.False(t, response.MsgHdr.Truncated) + require.Equal(t, len(response.Answer), 100) + } + + // do a tcp query, ensure we don't get a truncated response + { + tcpClient := dns.Client{Net: "tcp"} + response := doRequest(&tcpClient, &dns.Msg{}, tcpPort) + require.False(t, response.MsgHdr.Truncated) + require.Equal(t, len(response.Answer), 100) + } +} + +func TestTruncateResponse(t *testing.T) { + + header := dns.RR_Header{ + Name: "host.domain.com", + Rrtype: dns.TypePTR, + Class: dns.ClassINET, + Ttl: 30, + } + + for i := 0; i < 10000; i++ { + // generate a random msg + response := &dns.Msg{} + numAnswers := 40 + rand.Intn(200) + response.Answer = make([]dns.RR, numAnswers) + for j := 0; j < numAnswers; j++ { + response.Answer[j] = &dns.A{Hdr: header, A: address.Address(j).IP4()} + } + + // pick a random max size, truncate response to that, check it + maxSize := 512 + rand.Intn(response.Len()-512) + truncateResponse(response, maxSize) + require.True(t, response.Len() <= maxSize) + } +} diff --git a/nameserver/entry.go b/nameserver/entry.go new file mode 100644 index 00000000..0ab22008 --- /dev/null +++ b/nameserver/entry.go @@ -0,0 +1,184 @@ +package nameserver + +import ( + "bytes" + "encoding/gob" + "fmt" + "sort" + "time" + + "github.com/weaveworks/weave/net/address" + "github.com/weaveworks/weave/router" +) + +var now = func() int64 { return time.Now().Unix() } + +type Entry struct { + ContainerID string + Origin router.PeerName + Addr address.Address + Hostname string + Version int + Tombstone int64 // timestamp of when it was deleted +} + +type Entries []Entry + +func (e1 *Entry) equal(e2 *Entry) bool { + return e1.ContainerID == e2.ContainerID && + e1.Origin == e2.Origin && + e1.Addr == e2.Addr && + e1.Hostname == e2.Hostname +} + +func (e1 *Entry) less(e2 *Entry) bool { + // Entries are kept sorted by Hostname, Origin, ContainerID then address + switch { + case e1.Hostname != e2.Hostname: + return e1.Hostname < e2.Hostname + + case e1.Origin != e2.Origin: + return e1.Origin < e2.Origin + + case e1.ContainerID != e2.ContainerID: + return e1.ContainerID < e2.ContainerID + + default: + return e1.Addr < e2.Addr + } +} + +// returns true to indicate a change +func (e1 *Entry) merge(e2 *Entry) bool { + // we know container id, origin, add and hostname are equal + if e2.Version > e1.Version { + e1.Version = e2.Version + e1.Tombstone = e2.Tombstone + } + return e2.Version != e1.Version +} + +func (e1 *Entry) String() string { + return fmt.Sprintf("%s -> %s", e1.Hostname, e1.Addr.String()) +} + +func (es Entries) Len() int { return len(es) } +func (es Entries) Swap(i, j int) { panic("Swap") } +func (es Entries) Less(i, j int) bool { return es[i].less(&es[j]) } + +func (es *Entries) check() error { + if !sort.IsSorted(es) { + return fmt.Errorf("Not sorted!") + } + return nil +} + +func (es *Entries) checkAndPanic() { + if err := es.check(); err != nil { + panic(err) + } +} + +func (es *Entries) add(hostname, containerid string, origin router.PeerName, addr address.Address) Entry { + defer es.checkAndPanic() + + entry := Entry{Hostname: hostname, Origin: origin, ContainerID: containerid, Addr: addr} + i := sort.Search(len(*es), func(i int) bool { + return !(*es)[i].less(&entry) + }) + if i < len(*es) && (*es)[i].equal(&entry) { + (*es)[i].Tombstone = 0 + (*es)[i].Version++ + } else { + *es = append(*es, Entry{}) + copy((*es)[i+1:], (*es)[i:]) + (*es)[i] = entry + } + return (*es)[i] +} + +func (es *Entries) merge(incoming Entries) Entries { + defer es.checkAndPanic() + var ( + newEntries Entries + i = 0 + ) + + for _, entry := range incoming { + for i < len(*es) && (*es)[i].less(&entry) { + i++ + } + if i < len(*es) && (*es)[i].equal(&entry) { + if !(*es)[i].merge(&entry) { + continue + } + } + *es = append(*es, Entry{}) + copy((*es)[i+1:], (*es)[i:]) + (*es)[i] = entry + newEntries = append(newEntries, entry) + } + + return newEntries +} + +func (es *Entries) tombstone(ourname router.PeerName, f func(*Entry) bool) *Entries { + tombstoned := Entries{} + for i, e := range *es { + if f(&e) && e.Origin == ourname { + e.Version++ + e.Tombstone = now() + (*es)[i] = e + tombstoned = append(tombstoned, e) + } + } + return &tombstoned +} + +func (es *Entries) filter(f func(*Entry) bool) { + i := 0 + for _, e := range *es { + if !f(&e) { + continue + } + (*es)[i] = e + i++ + } + *es = (*es)[:i] +} + +func (es Entries) lookup(hostname string) Entries { + i := sort.Search(len(es), func(i int) bool { + return es[i].Hostname >= hostname + }) + if i >= len(es) || es[i].Hostname != hostname { + return Entries{} + } + + j := sort.Search(len(es)-i, func(j int) bool { + return es[i+j].Hostname > hostname + }) + + return es[i : i+j] +} + +func (es *Entries) first(f func(*Entry) bool) (*Entry, error) { + for _, e := range *es { + if f(&e) { + return &e, nil + } + } + return nil, fmt.Errorf("Not found") +} + +func (es *Entries) Merge(other router.GossipData) { + es.merge(*other.(*Entries)) +} + +func (es *Entries) Encode() [][]byte { + buf := &bytes.Buffer{} + if err := gob.NewEncoder(buf).Encode(es); err != nil { + panic(err) + } + return [][]byte{buf.Bytes()} +} diff --git a/nameserver/entry_test.go b/nameserver/entry_test.go new file mode 100644 index 00000000..5e8efa03 --- /dev/null +++ b/nameserver/entry_test.go @@ -0,0 +1,113 @@ +package nameserver + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/weaveworks/weave/net/address" + "github.com/weaveworks/weave/router" +) + +func TestAdd(t *testing.T) { + oldNow := now + defer func() { now = oldNow }() + now = func() int64 { return 1234 } + + entries := Entries{} + entries.add("A", "", router.UnknownPeerName, address.Address(0)) + expected := Entries{ + Entry{Hostname: "A", Origin: router.UnknownPeerName, Addr: address.Address(0)}, + } + require.Equal(t, entries, expected) + + entries.tombstone(router.UnknownPeerName, func(e *Entry) bool { return e.Hostname == "A" }) + expected = Entries{ + Entry{Hostname: "A", Origin: router.UnknownPeerName, Addr: address.Address(0), Version: 1, Tombstone: 1234}, + } + require.Equal(t, entries, expected) + + entries.add("A", "", router.UnknownPeerName, address.Address(0)) + expected = Entries{ + Entry{Hostname: "A", Origin: router.UnknownPeerName, Addr: address.Address(0), Version: 2}, + } + require.Equal(t, entries, expected) +} + +func TestMerge(t *testing.T) { + e1 := Entries{ + Entry{Hostname: "A"}, + Entry{Hostname: "C"}, + Entry{Hostname: "D"}, + Entry{Hostname: "F"}, + } + + e2 := Entries{ + Entry{Hostname: "B"}, + Entry{Hostname: "E"}, + Entry{Hostname: "F"}, + } + + e1.merge(e2) + expected := Entries{ + Entry{Hostname: "A"}, + Entry{Hostname: "B"}, + Entry{Hostname: "C"}, + Entry{Hostname: "D"}, + Entry{Hostname: "E"}, + Entry{Hostname: "F"}, + } + + require.Equal(t, e1, expected) +} + +func TestTombstone(t *testing.T) { + oldNow := now + defer func() { now = oldNow }() + now = func() int64 { return 1234 } + + es := Entries{ + Entry{Hostname: "A"}, + Entry{Hostname: "B"}, + } + + es.tombstone(router.UnknownPeerName, func(e *Entry) bool { + return e.Hostname == "B" + }) + expected := Entries{ + Entry{Hostname: "A"}, + Entry{Hostname: "B", Version: 1, Tombstone: 1234}, + } + require.Equal(t, es, expected) +} + +func TestDelete(t *testing.T) { + es := Entries{ + Entry{Hostname: "A"}, + Entry{Hostname: "B"}, + } + + es.filter(func(e *Entry) bool { + return e.Hostname != "A" + }) + expected := Entries{ + Entry{Hostname: "B"}, + } + require.Equal(t, es, expected) +} + +func TestLookup(t *testing.T) { + es := Entries{ + Entry{Hostname: "A"}, + Entry{Hostname: "B", ContainerID: "foo"}, + Entry{Hostname: "B", ContainerID: "bar"}, + Entry{Hostname: "C"}, + } + + have := es.lookup("B") + want := Entries{ + Entry{Hostname: "B", ContainerID: "foo"}, + Entry{Hostname: "B", ContainerID: "bar"}, + } + require.Equal(t, have, want) +} diff --git a/nameserver/http.go b/nameserver/http.go new file mode 100644 index 00000000..d85b9970 --- /dev/null +++ b/nameserver/http.go @@ -0,0 +1,85 @@ +package nameserver + +import ( + "fmt" + "net/http" + + "github.com/gorilla/mux" + "github.com/miekg/dns" + + "github.com/weaveworks/weave/common" + "github.com/weaveworks/weave/net/address" +) + +func badRequest(w http.ResponseWriter, err error) { + http.Error(w, err.Error(), http.StatusBadRequest) + common.Log.Warningf("[gossipdns]: %v", err) +} + +func (n *Nameserver) HandleHTTP(router *mux.Router) { + router.Methods("GET").Path("/domain").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, n.domain) + }) + + router.Methods("PUT").Path("/name/{container}/{ip}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var ( + vars = mux.Vars(r) + container = vars["container"] + ipStr = vars["ip"] + hostname = r.FormValue("fqdn") + ip, err = address.ParseIP(ipStr) + ) + if err != nil { + badRequest(w, err) + return + } + + if err := n.AddEntry(dns.Fqdn(hostname), container, n.ourName, ip); err != nil { + badRequest(w, fmt.Errorf("Unable to add entry: %v", err)) + return + } + + if r.FormValue("check-alive") == "true" && n.docker.IsContainerNotRunning(container) { + common.Log.Infof("[gossipdns] '%s' is not running: removing", container) + if err := n.Delete(hostname, container, ipStr, ip); err != nil { + common.Log.Infof("[gossipdns] failed to remove: %v", err) + } + } + + w.WriteHeader(204) + }) + + deleteHandler := func(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + + hostname := r.FormValue("fqdn") + if hostname == "" { + hostname = "*" + } else { + hostname = dns.Fqdn(hostname) + } + + container, ok := vars["container"] + if !ok { + container = "*" + } + + ipStr, ok := vars["ip"] + ip, err := address.ParseIP(ipStr) + if ok && err != nil { + badRequest(w, err) + return + } else if !ok { + ipStr = "*" + } + + if err := n.Delete(hostname, container, ipStr, ip); err != nil { + badRequest(w, fmt.Errorf("Unable to delete entries: %v", err)) + return + } + w.WriteHeader(204) + } + router.Methods("DELETE").Path("/name/{container}/{ip}").HandlerFunc(deleteHandler) + router.Methods("DELETE").Path("/name/{container}").HandlerFunc(deleteHandler) + router.Methods("DELETE").Path("/name").HandlerFunc(deleteHandler) +} diff --git a/nameserver/nameserver.go b/nameserver/nameserver.go new file mode 100644 index 00000000..43562ae5 --- /dev/null +++ b/nameserver/nameserver.go @@ -0,0 +1,251 @@ +package nameserver + +import ( + "bytes" + "encoding/gob" + "fmt" + "sync" + "time" + + "github.com/miekg/dns" + + . "github.com/weaveworks/weave/common" + "github.com/weaveworks/weave/common/docker" + "github.com/weaveworks/weave/net/address" + "github.com/weaveworks/weave/router" +) + +const ( + // Tombstones do not need to survive long periods of peer disconnection, as + // we delete entries for disconnected peers. Therefore they just need to hang + // around to account for propagation delay through gossip. 10 minutes sounds + // long enough. + tombstoneTimeout = time.Minute * 10 + + // Used by prog/weaver/main.go and proxy/create_container_interceptor.go + DefaultDomain = "weave.local." +) + +// Nameserver: gossip-based, in memory nameserver. +// - Holds a sorted list of (hostname, peer, container id, ip) tuples for the whole cluster. +// - This list is gossiped & merged around the cluser. +// - Lookup-by-hostname are O(nlogn), and return a (copy of a) slice of the entries +// - Update is O(n) for now +type Nameserver struct { + sync.RWMutex + ourName router.PeerName + domain string + gossip router.Gossip + docker *docker.Client + entries Entries + peers *router.Peers + quit chan struct{} +} + +func New(ourName router.PeerName, peers *router.Peers, docker *docker.Client, domain string) *Nameserver { + ns := &Nameserver{ + ourName: ourName, + domain: dns.Fqdn(domain), + peers: peers, + docker: docker, + quit: make(chan struct{}), + } + if peers != nil { + peers.OnGC(ns.PeerGone) + } + return ns +} + +func (n *Nameserver) SetGossip(gossip router.Gossip) { + n.gossip = gossip +} + +func (n *Nameserver) Start() { + go func() { + ticker := time.Tick(tombstoneTimeout) + for { + select { + case <-n.quit: + return + case <-ticker: + n.deleteTombstones() + } + } + }() +} + +func (n *Nameserver) Stop() { + n.quit <- struct{}{} +} + +func (n *Nameserver) AddEntry(hostname, containerid string, origin router.PeerName, addr address.Address) error { + n.infof("adding entry %s -> %s", hostname, addr.String()) + n.Lock() + entry := n.entries.add(hostname, containerid, origin, addr) + n.Unlock() + + if n.gossip != nil { + return n.gossip.GossipBroadcast(&Entries{entry}) + } + return nil +} + +func (n *Nameserver) Lookup(hostname string) []address.Address { + n.RLock() + defer n.RUnlock() + + entries := n.entries.lookup(hostname) + result := []address.Address{} + for _, e := range entries { + if e.Tombstone > 0 { + continue + } + result = append(result, e.Addr) + } + n.debugf("lookup %s -> %s", hostname, &result) + return result +} + +func (n *Nameserver) ReverseLookup(ip address.Address) (string, error) { + n.RLock() + defer n.RUnlock() + + match, err := n.entries.first(func(e *Entry) bool { + return e.Addr == ip + }) + if err != nil { + return "", err + } + n.debugf("reverse lookup %s -> %s", ip, match.Hostname) + return match.Hostname, nil +} + +func (n *Nameserver) ContainerDied(ident string) error { + n.Lock() + entries := n.entries.tombstone(n.ourName, func(e *Entry) bool { + if e.ContainerID == ident { + n.infof("container %s died, tombstoning entry %s", ident, e.String()) + return true + } + return false + }) + n.Unlock() + if n.gossip != nil { + return n.gossip.GossipBroadcast(entries) + } + return nil +} + +func (n *Nameserver) PeerGone(peer *router.Peer) { + n.Lock() + defer n.Unlock() + n.entries.filter(func(e *Entry) bool { + return e.Origin != peer.Name + }) +} + +func (n *Nameserver) Delete(hostname, containerid, ipStr string, ip address.Address) error { + n.Lock() + n.infof("tombstoning hostname=%s, containerid=%s, ip=%s", hostname, containerid, ipStr) + entries := n.entries.tombstone(n.ourName, func(e *Entry) bool { + if hostname != "*" && e.Hostname != hostname { + return false + } + + if containerid != "*" && e.ContainerID != containerid { + return false + } + + if ipStr != "*" && e.Addr != ip { + return false + } + return true + }) + n.Unlock() + if n.gossip != nil { + return n.gossip.GossipBroadcast(entries) + } + return nil +} + +func (n *Nameserver) deleteTombstones() { + n.Lock() + defer n.Unlock() + now := time.Now().Unix() + n.entries.filter(func(e *Entry) bool { + return now-e.Tombstone <= int64(tombstoneTimeout/time.Second) + }) +} + +func (n *Nameserver) String() string { + n.RLock() + defer n.RUnlock() + var buf bytes.Buffer + for _, entry := range n.entries { + if entry.Tombstone > 0 { + continue + } + containerid := entry.ContainerID + if len(containerid) > 12 { + containerid = containerid[:12] + } + fmt.Fprintf(&buf, "%s: %s [%s]\n", containerid, entry.Hostname, entry.Addr.String()) + } + return buf.String() +} + +func (n *Nameserver) Gossip() router.GossipData { + n.RLock() + defer n.RUnlock() + result := make(Entries, len(n.entries)) + copy(result, n.entries) + return &result +} + +func (n *Nameserver) OnGossipUnicast(sender router.PeerName, msg []byte) error { + return nil +} + +func (n *Nameserver) receiveGossip(msg []byte) (router.GossipData, router.GossipData, error) { + var entries Entries + if err := gob.NewDecoder(bytes.NewReader(msg)).Decode(&entries); err != nil { + return nil, nil, err + } + + if err := entries.check(); err != nil { + return nil, nil, err + } + + n.Lock() + defer n.Unlock() + + if n.peers != nil { + entries.filter(func(e *Entry) bool { + return n.peers.Fetch(e.Origin) != nil + }) + } + + newEntries := n.entries.merge(entries) + return &newEntries, &entries, nil +} + +// merge received data into state and return "everything new I've +// just learnt", or nil if nothing in the received data was new +func (n *Nameserver) OnGossip(msg []byte) (router.GossipData, error) { + newEntries, _, err := n.receiveGossip(msg) + return newEntries, err +} + +// merge received data into state and return a representation of +// the received data, for further propagation +func (n *Nameserver) OnGossipBroadcast(msg []byte) (router.GossipData, error) { + _, entries, err := n.receiveGossip(msg) + return entries, err +} + +func (n *Nameserver) infof(fmt string, args ...interface{}) { + Log.Infof("[nameserver %s] "+fmt, append([]interface{}{n.ourName}, args...)...) +} +func (n *Nameserver) debugf(fmt string, args ...interface{}) { + Log.Debugf("[nameserver %s] "+fmt, append([]interface{}{n.ourName}, args...)...) +} diff --git a/nameserver/nameserver_test.go b/nameserver/nameserver_test.go new file mode 100644 index 00000000..f1b091fe --- /dev/null +++ b/nameserver/nameserver_test.go @@ -0,0 +1,248 @@ +package nameserver + +import ( + "fmt" + "math/rand" + "reflect" + "sort" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/weaveworks/weave/net/address" + "github.com/weaveworks/weave/router" + "github.com/weaveworks/weave/testing/gossip" +) + +func makeNetwork(size int) ([]*Nameserver, *gossip.TestRouter) { + gossipRouter := gossip.NewTestRouter(0.0) + nameservers := make([]*Nameserver, size) + + for i := 0; i < size; i++ { + name, _ := router.PeerNameFromString(fmt.Sprintf("%02d:00:00:02:00:00", i)) + nameserver := New(name, nil, nil, "") + nameserver.SetGossip(gossipRouter.Connect(nameserver.ourName, nameserver)) + nameserver.Start() + nameservers[i] = nameserver + } + + return nameservers, gossipRouter +} + +func stopNetwork(nameservers []*Nameserver) { + for _, nameserver := range nameservers { + nameserver.Stop() + } +} + +type pair struct { + origin router.PeerName + addr address.Address +} + +type mapping struct { + hostname string + addrs []pair +} + +type addrs []address.Address + +func (a addrs) Len() int { return len(a) } +func (a addrs) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a addrs) Less(i, j int) bool { return a[i] < a[j] } +func (a addrs) String() string { + ss := []string{} + for _, addr := range a { + ss = append(ss, addr.String()) + } + return strings.Join(ss, " ") +} + +func (m mapping) Addrs() []address.Address { + want := addrs{} + for _, p := range m.addrs { + want = append(want, p.addr) + } + sort.Sort(want) + return want +} + +func TestNameservers(t *testing.T) { + //common.SetLogLevel("debug") + + lookupTimeout := 20 // ms + nameservers, grouter := makeNetwork(50) + defer stopNetwork(nameservers) + nameserversByName := map[router.PeerName]*Nameserver{} + for _, n := range nameservers { + nameserversByName[n.ourName] = n + } + mappings := []mapping{} + + check := func(nameserver *Nameserver, expected mapping) { + have := []address.Address{} + for i := 0; i < lookupTimeout; i++ { + have = nameserver.Lookup(expected.hostname) + sort.Sort(addrs(have)) + if reflect.DeepEqual(have, expected.Addrs()) { + return + } + time.Sleep(1 * time.Millisecond) + } + want := expected.Addrs() + require.Equal(t, addrs(want).String(), addrs(have).String()) + } + + addMapping := func() { + nameserver := nameservers[rand.Intn(len(nameservers))] + addr := address.Address(rand.Int31()) + hostname := fmt.Sprintf("hostname%d", rand.Int63()) + mapping := mapping{hostname, []pair{{nameserver.ourName, addr}}} + mappings = append(mappings, mapping) + + require.Nil(t, nameserver.AddEntry(hostname, "", nameserver.ourName, addr)) + check(nameserver, mapping) + } + + addExtraMapping := func() { + if len(mappings) <= 0 { + return + } + nameserver := nameservers[rand.Intn(len(nameservers))] + i := rand.Intn(len(mappings)) + mapping := mappings[i] + addr := address.Address(rand.Int31()) + mapping.addrs = append(mapping.addrs, pair{nameserver.ourName, addr}) + mappings[i] = mapping + + require.Nil(t, nameserver.AddEntry(mapping.hostname, "", nameserver.ourName, addr)) + check(nameserver, mapping) + } + + deleteMapping := func() { + if len(mappings) <= 0 { + return + } + i := rand.Intn(len(mappings)) + mapping := mappings[i] + if len(mapping.addrs) <= 0 { + return + } + j := rand.Intn(len(mapping.addrs)) + pair := mapping.addrs[j] + mapping.addrs = append(mapping.addrs[:j], mapping.addrs[j+1:]...) + mappings[i] = mapping + nameserver := nameserversByName[pair.origin] + + require.Nil(t, nameserver.Delete(mapping.hostname, "*", pair.addr.String(), pair.addr)) + check(nameserver, mapping) + } + + doLookup := func() { + if len(mappings) <= 0 { + return + } + mapping := mappings[rand.Intn(len(mappings))] + nameserver := nameservers[rand.Intn(len(nameservers))] + check(nameserver, mapping) + } + + doReverseLookup := func() { + if len(mappings) <= 0 { + return + } + mapping := mappings[rand.Intn(len(mappings))] + if len(mapping.addrs) <= 0 { + return + } + nameserver := nameservers[rand.Intn(len(nameservers))] + hostname := "" + var err error + for i := 0; i < lookupTimeout; i++ { + hostname, err = nameserver.ReverseLookup(mapping.addrs[0].addr) + if err != nil && mapping.hostname == hostname { + return + } + time.Sleep(1 * time.Millisecond) + } + require.Nil(t, err) + require.Equal(t, mapping.hostname, hostname) + } + + for i := 0; i < 1000; i++ { + if i%10 == 0 { + grouter.Flush() + } + + r := rand.Float32() + switch { + case r < 0.1: + addMapping() + + case 0.1 <= r && r < 0.2: + addExtraMapping() + + case 0.2 <= r && r < 0.3: + deleteMapping() + + case 0.3 <= r && r < 0.9: + doLookup() + + case 0.9 <= r: + doReverseLookup() + } + } +} + +func TestContainerAndPeerDeath(t *testing.T) { + peername, err := router.PeerNameFromString("00:00:00:02:00:00") + require.Nil(t, err) + nameserver := New(peername, nil, nil, "") + + err = nameserver.AddEntry("hostname", "containerid", peername, address.Address(0)) + require.Nil(t, err) + require.Equal(t, []address.Address{0}, nameserver.Lookup("hostname")) + + err = nameserver.ContainerDied("containerid") + require.Nil(t, err) + require.Equal(t, []address.Address{}, nameserver.Lookup("hostname")) + + err = nameserver.AddEntry("hostname", "containerid", peername, address.Address(0)) + require.Nil(t, err) + require.Equal(t, []address.Address{0}, nameserver.Lookup("hostname")) + + nameserver.PeerGone(&router.Peer{Name: peername}) + require.Equal(t, []address.Address{}, nameserver.Lookup("hostname")) +} + +func TestTombstoneDeletion(t *testing.T) { + oldNow := now + defer func() { now = oldNow }() + now = func() int64 { return 1234 } + + peername, err := router.PeerNameFromString("00:00:00:02:00:00") + require.Nil(t, err) + nameserver := New(peername, nil, nil, "") + + err = nameserver.AddEntry("hostname", "containerid", peername, address.Address(0)) + require.Nil(t, err) + require.Equal(t, []address.Address{0}, nameserver.Lookup("hostname")) + + err = nameserver.Delete("hostname", "containerid", "", address.Address(0)) + require.Nil(t, err) + require.Equal(t, []address.Address{}, nameserver.Lookup("hostname")) + require.Equal(t, Entries{{ + ContainerID: "containerid", + Origin: peername, + Addr: address.Address(0), + Hostname: "hostname", + Version: 1, + Tombstone: 1234, + }}, nameserver.entries) + + now = func() int64 { return 1234 + int64(tombstoneTimeout/time.Second) + 1 } + nameserver.deleteTombstones() + require.Equal(t, Entries{}, nameserver.entries) +} diff --git a/net/address/address.go b/net/address/address.go index f97d44c8..f3faf138 100644 --- a/net/address/address.go +++ b/net/address/address.go @@ -98,3 +98,10 @@ func Min(a, b Offset) Offset { } return a } + +func (addr Address) Reverse() Address { + return ((addr >> 24) & 0xff) | // move byte 3 to byte 0 + ((addr << 8) & 0xff0000) | // move byte 1 to byte 2 + ((addr >> 8) & 0xff00) | // move byte 2 to byte 1 + ((addr << 24) & 0xff000000) // byte 0 to byte 3 +} diff --git a/prog/weaver/main.go b/prog/weaver/main.go index 30a17d96..53aa3b20 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -8,6 +8,7 @@ import ( "os" "runtime" "strings" + "time" "github.com/davecheney/profile" "github.com/docker/docker/pkg/mflag" @@ -16,6 +17,7 @@ import ( . "github.com/weaveworks/weave/common" "github.com/weaveworks/weave/common/docker" "github.com/weaveworks/weave/ipam" + "github.com/weaveworks/weave/nameserver" weavenet "github.com/weaveworks/weave/net" "github.com/weaveworks/weave/net/address" weave "github.com/weaveworks/weave/router" @@ -52,6 +54,10 @@ func main() { peerCount int apiPath string peers []string + dnsDomain string + dnsPort int + dnsTTL int + dnsClientTimeout time.Duration ) mflag.BoolVar(&justVersion, []string{"#version", "-version"}, false, "print version and exit") @@ -73,6 +79,11 @@ func main() { mflag.StringVar(&ipsubnetCIDR, []string{"#ipsubnet", "#-ipsubnet", "-ipalloc-default-subnet"}, "", "subnet to allocate within by default, in CIDR notation") mflag.IntVar(&peerCount, []string{"#initpeercount", "#-initpeercount", "-init-peer-count"}, 0, "number of peers in network (for IP address allocation)") mflag.StringVar(&apiPath, []string{"#api", "-api"}, "unix:///var/run/docker.sock", "Path to Docker API socket") + mflag.StringVar(&dnsDomain, []string{"-dns-domain"}, nameserver.DefaultDomain, "local domain to server requests for") + mflag.IntVar(&dnsPort, []string{"-dns-port"}, nameserver.DefaultPort, "port to listen on for DNS requests") + mflag.IntVar(&dnsTTL, []string{"-dns-ttl"}, nameserver.DefaultTTL, "TTL for DNS request from our domain") + mflag.DurationVar(&dnsClientTimeout, []string{"-dns-fallback-timeout"}, nameserver.DefaultClientTimeout, "timeout for fallback DNS requests") + mflag.Parse() peers = mflag.Args() @@ -141,15 +152,15 @@ func main() { router := weave.NewRouter(config, name, nickName) Log.Println("Our name is", router.Ourself) + dockerCli, err := docker.NewClient(apiPath) + if err != nil { + Log.Fatal("Unable to start docker client: ", err) + } + var allocator *ipam.Allocator var defaultSubnet address.CIDR - var dockerCli *docker.Client if iprangeCIDR != "" { allocator, defaultSubnet = createAllocator(router, iprangeCIDR, ipsubnetCIDR, determineQuorum(peerCount, peers)) - dockerCli, err = docker.NewClient(apiPath) - if err != nil { - Log.Fatal("Unable to start docker client: ", err) - } if err = dockerCli.AddObserver(allocator); err != nil { Log.Fatal("Unable to start watcher", err) } @@ -157,6 +168,22 @@ func main() { Log.Fatal("--init-peer-count flag specified without --ipalloc-range") } + ns := nameserver.New(router.Ourself.Peer.Name, router.Peers, dockerCli, dnsDomain) + ns.SetGossip(router.NewGossip("nameserver", ns)) + if err = dockerCli.AddObserver(ns); err != nil { + Log.Fatal("Unable to start watcher", err) + } + ns.Start() + defer ns.Stop() + + dnsserver, err := nameserver.NewDNSServer(ns, dnsDomain, dnsPort, + uint32(dnsTTL), dnsClientTimeout) + if err != nil { + Log.Fatal("Unable to start dns server: ", err) + } + dnsserver.ActivateAndServe() + defer dnsserver.Stop() + router.Start() if errors := router.ConnectionMaker.InitiateConnections(peers, false); len(errors) > 0 { Log.Fatal(errorMessages(errors)) @@ -166,7 +193,7 @@ func main() { // so there is no point in doing "weave launch --http-addr ''". // This is here to support stand-alone use of weaver. if httpAddr != "" { - go handleHTTP(router, httpAddr, allocator, defaultSubnet, dockerCli) + go handleHTTP(router, httpAddr, allocator, defaultSubnet, dockerCli, ns, dnsserver) } SignalHandlerLoop(router) @@ -270,13 +297,15 @@ func determineQuorum(initPeerCountFlag int, peers []string) uint { return quorum } -func handleHTTP(router *weave.Router, httpAddr string, allocator *ipam.Allocator, defaultSubnet address.CIDR, docker *docker.Client) { +func handleHTTP(router *weave.Router, httpAddr string, allocator *ipam.Allocator, defaultSubnet address.CIDR, docker *docker.Client, ns *nameserver.Nameserver, dnsserver *nameserver.DNSServer) { muxRouter := mux.NewRouter() if allocator != nil { allocator.HandleHTTP(muxRouter, defaultSubnet, docker) } + ns.HandleHTTP(muxRouter) + muxRouter.Methods("GET").Path("/status").Headers("Accept", "application/json").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { json, _ := router.StatusJSON(version) w.Header().Set("Content-Type", "application/json") @@ -290,6 +319,9 @@ func handleHTTP(router *weave.Router, httpAddr string, allocator *ipam.Allocator fmt.Fprintln(w, allocator.String()) fmt.Fprintln(w, "Allocator default subnet:", defaultSubnet) } + fmt.Fprintln(w, "") + fmt.Fprintln(w, dnsserver.String()) + fmt.Fprintln(w, ns.String()) }) muxRouter.Methods("POST").Path("/connect").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/proxy/create_container_interceptor.go b/proxy/create_container_interceptor.go index 8e3cca04..f4c3ee06 100644 --- a/proxy/create_container_interceptor.go +++ b/proxy/create_container_interceptor.go @@ -9,8 +9,10 @@ import ( "strings" "github.com/fsouza/go-dockerclient" + . "github.com/weaveworks/weave/common" "github.com/weaveworks/weave/nameserver" + "github.com/weaveworks/weave/router" ) const MaxDockerHostname = 64 @@ -137,15 +139,15 @@ func (i *createContainerInterceptor) setWeaveDNS(container *createContainerReque } func (i *createContainerInterceptor) getDNSDomain() (domain string, running bool) { - domain = nameserver.DefaultLocalDomain - dnsContainer, err := i.proxy.client.InspectContainer("weavedns") + domain = nameserver.DefaultDomain + weaveContainer, err := i.proxy.client.InspectContainer("weave") if err != nil || - dnsContainer.NetworkSettings == nil || - dnsContainer.NetworkSettings.IPAddress == "" { + weaveContainer.NetworkSettings == nil || + weaveContainer.NetworkSettings.IPAddress == "" { return } - url := fmt.Sprintf("http://%s:%d/domain", dnsContainer.NetworkSettings.IPAddress, nameserver.DefaultHTTPPort) + url := fmt.Sprintf("http://%s:%d/domain", weaveContainer.NetworkSettings.IPAddress, router.HTTPPort) resp, err := http.Get(url) if err != nil || resp.StatusCode != http.StatusOK { return diff --git a/site/ipam.md b/site/ipam.md index 1c28978c..f3596f2a 100644 --- a/site/ipam.md +++ b/site/ipam.md @@ -8,7 +8,7 @@ layout: default Weave automatically assigns containers an IP address that is unique across the network, and releases that address when a container exits. This happens for all invocations of the `run`, `start`, -`attach`, `detach`, `expose`, `hide` and `launch-dns` commands, unless +`attach`, `detach`, `expose`, and `hide` commands, unless the user explictly specified an address. Weave can also assign addresses in multiple subnets. diff --git a/site/proxy.md b/site/proxy.md index 453c81de..2cd04a2b 100644 --- a/site/proxy.md +++ b/site/proxy.md @@ -30,7 +30,7 @@ start it simultaneously with the router and weaveDNS via `launch`: or independently via `launch-proxy`: - host1$ weave launch-router && weave launch-dns && weave launch-proxy + host1$ weave launch-router && weave launch-proxy The first form is more convenient, however you can only pass proxy related configuration arguments to `launch-proxy` so if you need to diff --git a/site/weavedns-design.md b/site/weavedns-design.md index fe70df18..9059505c 100644 --- a/site/weavedns-design.md +++ b/site/weavedns-design.md @@ -6,15 +6,36 @@ layout: default # WeaveDNS (service discovery) Design Notes The model is that each host has a service that is notified of -hostnames and weave addresses for containers on the host. It binds to -the host bridge to answer DNS queries from local containers; anything -it can't answer, it asks with mDNS on the weave network. +hostnames and weave addresses for containers on the host. Like IPAM, +this service is embedded within the router. It binds to +the host bridge to answer DNS queries from local containers; for +anything it can't answer, it uses the infomation in the host's +/etc/resolv.conf to query an 'fallback' server. + +The service is comprised of a DNS server, which answers all DNS queries +from containers, and a in-memory database of hostnames and IPs. The +database on each node contains a complete copy of the hostnames and IPs +for every containers in the cluster. + +For hostname queries in the local domain (default weave.local), the DNS +server will consult the in-memory database. For reverse queries, we +first consult the local database, and if not found we query the +upstream server. For all other queries, we consult the upstream +server. + +Updates to the in-memory database are broadcast to other DNS servers +within the cluster. The in-memory database only contains entries from +connected DNS servers; if a DNS server becomes partitioned from the +cluster, entries belonging to that server are removed from each node in +the cluster. When the partitioned DNS server reconnects, the entries +are re-broadcast around the cluster. + +The DNS server also listens to the Docker event stream, and removes +entries for containers when they die. Entries removed in this way are +tombstoned, and the tombstone lazily broadcast around the cluster. +After a short timeout the tombstones are independantly removed from +each host. -The service is divided into two components: the first is the DNS -server, which serves the records it is told about, and otherwise asks -via mDNS on the weave network. The other component is the DNS updater, -specific to Docker, which monitors the running containers so to add -and remove records from the DNS server. ## DNS server API @@ -24,60 +45,21 @@ and methods: `PUT /name//` Put a record for an IP, bound to a host-scoped identifier (e.g., a -container ID), in the DNS database. +container ID), in the DNS database. The request body must contain +a `fqdn=foo.weave.local` key pair. `DELETE /name//` -Remove a specific record for an IP and host-scoped identifier. +Remove a specific record for an IP and host-scoped identifier. The request +body can optionally contain a `fqdn=foo.weave.local` key pair. `DELETE /name/` Remove all records for the host-scoped identifier. -`GET /name/` +`GET /name/` -List all the records in the database. - -`GET /name/` - -List all the records for a host-scoped identifier. - -`GET /status` - -Give the server status. - -### Record structure - -The record structure is *either* form fields (useful for scripting -PUTs), or a JSON object with the same fields. The fields are: - -```js - { - "fqdn": string, - "routing_prefix": number, - "local_ip": string - } -``` - -> Also TTL? - -## DNS server behaviour - -The DNS server listens for the HTTP requests as above, and maintains a -database of host-local names accordingly. - -It also listens on the host-local network for DNS queries (port -53). Anything that is in its database it responds to immediately. For -other requests, it asks using multicast DNS on the weave network. - -Meanwhile, the DNS server is listening for mDNS queries, and responds -if it the queried name is in its database. - -### Caching mDNS answers - -The DNS server will hear the answers to its own and other servers' -mDNS queries sent on the weave network. It can cache these, to respond -to local DNS requests; the cache is consulted before using mDNS. +List of all IPs (in JSON format) for givne FQDN. ## DNS updater @@ -99,17 +81,3 @@ network attachment of the container, and updates the DNS server. > Perhaps it could put containers on a watch list when it's noticed > them. -## Container startup - -> TBD - -## Grouping and load balancing - -> TBD - -## Extensions - - * An updater that checks health more specifically, e.g., by testing - that a particular port is being listened to, or that it can get a - 200 OK. These could be encoded in the container environment - variables, as per registrator. diff --git a/site/weavedns.md b/site/weavedns.md index f41a7639..1166b00e 100644 --- a/site/weavedns.md +++ b/site/weavedns.md @@ -26,25 +26,15 @@ hosts. ## Using weaveDNS -WeaveDNS is deployed as a set of containers that communicate with each -other over the weave network. One such container needs to be started -on every weave host, either simultaneously with the Weave router and -Weave Docker API proxy via `launch`: +WeaveDNS is deployed as an embedded service within the Weave router. +The service is automatically started when the router is launched: ```bash host1$ weave launch host1$ eval $(weave env) ``` -or independently via `launch-dns`: -```bash -host1$ weave launch-router && weave launch-dns && weave launch-proxy -host1$ eval $(weave env) -``` - -The first form is more convenient, however you can only pass weaveDNS -related configuration arguments to `launch-dns` so if you need to -modify the default behaviour you will have to use the latter. +WeaveDNS related configuration arguments can be passed to `launch`. Application containers will use weaveDNS automatically if it is running at the point when they are started. They will use it for name @@ -61,53 +51,40 @@ root@ubuntu:/# ping pingme > **Please note** if both hostname and container name are specified at > the same time the hostname takes precedence; in this circumstance if -> the hostname is not in the weaveDNS domain the container will *not* be -> registered, but will still use weaveDNS for resolution. +> the hostname is not in the weaveDNS domain the container will *not* +> be registered, but will still use weaveDNS for resolution. It is also possible to force or forbid an application container's use of weaveDNS with the `--with-dns` and `--without-dns` options to -`weave run` and `weave launch-proxy`; these override the runtime -detection of the weaveDNS container. - -Each weaveDNS container started with `launch-dns` needs its own unique -IP address in a subnet that is common to all weaveDNS containers. In -the example above we did not specify such an address, so one was -allocated automatically from the default subnet; you can however -specify an address in CIDR format manually. In this case you are -responsible for ensuring that the IP addresses specified are uniquely -allocated and not in use by any other container. - -Finally, weaveDNS can be stopped independently with - - host1$ weave stop-dns - -or in conjunction with the router and proxy via `stop`. +`weave run` and `weave launch-proxy`. ## How it works -The weaveDNS container running on every host acts as the nameserver -for containers on that host. It learns about hostnames for local -containers from the Weave Docker API proxy and from the `weave run` -command. If a hostname is in the `.weave.local` domain then weaveDNS -records the association of that name with the container's weave IP -address(es). +The weaveDNS service running on every host acts as the nameserver for +containers on that host. It learns about hostnames for local containers +from the proxy and from the `weave run` command. If a hostname is in +the `.weave.local` domain then weaveDNS records the association of that +name with the container's weave IP address(es) in its in-memory +database, and broadcasts the association to other weave peers in the +cluster. When weaveDNS is queried for a name in the `.weave.local` domain, it -first checks its own records. If the name is not found there, it asks -the weaveDNS servers on the other hosts in the weave network. +looks up the hostname its in memory database and responds with the IPs +of all containers for that hostname across the entire cluster, in a +random order. When weaveDNS is queried for a name in a domain other than -`.weave.local`, it queries the host's configured nameserver, -which is the standard behaviour for Docker containers. +`.weave.local`, it queries the host's configured nameserver, which is +the standard behaviour for Docker containers. So that containers can connect to a stable and always routable IP -address, weaveDNS publishes its port 53 to the Docker bridge device, -which is assumed to be `docker0`. Some configurations may use a -different Docker bridge device. To supply a different bridge device, -use the environment variable `DOCKER_BRIDGE`, e.g., +address, weaveDNS listens on port 53 to the Docker bridge device, which +is assumed to be `docker0`. Some configurations may use a different +Docker bridge device. To supply a different bridge device, use the +environment variable `DOCKER_BRIDGE`, e.g., ```bash -$ sudo DOCKER_BRIDGE=someother weave launch-dns +$ sudo DOCKER_BRIDGE=someother weave launch ``` In the event that weaveDNS is launched in this way, it's important that @@ -120,8 +97,8 @@ $ sudo DOCKER_BRIDGE=someother weave run --with-dns ... ## Load balancing It is permissible to register multiple containers with the same name: -weaveDNS picks one address at random on each request. This provides a -basic load balancing capability. +weaveDNS returns all addresses, in a random order, for each request. +This provides a basic load balancing capability. Returning to our earlier example, let us start an additional `pingme` container, this time on the 2nd host, and then run some ping tests... @@ -152,8 +129,9 @@ Notice how the ping reaches different addresses. WeaveDNS removes the addresses of any container that dies. This offers a simple way to implement redundancy. E.g. if in our example we stop one of the `pingme` containers and re-run the ping tests, eventually -(within ~30s at most, since that is the weaveDNS [cache expiry time](#ttl)) we -will only be hitting the address of the container that is still alive. +(within ~30s at most, since that is the weaveDNS +[cache expiry time](#ttl)) we will only be hitting the address of the +container that is still alive. ## Adding and removing extra DNS entries @@ -169,14 +147,15 @@ $ weave dns-add 10.2.1.27 $C -h pingme2.weave.local You can also use `dns-add` to add the container's configured hostname and domain, simply by omitting `-h `. -The inverse operation can be carried out using the `dns-remove` command: +The inverse operation can be carried out using the `dns-remove` +command: ```bash $ weave dns-remove 10.2.1.27 $C ``` -When queried about a name with multiple IPs, weaveDNS returns a random -result from the set of IPs available. +When queried about a name with multiple IPs, weaveDNS returns all IPs +available in a random order. ## Hot-swapping service containers @@ -187,33 +166,20 @@ server container and then [remove](#add-remove) the entry for the old server container. Later, when all connections to the old server have terminated, stop the container as normal. -## Retaining DNS entries when containers stop - -By default, weaveDNS watches docker events and removes entries for any -containers that die. You can tell it not to, by adding `--watch=false` -to the container args: - -```bash -$ weave launch-dns --watch=false -``` ## Configuring a custom TTL -By default, weaveDNS specifies a TTL of 30 seconds in any reply sent to -another peer. Peers will honor the TTL received and cache the answer -until it is considered invalid. - -However, you can force a different TTL value by launching weaveDNS with -the `--ttl` argument: +By default, weaveDNS specifies a TTL of 30 seconds in responses to DNS +requests. However, you can force a different TTL value by launching +weave with the `--dns-ttl` argument: ```bash -$ weave launch-dns --ttl=10 +$ weave launch --dns-ttl=10 ``` -This will shorten the lifespan of answers sent to other peers, -so you will be effectively reducing the probability of them having stale -information, but you will also be increasing their resolution times (as -their cache hit rate will be reduced) and the number of request this +This will shorten the lifespan of answers sent to clients, so you will +be effectively reducing the probability of them having stale +information, but you will also be increasing the number of request this weaveDNS instance will receive. ## Configuring the domain search paths @@ -241,12 +207,12 @@ docker run -ti \ ## Using a different local domain By default, weaveDNS uses `weave.local.` as the domain for names on the -Weave network. In general users do not need to change this domain, but you -can force weaveDNS to use a different domain by launching it -with the `--domain` argument. For example, +Weave network. In general users do not need to change this domain, but +you can force weaveDNS to use a different domain by launching it with +the `--dns-domain` argument. For example, ```bash -$ weave launch-dns --domain="mycompany.local." +$ weave launch --dns-domain="mycompany.local." ``` The local domain should end with `local.`, since these names are @@ -265,16 +231,11 @@ DNS: ```` ... -weave DNS 1.0.0 -Listen address :53 -Fallback DNS config &{[10.0.2.3] [] 53 1 5 2} +WeaveDNS (d6:1b:df:42:af:7d) + listening on port 53, for domain weave.local. + response ttl 30 -Local domain weave.local. -Interface &{74 65535 ethwe 82:0c:92:84:0e:88 up|broadcast|multicast} -Zone database: -144b75a9b873: pingme.weave.local.[10.40.0.1]/OBS:1 -74a5510a91ad: ubuntu.weave.local.[10.40.0.2] -weave:remote: pingme.weave.local.[10.32.0.2]/TTL:30 +1d836dea2c56: foo.weave.local. [10.2.2.1] ... ```` @@ -284,17 +245,16 @@ guide](troubleshooting.html#status-report) for more detail. The second section is pertinent to weaveDNS, and includes: +* Port on which the DNS server is listening * The local domain suffix which is being served -* The address on which the DNS server is listening -* The interface being used for multicast DNS -* The fallback DNS which will be used to resolve non local names -* The names known to the local weaveDNS server. Each entry comprises +* The response ttl +* The names known to the weaveDNS server. Each entry comprises the container ID, IP address and its fully qualified domain name Information on the processing of queries, and the general operation of weaveDNS, can be obtained from the container logs with - docker logs weavedns + docker logs weave ## Present limitations @@ -304,6 +264,3 @@ weaveDNS, can be obtained from the container logs with * The server may give unreachable IPs as answers, since it doesn't try to filter by reachability. If you use subnets, align your hostnames with the subnets. - * We use UDP multicast to find out about remote names (from weaveDNS - servers on other hosts); this likely won't scale well beyond a - certain point T.B.D. diff --git a/test/200_dns_test.sh b/test/200_dns_test.sh index 49fa215a..f4d8c998 100755 --- a/test/200_dns_test.sh +++ b/test/200_dns_test.sh @@ -8,7 +8,7 @@ NAME=seetwo.weave.local start_suite "Resolve names on a single host" -launch_dns_on $HOST1 10.2.254.1/24 +weave_on $HOST1 launch start_container $HOST1 $C2/24 --name=c2 -h $NAME start_container_with_dns $HOST1 $C1/24 --name=c1 diff --git a/test/210_dns_cross_hosts_test.sh b/test/210_dns_cross_hosts_test.sh new file mode 100755 index 00000000..b3836bbc --- /dev/null +++ b/test/210_dns_cross_hosts_test.sh @@ -0,0 +1,35 @@ +#! /bin/bash + +. ./config.sh + +C1=10.2.3.78 +C2=10.2.3.34 +C2a=10.2.3.35 +C2b=10.2.3.36 +UNIVERSE=10.2.4.0/24 +NAME2=seetwo.weave.local +NAME4=seefour.weave.local + +start_suite "Resolve names across hosts" + +weave_on $HOST1 launch --ipalloc-range $UNIVERSE +weave_on $HOST2 launch --ipalloc-range $UNIVERSE $HOST1 + +start_container $HOST2 $C2/24 --name=c2 -h $NAME2 +start_container_with_dns $HOST1 $C1/24 --name=c1 + +assert_dns_record $HOST1 c1 $NAME2 $C2 + +# resolution for names mapped to multiple addresses +weave_on $HOST2 dns-add $C2a c2 -h $NAME2 +weave_on $HOST2 dns-add $C2b c2 -h $NAME2 +assert_dns_record $HOST1 c1 $NAME2 $C2 $C2a $C2b + +# resolution when containers addresses come from IPAM +start_container $HOST2 --name=c4 -h $NAME4 +start_container_with_dns $HOST1 --name=c3 +C4=$(container_ip $HOST2 c4) +assert_dns_record $HOST1 c3 $NAME4 $C4 +assert_raises "exec_on $HOST1 c3 getent hosts 8.8.8.8 | grep google" + +end_suite diff --git a/test/210_dns_multicast_test.sh b/test/210_dns_multicast_test.sh deleted file mode 100755 index e8619703..00000000 --- a/test/210_dns_multicast_test.sh +++ /dev/null @@ -1,61 +0,0 @@ -#! /bin/bash - -. ./config.sh - -C1=10.2.3.78 -C2=10.2.3.34 -C2a=10.2.3.35 -C2b=10.2.3.36 -UNIVERSE=10.2.4.0/24 -NAME2=seetwo.weave.local -NAME4=seefour.weave.local - -start() { - launch_dns_on $HOST1 $1 >/dev/null - launch_dns_on $HOST2 $2 >/dev/null -} - -stop() { - weave_on $HOST1 stop-dns - weave_on $HOST2 stop-dns -} - -check() { - start $1 $2 - assert_dns_record $HOST1 c1 $NAME2 $C2 - assert_raises "exec_on $HOST1 c1 getent hosts 8.8.8.8 | grep google" - stop -} - -start_suite "Resolve names across hosts (with and without IPAM), and repopulate on restart" - -weave_on $HOST1 launch-router --ipalloc-range $UNIVERSE -weave_on $HOST2 launch-router --ipalloc-range $UNIVERSE $HOST1 - -start_container $HOST2 $C2/24 --name=c2 -h $NAME2 -start_container_with_dns $HOST1 $C1/24 --name=c1 - -# resolution when DNS is launched with various ways of obtaining an IP -# NB: this also tests re-population on DNS restart -check ip:10.2.254.1/24 ip:10.2.254.2/24 -check net:10.2.4.128/25 net:10.2.4.128/25 -check net:default net:default -check - -start 10.2.254.1/24 10.2.254.2/24 - -# resolution for names mapped to multiple addresses -weave_on $HOST2 dns-add $C2a c2 -h $NAME2 -weave_on $HOST2 dns-add $C2b c2 -h $NAME2 -assert_dns_record $HOST1 c1 $NAME2 $C2 $C2a $C2b - -# resolution when containers addresses come from IPAM -start_container $HOST2 --name=c4 -h $NAME4 -start_container_with_dns $HOST1 --name=c3 -C4=$(container_ip $HOST2 c4) -assert_dns_record $HOST1 c3 $NAME4 $C4 -assert_raises "exec_on $HOST1 c3 getent hosts 8.8.8.8 | grep google" - -stop - -end_suite diff --git a/test/220_dns_custom_test.sh b/test/220_dns_custom_test.sh index ec1b1efe..bb404d32 100755 --- a/test/220_dns_custom_test.sh +++ b/test/220_dns_custom_test.sh @@ -9,7 +9,20 @@ NAME=seetwo.$DOMAIN start_suite "Resolve names in custom domain" -launch_dns_on $HOST1 10.2.254.1/24 --domain $DOMAIN. +# Check with a trailing dot on domain +weave_on $HOST1 launch --dns-domain $DOMAIN. + +start_container $HOST1 $C2/24 --name=c2 -h $NAME +start_container_with_dns $HOST1 $C1/24 --name=c1 + +assert_dns_record $HOST1 c1 $NAME $C2 + +rm_containers $HOST1 c1 +rm_containers $HOST1 c2 +weave_on $HOST1 stop + +# Check without a trailing dot on domain +weave_on $HOST1 launch --dns-domain $DOMAIN start_container $HOST1 $C2/24 --name=c2 -h $NAME start_container_with_dns $HOST1 $C1/24 --name=c1 diff --git a/test/230_dns_unqualified_test.sh b/test/230_dns_unqualified_test.sh index 38e30043..146af9c3 100755 --- a/test/230_dns_unqualified_test.sh +++ b/test/230_dns_unqualified_test.sh @@ -11,7 +11,7 @@ NAME=seeone.$DOMAIN start_suite "Resolve unqualified names" -launch_dns_on $HOST1 10.2.254.1/24 +weave_on $HOST1 launch start_container $HOST1 $C1/24 --name=c1 -h $NAME start_container_with_dns $HOST1 $C2/24 --name=c2 -h seetwo.$DOMAIN diff --git a/test/240_dns_add_name_test.sh b/test/240_dns_add_name_test.sh index ce76e225..87badd50 100755 --- a/test/240_dns_add_name_test.sh +++ b/test/240_dns_add_name_test.sh @@ -10,7 +10,7 @@ NAME3=seethree.weave.local start_suite "Add and remove names on a single host" -launch_dns_on $HOST1 10.2.254.1/24 +weave_on $HOST1 launch start_container $HOST1 --without-dns $C2/24 --name=c2 start_container_with_dns $HOST1 $C1/24 --name=c1 diff --git a/test/250_dns_restart_test.sh b/test/250_dns_restart_test.sh new file mode 100755 index 00000000..9c8a0e54 --- /dev/null +++ b/test/250_dns_restart_test.sh @@ -0,0 +1,25 @@ +#! /bin/bash + +. ./config.sh + +C1=10.2.0.78 +C2=10.2.0.34 +NAME=seetwo.weave.local + +start_suite "Restart Weave, check for repopulation" + +weave_on $HOST1 launch + +start_container $HOST1 $C2/24 --name=c2 -h $NAME +start_container_with_dns $HOST1 $C1/24 --name=c1 + +assert_dns_record $HOST1 c1 $NAME $C2 +assert_raises "exec_on $HOST1 c1 dig MX $NAME | grep -q 'status: NXDOMAIN'" + +weave_on $HOST1 stop +weave_on $HOST1 launch + +assert_dns_record $HOST1 c1 $NAME $C2 +assert_raises "exec_on $HOST1 c1 dig MX $NAME | grep -q 'status: NXDOMAIN'" + +end_suite diff --git a/test/260_dns_removal_test.sh b/test/260_dns_removal_test.sh index 78fa9bd9..bc951d40 100755 --- a/test/260_dns_removal_test.sh +++ b/test/260_dns_removal_test.sh @@ -16,7 +16,7 @@ check() { start_suite "Automatic DNS record removal on container death" -launch_dns_on $HOST1 10.2.254.1/24 +weave_on $HOST1 launch start_container_with_dns $HOST1 $C1/24 --name=c1 start_container $HOST1 $C2/24 $DOPTS diff --git a/test/280_with_or_without_dns_test.sh b/test/280_with_or_without_dns_test.sh index 2a58be95..f38e23e9 100755 --- a/test/280_with_or_without_dns_test.sh +++ b/test/280_with_or_without_dns_test.sh @@ -20,19 +20,12 @@ assert_resolution() { start_suite "With or without DNS test" +# Assert behaviour without weaveDNS running weave_on $HOST1 launch-router +start_container $HOST1 $TARGET_IP/24 --name c2 -h $TARGET DNS_IP=$(weave_on $HOST1 docker-bridge-ip) -# Assert behaviour without weaveDNS running -assert_no_resolution --without-dns -assert_no_resolution -assert_no_resolution --with-dns - -weave_on $HOST1 launch-dns 10.2.254.1/24 $WEAVEDNS_ARGS -start_container $HOST1 $TARGET_IP/24 --name c2 -h $TARGET - -# Assert behaviour with weaveDNS running assert_no_resolution --without-dns assert_resolution assert_resolution --with-dns diff --git a/test/290_dns_fallback_test.sh b/test/290_dns_fallback_test.sh index 9eee0f8f..1d897732 100755 --- a/test/290_dns_fallback_test.sh +++ b/test/290_dns_fallback_test.sh @@ -4,7 +4,7 @@ start_suite "Resolve a non-weave address" -launch_dns_on $HOST1 10.2.254.1/24 +weave_on $HOST1 launch-router start_container_with_dns $HOST1 10.2.1.5/24 --name=c1 diff --git a/test/295_dns_large_response.sh b/test/295_dns_large_response.sh new file mode 100755 index 00000000..daf7e82e --- /dev/null +++ b/test/295_dns_large_response.sh @@ -0,0 +1,23 @@ +#! /bin/bash + +. ./config.sh + +NAME=seetwo.weave.local + +N=50 +start_suite "Add $N dns entries and check we get the right response." + +weave_on $HOST1 launch + +CID=$(start_container_with_dns $HOST1 10.2.1.0/24 --name=c0) +IPS="" +for i in $(seq $N); do + IPS="$IPS 10.2.1.$i" +done +weave_on $HOST1 dns-add $IPS $CID -h $NAME + +assert_dns_record $HOST1 c0 $NAME $IPS + +assert_raises "exec_on $HOST1 c0 dig MX $NAME | grep -q 'status: NXDOMAIN'" + +end_suite diff --git a/test/500_weave_multi_cidr_test.sh b/test/500_weave_multi_cidr_test.sh index 83bc20c8..d3d09661 100755 --- a/test/500_weave_multi_cidr_test.sh +++ b/test/500_weave_multi_cidr_test.sh @@ -74,7 +74,6 @@ start_suite "Weave run/start/attach/detach/expose/hide with multiple cidr argume # lowest available address in the subnet weave_on $HOST1 launch-router --ipalloc-range 10.2.3.0/24 -launch_dns_on $HOST1 10.254.254.254/24 # Run container with three cidrs CID=$(start_container $HOST1 10.2.1.1/24 ip:10.2.2.1/24 net:10.2.3.0/24 --name=multicidr -h $NAME | cut -b 1-12) diff --git a/test/630_proxy_dns_test.sh b/test/630_proxy_dns_test.sh index d3b9ce04..8276be6c 100755 --- a/test/630_proxy_dns_test.sh +++ b/test/630_proxy_dns_test.sh @@ -37,7 +37,7 @@ weave_on $HOST1 launch-proxy assert_no_resolution # Assert behaviour with weaveDNS running -launch_dns_on $HOST1 10.2.254.1/24 +weave_on $HOST1 launch-router boot_containers assert_dns_record $HOST1 c1 $C2_NAME $C2 assert_dns_record $HOST1 c2 $C1_NAME $C1 diff --git a/test/635_proxy_dns_unqualified_test.sh b/test/635_proxy_dns_unqualified_test.sh index b4ffbd18..f81cbb3f 100755 --- a/test/635_proxy_dns_unqualified_test.sh +++ b/test/635_proxy_dns_unqualified_test.sh @@ -15,8 +15,7 @@ start_container() { start_suite "Resolve unqualified names" -weave_on $HOST1 launch-dns 10.2.254.1/24 -weave_on $HOST1 launch-proxy +weave_on $HOST1 launch start_container -e WEAVE_CIDR=$C1/24 --name=c1 -h $NAME start_container -e WEAVE_CIDR=$C2/24 --name=c2 -h seetwo.$DOMAIN diff --git a/test/640_proxy_restart_reattaches_test.sh b/test/640_proxy_restart_reattaches_test.sh index b85e87c4..1e468def 100755 --- a/test/640_proxy_restart_reattaches_test.sh +++ b/test/640_proxy_restart_reattaches_test.sh @@ -8,8 +8,7 @@ NAME=seetwo.weave.local start_suite "Proxy restart reattaches networking to containers" -launch_dns_on $HOST1 10.2.254.1/24 -weave_on $HOST1 launch-proxy --with-dns +weave_on $HOST1 launch proxy docker_on $HOST1 run -e WEAVE_CIDR=$C2/24 -dt --name=c2 -h $NAME $SMALL_IMAGE /bin/sh proxy docker_on $HOST1 run -e WEAVE_CIDR=$C1/24 -dt --name=c1 $DNS_IMAGE /bin/sh diff --git a/test/config.sh b/test/config.sh index 3d136999..e9e4a49b 100644 --- a/test/config.sh +++ b/test/config.sh @@ -133,12 +133,6 @@ exec_on() { docker -H tcp://$host:$DOCKER_PORT exec $container "$@" } -launch_dns_on() { - host=$1 - shift 1 - weave_on $host launch-dns $@ $WEAVEDNS_ARGS -} - start_container() { host=$1 shift 1 diff --git a/weave b/weave index c7983f68..1bedfb61 100755 --- a/weave +++ b/weave @@ -34,7 +34,6 @@ weave launch [--password ] [--nickname ] weave launch-router [--password ] [--nickname ] [--ipalloc-range [--ipalloc-default-subnet ]] [--no-discovery] [--init-peer-count ] ... -weave launch-dns [] weave launch-proxy [-H ] [--with-dns | --without-dns] [--no-default-ipalloc] [--no-rewrite-hosts] weave connect [--replace] [ ...] @@ -55,7 +54,6 @@ weave config weave version weave stop weave stop-router -weave stop-dns weave stop-proxy weave reset weave rmpeer @@ -78,7 +76,6 @@ exec_remote() { -e VERSION \ -e WEAVE_DEBUG \ -e WEAVE_DOCKER_ARGS \ - -e WEAVEDNS_DOCKER_ARGS \ -e WEAVEPROXY_DOCKER_ARGS \ -e WEAVE_PASSWORD \ -e WEAVE_PORT \ @@ -164,21 +161,16 @@ fi shift 1 BASE_IMAGE=$DOCKERHUB_USER/weave -BASE_DNS_IMAGE=$DOCKERHUB_USER/weavedns IMAGE=$BASE_IMAGE:$IMAGE_VERSION -DNS_IMAGE=$BASE_DNS_IMAGE:$IMAGE_VERSION PROCFS=${PROCFS:-/proc} DOCKER_BRIDGE=${DOCKER_BRIDGE:-docker0} CONTAINER_NAME=${WEAVE_CONTAINER_NAME:-weave} -DNS_CONTAINER_NAME=weavedns BRIDGE=weave CONTAINER_IFNAME=ethwe MTU=65535 PORT=${WEAVE_PORT:-6783} HTTP_PORT=6784 -DNS_HTTP_PORT=6785 -DNS_HTTP_IFNAME=eth0 PROXY_PORT=12375 PROXY_CONTAINER_NAME=weaveproxy COVERAGE_ARGS="" @@ -598,10 +590,6 @@ call_weave() { http_call $CONTAINER_NAME $HTTP_PORT "$@" } -call_dns() { - http_call $DNS_CONTAINER_NAME $DNS_HTTP_PORT "$@" -} - death_msg() { echo "The $1 container has died. Consult the logs with 'docker logs $1' for further details." } @@ -672,7 +660,7 @@ peer_args() { # Memoized function to query the weaveDNS server for our dns domain. dns_domain() { - DNS_DOMAIN=${DNS_DOMAIN:-$(call_dns GET /domain 2>/dev/null || true)} + DNS_DOMAIN=${DNS_DOMAIN:-$(call_weave GET /domain 2>/dev/null || true)} DNS_DOMAIN=${DNS_DOMAIN:-weave.local.} echo $DNS_DOMAIN } @@ -719,9 +707,6 @@ dns_args() { shift done [ -n "$WITHOUT_DNS" ] && return 0 - if [ -z "$WITH_DNS" ] ; then - check_running $DNS_CONTAINER_NAME 2>/dev/null || return 0 - fi DNS_ARGS="--dns $DOCKER_BRIDGE_IP" if [ -n "$NAME_ARG" -a -z "$HOSTNAME_SPECIFIED" ] ; then DOMAIN=$(dns_domain) @@ -754,9 +739,9 @@ check_running() { fi } -# Execute $@ only if the weaveDNS container is running -when_dns_running() { - if check_running $DNS_CONTAINER_NAME 2>/dev/null; then +# Execute $@ only if the weave container is running +when_weave_running() { + if check_running $CONTAINER_NAME 2>/dev/null; then "$@" fi } @@ -788,7 +773,7 @@ put_dns_fqdn() { shift 2 for ADDR in "$@" ; do - call_dns PUT /name/$CONTAINER_ID/${ADDR%/*} --data-urlencode fqdn=$FQDN $CHECK_ALIVE || true + call_weave PUT /name/$CONTAINER_ID/${ADDR%/*} --data-urlencode fqdn=$FQDN $CHECK_ALIVE || true done } @@ -798,7 +783,7 @@ delete_dns() { shift 1 for ADDR in "$@" ; do - call_dns DELETE /name/$CONTAINER_ID/${ADDR%/*} || true + call_weave DELETE /name/$CONTAINER_ID/${ADDR%/*} || true done } @@ -809,13 +794,13 @@ delete_dns_fqdn() { shift 2 for ADDR in "$@" ; do - call_dns DELETE /name/$CONTAINER_ID/${ADDR%/*}?fqdn=$FQDN || true + call_weave DELETE /name/$CONTAINER_ID/${ADDR%/*}?fqdn=$FQDN || true done } # Tell the newly-started weaveDNS about existing weave IPs populate_dns() { - if ! wait_for_status $DNS_CONTAINER_NAME $DNS_HTTP_PORT ; then + if ! wait_for_status $CONTAINER_NAME $HTTP_PORT ; then echo "If running, it will not be pre-populated." >&2 return 0 fi @@ -981,6 +966,7 @@ common_launch_args() { launch_router() { create_bridge + docker_bridge_ip # We set the router name to the bridge mac since that is # stable across re-creations of the containers. # @@ -1050,6 +1036,7 @@ launch_router() { # when launching the weave container. CONTAINER=$(docker run --privileged -d --name=$CONTAINER_NAME \ -p $PORT:$CONTAINER_PORT/tcp -p $PORT:$CONTAINER_PORT/udp \ + -p $DOCKER_BRIDGE_IP:53:53/udp -p $DOCKER_BRIDGE_IP:53:53/tcp \ -v /var/run/docker.sock:/var/run/docker.sock \ -e WEAVE_PASSWORD \ -e WEAVE_CIDR=none \ @@ -1060,24 +1047,6 @@ launch_router() { # Tell the newly-started weave IP allocator about existing weave IPs with_container_addresses ipam_claim weave:expose $(docker ps -q --no-trunc) fi -} - -launch_dns() { - collect_cidr_args "$@" - [ $CIDR_ARG_COUNT -lt 2 ] || usage - shift $CIDR_ARG_COUNT - create_bridge - docker_bridge_ip - # Set WEAVEDNS_DOCKER_ARGS in the environment in order to supply - # additional parameters, such as resource limits, to docker - # when launching the weave container. - DNS_CONTAINER=$(docker run --privileged -d --name=$DNS_CONTAINER_NAME \ - -p $DOCKER_BRIDGE_IP:53:53/udp \ - -e WEAVE_CIDR=none \ - -v /var/run/docker.sock:/var/run/docker.sock \ - $WEAVEDNS_DOCKER_ARGS $DNS_IMAGE --iface $CONTAINER_IFNAME --http-iface "$DNS_HTTP_IFNAME" "$@") - ipam_cidrs_or_die $DNS_CONTAINER $CIDR_ARGS - with_container_netns_or_die $DNS_CONTAINER attach $ALL_CIDRS populate_dns } @@ -1200,7 +1169,7 @@ shift 1 case "$COMMAND" in setup) - for img in $IMAGE $DNS_IMAGE $EXEC_IMAGE ; do + for img in $IMAGE $EXEC_IMAGE ; do docker pull $img done ;; @@ -1212,11 +1181,9 @@ case "$COMMAND" in launch) deprecation_warnings "$@" check_not_running $CONTAINER_NAME $BASE_IMAGE - check_not_running $DNS_CONTAINER_NAME $BASE_DNS_IMAGE check_not_running $PROXY_CONTAINER_NAME $EXEC_IMAGE COMMON_ARGS=$(common_launch_args "$@") launch_router "$@" - launch_dns $COMMON_ARGS launch_proxy $COMMON_ARGS ;; launch-router) @@ -1225,11 +1192,6 @@ case "$COMMAND" in launch_router "$@" echo $CONTAINER ;; - launch-dns) - check_not_running $DNS_CONTAINER_NAME $BASE_DNS_IMAGE - launch_dns "$@" - echo $DNS_CONTAINER - ;; launch-proxy) deprecation_warnings "$@" check_not_running $PROXY_CONTAINER_NAME $EXEC_IMAGE @@ -1260,7 +1222,6 @@ case "$COMMAND" in res=0 call_weave GET /status || res=1 echo - call_dns GET /status 2>/dev/null || true if check_running $PROXY_CONTAINER_NAME 2>/dev/null ; then echo echo "weave proxy is running" @@ -1275,7 +1236,6 @@ case "$COMMAND" in [ $# -eq 0 ] || usage echo weave script $SCRIPT_VERSION ask_version $CONTAINER_NAME $IMAGE || true - ask_version $DNS_CONTAINER_NAME $DNS_IMAGE || true if ! EXEC_IMAGE_ID=$(docker inspect --format='{{.Id}}' $EXEC_IMAGE 2>/dev/null) ; then echo "Unable to find $EXEC_IMAGE image." >&2 else @@ -1296,7 +1256,7 @@ case "$COMMAND" in create_bridge ipam_cidrs_or_die $CONTAINER $CIDR_ARGS with_container_netns_or_die $CONTAINER attach $ALL_CIDRS - when_dns_running with_container_fqdn $CONTAINER put_dns_fqdn $ALL_CIDRS + when_weave_running with_container_fqdn $CONTAINER put_dns_fqdn $ALL_CIDRS echo $CONTAINER ;; dns-args) @@ -1316,7 +1276,7 @@ case "$COMMAND" in create_bridge ipam_cidrs_or_die $CONTAINER $CIDR_ARGS with_container_netns_or_die $CONTAINER attach $ALL_CIDRS - when_dns_running with_container_fqdn $CONTAINER put_dns_fqdn $ALL_CIDRS + when_weave_running with_container_fqdn $CONTAINER put_dns_fqdn $ALL_CIDRS echo $RES ;; attach) @@ -1331,7 +1291,7 @@ case "$COMMAND" in create_bridge ipam_cidrs$ATTACH_TYPE $CONTAINER $CIDR_ARGS with_container_netns$ATTACH_TYPE $CONTAINER attach $ALL_CIDRS >/dev/null - when_dns_running with_container_fqdn $CONTAINER put_dns_fqdn $ALL_CIDRS + when_weave_running with_container_fqdn $CONTAINER put_dns_fqdn $ALL_CIDRS show_addrs $ALL_CIDRS ;; detach) @@ -1341,7 +1301,7 @@ case "$COMMAND" in CONTAINER=$(container_id $1) ipam_cidrs $CONTAINER $CIDR_ARGS with_container_netns $CONTAINER detach $ALL_CIDRS >/dev/null - when_dns_running with_container_fqdn $CONTAINER delete_dns_fqdn $ALL_CIDRS + when_weave_running with_container_fqdn $CONTAINER delete_dns_fqdn $ALL_CIDRS for CIDR in $IPAM_CIDRS ; do call_weave DELETE /ip/$CONTAINER/${CIDR%/*} done @@ -1351,7 +1311,7 @@ case "$COMMAND" in collect_ip_args "$@" shift $IP_COUNT [ $IP_COUNT -gt 0 -a \( $# -eq 1 -o \( $# -eq 3 -a "$2" = "-h" \) \) ] || usage - check_running $DNS_CONTAINER_NAME + check_running $CONTAINER_NAME CONTAINER=$(container_id $1) if [ $# -eq 1 ] ; then with_container_fqdn $CONTAINER put_dns_fqdn $IP_ARGS @@ -1363,7 +1323,7 @@ case "$COMMAND" in collect_ip_args "$@" shift $IP_COUNT [ $IP_COUNT -gt 0 -a \( $# -eq 1 -o \( $# -eq 3 -a "$2" = "-h" \) \) ] || usage - check_running $DNS_CONTAINER_NAME + check_running $CONTAINER_NAME CONTAINER=$(container_id $1) if [ $# -eq 1 ] ; then delete_dns $CONTAINER $IP_ARGS @@ -1389,7 +1349,7 @@ case "$COMMAND" in add_iptables_rule nat WEAVE -d $CIDR ! -s $CIDR -j MASQUERADE add_iptables_rule nat WEAVE -s $CIDR ! -d $CIDR -j MASQUERADE if [ -n "$FQDN" ] ; then - when_dns_running put_dns_fqdn --no-check-alive weave:expose $FQDN $CIDR + when_weave_running put_dns_fqdn --no-check-alive weave:expose $FQDN $CIDR fi fi done @@ -1405,7 +1365,7 @@ case "$COMMAND" in ip addr del dev $BRIDGE $CIDR delete_iptables_rule nat WEAVE -d $CIDR ! -s $CIDR -j MASQUERADE delete_iptables_rule nat WEAVE -s $CIDR ! -d $CIDR -j MASQUERADE - when_dns_running delete_dns weave:expose $CIDR + when_weave_running delete_dns weave:expose $CIDR fi done for CIDR in $IPAM_CIDRS ; do @@ -1416,7 +1376,6 @@ case "$COMMAND" in stop) [ $# -eq 0 ] || usage stop $PROXY_CONTAINER_NAME "Proxy" - stop $DNS_CONTAINER_NAME "WeaveDNS" stop $CONTAINER_NAME "Weave" conntrack -D -p udp --dport $PORT >/dev/null 2>&1 || true ;; @@ -1425,10 +1384,6 @@ case "$COMMAND" in stop $CONTAINER_NAME "Weave" conntrack -D -p udp --dport $PORT >/dev/null 2>&1 || true ;; - stop-dns) - [ $# -eq 0 ] || usage - stop $DNS_CONTAINER_NAME "WeaveDNS" - ;; stop-proxy) [ $# -eq 0 ] || usage stop $PROXY_CONTAINER_NAME "Proxy" @@ -1436,7 +1391,7 @@ case "$COMMAND" in reset) [ $# -eq 0 ] || usage call_weave DELETE /peer >/dev/null 2>&1 || true - for NAME in $CONTAINER_NAME $DNS_CONTAINER_NAME $PROXY_CONTAINER_NAME ; do + for NAME in $CONTAINER_NAME $PROXY_CONTAINER_NAME ; do docker stop $NAME >/dev/null 2>&1 || true docker rm -f $NAME >/dev/null 2>&1 || true done @@ -1451,6 +1406,14 @@ case "$COMMAND" in PEER=$1 call_weave DELETE /peer/$PEER ;; + launch-dns) + echo "The 'launch-dns' command has been removed; DNS is launched as part of 'launch' and 'launch-router'." >&2 + exit 0 + ;; + stop-dns) + echo "The 'stop-dns command has been removed; DNS is stopped as part of 'stop' and 'stop-router'." >&2 + exit 0 + ;; *) echo "Unknown weave command '$COMMAND'" >&2 usage