Merge pull request #2378 from weaveworks/issues/2364-observe-resolv-conf

Observe resolv.conf changes

Fixes #2364.

nameserver/dns.go

@@ -5,8 +5,10 @@ import (
 	"fmt"
 	"math/rand"
 	"net"
+	"os"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/miekg/dns"
@@ -17,7 +19,6 @@ import (
 const (
 	topDomain        = "."
 	reverseDNSdomain = "in-addr.arpa."
-	etcResolvConf    = "/etc/resolv.conf"
 	udpBuffSize      = uint16(4096)
 	minUDPSize       = 512
 
@@ -26,16 +27,48 @@ const (
 	DefaultClientTimeout = 5 * time.Second
 )
 
-type DNSServer struct {
-	ns      *Nameserver
-	domain  string
-	ttl     uint32
-	address string
+type Upstream interface {
+	Config() *dns.ClientConfig
+}
 
-	servers   []*dns.Server
-	upstream  *dns.ClientConfig
-	tcpClient *dns.Client
-	udpClient *dns.Client
+func NewUpstream(resolvConf, filterAddress string) Upstream {
+	return &upstream{
+		resolvConf:    resolvConf,
+		filterAddress: filterAddress,
+		cachedConfig:  &dns.ClientConfig{}}
+}
+
+type upstream struct {
+	sync.Mutex
+
+	resolvConf    string
+	filterAddress string
+	cachedConfig  *dns.ClientConfig
+	lastStat      time.Time
+	lastModified  time.Time
+}
+
+func (u *upstream) Config() *dns.ClientConfig {
+	u.Lock()
+	defer u.Unlock()
+
+	// If we checked less than five seconds ago return what we already have
+	now := time.Now()
+	if now.Sub(u.lastStat) < 5*time.Second {
+		return u.cachedConfig
+	}
+	u.lastStat = now
+
+	// Reread file if it has changed
+	if fi, err := os.Stat(u.resolvConf); err == nil && fi.ModTime() != u.lastModified {
+		if config, err := dns.ClientConfigFromFile(u.resolvConf); err == nil {
+			config.Servers = filter(config.Servers, u.filterAddress)
+			u.lastModified = fi.ModTime()
+			u.cachedConfig = config
+		}
+	}
+
+	return u.cachedConfig
 }
 
 func filter(ss []string, s string) []string {
@@ -49,24 +82,30 @@ func filter(ss []string, s string) []string {
 	return ss
 }
 
-func NewDNSServer(ns *Nameserver, domain, address, effectiveAddress string, ttl uint32, clientTimeout time.Duration) (*DNSServer, error) {
+type DNSServer struct {
+	ns      *Nameserver
+	domain  string
+	ttl     uint32
+	address string
+
+	servers   []*dns.Server
+	upstream  Upstream
+	tcpClient *dns.Client
+	udpClient *dns.Client
+}
+
+func NewDNSServer(ns *Nameserver, domain, address string, upstream Upstream, ttl uint32, clientTimeout time.Duration) (*DNSServer, error) {
 	s := &DNSServer{
 		ns:        ns,
 		domain:    dns.Fqdn(domain),
 		ttl:       ttl,
 		address:   address,
+		upstream:  upstream,
 		tcpClient: &dns.Client{Net: "tcp", ReadTimeout: clientTimeout},
 		udpClient: &dns.Client{Net: "udp", ReadTimeout: clientTimeout, UDPSize: udpBuffSize},
 	}
-	var err error
-	if s.upstream, err = dns.ClientConfigFromFile(etcResolvConf); err != nil {
-		return nil, err
-	}
-	if s.upstream != nil {
-		s.upstream.Servers = filter(s.upstream.Servers, effectiveAddress)
-	}
 
-	err = s.listen(address)
+	err := s.listen(address)
 	return s, err
 }
 
@@ -218,10 +257,11 @@ func (h *handler) handleRecursive(w dns.ResponseWriter, req *dns.Msg) {
 		}
 	}
 
-	for _, server := range h.upstream.Servers {
+	upstreamConfig := h.upstream.Config()
+	for _, server := range upstreamConfig.Servers {
 		reqCopy := req.Copy()
 		reqCopy.Id = dns.Id()
-		response, _, err := h.client.Exchange(reqCopy, fmt.Sprintf("%s:%s", server, h.upstream.Port))
+		response, _, err := h.client.Exchange(reqCopy, fmt.Sprintf("%s:%s", server, upstreamConfig.Port))
 		if (err != nil && err != dns.ErrTruncated) || response == nil {
 			h.ns.debugf("error trying %s: %v", server, err)
 			continue

nameserver/dns_test.go

@@ -15,17 +15,22 @@ import (
 	"github.com/weaveworks/weave/net/address"
 )
 
+type mockUpstream struct {
+	config *dns.ClientConfig
+}
+
+func (mu *mockUpstream) Config() *dns.ClientConfig {
+	return mu.config
+}
+
 func startServer(t *testing.T, upstream *dns.ClientConfig) (*DNSServer, *Nameserver, int, int) {
 	peername, err := mesh.PeerNameFromString("00:00:00:02:00:00")
 	require.Nil(t, err)
 	nameserver := New(peername, "", func(mesh.PeerName) bool { return true })
-	dnsserver, err := NewDNSServer(nameserver, "weave.local.", "0.0.0.0:0", "", 30, 5*time.Second)
+	dnsserver, err := NewDNSServer(nameserver, "weave.local.", "0.0.0.0:0", &mockUpstream{upstream}, 30, 5*time.Second)
 	require.Nil(t, err)
 	udpPort := dnsserver.servers[0].PacketConn.LocalAddr().(*net.UDPAddr).Port
 	tcpPort := dnsserver.servers[1].Listener.Addr().(*net.TCPAddr).Port
-	if upstream != nil {
-		dnsserver.upstream = upstream
-	}
 	go dnsserver.ActivateAndServe()
 	return dnsserver, nameserver, udpPort, tcpPort
 }

nameserver/status.go

@@ -38,7 +38,7 @@ func NewStatus(ns *Nameserver, dnsServer *DNSServer) *Status {
 
 	return &Status{
 		dnsServer.domain,
-		dnsServer.upstream.Servers,
+		dnsServer.upstream.Config().Servers,
 		dnsServer.address,
 		dnsServer.ttl,
 		entryStatusSlice}

prog/weaver/main.go

@@ -50,6 +50,7 @@ type dnsConfig struct {
 	TTL                    int
 	ClientTimeout          time.Duration
 	EffectiveListenAddress string
+	ResolvConf             string
 }
 
 const (
@@ -207,6 +208,7 @@ func main() {
 	mflag.IntVar(&dnsConfig.TTL, []string{"-dns-ttl"}, nameserver.DefaultTTL, "TTL for DNS request from our domain")
 	mflag.DurationVar(&dnsConfig.ClientTimeout, []string{"-dns-fallback-timeout"}, nameserver.DefaultClientTimeout, "timeout for fallback DNS requests")
 	mflag.StringVar(&dnsConfig.EffectiveListenAddress, []string{"-dns-effective-listen-address"}, "", "address DNS will actually be listening, after Docker port mapping")
+	mflag.StringVar(&dnsConfig.ResolvConf, []string{"-resolv-conf"}, "", "path to resolver configuration for fallback DNS lookups")
 	mflag.StringVar(&datapathName, []string{"-datapath"}, "", "ODP datapath name")
 	mflag.StringVar(&trustedSubnetStr, []string{"-trusted-subnets"}, "", "comma-separated list of trusted subnets in CIDR notation")
 	mflag.StringVar(&dbPrefix, []string{"-db-prefix"}, "/weavedb/weave", "pathname/prefix of filename to store data")
@@ -482,8 +484,9 @@ func createDNSServer(config dnsConfig, router *mesh.Router, isKnownPeer func(mes
 	ns := nameserver.New(router.Ourself.Peer.Name, config.Domain, isKnownPeer)
 	router.Peers.OnGC(func(peer *mesh.Peer) { ns.PeerGone(peer.Name) })
 	ns.SetGossip(router.NewGossip("nameserver", ns))
+	upstream := nameserver.NewUpstream(config.ResolvConf, config.EffectiveListenAddress)
 	dnsserver, err := nameserver.NewDNSServer(ns, config.Domain, config.ListenAddress,
-		config.EffectiveListenAddress, uint32(config.TTL), config.ClientTimeout)
+		upstream, uint32(config.TTL), config.ClientTimeout)
 	if err != nil {
 		Log.Fatal("Unable to start dns server: ", err)
 	}

weave

@@ -163,11 +163,16 @@ setup_cni() {
     fi
 }
 
+launch_router_options() {
+    [ "$1" = "launch" -o "$1" = "launch-router" ] && echo -v /:/host -e HOST_ROOT=/host
+}
+
 exec_remote() {
     docker $DOCKER_CLIENT_ARGS run --rm \
         $(docker_run_options) \
         --pid=host \
         $(cni_volume_options "$@") \
+        $(launch_router_options "$@") \
         -e DOCKERHUB_USER="$DOCKERHUB_USER" \
         -e WEAVE_VERSION \
         -e WEAVE_DEBUG \
@@ -1625,6 +1630,10 @@ launch_router() {
            --label=weavevolumes $WEAVEDB_IMAGE >/dev/null
     fi
 
+    RESOLV_CONF=$(chroot ${HOST_ROOT:-/} readlink -f /etc/resolv.conf)
+    RESOLV_CONF_DIR=$(dirname "$RESOLV_CONF")
+    RESOLV_CONF_BASE=$(basename "$RESOLV_CONF")
+
     # Set WEAVE_DOCKER_ARGS in the environment in order to supply
     # additional parameters, such as resource limits, to docker
     # when launching the weave container.
@@ -1632,6 +1641,7 @@ launch_router() {
         $(docker_run_options) \
         $RESTART_POLICY \
         --volumes-from $DB_CONTAINER_NAME \
+        -v $RESOLV_CONF_DIR:/var/run/weave/etc \
         -e WEAVE_PASSWORD \
         -e CHECKPOINT_DISABLE \
         $WEAVE_DOCKER_ARGS $IMAGE $COVERAGE_ARGS \
@@ -1642,6 +1652,7 @@ launch_router() {
         $DNS_ROUTER_OPTS $NO_DNS_OPT \
         $AWSVPC_ARGS \
         --http-addr $HTTP_ADDR \
+        --resolv-conf "/var/run/weave/etc/$RESOLV_CONF_BASE" \
         "$@")
     setup_router_iface_$BRIDGE_TYPE
     wait_for_status $CONTAINER_NAME http_call $HTTP_ADDR