From 2ebd358aeb97a42a969359c6a0e004f4ab57eb9f Mon Sep 17 00:00:00 2001 From: Martynas Pumputis Date: Thu, 4 May 2017 17:18:32 +0100 Subject: [PATCH] Workaround missing docker.sock in plugin-v2 "docker.sock" is used to access Docker API which is required to: 1. subscribe to Docker events to configure-arp, 2. query Docker network info to detect whether multicast should be enabled. 3. get list of Swarm peers before starting the plugin. (1) is currently not implemented. (2) is replaced by WEAVE_MULTICAST env var which can enable multicast for all Weave networks created by the plugin (not ideal, but better than nothing). (3) is fixed by not removing "restart.sentinel". --- plugin/net/driver.go | 64 ++++++++++++++++++++++++---------- plugin/plugin.go | 25 ++++++-------- prog/net-plugin/config.json | 8 +++++ prog/net-plugin/launch.sh | 55 +++++++++++++++++++----------- prog/weaver/main.go | 68 +++++++++++++++++++------------------ 5 files changed, 136 insertions(+), 84 deletions(-) diff --git a/plugin/net/driver.go b/plugin/net/driver.go index 6ddadb07..ede1f7a9 100644 --- a/plugin/net/driver.go +++ b/plugin/net/driver.go @@ -29,28 +29,39 @@ type network struct { type driver struct { sync.RWMutex - scope string - docker *docker.Client - dns bool - isPluginV2 bool - networks map[string]network - isNetworkOur func(driverName string) bool + name string + scope string + // Docker API is not available for plugin-v2 + docker *docker.Client + dns bool + isPluginV2 bool + // Enable multicast regardless whether multicast opt is passed to Docker; + // used only by plugin-v2 + forceMulticast bool + networks map[string]network } -func New(client *docker.Client, weave *weaveapi.Client, scope string, dns, isPluginV2 bool, isNetworkOur func(string) bool) (skel.Driver, error) { +func New(client *docker.Client, weave *weaveapi.Client, name, scope string, dns, isPluginV2, forceMulticast bool) (skel.Driver, error) { driver := &driver{ - scope: scope, - docker: client, - dns: dns, - isPluginV2: isPluginV2, - networks: make(map[string]network), - isNetworkOur: isNetworkOur, + name: name, + scope: scope, + docker: client, + dns: dns, + isPluginV2: isPluginV2, + // make sure that it's used only by plugin-v2 + forceMulticast: isPluginV2 && forceMulticast, + networks: make(map[string]network), } - _, err := NewWatcher(client, weave, driver) - if err != nil { - return nil, err + // Do not start watcher in the case of plugin v2, which prevents us from + // configuring arp settings of containers. + if client != nil { + _, err := NewWatcher(client, weave, driver) + if err != nil { + return nil, err + } } + return driver, nil } @@ -189,11 +200,30 @@ func (driver *driver) findNetworkInfo(id string) (network, error) { if found { return network, nil } + + // plugin-v2 does not have access to docker.sock, so we cannot call Docker + // API for the network info. + if driver.isPluginV2 { + // safe to set, as isOurs used only by the watcher which is disabled for plugin-v2 + network.isOurs = true + network.hasMulticastRoute = driver.forceMulticast + + driver.Lock() + driver.networks[id] = network + driver.Unlock() + + return network, nil + } + + if driver.docker == nil { + return network, fmt.Errorf("Docker client disabled; unable to get network info") + } + info, err := driver.docker.NetworkInfo(id) if err != nil { return network, err } - return driver.setupNetworkInfo(id, driver.isNetworkOur(info.Driver), info.Options) + return driver.setupNetworkInfo(id, info.Driver == driver.name, info.Options) } func (driver *driver) setupNetworkInfo(id string, isOurs bool, options map[string]string) (network, error) { diff --git a/plugin/plugin.go b/plugin/plugin.go index 3da45f0a..7c37e319 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -23,23 +23,23 @@ const ( var Log = common.Log -func Start(weaveAPIAddr string, dockerClient *docker.Client, address string, meshAddress string, dns bool, isPluginV2 bool) { +func Start(weaveAPIAddr string, dockerClient *docker.Client, address string, meshAddress string, dns bool, isPluginV2, forceMulticast bool) { weave := weaveapi.NewClient(weaveAPIAddr, Log) Log.Info("Waiting for Weave API Server...") weave.WaitAPIServer(30) Log.Info("Finished waiting for Weave API Server") - if err := run(dockerClient, weave, address, meshAddress, dns, isPluginV2); err != nil { + if err := run(dockerClient, weave, address, meshAddress, dns, isPluginV2, forceMulticast); err != nil { Log.Fatal(err) } } -func run(dockerClient *docker.Client, weave *weaveapi.Client, address, meshAddress string, dns, isPluginV2 bool) error { +func run(dockerClient *docker.Client, weave *weaveapi.Client, address, meshAddress string, dns, isPluginV2, forceMulticast bool) error { endChan := make(chan error, 1) if address != "" { - globalListener, err := listenAndServe(dockerClient, weave, address, endChan, "global", false, dns, isPluginV2) + globalListener, err := listenAndServe(dockerClient, weave, address, endChan, "global", false, dns, isPluginV2, forceMulticast) if err != nil { return err } @@ -47,7 +47,7 @@ func run(dockerClient *docker.Client, weave *weaveapi.Client, address, meshAddre defer globalListener.Close() } if meshAddress != "" { - meshListener, err := listenAndServe(dockerClient, weave, meshAddress, endChan, "local", true, dns, isPluginV2) + meshListener, err := listenAndServe(dockerClient, weave, meshAddress, endChan, "local", true, dns, isPluginV2, forceMulticast) if err != nil { return err } @@ -58,20 +58,15 @@ func run(dockerClient *docker.Client, weave *weaveapi.Client, address, meshAddre return <-endChan } -func listenAndServe(dockerClient *docker.Client, weave *weaveapi.Client, address string, endChan chan<- error, scope string, withIpam, dns bool, isPluginV2 bool) (net.Listener, error) { - var isNetworkOur func(string) bool +func listenAndServe(dockerClient *docker.Client, weave *weaveapi.Client, address string, endChan chan<- error, scope string, withIpam, dns bool, isPluginV2, forceMulticast bool) (net.Listener, error) { + var name string if isPluginV2 { - isNetworkOur = func(driverName string) bool { - return strings.Contains(driverName, pluginV2Name) - } + name = pluginV2Name } else { - name := strings.TrimSuffix(path.Base(address), ".sock") - isNetworkOur = func(driverName string) bool { - return driverName == name - } + name = strings.TrimSuffix(path.Base(address), ".sock") } - d, err := netplugin.New(dockerClient, weave, scope, dns, isPluginV2, isNetworkOur) + d, err := netplugin.New(dockerClient, weave, name, scope, dns, isPluginV2, forceMulticast) if err != nil { return nil, err } diff --git a/prog/net-plugin/config.json b/prog/net-plugin/config.json index 74601b08..c51d564f 100644 --- a/prog/net-plugin/config.json +++ b/prog/net-plugin/config.json @@ -66,6 +66,14 @@ "value" ], "value": "" + }, + { + "description": "Enable multicast for all Weave networks", + "name": "WEAVE_MULTICAST", + "settable": [ + "value" + ], + "value": "" } ], "network": { diff --git a/prog/net-plugin/launch.sh b/prog/net-plugin/launch.sh index 493a74c5..a39f7ced 100755 --- a/prog/net-plugin/launch.sh +++ b/prog/net-plugin/launch.sh @@ -16,18 +16,36 @@ echo "Starting launch.sh" # Check if the IP range overlaps anything existing on the host /usr/bin/weaveutil netcheck $IPALLOC_RANGE weave -STATUS=0 -/usr/bin/weaveutil is-swarm-manager 2>/dev/null || STATUS=$? -if [ $STATUS -eq 0 ]; then - IS_SWARM_MANAGER=1 -elif [ $STATUS -eq 20 ]; then - echo "Host swarm is not \"active\"; exiting." >&2 - exit 1 -fi +# We need to get a list of Swarm nodes which might run the net-plugin: +# - In the case of missing restart.sentinel, we assume that net-plugin has started +# for the first time via the docker-plugin cmd. So it's safe to request docker.sock. +# - If restart.sentinel present, let weaver restore from it as docker.sock is not +# available to any plugin in general (https://github.com/moby/moby/issues/32815). -SWARM_MANAGER_PEERS=$(/usr/bin/weaveutil swarm-manager-peers) -# Prevent from restoring from a persisted peers list -rm -f "/restart.sentinel" +PEERS= +IPALLOC_INIT= +if [ ! -f "/restart.sentinel" ]; then + STATUS=0 + /usr/bin/weaveutil is-swarm-manager 2>/dev/null || STATUS=$? + if [ $STATUS -eq 0 ]; then + IS_SWARM_MANAGER=1 + elif [ $STATUS -eq 20 ]; then + echo "Host swarm is not \"active\"; exiting." >&2 + exit 1 + fi + + SWARM_MANAGER_PEERS=$(/usr/bin/weaveutil swarm-manager-peers) + + if [ -z "$IPALLOC_INIT" ]; then + IPALLOC_INIT="observer" + if [ "$IS_SWARM_MANAGER" == "1" ]; then + IPALLOC_INIT="consensus=$(echo $SWARM_MANAGER_PEERS | wc -l)" + fi + fi + + PEERS=$(echo $SWARM_MANAGER_PEERS | tr '\n' ' ') + IPALLOC_INIT="--ipalloc-init $IPALLOC_INIT" +fi router_bridge_opts() { echo --datapath=datapath @@ -35,12 +53,9 @@ router_bridge_opts() { [ -z "$WEAVE_NO_FASTDP" ] || echo --no-fastdp } -if [ -z "$IPALLOC_INIT" ]; then - IPALLOC_INIT="observer" - if [ "$IS_SWARM_MANAGER" == "1" ]; then - IPALLOC_INIT="consensus=$(echo $SWARM_MANAGER_PEERS | wc -l)" - fi -fi +multicast_opt() { + [ -z "$WEAVE_MULTICAST" ] || echo "--plugin-v2-multicast" +} exec /home/weave/weaver $EXTRA_ARGS --port=6783 $(router_bridge_opts) \ --host-root=/host \ @@ -48,10 +63,12 @@ exec /home/weave/weaver $EXTRA_ARGS --port=6783 $(router_bridge_opts) \ --http-addr=$HTTP_ADDR --status-addr=$STATUS_ADDR \ --no-dns \ --ipalloc-range=$IPALLOC_RANGE \ - --ipalloc-init $IPALLOC_INIT \ + $IPALLOC_INIT \ --nickname "$(hostname)" \ --log-level=debug \ --db-prefix="$WEAVE_DIR/weave" \ --plugin-v2 \ + $(multicast_opt) \ --plugin-mesh-socket='' \ - $(echo $SWARM_MANAGER_PEERS | tr '\n' ' ') + --docker-api='' \ + $PEERS diff --git a/prog/weaver/main.go b/prog/weaver/main.go index fd364aa4..9a4898fe 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -123,38 +123,39 @@ func main() { runtime.GOMAXPROCS(procs) var ( - justVersion bool - config mesh.Config - bridgeConfig weavenet.BridgeConfig - networkConfig weave.NetworkConfig - protocolMinVersion int - resume bool - routerName string - nickName string - password string - pktdebug bool - logLevel string - prof string - bufSzMB int - noDiscovery bool - httpAddr string - statusAddr string - ipamConfig ipamConfig - dockerAPI string - peers []string - noDNS bool - dnsConfig dnsConfig - trustedSubnetStr string - dbPrefix string - hostRoot string - procPath string - discoveryEndpoint string - token string - advertiseAddress string - pluginSocket string - pluginMeshSocket string - enablePlugin bool - enablePluginV2 bool + justVersion bool + config mesh.Config + bridgeConfig weavenet.BridgeConfig + networkConfig weave.NetworkConfig + protocolMinVersion int + resume bool + routerName string + nickName string + password string + pktdebug bool + logLevel string + prof string + bufSzMB int + noDiscovery bool + httpAddr string + statusAddr string + ipamConfig ipamConfig + dockerAPI string + peers []string + noDNS bool + dnsConfig dnsConfig + trustedSubnetStr string + dbPrefix string + hostRoot string + procPath string + discoveryEndpoint string + token string + advertiseAddress string + pluginSocket string + pluginMeshSocket string + enablePlugin bool + enablePluginV2 bool + enablePluginV2Multicast bool defaultDockerHost = "unix:///var/run/docker.sock" ) @@ -208,6 +209,7 @@ func main() { mflag.BoolVar(&enablePlugin, []string{"-plugin"}, false, "enable Docker plugin (v1)") mflag.BoolVar(&enablePluginV2, []string{"-plugin-v2"}, false, "enable Docker plugin (v2)") + mflag.BoolVar(&enablePluginV2Multicast, []string{"-plugin-v2-multicast"}, false, "enable multicast for Docker plugin (v2)") mflag.StringVar(&pluginSocket, []string{"-plugin-socket"}, "/run/docker/plugins/weave.sock", "plugin socket on which to listen") mflag.StringVar(&pluginMeshSocket, []string{"-plugin-mesh-socket"}, "/run/docker/plugins/weavemesh.sock", "plugin socket on which to listen in mesh mode") @@ -427,7 +429,7 @@ func main() { } if enablePlugin || enablePluginV2 { - go plugin.Start(httpAddr, dockerCli, pluginSocket, pluginMeshSocket, !noDNS, enablePluginV2) + go plugin.Start(httpAddr, dockerCli, pluginSocket, pluginMeshSocket, !noDNS, enablePluginV2, enablePluginV2Multicast) } if enablePlugin { Log.Println("Creating default 'weave' network")