diff --git a/plugin/README.md b/plugin/README.md index 4af2efc7..1e246c1c 100644 --- a/plugin/README.md +++ b/plugin/README.md @@ -3,63 +3,7 @@ A [plugin](http://docs.docker.com/engine/extend/plugin_api/) to integrate [weave Net](http://weave.works/net/) with Docker. -## Setup: +The plugin runs automatically when you `weave launch`, provided your +Docker daemon is version 1.9 or newer. -Before you can use this plugin, you must [install weave](https://github.com/weaveworks/weave#installation) and [configure docker with a "cluster store"](https://docs.docker.com/engine/userguide/networking/get-started-overlay/#step-1-set-up-a-key-value-store). - - $ weave launch - $ weave launch-plugin - $ docker network create --driver=weave weave - -When you start subsequent weave peers you have to tell them about other peers, e.g. - - $ weave launch - $ weave launch-plugin - -## Starting a container: - - $ docker run --net=weave -ti ubuntu - -## Plugin command-line arguments - - * `--socket=`, which is the socket on which to listen for the - [plugin protocol](http://docs.docker.com/engine/extend/plugin_api/). - This defaults to what Docker expects, that is - `"/run/docker/plugins/weave.sock"`, but you may wish to - change it, for instance if you're running more than one instance of - the driver for some reason. - - * `--log-level=debug|info|warning|error`, which tells the plugin - how much information to emit for debugging. - -## Points to note - -Network plugins require Docker version 1.9 or later. - -### Docker needs a "cluster store" - -Weave operates as a "globally scoped" libnetwork driver, which means -libnetwork will assume all networks and endpoints are commonly known -to all hosts. It does this by using a shared database which you must -provide. - -As a consequence, you need to supply Docker with the address of a "cluster -store" when you start it; for example, an etcd installation. - -There's no specific documentation for using a cluster store, but the -first part of [this guide](https://github.com/docker/docker/blob/master/docs/userguide/networking/get-started-overlay.md) may help. - -### WeaveDNS not supported - -Weave's built-in service discovery mechanism is not currently -supported by the plugin. However, Docker provides a rudimentary -discovery mechanism by writing all user-provided container names and -hostnames into every container's `/etc/hosts`. - -### Restarting - -We start the plugin with a restart policy of 'always', because -Docker attempts to recreate networks on startup which means the plugin -container has to be started. If it isn't started automatically by Docker -with a restart policy, it's not possible to start it manually because -Docker is blocked. +More detail [here](http://docs.weave.works/weave/latest_release/plugin.html). diff --git a/plugin/ipam/driver.go b/plugin/ipam/driver.go index 1fbdf91b..766aa9a1 100644 --- a/plugin/ipam/driver.go +++ b/plugin/ipam/driver.go @@ -4,6 +4,7 @@ import ( "net" "github.com/docker/libnetwork/ipamapi" + "github.com/docker/libnetwork/netlabel" "github.com/weaveworks/weave/api" . "github.com/weaveworks/weave/common" "github.com/weaveworks/weave/common/docker" @@ -32,7 +33,10 @@ func (i *ipam) RequestPool(addressSpace, pool, subPool string, options map[strin Log.Debugln("RequestPool", addressSpace, pool, subPool, options) cidr, err := i.weave.DefaultSubnet() Log.Debugln("RequestPool returning ", cidr, err) - return "weavepool", cidr, nil, err + // Pass back a fake "gateway address"; we don't actually use it, + // so just give the network address. + data := map[string]string{netlabel.Gateway: cidr.String()} + return "weavepool", cidr, data, err } func (i *ipam) ReleasePool(poolID string) error { diff --git a/plugin/net/driver.go b/plugin/net/driver.go index 5eb5c984..34c5c306 100644 --- a/plugin/net/driver.go +++ b/plugin/net/driver.go @@ -23,14 +23,16 @@ const ( type driver struct { version string nameserver string + scope string sync.RWMutex endpoints map[string]struct{} } -func New(client *docker.Client, version string, nameserver string) (skel.Driver, error) { +func New(client *docker.Client, version string, nameserver string, scope string) (skel.Driver, error) { driver := &driver{ nameserver: nameserver, version: version, + scope: scope, endpoints: make(map[string]struct{}), } @@ -48,11 +50,10 @@ func errorf(format string, a ...interface{}) error { // === protocol handlers -var caps = &api.GetCapabilityResponse{ - Scope: "global", -} - func (driver *driver) GetCapabilities() (*api.GetCapabilityResponse, error) { + var caps = &api.GetCapabilityResponse{ + Scope: driver.scope, + } Log.Debugf("Get capabilities: responded with %+v", caps) return caps, nil } diff --git a/prog/plugin/main.go b/prog/plugin/main.go index 4ae75164..4da4c71e 100644 --- a/prog/plugin/main.go +++ b/prog/plugin/main.go @@ -6,8 +6,11 @@ import ( "net" "os" "os/signal" + "strings" "syscall" + "github.com/docker/libnetwork/ipamapi" + go_docker "github.com/fsouza/go-dockerclient" . "github.com/weaveworks/weave/common" "github.com/weaveworks/weave/common/docker" ipamplugin "github.com/weaveworks/weave/plugin/ipam" @@ -19,16 +22,20 @@ var version = "(unreleased version)" func main() { var ( - justVersion bool - address string - nameserver string - logLevel string + justVersion bool + address string + nameserver string + meshAddress string + logLevel string + meshNetworkName string ) flag.BoolVar(&justVersion, "version", false, "print version and exit") flag.StringVar(&logLevel, "log-level", "info", "logging level (debug, info, warning, error)") flag.StringVar(&address, "socket", "/run/docker/plugins/weave.sock", "socket on which to listen") flag.StringVar(&nameserver, "nameserver", "", "nameserver to provide to containers") + flag.StringVar(&meshAddress, "meshsocket", "/run/docker/plugins/weavemesh.sock", "socket on which to listen in mesh mode") + flag.StringVar(&meshNetworkName, "mesh-network-name", "weave", "network name to create in mesh mode") flag.Parse() @@ -39,51 +46,103 @@ func main() { SetLogLevel(logLevel) - Log.Println("Weave plugin", version, "Command line options:", os.Args) + Log.Println("Weave plugin", version, "Command line options:", os.Args[1:]) - dockerClient, err := docker.NewClient("unix:///var/run/docker.sock") + // API 1.21 is the first version that supports docker network commands + dockerClient, err := docker.NewVersionedClient("unix:///var/run/docker.sock", "1.21") if err != nil { Log.Fatalf("unable to connect to docker: %s", err) } - d, err := netplugin.New(dockerClient, version, nameserver) - if err != nil { - Log.Fatalf("unable to create driver: %s", err) + var globalListener, meshListener net.Listener + endChan := make(chan error, 1) + if address != "" { + globalListener, err := listenAndServe(dockerClient, address, nameserver, endChan, "global", false) + if err != nil { + Log.Fatalf("unable to create driver: %s", err) + } + defer globalListener.Close() + } + if meshAddress != "" { + meshListener, err := listenAndServe(dockerClient, meshAddress, nameserver, endChan, "local", true) + if err != nil { + Log.Fatalf("unable to create driver: %s", err) + } + defer meshListener.Close() } - i, err := ipamplugin.NewIpam(dockerClient, version) - if err != nil { - Log.Fatalf("unable to create driver: %s", err) + if meshNetworkName != "" { + createNetwork(dockerClient, meshNetworkName, meshAddress) } - var listener net.Listener - - // remove socket from last invocation - if err := os.Remove(address); err != nil && !os.IsNotExist(err) { - Log.Fatal(err) - } - listener, err = net.Listen("unix", address) - if err != nil { - Log.Fatal(err) - } - defer listener.Close() - sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, os.Interrupt, os.Kill, syscall.SIGTERM) - endChan := make(chan error, 1) - go func() { - endChan <- skel.Listen(listener, d, i) - }() - select { case sig := <-sigChan: Log.Debugf("Caught signal %s; shutting down", sig) + if err := dockerClient.Client.RemoveNetwork(meshNetworkName); err != nil { + Log.Fatal(err) + } case err := <-endChan: if err != nil { Log.Errorf("Error from listener: %s", err) - listener.Close() + globalListener.Close() + meshListener.Close() os.Exit(1) } } } + +func listenAndServe(dockerClient *docker.Client, address, nameserver string, endChan chan<- error, scope string, withIpam bool) (net.Listener, error) { + d, err := netplugin.New(dockerClient, version, nameserver, scope) + if err != nil { + return nil, err + } + + var i ipamapi.Ipam + if withIpam { + if i, err = ipamplugin.NewIpam(dockerClient, version); err != nil { + return nil, err + } + } + + var listener net.Listener + + // remove sockets from last invocation + if err := os.Remove(address); err != nil && !os.IsNotExist(err) { + return nil, err + } + listener, err = net.Listen("unix", address) + if err != nil { + return nil, err + } + Log.Printf("Listening on %s for %s scope", address, scope) + + go func() { + endChan <- skel.Listen(listener, d, i) + }() + + return listener, nil +} + +func createNetwork(dockerClient *docker.Client, networkName, address string) { + if _, err := dockerClient.Client.NetworkInfo(networkName); err == nil { + Log.Printf("Docker network '%s' already exists", networkName) + } else if _, ok := err.(*go_docker.NoSuchNetwork); ok { + driverName := strings.TrimSuffix(address, ".sock") + if i := strings.LastIndex(driverName, "/"); i >= 0 { + driverName = driverName[i+1:] + } + options := go_docker.CreateNetworkOptions{ + Name: networkName, + CheckDuplicate: true, + Driver: driverName, + IPAM: go_docker.IPAMOptions{Driver: driverName}, + } + _, err := dockerClient.Client.CreateNetwork(options) + if err != nil { + Log.Fatalf("Error creating network: %s", err) + } + } +} diff --git a/site/features.md b/site/features.md index e7e949a1..f69393b7 100644 --- a/site/features.md +++ b/site/features.md @@ -93,21 +93,12 @@ re-attached to the weave network by the weave Docker API proxy. ### Docker network plugin -Alternatively, you can use weave as a Docker plugin. First you must -configure the Docker daemon to use a cluster store as documented -[here](https://docs.docker.com/engine/userguide/networking/dockernetworks/#an-overlay-network); -you can then launch weave and create a network: - - $ weave launch - $ weave launch-plugin - $ docker network create --driver=weave weave - -and then start a container: +Alternatively, you can use weave as a Docker plugin. A Docker network +named `weave` is created by `weave launch`, which you can use like this: $ docker run --net=weave -ti ubuntu -For more details see the -[plugin README](https://github.com/weaveworks/docker-plugin). +For more details see the [plugin documentation](plugin.html). ### Address allocation diff --git a/site/plugin.md b/site/plugin.md new file mode 100644 index 00000000..eb4489b1 --- /dev/null +++ b/site/plugin.md @@ -0,0 +1,84 @@ +--- +title: Using Weave Net via Docker Networking +layout: default +--- + +New in Docker version 1.9 is a plugin mechanism to add different +network providers. + +To install, download the `weave` script as per the [readme][]. + +The Weave Net plugin runs automatically when you `weave launch`. You +must still tell the weave peers to connect to one another, either via +`weave launch` or `weave connect`. + +## Starting a container: + + $ docker run --net=weave -ti ubuntu + +## Using WeaveDNS for Service Discovery: + +You need to pass the additional arguments `--dns` and `-dns-search`, +for which we provide a helper in the weave script: + + $ docker run --net=weave -h foo.weave.local $(weave dns-args) -tdi ubuntu + $ docker run --net=weave -h bar.weave.local $(weave dns-args) -ti ubuntu + # ping foo + +For more details see the [Weave Service Discovery documentation][service-discovery]. + +## Under the hood + +The Weave Net plugin actually provides *two* network drivers to Docker +- one named `weavemesh` that can operate without a [cluster +store](#cluster-store) (like Docker's overlay driver) and one named +`weave` that can only work with one. + +### `weavemesh` driver + +* Weave handles all co-ordination between hosts (referred to by Docker as a "local scope" driver) +* Supports only a single network (we create one named `weave` for you automatically) +* Uses Weave's partition tolerant IPAM + +### `weave` driver + +* This runs in what Docker call "global scope"; requires a cluster store +* Supports multiple networks (these must be created with `docker network create --driver weave ...`) +* Used with Docker's cluster store based IPAM + +There's no specific documentation from Docker on using a cluster +store, but the first part of [Getting Started with Docker Multi-host Networking][docker-net] +should point the way. + +Note that in the case of multiple docker networks, all containers are +on the same virtual network but Docker allocates their addresses on +different subnets so they cannot talk to each other directly. + +## Plugin command-line arguments + +If you want to give some arguments to the plugin independently, don't +use `weave launch`; instead run: + + $ weave launch-router [other peers] + $ weave launch-plugin [plugin arguments] + +The plugin command-line arguments are: + + * `--log-level=debug|info|warning|error`, which tells the plugin + how much information to emit for debugging. + * `--mesh-network-name=`: set it to blank to disable creation + of a default network, or a name of your own choice. + +## Restarting + +We start the plugin with a restart policy, because Docker attempts to +recreate networks on startup. If it isn't started automatically by +Docker with a restart policy, it's not possible to start it manually +because Docker is blocked. + +Specifically, we use the policy `on-failure:3`, so Docker will give up +after three attempts if something just isn't working. + +[readme]: https://github.com/weaveworks/weave/blob/master/README.md#installation +[service-discovery]: weavedns.html +[docker-net]: https://github.com/docker/docker/blob/master/docs/userguide/networking/get-started-overlay.md diff --git a/site/troubleshooting.md b/site/troubleshooting.md index 30d5919e..e6272849 100644 --- a/site/troubleshooting.md +++ b/site/troubleshooting.md @@ -93,6 +93,9 @@ $ weave status Service: proxy Address: tcp://127.0.0.1:12375 + Service: plugin + DriverName: weave + ```` The terms used here are explained further at diff --git a/test/800_plugin_cross_hosts_2_test.sh b/test/800_plugin_cross_hosts_2_test.sh new file mode 100755 index 00000000..47f82d50 --- /dev/null +++ b/test/800_plugin_cross_hosts_2_test.sh @@ -0,0 +1,21 @@ +#! /bin/bash + +. ./config.sh + +C1_NAME=c1.weave.local +C2_NAME=seetwo.weave.local + +start_suite "Ping and DNS over cross-host weave network (via 'local' plugin)" + +DNS_IP=$(weave_on $host docker-bridge-ip) + +weave_on $HOST1 launch +weave_on $HOST2 launch $HOST1 + +start_container_local_plugin $HOST1 --name=c1 --hostname=$C1_NAME --dns=$DNS_IP +start_container_local_plugin $HOST2 --name=c2 --hostname=$C2_NAME --dns=$DNS_IP + +assert_raises "exec_on $HOST1 c1 $PING $C2_NAME" +assert_raises "exec_on $HOST2 c2 $PING $C1_NAME" + +end_suite diff --git a/test/config.sh b/test/config.sh index f50e03b5..6cecffab 100644 --- a/test/config.sh +++ b/test/config.sh @@ -154,6 +154,13 @@ start_container_with_dns() { weave_on $host run --with-dns "$@" -t $DNS_IMAGE /bin/sh } +start_container_local_plugin() { + host=$1 + shift 1 + # using ssh rather than docker -H because CircleCI docker client is older + $SSH $host docker run "$@" -dt --net=weave $SMALL_IMAGE /bin/sh +} + proxy_start_container() { host=$1 shift 1 @@ -217,8 +224,12 @@ assert_dns_ptr_record() { start_suite() { for host in $HOSTS; do [ -z "$DEBUG" ] || echo "Cleaning up on $host: removing all containers and resetting weave" + PLUGIN_ID=$(docker_on $host ps -aq --filter=name=weaveplugin) + PLUGIN_FILTER="cat" + [ -n "$PLUGIN_ID" ] && PLUGIN_FILTER="grep -v $PLUGIN_ID" + rm_containers $host $(docker_on $host ps -aq 2>/dev/null | $PLUGIN_FILTER) + run_on $host "docker network ls | grep -q ' weave ' && docker network rm weave" || true weave_on $host reset 2>/dev/null - rm_containers $host $(docker_on $host ps -aq 2>/dev/null) done whitely echo "$@" } diff --git a/weave b/weave index ac3e894c..aa76b409 100755 --- a/weave +++ b/weave @@ -57,6 +57,7 @@ weave launch-proxy [-H ] [--with-dns | --without-dns] weave launch-plugin weave env [--restore] weave config +weave dns-args weave connect [--replace] [ ...] weave forget ... weave status [targets | connections | peers | dns] @@ -111,6 +112,7 @@ exec_remote() { -e WEAVE_MTU \ -e WEAVE_NO_FASTDP \ -e WEAVE_NO_BRIDGED_FASTDP \ + -e WEAVE_NO_PLUGIN \ -e DOCKER_BRIDGE \ -e DOCKER_CLIENT_HOST="$DOCKER_CLIENT_HOST" \ -e DOCKER_CLIENT_ARGS \ @@ -120,30 +122,54 @@ exec_remote() { $WEAVEEXEC_DOCKER_ARGS $EXEC_IMAGE --local "$@" } +# Given $1 and $2 as semantic version numbers like 3.1.2, return [ $1 < $2 ] +version_lt() { + VERSION_MAJOR=${1%.*.*} + REST=${1%.*} VERSION_MINOR=${REST#*.} + VERSION_PATCH=${1#*.*.} + + MIN_VERSION_MAJOR=${2%.*.*} + REST=${2%.*} MIN_VERSION_MINOR=${REST#*.} + MIN_VERSION_PATCH=${2#*.*.} + + if [ \( "$VERSION_MAJOR" -lt "$MIN_VERSION_MAJOR" \) -o \ + \( "$VERSION_MAJOR" -eq "$MIN_VERSION_MAJOR" -a \ + \( "$VERSION_MINOR" -lt "$MIN_VERSION_MINOR" -o \ + \( "$VERSION_MINOR" -eq "$MIN_VERSION_MINOR" -a \ + \( "$VERSION_PATCH" -lt "$MIN_VERSION_PATCH" \) \) \) \) ] ; then + return 0 + fi + return 1 +} + check_docker_version() { if ! DOCKER_VERSION=$(docker -v | sed -n -e 's|^Docker version \([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\).*|\1|p') || [ -z "$DOCKER_VERSION" ] ; then echo "ERROR: Unable to parse docker version" >&2 exit 1 fi - DOCKER_VERSION_MAJOR=$(echo "$DOCKER_VERSION" | cut -d. -f 1) - DOCKER_VERSION_MINOR=$(echo "$DOCKER_VERSION" | cut -d. -f 2) - DOCKER_VERSION_PATCH=$(echo "$DOCKER_VERSION" | cut -d. -f 3) - - MIN_DOCKER_VERSION_MAJOR=$(echo "$MIN_DOCKER_VERSION" | cut -d. -f 1) - MIN_DOCKER_VERSION_MINOR=$(echo "$MIN_DOCKER_VERSION" | cut -d. -f 2) - MIN_DOCKER_VERSION_PATCH=$(echo "$MIN_DOCKER_VERSION" | cut -d. -f 3) - - if [ \( "$DOCKER_VERSION_MAJOR" -lt "$MIN_DOCKER_VERSION_MAJOR" \) -o \ - \( "$DOCKER_VERSION_MAJOR" -eq "$MIN_DOCKER_VERSION_MAJOR" -a \ - \( "$DOCKER_VERSION_MINOR" -lt "$MIN_DOCKER_VERSION_MINOR" -o \ - \( "$DOCKER_VERSION_MINOR" -eq "$MIN_DOCKER_VERSION_MINOR" -a \ - \( "$DOCKER_VERSION_PATCH" -lt "$MIN_DOCKER_VERSION_PATCH" \) \) \) \) ] ; then + if version_lt $DOCKER_VERSION $MIN_DOCKER_VERSION ; then echo "ERROR: weave requires Docker version $MIN_DOCKER_VERSION or later; you are running $DOCKER_VERSION" >&2 exit 1 fi } +check_docker_server_api_version() { + # Cope with various versions of `docker version` output format + if ! DOCKER_API_VERSION=$(docker version -f '{{.Server.APIVersion}}' 2> /dev/null ) ; then + if ! DOCKER_API_VERSION=$(docker version -f '{{.Server.ApiVersion}}' 2> /dev/null ) ; then + if ! DOCKER_API_VERSION=$(docker version | sed -n -e 's|^Server API version: *\([0-9][0-9]*\.[0-9][0-9]\).*|\1|p') ; then + echo "ERROR: Unable to determine docker version" >&2 + exit 1 + fi + fi + fi + + if version_lt ${DOCKER_API_VERSION}.0 ${1}.0 ; then + return 1 + fi +} + enforce_docker_bridge_addr_assign_type() { if ! ADDR_ASSIGN_TYPE=$(cat /sys/class/net/$DOCKER_BRIDGE/addr_assign_type 2>/dev/null) ; then echo "Could not determine address assignment type of $DOCKER_BRIDGE" >&2 @@ -1804,6 +1830,10 @@ EOF COMMON_ARGS=$(common_launch_args "$@") launch_router "$@" launch_proxy $COMMON_ARGS + if [ -z $WEAVE_NO_PLUGIN ] && check_docker_server_api_version 1.21 ; then + check_not_running $PLUGIN_CONTAINER_NAME $BASE_PLUGIN_IMAGE + launch_plugin $COMMON_ARGS + fi ;; launch-router) deprecation_warnings "$@" @@ -2080,6 +2110,7 @@ EOF ;; stop) [ $# -eq 0 ] || usage + stop_plugin stop_router stop_proxy ;; @@ -2099,7 +2130,7 @@ EOF [ $# -eq 0 ] || usage warn_if_stopping_proxy_in_env call_weave DELETE /peer >/dev/null 2>&1 || true - for NAME in $CONTAINER_NAME $PROXY_CONTAINER_NAME ; do + for NAME in $PLUGIN_CONTAINER_NAME $CONTAINER_NAME $PROXY_CONTAINER_NAME ; do docker stop $NAME >/dev/null 2>&1 || true docker rm -f $NAME >/dev/null 2>&1 || true done