diff --git a/.gitignore b/.gitignore index 3a7f66ad..6aece654 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,4 @@ test/coverage.* *qemu-*-static terraform.tfstate terraform.tfstate.backup +prog/net-plugin/rootfs/ diff --git a/.gitmodules b/.gitmodules index cb055f87..21de8218 100644 --- a/.gitmodules +++ b/.gitmodules @@ -124,9 +124,12 @@ [submodule "vendor/github.com/containernetworking/cni"] path = vendor/github.com/containernetworking/cni url = https://github.com/containernetworking/cni -[submodule "vendor/golang.org/x/net"] - path = vendor/golang.org/x/net - url = https://go.googlesource.com/net [submodule "vendor/google.golang.org/grpc"] path = vendor/google.golang.org/grpc url = https://github.com/grpc/grpc-go +[submodule "vendor/github.com/docker/go-units"] + path = vendor/github.com/docker/go-units + url = https://github.com/docker/go-units +[submodule "vendor/golang.org/x/net"] + path = vendor/golang.org/x/net + url = https://github.com/golang/net diff --git a/DEPENDENCIES b/DEPENDENCIES index c93ed026..32a099ba 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -1,3 +1,3 @@ -DOCKER_VERSION=1.11.2 +DOCKER_VERSION=1.13.1 KUBERNETES_VERSION=1.6.1 -KUBERNETES_CNI_VERSION=0.5.1 \ No newline at end of file +KUBERNETES_CNI_VERSION=0.5.1 diff --git a/Makefile b/Makefile index cf3bc69e..7b1bac40 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .DEFAULT: all -.PHONY: all exes testrunner update tests lint publish-one-arch $(PUBLISH) clean clean-bin prerequisites build run-smoketests +.PHONY: all exes testrunner update tests lint publish-one-arch $(PUBLISH) clean clean-bin clean-work-dir prerequisites build run-smoketests # If you can use docker without being root, you can do "make SUDO=" SUDO=$(shell docker info >/dev/null 2>&1 || echo "sudo -E") @@ -93,36 +93,38 @@ WEAVEWAIT_EXE=prog/weavewait/weavewait WEAVEWAIT_NOOP_EXE=prog/weavewait/weavewait_noop WEAVEWAIT_NOMCAST_EXE=prog/weavewait/weavewait_nomcast WEAVEUTIL_EXE=prog/weaveutil/weaveutil -PLUGIN_EXE=prog/plugin/plugin RUNNER_EXE=tools/runner/runner MANIFEST_TOOL_DIR=vendor/github.com/estesp/manifest-tool MANIFEST_TOOL_EXE=$(MANIFEST_TOOL_DIR)/manifest-tool TEST_TLS_EXE=test/tls/tls # All binaries together in a list -EXES=$(WEAVER_EXE) $(SIGPROXY_EXE) $(KUBEPEERS_EXE) $(WEAVENPC_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(WEAVEWAIT_NOMCAST_EXE) $(WEAVEUTIL_EXE) $(PLUGIN_EXE) $(RUNNER_EXE) $(TEST_TLS_EXE) $(MANIFEST_TOOL_EXE) +EXES=$(WEAVER_EXE) $(SIGPROXY_EXE) $(KUBEPEERS_EXE) $(WEAVENPC_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(WEAVEWAIT_NOMCAST_EXE) $(WEAVEUTIL_EXE) $(RUNNER_EXE) $(TEST_TLS_EXE) $(MANIFEST_TOOL_EXE) # These stamp files are used to mark the current state of the build; whether an image has been built or not BUILD_UPTODATE=.build.uptodate WEAVER_UPTODATE=.weaver$(ARCH_EXT).uptodate WEAVEEXEC_UPTODATE=.weaveexec$(ARCH_EXT).uptodate -PLUGIN_UPTODATE=.plugin$(ARCH_EXT).uptodate +PLUGIN_UPTODATE=.net-plugin$(ARCH_EXT).uptodate WEAVEKUBE_UPTODATE=.weavekube$(ARCH_EXT).uptodate WEAVENPC_UPTODATE=.weavenpc$(ARCH_EXT).uptodate WEAVEDB_UPTODATE=.weavedb.uptodate -IMAGES_UPTODATE=$(WEAVER_UPTODATE) $(WEAVEEXEC_UPTODATE) $(PLUGIN_UPTODATE) $(WEAVEKUBE_UPTODATE) $(WEAVENPC_UPTODATE) +IMAGES_UPTODATE=$(WEAVER_UPTODATE) $(WEAVEEXEC_UPTODATE) $(WEAVEKUBE_UPTODATE) $(WEAVENPC_UPTODATE) # The names of the images. Note that the images for other architectures than amd64 have a suffix in the image name. WEAVER_IMAGE=$(DOCKERHUB_USER)/weave$(ARCH_EXT) WEAVEEXEC_IMAGE=$(DOCKERHUB_USER)/weaveexec$(ARCH_EXT) -PLUGIN_IMAGE=$(DOCKERHUB_USER)/plugin$(ARCH_EXT) WEAVEKUBE_IMAGE=$(DOCKERHUB_USER)/weave-kube$(ARCH_EXT) WEAVENPC_IMAGE=$(DOCKERHUB_USER)/weave-npc$(ARCH_EXT) BUILD_IMAGE=weaveworks/weavebuild WEAVEDB_IMAGE=$(DOCKERHUB_USER)/weavedb +PLUGIN_IMAGE=$(DOCKERHUB_USER)/net-plugin -IMAGES=$(WEAVER_IMAGE) $(WEAVEEXEC_IMAGE) $(PLUGIN_IMAGE) $(WEAVEKUBE_IMAGE) $(WEAVENPC_IMAGE) $(WEAVEDB_IMAGE) +IMAGES=$(WEAVER_IMAGE) $(WEAVEEXEC_IMAGE) $(WEAVEKUBE_IMAGE) $(WEAVENPC_IMAGE) $(WEAVEDB_IMAGE) + +PLUGIN_WORK_DIR="prog/net-plugin/rootfs" +PLUGIN_BUILD_IMG="plugin-builder" PUBLISH=publish_weave publish_weaveexec publish_plugin publish_weave-kube publish_weave-npc PUSH_ML=push_ml_weave push_ml_weaveexec push_ml_plugin push_ml_weave-kube push_ml_weave-npc @@ -152,12 +154,12 @@ testrunner: $(RUNNER_EXE) $(TEST_TLS_EXE) $(WEAVER_EXE) $(WEAVEPROXY_EXE) $(WEAVEUTIL_EXE): common/*.go common/*/*.go net/*.go net/*/*.go $(WEAVER_EXE): router/*.go ipam/*.go ipam/*/*.go db/*.go nameserver/*.go prog/weaver/*.go +$(WEAVER_EXE): api/*.go plugin/*.go plugin/*/* $(WEAVEPROXY_EXE): proxy/*.go prog/weaveproxy/*.go $(WEAVEUTIL_EXE): prog/weaveutil/*.go net/*.go plugin/net/*.go plugin/ipam/*.go db/*.go $(SIGPROXY_EXE): prog/sigproxy/*.go $(KUBEPEERS_EXE): prog/kube-peers/*.go $(WEAVENPC_EXE): prog/weave-npc/*.go npc/*.go npc/*/*.go -$(PLUGIN_EXE): prog/plugin/*.go plugin/*/*.go api/*.go common/*.go common/docker/*.go net/*.go $(TEST_TLS_EXE): test/tls/*.go $(RUNNER_EXE): tools/runner/*.go $(MANIFEST_TOOL_EXE): $(MANIFEST_TOOL_DIR)/*.go @@ -187,7 +189,7 @@ else exes: $(EXES) -$(WEAVER_EXE) $(WEAVEPROXY_EXE) $(PLUGIN_EXE): +$(WEAVER_EXE) $(WEAVEPROXY_EXE): ifeq ($(COVERAGE),true) $(eval COVERAGE_MODULES := $(shell (go list ./$(@D); go list -f '{{join .Deps "\n"}}' ./$(@D) | grep "^$(PACKAGE_BASE)/") | grep -v "^$(PACKAGE_BASE)/vendor/" | paste -s -d,)) go test -c -o ./$@ $(BUILD_FLAGS) -v -covermode=atomic -coverpkg $(COVERAGE_MODULES) ./$(@D)/ @@ -264,8 +266,21 @@ $(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile.$(DOCKERHUB_USER) prog/weaveexe $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker build -f prog/weaveexec/Dockerfile.$(DOCKERHUB_USER) -t $(WEAVEEXEC_IMAGE) prog/weaveexec touch $@ -$(PLUGIN_UPTODATE): prog/plugin/Dockerfile.$(DOCKERHUB_USER) $(PLUGIN_EXE) $(WEAVER_UPTODATE) - $(SUDO) docker build -f prog/plugin/Dockerfile.$(DOCKERHUB_USER) -t $(PLUGIN_IMAGE) prog/plugin +# Builds Docker plugin. +$(PLUGIN_UPTODATE): prog/net-plugin/launch.sh prog/net-plugin/config.json $(WEAVER_UPTODATE) + -$(SUDO) docker rm -f $(PLUGIN_BUILD_IMG) 2>/dev/null + $(SUDO) docker create --name=$(PLUGIN_BUILD_IMG) $(WEAVER_IMAGE) true + rm -rf $(PLUGIN_WORK_DIR) + mkdir $(PLUGIN_WORK_DIR) + $(SUDO) docker export $(PLUGIN_BUILD_IMG) | tar -x -C $(PLUGIN_WORK_DIR) + $(SUDO) docker rm -f $(PLUGIN_BUILD_IMG) + cp prog/net-plugin/launch.sh $(PLUGIN_WORK_DIR)/home/weave/launch.sh + -$(SUDO) docker plugin disable $(PLUGIN_IMAGE):$(WEAVE_VERSION) 2>/dev/null + -$(SUDO) docker plugin rm $(PLUGIN_IMAGE):$(WEAVE_VERSION) 2>/dev/null + $(SUDO) docker plugin create $(PLUGIN_IMAGE):$(WEAVE_VERSION) prog/net-plugin + -$(SUDO) docker plugin disable $(PLUGIN_IMAGE):latest 2>/dev/null + -$(SUDO) docker plugin rm $(PLUGIN_IMAGE):latest 2>/dev/null + $(SUDO) docker plugin create $(PLUGIN_IMAGE):latest prog/net-plugin touch $@ $(WEAVEKUBE_UPTODATE): prog/weave-kube/Dockerfile.$(DOCKERHUB_USER) prog/weave-kube/launch.sh $(KUBEPEERS_EXE) $(WEAVER_UPTODATE) @@ -293,6 +308,13 @@ tools/.git $(MANIFEST_TOOL_DIR)/.git: # CODE FOR PUBLISHING THE IMAGES +# Push plugin +plugin_publish: $(PLUGIN_UPTODATE) + $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker plugin push $(PLUGIN_IMAGE):$(WEAVE_VERSION) +ifneq ($(UPDATE_LATEST),false) + $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker plugin push $(PLUGIN_IMAGE):latest +endif + # This target first runs "make publish" for each architecture # Then it pushes the manifest lists publish: $(addprefix sub-publish-,$(ALL_ARCHITECTURES)) $(WEAVEDB_UPTODATE) @@ -345,13 +367,16 @@ ifneq ($(UPDATE_LATEST),false) $(MANIFEST_TOOL_EXE) push from-args --platforms $(ML_PLATFORMS) --template $(DOCKERHUB_USER)/$*-ARCH:latest --target $(DOCKERHUB_USER)/$*:latest endif +clean-work-dir: + rm -rf $(PLUGIN_WORK_DIR) + clean-bin: -$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker rmi $(IMAGES) find prog -type f -name "Dockerfile.*" -not -name "Dockerfile.template" -print | xargs rm -f find prog -type f -name "*qemu-*" -print | xargs rm -f rm -rf $(EXES) $(IMAGES_UPTODATE) $(WEAVEDB_UPTODATE) weave*.tar.gz $(DOCKER_DISTRIB) prog/weaveexec/docker .pkg -clean: clean-bin +clean: clean-bin clean-work-dir -$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker rmi $(BUILD_IMAGE) rm -rf test/tls/*.pem test/coverage.* test/coverage $(BUILD_UPTODATE) $(MANIFEST_TOOL_EXE) diff --git a/api/weaveapi.go b/api/weaveapi.go index 210618a8..19a567a2 100644 --- a/api/weaveapi.go +++ b/api/weaveapi.go @@ -8,6 +8,7 @@ import ( "net/http" "net/url" "strings" + "time" ) const ( @@ -76,6 +77,20 @@ func (client *Client) Connect(remote string) error { return err } +// IsReady returns true if the API server is up and running +func (client *Client) IsReady() bool { + _, err := client.httpVerb("GET", "/status", nil) + + return err == nil +} + +// WaitAPIServer waits until the API server is ready to serve. +func (client *Client) WaitAPIServer(n int) { + for i := 0; i < n && !client.IsReady(); i++ { + time.Sleep(2 * time.Second) + } +} + type Logger interface { Infof(string, ...interface{}) Debugf(string, ...interface{}) diff --git a/bin/release b/bin/release index 6bfad988..89f1c68f 100755 --- a/bin/release +++ b/bin/release @@ -191,6 +191,10 @@ publish() { make UPDATE_LATEST=$IS_MAINLINE SUDO=$SUDO WEAVE_VERSION=$VERSION DOCKERHUB_USER=$DOCKERHUB_USER publish echo "** Docker images tagged and pushed" + echo "== Building and pushing plugin" + make UPDATE_LATEST=$IS_MAINLINE SUDO=$SUDO WEAVE_VERSION=$VERSION DOCKERHUB_USER=$DOCKERHUB_USER plugin_publish + echo "** Plugin built and pushed" + echo "== Publishing pre-release on GitHub" github-release publish \ @@ -223,6 +227,10 @@ publish() { make PUBLISH_WEAVEDB=$IS_MAINLINE UPDATE_LATEST=$IS_MAINLINE SUDO=$SUDO WEAVE_VERSION=$VERSION DOCKERHUB_USER=$DOCKERHUB_USER publish echo "** Docker images tagged and pushed" + echo "== Building and pushing plugin" + make UPDATE_LATEST=$IS_MAINLINE SUDO=$SUDO WEAVE_VERSION=$VERSION DOCKERHUB_USER=$DOCKERHUB_USER plugin_publish + echo "** Plugin built and pushed" + echo "== Publishing release on GitHub" github-release publish \ diff --git a/net/netns.go b/net/netns.go index 971baf63..3bd4a1c2 100644 --- a/net/netns.go +++ b/net/netns.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os/exec" + "path/filepath" "runtime" "github.com/vishvananda/netlink" @@ -80,5 +81,9 @@ func WithNetNSByPid(pid int, cmd string, args ...string) ([]byte, error) { } func NSPathByPid(pid int) string { - return fmt.Sprintf("/proc/%d/ns/net", pid) + return NSPathByPidWithRoot("/", pid) +} + +func NSPathByPidWithRoot(root string, pid int) string { + return filepath.Join(root, fmt.Sprintf("/proc/%d/ns/net", pid)) } diff --git a/plugin/ipam/driver.go b/plugin/ipam/driver.go index 417e468f..78a650e7 100644 --- a/plugin/ipam/driver.go +++ b/plugin/ipam/driver.go @@ -19,6 +19,11 @@ func NewIpam(weave *api.Client) *Ipam { return &Ipam{weave: weave} } +func (i *Ipam) GetCapabilities() (RequiresMACAddress bool, RequiresRequestReplay bool, err error) { + i.logReq("GetCapabilities") + return false, false, nil +} + func (i *Ipam) GetDefaultAddressSpaces() (string, string, error) { i.logReq("GetDefaultAddressSpaces") return "weavelocal", "weaveglobal", nil diff --git a/plugin/net/driver.go b/plugin/net/driver.go index 3a9740af..6ddadb07 100644 --- a/plugin/net/driver.go +++ b/plugin/net/driver.go @@ -28,19 +28,23 @@ type network struct { } type driver struct { - name string - scope string - docker *docker.Client sync.RWMutex - networks map[string]network + scope string + docker *docker.Client + dns bool + isPluginV2 bool + networks map[string]network + isNetworkOur func(driverName string) bool } -func New(client *docker.Client, weave *weaveapi.Client, name, scope string) (skel.Driver, error) { +func New(client *docker.Client, weave *weaveapi.Client, scope string, dns, isPluginV2 bool, isNetworkOur func(string) bool) (skel.Driver, error) { driver := &driver{ - name: name, - scope: scope, - docker: client, - networks: make(map[string]network), + scope: scope, + docker: client, + dns: dns, + isPluginV2: isPluginV2, + networks: make(map[string]network), + isNetworkOur: isNetworkOur, } _, err := NewWatcher(client, weave, driver) @@ -61,12 +65,27 @@ func (driver *driver) GetCapabilities() (*api.GetCapabilityResponse, error) { return caps, nil } +// In Swarm mode, CreateNetwork is called on each Swarm node when a new service +// is created. func (driver *driver) CreateNetwork(create *api.CreateNetworkRequest) error { driver.logReq("CreateNetwork", create, create.NetworkID) _, err := driver.setupNetworkInfo(create.NetworkID, true, stringOptions(create)) return err } +// NetworkAllocate is called on a Swarm node (master) which creates the network. +// The returned options are passed to CreateNetwork. +func (driver *driver) NetworkAllocate(alloc *api.AllocateNetworkRequest) (*api.AllocateNetworkResponse, error) { + driver.logReq("NetworkAllocate", alloc, alloc.NetworkID) + return &api.AllocateNetworkResponse{Options: alloc.Options}, nil +} + +// NetworkFree is called on a Swarm master node which created the network. +func (driver *driver) NetworkFree(free *api.FreeNetworkRequest) (*api.FreeNetworkResponse, error) { + driver.logReq("NetworkFree", free, free.NetworkID) + return nil, nil +} + // Deal with excessively-generic way the options get decoded from JSON func stringOptions(create *api.CreateNetworkRequest) map[string]string { if create.Options != nil { @@ -85,6 +104,7 @@ func stringOptions(create *api.CreateNetworkRequest) map[string]string { return nil } +// In Swarm mode, DeleteNetwork is called after a service has been removed. func (driver *driver) DeleteNetwork(delreq *api.DeleteNetworkRequest) error { driver.logReq("DeleteNetwork", delreq, delreq.NetworkID) driver.Lock() @@ -173,7 +193,7 @@ func (driver *driver) findNetworkInfo(id string) (network, error) { if err != nil { return network, err } - return driver.setupNetworkInfo(id, info.Driver == driver.name, info.Options) + return driver.setupNetworkInfo(id, driver.isNetworkOur(info.Driver), info.Options) } func (driver *driver) setupNetworkInfo(id string, isOurs bool, options map[string]string) (network, error) { diff --git a/plugin/net/watcher.go b/plugin/net/watcher.go index b9d04f5a..736751d7 100644 --- a/plugin/net/watcher.go +++ b/plugin/net/watcher.go @@ -41,11 +41,19 @@ func (w *watcher) ContainerStarted(id string) { continue } if network.isOurs { - fqdn := fmt.Sprintf("%s.%s", info.Config.Hostname, info.Config.Domainname) - if err := w.weave.RegisterWithDNS(id, fqdn, net.IPAddress); err != nil { - w.driver.warn("ContainerStarted", "unable to register %s with weaveDNS: %s", id, err) + if w.driver.dns { + fqdn := fmt.Sprintf("%s.%s", info.Config.Hostname, info.Config.Domainname) + if err := w.weave.RegisterWithDNS(id, fqdn, net.IPAddress); err != nil { + w.driver.warn("ContainerStarted", "unable to register %s with weaveDNS: %s", id, err) + } } - if _, err := weavenet.WithNetNSByPid(info.State.Pid, "configure-arp", weavenet.VethName); err != nil { + rootDir := "/" + if w.driver.isPluginV2 { + // We bind mount host's /proc to /host/proc for plugin-v2 + rootDir = "/host" + } + netNSPath := weavenet.NSPathByPidWithRoot(rootDir, info.State.Pid) + if _, err := weavenet.WithNetNS(netNSPath, "configure-arp", weavenet.VethName); err != nil { w.driver.warn("ContainerStarted", "unable to configure interfaces: %s", err) } } diff --git a/plugin/plugin.go b/plugin/plugin.go new file mode 100644 index 00000000..3343a902 --- /dev/null +++ b/plugin/plugin.go @@ -0,0 +1,109 @@ +package plugin + +import ( + "fmt" + "net" + "net/http" + "os" + "path" + "strings" + + "github.com/docker/libnetwork/ipamapi" + weaveapi "github.com/weaveworks/weave/api" + "github.com/weaveworks/weave/common" + "github.com/weaveworks/weave/common/docker" + weavenet "github.com/weaveworks/weave/net" + ipamplugin "github.com/weaveworks/weave/plugin/ipam" + netplugin "github.com/weaveworks/weave/plugin/net" + "github.com/weaveworks/weave/plugin/skel" +) + +const pluginV2Name = "net-plugin" + +var Log = common.Log + +func Start(weaveAPIAddr string, dockerClient *docker.Client, address string, meshAddress string, dns bool, isPluginV2 bool) { + weave := weaveapi.NewClient(weaveAPIAddr, Log) + + Log.Info("Waiting for Weave API Server...") + weave.WaitAPIServer(30) + Log.Info("Finished waiting for Weave API Server") + + if err := run(dockerClient, weave, address, meshAddress, dns, isPluginV2); err != nil { + Log.Fatal(err) + } +} + +func run(dockerClient *docker.Client, weave *weaveapi.Client, address, meshAddress string, dns, isPluginV2 bool) error { + endChan := make(chan error, 1) + + if address != "" { + globalListener, err := listenAndServe(dockerClient, weave, address, endChan, "global", false, dns, isPluginV2) + if err != nil { + return err + } + defer os.Remove(address) + defer globalListener.Close() + } + if meshAddress != "" { + meshListener, err := listenAndServe(dockerClient, weave, meshAddress, endChan, "local", true, dns, isPluginV2) + if err != nil { + return err + } + defer os.Remove(meshAddress) + defer meshListener.Close() + } + + statusListener, err := weavenet.ListenUnixSocket("/home/weave/plugin-status.sock") + if err != nil { + return err + } + go serveStatus(statusListener) + + return <-endChan +} + +func listenAndServe(dockerClient *docker.Client, weave *weaveapi.Client, address string, endChan chan<- error, scope string, withIpam, dns bool, isPluginV2 bool) (net.Listener, error) { + var isNetworkOur func(string) bool + if isPluginV2 { + isNetworkOur = func(driverName string) bool { + return strings.Contains(driverName, pluginV2Name) + } + } else { + name := strings.TrimSuffix(path.Base(address), ".sock") + isNetworkOur = func(driverName string) bool { + return driverName == name + } + } + + d, err := netplugin.New(dockerClient, weave, scope, dns, isPluginV2, isNetworkOur) + if err != nil { + return nil, err + } + + var i ipamapi.Ipam + if withIpam { + i = ipamplugin.NewIpam(weave) + } + + listener, err := weavenet.ListenUnixSocket(address) + if err != nil { + return nil, err + } + Log.Printf("Listening on %s for %s scope", address, scope) + + go func() { + endChan <- skel.Listen(listener, d, i) + }() + + return listener, nil +} + +func serveStatus(listener net.Listener) { + server := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "ok") + })} + if err := server.Serve(listener); err != nil { + Log.Fatalf("ListenAndServeStatus failed: %s", err) + } +} diff --git a/plugin/skel/listener.go b/plugin/skel/listener.go index afe0584e..1ad98ba8 100644 --- a/plugin/skel/listener.go +++ b/plugin/skel/listener.go @@ -26,11 +26,17 @@ type Driver interface { DeleteEndpoint(delete *api.DeleteEndpointRequest) error EndpointInfo(req *api.EndpointInfoRequest) (*api.EndpointInfoResponse, error) JoinEndpoint(j *api.JoinRequest) (response *api.JoinResponse, error error) + NetworkAllocate(*api.AllocateNetworkRequest) (*api.AllocateNetworkResponse, error) + NetworkFree(*api.FreeNetworkRequest) (*api.FreeNetworkResponse, error) LeaveEndpoint(leave *api.LeaveRequest) error DiscoverNew(discover *api.DiscoveryNotification) error DiscoverDelete(delete *api.DiscoveryNotification) error } +type IpamCapability interface { + GetCapabilities() (RequiresMACAddress bool, RequiresRequestReplay bool, err error) +} + type listener struct { d Driver i ipamapi.Ipam @@ -58,9 +64,12 @@ func Listen(socket net.Listener, driver Driver, ipamDriver ipamapi.Ipam) error { handleMethod(networkReceiver, "EndpointOperInfo", listener.infoEndpoint) handleMethod(networkReceiver, "Join", listener.joinEndpoint) handleMethod(networkReceiver, "Leave", listener.leaveEndpoint) + handleMethod(networkReceiver, "AllocateNetwork", listener.networkAllocate) + handleMethod(networkReceiver, "FreeNetwork", listener.networkFree) } if ipamDriver != nil { + handleMethod(ipamReceiver, "GetCapabilities", listener.getIpamCapabilities) handleMethod(ipamReceiver, "GetDefaultAddressSpaces", listener.getDefaultAddressSpaces) handleMethod(ipamReceiver, "RequestPool", listener.requestPool) handleMethod(ipamReceiver, "ReleasePool", listener.releasePool) @@ -163,6 +172,26 @@ func (listener *listener) joinEndpoint(w http.ResponseWriter, r *http.Request) { objectOrErrorResponse(w, res, err) } +func (listener *listener) networkAllocate(w http.ResponseWriter, r *http.Request) { + var alloc api.AllocateNetworkRequest + if err := json.NewDecoder(r.Body).Decode(&alloc); err != nil { + sendError(w, "Could not decode JSON encode payload", http.StatusBadRequest) + return + } + res, err := listener.d.NetworkAllocate(&alloc) + objectOrErrorResponse(w, res, err) +} + +func (listener *listener) networkFree(w http.ResponseWriter, r *http.Request) { + var free api.FreeNetworkRequest + if err := json.NewDecoder(r.Body).Decode(&free); err != nil { + sendError(w, "Could not decode JSON encode payload", http.StatusBadRequest) + return + } + res, err := listener.d.NetworkFree(&free) + objectOrErrorResponse(w, res, err) +} + func (listener *listener) leaveEndpoint(w http.ResponseWriter, r *http.Request) { var l api.LeaveRequest if err := json.NewDecoder(r.Body).Decode(&l); err != nil { @@ -192,6 +221,20 @@ func (listener *listener) discoverDelete(w http.ResponseWriter, r *http.Request) // === +func (listener *listener) getIpamCapabilities(w http.ResponseWriter, r *http.Request) { + c, ok := listener.i.(IpamCapability) + if !ok { + http.NotFound(w, r) + return + } + requiresMACAddress, requiresRequestReplay, err := c.GetCapabilities() + response := &ipamapi.Capability{ + RequiresMACAddress: requiresMACAddress, + RequiresRequestReplay: requiresRequestReplay, + } + objectOrErrorResponse(w, response, err) +} + func (listener *listener) getDefaultAddressSpaces(w http.ResponseWriter, r *http.Request) { local, global, err := listener.i.GetDefaultAddressSpaces() response := &iapi.GetAddressSpacesResponse{ diff --git a/prog/net-plugin/config.json b/prog/net-plugin/config.json new file mode 100644 index 00000000..74601b08 --- /dev/null +++ b/prog/net-plugin/config.json @@ -0,0 +1,75 @@ +{ + "description": "Weave Net plugin for Docker", + "documentation": "https://weave.works", + "entrypoint": [ + "/home/weave/launch.sh" + ], + "interface": { + "socket": "weave.sock", + "types": [ + "docker.networkdriver/1.0" + ] + }, + "linux": { + "capabilities": [ + "CAP_SYS_ADMIN", + "CAP_NET_ADMIN" + ] + }, + "mounts": [ + { + "destination": "/host/proc/", + "type": "bind", + "source": "/proc/", + "options": ["rbind", "rw"] + }, + { + "destination": "/var/run/docker.sock", + "source": "/var/run/docker.sock", + "type": "bind", + "options": ["rbind"] + }, + { + "destination": "/host/var/lib/", + "source": "/var/lib/", + "type": "bind", + "options": ["rbind"] + }, + { + "destination": "/host/etc/", + "source": "/etc/", + "type": "bind", + "options": ["rbind"] + } + ], + "env": [ + { + "description": "Extra args to `weaver` and `plugin`", + "name": "EXTRA_ARGS", + "settable": [ + "value" + ], + "value": "" + }, + { + "description": "Encryption password", + "name": "WEAVE_PASSWORD", + "settable": [ + "value" + ], + "value": "" + }, + { + "description": "MTU", + "name": "WEAVE_MTU", + "settable": [ + "value" + ], + "value": "" + } + ], + "network": { + "type": "host" + }, + "workdir": "" +} diff --git a/prog/net-plugin/launch.sh b/prog/net-plugin/launch.sh new file mode 100755 index 00000000..a372be13 --- /dev/null +++ b/prog/net-plugin/launch.sh @@ -0,0 +1,66 @@ +#!/bin/sh + +set -e + +# Default if not supplied - same as weave net default +IPALLOC_RANGE=${IPALLOC_RANGE:-10.32.0.0/12} +HTTP_ADDR=${WEAVE_HTTP_ADDR:-127.0.0.1:6784} +STATUS_ADDR=${WEAVE_STATUS_ADDR:-0.0.0.0:6782} +HOST_ROOT=${HOST_ROOT:-/host} +WEAVE_DIR="/host/var/lib/weave" + +mkdir $WEAVE_DIR || true + +echo "Starting launch.sh" + +# Check if the IP range overlaps anything existing on the host +/usr/bin/weaveutil netcheck $IPALLOC_RANGE weave + +STATUS=0 +/usr/bin/weaveutil is-swarm-manager 2>/dev/null || STATUS=$? +if [ $STATUS -eq 0 ]; then + IS_SWARM_MANAGER=1 +elif [ $STATUS -eq 20 ]; then + echo "Host swarm is not \"active\"; exiting." >&2 + exit 1 +fi + +SWARM_MANAGER_PEERS=$(/usr/bin/weaveutil swarm-manager-peers) +# Prevent from restoring from a persisted peers list +rm -f "/restart.sentinel" + +/home/weave/weave --local create-bridge \ + --proc-path=/host/proc \ + --weavedb-dir-path=$WEAVE_DIR \ + --force + +BRIDGE_OPTIONS="--datapath=datapath" +if [ "$(/home/weave/weave --local bridge-type)" = "bridge" ]; then + # TODO: Call into weave script to do this + if ! ip link show vethwe-pcap >/dev/null 2>&1; then + ip link add name vethwe-bridge type veth peer name vethwe-pcap + ip link set vethwe-bridge up + ip link set vethwe-pcap up + ip link set vethwe-bridge master weave + fi + BRIDGE_OPTIONS="--iface=vethwe-pcap" +fi + +if [ -z "$IPALLOC_INIT" ]; then + IPALLOC_INIT="observer" + if [ "$IS_SWARM_MANAGER" == "1" ]; then + IPALLOC_INIT="consensus=$(echo $SWARM_MANAGER_PEERS | wc -l)" + fi +fi + +exec /home/weave/weaver $EXTRA_ARGS --port=6783 $BRIDGE_OPTIONS \ + --http-addr=$HTTP_ADDR --status-addr=$STATUS_ADDR \ + --no-dns \ + --ipalloc-range=$IPALLOC_RANGE \ + --ipalloc-init $IPALLOC_INIT \ + --nickname "$(hostname)" \ + --log-level=debug \ + --db-prefix="$WEAVE_DIR/weave" \ + --plugin-v2 \ + --plugin-mesh-socket='' \ + $(echo $SWARM_MANAGER_PEERS | tr '\n' ' ') diff --git a/prog/plugin/Dockerfile.template b/prog/plugin/Dockerfile.template deleted file mode 100644 index 4de1c580..00000000 --- a/prog/plugin/Dockerfile.template +++ /dev/null @@ -1,3 +0,0 @@ -FROM DOCKERHUB_USER/weaveARCH_EXT -ADD ./plugin /home/weave/ -ENTRYPOINT ["/home/weave/plugin"] diff --git a/prog/plugin/main.go b/prog/plugin/main.go deleted file mode 100644 index 6e36d8ec..00000000 --- a/prog/plugin/main.go +++ /dev/null @@ -1,140 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "net" - "net/http" - "os" - "os/signal" - "path" - "strings" - "syscall" - - "github.com/docker/libnetwork/ipamapi" - weaveapi "github.com/weaveworks/weave/api" - "github.com/weaveworks/weave/common" - "github.com/weaveworks/weave/common/docker" - weavenet "github.com/weaveworks/weave/net" - ipamplugin "github.com/weaveworks/weave/plugin/ipam" - netplugin "github.com/weaveworks/weave/plugin/net" - "github.com/weaveworks/weave/plugin/skel" -) - -var version = "unreleased" - -var Log = common.Log - -func main() { - var ( - justVersion bool - address string - meshAddress string - logLevel string - noMulticastRoute bool - ) - - flag.BoolVar(&justVersion, "version", false, "print version and exit") - flag.StringVar(&logLevel, "log-level", "info", "logging level (debug, info, warning, error)") - flag.StringVar(&address, "socket", "/run/docker/plugins/weave.sock", "socket on which to listen") - flag.StringVar(&meshAddress, "meshsocket", "/run/docker/plugins/weavemesh.sock", "socket on which to listen in mesh mode") - flag.BoolVar(&noMulticastRoute, "no-multicast-route", false, "deprecated (this is now the default)") - - flag.Parse() - - if justVersion { - fmt.Printf("weave plugin %s\n", version) - os.Exit(0) - } - - common.SetLogLevel(logLevel) - - weave := weaveapi.NewClient(os.Getenv("WEAVE_HTTP_ADDR"), Log) - - // API 1.21 is the first version that supports docker network commands - dockerClient, err := docker.NewVersionedClientFromEnv("1.21") - if err != nil { - Log.Fatalf("unable to connect to docker: %s", err) - } - - Log.Println("Weave plugin", version, "Command line options:", os.Args[1:]) - if noMulticastRoute { - Log.Warning("--no-multicast-route option has been removed; multicast is off by default") - } - Log.Info(dockerClient.Info()) - - err = run(dockerClient, weave, address, meshAddress) - if err != nil { - Log.Fatal(err) - } -} - -func run(dockerClient *docker.Client, weave *weaveapi.Client, address, meshAddress string) error { - endChan := make(chan error, 1) - if address != "" { - globalListener, err := listenAndServe(dockerClient, weave, address, endChan, "global", false) - if err != nil { - return err - } - defer os.Remove(address) - defer globalListener.Close() - } - if meshAddress != "" { - meshListener, err := listenAndServe(dockerClient, weave, meshAddress, endChan, "local", true) - if err != nil { - return err - } - defer os.Remove(meshAddress) - defer meshListener.Close() - } - - statusListener, err := weavenet.ListenUnixSocket("/home/weave/status.sock") - if err != nil { - return err - } - go serveStatus(statusListener) - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) - - select { - case sig := <-sigChan: - Log.Debugf("Caught signal %s; shutting down", sig) - return nil - case err := <-endChan: - return err - } -} - -func listenAndServe(dockerClient *docker.Client, weave *weaveapi.Client, address string, endChan chan<- error, scope string, withIpam bool) (net.Listener, error) { - name := strings.TrimSuffix(path.Base(address), ".sock") - d, err := netplugin.New(dockerClient, weave, name, scope) - if err != nil { - return nil, err - } - - var i ipamapi.Ipam - if withIpam { - i = ipamplugin.NewIpam(weave) - } - - listener, err := weavenet.ListenUnixSocket(address) - if err != nil { - return nil, err - } - Log.Printf("Listening on %s for %s scope", address, scope) - - go func() { - endChan <- skel.Listen(listener, d, i) - }() - - return listener, nil -} - -func serveStatus(listener net.Listener) { - server := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, "ok") - })} - if err := server.Serve(listener); err != nil { - Log.Fatalf("ListenAndServeStatus failed: %s", err) - } -} diff --git a/prog/plugin/main_test.go b/prog/plugin/main_test.go deleted file mode 100644 index 1873f4a3..00000000 --- a/prog/plugin/main_test.go +++ /dev/null @@ -1,12 +0,0 @@ -package main - -import ( - "testing" - - weavetest "github.com/weaveworks/weave/testing" -) - -func TestMain(t *testing.T) { - weavetest.TrimTestArgs() - main() -} diff --git a/prog/weaver/main.go b/prog/weaver/main.go index 8b077da3..7982a2fb 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -25,6 +25,7 @@ import ( "github.com/weaveworks/weave/nameserver" weavenet "github.com/weaveworks/weave/net" "github.com/weaveworks/weave/net/address" + "github.com/weaveworks/weave/plugin" weave "github.com/weaveworks/weave/router" ) @@ -150,6 +151,10 @@ func main() { discoveryEndpoint string token string advertiseAddress string + pluginSocket string + pluginMeshSocket string + enablePlugin bool + enablePluginV2 bool defaultDockerHost = "unix:///var/run/docker.sock" ) @@ -195,6 +200,11 @@ func main() { mflag.StringVar(&token, []string{"-token"}, "", "token for peer discovery") mflag.StringVar(&advertiseAddress, []string{"-advertise-address"}, "", "address to advertise for peer discovery") + mflag.BoolVar(&enablePlugin, []string{"-plugin"}, false, "enable Docker plugin (v1)") + mflag.BoolVar(&enablePluginV2, []string{"-plugin-v2"}, false, "enable Docker plugin (v2)") + mflag.StringVar(&pluginSocket, []string{"-plugin-socket"}, "/run/docker/plugins/weave.sock", "plugin socket on which to listen") + mflag.StringVar(&pluginMeshSocket, []string{"-plugin-mesh-socket"}, "/run/docker/plugins/weavemesh.sock", "plugin socket on which to listen in mesh mode") + // crude way of detecting that we probably have been started in a // container, with `weave launch` --> suppress misleading paths in // mflags error messages. @@ -396,6 +406,10 @@ func main() { go listenAndServeHTTP(statusAddr, statusMux) } + if enablePlugin || enablePluginV2 { + go plugin.Start(httpAddr, dockerCli, pluginSocket, pluginMeshSocket, !noDNS, enablePluginV2) + } + signals.SignalHandlerLoop(common.Log, router) } diff --git a/prog/weaveutil/main.go b/prog/weaveutil/main.go index 9e8ea815..bc954761 100644 --- a/prog/weaveutil/main.go +++ b/prog/weaveutil/main.go @@ -14,29 +14,32 @@ var commands map[string]func([]string) error func init() { commands = map[string]func([]string) error{ - "help": help, - "netcheck": netcheck, - "docker-tls-args": dockerTLSArgs, - "create-datapath": createDatapath, - "delete-datapath": deleteDatapath, - "check-datapath": checkDatapath, - "add-datapath-interface": addDatapathInterface, - "create-plugin-network": createPluginNetwork, - "remove-plugin-network": removePluginNetwork, - "container-addrs": containerAddrs, - "process-addrs": processAddrs, - "attach-container": attach, - "detach-container": detach, - "configure-arp": configureARP, - "check-iface": checkIface, - "del-iface": delIface, - "setup-iface": setupIface, - "list-netdevs": listNetDevs, - "cni-net": cniNet, - "cni-ipam": cniIPAM, - "expose-nat": exposeNAT, - "bridge-ip": bridgeIP, - "unique-id": uniqueID, + "help": help, + "netcheck": netcheck, + "docker-tls-args": dockerTLSArgs, + "create-datapath": createDatapath, + "delete-datapath": deleteDatapath, + "check-datapath": checkDatapath, + "add-datapath-interface": addDatapathInterface, + "create-plugin-network": createPluginNetwork, + "remove-plugin-network": removePluginNetwork, + "container-addrs": containerAddrs, + "process-addrs": processAddrs, + "attach-container": attach, + "detach-container": detach, + "configure-arp": configureARP, + "check-iface": checkIface, + "del-iface": delIface, + "setup-iface": setupIface, + "list-netdevs": listNetDevs, + "cni-net": cniNet, + "cni-ipam": cniIPAM, + "expose-nat": exposeNAT, + "bridge-ip": bridgeIP, + "unique-id": uniqueID, + "swarm-manager-peers": swarmManagerPeers, + "is-swarm-manager": isSwarmManager, + "is-docker-plugin-enabled": isDockerPluginEnabled, } } diff --git a/prog/weaveutil/plugin_network.go b/prog/weaveutil/plugin_network.go index 33c3a79a..79d96b65 100644 --- a/prog/weaveutil/plugin_network.go +++ b/prog/weaveutil/plugin_network.go @@ -2,8 +2,10 @@ package main import ( + "context" "fmt" + "github.com/docker/docker/client" docker "github.com/fsouza/go-dockerclient" "github.com/weaveworks/weave/plugin/net" @@ -71,6 +73,37 @@ while Weave is not running`, networkName, containers) return nil } +// Exits with 0 if the given plugin (v2) is enabled. +// +// Any failure due to missing plugin support is non-harmful as plugin (v2) +// cannot be enabled when Docker does not support it. +func isDockerPluginEnabled(args []string) error { + if len(args) != 1 { + cmdUsage("is-docker-plugin-enabled", "") + } + + pluginName := args[0] + + // This is messed up: we are using docker/docker/client instead of + // fsouza/go-dockerclient because the latter does not support plugins. + c, err := client.NewEnvClient() + if err != nil { + return fmt.Errorf("unable to connect to docker: %s", err) + } + + ctx := context.Background() + p, _, err := c.PluginInspectWithRaw(ctx, pluginName) + if err != nil { + return err + } + + if !p.Enabled { + return fmt.Errorf("plugin %q is disabled", pluginName) + } + + return nil +} + func newDockerClient() (*docker.Client, error) { // API 1.21 is the first version that supports docker network // commands @@ -82,5 +115,6 @@ func newDockerClient() (*docker.Client, error) { if err != nil { return nil, fmt.Errorf("unable to connect to docker: %s", err) } + return c, nil } diff --git a/prog/weaveutil/swarm.go b/prog/weaveutil/swarm.go new file mode 100644 index 00000000..6131b3a9 --- /dev/null +++ b/prog/weaveutil/swarm.go @@ -0,0 +1,71 @@ +package main + +import ( + "fmt" + "os" + "strings" + + "github.com/docker/docker/api/types/swarm" + "github.com/fsouza/go-dockerclient" + "github.com/pkg/errors" +) + +const DOCKER_API_VERSION = "1.26" + +func isSwarmManager(args []string) error { + info, err := dockerInfo() + if err != nil { + return err + } + + // hack-y way to denote that a node does not belong to any swarm + if info.Swarm.LocalNodeState != swarm.LocalNodeStateActive { + os.Exit(20) + } + + if !info.Swarm.ControlAvailable { + return fmt.Errorf("node is not a manager") + } + + return nil +} + +func swarmManagerPeers(args []string) error { + info, err := dockerInfo() + if err != nil { + return err + } + + for _, managerNode := range info.Swarm.RemoteManagers { + ip, err := ipFromAddr(managerNode.Addr) + if err != nil { + return errors.Wrap(err, "ipFromAddr") + } + fmt.Println(ip) + } + + return nil +} + +func dockerInfo() (*docker.DockerInfo, error) { + client, err := docker.NewVersionedClientFromEnv(DOCKER_API_VERSION) + if err != nil { + return nil, errors.Wrap(err, "docker.NewVersionedClientFromEnv") + } + + info, err := client.Info() + if err != nil { + return nil, errors.Wrap(err, "docker.Info") + } + + return info, nil +} + +func ipFromAddr(addr string) (string, error) { + parts := strings.Split(addr, ":") + if len(parts) != 2 { + return "", fmt.Errorf("invalid address: %q", addr) + } + + return parts[0], nil +} diff --git a/site/plugin.md b/site/plugin.md index 832387c8..32a4a452 100644 --- a/site/plugin.md +++ b/site/plugin.md @@ -59,8 +59,29 @@ then run your containers using the Docker command-line: Docker enables you to create multiple independent networks and attach different sets of containers to each network. However, coordinating -this between hosts, Docker requires that you configure a -["key-value store"](https://docs.docker.com/engine/userguide/networking/get-started-overlay/#step-1-set-up-a-key-value-store). +this between hosts requires that you run Docker in ["swarm mode"](https://docs.docker.com/engine/swarm/swarm-mode/) or configure a +["key-value store"](https://docs.docker.com/engine/userguide/networking/get-started-overlay/#/set-up-a-key-value-store). + +Docker swarm mode requires Docker version 1.13 or later to work with +plugins such as Weave Net. + +To create a new network for services in swarm mode, run: + + $ docker network create --driver=weaveworks/net-plugin:2.0.0 mynetwork + +then use it to create a service: + + $ docker service create --network=mynetwork --name myservice ... + + +To create a new network to attach containers in swarm mode, run: + + $ docker network create --driver=weave --attachable mynetwork + +then use it to connect a container: + + $ docker run --net=mynetwork ... + If your Docker installation has a key-value store, create a network based on Weave Net as follows: diff --git a/test/090_docker_restart_policy_2_test.sh b/test/090_docker_restart_policy_2_test.sh index b4cc76cd..e1630bc9 100755 --- a/test/090_docker_restart_policy_2_test.sh +++ b/test/090_docker_restart_policy_2_test.sh @@ -6,22 +6,20 @@ start_suite "Test docker restart policy" weave_on $HOST1 launch -assert "docker_on $HOST1 inspect -f '{{.HostConfig.RestartPolicy.Name}}' weave weaveproxy weaveplugin" "always\nalways\nalways" +assert "docker_on $HOST1 inspect -f '{{.HostConfig.RestartPolicy.Name}}' weave weaveproxy" "always\nalways" assert_raises "check_restart $HOST1 weave" assert_raises "check_restart $HOST1 weaveproxy" -assert_raises "check_restart $HOST1 weaveplugin" # stop + launch tests that restart policy changes result # in the old containers being removed and new ones created weave_on $HOST1 stop weave_on $HOST1 launch --no-restart -assert "docker_on $HOST1 inspect -f '{{.HostConfig.RestartPolicy.Name}}' weave weaveproxy weaveplugin" "no\nno\nno" +assert "docker_on $HOST1 inspect -f '{{.HostConfig.RestartPolicy.Name}}' weave weaveproxy" "no\nno" assert_raises "! check_restart $HOST1 weave" assert_raises "! check_restart $HOST1 weaveproxy" -assert_raises "! check_restart $HOST1 weaveplugin" # Relaunch to prevent the `weave stop` in `end_suite` # timing out trying to remove the plugin network diff --git a/test/195_no_multicast_route_1_test.sh b/test/195_no_multicast_route_1_test.sh index dee59b0b..2b057e1a 100755 --- a/test/195_no_multicast_route_1_test.sh +++ b/test/195_no_multicast_route_1_test.sh @@ -12,9 +12,8 @@ start_suite "--no-multicast-route operation" $SSH $HOST1 docker network rm testmcasttrue testmcastfalse >/dev/null 2>&1 || true # Ensure containers run either way have no multicast route -weave_on $HOST1 launch-router +weave_on $HOST1 launch-router --plugin weave_on $HOST1 launch-proxy --no-multicast-route -weave_on $HOST1 launch-plugin # plugin defaults to multicast off except for 'net=weave' docker_on $HOST1 run --name c1 -dt $SMALL_IMAGE /bin/sh weave_on $HOST1 attach --no-multicast-route c1 diff --git a/test/335_rmpeer_3_test.sh b/test/335_rmpeer_3_test.sh index ffdfc85b..eb8f916e 100755 --- a/test/335_rmpeer_3_test.sh +++ b/test/335_rmpeer_3_test.sh @@ -14,7 +14,10 @@ weave_on $HOST3 launch $HOST1 weave_on $HOST3 prime # nuke 1st peer -docker_on $HOST1 kill weave +# NOTE: docker-kill hangs (https://github.com/docker/docker/issues/31447), so we +# kill directly the weaver process instead. +WEAVER_PID=$(container_pid $HOST1 weave) +run_on $HOST1 "sudo kill -9 $WEAVER_PID" # transfer its space to the 2nd weave_on $HOST2 rmpeer $PEER1 diff --git a/test/600_proxy_docker_py.sh b/test/600_proxy_docker_py.sh index f0a55222..8e54f2c1 100755 --- a/test/600_proxy_docker_py.sh +++ b/test/600_proxy_docker_py.sh @@ -3,7 +3,7 @@ . "$(dirname "$0")/config.sh" -IMAGE="joffrey/docker-py:1.8.1" +IMAGE="joffrey/docker-py:1.9.0-rc2" docker_py_test() { SHARD=$1 diff --git a/test/620_proxy_entrypoint_command_test.sh b/test/620_proxy_entrypoint_command_test.sh index 1085bcac..748c415a 100755 --- a/test/620_proxy_entrypoint_command_test.sh +++ b/test/620_proxy_entrypoint_command_test.sh @@ -15,7 +15,7 @@ run_container() { } start_suite "Proxy uses correct entrypoint and command with weavewait" -weave_on $HOST1 launch-proxy +weave_on $HOST1 launch-proxy --no-restart build_image check-ethwe-up '["grep"]' '["^1$", "/sys/class/net/ethwe/carrier"]' run_container "check-ethwe-up" @@ -27,7 +27,11 @@ build_image false '["/bin/false"]' '' run_container "--entrypoint='grep' false ^1$ /sys/class/net/ethwe/carrier" weave_on $HOST1 launch-router --ipalloc-range 10.2.2.0/24 -docker_on $HOST1 kill weaveproxy +# NOTE: docker-kill hangs (https://github.com/docker/docker/issues/31447), so we +# kill directly the weaveproxy process instead. +WEAVEPROXY_PID=$(container_pid $HOST1 weaveproxy) +$SSH $HOST1 "ps aux | grep weaveproxy" +run_on $HOST1 "sudo kill -9 $WEAVEPROXY_PID" weave_on $HOST1 launch-proxy assert_raises "proxy docker_on $HOST1 run check-ethwe-up" diff --git a/test/850_plugin_v2_cross_hosts_2_test.sh b/test/850_plugin_v2_cross_hosts_2_test.sh new file mode 100755 index 00000000..64e5e2b6 --- /dev/null +++ b/test/850_plugin_v2_cross_hosts_2_test.sh @@ -0,0 +1,128 @@ +#! /bin/bash + +. "$(dirname "$0")/config.sh" + +PLUGIN_NAME="weave-ci-registry:5000/weaveworks/net-plugin" +HOST1_IP=$($SSH $HOST1 getent ahosts $HOST1 | grep "RAW" | cut -d' ' -f1) +SERVICE="weave-850-service" +NETWORK="weave-850-network" + +setup_master() { + # Setup Docker image registry on $HOST1 + docker_on $HOST1 run -p 5000:5000 -d registry:2 + + # Build plugin-v2 on $HOST1, because Circle "CI" runs only ancient + # version of Docker which does not support v2 plugins. + build_plugin_img="buildpluginv2-ci" + rsync -az -e "$SSH" "$(dirname $0)/../prog/net-plugin" $HOST1:~/ + $SSH $HOST1</dev/null + docker plugin disable $PLUGIN_NAME >/dev/null + docker plugin remove $PLUGIN_NAME 2>/dev/null + + WORK_DIR=$(mktemp -d) + mkdir -p \${WORK_DIR}/rootfs + + docker create --name=$build_plugin_img weaveworks/weave true + docker export $build_plugin_img | tar -x -C \${WORK_DIR}/rootfs + cp \${HOME}/net-plugin/launch.sh \${WORK_DIR}/rootfs/home/weave/launch.sh + cp \${HOME}/net-plugin/config.json \${WORK_DIR} + docker plugin create $PLUGIN_NAME \${WORK_DIR} + + echo "$HOST1_IP weave-ci-registry" | sudo tee -a /etc/hosts + docker plugin push $PLUGIN_NAME + + # Start Swarm Manager and enable the plugin + docker swarm init --advertise-addr=$HOST1_IP + + [ -n "$COVERAGE" ] && docker plugin set $PLUGIN_NAME EXTRA_ARGS="-test.coverprofile=/home/weave/cover.prof --" + #docker plugin set $PLUGIN_NAME WEAVE_PASSWORD="foobar" + docker plugin enable $PLUGIN_NAME +EOF +} + +setup_worker() { + $SSH $HOST2</dev/null 2>&1; then + return + fi + echo "Waiting for \"$2\" network" + sleep 2 + done + echo "Failed to wait for \"$2\" network" >&2 + return 1 +} + +wait_for_service() { + for i in $(seq 60); do + replicas=$($SSH $1 docker service ls -f="name=$2" | awk '{print $4}' | grep -v REPLICAS) + if [ "$replicas" == "$3/$3" ]; then + return + fi + echo "Waiting for \"$2\" service" + sleep 2 + done + echo "Failed to wait for \"$2\" service" >&2 + return 1 +} + +start_suite "Test Docker plugin-v2" + +#cleanup $HOST1 $HOST2 +#exit 1 + +setup_master +setup_worker $($SSH $HOST1 docker swarm join-token --quiet worker) + +echo "Creating network and service..." + +# Create network and service +$SSH $HOST1< [--no-restart] [--resume] [--no-discovery] [--no-dns] [--ipalloc-init ] [--ipalloc-range [--ipalloc-default-subnet ]] + [--plugin] [--log-level=debug|info|warning|error] ... launch-proxy [-H ] [--without-dns] [--no-multicast-route] @@ -58,8 +59,6 @@ weave launch [--hostname-replacement ] [--rewrite-inspect] [--log-level=debug|info|warning|error] - launch-plugin [--no-restart] [--no-multicast-route] - [--log-level=debug|info|warning|error] weave prime @@ -90,7 +89,6 @@ weave status [targets | connections | peers | dns | ipam] weave stop stop-router stop-proxy - stop-plugin weave reset [--force] rmpeer ... @@ -163,7 +161,6 @@ exec_remote() { -e WEAVE_MTU \ -e WEAVE_NO_FASTDP \ -e WEAVE_NO_BRIDGED_FASTDP \ - -e WEAVE_NO_PLUGIN \ -e DOCKER_BRIDGE \ -e DOCKER_CLIENT_HOST="$DOCKER_CLIENT_HOST" \ -e DOCKER_CLIENT_ARGS \ @@ -262,15 +259,13 @@ fi # main (remote and --local) - settings ###################################################################### -# Default restart policy for router/proxy/plugin +# Default restart policy for router/proxy RESTART_POLICY="--restart=always" BASE_IMAGE=$DOCKERHUB_USER/weave IMAGE=$BASE_IMAGE:$IMAGE_VERSION CONTAINER_NAME=${WEAVE_CONTAINER_NAME:-weave} -BASE_PLUGIN_IMAGE=$DOCKERHUB_USER/plugin -PLUGIN_IMAGE=$BASE_PLUGIN_IMAGE:$IMAGE_VERSION -PLUGIN_CONTAINER_NAME=weaveplugin +PLUGIN_NAME="$DOCKERHUB_USER/net-plugin" CNI_PLUGIN_NAME="weave-plugin-$IMAGE_VERSION" CNI_PLUGIN_DIR=${WEAVE_CNI_PLUGIN_DIR:-$HOST_ROOT/opt/cni/bin} # Note VOLUMES_CONTAINER which is for weavewait should change when you upgrade Weave @@ -291,6 +286,7 @@ HTTP_ADDR=${WEAVE_HTTP_ADDR:-127.0.0.1:6784} STATUS_ADDR=${WEAVE_STATUS_ADDR:-127.0.0.1:6782} PROXY_PORT=12375 PROXY_CONTAINER_NAME=weaveproxy +PROC_PATH="/proc" COVERAGE_ARGS="" [ -n "$COVERAGE" ] && COVERAGE_ARGS="-test.coverprofile=/home/weave/cover.prof --" @@ -372,9 +368,9 @@ delete_iptables_rule() { # makes containers react more quickly to a change in the MAC address # associated with an IP address. configure_arp_cache() { - $2 sh -c "echo 5 >/proc/sys/net/ipv4/neigh/$1/base_reachable_time && - echo 2 >/proc/sys/net/ipv4/neigh/$1/delay_first_probe_time && - echo 1 >/proc/sys/net/ipv4/neigh/$1/ucast_solicit" + $2 sh -c "echo 5 >$PROC_PATH/sys/net/ipv4/neigh/$1/base_reachable_time && + echo 2 >$PROC_PATH/sys/net/ipv4/neigh/$1/delay_first_probe_time && + echo 1 >$PROC_PATH/sys/net/ipv4/neigh/$1/ucast_solicit" } # Send out an ARP announcement @@ -655,8 +651,10 @@ init_bridge_prep() { # Running this from 'weave --local' when weaveexec is not on the # path will run it as a Docker container that does not have access # to /etc/machine-id, so will not give the full range of persistent IDs - WEAVEDB_PATH=$(docker inspect -f '{{with index .Mounts 0}}{{.Source}}{{end}}' $DB_CONTAINER_NAME 2>/dev/null) - MAC=$(util_op unique-id "$HOST_ROOT/$WEAVEDB_PATH/weave" "$HOST_ROOT") + if [ -z "$WEAVEDB_DIR_PATH" ]; then + WEAVEDB_DIR_PATH="$HOST_ROOT/$(docker inspect -f '{{with index .Mounts 0}}{{.Source}}{{end}}' $DB_CONTAINER_NAME 2>/dev/null)" + fi + MAC=$(util_op unique-id "$WEAVEDB_DIR_PATH/weave" "$HOST_ROOT") ip link set dev $BRIDGE address $MAC @@ -810,6 +808,25 @@ attach() { # functions for interacting with containers ###################################################################### +# Docker Plugin V1 runs in a Docker container. +is_plugin_v1_enabled() { + docker inspect -f '{{.Config.Cmd}}' $CONTAINER_NAME 2>/dev/null | grep -q -- "--plugin" +} + +# The presence of Docker Plugin V2 can be checked only via 'docker plugin', +# as v2 plugins do not run as regular containers. +is_plugin_v2_enabled() { + util_op is-docker-plugin-enabled $PLUGIN_NAME 2>/dev/null +} + +# Check that plugin (v2) is not running. +check_not_running_plugin_v2() { + if is_plugin_v2_enabled; then + echo "Docker plugin \"$PLUGIN_NAME\" is running; Aborting." >&2 + return 1 + fi +} + # Check that a container for component $1 named $2 with image $3 is not running check_not_running() { RUN_STATUS=$(docker inspect --format='{{.State.Running}} {{.State.Status}} {{.Config.Image}}' $2 2>/dev/null) || true @@ -1527,6 +1544,10 @@ launch_router() { AWSVPC_ARGS="--awsvpc" AWSVPC=1 ;; + --plugin) + PLUGIN_ARGS="--plugin" + ENABLE_PLUGIN=1 + ;; *) ARGS="$ARGS '$(echo "$1" | sed "s|'|'\"'\"'|g")'" ;; @@ -1589,6 +1610,7 @@ launch_router() { --pid=host \ --volumes-from $DB_CONTAINER_NAME \ -v $RESOLV_CONF_DIR:/var/run/weave/etc \ + -v /run/docker/plugins:/run/docker/plugins \ -e WEAVE_PASSWORD \ -e CHECKPOINT_DISABLE \ $WEAVE_DOCKER_ARGS $IMAGE $COVERAGE_ARGS \ @@ -1598,6 +1620,7 @@ launch_router() { --dns-effective-listen-address $DOCKER_BRIDGE_IP \ $DNS_ROUTER_OPTS $NO_DNS_OPT \ $AWSVPC_ARGS \ + $PLUGIN_ARGS \ --http-addr $HTTP_ADDR \ --status-addr $STATUS_ADDR \ --resolv-conf "/var/run/weave/etc/$RESOLV_CONF_BASE" \ @@ -1605,6 +1628,11 @@ launch_router() { wait_for_status $CONTAINER_NAME http_call $HTTP_ADDR setup_awsvpc populate_router + + if [ -n "$ENABLE_PLUGIN" ]; then + wait_for_status $CONTAINER_NAME http_call_unix $CONTAINER_NAME plugin-status.sock + util_op create-plugin-network weave weavemesh $IPRANGE + fi } setup_awsvpc() { @@ -1612,11 +1640,11 @@ setup_awsvpc() { # Set proxy_arp on the bridge, so that it could accept packets destined # to containers within the same subnet but running on remote hosts. # Without it, exact routes on each container are required. - echo 1 >/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp + echo 1 >$PROC_PATH/sys/net/ipv4/conf/$BRIDGE/proxy_arp # Avoid delaying the first ARP request. Also, setting it to 0 avoids # placing the request into a bounded queue as it can be seen: # https://git.kernel.org/cgit/linux/kernel/git/stable/linux-stable.git/tree/net/ipv4/arp.c?id=refs/tags/v4.6.1#n819 - echo 0 >/proc/sys/net/ipv4/neigh/$BRIDGE/proxy_delay + echo 0 >$PROC_PATH/sys/net/ipv4/neigh/$BRIDGE/proxy_delay expose_ip fi } @@ -1639,6 +1667,7 @@ populate_router() { } stop_router() { + ! is_plugin_v1_enabled || util_op remove-plugin-network weave || true stop $CONTAINER_NAME "Weave" conntrack -D -p udp --dport $PORT >/dev/null 2>&1 || true } @@ -1680,53 +1709,9 @@ stop_proxy() { stop $PROXY_CONTAINER_NAME "Proxy" } -launch_plugin_if_not_running() { - while [ $# -gt 0 ]; do - case "$1" in - --no-restart) - RESTART_POLICY= - ;; - *) - break - ;; - esac - shift - done - - retval=0 - check_not_running plugin $PLUGIN_CONTAINER_NAME $BASE_PLUGIN_IMAGE || retval=$? - # If an existing plugin is running (we start it with restart=always), return its ID - [ $retval = 1 ] && PLUGIN_CONTAINER=$(container_id $PLUGIN_CONTAINER_NAME) && return 0 - # Any other kind of error code from check_not_running is a failure. - [ $retval -gt 0 ] && return $retval - - if ! PLUGIN_CONTAINER=$(docker run -d --name=$PLUGIN_CONTAINER_NAME \ - $(docker_run_options) \ - $RESTART_POLICY \ - --pid=host \ - -v /run/docker/plugins:/run/docker/plugins \ - -e WEAVE_HTTP_ADDR \ - $WEAVEPLUGIN_DOCKER_ARGS $PLUGIN_IMAGE $COVERAGE_ARGS \ - "$@") ; then - return 1 - fi - wait_for_status $PLUGIN_CONTAINER_NAME http_call_unix $PLUGIN_CONTAINER_NAME status.sock - WEAVE_IPAM_SUBNET=$(call_weave GET /ipinfo/defaultsubnet) - util_op create-plugin-network weave weavemesh $WEAVE_IPAM_SUBNET -} - -plugin_disabled() { - [ -n "$WEAVE_NO_PLUGIN" ] || ! check_docker_server_api_version 1.21 -} - -stop_plugin() { - util_op remove-plugin-network weave || true - stop $PLUGIN_CONTAINER_NAME "Plugin" -} - protect_against_docker_hang() { # If the plugin is not running, remove its socket so Docker doesn't try to talk to it - if ! check_running $PLUGIN_CONTAINER_NAME 2>/dev/null ; then + if ! is_plugin_v2_enabled 2>/dev/null ; then rm -f /run/docker/plugins/weave.sock /run/docker/plugins/weavemesh.sock fi } @@ -1842,7 +1827,7 @@ handle_help_arg "$1" case "$COMMAND" in setup) - for img in $IMAGE $EXEC_IMAGE $PLUGIN_IMAGE $WEAVEDB_IMAGE ; do + for img in $IMAGE $EXEC_IMAGE $WEAVEDB_IMAGE ; do docker pull $img done setup_cni @@ -1854,11 +1839,26 @@ case "$COMMAND" in [ $# -eq 0 ] || usage ask_version $CONTAINER_NAME $IMAGE || true ask_version $PROXY_CONTAINER_NAME $EXEC_IMAGE --entrypoint=/home/weave/weaveproxy || true - ask_version $PLUGIN_CONTAINER_NAME $PLUGIN_IMAGE || true ;; # intentionally undocumented since it assumes knowledge of weave # internals create-bridge) + while [ $# -gt 0 ]; do + case "$1" in + --proc-path=*) + PROC_PATH="${1#*=}" + shift + ;; + --weavedb-dir-path=*) + WEAVEDB_DIR_PATH="${1#*=}" + shift + ;; + *) + break + ;; + esac + done + if [ "$1" != "--force" ] ; then cat 1>&2 </dev/null ; then - echo "ERROR:" $CONTAINER_NAME "container must be running before plugin can be launched" >&2 - exit 1 - fi - launch_plugin_if_not_running "$@" - echo $PLUGIN_CONTAINER - ;; env|proxy-env) [ "$COMMAND" = "env" ] || deprecation_warning "$COMMAND" "'weave env'" if PROXY_ADDR=$(proxy_addr) ; then @@ -1976,10 +1970,14 @@ EOF echo " Service: proxy" echo " Address: $PROXY_ADDRS" fi - if [ -z "$SUB_STATUS" ] && check_running $PLUGIN_CONTAINER_NAME 2>/dev/null ; then + if [ -z "$SUB_STATUS" ] && is_plugin_v1_enabled ; then echo - echo " Service: plugin" + echo " Service: plugin (v1)" echo " DriverName: weave" + elif [ -z "$SUB_STATUS" ] && is_plugin_v2_enabled ; then + echo + echo " Service: plugin (v2)" + echo " DriverName: $PLUGIN_NAME" fi [ -n "$SUB_STATUS" ] || echo [ $res -eq 0 ] @@ -2119,7 +2117,6 @@ EOF ;; stop) [ $# -eq 0 ] || usage - plugin_disabled || stop_plugin stop_router stop_proxy ;; @@ -2131,13 +2128,9 @@ EOF [ $# -eq 0 ] || usage stop_proxy ;; - stop-plugin) - [ $# -eq 0 ] || usage - stop_plugin - ;; reset) [ $# -eq 0 ] || [ $# -eq 1 -a "$1" = "--force" ] || usage - plugin_disabled || util_op remove-plugin-network weave || true + ! is_plugin_v1_enabled || util_op remove-plugin-network weave || true warn_if_stopping_proxy_in_env res=0 [ "$1" = "--force" ] || check_running $CONTAINER_NAME 2>/dev/null || res=$? @@ -2155,7 +2148,7 @@ EOF exit 1 ;; esac - for NAME in $PLUGIN_CONTAINER_NAME $CONTAINER_NAME $PROXY_CONTAINER_NAME ; do + for NAME in $CONTAINER_NAME $PROXY_CONTAINER_NAME ; do docker stop $NAME >/dev/null 2>&1 || true docker rm -f $NAME >/dev/null 2>&1 || true done