diff --git a/.gitmodules b/.gitmodules index d653c81a..918c7914 100644 --- a/.gitmodules +++ b/.gitmodules @@ -61,9 +61,6 @@ [submodule "vendor/github.com/hashicorp/go-cleanhttp"] path = vendor/github.com/hashicorp/go-cleanhttp url = http://github.com/hashicorp/go-cleanhttp -[submodule "vendor/github.com/appc/cni"] - path = vendor/github.com/appc/cni - url = https://github.com/appc/cni [submodule "vendor/github.com/coreos/go-iptables"] path = vendor/github.com/coreos/go-iptables url = https://github.com/coreos/go-iptables @@ -124,3 +121,6 @@ [submodule "vendor/github.com/weaveworks/common"] path = vendor/github.com/weaveworks/common url = https://github.com/weaveworks/common +[submodule "vendor/github.com/containernetworking/cni"] + path = vendor/github.com/containernetworking/cni + url = https://github.com/containernetworking/cni diff --git a/Makefile b/Makefile index 8f7d62df..cf3bc69e 100644 --- a/Makefile +++ b/Makefile @@ -173,6 +173,8 @@ ifeq ($(BUILD_IN_CONTAINER),true) # It bind-mounts the source into the container and passes all important variables exes $(EXES) tests lint: $(BUILD_UPTODATE) git submodule update --init +# Containernetworking has another copy of vishvananda/netlink which leads to duplicate definitions + -@rm -r vendor/github.com/containernetworking/cni/vendor @mkdir -p $(shell pwd)/.pkg $(SUDO) docker run $(RM) $(RUN_FLAGS) \ -v $(shell pwd):/go/src/github.com/weaveworks/weave \ diff --git a/npc/constants.go b/npc/constants.go index 64565484..7ab01f7b 100644 --- a/npc/constants.go +++ b/npc/constants.go @@ -6,4 +6,6 @@ const ( MainChain = "WEAVE-NPC" DefaultChain = "WEAVE-NPC-DEFAULT" IngressChain = "WEAVE-NPC-INGRESS" + + IpsetNamePrefix = "weave-" ) diff --git a/npc/ipset/ipset.go b/npc/ipset/ipset.go index 3a27327b..d4dd5160 100644 --- a/npc/ipset/ipset.go +++ b/npc/ipset/ipset.go @@ -2,6 +2,7 @@ package ipset import ( "os/exec" + "strings" "github.com/pkg/errors" ) @@ -22,6 +23,8 @@ type Interface interface { Flush(ipsetName Name) error Destroy(ipsetName Name) error + List(prefix string) ([]Name, error) + FlushAll() error DestroyAll() error } @@ -72,6 +75,24 @@ func (i *ipset) DestroyAll() error { return doExec("destroy") } +// Fetch a list of all existing sets with a given prefix +func (i *ipset) List(prefix string) ([]Name, error) { + output, err := exec.Command("ipset", "list", "-name", "-output", "plain").Output() + if err != nil { + return nil, err + } + + var selected []Name + sets := strings.Split(string(output), "\n") + for _, v := range sets { + if strings.HasPrefix(v, prefix) { + selected = append(selected, Name(v)) + } + } + + return selected, err +} + func doExec(args ...string) error { if output, err := exec.Command("ipset", args...).CombinedOutput(); err != nil { return errors.Wrapf(err, "ipset %v failed: %s", args, output) diff --git a/npc/selector.go b/npc/selector.go index b03c97b1..1f34fa3b 100644 --- a/npc/selector.go +++ b/npc/selector.go @@ -29,7 +29,7 @@ func newSelectorSpec(json *unversioned.LabelSelector, nsName string, ipsetType i // We prefix the selector string with the namespace name when generating // the shortname because you can specify the same selector in multiple // namespaces - we need those to map to distinct ipsets - ipsetName: ipset.Name("weave-" + shortName(nsName+":"+key)), + ipsetName: ipset.Name(IpsetNamePrefix + shortName(nsName+":"+key)), ipsetType: ipsetType}, nil } diff --git a/plugin/ipam/cni.go b/plugin/ipam/cni.go index 4c9a45a8..628cb1db 100644 --- a/plugin/ipam/cni.go +++ b/plugin/ipam/cni.go @@ -5,19 +5,24 @@ import ( "fmt" "net" - "github.com/appc/cni/pkg/skel" - "github.com/appc/cni/pkg/types" + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/types/current" ) func (i *Ipam) CmdAdd(args *skel.CmdArgs) error { + var conf types.NetConf + if err := json.Unmarshal(args.StdinData, &conf); err != nil { + return fmt.Errorf("failed to load netconf: %v", err) + } result, err := i.Allocate(args) if err != nil { return err } - return result.Print() + return types.PrintResult(result, conf.CNIVersion) } -func (i *Ipam) Allocate(args *skel.CmdArgs) (*types.Result, error) { +func (i *Ipam) Allocate(args *skel.CmdArgs) (types.Result, error) { // extract the things we care about conf, err := loadIPAMConf(args.StdinData) if err != nil { @@ -46,12 +51,13 @@ func (i *Ipam) Allocate(args *skel.CmdArgs) (*types.Result, error) { if err != nil { return nil, err } - result := &types.Result{ - IP4: &types.IPConfig{ - IP: *ipnet, + result := ¤t.Result{ + IPs: []*current.IPConfig{{ + Version: "4", + Address: *ipnet, Gateway: conf.Gateway, - Routes: conf.Routes, - }, + }}, + Routes: conf.Routes, } return result, nil } @@ -65,9 +71,9 @@ func (i *Ipam) Release(args *skel.CmdArgs) error { } type ipamConf struct { - Subnet string `json:"subnet,omitempty"` - Gateway net.IP `json:"gateway,omitempty"` - Routes []types.Route `json:"routes"` + Subnet string `json:"subnet,omitempty"` + Gateway net.IP `json:"gateway,omitempty"` + Routes []*types.Route `json:"routes"` } type netConf struct { diff --git a/plugin/net/cni.go b/plugin/net/cni.go index 5ed1c35a..fcf208fc 100644 --- a/plugin/net/cni.go +++ b/plugin/net/cni.go @@ -5,11 +5,13 @@ import ( "encoding/json" "fmt" "net" + "strings" "syscall" - "github.com/appc/cni/pkg/ipam" - "github.com/appc/cni/pkg/skel" - "github.com/appc/cni/pkg/types" + "github.com/containernetworking/cni/pkg/ipam" + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/types/current" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" weaveapi "github.com/weaveworks/weave/api" @@ -40,17 +42,23 @@ func loadNetConf(bytes []byte) (*NetConf, error) { return n, nil } -func (c *CNIPlugin) getIP(ipamType string, args *skel.CmdArgs) (result *types.Result, err error) { +func (c *CNIPlugin) getIP(ipamType string, args *skel.CmdArgs) (newResult *current.Result, err error) { + var result types.Result // Default IPAM is Weave's own if ipamType == "" { result, err = ipamplugin.NewIpam(c.weave).Allocate(args) } else { result, err = ipam.ExecAdd(ipamType, args.StdinData) } - if err == nil && result.IP4 == nil { + if err != nil { + return nil, err + } + newResult, err = current.NewResultFromResult(result) + // Check if ipam returned no results without error + if err == nil && len(newResult.IPs) == 0 { return nil, fmt.Errorf("IPAM plugin failed to allocate IP address") } - return result, err + return newResult, err } func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error { @@ -70,10 +78,12 @@ func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error { if err != nil { return fmt.Errorf("unable to allocate IP address: %s", err) } + // Only expecting one address + ip := result.IPs[0] // If config says nothing about routes or gateway, default one will be via the bridge - if result.IP4.Routes == nil && result.IP4.Gateway == nil { - bridgeIP, err := weavenet.FindBridgeIP(conf.BrName, &result.IP4.IP) + if len(result.Routes) == 0 && ip.Gateway == nil { + bridgeIP, err := weavenet.FindBridgeIP(conf.BrName, &ip.Address) if err == weavenet.ErrBridgeNoIP { bridgeArgs := *args bridgeArgs.ContainerID = "weave:expose" @@ -81,17 +91,18 @@ func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error { if err != nil { return fmt.Errorf("unable to allocate IP address for bridge: %s", err) } - if err := assignBridgeIP(conf.BrName, bridgeIPResult.IP4.IP); err != nil { + bridgeCIDR := bridgeIPResult.IPs[0].Address + if err := assignBridgeIP(conf.BrName, bridgeCIDR); err != nil { return fmt.Errorf("unable to assign IP address to bridge: %s", err) } - if err := weavenet.ExposeNAT(bridgeIPResult.IP4.IP); err != nil { + if err := weavenet.ExposeNAT(bridgeCIDR); err != nil { return fmt.Errorf("unable to create NAT rules: %s", err) } - bridgeIP = bridgeIPResult.IP4.IP.IP + bridgeIP = bridgeCIDR.IP } else if err != nil { return err } - result.IP4.Gateway = bridgeIP + result.IPs[0].Gateway = bridgeIP } ns, err := netns.GetFromPath(args.Netns) @@ -110,20 +121,20 @@ func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error { id = fmt.Sprintf("%x", data) } - if err := weavenet.AttachContainer(args.Netns, id, args.IfName, conf.BrName, conf.MTU, false, []*net.IPNet{&result.IP4.IP}, false); err != nil { + if err := weavenet.AttachContainer(args.Netns, id, args.IfName, conf.BrName, conf.MTU, false, []*net.IPNet{&ip.Address}, false); err != nil { return err } if err := weavenet.WithNetNSLinkUnsafe(ns, args.IfName, func(link netlink.Link) error { - return setupRoutes(link, args.IfName, result.IP4.IP, result.IP4.Gateway, result.IP4.Routes) + return setupRoutes(link, args.IfName, ip.Address, ip.Gateway, result.Routes) }); err != nil { return fmt.Errorf("error setting up routes: %s", err) } result.DNS = conf.DNS - return result.Print() + return types.PrintResult(result, conf.CNIVersion) } -func setupRoutes(link netlink.Link, name string, ipnet net.IPNet, gw net.IP, routes []types.Route) error { +func setupRoutes(link netlink.Link, name string, ipnet net.IPNet, gw net.IP, routes []*types.Route) error { var err error if routes == nil { // If config says nothing about routes, add a default one if !ipnet.Contains(gw) { @@ -133,7 +144,7 @@ func setupRoutes(link netlink.Link, name string, ipnet net.IPNet, gw net.IP, rou return err } } - routes = []types.Route{{Dst: zeroNetwork}} + routes = []*types.Route{{Dst: zeroNetwork}} } for _, r := range routes { if r.GW != nil { @@ -170,8 +181,11 @@ func (c *CNIPlugin) CmdDel(args *skel.CmdArgs) error { return err } - if _, err = weavenet.WithNetNS(args.Netns, "del-iface", args.IfName); err != nil { - return fmt.Errorf("error removing interface %q: %s", args.IfName, err) + // As of CNI 0.3 spec, runtimes can send blank if they just want the address deallocated + if args.Netns != "" { + if _, err = weavenet.WithNetNS(args.Netns, "del-iface", args.IfName); err != nil { + return fmt.Errorf("error removing interface %q: %s", args.IfName, err) + } } // Default IPAM is Weave's own @@ -180,6 +194,10 @@ func (c *CNIPlugin) CmdDel(args *skel.CmdArgs) error { } else { err = ipam.ExecDel(conf.IPAM.Type, args.StdinData) } + // Hack - don't know how we should detect this situation properly + if args.Netns == "" && strings.Contains(err.Error(), "no addresses") { + err = nil + } if err != nil { return fmt.Errorf("unable to release IP address: %s", err) } diff --git a/prog/weave-kube/launch.sh b/prog/weave-kube/launch.sh index e7f79331..1b851d49 100755 --- a/prog/weave-kube/launch.sh +++ b/prog/weave-kube/launch.sh @@ -117,7 +117,7 @@ post_start_actions() { post_start_actions & -exec /home/weave/weaver $EXTRA_ARGS --port=6783 $BRIDGE_OPTIONS \ +/home/weave/weaver $EXTRA_ARGS --port=6783 $BRIDGE_OPTIONS \ --http-addr=$HTTP_ADDR --status-addr=$STATUS_ADDR --docker-api='' --no-dns \ --ipalloc-range=$IPALLOC_RANGE $NICKNAME_ARG \ --ipalloc-init $IPALLOC_INIT \ diff --git a/prog/weave-npc/main.go b/prog/weave-npc/main.go index 2c0623a2..4a2f205f 100644 --- a/prog/weave-npc/main.go +++ b/prog/weave-npc/main.go @@ -80,14 +80,22 @@ func resetIPTables(ipt *iptables.IPTables) error { } func resetIPSets(ips ipset.Interface) error { - // TODO should restrict ipset operations to the `weave-` prefix: + // Remove ipsets prefixed `weave-` only - if err := ips.FlushAll(); err != nil { + sets, err := ips.List(npc.IpsetNamePrefix) + if err != nil { + common.Log.Errorf("Failed to retrieve list of ipsets") return err } - if err := ips.DestroyAll(); err != nil { - return err + common.Log.Debugf("Got list of ipsets: %v", sets) + + for _, s := range sets { + common.Log.Debugf("Destroying ipsets '%s'", string(s)) + if err := ips.Destroy(s); err != nil { + common.Log.Errorf("Failed to destroy ipset '%s'", string(s)) + return err + } } return nil diff --git a/prog/weaveutil/cni.go b/prog/weaveutil/cni.go index 3f5dba4d..e6157261 100644 --- a/prog/weaveutil/cni.go +++ b/prog/weaveutil/cni.go @@ -3,7 +3,8 @@ package main import ( "os" - cni "github.com/appc/cni/pkg/skel" + cni "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/version" weaveapi "github.com/weaveworks/weave/api" "github.com/weaveworks/weave/common" @@ -14,13 +15,13 @@ import ( func cniNet(args []string) error { weave := weaveapi.NewClient(os.Getenv("WEAVE_HTTP_ADDR"), common.Log) n := netplugin.NewCNIPlugin(weave) - cni.PluginMain(n.CmdAdd, n.CmdDel) + cni.PluginMain(n.CmdAdd, n.CmdDel, version.PluginSupports("0.1.0", "0.2.0", "0.3.0")) return nil } func cniIPAM(args []string) error { weave := weaveapi.NewClient(os.Getenv("WEAVE_HTTP_ADDR"), common.Log) i := ipamplugin.NewIpam(weave) - cni.PluginMain(i.CmdAdd, i.CmdDel) + cni.PluginMain(i.CmdAdd, i.CmdDel, version.PluginSupports("0.1.0", "0.2.0", "0.3.0")) return nil } diff --git a/test/830_cni_plugin_test.sh b/test/830_cni_plugin_test.sh index b5d2cd4a..f33485fe 100755 --- a/test/830_cni_plugin_test.sh +++ b/test/830_cni_plugin_test.sh @@ -7,7 +7,7 @@ start_suite "Test CNI plugin" cni_connect() { pid=$(container_pid $1 $2) id=$(docker_on $1 inspect -f '{{.Id}}' $2) - run_on $1 sudo CNI_VERSION=1 CNI_COMMAND=ADD CNI_CONTAINERID=$id CNI_IFNAME=eth0 \ + run_on $1 sudo CNI_COMMAND=ADD CNI_CONTAINERID=$id CNI_IFNAME=eth0 \ CNI_NETNS=/proc/$pid/ns/net CNI_PATH=/opt/cni/bin /opt/cni/bin/weave-net } @@ -30,6 +30,7 @@ EOF cni_connect $HOST1 c2 <