Merge pull request #2475 from weaveworks/issues/2419-withnetns

Add a safer version of WithNetNS*
This commit is contained in:
Adam Harrison authored and GitHub committed 2016-10-26 13:50:34 +01:00
commit a5ff7ab8df
11 files changed
+280 -234

No files matched your search

+1 -151
View File
@@ -1,16 +1,6 @@
package common
import (
"fmt"
"net"
"os"
"strings"
"github.com/vishvananda/netlink"
"github.com/vishvananda/netns"
weavenet "github.com/weaveworks/weave/net"
)
import "strings"
// Assert test is true, panic otherwise
func Assert(test bool) {
@@ -26,143 +16,3 @@ func ErrorMessages(errors []error) string {
}
return strings.Join(result, "\n")
}
type NetDev struct {
Name string
MAC net.HardwareAddr
CIDRs []*net.IPNet
}
// Search the network namespace of a process for interfaces matching a predicate
// Note that the predicate is called while the goroutine is inside the process' netns
func FindNetDevs(processID int, match func(link netlink.Link) bool) ([]NetDev, error) {
var netDevs []NetDev
ns, err := netns.GetFromPid(processID)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
defer ns.Close()
err = weavenet.WithNetNSUnsafe(ns, func() error {
return forEachLink(func(link netlink.Link) error {
if match(link) {
netDev, err := linkToNetDev(link)
if err != nil {
return err
}
netDevs = append(netDevs, netDev)
}
return nil
})
})
return netDevs, err
}
func forEachLink(f func(netlink.Link) error) error {
links, err := netlink.LinkList()
if err != nil {
return err
}
for _, link := range links {
if err := f(link); err != nil {
return err
}
}
return nil
}
func linkToNetDev(link netlink.Link) (NetDev, error) {
addrs, err := netlink.AddrList(link, netlink.FAMILY_V4)
if err != nil {
return NetDev{}, err
}
netDev := NetDev{Name: link.Attrs().Name, MAC: link.Attrs().HardwareAddr}
for _, addr := range addrs {
netDev.CIDRs = append(netDev.CIDRs, addr.IPNet)
}
return netDev, nil
}
// ConnectedToBridgePredicate returns a function which is used to query whether
// a given link is a veth interface which one end is connected to a bridge.
// The returned function should be called from a container network namespace which
// the bridge does NOT belong to.
func ConnectedToBridgePredicate(bridgeName string) (func(link netlink.Link) bool, error) {
indexes := make(map[int]struct{})
// Scan devices in root namespace to find those attached to weave bridge
err := weavenet.WithNetNSLinkByPidUnsafe(1, bridgeName,
func(br netlink.Link) error {
return forEachLink(func(link netlink.Link) error {
if link.Attrs().MasterIndex == br.Attrs().Index {
peerIndex := link.Attrs().ParentIndex
if peerIndex == 0 {
// perhaps running on an older kernel where ParentIndex doesn't work.
// as fall-back, assume the indexes are consecutive
peerIndex = link.Attrs().Index - 1
}
indexes[peerIndex] = struct{}{}
}
return nil
})
})
if err != nil {
return nil, err
}
return func(link netlink.Link) bool {
_, isveth := link.(*netlink.Veth)
_, found := indexes[link.Attrs().Index]
return isveth && found
}, nil
}
func GetNetDevsWithPredicate(processID int, predicate func(link netlink.Link) bool) ([]NetDev, error) {
// Bail out if this process is running in the root namespace
nsToplevel, err := netns.GetFromPid(1)
if err != nil {
return nil, fmt.Errorf("unable to open root namespace: %s", err)
}
defer nsToplevel.Close()
nsContainr, err := netns.GetFromPid(processID)
if err != nil {
// Unable to find a namespace for this process - just return nothing
if os.IsNotExist(err) {
return nil, nil
}
return nil, fmt.Errorf("unable to open process %d namespace: %s", processID, err)
}
defer nsContainr.Close()
if nsToplevel.Equal(nsContainr) {
return nil, nil
}
return FindNetDevs(processID, predicate)
}
// Lookup the weave interface of a container
func GetWeaveNetDevs(processID int) ([]NetDev, error) {
p, err := ConnectedToBridgePredicate("weave")
if err != nil {
return nil, err
}
return GetNetDevsWithPredicate(processID, p)
}
// Get the weave bridge interface.
// NB: Should be called from the root network namespace.
func GetBridgeNetDev(bridgeName string) (NetDev, error) {
link, err := netlink.LinkByName(bridgeName)
if err != nil {
return NetDev{}, err
}
return linkToNetDev(link)
}
+116
View File
@@ -0,0 +1,116 @@
package net
import (
"encoding/json"
"fmt"
"net"
"os"
"strconv"
"github.com/vishvananda/netlink"
"github.com/vishvananda/netns"
)
type Dev struct {
Name string `json:"Name,omitempty"`
MAC net.HardwareAddr `json:"MAC,omitempty"`
CIDRs []*net.IPNet `json:"CIDRs,omitempty"`
}
func LinkToNetDev(link netlink.Link) (Dev, error) {
addrs, err := netlink.AddrList(link, netlink.FAMILY_V4)
if err != nil {
return Dev{}, err
}
netDev := Dev{Name: link.Attrs().Name, MAC: link.Attrs().HardwareAddr}
for _, addr := range addrs {
netDev.CIDRs = append(netDev.CIDRs, addr.IPNet)
}
return netDev, nil
}
// ConnectedToBridgeVethPeerIds returns peer indexes of veth links connected to
// the given bridge. The peer index is used to query from a container netns
// whether the container is connected to the bridge.
func ConnectedToBridgeVethPeerIds(bridgeName string) ([]int, error) {
var ids []int
br, err := netlink.LinkByName(bridgeName)
if err != nil {
return nil, err
}
links, err := netlink.LinkList()
if err != nil {
return nil, err
}
for _, link := range links {
if _, isveth := link.(*netlink.Veth); isveth && link.Attrs().MasterIndex == br.Attrs().Index {
peerID := link.Attrs().ParentIndex
if peerID == 0 {
// perhaps running on an older kernel where ParentIndex doesn't work.
// as fall-back, assume the peers are consecutive
peerID = link.Attrs().Index - 1
}
ids = append(ids, peerID)
}
}
return ids, nil
}
// Lookup the weave interface of a container
func GetWeaveNetDevs(processID int) ([]Dev, error) {
peerIDs, err := ConnectedToBridgeVethPeerIds("weave")
if err != nil {
return nil, err
}
return GetNetDevsByVethPeerIds(processID, peerIDs)
}
func GetNetDevsByVethPeerIds(processID int, peerIDs []int) ([]Dev, error) {
// Bail out if this container is running in the root namespace
netnsRoot, err := netns.GetFromPid(1)
if err != nil {
return nil, fmt.Errorf("unable to open root namespace: %s", err)
}
defer netnsRoot.Close()
netnsContainer, err := netns.GetFromPid(processID)
if err != nil {
// Unable to find a namespace for this process - just return nothing
if os.IsNotExist(err) {
return nil, nil
}
return nil, fmt.Errorf("unable to open process %d namespace: %s", processID, err)
}
defer netnsContainer.Close()
if netnsRoot.Equal(netnsContainer) {
return nil, nil
}
var netdevs []Dev
peersStr := make([]string, len(peerIDs))
for i, id := range peerIDs {
peersStr[i] = strconv.Itoa(id)
}
nds, err := WithNetNSByPid(processID, "list-netdevs", peersStr...)
if err != nil {
return nil, fmt.Errorf("list-netdevs failed: %s", err)
}
err = json.Unmarshal(nds, &netdevs)
return netdevs, err
}
// Get the weave bridge interface.
// NB: Should be called from the root network namespace.
func GetBridgeNetDev(bridgeName string) (Dev, error) {
link, err := netlink.LinkByName(bridgeName)
if err != nil {
return Dev{}, err
}
return LinkToNetDev(link)
}
+4 -14
View File
@@ -57,19 +57,9 @@ func WithNetNSLinkUnsafe(ns netns.NsHandle, ifName string, work func(link netlin
})
}
func WithNetNSLinkByPidUnsafe(pid int, ifName string, work func(link netlink.Link) error) error {
ns, err := netns.GetFromPid(pid)
if err != nil {
return err
}
defer ns.Close()
return WithNetNSLinkUnsafe(ns, ifName, work)
}
// A safe version of WithNetNS* which creates a process executing
// "nsenter --net=<ns-path> weaveutil <cmd> [args]".
func WithNetNS(nsPath string, cmd string, args ...string) (string, error) {
func WithNetNS(nsPath string, cmd string, args ...string) ([]byte, error) {
var stdout, stderr bytes.Buffer
args = append([]string{"--net=" + nsPath, "weaveutil", cmd}, args...)
@@ -77,13 +67,13 @@ func WithNetNS(nsPath string, cmd string, args ...string) (string, error) {
c.Stdout = &stdout
c.Stderr = &stderr
if err := c.Run(); err != nil {
return "", fmt.Errorf("%s: %s", string(stderr.Bytes()), err)
return nil, fmt.Errorf("%s: %s", string(stderr.Bytes()), err)
}
return string(stdout.Bytes()), nil
return stdout.Bytes(), nil
}
func WithNetNSByPid(pid int, cmd string, args ...string) (string, error) {
func WithNetNSByPid(pid int, cmd string, args ...string) ([]byte, error) {
return WithNetNS(NSPathByPid(pid), cmd, args...)
}
+22 -20
View File
@@ -105,21 +105,26 @@ const (
vethPrefix = "v" + VethName // starts with "veth" to suppress UI notifications
)
func interfaceExistsInNamespace(ns netns.NsHandle, ifName string) bool {
err := WithNetNSUnsafe(ns, func() error {
_, err := netlink.LinkByName(ifName)
return err
})
func interfaceExistsInNamespace(netNSPath string, ifName string) bool {
_, err := WithNetNS(netNSPath, "check-iface", ifName)
return err == nil
}
func AttachContainer(ns netns.NsHandle, id, ifName, bridgeName string, mtu int, withMulticastRoute bool, cidrs []*net.IPNet, keepTXOn bool) error {
// NB: This function can be used only by a process that terminates immediately
// after calling the function as it changes netns via WithNetNSLinkUnsafe.
func AttachContainer(netNSPath, id, ifName, bridgeName string, mtu int, withMulticastRoute bool, cidrs []*net.IPNet, keepTXOn bool) error {
ns, err := netns.GetFromPath(netNSPath)
if err != nil {
return err
}
defer ns.Close()
ipt, err := iptables.New()
if err != nil {
return err
}
if !interfaceExistsInNamespace(ns, ifName) {
if !interfaceExistsInNamespace(netNSPath, ifName) {
maxIDLen := IFNAMSIZ - 1 - len(vethPrefix+"pl")
if len(id) > maxIDLen {
id = id[:maxIDLen] // trim passed ID if too long
@@ -129,18 +134,7 @@ func AttachContainer(ns netns.NsHandle, id, ifName, bridgeName string, mtu int,
if err := netlink.LinkSetNsFd(veth, int(ns)); err != nil {
return fmt.Errorf("failed to move veth to container netns: %s", err)
}
if err := WithNetNSUnsafe(ns, func() error {
if err := netlink.LinkSetName(veth, ifName); err != nil {
return err
}
if err := ConfigureARPCache(ifName); err != nil {
return err
}
if err := ipt.Append("filter", "INPUT", "-i", ifName, "-d", "224.0.0.0/4", "-j", "DROP"); err != nil {
return err
}
return nil
}); err != nil {
if _, err := WithNetNS(netNSPath, "setup-iface", peerName, ifName); err != nil {
return fmt.Errorf("error setting up interface: %s", err)
}
return nil
@@ -201,7 +195,15 @@ func AttachContainer(ns netns.NsHandle, id, ifName, bridgeName string, mtu int,
return nil
}
func DetachContainer(ns netns.NsHandle, id, ifName string, cidrs []*net.IPNet) error {
// NB: This function can be used only by a process that terminates immediately
// after calling the function as it changes netns via WithNetNSLinkUnsafe.
func DetachContainer(netNSPath, id, ifName string, cidrs []*net.IPNet) error {
ns, err := netns.GetFromPath(netNSPath)
if err != nil {
return err
}
defer ns.Close()
ipt, err := iptables.New()
if err != nil {
return err
+3 -16
View File
@@ -12,7 +12,6 @@ import (
"github.com/vishvananda/netlink"
"github.com/vishvananda/netns"
weaveapi "github.com/weaveworks/weave/api"
"github.com/weaveworks/weave/common"
weavenet "github.com/weaveworks/weave/net"
ipamplugin "github.com/weaveworks/weave/plugin/ipam"
)
@@ -107,7 +106,7 @@ func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error {
id = fmt.Sprintf("%x", data)
}
if err := weavenet.AttachContainer(ns, id, args.IfName, conf.BrName, conf.MTU, false, []*net.IPNet{&result.IP4.IP}, false); err != nil {
if err := weavenet.AttachContainer(args.Netns, id, args.IfName, conf.BrName, conf.MTU, false, []*net.IPNet{&result.IP4.IP}, false); err != nil {
return err
}
if err := weavenet.WithNetNSLinkUnsafe(ns, args.IfName, func(link netlink.Link) error {
@@ -159,7 +158,7 @@ func assignBridgeIP(bridgeName string, ipnet net.IPNet) error {
var errBridgeNoIP = fmt.Errorf("Bridge has no IP address")
func findBridgeIP(bridgeName string, subnet net.IPNet) (net.IP, error) {
netdev, err := common.GetBridgeNetDev(bridgeName)
netdev, err := weavenet.GetBridgeNetDev(bridgeName)
if err != nil {
return nil, fmt.Errorf("Failed to get netdev for %q bridge: %s", bridgeName, err)
}
@@ -181,19 +180,7 @@ func (c *CNIPlugin) CmdDel(args *skel.CmdArgs) error {
return err
}
ns, err := netns.GetFromPath(args.Netns)
if err != nil {
return fmt.Errorf("error accessing namespace %q: %s", args.Netns, err)
}
defer ns.Close()
err = weavenet.WithNetNSUnsafe(ns, func() error {
link, err := netlink.LinkByName(args.IfName)
if err != nil {
return err
}
return netlink.LinkDel(link)
})
if err != nil {
if _, err = weavenet.WithNetNS(args.Netns, "del-iface", args.IfName); err != nil {
return fmt.Errorf("error removing interface %q: %s", args.IfName, err)
}
+7 -12
View File
@@ -4,9 +4,7 @@ import (
"fmt"
"github.com/fsouza/go-dockerclient"
"github.com/vishvananda/netlink"
"github.com/weaveworks/weave/common"
weavenet "github.com/weaveworks/weave/net"
)
@@ -21,7 +19,7 @@ func containerAddrs(args []string) error {
return err
}
pred, err := common.ConnectedToBridgePredicate(bridgeName)
peerIDs, err := weavenet.ConnectedToBridgeVethPeerIds(bridgeName)
if err != nil {
if err == weavenet.ErrLinkNotFound {
return nil
@@ -34,11 +32,11 @@ func containerAddrs(args []string) error {
for _, cid := range args[1:] {
if cid == "weave:expose" {
netDev, err := common.GetBridgeNetDev(bridgeName)
netDev, err := weavenet.GetBridgeNetDev(bridgeName)
if err != nil {
return err
}
printNetDevs(cid, []common.NetDev{netDev})
printNetDevs(cid, []weavenet.Dev{netDev})
continue
}
if containers[cid], err = client.InspectContainer(cid); err != nil {
@@ -51,11 +49,8 @@ func containerAddrs(args []string) error {
containerIDs = append(containerIDs, cid)
}
// NB: Because network namespaces (netns) are changed many times inside the loop,
// it's NOT safe to exec any code depending on the root netns without
// wrapping with WithNetNS*.
for _, cid := range containerIDs {
netDevs, err := getNetDevs(client, containers[cid], pred)
netDevs, err := getNetDevs(client, containers[cid], peerIDs)
if err != nil {
return err
}
@@ -65,14 +60,14 @@ func containerAddrs(args []string) error {
return nil
}
func getNetDevs(c *docker.Client, container *docker.Container, pred func(netlink.Link) bool) ([]common.NetDev, error) {
func getNetDevs(c *docker.Client, container *docker.Container, peerIDs []int) ([]weavenet.Dev, error) {
if container.State.Pid == 0 {
return nil, nil
}
return common.GetNetDevsWithPredicate(container.State.Pid, pred)
return weavenet.GetNetDevsByVethPeerIds(container.State.Pid, peerIDs)
}
func printNetDevs(cid string, netDevs []common.NetDev) {
func printNetDevs(cid string, netDevs []weavenet.Dev) {
for _, netDev := range netDevs {
fmt.Printf("%12s %s %s", cid, netDev.Name, netDev.MAC.String())
for _, cidr := range netDev.CIDRs {
+14 -13
View File
@@ -32,10 +32,15 @@ func attach(args []string) error {
}
}
pid, nsContainer, err := containerPidAndNs(args[0])
pid, err := containerPid(args[0])
if err != nil {
return err
}
nsContainer, err := netns.GetFromPid(pid)
if err != nil {
return fmt.Errorf("unable to open namespace for container %s: %s", args[0], err)
}
if nsHost, err := netns.GetFromPid(1); err != nil {
return fmt.Errorf("unable to open host namespace: %s", err)
} else if nsHost.Equal(nsContainer) {
@@ -50,7 +55,7 @@ func attach(args []string) error {
return err
}
err = weavenet.AttachContainer(nsContainer, fmt.Sprint(pid), weavenet.VethName, args[1], mtu, withMulticastRoute, cidrs, keepTXOn)
err = weavenet.AttachContainer(weavenet.NSPathByPid(pid), fmt.Sprint(pid), weavenet.VethName, args[1], mtu, withMulticastRoute, cidrs, keepTXOn)
// If we detected an error but the container has died, tell the user that instead.
if err != nil && !processExists(pid) {
err = fmt.Errorf("Container %s died", args[0])
@@ -58,23 +63,19 @@ func attach(args []string) error {
return err
}
func containerPidAndNs(containerID string) (int, netns.NsHandle, error) {
func containerPid(containerID string) (int, error) {
c, err := docker.NewVersionedClientFromEnv("1.18")
if err != nil {
return 0, 0, fmt.Errorf("unable to connect to docker: %s", err)
return 0, fmt.Errorf("unable to connect to docker: %s", err)
}
container, err := c.InspectContainer(containerID)
if err != nil {
return 0, 0, fmt.Errorf("unable to inspect container %s: %s", containerID, err)
return 0, fmt.Errorf("unable to inspect container %s: %s", containerID, err)
}
if container.State.Pid == 0 {
return 0, 0, fmt.Errorf("container %s not running", containerID)
return 0, fmt.Errorf("container %s not running", containerID)
}
ns, err := netns.GetFromPid(container.State.Pid)
if err != nil {
return 0, 0, fmt.Errorf("unable to open namespace for container %s: %s", containerID, err)
}
return container.State.Pid, ns, nil
return container.State.Pid, nil
}
func processExists(pid int) bool {
@@ -99,7 +100,7 @@ func detach(args []string) error {
cmdUsage("detach-container", "<container-id> <cidr>...")
}
_, ns, err := containerPidAndNs(args[0])
pid, err := containerPid(args[0])
if err != nil {
return err
}
@@ -107,5 +108,5 @@ func detach(args []string) error {
if err != nil {
return err
}
return weavenet.DetachContainer(ns, args[0], weavenet.VethName, cidrs)
return weavenet.DetachContainer(weavenet.NSPathByPid(pid), args[0], weavenet.VethName, cidrs)
}
+4
View File
@@ -24,6 +24,10 @@ func init() {
"attach-container": attach,
"detach-container": detach,
"configure-arp": configureARP,
"check-iface": checkIface,
"del-iface": delIface,
"setup-iface": setupIface,
"list-netdevs": listNetDevs,
}
}
+105
View File
@@ -1,14 +1,75 @@
package main
import (
"encoding/json"
"fmt"
"strconv"
"strings"
"github.com/coreos/go-iptables/iptables"
"github.com/j-keck/arping"
"github.com/vishvananda/netlink"
weavenet "github.com/weaveworks/weave/net"
)
// checkIface returns an error if the given interface cannot be found.
func checkIface(args []string) error {
if len(args) != 1 {
cmdUsage("check-iface", "<iface-name>")
}
ifaceName := args[0]
if _, err := netlink.LinkByName(ifaceName); err != nil {
return err
}
return nil
}
func delIface(args []string) error {
if len(args) != 1 {
cmdUsage("del-iface", "<iface-name>")
}
ifName := args[0]
link, err := netlink.LinkByName(ifName)
if err != nil {
return err
}
return netlink.LinkDel(link)
}
// setupIface renames the given iface and configures its ARP cache settings.
func setupIface(args []string) error {
if len(args) != 2 {
cmdUsage("setup-iface", "<iface-name> <new-iface-name>")
}
ifaceName := args[0]
newIfName := args[1]
ipt, err := iptables.New()
if err != nil {
return err
}
link, err := netlink.LinkByName(ifaceName)
if err != nil {
return err
}
if err := netlink.LinkSetName(link, newIfName); err != nil {
return err
}
if err := weavenet.ConfigureARPCache(newIfName); err != nil {
return err
}
if err := ipt.Append("filter", "INPUT", "-i", newIfName, "-d", "224.0.0.0/4", "-j", "DROP"); err != nil {
return err
}
return nil
}
func configureARP(args []string) error {
if len(args) != 1 {
cmdUsage("configure-arp", "<iface-name-prefix>")
@@ -33,3 +94,47 @@ func configureARP(args []string) error {
return nil
}
// listNetDevs outputs network ifaces identified by the given indexes
// in the format of weavenet.Dev.
func listNetDevs(args []string) error {
if len(args) == 0 {
cmdUsage("list-netdevs", "<iface-index>[ <iface-index>]")
}
indexes := make(map[int]struct{})
for _, index := range args {
if index != "" {
id, err := strconv.Atoi(index)
if err != nil {
return err
}
indexes[id] = struct{}{}
}
}
links, err := netlink.LinkList()
if err != nil {
return err
}
var netdevs []weavenet.Dev
for _, link := range links {
if _, found := indexes[link.Attrs().Index]; found {
netdev, err := weavenet.LinkToNetDev(link)
if err != nil {
return err
}
netdevs = append(netdevs, netdev)
}
}
nds, err := json.Marshal(netdevs)
if err != nil {
return err
}
fmt.Println(string(nds))
return nil
}
+2 -5
View File
@@ -13,7 +13,7 @@ func processAddrs(args []string) error {
}
bridgeName := args[0]
pred, err := common.ConnectedToBridgePredicate(bridgeName)
peerIDs, err := weavenet.ConnectedToBridgeVethPeerIds(bridgeName)
if err != nil {
if err == weavenet.ErrLinkNotFound {
return nil
@@ -26,11 +26,8 @@ func processAddrs(args []string) error {
return err
}
// NB: Because network namespaces (netns) are changed many times inside the loop,
// it's NOT safe to exec any code depending on the root netns without
// wrapping with WithNetNS*.
for _, pid := range pids {
netDevs, err := common.GetNetDevsWithPredicate(pid, pred)
netDevs, err := weavenet.GetNetDevsByVethPeerIds(pid, peerIDs)
if err != nil {
return err
}
+2 -3
View File
@@ -17,7 +17,6 @@ import (
docker "github.com/fsouza/go-dockerclient"
weaveapi "github.com/weaveworks/weave/api"
"github.com/weaveworks/weave/common"
weavedocker "github.com/weaveworks/weave/common/docker"
weavenet "github.com/weaveworks/weave/net"
"github.com/weaveworks/weave/net/address"
@@ -149,7 +148,7 @@ func NewProxy(c Config) (*Proxy, error) {
p.client = client.Client
if !p.WithoutDNS {
netDev, err := common.GetBridgeNetDev(c.DockerBridge)
netDev, err := weavenet.GetBridgeNetDev(c.DockerBridge)
if err != nil {
return nil, err
}
@@ -625,7 +624,7 @@ func (proxy *Proxy) updateContainerNetworkSettings(container jsonObject) error {
if err := proxy.waitForStartByIdent(containerID); err != nil {
return err
}
netDevs, err := common.GetWeaveNetDevs(pid)
netDevs, err := weavenet.GetWeaveNetDevs(pid)
if err != nil || len(netDevs) == 0 || len(netDevs[0].CIDRs) == 0 {
return err
}