From 30d6639a9829c1415ee85ec26da4c6d6fdca1c6e Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 20 Dec 2016 18:10:36 +0000 Subject: [PATCH 1/7] Refactor: move peer name identifier to DB package --- db/boltdb.go | 4 ++++ ipam/allocator.go | 5 ++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/db/boltdb.go b/db/boltdb.go index 26b4a651..41021f4d 100644 --- a/db/boltdb.go +++ b/db/boltdb.go @@ -23,6 +23,10 @@ var ( topBucket = []byte("top") ) +const ( + NameIdent = "peername" +) + func NewBoltDB(dbPathname string) (*BoltDB, error) { db, err := bolt.Open(dbPathname, 0660, nil) if err != nil { diff --git a/ipam/allocator.go b/ipam/allocator.go index 1a846e8b..fa7ddb26 100644 --- a/ipam/allocator.go +++ b/ipam/allocator.go @@ -920,13 +920,12 @@ func (alloc *Allocator) reportFreeSpace() { // Persistent data const ( ringIdent = "ring" - nameIdent = "peername" ownedIdent = "ownedAddresses" ) func (alloc *Allocator) persistRing() { // It would be better if these two Save operations happened in the same transaction - if err := alloc.db.Save(nameIdent, alloc.ourName); err != nil { + if err := alloc.db.Save(db.NameIdent, alloc.ourName); err != nil { alloc.fatalf("Error persisting ring data: %s", err) return } @@ -938,7 +937,7 @@ func (alloc *Allocator) persistRing() { // Returns true if persisted data is to be used, otherwise false func (alloc *Allocator) loadPersistedData() bool { var checkPeerName mesh.PeerName - nameFound, err := alloc.db.Load(nameIdent, &checkPeerName) + nameFound, err := alloc.db.Load(db.NameIdent, &checkPeerName) if err != nil { alloc.fatalf("Error loading persisted peer name: %s", err) } From d143470d7758ae830f6b6d8159a197f2adb1e2f0 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 11 Jan 2017 12:48:21 +0000 Subject: [PATCH 2/7] Refactor: extract db filename to a constant --- db/boltdb.go | 1 + prog/weaver/main.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/db/boltdb.go b/db/boltdb.go index 41021f4d..f8702e45 100644 --- a/db/boltdb.go +++ b/db/boltdb.go @@ -25,6 +25,7 @@ var ( const ( NameIdent = "peername" + FileName = "data.db" ) func NewBoltDB(dbPathname string) (*BoltDB, error) { diff --git a/prog/weaver/main.go b/prog/weaver/main.go index 2d47a8fb..5c57c7f8 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -248,7 +248,7 @@ func main() { Log.Fatalf("--awsvpc mode is not compatible with the --password option") } - db, err := db.NewBoltDB(dbPrefix + "data.db") + db, err := db.NewBoltDB(dbPrefix + db.FileName) checkFatal(err) defer db.Close() From 70a20b4d261036c01dad38134097ba34afa60461 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 19 Dec 2016 18:10:26 +0000 Subject: [PATCH 3/7] Generate peer name / mac address in Go --- net/mac.go | 38 +++++++++++++++++++++++++++++++++ net/uuid.go | 42 +++++++++++++++++++++++++++++++++++++ prog/weaveutil/main.go | 1 + prog/weaveutil/unique_id.go | 20 ++++++++++++++++++ 4 files changed, 101 insertions(+) create mode 100644 net/mac.go create mode 100644 net/uuid.go create mode 100644 prog/weaveutil/unique_id.go diff --git a/net/mac.go b/net/mac.go new file mode 100644 index 00000000..05864970 --- /dev/null +++ b/net/mac.go @@ -0,0 +1,38 @@ +package net + +import ( + "crypto/rand" + "crypto/sha256" + "net" +) + +func RandomMAC() (net.HardwareAddr, error) { + mac := make([]byte, 6) + if _, err := rand.Read(mac); err != nil { + return nil, err + } + + setUnicastAndLocal(mac) + + return net.HardwareAddr(mac), nil +} + +func MACfromUUID(uuid []byte) net.HardwareAddr { + hash := sha256.New() + // We salt the input just as a precaution to avoid clashes with + // other applications who might have had the bright idea of + // generating MACs in the same way. + hash.Write([]byte("9oBJ0Jmip-")) + hash.Write(uuid) + sum := hash.Sum(nil) + + setUnicastAndLocal(sum) + + return net.HardwareAddr(sum[:6]) +} + +// In the first byte of the MAC, the 'multicast' bit should be +// clear and 'locally administered' bit should be set. +func setUnicastAndLocal(mac []byte) { + mac[0] = (mac[0] & 0xFE) | 0x02 +} diff --git a/net/uuid.go b/net/uuid.go new file mode 100644 index 00000000..f3cd2ebc --- /dev/null +++ b/net/uuid.go @@ -0,0 +1,42 @@ +package net + +import ( + "fmt" + "io/ioutil" + "net" + "os" +) + +func getSystemUUID(hostRoot string) ([]byte, error) { + machineid, err := ioutil.ReadFile(hostRoot + "/etc/machine-id") + if os.IsNotExist(err) { + machineid, _ = ioutil.ReadFile(hostRoot + "/var/lib/dbus/machine-id") + } + uuid, err := ioutil.ReadFile(hostRoot + "/sys/class/dmi/id/product_uuid") + if os.IsNotExist(err) { + uuid, _ = ioutil.ReadFile(hostRoot + "/sys/hypervisor/uuid") + } + if err != nil && !os.IsNotExist(err) { + return nil, err + } + if len(machineid)+len(uuid) == 0 { + return nil, fmt.Errorf("Empty system uuid") + } + return append(machineid, uuid...), nil +} + +// GetSystemPeerName returns an ID derived from concatenated machine-id +// (either systemd or dbus), the system (aka bios) UUID and the +// hypervisor UUID. It is tweaked and formatted to be usable as a mac address +func GetSystemPeerName(hostRoot string) (string, error) { + var mac net.HardwareAddr + if uuid, err := getSystemUUID(hostRoot); err == nil { + mac = MACfromUUID(uuid) + } else { + mac, err = RandomMAC() + if err != nil { + return "", err + } + } + return mac.String(), nil +} diff --git a/prog/weaveutil/main.go b/prog/weaveutil/main.go index 8322a013..10e771f3 100644 --- a/prog/weaveutil/main.go +++ b/prog/weaveutil/main.go @@ -36,6 +36,7 @@ func init() { "cni-ipam": cniIPAM, "expose-nat": exposeNAT, "bridge-ip": bridgeIP, + "unique-id": uniqueID, } } diff --git a/prog/weaveutil/unique_id.go b/prog/weaveutil/unique_id.go new file mode 100644 index 00000000..456c69eb --- /dev/null +++ b/prog/weaveutil/unique_id.go @@ -0,0 +1,20 @@ +package main + +import ( + "fmt" + + weavenet "github.com/weaveworks/weave/net" +) + +func uniqueID(args []string) error { + if len(args) != 1 { + cmdUsage("unique-id", "") + } + hostRoot := args[0] + uid, err := weavenet.GetSystemPeerName(hostRoot) + if err != nil { + return err + } + fmt.Printf(uid) + return nil +} From e03c7b1b7ca03bc3b2bcef8daf42257ebcce47ac Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 20 Dec 2016 18:09:53 +0000 Subject: [PATCH 4/7] Allow opening persistence file read-only --- db/boltdb.go | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/db/boltdb.go b/db/boltdb.go index f8702e45..575f5883 100644 --- a/db/boltdb.go +++ b/db/boltdb.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/gob" "fmt" + "time" "github.com/boltdb/bolt" ) @@ -33,9 +34,29 @@ func NewBoltDB(dbPathname string) (*BoltDB, error) { if err != nil { return nil, fmt.Errorf("[boltDB] Unable to open %s: %s", dbPathname, err) } - // check the persistence version - err = db.Update(func(tx *bolt.Tx) error { + err = db.Update(checkVersion(false)) + return &BoltDB{db: db}, err +} + +func NewBoltDBReadOnly(dbPathname string) (*BoltDB, error) { + options := bolt.Options{Timeout: time.Millisecond * 50, ReadOnly: true} + db, err := bolt.Open(dbPathname, 0660, &options) + if err != nil { + return nil, fmt.Errorf("[boltDB] Unable to open %s: %s", dbPathname, err) + } + err = db.View(checkVersion(true)) + if err != nil { + return nil, fmt.Errorf("[boltDB] Cannot use persistence file %s: %s", dbPathname, err) + } + return &BoltDB{db: db}, nil +} + +func checkVersion(readOnly bool) func(tx *bolt.Tx) error { + return func(tx *bolt.Tx) error { if top := tx.Bucket(topBucket); top == nil { + if readOnly { + return fmt.Errorf("no top bucket") + } top, err := tx.CreateBucket(topBucket) if err != nil { return err @@ -46,13 +67,12 @@ func NewBoltDB(dbPathname string) (*BoltDB, error) { } else { if checkVersion := top.Get(versionIdent); checkVersion != nil { if checkVersion[0] != persistenceVersion[0] { - return fmt.Errorf("[boltDB] Cannot use persistence file %s - version %x", dbPathname, checkVersion) + return fmt.Errorf("mismatched version %x", checkVersion) } } } return nil - }) - return &BoltDB{db: db}, err + } } func (d *BoltDB) Load(ident string, data interface{}) (bool, error) { From ebcb7f61087cb118c40b04e008d308d918e85ae6 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 20 Dec 2016 18:12:24 +0000 Subject: [PATCH 5/7] Use persisted peer name if it matches old-style ID --- Makefile | 2 +- net/uuid.go | 60 +++++++++++++++++++++++++++++++------ prog/weaveutil/unique_id.go | 9 +++--- 3 files changed, 57 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 720901e8..2ea6402b 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,7 @@ testrunner: $(RUNNER_EXE) $(TEST_TLS_EXE) $(WEAVER_EXE) $(WEAVEPROXY_EXE) $(WEAVEUTIL_EXE): common/*.go common/*/*.go net/*.go net/*/*.go $(WEAVER_EXE): router/*.go ipam/*.go ipam/*/*.go db/*.go nameserver/*.go prog/weaver/*.go $(WEAVEPROXY_EXE): proxy/*.go prog/weaveproxy/*.go -$(WEAVEUTIL_EXE): prog/weaveutil/*.go net/*.go plugin/net/*.go plugin/ipam/*.go +$(WEAVEUTIL_EXE): prog/weaveutil/*.go net/*.go plugin/net/*.go plugin/ipam/*.go db/*.go $(SIGPROXY_EXE): prog/sigproxy/*.go $(KUBEPEERS_EXE): prog/kube-peers/*.go $(WEAVENPC_EXE): prog/weave-npc/*.go npc/*.go npc/*/*.go diff --git a/net/uuid.go b/net/uuid.go index f3cd2ebc..09bd7753 100644 --- a/net/uuid.go +++ b/net/uuid.go @@ -1,37 +1,79 @@ package net import ( - "fmt" + "errors" "io/ioutil" "net" "os" + + "github.com/weaveworks/mesh" + "github.com/weaveworks/weave/db" ) +func getOldStyleSystemUUID() ([]byte, error) { + uuid, err := ioutil.ReadFile("/sys/class/dmi/id/product_uuid") + if os.IsNotExist(err) { + uuid, err = ioutil.ReadFile("/sys/hypervisor/uuid") + } + return uuid, err +} + func getSystemUUID(hostRoot string) ([]byte, error) { + uuid, err := getOldStyleSystemUUID() + if err != nil && !os.IsNotExist(err) { + return nil, err + } machineid, err := ioutil.ReadFile(hostRoot + "/etc/machine-id") if os.IsNotExist(err) { - machineid, _ = ioutil.ReadFile(hostRoot + "/var/lib/dbus/machine-id") - } - uuid, err := ioutil.ReadFile(hostRoot + "/sys/class/dmi/id/product_uuid") - if os.IsNotExist(err) { - uuid, _ = ioutil.ReadFile(hostRoot + "/sys/hypervisor/uuid") + machineid, err = ioutil.ReadFile(hostRoot + "/var/lib/dbus/machine-id") } if err != nil && !os.IsNotExist(err) { return nil, err } - if len(machineid)+len(uuid) == 0 { - return nil, fmt.Errorf("Empty system uuid") + if len(uuid) == 0 { + return nil, errors.New("All system IDs are blank") } return append(machineid, uuid...), nil } +func getPersistedPeerName(pathName string) (mesh.PeerName, error) { + d, err := db.NewBoltDBReadOnly(pathName) + if err != nil { + return mesh.UnknownPeerName, err + } + defer d.Close() + var peerName mesh.PeerName + nameFound, err := d.Load(db.NameIdent, &peerName) + if err != nil || !nameFound { + return mesh.UnknownPeerName, err + } + return peerName, nil +} + // GetSystemPeerName returns an ID derived from concatenated machine-id // (either systemd or dbus), the system (aka bios) UUID and the // hypervisor UUID. It is tweaked and formatted to be usable as a mac address -func GetSystemPeerName(hostRoot string) (string, error) { +func GetSystemPeerName(dbPrefix, hostRoot string) (string, error) { + // Check if we have a persisted name that matches the old-style ID for this host + if oldUUID, err := getOldStyleSystemUUID(); err == nil { + pathName := dbPrefix + db.FileName + if _, err := os.Stat(pathName); err == nil { + persistedPeerName, err := getPersistedPeerName(pathName) + if err != nil && !os.IsNotExist(err) { + return "", err + } + if persistedPeerName == mesh.PeerNameFromBin(MACfromUUID(oldUUID)) { + return persistedPeerName.String(), nil + } + } + } else if !os.IsNotExist(err) { + return "", err + } var mac net.HardwareAddr if uuid, err := getSystemUUID(hostRoot); err == nil { mac = MACfromUUID(uuid) + } else if !os.IsNotExist(err) { + return "", err } else { mac, err = RandomMAC() if err != nil { diff --git a/prog/weaveutil/unique_id.go b/prog/weaveutil/unique_id.go index 456c69eb..2f12d71e 100644 --- a/prog/weaveutil/unique_id.go +++ b/prog/weaveutil/unique_id.go @@ -7,11 +7,12 @@ import ( ) func uniqueID(args []string) error { - if len(args) != 1 { - cmdUsage("unique-id", "") + if len(args) != 2 { + cmdUsage("unique-id", " ") } - hostRoot := args[0] - uid, err := weavenet.GetSystemPeerName(hostRoot) + dbPrefix := args[0] + hostRoot := args[1] + uid, err := weavenet.GetSystemPeerName(dbPrefix, hostRoot) if err != nil { return err } From 7d308a4ac57f9f689663f19041c1a7acb096e07f Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 20 Dec 2016 18:13:23 +0000 Subject: [PATCH 6/7] Wire weaveutil unique-id into weave script Also ensure the right host directories are mounted when create-bridge is called from either the weave script or weave-kube. --- prog/weave-kube/launch.sh | 2 +- prog/weave-kube/weave-daemonset.yaml | 5 +++++ site/operational-guide/concepts.md | 20 +++++++++++-------- weave | 29 +++++++++++----------------- 4 files changed, 29 insertions(+), 27 deletions(-) diff --git a/prog/weave-kube/launch.sh b/prog/weave-kube/launch.sh index 34a997e7..080a11b0 100755 --- a/prog/weave-kube/launch.sh +++ b/prog/weave-kube/launch.sh @@ -29,6 +29,7 @@ if [ "${EXPECT_NPC}" = "0" ]; then WEAVE_NPC_OPTS="" fi +export HOST_ROOT=/host /home/weave/weave --local create-bridge --force $WEAVE_NPC_OPTS # Kubernetes sets HOSTNAME to the host's hostname @@ -100,7 +101,6 @@ post_start_actions() { fi fi mkdir -p $HOST_ROOT/etc/cni/net.d - export HOST_ROOT /home/weave/weave --local setup-cni # Expose the weave network so host processes can communicate with pods diff --git a/prog/weave-kube/weave-daemonset.yaml b/prog/weave-kube/weave-daemonset.yaml index 67fd64be..15140679 100644 --- a/prog/weave-kube/weave-daemonset.yaml +++ b/prog/weave-kube/weave-daemonset.yaml @@ -44,6 +44,8 @@ spec: mountPath: /host/home - name: cni-conf mountPath: /host/etc + - name: dbus + mountPath: /host/var/lib/dbus resources: requests: cpu: 10m @@ -68,3 +70,6 @@ spec: - name: cni-conf hostPath: path: /etc + - name: dbus + hostPath: + path: /var/lib/dbus diff --git a/site/operational-guide/concepts.md b/site/operational-guide/concepts.md index c2584120..be306573 100644 --- a/site/operational-guide/concepts.md +++ b/site/operational-guide/concepts.md @@ -49,14 +49,18 @@ When the router is launched on a host, it derives its peer name in order of pref * From the command line, where the user is responsible for uniqueness and stability -* From the BIOS product UUID, which is generally stable across - restarts and unique across different physical hardware and certain - cloned VMs -* From the hypervisor UUID, which is generally stable across restarts - and unique across VMs which do not provide access to a BIOS product - UUID -* From a random value, practically unique across different physical - hardware and cloned VMs but not stable across restarts +* From Weave's data volume container (`weavedb`), if you upgrade your + Weave Net install from a pre-1.9 version and the underlying machine + IDs still match. +* From a unique machine ID. This is generated by combining IDs from + [systemd](https://www.freedesktop.org/software/systemd/man/machine-id.html) + or [D-Bus](https://dbus.freedesktop.org/doc/dbus-uuidgen.1.html) + with the BIOS product UUID or hypervisor UUID. Most machines have + one or two of these IDs, and the combination is practically stable + across restarts unless a user re-generates it. +* If none of the machine IDs are found, a random value is generated. + This is practically unique across different physical hardware and + cloned VMs but not stable across restarts. The appropriate strategy for assigning peer names depends on the type and method of your particular deployment and is discussed in more diff --git a/weave b/weave index ead763ce..f0fffece 100755 --- a/weave +++ b/weave @@ -140,6 +140,11 @@ exec_options() { setup|setup-cni|launch|launch-router) echo -v /:/host -e HOST_ROOT=/host ;; + # All the other commands that may create the bridge and need machine id files. + # We don't mount '/' to avoid recursive mounts of '/var' + create-bridge|attach-router|run|start|attach|restart|expose|hide) + echo -v /etc:/host/etc -v /var/lib/dbus:/host/var/lib/dbus -e HOST_ROOT=/host + ;; esac } @@ -466,14 +471,6 @@ mac_from_hex() { read a b c d e f && printf "%02x:$b:$c:$d:$e:$f" $((0x$a & ~1 | 2)) } -mac_from_file() { - UUID=$(cat $1 2>/dev/null) || return 1 - # We salt the input just as a precaution to avoid clashes with - # other applications who might have had the bright idea of - # generating MACs in the same way. - echo "9oBJ0Jmip-$UUID" | sha256sum | sed 's/\(..\)/\1 /g' | cut -c1-17 | mac_from_hex -} - # Generate a random MAC value random_mac() { od -txC -An -N6 /dev/urandom | mac_from_hex @@ -721,16 +718,12 @@ init_bridge_prep() { ip link add name $BRIDGE type bridge - # Derive the bridge MAC from the system (aka bios) UUID, or, - # failing that, the hypervisor UUID. Elsewhere we in turn derive - # the peer name from that, which we want to be stable across - # reboots but otherwise unique. The system/hypervisor UUID fits - # that bill, unlike, say, /etc/machine-id, which is often - # identical on VMs created from cloned filesystems. If we cannot - # determine the system/hypervisor UUID we just generate a random MAC. - MAC=$(mac_from_file /sys/class/dmi/id/product_uuid) || \ - MAC=$(mac_from_file /sys/hypervisor/uuid) || \ - MAC=$(random_mac) + # for Docker 1.12 this needs to be docker inspect -f '{{with index .Mounts 0}}{{.Source}}{{end}}' + # so running this from 'weave --local' may fail. + # also if weaveexec is not on the path it will be run as a Docker container + # that does not have access to /etc/machine-id + WEAVEDB_PATH=$(docker inspect -f '{{index .Volumes "/weavedb"}}' $DB_CONTAINER_NAME 2>/dev/null) + MAC=$(util_op unique-id "$HOST_ROOT/$WEAVEDB_PATH/weave" "$HOST_ROOT") ip link set dev $BRIDGE address $MAC From e7eeecd762a16e215708b03248d5b5faae488aae Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 22 Dec 2016 18:07:05 +0000 Subject: [PATCH 7/7] Smoke test for peer names --- test/390_peer_name_test.sh | 69 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100755 test/390_peer_name_test.sh diff --git a/test/390_peer_name_test.sh b/test/390_peer_name_test.sh new file mode 100755 index 00000000..ab97e3d9 --- /dev/null +++ b/test/390_peer_name_test.sh @@ -0,0 +1,69 @@ +#! /bin/bash + +. "$(dirname "$0")/config.sh" + +# rot8 the hex digits so if you do this twice you get back the original +rot8() { + tr '[0-9a-f]' '[8-9a-f0-7]' +} + +get_peername() { + weave_on $HOST1 report -f '{{.Router.Name}}' +} + +remove_bridge() { + run_on $HOST1 sudo ip link del weave + run_on $HOST1 sudo ip link del vethwe-datapath + docker_on $HOST1 run --rm --privileged --net=host --entrypoint=/usr/bin/weaveutil weaveworks/weaveexec delete-datapath datapath +} + +start_suite "peer name derivation" + +# Save the original machine ID +ORIG_MACHINE_ID=$($SSH $HOST1 cat /etc/machine-id) +[ -n "$ORIG_MACHINE_ID" ] || echo "Machine ID is blank!" >&2 + +# Blank machine-id to begin, so peername will work the same as pre-1.9 +$SSH $HOST1 sudo cp /dev/null /etc/machine-id + +weave_on $HOST1 launch-router +weave_on $HOST1 prime +PEERNAME1=$(get_peername) + +# Now put the machine ID back +echo $ORIG_MACHINE_ID | $SSH $HOST1 sudo tee /etc/machine-id >/dev/null + +# Stop and restart; it should take the ID from the bridge +weave_on $HOST1 stop-router +weave_on $HOST1 launch-router + +assert "get_peername" "$PEERNAME1" + +# Stop and remove the bridge, then restart; it should take the ID from persistence +weave_on $HOST1 stop-router +remove_bridge +weave_on $HOST1 launch-router + +assert "get_peername" "$PEERNAME1" + +# Now remove everything and restart: should get a different peer name +weave_on $HOST1 reset +weave_on $HOST1 launch-router + +PEERNAME3=$(get_peername) +assert_raises "[ $PEERNAME3 != $PEERNAME1 ]" + +# Change the machine ID +echo $ORIG_MACHINE_ID | rot8 | $SSH $HOST1 sudo tee /etc/machine-id >/dev/null + +# Remove bridge and restart; it should get a different ID +weave_on $HOST1 stop-router +remove_bridge +weave_on $HOST1 launch-router + +assert_raises "[ $(get_peername) != $PEERNAME3 ]" + +# Put the original machine ID back +echo $ORIG_MACHINE_ID | $SSH $HOST1 sudo tee /etc/machine-id >/dev/null + +end_suite