diff --git a/Makefile b/Makefile index 720901e8..2ea6402b 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,7 @@ testrunner: $(RUNNER_EXE) $(TEST_TLS_EXE) $(WEAVER_EXE) $(WEAVEPROXY_EXE) $(WEAVEUTIL_EXE): common/*.go common/*/*.go net/*.go net/*/*.go $(WEAVER_EXE): router/*.go ipam/*.go ipam/*/*.go db/*.go nameserver/*.go prog/weaver/*.go $(WEAVEPROXY_EXE): proxy/*.go prog/weaveproxy/*.go -$(WEAVEUTIL_EXE): prog/weaveutil/*.go net/*.go plugin/net/*.go plugin/ipam/*.go +$(WEAVEUTIL_EXE): prog/weaveutil/*.go net/*.go plugin/net/*.go plugin/ipam/*.go db/*.go $(SIGPROXY_EXE): prog/sigproxy/*.go $(KUBEPEERS_EXE): prog/kube-peers/*.go $(WEAVENPC_EXE): prog/weave-npc/*.go npc/*.go npc/*/*.go diff --git a/db/boltdb.go b/db/boltdb.go index 26b4a651..575f5883 100644 --- a/db/boltdb.go +++ b/db/boltdb.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/gob" "fmt" + "time" "github.com/boltdb/bolt" ) @@ -23,14 +24,39 @@ var ( topBucket = []byte("top") ) +const ( + NameIdent = "peername" + FileName = "data.db" +) + func NewBoltDB(dbPathname string) (*BoltDB, error) { db, err := bolt.Open(dbPathname, 0660, nil) if err != nil { return nil, fmt.Errorf("[boltDB] Unable to open %s: %s", dbPathname, err) } - // check the persistence version - err = db.Update(func(tx *bolt.Tx) error { + err = db.Update(checkVersion(false)) + return &BoltDB{db: db}, err +} + +func NewBoltDBReadOnly(dbPathname string) (*BoltDB, error) { + options := bolt.Options{Timeout: time.Millisecond * 50, ReadOnly: true} + db, err := bolt.Open(dbPathname, 0660, &options) + if err != nil { + return nil, fmt.Errorf("[boltDB] Unable to open %s: %s", dbPathname, err) + } + err = db.View(checkVersion(true)) + if err != nil { + return nil, fmt.Errorf("[boltDB] Cannot use persistence file %s: %s", dbPathname, err) + } + return &BoltDB{db: db}, nil +} + +func checkVersion(readOnly bool) func(tx *bolt.Tx) error { + return func(tx *bolt.Tx) error { if top := tx.Bucket(topBucket); top == nil { + if readOnly { + return fmt.Errorf("no top bucket") + } top, err := tx.CreateBucket(topBucket) if err != nil { return err @@ -41,13 +67,12 @@ func NewBoltDB(dbPathname string) (*BoltDB, error) { } else { if checkVersion := top.Get(versionIdent); checkVersion != nil { if checkVersion[0] != persistenceVersion[0] { - return fmt.Errorf("[boltDB] Cannot use persistence file %s - version %x", dbPathname, checkVersion) + return fmt.Errorf("mismatched version %x", checkVersion) } } } return nil - }) - return &BoltDB{db: db}, err + } } func (d *BoltDB) Load(ident string, data interface{}) (bool, error) { diff --git a/ipam/allocator.go b/ipam/allocator.go index 1a846e8b..fa7ddb26 100644 --- a/ipam/allocator.go +++ b/ipam/allocator.go @@ -920,13 +920,12 @@ func (alloc *Allocator) reportFreeSpace() { // Persistent data const ( ringIdent = "ring" - nameIdent = "peername" ownedIdent = "ownedAddresses" ) func (alloc *Allocator) persistRing() { // It would be better if these two Save operations happened in the same transaction - if err := alloc.db.Save(nameIdent, alloc.ourName); err != nil { + if err := alloc.db.Save(db.NameIdent, alloc.ourName); err != nil { alloc.fatalf("Error persisting ring data: %s", err) return } @@ -938,7 +937,7 @@ func (alloc *Allocator) persistRing() { // Returns true if persisted data is to be used, otherwise false func (alloc *Allocator) loadPersistedData() bool { var checkPeerName mesh.PeerName - nameFound, err := alloc.db.Load(nameIdent, &checkPeerName) + nameFound, err := alloc.db.Load(db.NameIdent, &checkPeerName) if err != nil { alloc.fatalf("Error loading persisted peer name: %s", err) } diff --git a/net/mac.go b/net/mac.go new file mode 100644 index 00000000..05864970 --- /dev/null +++ b/net/mac.go @@ -0,0 +1,38 @@ +package net + +import ( + "crypto/rand" + "crypto/sha256" + "net" +) + +func RandomMAC() (net.HardwareAddr, error) { + mac := make([]byte, 6) + if _, err := rand.Read(mac); err != nil { + return nil, err + } + + setUnicastAndLocal(mac) + + return net.HardwareAddr(mac), nil +} + +func MACfromUUID(uuid []byte) net.HardwareAddr { + hash := sha256.New() + // We salt the input just as a precaution to avoid clashes with + // other applications who might have had the bright idea of + // generating MACs in the same way. + hash.Write([]byte("9oBJ0Jmip-")) + hash.Write(uuid) + sum := hash.Sum(nil) + + setUnicastAndLocal(sum) + + return net.HardwareAddr(sum[:6]) +} + +// In the first byte of the MAC, the 'multicast' bit should be +// clear and 'locally administered' bit should be set. +func setUnicastAndLocal(mac []byte) { + mac[0] = (mac[0] & 0xFE) | 0x02 +} diff --git a/net/uuid.go b/net/uuid.go new file mode 100644 index 00000000..09bd7753 --- /dev/null +++ b/net/uuid.go @@ -0,0 +1,84 @@ +package net + +import ( + "errors" + "io/ioutil" + "net" + "os" + + "github.com/weaveworks/mesh" + "github.com/weaveworks/weave/db" +) + +func getOldStyleSystemUUID() ([]byte, error) { + uuid, err := ioutil.ReadFile("/sys/class/dmi/id/product_uuid") + if os.IsNotExist(err) { + uuid, err = ioutil.ReadFile("/sys/hypervisor/uuid") + } + return uuid, err +} + +func getSystemUUID(hostRoot string) ([]byte, error) { + uuid, err := getOldStyleSystemUUID() + if err != nil && !os.IsNotExist(err) { + return nil, err + } + machineid, err := ioutil.ReadFile(hostRoot + "/etc/machine-id") + if os.IsNotExist(err) { + machineid, err = ioutil.ReadFile(hostRoot + "/var/lib/dbus/machine-id") + } + if err != nil && !os.IsNotExist(err) { + return nil, err + } + if len(uuid) == 0 { + return nil, errors.New("All system IDs are blank") + } + return append(machineid, uuid...), nil +} + +func getPersistedPeerName(pathName string) (mesh.PeerName, error) { + d, err := db.NewBoltDBReadOnly(pathName) + if err != nil { + return mesh.UnknownPeerName, err + } + defer d.Close() + var peerName mesh.PeerName + nameFound, err := d.Load(db.NameIdent, &peerName) + if err != nil || !nameFound { + return mesh.UnknownPeerName, err + } + return peerName, nil +} + +// GetSystemPeerName returns an ID derived from concatenated machine-id +// (either systemd or dbus), the system (aka bios) UUID and the +// hypervisor UUID. It is tweaked and formatted to be usable as a mac address +func GetSystemPeerName(dbPrefix, hostRoot string) (string, error) { + // Check if we have a persisted name that matches the old-style ID for this host + if oldUUID, err := getOldStyleSystemUUID(); err == nil { + pathName := dbPrefix + db.FileName + if _, err := os.Stat(pathName); err == nil { + persistedPeerName, err := getPersistedPeerName(pathName) + if err != nil && !os.IsNotExist(err) { + return "", err + } + if persistedPeerName == mesh.PeerNameFromBin(MACfromUUID(oldUUID)) { + return persistedPeerName.String(), nil + } + } + } else if !os.IsNotExist(err) { + return "", err + } + var mac net.HardwareAddr + if uuid, err := getSystemUUID(hostRoot); err == nil { + mac = MACfromUUID(uuid) + } else if !os.IsNotExist(err) { + return "", err + } else { + mac, err = RandomMAC() + if err != nil { + return "", err + } + } + return mac.String(), nil +} diff --git a/prog/weave-kube/launch.sh b/prog/weave-kube/launch.sh index 34a997e7..080a11b0 100755 --- a/prog/weave-kube/launch.sh +++ b/prog/weave-kube/launch.sh @@ -29,6 +29,7 @@ if [ "${EXPECT_NPC}" = "0" ]; then WEAVE_NPC_OPTS="" fi +export HOST_ROOT=/host /home/weave/weave --local create-bridge --force $WEAVE_NPC_OPTS # Kubernetes sets HOSTNAME to the host's hostname @@ -100,7 +101,6 @@ post_start_actions() { fi fi mkdir -p $HOST_ROOT/etc/cni/net.d - export HOST_ROOT /home/weave/weave --local setup-cni # Expose the weave network so host processes can communicate with pods diff --git a/prog/weave-kube/weave-daemonset.yaml b/prog/weave-kube/weave-daemonset.yaml index 67fd64be..15140679 100644 --- a/prog/weave-kube/weave-daemonset.yaml +++ b/prog/weave-kube/weave-daemonset.yaml @@ -44,6 +44,8 @@ spec: mountPath: /host/home - name: cni-conf mountPath: /host/etc + - name: dbus + mountPath: /host/var/lib/dbus resources: requests: cpu: 10m @@ -68,3 +70,6 @@ spec: - name: cni-conf hostPath: path: /etc + - name: dbus + hostPath: + path: /var/lib/dbus diff --git a/prog/weaver/main.go b/prog/weaver/main.go index 2d47a8fb..5c57c7f8 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -248,7 +248,7 @@ func main() { Log.Fatalf("--awsvpc mode is not compatible with the --password option") } - db, err := db.NewBoltDB(dbPrefix + "data.db") + db, err := db.NewBoltDB(dbPrefix + db.FileName) checkFatal(err) defer db.Close() diff --git a/prog/weaveutil/main.go b/prog/weaveutil/main.go index 8322a013..10e771f3 100644 --- a/prog/weaveutil/main.go +++ b/prog/weaveutil/main.go @@ -36,6 +36,7 @@ func init() { "cni-ipam": cniIPAM, "expose-nat": exposeNAT, "bridge-ip": bridgeIP, + "unique-id": uniqueID, } } diff --git a/prog/weaveutil/unique_id.go b/prog/weaveutil/unique_id.go new file mode 100644 index 00000000..2f12d71e --- /dev/null +++ b/prog/weaveutil/unique_id.go @@ -0,0 +1,21 @@ +package main + +import ( + "fmt" + + weavenet "github.com/weaveworks/weave/net" +) + +func uniqueID(args []string) error { + if len(args) != 2 { + cmdUsage("unique-id", " ") + } + dbPrefix := args[0] + hostRoot := args[1] + uid, err := weavenet.GetSystemPeerName(dbPrefix, hostRoot) + if err != nil { + return err + } + fmt.Printf(uid) + return nil +} diff --git a/site/operational-guide/concepts.md b/site/operational-guide/concepts.md index c2584120..be306573 100644 --- a/site/operational-guide/concepts.md +++ b/site/operational-guide/concepts.md @@ -49,14 +49,18 @@ When the router is launched on a host, it derives its peer name in order of pref * From the command line, where the user is responsible for uniqueness and stability -* From the BIOS product UUID, which is generally stable across - restarts and unique across different physical hardware and certain - cloned VMs -* From the hypervisor UUID, which is generally stable across restarts - and unique across VMs which do not provide access to a BIOS product - UUID -* From a random value, practically unique across different physical - hardware and cloned VMs but not stable across restarts +* From Weave's data volume container (`weavedb`), if you upgrade your + Weave Net install from a pre-1.9 version and the underlying machine + IDs still match. +* From a unique machine ID. This is generated by combining IDs from + [systemd](https://www.freedesktop.org/software/systemd/man/machine-id.html) + or [D-Bus](https://dbus.freedesktop.org/doc/dbus-uuidgen.1.html) + with the BIOS product UUID or hypervisor UUID. Most machines have + one or two of these IDs, and the combination is practically stable + across restarts unless a user re-generates it. +* If none of the machine IDs are found, a random value is generated. + This is practically unique across different physical hardware and + cloned VMs but not stable across restarts. The appropriate strategy for assigning peer names depends on the type and method of your particular deployment and is discussed in more diff --git a/test/390_peer_name_test.sh b/test/390_peer_name_test.sh new file mode 100755 index 00000000..ab97e3d9 --- /dev/null +++ b/test/390_peer_name_test.sh @@ -0,0 +1,69 @@ +#! /bin/bash + +. "$(dirname "$0")/config.sh" + +# rot8 the hex digits so if you do this twice you get back the original +rot8() { + tr '[0-9a-f]' '[8-9a-f0-7]' +} + +get_peername() { + weave_on $HOST1 report -f '{{.Router.Name}}' +} + +remove_bridge() { + run_on $HOST1 sudo ip link del weave + run_on $HOST1 sudo ip link del vethwe-datapath + docker_on $HOST1 run --rm --privileged --net=host --entrypoint=/usr/bin/weaveutil weaveworks/weaveexec delete-datapath datapath +} + +start_suite "peer name derivation" + +# Save the original machine ID +ORIG_MACHINE_ID=$($SSH $HOST1 cat /etc/machine-id) +[ -n "$ORIG_MACHINE_ID" ] || echo "Machine ID is blank!" >&2 + +# Blank machine-id to begin, so peername will work the same as pre-1.9 +$SSH $HOST1 sudo cp /dev/null /etc/machine-id + +weave_on $HOST1 launch-router +weave_on $HOST1 prime +PEERNAME1=$(get_peername) + +# Now put the machine ID back +echo $ORIG_MACHINE_ID | $SSH $HOST1 sudo tee /etc/machine-id >/dev/null + +# Stop and restart; it should take the ID from the bridge +weave_on $HOST1 stop-router +weave_on $HOST1 launch-router + +assert "get_peername" "$PEERNAME1" + +# Stop and remove the bridge, then restart; it should take the ID from persistence +weave_on $HOST1 stop-router +remove_bridge +weave_on $HOST1 launch-router + +assert "get_peername" "$PEERNAME1" + +# Now remove everything and restart: should get a different peer name +weave_on $HOST1 reset +weave_on $HOST1 launch-router + +PEERNAME3=$(get_peername) +assert_raises "[ $PEERNAME3 != $PEERNAME1 ]" + +# Change the machine ID +echo $ORIG_MACHINE_ID | rot8 | $SSH $HOST1 sudo tee /etc/machine-id >/dev/null + +# Remove bridge and restart; it should get a different ID +weave_on $HOST1 stop-router +remove_bridge +weave_on $HOST1 launch-router + +assert_raises "[ $(get_peername) != $PEERNAME3 ]" + +# Put the original machine ID back +echo $ORIG_MACHINE_ID | $SSH $HOST1 sudo tee /etc/machine-id >/dev/null + +end_suite diff --git a/weave b/weave index ead763ce..f0fffece 100755 --- a/weave +++ b/weave @@ -140,6 +140,11 @@ exec_options() { setup|setup-cni|launch|launch-router) echo -v /:/host -e HOST_ROOT=/host ;; + # All the other commands that may create the bridge and need machine id files. + # We don't mount '/' to avoid recursive mounts of '/var' + create-bridge|attach-router|run|start|attach|restart|expose|hide) + echo -v /etc:/host/etc -v /var/lib/dbus:/host/var/lib/dbus -e HOST_ROOT=/host + ;; esac } @@ -466,14 +471,6 @@ mac_from_hex() { read a b c d e f && printf "%02x:$b:$c:$d:$e:$f" $((0x$a & ~1 | 2)) } -mac_from_file() { - UUID=$(cat $1 2>/dev/null) || return 1 - # We salt the input just as a precaution to avoid clashes with - # other applications who might have had the bright idea of - # generating MACs in the same way. - echo "9oBJ0Jmip-$UUID" | sha256sum | sed 's/\(..\)/\1 /g' | cut -c1-17 | mac_from_hex -} - # Generate a random MAC value random_mac() { od -txC -An -N6 /dev/urandom | mac_from_hex @@ -721,16 +718,12 @@ init_bridge_prep() { ip link add name $BRIDGE type bridge - # Derive the bridge MAC from the system (aka bios) UUID, or, - # failing that, the hypervisor UUID. Elsewhere we in turn derive - # the peer name from that, which we want to be stable across - # reboots but otherwise unique. The system/hypervisor UUID fits - # that bill, unlike, say, /etc/machine-id, which is often - # identical on VMs created from cloned filesystems. If we cannot - # determine the system/hypervisor UUID we just generate a random MAC. - MAC=$(mac_from_file /sys/class/dmi/id/product_uuid) || \ - MAC=$(mac_from_file /sys/hypervisor/uuid) || \ - MAC=$(random_mac) + # for Docker 1.12 this needs to be docker inspect -f '{{with index .Mounts 0}}{{.Source}}{{end}}' + # so running this from 'weave --local' may fail. + # also if weaveexec is not on the path it will be run as a Docker container + # that does not have access to /etc/machine-id + WEAVEDB_PATH=$(docker inspect -f '{{index .Volumes "/weavedb"}}' $DB_CONTAINER_NAME 2>/dev/null) + MAC=$(util_op unique-id "$HOST_ROOT/$WEAVEDB_PATH/weave" "$HOST_ROOT") ip link set dev $BRIDGE address $MAC