reweave/weave
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #2305 from weaveworks/issues/2186-peer-list-persistence

Browse Source 
Peer list persistence; Fixes #2186.
This commit is contained in:
Bryan Boreham committed 2016-06-03 14:31:01 +01:00
2 parents 1d91c1a23b 37f5a17953
commit 2728bc0f9d
8 files changed
+169 -173
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

prog/weaver/main.go
+6 -1
View File
@@ -151,6 +151,7 @@ func main() {
config mesh.Config
networkConfig weave.NetworkConfig
protocolMinVersion int
resume bool
ifaceName string
routerName string
nickName string
@@ -181,6 +182,7 @@ func main() {
mflag.StringVar(&config.Host, []string{"-host"}, "", "router host")
mflag.IntVar(&config.Port, []string{"#port", "-port"}, mesh.Port, "router port")
mflag.IntVar(&protocolMinVersion, []string{"-min-protocol-version"}, mesh.ProtocolMinVersion, "minimum weave protocol version")
mflag.BoolVar(&resume, []string{"-resume"}, false, "resume connections to previous peers")
mflag.StringVar(&ifaceName, []string{"#iface", "-iface"}, "", "name of interface to capture/inject from (disabled if blank)")
mflag.StringVar(&routerName, []string{"#name", "-name"}, "", "name of router (defaults to MAC of interface)")
mflag.StringVar(&nickName, []string{"#nickname", "-nickname"}, "", "nickname of peer (defaults to hostname)")
@@ -218,6 +220,9 @@ func main() {
mflag.Parse()
peers = mflag.Args()
if resume && len(peers) > 0 {
Log.Fatalf("You must not specify an initial peer list in conjuction with --resume")
}
common.SetLogLevel(logLevel)
@@ -267,7 +272,7 @@ func main() {
router := weave.NewNetworkRouter(config, networkConfig, name, nickName, overlay, db)
Log.Println("Our name is", router.Ourself)
if peers, err = router.InitialPeers(peers); err != nil {
if peers, err = router.InitialPeers(resume, peers); err != nil {
Log.Fatal("Unable to get initial peer set: ", err)
}
router/network_router.go
+20 -27
View File
@@ -1,8 +1,10 @@
package router
import (
"fmt"
"math"
"net"
"os"
"time"
"github.com/weaveworks/mesh"
@@ -211,11 +213,10 @@ func (router *NetworkRouter) relayBroadcast(srcPeer *mesh.Peer, key PacketKey) F
const peersIdent = "directPeers"
func (router *NetworkRouter) persistPeers() {
// Persistence disabled pending rationalisation of the expected behaviour
/*if err := router.db.Save(peersIdent, router.ConnectionMaker.Targets(false)); err != nil {
if err := router.db.Save(peersIdent, router.ConnectionMaker.Targets(false)); err != nil {
log.Errorf("Error persisting peers: %s", err)
return
}*/
}
}
func (router *NetworkRouter) InitiateConnections(peers []string, replace bool) []error {
@@ -229,30 +230,22 @@ func (router *NetworkRouter) ForgetConnections(peers []string) {
router.persistPeers()
}
func (router *NetworkRouter) InitialPeers(peers []string) ([]string, error) {
var storedPeers []string
// Persistence disabled pending rationalisation of the expected behaviour
/*if _, err := router.db.Load(peersIdent, &storedPeers); err != nil {
return nil, err
}*/
if storedPeers != nil && !equal(peers, storedPeers) {
log.Println("Overriding initial peer list with stored list:", storedPeers)
peers = storedPeers
} else {
log.Println("Initial set of peers:", peers)
func (router *NetworkRouter) InitialPeers(resume bool, peers []string) ([]string, error) {
if _, err := os.Stat("restart.sentinel"); err == nil || resume {
var storedPeers []string
if _, err := router.db.Load(peersIdent, &storedPeers); err != nil {
return nil, err
}
log.Println("Restart/resume detected - using persisted peer list:", storedPeers)
return storedPeers, nil
}
sentinel, err := os.Create("restart.sentinel")
if err != nil {
return nil, fmt.Errorf("error creating sentinel: %v", err)
}
sentinel.Close()
log.Println("Launch detected - using supplied peer list:", peers)
return peers, nil
}
func equal(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i, v := range a {
if v != b[i] {
return false
}
}
return true
}
site/using-weave/finding-adding-hosts-dynamically.md
+45 -34
View File
@@ -3,58 +3,69 @@ title: Adding and Removing Hosts Dynamically
menu_order: 90
---
To add a host to an existing Weave network, launch Weave Net on the
host, and supply the address of at least one host. Weave Net
automatically discovers any other hosts in the network and establishes
connections with them if it can (in order to avoid unnecessary
multi-hop routing).
To add a host to an existing Weave network, simply launch
Weave Net on the host, and then supply the address of at least
one host. Weave Net automatically discovers any other hosts in
the network and establishes connections with them if it
can (in order to avoid unnecessary multi-hop routing).
In some situations existing Weave Net hosts may be unreachable from
the new host due to firewalls, etc. However, it is still possible to
add the new host, provided that inverse connections, for example, from
existing hosts to the new hosts, are available.
In some situations all existing Weave Net hosts may be
unreachable from the new host due to firewalls, etc.
However, it is still possible to add the new host,
provided that inverse connections, for example,
from existing hosts to the new hosts, are available.
To accomplish this, launch Weave Net onto the new host
without supplying any additional addresses. And then, from one
of the existing hosts run:
To accomplish this, launch Weave Net onto the new host without
supplying any additional addresses and then, from one of the existing
hosts run:
host# weave connect $NEW_HOST
Other hosts on the Weave network will automatically attempt
to establish connections to the new host as well.
Any other existing hosts on the Weave network will attempt to
establish connections to the new host as well.
Alternatively, you can also instruct a peer to forget a
particular host specified to it via `weave launch` or
`weave connect` by running:
###Instructing Peers to Forget a Host
To instruct a peer to forget a particular host specified to it via
`weave launch` or `weave connect` run:
host# weave forget $DECOMMISSIONED_HOST
This prevents the peer from trying to reconnect to that host
once connectivity to it is lost, and therefore can be used
to administratively remove any decommissioned peers
from the network.
This prevents the peer from reconnecting to that host once
connectivity to it is lost, and can be used to administratively remove
any decommissioned peers from the network.
Hosts may also be bulk-replaced. All existing hosts
will be forgotten, and the new hosts will be added:
###Bulk Replacing Hosts
Hosts can also be bulk-replaced. All existing hosts will be forgotten,
and the new hosts added:
host# weave connect --replace $NEW_HOST1 $NEW_HOST2
For complete control over the peer topology, automatic
discovery can be disabled using the `--no-discovery`
option with `weave launch`.
###Restarting Docker and Weave Net
If discovery if disabled, Weave Net only connects to the
addresses specified at launch time and with `weave connect`.
If Weave Net is restarted by Docker it automatically remembers any
previous connect and forget operations, however if you stop it
manually and launch it again, it will not remember any prior connects.
If you want to launch again and retain the results of those operations
use `--resume`:
A list of all hosts that a peer has been asked to connect
to with `weave launch` and `weave connect`
can be obtained using:
host# weave launch --resume
> **Note:** In this case, you cannot specify a list of addresses,
> since the previous peer list is used exclusively.
For complete control over the peer topology, disable automatic
discovery using the `--no-discovery` option with `weave launch`.
If discovery if disabled, Weave Net only connects to the addresses
specified at launch time and with `weave connect`.
To return a list of all hosts and their peer connections established
with `weave launch` and `weave connect` run:
host# weave status targets
**See Also**
**See Also**
* [Enabling Multi-Cloud, Multi-Hop Networking and Routing](/site/using-weave/multi-cloud-multi-hop.md)
* [Stopping and Removing Peers](/site/ipam/stop-remove-peers-ipam.md)
test/090_docker_restart_policy_2_test.sh
-18
View File
@@ -4,24 +4,6 @@
start_suite "Test docker restart policy"
check_restart() {
OLD_PID=$(container_pid $1 $2)
run_on $1 sudo kill $OLD_PID
for i in $(seq 1 10); do
NEW_PID=$(container_pid $1 $2)
if [ $NEW_PID != 0 -a $NEW_PID != $OLD_PID ] ; then
return 0
fi
sleep 1
done
return 1
}
weave_on $HOST1 launch
assert "docker_on $HOST1 inspect -f '{{.HostConfig.RestartPolicy.Name}}' weave weaveproxy weaveplugin" "always\nalways\nalways"
test/175_stop_start_test.sh
-30
View File
@@ -1,30 +0,0 @@
#! /bin/bash
. ./config.sh
start_suite "Check restart re-uses same container when possible"
ID=$(weave_on $HOST1 launch-router)
weave_on $HOST1 stop-router
assert "weave_on $HOST1 launch-router" "$ID"
# Can't start if already running with different args
assert_raises "weave_on $HOST1 launch-router --log-level=debug" 1
# Stop then start with different arg
weave_on $HOST1 stop-router
ID2=$(weave_on $HOST1 launch-router --log-level=debug)
assert "[ $ID != $ID2 ]"
# Stop then start with different password
weave_on $HOST1 stop-router
ID3=$(weave_on $HOST1 launch-router --password=xyzzy)
assert "[ $ID2 != $ID3 ]"
# Stop then start with same password
weave_on $HOST1 stop-router
assert "weave_on $HOST1 launch-router --password=xyzzy" "$ID3"
# Fails on bad argument
assert_raises "weave_on $HOST1 launch-plugin --foo" 1
# Fails again on bad argument, although there is a dead container
assert_raises "weave_on $HOST1 launch-plugin --foo" 1
end_suite
test/175_target_peer_persistence_3_test.sh
Executable
+36
View File
@@ -0,0 +1,36 @@
#! /bin/bash
. ./config.sh
assert_targets() {
HOST=$1
shift
EXPECTED=$(for TARGET in $@; do echo $TARGET; done | sort)
assert "weave_on $HOST report | jq -r '.Router.Targets[] | tostring' | sort" "$EXPECTED"
}
start_suite "Check Docker restart uses persisted peer list"
# Launch router and modify initial peer list
weave_on $HOST1 launch $HOST1 $HOST2
weave_on $HOST1 forget $HOST1
weave_on $HOST1 connect $HOST3
# Ensure modified peer list is still in effect after restart
check_restart $HOST1 weave
assert_targets $HOST1 $HOST2 $HOST3
# Ensure persisted peer changes are still in effect after --resume
weave_on $HOST1 stop
weave_on $HOST1 launch --resume
assert_targets $HOST1 $HOST2 $HOST3
# Ensure persisted peer changes are ignored after stop and subsequent restart
weave_on $HOST1 stop
weave_on $HOST1 launch $HOST1 $HOST2
assert_targets $HOST1 $HOST1 $HOST2
check_restart $HOST1 weave
assert_targets $HOST1 $HOST1 $HOST2
end_suite
test/config.sh
+19
View File
@@ -249,6 +249,25 @@ assert_dns_ptr_record() {
assert "exec_on $1 $2 getent hosts $4 | tr -s ' '" "$4 $3"
}
# Kill a container process and make sure it's restarted by Docker
check_restart() {
OLD_PID=$(container_pid $1 $2)
run_on $1 sudo kill $OLD_PID
for i in $(seq 1 10); do
NEW_PID=$(container_pid $1 $2)
if [ $NEW_PID != 0 -a $NEW_PID != $OLD_PID ] ; then
return 0
fi
sleep 1
done
return 1
}
start_suite() {
for host in $HOSTS; do
[ -z "$DEBUG" ] || echo "Cleaning up on $host: removing all containers and resetting weave"
weave
+43 -63
View File
@@ -45,7 +45,7 @@ weave launch <same arguments as 'weave launch-router'>
[--no-restart] [--ipalloc-init <mode>]
[--ipalloc-range <cidr> [--ipalloc-default-subnet <cidr>]]
[--no-discovery] [--no-dns]
[--trusted-subnets <cidr>,...] <peer> ...
[--trusted-subnets <cidr>,...] [--resume] <peer> ...
launch-proxy [-H <endpoint>] [--without-dns] [--no-multicast-route]
[--log-level=debug|info|warning|error]
[--no-rewrite-hosts] [--no-default-ipalloc] [--no-restart]
@@ -893,65 +893,28 @@ attach() {
# functions for interacting with containers
######################################################################
pertinent_match_args() {
docker inspect -f '{{.Args}}
{{.Config.Env}}
{{.HostConfig.RestartPolicy.Name}}
{{.HostConfig.RestartPolicy.MaximumRetryCount}}' $1
}
args_match() {
prev_args="$(pertinent_match_args $1)" || return 1
shift
dummy_container=$(docker create "$@" 2>/dev/null) || return 1
dummy_args="$(pertinent_match_args $dummy_container)" || return 1
docker rm $dummy_container >/dev/null
[ "$prev_args" = "$dummy_args" ]
}
# Check if a container for component $1 named $2 with image $3 and args $4... is running,
# and start it if necessary
start_if_necessary() {
component=$1
name=$2
image=$3
shift 3
image_base=${image%:*}
IMAGE_ID=$(docker inspect --format='{{.Id}}' $image 2>/dev/null) || true
RUN_STATUS=$(docker inspect --format='{{.State.Running}} {{.Image}} {{.Config.Image}} {{.Id}}' $name 2>/dev/null) || true
CONTAINER_ID=${RUN_STATUS##* }
# Check that a container for component $1 named $2 with image $3 is not running
check_not_running() {
RUN_STATUS=$(docker inspect --format='{{.State.Running}} {{.State.Status}} {{.Config.Image}}' $2 2>/dev/null) || true
case ${RUN_STATUS%:*} in
"true $IMAGE_ID "*)
if args_match $CONTAINER_ID "$@" ; then
ALREADY_RUNNING_CONTAINER=$CONTAINER_ID
echo $CONTAINER_ID
else
echo "$name is already running with different arguments; you can stop it with 'weave stop-$component'." >&2
return 1
fi
"true restarting $3")
echo "$2 is restarting; you can stop it with 'weave stop-$1'." >&2
return 3
;;
"false $IMAGE_ID "*)
if args_match $CONTAINER_ID "$@" ; then
docker start $CONTAINER_ID
else
docker rm $name >/dev/null
docker run -d --name=$name "$@"
fi
"true "*" $3")
echo "$2 is already running; you can stop it with 'weave stop-$1'." >&2
return 1
;;
true*$image_base)
echo "Found another version of $name running. Aborting." >&2
return 2
"false "*" $3")
docker rm $2 >/dev/null
;;
true*)
echo "Found another running container named '$name'. Aborting." >&2
echo "Found another running container named '$2'. Aborting." >&2
return 2
;;
false*)
docker rm $name >/dev/null
docker run -d --name=$name "$@"
;;
"")
docker run -d --name=$name "$@"
echo "Found another container named '$2'. Aborting." >&2
return 2
;;
esac
}
@@ -1189,7 +1152,6 @@ check_running() {
echo "$1 container is not present. Have you launched it?" >&2
return 1
elif [ "$STATUS" = "true true" ] ; then
[ -n "$ALREADY_RUNNING_CONTAINER" ] && return 0
echo "$1 container is restarting." >&2
return 2
elif [ "$STATUS" != "true false" ] ; then
@@ -1627,7 +1589,7 @@ launch_router() {
# Set WEAVE_DOCKER_ARGS in the environment in order to supply
# additional parameters, such as resource limits, to docker
# when launching the weave container.
start_if_necessary router $CONTAINER_NAME $IMAGE \
ROUTER_CONTAINER=$(docker run -d --name=$CONTAINER_NAME \
$(docker_run_options) \
$RESTART_POLICY \
--volumes-from $DB_CONTAINER_NAME \
@@ -1640,10 +1602,10 @@ launch_router() {
--dns-effective-listen-address $DOCKER_BRIDGE_IP \
$DNS_ROUTER_OPTS $NO_DNS_OPT \
--http-addr $HTTP_ADDR \
"$@"
"$@")
setup_router_iface_$BRIDGE_TYPE
wait_for_status $CONTAINER_NAME http_call $HTTP_ADDR
[ -n "$ALREADY_RUNNING_CONTAINER" ] || populate_router
populate_router
}
# Recreate the parameter values that are set when the router is first launched
@@ -1687,7 +1649,7 @@ launch_proxy() {
docker create -v /w -v /w-noop -v /w-nomcast --name=$VOLUMES_CONTAINER_NAME \
--label=weavevolumes --entrypoint=/bin/false $EXEC_IMAGE >/dev/null
fi
start_if_necessary proxy $PROXY_CONTAINER_NAME $EXEC_IMAGE \
PROXY_CONTAINER=$(docker run -d --name=$PROXY_CONTAINER_NAME \
$(docker_run_options) \
$RESTART_POLICY \
--pid=host \
@@ -1700,7 +1662,7 @@ launch_proxy() {
-e WEAVE_HTTP_ADDR \
-e EXEC_IMAGE=$EXEC_IMAGE \
--entrypoint=/home/weave/weaveproxy \
$WEAVEPROXY_DOCKER_ARGS $EXEC_IMAGE $COVERAGE_ARGS $PROXY_ARGS
$WEAVEPROXY_DOCKER_ARGS $EXEC_IMAGE $COVERAGE_ARGS $PROXY_ARGS)
wait_for_status $PROXY_CONTAINER_NAME http_call_unix $PROXY_CONTAINER_NAME status.sock
}
@@ -1721,13 +1683,23 @@ launch_plugin_if_not_running() {
esac
shift
done
start_if_necessary plugin $PLUGIN_CONTAINER_NAME $PLUGIN_IMAGE \
retval=0
check_not_running plugin $PLUGIN_CONTAINER_NAME $BASE_PLUGIN_IMAGE || retval=$?
# If an existing plugin is running (we start it with restart=always), return its ID
[ $retval = 1 ] && PLUGIN_CONTAINER=$(container_id $PLUGIN_CONTAINER_NAME) && return 0
# Any other kind of error code from check_not_running is a failure.
[ $retval -gt 0 ] && return $retval
if ! PLUGIN_CONTAINER=$(docker run -d --name=$PLUGIN_CONTAINER_NAME \
$(docker_run_options) \
$RESTART_POLICY \
-v /run/docker/plugins:/run/docker/plugins \
-e WEAVE_HTTP_ADDR \
$WEAVEPLUGIN_DOCKER_ARGS $PLUGIN_IMAGE $COVERAGE_ARGS \
"$@"
"$@") ; then
return 1
fi
wait_for_status $PLUGIN_CONTAINER_NAME http_call_unix $PLUGIN_CONTAINER_NAME status.sock
util_op create-plugin-network weave weavemesh
}
@@ -1905,14 +1877,19 @@ EOF
;;
launch)
deprecation_warnings "$@"
check_not_running router $CONTAINER_NAME $BASE_IMAGE
check_not_running proxy $PROXY_CONTAINER_NAME $BASE_EXEC_IMAGE
check_not_running plugin $PLUGIN_CONTAINER_NAME $BASE_PLUGIN_IMAGE
COMMON_ARGS=$(common_launch_args "$@")
launch_router "$@" >/dev/null
launch_proxy $COMMON_ARGS >/dev/null
plugin_disabled || launch_plugin_if_not_running $COMMON_ARGS >/dev/null
launch_router "$@"
launch_proxy $COMMON_ARGS
plugin_disabled || launch_plugin_if_not_running $COMMON_ARGS
;;
launch-router)
deprecation_warnings "$@"
check_not_running router $CONTAINER_NAME $BASE_IMAGE
launch_router "$@"
echo $ROUTER_CONTAINER
;;
attach-router)
check_running $CONTAINER_NAME
@@ -1925,10 +1902,13 @@ EOF
;;
launch-proxy)
deprecation_warnings "$@"
check_not_running proxy $PROXY_CONTAINER_NAME $BASE_EXEC_IMAGE
launch_proxy "$@"
echo $PROXY_CONTAINER
;;
launch-plugin)
launch_plugin_if_not_running "$@"
echo $PLUGIN_CONTAINER
;;
env|proxy-env)
[ "$COMMAND" = "env" ] || deprecation_warning "$COMMAND" "'weave env'"