diff --git a/prog/weaver/main.go b/prog/weaver/main.go index e185abde..2846fa29 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -151,6 +151,7 @@ func main() { config mesh.Config networkConfig weave.NetworkConfig protocolMinVersion int + resume bool ifaceName string routerName string nickName string @@ -181,6 +182,7 @@ func main() { mflag.StringVar(&config.Host, []string{"-host"}, "", "router host") mflag.IntVar(&config.Port, []string{"#port", "-port"}, mesh.Port, "router port") mflag.IntVar(&protocolMinVersion, []string{"-min-protocol-version"}, mesh.ProtocolMinVersion, "minimum weave protocol version") + mflag.BoolVar(&resume, []string{"-resume"}, false, "resume connections to previous peers") mflag.StringVar(&ifaceName, []string{"#iface", "-iface"}, "", "name of interface to capture/inject from (disabled if blank)") mflag.StringVar(&routerName, []string{"#name", "-name"}, "", "name of router (defaults to MAC of interface)") mflag.StringVar(&nickName, []string{"#nickname", "-nickname"}, "", "nickname of peer (defaults to hostname)") @@ -218,6 +220,9 @@ func main() { mflag.Parse() peers = mflag.Args() + if resume && len(peers) > 0 { + Log.Fatalf("You must not specify an initial peer list in conjuction with --resume") + } common.SetLogLevel(logLevel) @@ -267,7 +272,7 @@ func main() { router := weave.NewNetworkRouter(config, networkConfig, name, nickName, overlay, db) Log.Println("Our name is", router.Ourself) - if peers, err = router.InitialPeers(peers); err != nil { + if peers, err = router.InitialPeers(resume, peers); err != nil { Log.Fatal("Unable to get initial peer set: ", err) } diff --git a/router/network_router.go b/router/network_router.go index 48f6334f..af6128d6 100644 --- a/router/network_router.go +++ b/router/network_router.go @@ -1,8 +1,10 @@ package router import ( + "fmt" "math" "net" + "os" "time" "github.com/weaveworks/mesh" @@ -211,11 +213,10 @@ func (router *NetworkRouter) relayBroadcast(srcPeer *mesh.Peer, key PacketKey) F const peersIdent = "directPeers" func (router *NetworkRouter) persistPeers() { - // Persistence disabled pending rationalisation of the expected behaviour - /*if err := router.db.Save(peersIdent, router.ConnectionMaker.Targets(false)); err != nil { + if err := router.db.Save(peersIdent, router.ConnectionMaker.Targets(false)); err != nil { log.Errorf("Error persisting peers: %s", err) return - }*/ + } } func (router *NetworkRouter) InitiateConnections(peers []string, replace bool) []error { @@ -229,30 +230,22 @@ func (router *NetworkRouter) ForgetConnections(peers []string) { router.persistPeers() } -func (router *NetworkRouter) InitialPeers(peers []string) ([]string, error) { - var storedPeers []string - // Persistence disabled pending rationalisation of the expected behaviour - /*if _, err := router.db.Load(peersIdent, &storedPeers); err != nil { - return nil, err - }*/ - - if storedPeers != nil && !equal(peers, storedPeers) { - log.Println("Overriding initial peer list with stored list:", storedPeers) - peers = storedPeers - } else { - log.Println("Initial set of peers:", peers) +func (router *NetworkRouter) InitialPeers(resume bool, peers []string) ([]string, error) { + if _, err := os.Stat("restart.sentinel"); err == nil || resume { + var storedPeers []string + if _, err := router.db.Load(peersIdent, &storedPeers); err != nil { + return nil, err + } + log.Println("Restart/resume detected - using persisted peer list:", storedPeers) + return storedPeers, nil } + + sentinel, err := os.Create("restart.sentinel") + if err != nil { + return nil, fmt.Errorf("error creating sentinel: %v", err) + } + sentinel.Close() + + log.Println("Launch detected - using supplied peer list:", peers) return peers, nil } - -func equal(a, b []string) bool { - if len(a) != len(b) { - return false - } - for i, v := range a { - if v != b[i] { - return false - } - } - return true -} diff --git a/site/using-weave/finding-adding-hosts-dynamically.md b/site/using-weave/finding-adding-hosts-dynamically.md index 9c4bf072..f3f7bf86 100644 --- a/site/using-weave/finding-adding-hosts-dynamically.md +++ b/site/using-weave/finding-adding-hosts-dynamically.md @@ -3,58 +3,69 @@ title: Adding and Removing Hosts Dynamically menu_order: 90 --- +To add a host to an existing Weave network, launch Weave Net on the +host, and supply the address of at least one host. Weave Net +automatically discovers any other hosts in the network and establishes +connections with them if it can (in order to avoid unnecessary +multi-hop routing). -To add a host to an existing Weave network, simply launch -Weave Net on the host, and then supply the address of at least -one host. Weave Net automatically discovers any other hosts in -the network and establishes connections with them if it -can (in order to avoid unnecessary multi-hop routing). +In some situations existing Weave Net hosts may be unreachable from +the new host due to firewalls, etc. However, it is still possible to +add the new host, provided that inverse connections, for example, from +existing hosts to the new hosts, are available. -In some situations all existing Weave Net hosts may be -unreachable from the new host due to firewalls, etc. -However, it is still possible to add the new host, -provided that inverse connections, for example, -from existing hosts to the new hosts, are available. - -To accomplish this, launch Weave Net onto the new host -without supplying any additional addresses. And then, from one -of the existing hosts run: +To accomplish this, launch Weave Net onto the new host without +supplying any additional addresses and then, from one of the existing +hosts run: host# weave connect $NEW_HOST -Other hosts on the Weave network will automatically attempt -to establish connections to the new host as well. +Any other existing hosts on the Weave network will attempt to +establish connections to the new host as well. -Alternatively, you can also instruct a peer to forget a -particular host specified to it via `weave launch` or -`weave connect` by running: +###Instructing Peers to Forget a Host + +To instruct a peer to forget a particular host specified to it via +`weave launch` or `weave connect` run: host# weave forget $DECOMMISSIONED_HOST -This prevents the peer from trying to reconnect to that host -once connectivity to it is lost, and therefore can be used -to administratively remove any decommissioned peers -from the network. +This prevents the peer from reconnecting to that host once +connectivity to it is lost, and can be used to administratively remove +any decommissioned peers from the network. -Hosts may also be bulk-replaced. All existing hosts -will be forgotten, and the new hosts will be added: +###Bulk Replacing Hosts + +Hosts can also be bulk-replaced. All existing hosts will be forgotten, +and the new hosts added: host# weave connect --replace $NEW_HOST1 $NEW_HOST2 -For complete control over the peer topology, automatic -discovery can be disabled using the `--no-discovery` -option with `weave launch`. +###Restarting Docker and Weave Net -If discovery if disabled, Weave Net only connects to the -addresses specified at launch time and with `weave connect`. +If Weave Net is restarted by Docker it automatically remembers any +previous connect and forget operations, however if you stop it +manually and launch it again, it will not remember any prior connects. +If you want to launch again and retain the results of those operations +use `--resume`: -A list of all hosts that a peer has been asked to connect -to with `weave launch` and `weave connect` -can be obtained using: + host# weave launch --resume + +> **Note:** In this case, you cannot specify a list of addresses, +> since the previous peer list is used exclusively. + +For complete control over the peer topology, disable automatic +discovery using the `--no-discovery` option with `weave launch`. + +If discovery if disabled, Weave Net only connects to the addresses +specified at launch time and with `weave connect`. + +To return a list of all hosts and their peer connections established +with `weave launch` and `weave connect` run: host# weave status targets -**See Also** +**See Also** * [Enabling Multi-Cloud, Multi-Hop Networking and Routing](/site/using-weave/multi-cloud-multi-hop.md) * [Stopping and Removing Peers](/site/ipam/stop-remove-peers-ipam.md) diff --git a/test/090_docker_restart_policy_2_test.sh b/test/090_docker_restart_policy_2_test.sh index f7cc04c2..f43428d4 100755 --- a/test/090_docker_restart_policy_2_test.sh +++ b/test/090_docker_restart_policy_2_test.sh @@ -4,24 +4,6 @@ start_suite "Test docker restart policy" -check_restart() { - OLD_PID=$(container_pid $1 $2) - - run_on $1 sudo kill $OLD_PID - - for i in $(seq 1 10); do - NEW_PID=$(container_pid $1 $2) - - if [ $NEW_PID != 0 -a $NEW_PID != $OLD_PID ] ; then - return 0 - fi - - sleep 1 - done - - return 1 -} - weave_on $HOST1 launch assert "docker_on $HOST1 inspect -f '{{.HostConfig.RestartPolicy.Name}}' weave weaveproxy weaveplugin" "always\nalways\nalways" diff --git a/test/175_stop_start_test.sh b/test/175_stop_start_test.sh deleted file mode 100755 index 791d7a8c..00000000 --- a/test/175_stop_start_test.sh +++ /dev/null @@ -1,30 +0,0 @@ -#! /bin/bash - -. ./config.sh - -start_suite "Check restart re-uses same container when possible" - -ID=$(weave_on $HOST1 launch-router) -weave_on $HOST1 stop-router -assert "weave_on $HOST1 launch-router" "$ID" -# Can't start if already running with different args -assert_raises "weave_on $HOST1 launch-router --log-level=debug" 1 -# Stop then start with different arg -weave_on $HOST1 stop-router -ID2=$(weave_on $HOST1 launch-router --log-level=debug) -assert "[ $ID != $ID2 ]" - -# Stop then start with different password -weave_on $HOST1 stop-router -ID3=$(weave_on $HOST1 launch-router --password=xyzzy) -assert "[ $ID2 != $ID3 ]" -# Stop then start with same password -weave_on $HOST1 stop-router -assert "weave_on $HOST1 launch-router --password=xyzzy" "$ID3" - -# Fails on bad argument -assert_raises "weave_on $HOST1 launch-plugin --foo" 1 -# Fails again on bad argument, although there is a dead container -assert_raises "weave_on $HOST1 launch-plugin --foo" 1 - -end_suite diff --git a/test/175_target_peer_persistence_3_test.sh b/test/175_target_peer_persistence_3_test.sh new file mode 100755 index 00000000..d880bb04 --- /dev/null +++ b/test/175_target_peer_persistence_3_test.sh @@ -0,0 +1,36 @@ +#! /bin/bash + +. ./config.sh + + +assert_targets() { + HOST=$1 + shift + EXPECTED=$(for TARGET in $@; do echo $TARGET; done | sort) + assert "weave_on $HOST report | jq -r '.Router.Targets[] | tostring' | sort" "$EXPECTED" +} + +start_suite "Check Docker restart uses persisted peer list" + +# Launch router and modify initial peer list +weave_on $HOST1 launch $HOST1 $HOST2 +weave_on $HOST1 forget $HOST1 +weave_on $HOST1 connect $HOST3 + +# Ensure modified peer list is still in effect after restart +check_restart $HOST1 weave +assert_targets $HOST1 $HOST2 $HOST3 + +# Ensure persisted peer changes are still in effect after --resume +weave_on $HOST1 stop +weave_on $HOST1 launch --resume +assert_targets $HOST1 $HOST2 $HOST3 + +# Ensure persisted peer changes are ignored after stop and subsequent restart +weave_on $HOST1 stop +weave_on $HOST1 launch $HOST1 $HOST2 +assert_targets $HOST1 $HOST1 $HOST2 +check_restart $HOST1 weave +assert_targets $HOST1 $HOST1 $HOST2 + +end_suite diff --git a/test/config.sh b/test/config.sh index 57ceca69..97574b27 100644 --- a/test/config.sh +++ b/test/config.sh @@ -249,6 +249,25 @@ assert_dns_ptr_record() { assert "exec_on $1 $2 getent hosts $4 | tr -s ' '" "$4 $3" } +# Kill a container process and make sure it's restarted by Docker +check_restart() { + OLD_PID=$(container_pid $1 $2) + + run_on $1 sudo kill $OLD_PID + + for i in $(seq 1 10); do + NEW_PID=$(container_pid $1 $2) + + if [ $NEW_PID != 0 -a $NEW_PID != $OLD_PID ] ; then + return 0 + fi + + sleep 1 + done + + return 1 +} + start_suite() { for host in $HOSTS; do [ -z "$DEBUG" ] || echo "Cleaning up on $host: removing all containers and resetting weave" diff --git a/weave b/weave index bd9de658..158c0da1 100755 --- a/weave +++ b/weave @@ -45,7 +45,7 @@ weave launch [--no-restart] [--ipalloc-init ] [--ipalloc-range [--ipalloc-default-subnet ]] [--no-discovery] [--no-dns] - [--trusted-subnets ,...] ... + [--trusted-subnets ,...] [--resume] ... launch-proxy [-H ] [--without-dns] [--no-multicast-route] [--log-level=debug|info|warning|error] [--no-rewrite-hosts] [--no-default-ipalloc] [--no-restart] @@ -893,65 +893,28 @@ attach() { # functions for interacting with containers ###################################################################### -pertinent_match_args() { - docker inspect -f '{{.Args}} - {{.Config.Env}} - {{.HostConfig.RestartPolicy.Name}} - {{.HostConfig.RestartPolicy.MaximumRetryCount}}' $1 -} - -args_match() { - prev_args="$(pertinent_match_args $1)" || return 1 - shift - dummy_container=$(docker create "$@" 2>/dev/null) || return 1 - dummy_args="$(pertinent_match_args $dummy_container)" || return 1 - docker rm $dummy_container >/dev/null - [ "$prev_args" = "$dummy_args" ] -} - -# Check if a container for component $1 named $2 with image $3 and args $4... is running, -# and start it if necessary -start_if_necessary() { - component=$1 - name=$2 - image=$3 - shift 3 - image_base=${image%:*} - IMAGE_ID=$(docker inspect --format='{{.Id}}' $image 2>/dev/null) || true - RUN_STATUS=$(docker inspect --format='{{.State.Running}} {{.Image}} {{.Config.Image}} {{.Id}}' $name 2>/dev/null) || true - CONTAINER_ID=${RUN_STATUS##* } +# Check that a container for component $1 named $2 with image $3 is not running +check_not_running() { + RUN_STATUS=$(docker inspect --format='{{.State.Running}} {{.State.Status}} {{.Config.Image}}' $2 2>/dev/null) || true case ${RUN_STATUS%:*} in - "true $IMAGE_ID "*) - if args_match $CONTAINER_ID "$@" ; then - ALREADY_RUNNING_CONTAINER=$CONTAINER_ID - echo $CONTAINER_ID - else - echo "$name is already running with different arguments; you can stop it with 'weave stop-$component'." >&2 - return 1 - fi + "true restarting $3") + echo "$2 is restarting; you can stop it with 'weave stop-$1'." >&2 + return 3 ;; - "false $IMAGE_ID "*) - if args_match $CONTAINER_ID "$@" ; then - docker start $CONTAINER_ID - else - docker rm $name >/dev/null - docker run -d --name=$name "$@" - fi + "true "*" $3") + echo "$2 is already running; you can stop it with 'weave stop-$1'." >&2 + return 1 ;; - true*$image_base) - echo "Found another version of $name running. Aborting." >&2 - return 2 + "false "*" $3") + docker rm $2 >/dev/null ;; true*) - echo "Found another running container named '$name'. Aborting." >&2 + echo "Found another running container named '$2'. Aborting." >&2 return 2 ;; false*) - docker rm $name >/dev/null - docker run -d --name=$name "$@" - ;; - "") - docker run -d --name=$name "$@" + echo "Found another container named '$2'. Aborting." >&2 + return 2 ;; esac } @@ -1189,7 +1152,6 @@ check_running() { echo "$1 container is not present. Have you launched it?" >&2 return 1 elif [ "$STATUS" = "true true" ] ; then - [ -n "$ALREADY_RUNNING_CONTAINER" ] && return 0 echo "$1 container is restarting." >&2 return 2 elif [ "$STATUS" != "true false" ] ; then @@ -1627,7 +1589,7 @@ launch_router() { # Set WEAVE_DOCKER_ARGS in the environment in order to supply # additional parameters, such as resource limits, to docker # when launching the weave container. - start_if_necessary router $CONTAINER_NAME $IMAGE \ + ROUTER_CONTAINER=$(docker run -d --name=$CONTAINER_NAME \ $(docker_run_options) \ $RESTART_POLICY \ --volumes-from $DB_CONTAINER_NAME \ @@ -1640,10 +1602,10 @@ launch_router() { --dns-effective-listen-address $DOCKER_BRIDGE_IP \ $DNS_ROUTER_OPTS $NO_DNS_OPT \ --http-addr $HTTP_ADDR \ - "$@" + "$@") setup_router_iface_$BRIDGE_TYPE wait_for_status $CONTAINER_NAME http_call $HTTP_ADDR - [ -n "$ALREADY_RUNNING_CONTAINER" ] || populate_router + populate_router } # Recreate the parameter values that are set when the router is first launched @@ -1687,7 +1649,7 @@ launch_proxy() { docker create -v /w -v /w-noop -v /w-nomcast --name=$VOLUMES_CONTAINER_NAME \ --label=weavevolumes --entrypoint=/bin/false $EXEC_IMAGE >/dev/null fi - start_if_necessary proxy $PROXY_CONTAINER_NAME $EXEC_IMAGE \ + PROXY_CONTAINER=$(docker run -d --name=$PROXY_CONTAINER_NAME \ $(docker_run_options) \ $RESTART_POLICY \ --pid=host \ @@ -1700,7 +1662,7 @@ launch_proxy() { -e WEAVE_HTTP_ADDR \ -e EXEC_IMAGE=$EXEC_IMAGE \ --entrypoint=/home/weave/weaveproxy \ - $WEAVEPROXY_DOCKER_ARGS $EXEC_IMAGE $COVERAGE_ARGS $PROXY_ARGS + $WEAVEPROXY_DOCKER_ARGS $EXEC_IMAGE $COVERAGE_ARGS $PROXY_ARGS) wait_for_status $PROXY_CONTAINER_NAME http_call_unix $PROXY_CONTAINER_NAME status.sock } @@ -1721,13 +1683,23 @@ launch_plugin_if_not_running() { esac shift done - start_if_necessary plugin $PLUGIN_CONTAINER_NAME $PLUGIN_IMAGE \ + + retval=0 + check_not_running plugin $PLUGIN_CONTAINER_NAME $BASE_PLUGIN_IMAGE || retval=$? + # If an existing plugin is running (we start it with restart=always), return its ID + [ $retval = 1 ] && PLUGIN_CONTAINER=$(container_id $PLUGIN_CONTAINER_NAME) && return 0 + # Any other kind of error code from check_not_running is a failure. + [ $retval -gt 0 ] && return $retval + + if ! PLUGIN_CONTAINER=$(docker run -d --name=$PLUGIN_CONTAINER_NAME \ $(docker_run_options) \ $RESTART_POLICY \ -v /run/docker/plugins:/run/docker/plugins \ -e WEAVE_HTTP_ADDR \ $WEAVEPLUGIN_DOCKER_ARGS $PLUGIN_IMAGE $COVERAGE_ARGS \ - "$@" + "$@") ; then + return 1 + fi wait_for_status $PLUGIN_CONTAINER_NAME http_call_unix $PLUGIN_CONTAINER_NAME status.sock util_op create-plugin-network weave weavemesh } @@ -1905,14 +1877,19 @@ EOF ;; launch) deprecation_warnings "$@" + check_not_running router $CONTAINER_NAME $BASE_IMAGE + check_not_running proxy $PROXY_CONTAINER_NAME $BASE_EXEC_IMAGE + check_not_running plugin $PLUGIN_CONTAINER_NAME $BASE_PLUGIN_IMAGE COMMON_ARGS=$(common_launch_args "$@") - launch_router "$@" >/dev/null - launch_proxy $COMMON_ARGS >/dev/null - plugin_disabled || launch_plugin_if_not_running $COMMON_ARGS >/dev/null + launch_router "$@" + launch_proxy $COMMON_ARGS + plugin_disabled || launch_plugin_if_not_running $COMMON_ARGS ;; launch-router) deprecation_warnings "$@" + check_not_running router $CONTAINER_NAME $BASE_IMAGE launch_router "$@" + echo $ROUTER_CONTAINER ;; attach-router) check_running $CONTAINER_NAME @@ -1925,10 +1902,13 @@ EOF ;; launch-proxy) deprecation_warnings "$@" + check_not_running proxy $PROXY_CONTAINER_NAME $BASE_EXEC_IMAGE launch_proxy "$@" + echo $PROXY_CONTAINER ;; launch-plugin) launch_plugin_if_not_running "$@" + echo $PLUGIN_CONTAINER ;; env|proxy-env) [ "$COMMAND" = "env" ] || deprecation_warning "$COMMAND" "'weave env'"