From e4683996168ba3199e6b08bd060efbf7eb89e128 Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Tue, 1 Sep 2015 11:09:43 +0100
Subject: [PATCH] Extend the timeout for dead containers to 30 seconds, but
 remove them if they are destroyed, to reduce starvation when users are
 creating and destroying containers quickly.

---
 ipam/allocator.go                         |  2 +-
 site/features.md                          |  2 +-
 test/100_cross_hosts_2_test.sh            |  2 +-
 test/640_proxy_restart_reattaches_test.sh | 12 ++++++++++++
 4 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/ipam/allocator.go b/ipam/allocator.go
index fb5e6cd2..ae0565c3 100644
--- a/ipam/allocator.go
+++ b/ipam/allocator.go
@@ -23,7 +23,7 @@ const (
 
 	tickInterval         = time.Second * 5
 	MinSubnetSize        = 4 // first and last addresses are excluded, so 2 would be too small
-	containerDiedTimeout = time.Second * 5
+	containerDiedTimeout = time.Second * 30
 )
 
 // operation represents something which Allocator wants to do, but
diff --git a/site/features.md b/site/features.md
index 3612a151..766bd76e 100644
--- a/site/features.md
+++ b/site/features.md
@@ -181,7 +181,7 @@ current IP addresses:
     host1$ weave restart b1
 
 (note that the IP addresses are held for a limited time - currently
-five seconds)
+30 seconds)
 
 ### <a name="naming-and-discovery"></a>Naming and discovery
 
diff --git a/test/100_cross_hosts_2_test.sh b/test/100_cross_hosts_2_test.sh
index 203cb0b2..0eab1027 100755
--- a/test/100_cross_hosts_2_test.sh
+++ b/test/100_cross_hosts_2_test.sh
@@ -30,7 +30,7 @@ assert_raises "exec_on $HOST1 c5 $PING $C6"
 # check that restart retains the same IP, and reclaims it in IPAM
 weave_on $HOST2 restart c6
 assert_raises "exec_on $HOST1 c5 $PING $C6"
-sleep 6 # past the IPAM timeout
+sleep 31 # past the IPAM timeout
 start_container $HOST2 net:$SUBNET_2 --name=c7
 C7=$(container_ip $HOST2 c7)
 assert_raises "[ $C6 != $C7 ]"
diff --git a/test/640_proxy_restart_reattaches_test.sh b/test/640_proxy_restart_reattaches_test.sh
index 584b3be2..f02889c5 100755
--- a/test/640_proxy_restart_reattaches_test.sh
+++ b/test/640_proxy_restart_reattaches_test.sh
@@ -21,6 +21,16 @@ wait_for_proxy() {
     exit 1
 }
 
+N=50
+# Create and remove a lot of containers in a small subnet; the failure
+# mode is that this takes a long time as it has to wait for the old
+# ones to time out, so we run this function inside 'timeout'
+run_many() {
+    for i in $(seq $N); do
+        proxy docker_on $HOST1 run -e WEAVE_CIDR=net:10.32.4.0/28 --rm -t $SMALL_IMAGE /bin/true
+    done
+}
+
 start_suite "Proxy restart reattaches networking to containers"
 
 WEAVE_DOCKER_ARGS=--restart=always WEAVEPROXY_DOCKER_ARGS=--restart=always weave_on $HOST1 launch
@@ -49,6 +59,8 @@ sleep 5 # allow for re-tries of attach
 C2=$(container_ip $HOST1 c2) || (echo "container c2 has no IP address" 2>&1; exit 1)
 check_attached
 
+assert_raises "timeout 90 cat <( run_many )"
+
 # Restarting proxy shouldn't kill unattachable containers
 proxy_start_container $HOST1 -di --name=c3 --restart=always # Use ipam, so it won't be attachable w/o weave
 weave_on $HOST1 stop