Merge pull request #3356 from weaveworks/portmap

support for portmap as additional plugin with wave-net

docs/entrypoints.md

@@ -35,21 +35,21 @@ weave-kube is an image with the weave binaries and a wrapper script installed. I
 
 1. Setting up the weave network
 2. Connecting to peers
-3. Copy the weave CNI-compatible binary plugin `weaveutil` to `/opt/cni/bin/weave-net` and weave config in `/etc/cni/net.d/10-weave.conf`
+3. Copy the weave CNI-compatible binary plugin `weaveutil` to `/opt/cni/bin/weave-net` and weave config in `/etc/cni/net.d/10-weave.conflist`
 4. Running the weave network policy controller (weave-npc) to implement kubernetes' `NetworkPolicy`
 
 Once installation is complete, each network-relevant change in a container leads `kubelet` to:
 
 1. Set certain environment variables to configure the CNI plugin, mostly related to the container ID
 2. Launch `/opt/cni/bin/weave-net`
-3. Pass the CNI config - the contents of `/etc/cni/net.d/10-weave.conf` to `weave-net`
+3. Pass the CNI config - the contents of `/etc/cni/net.d/10-weave.conflist` to `weave-net`
 
 Thus, when each container is changed, and `kubelet` calls weave as a CNI plugin, it really just is launching `weaveutil` as `weave-net`.
 
 The installation and setup of all of the above - and therefore the entrypoint to the weave-kube image - is the script `prog/weave-kube/launch.sh`. `launch.sh` does the following:
 
 1. Read configuration variables from the environment. When the documentation for `weave-kube` describes configuring the weave network by changing the environment variables in the daemonset in the `.yml` file, `launch.sh` reads these environment variables.
-2. Set up the config file at `/etc/cni/net.d/10-weave.conf`
+2. Set up the config file at `/etc/cni/net.d/10-weave.conflist`
 3. Run the `weave` initialization script
 4. Run `weaver` with the correct configuration passed as command-line options
 
@@ -66,5 +66,5 @@ To add new options to how weave should run with each invocation, you would do th
     * have `prog/weave-kube/launch.sh` read it as an environment variable and set inside
     * set a default for the environment variable in `prog/weave-kube/weave-daemonset-k8s-1.6.yaml` and `weave-daemonset.yaml`
     * if it should be set via `weaver` globally on its one-time initialization invocation, pass it on in `launch.sh`
-    * if it needs to be set via `weaveutil` on each invocation of `weave-net`, have `launch.sh` save it as an option in the CNI config file `/etc/cni/net.d/10-weave.conf` and then have the CNI code in `plugin/net/cni.go` in `weaveutil` read it and use where appropriate
+    * if it needs to be set via `weaveutil` on each invocation of `weave-net`, have `launch.sh` save it as an option in the CNI config file `/etc/cni/net.d/10-weave.conflist` and then have the CNI code in `plugin/net/cni.go` in `weaveutil` read it and use where appropriate
 7. Document it!

site/kubernetes.md

@@ -30,6 +30,8 @@ Then run:
 
     weave setup
 
+Weave net depends on the *portmap* [standard](https://github.com/containernetworking/plugins) CNI plugin to support *hostport* functionality. Please ensure that *portmap* CNI plugin is installed in `/opt/cni/bin` directory.
+
 #### Launching Weave Net
 
 To create a network that spans multiple hosts, the Weave peers must be connected to each other.  
@@ -45,7 +47,7 @@ for a discussion on peer connections.
 
 All CNI plugins are configured by a JSON file in the directory
 `/etc/cni/net.d/`.  `weave setup` installs a minimal configuration
-file named `10-weave.conf`, which you can alter to suit your needs.
+file named `10-weave.conflist`, which you can alter to suit your needs.
 
 See the [CNI Spec](https://github.com/appc/cni/blob/master/SPEC.md#network-configuration)
 for details on the format and contents of this file.

site/kubernetes/kube-addon.md

@@ -49,6 +49,11 @@ recommend your master node has at least two CPU cores.
 > [kubeadm](http://kubernetes.io/docs/getting-started-guides/kubeadm/).
 >
 > Alternatively, you can [configure CNI yourself](http://kubernetes.io/docs/admin/network-plugins/#cni)
+>
+> Weave net depends on the *portmap* [standard](https://github.com/containernetworking/plugins) CNI plugin
+> to support *hostport* functionality. Please ensure that *portmap* CNI plugin is installed 
+> (either by cluster installers like kubeadm or manually if you have configured CNI yourself) in `/opt/cni/bin` directory.
+
 
 **Note:** If using the [Weave CNI
 Plugin](/site/kubernetes.md) from a prior full install of Weave Net with your

test/860_weave_kube_portmap_3_test.sh

@@ -0,0 +1,115 @@
+#! /bin/bash
+
+. "$(dirname "$0")/config.sh"
+
+function howmany { echo $#; }
+
+#
+# Test vars
+#
+TOKEN=112233.4455667788990000
+HOST1IP=$($SSH $HOST1 "getent hosts $HOST1 | cut -f 1 -d ' '")
+NUM_HOSTS=$(howmany $HOSTS)
+SUCCESS="$(( $NUM_HOSTS * ($NUM_HOSTS-1) )) established"
+KUBECTL="sudo kubectl --kubeconfig /etc/kubernetes/admin.conf"
+KUBE_PORT=6443
+IMAGE=weaveworks/network-tester:latest
+
+if [ -n "$COVERAGE" ]; then
+    COVERAGE_ARGS="env:\\n                - name: EXTRA_ARGS\\n                  value: \"-test.coverprofile=/home/weave/cover.prof --\""
+else
+    COVERAGE_ARGS="env:"
+fi
+
+#
+# Utility functions
+#
+function teardown_kubernetes_cluster {
+    greyly echo "Tearing down kubernetes cluster"
+    for host in $HOSTS; do
+        run_on $host "sudo kubeadm reset --force && sudo rm -r -f /opt/cni/bin/*weave*"
+    done
+}
+
+function setup_kubernetes_cluster {
+    teardown_kubernetes_cluster;
+
+    greyly echo "Setting up kubernetes cluster"
+
+    # kubeadm init upgrades to latest Kubernetes version by default, therefore we try to lock the version using the below option:
+    k8s_version="$(run_on $HOST1 "kubelet --version" | grep -oP "(?<=Kubernetes )v[\d\.\-beta]+")"
+
+    for host in $HOSTS; do
+        if [ $host = $HOST1 ] ; then
+        run_on $host "sudo systemctl start kubelet && sudo kubeadm init --kubernetes-version=$k8s_version --token=$TOKEN"
+        else
+        run_on $host "sudo systemctl start kubelet && sudo kubeadm join --token=$TOKEN --discovery-token-unsafe-skip-ca-verification $HOST1IP:$KUBE_PORT"
+        fi
+    done
+
+    # Ensure Kubernetes uses locally built container images and inject code coverage environment variable (or do nothing depending on $COVERAGE):
+    sed -e "s%imagePullPolicy: Always%imagePullPolicy: Never%" \
+        -e "s%env:%$COVERAGE_ARGS%" \
+        "$(dirname "$0")/../prog/weave-kube/weave-daemonset-k8s-1.7.yaml" | run_on "$HOST1" "$KUBECTL apply -n kube-system -f -"
+}
+
+function weave_connected {
+    run_on $HOST1 "curl -sS http://127.0.0.1:6784/status | grep \"$SUCCESS\""
+}
+
+# setup a daemonset that uses hostport 80
+function setup_hostport_daemonSet {
+    greyly echo "Deploying daemonset that use hostport 80"
+    run_on $HOST1 "$KUBECTL create -f -" <<EOF
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: frontend
+spec:
+  template:
+    metadata:
+      labels:
+        app: frontend-webserver
+    spec:
+      containers:
+        - name: webserver
+          image: nginx
+          ports:
+          - containerPort: 80
+            hostPort: 80
+EOF
+}
+
+function check_daemonset_ready {
+    ready=$($SSH $HOST1 "$KUBECTL get ds frontend  -o go-template='{{.status.numberReady}}'")
+    if [ $ready = "2" ] ; then
+        return 0
+    fi
+    return 1
+}
+
+#
+# Suite
+#
+function main {
+    start_suite "Test weave-net support for Kubernetes HostPort";
+
+    setup_kubernetes_cluster;
+
+    # Need to wait until all pods have come up
+    assert_raises "wait_for_x weave_connected 'pods to be running weave net'"
+
+    setup_hostport_daemonSet
+
+    assert_raises 'wait_for_x check_daemonset_ready "wait for pods to be ready" 200'
+
+    # verify hostport is accessible on each node
+    assert_raises "run_on $HOST2 curl 127.0.0.1:80"
+    assert_raises "run_on $HOST3 curl 127.0.0.1:80"
+
+    teardown_kubernetes_cluster;
+
+    end_suite;
+}
+
+main

test/config.sh

@@ -329,7 +329,7 @@ start_suite() {
         [ -z "$DEBUG" ] || echo "Cleaning up on $host: removing all containers and resetting weave"
         rm_containers $host $(docker_on $host ps -aq 2>/dev/null)
         weave_on $host reset 2>/dev/null
-        run_on $host sudo rm -f /opt/cni/bin/weave-plugin-latest /opt/cni/bin/weave-net /opt/cni/bin/weave-ipam /etc/cni/net.d/10-weave.conf
+        run_on $host sudo rm -f /opt/cni/bin/weave-plugin-latest /opt/cni/bin/weave-net /opt/cni/bin/weave-ipam /etc/cni/net.d/10-weave.conflist
     done
     whitely echo "$@"
 }

weave

@@ -729,9 +729,20 @@ upgrade_cni_plugin() {
 create_cni_config() {
     cat >"$1" <<EOF
 {
+    "cniVersion": "0.3.0",
     "name": "weave",
-    "type": "weave-net",
-    "hairpinMode": $2
+    "plugins": [
+        {
+            "name": "weave",
+            "type": "weave-net",
+            "hairpinMode": $2
+        },
+        {
+            "type": "portmap",
+            "capabilities": {"portMappings": true},
+            "snat": true
+        }
+    ]
 }
 EOF
 }
@@ -742,8 +753,13 @@ setup_cni() {
     if install_cni_plugin $CNI_PLUGIN_DIR ; then
         upgrade_cni_plugin $CNI_PLUGIN_DIR
     fi
-    if [ -d $HOST_ROOT/etc/cni/net.d -a ! -f $HOST_ROOT/etc/cni/net.d/10-weave.conf ] ; then
-        create_cni_config $HOST_ROOT/etc/cni/net.d/10-weave.conf $HAIRPIN_MODE
+
+    # Weave will use .conflist for versions > 2.4.0. So in case of upgrade of old
+    # clusters with Weave version <= 2.4.0 remove existing 10-weave.conf
+    rm -f $HOST_ROOT/etc/cni/net.d/10-weave.conf
+
+    if [ -d $HOST_ROOT/etc/cni/net.d -a ! -f $HOST_ROOT/etc/cni/net.d/10-weave.conflist ] ; then
+        create_cni_config $HOST_ROOT/etc/cni/net.d/10-weave.conflist $HAIRPIN_MODE
     fi
 }