From 8eeb3ca4f2cb067bc7e4afd4c13ac28bcb6a48d6 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 11 Jan 2021 14:34:38 +0000 Subject: [PATCH] Kubernetes: move part of launch to init container This allows the main container to run without write access to the host filesystem, except for two specific areas, thus trimming the attack surface. We lose the behaviour that the CNI plugin is only installed if Weave Net gets up and running at least once, so the error reports from kubelet will be slightly different in the case that it never manages to run. --- prog/weave-kube/Dockerfile.template | 2 +- prog/weave-kube/init.sh | 62 +++++++++++++++++++ prog/weave-kube/launch.sh | 55 +--------------- prog/weave-kube/weave-daemonset-k8s-1.11.yaml | 30 ++++++--- prog/weave-kube/weave-daemonset-k8s-1.8.yaml | 30 ++++++--- prog/weave-kube/weave-daemonset-k8s-1.9.yaml | 30 ++++++--- 6 files changed, 130 insertions(+), 79 deletions(-) create mode 100755 prog/weave-kube/init.sh diff --git a/prog/weave-kube/Dockerfile.template b/prog/weave-kube/Dockerfile.template index 0684edcf..771cf1cd 100644 --- a/prog/weave-kube/Dockerfile.template +++ b/prog/weave-kube/Dockerfile.template @@ -6,7 +6,7 @@ LABEL maintainer="Weaveworks " \ org.opencontainers.image.source="https://github.com/weaveworks/weave" \ org.opencontainers.image.vendor="Weaveworks" -ADD ./launch.sh ./kube-utils /home/weave/ +ADD ./init.sh ./launch.sh ./kube-utils /home/weave/ ENTRYPOINT ["/home/weave/launch.sh"] # This label will change for every build, and should therefore be the last layer of the image: diff --git a/prog/weave-kube/init.sh b/prog/weave-kube/init.sh new file mode 100755 index 00000000..232fda82 --- /dev/null +++ b/prog/weave-kube/init.sh @@ -0,0 +1,62 @@ +#!/bin/sh +# Initialisation of Weave Net pod: check Linux settings and install CNI plugin + +set -e + +modprobe_safe() { + modprobe $1 || echo "Ignore the error if \"$1\" is built-in in the kernel" >&2 +} + +# Check whether xt_set actually exists +xt_set_exists() { + # Clean everything up in advance, in case there's leftovers + iptables -w -F WEAVE-KUBE-TEST 2>/dev/null || true + iptables -w -X WEAVE-KUBE-TEST 2>/dev/null || true + ipset destroy weave-kube-test 2>/dev/null || true + + ipset create weave-kube-test hash:ip + iptables -w -t filter -N WEAVE-KUBE-TEST + if ! iptables -w -A WEAVE-KUBE-TEST -m set --match-set weave-kube-test src -j DROP; then + NOT_EXIST=1 + fi + iptables -w -F WEAVE-KUBE-TEST + iptables -w -X WEAVE-KUBE-TEST + ipset destroy weave-kube-test + [ -z "$NOT_EXIST" ] || (echo "\"xt_set\" does not exist" >&2 && return 1) +} + +# Default for network policy +EXPECT_NPC=${EXPECT_NPC:-1} + +# Ensure we have the required modules for NPC +if [ "${EXPECT_NPC}" != "0" ]; then + modprobe_safe br_netfilter + modprobe_safe xt_set + xt_set_exists +fi + +# kube-proxy requires that bridged traffic passes through netfilter +if ! BRIDGE_NF_ENABLED=$(cat /proc/sys/net/bridge/bridge-nf-call-iptables); then + echo "Cannot detect bridge-nf support - network policy and iptables mode kubeproxy may not work reliably" >&2 +else + if [ "$BRIDGE_NF_ENABLED" != "1" ]; then + echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables + fi +fi + +# This is where we expect the manifest to map host directories +HOST_ROOT=${HOST_ROOT:-/host} + +# Install CNI plugin binary to typical CNI bin location +# with fall-back to CNI directory used by kube-up on GCI OS +if ! mkdir -p $HOST_ROOT/opt/cni/bin ; then + if mkdir -p $HOST_ROOT/home/kubernetes/bin ; then + export WEAVE_CNI_PLUGIN_DIR=$HOST_ROOT/home/kubernetes/bin + else + echo "Failed to install the Weave CNI plugin" >&2 + exit 1 + fi +fi +mkdir -p $HOST_ROOT/etc/cni/net.d +export HOST_ROOT +/home/weave/weave --local setup-cni \ No newline at end of file diff --git a/prog/weave-kube/launch.sh b/prog/weave-kube/launch.sh index 01ea699b..aec5b1e8 100755 --- a/prog/weave-kube/launch.sh +++ b/prog/weave-kube/launch.sh @@ -1,11 +1,8 @@ #!/bin/sh +# Launch of Weave Net pod - requires that init.sh has been run previously set -e -modprobe_safe() { - modprobe $1 || echo "Ignore the error if \"$1\" is built-in in the kernel" >&2 -} - # Setup iptables backend to be legacy or nftable setup_iptables_backend() { if [ -n "${IPTABLES_BACKEND}" ]; then @@ -34,24 +31,6 @@ setup_iptables_backend() { fi } -# Check whether xt_set actually exists -xt_set_exists() { - # Clean everything up in advance, in case there's leftovers - iptables -w -F WEAVE-KUBE-TEST 2>/dev/null || true - iptables -w -X WEAVE-KUBE-TEST 2>/dev/null || true - ipset destroy weave-kube-test 2>/dev/null || true - - ipset create weave-kube-test hash:ip - iptables -w -t filter -N WEAVE-KUBE-TEST - if ! iptables -w -A WEAVE-KUBE-TEST -m set --match-set weave-kube-test src -j DROP; then - NOT_EXIST=1 - fi - iptables -w -F WEAVE-KUBE-TEST - iptables -w -X WEAVE-KUBE-TEST - ipset destroy weave-kube-test - [ -z "$NOT_EXIST" ] || (echo "\"xt_set\" does not exist" >&2 && return 1) -} - setup_iptables_backend # Default if not supplied - same as weave net default @@ -67,23 +46,6 @@ DB_PREFIX=${DB_PREFIX:-/weavedb/weave-net} # Default for network policy EXPECT_NPC=${EXPECT_NPC:-1} -NO_MASQ_LOCAL=${NO_MASQ_LOCAL:-1} - -# Ensure we have the required modules for NPC -if [ "${EXPECT_NPC}" != "0" ]; then - modprobe_safe br_netfilter - modprobe_safe xt_set - xt_set_exists -fi - -# kube-proxy requires that bridged traffic passes through netfilter -if ! BRIDGE_NF_ENABLED=$(cat /proc/sys/net/bridge/bridge-nf-call-iptables); then - echo "Cannot detect bridge-nf support - network policy and iptables mode kubeproxy may not work reliably" >&2 -else - if [ "$BRIDGE_NF_ENABLED" != "1" ]; then - echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables - fi -fi STATUS_OPTS="--metrics-addr=$METRICS_ADDR" # --status-addr exposes internal information, so only turn it on if asked to. @@ -160,20 +122,6 @@ post_start_actions() { sleep 1 done - # Install CNI plugin binary to typical CNI bin location - # with fall-back to CNI directory used by kube-up on GCI OS - if ! mkdir -p $HOST_ROOT/opt/cni/bin ; then - if mkdir -p $HOST_ROOT/home/kubernetes/bin ; then - export WEAVE_CNI_PLUGIN_DIR=$HOST_ROOT/home/kubernetes/bin - else - echo "Failed to install the Weave CNI plugin" >&2 - exit 1 - fi - fi - mkdir -p $HOST_ROOT/etc/cni/net.d - export HOST_ROOT - /home/weave/weave --local setup-cni - # Attempt to run the reclaim process, but don't halt the script if it fails /home/weave/kube-utils -reclaim -node-name="$HOSTNAME" -peer-name="$PEERNAME" -log-level=debug || true @@ -190,7 +138,6 @@ post_start_actions & /home/weave/weaver $EXTRA_ARGS --port=6783 $(router_bridge_opts) \ --name="$PEERNAME" \ - --host-root=$HOST_ROOT \ --http-addr=$HTTP_ADDR $STATUS_OPTS --docker-api='' --no-dns \ --db-prefix="$DB_PREFIX" \ --ipalloc-range=$IPALLOC_RANGE $NICKNAME_ARG \ diff --git a/prog/weave-kube/weave-daemonset-k8s-1.11.yaml b/prog/weave-kube/weave-daemonset-k8s-1.11.yaml index 4561b072..0ba49b72 100644 --- a/prog/weave-kube/weave-daemonset-k8s-1.11.yaml +++ b/prog/weave-kube/weave-daemonset-k8s-1.11.yaml @@ -118,6 +118,27 @@ items: labels: name: weave-net spec: + initContainers: + - name: weave-init + image: 'weaveworks/weave-kube:latest' + imagePullPolicy: Always + command: + - /home/weave/init.sh + env: + securityContext: + privileged: true + volumeMounts: + - name: cni-bin + mountPath: /host/opt + - name: cni-bin2 + mountPath: /host/home + - name: cni-conf + mountPath: /host/etc + - name: lib-modules + mountPath: /lib/modules + - name: xtables-lock + mountPath: /run/xtables.lock + readOnly: false containers: - name: weave command: @@ -143,16 +164,9 @@ items: volumeMounts: - name: weavedb mountPath: /weavedb - - name: cni-bin - mountPath: /host/opt - - name: cni-bin2 - mountPath: /host/home - - name: cni-conf - mountPath: /host/etc - name: dbus mountPath: /host/var/lib/dbus - - name: lib-modules - mountPath: /lib/modules + readOnly: true - name: xtables-lock mountPath: /run/xtables.lock readOnly: false diff --git a/prog/weave-kube/weave-daemonset-k8s-1.8.yaml b/prog/weave-kube/weave-daemonset-k8s-1.8.yaml index 110a4a88..21c99d8e 100644 --- a/prog/weave-kube/weave-daemonset-k8s-1.8.yaml +++ b/prog/weave-kube/weave-daemonset-k8s-1.8.yaml @@ -115,6 +115,27 @@ items: labels: name: weave-net spec: + initContainers: + - name: weave-init + image: 'weaveworks/weave-kube:latest' + imagePullPolicy: Always + command: + - /home/weave/init.sh + env: + securityContext: + privileged: true + volumeMounts: + - name: cni-bin + mountPath: /host/opt + - name: cni-bin2 + mountPath: /host/home + - name: cni-conf + mountPath: /host/etc + - name: lib-modules + mountPath: /lib/modules + - name: xtables-lock + mountPath: /run/xtables.lock + readOnly: false containers: - name: weave command: @@ -140,16 +161,9 @@ items: volumeMounts: - name: weavedb mountPath: /weavedb - - name: cni-bin - mountPath: /host/opt - - name: cni-bin2 - mountPath: /host/home - - name: cni-conf - mountPath: /host/etc - name: dbus mountPath: /host/var/lib/dbus - - name: lib-modules - mountPath: /lib/modules + readOnly: true - name: xtables-lock mountPath: /run/xtables.lock readOnly: false diff --git a/prog/weave-kube/weave-daemonset-k8s-1.9.yaml b/prog/weave-kube/weave-daemonset-k8s-1.9.yaml index 782ec3de..89c9d698 100644 --- a/prog/weave-kube/weave-daemonset-k8s-1.9.yaml +++ b/prog/weave-kube/weave-daemonset-k8s-1.9.yaml @@ -118,6 +118,27 @@ items: labels: name: weave-net spec: + initContainers: + - name: weave-init + image: 'weaveworks/weave-kube:latest' + imagePullPolicy: Always + command: + - /home/weave/init.sh + env: + securityContext: + privileged: true + volumeMounts: + - name: cni-bin + mountPath: /host/opt + - name: cni-bin2 + mountPath: /host/home + - name: cni-conf + mountPath: /host/etc + - name: lib-modules + mountPath: /lib/modules + - name: xtables-lock + mountPath: /run/xtables.lock + readOnly: false containers: - name: weave command: @@ -143,16 +164,9 @@ items: volumeMounts: - name: weavedb mountPath: /weavedb - - name: cni-bin - mountPath: /host/opt - - name: cni-bin2 - mountPath: /host/home - - name: cni-conf - mountPath: /host/etc - name: dbus mountPath: /host/var/lib/dbus - - name: lib-modules - mountPath: /lib/modules + readOnly: true - name: xtables-lock mountPath: /run/xtables.lock readOnly: false