Merge pull request #3880 from weaveworks/init-container

Kubernetes: move kernel and CNI setup to init container

This allows the main container to run without write access to the host filesystem,
except for two specific areas, thus trimming the attack surface.

Also remove manifests for Kubernetes 1.6 and 1.7 to reduce maintenance load -
they are ancient history.
This commit is contained in:
Bryan Boreham authored and GitHub committed 2021-01-14 12:10:25 +00:00
commit e91900cc9e
10 files changed
+139 -490

No files matched your search

-30
View File
@@ -89,8 +89,6 @@ build() {
## Inject the version numbers and build the distributables
## (library versions?)
sed -i.bak "/SCRIPT_VERSION=/ c\SCRIPT_VERSION=\"$VERSION\"" ./weave
sed -i.bak -e "s/:latest/:$VERSION/" -e "/imagePullPolicy: Always/d" ./prog/weave-kube/weave-daemonset-k8s-1.6.yaml
sed -i.bak -e "s/:latest/:$VERSION/" -e "/imagePullPolicy: Always/d" ./prog/weave-kube/weave-daemonset-k8s-1.7.yaml
sed -i.bak -e "s/:latest/:$VERSION/" -e "/imagePullPolicy: Always/d" ./prog/weave-kube/weave-daemonset-k8s-1.8.yaml
sed -i.bak -e "s/:latest/:$VERSION/" -e "/imagePullPolicy: Always/d" ./prog/weave-kube/weave-daemonset-k8s-1.9.yaml
sed -i.bak -e "s/:latest/:$VERSION/" -e "/imagePullPolicy: Always/d" ./prog/weave-kube/weave-daemonset-k8s-1.11.yaml
@@ -162,20 +160,6 @@ draft() {
--name "weave" \
--file "./weave"
github-release upload \
--user $GITHUB_USER \
--repo weave \
--tag $LATEST_TAG \
--name "weave-daemonset-k8s-1.6.yaml" \
--file "./prog/weave-kube/weave-daemonset-k8s-1.6.yaml"
github-release upload \
--user $GITHUB_USER \
--repo weave \
--tag $LATEST_TAG \
--name "weave-daemonset-k8s-1.7.yaml" \
--file "./prog/weave-kube/weave-daemonset-k8s-1.7.yaml"
github-release upload \
--user $GITHUB_USER \
--repo weave \
@@ -284,20 +268,6 @@ publish() {
--name "weave" \
--file "./weave"
github-release upload \
--user $GITHUB_USER \
--repo weave \
--tag latest_release \
--name "weave-daemonset-k8s-1.6.yaml" \
--file "./prog/weave-kube/weave-daemonset-k8s-1.6.yaml"
github-release upload \
--user $GITHUB_USER \
--repo weave \
--tag latest_release \
--name "weave-daemonset-k8s-1.7.yaml" \
--file "./prog/weave-kube/weave-daemonset-k8s-1.7.yaml"
github-release upload \
--user $GITHUB_USER \
--repo weave \
+1 -1
View File
@@ -6,7 +6,7 @@ LABEL maintainer="Weaveworks <help@weave.works>" \
org.opencontainers.image.source="https://github.com/weaveworks/weave" \
org.opencontainers.image.vendor="Weaveworks"
ADD ./launch.sh ./kube-utils /home/weave/
ADD ./init.sh ./launch.sh ./kube-utils /home/weave/
ENTRYPOINT ["/home/weave/launch.sh"]
# This label will change for every build, and should therefore be the last layer of the image:
+62
View File
@@ -0,0 +1,62 @@
#!/bin/sh
# Initialisation of Weave Net pod: check Linux settings and install CNI plugin
set -e
modprobe_safe() {
modprobe $1 || echo "Ignore the error if \"$1\" is built-in in the kernel" >&2
}
# Check whether xt_set actually exists
xt_set_exists() {
# Clean everything up in advance, in case there's leftovers
iptables -w -F WEAVE-KUBE-TEST 2>/dev/null || true
iptables -w -X WEAVE-KUBE-TEST 2>/dev/null || true
ipset destroy weave-kube-test 2>/dev/null || true
ipset create weave-kube-test hash:ip
iptables -w -t filter -N WEAVE-KUBE-TEST
if ! iptables -w -A WEAVE-KUBE-TEST -m set --match-set weave-kube-test src -j DROP; then
NOT_EXIST=1
fi
iptables -w -F WEAVE-KUBE-TEST
iptables -w -X WEAVE-KUBE-TEST
ipset destroy weave-kube-test
[ -z "$NOT_EXIST" ] || (echo "\"xt_set\" does not exist" >&2 && return 1)
}
# Default for network policy
EXPECT_NPC=${EXPECT_NPC:-1}
# Ensure we have the required modules for NPC
if [ "${EXPECT_NPC}" != "0" ]; then
modprobe_safe br_netfilter
modprobe_safe xt_set
xt_set_exists
fi
# kube-proxy requires that bridged traffic passes through netfilter
if ! BRIDGE_NF_ENABLED=$(cat /proc/sys/net/bridge/bridge-nf-call-iptables); then
echo "Cannot detect bridge-nf support - network policy and iptables mode kubeproxy may not work reliably" >&2
else
if [ "$BRIDGE_NF_ENABLED" != "1" ]; then
echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables
fi
fi
# This is where we expect the manifest to map host directories
HOST_ROOT=${HOST_ROOT:-/host}
# Install CNI plugin binary to typical CNI bin location
# with fall-back to CNI directory used by kube-up on GCI OS
if ! mkdir -p $HOST_ROOT/opt/cni/bin ; then
if mkdir -p $HOST_ROOT/home/kubernetes/bin ; then
export WEAVE_CNI_PLUGIN_DIR=$HOST_ROOT/home/kubernetes/bin
else
echo "Failed to install the Weave CNI plugin" >&2
exit 1
fi
fi
mkdir -p $HOST_ROOT/etc/cni/net.d
export HOST_ROOT
/home/weave/weave --local setup-cni
+3 -53
View File
@@ -1,10 +1,10 @@
#!/bin/sh
# Launch of Weave Net pod - requires that init.sh has been run previously
set -e
modprobe_safe() {
modprobe $1 || echo "Ignore the error if \"$1\" is built-in in the kernel" >&2
}
# If this is run from an older manifest, run the init script here
[ "${INIT_CONTAINER}" = "true" ] || "$(dirname "$0")/init.sh"
# Setup iptables backend to be legacy or nftable
setup_iptables_backend() {
@@ -34,24 +34,6 @@ setup_iptables_backend() {
fi
}
# Check whether xt_set actually exists
xt_set_exists() {
# Clean everything up in advance, in case there's leftovers
iptables -w -F WEAVE-KUBE-TEST 2>/dev/null || true
iptables -w -X WEAVE-KUBE-TEST 2>/dev/null || true
ipset destroy weave-kube-test 2>/dev/null || true
ipset create weave-kube-test hash:ip
iptables -w -t filter -N WEAVE-KUBE-TEST
if ! iptables -w -A WEAVE-KUBE-TEST -m set --match-set weave-kube-test src -j DROP; then
NOT_EXIST=1
fi
iptables -w -F WEAVE-KUBE-TEST
iptables -w -X WEAVE-KUBE-TEST
ipset destroy weave-kube-test
[ -z "$NOT_EXIST" ] || (echo "\"xt_set\" does not exist" >&2 && return 1)
}
setup_iptables_backend
# Default if not supplied - same as weave net default
@@ -67,23 +49,6 @@ DB_PREFIX=${DB_PREFIX:-/weavedb/weave-net}
# Default for network policy
EXPECT_NPC=${EXPECT_NPC:-1}
NO_MASQ_LOCAL=${NO_MASQ_LOCAL:-1}
# Ensure we have the required modules for NPC
if [ "${EXPECT_NPC}" != "0" ]; then
modprobe_safe br_netfilter
modprobe_safe xt_set
xt_set_exists
fi
# kube-proxy requires that bridged traffic passes through netfilter
if ! BRIDGE_NF_ENABLED=$(cat /proc/sys/net/bridge/bridge-nf-call-iptables); then
echo "Cannot detect bridge-nf support - network policy and iptables mode kubeproxy may not work reliably" >&2
else
if [ "$BRIDGE_NF_ENABLED" != "1" ]; then
echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables
fi
fi
STATUS_OPTS="--metrics-addr=$METRICS_ADDR"
# --status-addr exposes internal information, so only turn it on if asked to.
@@ -160,20 +125,6 @@ post_start_actions() {
sleep 1
done
# Install CNI plugin binary to typical CNI bin location
# with fall-back to CNI directory used by kube-up on GCI OS
if ! mkdir -p $HOST_ROOT/opt/cni/bin ; then
if mkdir -p $HOST_ROOT/home/kubernetes/bin ; then
export WEAVE_CNI_PLUGIN_DIR=$HOST_ROOT/home/kubernetes/bin
else
echo "Failed to install the Weave CNI plugin" >&2
exit 1
fi
fi
mkdir -p $HOST_ROOT/etc/cni/net.d
export HOST_ROOT
/home/weave/weave --local setup-cni
# Attempt to run the reclaim process, but don't halt the script if it fails
/home/weave/kube-utils -reclaim -node-name="$HOSTNAME" -peer-name="$PEERNAME" -log-level=debug || true
@@ -190,7 +141,6 @@ post_start_actions &
/home/weave/weaver $EXTRA_ARGS --port=6783 $(router_bridge_opts) \
--name="$PEERNAME" \
--host-root=$HOST_ROOT \
--http-addr=$HTTP_ADDR $STATUS_OPTS --docker-api='' --no-dns \
--db-prefix="$DB_PREFIX" \
--ipalloc-range=$IPALLOC_RANGE $NICKNAME_ARG \
+24 -8
View File
@@ -118,11 +118,34 @@ items:
labels:
name: weave-net
spec:
initContainers:
- name: weave-init
image: 'weaveworks/weave-kube:latest'
imagePullPolicy: Always
command:
- /home/weave/init.sh
env:
securityContext:
privileged: true
volumeMounts:
- name: cni-bin
mountPath: /host/opt
- name: cni-bin2
mountPath: /host/home
- name: cni-conf
mountPath: /host/etc
- name: lib-modules
mountPath: /lib/modules
- name: xtables-lock
mountPath: /run/xtables.lock
readOnly: false
containers:
- name: weave
command:
- /home/weave/launch.sh
env:
- name: INIT_CONTAINER
value: "true"
- name: HOSTNAME
valueFrom:
fieldRef:
@@ -143,16 +166,9 @@ items:
volumeMounts:
- name: weavedb
mountPath: /weavedb
- name: cni-bin
mountPath: /host/opt
- name: cni-bin2
mountPath: /host/home
- name: cni-conf
mountPath: /host/etc
- name: dbus
mountPath: /host/var/lib/dbus
- name: lib-modules
mountPath: /lib/modules
readOnly: true
- name: xtables-lock
mountPath: /run/xtables.lock
readOnly: false
@@ -1,181 +0,0 @@
apiVersion: v1
kind: List
items:
- apiVersion: v1
kind: ServiceAccount
metadata:
name: weave-net
labels:
name: weave-net
namespace: kube-system
- apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: weave-net
labels:
name: weave-net
rules:
- apiGroups:
- ''
resources:
- pods
- namespaces
- nodes
verbs:
- get
- list
- watch
- apiGroups:
- extensions
resources:
- networkpolicies
verbs:
- get
- list
- watch
- apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: weave-net
labels:
name: weave-net
roleRef:
kind: ClusterRole
name: weave-net
apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
name: weave-net
namespace: kube-system
- apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
name: weave-net-kube-peer
namespace: kube-system
labels:
name: weave-net-kube-peer
rules:
- apiGroups:
- ''
resources:
- configmaps
resourceNames:
- weave-net
verbs:
- get
- update
- apiGroups:
- ''
resources:
- configmaps
verbs:
- create
- apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
name: weave-net-kube-peer
namespace: kube-system
labels:
name: weave-net-kube-peer
roleRef:
kind: Role
name: weave-net-kube-peer
apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
name: weave-net
namespace: kube-system
- apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: weave-net
labels:
name: weave-net
namespace: kube-system
spec:
template:
metadata:
labels:
name: weave-net
spec:
containers:
- name: weave
command:
- /home/weave/launch.sh
env:
- name: HOSTNAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
image: 'weaveworks/weave-kube:latest'
imagePullPolicy: Always
readinessProbe:
httpGet:
host: 127.0.0.1
path: /status
port: 6784
resources:
requests:
cpu: 50m
securityContext:
privileged: true
volumeMounts:
- name: weavedb
mountPath: /weavedb
- name: cni-bin
mountPath: /host/opt
- name: cni-bin2
mountPath: /host/home
- name: cni-conf
mountPath: /host/etc
- name: dbus
mountPath: /host/var/lib/dbus
- name: lib-modules
mountPath: /lib/modules
- name: weave-npc
env:
- name: HOSTNAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
image: 'weaveworks/weave-npc:latest'
imagePullPolicy: Always
resources:
requests:
cpu: 50m
securityContext:
privileged: true
hostNetwork: true
hostPID: false
restartPolicy: Always
securityContext:
seLinuxOptions: {}
serviceAccountName: weave-net
tolerations:
- effect: NoSchedule
operator: Exists
- effect: NoExecute
operator: Exists
volumes:
- name: weavedb
hostPath:
path: /var/lib/weave
- name: cni-bin
hostPath:
path: /opt
- name: cni-bin2
hostPath:
path: /home
- name: cni-conf
hostPath:
path: /etc
- name: dbus
hostPath:
path: /var/lib/dbus
- name: lib-modules
hostPath:
path: /lib/modules
updateStrategy:
type: RollingUpdate
@@ -1,200 +0,0 @@
apiVersion: v1
kind: List
items:
- apiVersion: v1
kind: ServiceAccount
metadata:
name: weave-net
labels:
name: weave-net
namespace: kube-system
- apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: weave-net
labels:
name: weave-net
rules:
- apiGroups:
- ''
resources:
- pods
- namespaces
- nodes
verbs:
- get
- list
- watch
- apiGroups:
- 'networking.k8s.io'
resources:
- networkpolicies
verbs:
- get
- list
- watch
- apiGroups:
- ''
resources:
- nodes/status
verbs:
- patch
- update
- apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: weave-net
labels:
name: weave-net
roleRef:
kind: ClusterRole
name: weave-net
apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
name: weave-net
namespace: kube-system
- apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
name: weave-net
namespace: kube-system
labels:
name: weave-net
rules:
- apiGroups:
- ''
resources:
- configmaps
resourceNames:
- weave-net
verbs:
- get
- update
- apiGroups:
- ''
resources:
- configmaps
verbs:
- create
- apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
name: weave-net
namespace: kube-system
labels:
name: weave-net
roleRef:
kind: Role
name: weave-net
apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
name: weave-net
namespace: kube-system
- apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: weave-net
labels:
name: weave-net
namespace: kube-system
spec:
# Wait 5 seconds to let pod connect before rolling next pod
minReadySeconds: 5
template:
metadata:
labels:
name: weave-net
spec:
containers:
- name: weave
command:
- /home/weave/launch.sh
env:
- name: HOSTNAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
image: 'weaveworks/weave-kube:latest'
imagePullPolicy: Always
readinessProbe:
httpGet:
host: 127.0.0.1
path: /status
port: 6784
resources:
requests:
cpu: 50m
securityContext:
privileged: true
volumeMounts:
- name: weavedb
mountPath: /weavedb
- name: cni-bin
mountPath: /host/opt
- name: cni-bin2
mountPath: /host/home
- name: cni-conf
mountPath: /host/etc
- name: dbus
mountPath: /host/var/lib/dbus
- name: lib-modules
mountPath: /lib/modules
- name: xtables-lock
mountPath: /run/xtables.lock
readOnly: false
- name: weave-npc
env:
- name: HOSTNAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
image: 'weaveworks/weave-npc:latest'
imagePullPolicy: Always
resources:
requests:
cpu: 50m
securityContext:
privileged: true
volumeMounts:
- name: xtables-lock
mountPath: /run/xtables.lock
readOnly: false
hostNetwork: true
hostPID: false
restartPolicy: Always
securityContext:
seLinuxOptions: {}
serviceAccountName: weave-net
tolerations:
- effect: NoSchedule
operator: Exists
- effect: NoExecute
operator: Exists
volumes:
- name: weavedb
hostPath:
path: /var/lib/weave
- name: cni-bin
hostPath:
path: /opt
- name: cni-bin2
hostPath:
path: /home
- name: cni-conf
hostPath:
path: /etc
- name: dbus
hostPath:
path: /var/lib/dbus
- name: lib-modules
hostPath:
path: /lib/modules
- name: xtables-lock
hostPath:
path: /run/xtables.lock
updateStrategy:
type: RollingUpdate
+24 -8
View File
@@ -115,11 +115,34 @@ items:
labels:
name: weave-net
spec:
initContainers:
- name: weave-init
image: 'weaveworks/weave-kube:latest'
imagePullPolicy: Always
command:
- /home/weave/init.sh
env:
securityContext:
privileged: true
volumeMounts:
- name: cni-bin
mountPath: /host/opt
- name: cni-bin2
mountPath: /host/home
- name: cni-conf
mountPath: /host/etc
- name: lib-modules
mountPath: /lib/modules
- name: xtables-lock
mountPath: /run/xtables.lock
readOnly: false
containers:
- name: weave
command:
- /home/weave/launch.sh
env:
- name: INIT_CONTAINER
value: "true"
- name: HOSTNAME
valueFrom:
fieldRef:
@@ -140,16 +163,9 @@ items:
volumeMounts:
- name: weavedb
mountPath: /weavedb
- name: cni-bin
mountPath: /host/opt
- name: cni-bin2
mountPath: /host/home
- name: cni-conf
mountPath: /host/etc
- name: dbus
mountPath: /host/var/lib/dbus
- name: lib-modules
mountPath: /lib/modules
readOnly: true
- name: xtables-lock
mountPath: /run/xtables.lock
readOnly: false
+24 -8
View File
@@ -118,11 +118,34 @@ items:
labels:
name: weave-net
spec:
initContainers:
- name: weave-init
image: 'weaveworks/weave-kube:latest'
imagePullPolicy: Always
command:
- /home/weave/init.sh
env:
securityContext:
privileged: true
volumeMounts:
- name: cni-bin
mountPath: /host/opt
- name: cni-bin2
mountPath: /host/home
- name: cni-conf
mountPath: /host/etc
- name: lib-modules
mountPath: /lib/modules
- name: xtables-lock
mountPath: /run/xtables.lock
readOnly: false
containers:
- name: weave
command:
- /home/weave/launch.sh
env:
- name: INIT_CONTAINER
value: "true"
- name: HOSTNAME
valueFrom:
fieldRef:
@@ -143,16 +166,9 @@ items:
volumeMounts:
- name: weavedb
mountPath: /weavedb
- name: cni-bin
mountPath: /host/opt
- name: cni-bin2
mountPath: /host/home
- name: cni-conf
mountPath: /host/etc
- name: dbus
mountPath: /host/var/lib/dbus
- name: lib-modules
mountPath: /lib/modules
readOnly: true
- name: xtables-lock
mountPath: /run/xtables.lock
readOnly: false
@@ -50,7 +50,7 @@ function setup_kubernetes_cluster {
# Ensure Kubernetes uses locally built container images and inject code coverage environment variable (or do nothing depending on $COVERAGE):
sed -e "s%imagePullPolicy: Always%imagePullPolicy: Never%" \
-e "s%env:%$COVERAGE_ARGS%" \
"$(dirname "$0")/../prog/weave-kube/weave-daemonset-k8s-1.7.yaml" | run_on "$HOST1" "$KUBECTL apply -n kube-system -f -"
"$(dirname "$0")/../prog/weave-kube/weave-daemonset-k8s-1.11.yaml" | run_on "$HOST1" "$KUBECTL apply -n kube-system -f -"
}
function force_drop_node {