lb: EgressService - a LoadBalancer Service's pods leave from its address
images / images (push) Successful in 18m34s
images / images (push) Successful in 18m34s
EgressService (reweave.levk.dev/v1alpha1, named after a LoadBalancer Service of class reweave.levk.dev/node-address) makes every pod the Service selects (by pod IP, never readiness) leave the cluster from the Service's address, or not at all. spec.emit is explicit: Holder (the only mode so far) tunnels members' packets to the node holding the address, which emits them. The holder (S1's: the one node with the address on an interface) is the status' sole writer: host, address, and a 512-port block per node in 61000-65535 (above the default ephemeral range), stable across churn. A member's node SNATs it to address:its-block before it leaves, so the holder forwards it unchanged and routes replies back by port alone: ip rules (to address, ipproto tcp/udp, dport block) ahead of the local table, which moves from 0 to 100, since the address is local on the holder. Members' packets travel both ways in reweave-eg, an ipip tunnel between weave bridges (NPC drops new routed packets into weave), with accept_local and loose rp_filter on that device only. Members' new connections are classed at the tail of nat PREROUTING, after kube-proxy's DNAT has ended the nat walk for Services: the pod network, multicast and the nodes' addresses are not outside. Marked packets follow a per-member rule to the holder's table (or to an unreachable one), are SNATed into the tunnel, and a FORWARD guard drops anything else a member sends outside (other protocols, other ways out). The tunnel is accepted only from the nodes' bridges, a holder accepts from it only the blocks' ports, and TCP MSS is clamped into it. Fail closed throughout: until the holder has published the status, without a block, with no single holder, or for a pod two EgressServices select, a member's outside traffic is dropped. The CNI plugin asks weave-npc to admit every pod (unix socket /run/weave-npc/egress.sock, conflist "egressAdmission": true, now rewritten when it differs) after IPAM and before the pod has an interface, so a member's first packet already leaves from the address; no answer, no pod. weave-npc applies nothing about egress from caches that are not synced (unless the API is absent): a restart never strips members' rules. reweave/test/egress/run.sh checks the design's assertions against a live cluster; both end-to-end scripts now re-evaluate each check on every retry (it was expanded once, before the loop). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UHXzGrsxfkGk8LmwrBbHNZ
This commit is contained in:
1 parent
8018ea4bf9
commit
305ac4f3d3
15 files changed
+1866
-64
No files matched your search
@@ -0,0 +1,88 @@
|
||||
// Package admit is the protocol between the weave CNI plugin and weave-npc's egress agent: at CNI ADD the
|
||||
// plugin tells the agent which pod got which IP, and the agent answers once the pod's egress rules are in
|
||||
// place, so an EgressService member's first packet already leaves from the Service's address (or not at all).
|
||||
//
|
||||
// It depends on the standard library only: the plugin binary must not grow a Kubernetes client.
|
||||
package admit
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SocketPath is where the agent listens (a hostPath directory shared with the host, where the plugin runs).
|
||||
const SocketPath = "/run/weave-npc/egress.sock"
|
||||
|
||||
// Request is one pod's admission: the IP the plugin just allocated to it.
|
||||
type Request struct {
|
||||
Namespace string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
IP string `json:"ip"`
|
||||
}
|
||||
|
||||
// Response says whether the pod is an EgressService member (its rules are installed either way when 200).
|
||||
type Response struct {
|
||||
Member bool `json:"member"`
|
||||
}
|
||||
|
||||
// Timeout bounds one admission (the agent may have to fetch the pod from the API server).
|
||||
const Timeout = 20 * time.Second
|
||||
|
||||
// PodFromArgs reads the pod's namespace and name from CNI_ARGS ("K8S_POD_NAMESPACE=..;K8S_POD_NAME=..").
|
||||
func PodFromArgs(args string) (namespace, name string) {
|
||||
for _, kv := range strings.Split(args, ";") {
|
||||
k, v, ok := strings.Cut(kv, "=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
switch k {
|
||||
case "K8S_POD_NAMESPACE":
|
||||
namespace = v
|
||||
case "K8S_POD_NAME":
|
||||
name = v
|
||||
}
|
||||
}
|
||||
return namespace, name
|
||||
}
|
||||
|
||||
// Admit asks the agent at socket to admit req, and returns only once the agent has installed the pod's rules.
|
||||
func Admit(ctx context.Context, socket string, req Request) (Response, error) {
|
||||
var resp Response
|
||||
body, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return resp, err
|
||||
}
|
||||
client := &http.Client{Transport: &http.Transport{
|
||||
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
var d net.Dialer
|
||||
return d.DialContext(ctx, "unix", socket)
|
||||
},
|
||||
}}
|
||||
ctx, cancel := context.WithTimeout(ctx, Timeout)
|
||||
defer cancel()
|
||||
hreq, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://weave-npc/admit", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return resp, err
|
||||
}
|
||||
hreq.Header.Set("Content-Type", "application/json")
|
||||
hresp, err := client.Do(hreq)
|
||||
if err != nil {
|
||||
return resp, fmt.Errorf("egress admission (weave-npc at %s): %v", socket, err)
|
||||
}
|
||||
defer hresp.Body.Close()
|
||||
out, _ := io.ReadAll(io.LimitReader(hresp.Body, 1<<16))
|
||||
if hresp.StatusCode != http.StatusOK {
|
||||
return resp, fmt.Errorf("egress admission refused (%s): %s", hresp.Status, strings.TrimSpace(string(out)))
|
||||
}
|
||||
if err := json.Unmarshal(out, &resp); err != nil {
|
||||
return resp, fmt.Errorf("egress admission: bad answer: %v", err)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
Reference in new issue
Block a user