images / images (push) Successful in 18m34s
EgressService (reweave.levk.dev/v1alpha1, named after a LoadBalancer Service of class reweave.levk.dev/node-address) makes every pod the Service selects (by pod IP, never readiness) leave the cluster from the Service's address, or not at all. spec.emit is explicit: Holder (the only mode so far) tunnels members' packets to the node holding the address, which emits them. The holder (S1's: the one node with the address on an interface) is the status' sole writer: host, address, and a 512-port block per node in 61000-65535 (above the default ephemeral range), stable across churn. A member's node SNATs it to address:its-block before it leaves, so the holder forwards it unchanged and routes replies back by port alone: ip rules (to address, ipproto tcp/udp, dport block) ahead of the local table, which moves from 0 to 100, since the address is local on the holder. Members' packets travel both ways in reweave-eg, an ipip tunnel between weave bridges (NPC drops new routed packets into weave), with accept_local and loose rp_filter on that device only. Members' new connections are classed at the tail of nat PREROUTING, after kube-proxy's DNAT has ended the nat walk for Services: the pod network, multicast and the nodes' addresses are not outside. Marked packets follow a per-member rule to the holder's table (or to an unreachable one), are SNATed into the tunnel, and a FORWARD guard drops anything else a member sends outside (other protocols, other ways out). The tunnel is accepted only from the nodes' bridges, a holder accepts from it only the blocks' ports, and TCP MSS is clamped into it. Fail closed throughout: until the holder has published the status, without a block, with no single holder, or for a pod two EgressServices select, a member's outside traffic is dropped. The CNI plugin asks weave-npc to admit every pod (unix socket /run/weave-npc/egress.sock, conflist "egressAdmission": true, now rewritten when it differs) after IPAM and before the pod has an interface, so a member's first packet already leaves from the address; no answer, no pod. weave-npc applies nothing about egress from caches that are not synced (unless the API is absent): a restart never strips members' rules. reweave/test/egress/run.sh checks the design's assertions against a live cluster; both end-to-end scripts now re-evaluate each check on every retry (it was expanded once, before the loop). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UHXzGrsxfkGk8LmwrBbHNZ
297 lines
13 KiB
Go
297 lines
13 KiB
Go
package lb
|
|
|
|
import (
|
|
"net"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"golang.org/x/sys/unix"
|
|
coreapi "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
)
|
|
|
|
// the S1 fixtures: cp, w1 (holds 86.107.168.15), w2
|
|
|
|
func egressFor(name string, status EgressServiceStatus) *EgressService {
|
|
return &EgressService{ObjectMeta: metav1.ObjectMeta{Namespace: "gitea", Name: name},
|
|
Spec: EgressServiceSpec{Emit: EmitHolder}, Status: status}
|
|
}
|
|
|
|
func mailSvc() *coreapi.Service {
|
|
return svc("mail", "86.107.168.15", func(s *coreapi.Service) {
|
|
s.Spec.Selector = map[string]string{"app": "mail"}
|
|
})
|
|
}
|
|
|
|
func pod(name, node, ip string, labels map[string]string, mutate ...func(*coreapi.Pod)) *coreapi.Pod {
|
|
p := &coreapi.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "gitea", Name: name, Labels: labels,
|
|
CreationTimestamp: metav1.NewTime(time.Unix(1000, 0))},
|
|
Spec: coreapi.PodSpec{NodeName: node}, Status: coreapi.PodStatus{PodIP: ip, Phase: coreapi.PodRunning}}
|
|
for _, m := range mutate {
|
|
m(p)
|
|
}
|
|
return p
|
|
}
|
|
|
|
var mailLabels = map[string]string{"app": "mail"}
|
|
|
|
// the status the holder (w1) writes for three nodes, blocks in name order
|
|
func written() EgressServiceStatus {
|
|
return EgressServiceStatus{Host: "w1", Address: "86.107.168.15", Blocks: []PortBlock{
|
|
{Node: "cp", From: 61000, To: 61511}, {Node: "w1", From: 61512, To: 62023}, {Node: "w2", From: 62024, To: 62535}}}
|
|
}
|
|
|
|
func TestBlocksAreStableAndCapped(t *testing.T) {
|
|
got := AllocateBlocks(nil, []string{"w2", "cp", "w1"})
|
|
if !sameStatus(EgressServiceStatus{Blocks: got}, EgressServiceStatus{Blocks: written().Blocks}) {
|
|
t.Fatalf("fresh blocks: %+v", got)
|
|
}
|
|
// a node leaves, another joins: survivors keep their blocks, the newcomer takes the freed lowest one
|
|
got = AllocateBlocks(got, []string{"w1", "w2", "w3"})
|
|
want := []PortBlock{{"w1", 61512, 62023}, {"w2", 62024, 62535}, {"w3", 61000, 61511}}
|
|
if !sameStatus(EgressServiceStatus{Blocks: got}, EgressServiceStatus{Blocks: want}) {
|
|
t.Fatalf("after churn: %+v", got)
|
|
}
|
|
// misaligned or duplicate blocks are not kept
|
|
got = AllocateBlocks([]PortBlock{{"w1", 61001, 61512}, {"w2", 61000, 61511}, {"w3", 61000, 61511}}, []string{"w1", "w2", "w3"})
|
|
for _, b := range got {
|
|
if (b.From-BlockBase)%BlockSize != 0 || b.To != b.From+BlockSize-1 {
|
|
t.Fatalf("invalid block kept: %+v", got)
|
|
}
|
|
}
|
|
if got[1].Node != "w2" || got[1].From != 61000 {
|
|
t.Fatalf("first valid claim wins: %+v", got)
|
|
}
|
|
var many []string
|
|
for i := 0; i < BlockCount+2; i++ {
|
|
many = append(many, string(rune('a'+i)))
|
|
}
|
|
if n := len(AllocateBlocks(nil, many)); n != BlockCount {
|
|
t.Fatalf("more blocks than exist: %d", n)
|
|
}
|
|
for _, b := range AllocateBlocks(nil, many) {
|
|
if b.To > 65535 || b.From < 61000 {
|
|
t.Fatalf("block outside the range: %+v", b)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestHolderWritesStatusAndMembersWaitForIt(t *testing.T) {
|
|
e := egressFor("mail", EgressServiceStatus{})
|
|
pods := []*coreapi.Pod{pod("mail-0", "w2", "10.44.0.9", mailLabels)}
|
|
d := ComputeEgress("w1", nodes, []*coreapi.Service{mailSvc()}, []*EgressService{e}, nil, nil)
|
|
if len(d.Status) != 1 || !sameStatus(d.Status[0].Status, written()) {
|
|
t.Fatalf("holder's status: %+v", d.Status)
|
|
}
|
|
// until the status is in the API, a member's outside traffic goes nowhere (fail closed)
|
|
d = ComputeEgress("w2", nodes, []*coreapi.Service{mailSvc()}, []*EgressService{e}, pods, nil)
|
|
if len(d.Status) != 0 || len(d.Members) != 1 || !d.Members[0].Unreachable {
|
|
t.Fatalf("member before status: %+v", d)
|
|
}
|
|
if d.Tunnelled("w2") {
|
|
t.Fatalf("no tunnel for an unreachable member")
|
|
}
|
|
}
|
|
|
|
func TestMemberTunnelsToHolderWhichRoutesItsBlockBack(t *testing.T) {
|
|
e := egressFor("mail", written())
|
|
svcs := []*coreapi.Service{mailSvc()}
|
|
pods := []*coreapi.Pod{
|
|
pod("mail-0", "w2", "10.44.0.9", mailLabels),
|
|
pod("web", "w2", "10.44.0.10", map[string]string{"app": "web"}), // not a member (A4)
|
|
}
|
|
d := ComputeEgress("w2", nodes, svcs, []*EgressService{e}, pods, nil)
|
|
if len(d.Members) != 1 || d.Members[0].Pod != "gitea/mail-0" {
|
|
t.Fatalf("members: %+v", d.Members)
|
|
}
|
|
m := d.Members[0]
|
|
if m.Unreachable || !m.Address.Equal(net.ParseIP("86.107.168.15")) || m.From != 62024 || m.To != 62535 || m.Holder.Name != "w1" {
|
|
t.Fatalf("member path: %+v", m)
|
|
}
|
|
if !d.Tunnelled("w2") || len(d.Peers("w2")) != 1 || d.Peers("w2")[0].Name != "w1" {
|
|
t.Fatalf("w2 tunnels to w1 only: %+v", d.Peers("w2"))
|
|
}
|
|
if len(d.Status) != 0 || len(d.Routes) != 0 {
|
|
t.Fatalf("only the holder writes and routes: %+v", d)
|
|
}
|
|
// the holder: blocks of the other nodes route back to them; its own block is plain conntrack
|
|
h := ComputeEgress("w1", nodes, svcs, []*EgressService{e}, nil, nil)
|
|
if len(h.Status) != 0 {
|
|
t.Fatalf("status already right: %+v", h.Status)
|
|
}
|
|
if len(h.Routes) != 2 || h.Routes[0].Peer.Name != "cp" || h.Routes[1].Peer.Name != "w2" || h.Routes[1].From != 62024 {
|
|
t.Fatalf("holder routes: %+v", h.Routes)
|
|
}
|
|
// a member on the holder leaves from there
|
|
hm := ComputeEgress("w1", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{pod("mail-1", "w1", "10.40.0.4", mailLabels)}, nil)
|
|
if len(hm.Members) != 1 || hm.Members[0].Holder.Name != "w1" || hm.Members[0].From != 61512 {
|
|
t.Fatalf("member on the holder: %+v", hm.Members)
|
|
}
|
|
}
|
|
|
|
func TestEgressFailsClosed(t *testing.T) {
|
|
pods := []*coreapi.Pod{pod("mail-0", "w2", "10.44.0.9", mailLabels)}
|
|
// no node holds the address: status cleared (by the first node by name), member dropped
|
|
unheld := svc("mail", "86.107.168.99", func(s *coreapi.Service) { s.Spec.Selector = mailLabels })
|
|
for _, self := range []string{"cp", "w2"} {
|
|
d := ComputeEgress(self, nodes, []*coreapi.Service{unheld}, []*EgressService{egressFor("mail", written())}, pods, nil)
|
|
clears := len(d.Status) == 1 && d.Status[0].Status.Host == ""
|
|
if clears != (self == "cp") {
|
|
t.Fatalf("%s clears=%v: %+v", self, clears, d.Status)
|
|
}
|
|
if self == "w2" && (len(d.Members) != 1 || !d.Members[0].Unreachable) {
|
|
t.Fatalf("member of an unheld address: %+v", d.Members)
|
|
}
|
|
}
|
|
// two EgressServices select the pod: dropped, reported
|
|
other := svc("mail2", "86.107.168.15", func(s *coreapi.Service) { s.Spec.Selector = mailLabels })
|
|
d := ComputeEgress("w2", nodes, []*coreapi.Service{mailSvc(), other},
|
|
[]*EgressService{egressFor("mail", written()), egressFor("mail2", written())}, pods, nil)
|
|
if len(d.Members) != 1 || !d.Members[0].Unreachable || !strings.Contains(strings.Join(d.Problems, "\n"), "several EgressServices") {
|
|
t.Fatalf("double member: %+v", d)
|
|
}
|
|
// a spec this agent does not implement is refused, not guessed at
|
|
bad := egressFor("mail", written())
|
|
bad.Spec.Emit = "Direct"
|
|
d = ComputeEgress("w2", nodes, []*coreapi.Service{mailSvc()}, []*EgressService{bad}, pods, nil)
|
|
if len(d.Members) != 1 || !d.Members[0].Unreachable {
|
|
t.Fatalf("unknown emit mode: %+v", d)
|
|
}
|
|
// hostNetwork pods are never members (their traffic is the node's)
|
|
hn := pod("mail-h", "w2", "86.107.168.88", mailLabels, func(p *coreapi.Pod) { p.Spec.HostNetwork = true })
|
|
d = ComputeEgress("w2", nodes, []*coreapi.Service{mailSvc()}, []*EgressService{egressFor("mail", written())}, []*coreapi.Pod{hn}, nil)
|
|
if len(d.Members) != 0 {
|
|
t.Fatalf("hostNetwork pod became a member: %+v", d.Members)
|
|
}
|
|
}
|
|
|
|
func TestAdmissionDecidesWhoOwnsAnIP(t *testing.T) {
|
|
e := egressFor("mail", written())
|
|
svcs := []*coreapi.Service{mailSvc()}
|
|
old := pod("mail-0", "w2", "10.44.0.9", mailLabels, func(p *coreapi.Pod) {
|
|
now := metav1.Now()
|
|
p.DeletionTimestamp = &now
|
|
})
|
|
newer := pod("web", "w2", "", map[string]string{"app": "web"}) // API has no IP for it yet
|
|
// the CNI plugin just gave 10.44.0.9 to web: the terminating member must not keep it (A4)
|
|
d := ComputeEgress("w2", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{old, newer},
|
|
map[string]Admission{"10.44.0.9": {Namespace: "gitea", Name: "web"}})
|
|
if len(d.Members) != 0 {
|
|
t.Fatalf("reused IP kept the old member's rules: %+v", d.Members)
|
|
}
|
|
// and the other way round: a member admitted before the API shows its IP is protected at once (A1)
|
|
fresh := pod("mail-1", "w2", "", mailLabels)
|
|
d = ComputeEgress("w2", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{fresh},
|
|
map[string]Admission{"10.44.0.11": {Namespace: "gitea", Name: "mail-1"}})
|
|
if len(d.Members) != 1 || d.Members[0].IP.String() != "10.44.0.11" || d.Members[0].Unreachable {
|
|
t.Fatalf("admitted member: %+v", d.Members)
|
|
}
|
|
// without an admission, a live pod wins a shared IP over a terminating one
|
|
live := pod("web", "w2", "10.44.0.9", map[string]string{"app": "web"})
|
|
d = ComputeEgress("w2", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{old, live}, nil)
|
|
if len(d.Members) != 0 {
|
|
t.Fatalf("terminating member won the IP: %+v", d.Members)
|
|
}
|
|
}
|
|
|
|
func TestRenderedEgressRules(t *testing.T) {
|
|
_, podNet, _ := net.ParseCIDR("10.32.0.0/12")
|
|
e := egressFor("mail", written())
|
|
svcs := []*coreapi.Service{mailSvc()}
|
|
member := ComputeEgress("w2", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{pod("mail-0", "w2", "10.44.0.9", mailLabels)}, nil)
|
|
r := Render(Desired{}, member, "w2", podNet, nil)
|
|
for _, want := range []string{
|
|
"-A REWEAVE-LB-CLASS -d 10.32.0.0/12 -j RETURN\n",
|
|
"-A REWEAVE-LB-CLASS -d 86.107.168.24/32 -j RETURN\n", // a node is not outside
|
|
"-A REWEAVE-LB-CLASS -s 10.44.0.9/32 -m comment --comment \"gitea/mail-0\" -j CONNMARK --set-xmark 0x100000/0x100000\n",
|
|
"-A REWEAVE-LB-CLASS -s 10.44.0.9/32 -j MARK --set-xmark 0x100000/0x100000\n",
|
|
"-A REWEAVE-LB-POST -s 10.44.0.9/32 -o reweave-eg -m mark --mark 0x100000/0x100000 -p tcp -j SNAT --to-source 86.107.168.15:62024-62535\n",
|
|
"-A REWEAVE-LB-POST -s 10.44.0.9/32 -o reweave-eg -m mark --mark 0x100000/0x100000 -p udp -j SNAT --to-source 86.107.168.15:62024-62535\n",
|
|
} {
|
|
if !strings.Contains(r.Nat, want) {
|
|
t.Fatalf("member nat lacks %q:\n%s", want, r.Nat)
|
|
}
|
|
}
|
|
for _, want := range []string{
|
|
"-A REWEAVE-LB-FWD -s 10.44.0.9/32 -o reweave-eg -m mark --mark 0x100000/0x100000 -p tcp -j RETURN\n",
|
|
"-A REWEAVE-LB-FWD -s 10.44.0.9/32 -m mark --mark 0x100000/0x100000 -m comment --comment \"gitea/mail-0\" -j DROP\n",
|
|
"-A REWEAVE-LB-FWD -i reweave-eg -d 86.107.168.15/32 -j DROP\n",
|
|
"-A REWEAVE-LB-FWD -i reweave-eg -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n",
|
|
"-A REWEAVE-LB-FWD -i reweave-eg -j DROP\n",
|
|
"-A REWEAVE-LB-IN -p 4 -s 10.40.0.0/32 -m mac --mac-source aa:aa:aa:aa:aa:02 -j RETURN\n",
|
|
"-A REWEAVE-LB-IN -p 4 -j DROP\n",
|
|
} {
|
|
if !strings.Contains(r.Filter, want) {
|
|
t.Fatalf("member filter lacks %q:\n%s", want, r.Filter)
|
|
}
|
|
}
|
|
for _, want := range []string{
|
|
"-A REWEAVE-LB-MARK -m conntrack --ctdir ORIGINAL -m connmark --mark 0x100000/0x100000 -j MARK --set-xmark 0x100000/0x100000\n",
|
|
"-A REWEAVE-LB-MSS -o reweave-eg -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu\n",
|
|
} {
|
|
if !strings.Contains(r.Mangle, want) {
|
|
t.Fatalf("member mangle lacks %q:\n%s", want, r.Mangle)
|
|
}
|
|
}
|
|
// the guard's DROP comes after the member's own RETURNs: nothing else of its leaves
|
|
if strings.Index(r.Filter, "-p tcp -j RETURN") > strings.Index(r.Filter, "\"gitea/mail-0\" -j DROP") {
|
|
t.Fatalf("guard order:\n%s", r.Filter)
|
|
}
|
|
// the holder: accepts each node's block from the tunnel, drops the rest; a member on it SNATs to its own block
|
|
h := ComputeEgress("w1", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{pod("mail-1", "w1", "10.40.0.4", mailLabels)}, nil)
|
|
r = Render(Desired{}, h, "w1", podNet, nil)
|
|
for _, want := range []string{
|
|
"-A REWEAVE-LB-FWD -i reweave-eg -s 86.107.168.15/32 -p tcp -m tcp --sport 62024:62535 -m comment --comment \"block of w2\" -j RETURN\n",
|
|
"-A REWEAVE-LB-FWD -i reweave-eg -s 86.107.168.15/32 -p udp -m udp --sport 61000:61511 -m comment --comment \"block of cp\" -j RETURN\n",
|
|
"-A REWEAVE-LB-FWD -s 10.40.0.4/32 -o weave -m mark --mark 0x100000/0x100000 -j DROP\n",
|
|
} {
|
|
if !strings.Contains(r.Filter, want) {
|
|
t.Fatalf("holder filter lacks %q:\n%s", want, r.Filter)
|
|
}
|
|
}
|
|
if !strings.Contains(r.Nat, "-A REWEAVE-LB-POST -s 10.40.0.4/32 ! -o weave -m mark --mark 0x100000/0x100000 -p tcp -j SNAT --to-source 86.107.168.15:61512-62023\n") {
|
|
t.Fatalf("holder's own member:\n%s", r.Nat)
|
|
}
|
|
for _, line := range strings.Split(r.Nat+r.Filter, "\n") {
|
|
if strings.Contains(line, "10.44.0.10") {
|
|
t.Fatalf("a rule names a non-member: %s", line)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPolicyRules(t *testing.T) {
|
|
e := egressFor("mail", written())
|
|
svcs := []*coreapi.Service{mailSvc()}
|
|
h := ComputeEgress("w1", nodes, svcs, []*EgressService{e}, nil, nil)
|
|
rules := wantRules(h, "w1")
|
|
tables := peerTables(h, "w1")
|
|
if len(rules) != 4 {
|
|
t.Fatalf("2 peers x tcp,udp: %+v", rules)
|
|
}
|
|
for _, r := range rules {
|
|
if r.priority != HolderRulePriority || r.dst != "86.107.168.15/32" || r.src != "" || r.mark != 0 {
|
|
t.Fatalf("holder rule: %+v", r)
|
|
}
|
|
if r.ipproto != unix.IPPROTO_TCP && r.ipproto != unix.IPPROTO_UDP {
|
|
t.Fatalf("protocol: %+v", r)
|
|
}
|
|
if (r.from == 62024) != (r.table == tables["w2"]) || (r.from == 61000) != (r.table == tables["cp"]) {
|
|
t.Fatalf("block routed to the wrong node: %+v %v", r, tables)
|
|
}
|
|
}
|
|
if HolderRulePriority >= LocalRulePriority {
|
|
t.Fatalf("holder rules must precede the local table")
|
|
}
|
|
m := ComputeEgress("w2", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{pod("mail-0", "w2", "10.44.0.9", mailLabels)}, nil)
|
|
rules = wantRules(m, "w2")
|
|
if len(rules) != 1 || rules[0].src != "10.44.0.9/32" || rules[0].mark != EgressMark || rules[0].table != peerTables(m, "w2")["w1"] {
|
|
t.Fatalf("member rule: %+v", rules)
|
|
}
|
|
u := ComputeEgress("w2", nodes, svcs, []*EgressService{egressFor("mail", EgressServiceStatus{})}, []*coreapi.Pod{pod("mail-0", "w2", "10.44.0.9", mailLabels)}, nil)
|
|
rules = wantRules(u, "w2")
|
|
if len(rules) != 1 || rules[0].table != EgressTableBase {
|
|
t.Fatalf("unreachable member must hit the unreachable table: %+v", rules)
|
|
}
|
|
}
|