lb: serve Gateways (Gateway API v1) from the node holding their address
images / images (push) Successful in 6m32s
images / images (push) Successful in 6m32s
Gateways of a GatewayClass whose controllerName is reweave.levk.dev/gateway are served like LoadBalancer Services: a Gateway names one IPv4 address in spec.addresses, and the node holding that address serves all of it and writes its status. - TCP listeners forward the oldest TCPRoute attached to them in the kernel, through the same DNAT and reply steering as LoadBalancer Services, so the backend sees the client's own address. - HTTP and HTTPS listeners are served by the agent itself on the holder: TLS terminated with the listener's certificate (chosen by SNI, swapped in place when its Secret changes, HTTP/2 offered), requests routed by the attached HTTPRoutes in the order the spec prescribes (hostname, exact path, longest prefix, method, headers, query, age, name, rule), and passed to a ready endpoint of the backend with X-Forwarded-For built from this hop only. RequestRedirect, URLRewrite and both header modifiers are implemented; other filters and regular expressions are refused (Accepted False, UnsupportedValue). Bodies are streamed, never limited. Unresolved backends answer 500, backends without ready endpoints 503, requests nothing matches 404. Those ports are let in ahead of the host firewall (REWEAVE-LB-IN). Listeners are checked for protocol, route kinds, certificates (Secrets in the Gateway's namespace or granted by a ReferenceGrant, and only in the namespaces the agent may read: --gateway-namespaces, default gateway-system), conflicts on a port, and ports LoadBalancer Services already forward. Routes attach by parentRef (sectionName, port), allowedRoutes (namespaces Same, All or Selector; kinds) and hostname intersection, with the spec's conditions and reasons. One writer per status: the holder writes its Gateway's (the first node by name when nobody holds the address, so failures are reported once), the first node by name writes GatewayClass and route statuses, keeping other controllers' route parents. Writes are conditional on the resourceVersion they were computed from, and a condition keeps its transition time while its status stands, so an unchanged status is never rewritten. The types are plain structs for the subset implemented (the upstream module needs a newer client-go than the fork's); the CRDs are installed apart from this manifest, which grants the agent read access to the Gateway API kinds, patch on their status, and Secrets in gateway-system only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UHXzGrsxfkGk8LmwrBbHNZ
This commit is contained in:
1 parent
685d084af7
commit
7405b51e53
10 files changed
+3117
-9
No files matched your search
@@ -53,6 +53,15 @@ type Controller struct {
|
||||
// lastEgress is what was last applied, reused while the EgressService caches are not synced (never wipe
|
||||
// members' rules on a partial view)
|
||||
lastEgress EgressDesired
|
||||
|
||||
// Gateways (nil gateway client: not served). Secrets are read only in gwNamespaces.
|
||||
gateway rest.Interface
|
||||
gwNamespaces []string
|
||||
proxy *Proxy
|
||||
classes, gateways, httpRoutes, tcpRoutes, grants, namespaces cache.Store
|
||||
secrets []cache.Store
|
||||
gatewaySynced []cache.InformerSynced // empty while the Gateway API is absent
|
||||
unbound map[string]string // listeners the proxy could not bind, last pass
|
||||
}
|
||||
|
||||
// Resync is how often the node re-reads its interfaces and re-applies everything even without API events.
|
||||
@@ -93,6 +102,86 @@ func (c *Controller) egressAPI() (bool, error) {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// WithGateway has the controller serve Gateways of the classes whose controllerName is GatewayControllerName,
|
||||
// reading their certificates from Secrets in these namespaces only.
|
||||
func (c *Controller) WithGateway(client rest.Interface, namespaces []string) *Controller {
|
||||
c.gateway, c.gwNamespaces, c.proxy, c.unbound = client, namespaces, NewProxy(), map[string]string{}
|
||||
return c
|
||||
}
|
||||
|
||||
// gatewayAPI is which Gateway API resources are served (nil: none, the CRDs are not installed).
|
||||
func (c *Controller) gatewayAPI() (map[string]bool, error) {
|
||||
l, err := c.client.Discovery().ServerResourcesForGroupVersion(GatewayGroupVersion.String())
|
||||
if apierrors.IsNotFound(err) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
served := map[string]bool{}
|
||||
for _, r := range l.APIResources {
|
||||
served[r.Name] = true
|
||||
}
|
||||
return served, nil
|
||||
}
|
||||
|
||||
func (c *Controller) startGateway(ctx context.Context, served map[string]bool) {
|
||||
run := func(store *cache.Store, resource string, obj runtime.Object) {
|
||||
if !served[resource] {
|
||||
return // an older bundle without this kind: nothing of it to serve
|
||||
}
|
||||
var ctl cache.Controller
|
||||
*store, ctl = newInformer(c, cache.NewListWatchFromClient(c.gateway, resource, "", fields.Everything()), obj)
|
||||
go ctl.Run(ctx.Done())
|
||||
c.gatewaySynced = append(c.gatewaySynced, ctl.HasSynced)
|
||||
}
|
||||
run(&c.classes, GatewayClasses, &GatewayClass{})
|
||||
run(&c.gateways, Gateways, &Gateway{})
|
||||
run(&c.httpRoutes, HTTPRoutes, &HTTPRoute{})
|
||||
run(&c.tcpRoutes, TCPRoutes, &TCPRoute{})
|
||||
run(&c.grants, ReferenceGrants, &ReferenceGrant{})
|
||||
var ctl cache.Controller
|
||||
c.namespaces, ctl = newInformer(c, cache.NewListWatchFromClient(c.client.CoreV1().RESTClient(), "namespaces", "", fields.Everything()), &coreapi.Namespace{})
|
||||
go ctl.Run(ctx.Done())
|
||||
c.gatewaySynced = append(c.gatewaySynced, ctl.HasSynced)
|
||||
for _, ns := range c.gwNamespaces {
|
||||
var store cache.Store
|
||||
store, ctl = newInformer(c, cache.NewListWatchFromClient(c.client.CoreV1().RESTClient(), "secrets", ns, fields.Everything()), &coreapi.Secret{})
|
||||
go ctl.Run(ctx.Done())
|
||||
c.secrets = append(c.secrets, store)
|
||||
c.gatewaySynced = append(c.gatewaySynced, ctl.HasSynced)
|
||||
}
|
||||
common.Log.Infof("lb: serving Gateways of controller %s (certificates from %v)", GatewayControllerName, c.gwNamespaces)
|
||||
}
|
||||
|
||||
// gatewayReady: the Gateway view can be trusted (the API is served and every informer has synced).
|
||||
func (c *Controller) gatewayReady() bool {
|
||||
if c.gateway == nil || len(c.gatewaySynced) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, s := range c.gatewaySynced {
|
||||
if !s() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// tryStartGateway starts the Gateway informers once the API is served.
|
||||
func (c *Controller) tryStartGateway(ctx context.Context) {
|
||||
if c.gateway == nil || len(c.gatewaySynced) != 0 {
|
||||
return
|
||||
}
|
||||
served, err := c.gatewayAPI()
|
||||
if err != nil {
|
||||
common.Log.Warnf("lb: cannot tell whether the Gateway API is served: %v", err)
|
||||
return
|
||||
}
|
||||
if served[Gateways] {
|
||||
c.startGateway(ctx, served)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Controller) startEgress(ctx context.Context) {
|
||||
var ctl cache.Controller
|
||||
c.egresses, ctl = newInformer(c, cache.NewListWatchFromClient(c.egress, EgressServices, "", fields.Everything()), &EgressService{})
|
||||
@@ -130,6 +219,10 @@ func (c *Controller) Run(ctx context.Context) {
|
||||
case <-time.After(5 * time.Second):
|
||||
}
|
||||
}
|
||||
c.tryStartGateway(ctx)
|
||||
if c.proxy != nil {
|
||||
defer c.proxy.Close()
|
||||
}
|
||||
go c.serveAdmission(ctx)
|
||||
if !cache.WaitForCacheSync(ctx.Done(), c.synced...) {
|
||||
return
|
||||
@@ -143,6 +236,9 @@ func (c *Controller) Run(ctx context.Context) {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
c.mu.Lock()
|
||||
c.tryStartGateway(ctx)
|
||||
c.mu.Unlock()
|
||||
if c.egressSynced == nil {
|
||||
// the API may have been installed since
|
||||
if present, err := c.egressAPI(); err == nil && present {
|
||||
@@ -208,6 +304,14 @@ func (c *Controller) sync(ctx context.Context) error {
|
||||
slices = append(slices, o.(*discovery.EndpointSlice))
|
||||
}
|
||||
d := Compute(c.self, nodes, svcs, slices)
|
||||
var g *GatewayDesired
|
||||
if c.gatewayReady() {
|
||||
gd := ComputeGateways(c.self, nodes, c.gatewayInput(svcs, slices))
|
||||
d = MergeGateways(d, gd, c.self, nodes)
|
||||
g = &gd
|
||||
} else if c.gateway != nil && len(c.gatewaySynced) != 0 {
|
||||
problems = append(problems, "Gateway view not synced yet: Gateways not served")
|
||||
}
|
||||
// rewrite once after start what an earlier run may have written without ipMode
|
||||
for _, f := range d.Forwards {
|
||||
if s := c.service(f.Namespace, f.Name); s != nil && c.written[s.UID] != f.Address.String() && !hasStatusFor(d, f) {
|
||||
@@ -236,6 +340,18 @@ func (c *Controller) sync(ctx context.Context) error {
|
||||
return fmt.Errorf("applying datapath: %v", err)
|
||||
}
|
||||
c.lastEgress = e
|
||||
if g != nil {
|
||||
// the ports are let in by now; bind what this node serves, close what it no longer does
|
||||
unbound := map[string]string{}
|
||||
for k, err := range c.proxy.Apply(g.Serve) {
|
||||
unbound[k] = err.Error()
|
||||
}
|
||||
if !sameJSON(unbound, c.unbound) {
|
||||
c.unbound = unbound
|
||||
c.poke() // the statuses say so on the next pass
|
||||
}
|
||||
c.writeGatewayStatuses(ctx, *g)
|
||||
}
|
||||
for _, st := range d.Status {
|
||||
if err := c.writeStatus(ctx, st); err != nil {
|
||||
return err
|
||||
@@ -458,3 +574,56 @@ func (c *Controller) writeStatus(ctx context.Context, st Status) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// gatewayInput is the Gateway view the policy reads.
|
||||
func (c *Controller) gatewayInput(svcs []*coreapi.Service, slices []*discovery.EndpointSlice) GatewayInput {
|
||||
in := GatewayInput{Services: svcs, Slices: slices, SecretNamespaces: map[string]bool{}, Unbound: c.unbound, Now: metav1.Now()}
|
||||
list := func(s cache.Store, each func(interface{})) {
|
||||
if s != nil {
|
||||
for _, o := range s.List() {
|
||||
each(o)
|
||||
}
|
||||
}
|
||||
}
|
||||
list(c.classes, func(o interface{}) { in.Classes = append(in.Classes, o.(*GatewayClass)) })
|
||||
list(c.gateways, func(o interface{}) { in.Gateways = append(in.Gateways, o.(*Gateway)) })
|
||||
list(c.httpRoutes, func(o interface{}) { in.HTTPRoutes = append(in.HTTPRoutes, o.(*HTTPRoute)) })
|
||||
list(c.tcpRoutes, func(o interface{}) { in.TCPRoutes = append(in.TCPRoutes, o.(*TCPRoute)) })
|
||||
list(c.grants, func(o interface{}) { in.Grants = append(in.Grants, o.(*ReferenceGrant)) })
|
||||
list(c.namespaces, func(o interface{}) { in.Namespaces = append(in.Namespaces, o.(*coreapi.Namespace)) })
|
||||
for _, s := range c.secrets {
|
||||
list(s, func(o interface{}) { in.Secrets = append(in.Secrets, o.(*coreapi.Secret)) })
|
||||
}
|
||||
for _, ns := range c.gwNamespaces {
|
||||
in.SecretNamespaces[ns] = true
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
// writeGatewayStatuses writes what the policy says this node writes. Each write is conditional on the version it
|
||||
// was computed from: one built from a stale view fails, and the next pass recomputes it.
|
||||
func (c *Controller) writeGatewayStatuses(ctx context.Context, g GatewayDesired) {
|
||||
for _, w := range g.Classes {
|
||||
if err := patchStatus(ctx, c.gateway, GatewayClasses, "", w.Name, w.ResourceVersion, w.Status); err != nil {
|
||||
common.Log.Warnf("lb: status of GatewayClass %s: %v", w.Name, err)
|
||||
continue
|
||||
}
|
||||
common.Log.Infof("lb: GatewayClass %s accepted", w.Name)
|
||||
}
|
||||
for _, w := range g.Gateways {
|
||||
if err := patchStatus(ctx, c.gateway, Gateways, w.Namespace, w.Name, w.ResourceVersion, w.Status); err != nil {
|
||||
common.Log.Warnf("lb: status of Gateway %s/%s: %v", w.Namespace, w.Name, err)
|
||||
continue
|
||||
}
|
||||
var conds []string
|
||||
for _, cd := range w.Status.Conditions {
|
||||
conds = append(conds, fmt.Sprintf("%s=%s (%s)", cd.Type, cd.Status, cd.Reason))
|
||||
}
|
||||
common.Log.Infof("lb: Gateway %s/%s: %v", w.Namespace, w.Name, conds)
|
||||
}
|
||||
for _, w := range g.Routes {
|
||||
if err := patchStatus(ctx, c.gateway, w.Resource, w.Namespace, w.Name, w.ResourceVersion, w.Status); err != nil {
|
||||
common.Log.Warnf("lb: status of %s %s/%s: %v", w.Resource, w.Namespace, w.Name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -117,6 +117,10 @@ func Render(d Desired, e EgressDesired, self string, podNet *net.IPNet, stale []
|
||||
for _, c := range []string{FwdChain, InChain} {
|
||||
fmt.Fprintf(&filter, ":%s - [0:0]\n", c)
|
||||
}
|
||||
for _, a := range d.Accept {
|
||||
// served by the agent itself (a Gateway's HTTP or HTTPS listener): in, whatever the host firewall says
|
||||
fmt.Fprintf(&filter, "-A %s -d %s/32 -p tcp -m tcp --dport %d -m comment --comment %q -j ACCEPT\n", InChain, a.Address, a.Port, a.Comment)
|
||||
}
|
||||
renderEgress(e, self, podNet, &body, &mangle, &filter)
|
||||
|
||||
nat.Write(body.Bytes())
|
||||
|
||||
+1009
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,415 @@
|
||||
package lb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/runtime/serializer"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/rest"
|
||||
)
|
||||
|
||||
// The Gateway API (gateway.networking.k8s.io/v1, as the v1.6.3 standard channel serves it): the subset this agent
|
||||
// implements, as plain types. The upstream Go module needs a newer client-go than the fork's; the JSON is the
|
||||
// contract, and fields this agent does not implement are simply not decoded.
|
||||
|
||||
const (
|
||||
GatewayGroup = "gateway.networking.k8s.io"
|
||||
GatewayVersion = "v1"
|
||||
// GatewayControllerName is the spec.controllerName of the GatewayClasses this agent implements.
|
||||
GatewayControllerName = "reweave.levk.dev/gateway"
|
||||
|
||||
GatewayClasses = "gatewayclasses"
|
||||
Gateways = "gateways"
|
||||
HTTPRoutes = "httproutes"
|
||||
TCPRoutes = "tcproutes"
|
||||
ReferenceGrants = "referencegrants"
|
||||
)
|
||||
|
||||
// GatewayGroupVersion of the Gateway API.
|
||||
var GatewayGroupVersion = schema.GroupVersion{Group: GatewayGroup, Version: GatewayVersion}
|
||||
|
||||
type GatewayClass struct {
|
||||
metav1.TypeMeta `json:",inline"`
|
||||
metav1.ObjectMeta `json:"metadata,omitempty"`
|
||||
Spec GatewayClassSpec `json:"spec"`
|
||||
Status GatewayClassStatus `json:"status,omitempty"`
|
||||
}
|
||||
|
||||
type GatewayClassSpec struct {
|
||||
ControllerName string `json:"controllerName"`
|
||||
}
|
||||
|
||||
type GatewayClassStatus struct {
|
||||
Conditions []metav1.Condition `json:"conditions,omitempty"`
|
||||
}
|
||||
|
||||
type Gateway struct {
|
||||
metav1.TypeMeta `json:",inline"`
|
||||
metav1.ObjectMeta `json:"metadata,omitempty"`
|
||||
Spec GatewaySpec `json:"spec"`
|
||||
Status GatewayStatus `json:"status,omitempty"`
|
||||
}
|
||||
|
||||
type GatewaySpec struct {
|
||||
GatewayClassName string `json:"gatewayClassName"`
|
||||
Listeners []Listener `json:"listeners"`
|
||||
Addresses []GatewayAddress `json:"addresses,omitempty"`
|
||||
}
|
||||
|
||||
// GatewayAddress is a spec or status address; Type defaults to IPAddress.
|
||||
type GatewayAddress struct {
|
||||
Type *string `json:"type,omitempty"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
type Listener struct {
|
||||
Name string `json:"name"`
|
||||
Hostname *string `json:"hostname,omitempty"`
|
||||
Port int32 `json:"port"`
|
||||
Protocol string `json:"protocol"`
|
||||
TLS *ListenerTLSConfig `json:"tls,omitempty"`
|
||||
AllowedRoutes *AllowedRoutes `json:"allowedRoutes,omitempty"`
|
||||
}
|
||||
|
||||
type ListenerTLSConfig struct {
|
||||
Mode *string `json:"mode,omitempty"`
|
||||
CertificateRefs []SecretObjectReference `json:"certificateRefs,omitempty"`
|
||||
}
|
||||
|
||||
type SecretObjectReference struct {
|
||||
Group *string `json:"group,omitempty"`
|
||||
Kind *string `json:"kind,omitempty"`
|
||||
Name string `json:"name"`
|
||||
Namespace *string `json:"namespace,omitempty"`
|
||||
}
|
||||
|
||||
type AllowedRoutes struct {
|
||||
Namespaces *RouteNamespaces `json:"namespaces,omitempty"`
|
||||
Kinds []RouteGroupKind `json:"kinds,omitempty"`
|
||||
}
|
||||
|
||||
type RouteNamespaces struct {
|
||||
From *string `json:"from,omitempty"`
|
||||
Selector *metav1.LabelSelector `json:"selector,omitempty"`
|
||||
}
|
||||
|
||||
type RouteGroupKind struct {
|
||||
Group *string `json:"group,omitempty"`
|
||||
Kind string `json:"kind"`
|
||||
}
|
||||
|
||||
type GatewayStatus struct {
|
||||
Addresses []GatewayAddress `json:"addresses,omitempty"`
|
||||
Conditions []metav1.Condition `json:"conditions,omitempty"`
|
||||
Listeners []ListenerStatus `json:"listeners,omitempty"`
|
||||
}
|
||||
|
||||
type ListenerStatus struct {
|
||||
Name string `json:"name"`
|
||||
SupportedKinds []RouteGroupKind `json:"supportedKinds"`
|
||||
AttachedRoutes int32 `json:"attachedRoutes"`
|
||||
Conditions []metav1.Condition `json:"conditions"`
|
||||
}
|
||||
|
||||
type ParentReference struct {
|
||||
Group *string `json:"group,omitempty"`
|
||||
Kind *string `json:"kind,omitempty"`
|
||||
Namespace *string `json:"namespace,omitempty"`
|
||||
Name string `json:"name"`
|
||||
SectionName *string `json:"sectionName,omitempty"`
|
||||
Port *int32 `json:"port,omitempty"`
|
||||
}
|
||||
|
||||
type BackendRef struct {
|
||||
Group *string `json:"group,omitempty"`
|
||||
Kind *string `json:"kind,omitempty"`
|
||||
Name string `json:"name"`
|
||||
Namespace *string `json:"namespace,omitempty"`
|
||||
Port *int32 `json:"port,omitempty"`
|
||||
Weight *int32 `json:"weight,omitempty"`
|
||||
}
|
||||
|
||||
type RouteStatus struct {
|
||||
Parents []RouteParentStatus `json:"parents"`
|
||||
}
|
||||
|
||||
type RouteParentStatus struct {
|
||||
ParentRef ParentReference `json:"parentRef"`
|
||||
ControllerName string `json:"controllerName"`
|
||||
Conditions []metav1.Condition `json:"conditions,omitempty"`
|
||||
}
|
||||
|
||||
type HTTPRoute struct {
|
||||
metav1.TypeMeta `json:",inline"`
|
||||
metav1.ObjectMeta `json:"metadata,omitempty"`
|
||||
Spec HTTPRouteSpec `json:"spec"`
|
||||
Status RouteStatus `json:"status,omitempty"`
|
||||
}
|
||||
|
||||
type HTTPRouteSpec struct {
|
||||
ParentRefs []ParentReference `json:"parentRefs,omitempty"`
|
||||
Hostnames []string `json:"hostnames,omitempty"`
|
||||
Rules []HTTPRouteRule `json:"rules,omitempty"`
|
||||
}
|
||||
|
||||
type HTTPRouteRule struct {
|
||||
Matches []HTTPRouteMatch `json:"matches,omitempty"`
|
||||
Filters []HTTPRouteFilter `json:"filters,omitempty"`
|
||||
BackendRefs []HTTPBackendRef `json:"backendRefs,omitempty"`
|
||||
}
|
||||
|
||||
type HTTPRouteMatch struct {
|
||||
Path *HTTPPathMatch `json:"path,omitempty"`
|
||||
Headers []HTTPHeaderMatch `json:"headers,omitempty"`
|
||||
QueryParams []HTTPQueryParamMatch `json:"queryParams,omitempty"`
|
||||
Method *string `json:"method,omitempty"`
|
||||
}
|
||||
|
||||
type HTTPPathMatch struct {
|
||||
Type *string `json:"type,omitempty"`
|
||||
Value *string `json:"value,omitempty"`
|
||||
}
|
||||
|
||||
type HTTPHeaderMatch struct {
|
||||
Type *string `json:"type,omitempty"`
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
type HTTPQueryParamMatch struct {
|
||||
Type *string `json:"type,omitempty"`
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
// HTTPRouteFilter: the filters this agent implements are decoded; for the others only their presence is known
|
||||
// (through Type), which is enough to refuse them.
|
||||
type HTTPRouteFilter struct {
|
||||
Type string `json:"type"`
|
||||
RequestHeaderModifier *HTTPHeaderFilter `json:"requestHeaderModifier,omitempty"`
|
||||
ResponseHeaderModifier *HTTPHeaderFilter `json:"responseHeaderModifier,omitempty"`
|
||||
RequestRedirect *HTTPRequestRedirectFilter `json:"requestRedirect,omitempty"`
|
||||
URLRewrite *HTTPURLRewriteFilter `json:"urlRewrite,omitempty"`
|
||||
}
|
||||
|
||||
type HTTPHeaderFilter struct {
|
||||
Set []HTTPHeader `json:"set,omitempty"`
|
||||
Add []HTTPHeader `json:"add,omitempty"`
|
||||
Remove []string `json:"remove,omitempty"`
|
||||
}
|
||||
|
||||
type HTTPHeader struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
type HTTPRequestRedirectFilter struct {
|
||||
Scheme *string `json:"scheme,omitempty"`
|
||||
Hostname *string `json:"hostname,omitempty"`
|
||||
Path *HTTPPathModifier `json:"path,omitempty"`
|
||||
Port *int32 `json:"port,omitempty"`
|
||||
StatusCode *int `json:"statusCode,omitempty"`
|
||||
}
|
||||
|
||||
type HTTPURLRewriteFilter struct {
|
||||
Hostname *string `json:"hostname,omitempty"`
|
||||
Path *HTTPPathModifier `json:"path,omitempty"`
|
||||
}
|
||||
|
||||
type HTTPPathModifier struct {
|
||||
Type string `json:"type"`
|
||||
ReplaceFullPath *string `json:"replaceFullPath,omitempty"`
|
||||
ReplacePrefixMatch *string `json:"replacePrefixMatch,omitempty"`
|
||||
}
|
||||
|
||||
type HTTPBackendRef struct {
|
||||
BackendRef `json:",inline"`
|
||||
Filters []HTTPRouteFilter `json:"filters,omitempty"`
|
||||
}
|
||||
|
||||
type TCPRoute struct {
|
||||
metav1.TypeMeta `json:",inline"`
|
||||
metav1.ObjectMeta `json:"metadata,omitempty"`
|
||||
Spec TCPRouteSpec `json:"spec"`
|
||||
Status RouteStatus `json:"status,omitempty"`
|
||||
}
|
||||
|
||||
type TCPRouteSpec struct {
|
||||
ParentRefs []ParentReference `json:"parentRefs,omitempty"`
|
||||
Rules []TCPRouteRule `json:"rules,omitempty"`
|
||||
}
|
||||
|
||||
type TCPRouteRule struct {
|
||||
BackendRefs []BackendRef `json:"backendRefs,omitempty"`
|
||||
}
|
||||
|
||||
type ReferenceGrant struct {
|
||||
metav1.TypeMeta `json:",inline"`
|
||||
metav1.ObjectMeta `json:"metadata,omitempty"`
|
||||
Spec ReferenceGrantSpec `json:"spec"`
|
||||
}
|
||||
|
||||
type ReferenceGrantSpec struct {
|
||||
From []ReferenceGrantFrom `json:"from"`
|
||||
To []ReferenceGrantTo `json:"to"`
|
||||
}
|
||||
|
||||
type ReferenceGrantFrom struct {
|
||||
Group string `json:"group"`
|
||||
Kind string `json:"kind"`
|
||||
Namespace string `json:"namespace"`
|
||||
}
|
||||
|
||||
type ReferenceGrantTo struct {
|
||||
Group string `json:"group"`
|
||||
Kind string `json:"kind"`
|
||||
Name *string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
type GatewayClassList struct {
|
||||
metav1.TypeMeta `json:",inline"`
|
||||
metav1.ListMeta `json:"metadata,omitempty"`
|
||||
Items []GatewayClass `json:"items"`
|
||||
}
|
||||
|
||||
type GatewayList struct {
|
||||
metav1.TypeMeta `json:",inline"`
|
||||
metav1.ListMeta `json:"metadata,omitempty"`
|
||||
Items []Gateway `json:"items"`
|
||||
}
|
||||
|
||||
type HTTPRouteList struct {
|
||||
metav1.TypeMeta `json:",inline"`
|
||||
metav1.ListMeta `json:"metadata,omitempty"`
|
||||
Items []HTTPRoute `json:"items"`
|
||||
}
|
||||
|
||||
type TCPRouteList struct {
|
||||
metav1.TypeMeta `json:",inline"`
|
||||
metav1.ListMeta `json:"metadata,omitempty"`
|
||||
Items []TCPRoute `json:"items"`
|
||||
}
|
||||
|
||||
type ReferenceGrantList struct {
|
||||
metav1.TypeMeta `json:",inline"`
|
||||
metav1.ListMeta `json:"metadata,omitempty"`
|
||||
Items []ReferenceGrant `json:"items"`
|
||||
}
|
||||
|
||||
// jsonCopy deep-copies through JSON: these types are only what the JSON says, so nothing is lost.
|
||||
func jsonCopy(in, out interface{}) {
|
||||
b, err := json.Marshal(in)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := json.Unmarshal(b, out); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (o *GatewayClass) DeepCopyObject() runtime.Object {
|
||||
out := &GatewayClass{}
|
||||
jsonCopy(o, out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (o *Gateway) DeepCopyObject() runtime.Object {
|
||||
out := &Gateway{}
|
||||
jsonCopy(o, out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (o *HTTPRoute) DeepCopyObject() runtime.Object {
|
||||
out := &HTTPRoute{}
|
||||
jsonCopy(o, out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (o *TCPRoute) DeepCopyObject() runtime.Object {
|
||||
out := &TCPRoute{}
|
||||
jsonCopy(o, out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (o *ReferenceGrant) DeepCopyObject() runtime.Object {
|
||||
out := &ReferenceGrant{}
|
||||
jsonCopy(o, out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (o *GatewayClassList) DeepCopyObject() runtime.Object {
|
||||
out := &GatewayClassList{}
|
||||
jsonCopy(o, out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (o *GatewayList) DeepCopyObject() runtime.Object {
|
||||
out := &GatewayList{}
|
||||
jsonCopy(o, out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (o *HTTPRouteList) DeepCopyObject() runtime.Object {
|
||||
out := &HTTPRouteList{}
|
||||
jsonCopy(o, out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (o *TCPRouteList) DeepCopyObject() runtime.Object {
|
||||
out := &TCPRouteList{}
|
||||
jsonCopy(o, out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (o *ReferenceGrantList) DeepCopyObject() runtime.Object {
|
||||
out := &ReferenceGrantList{}
|
||||
jsonCopy(o, out)
|
||||
return out
|
||||
}
|
||||
|
||||
func addGatewayTypes(s *runtime.Scheme) error {
|
||||
s.AddKnownTypes(GatewayGroupVersion,
|
||||
&GatewayClass{}, &GatewayClassList{}, &Gateway{}, &GatewayList{}, &HTTPRoute{}, &HTTPRouteList{},
|
||||
&TCPRoute{}, &TCPRouteList{}, &ReferenceGrant{}, &ReferenceGrantList{})
|
||||
metav1.AddToGroupVersion(s, GatewayGroupVersion)
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewGatewayClient is a REST client for the Gateway API kinds this agent reads and writes the status of.
|
||||
func NewGatewayClient(base *rest.Config) (*rest.RESTClient, error) {
|
||||
scheme := runtime.NewScheme()
|
||||
if err := addGatewayTypes(scheme); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg := *base
|
||||
cfg.GroupVersion = &GatewayGroupVersion
|
||||
cfg.APIPath = "/apis"
|
||||
cfg.ContentType = runtime.ContentTypeJSON
|
||||
cfg.NegotiatedSerializer = serializer.NewCodecFactory(scheme).WithoutConversion()
|
||||
if cfg.UserAgent == "" {
|
||||
cfg.UserAgent = rest.DefaultKubernetesUserAgent()
|
||||
}
|
||||
return rest.RESTClientFor(&cfg)
|
||||
}
|
||||
|
||||
// patchStatus merge-patches an object's status. resourceVersion, when given, makes it conditional: a status built
|
||||
// from a stale read fails instead of overwriting what someone else wrote since.
|
||||
func patchStatus(ctx context.Context, c rest.Interface, resource, namespace, name, resourceVersion string, status interface{}) error {
|
||||
body := map[string]interface{}{"status": status}
|
||||
if resourceVersion != "" {
|
||||
body["metadata"] = map[string]interface{}{"resourceVersion": resourceVersion}
|
||||
}
|
||||
patch, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req := c.Patch(types.MergePatchType).Resource(resource).Name(name).SubResource("status").Body(patch)
|
||||
if namespace != "" {
|
||||
req = req.Namespace(namespace)
|
||||
}
|
||||
return req.Do(ctx).Error()
|
||||
}
|
||||
@@ -0,0 +1,542 @@
|
||||
package lb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/rand"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httputil"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
coreapi "k8s.io/api/core/v1"
|
||||
|
||||
"github.com/rajch/weave/common"
|
||||
)
|
||||
|
||||
// Proxy serves the HTTP and HTTPS listeners of the Gateways this node holds. TLS is terminated with the
|
||||
// listener's certificate, chosen by SNI and swapped in place when its Secret changes (no restart, no rebind).
|
||||
// Requests are routed by the attached HTTPRoutes in the order the spec prescribes and passed to a ready endpoint
|
||||
// of the chosen backend, with the client's own address in X-Forwarded-For. Bodies are streamed, never limited.
|
||||
type Proxy struct {
|
||||
mu sync.Mutex
|
||||
servers map[string]*portServer // by "address:port"
|
||||
// listen binds an address (swappable in tests)
|
||||
listen func(addr string) (net.Listener, error)
|
||||
transport http.RoundTripper
|
||||
certs map[string]*tls.Certificate // parsed certificates, by Secret uid@resourceVersion
|
||||
rr sync.Map // backend key -> *uint64, for round robin over its endpoints
|
||||
}
|
||||
|
||||
func NewProxy() *Proxy {
|
||||
return &Proxy{
|
||||
servers: map[string]*portServer{},
|
||||
certs: map[string]*tls.Certificate{},
|
||||
listen: func(addr string) (net.Listener, error) { return net.Listen("tcp", addr) },
|
||||
transport: &http.Transport{
|
||||
DialContext: (&net.Dialer{Timeout: 10 * time.Second, KeepAlive: 30 * time.Second}).DialContext,
|
||||
MaxIdleConns: 256,
|
||||
MaxIdleConnsPerHost: 32,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
ExpectContinueTimeout: time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// portServer is one bound address:port and what it currently serves.
|
||||
type portServer struct {
|
||||
addr string
|
||||
https bool
|
||||
ln net.Listener
|
||||
srv *http.Server
|
||||
cfg atomic.Pointer[portConfig]
|
||||
}
|
||||
|
||||
// portConfig is everything one address:port serves: its listeners, most specific hostname first.
|
||||
type portConfig struct {
|
||||
listeners []*liveListener
|
||||
}
|
||||
|
||||
type liveListener struct {
|
||||
spec ServeListener
|
||||
cert *tls.Certificate
|
||||
}
|
||||
|
||||
// Apply serves exactly these listeners: ports no longer listed are closed, new ones bound, and every served
|
||||
// port's routes and certificates replaced at once. It returns, by "namespace/gateway/listener", the listeners
|
||||
// it could not bind.
|
||||
func (p *Proxy) Apply(listeners []ServeListener) map[string]error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
failed := map[string]error{}
|
||||
byAddr := map[string][]ServeListener{}
|
||||
for _, l := range listeners {
|
||||
a := net.JoinHostPort(l.Address.String(), strconv.Itoa(int(l.Port)))
|
||||
byAddr[a] = append(byAddr[a], l)
|
||||
}
|
||||
for a, s := range p.servers {
|
||||
if group, ok := byAddr[a]; !ok || group[0].HTTPS != s.https {
|
||||
s.close()
|
||||
delete(p.servers, a)
|
||||
}
|
||||
}
|
||||
used := map[string]bool{}
|
||||
for a, group := range byAddr {
|
||||
cfg := p.compile(group, used)
|
||||
s := p.servers[a]
|
||||
if s == nil {
|
||||
var err error
|
||||
if s, err = p.start(a, group[0].HTTPS); err != nil {
|
||||
for _, l := range group {
|
||||
failed[l.Gateway+"/"+l.Name] = err
|
||||
}
|
||||
continue
|
||||
}
|
||||
p.servers[a] = s
|
||||
}
|
||||
s.cfg.Store(cfg)
|
||||
}
|
||||
// forget certificates no listener uses any more
|
||||
for k := range p.certs {
|
||||
if !used[k] {
|
||||
delete(p.certs, k)
|
||||
}
|
||||
}
|
||||
return failed
|
||||
}
|
||||
|
||||
// Close stops serving everything.
|
||||
func (p *Proxy) Close() {
|
||||
p.Apply(nil)
|
||||
}
|
||||
|
||||
func (p *Proxy) compile(group []ServeListener, used map[string]bool) *portConfig {
|
||||
cfg := &portConfig{}
|
||||
for _, l := range group {
|
||||
ll := &liveListener{spec: l}
|
||||
if l.HTTPS && l.Cert != nil {
|
||||
ll.cert = p.cert(l.Cert)
|
||||
used[certKey(l.Cert)] = true
|
||||
}
|
||||
cfg.listeners = append(cfg.listeners, ll)
|
||||
}
|
||||
// most specific first: exact hostnames, then wildcards (longest first), then the catch-all
|
||||
sort.SliceStable(cfg.listeners, func(i, j int) bool {
|
||||
return hostRank(cfg.listeners[i].spec.Hostname) > hostRank(cfg.listeners[j].spec.Hostname)
|
||||
})
|
||||
return cfg
|
||||
}
|
||||
|
||||
func certKey(s *coreapi.Secret) string {
|
||||
return string(s.UID) + "@" + s.ResourceVersion + "/" + s.Namespace + "/" + s.Name
|
||||
}
|
||||
|
||||
// cert parses a Secret's certificate once per version of the Secret.
|
||||
func (p *Proxy) cert(s *coreapi.Secret) *tls.Certificate {
|
||||
k := certKey(s)
|
||||
if c, ok := p.certs[k]; ok {
|
||||
return c
|
||||
}
|
||||
c, err := tls.X509KeyPair(s.Data[coreapi.TLSCertKey], s.Data[coreapi.TLSPrivateKeyKey])
|
||||
if err != nil {
|
||||
common.Log.Warnf("lb: gateway certificate %s/%s: %v", s.Namespace, s.Name, err)
|
||||
return nil
|
||||
}
|
||||
p.certs[k] = &c
|
||||
return &c
|
||||
}
|
||||
|
||||
// hostRank orders listener hostnames: exact (by length) above wildcards (by length) above none.
|
||||
func hostRank(h string) int {
|
||||
switch {
|
||||
case h == "":
|
||||
return 0
|
||||
case strings.HasPrefix(h, "*."):
|
||||
return 1000 + len(h)
|
||||
}
|
||||
return 100000 + len(h)
|
||||
}
|
||||
|
||||
func (p *Proxy) start(addr string, https bool) (*portServer, error) {
|
||||
ln, err := p.listen(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s := &portServer{addr: addr, https: https, ln: ln}
|
||||
s.cfg.Store(&portConfig{})
|
||||
s.srv = &http.Server{
|
||||
Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { p.serve(s, w, r) }),
|
||||
ReadHeaderTimeout: 30 * time.Second,
|
||||
IdleTimeout: 2 * time.Minute,
|
||||
// handshakes from scanners and the like would flood the log
|
||||
ErrorLog: log.New(debugWriter{}, "lb: gateway "+addr+": ", 0),
|
||||
}
|
||||
if https {
|
||||
s.srv.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS12, GetCertificate: s.certificate}
|
||||
go func() {
|
||||
if err := s.srv.ServeTLS(ln, "", ""); err != nil && err != http.ErrServerClosed {
|
||||
common.Log.Errorf("lb: gateway %s: %v", addr, err)
|
||||
}
|
||||
}()
|
||||
} else {
|
||||
go func() {
|
||||
if err := s.srv.Serve(ln); err != nil && err != http.ErrServerClosed {
|
||||
common.Log.Errorf("lb: gateway %s: %v", addr, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
common.Log.Infof("lb: gateway serving %s (%s)", addr, map[bool]string{true: "HTTPS", false: "HTTP"}[https])
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *portServer) close() {
|
||||
srv := s.srv
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = srv.Shutdown(ctx)
|
||||
_ = srv.Close()
|
||||
}()
|
||||
common.Log.Infof("lb: gateway no longer serving %s", s.addr)
|
||||
}
|
||||
|
||||
type debugWriter struct{}
|
||||
|
||||
func (debugWriter) Write(b []byte) (int, error) {
|
||||
common.Log.Debugf("%s", strings.TrimRight(string(b), "\n"))
|
||||
return len(b), nil
|
||||
}
|
||||
|
||||
// listenerFor picks the listener a name (SNI or Host) reaches on a port, most specific hostname first.
|
||||
func (c *portConfig) listenerFor(name string) *liveListener {
|
||||
name = strings.ToLower(name)
|
||||
for _, l := range c.listeners {
|
||||
if l.spec.Hostname == "" || hostMatches(l.spec.Hostname, name) {
|
||||
return l
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// certificate answers a TLS handshake with the certificate of the listener its SNI names.
|
||||
func (s *portServer) certificate(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
|
||||
cfg := s.cfg.Load()
|
||||
l := cfg.listenerFor(hello.ServerName)
|
||||
if l == nil || l.cert == nil {
|
||||
return nil, fmt.Errorf("no listener for %q", hello.ServerName)
|
||||
}
|
||||
return l.cert, nil
|
||||
}
|
||||
|
||||
// candidate is one match of one rule that a request satisfies, with what decides its precedence.
|
||||
type candidate struct {
|
||||
route *ServeRoute
|
||||
rule *ServeRule
|
||||
ruleIdx int
|
||||
// the matched hostname: characters if not a wildcard, and characters
|
||||
hostExact, hostLen int
|
||||
exact bool
|
||||
prefixLen int
|
||||
method bool
|
||||
headers, query int
|
||||
prefix string // the path prefix matched (for ReplacePrefixMatch)
|
||||
}
|
||||
|
||||
func (p *Proxy) serve(s *portServer, w http.ResponseWriter, r *http.Request) {
|
||||
cfg := s.cfg.Load()
|
||||
host := hostOnly(r.Host)
|
||||
name := host
|
||||
if r.TLS != nil && r.TLS.ServerName != "" {
|
||||
name = r.TLS.ServerName
|
||||
}
|
||||
l := cfg.listenerFor(name)
|
||||
if l == nil {
|
||||
http.Error(w, "no listener for this host", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
c := pick(l, host, r)
|
||||
if c == nil {
|
||||
http.Error(w, "no route", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
for _, f := range c.rule.Filters {
|
||||
if f.Type == filterRequestRedirect && f.RequestRedirect != nil {
|
||||
to, code := redirect(r, f.RequestRedirect, l.spec.Port, c.prefix)
|
||||
http.Redirect(w, r, to, code)
|
||||
return
|
||||
}
|
||||
}
|
||||
b := chooseBackend(c.rule.Backends)
|
||||
if b == nil {
|
||||
// no resolved backend to send it to: the spec's answer for unresolved backends
|
||||
http.Error(w, "no usable backend", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if len(b.Endpoints) == 0 {
|
||||
http.Error(w, "no ready endpoint", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
n, _ := p.rr.LoadOrStore(b.Key, new(uint64))
|
||||
ep := b.Endpoints[int(atomic.AddUint64(n.(*uint64), 1)-1)%len(b.Endpoints)]
|
||||
filters := append(append([]HTTPRouteFilter(nil), c.rule.Filters...), b.Filters...)
|
||||
rp := &httputil.ReverseProxy{
|
||||
Transport: p.transport,
|
||||
FlushInterval: -1, // stream as it comes (git, server-sent events)
|
||||
Rewrite: func(pr *httputil.ProxyRequest) {
|
||||
pr.SetURL(&url.URL{Scheme: "http", Host: ep.String()})
|
||||
pr.Out.Host = pr.In.Host // forwarded unmodified unless a filter says otherwise
|
||||
// SetXForwarded replaces whatever the client sent: only this hop's view is trusted
|
||||
pr.SetXForwarded()
|
||||
if ip, _, err := net.SplitHostPort(pr.In.RemoteAddr); err == nil {
|
||||
pr.Out.Header.Set("X-Real-IP", ip)
|
||||
}
|
||||
for _, f := range filters {
|
||||
switch {
|
||||
case f.Type == filterRequestHeaderModifier && f.RequestHeaderModifier != nil:
|
||||
modifyHeaders(pr.Out.Header, f.RequestHeaderModifier)
|
||||
case f.Type == filterURLRewrite && f.URLRewrite != nil:
|
||||
if f.URLRewrite.Hostname != nil {
|
||||
pr.Out.Host = *f.URLRewrite.Hostname
|
||||
}
|
||||
if f.URLRewrite.Path != nil {
|
||||
pr.Out.URL.Path = rewritePath(pr.In.URL.Path, f.URLRewrite.Path, c.prefix)
|
||||
pr.Out.URL.RawPath = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
ModifyResponse: func(resp *http.Response) error {
|
||||
for _, f := range filters {
|
||||
if f.Type == filterResponseHeaderModifier && f.ResponseHeaderModifier != nil {
|
||||
modifyHeaders(resp.Header, f.ResponseHeaderModifier)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
|
||||
common.Log.Debugf("lb: gateway %s -> %s: %v", s.addr, ep, err)
|
||||
w.WriteHeader(http.StatusBadGateway)
|
||||
},
|
||||
}
|
||||
rp.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
// pick is the match that takes the request, by the spec's precedence: the most specific matching hostname, then
|
||||
// an exact path, the longest prefix, a method, the most header matches, the most query matches, then the oldest
|
||||
// route, its namespace/name, and the first rule.
|
||||
func pick(l *liveListener, host string, r *http.Request) *candidate {
|
||||
var best *candidate
|
||||
for ri := range l.spec.Routes {
|
||||
route := &l.spec.Routes[ri]
|
||||
hostExact, hostLen, ok := routeHost(route, l.spec.Hostname, host)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for i := range route.Rules {
|
||||
rule := &route.Rules[i]
|
||||
for _, m := range rule.Matches {
|
||||
c, ok := matches(m, r)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
c.route, c.rule, c.ruleIdx, c.hostExact, c.hostLen = route, rule, i, hostExact, hostLen
|
||||
if best == nil || better(c, *best) {
|
||||
cc := c
|
||||
best = &cc
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
func better(a, b candidate) bool {
|
||||
switch {
|
||||
case a.hostExact != b.hostExact:
|
||||
return a.hostExact > b.hostExact
|
||||
case a.hostLen != b.hostLen:
|
||||
return a.hostLen > b.hostLen
|
||||
case a.exact != b.exact:
|
||||
return a.exact
|
||||
case a.prefixLen != b.prefixLen:
|
||||
return a.prefixLen > b.prefixLen
|
||||
case a.method != b.method:
|
||||
return a.method
|
||||
case a.headers != b.headers:
|
||||
return a.headers > b.headers
|
||||
case a.query != b.query:
|
||||
return a.query > b.query
|
||||
case !a.route.Created.Equal(b.route.Created):
|
||||
return a.route.Created.Before(b.route.Created)
|
||||
case a.route.Key != b.route.Key:
|
||||
return a.route.Key < b.route.Key
|
||||
}
|
||||
return a.ruleIdx < b.ruleIdx
|
||||
}
|
||||
|
||||
// routeHost reports whether a route answers this host on its listener, and how specifically.
|
||||
func routeHost(route *ServeRoute, listenerHost, host string) (int, int, bool) {
|
||||
names := route.Hostnames
|
||||
if len(names) == 0 {
|
||||
if listenerHost == "" {
|
||||
return 0, 0, true
|
||||
}
|
||||
names = []string{listenerHost}
|
||||
}
|
||||
exact, length, ok := 0, 0, false
|
||||
for _, h := range names {
|
||||
if !hostMatches(h, host) {
|
||||
continue
|
||||
}
|
||||
ok = true
|
||||
e := 0
|
||||
if !strings.HasPrefix(h, "*.") {
|
||||
e = len(h)
|
||||
}
|
||||
if e > exact || (e == exact && len(h) > length) {
|
||||
exact, length = e, len(h)
|
||||
}
|
||||
}
|
||||
return exact, length, ok
|
||||
}
|
||||
|
||||
func matches(m HTTPRouteMatch, r *http.Request) (candidate, bool) {
|
||||
var c candidate
|
||||
path, value := r.URL.Path, str(m.Path.Value)
|
||||
switch str(m.Path.Type) {
|
||||
case pathExact:
|
||||
if path != value {
|
||||
return c, false
|
||||
}
|
||||
c.exact = true
|
||||
c.prefixLen = len(value)
|
||||
default:
|
||||
trimmed := strings.TrimSuffix(value, "/")
|
||||
if trimmed != "" && path != trimmed && !strings.HasPrefix(path, trimmed+"/") {
|
||||
return c, false
|
||||
}
|
||||
c.prefixLen = len(value)
|
||||
c.prefix = value
|
||||
}
|
||||
if m.Method != nil {
|
||||
if r.Method != *m.Method {
|
||||
return c, false
|
||||
}
|
||||
c.method = true
|
||||
}
|
||||
for _, h := range m.Headers {
|
||||
if r.Header.Get(h.Name) != h.Value {
|
||||
return c, false
|
||||
}
|
||||
}
|
||||
c.headers = len(m.Headers)
|
||||
q := r.URL.Query()
|
||||
for _, qp := range m.QueryParams {
|
||||
if q.Get(qp.Name) != qp.Value {
|
||||
return c, false
|
||||
}
|
||||
}
|
||||
c.query = len(m.QueryParams)
|
||||
return c, true
|
||||
}
|
||||
|
||||
// chooseBackend picks a backend by weight (nil when there is none to pick).
|
||||
func chooseBackend(bs []ServeBackend) *ServeBackend {
|
||||
total := int32(0)
|
||||
for _, b := range bs {
|
||||
if b.Weight > 0 {
|
||||
total += b.Weight
|
||||
}
|
||||
}
|
||||
if total == 0 {
|
||||
return nil
|
||||
}
|
||||
n := rand.Int31n(total)
|
||||
for i := range bs {
|
||||
if bs[i].Weight <= 0 {
|
||||
continue
|
||||
}
|
||||
if n < bs[i].Weight {
|
||||
return &bs[i]
|
||||
}
|
||||
n -= bs[i].Weight
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// redirect is where a RequestRedirect filter sends a request, and with which status.
|
||||
func redirect(r *http.Request, f *HTTPRequestRedirectFilter, listenerPort int32, prefix string) (string, int) {
|
||||
scheme := "http"
|
||||
if r.TLS != nil {
|
||||
scheme = "https"
|
||||
}
|
||||
port := int(listenerPort)
|
||||
if f.Scheme != nil {
|
||||
// a new scheme brings its well-known port, unless a port is given
|
||||
scheme = *f.Scheme
|
||||
port = map[string]int{"http": 80, "https": 443}[scheme]
|
||||
}
|
||||
if f.Port != nil {
|
||||
port = int(*f.Port)
|
||||
}
|
||||
host := hostOnly(r.Host)
|
||||
if f.Hostname != nil {
|
||||
host = *f.Hostname
|
||||
}
|
||||
if !(scheme == "http" && port == 80) && !(scheme == "https" && port == 443) && port != 0 {
|
||||
host = net.JoinHostPort(host, strconv.Itoa(port))
|
||||
}
|
||||
path := r.URL.Path
|
||||
if f.Path != nil {
|
||||
path = rewritePath(path, f.Path, prefix)
|
||||
}
|
||||
code := http.StatusFound
|
||||
if f.StatusCode != nil {
|
||||
code = *f.StatusCode
|
||||
}
|
||||
return (&url.URL{Scheme: scheme, Host: host, Path: path, RawQuery: r.URL.RawQuery}).String(), code
|
||||
}
|
||||
|
||||
// rewritePath applies a path modifier: the whole path, or the matched prefix replaced.
|
||||
func rewritePath(path string, m *HTTPPathModifier, prefix string) string {
|
||||
switch m.Type {
|
||||
case "ReplaceFullPath":
|
||||
return str(m.ReplaceFullPath)
|
||||
case "ReplacePrefixMatch":
|
||||
rest := strings.TrimPrefix(path, strings.TrimSuffix(prefix, "/"))
|
||||
out := strings.TrimSuffix(str(m.ReplacePrefixMatch), "/") + rest
|
||||
if !strings.HasPrefix(out, "/") {
|
||||
out = "/" + out
|
||||
}
|
||||
return out
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func modifyHeaders(h http.Header, f *HTTPHeaderFilter) {
|
||||
for _, s := range f.Set {
|
||||
h.Set(s.Name, s.Value)
|
||||
}
|
||||
for _, a := range f.Add {
|
||||
h.Add(a.Name, a.Value)
|
||||
}
|
||||
for _, name := range f.Remove {
|
||||
h.Del(name)
|
||||
}
|
||||
}
|
||||
|
||||
// hostOnly is a Host header without its port, lowercased (the spec ignores the port when matching).
|
||||
func hostOnly(h string) string {
|
||||
if host, _, err := net.SplitHostPort(h); err == nil {
|
||||
h = host
|
||||
}
|
||||
return strings.ToLower(strings.TrimSuffix(h, "."))
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
package lb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
coreapi "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
)
|
||||
|
||||
// testProxy binds every requested address on loopback and remembers where.
|
||||
type testProxy struct {
|
||||
*Proxy
|
||||
mu sync.Mutex
|
||||
actual map[string]string
|
||||
}
|
||||
|
||||
func newTestProxy(t *testing.T) *testProxy {
|
||||
tp := &testProxy{Proxy: NewProxy(), actual: map[string]string{}}
|
||||
tp.listen = func(addr string) (net.Listener, error) {
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err == nil {
|
||||
tp.mu.Lock()
|
||||
tp.actual[addr] = ln.Addr().String()
|
||||
tp.mu.Unlock()
|
||||
}
|
||||
return ln, err
|
||||
}
|
||||
t.Cleanup(tp.Close)
|
||||
return tp
|
||||
}
|
||||
|
||||
func (tp *testProxy) at(port int32) string {
|
||||
tp.mu.Lock()
|
||||
defer tp.mu.Unlock()
|
||||
return tp.actual[net.JoinHostPort("192.0.2.1", strconv.Itoa(int(port)))]
|
||||
}
|
||||
|
||||
// named is a backend that answers with its name and the request it saw.
|
||||
func named(t *testing.T, name string) (*httptest.Server, Endpoint) {
|
||||
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("X-Backend", name)
|
||||
n, _ := io.Copy(io.Discard, r.Body)
|
||||
fmt.Fprintf(w, "%s path=%s host=%s xff=%s proto=%s xhost=%s realip=%s added=%s removed=%s body=%d",
|
||||
name, r.URL.Path, r.Host, r.Header.Get("X-Forwarded-For"), r.Header.Get("X-Forwarded-Proto"),
|
||||
r.Header.Get("X-Forwarded-Host"), r.Header.Get("X-Real-IP"), strings.Join(r.Header.Values("X-Added"), ","),
|
||||
r.Header.Get("X-Remove-Me"), n)
|
||||
}))
|
||||
t.Cleanup(s.Close)
|
||||
_, port, _ := net.SplitHostPort(s.Listener.Addr().String())
|
||||
p, _ := strconv.Atoi(port)
|
||||
return s, Endpoint{IP: net.ParseIP("127.0.0.1").To4(), Port: int32(p)}
|
||||
}
|
||||
|
||||
func backend(name string, ep Endpoint) []ServeBackend {
|
||||
return []ServeBackend{{Key: name, Weight: 1, Endpoints: []Endpoint{ep}}}
|
||||
}
|
||||
|
||||
func prefix(v string) HTTPRouteMatch {
|
||||
return HTTPRouteMatch{Path: &HTTPPathMatch{Type: ptr(pathPrefix), Value: ptr(v)}}
|
||||
}
|
||||
|
||||
func exact(v string) HTTPRouteMatch {
|
||||
return HTTPRouteMatch{Path: &HTTPPathMatch{Type: ptr(pathExact), Value: ptr(v)}}
|
||||
}
|
||||
|
||||
func get(t *testing.T, url, host string, hdr map[string]string) (int, string, http.Header) {
|
||||
t.Helper()
|
||||
req, _ := http.NewRequest("GET", url, nil)
|
||||
if host != "" {
|
||||
req.Host = host
|
||||
}
|
||||
for k, v := range hdr {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
c := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
|
||||
resp, err := c.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return resp.StatusCode, string(b), resp.Header
|
||||
}
|
||||
|
||||
func TestProxyRoutesByPrecedence(t *testing.T) {
|
||||
tp := newTestProxy(t)
|
||||
_, a := named(t, "A")
|
||||
_, b := named(t, "B")
|
||||
_, c := named(t, "C")
|
||||
_, d := named(t, "D")
|
||||
_, e := named(t, "E")
|
||||
t0 := time.Date(2026, 10, 14, 1, 0, 0, 0, time.UTC)
|
||||
routes := []ServeRoute{
|
||||
{Key: "ns/first", Created: t0, Rules: []ServeRule{
|
||||
{Matches: []HTTPRouteMatch{exact("/exact")}, Backends: backend("A", a)},
|
||||
{Matches: []HTTPRouteMatch{prefix("/api")}, Backends: backend("B", b)},
|
||||
{Matches: []HTTPRouteMatch{prefix("/")}, Backends: backend("C", c)},
|
||||
}},
|
||||
{Key: "ns/second", Created: t0.Add(time.Minute), Rules: []ServeRule{
|
||||
{Matches: []HTTPRouteMatch{prefix("/api/v2")}, Backends: backend("D", d)},
|
||||
{Matches: []HTTPRouteMatch{{Path: &HTTPPathMatch{Type: ptr(pathPrefix), Value: ptr("/api")},
|
||||
Headers: []HTTPHeaderMatch{{Name: "X-Ver", Value: "2"}}}}, Backends: backend("E", e)},
|
||||
}},
|
||||
}
|
||||
if f := tp.Apply([]ServeListener{{Gateway: "ns/gw", Name: "http", Address: net.ParseIP("192.0.2.1"), Port: 8080, Routes: routes}}); len(f) != 0 {
|
||||
t.Fatal(f)
|
||||
}
|
||||
base := "http://" + tp.at(8080)
|
||||
for path, want := range map[string]string{
|
||||
"/exact": "A", "/exactly": "C", "/api/x": "B", "/api": "B", "/api/v2/y": "D", "/apix": "C", "/": "C",
|
||||
} {
|
||||
if _, body, _ := get(t, base+path, "example.com", nil); !strings.HasPrefix(body, want+" ") {
|
||||
t.Errorf("%s went to %q, want %s", path, body, want)
|
||||
}
|
||||
}
|
||||
// same prefix length: the match with more headers takes it, though its route is younger
|
||||
if _, body, _ := get(t, base+"/api/x", "example.com", map[string]string{"X-Ver": "2"}); !strings.HasPrefix(body, "E ") {
|
||||
t.Errorf("header match lost: %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyHostPrecedence(t *testing.T) {
|
||||
tp := newTestProxy(t)
|
||||
_, w := named(t, "W")
|
||||
_, x := named(t, "X")
|
||||
routes := []ServeRoute{
|
||||
{Key: "ns/wild", Hostnames: []string{"*.example.com"}, Rules: []ServeRule{{Matches: []HTTPRouteMatch{prefix("/")}, Backends: backend("W", w)}}},
|
||||
{Key: "ns/exact", Hostnames: []string{"a.example.com"}, Rules: []ServeRule{{Matches: []HTTPRouteMatch{prefix("/")}, Backends: backend("X", x)}}},
|
||||
}
|
||||
tp.Apply([]ServeListener{{Gateway: "ns/gw", Name: "http", Address: net.ParseIP("192.0.2.1"), Port: 8081, Routes: routes}})
|
||||
base := "http://" + tp.at(8081)
|
||||
for host, want := range map[string]string{"a.example.com": "X", "A.Example.com:8081": "X", "b.example.com": "W"} {
|
||||
if _, body, _ := get(t, base+"/", host, nil); !strings.HasPrefix(body, want+" ") {
|
||||
t.Errorf("host %s went to %q, want %s", host, body, want)
|
||||
}
|
||||
}
|
||||
if code, _, _ := get(t, base+"/", "example.org", nil); code != http.StatusNotFound {
|
||||
t.Errorf("a host no route names: %d, want 404", code)
|
||||
}
|
||||
}
|
||||
|
||||
func certSecret(t *testing.T, name, rv string) *coreapi.Secret {
|
||||
s := tlsSecret(t, "gateway-system", strings.ReplaceAll(name, ".", "-"), name)
|
||||
s.UID = types.UID("uid-" + name)
|
||||
s.ResourceVersion = rv
|
||||
return s
|
||||
}
|
||||
|
||||
func dialTLS(t *testing.T, addr, sni string) (*x509.Certificate, error) {
|
||||
conn, err := tls.Dial("tcp", addr, &tls.Config{ServerName: sni, InsecureSkipVerify: true})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer conn.Close()
|
||||
return conn.ConnectionState().PeerCertificates[0], nil
|
||||
}
|
||||
|
||||
func TestProxyTLSBySNIAndSwapsCertificates(t *testing.T) {
|
||||
tp := newTestProxy(t)
|
||||
_, a := named(t, "A")
|
||||
_, b := named(t, "B")
|
||||
listeners := func(certA *coreapi.Secret) []ServeListener {
|
||||
return []ServeListener{
|
||||
{Gateway: "ns/gw", Name: "a", Address: net.ParseIP("192.0.2.1"), Port: 8443, HTTPS: true, Hostname: "a.test", Cert: certA,
|
||||
Routes: []ServeRoute{{Key: "ns/a", Rules: []ServeRule{{Matches: []HTTPRouteMatch{prefix("/")}, Backends: backend("A", a)}}}}},
|
||||
{Gateway: "ns/gw", Name: "b", Address: net.ParseIP("192.0.2.1"), Port: 8443, HTTPS: true, Hostname: "b.test", Cert: certSecret(t, "b.test", "1"),
|
||||
Routes: []ServeRoute{{Key: "ns/b", Rules: []ServeRule{{Matches: []HTTPRouteMatch{prefix("/")}, Backends: backend("B", b)}}}}},
|
||||
}
|
||||
}
|
||||
first := certSecret(t, "a.test", "1")
|
||||
if f := tp.Apply(listeners(first)); len(f) != 0 {
|
||||
t.Fatal(f)
|
||||
}
|
||||
addr := tp.at(8443)
|
||||
ca, err := dialTLS(t, addr, "a.test")
|
||||
if err != nil || ca.Subject.CommonName != "a.test" {
|
||||
t.Fatalf("SNI a.test: %v %v", ca, err)
|
||||
}
|
||||
cb, err := dialTLS(t, addr, "b.test")
|
||||
if err != nil || cb.Subject.CommonName != "b.test" {
|
||||
t.Fatalf("SNI b.test: %v %v", cb, err)
|
||||
}
|
||||
if _, err := dialTLS(t, addr, "c.test"); err == nil {
|
||||
t.Fatal("a name no listener has must not get a certificate")
|
||||
}
|
||||
// requests go by listener, over HTTP/2 when the client offers it
|
||||
client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true, ServerName: "b.test"}, ForceAttemptHTTP2: true}}
|
||||
req, _ := http.NewRequest("GET", "https://"+addr+"/", nil)
|
||||
req.Host = "b.test"
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if !strings.HasPrefix(string(body), "B ") || !strings.Contains(string(body), "proto=https") || resp.ProtoMajor != 2 {
|
||||
t.Fatalf("over HTTPS to b.test: %s %q", resp.Proto, body)
|
||||
}
|
||||
// a renewed Secret: new handshakes get the new certificate on the same, still-bound port
|
||||
renewed := certSecret(t, "a.test", "2")
|
||||
tp.Apply(listeners(renewed))
|
||||
if tp.at(8443) != addr {
|
||||
t.Fatal("the port was rebound for a certificate change")
|
||||
}
|
||||
cn, err := dialTLS(t, addr, "a.test")
|
||||
if err != nil || cn.SerialNumber.Cmp(ca.SerialNumber) == 0 {
|
||||
t.Fatalf("still the old certificate after the swap: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyRedirectsAndForwards(t *testing.T) {
|
||||
tp := newTestProxy(t)
|
||||
_, a := named(t, "A")
|
||||
routes := []ServeRoute{{Key: "ns/r", Rules: []ServeRule{
|
||||
{Matches: []HTTPRouteMatch{prefix("/old")}, Filters: []HTTPRouteFilter{{Type: filterRequestRedirect,
|
||||
RequestRedirect: &HTTPRequestRedirectFilter{Scheme: ptr("https"), StatusCode: ptr(301)}}}},
|
||||
{Matches: []HTTPRouteMatch{prefix("/moved")}, Filters: []HTTPRouteFilter{{Type: filterRequestRedirect,
|
||||
RequestRedirect: &HTTPRequestRedirectFilter{Path: &HTTPPathModifier{Type: "ReplacePrefixMatch", ReplacePrefixMatch: ptr("/new")}}}}},
|
||||
{Matches: []HTTPRouteMatch{prefix("/v1")}, Backends: backend("A", a), Filters: []HTTPRouteFilter{
|
||||
{Type: filterRequestHeaderModifier, RequestHeaderModifier: &HTTPHeaderFilter{
|
||||
Set: []HTTPHeader{{Name: "X-Set", Value: "s"}}, Add: []HTTPHeader{{Name: "X-Added", Value: "two"}}, Remove: []string{"X-Remove-Me"}}},
|
||||
{Type: filterResponseHeaderModifier, ResponseHeaderModifier: &HTTPHeaderFilter{Set: []HTTPHeader{{Name: "X-Resp", Value: "r"}}}},
|
||||
{Type: filterURLRewrite, URLRewrite: &HTTPURLRewriteFilter{Path: &HTTPPathModifier{Type: "ReplacePrefixMatch", ReplacePrefixMatch: ptr("/v2")}}},
|
||||
}},
|
||||
{Matches: []HTTPRouteMatch{prefix("/")}, Backends: backend("A", a)},
|
||||
}}}
|
||||
tp.Apply([]ServeListener{{Gateway: "ns/gw", Name: "http", Address: net.ParseIP("192.0.2.1"), Port: 80, Routes: routes}})
|
||||
base := "http://" + tp.at(80)
|
||||
code, _, h := get(t, base+"/old/page?q=1", "git.levk.dev", nil)
|
||||
if code != http.StatusMovedPermanently || h.Get("Location") != "https://git.levk.dev/old/page?q=1" {
|
||||
t.Fatalf("scheme redirect: %d %q", code, h.Get("Location"))
|
||||
}
|
||||
code, _, h = get(t, base+"/moved/x", "git.levk.dev", nil)
|
||||
if code != http.StatusFound || h.Get("Location") != "http://git.levk.dev/new/x" {
|
||||
t.Fatalf("prefix redirect: %d %q", code, h.Get("Location"))
|
||||
}
|
||||
// the client's own claims about who it is are not passed on; the backend sees this hop's view
|
||||
_, body, _ := get(t, base+"/", "git.levk.dev", map[string]string{"X-Forwarded-For": "6.6.6.6", "X-Real-IP": "6.6.6.6"})
|
||||
if !strings.Contains(body, "host=git.levk.dev") || !strings.Contains(body, "xff=127.0.0.1 ") ||
|
||||
!strings.Contains(body, "proto=http ") || !strings.Contains(body, "xhost=git.levk.dev") || !strings.Contains(body, "realip=127.0.0.1") {
|
||||
t.Fatalf("forwarded headers: %q", body)
|
||||
}
|
||||
_, body, h = get(t, base+"/v1/thing", "git.levk.dev", map[string]string{"X-Added": "one", "X-Remove-Me": "x"})
|
||||
if !strings.Contains(body, "path=/v2/thing") || !strings.Contains(body, "added=one,two") || !strings.Contains(body, "removed= ") || h.Get("X-Resp") != "r" {
|
||||
t.Fatalf("filters: %q %v", body, h)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyAnswersWhatItCannotRoute(t *testing.T) {
|
||||
tp := newTestProxy(t)
|
||||
routes := []ServeRoute{{Key: "ns/r", Rules: []ServeRule{
|
||||
{Matches: []HTTPRouteMatch{prefix("/broken")}, Broken: true},
|
||||
{Matches: []HTTPRouteMatch{prefix("/empty")}, Backends: []ServeBackend{{Key: "ns/svc:80", Weight: 1}}},
|
||||
}}}
|
||||
tp.Apply([]ServeListener{{Gateway: "ns/gw", Name: "http", Address: net.ParseIP("192.0.2.1"), Port: 8082, Routes: routes}})
|
||||
base := "http://" + tp.at(8082)
|
||||
for path, want := range map[string]int{"/broken": 500, "/empty": 503, "/nothing": 404} {
|
||||
if code, _, _ := get(t, base+path, "x", nil); code != want {
|
||||
t.Errorf("%s: %d, want %d", path, code, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyStreamsLargeBodies(t *testing.T) {
|
||||
tp := newTestProxy(t)
|
||||
_, a := named(t, "A")
|
||||
tp.Apply([]ServeListener{{Gateway: "ns/gw", Name: "http", Address: net.ParseIP("192.0.2.1"), Port: 8083,
|
||||
Routes: []ServeRoute{{Key: "ns/r", Rules: []ServeRule{{Matches: []HTTPRouteMatch{prefix("/")}, Backends: backend("A", a)}}}}}})
|
||||
const size = 32 << 20
|
||||
resp, err := http.Post("http://"+tp.at(8083)+"/upload", "application/octet-stream", bytes.NewReader(make([]byte, size)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if !strings.Contains(string(body), fmt.Sprintf("body=%d", size)) {
|
||||
t.Fatalf("upload: %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyBindFailureAndClose(t *testing.T) {
|
||||
tp := newTestProxy(t)
|
||||
tp.listen = func(addr string) (net.Listener, error) { return nil, fmt.Errorf("address in use") }
|
||||
f := tp.Apply([]ServeListener{{Gateway: "ns/gw", Name: "http", Address: net.ParseIP("192.0.2.1"), Port: 8084}})
|
||||
if f["ns/gw/http"] == nil {
|
||||
t.Fatalf("bind failure not reported: %v", f)
|
||||
}
|
||||
tp2 := newTestProxy(t)
|
||||
tp2.Apply([]ServeListener{{Gateway: "ns/gw", Name: "http", Address: net.ParseIP("192.0.2.1"), Port: 8085}})
|
||||
addr := tp2.at(8085)
|
||||
tp2.Apply(nil)
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
c, err := net.DialTimeout("tcp", addr, 200*time.Millisecond)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
c.Close()
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("a port no longer served still accepts connections")
|
||||
}
|
||||
@@ -0,0 +1,576 @@
|
||||
package lb
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"math/big"
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
coreapi "k8s.io/api/core/v1"
|
||||
discovery "k8s.io/api/discovery/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
var gwNow = metav1.NewTime(time.Date(2026, 10, 14, 4, 0, 0, 0, time.UTC))
|
||||
|
||||
// selfSigned is a certificate and key for the given names, PEM-encoded.
|
||||
func selfSigned(t *testing.T, names ...string) ([]byte, []byte) {
|
||||
t.Helper()
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tmpl := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: names[0]}, DNSNames: names,
|
||||
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(24 * time.Hour),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kb, _ := x509.MarshalECPrivateKey(key)
|
||||
return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: kb})
|
||||
}
|
||||
|
||||
func tlsSecret(t *testing.T, namespace, name string, names ...string) *coreapi.Secret {
|
||||
crt, key := selfSigned(t, names...)
|
||||
return &coreapi.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}, Type: coreapi.SecretTypeTLS,
|
||||
Data: map[string][]byte{coreapi.TLSCertKey: crt, coreapi.TLSPrivateKeyKey: key}}
|
||||
}
|
||||
|
||||
func gwNodes() []Node {
|
||||
return []Node{
|
||||
{Name: "cp", Addresses: []net.IP{net.ParseIP("86.107.168.24").To4()}, WeaveIP: net.ParseIP("10.32.0.1").To4(), WeaveMAC: mustMAC("02:00:00:00:00:01")},
|
||||
{Name: "w15", Addresses: []net.IP{net.ParseIP("86.107.168.15").To4()}, WeaveIP: net.ParseIP("10.44.0.0").To4(), WeaveMAC: mustMAC("02:00:00:00:00:02")},
|
||||
}
|
||||
}
|
||||
|
||||
func mustMAC(s string) net.HardwareAddr {
|
||||
m, _ := net.ParseMAC(s)
|
||||
return m
|
||||
}
|
||||
|
||||
func ourClass() *GatewayClass {
|
||||
return &GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: "reweave", Generation: 1}, Spec: GatewayClassSpec{ControllerName: GatewayControllerName}}
|
||||
}
|
||||
|
||||
// giteaGateway is the shape Gitea's documentation prescribes: HTTPS with a certificate, plain HTTP, and SSH over TCP.
|
||||
func giteaGateway() *Gateway {
|
||||
return &Gateway{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "gateway-system", Name: "shared", Generation: 3},
|
||||
Spec: GatewaySpec{
|
||||
GatewayClassName: "reweave",
|
||||
Addresses: []GatewayAddress{{Value: "86.107.168.15"}},
|
||||
Listeners: []Listener{
|
||||
{Name: "https-gitea", Port: 443, Protocol: ProtocolHTTPS, Hostname: ptr("git.levk.dev"),
|
||||
TLS: &ListenerTLSConfig{CertificateRefs: []SecretObjectReference{{Name: "git-levk-dev"}}},
|
||||
AllowedRoutes: &AllowedRoutes{Namespaces: &RouteNamespaces{From: ptr("Selector"), Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": "gitea"}}}}},
|
||||
{Name: "http", Port: 80, Protocol: ProtocolHTTP, Hostname: ptr("git.levk.dev"),
|
||||
AllowedRoutes: &AllowedRoutes{Namespaces: &RouteNamespaces{From: ptr("All")}}},
|
||||
{Name: "ssh", Port: 22, Protocol: ProtocolTCP,
|
||||
AllowedRoutes: &AllowedRoutes{Namespaces: &RouteNamespaces{From: ptr("All")}}},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func giteaService() *coreapi.Service {
|
||||
return &coreapi.Service{ObjectMeta: metav1.ObjectMeta{Namespace: "gitea", Name: "gitea"}, Spec: coreapi.ServiceSpec{
|
||||
Ports: []coreapi.ServicePort{{Name: "http", Port: 3000}, {Name: "ssh", Port: 22}},
|
||||
}}
|
||||
}
|
||||
|
||||
func giteaSlice() *discovery.EndpointSlice {
|
||||
return &discovery.EndpointSlice{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "gitea", Name: "gitea-abc", Labels: map[string]string{serviceNameLabel: "gitea"}},
|
||||
AddressType: discovery.AddressTypeIPv4,
|
||||
Ports: []discovery.EndpointPort{{Name: ptr("http"), Port: ptr(int32(3000))}, {Name: ptr("ssh"), Port: ptr(int32(22))}},
|
||||
Endpoints: []discovery.Endpoint{{Addresses: []string{"10.44.0.12"}, Conditions: discovery.EndpointConditions{Ready: ptr(true)}}},
|
||||
}
|
||||
}
|
||||
|
||||
func giteaRoutes() ([]*HTTPRoute, []*TCPRoute) {
|
||||
t0 := metav1.NewTime(time.Date(2026, 10, 14, 1, 0, 0, 0, time.UTC))
|
||||
https := &HTTPRoute{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "gitea", Name: "gitea", Generation: 1, CreationTimestamp: t0},
|
||||
Spec: HTTPRouteSpec{
|
||||
ParentRefs: []ParentReference{{Namespace: ptr("gateway-system"), Name: "shared", SectionName: ptr("https-gitea")}},
|
||||
Hostnames: []string{"git.levk.dev"},
|
||||
Rules: []HTTPRouteRule{{BackendRefs: []HTTPBackendRef{{BackendRef: BackendRef{Name: "gitea", Port: ptr(int32(3000))}}}}},
|
||||
},
|
||||
}
|
||||
redirect := &HTTPRoute{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "gitea", Name: "gitea-redirect", Generation: 1, CreationTimestamp: t0},
|
||||
Spec: HTTPRouteSpec{
|
||||
ParentRefs: []ParentReference{{Namespace: ptr("gateway-system"), Name: "shared", SectionName: ptr("http")}},
|
||||
Rules: []HTTPRouteRule{{Filters: []HTTPRouteFilter{{Type: filterRequestRedirect,
|
||||
RequestRedirect: &HTTPRequestRedirectFilter{Scheme: ptr("https"), StatusCode: ptr(301)}}}}},
|
||||
},
|
||||
}
|
||||
ssh := &TCPRoute{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "gitea", Name: "gitea-ssh", Generation: 1, CreationTimestamp: t0},
|
||||
Spec: TCPRouteSpec{
|
||||
ParentRefs: []ParentReference{{Namespace: ptr("gateway-system"), Name: "shared", SectionName: ptr("ssh")}},
|
||||
Rules: []TCPRouteRule{{BackendRefs: []BackendRef{{Name: "gitea", Port: ptr(int32(22))}}}},
|
||||
},
|
||||
}
|
||||
return []*HTTPRoute{https, redirect}, []*TCPRoute{ssh}
|
||||
}
|
||||
|
||||
func giteaInput(t *testing.T) GatewayInput {
|
||||
hr, tr := giteaRoutes()
|
||||
return GatewayInput{
|
||||
Classes: []*GatewayClass{ourClass()}, Gateways: []*Gateway{giteaGateway()}, HTTPRoutes: hr, TCPRoutes: tr,
|
||||
Namespaces: []*coreapi.Namespace{
|
||||
{ObjectMeta: metav1.ObjectMeta{Name: "gitea", Labels: map[string]string{"kubernetes.io/metadata.name": "gitea"}}},
|
||||
{ObjectMeta: metav1.ObjectMeta{Name: "gateway-system", Labels: map[string]string{"kubernetes.io/metadata.name": "gateway-system"}}},
|
||||
},
|
||||
Services: []*coreapi.Service{giteaService()}, Slices: []*discovery.EndpointSlice{giteaSlice()},
|
||||
Secrets: []*coreapi.Secret{tlsSecret(t, "gateway-system", "git-levk-dev", "git.levk.dev")},
|
||||
SecretNamespaces: map[string]bool{"gateway-system": true},
|
||||
Now: gwNow,
|
||||
}
|
||||
}
|
||||
|
||||
func condOf(cs []metav1.Condition, typ string) metav1.Condition {
|
||||
for _, c := range cs {
|
||||
if c.Type == typ {
|
||||
return c
|
||||
}
|
||||
}
|
||||
return metav1.Condition{}
|
||||
}
|
||||
|
||||
func TestGatewayServedByItsHolder(t *testing.T) {
|
||||
in := giteaInput(t)
|
||||
d := ComputeGateways("w15", gwNodes(), in)
|
||||
if len(d.Problems) != 0 {
|
||||
t.Fatalf("problems: %v", d.Problems)
|
||||
}
|
||||
// HTTPS and HTTP served by the agent; SSH forwarded in the kernel to Gitea's pod with the client address intact
|
||||
if len(d.Serve) != 2 {
|
||||
t.Fatalf("want the https and http listeners served, got %+v", d.Serve)
|
||||
}
|
||||
https := d.Serve[0]
|
||||
if !https.HTTPS || https.Port != 443 || https.Hostname != "git.levk.dev" || https.Cert == nil || len(https.Routes) != 1 {
|
||||
t.Fatalf("https listener: %+v", https)
|
||||
}
|
||||
be := https.Routes[0].Rules[0].Backends
|
||||
if len(be) != 1 || be[0].Key != "gitea/gitea:3000" || len(be[0].Endpoints) != 1 || be[0].Endpoints[0].String() != "10.44.0.12:3000" {
|
||||
t.Fatalf("https backends: %+v", be)
|
||||
}
|
||||
if len(d.Forwards) != 1 || d.Forwards[0].Ports[0].Port != 22 || d.Forwards[0].Ports[0].Endpoints[0].String() != "10.44.0.12:22" {
|
||||
t.Fatalf("ssh forward: %+v", d.Forwards)
|
||||
}
|
||||
if len(d.Holding) != 1 || d.Holding[0] != "w15" {
|
||||
t.Fatalf("holding: %v", d.Holding)
|
||||
}
|
||||
// the holder writes the Gateway's status: accepted, programmed, its address, each listener with one route
|
||||
if len(d.Gateways) != 1 {
|
||||
t.Fatalf("gateway status writes: %+v", d.Gateways)
|
||||
}
|
||||
st := d.Gateways[0].Status
|
||||
if c := condOf(st.Conditions, condProgrammed); c.Status != metav1.ConditionTrue || c.ObservedGeneration != 3 {
|
||||
t.Fatalf("programmed: %+v", c)
|
||||
}
|
||||
if len(st.Addresses) != 1 || st.Addresses[0].Value != "86.107.168.15" {
|
||||
t.Fatalf("addresses: %+v", st.Addresses)
|
||||
}
|
||||
for _, l := range st.Listeners {
|
||||
if l.AttachedRoutes != 1 || condOf(l.Conditions, condProgrammed).Status != metav1.ConditionTrue {
|
||||
t.Fatalf("listener %s: %+v", l.Name, l)
|
||||
}
|
||||
}
|
||||
// route statuses come from the first node by name, not the holder
|
||||
if len(d.Routes) != 0 || len(d.Classes) != 0 {
|
||||
t.Fatalf("w15 wrote route or class statuses: %+v %+v", d.Routes, d.Classes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGatewayElsewhere(t *testing.T) {
|
||||
in := giteaInput(t)
|
||||
d := ComputeGateways("cp", gwNodes(), in)
|
||||
if len(d.Serve) != 0 || len(d.Forwards) != 0 || len(d.Gateways) != 0 {
|
||||
t.Fatalf("cp serves or writes what w15 holds: %+v %+v %+v", d.Serve, d.Forwards, d.Gateways)
|
||||
}
|
||||
// but steers replies of w15's forwards back to it, and writes class and route statuses (first by name)
|
||||
if len(d.Holding) != 1 || d.Holding[0] != "w15" {
|
||||
t.Fatalf("holding: %v", d.Holding)
|
||||
}
|
||||
if len(d.Classes) != 1 || condOf(d.Classes[0].Status.Conditions, condAccepted).Status != metav1.ConditionTrue {
|
||||
t.Fatalf("class status: %+v", d.Classes)
|
||||
}
|
||||
if len(d.Routes) != 3 {
|
||||
t.Fatalf("want 3 route statuses, got %+v", d.Routes)
|
||||
}
|
||||
for _, r := range d.Routes {
|
||||
if len(r.Status.Parents) != 1 || condOf(r.Status.Parents[0].Conditions, condAccepted).Status != metav1.ConditionTrue ||
|
||||
condOf(r.Status.Parents[0].Conditions, condResolvedRefs).Status != metav1.ConditionTrue {
|
||||
t.Fatalf("route %s/%s status: %+v", r.Namespace, r.Name, r.Status)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGatewayStatusIsWrittenOnce(t *testing.T) {
|
||||
in := giteaInput(t)
|
||||
d := ComputeGateways("w15", gwNodes(), in)
|
||||
in.Gateways[0].Status = d.Gateways[0].Status
|
||||
in.Now = metav1.NewTime(gwNow.Add(time.Hour)) // later passes see the status they wrote
|
||||
if again := ComputeGateways("w15", gwNodes(), in); len(again.Gateways) != 0 {
|
||||
t.Fatalf("unchanged status rewritten: %+v", again.Gateways)
|
||||
}
|
||||
c := ComputeGateways("cp", gwNodes(), in)
|
||||
for _, r := range c.Routes {
|
||||
for _, hr := range in.HTTPRoutes {
|
||||
if hr.Name == r.Name {
|
||||
hr.Status = r.Status
|
||||
}
|
||||
}
|
||||
for _, tr := range in.TCPRoutes {
|
||||
if tr.Name == r.Name {
|
||||
tr.Status = r.Status
|
||||
}
|
||||
}
|
||||
}
|
||||
in.Classes[0].Status = c.Classes[0].Status
|
||||
if again := ComputeGateways("cp", gwNodes(), in); len(again.Routes) != 0 || len(again.Classes) != 0 {
|
||||
t.Fatalf("unchanged route or class status rewritten: %+v %+v", again.Routes, again.Classes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGatewayFailsClosedWithoutHolder(t *testing.T) {
|
||||
in := giteaInput(t)
|
||||
in.Gateways[0].Spec.Addresses[0].Value = "198.51.100.7"
|
||||
for _, self := range []string{"cp", "w15"} {
|
||||
d := ComputeGateways(self, gwNodes(), in)
|
||||
if len(d.Serve) != 0 || len(d.Forwards) != 0 || len(d.Holding) != 0 {
|
||||
t.Fatalf("%s serves an address nobody holds: %+v", self, d)
|
||||
}
|
||||
}
|
||||
d := ComputeGateways("cp", gwNodes(), in)
|
||||
if len(d.Gateways) != 1 {
|
||||
t.Fatalf("the first node by name must write the failure: %+v", d.Gateways)
|
||||
}
|
||||
c := condOf(d.Gateways[0].Status.Conditions, condProgrammed)
|
||||
if c.Status != metav1.ConditionFalse || c.Reason != "AddressNotUsable" || len(d.Gateways[0].Status.Addresses) != 0 {
|
||||
t.Fatalf("status: %+v", d.Gateways[0].Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGatewayAddressMustBeOneIPv4(t *testing.T) {
|
||||
in := giteaInput(t)
|
||||
in.Gateways[0].Spec.Addresses = nil
|
||||
d := ComputeGateways("cp", gwNodes(), in)
|
||||
if c := condOf(d.Gateways[0].Status.Conditions, condAccepted); c.Status != metav1.ConditionFalse || c.Reason != "UnsupportedAddress" {
|
||||
t.Fatalf("accepted: %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListenerCertificateReferences(t *testing.T) {
|
||||
cases := map[string]struct {
|
||||
mutate func(in *GatewayInput)
|
||||
reason string
|
||||
}{
|
||||
"missing Secret": {func(in *GatewayInput) { in.Secrets = nil }, "InvalidCertificateRef"},
|
||||
"other namespace without grant": {func(in *GatewayInput) {
|
||||
in.Gateways[0].Spec.Listeners[0].TLS.CertificateRefs[0].Namespace = ptr("gitea")
|
||||
in.SecretNamespaces["gitea"] = true
|
||||
}, "RefNotPermitted"},
|
||||
"namespace the agent cannot read": {func(in *GatewayInput) {
|
||||
in.SecretNamespaces = map[string]bool{}
|
||||
}, "RefNotPermitted"},
|
||||
"not a certificate": {func(in *GatewayInput) {
|
||||
in.Secrets[0].Data[coreapi.TLSCertKey] = []byte("nope")
|
||||
}, "InvalidCertificateRef"},
|
||||
}
|
||||
for name, c := range cases {
|
||||
in := giteaInput(t)
|
||||
c.mutate(&in)
|
||||
d := ComputeGateways("w15", gwNodes(), in)
|
||||
var https ListenerStatus
|
||||
for _, l := range d.Gateways[0].Status.Listeners {
|
||||
if l.Name == "https-gitea" {
|
||||
https = l
|
||||
}
|
||||
}
|
||||
if r := condOf(https.Conditions, condResolvedRefs); r.Status != metav1.ConditionFalse || r.Reason != c.reason {
|
||||
t.Errorf("%s: resolvedRefs %+v", name, r)
|
||||
}
|
||||
if p := condOf(https.Conditions, condProgrammed); p.Status != metav1.ConditionFalse {
|
||||
t.Errorf("%s: an HTTPS listener without its certificate must not be programmed", name)
|
||||
}
|
||||
for _, s := range d.Serve {
|
||||
if s.HTTPS {
|
||||
t.Errorf("%s: HTTPS listener served without a certificate", name)
|
||||
}
|
||||
}
|
||||
if a := condOf(d.Gateways[0].Status.Conditions, condAccepted); a.Reason != "ListenersNotValid" || a.Status != metav1.ConditionTrue {
|
||||
t.Errorf("%s: gateway accepted %+v", name, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrossNamespaceCertificateWithGrant(t *testing.T) {
|
||||
in := giteaInput(t)
|
||||
in.Gateways[0].Spec.Listeners[0].TLS.CertificateRefs[0].Namespace = ptr("gitea")
|
||||
in.Secrets = []*coreapi.Secret{tlsSecret(t, "gitea", "git-levk-dev", "git.levk.dev")}
|
||||
in.SecretNamespaces = map[string]bool{"gitea": true}
|
||||
in.Grants = []*ReferenceGrant{{ObjectMeta: metav1.ObjectMeta{Namespace: "gitea", Name: "gw"}, Spec: ReferenceGrantSpec{
|
||||
From: []ReferenceGrantFrom{{Group: GatewayGroup, Kind: kindGateway, Namespace: "gateway-system"}},
|
||||
To: []ReferenceGrantTo{{Group: "", Kind: kindSecret}},
|
||||
}}}
|
||||
d := ComputeGateways("w15", gwNodes(), in)
|
||||
if len(d.Serve) != 2 || d.Serve[0].Cert == nil {
|
||||
t.Fatalf("a granted certificate in another namespace must be served: %+v", d.Serve)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListenerConflicts(t *testing.T) {
|
||||
in := giteaInput(t)
|
||||
l := in.Gateways[0].Spec.Listeners
|
||||
in.Gateways[0].Spec.Listeners = append(l,
|
||||
Listener{Name: "ssh2", Port: 22, Protocol: ProtocolTCP}, // two TCP listeners on 22
|
||||
Listener{Name: "http-on-443", Port: 443, Protocol: ProtocolHTTP}, // HTTP and HTTPS on 443
|
||||
)
|
||||
d := ComputeGateways("w15", gwNodes(), in)
|
||||
want := map[string]string{"ssh": "HostnameConflict", "ssh2": "HostnameConflict", "https-gitea": "ProtocolConflict", "http-on-443": "ProtocolConflict", "http": ""}
|
||||
for _, ls := range d.Gateways[0].Status.Listeners {
|
||||
c := condOf(ls.Conditions, condConflicted)
|
||||
if want[ls.Name] == "" {
|
||||
if c.Status != metav1.ConditionFalse {
|
||||
t.Errorf("%s: conflicted %+v", ls.Name, c)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if c.Status != metav1.ConditionTrue || c.Reason != want[ls.Name] {
|
||||
t.Errorf("%s: want conflicted %s, got %+v", ls.Name, want[ls.Name], c)
|
||||
}
|
||||
}
|
||||
if len(d.Forwards) != 0 {
|
||||
t.Fatalf("conflicted TCP listeners forwarded: %+v", d.Forwards)
|
||||
}
|
||||
if len(d.Serve) != 1 || d.Serve[0].Name != "http" {
|
||||
t.Fatalf("only the plain http listener on 80 is unconflicted: %+v", d.Serve)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListenerPortTakenByLoadBalancerService(t *testing.T) {
|
||||
in := giteaInput(t)
|
||||
class := Class
|
||||
in.Services = append(in.Services, &coreapi.Service{ObjectMeta: metav1.ObjectMeta{Namespace: "mail", Name: "smtp"}, Spec: coreapi.ServiceSpec{
|
||||
Type: coreapi.ServiceTypeLoadBalancer, LoadBalancerClass: &class, LoadBalancerIP: "86.107.168.15",
|
||||
Ports: []coreapi.ServicePort{{Port: 22}},
|
||||
}})
|
||||
d := ComputeGateways("w15", gwNodes(), in)
|
||||
for _, ls := range d.Gateways[0].Status.Listeners {
|
||||
if ls.Name == "ssh" {
|
||||
if c := condOf(ls.Conditions, condAccepted); c.Status != metav1.ConditionFalse || c.Reason != "PortUnavailable" {
|
||||
t.Fatalf("ssh accepted: %+v", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(d.Forwards) != 0 {
|
||||
t.Fatalf("a listener on a port a Service forwards must not be forwarded: %+v", d.Forwards)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRouteAttachment(t *testing.T) {
|
||||
cases := map[string]struct {
|
||||
mutate func(in *GatewayInput)
|
||||
route string
|
||||
reason string
|
||||
}{
|
||||
"unknown section": {func(in *GatewayInput) { in.HTTPRoutes[0].Spec.ParentRefs[0].SectionName = ptr("nope") }, "gitea", "NoMatchingParent"},
|
||||
"namespace not selected": {func(in *GatewayInput) { in.Namespaces[0].Labels = nil }, "gitea", "NotAllowedByListeners"},
|
||||
"hostname outside the listener's": {func(in *GatewayInput) { in.HTTPRoutes[0].Spec.Hostnames = []string{"git.example.net"} }, "gitea", "NoMatchingListenerHostname"},
|
||||
"unsupported filter": {func(in *GatewayInput) {
|
||||
in.HTTPRoutes[0].Spec.Rules[0].Filters = []HTTPRouteFilter{{Type: "RequestMirror"}}
|
||||
}, "gitea", "UnsupportedValue"},
|
||||
"regular expression path": {func(in *GatewayInput) {
|
||||
in.HTTPRoutes[0].Spec.Rules[0].Matches = []HTTPRouteMatch{{Path: &HTTPPathMatch{Type: ptr("RegularExpression"), Value: ptr("/.*")}}}
|
||||
}, "gitea", "UnsupportedValue"},
|
||||
"wrong port": {func(in *GatewayInput) { in.TCPRoutes[0].Spec.ParentRefs[0].Port = ptr(int32(2222)) }, "gitea-ssh", "NoMatchingParent"},
|
||||
}
|
||||
for name, c := range cases {
|
||||
in := giteaInput(t)
|
||||
c.mutate(&in)
|
||||
d := ComputeGateways("cp", gwNodes(), in)
|
||||
found := false
|
||||
for _, r := range d.Routes {
|
||||
if r.Name != c.route {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
a := condOf(r.Status.Parents[0].Conditions, condAccepted)
|
||||
if a.Status != metav1.ConditionFalse || a.Reason != c.reason {
|
||||
t.Errorf("%s: accepted %+v", name, a)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("%s: no status for route %s", name, c.route)
|
||||
}
|
||||
h := ComputeGateways("w15", gwNodes(), in)
|
||||
if c.route == "gitea" {
|
||||
for _, s := range h.Serve {
|
||||
if s.Name == "https-gitea" && len(s.Routes) != 0 {
|
||||
t.Errorf("%s: a route that is not accepted was attached", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendNotFoundBreaksTheRule(t *testing.T) {
|
||||
in := giteaInput(t)
|
||||
in.HTTPRoutes[0].Spec.Rules[0].BackendRefs[0].Name = "nope"
|
||||
d := ComputeGateways("cp", gwNodes(), in)
|
||||
for _, r := range d.Routes {
|
||||
if r.Name == "gitea" {
|
||||
if c := condOf(r.Status.Parents[0].Conditions, condResolvedRefs); c.Status != metav1.ConditionFalse || c.Reason != "BackendNotFound" {
|
||||
t.Fatalf("resolvedRefs: %+v", c)
|
||||
}
|
||||
if c := condOf(r.Status.Parents[0].Conditions, condAccepted); c.Status != metav1.ConditionTrue {
|
||||
t.Fatalf("a route with an unresolved backend is still accepted: %+v", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
h := ComputeGateways("w15", gwNodes(), in)
|
||||
for _, s := range h.Serve {
|
||||
if s.Name == "https-gitea" && (len(s.Routes) != 1 || !s.Routes[0].Rules[0].Broken) {
|
||||
t.Fatalf("the rule must be served broken (500): %+v", s.Routes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTCPListenerForwardsTheOldestRoute(t *testing.T) {
|
||||
in := giteaInput(t)
|
||||
later := in.TCPRoutes[0].DeepCopyObject().(*TCPRoute)
|
||||
later.Name = "other-ssh"
|
||||
later.CreationTimestamp = metav1.NewTime(later.CreationTimestamp.Add(time.Minute))
|
||||
later.Spec.Rules[0].BackendRefs[0].Name = "other"
|
||||
in.TCPRoutes = append(in.TCPRoutes, later)
|
||||
d := ComputeGateways("cp", gwNodes(), in)
|
||||
for _, r := range d.Routes {
|
||||
a := condOf(r.Status.Parents[0].Conditions, condAccepted)
|
||||
switch r.Name {
|
||||
case "gitea-ssh":
|
||||
if a.Status != metav1.ConditionTrue {
|
||||
t.Fatalf("oldest route: %+v", a)
|
||||
}
|
||||
case "other-ssh":
|
||||
if a.Status != metav1.ConditionFalse || !strings.Contains(a.Message, "gitea/gitea-ssh") {
|
||||
t.Fatalf("second route: %+v", a)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOtherControllersParentsKept(t *testing.T) {
|
||||
in := giteaInput(t)
|
||||
theirs := RouteParentStatus{ParentRef: ParentReference{Name: "elsewhere"}, ControllerName: "example.com/other",
|
||||
Conditions: []metav1.Condition{{Type: condAccepted, Status: metav1.ConditionTrue, Reason: "Accepted", LastTransitionTime: gwNow}}}
|
||||
in.HTTPRoutes[0].Status.Parents = []RouteParentStatus{theirs}
|
||||
d := ComputeGateways("cp", gwNodes(), in)
|
||||
for _, r := range d.Routes {
|
||||
if r.Name == "gitea" {
|
||||
if len(r.Status.Parents) != 2 || r.Status.Parents[0].ControllerName != "example.com/other" {
|
||||
t.Fatalf("another controller's entry must be kept: %+v", r.Status.Parents)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOtherClassesIgnored(t *testing.T) {
|
||||
in := giteaInput(t)
|
||||
in.Classes[0].Spec.ControllerName = "example.com/other"
|
||||
for _, self := range []string{"cp", "w15"} {
|
||||
d := ComputeGateways(self, gwNodes(), in)
|
||||
if len(d.Serve)+len(d.Forwards)+len(d.Gateways)+len(d.Routes)+len(d.Classes) != 0 {
|
||||
t.Fatalf("%s acted on another controller's class: %+v", self, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNarrowHostnames(t *testing.T) {
|
||||
cases := []struct {
|
||||
listener string
|
||||
route []string
|
||||
want []string
|
||||
ok bool
|
||||
}{
|
||||
{"", nil, nil, true},
|
||||
{"git.levk.dev", nil, []string{"git.levk.dev"}, true},
|
||||
{"", []string{"a.example.com"}, []string{"a.example.com"}, true},
|
||||
{"test.example.com", []string{"*.example.com"}, []string{"test.example.com"}, true},
|
||||
{"*.example.com", []string{"test.example.com", "test.example.net", "foo.test.example.com"}, []string{"test.example.com", "foo.test.example.com"}, true},
|
||||
{"*.example.com", []string{"example.com"}, nil, false},
|
||||
{"*.example.com", []string{"*.a.example.com"}, []string{"*.a.example.com"}, true},
|
||||
{"*.a.example.com", []string{"*.example.com"}, []string{"*.a.example.com"}, true},
|
||||
{"git.levk.dev", []string{"git.example.net"}, nil, false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got, ok := narrowHostnames(c.listener, c.route)
|
||||
if ok != c.ok || strings.Join(got, ",") != strings.Join(c.want, ",") {
|
||||
t.Errorf("listener %q route %v: got %v %v, want %v %v", c.listener, c.route, got, ok, c.want, c.ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeGateways(t *testing.T) {
|
||||
in := giteaInput(t)
|
||||
nodes := gwNodes()
|
||||
// on the holder: Gitea's SSH forwarded, its HTTP(S) ports let in
|
||||
d := MergeGateways(Compute("w15", nodes, nil, nil), ComputeGateways("w15", nodes, in), "w15", nodes)
|
||||
if len(d.Forwards) != 1 || len(d.Accept) != 2 || len(d.Holders) != 0 {
|
||||
t.Fatalf("holder: forwards %+v accept %+v holders %+v", d.Forwards, d.Accept, d.Holders)
|
||||
}
|
||||
r := Render(d, EgressDesired{}, "w15", &net.IPNet{IP: net.ParseIP("10.32.0.0"), Mask: net.CIDRMask(12, 32)}, nil)
|
||||
for _, want := range []string{
|
||||
"-A REWEAVE-LB-IN -d 86.107.168.15/32 -p tcp -m tcp --dport 443",
|
||||
"-A REWEAVE-LB-IN -d 86.107.168.15/32 -p tcp -m tcp --dport 80",
|
||||
} {
|
||||
if !strings.Contains(r.Filter, want) {
|
||||
t.Errorf("filter lacks %q:\n%s", want, r.Filter)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(r.Nat, "-A REWEAVE-LB -d 86.107.168.15/32 -p tcp -m tcp --dport 22") || !strings.Contains(r.Nat, "--to-destination 10.44.0.12:22") {
|
||||
t.Errorf("nat lacks the SSH forward:\n%s", r.Nat)
|
||||
}
|
||||
// elsewhere: w15 is a holder whose replies come back to it
|
||||
e := MergeGateways(Compute("cp", nodes, nil, nil), ComputeGateways("cp", nodes, in), "cp", nodes)
|
||||
if len(e.Forwards) != 0 || len(e.Accept) != 0 || len(e.Holders) != 1 || e.Holders[0].Node.Name != "w15" {
|
||||
t.Fatalf("cp: forwards %+v accept %+v holders %+v", e.Forwards, e.Accept, e.Holders)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnboundListenerReported(t *testing.T) {
|
||||
in := giteaInput(t)
|
||||
in.Unbound = map[string]string{"gateway-system/shared/https-gitea": "address already in use"}
|
||||
d := ComputeGateways("w15", gwNodes(), in)
|
||||
for _, l := range d.Gateways[0].Status.Listeners {
|
||||
if l.Name != "https-gitea" {
|
||||
continue
|
||||
}
|
||||
if a := condOf(l.Conditions, condAccepted); a.Status != metav1.ConditionFalse || a.Reason != "PortUnavailable" {
|
||||
t.Fatalf("accepted: %+v", a)
|
||||
}
|
||||
if p := condOf(l.Conditions, condProgrammed); p.Status != metav1.ConditionFalse {
|
||||
t.Fatalf("programmed: %+v", p)
|
||||
}
|
||||
}
|
||||
served := false
|
||||
for _, s := range d.Serve {
|
||||
served = served || s.Name == "https-gitea"
|
||||
}
|
||||
if !served {
|
||||
t.Fatal("an unbound listener is still tried on every pass")
|
||||
}
|
||||
}
|
||||
+10
@@ -136,9 +136,19 @@ type Desired struct {
|
||||
Forwards []Forward
|
||||
Holders []Holder
|
||||
Status []Status
|
||||
// Accept are the address:ports this node serves itself (Gateways' HTTP and HTTPS listeners), let in ahead of
|
||||
// the host firewall
|
||||
Accept []Accept
|
||||
Problems []string // what an operator should hear about (logged and evented by the controller)
|
||||
}
|
||||
|
||||
// Accept is a TCP address:port this node serves itself.
|
||||
type Accept struct {
|
||||
Address net.IP
|
||||
Port int32
|
||||
Comment string
|
||||
}
|
||||
|
||||
// eligible reports whether the Service asks for this agent.
|
||||
func eligible(s *coreapi.Service) bool {
|
||||
return s.Spec.Type == coreapi.ServiceTypeLoadBalancer && s.Spec.LoadBalancerClass != nil &&
|
||||
|
||||
@@ -154,6 +154,28 @@ items:
|
||||
- egressservices/status
|
||||
verbs:
|
||||
- patch
|
||||
# Gateways of controller reweave.levk.dev/gateway (the Gateway API CRDs are installed separately)
|
||||
- apiGroups:
|
||||
- gateway.networking.k8s.io
|
||||
resources:
|
||||
- gatewayclasses
|
||||
- gateways
|
||||
- httproutes
|
||||
- tcproutes
|
||||
- referencegrants
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- apiGroups:
|
||||
- gateway.networking.k8s.io
|
||||
resources:
|
||||
- gatewayclasses/status
|
||||
- gateways/status
|
||||
- httproutes/status
|
||||
- tcproutes/status
|
||||
verbs:
|
||||
- patch
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
@@ -206,6 +228,42 @@ items:
|
||||
- kind: ServiceAccount
|
||||
name: weave-net
|
||||
namespace: kube-system
|
||||
# Gateways' certificates: the agent reads Secrets in gateway-system only (weave-npc --gateway-namespaces)
|
||||
- apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: gateway-system
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: weave-net-gateway-certificates
|
||||
namespace: gateway-system
|
||||
labels:
|
||||
name: weave-net
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ''
|
||||
resources:
|
||||
- secrets
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: weave-net-gateway-certificates
|
||||
namespace: gateway-system
|
||||
labels:
|
||||
name: weave-net
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: weave-net-gateway-certificates
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: weave-net
|
||||
namespace: kube-system
|
||||
- apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
|
||||
+20
-9
@@ -6,6 +6,7 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
goruntime "runtime"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/coreos/go-iptables/iptables"
|
||||
@@ -30,14 +31,15 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
version = "unreleased"
|
||||
metricsAddr string
|
||||
logLevel string
|
||||
allowMcast bool
|
||||
nodeName string
|
||||
maxList int
|
||||
bridgePortName string
|
||||
serveLB bool
|
||||
version = "unreleased"
|
||||
metricsAddr string
|
||||
logLevel string
|
||||
allowMcast bool
|
||||
nodeName string
|
||||
maxList int
|
||||
bridgePortName string
|
||||
serveLB bool
|
||||
gatewayNamespaces string
|
||||
)
|
||||
|
||||
func handleError(err error) {
|
||||
@@ -344,7 +346,14 @@ func root(cmd *cobra.Command, args []string) {
|
||||
// EgressServices: those Services' pods leave the cluster from that address (or not at all)
|
||||
egress, err := lb.NewEgressClient(config)
|
||||
handleFatal(err)
|
||||
go lb.NewController(nodeName, client, egress, lb.NewDatapath(ipt)).Run(context.Background())
|
||||
ctl := lb.NewController(nodeName, client, egress, lb.NewDatapath(ipt))
|
||||
if gatewayNamespaces != "" {
|
||||
// and Gateways of class controller lb.GatewayControllerName, from the node holding their address
|
||||
gw, err := lb.NewGatewayClient(config)
|
||||
handleFatal(err)
|
||||
ctl = ctl.WithGateway(gw, strings.Split(gatewayNamespaces, ","))
|
||||
}
|
||||
go ctl.Run(context.Background())
|
||||
}
|
||||
|
||||
signals := make(chan os.Signal, 1)
|
||||
@@ -365,6 +374,8 @@ func main() {
|
||||
rootCmd.PersistentFlags().IntVar(&maxList, "max-list-size", 1024, "maximum size of ipset list (for namespaces)")
|
||||
rootCmd.PersistentFlags().StringVar(&bridgePortName, "bridge-port-name", "vethwe-bridge", "name of the brige port on which packets are received and sent")
|
||||
rootCmd.PersistentFlags().BoolVar(&serveLB, "lb", true, "serve LoadBalancer Services of class "+lb.Class+" from addresses nodes hold")
|
||||
rootCmd.PersistentFlags().StringVar(&gatewayNamespaces, "gateway-namespaces", "gateway-system",
|
||||
"serve Gateways of controller "+lb.GatewayControllerName+", with certificates from Secrets in these namespaces (comma-separated; empty: no Gateways)")
|
||||
|
||||
handleFatal(rootCmd.Execute())
|
||||
}
|
||||
Reference in new issue
Block a user