From 9d4436c8ffa34a186cbce9a766006e1d8e9c865c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 15:38:10 +0000 Subject: [PATCH 1/2] feat: Health-check load balancer backends Each location checks its load balancer backends with TCP, HTTP or HTTPS probes, and only backends with a recent successful check are eligible for traffic, so a failed backend stops receiving new connections on its own. Key changes: - Add a cell-local backend health record and probe runner - Fail closed when a probe fails, is missing or goes stale - Require a matching attachment UID before probing a backend - Report that probing is unavailable until a location can reach VPC backends --- api/v1alpha/groupversion_info.go | 2 + api/v1alpha/networkinterface_types.go | 6 + .../networkloadbalancerbackendhealth_types.go | 143 ++++++++ api/v1alpha/zz_generated.deepcopy.go | 188 ++++++++++ ...rking.datumapis.com_networkinterfaces.yaml | 5 + ...com_networkloadbalancerbackendhealths.yaml | 329 ++++++++++++++++++ config/crd/downstream/kustomization.yaml | 1 + config/rbac/role.yaml | 2 + docs/api/networkinterfaces.md | 8 + docs/plans/network-load-balancer-health.md | 49 +++ internal/agent/catalog_test.go | 33 +- internal/cmd/cell/cell.go | 3 + internal/cmd/manager/manager_test.go | 1 + ...orkloadbalancerbackendhealth_controller.go | 164 +++++++++ ...adbalancerbackendhealth_controller_test.go | 307 ++++++++++++++++ internal/networkagent/catalog_test.go | 23 +- internal/nlbhealth/health.go | 188 ++++++++++ internal/nlbhealth/health_test.go | 230 ++++++++++++ 18 files changed, 1655 insertions(+), 27 deletions(-) create mode 100644 api/v1alpha/networkloadbalancerbackendhealth_types.go create mode 100644 config/crd/bases/networking.datumapis.com_networkloadbalancerbackendhealths.yaml create mode 100644 docs/plans/network-load-balancer-health.md create mode 100644 internal/controller/networkloadbalancerbackendhealth_controller.go create mode 100644 internal/controller/networkloadbalancerbackendhealth_controller_test.go create mode 100644 internal/nlbhealth/health.go create mode 100644 internal/nlbhealth/health_test.go diff --git a/api/v1alpha/groupversion_info.go b/api/v1alpha/groupversion_info.go index 4f890ace..c2634bfe 100644 --- a/api/v1alpha/groupversion_info.go +++ b/api/v1alpha/groupversion_info.go @@ -42,6 +42,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &NetworkInterfaceList{}, &NetworkInterfaceClaim{}, &NetworkInterfaceClaimList{}, + &NetworkLoadBalancerBackendHealth{}, + &NetworkLoadBalancerBackendHealthList{}, &NetworkLoadBalancer{}, &NetworkLoadBalancerList{}, &NetworkLoadBalancerRoute{}, diff --git a/api/v1alpha/networkinterface_types.go b/api/v1alpha/networkinterface_types.go index 2f1fa994..89cb9220 100644 --- a/api/v1alpha/networkinterface_types.go +++ b/api/v1alpha/networkinterface_types.go @@ -4,6 +4,7 @@ package v1alpha import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) // NetworkInterfaceReclaimPolicy decides what becomes of an interface, and the @@ -262,6 +263,11 @@ type NetworkInterfaceAttachmentRef struct { // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:MaxLength=253 Name string `json:"name"` + + // uid identifies the exact attachment incarnation. Probe consumers require + // this value so a recreated attachment cannot inherit the prior health. + // +kubebuilder:validation:Optional + UID types.UID `json:"uid,omitempty"` } // NetworkInterfaceSpec defines the desired state of NetworkInterface. It is diff --git a/api/v1alpha/networkloadbalancerbackendhealth_types.go b/api/v1alpha/networkloadbalancerbackendhealth_types.go new file mode 100644 index 00000000..d6db9b01 --- /dev/null +++ b/api/v1alpha/networkloadbalancerbackendhealth_types.go @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package v1alpha + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// +kubebuilder:validation:Enum=TCP;HTTP;HTTPS +type NetworkLoadBalancerProbeProtocol string + +const ( + NetworkLoadBalancerProbeTCP NetworkLoadBalancerProbeProtocol = "TCP" + NetworkLoadBalancerProbeHTTP NetworkLoadBalancerProbeProtocol = "HTTP" + NetworkLoadBalancerProbeHTTPS NetworkLoadBalancerProbeProtocol = "HTTPS" +) + +type NetworkLoadBalancerBackendHealthRouteRef struct { + // +kubebuilder:validation:MinLength=1 + Namespace string `json:"namespace"` + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + UID types.UID `json:"uid"` + // +kubebuilder:validation:Minimum=1 + Generation int64 `json:"generation"` +} + +type NetworkLoadBalancerBackendHealthParentRef struct { + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + UID types.UID `json:"uid"` + // +kubebuilder:validation:Minimum=1 + Generation int64 `json:"generation"` + // +kubebuilder:validation:MinLength=1 + Listener string `json:"listener"` +} + +// NetworkLoadBalancerProbe describes a resolved active check. Its port is a +// backend port number, not a listener port or a NetworkService port name. +// +kubebuilder:validation:XValidation:message="path is only valid for HTTP and HTTPS",rule="has(self.path) ? self.protocol in ['HTTP', 'HTTPS'] : true" +// +kubebuilder:validation:XValidation:message="serverName is only valid for HTTPS",rule="has(self.serverName) ? self.protocol == 'HTTPS' : true" +type NetworkLoadBalancerProbe struct { + Protocol NetworkLoadBalancerProbeProtocol `json:"protocol"` + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + Port int32 `json:"port"` + // +kubebuilder:validation:Optional + // +kubebuilder:validation:Pattern=`^/.*$` + // +kubebuilder:validation:MaxLength=256 + Path string `json:"path,omitempty"` + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MaxLength=253 + ServerName string `json:"serverName,omitempty"` +} + +// NetworkLoadBalancerBackendHealthTarget identifies one resolved interface +// endpoint and its probe. The source UID prevents a replacement interface +// reusing a name from inheriting a previous endpoint's health. +type NetworkLoadBalancerBackendHealthTarget struct { + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + Name string `json:"name"` + // +kubebuilder:validation:MinLength=1 + SourceNamespace string `json:"sourceNamespace"` + // +kubebuilder:validation:MinLength=1 + SourceInterface string `json:"sourceInterface"` + SourceUID types.UID `json:"sourceUID"` + // +kubebuilder:validation:MinLength=1 + AttachmentName string `json:"attachmentName"` + AttachmentUID types.UID `json:"attachmentUID"` + // +kubebuilder:validation:MinLength=1 + Address string `json:"address"` + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + ServingPort int32 `json:"servingPort"` + Probe NetworkLoadBalancerProbe `json:"probe"` +} + +// NetworkLoadBalancerBackendHealthSpec carries one route parent and location's +// resolved targets into the cell that can reach them. +type NetworkLoadBalancerBackendHealthSpec struct { + RouteRef NetworkLoadBalancerBackendHealthRouteRef `json:"routeRef"` + ParentRef NetworkLoadBalancerBackendHealthParentRef `json:"parentRef"` + // +kubebuilder:validation:MinLength=1 + Location string `json:"location"` + // +kubebuilder:validation:MinLength=1 + NetworkName string `json:"networkName"` + // +kubebuilder:validation:MinLength=1 + FabricVPCID string `json:"fabricVPCID"` + // +kubebuilder:validation:MaxItems=64 + // +listType=map + // +listMapKey=name + Targets []NetworkLoadBalancerBackendHealthTarget `json:"targets"` +} + +// NetworkLoadBalancerBackendHealthTargetStatus is usable only while its route, +// object generation, target fingerprint, and expiry match current intent. +type NetworkLoadBalancerBackendHealthTargetStatus struct { + Name string `json:"name"` + ObservedGeneration int64 `json:"observedGeneration"` + RouteUID types.UID `json:"routeUID"` + RouteGeneration int64 `json:"routeGeneration"` + TargetFingerprint string `json:"targetFingerprint"` + Healthy bool `json:"healthy"` + ConsecutiveSuccesses int32 `json:"consecutiveSuccesses"` + LastProbeTime metav1.Time `json:"lastProbeTime"` + ValidUntil metav1.Time `json:"validUntil"` + Reason string `json:"reason,omitempty"` +} + +type NetworkLoadBalancerBackendHealthStatus struct { + // ProbesEvaluated remains Unknown until a VPC-capable runner reports results. + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty"` + + // +kubebuilder:validation:MaxItems=64 + // +listType=map + // +listMapKey=name + Targets []NetworkLoadBalancerBackendHealthTargetStatus `json:"targets,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status + +// NetworkLoadBalancerBackendHealth is an internal, cell-local probe contract. +// A target is eligible for data plane programming only after a fresh success. +type NetworkLoadBalancerBackendHealth struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec NetworkLoadBalancerBackendHealthSpec `json:"spec"` + // +kubebuilder:default={conditions:{{type:"ProbesEvaluated",status:"Unknown",reason:"Pending",message:"Waiting for a VPC-capable probe runner",lastTransitionTime:"1970-01-01T00:00:00Z"}}} + Status NetworkLoadBalancerBackendHealthStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +type NetworkLoadBalancerBackendHealthList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NetworkLoadBalancerBackendHealth `json:"items"` +} diff --git a/api/v1alpha/zz_generated.deepcopy.go b/api/v1alpha/zz_generated.deepcopy.go index f46f7b34..a9a26d0d 100644 --- a/api/v1alpha/zz_generated.deepcopy.go +++ b/api/v1alpha/zz_generated.deepcopy.go @@ -1948,6 +1948,179 @@ func (in *NetworkLoadBalancerAddressStatus) DeepCopy() *NetworkLoadBalancerAddre return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerBackendHealth) DeepCopyInto(out *NetworkLoadBalancerBackendHealth) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerBackendHealth. +func (in *NetworkLoadBalancerBackendHealth) DeepCopy() *NetworkLoadBalancerBackendHealth { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerBackendHealth) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetworkLoadBalancerBackendHealth) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerBackendHealthList) DeepCopyInto(out *NetworkLoadBalancerBackendHealthList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NetworkLoadBalancerBackendHealth, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerBackendHealthList. +func (in *NetworkLoadBalancerBackendHealthList) DeepCopy() *NetworkLoadBalancerBackendHealthList { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerBackendHealthList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetworkLoadBalancerBackendHealthList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerBackendHealthParentRef) DeepCopyInto(out *NetworkLoadBalancerBackendHealthParentRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerBackendHealthParentRef. +func (in *NetworkLoadBalancerBackendHealthParentRef) DeepCopy() *NetworkLoadBalancerBackendHealthParentRef { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerBackendHealthParentRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerBackendHealthRouteRef) DeepCopyInto(out *NetworkLoadBalancerBackendHealthRouteRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerBackendHealthRouteRef. +func (in *NetworkLoadBalancerBackendHealthRouteRef) DeepCopy() *NetworkLoadBalancerBackendHealthRouteRef { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerBackendHealthRouteRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerBackendHealthSpec) DeepCopyInto(out *NetworkLoadBalancerBackendHealthSpec) { + *out = *in + out.RouteRef = in.RouteRef + out.ParentRef = in.ParentRef + if in.Targets != nil { + in, out := &in.Targets, &out.Targets + *out = make([]NetworkLoadBalancerBackendHealthTarget, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerBackendHealthSpec. +func (in *NetworkLoadBalancerBackendHealthSpec) DeepCopy() *NetworkLoadBalancerBackendHealthSpec { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerBackendHealthSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerBackendHealthStatus) DeepCopyInto(out *NetworkLoadBalancerBackendHealthStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Targets != nil { + in, out := &in.Targets, &out.Targets + *out = make([]NetworkLoadBalancerBackendHealthTargetStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerBackendHealthStatus. +func (in *NetworkLoadBalancerBackendHealthStatus) DeepCopy() *NetworkLoadBalancerBackendHealthStatus { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerBackendHealthStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerBackendHealthTarget) DeepCopyInto(out *NetworkLoadBalancerBackendHealthTarget) { + *out = *in + out.Probe = in.Probe +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerBackendHealthTarget. +func (in *NetworkLoadBalancerBackendHealthTarget) DeepCopy() *NetworkLoadBalancerBackendHealthTarget { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerBackendHealthTarget) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerBackendHealthTargetStatus) DeepCopyInto(out *NetworkLoadBalancerBackendHealthTargetStatus) { + *out = *in + in.LastProbeTime.DeepCopyInto(&out.LastProbeTime) + in.ValidUntil.DeepCopyInto(&out.ValidUntil) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerBackendHealthTargetStatus. +func (in *NetworkLoadBalancerBackendHealthTargetStatus) DeepCopy() *NetworkLoadBalancerBackendHealthTargetStatus { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerBackendHealthTargetStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkLoadBalancerBackendTransport) DeepCopyInto(out *NetworkLoadBalancerBackendTransport) { *out = *in @@ -2154,6 +2327,21 @@ func (in *NetworkLoadBalancerNetworkServiceRef) DeepCopy() *NetworkLoadBalancerN return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerProbe) DeepCopyInto(out *NetworkLoadBalancerProbe) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerProbe. +func (in *NetworkLoadBalancerProbe) DeepCopy() *NetworkLoadBalancerProbe { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerProbe) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkLoadBalancerRoute) DeepCopyInto(out *NetworkLoadBalancerRoute) { *out = *in diff --git a/config/crd/bases/networking.datumapis.com_networkinterfaces.yaml b/config/crd/bases/networking.datumapis.com_networkinterfaces.yaml index 4973ac3a..aae8ac3a 100644 --- a/config/crd/bases/networking.datumapis.com_networkinterfaces.yaml +++ b/config/crd/bases/networking.datumapis.com_networkinterfaces.yaml @@ -316,6 +316,11 @@ spec: maxLength: 253 minLength: 1 type: string + uid: + description: |- + uid identifies the exact attachment incarnation. Probe consumers require + this value so a recreated attachment cannot inherit the prior health. + type: string required: - apiGroup - kind diff --git a/config/crd/bases/networking.datumapis.com_networkloadbalancerbackendhealths.yaml b/config/crd/bases/networking.datumapis.com_networkloadbalancerbackendhealths.yaml new file mode 100644 index 00000000..24aa6c70 --- /dev/null +++ b/config/crd/bases/networking.datumapis.com_networkloadbalancerbackendhealths.yaml @@ -0,0 +1,329 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: networkloadbalancerbackendhealths.networking.datumapis.com +spec: + group: networking.datumapis.com + names: + kind: NetworkLoadBalancerBackendHealth + listKind: NetworkLoadBalancerBackendHealthList + plural: networkloadbalancerbackendhealths + singular: networkloadbalancerbackendhealth + scope: Namespaced + versions: + - name: v1alpha + schema: + openAPIV3Schema: + description: |- + NetworkLoadBalancerBackendHealth is an internal, cell-local probe contract. + A target is eligible for data plane programming only after a fresh success. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NetworkLoadBalancerBackendHealthSpec carries one route parent and location's + resolved targets into the cell that can reach them. + properties: + fabricVPCID: + minLength: 1 + type: string + location: + minLength: 1 + type: string + networkName: + minLength: 1 + type: string + parentRef: + properties: + generation: + format: int64 + minimum: 1 + type: integer + listener: + minLength: 1 + type: string + name: + minLength: 1 + type: string + uid: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + required: + - generation + - listener + - name + - uid + type: object + routeRef: + properties: + generation: + format: int64 + minimum: 1 + type: integer + name: + minLength: 1 + type: string + namespace: + minLength: 1 + type: string + uid: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + required: + - generation + - name + - namespace + - uid + type: object + targets: + items: + description: |- + NetworkLoadBalancerBackendHealthTarget identifies one resolved interface + endpoint and its probe. The source UID prevents a replacement interface + reusing a name from inheriting a previous endpoint's health. + properties: + address: + minLength: 1 + type: string + attachmentName: + minLength: 1 + type: string + attachmentUID: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + name: + maxLength: 63 + minLength: 1 + type: string + probe: + description: |- + NetworkLoadBalancerProbe describes a resolved active check. Its port is a + backend port number, not a listener port or a NetworkService port name. + properties: + path: + maxLength: 256 + pattern: ^/.*$ + type: string + port: + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + enum: + - TCP + - HTTP + - HTTPS + type: string + serverName: + maxLength: 253 + type: string + required: + - port + - protocol + type: object + x-kubernetes-validations: + - message: path is only valid for HTTP and HTTPS + rule: 'has(self.path) ? self.protocol in [''HTTP'', ''HTTPS''] + : true' + - message: serverName is only valid for HTTPS + rule: 'has(self.serverName) ? self.protocol == ''HTTPS'' : + true' + servingPort: + format: int32 + maximum: 65535 + minimum: 1 + type: integer + sourceInterface: + minLength: 1 + type: string + sourceNamespace: + minLength: 1 + type: string + sourceUID: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + required: + - address + - attachmentName + - attachmentUID + - name + - probe + - servingPort + - sourceInterface + - sourceNamespace + - sourceUID + type: object + maxItems: 64 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - fabricVPCID + - location + - networkName + - parentRef + - routeRef + - targets + type: object + status: + default: + conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for a VPC-capable probe runner + reason: Pending + status: Unknown + type: ProbesEvaluated + properties: + conditions: + description: ProbesEvaluated remains Unknown until a VPC-capable runner + reports results. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + targets: + items: + description: |- + NetworkLoadBalancerBackendHealthTargetStatus is usable only while its route, + object generation, target fingerprint, and expiry match current intent. + properties: + consecutiveSuccesses: + format: int32 + type: integer + healthy: + type: boolean + lastProbeTime: + format: date-time + type: string + name: + type: string + observedGeneration: + format: int64 + type: integer + reason: + type: string + routeGeneration: + format: int64 + type: integer + routeUID: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + targetFingerprint: + type: string + validUntil: + format: date-time + type: string + required: + - consecutiveSuccesses + - healthy + - lastProbeTime + - name + - observedGeneration + - routeGeneration + - routeUID + - targetFingerprint + - validUntil + type: object + maxItems: 64 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/downstream/kustomization.yaml b/config/crd/downstream/kustomization.yaml index 215f6511..123da524 100644 --- a/config/crd/downstream/kustomization.yaml +++ b/config/crd/downstream/kustomization.yaml @@ -46,4 +46,5 @@ resources: # Network interface claim path, served on cells - ../bases/networking.datumapis.com_networkinterfaceclaims.yaml - ../bases/networking.datumapis.com_networkinterfaces.yaml + - ../bases/networking.datumapis.com_networkloadbalancerbackendhealths.yaml - ../bases/networking.datumapis.com_subnets.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index d556090b..b3dee038 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -362,6 +362,7 @@ rules: - networkcontexts/status - networkinterfaceclaims/status - networkinterfaces/status + - networkloadbalancerbackendhealths/status - networkloadbalancerroutes/status - networkloadbalancers/status - networkpolicies/status @@ -378,6 +379,7 @@ rules: - networking.datumapis.com resources: - connectorclasses + - networkloadbalancerbackendhealths - networkloadbalancerroutes - networkservices verbs: diff --git a/docs/api/networkinterfaces.md b/docs/api/networkinterfaces.md index 84c5ee2b..4ea9d402 100644 --- a/docs/api/networkinterfaces.md +++ b/docs/api/networkinterfaces.md @@ -497,6 +497,14 @@ compute.datumapis.com.
name is the name of the referent.
true + + uid + string + + uid identifies the exact attachment incarnation. Probe consumers require +this value so a recreated attachment cannot inherit the prior health.
+ + false diff --git a/docs/plans/network-load-balancer-health.md b/docs/plans/network-load-balancer-health.md new file mode 100644 index 00000000..9859c283 --- /dev/null +++ b/docs/plans/network-load-balancer-health.md @@ -0,0 +1,49 @@ +# Cell-local network load balancer health + +The project controller cannot prove that a backend answers on its application +port. It sees a projected interface address, but the cell controller manager's +pod has no established route into each tenant VPC. Host-network connection +failures must not be interpreted as unhealthy applications. + +`NetworkLoadBalancerBackendHealth` is an internal cell-local contract for a +runner on the gateway's VPC-aware socket path. The cell manager registers the +probe reconciler without a dialer today. Its `ProbesEvaluated` condition is +`False/NoDataPlaneDialer`, and no target is eligible from status alone. +Serving requires a dialer that selects the validated fabric VPC's VRF or +network namespace for each outgoing connection. It must never fall back to the +controller pod's ordinary network. + +The cell intent controller must first verify the live source interface, +attachment UID, VPC identity, and project ownership described in the +[cell contract](network-load-balancer-cell-contract.md). It can then create one +health object for a route parent and location. The parent UID and generation +bind the health object to a particular load balancer instance. Each target +records the source interface UID, attachment UID, address, serving port, +and resolved probe. The probe runner also rechecks the live interface +UID, network, location, holder availability, attachment name, VPC ID, and +address before each check. The eventual traffic programmer must revalidate +the attachment UID and VPC association when it consumes healthy targets; probe success is +not an authorization to program or announce a VIP. + +A VPC-aware runner evaluates up to 64 targets per object, with at most eight +concurrent probes per object and 32 across the process. Queueing for a probe +slot does not consume the probe deadline. Each TCP connection or HTTP/HTTPS GET +has a two-second deadline. A full 64-target sweep can take eight waves, or +16 seconds, and the next sweep starts five seconds after completion. Five +seconds is therefore a pause between sweeps, not a per-target interval. +Two consecutive successes are needed to mark a target healthy; a failure +removes it when that sweep is published. Results expire after 39 seconds: +two maximum sweeps, the five-second pause, and one probe timeout of scheduling +slack. A delayed sweep beyond this bound causes health to expire closed. +Consumers use `HealthyTargets`, which rejects a +result when the object generation, route UID or generation, target fingerprint, +probe timestamp, or expiry no longer matches. A removed or replaced source +cannot inherit an earlier success. + +HTTP and HTTPS accept only 2xx responses and do not follow redirects. HTTPS +always validates the server certificate. With the current native route API, +which has no probe hostname, the certificate must have an IP SAN for the +backend IPv6 address. The cell-local contract can carry an explicit server +name once the public route API exposes one; until then a hostname is not +invented by the runner. UDP traffic uses a separate TCP, HTTP, or HTTPS probe +port and is never treated as healthy from a UDP packet alone. diff --git a/internal/agent/catalog_test.go b/internal/agent/catalog_test.go index 9304e31a..2f3c8930 100644 --- a/internal/agent/catalog_test.go +++ b/internal/agent/catalog_test.go @@ -32,22 +32,23 @@ var inScopeAPIFiles = []string{ // rather than ignored so that "scoped" cannot quietly become "stale": a new // file has to be argued into one list or the other. var outOfScopeAPIFiles = map[string]string{ - "edgereachability_types.go": "hub-side record of which addresses an edge should reach; no customer-facing status", - "groupversion_info.go": "scheme registration, no reasons", - "internaldns_types.go": "private resolver access is a network surface, not load balancer status", - "locationbinding_types.go": "placement plumbing behind a location, never named by a load balancer", - "network_types.go": "the network a service sits on; its own product surface", - "networkbinding_types.go": "network attachment plumbing", - "networkcontext_types.go": "network attachment plumbing", - "networkinterface_types.go": "an instance's interface; compute's surface", - "networkinterfaceclaim_types.go": "an instance's interface; compute's surface", - "networkloadbalancer_types.go": "its own Layer 4 product surface, not reached through an application load balancer", - "networkloadbalancerroute_types.go": "its own Layer 4 routing surface, not reached through an application load balancer", - "networkpolicy_types.go": "its own product surface, not reached through a load balancer", - "subnet_types.go": "address management below a network", - "subnetclaim_types.go": "address management below a network", - "trafficprotectionpolicy_types.go": "declares no reasons of its own; the controller writes upstream Gateway API policy reasons", - "zz_generated.deepcopy.go": "generated", + "edgereachability_types.go": "hub-side record of which addresses an edge should reach; no customer-facing status", + "groupversion_info.go": "scheme registration, no reasons", + "internaldns_types.go": "private resolver access is a network surface, not load balancer status", + "locationbinding_types.go": "placement plumbing behind a location, never named by a load balancer", + "network_types.go": "the network a service sits on; its own product surface", + "networkbinding_types.go": "network attachment plumbing", + "networkcontext_types.go": "network attachment plumbing", + "networkinterface_types.go": "an instance's interface; compute's surface", + "networkinterfaceclaim_types.go": "an instance's interface; compute's surface", + "networkloadbalancer_types.go": "its own Layer 4 product surface, not reached through an application load balancer", + "networkloadbalancerbackendhealth_types.go": "cell-local probe state, not an application load balancer customer surface", + "networkloadbalancerroute_types.go": "its own Layer 4 routing surface, not reached through an application load balancer", + "networkpolicy_types.go": "its own product surface, not reached through a load balancer", + "subnet_types.go": "address management below a network", + "subnetclaim_types.go": "address management below a network", + "trafficprotectionpolicy_types.go": "declares no reasons of its own; the controller writes upstream Gateway API policy reasons", + "zz_generated.deepcopy.go": "generated", } // extraReasonConsts are reason constants whose NAME does not contain "Reason", diff --git a/internal/cmd/cell/cell.go b/internal/cmd/cell/cell.go index c862fd21..7ee15531 100644 --- a/internal/cmd/cell/cell.go +++ b/internal/cmd/cell/cell.go @@ -352,6 +352,9 @@ func controllerRegistrations( HubCluster: hubCluster, }).SetupWithManager(mgr) }}) + registrations = append(registrations, namedSetup{"networkloadbalancerbackendhealth", func() (bool, error) { + return true, (&controller.NetworkLoadBalancerBackendHealthReconciler{}).SetupWithManager(mgr.GetLocalManager()) + }}) registrations = append(registrations, namedSetup{"internaldns-cell", func() (bool, error) { if !serverConfig.InternalDNS.Enabled { diff --git a/internal/cmd/manager/manager_test.go b/internal/cmd/manager/manager_test.go index 19f3065f..d97e4fe3 100644 --- a/internal/cmd/manager/manager_test.go +++ b/internal/cmd/manager/manager_test.go @@ -71,6 +71,7 @@ var reconcilerControllerNames = map[string]string{ "NetworkReconciler": "network", "NetworkServiceReconciler": "networkservice", "NetworkLoadBalancerRouteReconciler": "networkloadbalancerroute", + "NetworkLoadBalancerBackendHealthReconciler": "networkloadbalancerbackendhealth", "SubnetClaimReconciler": "subnetclaim", "SubnetReconciler": "subnet", "TrafficProtectionPolicyReconciler": "trafficprotectionpolicy", diff --git a/internal/controller/networkloadbalancerbackendhealth_controller.go b/internal/controller/networkloadbalancerbackendhealth_controller.go new file mode 100644 index 00000000..3ab5e494 --- /dev/null +++ b/internal/controller/networkloadbalancerbackendhealth_controller.go @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + "sync" + "time" + + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + "go.datum.net/network-services-operator/internal/nlbhealth" +) + +const probesPerHealthObject = nlbhealth.ProbeSlotsPerObject + +var cellProbeSlots = make(chan struct{}, 32) + +type NetworkLoadBalancerBackendHealthReconciler struct { + Client client.Client + Prober nlbhealth.Prober + Now func() time.Time +} + +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkloadbalancerbackendhealths,verbs=get;list;watch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkloadbalancerbackendhealths/status,verbs=get;update;patch + +func (r *NetworkLoadBalancerBackendHealthReconciler) Reconcile(ctx context.Context, req reconcile.Request) (ctrl.Result, error) { + var health networkingv1alpha.NetworkLoadBalancerBackendHealth + if err := r.Client.Get(ctx, req.NamespacedName, &health); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + if !health.DeletionTimestamp.IsZero() { + return ctrl.Result{}, nil + } + if len(health.Spec.Targets) > nlbhealth.MaxTargets { + return ctrl.Result{}, fmt.Errorf("health object %s has too many targets", req.NamespacedName) + } + + now := time.Now + if r.Now != nil { + now = r.Now + } + previous := make(map[string]*networkingv1alpha.NetworkLoadBalancerBackendHealthTargetStatus, len(health.Status.Targets)) + for i := range health.Status.Targets { + previous[health.Status.Targets[i].Name] = &health.Status.Targets[i] + } + statuses := make([]networkingv1alpha.NetworkLoadBalancerBackendHealthTargetStatus, len(health.Spec.Targets)) + localSlots := make(chan struct{}, probesPerHealthObject) + var workers sync.WaitGroup + for i := range health.Spec.Targets { + workers.Add(1) + go func(i int) { + defer workers.Done() + target := health.Spec.Targets[i] + var probeErr error + if health.Spec.RouteRef.UID == "" || health.Spec.RouteRef.Generation < 1 || + health.Spec.ParentRef.UID == "" || health.Spec.ParentRef.Generation < 1 || + target.SourceUID == "" || target.AttachmentUID == "" { + probeErr = errors.New("source identity is incomplete") + } else { + probeErr = r.validateLiveTarget(ctx, &health, target) + } + if probeErr == nil { + probeErr = takeProbeSlot(ctx, localSlots) + } + if probeErr == nil { + defer func() { <-localSlots }() + if probeErr = takeProbeSlot(ctx, cellProbeSlots); probeErr == nil { + defer func() { <-cellProbeSlots }() + // Probe starts its own timeout after both concurrency slots are held. + probeErr = r.Prober.Probe(ctx, health.Spec.FabricVPCID, target) + } + } + statuses[i] = nlbhealth.NextTargetStatus(&health, target, previous[target.Name], probeErr, now()) + }(i) + } + workers.Wait() + if err := ctx.Err(); err != nil { + return ctrl.Result{}, err + } + + wanted := networkingv1alpha.NetworkLoadBalancerBackendHealthStatus{ + Conditions: slices.Clone(health.Status.Conditions), Targets: statuses, + } + condition := metav1.Condition{ + Type: "ProbesEvaluated", Status: metav1.ConditionTrue, Reason: "Evaluated", + Message: fmt.Sprintf("Evaluated %d targets from the cell network path", len(statuses)), + ObservedGeneration: health.Generation, + } + if r.Prober.DialContext == nil { + condition.Status = metav1.ConditionFalse + condition.Reason = "NoDataPlaneDialer" + condition.Message = "No VPC-aware probe dialer is configured" + } + apimeta.SetStatusCondition(&wanted.Conditions, condition) + if !equality.Semantic.DeepEqual(health.Status, wanted) { + health.Status = wanted + if err := r.Client.Status().Update(ctx, &health); err != nil { + if apierrors.IsConflict(err) { + return ctrl.Result{Requeue: true}, nil + } + return ctrl.Result{}, fmt.Errorf("update backend health status: %w", err) + } + } + return ctrl.Result{RequeueAfter: nlbhealth.ProbeInterval}, nil +} + +func (r *NetworkLoadBalancerBackendHealthReconciler) validateLiveTarget( + ctx context.Context, + health *networkingv1alpha.NetworkLoadBalancerBackendHealth, + target networkingv1alpha.NetworkLoadBalancerBackendHealthTarget, +) error { + var iface networkingv1alpha.NetworkInterface + if err := r.Client.Get(ctx, client.ObjectKey{Namespace: target.SourceNamespace, Name: target.SourceInterface}, &iface); err != nil { + return fmt.Errorf("read source interface: %w", err) + } + if iface.UID != target.SourceUID || iface.Labels[networkingv1alpha.NetworkInterfaceProjectionLabel] == "true" || + iface.Spec.Network.Name != health.Spec.NetworkName || + iface.Labels[networkingv1alpha.NetworkInterfaceLocationLabel] != health.Spec.Location || + iface.Status.Phase != networkingv1alpha.NetworkInterfacePhaseBound || !isServiceMemberHealthy(&iface) || + iface.Status.VPC != health.Spec.FabricVPCID || iface.Status.AttachmentRef == nil || + iface.Status.AttachmentRef.Name != target.AttachmentName || + iface.Status.AttachmentRef.UID == "" || iface.Status.AttachmentRef.UID != target.AttachmentUID { + return errors.New("source interface no longer matches this target") + } + if !slices.ContainsFunc(iface.Spec.Addresses, func(a networkingv1alpha.NetworkInterfaceAddress) bool { + address, _, _ := strings.Cut(a.Address, "/") + return address == target.Address + }) { + return errors.New("source interface no longer holds this address") + } + return nil +} + +func takeProbeSlot(ctx context.Context, slots chan struct{}) error { + select { + case slots <- struct{}{}: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (r *NetworkLoadBalancerBackendHealthReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.Client = mgr.GetClient() + return ctrl.NewControllerManagedBy(mgr). + For(&networkingv1alpha.NetworkLoadBalancerBackendHealth{}). + WithOptions(controller.Options{MaxConcurrentReconciles: 4}). + Named("networkloadbalancerbackendhealth"). + Complete(r) +} diff --git a/internal/controller/networkloadbalancerbackendhealth_controller_test.go b/internal/controller/networkloadbalancerbackendhealth_controller_test.go new file mode 100644 index 00000000..7b1fab2e --- /dev/null +++ b/internal/controller/networkloadbalancerbackendhealth_controller_test.go @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "fmt" + "net" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + "go.datum.net/network-services-operator/internal/nlbhealth" +) + +func TestBackendHealthFullProbeSweepPublishesFreshConsecutiveSuccesses(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, networkingv1alpha.AddToScheme(scheme)) + iface := &networkingv1alpha.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "cell-project", Name: "nic-1", UID: types.UID("interface-1"), + Labels: map[string]string{networkingv1alpha.NetworkInterfaceLocationLabel: "us-east-1"}, + }, + Spec: networkingv1alpha.NetworkInterfaceSpec{ + Network: networkingv1alpha.LocalNetworkRef{Name: "default"}, + Addresses: []networkingv1alpha.NetworkInterfaceAddress{{Address: "fd20::1/128"}}, + }, + Status: networkingv1alpha.NetworkInterfaceStatus{ + Phase: networkingv1alpha.NetworkInterfacePhaseBound, VPC: "vpc-1", + AttachmentRef: &networkingv1alpha.NetworkInterfaceAttachmentRef{Name: "attach-1", UID: types.UID("attachment-1")}, + Conditions: []metav1.Condition{{Type: networkingv1alpha.NetworkInterfaceHolderAvailable, + Status: metav1.ConditionTrue, Reason: "Ready", LastTransitionTime: metav1.Now()}}, + }, + } + health := &networkingv1alpha.NetworkLoadBalancerBackendHealth{ + ObjectMeta: metav1.ObjectMeta{Namespace: "network-services-operator-system", Name: "health-64", Generation: 1}, + Spec: networkingv1alpha.NetworkLoadBalancerBackendHealthSpec{ + RouteRef: networkingv1alpha.NetworkLoadBalancerBackendHealthRouteRef{ + Namespace: "project", Name: "route-1", UID: types.UID("route-1"), Generation: 1, + }, + ParentRef: networkingv1alpha.NetworkLoadBalancerBackendHealthParentRef{ + Name: "load-balancer", UID: types.UID("load-balancer-1"), Generation: 1, Listener: "game", + }, + Location: "us-east-1", NetworkName: "default", FabricVPCID: "vpc-1", + }, + } + for i := 0; i < 64; i++ { + health.Spec.Targets = append(health.Spec.Targets, networkingv1alpha.NetworkLoadBalancerBackendHealthTarget{ + Name: fmt.Sprintf("origin-%02d", i), SourceNamespace: "cell-project", SourceInterface: "nic-1", SourceUID: iface.UID, + AttachmentName: "attach-1", AttachmentUID: types.UID("attachment-1"), Address: "fd20::1", + ServingPort: 8443, + Probe: networkingv1alpha.NetworkLoadBalancerProbe{Protocol: networkingv1alpha.NetworkLoadBalancerProbeTCP, Port: 8443}, + }) + } + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(iface, health). + WithStatusSubresource(iface, health).Build() + start := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC) + var clock atomic.Int64 + clock.Store(start.UnixNano()) + var dials, active, maximum atomic.Int32 + waves := make([]chan struct{}, 16) + for i := range waves { + waves[i] = make(chan struct{}) + } + r := &NetworkLoadBalancerBackendHealthReconciler{ + Client: cl, Now: func() time.Time { return time.Unix(0, clock.Load()) }, + Prober: nlbhealth.Prober{DialContext: func(ctx context.Context, _, _, _ string) (net.Conn, error) { + current := active.Add(1) + defer active.Add(-1) + for old := maximum.Load(); current > old; old = maximum.Load() { + if maximum.CompareAndSwap(old, current) { + break + } + } + n := int(dials.Add(1)) + wave := waves[(n-1)/probesPerHealthObject] + if n%probesPerHealthObject == 0 { + clock.Add(int64(nlbhealth.ProbeTimeout)) + close(wave) + } + select { + case <-wave: + case <-ctx.Done(): + return nil, ctx.Err() + } + a, b := net.Pipe() + _ = b.Close() + return a, nil + }}, + } + key := client.ObjectKeyFromObject(health) + for sweep := 0; sweep < 2; sweep++ { + result, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.Equal(t, nlbhealth.ProbeInterval, result.RequeueAfter) + require.NoError(t, cl.Get(context.Background(), key, health)) + require.Len(t, health.Status.Targets, 64) + if sweep == 0 { + publishedAt := time.Unix(0, clock.Load()) + for _, status := range health.Status.Targets { + require.True(t, status.ValidUntil.After(publishedAt), "first-sweep result expired before publication: %s", status.Name) + } + require.Empty(t, nlbhealth.HealthyTargets(health, time.Unix(0, clock.Load()))) + } else { + require.Len(t, nlbhealth.HealthyTargets(health, time.Unix(0, clock.Load())), 64) + // The next 64-target sweep consumes eight two-second waves before + // publishing. Every previously healthy target must remain eligible + // immediately before that publication, including the first wave. + beforeNextPublication := time.Unix(0, clock.Load()).Add(nlbhealth.ProbeInterval + nlbhealth.MaxSweepDuration - time.Nanosecond) + require.Len(t, nlbhealth.HealthyTargets(health, beforeNextPublication), 64) + } + clock.Add(int64(nlbhealth.ProbeInterval)) + } + require.EqualValues(t, 128, dials.Load()) + require.LessOrEqual(t, maximum.Load(), int32(probesPerHealthObject)) + // Results are never reusable after the freshness window, even after success. + require.Empty(t, nlbhealth.HealthyTargets(health, time.Unix(0, clock.Load()).Add(nlbhealth.ResultLifetime))) +} + +func TestBackendHealthReconcileRequiresLiveSourceAndVPCDialer(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, networkingv1alpha.AddToScheme(scheme)) + iface := &networkingv1alpha.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "cell-project", Name: "nic-1", UID: types.UID("interface-1"), + Labels: map[string]string{networkingv1alpha.NetworkInterfaceLocationLabel: "us-east-1"}, + }, + Spec: networkingv1alpha.NetworkInterfaceSpec{ + Network: networkingv1alpha.LocalNetworkRef{Name: "default"}, + Addresses: []networkingv1alpha.NetworkInterfaceAddress{{Address: "fd20::1/128"}}, + }, + Status: networkingv1alpha.NetworkInterfaceStatus{ + Phase: networkingv1alpha.NetworkInterfacePhaseBound, VPC: "vpc-1", + AttachmentRef: &networkingv1alpha.NetworkInterfaceAttachmentRef{Name: "attach-1", UID: types.UID("attachment-1")}, + Conditions: []metav1.Condition{{Type: networkingv1alpha.NetworkInterfaceHolderAvailable, + Status: metav1.ConditionTrue, Reason: "Ready", LastTransitionTime: metav1.Now()}}, + }, + } + health := &networkingv1alpha.NetworkLoadBalancerBackendHealth{ + ObjectMeta: metav1.ObjectMeta{Namespace: "network-services-operator-system", Name: "route-1-us-east-1", Generation: 4}, + Spec: networkingv1alpha.NetworkLoadBalancerBackendHealthSpec{ + RouteRef: networkingv1alpha.NetworkLoadBalancerBackendHealthRouteRef{Namespace: "project", Name: "route-1", UID: types.UID("route-1"), Generation: 2}, + ParentRef: networkingv1alpha.NetworkLoadBalancerBackendHealthParentRef{Name: "load-balancer", UID: types.UID("load-balancer-1"), Generation: 3, Listener: "game"}, + Location: "us-east-1", NetworkName: "default", FabricVPCID: "vpc-1", + Targets: []networkingv1alpha.NetworkLoadBalancerBackendHealthTarget{{ + Name: "origin-1", SourceNamespace: "cell-project", SourceInterface: "nic-1", SourceUID: iface.UID, + AttachmentName: "attach-1", AttachmentUID: types.UID("attachment-1"), Address: "fd20::1", + ServingPort: 8443, + Probe: networkingv1alpha.NetworkLoadBalancerProbe{Protocol: networkingv1alpha.NetworkLoadBalancerProbeTCP, Port: 8443}, + }}, + }, + } + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(iface, health). + WithStatusSubresource(iface, health).Build() + now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC) + dials := 0 + r := &NetworkLoadBalancerBackendHealthReconciler{ + Client: cl, Now: func() time.Time { return now }, + Prober: nlbhealth.Prober{DialContext: func(_ context.Context, vpc, _, _ string) (net.Conn, error) { + require.Equal(t, "vpc-1", vpc) + dials++ + a, b := net.Pipe() + if err := b.Close(); err != nil { + return nil, err + } + return a, nil + }}, + } + key := client.ObjectKeyFromObject(health) + for i := 0; i < 2; i++ { + result, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.Equal(t, nlbhealth.ProbeInterval, result.RequeueAfter) + now = now.Add(nlbhealth.ProbeInterval) + } + require.Equal(t, 2, dials) + require.NoError(t, cl.Get(context.Background(), key, health)) + require.Len(t, nlbhealth.HealthyTargets(health, now), 1) + require.True(t, apimeta.IsStatusConditionTrue(health.Status.Conditions, "ProbesEvaluated")) + + // Replacing an attachment under the same name must not reuse prior health. + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(iface), iface)) + iface.Status.AttachmentRef.UID = types.UID("attachment-2") + require.NoError(t, cl.Status().Update(context.Background(), iface)) + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.Equal(t, 2, dials) + require.NoError(t, cl.Get(context.Background(), key, health)) + require.Empty(t, nlbhealth.HealthyTargets(health, now)) + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(iface), iface)) + iface.Status.AttachmentRef.UID = "" + require.NoError(t, cl.Status().Update(context.Background(), iface)) + _, err = r.Reconcile(context.Background(), ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.Equal(t, 2, dials) + require.NoError(t, cl.Get(context.Background(), key, health)) + require.Empty(t, nlbhealth.HealthyTargets(health, now)) + + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(iface), iface)) + iface.Status.Phase = networkingv1alpha.NetworkInterfacePhaseAvailable + require.NoError(t, cl.Status().Update(context.Background(), iface)) + _, err = r.Reconcile(context.Background(), ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.Equal(t, 2, dials) + require.NoError(t, cl.Get(context.Background(), key, health)) + require.Empty(t, nlbhealth.HealthyTargets(health, now)) + + noDialer := &NetworkLoadBalancerBackendHealthReconciler{Client: cl} + _, err = noDialer.Reconcile(context.Background(), ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.NoError(t, cl.Get(context.Background(), key, health)) + require.Equal(t, "NoDataPlaneDialer", apimeta.FindStatusCondition(health.Status.Conditions, "ProbesEvaluated").Reason) +} + +func TestBackendHealthProbeDeadlineStartsAfterGlobalSlotAcquisition(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, networkingv1alpha.AddToScheme(scheme)) + iface := &networkingv1alpha.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "cell-project", Name: "nic-1", UID: types.UID("interface-1"), + Labels: map[string]string{networkingv1alpha.NetworkInterfaceLocationLabel: "us-east-1"}, + }, + Spec: networkingv1alpha.NetworkInterfaceSpec{ + Network: networkingv1alpha.LocalNetworkRef{Name: "default"}, + Addresses: []networkingv1alpha.NetworkInterfaceAddress{{Address: "fd20::1/128"}}, + }, + Status: networkingv1alpha.NetworkInterfaceStatus{ + Phase: networkingv1alpha.NetworkInterfacePhaseBound, VPC: "vpc-1", + AttachmentRef: &networkingv1alpha.NetworkInterfaceAttachmentRef{Name: "attach-1", UID: types.UID("attachment-1")}, + Conditions: []metav1.Condition{{Type: networkingv1alpha.NetworkInterfaceHolderAvailable, + Status: metav1.ConditionTrue, Reason: "Ready", LastTransitionTime: metav1.Now()}}, + }, + } + health := &networkingv1alpha.NetworkLoadBalancerBackendHealth{ + ObjectMeta: metav1.ObjectMeta{Namespace: "network-services-operator-system", Name: "health-1", Generation: 1}, + Spec: networkingv1alpha.NetworkLoadBalancerBackendHealthSpec{ + RouteRef: networkingv1alpha.NetworkLoadBalancerBackendHealthRouteRef{ + Namespace: "project", Name: "route-1", UID: types.UID("route-1"), Generation: 1, + }, + ParentRef: networkingv1alpha.NetworkLoadBalancerBackendHealthParentRef{ + Name: "load-balancer", UID: types.UID("load-balancer-1"), Generation: 1, Listener: "game", + }, + Location: "us-east-1", NetworkName: "default", FabricVPCID: "vpc-1", + Targets: []networkingv1alpha.NetworkLoadBalancerBackendHealthTarget{{ + Name: "origin-1", SourceNamespace: "cell-project", SourceInterface: "nic-1", SourceUID: iface.UID, + AttachmentName: "attach-1", AttachmentUID: types.UID("attachment-1"), Address: "fd20::1", + ServingPort: 8443, + Probe: networkingv1alpha.NetworkLoadBalancerProbe{Protocol: networkingv1alpha.NetworkLoadBalancerProbeTCP, Port: 8443}, + }}, + }, + } + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(iface, health). + WithStatusSubresource(iface, health).Build() + probed := make(chan struct{}, 1) + r := &NetworkLoadBalancerBackendHealthReconciler{ + Client: cl, + Prober: nlbhealth.Prober{DialContext: func(ctx context.Context, _, _, _ string) (net.Conn, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + probed <- struct{}{} + a, b := net.Pipe() + _ = b.Close() + return a, nil + }}, + } + for range cap(cellProbeSlots) { + cellProbeSlots <- struct{}{} + } + t.Cleanup(func() { + for len(cellProbeSlots) > 0 { + <-cellProbeSlots + } + }) + finished := make(chan error, 1) + go func() { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(health)}) + finished <- err + }() + select { + case <-probed: + t.Fatal("probe ran without acquiring a global slot") + case err := <-finished: + t.Fatalf("reconcile finished before a slot was released: %v", err) + case <-time.After(nlbhealth.ProbeTimeout + 100*time.Millisecond): + } + <-cellProbeSlots + select { + case err := <-finished: + require.NoError(t, err) + case <-time.After(2 * time.Second): + t.Fatal("reconcile did not complete after a slot was released") + } + require.Len(t, probed, 1) + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(health), health)) + require.Equal(t, "Warming", health.Status.Targets[0].Reason) +} diff --git a/internal/networkagent/catalog_test.go b/internal/networkagent/catalog_test.go index 26796cb9..1ed7176f 100644 --- a/internal/networkagent/catalog_test.go +++ b/internal/networkagent/catalog_test.go @@ -30,17 +30,18 @@ var inScopeAPIFiles = []string{ } var outOfScopeAPIFiles = map[string]string{ - "domain_types.go": "load balancer hostnames; the ALB agent's", - "edgereachability_types.go": "hub-side record with no customer-facing status", - "groupversion_info.go": "scheme registration, no reasons", - "httpproxy_types.go": "load balancers; the ALB agent's", - "internaldns_types.go": "internal DNS service configuration, no status reasons", - "locationbinding_types.go": "which locations a project is offered; surfaces here only as LocationNotAvailable", - "networkloadbalancer_types.go": "Network Load Balancer has its own product status, separate from networks and network services", - "networkloadbalancerroute_types.go": "Network Load Balancer route status belongs to the load balancer product", - "networkpolicy_types.go": "declares no status reasons", - "trafficprotectionpolicy_types.go": "load balancer protection; the ALB agent's", - "zz_generated.deepcopy.go": "generated", + "domain_types.go": "load balancer hostnames; the ALB agent's", + "edgereachability_types.go": "hub-side record with no customer-facing status", + "groupversion_info.go": "scheme registration, no reasons", + "httpproxy_types.go": "load balancers; the ALB agent's", + "internaldns_types.go": "internal DNS service configuration, no status reasons", + "locationbinding_types.go": "which locations a project is offered; surfaces here only as LocationNotAvailable", + "networkloadbalancer_types.go": "Network Load Balancer has its own product status, separate from networks and network services", + "networkloadbalancerbackendhealth_types.go": "cell-local NLB probe status, not a network-agent customer API", + "networkloadbalancerroute_types.go": "Network Load Balancer route status belongs to the load balancer product", + "networkpolicy_types.go": "declares no status reasons", + "trafficprotectionpolicy_types.go": "load balancer protection; the ALB agent's", + "zz_generated.deepcopy.go": "generated", } var controllerFiles = []string{ diff --git a/internal/nlbhealth/health.go b/internal/nlbhealth/health.go new file mode 100644 index 00000000..05830187 --- /dev/null +++ b/internal/nlbhealth/health.go @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package nlbhealth + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "net/netip" + "net/url" + "strconv" + "time" + + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +const ( + ProbeTimeout = 2 * time.Second + ProbeInterval = 5 * time.Second + MaxTargets = 64 + ProbeSlotsPerObject = 8 + MaxSweepDuration = (MaxTargets / ProbeSlotsPerObject) * ProbeTimeout + // Status is published after an entire sweep. An early result is already up + // to MaxSweepDuration old at publication, then waits ProbeInterval and up + // to another full sweep for replacement. One timeout of scheduling slack + // preserves eligibility under the bounded 64-target, eight-slot workload; + // a longer delay expires closed rather than extending stale health forever. + ResultLifetime = 2*MaxSweepDuration + ProbeInterval + ProbeTimeout + ConsecutiveSuccessWindow = ResultLifetime + ProbeTimeout + HealthyAfterSuccesses = 2 +) + +type Prober struct { + DialContext func(context.Context, string, string, string) (net.Conn, error) + RootCAs *x509.CertPool +} + +func (p Prober) Probe(ctx context.Context, fabricVPCID string, target networkingv1alpha.NetworkLoadBalancerBackendHealthTarget) error { + if p.DialContext == nil || fabricVPCID == "" { + return errors.New("VPC-aware probe dialer is not configured") + } + address, err := netip.ParseAddr(target.Address) + if err != nil || !address.Is6() || address.Is4In6() || address.IsUnspecified() || address.IsLoopback() || address.IsLinkLocalUnicast() || address.IsMulticast() { + return errors.New("target must be a routable IPv6 unicast address") + } + if target.Probe.Port < 1 || target.Probe.Port > 65535 { + return errors.New("invalid probe port") + } + + ctx, cancel := context.WithTimeout(ctx, ProbeTimeout) + defer cancel() + endpoint := net.JoinHostPort(address.String(), strconv.Itoa(int(target.Probe.Port))) + dial := func(ctx context.Context, network, endpoint string) (net.Conn, error) { + return p.DialContext(ctx, fabricVPCID, network, endpoint) + } + + switch target.Probe.Protocol { + case networkingv1alpha.NetworkLoadBalancerProbeTCP: + conn, err := dial(ctx, "tcp", endpoint) + if err != nil { + return err + } + return conn.Close() + case networkingv1alpha.NetworkLoadBalancerProbeHTTP, networkingv1alpha.NetworkLoadBalancerProbeHTTPS: + path := target.Probe.Path + if path == "" { + path = "/" + } + if path[0] != '/' { + return errors.New("probe path must begin with /") + } + scheme := "http" + if target.Probe.Protocol == networkingv1alpha.NetworkLoadBalancerProbeHTTPS { + scheme = "https" + } + transport := &http.Transport{ + Proxy: nil, + DialContext: dial, + DisableKeepAlives: true, + TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: p.RootCAs, ServerName: target.Probe.ServerName}, + } + defer transport.CloseIdleConnections() + requestURL := url.URL{Scheme: scheme, Host: endpoint, Path: path} + request, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL.String(), nil) + if err != nil { + return err + } + if target.Probe.ServerName != "" { + request.Host = target.Probe.ServerName + } + response, err := (&http.Client{ + Transport: transport, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }).Do(request) + if err != nil { + return err + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode < 200 || response.StatusCode >= 300 { + return fmt.Errorf("probe returned HTTP %d", response.StatusCode) + } + return nil + default: + return errors.New("unsupported probe protocol") + } +} + +func TargetFingerprint(target networkingv1alpha.NetworkLoadBalancerBackendHealthTarget) string { + encoded, _ := json.Marshal(target) + sum := sha256.Sum256(encoded) + return hex.EncodeToString(sum[:]) +} + +func NextTargetStatus( + health *networkingv1alpha.NetworkLoadBalancerBackendHealth, + target networkingv1alpha.NetworkLoadBalancerBackendHealthTarget, + previous *networkingv1alpha.NetworkLoadBalancerBackendHealthTargetStatus, + probeErr error, + now time.Time, +) networkingv1alpha.NetworkLoadBalancerBackendHealthTargetStatus { + fingerprint := TargetFingerprint(target) + status := networkingv1alpha.NetworkLoadBalancerBackendHealthTargetStatus{ + Name: target.Name, + ObservedGeneration: health.Generation, + RouteUID: health.Spec.RouteRef.UID, + RouteGeneration: health.Spec.RouteRef.Generation, + TargetFingerprint: fingerprint, + LastProbeTime: metav1.NewTime(now), + ValidUntil: metav1.NewTime(now), + } + if probeErr != nil { + status.Reason = "ProbeFailed" + return status + } + if previous != nil && previous.ObservedGeneration == health.Generation && + previous.RouteUID == health.Spec.RouteRef.UID && previous.RouteGeneration == health.Spec.RouteRef.Generation && + previous.TargetFingerprint == fingerprint && previous.ConsecutiveSuccesses > 0 && + (previous.Reason == "Warming" || previous.Reason == "Healthy") && + previous.ValidUntil.Time.Equal(previous.LastProbeTime.Add(ResultLifetime)) && + !previous.LastProbeTime.After(now) && now.Sub(previous.LastProbeTime.Time) < ConsecutiveSuccessWindow { + status.ConsecutiveSuccesses = previous.ConsecutiveSuccesses + } + if status.ConsecutiveSuccesses < HealthyAfterSuccesses { + status.ConsecutiveSuccesses++ + } + status.Healthy = status.ConsecutiveSuccesses >= HealthyAfterSuccesses + if status.Healthy { + status.Reason = "Healthy" + } else { + status.Reason = "Warming" + } + status.ValidUntil = metav1.NewTime(now.Add(ResultLifetime)) + return status +} + +func HealthyTargets(health *networkingv1alpha.NetworkLoadBalancerBackendHealth, now time.Time) []networkingv1alpha.NetworkLoadBalancerBackendHealthTarget { + condition := apimeta.FindStatusCondition(health.Status.Conditions, "ProbesEvaluated") + if condition == nil || condition.Status != metav1.ConditionTrue || condition.ObservedGeneration != health.Generation { + return nil + } + byName := make(map[string]networkingv1alpha.NetworkLoadBalancerBackendHealthTargetStatus, len(health.Status.Targets)) + for _, status := range health.Status.Targets { + byName[status.Name] = status + } + var eligible []networkingv1alpha.NetworkLoadBalancerBackendHealthTarget + for _, target := range health.Spec.Targets { + status, found := byName[target.Name] + if !found || !status.Healthy || status.ObservedGeneration != health.Generation || + status.RouteUID != health.Spec.RouteRef.UID || status.RouteGeneration != health.Spec.RouteRef.Generation || + status.TargetFingerprint != TargetFingerprint(target) || status.LastProbeTime.IsZero() || + status.LastProbeTime.After(now) || !status.ValidUntil.After(now) || + status.ValidUntil.Sub(status.LastProbeTime.Time) > ResultLifetime { + continue + } + eligible = append(eligible, target) + } + return eligible +} diff --git a/internal/nlbhealth/health_test.go b/internal/nlbhealth/health_test.go new file mode 100644 index 00000000..51601866 --- /dev/null +++ b/internal/nlbhealth/health_test.go @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package nlbhealth + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "errors" + "math/big" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +func healthFixture() *networkingv1alpha.NetworkLoadBalancerBackendHealth { + return &networkingv1alpha.NetworkLoadBalancerBackendHealth{ + Spec: networkingv1alpha.NetworkLoadBalancerBackendHealthSpec{ + RouteRef: networkingv1alpha.NetworkLoadBalancerBackendHealthRouteRef{UID: types.UID("route-1"), Generation: 2}, + Targets: []networkingv1alpha.NetworkLoadBalancerBackendHealthTarget{{ + Name: "origin", SourceUID: types.UID("interface-1"), AttachmentUID: types.UID("attachment-1"), + Address: "fd20::1", Probe: networkingv1alpha.NetworkLoadBalancerProbe{Protocol: networkingv1alpha.NetworkLoadBalancerProbeTCP, Port: 8080}, + }}, + }, + } +} + +func TestHealthyTargetsRequiresFreshMatchingObservation(t *testing.T) { + now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC) + health := healthFixture() + health.Generation = 4 + target := health.Spec.Targets[0] + first := NextTargetStatus(health, target, nil, nil, now) + require.False(t, first.Healthy) + second := NextTargetStatus(health, target, &first, nil, now.Add(ProbeInterval)) + require.True(t, second.Healthy) + health.Status.Targets = []networkingv1alpha.NetworkLoadBalancerBackendHealthTargetStatus{second} + require.Empty(t, HealthyTargets(health, now.Add(ProbeInterval))) + health.Status.Conditions = []metav1.Condition{{Type: "ProbesEvaluated", Status: metav1.ConditionTrue, Reason: "Evaluated", ObservedGeneration: health.Generation, LastTransitionTime: metav1.NewTime(now)}} + require.Len(t, HealthyTargets(health, now.Add(ProbeInterval)), 1) + + cases := []struct { + name string + change func(*networkingv1alpha.NetworkLoadBalancerBackendHealth) + at time.Time + }{ + {name: "expired", at: now.Add(ProbeInterval + ResultLifetime)}, + {name: "pending evaluation", change: func(h *networkingv1alpha.NetworkLoadBalancerBackendHealth) { + h.Status.Conditions[0].Status = metav1.ConditionUnknown + }}, + {name: "stale evaluation", change: func(h *networkingv1alpha.NetworkLoadBalancerBackendHealth) { + h.Status.Conditions[0].ObservedGeneration-- + }}, + {name: "new object generation", change: func(h *networkingv1alpha.NetworkLoadBalancerBackendHealth) { h.Generation++ }}, + {name: "new route UID", change: func(h *networkingv1alpha.NetworkLoadBalancerBackendHealth) { h.Spec.RouteRef.UID = "route-2" }}, + {name: "new route generation", change: func(h *networkingv1alpha.NetworkLoadBalancerBackendHealth) { h.Spec.RouteRef.Generation++ }}, + {name: "replacement interface", change: func(h *networkingv1alpha.NetworkLoadBalancerBackendHealth) { + h.Spec.Targets[0].SourceUID = "interface-2" + }}, + {name: "replacement attachment", change: func(h *networkingv1alpha.NetworkLoadBalancerBackendHealth) { + h.Spec.Targets[0].AttachmentUID = "attachment-2" + }}, + {name: "probe changed", change: func(h *networkingv1alpha.NetworkLoadBalancerBackendHealth) { h.Spec.Targets[0].Probe.Port++ }}, + {name: "future probe timestamp", change: func(h *networkingv1alpha.NetworkLoadBalancerBackendHealth) { + h.Status.Targets[0].LastProbeTime = h.Status.Targets[0].ValidUntil + }}, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + copy := health.DeepCopy() + if tt.change != nil { + tt.change(copy) + } + at := now.Add(ProbeInterval) + if !tt.at.IsZero() { + at = tt.at + } + require.Empty(t, HealthyTargets(copy, at)) + }) + } + + failed := NextTargetStatus(health, target, &second, errors.New("connection refused"), now.Add(2*ProbeInterval)) + require.False(t, failed.Healthy) + require.Equal(t, int32(0), failed.ConsecutiveSuccesses) + health.Status.Targets = []networkingv1alpha.NetworkLoadBalancerBackendHealthTargetStatus{failed} + require.Empty(t, HealthyTargets(health, now.Add(2*ProbeInterval))) +} + +func TestSuccessContinuityAcrossDifferentProbeWavesDoesNotExtendServingFreshness(t *testing.T) { + now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC) + health := healthFixture() + health.Generation = 4 + health.Status.Conditions = []metav1.Condition{{ + Type: "ProbesEvaluated", Status: metav1.ConditionTrue, Reason: "Evaluated", + ObservedGeneration: health.Generation, LastTransitionTime: metav1.NewTime(now), + }} + target := health.Spec.Targets[0] + warming := NextTargetStatus(health, target, nil, nil, now.Add(-ProbeInterval)) + first := NextTargetStatus(health, target, &warming, nil, now) + require.True(t, first.Healthy) + health.Status.Targets = []networkingv1alpha.NetworkLoadBalancerBackendHealthTargetStatus{first} + require.Len(t, HealthyTargets(health, now.Add(ResultLifetime-time.Nanosecond)), 1) + require.Empty(t, HealthyTargets(health, now.Add(ResultLifetime))) + + // A target may move from an early wave to a late wave of the next sweep. + // Its earlier observation remains unservable, but a fresh success can count. + late := now.Add(ResultLifetime + time.Second) + require.Empty(t, HealthyTargets(health, late)) + second := NextTargetStatus(health, target, &first, nil, late) + require.True(t, second.Healthy) + require.Equal(t, int32(HealthyAfterSuccesses), second.ConsecutiveSuccesses) + health.Status.Targets = []networkingv1alpha.NetworkLoadBalancerBackendHealthTargetStatus{second} + require.Len(t, HealthyTargets(health, late), 1) + + tooLate := NextTargetStatus(health, target, &first, nil, now.Add(ConsecutiveSuccessWindow)) + require.False(t, tooLate.Healthy) + require.Equal(t, int32(1), tooLate.ConsecutiveSuccesses) +} + +func TestProbeRequiresVPCDialerAndRejectsUnsafeTarget(t *testing.T) { + target := healthFixture().Spec.Targets[0] + require.ErrorContains(t, (Prober{}).Probe(context.Background(), "vpc-1", target), "VPC-aware") + prober := Prober{DialContext: func(context.Context, string, string, string) (net.Conn, error) { + a, b := net.Pipe() + if err := b.Close(); err != nil { + return nil, err + } + return a, nil + }} + require.NoError(t, prober.Probe(context.Background(), "vpc-1", target)) + target.Address = "::1" + require.ErrorContains(t, prober.Probe(context.Background(), "vpc-1", target), "routable IPv6") + target.Address = "192.0.2.1" + require.ErrorContains(t, prober.Probe(context.Background(), "vpc-1", target), "routable IPv6") +} + +func TestHTTPAndHTTPSProbeResults(t *testing.T) { + target := healthFixture().Spec.Targets[0] + target.Probe = networkingv1alpha.NetworkLoadBalancerProbe{Protocol: networkingv1alpha.NetworkLoadBalancerProbeHTTP, Port: 8080, Path: "/health"} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/health" { + w.WriteHeader(http.StatusNoContent) + } else { + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + dial := func(ctx context.Context, vpc, network, _ string) (net.Conn, error) { + require.Equal(t, "vpc-1", vpc) + return (&net.Dialer{}).DialContext(ctx, network, server.Listener.Addr().String()) + } + require.NoError(t, (Prober{DialContext: dial}).Probe(context.Background(), "vpc-1", target)) + target.Probe.Path = "/missing" + require.ErrorContains(t, (Prober{DialContext: dial}).Probe(context.Background(), "vpc-1", target), "404") + + tlsServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) })) + defer tlsServer.Close() + cert := tlsServer.Certificate() + serverName := "" + if len(cert.DNSNames) > 0 { + serverName = cert.DNSNames[0] + } else { + serverName = cert.IPAddresses[0].String() + } + target.Probe = networkingv1alpha.NetworkLoadBalancerProbe{Protocol: networkingv1alpha.NetworkLoadBalancerProbeHTTPS, Port: 8080, ServerName: serverName} + tlsDial := func(ctx context.Context, _, network, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, tlsServer.Listener.Addr().String()) + } + require.Error(t, (Prober{DialContext: tlsDial}).Probe(context.Background(), "vpc-1", target)) + roots := x509.NewCertPool() + roots.AddCert(cert) + require.NoError(t, (Prober{DialContext: tlsDial, RootCAs: roots}).Probe(context.Background(), "vpc-1", target)) +} + +func TestProbeHonorsCallerDeadline(t *testing.T) { + target := healthFixture().Spec.Targets[0] + prober := Prober{DialContext: func(ctx context.Context, _, _, _ string) (net.Conn, error) { + <-ctx.Done() + return nil, ctx.Err() + }} + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + start := time.Now() + require.ErrorIs(t, prober.Probe(ctx, "vpc-1", target), context.DeadlineExceeded) + require.Less(t, time.Since(start), time.Second) +} + +func TestHTTPSDefaultsToVerifiedIPSAN(t *testing.T) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + certificate := &x509.Certificate{ + SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "health"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + IPAddresses: []net.IP{net.ParseIP("fd20::1")}, + } + der, err := x509.CreateCertificate(rand.Reader, certificate, certificate, &key.PublicKey, key) + require.NoError(t, err) + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) })) + server.TLS = &tls.Config{Certificates: []tls.Certificate{{Certificate: [][]byte{der}, PrivateKey: key}}} + server.StartTLS() + defer server.Close() + roots := x509.NewCertPool() + parsed, err := x509.ParseCertificate(der) + require.NoError(t, err) + roots.AddCert(parsed) + prober := Prober{ + RootCAs: roots, + DialContext: func(ctx context.Context, _, network, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, server.Listener.Addr().String()) + }, + } + target := healthFixture().Spec.Targets[0] + target.Probe = networkingv1alpha.NetworkLoadBalancerProbe{Protocol: networkingv1alpha.NetworkLoadBalancerProbeHTTPS, Port: 8080} + require.NoError(t, prober.Probe(context.Background(), "vpc-1", target)) + target.Address = "fd20::2" + require.Error(t, prober.Probe(context.Background(), "vpc-1", target)) +} From f801336000bac4ce6d6c5d257df361e3ee864b24 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 15:39:30 +0000 Subject: [PATCH 2/2] feat: Tell serving locations what load balancers need Each location that will serve a load balancer now receives what it needs to serve: the address, port, backends and health check settings. Locations validate this request but do not program traffic yet. Key changes: - Publish scoped cell intent per listener and per-location IPv6 address - Hold back setups the data plane cannot serve yet, such as several addresses on one listener or more than 64 backends - Withdraw old intent when a route is removed or an address changes --- api/v1alpha/groupversion_info.go | 2 + .../networkloadbalancercellintent_types.go | 94 +++++ api/v1alpha/zz_generated.deepcopy.go | 125 ++++++ ...is.com_networkloadbalancercellintents.yaml | 191 +++++++++ config/crd/downstream/kustomization.yaml | 1 + config/crd/kustomization.yaml | 1 + .../federation/clusterpropagationpolicy.yaml | 6 + config/rbac/role.yaml | 14 +- docs/api/networkloadbalancercellintents.md | 369 +++++++++++++++++ .../network-load-balancer-cell-contract.md | 46 ++- internal/agent/catalog_test.go | 1 + internal/cmd/manager/manager.go | 5 + internal/cmd/manager/manager_test.go | 1 + ...etworkloadbalancercellintent_controller.go | 378 ++++++++++++++++++ ...kloadbalancercellintent_controller_test.go | 335 ++++++++++++++++ .../networkloadbalancerroute_controller.go | 13 +- internal/networkagent/catalog_test.go | 1 + 17 files changed, 1571 insertions(+), 12 deletions(-) create mode 100644 api/v1alpha/networkloadbalancercellintent_types.go create mode 100644 config/crd/bases/networking.datumapis.com_networkloadbalancercellintents.yaml create mode 100644 docs/api/networkloadbalancercellintents.md create mode 100644 internal/controller/networkloadbalancercellintent_controller.go create mode 100644 internal/controller/networkloadbalancercellintent_controller_test.go diff --git a/api/v1alpha/groupversion_info.go b/api/v1alpha/groupversion_info.go index c2634bfe..c443e4a7 100644 --- a/api/v1alpha/groupversion_info.go +++ b/api/v1alpha/groupversion_info.go @@ -46,6 +46,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &NetworkLoadBalancerBackendHealthList{}, &NetworkLoadBalancer{}, &NetworkLoadBalancerList{}, + &NetworkLoadBalancerCellIntent{}, + &NetworkLoadBalancerCellIntentList{}, &NetworkLoadBalancerRoute{}, &NetworkLoadBalancerRouteList{}, &NetworkPolicy{}, diff --git a/api/v1alpha/networkloadbalancercellintent_types.go b/api/v1alpha/networkloadbalancercellintent_types.go new file mode 100644 index 00000000..b74dfc89 --- /dev/null +++ b/api/v1alpha/networkloadbalancercellintent_types.go @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package v1alpha + +import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + +// NetworkLoadBalancerCellIntentState distinguishes desired backend validation +// from an explicit withdrawal. Neither state authorizes traffic programming. +// +kubebuilder:validation:Enum=Validate;Withdrawn +type NetworkLoadBalancerCellIntentState string + +const ( + NetworkLoadBalancerCellIntentValidate NetworkLoadBalancerCellIntentState = "Validate" + NetworkLoadBalancerCellIntentWithdrawn NetworkLoadBalancerCellIntentState = "Withdrawn" +) + +type NetworkLoadBalancerCellIntentSource struct { + // +kubebuilder:validation:MinLength=1 + ProjectCluster string `json:"projectCluster"` + // +kubebuilder:validation:MinLength=1 + Namespace string `json:"namespace"` + // +kubebuilder:validation:MinLength=1 + RouteName string `json:"routeName"` + // +kubebuilder:validation:MinLength=1 + RouteUID string `json:"routeUID"` + RouteGeneration int64 `json:"routeGeneration"` + // +kubebuilder:validation:MinLength=1 + ParentName string `json:"parentName"` + // +kubebuilder:validation:MinLength=1 + ParentUID string `json:"parentUID"` + ParentGeneration int64 `json:"parentGeneration"` + // +kubebuilder:validation:MinLength=1 + ListenerName string `json:"listenerName"` + // +kubebuilder:validation:MinLength=1 + AddressRequest string `json:"addressRequest"` +} + +type NetworkLoadBalancerCellIntentProbe struct { + Protocol string `json:"protocol"` + Port int32 `json:"port"` + Path string `json:"path,omitempty"` +} + +type NetworkLoadBalancerCellIntentBackend struct { + // +kubebuilder:validation:MinLength=1 + SourceNamespace string `json:"sourceNamespace"` + // +kubebuilder:validation:MinLength=1 + SourceName string `json:"sourceName"` + // +kubebuilder:validation:MinLength=1 + SourceUID string `json:"sourceUID"` + // Family identifies the backend transport address independently of the + // public VIP family. IPv4 ingress can use an IPv6 backend through a proxy. + Family IPFamily `json:"family"` + // +kubebuilder:validation:MinLength=1 + Address string `json:"address"` + Port int32 `json:"port"` + Probe NetworkLoadBalancerCellIntentProbe `json:"probe"` +} + +// NetworkLoadBalancerCellIntentSpec is a validation request to one location. +// The cell must independently verify the live interface, attachment, and VPC. +// +kubebuilder:validation:XValidation:message="Validate needs backends; Withdrawn must have none",rule="self.state == 'Validate' ? has(self.backends) && size(self.backends) > 0 : !has(self.backends) || size(self.backends) == 0" +type NetworkLoadBalancerCellIntentSpec struct { + Source NetworkLoadBalancerCellIntentSource `json:"source"` + Location string `json:"location"` + NetworkName string `json:"networkName"` + // VIPFamily selects the ingress packet path. It may differ from a backend's + // family; the serving cell must support that family pair before programming. + VIPFamily IPFamily `json:"vipFamily"` + VIP string `json:"vip"` + Protocol NetworkLoadBalancerProtocol `json:"protocol"` + ListenerPort int32 `json:"listenerPort"` + State NetworkLoadBalancerCellIntentState `json:"state"` + // +kubebuilder:validation:MaxItems=1024 + Backends []NetworkLoadBalancerCellIntentBackend `json:"backends,omitempty"` +} + +// +kubebuilder:object:root=true +// NetworkLoadBalancerCellIntent carries a scoped, non-serving desired state +// from a project control plane through the hub to the serving cell. +type NetworkLoadBalancerCellIntent struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NetworkLoadBalancerCellIntentSpec `json:"spec"` +} + +// +kubebuilder:object:root=true + +type NetworkLoadBalancerCellIntentList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NetworkLoadBalancerCellIntent `json:"items"` +} diff --git a/api/v1alpha/zz_generated.deepcopy.go b/api/v1alpha/zz_generated.deepcopy.go index a9a26d0d..05fa91b1 100644 --- a/api/v1alpha/zz_generated.deepcopy.go +++ b/api/v1alpha/zz_generated.deepcopy.go @@ -2141,6 +2141,131 @@ func (in *NetworkLoadBalancerBackendTransport) DeepCopy() *NetworkLoadBalancerBa return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerCellIntent) DeepCopyInto(out *NetworkLoadBalancerCellIntent) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerCellIntent. +func (in *NetworkLoadBalancerCellIntent) DeepCopy() *NetworkLoadBalancerCellIntent { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerCellIntent) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetworkLoadBalancerCellIntent) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerCellIntentBackend) DeepCopyInto(out *NetworkLoadBalancerCellIntentBackend) { + *out = *in + out.Probe = in.Probe +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerCellIntentBackend. +func (in *NetworkLoadBalancerCellIntentBackend) DeepCopy() *NetworkLoadBalancerCellIntentBackend { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerCellIntentBackend) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerCellIntentList) DeepCopyInto(out *NetworkLoadBalancerCellIntentList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NetworkLoadBalancerCellIntent, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerCellIntentList. +func (in *NetworkLoadBalancerCellIntentList) DeepCopy() *NetworkLoadBalancerCellIntentList { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerCellIntentList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetworkLoadBalancerCellIntentList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerCellIntentProbe) DeepCopyInto(out *NetworkLoadBalancerCellIntentProbe) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerCellIntentProbe. +func (in *NetworkLoadBalancerCellIntentProbe) DeepCopy() *NetworkLoadBalancerCellIntentProbe { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerCellIntentProbe) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerCellIntentSource) DeepCopyInto(out *NetworkLoadBalancerCellIntentSource) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerCellIntentSource. +func (in *NetworkLoadBalancerCellIntentSource) DeepCopy() *NetworkLoadBalancerCellIntentSource { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerCellIntentSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerCellIntentSpec) DeepCopyInto(out *NetworkLoadBalancerCellIntentSpec) { + *out = *in + out.Source = in.Source + if in.Backends != nil { + in, out := &in.Backends, &out.Backends + *out = make([]NetworkLoadBalancerCellIntentBackend, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerCellIntentSpec. +func (in *NetworkLoadBalancerCellIntentSpec) DeepCopy() *NetworkLoadBalancerCellIntentSpec { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerCellIntentSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkLoadBalancerEndpointStatus) DeepCopyInto(out *NetworkLoadBalancerEndpointStatus) { *out = *in diff --git a/config/crd/bases/networking.datumapis.com_networkloadbalancercellintents.yaml b/config/crd/bases/networking.datumapis.com_networkloadbalancercellintents.yaml new file mode 100644 index 00000000..6f200c27 --- /dev/null +++ b/config/crd/bases/networking.datumapis.com_networkloadbalancercellintents.yaml @@ -0,0 +1,191 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: networkloadbalancercellintents.networking.datumapis.com +spec: + group: networking.datumapis.com + names: + kind: NetworkLoadBalancerCellIntent + listKind: NetworkLoadBalancerCellIntentList + plural: networkloadbalancercellintents + singular: networkloadbalancercellintent + scope: Namespaced + versions: + - name: v1alpha + schema: + openAPIV3Schema: + description: |- + NetworkLoadBalancerCellIntent carries a scoped, non-serving desired state + from a project control plane through the hub to the serving cell. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NetworkLoadBalancerCellIntentSpec is a validation request to one location. + The cell must independently verify the live interface, attachment, and VPC. + properties: + backends: + items: + properties: + address: + minLength: 1 + type: string + family: + description: |- + Family identifies the backend transport address independently of the + public VIP family. IPv4 ingress can use an IPv6 backend through a proxy. + enum: + - IPv4 + - IPv6 + type: string + port: + format: int32 + type: integer + probe: + properties: + path: + type: string + port: + format: int32 + type: integer + protocol: + type: string + required: + - port + - protocol + type: object + sourceName: + minLength: 1 + type: string + sourceNamespace: + minLength: 1 + type: string + sourceUID: + minLength: 1 + type: string + required: + - address + - family + - port + - probe + - sourceName + - sourceNamespace + - sourceUID + type: object + maxItems: 1024 + type: array + listenerPort: + format: int32 + type: integer + location: + type: string + networkName: + type: string + protocol: + description: NetworkLoadBalancerProtocol is the transport accepted + by a listener. + enum: + - TCP + - UDP + - TLS + type: string + source: + properties: + addressRequest: + minLength: 1 + type: string + listenerName: + minLength: 1 + type: string + namespace: + minLength: 1 + type: string + parentGeneration: + format: int64 + type: integer + parentName: + minLength: 1 + type: string + parentUID: + minLength: 1 + type: string + projectCluster: + minLength: 1 + type: string + routeGeneration: + format: int64 + type: integer + routeName: + minLength: 1 + type: string + routeUID: + minLength: 1 + type: string + required: + - addressRequest + - listenerName + - namespace + - parentGeneration + - parentName + - parentUID + - projectCluster + - routeGeneration + - routeName + - routeUID + type: object + state: + description: |- + NetworkLoadBalancerCellIntentState distinguishes desired backend validation + from an explicit withdrawal. Neither state authorizes traffic programming. + enum: + - Validate + - Withdrawn + type: string + vip: + type: string + vipFamily: + description: |- + VIPFamily selects the ingress packet path. It may differ from a backend's + family; the serving cell must support that family pair before programming. + enum: + - IPv4 + - IPv6 + type: string + required: + - listenerPort + - location + - networkName + - protocol + - source + - state + - vip + - vipFamily + type: object + x-kubernetes-validations: + - message: Validate needs backends; Withdrawn must have none + rule: 'self.state == ''Validate'' ? has(self.backends) && size(self.backends) + > 0 : !has(self.backends) || size(self.backends) == 0' + required: + - spec + type: object + served: true + storage: true diff --git a/config/crd/downstream/kustomization.yaml b/config/crd/downstream/kustomization.yaml index 123da524..096c7072 100644 --- a/config/crd/downstream/kustomization.yaml +++ b/config/crd/downstream/kustomization.yaml @@ -47,4 +47,5 @@ resources: - ../bases/networking.datumapis.com_networkinterfaceclaims.yaml - ../bases/networking.datumapis.com_networkinterfaces.yaml - ../bases/networking.datumapis.com_networkloadbalancerbackendhealths.yaml + - ../bases/networking.datumapis.com_networkloadbalancercellintents.yaml - ../bases/networking.datumapis.com_subnets.yaml diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index cedb4a3a..ccbd42bf 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -19,6 +19,7 @@ resources: - bases/networking.datumapis.com_networkinterfaceclaims.yaml - bases/networking.datumapis.com_networkservices.yaml - bases/networking.datumapis.com_networkloadbalancers.yaml +- bases/networking.datumapis.com_networkloadbalancercellintents.yaml - bases/networking.datumapis.com_networkloadbalancerroutes.yaml # +kubebuilder:scaffold:crdkustomizeresource diff --git a/config/federation/clusterpropagationpolicy.yaml b/config/federation/clusterpropagationpolicy.yaml index 7cbefd9d..dd88a7c8 100644 --- a/config/federation/clusterpropagationpolicy.yaml +++ b/config/federation/clusterpropagationpolicy.yaml @@ -154,3 +154,9 @@ spec: matchExpressions: - key: meta.datumapis.com/upstream-cluster-name operator: Exists + - apiVersion: networking.datumapis.com/v1alpha + kind: NetworkLoadBalancerCellIntent + labelSelector: + matchExpressions: + - key: meta.datumapis.com/upstream-cluster-name + operator: Exists diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index b3dee038..4c9c511f 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -343,6 +343,7 @@ rules: - networkcontexts/finalizers - networkinterfaceclaims/finalizers - networkinterfaces/finalizers + - networkloadbalancerroutes/finalizers - networkloadbalancers/finalizers - networkpolicies/finalizers - networks/finalizers @@ -380,7 +381,6 @@ rules: resources: - connectorclasses - networkloadbalancerbackendhealths - - networkloadbalancerroutes - networkservices verbs: - get @@ -389,6 +389,18 @@ rules: - apiGroups: - networking.datumapis.com resources: + - networkloadbalancercellintents + verbs: + - create + - get + - list + - patch + - update + - watch +- apiGroups: + - networking.datumapis.com + resources: + - networkloadbalancerroutes - networkloadbalancers verbs: - get diff --git a/docs/api/networkloadbalancercellintents.md b/docs/api/networkloadbalancercellintents.md new file mode 100644 index 00000000..30b3b29f --- /dev/null +++ b/docs/api/networkloadbalancercellintents.md @@ -0,0 +1,369 @@ +# API Reference + +Packages: + +- [networking.datumapis.com/v1alpha](#networkingdatumapiscomv1alpha) + +# networking.datumapis.com/v1alpha + +Resource Types: + +- [NetworkLoadBalancerCellIntent](#networkloadbalancercellintent) + + + + +## NetworkLoadBalancerCellIntent +[↩ Parent](#networkingdatumapiscomv1alpha ) + + + + + + +NetworkLoadBalancerCellIntent carries a scoped, non-serving desired state +from a project control plane through the hub to the serving cell. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
apiVersionstringnetworking.datumapis.com/v1alphatrue
kindstringNetworkLoadBalancerCellIntenttrue
metadataobjectRefer to the Kubernetes API documentation for the fields of the `metadata` field.true
specobject + NetworkLoadBalancerCellIntentSpec is a validation request to one location. +The cell must independently verify the live interface, attachment, and VPC.
+
+ Validations:
  • self.state == 'Validate' ? has(self.backends) && size(self.backends) > 0 : !has(self.backends) || size(self.backends) == 0: Validate needs backends; Withdrawn must have none
    • +     		true
    + + +### NetworkLoadBalancerCellIntent.spec +[↩ Parent](#networkloadbalancercellintent) + + + +NetworkLoadBalancerCellIntentSpec is a validation request to one location. +The cell must independently verify the live interface, attachment, and VPC. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    listenerPortinteger +
    +
    + Format: int32
    +     		true
    locationstring +
    +     		true
    networkNamestring +
    +     		true
    protocolenum + NetworkLoadBalancerProtocol is the transport accepted by a listener.
    +
    + Enum: TCP, UDP, TLS
    +     		true
    sourceobject +
    +     		true
    stateenum + NetworkLoadBalancerCellIntentState distinguishes desired backend validation +from an explicit withdrawal. Neither state authorizes traffic programming.
    +
    + Enum: Validate, Withdrawn
    +     		true
    vipstring +
    +     		true
    vipFamilyenum + VIPFamily selects the ingress packet path. It may differ from a backend's +family; the serving cell must support that family pair before programming.
    +
    + Enum: IPv4, IPv6
    +     		true
    backends[]object +
    +     		false
    + + +### NetworkLoadBalancerCellIntent.spec.source +[↩ Parent](#networkloadbalancercellintentspec) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    addressRequeststring +
    +     		true
    listenerNamestring +
    +     		true
    namespacestring +
    +     		true
    parentGenerationinteger +
    +
    + Format: int64
    +     		true
    parentNamestring +
    +     		true
    parentUIDstring +
    +     		true
    projectClusterstring +
    +     		true
    routeGenerationinteger +
    +
    + Format: int64
    +     		true
    routeNamestring +
    +     		true
    routeUIDstring +
    +     		true
    + + +### NetworkLoadBalancerCellIntent.spec.backends[index] +[↩ Parent](#networkloadbalancercellintentspec) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    addressstring +
    +     		true
    familyenum + Family identifies the backend transport address independently of the +public VIP family. IPv4 ingress can use an IPv6 backend through a proxy.
    +
    + Enum: IPv4, IPv6
    +     		true
    portinteger +
    +
    + Format: int32
    +     		true
    probeobject +
    +     		true
    sourceNamestring +
    +     		true
    sourceNamespacestring +
    +     		true
    sourceUIDstring +
    +     		true
    + + +### NetworkLoadBalancerCellIntent.spec.backends[index].probe +[↩ Parent](#networkloadbalancercellintentspecbackendsindex) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    portinteger +
    +
    + Format: int32
    +     		true
    protocolstring +
    +     		true
    pathstring +
    +     		false
    diff --git a/docs/plans/network-load-balancer-cell-contract.md b/docs/plans/network-load-balancer-cell-contract.md index 500e4f86..ec97244c 100644 --- a/docs/plans/network-load-balancer-cell-contract.md +++ b/docs/plans/network-load-balancer-cell-contract.md @@ -21,12 +21,22 @@ namespace. The source interface reports the attachment in reads attachment addresses, `status.vpc`, and `status.node`. Matching an address alone is unsafe because different VPCs can use the same address. -## Proposed intent +## Published intent -The project route reconciler should publish a **versioned, location-scoped -intent** through the hub to the target cell. This resource does not exist yet. -The owner reference must identify the route by UID and generation so a deleted -or replaced route cannot keep a VIP active. Its minimum schema is: +The project reconciler writes a `NetworkLoadBalancerCellIntent` into the mapped +hub namespace. The federation policy carries its spec to gateway-enabled +cells. Only the cell whose configured location equals `spec.location` may +process it. The route's UID and generation and the load balancer's UID and +generation identify the source objects; cross-cluster owner references cannot +provide that guarantee. + +An intent is published only for the oldest route on an existing listener with +resolved backend references, a ready network, and an allocated IPv6 address +whose `Allocated=True` condition observed the current load balancer +generation. Every published address names one location; an empty location or +an address shared by two allocation entries is excluded. +The source interface namespace and UID come from the projected interface; +the cell must verify them against live state. The published shape is: ```yaml apiVersion: networking.datumapis.com/v1alpha @@ -39,22 +49,36 @@ spec: routeUID: routeGeneration: parentName: + parentUID: + parentGeneration: listenerName: + addressRequest:
    location: networkName: vip: protocol: TCP # or UDP listenerPort: 443 + state: Validate backends: - sourceNamespace: sourceName: sourceUID: address: port: 8443 - weight: 1 - healthCheck: + probe: + protocol: TCP + port: 8443 ``` +`Validate` requests identity and health validation only. It never authorizes +`NetworkRule` creation, `Programmed=True`, or VIP announcement. Loss of +eligibility or removal of a route changes the old intent to `Withdrawn` with +an empty backend list. The route holds a finalizer until that withdrawal is +written. Withdrawn objects remain on the hub so a later cell acknowledgment +can prove the old VIP is no longer announced before they are deleted. A new +VIP, listener port, parent UID, or route UID gets a distinct intent name, so +the previous identity is explicitly withdrawn. + Only a platform controller may write this intent. The cell must treat the projected source tuple as a lookup hint, not as proof. For every backend it must read the cell `NetworkInterface` and require the UID, network, bound phase, @@ -70,7 +94,9 @@ fabric ID matches both statuses. A source UID change invalidates old intent even when the name and IP are reused. The cell should report resolved attachment identities and probe results per -backend, with the intent generation it observed. A project controller may set +backend, with the intent generation it observed. Cell status is not propagated +back by Karmada, so a dedicated cell-to-hub acknowledgment is still needed. +A project controller may set route `Programmed=True` only after the cell has acknowledged that generation and every announcing gateway node confirms the same healthy backend set. VIP announcement follows that acknowledgment; an empty healthy set withdraws the @@ -78,10 +104,10 @@ VIP but retains the allocated address. ## Galactic API dependencies -The current `NetworkRule` requires a nonempty `backendSelector`. Cloud's +The deployed `NetworkRule` requires a nonempty `backendSelector`. Cloud's `VPCAttachment` writer does not provide a trusted label that selects all and only this route's attachments. `backendAddresses` narrows a selector, so it -cannot be used alone today. The Network API should permit an omitted selector +cannot be used alone in the deployed API. The Network API follow-up permits an omitted selector **only when `backendAddresses` is explicitly set**. Galactic should then select attachments by authoritative `status.vpc == spec.vpcRef` and intersect their IPv6 addresses with `backendAddresses`. An explicit empty list means no diff --git a/internal/agent/catalog_test.go b/internal/agent/catalog_test.go index 2f3c8930..714136e4 100644 --- a/internal/agent/catalog_test.go +++ b/internal/agent/catalog_test.go @@ -43,6 +43,7 @@ var outOfScopeAPIFiles = map[string]string{ "networkinterfaceclaim_types.go": "an instance's interface; compute's surface", "networkloadbalancer_types.go": "its own Layer 4 product surface, not reached through an application load balancer", "networkloadbalancerbackendhealth_types.go": "cell-local probe state, not an application load balancer customer surface", + "networkloadbalancercellintent_types.go": "internal cell validation request; no customer-facing condition reasons", "networkloadbalancerroute_types.go": "its own Layer 4 routing surface, not reached through an application load balancer", "networkpolicy_types.go": "its own product surface, not reached through a load balancer", "subnet_types.go": "address management below a network", diff --git a/internal/cmd/manager/manager.go b/internal/cmd/manager/manager.go index 09de377b..baa9371d 100644 --- a/internal/cmd/manager/manager.go +++ b/internal/cmd/manager/manager.go @@ -812,6 +812,11 @@ func controllerRegistrations( {"networkloadbalancerroute", true, func() error { return (&controller.NetworkLoadBalancerRouteReconciler{}).SetupWithManager(mgr) }}, + {"networkloadbalancercellintent", true, func() error { + return (&controller.NetworkLoadBalancerCellIntentReconciler{ + HubCluster: deps.downstreamCluster, + }).SetupWithManager(mgr) + }}, // Which workloads are behind a proxy is only answerable in a project // control plane, and only a cell can act on it. The record this writes to // the hub is what carries the answer between them. diff --git a/internal/cmd/manager/manager_test.go b/internal/cmd/manager/manager_test.go index d97e4fe3..6f3185b5 100644 --- a/internal/cmd/manager/manager_test.go +++ b/internal/cmd/manager/manager_test.go @@ -72,6 +72,7 @@ var reconcilerControllerNames = map[string]string{ "NetworkServiceReconciler": "networkservice", "NetworkLoadBalancerRouteReconciler": "networkloadbalancerroute", "NetworkLoadBalancerBackendHealthReconciler": "networkloadbalancerbackendhealth", + "NetworkLoadBalancerCellIntentReconciler": "networkloadbalancercellintent", "SubnetClaimReconciler": "subnetclaim", "SubnetReconciler": "subnet", "TrafficProtectionPolicyReconciler": "trafficprotectionpolicy", diff --git a/internal/controller/networkloadbalancercellintent_controller.go b/internal/controller/networkloadbalancercellintent_controller.go new file mode 100644 index 00000000..86571540 --- /dev/null +++ b/internal/controller/networkloadbalancercellintent_controller.go @@ -0,0 +1,378 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/netip" + "slices" + "strings" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + "sigs.k8s.io/multicluster-runtime/pkg/multicluster" + mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + "go.datum.net/network-services-operator/internal/downstreamclient" +) + +const ( + nlbIntentFinalizer = "networking.datumapis.com/nlb-cell-intent-withdrawal" + nlbIntentRouteUIDLabel = "networking.datumapis.com/route-uid" + nlbIntentRetry = 30 * time.Second +) + +type NetworkLoadBalancerCellIntentReconciler struct { + HubCluster cluster.Cluster + mgr mcmanager.Manager +} + +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkloadbalancerroutes,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkloadbalancerroutes/finalizers,verbs=update +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkloadbalancers;networkservices;networkinterfaces;networks,verbs=get;list;watch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkloadbalancercellintents,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch + +func (r *NetworkLoadBalancerCellIntentReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { + cl, err := r.mgr.GetCluster(ctx, req.ClusterName) + if err != nil { + return ctrl.Result{}, err + } + return r.reconcileIntent(ctx, string(req.ClusterName), cl.GetClient(), req.NamespacedName) +} + +func (r *NetworkLoadBalancerCellIntentReconciler) reconcileIntent(ctx context.Context, project string, projectClient client.Client, key client.ObjectKey) (ctrl.Result, error) { + var route networkingv1alpha.NetworkLoadBalancerRoute + if err := projectClient.Get(ctx, key, &route); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + if route.UID == "" { + return ctrl.Result{}, fmt.Errorf("route %s has no UID", key) + } + + if route.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(&route, nlbIntentFinalizer) { + if err := projectClient.Update(ctx, &route); err != nil { + return ctrl.Result{}, fmt.Errorf("hold route until cell intent withdrawal: %w", err) + } + } + if !route.DeletionTimestamp.IsZero() && !controllerutil.ContainsFinalizer(&route, nlbIntentFinalizer) { + return ctrl.Result{}, nil + } + + strategy := downstreamclient.NewMappedNamespaceResourceStrategy(project, projectClient, r.HubCluster.GetClient()) + hubNamespace, err := strategy.GetDownstreamNamespaceNameForUpstreamNamespace(ctx, route.Namespace) + if apierrors.IsNotFound(err) { + if !route.DeletionTimestamp.IsZero() { + controllerutil.RemoveFinalizer(&route, nlbIntentFinalizer) + return ctrl.Result{}, projectClient.Update(ctx, &route) + } + return ctrl.Result{RequeueAfter: nlbIntentRetry}, nil + } + if err != nil { + return ctrl.Result{}, fmt.Errorf("resolve hub namespace: %w", err) + } + + var desired []networkingv1alpha.NetworkLoadBalancerCellIntent + if route.DeletionTimestamp.IsZero() { + desired, err = buildNetworkLoadBalancerCellIntents(ctx, projectClient, project, hubNamespace, &route) + if err != nil { + return ctrl.Result{}, err + } + } + if err := reconcileHubCellIntents(ctx, r.HubCluster.GetClient(), hubNamespace, project, &route, desired); err != nil { + return ctrl.Result{}, err + } + if !route.DeletionTimestamp.IsZero() { + controllerutil.RemoveFinalizer(&route, nlbIntentFinalizer) + if err := projectClient.Update(ctx, &route); err != nil { + return ctrl.Result{}, fmt.Errorf("release withdrawn route: %w", err) + } + } + return ctrl.Result{}, nil +} + +func buildNetworkLoadBalancerCellIntents(ctx context.Context, cl client.Client, project, hubNamespace string, route *networkingv1alpha.NetworkLoadBalancerRoute) ([]networkingv1alpha.NetworkLoadBalancerCellIntent, error) { + var peers networkingv1alpha.NetworkLoadBalancerRouteList + if err := cl.List(ctx, &peers, client.InNamespace(route.Namespace)); err != nil { + return nil, fmt.Errorf("list route owners: %w", err) + } + var desired []networkingv1alpha.NetworkLoadBalancerCellIntent + for _, ref := range route.Spec.ParentRefs { + if ref.Namespace != "" && ref.Namespace != route.Namespace { + continue + } + var parent networkingv1alpha.NetworkLoadBalancer + if err := cl.Get(ctx, client.ObjectKey{Namespace: route.Namespace, Name: ref.Name}, &parent); err != nil { + if apierrors.IsNotFound(err) { + continue + } + return nil, err + } + if !parent.DeletionTimestamp.IsZero() || parent.UID == "" { + continue + } + listenerName, ok := ResolveNetworkLoadBalancerRouteListener(&parent, ref) + if !ok { + continue + } + winner := oldestRouteForListener(peers.Items, &parent, listenerName) + if winner == nil || winner.UID != route.UID { + continue + } + listenerIndex := slices.IndexFunc(parent.Spec.Listeners, func(l networkingv1alpha.NetworkLoadBalancerListener) bool { return l.Name == listenerName }) + if listenerIndex < 0 { + continue + } + listener := &parent.Spec.Listeners[listenerIndex] + if unsupportedRouteListenerCapability(route, &parent, listener) != "" || len(listener.AddressRefs) != 1 { + continue + } + var network networkingv1alpha.Network + if err := cl.Get(ctx, client.ObjectKey{Namespace: route.Namespace, Name: parent.Spec.NetworkRef.Name}, &network); err != nil { + if apierrors.IsNotFound(err) { + continue + } + return nil, err + } + ready := apimeta.FindStatusCondition(network.Status.Conditions, networkingv1alpha.NetworkReady) + if !network.DeletionTimestamp.IsZero() || ready == nil || ready.Status != metav1.ConditionTrue || ready.ObservedGeneration != network.Generation { + continue + } + endpoints, err := resolveNetworkLoadBalancerRouteBackends(ctx, cl, route, &parent, listener) + var refError *networkLoadBalancerRouteReferenceError + if errors.As(err, &refError) { + continue + } + if err != nil { + return nil, err + } + owner, err := olderRouteClaimingBackend(ctx, cl, route, &parent, listener, endpoints, peers.Items) + if err != nil { + return nil, err + } + if owner != "" { + continue + } + sharedVIPs := map[string]bool{} + seenVIPs := map[string]bool{} + for _, endpoint := range parent.Status.Endpoints { + for _, allocation := range endpoint.Addresses { + vip, parseErr := netip.ParseAddr(allocation.Value) + if parseErr != nil { + continue + } + address := vip.String() + if seenVIPs[address] { + sharedVIPs[address] = true + } + seenVIPs[address] = true + } + } + for _, addressRef := range listener.AddressRefs { + requestIndex := slices.IndexFunc(parent.Spec.Exposure.Addresses, func(a networkingv1alpha.NetworkLoadBalancerAddressRequest) bool { return a.Name == addressRef }) + if requestIndex < 0 { + continue + } + request := parent.Spec.Exposure.Addresses[requestIndex] + for _, endpointStatus := range parent.Status.Endpoints { + if endpointStatus.RequestName != addressRef { + continue + } + for _, allocation := range endpointStatus.Addresses { + // The public ingress path currently supports IPv6 only. + if !slices.Contains(request.Families, allocation.Family) || + allocation.Family != networkingv1alpha.IPv6Protocol || allocation.Value == "" || + allocation.ClaimRef == nil || allocation.ClaimRef.Name == "" { + continue + } + allocated := apimeta.FindStatusCondition(allocation.Conditions, networkingv1alpha.NetworkLoadBalancerAllocated) + if allocated == nil || allocated.Status != metav1.ConditionTrue || allocated.ObservedGeneration != parent.Generation { + continue + } + vip, parseErr := netip.ParseAddr(allocation.Value) + if parseErr != nil || !vip.Is6() || vip.Is4In6() || vip.Zone() != "" || !vip.IsGlobalUnicast() || + endpointStatus.Location == "" || sharedVIPs[vip.String()] { + continue + } + location := endpointStatus.Location + if request.Locations != nil && request.Locations.From == networkingv1alpha.NetworkLoadBalancerLocationSourceNames && !slices.Contains(request.Locations.Names, location) { + continue + } + intent := networkingv1alpha.NetworkLoadBalancerCellIntent{ObjectMeta: metav1.ObjectMeta{ + Namespace: hubNamespace, + Labels: map[string]string{ + downstreamclient.UpstreamOwnerClusterNameLabel: "cluster-" + strings.ReplaceAll(project, "/", "_"), + downstreamclient.UpstreamOwnerNamespaceLabel: route.Namespace, + nlbIntentRouteUIDLabel: string(route.UID), + networkingv1alpha.LocationLabel: location, + }, + }, Spec: networkingv1alpha.NetworkLoadBalancerCellIntentSpec{ + Source: networkingv1alpha.NetworkLoadBalancerCellIntentSource{ + ProjectCluster: project, Namespace: route.Namespace, RouteName: route.Name, + RouteUID: string(route.UID), RouteGeneration: route.Generation, + ParentName: parent.Name, ParentUID: string(parent.UID), ParentGeneration: parent.Generation, + ListenerName: listener.Name, AddressRequest: addressRef, + }, + Location: location, NetworkName: parent.Spec.NetworkRef.Name, + VIPFamily: allocation.Family, VIP: vip.String(), Protocol: listener.Protocol, ListenerPort: listener.Port, + State: networkingv1alpha.NetworkLoadBalancerCellIntentWithdrawn, + }} + for _, endpoint := range endpoints { + if endpoint.Location != location { + continue + } + intent.Spec.Backends = append(intent.Spec.Backends, networkingv1alpha.NetworkLoadBalancerCellIntentBackend{ + SourceNamespace: endpoint.SourceNamespace, SourceName: endpoint.Interface, + SourceUID: endpoint.SourceUID, Family: networkingv1alpha.IPv6Protocol, + Address: endpoint.Address, Port: endpoint.Port, + Probe: endpoint.Probe, + }) + } + // Galactic currently accepts at most 64 explicit backend addresses. + // Keep the consumer and hub APIs open for larger endpoint sets, + // but do not mark such an intent ready for this cell adapter. + if len(intent.Spec.Backends) > 64 { + intent.Spec.Backends = nil + } + if len(intent.Spec.Backends) != 0 { + intent.Spec.State = networkingv1alpha.NetworkLoadBalancerCellIntentValidate + } + intent.Name = networkLoadBalancerCellIntentName(intent.Spec) + desired = append(desired, intent) + } + } + } + } + withdrawConflictingCellIntents(desired) + return desired, nil +} + +// Status can temporarily contain two distinct VIP allocations for one +// address request during replacement. Withdraw all conflicting intents +// until Galactic can reuse backends across VIPs. +func withdrawConflictingCellIntents(desired []networkingv1alpha.NetworkLoadBalancerCellIntent) { + claims := make(map[string]int) + for _, intent := range desired { + if intent.Spec.State != networkingv1alpha.NetworkLoadBalancerCellIntentValidate { + continue + } + for _, backend := range intent.Spec.Backends { + claims[cellIntentBackendClaimKey(&intent, &backend)]++ + } + } + for i := range desired { + intent := &desired[i] + if intent.Spec.State != networkingv1alpha.NetworkLoadBalancerCellIntentValidate { + continue + } + if slices.ContainsFunc(intent.Spec.Backends, func(backend networkingv1alpha.NetworkLoadBalancerCellIntentBackend) bool { + return claims[cellIntentBackendClaimKey(intent, &backend)] > 1 + }) { + intent.Spec.State = networkingv1alpha.NetworkLoadBalancerCellIntentWithdrawn + intent.Spec.Backends = nil + } + } +} + +func cellIntentBackendClaimKey(intent *networkingv1alpha.NetworkLoadBalancerCellIntent, + backend *networkingv1alpha.NetworkLoadBalancerCellIntentBackend, +) string { + return fmt.Sprintf("%s/%s/%s/%s/%d", intent.Spec.NetworkName, intent.Spec.Location, + intent.Spec.Protocol, backend.Address, backend.Port) +} + +func networkLoadBalancerCellIntentName(spec networkingv1alpha.NetworkLoadBalancerCellIntentSpec) string { + identity := fmt.Sprintf("%s/%s/%s/%s/%s/%s/%s/%s/%d", spec.Source.ProjectCluster, spec.Source.Namespace, + spec.Source.RouteUID, spec.Source.ParentUID, spec.Source.ListenerName, spec.Source.AddressRequest, + spec.Location, spec.VIP, spec.ListenerPort) + identity += "/" + string(spec.Protocol) + sum := sha256.Sum256([]byte(identity)) + return "nlb-" + hex.EncodeToString(sum[:16]) +} + +func reconcileHubCellIntents(ctx context.Context, hub client.Client, namespace, project string, route *networkingv1alpha.NetworkLoadBalancerRoute, desired []networkingv1alpha.NetworkLoadBalancerCellIntent) error { + labels := client.MatchingLabels{ + downstreamclient.UpstreamOwnerClusterNameLabel: "cluster-" + strings.ReplaceAll(project, "/", "_"), + downstreamclient.UpstreamOwnerNamespaceLabel: route.Namespace, + nlbIntentRouteUIDLabel: string(route.UID), + } + var existing networkingv1alpha.NetworkLoadBalancerCellIntentList + if err := hub.List(ctx, &existing, client.InNamespace(namespace), labels); err != nil { + return fmt.Errorf("list published cell intents: %w", err) + } + wanted := make(map[string]networkingv1alpha.NetworkLoadBalancerCellIntent, len(desired)) + for _, intent := range desired { + wanted[intent.Name] = intent + } + for i := range existing.Items { + intent := &existing.Items[i] + if _, ok := wanted[intent.Name]; ok { + continue + } + if intent.Spec.State == networkingv1alpha.NetworkLoadBalancerCellIntentWithdrawn && len(intent.Spec.Backends) == 0 { + continue + } + intent.Spec.State = networkingv1alpha.NetworkLoadBalancerCellIntentWithdrawn + intent.Spec.Backends = nil + if err := hub.Update(ctx, intent); err != nil { + return fmt.Errorf("withdraw old cell intent %s: %w", intent.Name, err) + } + } + for _, wantedIntent := range desired { + intent := &networkingv1alpha.NetworkLoadBalancerCellIntent{ObjectMeta: metav1.ObjectMeta{Name: wantedIntent.Name, Namespace: namespace}} + _, err := controllerutil.CreateOrUpdate(ctx, hub, intent, func() error { + if intent.CreationTimestamp.IsZero() && intent.UID == "" || intent.Spec.Source.RouteUID == wantedIntent.Spec.Source.RouteUID { + intent.Labels = wantedIntent.Labels + intent.Spec = wantedIntent.Spec + return nil + } + return fmt.Errorf("cell intent name %s belongs to another route", intent.Name) + }) + if err != nil { + return fmt.Errorf("publish cell intent %s: %w", wantedIntent.Name, err) + } + } + return nil +} + +func (r *NetworkLoadBalancerCellIntentReconciler) SetupWithManager(mgr mcmanager.Manager) error { + if r.HubCluster == nil { + return errors.New("a hub cluster is required") + } + r.mgr = mgr + watch := func(mapper func(context.Context, client.Client, client.Object) []ctrl.Request) func(multicluster.ClusterName, cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { + return func(clusterName multicluster.ClusterName, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { + return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []mcreconcile.Request { + requests := mapper(ctx, cl.GetClient(), obj) + out := make([]mcreconcile.Request, 0, len(requests)) + for _, req := range requests { + out = append(out, mcreconcile.Request{ClusterName: clusterName, Request: req}) + } + return out + }) + } + } + return mcbuilder.ControllerManagedBy(mgr). + For(&networkingv1alpha.NetworkLoadBalancerRoute{}, mcbuilder.WithEngageWithLocalCluster(false)). + Watches(&networkingv1alpha.NetworkLoadBalancerRoute{}, watch(siblingRoutesForRouteEvent)). + Watches(&networkingv1alpha.NetworkLoadBalancer{}, watch(routesForLoadBalancerEvent)). + Watches(&networkingv1alpha.NetworkService{}, watch(routesForNetworkServiceEvent)). + Watches(&networkingv1alpha.NetworkInterface{}, watch(routesForInterfaceEvent)). + Watches(&networkingv1alpha.Network{}, watch(routesForInterfaceEvent)). + Named("networkloadbalancercellintent"). + Complete(r) +} diff --git a/internal/controller/networkloadbalancercellintent_controller_test.go b/internal/controller/networkloadbalancercellintent_controller_test.go new file mode 100644 index 00000000..3066ee3e --- /dev/null +++ b/internal/controller/networkloadbalancercellintent_controller_test.go @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +func allocatedCellIntentAddress(value string, generation int64) networkingv1alpha.NetworkLoadBalancerAddressStatus { + return networkingv1alpha.NetworkLoadBalancerAddressStatus{ + Family: networkingv1alpha.IPv6Protocol, Value: value, + ClaimRef: &networkingv1alpha.NetworkLoadBalancerIPClaimRef{Project: "project", Namespace: "test", Name: "vip-claim"}, + Conditions: []metav1.Condition{{Type: networkingv1alpha.NetworkLoadBalancerAllocated, + Status: metav1.ConditionTrue, Reason: "Allocated", ObservedGeneration: generation, + LastTransitionTime: metav1.Now()}}, + } +} + +func allocatedCellIntentEndpoint(value string, generation int64) networkingv1alpha.NetworkLoadBalancerEndpointStatus { + return networkingv1alpha.NetworkLoadBalancerEndpointStatus{ + RequestName: "regional", Location: "dfw", + Addresses: []networkingv1alpha.NetworkLoadBalancerAddressStatus{allocatedCellIntentAddress(value, generation)}, + } +} + +func TestCellIntentPublishesAndWithdrawsWithoutDeletingHold(t *testing.T) { + ctx := context.Background() + route := testLoadBalancerRoute("storefront", time.Unix(10, 0)) + route.UID = types.UID("route-1") + parent := testRouteLoadBalancer() + parent.UID = types.UID("parent-1") + parent.Generation = 5 + parent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{allocatedCellIntentEndpoint("2001:db8::1", parent.Generation)} + network := readyIntentNetwork() + namespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test", UID: types.UID("project-ns")}} + project := fake.NewClientBuilder().WithScheme(networkServiceTestScheme(t)). + WithObjects(namespace, route, parent, network, testRouteNetworkService(), testRouteMember("member", true, "fd00::1/128")). + WithStatusSubresource(route, parent).Build() + hub := fake.NewClientBuilder().WithScheme(project.Scheme()). + WithObjects(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ns-project-ns"}}).Build() + reconciler := &NetworkLoadBalancerCellIntentReconciler{HubCluster: &fakeCluster{cl: hub}} + key := client.ObjectKeyFromObject(route) + + _, err := reconciler.reconcileIntent(ctx, "project", project, key) + require.NoError(t, err) + var first networkingv1alpha.NetworkLoadBalancerCellIntentList + require.NoError(t, hub.List(ctx, &first)) + require.Len(t, first.Items, 1) + require.Equal(t, networkingv1alpha.NetworkLoadBalancerCellIntentValidate, first.Items[0].Spec.State) + require.Equal(t, "2001:db8::1", first.Items[0].Spec.VIP) + require.Equal(t, networkingv1alpha.IPv6Protocol, first.Items[0].Spec.VIPFamily) + require.Equal(t, networkingv1alpha.IPv6Protocol, first.Items[0].Spec.Backends[0].Family) + require.Equal(t, "project", first.Items[0].Spec.Source.ProjectCluster) + require.Equal(t, "route-1", first.Items[0].Spec.Source.RouteUID) + require.Equal(t, "cell-test", first.Items[0].Spec.Backends[0].SourceNamespace) + require.Equal(t, "source-member", first.Items[0].Spec.Backends[0].SourceUID) + require.Equal(t, int32(8080), first.Items[0].Spec.Backends[0].Probe.Port) + + require.NoError(t, project.Get(ctx, client.ObjectKeyFromObject(parent), parent)) + parent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{allocatedCellIntentEndpoint("2001:db8::2", parent.Generation)} + require.NoError(t, project.Status().Update(ctx, parent)) + _, err = reconciler.reconcileIntent(ctx, "project", project, key) + require.NoError(t, err) + var changed networkingv1alpha.NetworkLoadBalancerCellIntentList + require.NoError(t, hub.List(ctx, &changed)) + require.Len(t, changed.Items, 2) + for _, intent := range changed.Items { + if intent.Spec.VIP == "2001:db8::1" { + require.Equal(t, networkingv1alpha.NetworkLoadBalancerCellIntentWithdrawn, intent.Spec.State) + require.Empty(t, intent.Spec.Backends) + } else { + require.Equal(t, "2001:db8::2", intent.Spec.VIP) + require.Equal(t, networkingv1alpha.NetworkLoadBalancerCellIntentValidate, intent.Spec.State) + } + } + + require.NoError(t, project.Get(ctx, key, route)) + require.NoError(t, project.Delete(ctx, route)) + _, err = reconciler.reconcileIntent(ctx, "project", project, key) + require.NoError(t, err) + var withdrawn networkingv1alpha.NetworkLoadBalancerCellIntentList + require.NoError(t, hub.List(ctx, &withdrawn)) + require.Len(t, withdrawn.Items, 2) + for _, intent := range withdrawn.Items { + require.Equal(t, networkingv1alpha.NetworkLoadBalancerCellIntentWithdrawn, intent.Spec.State) + require.Empty(t, intent.Spec.Backends) + } +} + +func TestCellIntentRequiresFreshPerLocationAllocation(t *testing.T) { + ctx := context.Background() + route := testLoadBalancerRoute("storefront", time.Unix(10, 0)) + parent := testRouteLoadBalancer() + parent.UID = types.UID("parent-1") + parent.Generation = 5 + parent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{ + allocatedCellIntentEndpoint("2001:db8::1", 4), + } + parent.Status.Endpoints[0].Location = "" + cl := fake.NewClientBuilder().WithScheme(networkServiceTestScheme(t)). + WithObjects(route, parent, readyIntentNetwork(), + testRouteNetworkService(), testRouteMember("member", true, "fd00::1/128")).Build() + desired, err := buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Empty(t, desired) + + parent.Status.Endpoints[0].Addresses[0].Conditions[0].ObservedGeneration = 5 + require.NoError(t, cl.Update(ctx, parent)) + desired, err = buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Empty(t, desired) + + parent.Status.Endpoints[0].Location = "dfw" + require.NoError(t, cl.Update(ctx, parent)) + desired, err = buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Len(t, desired, 1) + require.Equal(t, "dfw", desired[0].Spec.Location) + require.Equal(t, networkingv1alpha.NetworkLoadBalancerCellIntentValidate, desired[0].Spec.State) + + duplicate := allocatedCellIntentEndpoint("2001:db8::1", 5) + duplicate.Location = "ord" + parent.Status.Endpoints = append(parent.Status.Endpoints, duplicate) + require.NoError(t, cl.Update(ctx, parent)) + desired, err = buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Empty(t, desired) +} + +func TestCellIntentRejectsUnsupportedVIPFamily(t *testing.T) { + ctx := context.Background() + route := testLoadBalancerRoute("storefront", time.Unix(10, 0)) + parent := testRouteLoadBalancer() + parent.UID = types.UID("parent-1") + parent.Generation = 5 + parent.Spec.Exposure.Addresses[0].Families = []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol} + allocation := allocatedCellIntentAddress("192.0.2.10", parent.Generation) + allocation.Family = networkingv1alpha.IPv4Protocol + parent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{{RequestName: "regional", Location: "dfw", Addresses: []networkingv1alpha.NetworkLoadBalancerAddressStatus{allocation}}} + cl := fake.NewClientBuilder().WithScheme(networkServiceTestScheme(t)). + WithObjects(route, parent, readyIntentNetwork(), testRouteNetworkService(), + testRouteMember("member", true, "fd00::1/128")).Build() + + desired, err := buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Empty(t, desired) +} + +func TestCellIntentResolvesPortParentAndEquivalentNamespace(t *testing.T) { + ctx := context.Background() + port := int32(443) + older := testLoadBalancerRoute("older", time.Unix(10, 0)) + older.Spec.ParentRefs[0] = networkingv1alpha.NetworkLoadBalancerRouteParentRef{ + Name: "edge", Port: &port, Namespace: "test", + } + parent := testRouteLoadBalancer() + parent.UID = types.UID("parent-1") + parent.Generation = 5 + parent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{allocatedCellIntentEndpoint("2001:db8::1", 5)} + cl := fake.NewClientBuilder().WithScheme(networkServiceTestScheme(t)). + WithObjects(older, parent, readyIntentNetwork(), testRouteNetworkService(), + testRouteMember("member", true, "fd00::1/128")).Build() + + desired, err := buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", older) + require.NoError(t, err) + require.Len(t, desired, 1) + require.Equal(t, "tcp", desired[0].Spec.Source.ListenerName) + require.Equal(t, networkingv1alpha.NetworkLoadBalancerCellIntentValidate, desired[0].Spec.State) + + newer := testLoadBalancerRoute("newer", time.Unix(20, 0)) + require.NoError(t, cl.Create(ctx, newer)) + desired, err = buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", newer) + require.NoError(t, err) + require.Empty(t, desired) +} + +func TestCellIntentWithholdsOversizedGalacticBackendSet(t *testing.T) { + ctx := context.Background() + route := testLoadBalancerRoute("storefront", time.Unix(10, 0)) + parent := testRouteLoadBalancer() + parent.UID = types.UID("parent-1") + parent.Generation = 5 + parent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{allocatedCellIntentEndpoint("2001:db8::1", 5)} + objects := make([]client.Object, 0, 69) + objects = append(objects, route, parent, readyIntentNetwork(), testRouteNetworkService()) + for i := range 65 { + objects = append(objects, testRouteMember(fmt.Sprintf("member-%d", i), true, + fmt.Sprintf("fd00::%x/128", i+1))) + } + cl := fake.NewClientBuilder().WithScheme(networkServiceTestScheme(t)).WithObjects(objects...).Build() + desired, err := buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Len(t, desired, 1) + require.Equal(t, networkingv1alpha.NetworkLoadBalancerCellIntentWithdrawn, desired[0].Spec.State) + require.Empty(t, desired[0].Spec.Backends) +} + +func readyIntentNetwork() *networkingv1alpha.Network { + return &networkingv1alpha.Network{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "default", Generation: 1}, + Status: networkingv1alpha.NetworkStatus{Conditions: []metav1.Condition{{ + Type: networkingv1alpha.NetworkReady, Status: metav1.ConditionTrue, + Reason: "Ready", ObservedGeneration: 1, LastTransitionTime: metav1.Now(), + }}}, + } +} + +func TestCellIntentFailsClosedOnMissingIdentityAndOlderRoute(t *testing.T) { + ctx := context.Background() + route := testLoadBalancerRoute("newer", time.Unix(20, 0)) + parent := testRouteLoadBalancer() + parent.UID = types.UID("parent-1") + parent.Generation = 5 + parent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{allocatedCellIntentEndpoint("2001:db8::1", 5)} + member := testRouteMember("member", true, "fd00::1/128") + delete(member.Labels, networkingv1alpha.NetworkInterfaceSourceUIDLabel) + cl := fake.NewClientBuilder().WithScheme(networkServiceTestScheme(t)). + WithObjects(route, parent, readyIntentNetwork(), testRouteNetworkService(), member).Build() + + desired, err := buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Len(t, desired, 1) + require.Equal(t, networkingv1alpha.NetworkLoadBalancerCellIntentWithdrawn, desired[0].Spec.State) + require.Empty(t, desired[0].Spec.Backends) + + member.Labels[networkingv1alpha.NetworkInterfaceSourceUIDLabel] = "source-member" + require.NoError(t, cl.Update(ctx, member)) + require.NoError(t, cl.Create(ctx, testLoadBalancerRoute("older", time.Unix(10, 0)))) + desired, err = buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Empty(t, desired) +} + +func TestCellIntentDoesNotValidateConflictingBackendSocket(t *testing.T) { + ctx := context.Background() + older := testLoadBalancerRoute("older", time.Unix(10, 0)) + newer := testLoadBalancerRoute("newer", time.Unix(20, 0)) + newer.Spec.ParentRefs[0].Name = "edge-2" + firstParent := testRouteLoadBalancer() + firstParent.UID = types.UID("first-parent") + firstParent.Generation = 5 + firstParent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{allocatedCellIntentEndpoint("2001:db8::1", 5)} + secondParent := firstParent.DeepCopy() + secondParent.Name = "edge-2" + secondParent.UID = types.UID("second-parent") + secondParent.Status.Endpoints[0].Addresses[0].Value = "2001:db8::2" + cl := fake.NewClientBuilder().WithScheme(networkServiceTestScheme(t)). + WithObjects(older, newer, firstParent, secondParent, readyIntentNetwork(), testRouteNetworkService(), + testRouteMember("member", true, "fd00::1/128")).Build() + desired, err := buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", newer) + require.NoError(t, err) + require.Empty(t, desired) +} + +func TestCellIntentWithholdsMultipleVIPsOnOneListener(t *testing.T) { + ctx := context.Background() + route := testLoadBalancerRoute("route", time.Unix(10, 0)) + parent := testRouteLoadBalancer() + parent.UID = types.UID("parent-1") + parent.Generation = 5 + second := parent.Spec.Exposure.Addresses[0] + second.Name = "second" + parent.Spec.Exposure.Addresses = append(parent.Spec.Exposure.Addresses, second) + parent.Spec.Listeners[0].AddressRefs = []string{"regional", "second"} + parent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{ + allocatedCellIntentEndpoint("2001:db8::1", 5), + allocatedCellIntentEndpoint("2001:db8::2", 5), + } + parent.Status.Endpoints[1].RequestName = "second" + cl := fake.NewClientBuilder().WithScheme(networkServiceTestScheme(t)). + WithObjects(route, parent, readyIntentNetwork(), testRouteNetworkService(), + testRouteMember("member", true, "fd00::1/128")).Build() + desired, err := buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Empty(t, desired) +} + +func TestCellIntentWithdrawsTwoAllocatedVIPsForOneRequest(t *testing.T) { + ctx := context.Background() + route := testLoadBalancerRoute("route", time.Unix(10, 0)) + parent := testRouteLoadBalancer() + parent.UID = types.UID("parent-1") + parent.Generation = 5 + parent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{ + allocatedCellIntentEndpoint("2001:db8::1", 5), + allocatedCellIntentEndpoint("2001:db8::2", 5), + } + cl := fake.NewClientBuilder().WithScheme(networkServiceTestScheme(t)). + WithObjects(route, parent, readyIntentNetwork(), testRouteNetworkService(), + testRouteMember("member", true, "fd00::1/128")).Build() + desired, err := buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Len(t, desired, 2) + for _, intent := range desired { + require.Equal(t, networkingv1alpha.NetworkLoadBalancerCellIntentWithdrawn, intent.Spec.State) + require.Empty(t, intent.Spec.Backends) + } +} + +func TestCellIntentWithholdsSameRouteSecondParentBackendClaim(t *testing.T) { + ctx := context.Background() + route := testLoadBalancerRoute("route", time.Unix(10, 0)) + route.Spec.ParentRefs = append(route.Spec.ParentRefs, + networkingv1alpha.NetworkLoadBalancerRouteParentRef{Name: "edge-2", Listener: "tcp"}) + firstParent := testRouteLoadBalancer() + firstParent.UID = types.UID("first-parent") + firstParent.Generation = 5 + firstParent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{ + allocatedCellIntentEndpoint("2001:db8::1", 5), + } + secondParent := firstParent.DeepCopy() + secondParent.Name = "edge-2" + secondParent.UID = types.UID("second-parent") + secondParent.Status.Endpoints[0].Addresses[0].Value = "2001:db8::2" + cl := fake.NewClientBuilder().WithScheme(networkServiceTestScheme(t)). + WithObjects(route, firstParent, secondParent, readyIntentNetwork(), testRouteNetworkService(), + testRouteMember("member", true, "fd00::1/128")).Build() + desired, err := buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Len(t, desired, 1) + require.Equal(t, "edge", desired[0].Spec.Source.ParentName) + require.Equal(t, networkingv1alpha.NetworkLoadBalancerCellIntentValidate, desired[0].Spec.State) +} diff --git a/internal/controller/networkloadbalancerroute_controller.go b/internal/controller/networkloadbalancerroute_controller.go index 5bb5f751..5d05adb1 100644 --- a/internal/controller/networkloadbalancerroute_controller.go +++ b/internal/controller/networkloadbalancerroute_controller.go @@ -60,6 +60,7 @@ type networkLoadBalancerRouteEndpoint struct { Location string Address string Port int32 + Probe networkingv1alpha.NetworkLoadBalancerCellIntentProbe } type networkLoadBalancerRouteReferenceError struct { @@ -492,12 +493,21 @@ func resolveNetworkLoadBalancerRouteBackends( fmt.Sprintf("NetworkService %q selects an interface outside network %q", ref.Name, parent.Spec.NetworkRef.Name)} } } + probe := networkingv1alpha.NetworkLoadBalancerCellIntentProbe{Protocol: "TCP", Port: port.Port} + if check := backend.HealthCheck; check != nil { + probe.Protocol = check.Protocol + probe.Path = check.Path + if check.Port != "" { + probeIndex := slices.IndexFunc(service.Spec.Ports, func(p networkingv1alpha.NetworkServicePort) bool { return p.Name == check.Port }) + probe.Port = service.Spec.Ports[probeIndex].Port + } + } for i := range members { member := &members[i] location := member.Labels[networkingv1alpha.NetworkInterfaceLocationLabel] sourceNamespace := member.Labels[networkingv1alpha.NetworkInterfaceSourceNamespaceLabel] sourceUID := member.Labels[networkingv1alpha.NetworkInterfaceSourceUIDLabel] - if !isProjection(member) || sourceNamespace == "" || sourceUID == "" || + if !isProjection(member) || member.Status.Phase != networkingv1alpha.NetworkInterfacePhaseBound || sourceNamespace == "" || sourceUID == "" || location == "" || !routeLocationSelected(parent, listener, location) || !isServiceMemberHealthy(member) { continue } @@ -513,6 +523,7 @@ func resolveNetworkLoadBalancerRouteBackends( Location: location, Address: address, Port: port.Port, + Probe: probe, }) } } diff --git a/internal/networkagent/catalog_test.go b/internal/networkagent/catalog_test.go index 1ed7176f..50cee86b 100644 --- a/internal/networkagent/catalog_test.go +++ b/internal/networkagent/catalog_test.go @@ -38,6 +38,7 @@ var outOfScopeAPIFiles = map[string]string{ "locationbinding_types.go": "which locations a project is offered; surfaces here only as LocationNotAvailable", "networkloadbalancer_types.go": "Network Load Balancer has its own product status, separate from networks and network services", "networkloadbalancerbackendhealth_types.go": "cell-local NLB probe status, not a network-agent customer API", + "networkloadbalancercellintent_types.go": "hub-side NLB serving intent, not a network-agent customer API", "networkloadbalancerroute_types.go": "Network Load Balancer route status belongs to the load balancer product", "networkpolicy_types.go": "declares no status reasons", "trafficprotectionpolicy_types.go": "load balancer protection; the ALB agent's",