diff --git a/NOTICE b/NOTICE index 557e25eb..95b6ebb7 100644 --- a/NOTICE +++ b/NOTICE @@ -9355,10 +9355,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ``` -## go.datum.net/galactic/api/cloud/v1alpha1 +## go.datum.net/galactic/api -* Name: go.datum.net/galactic/api/cloud/v1alpha1 -* Version: v0.5.8-0.20261016143526-45d3c3514bec +* Name: go.datum.net/galactic/api +* Version: v0.6.3-0.20261021151056-bb0b4c057338 * License: AGPL-3.0 ``` @@ -10038,7 +10038,7 @@ For more information on this, and how to apply and follow the GNU AGPL, see ## go.miloapis.com/ipam/pkg/apis/ipam * Name: go.miloapis.com/ipam/pkg/apis/ipam -* Version: v0.4.0 +* Version: v0.5.1 * License: Unknown ``` @@ -10048,7 +10048,7 @@ For more information on this, and how to apply and follow the GNU AGPL, see ## go.miloapis.com/ipam/pkg/apis/ipam/v1alpha1 * Name: go.miloapis.com/ipam/pkg/apis/ipam/v1alpha1 -* Version: v0.4.0 +* Version: v0.5.1 * License: Unknown ``` @@ -10058,7 +10058,7 @@ For more information on this, and how to apply and follow the GNU AGPL, see ## go.miloapis.com/ipam/pkg/ipamerrors * Name: go.miloapis.com/ipam/pkg/ipamerrors -* Version: v0.4.0 +* Version: v0.5.1 * License: Unknown ``` @@ -15151,7 +15151,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ## google.golang.org/genproto/googleapis/rpc * Name: google.golang.org/genproto/googleapis/rpc -* Version: v0.0.0-20261005182115-fad411399dd8 +* Version: v0.0.0-20261019180522-ce908d29299b * License: Apache-2.0 ``` @@ -19626,7 +19626,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ## sigs.k8s.io/json * Name: sigs.k8s.io/json -* Version: v0.0.0-20250730193827-2d320260d730 +* Version: v0.0.0-20260909141634-11ed52e25bc5 * License: Apache-2.0 ``` diff --git a/api/v1alpha/internaldns_types.go b/api/v1alpha/internaldns_types.go new file mode 100644 index 00000000..1ebb49fa --- /dev/null +++ b/api/v1alpha/internaldns_types.go @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package v1alpha + +// +kubebuilder:validation:Enum=Managed;Disabled +type NetworkDNSMode string + +const ( + NetworkDNSModeManaged NetworkDNSMode = "Managed" + NetworkDNSModeDisabled NetworkDNSMode = "Disabled" +) + +// NetworkDNSPolicy selects managed resolver access for a network. +type NetworkDNSPolicy struct { + // Mode controls whether network services maintain private resolver access. + // An empty value uses Managed when the Internal DNS feature is enabled. + // + // +kubebuilder:validation:Optional + Mode NetworkDNSMode `json:"mode,omitempty"` +} + +// NetworkDNSObjectReference identifies a DNS resource by name. +type NetworkDNSObjectReference struct { + // Name is the resource name in the network's namespace. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` +} + +type NetworkDNSResolverSettings struct { + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=3 + // +kubebuilder:validation:XValidation:rule="self.all(n, isIP(n))",message="nameservers must contain IP addresses" + Nameservers []string `json:"nameservers"` + // +kubebuilder:validation:MaxItems=6 + Searches []string `json:"searches,omitempty"` +} + +// NetworkDNSSettings are verified resolver settings inherited by interfaces. +type NetworkDNSSettings struct { + // ContextRef identifies the DNS serving and namespace lifetime. + // + // +kubebuilder:validation:Required + ContextRef NetworkDNSObjectReference `json:"contextRef"` + + // AccessBindingRef identifies the regional DNS configuration. + // + // +kubebuilder:validation:Required + AccessBindingRef NetworkDNSObjectReference `json:"accessBindingRef"` + + // Nameservers are resolver addresses reachable from this network. + // + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=3 + // +kubebuilder:validation:XValidation:rule="self.all(n, isIP(n))",message="nameservers must contain IP addresses" + Nameservers []string `json:"nameservers"` + + // Searches contains DNS search suffixes allocated to this network. + // + // +kubebuilder:validation:MaxItems=6 + Searches []string `json:"searches,omitempty"` +} + +// NetworkDNSAccessStatus reports the private path verified for one context. +type NetworkDNSAccessStatus struct { + // ContextRef identifies the DNS context used for the proof. + ContextRef NetworkDNSObjectReference `json:"contextRef"` + + // AccessBindingRef identifies the DNS configuration used for the proof. + AccessBindingRef NetworkDNSObjectReference `json:"accessBindingRef"` + + // ServiceDestination identifies this DNS context inside the shared service. + // + // +kubebuilder:validation:XValidation:rule="isIP(self)",message="serviceDestination must be an IP address" + ServiceDestination string `json:"serviceDestination"` +} + +const ( + NetworkContextDNSReady = "DNSReady" + + NetworkContextDNSReadyReasonReady = "Ready" + NetworkContextDNSReadyReasonDisabled = "Disabled" + NetworkContextDNSReadyReasonPending = "Pending" + NetworkContextDNSReadyReasonAccessAbsent = "AccessAbsent" + NetworkContextDNSReadyReasonNetworkPathNotReady = "NetworkPathNotReady" + NetworkContextDNSReadyReasonNetworkIdentityAbsent = "NetworkIdentityAbsent" + + NetworkInterfaceDNSConfiguredReasonApplied = "Applied" + NetworkInterfaceDNSConfiguredReasonUnsupportedMultipleContexts = "UnsupportedMultipleContexts" +) diff --git a/api/v1alpha/network_types.go b/api/v1alpha/network_types.go index 8dc07ecf..b2a5b310 100644 --- a/api/v1alpha/network_types.go +++ b/api/v1alpha/network_types.go @@ -16,6 +16,10 @@ const ( // NetworkSpec defines the desired state of a Network type NetworkSpec struct { + // DNS configures private resolver access for the network. + // + // +kubebuilder:validation:Optional + DNS NetworkDNSPolicy `json:"dns,omitempty"` // IPAM settings for the network. // diff --git a/api/v1alpha/networkcontext_types.go b/api/v1alpha/networkcontext_types.go index 7f5d7fc8..ace98918 100644 --- a/api/v1alpha/networkcontext_types.go +++ b/api/v1alpha/networkcontext_types.go @@ -43,6 +43,12 @@ type NetworkContextSpec struct { // // +kubebuilder:validation:Optional NetworkGeneration int64 `json:"networkGeneration,omitempty"` + + // DNS contains resolver settings whose DNS authorization and private path + // have both been verified for this location. + // + // +kubebuilder:validation:Optional + DNS *NetworkDNSSettings `json:"dns,omitempty"` } // NetworkContextStatus defines the observed state of NetworkContext @@ -55,6 +61,12 @@ type NetworkContextStatus struct { // // +kubebuilder:validation:Optional IPAM *NetworkContextIPAMStatus `json:"ipam,omitempty"` + + // DNSAccess identifies the current private path proof. DNSReady=True applies + // only to this exact context, binding revision, and deadline. + // + // +kubebuilder:validation:Optional + DNSAccess *NetworkDNSAccessStatus `json:"dnsAccess,omitempty"` } // NetworkContextIPAMStatus reports what IPAM holds for a network in one diff --git a/api/v1alpha/networkinterface_types.go b/api/v1alpha/networkinterface_types.go index 14552ffa..7319a3a1 100644 --- a/api/v1alpha/networkinterface_types.go +++ b/api/v1alpha/networkinterface_types.go @@ -113,6 +113,10 @@ const ( // interface at all. A phase of Available means no claim holds it, which is // the opposite of anything being available to serve. NetworkInterfaceHolderAvailable = "HolderAvailable" + + // NetworkInterfaceDNSConfigured is reported by the provider after applying + // the current settings in spec.dns to the workload. + NetworkInterfaceDNSConfigured = "DNSConfigured" ) const ( @@ -342,6 +346,9 @@ type NetworkInterfaceSpec struct { // claim holds it, what realizes it on the data plane, and whether programming // has succeeded. type NetworkInterfaceStatus struct { + // DNS contains effective resolver settings inherited from the network. + // +kubebuilder:validation:Optional + DNS *NetworkDNSResolverSettings `json:"dns,omitempty"` // phase reports whether a claim holds the interface. Bound means the claim in // spec.claimRef holds it. Available means it is retained and holding its // addresses with no claim bound. diff --git a/api/v1alpha/zz_generated.deepcopy.go b/api/v1alpha/zz_generated.deepcopy.go index a2e146b3..bf60acf9 100644 --- a/api/v1alpha/zz_generated.deepcopy.go +++ b/api/v1alpha/zz_generated.deepcopy.go @@ -1263,6 +1263,11 @@ func (in *NetworkContextSpec) DeepCopyInto(out *NetworkContextSpec) { *out = make([]IPFamily, len(*in)) copy(*out, *in) } + if in.DNS != nil { + in, out := &in.DNS, &out.DNS + *out = new(NetworkDNSSettings) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkContextSpec. @@ -1290,6 +1295,11 @@ func (in *NetworkContextStatus) DeepCopyInto(out *NetworkContextStatus) { *out = new(NetworkContextIPAMStatus) (*in).DeepCopyInto(*out) } + if in.DNSAccess != nil { + in, out := &in.DNSAccess, &out.DNSAccess + *out = new(NetworkDNSAccessStatus) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkContextStatus. @@ -1302,6 +1312,105 @@ func (in *NetworkContextStatus) DeepCopy() *NetworkContextStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkDNSAccessStatus) DeepCopyInto(out *NetworkDNSAccessStatus) { + *out = *in + out.ContextRef = in.ContextRef + out.AccessBindingRef = in.AccessBindingRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkDNSAccessStatus. +func (in *NetworkDNSAccessStatus) DeepCopy() *NetworkDNSAccessStatus { + if in == nil { + return nil + } + out := new(NetworkDNSAccessStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkDNSObjectReference) DeepCopyInto(out *NetworkDNSObjectReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkDNSObjectReference. +func (in *NetworkDNSObjectReference) DeepCopy() *NetworkDNSObjectReference { + if in == nil { + return nil + } + out := new(NetworkDNSObjectReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkDNSPolicy) DeepCopyInto(out *NetworkDNSPolicy) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkDNSPolicy. +func (in *NetworkDNSPolicy) DeepCopy() *NetworkDNSPolicy { + if in == nil { + return nil + } + out := new(NetworkDNSPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkDNSResolverSettings) DeepCopyInto(out *NetworkDNSResolverSettings) { + *out = *in + if in.Nameservers != nil { + in, out := &in.Nameservers, &out.Nameservers + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Searches != nil { + in, out := &in.Searches, &out.Searches + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkDNSResolverSettings. +func (in *NetworkDNSResolverSettings) DeepCopy() *NetworkDNSResolverSettings { + if in == nil { + return nil + } + out := new(NetworkDNSResolverSettings) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkDNSSettings) DeepCopyInto(out *NetworkDNSSettings) { + *out = *in + out.ContextRef = in.ContextRef + out.AccessBindingRef = in.AccessBindingRef + if in.Nameservers != nil { + in, out := &in.Nameservers, &out.Nameservers + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Searches != nil { + in, out := &in.Searches, &out.Searches + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkDNSSettings. +func (in *NetworkDNSSettings) DeepCopy() *NetworkDNSSettings { + if in == nil { + return nil + } + out := new(NetworkDNSSettings) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkIPAM) DeepCopyInto(out *NetworkIPAM) { *out = *in @@ -1654,6 +1763,11 @@ func (in *NetworkInterfaceSpec) DeepCopy() *NetworkInterfaceSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkInterfaceStatus) DeepCopyInto(out *NetworkInterfaceStatus) { *out = *in + if in.DNS != nil { + in, out := &in.DNS, &out.DNS + *out = new(NetworkDNSResolverSettings) + (*in).DeepCopyInto(*out) + } if in.NetworkContextRef != nil { in, out := &in.NetworkContextRef, &out.NetworkContextRef *out = new(LocalNetworkContextRef) @@ -2123,6 +2237,7 @@ func (in *NetworkServiceTrafficDistribution) DeepCopy() *NetworkServiceTrafficDi // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkSpec) DeepCopyInto(out *NetworkSpec) { *out = *in + out.DNS = in.DNS in.IPAM.DeepCopyInto(&out.IPAM) if in.IPFamilies != nil { in, out := &in.IPFamilies, &out.IPFamilies diff --git a/config/components/cell-controllers/config.yaml b/config/components/cell-controllers/config.yaml index 057a1b67..fbd500cf 100644 --- a/config/components/cell-controllers/config.yaml +++ b/config/components/cell-controllers/config.yaml @@ -1,5 +1,17 @@ apiVersion: apiserver.config.datumapis.com/v1alpha1 kind: CellControllerManager +# Internal DNS is an alpha network integration and remains off until the +# shared resolver attachments are configured for this cell. +internalDNS: + # Enables private resolver route programming and readiness reporting. + enabled: false + # Presents the same resolver address inside each consumer VPC. + # frontend: fd53::53 + # Allocates one stable service destination from this prefix per DNS context. + # servicePrefix: fd70:100::/64 + # Selects the shared resolver attachments that serve authorized VPCs. + # producerAttachmentSelector: + # service.datum.net/internal-dns: "true" metricsServer: bindAddress: ":8443" ipam: diff --git a/config/crd/bases/networking.datumapis.com_networkcontexts.yaml b/config/crd/bases/networking.datumapis.com_networkcontexts.yaml index 5c939214..492830bc 100644 --- a/config/crd/bases/networking.datumapis.com_networkcontexts.yaml +++ b/config/crd/bases/networking.datumapis.com_networkcontexts.yaml @@ -52,6 +52,55 @@ spec: spec: description: NetworkContextSpec defines the desired state of NetworkContext properties: + dns: + description: |- + DNS contains resolver settings whose DNS authorization and private path + have both been verified for this location. + properties: + accessBindingRef: + description: AccessBindingRef identifies the regional DNS configuration. + properties: + name: + description: Name is the resource name in the network's namespace. + minLength: 1 + type: string + required: + - name + type: object + contextRef: + description: ContextRef identifies the DNS serving and namespace + lifetime. + properties: + name: + description: Name is the resource name in the network's namespace. + minLength: 1 + type: string + required: + - name + type: object + nameservers: + description: Nameservers are resolver addresses reachable from + this network. + items: + type: string + maxItems: 3 + minItems: 1 + type: array + x-kubernetes-validations: + - message: nameservers must contain IP addresses + rule: self.all(n, isIP(n)) + searches: + description: Searches contains DNS search suffixes allocated to + this network. + items: + type: string + maxItems: 6 + type: array + required: + - accessBindingRef + - contextRef + - nameservers + type: object ipFamilies: description: |- IP families the network carries, projected from the Network. @@ -170,6 +219,45 @@ spec: - type type: object type: array + dnsAccess: + description: |- + DNSAccess identifies the current private path proof. DNSReady=True applies + only to this exact context, binding revision, and deadline. + properties: + accessBindingRef: + description: AccessBindingRef identifies the DNS configuration + used for the proof. + properties: + name: + description: Name is the resource name in the network's namespace. + minLength: 1 + type: string + required: + - name + type: object + contextRef: + description: ContextRef identifies the DNS context used for the + proof. + properties: + name: + description: Name is the resource name in the network's namespace. + minLength: 1 + type: string + required: + - name + type: object + serviceDestination: + description: ServiceDestination identifies this DNS context inside + the shared service. + type: string + x-kubernetes-validations: + - message: serviceDestination must be an IP address + rule: isIP(self) + required: + - accessBindingRef + - contextRef + - serviceDestination + type: object ipam: description: |- IPAM reports the address space IPAM holds for this network in this diff --git a/config/crd/bases/networking.datumapis.com_networkinterfaces.yaml b/config/crd/bases/networking.datumapis.com_networkinterfaces.yaml index 6d91187a..4973ac3a 100644 --- a/config/crd/bases/networking.datumapis.com_networkinterfaces.yaml +++ b/config/crd/bases/networking.datumapis.com_networkinterfaces.yaml @@ -383,6 +383,27 @@ spec: - type type: object type: array + dns: + description: DNS contains effective resolver settings inherited from + the network. + properties: + nameservers: + items: + type: string + maxItems: 3 + minItems: 1 + type: array + x-kubernetes-validations: + - message: nameservers must contain IP addresses + rule: self.all(n, isIP(n)) + searches: + items: + type: string + maxItems: 6 + type: array + required: + - nameservers + type: object networkContextRef: description: |- networkContextRef is the network's presence in this location, resolved or diff --git a/config/crd/bases/networking.datumapis.com_networks.yaml b/config/crd/bases/networking.datumapis.com_networks.yaml index fe83e191..712d104f 100644 --- a/config/crd/bases/networking.datumapis.com_networks.yaml +++ b/config/crd/bases/networking.datumapis.com_networks.yaml @@ -64,6 +64,18 @@ spec: spec: description: NetworkSpec defines the desired state of a Network properties: + dns: + description: DNS configures private resolver access for the network. + properties: + mode: + description: |- + Mode controls whether network services maintain private resolver access. + An empty value uses Managed when the Internal DNS feature is enabled. + enum: + - Managed + - Disabled + type: string + type: object ipFamilies: default: - IPv6 diff --git a/config/manager/config.yaml b/config/manager/config.yaml index 5cf7dcde..fe4762d3 100644 --- a/config/manager/config.yaml +++ b/config/manager/config.yaml @@ -1,5 +1,19 @@ apiVersion: apiserver.config.datumapis.com/v1alpha1 kind: NetworkServicesOperator +# Internal DNS is an alpha network integration and remains off until every +# serving cell has matching service attachment configuration. +internalDNS: + # Enables DNS context and regional access lifecycle management. + enabled: false + # Presents the same resolver address inside each consumer VPC. + # frontend: fd53::53 + # Allocates one stable service destination from this prefix per DNS context. + # servicePrefix: fd70:100::/64 + # Bounds each DNS authorization; Galactic accepts at most two minutes. + # accessLeaseDuration: 2m + # Selects the shared resolver attachments that serve authorized VPCs. + # producerAttachmentSelector: + # service.datum.net/internal-dns: "true" gateway: targetDomain: example.com # enableDNSIntegration controls automatic DNSRecordSet creation for Gateway diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index f073056f..d75765d3 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -121,6 +121,8 @@ rules: - dns.networking.miloapis.com resources: - dnsrecordsets + - dnsresolveraccessbindings + - dnsresolvercontexts verbs: - create - delete @@ -187,6 +189,8 @@ rules: - apiGroups: - galactic.datumapis.com resources: + - serviceendpoints + - serviceroutepolicies - vpcattachments - vpcs verbs: @@ -197,6 +201,12 @@ rules: - patch - update - watch +- apiGroups: + - galactic.datumapis.com + resources: + - serviceroutepolicies/status + verbs: + - get - apiGroups: - galactic.datumapis.com resources: diff --git a/docs/api/networkcontexts.md b/docs/api/networkcontexts.md index 835a883f..0989cabb 100644 --- a/docs/api/networkcontexts.md +++ b/docs/api/networkcontexts.md @@ -99,6 +99,14 @@ NetworkContextSpec defines the desired state of NetworkContext The attached network
true + + dns + object + + DNS contains resolver settings whose DNS authorization and private path +have both been verified for this location.
+ + false ipFamilies []enum @@ -191,6 +199,111 @@ The attached network +### NetworkContext.spec.dns +[↩ Parent](#networkcontextspec) + + + +DNS contains resolver settings whose DNS authorization and private path +have both been verified for this location. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
accessBindingRefobject + AccessBindingRef identifies the regional DNS configuration.
+ 		true
contextRefobject + ContextRef identifies the DNS serving and namespace lifetime.
+ 		true
nameservers[]string + Nameservers are resolver addresses reachable from this network.
+
+ Validations:
  • self.all(n, isIP(n)): nameservers must contain IP addresses
    • +     		true
    searches[]string + Searches contains DNS search suffixes allocated to this network.
    +     		false
    + + +### NetworkContext.spec.dns.accessBindingRef +[↩ Parent](#networkcontextspecdns) + + + +AccessBindingRef identifies the regional DNS configuration. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + Name is the resource name in the network's namespace.
    +     		true
    + + +### NetworkContext.spec.dns.contextRef +[↩ Parent](#networkcontextspecdns) + + + +ContextRef identifies the DNS serving and namespace lifetime. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + Name is the resource name in the network's namespace.
    +     		true
    + + ### NetworkContext.status [↩ Parent](#networkcontext) @@ -214,6 +327,14 @@ NetworkContextStatus defines the observed state of NetworkContext Represents the observations of a network context's current state.
    false + + dnsAccess + object + + DNSAccess identifies the current private path proof. DNSReady=True applies +only to this exact context, binding revision, and deadline.
    + + false ipam object @@ -303,6 +424,104 @@ with respect to the current state of the instance.
    +### NetworkContext.status.dnsAccess +[↩ Parent](#networkcontextstatus) + + + +DNSAccess identifies the current private path proof. DNSReady=True applies +only to this exact context, binding revision, and deadline. + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    accessBindingRefobject + AccessBindingRef identifies the DNS configuration used for the proof.
    +     		true
    contextRefobject + ContextRef identifies the DNS context used for the proof.
    +     		true
    serviceDestinationstring + ServiceDestination identifies this DNS context inside the shared service.
    +
    + Validations:
  • isIP(self): serviceDestination must be an IP address
    • +     		true
    + + +### NetworkContext.status.dnsAccess.accessBindingRef +[↩ Parent](#networkcontextstatusdnsaccess) + + + +AccessBindingRef identifies the DNS configuration used for the proof. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + Name is the resource name in the network's namespace.
    +     		true
    + + +### NetworkContext.status.dnsAccess.contextRef +[↩ Parent](#networkcontextstatusdnsaccess) + + + +ContextRef identifies the DNS context used for the proof. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + Name is the resource name in the network's namespace.
    +     		true
    + + ### NetworkContext.status.ipam [↩ Parent](#networkcontextstatus) diff --git a/docs/api/networkinterfaces.md b/docs/api/networkinterfaces.md index 4df452e1..84c5ee2b 100644 --- a/docs/api/networkinterfaces.md +++ b/docs/api/networkinterfaces.md @@ -418,6 +418,13 @@ HolderAvailable means whatever holds the interface reports itself available to serve, and it is the only one of the four a service reads.
    false + + dns + object + + DNS contains effective resolver settings inherited from the network.
    + + false networkContextRef object @@ -571,6 +578,42 @@ with respect to the current state of the instance.
    +### NetworkInterface.status.dns +[↩ Parent](#networkinterfacestatus) + + + +DNS contains effective resolver settings inherited from the network. + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    nameservers[]string +
    +
    + Validations:
  • self.all(n, isIP(n)): nameservers must contain IP addresses
    • +     		true
    searches[]string +
    +     		false
    + + ### NetworkInterface.status.networkContextRef [↩ Parent](#networkinterfacestatus) diff --git a/docs/api/networks.md b/docs/api/networks.md index 1a1c65b8..e79da65e 100644 --- a/docs/api/networks.md +++ b/docs/api/networks.md @@ -92,6 +92,13 @@ NetworkSpec defines the desired state of a Network IPAM settings for the network.
    true + + dns + object + + DNS configures private resolver access for the network.
    + + false ipFamilies []enum @@ -172,6 +179,36 @@ networks created before this rule stay writable.
    +### Network.spec.dns +[↩ Parent](#networkspec) + + + +DNS configures private resolver access for the network. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    modeenum + Mode controls whether network services maintain private resolver access. +An empty value uses Managed when the Internal DNS feature is enabled.
    +
    + Enum: Managed, Disabled
    +     		false
    + + ### Network.status [↩ Parent](#network) diff --git a/go.mod b/go.mod index 2eccabe3..602e3d62 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/spf13/cobra v1.10.2 github.com/stretchr/testify v1.12.1 go.datum.net/datumctl v0.18.1 - go.datum.net/galactic v0.5.8-0.20261016143526-45d3c3514bec + go.datum.net/galactic v0.6.3-0.20261021151056-bb0b4c057338 go.miloapis.com/activity v0.7.1 go.miloapis.com/dns-operator v0.5.1 go.miloapis.com/locations v0.0.1 @@ -272,7 +272,7 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect github.com/tklauser/go-sysconf v0.4.0 // indirect github.com/tklauser/numcpus v0.12.0 // indirect - github.com/vishvananda/netlink v1.3.2-0.20261002201547-1d1c62507e3c // indirect + github.com/vishvananda/netlink v1.3.2-0.20261015184244-3fdd74264153 // indirect github.com/vishvananda/netns v0.0.5 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xhit/go-str2duration/v2 v2.1.0 // indirect @@ -283,7 +283,7 @@ require ( go.etcd.io/etcd/api/v3 v3.6.8 // indirect go.etcd.io/etcd/client/pkg/v3 v3.6.8 // indirect go.etcd.io/etcd/client/v3 v3.6.8 // indirect - go.miloapis.com/ipam v0.4.0 + go.miloapis.com/ipam v0.5.1 go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/exporters/autoexport v0.67.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect @@ -310,7 +310,7 @@ require ( golang.org/x/tools v0.51.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20261005182115-fad411399dd8 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect @@ -327,7 +327,7 @@ require ( oras.land/oras-go/v2 v2.6.0 // indirect periph.io/x/host/v3 v3.8.5 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 // indirect - sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/json v0.0.0-20260909141634-11ed52e25bc5 // indirect sigs.k8s.io/kubectl-validate v0.0.5-0.20250915070809-d2f2d68fba09 // indirect sigs.k8s.io/kustomize/api v0.21.1 // indirect sigs.k8s.io/kustomize/kyaml v0.21.1 // indirect diff --git a/go.sum b/go.sum index c54671b3..7b7f3381 100644 --- a/go.sum +++ b/go.sum @@ -655,8 +655,8 @@ github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqo github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= -github.com/vishvananda/netlink v1.3.2-0.20261002201547-1d1c62507e3c h1:88GLXNKgc6pwIU/4yq+gleF5ISv3AsJkYmFlS1GjHvo= -github.com/vishvananda/netlink v1.3.2-0.20261002201547-1d1c62507e3c/go.mod h1:lEui7SPMd9fgxzHVGRAvTxsBGCF6PRH81o2kLWLWHgw= +github.com/vishvananda/netlink v1.3.2-0.20261015184244-3fdd74264153 h1:h+5tbko9K2pYcDASxNj0GYkEJnWmdtpF0GN7PSuY9oM= +github.com/vishvananda/netlink v1.3.2-0.20261015184244-3fdd74264153/go.mod h1:lEui7SPMd9fgxzHVGRAvTxsBGCF6PRH81o2kLWLWHgw= github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY= github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= @@ -687,8 +687,8 @@ github.com/zonedb/zonedb v1.0.3544 h1:u5a3xOaI338FclecJ/H6J7fiImVEZ/qZomJnVYIlTe github.com/zonedb/zonedb v1.0.3544/go.mod h1:h9mfHV/S6lboOkltULrbNY52cd7JZo6MbxIiqKMWPLg= go.datum.net/datumctl v0.18.1 h1:MP83cRkJ94aq4lYyOm5dSLU60JNnmR486LIeZnaCw8s= go.datum.net/datumctl v0.18.1/go.mod h1:0L0BBgFfud8ZuV9sbsSmeEBUPoK2q9g4boU5yczF92o= -go.datum.net/galactic v0.5.8-0.20261016143526-45d3c3514bec h1:voR/MqaNhUuh9OylFZ3IamgoSkWGsKi+9Rta+PqlHc4= -go.datum.net/galactic v0.5.8-0.20261016143526-45d3c3514bec/go.mod h1:g9KA3v6fQ4fMFBoWxFNA2ABlacKecn2l6ftJQeUdVE4= +go.datum.net/galactic v0.6.3-0.20261021151056-bb0b4c057338 h1:eK8GYG7Z2D9JThUIDuxBGSG3Ur/Jx9F3AL0rVjhdnuk= +go.datum.net/galactic v0.6.3-0.20261021151056-bb0b4c057338/go.mod h1:IJDvHOz3iyXb2eWRxvyH5P5Crudz10mMs4oE3WcMn4Y= go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= go.etcd.io/etcd/api/v3 v3.6.8 h1:gqb1VN92TAI6G2FiBvWcqKtHiIjr4SU2GdXxTwyexbM= @@ -707,8 +707,8 @@ go.miloapis.com/activity v0.7.1 h1:e2c4xVISMA10QQJwPv8pT8U1T1xS3eTGhy8FJHXOKvY= go.miloapis.com/activity v0.7.1/go.mod h1:Sh2Irbq6siJcfq17nLjHvm4JHN/2Csc5YCHB+ycz20c= go.miloapis.com/dns-operator v0.5.1 h1:3HmturI/Opl+l4/KurN6B0/XEeVefoSvYU0ewcZ8BZ8= go.miloapis.com/dns-operator v0.5.1/go.mod h1:yQSBOx1ZhSzZgg5RyNKyNHfZgjXRKhxkcGD+jvbebQE= -go.miloapis.com/ipam v0.4.0 h1:U+mg3RMFXj0c2eQ0Lm15CI0bYBPFDqy8IsGLN8v/eGs= -go.miloapis.com/ipam v0.4.0/go.mod h1:Jj7xg4lJi9psE0+4PuOg/GQOG8rG13h112xYoM994rc= +go.miloapis.com/ipam v0.5.1 h1:jb8jghj0jSOIDEPNo6b2gg2S4SWhNVfW4GDstb0zSKQ= +go.miloapis.com/ipam v0.5.1/go.mod h1:Jj7xg4lJi9psE0+4PuOg/GQOG8rG13h112xYoM994rc= go.miloapis.com/locations v0.0.1 h1:voJKqBzyLX5x96M3+5y/ga7fgq9/+vypTizd+bBvqUY= go.miloapis.com/locations v0.0.1/go.mod h1:gzfAfHhSMwl/N68k/uSNYXOKK3IOBJCdXYaBgCE3gdE= go.miloapis.com/milo v0.36.2 h1:ZXKX6KIQ9O6KF87IVHfXoQ0xmF9MztiP91fFLWasznQ= @@ -864,8 +864,8 @@ gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE= google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:K/n0XLazCJHjD73YuMyNJsN38BPPy1kfFnKxUXaThFU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20261005182115-fad411399dd8 h1:0SXBqli5dnuCKMjI/X3FM226zEaEa0GUcSCJtN1vL2o= -google.golang.org/genproto/googleapis/rpc v0.0.0-20261005182115-fad411399dd8/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b h1:6WNWHZk91dp4gZ25642OALkuG1UVNZbwENJD19DnB8k= +google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= @@ -940,8 +940,8 @@ sigs.k8s.io/gateway-api v1.5.1 h1:RqVRIlkhLhUO8wOHKTLnTJA6o/1un4po4/6M1nRzdd0= sigs.k8s.io/gateway-api v1.5.1/go.mod h1:GvCETiaMAlLym5CovLxGjS0NysqFk3+Yuq3/rh6QL2o= sigs.k8s.io/gateway-api/conformance v1.5.1 h1:5eruSMKcwKnkX42PFek8oO6BgPNBD5FbWbTcRV76KIw= sigs.k8s.io/gateway-api/conformance v1.5.1/go.mod h1:mcvYR0Zll1i5hmcKn+jNbWdZTBls6s5GU+FPUFIceXw= -sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= -sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/json v0.0.0-20260909141634-11ed52e25bc5 h1:k9jo7ED/fsfOcz3L1P20P5tjTNtsdQNwMOG2aKzHrl8= +sigs.k8s.io/json v0.0.0-20260909141634-11ed52e25bc5/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/kubectl-validate v0.0.5-0.20250915070809-d2f2d68fba09 h1:JQbPOwLjSztom+aSDQIi6UZq8V0Gbv7BjAlYQSgycCI= sigs.k8s.io/kubectl-validate v0.0.5-0.20250915070809-d2f2d68fba09/go.mod h1:n5AZlk9qPlp2JHChA4a4N/+lU3bJJ3X1Cig6VGQeqwE= sigs.k8s.io/kustomize/api v0.21.1 h1:lzqbzvz2CSvsjIUZUBNFKtIMsEw7hVLJp0JeSIVmuJs= diff --git a/internal/agent/catalog_test.go b/internal/agent/catalog_test.go index 39505281..09541e37 100644 --- a/internal/agent/catalog_test.go +++ b/internal/agent/catalog_test.go @@ -34,6 +34,7 @@ var inScopeAPIFiles = []string{ var outOfScopeAPIFiles = map[string]string{ "edgereachability_types.go": "hub-side record of which addresses an edge should reach; no customer-facing status", "groupversion_info.go": "scheme registration, no reasons", + "internaldns_types.go": "private resolver access is a network surface, not load balancer status", "locationbinding_types.go": "placement plumbing behind a location, never named by a load balancer", "network_types.go": "the network a service sits on; its own product surface", "networkbinding_types.go": "network attachment plumbing", diff --git a/internal/cmd/cell/cell.go b/internal/cmd/cell/cell.go index b4c490b3..22c55d2a 100644 --- a/internal/cmd/cell/cell.go +++ b/internal/cmd/cell/cell.go @@ -20,6 +20,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + networkv1alpha1 "go.datum.net/galactic/api/network/v1alpha1" ipamv1alpha1 "go.miloapis.com/ipam/pkg/apis/ipam/v1alpha1" locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" @@ -50,6 +51,7 @@ func init() { utilruntime.Must(networkingv1alpha1.AddToScheme(scheme)) utilruntime.Must(ipamv1alpha1.AddToScheme(scheme)) utilruntime.Must(cloudv1alpha1.AddToScheme(scheme)) + utilruntime.Must(networkv1alpha1.AddToScheme(scheme)) utilruntime.Must(nadv1.AddToScheme(scheme)) } @@ -349,6 +351,16 @@ func controllerRegistrations( }).SetupWithManager(mgr) }}) + registrations = append(registrations, namedSetup{"internaldns-cell", func() (bool, error) { + if !serverConfig.InternalDNS.Enabled { + return false, nil + } + return true, (&controller.InternalDNSCellReconciler{ + Config: serverConfig.InternalDNS, + HubCluster: hubCluster, + }).SetupWithManager(mgr) + }}) + return registrations } diff --git a/internal/cmd/manager/manager.go b/internal/cmd/manager/manager.go index c86a3955..a8e49971 100644 --- a/internal/cmd/manager/manager.go +++ b/internal/cmd/manager/manager.go @@ -24,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" @@ -150,13 +151,13 @@ func NewCommand(build BuildInfo) *cobra.Command { &singletonControllersLeaderElection, "singleton-controllers-leader-elect", true, - "Enable leader election for singleton downstream controllers (Challenge and GatewayDownstreamCertificateSolver).", + "Enable leader election for controllers that watch the federation hub.", ) fs.StringVar( &singletonControllersLeaderElectionID, "singleton-controllers-leader-election-id", "", - "Leader election ID for singleton downstream controllers. When empty, it is derived from the enabled controller sets.", + "Leader election ID for controllers that watch the federation hub. When empty, it is derived from the enabled controller sets.", ) opts := zap.Options{} @@ -349,26 +350,29 @@ func NewCommand(build BuildInfo) *cobra.Command { os.Exit(1) } - var singletonMgr manager.Manager - singletonControllerMgr := mgr.GetLocalManager() - if enableClusterSharding { - singletonMgr, err = manager.New(cfg, manager.Options{ - Scheme: scheme, - Metrics: metricsserver.Options{BindAddress: "0"}, - WebhookServer: webhook.NewServer(webhook.Options{Port: 0}), - HealthProbeBindAddress: "0", - LeaderElection: singletonControllersLeaderElection, - LeaderElectionID: singletonControllersLeaderElectionID, - LeaderElectionNamespace: leaderElectionNamespace, - LeaseDuration: &leaseDuration, - RenewDeadline: &renewDeadline, - RetryPeriod: &retryPeriod, - }) - if err != nil { - setupLog.Error(err, "unable to create singleton controller manager") - os.Exit(1) - } - singletonControllerMgr = singletonMgr + // Hub controllers reconcile NetworkBindings and resources published by + // cells. Keep their cache and leader election on the federation API, + // alongside the authoritative copies they watch. + federationConfig, err := federationRESTConfig(serverConfig.Federation, cfg) + if err != nil { + setupLog.Error(err, "unable to load federation kubeconfig") + os.Exit(1) + } + federationMgr, err := manager.New(federationConfig, manager.Options{ + Scheme: scheme, + Metrics: metricsserver.Options{BindAddress: "0"}, + WebhookServer: webhook.NewServer(webhook.Options{Port: 0}), + HealthProbeBindAddress: "0", + LeaderElection: singletonControllersLeaderElection, + LeaderElectionID: singletonControllersLeaderElectionID, + LeaderElectionNamespace: leaderElectionNamespace, + LeaseDuration: &leaseDuration, + RenewDeadline: &renewDeadline, + RetryPeriod: &retryPeriod, + }) + if err != nil { + setupLog.Error(err, "unable to create federation controller manager") + os.Exit(1) } var irohDownstream cluster.Cluster @@ -409,15 +413,21 @@ func NewCommand(build BuildInfo) *cobra.Command { setupLog.Error(err, "unable to build wildcard entitlement client") os.Exit(1) } + projectIdentities, err := newProjectIdentityResolver(serverConfig) + if err != nil { + setupLog.Error(err, "unable to build project identity client") + os.Exit(1) + } registeredControllers, err := setupControllers(mgr, serverConfig, controllerDeps{ downstreamCluster: downstreamCluster, - singletonManager: singletonControllerMgr, + federationManager: federationMgr, irohDownstream: irohDownstream, ipamClients: ipamClients, certificateServiceReader: certificateServiceReader, certificateServiceRoots: certificateServiceRoots, wildcardEntitlements: wildcardEntitlements, + projectIdentities: projectIdentities, }) if err != nil { setupLog.Error(err, "unable to set up controllers") @@ -490,12 +500,10 @@ func NewCommand(build BuildInfo) *cobra.Command { return ignoreCanceled(mgr.Start(ctx)) }) - if singletonMgr != nil { - setupLog.Info("starting singleton controller manager") - g.Go(func() error { - return ignoreCanceled(singletonMgr.Start(ctx)) - }) - } + setupLog.Info("starting federation controller manager") + g.Go(func() error { + return ignoreCanceled(federationMgr.Start(ctx)) + }) if err := g.Wait(); err != nil { setupLog.Error(err, "unable to start") @@ -591,12 +599,35 @@ func setupWebhooks(mgr mcmanager.Manager, serverConfig config.NetworkServicesOpe // against. Fields are only populated for the sets that need them. type controllerDeps struct { downstreamCluster cluster.Cluster - singletonManager manager.Manager + federationManager manager.Manager irohDownstream cluster.Cluster ipamClients controller.IPAMClientFactory certificateServiceReader client.Reader certificateServiceRoots *x509.CertPool wildcardEntitlements controller.WildcardEntitlementChecker + projectIdentities controller.ProjectIdentityResolver +} + +func federationRESTConfig(federation config.FederationConfig, deployment *rest.Config) (*rest.Config, error) { + if federation.KubeconfigPath == "" { + return rest.CopyConfig(deployment), nil + } + return federation.RestConfig() +} + +func newProjectIdentityResolver(serverConfig config.NetworkServicesOperator) (controller.ProjectIdentityResolver, error) { + if !serverConfig.InternalDNS.Enabled { + return nil, nil + } + restConfig, err := serverConfig.Discovery.DiscoveryRestConfig() + if err != nil { + return nil, err + } + reader, err := client.New(restConfig, client.Options{Scheme: scheme}) + if err != nil { + return nil, err + } + return &controller.APIProjectIdentityResolver{Reader: reader}, nil } // newCertificateServiceReader returns an uncached client for the cluster the @@ -680,17 +711,14 @@ func controllerRegistrations( SubnetClass: serverConfig.IPAM.Classes.Subnet, }).SetupWithManager(mgr) }}, - // The deployment cluster is the hub, and the milo provider engages - // project control planes in the same process. The presence controller - // goes on the singleton manager because the sharded managers run three - // replicas with leader election disabled, which would reconcile every - // hub object three times. + // Bindings are authoritative on the federation hub. Project control + // planes are reached through the multicluster manager in the same process. {"networkpresence", true, func() error { return (&controller.NetworkPresenceReconciler{ Projects: controller.NewProjectClusterResolver(mgr), Events: presenceEvents, UnclaimedGracePeriod: serverConfig.NetworkPresence.UnclaimedGracePeriod.Duration, - }).SetupWithManager(deps.singletonManager) + }).SetupWithManager(deps.federationManager) }}, // Everything the presence controller reads besides the binding lives in a // project control plane, which only the multicluster manager watches. The @@ -699,17 +727,26 @@ func controllerRegistrations( {"networkpresencesync", true, func() error { return (&controller.NetworkPresenceSyncReconciler{ Events: presenceEvents, - }).SetupWithManager(mgr, deps.singletonManager) + }).SetupWithManager(mgr, deps.federationManager) + }}, + {"internaldns-project", serverConfig.InternalDNS.Enabled, func() error { + return (&controller.InternalDNSProjectReconciler{ + Config: serverConfig.InternalDNS, + Projects: deps.projectIdentities, + }).SetupWithManager(mgr) + }}, + {"internaldns-readiness", serverConfig.InternalDNS.Enabled, func() error { + return (&controller.InternalDNSReadinessReconciler{ + Config: serverConfig.InternalDNS, + Projects: controller.NewProjectClusterResolver(mgr), + }).SetupWithManager(deps.federationManager) }}, - // The interfaces a cell publishes arrive on the hub, and only the + // The interfaces a cell publishes arrive on the federation hub, and the // multicluster manager reaches the project control planes they are for. - // The projector goes on the singleton manager for the same reason the - // presence controller does: three replicas would project every one of - // them three times. {"networkinterfaceprojector", true, func() error { return (&controller.NetworkInterfaceProjector{ Projects: controller.NewProjectClusterResolver(mgr), - }).SetupWithManager(deps.singletonManager) + }).SetupWithManager(deps.federationManager) }}, // Removing a copy is the other half, and it has to watch project control // planes: a published interface that has gone says nothing about where @@ -781,7 +818,7 @@ func controllerRegistrations( return (&controller.GatewayDownstreamCertificateSolverReconciler{ Config: serverConfig, DownstreamCluster: deps.downstreamCluster, - }).SetupWithManager(deps.singletonManager) + }).SetupWithManager(deps.federationManager) }}, {"domain", true, func() error { return (&controller.DomainReconciler{ @@ -806,7 +843,7 @@ func controllerRegistrations( return (&controller.ChallengeReconciler{ Config: serverConfig, DownstreamCluster: deps.downstreamCluster, - }).SetupWithManager(deps.singletonManager) + }).SetupWithManager(deps.federationManager) }}, } } diff --git a/internal/cmd/manager/manager_test.go b/internal/cmd/manager/manager_test.go index f54a54d5..eb0eef7b 100644 --- a/internal/cmd/manager/manager_test.go +++ b/internal/cmd/manager/manager_test.go @@ -12,12 +12,34 @@ import ( "strings" "testing" + "k8s.io/client-go/rest" + "go.datum.net/network-services-operator/internal/cmd/cell" "go.datum.net/network-services-operator/internal/config" ) var setupWithManagerPattern = regexp.MustCompile(`func \(\w+ \*(\w+)\) SetupWithManager\(`) +func TestFederationRESTConfig(t *testing.T) { + deployment := &rest.Config{Host: "https://deployment.example"} + got, err := federationRESTConfig(config.FederationConfig{}, deployment) + if err != nil || got.Host != deployment.Host || got == deployment { + t.Fatalf("deployment hub configuration: got %v, error %v", got, err) + } + kubeconfig := filepath.Join(t.TempDir(), "hub.yaml") + data := []byte("apiVersion: v1\nkind: Config\nclusters:\n- name: hub\n cluster:\n server: https://hub.example\ncontexts:\n- name: hub\n context:\n cluster: hub\ncurrent-context: hub\n") + if err := os.WriteFile(kubeconfig, data, 0600); err != nil { + t.Fatal(err) + } + got, err = federationRESTConfig(config.FederationConfig{KubeconfigPath: kubeconfig}, deployment) + if err != nil || got.Host != "https://hub.example" { + t.Fatalf("explicit hub configuration: got %v, error %v", got, err) + } + if _, err := federationRESTConfig(config.FederationConfig{KubeconfigPath: kubeconfig + ".missing"}, deployment); err == nil { + t.Fatal("missing federation kubeconfig was accepted") + } +} + var reconcilerControllerNames = map[string]string{ "ChallengeReconciler": "challenge", "ConnectorAdvertisementReconciler": "connectoradvertisement", @@ -30,6 +52,9 @@ var reconcilerControllerNames = map[string]string{ "GatewayReconciler": "gateway", "GatewayResourceReplicatorReconciler": "gateway_resource_replicator", "HTTPProxyReconciler": "httpproxy", + "InternalDNSCellReconciler": "internaldns-cell", + "InternalDNSProjectReconciler": "internaldns-project", + "InternalDNSReadinessReconciler": "internaldns-readiness", "IrohDNSReconciler": "iroh-dns", "NetworkBindingReconciler": "networkbinding", "NetworkContextReconciler": "networkcontext", diff --git a/internal/config/cell.go b/internal/config/cell.go index 20459b4e..8962b79b 100644 --- a/internal/config/cell.go +++ b/internal/config/cell.go @@ -59,6 +59,9 @@ type CellControllerManager struct { // nso_cell_location_identity_source. Location LocationConfig `json:"location,omitempty"` + // InternalDNS configures managed resolver access. It is disabled unless + // explicitly enabled. + InternalDNS InternalDNSConfig `json:"internalDNS,omitempty"` // Galactic enables this cell to reconcile downstream Galactic VPCs and // attachments once the Galactic CRDs are installed. Galactic GalacticCellConfig `json:"galactic,omitempty"` @@ -77,6 +80,9 @@ func (c *CellControllerManager) Validate() error { if err := c.Federation.validate(); err != nil { return fmt.Errorf("federation: %w", err) } + if err := c.InternalDNS.validate(); err != nil { + return fmt.Errorf("internalDNS: %w", err) + } if c.Galactic.Enabled { switch c.Galactic.AttachmentMode { case "Netns", "Hypervisor", "HypervisorDeclared": diff --git a/internal/config/config.go b/internal/config/config.go index 1d8c5091..e5a4af51 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -6,6 +6,7 @@ import ( "crypto/x509" "errors" "fmt" + "net/netip" "os" "path/filepath" "slices" @@ -64,6 +65,10 @@ type NetworkServicesOperator struct { DownstreamResourceManagement DownstreamResourceManagementConfig `json:"downstreamResourceManagement"` + // Federation selects the hub for network bindings and cell observations. + // An empty kubeconfigPath uses the deployment control plane. + Federation FederationConfig `json:"federation,omitempty"` + // Redis provides shared Redis connection settings. Redis RedisConfig `json:"redis"` @@ -108,6 +113,10 @@ type NetworkServicesOperator struct { // NetworkPresence configures how a network's presence in a location is // maintained. NetworkPresence NetworkPresenceConfig `json:"networkPresence,omitempty"` + + // InternalDNS configures managed resolver access. It is disabled unless + // explicitly enabled. + InternalDNS InternalDNSConfig `json:"internalDNS,omitempty"` } // +k8s:deepcopy-gen=true @@ -148,6 +157,50 @@ func (c *NetworkPresenceConfig) validate() error { // +k8s:deepcopy-gen=true +// InternalDNSConfig configures managed resolver access and private service +// programming. +type InternalDNSConfig struct { + // Enabled activates the alpha Internal DNS integration. + Enabled bool `json:"enabled,omitempty"` + + // Frontend is the resolver address presented inside every consumer VPC. + Frontend string `json:"frontend,omitempty"` + + // ServicePrefix allocates a stable service-side destination per DNS context. + ServicePrefix string `json:"servicePrefix,omitempty"` + + // ProducerAttachmentSelector selects shared DNS service attachments. + ProducerAttachmentSelector map[string]string `json:"producerAttachmentSelector,omitempty"` +} + +func SetDefaults_InternalDNSConfig(obj *InternalDNSConfig) { + if obj.Frontend == "" { + obj.Frontend = "fd53::53" + } + if obj.ServicePrefix == "" { + obj.ServicePrefix = "fd70:100::/64" + } + +} + +func (c *InternalDNSConfig) validate() error { + if !c.Enabled { + return nil + } + frontend, err := netip.ParseAddr(c.Frontend) + if err != nil || !frontend.Is6() { + return fmt.Errorf("frontend must be an IPv6 address, got %q", c.Frontend) + } + prefix, err := netip.ParsePrefix(c.ServicePrefix) + if err != nil || !prefix.Addr().Is6() || prefix.Bits() > 64 { + return fmt.Errorf("servicePrefix must be an IPv6 prefix of /64 or larger, got %q", c.ServicePrefix) + } + if len(c.ProducerAttachmentSelector) == 0 { + return errors.New("producerAttachmentSelector is required") + } + return nil +} + // +k8s:deepcopy-gen=true // IPAMConfig describes how the operator reaches the IPAM API server. One @@ -228,12 +281,9 @@ func (c *IPAMConfig) RestConfig() (*rest.Config, error) { // +k8s:deepcopy-gen=true -// FederationConfig names the federation hub a cell publishes to. It is the only -// plane a cell and the control planes serving projects both reach. +// FederationConfig names the hub shared by cells and project controllers. type FederationConfig struct { // KubeconfigPath is the path to a kubeconfig file pointing at the hub. - // Required: a cell that cannot reach the hub cannot show a consumer the - // interface behind their instance. KubeconfigPath string `json:"kubeconfigPath,omitempty"` // Client configures the Kubernetes client connection to the hub. @@ -1667,6 +1717,9 @@ func (c *NetworkServicesOperator) Validate() error { if err := c.NetworkPresence.validate(); err != nil { return fmt.Errorf("networkPresence: %w", err) } + if err := c.InternalDNS.validate(); err != nil { + return fmt.Errorf("internalDNS: %w", err) + } return nil } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 55e09a4e..a1298700 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -14,6 +14,28 @@ import ( "k8s.io/utils/ptr" ) +func TestInternalDNSDefaultsRemainDisabled(t *testing.T) { + cfg := &NetworkServicesOperator{} + SetObjectDefaults_NetworkServicesOperator(cfg) + if cfg.InternalDNS.Enabled { + t.Fatal("internal DNS must require an explicit alpha feature gate") + } + if cfg.InternalDNS.Frontend != "fd53::53" || cfg.InternalDNS.ServicePrefix != "fd70:100::/64" { + t.Fatalf("unexpected internal DNS defaults: %#v", cfg.InternalDNS) + } +} + +func TestInternalDNSValidation(t *testing.T) { + cfg := InternalDNSConfig{ + Enabled: true, Frontend: "fd53::53", ServicePrefix: "fd70:100::/64", + ProducerAttachmentSelector: map[string]string{"service.datum.net/internal-dns": "true"}, + } + if err := cfg.validate(); err != nil { + t.Fatalf("valid config rejected: %v", err) + } + +} + func TestNetworkServicesOperator_Validate_IrohDisabled(t *testing.T) { // When DNSEnabled is false the rest of the iroh config is allowed to // be empty — nothing depends on it. diff --git a/internal/config/zz_generated.deepcopy.go b/internal/config/zz_generated.deepcopy.go index 685c5de8..b639bb7f 100644 --- a/internal/config/zz_generated.deepcopy.go +++ b/internal/config/zz_generated.deepcopy.go @@ -43,6 +43,7 @@ func (in *CellControllerManager) DeepCopyInto(out *CellControllerManager) { out.IPAM = in.IPAM out.Federation = in.Federation out.Location = in.Location + in.InternalDNS.DeepCopyInto(&out.InternalDNS) out.Galactic = in.Galactic } @@ -586,6 +587,28 @@ func (in *IPAMConfig) DeepCopy() *IPAMConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InternalDNSConfig) DeepCopyInto(out *InternalDNSConfig) { + *out = *in + if in.ProducerAttachmentSelector != nil { + in, out := &in.ProducerAttachmentSelector, &out.ProducerAttachmentSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InternalDNSConfig. +func (in *InternalDNSConfig) DeepCopy() *InternalDNSConfig { + if in == nil { + return nil + } + out := new(InternalDNSConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IrohConnectorConfig) DeepCopyInto(out *IrohConnectorConfig) { *out = *in @@ -714,6 +737,7 @@ func (in *NetworkServicesOperator) DeepCopyInto(out *NetworkServicesOperator) { out.Connector = in.Connector out.Discovery = in.Discovery out.DownstreamResourceManagement = in.DownstreamResourceManagement + out.Federation = in.Federation in.Redis.DeepCopyInto(&out.Redis) out.LeaderElection = in.LeaderElection in.DomainVerification.DeepCopyInto(&out.DomainVerification) @@ -724,6 +748,7 @@ func (in *NetworkServicesOperator) DeepCopyInto(out *NetworkServicesOperator) { out.IPAM = in.IPAM out.NetworkInterface = in.NetworkInterface out.NetworkPresence = in.NetworkPresence + in.InternalDNS.DeepCopyInto(&out.InternalDNS) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkServicesOperator. diff --git a/internal/config/zz_generated.defaults.go b/internal/config/zz_generated.defaults.go index f0d42f54..b6340abd 100644 --- a/internal/config/zz_generated.defaults.go +++ b/internal/config/zz_generated.defaults.go @@ -53,6 +53,7 @@ func SetObjectDefaults_CellControllerManager(in *CellControllerManager) { if in.Federation.Client.Burst == 0 { in.Federation.Client.Burst = 100 } + SetDefaults_InternalDNSConfig(&in.InternalDNS) } func SetObjectDefaults_NetworkServicesOperator(in *NetworkServicesOperator) { @@ -317,6 +318,13 @@ func SetObjectDefaults_NetworkServicesOperator(in *NetworkServicesOperator) { in.Connector.Iroh.TTLSeconds = 5 } SetDefaults_DiscoveryConfig(&in.Discovery) + SetDefaults_ClientConnectionConfig(&in.Federation.Client) + if in.Federation.Client.QPS == 0 { + in.Federation.Client.QPS = 50 + } + if in.Federation.Client.Burst == 0 { + in.Federation.Client.Burst = 100 + } if in.Redis.DialTimeout == nil { if err := json.Unmarshal([]byte(`"5s"`), &in.Redis.DialTimeout); err != nil { panic(err) @@ -432,4 +440,5 @@ func SetObjectDefaults_NetworkServicesOperator(in *NetworkServicesOperator) { in.IPAM.Client.Burst = 100 } SetDefaults_NetworkPresenceConfig(&in.NetworkPresence) + SetDefaults_InternalDNSConfig(&in.InternalDNS) } diff --git a/internal/controller/gateway_resource_replicator_controller.go b/internal/controller/gateway_resource_replicator_controller.go index 0a0ed564..1ae3cafb 100644 --- a/internal/controller/gateway_resource_replicator_controller.go +++ b/internal/controller/gateway_resource_replicator_controller.go @@ -79,9 +79,10 @@ type reasonHandler struct { type conditionReasonHandlers map[string]map[string]reasonHandler type replicationResourceConfig struct { - statusGVK *schema.GroupVersionKind - statusTransform statusTransformFunc - conditionHandlers conditionReasonHandlers + statusGVK *schema.GroupVersionKind + statusTransform statusTransformFunc + conditionHandlers conditionReasonHandlers + propagatedAnnotations []string // mirrorStatusToAnnotation, when true, copies the upstream resource's full // .status verbatim into the UpstreamStatusAnnotation on the downstream @@ -197,8 +198,9 @@ func initReplicationResourceConfigs() map[string]replicationResourceConfig { // The Subnet copy also carries the upstream generation: the cells' data plane // reports Programmed against it, and the subnet controller reads that report // off the hub copy itself rather than through a wholesale status sync. + networkContextGVK := schema.GroupVersionKind{Group: groupNetworkingDatumAPIs, Version: versionV1Alpha, Kind: KindNetworkContext} presenceGVKs := []schema.GroupVersionKind{ - {Group: groupNetworkingDatumAPIs, Version: versionV1Alpha, Kind: KindNetworkContext}, + networkContextGVK, {Group: groupNetworkingDatumAPIs, Version: versionV1Alpha, Kind: KindSubnet}, } for _, gvk := range presenceGVKs { @@ -211,6 +213,9 @@ func initReplicationResourceConfigs() map[string]replicationResourceConfig { }, } } + contextConfig := configs[gvkKey(networkContextGVK)] + contextConfig.propagatedAnnotations = internalDNSIntentAnnotations() + configs[gvkKey(networkContextGVK)] = contextConfig // The network itself, mirrored onto the hub so the central allocator of a // network's fabric identity can be driven by the network rather than infer @@ -405,6 +410,7 @@ func (r *GatewayResourceReplicatorReconciler) ensureDownstreamResource( } propagateLabels(upstreamObj, downstreamObj, resource.replicationResourceConfig.propagatedLabels) + propagateAnnotations(upstreamObj, downstreamObj, resource.replicationResourceConfig.propagatedAnnotations) if resource.replicationResourceConfig.stampUpstreamGeneration { setUpstreamGenerationAnnotation(downstreamObj, upstreamObj) @@ -477,6 +483,27 @@ func propagateLabels(upstreamObj, downstreamObj *unstructured.Unstructured, name downstreamObj.SetLabels(downstreamLabels) } +func propagateAnnotations(upstreamObj, downstreamObj *unstructured.Unstructured, names []string) { + if len(names) == 0 { + return + } + + upstreamAnnotations := upstreamObj.GetAnnotations() + downstreamAnnotations := downstreamObj.GetAnnotations() + if downstreamAnnotations == nil { + downstreamAnnotations = map[string]string{} + } + + for _, name := range names { + if value, ok := upstreamAnnotations[name]; ok { + downstreamAnnotations[name] = value + } else { + delete(downstreamAnnotations, name) + } + } + downstreamObj.SetAnnotations(downstreamAnnotations) +} + // setUpstreamGenerationAnnotation stamps the upstream object's generation on the // downstream object. It changes only when the upstream spec does, so it adds no // write of its own. diff --git a/internal/controller/gateway_resource_replicator_controller_test.go b/internal/controller/gateway_resource_replicator_controller_test.go index 7520d8f0..a8d534eb 100644 --- a/internal/controller/gateway_resource_replicator_controller_test.go +++ b/internal/controller/gateway_resource_replicator_controller_test.go @@ -1132,3 +1132,60 @@ func TestNetworkIsReplicatedToTheHubWithoutBeingPlaced(t *testing.T) { assert.False(t, cfg.mirrorStatusToAnnotation, "nothing downstream reads a Network's status") } + +func TestNetworkContextReplicationCarriesOnlyDNSIntentAnnotations(t *testing.T) { + configs := initReplicationResourceConfigs() + gvk := schema.GroupVersionKind{ + Group: groupNetworkingDatumAPIs, Version: versionV1Alpha, Kind: KindNetworkContext, + } + cfg := configs[gvkKey(gvk)] + assert.ElementsMatch(t, internalDNSIntentAnnotations(), cfg.propagatedAnnotations) + + upstream := &unstructured.Unstructured{} + upstream.SetAnnotations(map[string]string{ + InternalDNSContextNameAnnotation: "resolver-a", + "example.test/untrusted": "discard", + }) + downstream := &unstructured.Unstructured{} + downstream.SetAnnotations(map[string]string{ + InternalDNSContextNameAnnotation: "stale", + InternalDNSAccessNameAnnotation: "removed-upstream", + "karmada.io/managed": "preserve", + }) + + propagateAnnotations(upstream, downstream, cfg.propagatedAnnotations) + assert.Equal(t, "resolver-a", downstream.GetAnnotations()[InternalDNSContextNameAnnotation]) + assert.NotContains(t, downstream.GetAnnotations(), InternalDNSAccessNameAnnotation) + assert.NotContains(t, downstream.GetAnnotations(), "example.test/untrusted") + assert.Equal(t, "preserve", downstream.GetAnnotations()["karmada.io/managed"]) +} + +func TestNetworkContextReplicationProducesUsableDNSIntent(t *testing.T) { + gvk := schema.GroupVersionKind{ + Group: groupNetworkingDatumAPIs, Version: versionV1Alpha, Kind: KindNetworkContext, + } + cfg := initReplicationResourceConfigs()[gvkKey(gvk)] + upstream := &unstructured.Unstructured{} + upstream.SetAnnotations(map[string]string{ + InternalDNSRegionAnnotation: "us-central1", + InternalDNSContextNameAnnotation: "resolver-a", + InternalDNSAccessNameAnnotation: "access-a", + InternalDNSServiceDestinationAnnotation: "fd70:100::1", + InternalDNSSuffixAnnotation: "tenant.managed.internal", + }) + downstream := &unstructured.Unstructured{} + propagateAnnotations(upstream, downstream, cfg.propagatedAnnotations) + + annotations := downstream.GetAnnotations() + assert.Equal(t, "us-central1", annotations[InternalDNSRegionAnnotation]) + assert.Equal(t, "fd70:100::1", annotations[InternalDNSServiceDestinationAnnotation]) + settings, err := internalDNSSettingsFromAnnotations(annotations, "fd53::53") + assert.NoError(t, err) + if settings == nil { + t.Fatal("replicated DNS intent did not produce resolver settings") + } + assert.Equal(t, "resolver-a", settings.ContextRef.Name) + assert.Equal(t, "access-a", settings.AccessBindingRef.Name) + assert.Equal(t, []string{"fd53::53"}, settings.Nameservers) + assert.Equal(t, []string{"tenant.managed.internal"}, settings.Searches) +} diff --git a/internal/controller/internaldns_cell_controller.go b/internal/controller/internaldns_cell_controller.go new file mode 100644 index 00000000..1650fdc3 --- /dev/null +++ b/internal/controller/internaldns_cell_controller.go @@ -0,0 +1,375 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "errors" + "fmt" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + + cloudv1alpha1 "go.datum.net/galactic/api/cloud/v1alpha1" + networkv1alpha1 "go.datum.net/galactic/api/network/v1alpha1" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + "go.datum.net/network-services-operator/internal/config" + "go.datum.net/network-services-operator/internal/dnsaccess" +) + +const ( + internalDNSConsumerVPCLabel = "networking.datumapis.com/internal-dns-vpc-uid" +) + +type InternalDNSCellReconciler struct { + Config config.InternalDNSConfig + HubCluster cluster.Cluster + Now func() time.Time + + mgr mcmanager.Manager +} + +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkcontexts;networkinterfaces,verbs=get;list;watch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkcontexts/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=galactic.datumapis.com,resources=vpcs,verbs=get;list;watch +// +kubebuilder:rbac:groups=galactic.datumapis.com,resources=vpcattachments,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=galactic.datumapis.com,resources=serviceendpoints;serviceroutepolicies,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=galactic.datumapis.com,resources=serviceroutepolicies/status,verbs=get + +func (r *InternalDNSCellReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { + cl, err := r.mgr.GetCluster(ctx, req.ClusterName) + if err != nil { + return ctrl.Result{}, err + } + networkContext := &networkingv1alpha.NetworkContext{} + if err := cl.GetClient().Get(ctx, req.NamespacedName, networkContext); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + return r.reconcile(ctx, cl.GetClient(), networkContext) +} + +func (r *InternalDNSCellReconciler) reconcile( + ctx context.Context, + cl client.Client, + networkContext *networkingv1alpha.NetworkContext, +) (ctrl.Result, error) { + settings, err := internalDNSSettingsFromAnnotations(networkContext.Annotations, r.Config.Frontend) + serviceDestination := networkContext.Annotations[InternalDNSServiceDestinationAnnotation] + region := networkContext.Annotations[InternalDNSRegionAnnotation] + if err != nil || serviceDestination == "" || region == "" || + !networkContext.DeletionTimestamp.IsZero() { + if cleanupErr := r.cleanup(ctx, cl, networkContext); cleanupErr != nil { + return ctrl.Result{}, cleanupErr + } + return ctrl.Result{RequeueAfter: internalDNSRetryInterval}, r.report(ctx, networkContext, nil, metav1.Condition{ + Type: networkingv1alpha.NetworkContextDNSReady, Status: metav1.ConditionFalse, + Reason: networkingv1alpha.NetworkContextDNSReadyReasonAccessAbsent, + Message: "No DNS access configuration is available", + }) + } + + vpc, consumerAttachments, err := r.consumerVPC(ctx, cl, networkContext) + if err != nil { + return ctrl.Result{RequeueAfter: internalDNSRetryInterval}, r.report(ctx, networkContext, nil, metav1.Condition{ + Type: networkingv1alpha.NetworkContextDNSReady, Status: metav1.ConditionUnknown, + Reason: networkingv1alpha.NetworkContextDNSReadyReasonNetworkIdentityAbsent, + Message: err.Error(), + }) + } + for _, attachment := range consumerAttachments { + if attachment.Labels[internalDNSConsumerVPCLabel] == string(vpc.UID) { + continue + } + patch := client.MergeFrom(attachment.DeepCopy()) + if attachment.Labels == nil { + attachment.Labels = map[string]string{} + } + attachment.Labels[internalDNSConsumerVPCLabel] = string(vpc.UID) + if err := cl.Patch(ctx, attachment, patch); err != nil { + return ctrl.Result{}, err + } + } + + var attachments cloudv1alpha1.VPCAttachmentList + if err := cl.List(ctx, &attachments); err != nil { + return ctrl.Result{}, err + } + attachmentPointers := make([]*cloudv1alpha1.VPCAttachment, 0, len(attachments.Items)) + for i := range attachments.Items { + attachmentPointers = append(attachmentPointers, &attachments.Items[i]) + } + + for _, protocol := range []networkv1alpha1.NetworkServiceProtocol{ + networkv1alpha1.NetworkServiceProtocolUDP, networkv1alpha1.NetworkServiceProtocolTCP, + } { + endpoint, policy, err := r.ensureServicePath( + ctx, cl, networkContext, vpc, protocol, serviceDestination, region, + ) + if err != nil { + return ctrl.Result{}, err + } + _, ready := policyProgrammed(policy, endpoint, attachmentPointers, vpc.Status.VPC, r.now()) + if !ready { + if err := r.report(ctx, networkContext, nil, metav1.Condition{ + Type: networkingv1alpha.NetworkContextDNSReady, Status: metav1.ConditionUnknown, + Reason: networkingv1alpha.NetworkContextDNSReadyReasonNetworkPathNotReady, + Message: "Galactic has not acknowledged the current DNS service path on every required node", + }); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: internalDNSRetryInterval}, nil + } + } + + access := &networkingv1alpha.NetworkDNSAccessStatus{ + ContextRef: settings.ContextRef, + AccessBindingRef: settings.AccessBindingRef, + ServiceDestination: serviceDestination, + } + if err := r.report(ctx, networkContext, access, metav1.Condition{ + Type: networkingv1alpha.NetworkContextDNSReady, Status: metav1.ConditionTrue, + Reason: networkingv1alpha.NetworkContextDNSReadyReasonReady, + Message: "DNS configuration and Galactic programming are current", + }); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: internalDNSRetryInterval}, nil +} + +func (r *InternalDNSCellReconciler) now() time.Time { + if r.Now != nil { + return r.Now() + } + return time.Now() +} + +func (r *InternalDNSCellReconciler) consumerVPC( + ctx context.Context, + cl client.Client, + networkContext *networkingv1alpha.NetworkContext, +) (*cloudv1alpha1.VPC, []*cloudv1alpha1.VPCAttachment, error) { + var interfaces networkingv1alpha.NetworkInterfaceList + if err := cl.List(ctx, &interfaces, client.InNamespace(networkContext.Namespace)); err != nil { + return nil, nil, err + } + attachments := []*cloudv1alpha1.VPCAttachment{} + var vpc *cloudv1alpha1.VPC + for i := range interfaces.Items { + iface := &interfaces.Items[i] + if iface.Status.NetworkContextRef == nil || iface.Status.NetworkContextRef.Name != networkContext.Name || + iface.Status.AttachmentRef == nil || iface.Status.AttachmentRef.APIGroup != "galactic.datumapis.com" || + iface.Status.AttachmentRef.Kind != "VPCAttachment" || !iface.DeletionTimestamp.IsZero() { + continue + } + attachment := &cloudv1alpha1.VPCAttachment{} + key := client.ObjectKey{Namespace: iface.Namespace, Name: iface.Status.AttachmentRef.Name} + if err := cl.Get(ctx, key, attachment); err != nil { + return nil, nil, fmt.Errorf("read VPC attachment %s: %w", key, err) + } + candidate := &cloudv1alpha1.VPC{} + vpcKey := client.ObjectKey{Namespace: attachment.Namespace, Name: attachment.Spec.VPC.Name} + if err := cl.Get(ctx, vpcKey, candidate); err != nil { + return nil, nil, fmt.Errorf("read consumer VPC %s: %w", vpcKey, err) + } + if candidate.UID == "" || candidate.Status.VPC == "" { + return nil, nil, fmt.Errorf("consumer VPC %s has no current network identity", vpcKey) + } + if vpc != nil && vpc.UID != candidate.UID { + return nil, nil, errors.New("one network context resolves to more than one consumer VPC lifetime") + } + vpc = candidate + attachments = append(attachments, attachment) + } + if vpc == nil || len(attachments) == 0 { + return nil, nil, errors.New("no realized interface identifies the consumer VPC lifetime") + } + return vpc, attachments, nil +} + +func (r *InternalDNSCellReconciler) ensureServicePath( + ctx context.Context, + cl client.Client, + networkContext *networkingv1alpha.NetworkContext, + vpc *cloudv1alpha1.VPC, + protocol networkv1alpha1.NetworkServiceProtocol, + destination string, + region string, +) (*networkv1alpha1.ServiceEndpoint, *networkv1alpha1.ServiceRoutePolicy, error) { + name := boundedDNSName(networkContext.Name+"-dns-"+string(protocol), string(networkContext.UID)+"/"+string(protocol)) + endpoint := &networkv1alpha1.ServiceEndpoint{ObjectMeta: metav1.ObjectMeta{ + Namespace: networkContext.Namespace, Name: name, + }} + _, err := controllerutil.CreateOrUpdate(ctx, cl, endpoint, func() error { + endpoint.Spec = networkv1alpha1.ServiceEndpointSpec{ + ServiceClass: "internal-dns", Address: destination, Port: 53, Protocol: protocol, + Region: region, DeliveryMode: networkv1alpha1.ServiceEndpointDeliveryModePreferNodeLocal, + AttachmentSelector: &metav1.LabelSelector{MatchLabels: r.Config.ProducerAttachmentSelector}, + } + return controllerutil.SetControllerReference(networkContext, endpoint, cl.Scheme()) + }) + if err != nil { + return nil, nil, err + } + + policy := &networkv1alpha1.ServiceRoutePolicy{ObjectMeta: metav1.ObjectMeta{ + Namespace: networkContext.Namespace, Name: name, + }} + _, err = controllerutil.CreateOrUpdate(ctx, cl, policy, func() error { + policy.Spec = networkv1alpha1.ServiceRoutePolicySpec{ + ServiceRef: networkv1alpha1.ServiceEndpointReference{Name: endpoint.Name}, + ConsumerVPCRef: &networkv1alpha1.ServiceRouteVPCReference{Name: vpc.Name, UID: string(vpc.UID)}, + Frontend: &networkv1alpha1.ServiceRouteFrontend{Address: r.Config.Frontend}, + Authorization: &networkv1alpha1.ServiceRouteAuthorization{Persistent: true}, + AttachmentSelector: metav1.LabelSelector{MatchLabels: map[string]string{ + internalDNSConsumerVPCLabel: string(vpc.UID), + }}, + ProtocolPorts: []networkv1alpha1.ServiceRouteProtocolPort{{Protocol: protocol, Port: 53}}, + Region: region, + } + return controllerutil.SetControllerReference(networkContext, policy, cl.Scheme()) + }) + if err != nil { + return nil, nil, err + } + return endpoint, policy, nil +} + +func policyProgrammed( + policy *networkv1alpha1.ServiceRoutePolicy, + endpoint *networkv1alpha1.ServiceEndpoint, + attachments []*cloudv1alpha1.VPCAttachment, + consumerVPCIdentity string, + now time.Time, +) (time.Time, bool) { + if policy.UID == "" || endpoint.UID == "" || policy.Status.ObservedGeneration != policy.Generation || + !apimeta.IsStatusConditionTrue(policy.Status.Conditions, networkv1alpha1.ConditionTypeServiceRouteAccepted) || + policy.Spec.Authorization == nil || !policy.Spec.Authorization.Persistent { + return time.Time{}, false + } + plan, err := dnsaccess.ProgrammingPlan(policy, endpoint, attachments, consumerVPCIdentity) + if err != nil { + return time.Time{}, false + } + validUntil := now.Add(time.Minute) + for _, node := range plan.Nodes { + matched := 0 + for _, report := range policy.Status.Nodes { + if report.NodeName != node { + continue + } + matched++ + if !report.Ready || report.PolicyUID != string(policy.UID) || + report.ObservedGeneration != policy.Generation || report.InputDigest != plan.InputDigest || + !report.ValidUntil.After(now) { + return time.Time{}, false + } + validUntil = minTime(validUntil, report.ValidUntil.Time) + } + if matched != 1 { + return time.Time{}, false + } + } + return validUntil, true +} + +func (r *InternalDNSCellReconciler) report( + ctx context.Context, + networkContext *networkingv1alpha.NetworkContext, + access *networkingv1alpha.NetworkDNSAccessStatus, + condition metav1.Condition, +) error { + hub := r.HubCluster.GetClient() + current := &networkingv1alpha.NetworkContext{} + key := client.ObjectKeyFromObject(networkContext) + if err := hub.Get(ctx, key, current); err != nil { + return client.IgnoreNotFound(err) + } + patch := client.MergeFrom(current.DeepCopy()) + if access != nil || condition.Status != metav1.ConditionUnknown { + current.Status.DNSAccess = access + } + condition.ObservedGeneration = current.Generation + apimeta.SetStatusCondition(¤t.Status.Conditions, condition) + return hub.Status().Patch(ctx, current, patch) +} + +func (r *InternalDNSCellReconciler) cleanup( + ctx context.Context, + cl client.Client, + networkContext *networkingv1alpha.NetworkContext, +) error { + var errs []error + var interfaces networkingv1alpha.NetworkInterfaceList + if err := cl.List(ctx, &interfaces, client.InNamespace(networkContext.Namespace)); err != nil { + errs = append(errs, err) + } else { + for i := range interfaces.Items { + iface := &interfaces.Items[i] + if iface.Status.NetworkContextRef == nil || iface.Status.NetworkContextRef.Name != networkContext.Name || + iface.Status.AttachmentRef == nil || iface.Status.AttachmentRef.APIGroup != "galactic.datumapis.com" || + iface.Status.AttachmentRef.Kind != "VPCAttachment" { + continue + } + attachment := &cloudv1alpha1.VPCAttachment{} + key := client.ObjectKey{Namespace: iface.Namespace, Name: iface.Status.AttachmentRef.Name} + if err := cl.Get(ctx, key, attachment); err != nil { + if !apierrors.IsNotFound(err) { + errs = append(errs, err) + } + continue + } + if _, present := attachment.Labels[internalDNSConsumerVPCLabel]; !present { + continue + } + patch := client.MergeFrom(attachment.DeepCopy()) + delete(attachment.Labels, internalDNSConsumerVPCLabel) + if err := cl.Patch(ctx, attachment, patch); err != nil { + errs = append(errs, err) + } + } + } + for _, protocol := range []string{"udp", "tcp"} { + name := boundedDNSName(networkContext.Name+"-dns-"+protocol, string(networkContext.UID)+"/"+protocol) + policy := &networkv1alpha1.ServiceRoutePolicy{ObjectMeta: metav1.ObjectMeta{ + Namespace: networkContext.Namespace, Name: name, + }} + if err := cl.Delete(ctx, policy); err != nil && !apierrors.IsNotFound(err) { + errs = append(errs, err) + } + endpoint := &networkv1alpha1.ServiceEndpoint{ObjectMeta: metav1.ObjectMeta{ + Namespace: networkContext.Namespace, Name: name, + }} + if err := cl.Delete(ctx, endpoint); err != nil && !apierrors.IsNotFound(err) { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func (r *InternalDNSCellReconciler) SetupWithManager(mgr mcmanager.Manager) error { + if r.HubCluster == nil { + return errors.New("a federation hub cluster is required") + } + r.mgr = mgr + return mcbuilder.ControllerManagedBy(mgr). + For(&networkingv1alpha.NetworkContext{}, mcbuilder.WithEngageWithLocalCluster(false)). + Named("internaldns-cell"). + Complete(r) +} + +func minTime(a, b time.Time) time.Time { + if a.Before(b) { + return a + } + return b +} diff --git a/internal/controller/internaldns_cell_controller_test.go b/internal/controller/internaldns_cell_controller_test.go new file mode 100644 index 00000000..2598003a --- /dev/null +++ b/internal/controller/internaldns_cell_controller_test.go @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + cloudv1alpha1 "go.datum.net/galactic/api/cloud/v1alpha1" + networkv1alpha1 "go.datum.net/galactic/api/network/v1alpha1" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + "go.datum.net/network-services-operator/internal/dnsaccess" +) + +func TestPolicyProgrammedRequiresExactCurrentNodeReports(t *testing.T) { + t.Parallel() + now := time.Date(2026, time.October, 9, 12, 0, 0, 0, time.UTC) + deadline := metav1.NewTime(now.Add(time.Minute)) + consumer := readyAttachment("tenant", "consumer", "consumer-uid", "blue", "vpc-fabric-id", "node-a", "consumer0", map[string]string{ + internalDNSConsumerVPCLabel: "vpc-uid", + }) + producer := readyAttachment("tenant", "producer", "producer-uid", "dns-service", "service-vpc-id", "node-b", "producer0", map[string]string{ + "service.datum.net/internal-dns": "true", + }) + endpoint := &networkv1alpha1.ServiceEndpoint{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "dns", UID: "endpoint-uid", Generation: 2}, + Spec: networkv1alpha1.ServiceEndpointSpec{ + ServiceClass: "internal-dns", Address: "fd70:100::10", Port: 53, + Protocol: networkv1alpha1.NetworkServiceProtocolUDP, + DeliveryMode: networkv1alpha1.ServiceEndpointDeliveryModePreferNodeLocal, + AttachmentSelector: &metav1.LabelSelector{MatchLabels: map[string]string{ + "service.datum.net/internal-dns": "true", + }}, + }, + } + policy := &networkv1alpha1.ServiceRoutePolicy{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "dns", UID: "policy-uid", Generation: 3}, + Spec: networkv1alpha1.ServiceRoutePolicySpec{ + ServiceRef: networkv1alpha1.ServiceEndpointReference{Name: endpoint.Name}, + ConsumerVPCRef: &networkv1alpha1.ServiceRouteVPCReference{Name: "blue", UID: "vpc-uid"}, + Frontend: &networkv1alpha1.ServiceRouteFrontend{Address: "fd53::53"}, + Authorization: &networkv1alpha1.ServiceRouteAuthorization{Persistent: true}, + AttachmentSelector: metav1.LabelSelector{MatchLabels: map[string]string{ + internalDNSConsumerVPCLabel: "vpc-uid", + }}, + ProtocolPorts: []networkv1alpha1.ServiceRouteProtocolPort{{ + Protocol: networkv1alpha1.NetworkServiceProtocolUDP, Port: 53, + }}, + }, + Status: networkv1alpha1.ServiceRoutePolicyStatus{ + ObservedGeneration: 3, + Conditions: []metav1.Condition{{ + Type: networkv1alpha1.ConditionTypeServiceRouteAccepted, Status: metav1.ConditionTrue, + Reason: "Accepted", + }}, + }, + } + attachments := []*cloudv1alpha1.VPCAttachment{consumer, producer} + plan, err := dnsaccess.ProgrammingPlan(policy, endpoint, attachments, "vpc-fabric-id") + if err != nil { + t.Fatal(err) + } + for _, node := range plan.Nodes { + policy.Status.Nodes = append(policy.Status.Nodes, networkv1alpha1.ServiceRouteNodeStatus{ + NodeName: node, PolicyUID: string(policy.UID), ObservedGeneration: policy.Generation, + InputDigest: plan.InputDigest, Ready: true, ValidUntil: deadline, + }) + } + if until, ready := policyProgrammed(policy, endpoint, attachments, "vpc-fabric-id", now); !ready || !until.Equal(deadline.Time) { + t.Fatalf("current exact reports not accepted: ready=%v until=%v", ready, until) + } + + policy.Status.Nodes[0].InputDigest = "stale" + if _, ready := policyProgrammed(policy, endpoint, attachments, "vpc-fabric-id", now); ready { + t.Fatal("stale input digest was accepted") + } + policy.Status.Nodes[0].InputDigest = plan.InputDigest + policy.Status.Nodes[0].ValidUntil = metav1.NewTime(now.Add(-time.Second)) + if _, ready := policyProgrammed(policy, endpoint, attachments, "vpc-fabric-id", now); ready { + t.Fatal("expired node report was accepted") + } +} + +func TestConsumerVPCRejectsMultipleVPCLifetimes(t *testing.T) { + t.Parallel() + scheme := runtime.NewScheme() + if err := networkingv1alpha.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := cloudv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + contextObject := testNetworkContext("context", "context-uid") + firstInterface, firstAttachment, firstVPC := consumerPath("one", contextObject.Name, "vpc-one", "vpc-one-uid", "fabric-one") + secondInterface, secondAttachment, secondVPC := consumerPath("two", contextObject.Name, "vpc-two", "vpc-two-uid", "fabric-two") + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects( + firstInterface, firstAttachment, firstVPC, + secondInterface, secondAttachment, secondVPC, + ).Build() + r := &InternalDNSCellReconciler{} + if _, _, err := r.consumerVPC(context.Background(), cl, contextObject); err == nil { + t.Fatal("multiple VPC lifetimes were accepted for one network context") + } +} + +func readyAttachment( + namespace, name, uid, vpcName, vpcIdentity, node, hostInterface string, + labels map[string]string, +) *cloudv1alpha1.VPCAttachment { + return &cloudv1alpha1.VPCAttachment{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name, UID: types.UID(uid), Generation: 1, Labels: labels}, + Spec: cloudv1alpha1.VPCAttachmentSpec{VPC: cloudv1alpha1.VPCRef{Name: vpcName}}, + Status: cloudv1alpha1.VPCAttachmentStatus{ + ObservedGeneration: 1, VPC: vpcIdentity, VPCAttachment: name + "-fabric", + Node: node, HostInterface: hostInterface, + Conditions: []metav1.Condition{ + {Type: cloudv1alpha1.ConditionTypeReady, Status: metav1.ConditionTrue, Reason: "Ready"}, + {Type: cloudv1alpha1.ConditionTypeProgrammed, Status: metav1.ConditionTrue, Reason: "Programmed"}, + }, + }, + } +} + +func consumerPath( + name, contextName, vpcName string, + vpcUID types.UID, + vpcIdentity string, +) (*networkingv1alpha.NetworkInterface, *cloudv1alpha1.VPCAttachment, *cloudv1alpha1.VPC) { + iface := &networkingv1alpha.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "interface-" + name}, + Status: networkingv1alpha.NetworkInterfaceStatus{ + NetworkContextRef: &networkingv1alpha.LocalNetworkContextRef{Name: contextName}, + AttachmentRef: &networkingv1alpha.NetworkInterfaceAttachmentRef{ + APIGroup: "galactic.datumapis.com", Kind: "VPCAttachment", Name: "attachment-" + name, + }, + }, + } + attachment := &cloudv1alpha1.VPCAttachment{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "attachment-" + name}, + Spec: cloudv1alpha1.VPCAttachmentSpec{VPC: cloudv1alpha1.VPCRef{Name: vpcName}}, + } + vpc := &cloudv1alpha1.VPC{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: vpcName, UID: vpcUID}, + Status: cloudv1alpha1.VPCStatus{VPC: vpcIdentity}, + } + return iface, attachment, vpc +} diff --git a/internal/controller/internaldns_controller.go b/internal/controller/internaldns_controller.go new file mode 100644 index 00000000..c79d1a27 --- /dev/null +++ b/internal/controller/internaldns_controller.go @@ -0,0 +1,660 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/netip" + "strings" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/manager" + mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" + mccontext "sigs.k8s.io/multicluster-runtime/pkg/context" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + "go.datum.net/network-services-operator/internal/config" +) + +const ( + internalDNSFinalizer = "networking.datumapis.com/internal-dns-access" + + InternalDNSContextNameAnnotation = "networking.datumapis.com/internal-dns-context-name" + InternalDNSAccessNameAnnotation = "networking.datumapis.com/internal-dns-access-name" + InternalDNSServiceDestinationAnnotation = "networking.datumapis.com/internal-dns-service-destination" + InternalDNSSuffixAnnotation = "networking.datumapis.com/internal-dns-suffix" + InternalDNSRegionAnnotation = "networking.datumapis.com/internal-dns-region" + + internalDNSOwnerNetworkUIDAnnotation = "networking.datumapis.com/internal-dns-network-uid" + internalDNSOwnerProjectUIDAnnotation = "networking.datumapis.com/internal-dns-project-uid" + + internalDNSRegionTopologyKey = "topology.datum.net/region" + internalDNSRetryInterval = 10 * time.Second + internalDNSDefaultDomain = "datum.internal" +) + +var ( + internalDNSContextGVK = schema.GroupVersionKind{ + Group: "dns.networking.miloapis.com", Version: "v1alpha1", Kind: "DNSResolverContext", + } + internalDNSAccessGVK = schema.GroupVersionKind{ + Group: "dns.networking.miloapis.com", Version: "v1alpha1", Kind: "DNSResolverAccessBinding", + } +) + +type ProjectIdentityResolver interface { + ProjectUID(context.Context, string) (types.UID, error) +} + +type APIProjectIdentityResolver struct { + Reader client.Reader +} + +func (r *APIProjectIdentityResolver) ProjectUID(ctx context.Context, name string) (types.UID, error) { + project := &unstructured.Unstructured{} + project.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "resourcemanager.miloapis.com", Version: "v1alpha1", Kind: "Project", + }) + if err := r.Reader.Get(ctx, client.ObjectKey{Name: name}, project); err != nil { + return "", err + } + if project.GetUID() == "" { + return "", fmt.Errorf("project %q has no UID", name) + } + return project.GetUID(), nil +} + +type InternalDNSProjectReconciler struct { + Config config.InternalDNSConfig + Projects ProjectIdentityResolver + Now func() time.Time + + mgr mcmanager.Manager +} + +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkcontexts,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkcontexts/finalizers,verbs=update +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networks,verbs=get;list;watch +// +kubebuilder:rbac:groups=locations.miloapis.com,resources=locations,verbs=get;list;watch +// +kubebuilder:rbac:groups=dns.networking.miloapis.com,resources=dnsresolvercontexts;dnsresolveraccessbindings,verbs=get;list;watch;create;update;patch;delete + +func (r *InternalDNSProjectReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { + cl, err := r.mgr.GetCluster(ctx, req.ClusterName) + if err != nil { + return ctrl.Result{}, err + } + ctx = mccontext.WithCluster(ctx, req.ClusterName) + return r.reconcile(ctx, cl.GetClient(), string(req.ClusterName), req.NamespacedName) +} + +func (r *InternalDNSProjectReconciler) reconcile( + ctx context.Context, + cl client.Client, + project string, + key client.ObjectKey, +) (ctrl.Result, error) { + networkContext := &networkingv1alpha.NetworkContext{} + if err := cl.Get(ctx, key, networkContext); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + region, err := r.region(ctx, cl, networkContext) + if err != nil { + return ctrl.Result{}, err + } + + if !networkContext.DeletionTimestamp.IsZero() { + if controllerutil.ContainsFinalizer(networkContext, internalDNSFinalizer) { + if err := r.cleanupRegion(ctx, cl, networkContext); err != nil { + return ctrl.Result{}, err + } + controllerutil.RemoveFinalizer(networkContext, internalDNSFinalizer) + if err := cl.Update(ctx, networkContext); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil + } + previousRegion := networkContext.Annotations[InternalDNSRegionAnnotation] + if previousRegion != "" && previousRegion != region { + if err := r.cleanupRegion(ctx, cl, networkContext); err != nil { + return ctrl.Result{}, err + } + } + if previousRegion != region { + patch := client.MergeFrom(networkContext.DeepCopy()) + if networkContext.Annotations == nil { + networkContext.Annotations = map[string]string{} + } + if region == "" { + delete(networkContext.Annotations, InternalDNSRegionAnnotation) + } else { + networkContext.Annotations[InternalDNSRegionAnnotation] = region + } + if err := cl.Patch(ctx, networkContext, patch); err != nil { + return ctrl.Result{}, err + } + } + + network := &networkingv1alpha.Network{} + if err := cl.Get(ctx, client.ObjectKey{Namespace: key.Namespace, Name: networkContext.Spec.Network.Name}, network); err != nil { + return ctrl.Result{}, err + } + if network.Spec.DNS.Mode == networkingv1alpha.NetworkDNSModeDisabled { + return ctrl.Result{}, r.disable(ctx, cl, networkContext, network) + } + if region == "" { + return ctrl.Result{RequeueAfter: internalDNSRetryInterval}, nil + } + + if controllerutil.AddFinalizer(networkContext, internalDNSFinalizer) { + if err := cl.Update(ctx, networkContext); err != nil { + return ctrl.Result{}, err + } + } + + projectUID, err := r.Projects.ProjectUID(ctx, project) + if err != nil { + return ctrl.Result{}, fmt.Errorf("resolve project identity: %w", err) + } + consumerID := string(projectUID) + "/" + string(network.UID) + dnsContext, err := r.ensureContext(ctx, cl, network, projectUID, consumerID) + if err != nil { + return ctrl.Result{}, err + } + suffix, _, _ := unstructured.NestedString(dnsContext.Object, "status", "managedNamespace", "suffix") + zoneName, _, _ := unstructured.NestedString(dnsContext.Object, "status", "managedNamespace", "dnsZoneRef", "name") + if dnsContext.GetUID() == "" || suffix != internalDNSDefaultDomain || zoneName == "" || !resolverContextReady(dnsContext) { + return ctrl.Result{RequeueAfter: internalDNSRetryInterval}, nil + } + + destination, err := internalDNSDestination(r.Config.ServicePrefix, dnsContext.GetUID()) + if err != nil { + return ctrl.Result{}, err + } + binding, ready, err := r.ensureAccess(ctx, cl, network, projectUID, dnsContext, region, destination) + if err != nil { + return ctrl.Result{}, err + } + if !ready { + return ctrl.Result{RequeueAfter: internalDNSRetryInterval}, nil + } + + if err := r.publishIntent(ctx, cl, networkContext, map[string]string{ + InternalDNSContextNameAnnotation: dnsContext.GetName(), + InternalDNSAccessNameAnnotation: binding.GetName(), + InternalDNSServiceDestinationAnnotation: destination, + InternalDNSSuffixAnnotation: suffix, + InternalDNSRegionAnnotation: region, + }); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{RequeueAfter: internalDNSRetryInterval}, nil +} + +func (r *InternalDNSProjectReconciler) region( + ctx context.Context, + cl client.Client, + networkContext *networkingv1alpha.NetworkContext, +) (string, error) { + location := &locationsv1alpha1.Location{} + if err := cl.Get(ctx, client.ObjectKey{Name: networkContext.Spec.Location.Name}, location); err != nil { + return "", client.IgnoreNotFound(err) + } + return location.Spec.Topology[internalDNSRegionTopologyKey], nil +} + +func (r *InternalDNSProjectReconciler) ensureContext( + ctx context.Context, + cl client.Client, + network *networkingv1alpha.Network, + projectUID types.UID, + consumerID string, +) (*unstructured.Unstructured, error) { + name := internalDNSResourceName(network.Name, network.UID) + current := newInternalDNSObject(internalDNSContextGVK, network.Namespace, name) + err := cl.Get(ctx, client.ObjectKeyFromObject(current), current) + if err != nil && !apierrors.IsNotFound(err) { + return nil, err + } + if err == nil { + ownedNetwork := current.GetAnnotations()[internalDNSOwnerNetworkUIDAnnotation] + existingConsumer, _, _ := unstructured.NestedString(current.Object, "spec", "consumerID") + if ownedNetwork != string(network.UID) || existingConsumer != consumerID { + return nil, fmt.Errorf("DNSResolverContext %s/%s belongs to another network lifetime", network.Namespace, name) + } + domain, _, _ := unstructured.NestedString(current.Object, "spec", "managedNamespace", "domainSuffix") + enabled, _, _ := unstructured.NestedBool(current.Object, "spec", "managedNamespace", "enabled") + if domain != internalDNSDefaultDomain || !enabled { + return nil, fmt.Errorf("DNSResolverContext %s/%s does not request the network default domain", network.Namespace, name) + } + return current, nil + } + + desired := newInternalDNSObject(internalDNSContextGVK, network.Namespace, name) + desired.SetAnnotations(map[string]string{ + internalDNSOwnerNetworkUIDAnnotation: string(network.UID), + internalDNSOwnerProjectUIDAnnotation: string(projectUID), + }) + desired.SetOwnerReferences([]metav1.OwnerReference{{ + APIVersion: networkingv1alpha.GroupVersion.String(), Kind: "Network", Name: network.Name, + UID: network.UID, Controller: boolPtr(true), BlockOwnerDeletion: boolPtr(true), + }}) + desired.Object["spec"] = map[string]any{ + "consumerID": consumerID, + "managedNamespace": map[string]any{"enabled": true, "domainSuffix": internalDNSDefaultDomain}, + } + if err := cl.Create(ctx, desired); err != nil { + return nil, err + } + return desired, nil +} + +func resolverContextReady(obj *unstructured.Unstructured) bool { + if !obj.GetDeletionTimestamp().IsZero() || obj.GetGeneration() < 1 { + return false + } + conditions, _, _ := unstructured.NestedSlice(obj.Object, "status", "conditions") + for _, raw := range conditions { + condition, _ := raw.(map[string]any) + if condition["type"] == "Ready" && condition["status"] == "True" && condition["observedGeneration"] == obj.GetGeneration() { + return true + } + } + return false +} + +func (r *InternalDNSProjectReconciler) ensureAccess( + ctx context.Context, + cl client.Client, + network *networkingv1alpha.Network, + projectUID types.UID, + dnsContext *unstructured.Unstructured, + region string, + destination string, +) (*unstructured.Unstructured, bool, error) { + name := internalDNSRegionalName(dnsContext.GetName(), region) + current := newInternalDNSObject(internalDNSAccessGVK, network.Namespace, name) + err := cl.Get(ctx, client.ObjectKeyFromObject(current), current) + if err != nil && !apierrors.IsNotFound(err) { + return nil, false, err + } + if err == nil { + if current.GetAnnotations()[internalDNSOwnerNetworkUIDAnnotation] != string(network.UID) { + return nil, false, fmt.Errorf("DNSResolverAccessBinding %s/%s belongs to another network lifetime", network.Namespace, name) + } + if !current.GetDeletionTimestamp().IsZero() { + return current, false, nil + } + if accessBindingMatches(current, dnsContext, region, destination) { + return current, bindingReady(current), nil + } + uid, resourceVersion := current.GetUID(), current.GetResourceVersion() + return current, false, client.IgnoreNotFound(cl.Delete(ctx, current, client.Preconditions{UID: &uid, ResourceVersion: &resourceVersion})) + } + current.Object["spec"] = map[string]any{ + "contextRef": map[string]any{"name": dnsContext.GetName(), "uid": string(dnsContext.GetUID())}, + "region": region, + "destinationAddress": destination, + } + current.SetAnnotations(map[string]string{ + internalDNSOwnerNetworkUIDAnnotation: string(network.UID), + internalDNSOwnerProjectUIDAnnotation: string(projectUID), + InternalDNSRegionAnnotation: region, + }) + current.SetOwnerReferences([]metav1.OwnerReference{{ + APIVersion: internalDNSContextGVK.GroupVersion().String(), Kind: internalDNSContextGVK.Kind, + Name: dnsContext.GetName(), UID: dnsContext.GetUID(), Controller: boolPtr(true), BlockOwnerDeletion: boolPtr(true), + }}) + if err := cl.Create(ctx, current); err != nil { + return nil, false, err + } + return current, false, nil +} + +func accessBindingMatches(binding *unstructured.Unstructured, dnsContext *unstructured.Unstructured, region, destination string) bool { + contextName, _, _ := unstructured.NestedString(binding.Object, "spec", "contextRef", "name") + contextUID, _, _ := unstructured.NestedString(binding.Object, "spec", "contextRef", "uid") + boundRegion, _, _ := unstructured.NestedString(binding.Object, "spec", "region") + boundDestination, _, _ := unstructured.NestedString(binding.Object, "spec", "destinationAddress") + return contextName == dnsContext.GetName() && contextUID == string(dnsContext.GetUID()) && boundRegion == region && boundDestination == destination +} + +func (r *InternalDNSProjectReconciler) disable( + ctx context.Context, + cl client.Client, + networkContext *networkingv1alpha.NetworkContext, + network *networkingv1alpha.Network, +) error { + var errs []error + if err := r.cleanupRegion(ctx, cl, networkContext); err != nil { + errs = append(errs, err) + } + contextObj := newInternalDNSObject(internalDNSContextGVK, network.Namespace, internalDNSResourceName(network.Name, network.UID)) + if err := cl.Delete(ctx, contextObj); err != nil && !apierrors.IsNotFound(err) { + errs = append(errs, err) + } + if err := r.clearIntent(ctx, cl, networkContext); err != nil { + errs = append(errs, err) + } + if controllerutil.RemoveFinalizer(networkContext, internalDNSFinalizer) { + if err := cl.Update(ctx, networkContext); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func (r *InternalDNSProjectReconciler) cleanupRegion( + ctx context.Context, + cl client.Client, + networkContext *networkingv1alpha.NetworkContext, +) error { + region := networkContext.Annotations[InternalDNSRegionAnnotation] + contextName := networkContext.Annotations[InternalDNSContextNameAnnotation] + if region == "" || contextName == "" { + return nil + } + var contexts networkingv1alpha.NetworkContextList + if err := cl.List(ctx, &contexts, client.InNamespace(networkContext.Namespace), client.MatchingLabels{ + networkingv1alpha.NetworkUIDLabel: networkContext.Labels[networkingv1alpha.NetworkUIDLabel], + }); err != nil { + return err + } + for i := range contexts.Items { + other := &contexts.Items[i] + if other.UID == networkContext.UID || !other.DeletionTimestamp.IsZero() { + continue + } + otherRegion := other.Annotations[InternalDNSRegionAnnotation] + if otherRegion == "" { + var err error + otherRegion, err = r.region(ctx, cl, other) + if err != nil { + return err + } + } + if otherRegion == region { + return nil + } + } + binding := newInternalDNSObject(internalDNSAccessGVK, networkContext.Namespace, internalDNSRegionalName(contextName, region)) + if err := cl.Delete(ctx, binding); err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil +} + +func (r *InternalDNSProjectReconciler) publishIntent( + ctx context.Context, + cl client.Client, + networkContext *networkingv1alpha.NetworkContext, + values map[string]string, +) error { + patch := client.MergeFrom(networkContext.DeepCopy()) + if networkContext.Annotations == nil { + networkContext.Annotations = map[string]string{} + } + changed := false + for key, value := range values { + if networkContext.Annotations[key] != value { + networkContext.Annotations[key] = value + changed = true + } + } + if !changed { + return nil + } + return cl.Patch(ctx, networkContext, patch) +} + +func (r *InternalDNSProjectReconciler) clearIntent( + ctx context.Context, + cl client.Client, + networkContext *networkingv1alpha.NetworkContext, +) error { + patch := client.MergeFrom(networkContext.DeepCopy()) + changed := false + for _, key := range internalDNSIntentAnnotations() { + if _, ok := networkContext.Annotations[key]; ok { + delete(networkContext.Annotations, key) + changed = true + } + } + if networkContext.Spec.DNS != nil { + networkContext.Spec.DNS = nil + changed = true + } + if !changed { + return nil + } + return cl.Patch(ctx, networkContext, patch) +} + +func (r *InternalDNSProjectReconciler) SetupWithManager(mgr mcmanager.Manager) error { + if r.Projects == nil { + return errors.New("a project identity resolver is required") + } + r.mgr = mgr + return mcbuilder.ControllerManagedBy(mgr). + For(&networkingv1alpha.NetworkContext{}, mcbuilder.WithEngageWithLocalCluster(false)). + Named("internaldns-project"). + Complete(r) +} + +type InternalDNSReadinessReconciler struct { + Config config.InternalDNSConfig + Projects ProjectClusterResolver + Now func() time.Time + + hub client.Client +} + +func (r *InternalDNSReadinessReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + hubContext := &networkingv1alpha.NetworkContext{} + if err := r.hub.Get(ctx, req.NamespacedName, hubContext); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + routing, err := resolveProjectRouting(ctx, r.hub, req.Namespace) + if err != nil { + var unresolved *projectUnresolvable + if errors.As(err, &unresolved) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + projectCluster, err := r.Projects.ClusterForProject(ctx, routing.project) + if err != nil { + return ctrl.Result{}, err + } + projectClient := projectCluster.GetClient() + projectContext := &networkingv1alpha.NetworkContext{} + key := client.ObjectKey{Namespace: routing.projectNamespace, Name: hubContext.Name} + if err := projectClient.Get(ctx, key, projectContext); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + return r.publish(ctx, projectClient, projectContext, hubContext) +} + +func (r *InternalDNSReadinessReconciler) publish( + ctx context.Context, + cl client.Client, + projectContext *networkingv1alpha.NetworkContext, + hubContext *networkingv1alpha.NetworkContext, +) (ctrl.Result, error) { + intent, err := internalDNSSettingsFromAnnotations(projectContext.Annotations, r.Config.Frontend) + if err != nil { + return ctrl.Result{}, r.updateProject(ctx, cl, projectContext, nil, metav1.Condition{ + Type: networkingv1alpha.NetworkContextDNSReady, Status: metav1.ConditionFalse, + Reason: networkingv1alpha.NetworkContextDNSReadyReasonPending, Message: err.Error(), + }) + } + + access := hubContext.Status.DNSAccess + ready := apimeta.IsStatusConditionTrue(hubContext.Status.Conditions, networkingv1alpha.NetworkContextDNSReady) + if !ready || !dnsAccessMatches(intent, access) || access == nil || access.ServiceDestination != projectContext.Annotations[InternalDNSServiceDestinationAnnotation] { + settings := projectContext.Spec.DNS + if !equalDNSSettings(settings, intent) { + settings = nil + } + err := r.updateProject(ctx, cl, projectContext, settings, metav1.Condition{ + Type: networkingv1alpha.NetworkContextDNSReady, Status: metav1.ConditionUnknown, + Reason: networkingv1alpha.NetworkContextDNSReadyReasonNetworkPathNotReady, + Message: "Current network programming health is unavailable", + }) + return ctrl.Result{RequeueAfter: internalDNSRetryInterval}, err + } + err = r.updateProject(ctx, cl, projectContext, intent, metav1.Condition{ + Type: networkingv1alpha.NetworkContextDNSReady, Status: metav1.ConditionTrue, + Reason: networkingv1alpha.NetworkContextDNSReadyReasonReady, + Message: "DNS configuration and private network programming are current", + }) + return ctrl.Result{RequeueAfter: internalDNSRetryInterval}, err +} + +func (r *InternalDNSReadinessReconciler) updateProject( + ctx context.Context, + cl client.Client, + networkContext *networkingv1alpha.NetworkContext, + settings *networkingv1alpha.NetworkDNSSettings, + condition metav1.Condition, +) error { + var errs []error + if !equalDNSSettings(networkContext.Spec.DNS, settings) { + patch := client.MergeFrom(networkContext.DeepCopy()) + networkContext.Spec.DNS = settings + if err := cl.Patch(ctx, networkContext, patch); err != nil { + return err + } + } + condition.ObservedGeneration = networkContext.Generation + if apimeta.SetStatusCondition(&networkContext.Status.Conditions, condition) { + if err := cl.Status().Update(ctx, networkContext); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func (r *InternalDNSReadinessReconciler) SetupWithManager(mgr manager.Manager) error { + if r.Projects == nil { + return errors.New("a project cluster resolver is required") + } + r.hub = mgr.GetClient() + return ctrl.NewControllerManagedBy(mgr). + For(&networkingv1alpha.NetworkContext{}). + Named("internaldns-readiness"). + Complete(r) +} + +func internalDNSSettingsFromAnnotations( + annotations map[string]string, + frontend string, +) (*networkingv1alpha.NetworkDNSSettings, error) { + contextName := annotations[InternalDNSContextNameAnnotation] + accessName := annotations[InternalDNSAccessNameAnnotation] + if contextName == "" || accessName == "" { + return nil, errors.New("DNS access intent is incomplete") + } + settings := &networkingv1alpha.NetworkDNSSettings{ + ContextRef: networkingv1alpha.NetworkDNSObjectReference{Name: contextName}, + AccessBindingRef: networkingv1alpha.NetworkDNSObjectReference{Name: accessName}, + Nameservers: []string{frontend}, + } + if suffix := annotations[InternalDNSSuffixAnnotation]; suffix != "" { + settings.Searches = []string{suffix} + } + return settings, nil +} + +func dnsAccessMatches(settings *networkingv1alpha.NetworkDNSSettings, access *networkingv1alpha.NetworkDNSAccessStatus) bool { + return settings != nil && access != nil && + settings.ContextRef == access.ContextRef && + settings.AccessBindingRef == access.AccessBindingRef +} + +func equalDNSSettings(a, b *networkingv1alpha.NetworkDNSSettings) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return a.ContextRef == b.ContextRef && a.AccessBindingRef == b.AccessBindingRef && + strings.Join(a.Nameservers, "\x00") == strings.Join(b.Nameservers, "\x00") && + strings.Join(a.Searches, "\x00") == strings.Join(b.Searches, "\x00") +} + +func internalDNSDestination(prefixText string, uid types.UID) (string, error) { + prefix, err := netip.ParsePrefix(prefixText) + if err != nil { + return "", err + } + bytes := prefix.Masked().Addr().As16() + digest := sha256.Sum256([]byte(uid)) + copy(bytes[8:], digest[:8]) + address := netip.AddrFrom16(bytes) + if !prefix.Contains(address) { + return "", fmt.Errorf("allocated DNS destination %s falls outside %s", address, prefix) + } + return address.String(), nil +} + +func internalDNSResourceName(network string, uid types.UID) string { + return boundedDNSName("dns-"+network, string(uid)) +} + +func internalDNSRegionalName(contextName, region string) string { + return boundedDNSName(contextName+"-"+region, contextName+"/"+region) +} + +func boundedDNSName(prefix, identity string) string { + sum := sha256.Sum256([]byte(identity)) + suffix := hex.EncodeToString(sum[:6]) + maxPrefix := 253 - len(suffix) - 1 + prefix = strings.Trim(prefix, "-") + if len(prefix) > maxPrefix { + prefix = strings.TrimRight(prefix[:maxPrefix], "-") + } + return prefix + "-" + suffix +} + +func bindingReady(binding *unstructured.Unstructured) bool { + observed, found, _ := unstructured.NestedInt64(binding.Object, "status", "observedGeneration") + return found && observed == binding.GetGeneration() && resolverContextReady(binding) +} + +func newInternalDNSObject(gvk schema.GroupVersionKind, namespace, name string) *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(gvk) + obj.SetNamespace(namespace) + obj.SetName(name) + return obj +} + +func internalDNSIntentAnnotations() []string { + return []string{ + InternalDNSRegionAnnotation, + InternalDNSContextNameAnnotation, + InternalDNSAccessNameAnnotation, + InternalDNSServiceDestinationAnnotation, + InternalDNSSuffixAnnotation, + } +} + +func boolPtr(value bool) *bool { return &value } diff --git a/internal/controller/internaldns_controller_test.go b/internal/controller/internaldns_controller_test.go new file mode 100644 index 00000000..c66d3df3 --- /dev/null +++ b/internal/controller/internaldns_controller_test.go @@ -0,0 +1,386 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "testing" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + "go.datum.net/network-services-operator/internal/config" +) + +type fixedProjectIdentity types.UID + +func (f fixedProjectIdentity) ProjectUID(context.Context, string) (types.UID, error) { + return types.UID(f), nil +} + +func TestInternalDNSProjectLifecycleFencesAccess(t *testing.T) { + t.Parallel() + ctx := context.Background() + now := time.Date(2026, time.October, 9, 12, 0, 0, 0, time.UTC) + scheme := runtime.NewScheme() + if err := networkingv1alpha.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := locationsv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + scheme.AddKnownTypeWithName(internalDNSContextGVK, &unstructured.Unstructured{}) + scheme.AddKnownTypeWithName(internalDNSContextGVK.GroupVersion().WithKind("DNSResolverContextList"), &unstructured.UnstructuredList{}) + scheme.AddKnownTypeWithName(internalDNSAccessGVK, &unstructured.Unstructured{}) + scheme.AddKnownTypeWithName(internalDNSAccessGVK.GroupVersion().WithKind("DNSResolverAccessBindingList"), &unstructured.UnstructuredList{}) + + network := &networkingv1alpha.Network{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "blue", UID: "network-uid"}, + } + location := &locationsv1alpha1.Location{ + ObjectMeta: metav1.ObjectMeta{Name: "ord-1"}, + Spec: locationsv1alpha1.LocationSpec{Topology: map[string]string{internalDNSRegionTopologyKey: "us-central"}}, + } + first := testNetworkContext("first", "context-1") + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(network, location, first).Build() + r := &InternalDNSProjectReconciler{ + Config: config.InternalDNSConfig{ + Frontend: "fd53::53", ServicePrefix: "fd70:100::/64", + }, + Projects: fixedProjectIdentity("project-uid"), + Now: func() time.Time { return now }, + } + + if _, err := r.reconcile(ctx, cl, "project", client.ObjectKeyFromObject(first)); err != nil { + t.Fatal(err) + } + dnsContext := newInternalDNSObject(internalDNSContextGVK, "tenant", internalDNSResourceName("blue", network.UID)) + if err := cl.Get(ctx, client.ObjectKeyFromObject(dnsContext), dnsContext); err != nil { + t.Fatal(err) + } + domain, _, _ := unstructured.NestedString(dnsContext.Object, "spec", "managedNamespace", "domainSuffix") + if domain != internalDNSDefaultDomain { + t.Fatalf("network default domain = %q", domain) + } + dnsContext.SetUID("dns-context-uid") + dnsContext.SetGeneration(1) + if err := unstructured.SetNestedSlice(dnsContext.Object, []any{map[string]any{"type": "Ready", "status": "True", "observedGeneration": int64(1)}}, "status", "conditions"); err != nil { + t.Fatal(err) + } + if err := unstructured.SetNestedField(dnsContext.Object, internalDNSDefaultDomain, "status", "managedNamespace", "suffix"); err != nil { + t.Fatal(err) + } + if err := unstructured.SetNestedMap(dnsContext.Object, map[string]any{ + "name": "blue-internal", + }, "status", "managedNamespace", "dnsZoneRef"); err != nil { + t.Fatal(err) + } + if err := cl.Update(ctx, dnsContext); err != nil { + t.Fatal(err) + } + + if _, err := r.reconcile(ctx, cl, "project", client.ObjectKeyFromObject(first)); err != nil { + t.Fatal(err) + } + bindingName := internalDNSRegionalName(dnsContext.GetName(), "us-central") + binding := newInternalDNSObject(internalDNSAccessGVK, "tenant", bindingName) + if err := cl.Get(ctx, client.ObjectKeyFromObject(binding), binding); err != nil { + t.Fatal(err) + } + binding.SetUID("access-uid") + binding.SetGeneration(1) + if err := unstructured.SetNestedField(binding.Object, int64(1), "status", "observedGeneration"); err != nil { + t.Fatal(err) + } + if err := unstructured.SetNestedField(binding.Object, "dns-binding-uid", "status", "bindingRef", "uid"); err != nil { + t.Fatal(err) + } + if err := unstructured.SetNestedSlice(binding.Object, []any{map[string]any{ + "type": "Ready", "status": "True", "reason": "Ready", "observedGeneration": int64(1), + }}, "status", "conditions"); err != nil { + t.Fatal(err) + } + if err := cl.Update(ctx, binding); err != nil { + t.Fatal(err) + } + + if _, err := r.reconcile(ctx, cl, "project", client.ObjectKeyFromObject(first)); err != nil { + t.Fatal(err) + } + assertDNSIntent(t, ctx, cl, "first", bindingName) + originalStatus, _, _ := unstructured.NestedMap(dnsContext.Object, "status") + for _, invalid := range []struct { + path []string + value any + }{ + {path: []string{"status", "managedNamespace", "suffix"}, value: "different.internal"}, + {path: []string{"status", "conditions"}, value: []any{map[string]any{"type": "Ready", "status": "True", "observedGeneration": int64(0)}}}, + } { + if err := unstructured.SetNestedField(dnsContext.Object, invalid.value, invalid.path...); err != nil { + t.Fatal(err) + } + if err := cl.Update(ctx, dnsContext); err != nil { + t.Fatal(err) + } + if _, err := r.reconcile(ctx, cl, "project", client.ObjectKeyFromObject(first)); err != nil { + t.Fatal(err) + } + current := &networkingv1alpha.NetworkContext{} + if err := cl.Get(ctx, client.ObjectKeyFromObject(first), current); err != nil { + t.Fatal(err) + } + if current.Annotations[InternalDNSSuffixAnnotation] != "datum.internal" { + t.Fatalf("unavailable allocation observation withdrew DNS intent: %#v", current.Annotations) + } + if err := unstructured.SetNestedMap(dnsContext.Object, originalStatus, "status"); err != nil { + t.Fatal(err) + } + if err := cl.Update(ctx, dnsContext); err != nil { + t.Fatal(err) + } + if _, err := r.reconcile(ctx, cl, "project", client.ObjectKeyFromObject(first)); err != nil { + t.Fatal(err) + } + assertDNSIntent(t, ctx, cl, "first", bindingName) + } + + second := testNetworkContext("second", "context-2") + if err := cl.Create(ctx, second); err != nil { + t.Fatal(err) + } + if _, err := r.reconcile(ctx, cl, "project", client.ObjectKeyFromObject(second)); err != nil { + t.Fatal(err) + } + assertDNSIntent(t, ctx, cl, "second", bindingName) + assertUnstructuredCount(t, ctx, cl, internalDNSContextGVK, "tenant", 1) + assertUnstructuredCount(t, ctx, cl, internalDNSAccessGVK, "tenant", 1) + firstCurrent := &networkingv1alpha.NetworkContext{} + if err := cl.Get(ctx, client.ObjectKey{Namespace: "tenant", Name: "first"}, firstCurrent); err != nil { + t.Fatal(err) + } + if err := r.cleanupRegion(ctx, cl, firstCurrent); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, client.ObjectKey{Namespace: "tenant", Name: bindingName}, binding); err != nil { + t.Fatalf("regional access was removed while another context remained: %v", err) + } + + now = now.Add(3 * time.Minute) + if _, err := r.reconcile(ctx, cl, "project", client.ObjectKeyFromObject(first)); err != nil { + t.Fatal(err) + } + current := &networkingv1alpha.NetworkContext{} + if err := cl.Get(ctx, client.ObjectKey{Namespace: "tenant", Name: "first"}, current); err != nil { + t.Fatal(err) + } + if current.Annotations[InternalDNSAccessNameAnnotation] != bindingName { + t.Fatalf("persistent access was withdrawn: %#v", current.Annotations) + } + if err := cl.Get(ctx, client.ObjectKey{Namespace: "tenant", Name: bindingName}, binding); err != nil { + t.Fatal(err) + } + if !bindingReady(binding) { + t.Fatal("unchanged access lost readiness") + } + secondCurrent := &networkingv1alpha.NetworkContext{} + if err := cl.Get(ctx, client.ObjectKey{Namespace: "tenant", Name: "second"}, secondCurrent); err != nil { + t.Fatal(err) + } + if err := cl.Delete(ctx, firstCurrent); err != nil { + t.Fatal(err) + } + if err := r.cleanupRegion(ctx, cl, secondCurrent); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, client.ObjectKey{Namespace: "tenant", Name: bindingName}, binding); !apierrors.IsNotFound(err) { + t.Fatalf("regional access survived the last context: %v", err) + } +} + +func TestInternalDNSReadinessPreservesAppliedSettings(t *testing.T) { + t.Parallel() + ctx := context.Background() + now := time.Date(2026, time.October, 9, 12, 0, 0, 0, time.UTC) + scheme := runtime.NewScheme() + if err := networkingv1alpha.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + projectContext := testNetworkContext("context", "project-context-uid") + projectContext.Annotations = testDNSIntent(now.Add(time.Minute)) + cl := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&networkingv1alpha.NetworkContext{}). + WithObjects(projectContext).Build() + hubContext := projectContext.DeepCopy() + hubContext.Status.DNSAccess = &networkingv1alpha.NetworkDNSAccessStatus{ + ContextRef: networkingv1alpha.NetworkDNSObjectReference{Name: "resolver"}, + AccessBindingRef: networkingv1alpha.NetworkDNSObjectReference{Name: "access"}, + ServiceDestination: projectContext.Annotations[InternalDNSServiceDestinationAnnotation], + } + hubContext.Status.Conditions = []metav1.Condition{{ + Type: networkingv1alpha.NetworkContextDNSReady, Status: metav1.ConditionTrue, + Reason: networkingv1alpha.NetworkContextDNSReadyReasonReady, + }} + r := &InternalDNSReadinessReconciler{ + Config: config.InternalDNSConfig{Frontend: "fd53::53"}, + Now: func() time.Time { return now }, + } + + if _, err := r.publish(ctx, cl, projectContext, hubContext); err != nil { + t.Fatal(err) + } + current := &networkingv1alpha.NetworkContext{} + if err := cl.Get(ctx, client.ObjectKeyFromObject(projectContext), current); err != nil { + t.Fatal(err) + } + if current.Spec.DNS == nil || current.Spec.DNS.AccessBindingRef.Name != "access" { + t.Fatalf("verified settings were not published: %#v", current.Spec.DNS) + } + + hubContext.Status.DNSAccess.ServiceDestination = "fd70:100::11" + if _, err := r.publish(ctx, cl, current, hubContext); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(projectContext), current); err != nil { + t.Fatal(err) + } + if current.Spec.DNS == nil { + t.Fatal("observation mismatch withdrew applied resolver settings") + } + + hubContext.Status.DNSAccess.ServiceDestination = projectContext.Annotations[InternalDNSServiceDestinationAnnotation] + hubContext.Status.Conditions[0].Status = metav1.ConditionUnknown + r.Now = func() time.Time { return now.Add(2 * time.Minute) } + if _, err := r.publish(ctx, cl, current, hubContext); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(projectContext), current); err != nil { + t.Fatal(err) + } + if current.Spec.DNS == nil { + t.Fatal("resolver settings expired") + } + if condition := apimeta.FindStatusCondition(current.Status.Conditions, networkingv1alpha.NetworkContextDNSReady); condition == nil || condition.Status != metav1.ConditionUnknown { + t.Fatalf("unavailable observation was not reported: %#v", current.Status.Conditions) + } +} + +func TestRecreatedContextReplacesAccessBinding(t *testing.T) { + t.Parallel() + ctx := context.Background() + scheme := runtime.NewScheme() + scheme.AddKnownTypeWithName(internalDNSAccessGVK, &unstructured.Unstructured{}) + network := &networkingv1alpha.Network{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "blue", UID: "network-uid"}} + dnsContext := newInternalDNSObject(internalDNSContextGVK, "tenant", "resolver") + dnsContext.SetUID("new-context") + binding := newInternalDNSObject(internalDNSAccessGVK, "tenant", internalDNSRegionalName("resolver", "us-central")) + binding.SetUID("access-uid") + binding.SetAnnotations(map[string]string{internalDNSOwnerNetworkUIDAnnotation: string(network.UID)}) + binding.Object["spec"] = map[string]any{ + "contextRef": map[string]any{"name": "resolver", "uid": "old-context"}, + "region": "us-central", "destinationAddress": "fd70:100::10", + } + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(binding).Build() + r := &InternalDNSProjectReconciler{} + _, ready, err := r.ensureAccess(ctx, cl, network, "project-uid", dnsContext, "us-central", "fd70:100::11") + if err != nil { + t.Fatal(err) + } + if ready { + t.Fatal("replaced access was reported ready") + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(binding), binding); !apierrors.IsNotFound(err) { + t.Fatalf("retired binding survived: %v", err) + } + current, ready, err := r.ensureAccess(ctx, cl, network, "project-uid", dnsContext, "us-central", "fd70:100::11") + if err != nil { + t.Fatal(err) + } + if ready || !accessBindingMatches(current, dnsContext, "us-central", "fd70:100::11") { + t.Fatal("new context did not receive a pending access binding") + } +} + +func TestAccessReadinessRequiresCurrentGeneration(t *testing.T) { + binding := newInternalDNSObject(internalDNSAccessGVK, "tenant", "access") + binding.SetGeneration(2) + binding.Object["status"] = map[string]any{"observedGeneration": int64(1), "conditions": []any{map[string]any{"type": "Ready", "status": "True", "observedGeneration": int64(1)}}} + if bindingReady(binding) { + t.Fatal("stale observation was accepted") + } + if err := unstructured.SetNestedField(binding.Object, int64(2), "status", "observedGeneration"); err != nil { + t.Fatal(err) + } + if bindingReady(binding) { + t.Fatal("stale condition was accepted") + } + if err := unstructured.SetNestedSlice(binding.Object, []any{map[string]any{"type": "Ready", "status": "True", "observedGeneration": int64(2)}}, "status", "conditions"); err != nil { + t.Fatal(err) + } + if !bindingReady(binding) { + t.Fatal("current observation was rejected") + } +} + +func testNetworkContext(name string, uid types.UID) *networkingv1alpha.NetworkContext { + return &networkingv1alpha.NetworkContext{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: name, UID: uid, + Labels: map[string]string{networkingv1alpha.NetworkUIDLabel: "network-uid"}, + }, + Spec: networkingv1alpha.NetworkContextSpec{ + Network: networkingv1alpha.LocalNetworkRef{Name: "blue"}, + Location: locationsv1alpha1.LocationReference{Name: "ord-1"}, + }, + } +} + +func testDNSIntent(_ time.Time) map[string]string { + return map[string]string{ + InternalDNSContextNameAnnotation: "resolver", + InternalDNSAccessNameAnnotation: "access", + InternalDNSSuffixAnnotation: "blue.internal", + InternalDNSRegionAnnotation: "us-central", + InternalDNSServiceDestinationAnnotation: "fd70:100::10", + } +} + +func assertDNSIntent(t *testing.T, ctx context.Context, cl client.Client, name, accessName string) { + t.Helper() + current := &networkingv1alpha.NetworkContext{} + if err := cl.Get(ctx, client.ObjectKey{Namespace: "tenant", Name: name}, current); err != nil { + t.Fatal(err) + } + if current.Annotations[InternalDNSAccessNameAnnotation] != accessName { + t.Fatalf("unexpected DNS intent: %#v", current.Annotations) + } +} + +func assertUnstructuredCount( + t *testing.T, + ctx context.Context, + cl client.Client, + gvk schema.GroupVersionKind, + namespace string, + want int, +) { + t.Helper() + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(gvk.GroupVersion().WithKind(gvk.Kind + "List")) + if err := cl.List(ctx, list, client.InNamespace(namespace)); err != nil { + t.Fatal(err) + } + if len(list.Items) != want { + t.Fatalf("%s count = %d, want %d", gvk.Kind, len(list.Items), want) + } +} diff --git a/internal/controller/networkinterface_projection.go b/internal/controller/networkinterface_projection.go index 2a128714..c7cf85f9 100644 --- a/internal/controller/networkinterface_projection.go +++ b/internal/controller/networkinterface_projection.go @@ -41,6 +41,7 @@ func projectedInterface(source *networkingv1alpha.NetworkInterface, location str ReclaimPolicy: source.Spec.ReclaimPolicy, }, Status: networkingv1alpha.NetworkInterfaceStatus{ + DNS: source.Status.DNS.DeepCopy(), Phase: source.Status.Phase, Conditions: append([]metav1.Condition(nil), source.Status.Conditions...), }, diff --git a/internal/controller/networkinterface_projection_test.go b/internal/controller/networkinterface_projection_test.go index 66947490..532c411b 100644 --- a/internal/controller/networkinterface_projection_test.go +++ b/internal/controller/networkinterface_projection_test.go @@ -367,6 +367,20 @@ func TestInterfaceReachesTheProjectControlPlane(t *testing.T) { require.Equal(t, boundInterfaceName, copied.Labels[networkingv1alpha.NetworkInterfaceHolderLabel]) } +func TestProjectedInterfaceReportsDNSSettings(t *testing.T) { + source := &networkingv1alpha.NetworkInterface{Status: networkingv1alpha.NetworkInterfaceStatus{ + DNS: &networkingv1alpha.NetworkDNSResolverSettings{ + Nameservers: []string{"fd53::53"}, + Searches: []string{"blue.internal"}, + }, + }} + + projection := projectedInterface(source, "ord-1") + require.Equal(t, source.Status.DNS, projection.Status.DNS) + projection.Status.DNS.Nameservers[0] = "fd53::54" + require.Equal(t, "fd53::53", source.Status.DNS.Nameservers[0], "the projection must own its DNS settings") +} + func TestCopyDropsWhatOnlyMeansSomethingOnTheCell(t *testing.T) { v := newVisibility(t) v.interfaceOnCell() diff --git a/internal/controller/networkinterfaceclaim_controller.go b/internal/controller/networkinterfaceclaim_controller.go index 114b0d00..91ea5d1f 100644 --- a/internal/controller/networkinterfaceclaim_controller.go +++ b/internal/controller/networkinterfaceclaim_controller.go @@ -14,6 +14,7 @@ import ( "time" corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -1023,6 +1024,18 @@ func (r *NetworkInterfaceClaimReconciler) syncInterface( iface.Spec.MTU = networkContext.Spec.MTU changed = true } + var dns *networkingv1alpha.NetworkDNSResolverSettings + if settings := networkContext.Spec.DNS; settings != nil { + dns = &networkingv1alpha.NetworkDNSResolverSettings{Nameservers: append([]string(nil), settings.Nameservers...), Searches: append([]string(nil), settings.Searches...)} + } + if !apiequality.Semantic.DeepEqual(iface.Status.DNS, dns) { + base := iface.DeepCopy() + iface.Status.DNS = dns + apimeta.RemoveStatusCondition(&iface.Status.Conditions, networkingv1alpha.NetworkInterfaceDNSConfigured) + if err := cl.Status().Patch(ctx, iface, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { + return fmt.Errorf("publish interface resolver settings: %w", err) + } + } // An interface adopted from before the field existed carries none. if claim.Spec.AttachmentMode != "" && iface.Spec.AttachmentMode != claim.Spec.AttachmentMode { iface.Spec.AttachmentMode = claim.Spec.AttachmentMode diff --git a/internal/controller/networkinterfaceclaim_controller_test.go b/internal/controller/networkinterfaceclaim_controller_test.go index 805d810f..98e73c71 100644 --- a/internal/controller/networkinterfaceclaim_controller_test.go +++ b/internal/controller/networkinterfaceclaim_controller_test.go @@ -23,6 +23,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/validation" @@ -1677,6 +1678,58 @@ func TestNetworkInterfaceClaimTakesMTUFromTheNetworkContext(t *testing.T) { "the context carries the MTU, and the network the cell cannot read carries 1440") } +func TestNetworkInterfaceDNSIsReadOnlyAndFollowsTheNetwork(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + contextObject, err := s.getNetworkContext() + require.NoError(t, err) + contextObject.Spec.DNS = &networkingv1alpha.NetworkDNSSettings{ + ContextRef: networkingv1alpha.NetworkDNSObjectReference{Name: "resolver"}, + AccessBindingRef: networkingv1alpha.NetworkDNSObjectReference{Name: "access"}, + Nameservers: []string{"fd53::53"}, Searches: []string{"datum.internal"}, + } + require.NoError(t, s.client.Update(s.ctx, contextObject)) + claim := s.createClaim("dns-interface", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + iface, err := s.getInterface(claim.Name) + require.NoError(t, err) + require.Equal(t, []string{"fd53::53"}, iface.Status.DNS.Nameservers) + require.Equal(t, []string{"datum.internal"}, iface.Status.DNS.Searches) + object := &unstructured.Unstructured{} + object.SetGroupVersionKind(networkingv1alpha.GroupVersion.WithKind("NetworkInterface")) + require.NoError(t, s.client.Get(s.ctx, client.ObjectKeyFromObject(iface), object)) + for _, field := range []string{"spec", "status"} { + require.NoError(t, unstructured.SetNestedMap(object.Object, map[string]any{"nameservers": []any{"fd53::54"}}, field, "dns")) + } + require.NoError(t, s.client.Update(s.ctx, object)) + require.NoError(t, s.client.Get(s.ctx, client.ObjectKeyFromObject(iface), object)) + _, editable, err := unstructured.NestedMap(object.Object, "spec", "dns") + require.NoError(t, err) + require.False(t, editable) + require.NoError(t, s.client.Get(s.ctx, client.ObjectKeyFromObject(iface), iface)) + require.Equal(t, []string{"fd53::53"}, iface.Status.DNS.Nameservers) + apimeta.SetStatusCondition(&iface.Status.Conditions, metav1.Condition{Type: networkingv1alpha.NetworkInterfaceDNSConfigured, Status: metav1.ConditionTrue, Reason: "Applied", LastTransitionTime: metav1.Now()}) + require.NoError(t, s.client.Status().Update(s.ctx, iface)) + contextObject, err = s.getNetworkContext() + require.NoError(t, err) + contextObject.Spec.DNS.Searches = []string{"apps.internal"} + require.NoError(t, s.client.Update(s.ctx, contextObject)) + s.reconcile(claim) + iface, err = s.getInterface(claim.Name) + require.NoError(t, err) + require.Equal(t, []string{"apps.internal"}, iface.Status.DNS.Searches) + require.Nil(t, apimeta.FindStatusCondition(iface.Status.Conditions, networkingv1alpha.NetworkInterfaceDNSConfigured)) + contextObject, err = s.getNetworkContext() + require.NoError(t, err) + contextObject.Spec.DNS = nil + require.NoError(t, s.client.Update(s.ctx, contextObject)) + s.reconcile(claim) + iface, err = s.getInterface(claim.Name) + require.NoError(t, err) + require.Nil(t, iface.Status.DNS) +} + // A network that has not reached the location is a different answer from a // consumer naming a network that does not exist. func TestNetworkInterfaceClaimRejectsAMissingNetworkContext(t *testing.T) { diff --git a/internal/dnsaccess/programming.go b/internal/dnsaccess/programming.go new file mode 100644 index 00000000..9bba366a --- /dev/null +++ b/internal/dnsaccess/programming.go @@ -0,0 +1,244 @@ +// Copyright 2026 Datum Cloud, Inc. +// SPDX-License-Identifier: AGPL-3.0-or-later + +package dnsaccess + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sort" + + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + + cloudv1alpha1 "go.datum.net/galactic/api/cloud/v1alpha1" + networkv1alpha1 "go.datum.net/galactic/api/network/v1alpha1" +) + +// ProgrammingRequirements fences readiness to the complete service topology. +type ProgrammingRequirements struct { + InputDigest string + Nodes []string +} + +// ProgrammingPlan returns the exact digest and node set Galactic must report +// before network services publishes resolver settings. +func ProgrammingPlan( + policy *networkv1alpha1.ServiceRoutePolicy, + endpoint *networkv1alpha1.ServiceEndpoint, + attachments []*cloudv1alpha1.VPCAttachment, + consumerVPCIdentity string, +) (ProgrammingRequirements, error) { + plan := ProgrammingRequirements{} + if policy == nil || endpoint == nil { + return plan, errors.New("policy and endpoint are required") + } + if policy.Spec.Frontend != nil && (policy.UID == "" || endpoint.UID == "" || consumerVPCIdentity == "") { + return plan, errors.New("translated programming requires current policy, endpoint, and VPC identities") + } + selector, err := metav1.LabelSelectorAsSelector(&policy.Spec.AttachmentSelector) + if err != nil { + return plan, err + } + producers, err := producerCandidates(endpoint, attachments) + if err != nil { + return plan, err + } + if err := validateProducerTopology(endpoint, producers); err != nil { + return plan, err + } + ready := map[types.NamespacedName]bool{} + for _, attachment := range readyAttachments(attachments) { + ready[types.NamespacedName{Namespace: attachment.Namespace, Name: attachment.Name}] = + attachment.Status.HostInterface != "" + } + relevant := map[types.NamespacedName]bool{} + for _, producer := range producers { + relevant[types.NamespacedName{Namespace: producer.Namespace, Name: producer.Name}] = true + } + nodes := map[string]bool{} + selected := 0 + sorted := nonNilAttachments(attachments) + for _, attachment := range sorted { + if !selector.Matches(labels.Set(attachment.Labels)) || !consumerMatchesVPC(policy, attachment, consumerVPCIdentity) { + continue + } + key := types.NamespacedName{Namespace: attachment.Namespace, Name: attachment.Name} + relevant[key] = true + selected++ + if attachment.DeletionTimestamp != nil || !ready[key] { + return plan, fmt.Errorf("consumer attachment %s is not ready", key) + } + producer := chooseProducer(endpoint.Spec.DeliveryMode, attachment.Status.Node, producers) + if producer == nil { + return plan, fmt.Errorf("consumer attachment %s has no eligible producer", key) + } + if policy.Spec.Frontend != nil && (attachment.UID == "" || producer.UID == "") { + return plan, errors.New("translated programming requires current attachment UIDs") + } + nodes[attachment.Status.Node] = true + nodes[producer.Status.Node] = true + } + if selected == 0 { + return plan, errors.New("no eligible consumer attachments") + } + + snapshot := struct { + PolicyUID types.UID + PolicyGeneration int64 + PolicySpec networkv1alpha1.ServiceRoutePolicySpec + EndpointUID types.UID + EndpointGeneration int64 + EndpointSpec networkv1alpha1.ServiceEndpointSpec + VPCIdentity string + Attachments []any + }{ + PolicyUID: policy.UID, PolicyGeneration: policy.Generation, PolicySpec: policy.Spec, + EndpointUID: endpoint.UID, EndpointGeneration: endpoint.Generation, EndpointSpec: endpoint.Spec, + VPCIdentity: consumerVPCIdentity, + } + for _, attachment := range sorted { + key := types.NamespacedName{Namespace: attachment.Namespace, Name: attachment.Name} + if !relevant[key] { + continue + } + snapshot.Attachments = append(snapshot.Attachments, struct { + Namespace, Name string + UID types.UID + Generation int64 + Labels map[string]string + Deleted bool + Spec cloudv1alpha1.VPCAttachmentSpec + Status cloudv1alpha1.VPCAttachmentStatus + }{ + attachment.Namespace, attachment.Name, attachment.UID, attachment.Generation, + attachment.Labels, attachment.DeletionTimestamp != nil, attachment.Spec, attachment.Status, + }) + } + encoded, err := json.Marshal(snapshot) + if err != nil { + return plan, err + } + sum := sha256.Sum256(encoded) + plan.InputDigest = hex.EncodeToString(sum[:]) + for node := range nodes { + plan.Nodes = append(plan.Nodes, node) + } + sort.Strings(plan.Nodes) + return plan, nil +} + +func nonNilAttachments(attachments []*cloudv1alpha1.VPCAttachment) []*cloudv1alpha1.VPCAttachment { + result := make([]*cloudv1alpha1.VPCAttachment, 0, len(attachments)) + for _, attachment := range attachments { + if attachment != nil { + result = append(result, attachment) + } + } + sort.Slice(result, func(i, j int) bool { + return result[i].Namespace+"/"+result[i].Name < result[j].Namespace+"/"+result[j].Name + }) + return result +} + +func readyAttachments(attachments []*cloudv1alpha1.VPCAttachment) []*cloudv1alpha1.VPCAttachment { + result := make([]*cloudv1alpha1.VPCAttachment, 0, len(attachments)) + for _, attachment := range attachments { + if attachment == nil || attachment.DeletionTimestamp != nil || + attachment.Status.ObservedGeneration != attachment.Generation || + attachment.Status.Node == "" || attachment.Status.VPC == "" || attachment.Status.VPCAttachment == "" || + !apimeta.IsStatusConditionTrue(attachment.Status.Conditions, cloudv1alpha1.ConditionTypeReady) || + !apimeta.IsStatusConditionTrue(attachment.Status.Conditions, cloudv1alpha1.ConditionTypeProgrammed) { + continue + } + result = append(result, attachment) + } + sort.Slice(result, func(i, j int) bool { + return types.NamespacedName{Namespace: result[i].Namespace, Name: result[i].Name}.String() < + types.NamespacedName{Namespace: result[j].Namespace, Name: result[j].Name}.String() + }) + return result +} + +func producerCandidates( + endpoint *networkv1alpha1.ServiceEndpoint, + attachments []*cloudv1alpha1.VPCAttachment, +) ([]*cloudv1alpha1.VPCAttachment, error) { + if (endpoint.Spec.AttachmentRef == nil) == (endpoint.Spec.AttachmentSelector == nil) { + return nil, errors.New("service endpoint must select one producer source") + } + ready := readyAttachments(attachments) + if endpoint.Spec.AttachmentRef != nil { + key := types.NamespacedName{ + Namespace: endpoint.Spec.AttachmentRef.Namespace, + Name: endpoint.Spec.AttachmentRef.Name, + } + for _, attachment := range ready { + if (types.NamespacedName{Namespace: attachment.Namespace, Name: attachment.Name}) == key && + attachment.Status.HostInterface != "" { + return []*cloudv1alpha1.VPCAttachment{attachment}, nil + } + } + return nil, nil + } + selector, err := metav1.LabelSelectorAsSelector(endpoint.Spec.AttachmentSelector) + if err != nil { + return nil, err + } + result := make([]*cloudv1alpha1.VPCAttachment, 0, len(ready)) + for _, attachment := range ready { + if attachment.Status.HostInterface != "" && selector.Matches(labels.Set(attachment.Labels)) { + result = append(result, attachment) + } + } + return result, nil +} + +func validateProducerTopology( + endpoint *networkv1alpha1.ServiceEndpoint, + producers []*cloudv1alpha1.VPCAttachment, +) error { + type location struct{ node, vpc string } + seen := map[location]bool{} + for _, producer := range producers { + key := location{node: producer.Status.Node, vpc: producer.Status.VPC} + if seen[key] { + return fmt.Errorf("service endpoint %s selects multiple producers on node %q in VPC %q", endpoint.Name, key.node, key.vpc) + } + seen[key] = true + } + return nil +} + +func chooseProducer( + mode networkv1alpha1.ServiceEndpointDeliveryMode, + consumerNode string, + producers []*cloudv1alpha1.VPCAttachment, +) *cloudv1alpha1.VPCAttachment { + for _, producer := range producers { + if producer.Status.Node == consumerNode { + return producer + } + } + if mode == networkv1alpha1.ServiceEndpointDeliveryModePreferNodeLocal && len(producers) > 0 { + return producers[0] + } + return nil +} + +func consumerMatchesVPC( + policy *networkv1alpha1.ServiceRoutePolicy, + consumer *cloudv1alpha1.VPCAttachment, + identity string, +) bool { + if ref := policy.Spec.ConsumerVPCRef; ref != nil && + (consumer.Namespace != policy.Namespace || consumer.Spec.VPC.Name != ref.Name) { + return false + } + return identity == "" || consumer.Status.VPC == identity +} diff --git a/internal/networkagent/catalog_test.go b/internal/networkagent/catalog_test.go index 17d93be9..c756fb1b 100644 --- a/internal/networkagent/catalog_test.go +++ b/internal/networkagent/catalog_test.go @@ -34,6 +34,7 @@ var outOfScopeAPIFiles = map[string]string{ "edgereachability_types.go": "hub-side record with no customer-facing status", "groupversion_info.go": "scheme registration, no reasons", "httpproxy_types.go": "load balancers; the ALB agent's", + "internaldns_types.go": "internal DNS service configuration, no status reasons", "locationbinding_types.go": "which locations a project is offered; surfaces here only as LocationNotAvailable", "networkpolicy_types.go": "declares no status reasons", "trafficprotectionpolicy_types.go": "load balancer protection; the ALB agent's",