From 18c28f70032a5dc8acab453a7c47b7dd6661719e Mon Sep 17 00:00:00 2001 From: Andrii Chubatiuk Date: Fri, 9 Oct 2026 10:51:27 +0300 Subject: [PATCH 1/5] cluster: don't wait for storage pods in a maintenance --- api/operator/v1/cluster_types_test.go | 37 ++++ api/operator/v1/vlcluster_types.go | 13 ++ api/operator/v1/vtcluster_types.go | 13 ++ api/operator/v1beta1/vmcluster_types.go | 13 ++ api/operator/v1beta1/vmcluster_types_test.go | 31 +++ config/crd/overlay/crd.yaml | 21 +- docs/CHANGELOG.md | 1 + docs/api.md | 6 +- docs/resources/vlcluster.md | 36 ++++ docs/resources/vmcluster.md | 38 ++++ docs/resources/vtcluster.md | 36 ++++ .../operator/factory/reconcile/statefulset.go | 89 ++++++-- .../factory/reconcile/statefulset_test.go | 203 ++++++++++++++++++ .../operator/factory/vlcluster/vlstorage.go | 1 + .../operator/factory/vmcluster/vmcluster.go | 1 + .../factory/vmcluster/vmcluster_test.go | 48 ++--- .../operator/factory/vtcluster/storage.go | 1 + 17 files changed, 539 insertions(+), 49 deletions(-) diff --git a/api/operator/v1/cluster_types_test.go b/api/operator/v1/cluster_types_test.go index 9bca6afe18..5c22883c44 100644 --- a/api/operator/v1/cluster_types_test.go +++ b/api/operator/v1/cluster_types_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/ptr" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" @@ -79,6 +80,42 @@ func TestVLCluster_AvailableStorageNodeIDs(t *testing.T) { }, vmv1beta1.ClusterComponentSelect, []int32{0, 1, 2}) } +func TestVTCluster_StorageNodeIDsUnderMaintenance(t *testing.T) { + f := func(cr *VTCluster, want sets.Set[int32]) { + t.Helper() + assert.Equal(t, want, cr.StorageNodeIDsUnderMaintenance()) + } + + f(&VTCluster{ + Spec: VTClusterSpec{ + Storage: &VTStorage{ + MaintenanceSelectNodeIDs: []int32{1, 3}, + MaintenanceInsertNodeIDs: []int32{0, 3}, + }, + }, + }, sets.New[int32](3)) + + f(&VTCluster{}, nil) +} + +func TestVLCluster_StorageNodeIDsUnderMaintenance(t *testing.T) { + f := func(cr *VLCluster, want sets.Set[int32]) { + t.Helper() + assert.Equal(t, want, cr.StorageNodeIDsUnderMaintenance()) + } + + f(&VLCluster{ + Spec: VLClusterSpec{ + VLStorage: &VLStorage{ + MaintenanceSelectNodeIDs: []int32{1, 3}, + MaintenanceInsertNodeIDs: []int32{0, 3}, + }, + }, + }, sets.New[int32](3)) + + f(&VLCluster{}, nil) +} + func TestVLSingle_PrefixedName(t *testing.T) { f := func(name string, omit bool, want string) { t.Helper() diff --git a/api/operator/v1/vlcluster_types.go b/api/operator/v1/vlcluster_types.go index 67d9f94c2f..1f8d6ac94c 100644 --- a/api/operator/v1/vlcluster_types.go +++ b/api/operator/v1/vlcluster_types.go @@ -652,6 +652,8 @@ type VLStorage struct { // +optional MaintenanceInsertNodeIDs []int32 `json:"maintenanceInsertNodeIDs,omitempty"` // MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. + // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs is also exempted from the storage pod readiness wait during reconcile. + // This only reliably prevents reconcile from blocking under the default OnDelete rollingUpdateStrategy; with RollingUpdate, Kubernetes' own OrderedReady pod management can still block on an unready node. // +optional MaintenanceSelectNodeIDs []int32 `json:"maintenanceSelectNodeIDs,omitempty"` @@ -902,6 +904,17 @@ func (cr *VLCluster) AvailableStorageNodeIDs(kind vmv1beta1.ClusterComponent) [] return result } +// StorageNodeIDsUnderMaintenance returns the vlstorage pod ordinals excluded from both +// insert and select routing via MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs. +func (cr *VLCluster) StorageNodeIDsUnderMaintenance() sets.Set[int32] { + if cr.Spec.VLStorage == nil { + return nil + } + insertNodes := sets.New(cr.Spec.VLStorage.MaintenanceInsertNodeIDs...) + selectNodes := sets.New(cr.Spec.VLStorage.MaintenanceSelectNodeIDs...) + return insertNodes.Intersection(selectNodes) +} + // LastSpecUpdated compares spec with last applied spec stored, replaces old spec and returns true if it's updated func (cr *VLCluster) LastSpecUpdated() bool { updated := cr.Status.LastAppliedSpec == nil || !equality.Semantic.DeepEqual(&cr.Spec, cr.Status.LastAppliedSpec) diff --git a/api/operator/v1/vtcluster_types.go b/api/operator/v1/vtcluster_types.go index 5efa0a9014..9348ff91f9 100644 --- a/api/operator/v1/vtcluster_types.go +++ b/api/operator/v1/vtcluster_types.go @@ -515,6 +515,8 @@ type VTStorage struct { // +optional MaintenanceInsertNodeIDs []int32 `json:"maintenanceInsertNodeIDs,omitempty"` // MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. + // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs is also exempted from the storage pod readiness wait during reconcile. + // This only reliably prevents reconcile from blocking under the default OnDelete rollingUpdateStrategy; with RollingUpdate, Kubernetes' own OrderedReady pod management can still block on an unready node. // +optional MaintenanceSelectNodeIDs []int32 `json:"maintenanceSelectNodeIDs,omitempty"` @@ -770,6 +772,17 @@ func (cr *VTCluster) AvailableStorageNodeIDs(kind vmv1beta1.ClusterComponent) [] return result } +// StorageNodeIDsUnderMaintenance returns the storage pod ordinals excluded from both +// insert and select routing via MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs. +func (cr *VTCluster) StorageNodeIDsUnderMaintenance() sets.Set[int32] { + if cr.Spec.Storage == nil { + return nil + } + insertNodes := sets.New(cr.Spec.Storage.MaintenanceInsertNodeIDs...) + selectNodes := sets.New(cr.Spec.Storage.MaintenanceSelectNodeIDs...) + return insertNodes.Intersection(selectNodes) +} + // LastSpecUpdated compares spec with last applied spec stored, replaces old spec and returns true if it's updated func (cr *VTCluster) LastSpecUpdated() bool { updated := cr.Status.LastAppliedSpec == nil || !equality.Semantic.DeepEqual(&cr.Spec, cr.Status.LastAppliedSpec) diff --git a/api/operator/v1beta1/vmcluster_types.go b/api/operator/v1beta1/vmcluster_types.go index 55184a062b..66787dec49 100644 --- a/api/operator/v1beta1/vmcluster_types.go +++ b/api/operator/v1beta1/vmcluster_types.go @@ -592,6 +592,8 @@ type VMStorage struct { // +optional MaintenanceInsertNodeIDs []int32 `json:"maintenanceInsertNodeIDs,omitempty"` // MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. + // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs is also exempted from the storage pod readiness wait during reconcile. + // This only reliably prevents reconcile from blocking under the default OnDelete rollingUpdateStrategy; with RollingUpdate, Kubernetes' own OrderedReady pod management can still block on an unready node. // +optional MaintenanceSelectNodeIDs []int32 `json:"maintenanceSelectNodeIDs,omitempty"` @@ -938,6 +940,17 @@ func (cr *VMCluster) AvailableStorageNodeIDs(kind ClusterComponent) []int32 { return result } +// StorageNodeIDsUnderMaintenance returns the vmstorage pod ordinals excluded from both +// insert and select routing via MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs. +func (cr *VMCluster) StorageNodeIDsUnderMaintenance() sets.Set[int32] { + if cr.Spec.VMStorage == nil { + return nil + } + insertNodes := sets.New(cr.Spec.VMStorage.MaintenanceInsertNodeIDs...) + selectNodes := sets.New(cr.Spec.VMStorage.MaintenanceSelectNodeIDs...) + return insertNodes.Intersection(selectNodes) +} + // FinalLabels adds cluster labels to the base labels and filters by prefix if needed func (cr *VMCluster) FinalLabels(kind ClusterComponent) map[string]string { v := AddClusterLabels(cr.SelectorLabels(kind), "vm") diff --git a/api/operator/v1beta1/vmcluster_types_test.go b/api/operator/v1beta1/vmcluster_types_test.go index dba5b9517a..fccb2ee054 100644 --- a/api/operator/v1beta1/vmcluster_types_test.go +++ b/api/operator/v1beta1/vmcluster_types_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/ptr" ) @@ -124,6 +125,36 @@ func TestVMCluster_AvailableStorageNodeIDs(t *testing.T) { }, ClusterComponentSelect, []int32{0, 1, 2}) } +func TestVMCluster_StorageNodeIDsUnderMaintenance(t *testing.T) { + f := func(cr *VMCluster, want sets.Set[int32]) { + t.Helper() + assert.Equal(t, want, cr.StorageNodeIDsUnderMaintenance()) + } + + // node 3 is excluded from both insert and select routing + f(&VMCluster{ + Spec: VMClusterSpec{ + VMStorage: &VMStorage{ + MaintenanceSelectNodeIDs: []int32{1, 3}, + MaintenanceInsertNodeIDs: []int32{0, 3}, + }, + }, + }, sets.New[int32](3)) + + // no overlap between the two lists + f(&VMCluster{ + Spec: VMClusterSpec{ + VMStorage: &VMStorage{ + MaintenanceSelectNodeIDs: []int32{1}, + MaintenanceInsertNodeIDs: []int32{0}, + }, + }, + }, sets.New[int32]()) + + // no VMStorage + f(&VMCluster{}, nil) +} + func TestVMCluster_Validate(t *testing.T) { f := func(spec VMClusterSpec, wantErr bool) { t.Helper() diff --git a/config/crd/overlay/crd.yaml b/config/crd/overlay/crd.yaml index 6be0b26dd3..6bad6553b8 100644 --- a/config/crd/overlay/crd.yaml +++ b/config/crd/overlay/crd.yaml @@ -7254,9 +7254,10 @@ spec: type: integer type: array maintenanceSelectNodeIDs: - description: MaintenanceSelectNodeIDs - excludes given node ids - from select requests routing, must contain pod suffixes - for - pod-0, id will be 0 and etc. + description: |- + MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. + A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs is also exempted from the storage pod readiness wait during reconcile. + This only reliably prevents reconcile from blocking under the default OnDelete rollingUpdateStrategy; with RollingUpdate, Kubernetes' own OrderedReady pod management can still block on an unready node. items: format: int32 type: integer @@ -49350,9 +49351,10 @@ spec: type: integer type: array maintenanceSelectNodeIDs: - description: MaintenanceSelectNodeIDs - excludes given node ids - from select requests routing, must contain pod suffixes - for - pod-0, id will be 0 and etc. + description: |- + MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. + A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs is also exempted from the storage pod readiness wait during reconcile. + This only reliably prevents reconcile from blocking under the default OnDelete rollingUpdateStrategy; with RollingUpdate, Kubernetes' own OrderedReady pod management can still block on an unready node. items: format: int32 type: integer @@ -93102,9 +93104,10 @@ spec: type: integer type: array maintenanceSelectNodeIDs: - description: MaintenanceSelectNodeIDs - excludes given node ids - from select requests routing, must contain pod suffixes - for - pod-0, id will be 0 and etc. + description: |- + MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. + A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs is also exempted from the storage pod readiness wait during reconcile. + This only reliably prevents reconcile from blocking under the default OnDelete rollingUpdateStrategy; with RollingUpdate, Kubernetes' own OrderedReady pod management can still block on an unready node. items: format: int32 type: integer diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 907bd8702b..c74d095593 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -34,6 +34,7 @@ aliases: * BUGFIX: [helm-converter](https://docs.victoriametrics.com/operator/helm-converter/): convert the `victoria-metrics-alert` chart's legacy values the same way the chart does: `server.remote.read`, `server.remote.write`, `server.notifier.alertmanager`, nested `basicAuth`/`bearer` auth keys and a list-valued `server.notifier.url`. Previously, they were silently dropped, so the generated `VMAlert` lost its remote storage, notifiers and credentials. Plaintext `server.remoteWrite` credentials, legacy or not, are now moved into a generated Secret like the other blocks; before, they were dropped too. The bug has existed since the converter was added in [v0.69.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.69.0). See [#2601](https://github.com/VictoriaMetrics/operator/issues/2601). * BUGFIX: [helm-converter](https://docs.victoriametrics.com/operator/helm-converter/): omit `spec.notifier` for the `victoria-metrics-alert` chart when no notifier URL is set. Previously, the converter generated `spec.notifier: {}`, which the operator webhook rejects, so the output could not be applied. The conversion now fails with an explicit error when the values contain alerting rules but no notifier, since the operator would silently ignore those rules, and when `alertmanager.enabled: true` would make the chart use its bundled Alertmanager, which has no CR equivalent. See [#2601](https://github.com/VictoriaMetrics/operator/issues/2601). * BUGFIX: [helm-converter](https://docs.victoriametrics.com/operator/helm-converter/): convert the charts' `license` and `global.license` blocks into `spec.license` of `VMSingle`, `VMCluster`, `VMAgent`, `VMAlert`, `VMAuth`, `VMAnomaly`, `VLCluster` and `VLAgent` (from both `victoria-logs-agent` and `victoria-logs-collector`). Previously, they were dropped, so the generated CRs started without the enterprise license. The precedence matches the charts: a plaintext key (`license.key`, then `global.license.key`) wins over any Secret reference, which is taken from `license.secret`, `global.license.secret`, `license.keyRef` and `global.license.keyRef`, in that order. See [#2601](https://github.com/VictoriaMetrics/operator/issues/2601). +* BUGFIX: [vmcluster](https://docs.victoriametrics.com/operator/resources/vmcluster/), [vlcluster](https://docs.victoriametrics.com/operator/resources/vlcluster/), [vtcluster](https://docs.victoriametrics.com/operator/resources/vtcluster/): skip the storage pod readiness check for storage node IDs listed in both `maintenanceInsertNodeIDs` and `maintenanceSelectNodeIDs`. Previously, a storage pod stuck `Pending` (e.g. during node maintenance) blocked every reconcile at the storage step, so no change to the insert or select components could be applied until the pod recovered, even though it was already excluded from both read and write routing. See [#2665](https://github.com/VictoriaMetrics/operator/issues/2665). ## [v0.75.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.75.0) **Release date:** 25 September 2026 diff --git a/docs/api.md b/docs/api.md index cb3a36914d..903e35b8b7 100644 --- a/docs/api.md +++ b/docs/api.md @@ -584,7 +584,7 @@ Appears in: [VLClusterSpec (v1)](#v1-vlclusterspec) | logLevel#
_string_ | _(Optional)_
LogLevel for VLStorage to be configured with. | | logNewStreams#
_boolean_ | _(Required)_
LogNewStreams Whether to log creation of new streams; this can be useful for debugging of high cardinality issues with log streams; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields | | maintenanceInsertNodeIDs#
_integer array_ | _(Optional)_
MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
Useful at storage expanding, when you want to rebalance some data at cluster. | -| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. | +| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs is also exempted from the storage pod readiness wait during reconcile.
This only reliably prevents reconcile from blocking under the default OnDelete rollingUpdateStrategy; with RollingUpdate, Kubernetes' own OrderedReady pod management can still block on an unready node. | | minReadySeconds#
_integer_ | _(Optional)_
MinReadySeconds defines a minimum number of seconds to wait before starting update next pod
if previous in healthy state
Has no effect for VLogs and VMSingle | | networkPolicy#
_[EmbeddedNetworkPolicy (v1beta1)](#v1beta1-embeddednetworkpolicy)_ | _(Optional)_
NetworkPolicy defines network access rules for pods created by this CR. | | nodeSelector#
_object (keys:string, values:string)_ | _(Optional)_
NodeSelector Define which Nodes the Pods are scheduled on. | @@ -1280,7 +1280,7 @@ Appears in: [VTClusterSpec (v1)](#v1-vtclusterspec) | logLevel#
_string_ | _(Optional)_
LogLevel for VTStorage to be configured with. | | logNewStreams#
_boolean_ | _(Required)_
LogNewStreams Whether to log creation of new streams; this can be useful for debugging of high cardinality issues with log streams
see https://docs.victoriametrics.com/victoriatraces/#configure-victoriatraces | | maintenanceInsertNodeIDs#
_integer array_ | _(Optional)_
MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
Useful at storage expanding, when you want to rebalance some data at cluster. | -| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. | +| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs is also exempted from the storage pod readiness wait during reconcile.
This only reliably prevents reconcile from blocking under the default OnDelete rollingUpdateStrategy; with RollingUpdate, Kubernetes' own OrderedReady pod management can still block on an unready node. | | minReadySeconds#
_integer_ | _(Optional)_
MinReadySeconds defines a minimum number of seconds to wait before starting update next pod
if previous in healthy state
Has no effect for VLogs and VMSingle | | networkPolicy#
_[EmbeddedNetworkPolicy (v1beta1)](#v1beta1-embeddednetworkpolicy)_ | _(Optional)_
NetworkPolicy defines network access rules for pods created by this CR. | | nodeSelector#
_object (keys:string, values:string)_ | _(Optional)_
NodeSelector Define which Nodes the Pods are scheduled on. | @@ -5755,7 +5755,7 @@ Appears in: [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec) | logFormat#
_string_ | _(Optional)_
LogFormat for VMStorage to be configured with.
default or json | | logLevel#
_string_ | _(Optional)_
LogLevel for VMStorage to be configured with. | | maintenanceInsertNodeIDs#
_integer array_ | _(Optional)_
MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
Useful at storage expanding, when you want to rebalance some data at cluster. | -| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. | +| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs is also exempted from the storage pod readiness wait during reconcile.
This only reliably prevents reconcile from blocking under the default OnDelete rollingUpdateStrategy; with RollingUpdate, Kubernetes' own OrderedReady pod management can still block on an unready node. | | minReadySeconds#
_integer_ | _(Optional)_
MinReadySeconds defines a minimum number of seconds to wait before starting update next pod
if previous in healthy state
Has no effect for VLogs and VMSingle | | networkPolicy#
_[EmbeddedNetworkPolicy (v1beta1)](#v1beta1-embeddednetworkpolicy)_ | _(Optional)_
NetworkPolicy defines network access rules for pods created by this CR. | | nodeSelector#
_object (keys:string, values:string)_ | _(Optional)_
NodeSelector Define which Nodes the Pods are scheduled on. | diff --git a/docs/resources/vlcluster.md b/docs/resources/vlcluster.md index bce71d7c20..e1a219d61f 100644 --- a/docs/resources/vlcluster.md +++ b/docs/resources/vlcluster.md @@ -166,6 +166,42 @@ at the cost of temporary unavailability. The timeout for each node to become ready is controlled by the Operator environment variable `VM_PODWAITREADYTIMEOUT` (default `80s`). The poll interval is `VM_PODWAITREADYINTERVALCHECK` (default `5s`). +### Storage node maintenance mode + +`vlstorage.maintenanceInsertNodeIDs` and `vlstorage.maintenanceSelectNodeIDs` exclude the listed pod ordinals +from `vlinsert`/`vlselect` routing, for example while the underlying Kubernetes node is cordoned or drained +for maintenance and the `vlstorage` pod is expected to stay `Pending`/`NotReady` for an extended period. + +A pod ordinal listed in **both** fields (excluded from insert and select routing at the same time) is also +skipped by the Operator's readiness wait during reconciliation. Previously, such a pod being not ready blocked +every reconcile of the cluster — including unrelated changes to `vlinsert` or `vlselect` — until it recovered, +even though it was already excluded from all routing. + +```yaml +apiVersion: operator.victoriametrics.com/v1 +kind: VLCluster +metadata: + name: example +spec: + vlstorage: + replicaCount: 3 + maintenanceInsertNodeIDs: [2] + maintenanceSelectNodeIDs: [2] # pod-2 excluded from routing and from the readiness wait +``` + +**Limitations:** + +- A pod ordinal must appear in *both* `maintenanceInsertNodeIDs` and `maintenanceSelectNodeIDs` to be exempted + from the readiness wait. An ordinal excluded from only one of the two still blocks reconcile until it becomes + ready. +- The skip only reliably prevents reconcile from blocking under the default `rollingUpdateStrategy: OnDelete`. + With `rollingUpdateStrategy: RollingUpdate`, the StatefulSet uses Kubernetes' native `OrderedReady` pod + management, which itself refuses to create or update any pod at a higher ordinal until a lower-ordinal pod + is ready. In that mode a maintenance pod can still stall the rollout — this is enforced by Kubernetes itself, + not the Operator, and cannot be bypassed from the Operator side. +- The skip only affects the Operator's own readiness wait; it does not change the pod's actual status or + affect routing, which is already controlled by the same two fields. + ### vlinsert and vlselect `vlinsert` and `vlselect` nodes support standard Deployment update parameters: diff --git a/docs/resources/vmcluster.md b/docs/resources/vmcluster.md index 6d623af04d..61865fffb4 100644 --- a/docs/resources/vmcluster.md +++ b/docs/resources/vmcluster.md @@ -408,6 +408,44 @@ spec: The timeout for each node to become ready is controlled by the Operator environment variable `VM_PODWAITREADYTIMEOUT` (default `80s`). The poll interval is `VM_PODWAITREADYINTERVALCHECK` (default `5s`). +### Storage node maintenance mode + +`vmstorage.maintenanceInsertNodeIDs` and `vmstorage.maintenanceSelectNodeIDs` exclude the listed pod ordinals +from `vminsert`/`vmselect` routing, for example while the underlying Kubernetes node is cordoned or drained +for maintenance and the `vmstorage` pod is expected to stay `Pending`/`NotReady` for an extended period. + +A pod ordinal listed in **both** fields (excluded from insert and select routing at the same time) is also +skipped by the Operator's readiness wait during reconciliation. Previously, such a pod being not ready blocked +every reconcile of the cluster — including unrelated changes to `vminsert` or `vmselect` — until it recovered, +even though it was already excluded from all routing. + +```yaml +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMCluster +metadata: + name: example +spec: + vmstorage: + replicaCount: 3 + maintenanceInsertNodeIDs: [2] + maintenanceSelectNodeIDs: [2] # pod-2 excluded from routing and from the readiness wait +``` + +**Limitations:** + +- A pod ordinal must appear in *both* `maintenanceInsertNodeIDs` and `maintenanceSelectNodeIDs` to be exempted + from the readiness wait. An ordinal excluded from only one of the two still blocks reconcile until it becomes + ready. +- The skip only reliably prevents reconcile from blocking under the default `rollingUpdateStrategy: OnDelete`. + With `rollingUpdateStrategy: RollingUpdate`, the StatefulSet uses Kubernetes' native `OrderedReady` pod + management, which itself refuses to create or update any pod at a higher ordinal until a lower-ordinal pod + is ready. In that mode a maintenance pod can still stall the rollout — this is enforced by Kubernetes itself, + not the Operator, and cannot be bypassed from the Operator side. +- These fields cannot be combined with [automatic vmstorage discovery](#automatic-vmstorage-discovery), since + node selection is delegated to the `filter` regexp there. +- The skip only affects the Operator's own readiness wait; it does not change the pod's actual status or + affect routing, which is already controlled by the same two fields. + ### vminsert `vminsert` nodes can be configured with standard Deployment update parameters: diff --git a/docs/resources/vtcluster.md b/docs/resources/vtcluster.md index 2c653a0a4f..11d9e969be 100644 --- a/docs/resources/vtcluster.md +++ b/docs/resources/vtcluster.md @@ -192,6 +192,42 @@ at the cost of temporary unavailability. The timeout for each node to become ready is controlled by the Operator environment variable `VM_PODWAITREADYTIMEOUT` (default `80s`). The poll interval is `VM_PODWAITREADYINTERVALCHECK` (default `5s`). +### Storage node maintenance mode + +`storage.maintenanceInsertNodeIDs` and `storage.maintenanceSelectNodeIDs` exclude the listed pod ordinals +from `insert`/`select` routing, for example while the underlying Kubernetes node is cordoned or drained for +maintenance and the `storage` pod is expected to stay `Pending`/`NotReady` for an extended period. + +A pod ordinal listed in **both** fields (excluded from insert and select routing at the same time) is also +skipped by the Operator's readiness wait during reconciliation. Previously, such a pod being not ready blocked +every reconcile of the cluster — including unrelated changes to `insert` or `select` — until it recovered, +even though it was already excluded from all routing. + +```yaml +apiVersion: operator.victoriametrics.com/v1 +kind: VTCluster +metadata: + name: example +spec: + storage: + replicaCount: 3 + maintenanceInsertNodeIDs: [2] + maintenanceSelectNodeIDs: [2] # pod-2 excluded from routing and from the readiness wait +``` + +**Limitations:** + +- A pod ordinal must appear in *both* `maintenanceInsertNodeIDs` and `maintenanceSelectNodeIDs` to be exempted + from the readiness wait. An ordinal excluded from only one of the two still blocks reconcile until it becomes + ready. +- The skip only reliably prevents reconcile from blocking under the default `rollingUpdateStrategy: OnDelete`. + With `rollingUpdateStrategy: RollingUpdate`, the StatefulSet uses Kubernetes' native `OrderedReady` pod + management, which itself refuses to create or update any pod at a higher ordinal until a lower-ordinal pod + is ready. In that mode a maintenance pod can still stall the rollout — this is enforced by Kubernetes itself, + not the Operator, and cannot be bypassed from the Operator side. +- The skip only affects the Operator's own readiness wait; it does not change the pod's actual status or + affect routing, which is already controlled by the same two fields. + ### insert and select `insert` and `select` nodes support standard Deployment update parameters: diff --git a/internal/controller/operator/factory/reconcile/statefulset.go b/internal/controller/operator/factory/reconcile/statefulset.go index bc3aaab490..723dece076 100644 --- a/internal/controller/operator/factory/reconcile/statefulset.go +++ b/internal/controller/operator/factory/reconcile/statefulset.go @@ -35,12 +35,13 @@ const podRevisionLabel = "controller-revision-hash" // StatefulSetOpts options for StatefulSet update // HPA and UpdateReplicaCount optional type StatefulSetOpts struct { - SelectorLabels map[string]string - PatchSpec func(existingSpec, newSpec *appsv1.StatefulSetSpec) - UpdateBehavior *vmv1beta1.StatefulSetUpdateStrategyBehavior + SelectorLabels map[string]string + PatchSpec func(existingSpec, newSpec *appsv1.StatefulSetSpec) + UpdateBehavior *vmv1beta1.StatefulSetUpdateStrategyBehavior + IgnoreNotReadyPodOrdinals sets.Set[int32] } -func waitForStatefulSetReady(ctx context.Context, rclient client.Client, newObj *appsv1.StatefulSet) error { +func waitForStatefulSetReady(ctx context.Context, rclient client.Client, newObj *appsv1.StatefulSet, ignoreNotReadyPodOrdinals sets.Set[int32]) error { err := wait.PollUntilContextTimeout(ctx, podWaitReadyInterval, appWaitReadyTimeout, true, func(ctx context.Context) (done bool, err error) { var existingObj appsv1.StatefulSet if err := rclient.Get(ctx, types.NamespacedName{Namespace: newObj.Namespace, Name: newObj.Name}, &existingObj); err != nil { @@ -64,9 +65,35 @@ func waitForStatefulSetReady(ctx context.Context, rclient client.Client, newObj if targetReplicas == nil { return true, nil } - if *targetReplicas != existingObj.Status.ReadyReplicas || *targetReplicas != existingObj.Status.UpdatedReplicas { + if *targetReplicas != existingObj.Status.UpdatedReplicas { return false, nil } + if ignoreNotReadyPodOrdinals.Len() == 0 { + return *targetReplicas == existingObj.Status.ReadyReplicas, nil + } + var podList corev1.PodList + if err := rclient.List(ctx, &podList, &client.ListOptions{ + Namespace: newObj.Namespace, + LabelSelector: labels.SelectorFromSet(newObj.Spec.Selector.MatchLabels), + }); err != nil { + return false, err + } + readyOrdinals := sets.New[int32]() + for _, pod := range podList.Items { + if PodIsReady(&pod, newObj.Spec.MinReadySeconds) { + if ordinal, err := podOrdinal(pod.Name); err == nil { + readyOrdinals.Insert(ordinal) + } + } + } + for i := int32(0); i < *targetReplicas; i++ { + if ignoreNotReadyPodOrdinals.Has(i) { + continue + } + if !readyOrdinals.Has(i) { + return false, nil + } + } return true, nil }) if err != nil { @@ -177,9 +204,10 @@ func StatefulSet(ctx context.Context, rclient client.Client, newObj, prevObj *ap switch updateStrategy { case appsv1.OnDeleteStatefulSetStrategyType: opts := rollingUpdateOpts{ - recreate: recreatePod, - selector: o.SelectorLabels, - maxUnavailable: 1, + recreate: recreatePod, + selector: o.SelectorLabels, + maxUnavailable: 1, + ignoreNotReadyOrdinals: o.IgnoreNotReadyPodOrdinals, } if o.UpdateBehavior != nil { if o.UpdateBehavior.MaxUnavailable.String() == "100%" { @@ -199,7 +227,7 @@ func StatefulSet(ctx context.Context, rclient client.Client, newObj, prevObj *ap if hpaManaged { newObj.Spec.Replicas = nil } - if err := waitForStatefulSetReady(ctx, rclient, newObj); err != nil { + if err := waitForStatefulSetReady(ctx, rclient, newObj, o.IgnoreNotReadyPodOrdinals); err != nil { return fmt.Errorf("cannot ensure that statefulset is ready with strategy=%q: %w", updateStrategy, err) } } @@ -229,10 +257,11 @@ func getLatestStsState(ctx context.Context, rclient client.Client, targetSTS typ } type rollingUpdateOpts struct { - recreate bool - maxUnavailable int - selector map[string]string - delete bool + recreate bool + maxUnavailable int + selector map[string]string + delete bool + ignoreNotReadyOrdinals sets.Set[int32] } // patchSTSCurrentRevision patches statefulset status.currentRevision to match status.updateRevision @@ -326,10 +355,27 @@ func performRollingUpdateOnSts(ctx context.Context, rclient client.Client, obj * return patchSTSCurrentRevision(ctx, rclient, nsn, stsVersion, int32(neededPodCount)) } + // defer updates for pods excluded by maintenance node IDs + var deferredUpdate bool + filteredPodsForUpdate := podsForUpdate[:0] + for _, pod := range podsForUpdate { + if ordinal, err := podOrdinal(pod.Name); err == nil && o.ignoreNotReadyOrdinals.Has(ordinal) { + l.Info(fmt.Sprintf("deferring update for pod %s, excluded by maintenance node IDs", pod.Name)) + deferredUpdate = true + continue + } + filteredPodsForUpdate = append(filteredPodsForUpdate, pod) + } + podsForUpdate = filteredPodsForUpdate + l.Info(fmt.Sprintf("discovered already updated pods=%d, pods needed to be update=%d", len(updatedPods), len(podsForUpdate))) // check updated, by not ready pods for _, pod := range updatedPods { + if ordinal, err := podOrdinal(pod.Name); err == nil && o.ignoreNotReadyOrdinals.Has(ordinal) { + l.Info(fmt.Sprintf("skipping not-ready check for pod %s, excluded by maintenance node IDs", pod.Name)) + continue + } l.Info(fmt.Sprintf("checking ready status for already updated pod %s to revision version=%q", pod.Name, stsVersion)) podNsn := types.NamespacedName{Namespace: obj.Namespace, Name: pod.Name} if err := waitForPodReady(ctx, rclient, podNsn, stsVersion, sts.Spec.MinReadySeconds); err != nil { @@ -395,6 +441,11 @@ func performRollingUpdateOnSts(ctx context.Context, rclient client.Client, obj * } } + if deferredUpdate { + l.Info("statefulset update is incomplete: some pods were deferred due to maintenance node IDs") + return nil + } + l.Info(fmt.Sprintf("finished statefulset update from revision=%q to revision=%q", sts.Status.CurrentRevision, stsVersion)) return patchSTSCurrentRevision(ctx, rclient, nsn, stsVersion, int32(neededPodCount)) @@ -495,6 +546,18 @@ func podStatusesToError(origin error, pod *corev1.Pod) error { return fmt.Errorf("%s: %w", msg, origin) } +func podOrdinal(name string) (int32, error) { + n := strings.LastIndexByte(name, '-') + if n <= 0 { + return 0, fmt.Errorf("cannot find - at the pod name: %s", name) + } + id, err := strconv.ParseInt(name[n+1:], 10, 32) + if err != nil { + return 0, fmt.Errorf("cannot parse pod id number: %s from name: %s", name[n+1:], name) + } + return int32(id), nil +} + func sortStsPodsByID(src []corev1.Pod) error { var firstParseError error sort.Slice(src, func(i, j int) bool { diff --git a/internal/controller/operator/factory/reconcile/statefulset_test.go b/internal/controller/operator/factory/reconcile/statefulset_test.go index 1b3b17ea01..82cfbb466e 100644 --- a/internal/controller/operator/factory/reconcile/statefulset_test.go +++ b/internal/controller/operator/factory/reconcile/statefulset_test.go @@ -18,6 +18,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" @@ -803,6 +804,141 @@ func Test_performRollingUpdateOnSts(t *testing.T) { }, wantErr: true, }) + + // not-ready pod at current revision times out + f(opts{ + sts: &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vmstorage-sts", + Namespace: "default", + }, + Status: appsv1.StatefulSetStatus{ + CurrentRevision: "rev1", + UpdateRevision: "rev1", + }, + }, + opts: rollingUpdateOpts{ + selector: map[string]string{"app": "vmstorage"}, + maxUnavailable: 1, + }, + predefinedObjects: []runtime.Object{ + &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vmstorage-sts", + Namespace: "default", + Labels: map[string]string{"app": "vmstorage"}, + }, + Spec: appsv1.StatefulSetSpec{Replicas: ptr.To(int32(1))}, + Status: appsv1.StatefulSetStatus{ + CurrentRevision: "rev1", + UpdateRevision: "rev1", + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vmstorage-sts-0", + Namespace: "default", + Labels: map[string]string{"app": "vmstorage", podRevisionLabel: "rev1"}, + OwnerReferences: []metav1.OwnerReference{{ + Kind: "StatefulSet", + }}, + }, + Status: corev1.PodStatus{}, + }, + }, + wantErr: true, + }) + + // not-ready pod excluded via ignoreNotReadyOrdinals does not block the update + f(opts{ + sts: &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vmstorage-sts", + Namespace: "default", + }, + Status: appsv1.StatefulSetStatus{ + CurrentRevision: "rev1", + UpdateRevision: "rev1", + }, + }, + opts: rollingUpdateOpts{ + selector: map[string]string{"app": "vmstorage"}, + maxUnavailable: 1, + ignoreNotReadyOrdinals: sets.New[int32](0), + }, + predefinedObjects: []runtime.Object{ + &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vmstorage-sts", + Namespace: "default", + Labels: map[string]string{"app": "vmstorage"}, + }, + Spec: appsv1.StatefulSetSpec{Replicas: ptr.To(int32(1))}, + Status: appsv1.StatefulSetStatus{ + CurrentRevision: "rev1", + UpdateRevision: "rev1", + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vmstorage-sts-0", + Namespace: "default", + Labels: map[string]string{"app": "vmstorage", podRevisionLabel: "rev1"}, + OwnerReferences: []metav1.OwnerReference{{ + Kind: "StatefulSet", + }}, + }, + Status: corev1.PodStatus{}, + }, + }, + actions: map[string][]string{}, + }) + + // a pod on the old revision excluded via ignoreNotReadyOrdinals is deferred, not + // evicted and waited on + f(opts{ + sts: &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vmstorage-sts", + Namespace: "default", + }, + Status: appsv1.StatefulSetStatus{ + CurrentRevision: "rev0", + UpdateRevision: "rev1", + }, + }, + opts: rollingUpdateOpts{ + selector: map[string]string{"app": "vmstorage"}, + maxUnavailable: 1, + ignoreNotReadyOrdinals: sets.New[int32](0), + }, + predefinedObjects: []runtime.Object{ + &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vmstorage-sts", + Namespace: "default", + Labels: map[string]string{"app": "vmstorage"}, + }, + Spec: appsv1.StatefulSetSpec{Replicas: ptr.To(int32(1))}, + Status: appsv1.StatefulSetStatus{ + CurrentRevision: "rev0", + UpdateRevision: "rev1", + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vmstorage-sts-0", + Namespace: "default", + Labels: map[string]string{"app": "vmstorage", podRevisionLabel: "rev0"}, + OwnerReferences: []metav1.OwnerReference{{ + Kind: "StatefulSet", + }}, + }, + Status: corev1.PodStatus{}, + }, + }, + actions: map[string][]string{}, + }) } func TestSortPodsByID(t *testing.T) { @@ -1363,6 +1499,73 @@ func TestStatefulsetReconcile(t *testing.T) { assert.Equal(t, ptr.To[int32](6), s.Spec.Replicas) }, }) + + // RollingUpdate strategy: excluded pod ordinal doesn't count against readiness + mkPod := func(idx int, ready bool) *corev1.Pod { + p := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("test-1-%d", idx), + Namespace: "default", + Labels: map[string]string{"label": "value"}, + }, + } + if ready { + p.Status = corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}, + } + } + return p + } + f(opts{ + new: getSts(func(s *appsv1.StatefulSet) { + s.Spec.UpdateStrategy.Type = appsv1.RollingUpdateStatefulSetStrategyType + }), + prev: getSts(), + predefinedObjects: []runtime.Object{ + getSts(func(s *appsv1.StatefulSet) { + s.Spec.UpdateStrategy.Type = appsv1.RollingUpdateStatefulSetStrategyType + s.Status.Replicas = 4 + s.Status.ReadyReplicas = 3 + s.Status.UpdatedReplicas = 4 + }), + mkPod(0, true), + mkPod(1, true), + mkPod(2, true), + mkPod(3, false), + }, + actions: []k8stools.ClientAction{ + {Verb: "Get", Kind: "StatefulSet", Resource: nn}, + {Verb: "Get", Kind: "StatefulSet", Resource: nn}, + }, + o: &StatefulSetOpts{ + IgnoreNotReadyPodOrdinals: sets.New[int32](3), + }, + }) + + // RollingUpdate strategy: an unready non-maintenance pod still blocks readiness + f(opts{ + new: getSts(func(s *appsv1.StatefulSet) { + s.Spec.UpdateStrategy.Type = appsv1.RollingUpdateStatefulSetStrategyType + }), + prev: getSts(), + predefinedObjects: []runtime.Object{ + getSts(func(s *appsv1.StatefulSet) { + s.Spec.UpdateStrategy.Type = appsv1.RollingUpdateStatefulSetStrategyType + s.Status.Replicas = 4 + s.Status.ReadyReplicas = 3 + s.Status.UpdatedReplicas = 4 + }), + mkPod(0, true), + mkPod(1, false), + mkPod(2, true), + mkPod(3, true), + }, + o: &StatefulSetOpts{ + IgnoreNotReadyPodOrdinals: sets.New[int32](3), + }, + wantErr: true, + }) } func TestStatefulsetReconcile_ExistingObjNotStaleAfterNotFoundRetry(t *testing.T) { diff --git a/internal/controller/operator/factory/vlcluster/vlstorage.go b/internal/controller/operator/factory/vlcluster/vlstorage.go index 016eb4237b..6842a7c352 100644 --- a/internal/controller/operator/factory/vlcluster/vlstorage.go +++ b/internal/controller/operator/factory/vlcluster/vlstorage.go @@ -189,6 +189,7 @@ func createOrUpdateVLStorageSTS(ctx context.Context, rclient client.Client, cr, cr.Spec.VLStorage.ReplicaCount = existingSpec.Replicas } }, + IgnoreNotReadyPodOrdinals: cr.StorageNodeIDsUnderMaintenance(), } owner := cr.AsOwner() return reconcile.StatefulSet(ctx, rclient, newSts, prevSts, &owner, &o) diff --git a/internal/controller/operator/factory/vmcluster/vmcluster.go b/internal/controller/operator/factory/vmcluster/vmcluster.go index b72d3b1ef7..37ca976ca9 100644 --- a/internal/controller/operator/factory/vmcluster/vmcluster.go +++ b/internal/controller/operator/factory/vmcluster/vmcluster.go @@ -458,6 +458,7 @@ func createOrUpdateVMStorage(ctx context.Context, rclient client.Client, cr, pre cr.Spec.VMStorage.ReplicaCount = existingSpec.Replicas } }, + IgnoreNotReadyPodOrdinals: cr.StorageNodeIDsUnderMaintenance(), } owner := cr.AsOwner() return reconcile.StatefulSet(ctx, rclient, newSts, prevSts, &owner, &o) diff --git a/internal/controller/operator/factory/vmcluster/vmcluster_test.go b/internal/controller/operator/factory/vmcluster/vmcluster_test.go index 33f3d57dea..e917d6a838 100644 --- a/internal/controller/operator/factory/vmcluster/vmcluster_test.go +++ b/internal/controller/operator/factory/vmcluster/vmcluster_test.go @@ -102,11 +102,11 @@ func TestCreateOrUpdate(t *testing.T) { predefinedObjects: []runtime.Object{ &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-0", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-1", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, }, }) @@ -197,11 +197,11 @@ func TestCreateOrUpdate(t *testing.T) { predefinedObjects: []runtime.Object{ &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "select-0", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmselect", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "select-1", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmselect", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, }, }) @@ -235,43 +235,43 @@ func TestCreateOrUpdate(t *testing.T) { predefinedObjects: []runtime.Object{ &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-0", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-1", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-2", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-3", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-4", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-5", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-6", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-7", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-8", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-9", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, }, }) @@ -305,43 +305,43 @@ func TestCreateOrUpdate(t *testing.T) { predefinedObjects: []runtime.Object{ &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-0", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-1", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-2", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-3", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-4", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-5", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-6", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-7", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-8", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-9", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, }, }) diff --git a/internal/controller/operator/factory/vtcluster/storage.go b/internal/controller/operator/factory/vtcluster/storage.go index 91009e6d99..49472c4ff3 100644 --- a/internal/controller/operator/factory/vtcluster/storage.go +++ b/internal/controller/operator/factory/vtcluster/storage.go @@ -186,6 +186,7 @@ func createOrUpdateVTStorageSTS(ctx context.Context, rclient client.Client, cr, cr.Spec.Storage.ReplicaCount = existingSpec.Replicas } }, + IgnoreNotReadyPodOrdinals: cr.StorageNodeIDsUnderMaintenance(), } owner := cr.AsOwner() return reconcile.StatefulSet(ctx, rclient, newSts, prevSts, &owner, &o) From 4bc910bb6e77c2c2f583dd1146dd4a469eb539ca Mon Sep 17 00:00:00 2001 From: Hui Wang Date: Wed, 21 Oct 2026 12:37:40 +0800 Subject: [PATCH 2/5] allow skip on terminating pod and skip pvc resize for maintenance pod as well --- api/operator/v1/vlcluster_types.go | 5 +- api/operator/v1/vtcluster_types.go | 5 +- api/operator/v1beta1/vmcluster_types.go | 5 +- config/crd/overlay/crd.yaml | 15 +-- docs/CHANGELOG.md | 2 +- docs/api.md | 6 +- docs/resources/vlcluster.md | 86 ++++++++------- docs/resources/vmcluster.md | 79 ++++++++------ docs/resources/vtcluster.md | 86 ++++++++------- .../operator/factory/reconcile/statefulset.go | 14 ++- .../reconcile/statefulset_pvc_expand.go | 15 ++- .../reconcile/statefulset_pvc_expand_test.go | 101 +++++++++++++++++- .../factory/reconcile/statefulset_test.go | 84 +++++++++++---- .../factory/vmcluster/vmcluster_test.go | 44 ++++---- 14 files changed, 378 insertions(+), 169 deletions(-) diff --git a/api/operator/v1/vlcluster_types.go b/api/operator/v1/vlcluster_types.go index 1f8d6ac94c..feb0492510 100644 --- a/api/operator/v1/vlcluster_types.go +++ b/api/operator/v1/vlcluster_types.go @@ -652,8 +652,9 @@ type VLStorage struct { // +optional MaintenanceInsertNodeIDs []int32 `json:"maintenanceInsertNodeIDs,omitempty"` // MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. - // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs is also exempted from the storage pod readiness wait during reconcile. - // This only reliably prevents reconcile from blocking under the default OnDelete rollingUpdateStrategy; with RollingUpdate, Kubernetes' own OrderedReady pod management can still block on an unready node. + // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic, so the operator doesn't wait for its pod to become ready + // and doesn't apply spec changes to the pod and its volumes until the id is removed from either list. + // See https://docs.victoriametrics.com/operator/resources/vlcluster/#pods-excluded-from-all-routing // +optional MaintenanceSelectNodeIDs []int32 `json:"maintenanceSelectNodeIDs,omitempty"` diff --git a/api/operator/v1/vtcluster_types.go b/api/operator/v1/vtcluster_types.go index 9348ff91f9..9c24c3876d 100644 --- a/api/operator/v1/vtcluster_types.go +++ b/api/operator/v1/vtcluster_types.go @@ -515,8 +515,9 @@ type VTStorage struct { // +optional MaintenanceInsertNodeIDs []int32 `json:"maintenanceInsertNodeIDs,omitempty"` // MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. - // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs is also exempted from the storage pod readiness wait during reconcile. - // This only reliably prevents reconcile from blocking under the default OnDelete rollingUpdateStrategy; with RollingUpdate, Kubernetes' own OrderedReady pod management can still block on an unready node. + // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic, so the operator doesn't wait for its pod to become ready + // and doesn't apply spec changes to the pod and its volumes until the id is removed from either list. + // See https://docs.victoriametrics.com/operator/resources/vtcluster/#pods-excluded-from-all-routing // +optional MaintenanceSelectNodeIDs []int32 `json:"maintenanceSelectNodeIDs,omitempty"` diff --git a/api/operator/v1beta1/vmcluster_types.go b/api/operator/v1beta1/vmcluster_types.go index 66787dec49..33c0cc2ae8 100644 --- a/api/operator/v1beta1/vmcluster_types.go +++ b/api/operator/v1beta1/vmcluster_types.go @@ -592,8 +592,9 @@ type VMStorage struct { // +optional MaintenanceInsertNodeIDs []int32 `json:"maintenanceInsertNodeIDs,omitempty"` // MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. - // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs is also exempted from the storage pod readiness wait during reconcile. - // This only reliably prevents reconcile from blocking under the default OnDelete rollingUpdateStrategy; with RollingUpdate, Kubernetes' own OrderedReady pod management can still block on an unready node. + // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic, so the operator doesn't wait for its pod to become ready + // and doesn't apply spec changes to the pod and its volumes until the id is removed from either list. + // See https://docs.victoriametrics.com/operator/resources/vmcluster/#pods-excluded-from-all-routing // +optional MaintenanceSelectNodeIDs []int32 `json:"maintenanceSelectNodeIDs,omitempty"` diff --git a/config/crd/overlay/crd.yaml b/config/crd/overlay/crd.yaml index 6bad6553b8..a695c08243 100644 --- a/config/crd/overlay/crd.yaml +++ b/config/crd/overlay/crd.yaml @@ -7256,8 +7256,9 @@ spec: maintenanceSelectNodeIDs: description: |- MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. - A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs is also exempted from the storage pod readiness wait during reconcile. - This only reliably prevents reconcile from blocking under the default OnDelete rollingUpdateStrategy; with RollingUpdate, Kubernetes' own OrderedReady pod management can still block on an unready node. + A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic, so the operator doesn't wait for its pod to become ready + and doesn't apply spec changes to the pod and its volumes until the id is removed from either list. + See https://docs.victoriametrics.com/operator/resources/vlcluster/#pods-excluded-from-all-routing items: format: int32 type: integer @@ -49353,8 +49354,9 @@ spec: maintenanceSelectNodeIDs: description: |- MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. - A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs is also exempted from the storage pod readiness wait during reconcile. - This only reliably prevents reconcile from blocking under the default OnDelete rollingUpdateStrategy; with RollingUpdate, Kubernetes' own OrderedReady pod management can still block on an unready node. + A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic, so the operator doesn't wait for its pod to become ready + and doesn't apply spec changes to the pod and its volumes until the id is removed from either list. + See https://docs.victoriametrics.com/operator/resources/vmcluster/#pods-excluded-from-all-routing items: format: int32 type: integer @@ -93106,8 +93108,9 @@ spec: maintenanceSelectNodeIDs: description: |- MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. - A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs is also exempted from the storage pod readiness wait during reconcile. - This only reliably prevents reconcile from blocking under the default OnDelete rollingUpdateStrategy; with RollingUpdate, Kubernetes' own OrderedReady pod management can still block on an unready node. + A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic, so the operator doesn't wait for its pod to become ready + and doesn't apply spec changes to the pod and its volumes until the id is removed from either list. + See https://docs.victoriametrics.com/operator/resources/vtcluster/#pods-excluded-from-all-routing items: format: int32 type: integer diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index c74d095593..43ff82979b 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -19,6 +19,7 @@ aliases: * FEATURE: [vmuser](https://docs.victoriametrics.com/operator/resources/vmuser/), [vmauth](https://docs.victoriametrics.com/operator/resources/vmauth/): allow `VTAgent` in `targetRef.crd.kind`, so `vmauth` can route to a `VTAgent` by name instead of a static URL. The CRDs must be updated to use it. See [#2687](https://github.com/VictoriaMetrics/operator/issues/2687). * FEATURE: [vmanomaly](https://docs.victoriametrics.com/operator/resources/vmanomaly/), [vmanomalyconfig](https://docs.victoriametrics.com/operator/resources/vmanomalyconfig/): support the experimental `peer_outlier` model and its peer-group parameters in managed configuration, available since vmanomaly [v1.31.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1310). See [#2714](https://github.com/VictoriaMetrics/operator/pull/2714). +* FEATURE: [vmcluster](https://docs.victoriametrics.com/operator/resources/vmcluster/), [vlcluster](https://docs.victoriametrics.com/operator/resources/vlcluster/), [vtcluster](https://docs.victoriametrics.com/operator/resources/vtcluster/): leave storage pods listed in both `maintenanceInsertNodeIDs` and `maintenanceSelectNodeIDs` alone during reconciliation: do not wait for them to become ready, and defer spec changes to them and their volumes until they leave maintenance. Previously, such a pod stuck `Pending` or `Terminating` (e.g. its node is under maintenance and its volume is bound to that node) failed every reconcile at the storage step, so changes to insert and select components were not applied until the pod came back. See [these docs](https://docs.victoriametrics.com/operator/resources/vmcluster/#pods-excluded-from-all-routing) and [#2665](https://github.com/VictoriaMetrics/operator/issues/2665). * BUGFIX: [vmsingle](https://docs.victoriametrics.com/operator/resources/vmsingle/), [vmcluster](https://docs.victoriametrics.com/operator/resources/vmcluster/), [vlsingle](https://docs.victoriametrics.com/operator/resources/vlsingle/), [vlcluster](https://docs.victoriametrics.com/operator/resources/vlcluster/), [vtsingle](https://docs.victoriametrics.com/operator/resources/vtsingle/), [vtcluster](https://docs.victoriametrics.com/operator/resources/vtcluster/): accept the months unit `M` in `retentionPeriod` and `futureRetention`. Previously, the CRD rejected values such as `3M`, even though VictoriaMetrics, VictoriaLogs and VictoriaTraces accept them. The CRDs must be updated to get this fix. See [#2693](https://github.com/VictoriaMetrics/operator/issues/2693). * BUGFIX: [vmanomaly](https://docs.victoriametrics.com/operator/resources/vmanomaly/): allow `spec.configRawYaml` and `spec.configSecret` without `reader.queries` when selected [VMAnomalyConfig](https://docs.victoriametrics.com/operator/resources/vmanomalyconfig/) objects provide the queries. Previously, such a config failed reconciliation with `reader.queries must be provided via configRawYaml or configSecret`, so queries could not come from `VMAnomalyConfig` objects alone. A config with no queries from any source is still rejected. This bug was introduced in [v0.69.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.69.0) with `VMAnomalyConfig`. See [#2685](https://github.com/VictoriaMetrics/operator/pull/2685). @@ -34,7 +35,6 @@ aliases: * BUGFIX: [helm-converter](https://docs.victoriametrics.com/operator/helm-converter/): convert the `victoria-metrics-alert` chart's legacy values the same way the chart does: `server.remote.read`, `server.remote.write`, `server.notifier.alertmanager`, nested `basicAuth`/`bearer` auth keys and a list-valued `server.notifier.url`. Previously, they were silently dropped, so the generated `VMAlert` lost its remote storage, notifiers and credentials. Plaintext `server.remoteWrite` credentials, legacy or not, are now moved into a generated Secret like the other blocks; before, they were dropped too. The bug has existed since the converter was added in [v0.69.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.69.0). See [#2601](https://github.com/VictoriaMetrics/operator/issues/2601). * BUGFIX: [helm-converter](https://docs.victoriametrics.com/operator/helm-converter/): omit `spec.notifier` for the `victoria-metrics-alert` chart when no notifier URL is set. Previously, the converter generated `spec.notifier: {}`, which the operator webhook rejects, so the output could not be applied. The conversion now fails with an explicit error when the values contain alerting rules but no notifier, since the operator would silently ignore those rules, and when `alertmanager.enabled: true` would make the chart use its bundled Alertmanager, which has no CR equivalent. See [#2601](https://github.com/VictoriaMetrics/operator/issues/2601). * BUGFIX: [helm-converter](https://docs.victoriametrics.com/operator/helm-converter/): convert the charts' `license` and `global.license` blocks into `spec.license` of `VMSingle`, `VMCluster`, `VMAgent`, `VMAlert`, `VMAuth`, `VMAnomaly`, `VLCluster` and `VLAgent` (from both `victoria-logs-agent` and `victoria-logs-collector`). Previously, they were dropped, so the generated CRs started without the enterprise license. The precedence matches the charts: a plaintext key (`license.key`, then `global.license.key`) wins over any Secret reference, which is taken from `license.secret`, `global.license.secret`, `license.keyRef` and `global.license.keyRef`, in that order. See [#2601](https://github.com/VictoriaMetrics/operator/issues/2601). -* BUGFIX: [vmcluster](https://docs.victoriametrics.com/operator/resources/vmcluster/), [vlcluster](https://docs.victoriametrics.com/operator/resources/vlcluster/), [vtcluster](https://docs.victoriametrics.com/operator/resources/vtcluster/): skip the storage pod readiness check for storage node IDs listed in both `maintenanceInsertNodeIDs` and `maintenanceSelectNodeIDs`. Previously, a storage pod stuck `Pending` (e.g. during node maintenance) blocked every reconcile at the storage step, so no change to the insert or select components could be applied until the pod recovered, even though it was already excluded from both read and write routing. See [#2665](https://github.com/VictoriaMetrics/operator/issues/2665). ## [v0.75.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.75.0) **Release date:** 25 September 2026 diff --git a/docs/api.md b/docs/api.md index 903e35b8b7..c092f67561 100644 --- a/docs/api.md +++ b/docs/api.md @@ -584,7 +584,7 @@ Appears in: [VLClusterSpec (v1)](#v1-vlclusterspec) | logLevel#
_string_ | _(Optional)_
LogLevel for VLStorage to be configured with. | | logNewStreams#
_boolean_ | _(Required)_
LogNewStreams Whether to log creation of new streams; this can be useful for debugging of high cardinality issues with log streams; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields | | maintenanceInsertNodeIDs#
_integer array_ | _(Optional)_
MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
Useful at storage expanding, when you want to rebalance some data at cluster. | -| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs is also exempted from the storage pod readiness wait during reconcile.
This only reliably prevents reconcile from blocking under the default OnDelete rollingUpdateStrategy; with RollingUpdate, Kubernetes' own OrderedReady pod management can still block on an unready node. | +| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic, so the operator doesn't wait for its pod to become ready
and doesn't apply spec changes to the pod and its volumes until the id is removed from either list.
See https://docs.victoriametrics.com/operator/resources/vlcluster/#pods-excluded-from-all-routing | | minReadySeconds#
_integer_ | _(Optional)_
MinReadySeconds defines a minimum number of seconds to wait before starting update next pod
if previous in healthy state
Has no effect for VLogs and VMSingle | | networkPolicy#
_[EmbeddedNetworkPolicy (v1beta1)](#v1beta1-embeddednetworkpolicy)_ | _(Optional)_
NetworkPolicy defines network access rules for pods created by this CR. | | nodeSelector#
_object (keys:string, values:string)_ | _(Optional)_
NodeSelector Define which Nodes the Pods are scheduled on. | @@ -1280,7 +1280,7 @@ Appears in: [VTClusterSpec (v1)](#v1-vtclusterspec) | logLevel#
_string_ | _(Optional)_
LogLevel for VTStorage to be configured with. | | logNewStreams#
_boolean_ | _(Required)_
LogNewStreams Whether to log creation of new streams; this can be useful for debugging of high cardinality issues with log streams
see https://docs.victoriametrics.com/victoriatraces/#configure-victoriatraces | | maintenanceInsertNodeIDs#
_integer array_ | _(Optional)_
MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
Useful at storage expanding, when you want to rebalance some data at cluster. | -| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs is also exempted from the storage pod readiness wait during reconcile.
This only reliably prevents reconcile from blocking under the default OnDelete rollingUpdateStrategy; with RollingUpdate, Kubernetes' own OrderedReady pod management can still block on an unready node. | +| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic, so the operator doesn't wait for its pod to become ready
and doesn't apply spec changes to the pod and its volumes until the id is removed from either list.
See https://docs.victoriametrics.com/operator/resources/vtcluster/#pods-excluded-from-all-routing | | minReadySeconds#
_integer_ | _(Optional)_
MinReadySeconds defines a minimum number of seconds to wait before starting update next pod
if previous in healthy state
Has no effect for VLogs and VMSingle | | networkPolicy#
_[EmbeddedNetworkPolicy (v1beta1)](#v1beta1-embeddednetworkpolicy)_ | _(Optional)_
NetworkPolicy defines network access rules for pods created by this CR. | | nodeSelector#
_object (keys:string, values:string)_ | _(Optional)_
NodeSelector Define which Nodes the Pods are scheduled on. | @@ -5755,7 +5755,7 @@ Appears in: [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec) | logFormat#
_string_ | _(Optional)_
LogFormat for VMStorage to be configured with.
default or json | | logLevel#
_string_ | _(Optional)_
LogLevel for VMStorage to be configured with. | | maintenanceInsertNodeIDs#
_integer array_ | _(Optional)_
MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
Useful at storage expanding, when you want to rebalance some data at cluster. | -| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs is also exempted from the storage pod readiness wait during reconcile.
This only reliably prevents reconcile from blocking under the default OnDelete rollingUpdateStrategy; with RollingUpdate, Kubernetes' own OrderedReady pod management can still block on an unready node. | +| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic, so the operator doesn't wait for its pod to become ready
and doesn't apply spec changes to the pod and its volumes until the id is removed from either list.
See https://docs.victoriametrics.com/operator/resources/vmcluster/#pods-excluded-from-all-routing | | minReadySeconds#
_integer_ | _(Optional)_
MinReadySeconds defines a minimum number of seconds to wait before starting update next pod
if previous in healthy state
Has no effect for VLogs and VMSingle | | networkPolicy#
_[EmbeddedNetworkPolicy (v1beta1)](#v1beta1-embeddednetworkpolicy)_ | _(Optional)_
NetworkPolicy defines network access rules for pods created by this CR. | | nodeSelector#
_object (keys:string, values:string)_ | _(Optional)_
NodeSelector Define which Nodes the Pods are scheduled on. | diff --git a/docs/resources/vlcluster.md b/docs/resources/vlcluster.md index e1a219d61f..a8df3b5dcf 100644 --- a/docs/resources/vlcluster.md +++ b/docs/resources/vlcluster.md @@ -166,42 +166,6 @@ at the cost of temporary unavailability. The timeout for each node to become ready is controlled by the Operator environment variable `VM_PODWAITREADYTIMEOUT` (default `80s`). The poll interval is `VM_PODWAITREADYINTERVALCHECK` (default `5s`). -### Storage node maintenance mode - -`vlstorage.maintenanceInsertNodeIDs` and `vlstorage.maintenanceSelectNodeIDs` exclude the listed pod ordinals -from `vlinsert`/`vlselect` routing, for example while the underlying Kubernetes node is cordoned or drained -for maintenance and the `vlstorage` pod is expected to stay `Pending`/`NotReady` for an extended period. - -A pod ordinal listed in **both** fields (excluded from insert and select routing at the same time) is also -skipped by the Operator's readiness wait during reconciliation. Previously, such a pod being not ready blocked -every reconcile of the cluster — including unrelated changes to `vlinsert` or `vlselect` — until it recovered, -even though it was already excluded from all routing. - -```yaml -apiVersion: operator.victoriametrics.com/v1 -kind: VLCluster -metadata: - name: example -spec: - vlstorage: - replicaCount: 3 - maintenanceInsertNodeIDs: [2] - maintenanceSelectNodeIDs: [2] # pod-2 excluded from routing and from the readiness wait -``` - -**Limitations:** - -- A pod ordinal must appear in *both* `maintenanceInsertNodeIDs` and `maintenanceSelectNodeIDs` to be exempted - from the readiness wait. An ordinal excluded from only one of the two still blocks reconcile until it becomes - ready. -- The skip only reliably prevents reconcile from blocking under the default `rollingUpdateStrategy: OnDelete`. - With `rollingUpdateStrategy: RollingUpdate`, the StatefulSet uses Kubernetes' native `OrderedReady` pod - management, which itself refuses to create or update any pod at a higher ordinal until a lower-ordinal pod - is ready. In that mode a maintenance pod can still stall the rollout — this is enforced by Kubernetes itself, - not the Operator, and cannot be bypassed from the Operator side. -- The skip only affects the Operator's own readiness wait; it does not change the pod's actual status or - affect routing, which is already controlled by the same two fields. - ### vlinsert and vlselect `vlinsert` and `vlselect` nodes support standard Deployment update parameters: @@ -226,6 +190,56 @@ spec: maxSurge: 1 ``` +## Storage node maintenance mode + +`vlstorage.maintenanceInsertNodeIDs` and `vlstorage.maintenanceSelectNodeIDs` exclude the listed `vlstorage` pods from `vlinsert` and `vlselect` routing. +Each ID is a pod ordinal, the numeric suffix of the pod name, e.g. `2` for the pod ending with `-2`. +Use them when a `vlstorage` pod is expected to be unavailable for a while, +for example when its Kubernetes node is under maintenance and the pod can't move, since its volume is bound to that node. + +Excluding a pod from routing affects the cluster: + +- with the pod in `maintenanceSelectNodeIDs`, logs stored on this pod are missing from query responses, + and `vlselect` doesn't mark them as partial, since it doesn't know about the pod; +- with the pod in `maintenanceInsertNodeIDs`, `vlinsert` spreads incoming logs across the remaining pods, + so they take over the excluded pod's share of writes. + +### Pods excluded from all routing + +Starting from [v0.75.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.75.1), +the Operator leaves a pod listed in **both** fields alone during reconciliation, since it receives no reads and no writes: + +- it doesn't wait for the pod to become ready, so a pod stuck `Pending` or `Terminating` no longer blocks + reconciliation of `vlinsert` and `vlselect`; +- it doesn't apply `vlstorage` spec changes to the pod or its volumes, e.g. a new image or a bigger volume size. + +A pod listed in only one of the two fields still receives the other kind of traffic, so the Operator handles it as usual. + +```yaml +apiVersion: operator.victoriametrics.com/v1 +kind: VLCluster +metadata: + name: example +spec: + vlstorage: + replicaCount: 3 + maintenanceInsertNodeIDs: [2] + maintenanceSelectNodeIDs: [2] # the pod with ordinal 2 gets no traffic and is left alone by the Operator +``` + +When the node is back, remove the ID from both fields before making other changes. +The Operator then applies the `vlstorage` changes deferred during maintenance to the pod, +and waits for it to become ready before adding it back to `vlinsert` and `vlselect`, so an unhealthy pod never receives traffic. + +**Limitations:** + +- Spec changes are deferred only with the default `rollingUpdateStrategy: OnDelete`, where the Operator updates `vlstorage` pods itself. + With `RollingUpdate`, Kubernetes rolls out all pods, including the pod under maintenance, using `OrderedReady` pod management, + which is also used when `minReadySeconds` is set. In this mode, while the pod under maintenance isn't ready, Kubernetes doesn't roll out updates, + scale the StatefulSet or recreate pods with a higher ordinal, so reconciliation that changes `vlstorage` pods can block until the pod is ready again. +- With `vlstorage.podDisruptionBudget.maxUnavailable: 1`, the pod under maintenance already uses the disruption budget, + so any `vlstorage` change that requires pod restarts fails until the pod is ready again. + ## Resource management You can specify resources for each `VLCluster` resource components in the `spec` section of the `VLCluster` CRD. diff --git a/docs/resources/vmcluster.md b/docs/resources/vmcluster.md index 61865fffb4..5b8ae0ec41 100644 --- a/docs/resources/vmcluster.md +++ b/docs/resources/vmcluster.md @@ -408,16 +408,9 @@ spec: The timeout for each node to become ready is controlled by the Operator environment variable `VM_PODWAITREADYTIMEOUT` (default `80s`). The poll interval is `VM_PODWAITREADYINTERVALCHECK` (default `5s`). -### Storage node maintenance mode - -`vmstorage.maintenanceInsertNodeIDs` and `vmstorage.maintenanceSelectNodeIDs` exclude the listed pod ordinals -from `vminsert`/`vmselect` routing, for example while the underlying Kubernetes node is cordoned or drained -for maintenance and the `vmstorage` pod is expected to stay `Pending`/`NotReady` for an extended period. +### vminsert -A pod ordinal listed in **both** fields (excluded from insert and select routing at the same time) is also -skipped by the Operator's readiness wait during reconciliation. Previously, such a pod being not ready blocked -every reconcile of the cluster — including unrelated changes to `vminsert` or `vmselect` — until it recovered, -even though it was already excluded from all routing. +`vminsert` nodes can be configured with standard Deployment update parameters: ```yaml apiVersion: operator.victoriametrics.com/v1beta1 @@ -425,30 +418,41 @@ kind: VMCluster metadata: name: example spec: - vmstorage: + vminsert: replicaCount: 3 - maintenanceInsertNodeIDs: [2] - maintenanceSelectNodeIDs: [2] # pod-2 excluded from routing and from the readiness wait + updateStrategy: RollingUpdate # default; set to Recreate for minimum downtime + rollingUpdate: + maxUnavailable: 1 + maxSurge: 1 ``` -**Limitations:** +## Storage node maintenance mode -- A pod ordinal must appear in *both* `maintenanceInsertNodeIDs` and `maintenanceSelectNodeIDs` to be exempted - from the readiness wait. An ordinal excluded from only one of the two still blocks reconcile until it becomes - ready. -- The skip only reliably prevents reconcile from blocking under the default `rollingUpdateStrategy: OnDelete`. - With `rollingUpdateStrategy: RollingUpdate`, the StatefulSet uses Kubernetes' native `OrderedReady` pod - management, which itself refuses to create or update any pod at a higher ordinal until a lower-ordinal pod - is ready. In that mode a maintenance pod can still stall the rollout — this is enforced by Kubernetes itself, - not the Operator, and cannot be bypassed from the Operator side. -- These fields cannot be combined with [automatic vmstorage discovery](#automatic-vmstorage-discovery), since - node selection is delegated to the `filter` regexp there. -- The skip only affects the Operator's own readiness wait; it does not change the pod's actual status or - affect routing, which is already controlled by the same two fields. +`vmstorage.maintenanceInsertNodeIDs` and `vmstorage.maintenanceSelectNodeIDs` exclude the listed `vmstorage` pods from `vminsert` and `vmselect` routing. +Each ID is a pod ordinal, the numeric suffix of the pod name, e.g. `2` for the pod ending with `-2`. +Use them when a `vmstorage` pod is expected to be unavailable for a while, +for example when its Kubernetes node is under maintenance and the pod can't move, since its volume is bound to that node. -### vminsert +Excluding a pod from routing affects the cluster: -`vminsert` nodes can be configured with standard Deployment update parameters: +- with the pod in `maintenanceSelectNodeIDs`, data stored only on this pod is missing from query responses, + and `vmselect` doesn't mark them as partial, since it doesn't know about the pod. + Use [replicationFactor](#high-availability) > 1 to keep such data available; +- with the pod in `maintenanceInsertNodeIDs`, `vminsert` spreads incoming series across the remaining pods, + so they take over the excluded pod's share of writes, and some series move to other pods, which increases churn there. + +These fields cannot be combined with [automatic vmstorage discovery](#automatic-vmstorage-discovery). + +### Pods excluded from all routing + +Starting from [v0.75.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.75.1), +the Operator leaves a pod listed in **both** fields alone during reconciliation, since it receives no reads and no writes: + +- it doesn't wait for the pod to become ready, so a pod stuck `Pending` or `Terminating` no longer blocks + reconciliation of `vminsert` and `vmselect`; +- it doesn't apply `vmstorage` spec changes to the pod or its volumes, e.g. a new image or a bigger volume size. + +A pod listed in only one of the two fields still receives the other kind of traffic, so the Operator handles it as usual. ```yaml apiVersion: operator.victoriametrics.com/v1beta1 @@ -456,14 +460,25 @@ kind: VMCluster metadata: name: example spec: - vminsert: + vmstorage: replicaCount: 3 - updateStrategy: RollingUpdate # default; set to Recreate for minimum downtime - rollingUpdate: - maxUnavailable: 1 - maxSurge: 1 + maintenanceInsertNodeIDs: [2] + maintenanceSelectNodeIDs: [2] # the pod with ordinal 2 gets no traffic and is left alone by the Operator ``` +When the node is back, remove the ID from both fields before making other changes. +The Operator then applies the `vmstorage` changes deferred during maintenance to the pod, +and waits for it to become ready before adding it back to `vminsert` and `vmselect`, so an unhealthy pod never receives traffic. + +**Limitations:** + +- Spec changes are deferred only with the default `rollingUpdateStrategy: OnDelete`, where the Operator updates `vmstorage` pods itself. + With `RollingUpdate`, Kubernetes rolls out all pods, including the pod under maintenance, using `OrderedReady` pod management, + which is also used when `minReadySeconds` is set. In this mode, while the pod under maintenance isn't ready, Kubernetes doesn't roll out updates, + scale the StatefulSet or recreate pods with a higher ordinal, so reconciliation that changes `vmstorage` pods can block until the pod is ready again. +- With `vmstorage.podDisruptionBudget.maxUnavailable: 1`, the pod under maintenance already uses the disruption budget, + so any `vmstorage` change that requires pod restarts fails until the pod is ready again. + ## Resource management You can specify resources for each component of `VMCluster` resource in the `spec` section of the `VMCluster` CRD. diff --git a/docs/resources/vtcluster.md b/docs/resources/vtcluster.md index 11d9e969be..c2d11de549 100644 --- a/docs/resources/vtcluster.md +++ b/docs/resources/vtcluster.md @@ -192,42 +192,6 @@ at the cost of temporary unavailability. The timeout for each node to become ready is controlled by the Operator environment variable `VM_PODWAITREADYTIMEOUT` (default `80s`). The poll interval is `VM_PODWAITREADYINTERVALCHECK` (default `5s`). -### Storage node maintenance mode - -`storage.maintenanceInsertNodeIDs` and `storage.maintenanceSelectNodeIDs` exclude the listed pod ordinals -from `insert`/`select` routing, for example while the underlying Kubernetes node is cordoned or drained for -maintenance and the `storage` pod is expected to stay `Pending`/`NotReady` for an extended period. - -A pod ordinal listed in **both** fields (excluded from insert and select routing at the same time) is also -skipped by the Operator's readiness wait during reconciliation. Previously, such a pod being not ready blocked -every reconcile of the cluster — including unrelated changes to `insert` or `select` — until it recovered, -even though it was already excluded from all routing. - -```yaml -apiVersion: operator.victoriametrics.com/v1 -kind: VTCluster -metadata: - name: example -spec: - storage: - replicaCount: 3 - maintenanceInsertNodeIDs: [2] - maintenanceSelectNodeIDs: [2] # pod-2 excluded from routing and from the readiness wait -``` - -**Limitations:** - -- A pod ordinal must appear in *both* `maintenanceInsertNodeIDs` and `maintenanceSelectNodeIDs` to be exempted - from the readiness wait. An ordinal excluded from only one of the two still blocks reconcile until it becomes - ready. -- The skip only reliably prevents reconcile from blocking under the default `rollingUpdateStrategy: OnDelete`. - With `rollingUpdateStrategy: RollingUpdate`, the StatefulSet uses Kubernetes' native `OrderedReady` pod - management, which itself refuses to create or update any pod at a higher ordinal until a lower-ordinal pod - is ready. In that mode a maintenance pod can still stall the rollout — this is enforced by Kubernetes itself, - not the Operator, and cannot be bypassed from the Operator side. -- The skip only affects the Operator's own readiness wait; it does not change the pod's actual status or - affect routing, which is already controlled by the same two fields. - ### insert and select `insert` and `select` nodes support standard Deployment update parameters: @@ -252,6 +216,56 @@ spec: maxSurge: 1 ``` +## Storage node maintenance mode + +`storage.maintenanceInsertNodeIDs` and `storage.maintenanceSelectNodeIDs` exclude the listed `storage` pods from `insert` and `select` routing. +Each ID is a pod ordinal, the numeric suffix of the pod name, e.g. `2` for the pod ending with `-2`. +Use them when a `storage` pod is expected to be unavailable for a while, +for example when its Kubernetes node is under maintenance and the pod can't move, since its volume is bound to that node. + +Excluding a pod from routing affects the cluster: + +- with the pod in `maintenanceSelectNodeIDs`, traces stored on this pod are missing from query responses, + and `select` doesn't mark them as partial, since it doesn't know about the pod; +- with the pod in `maintenanceInsertNodeIDs`, `insert` spreads incoming spans across the remaining pods, + so they take over the excluded pod's share of writes. + +### Pods excluded from all routing + +Starting from [v0.75.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.75.1), +the Operator leaves a pod listed in **both** fields alone during reconciliation, since it receives no reads and no writes: + +- it doesn't wait for the pod to become ready, so a pod stuck `Pending` or `Terminating` no longer blocks + reconciliation of `insert` and `select`; +- it doesn't apply `storage` spec changes to the pod or its volumes, e.g. a new image or a bigger volume size. + +A pod listed in only one of the two fields still receives the other kind of traffic, so the Operator handles it as usual. + +```yaml +apiVersion: operator.victoriametrics.com/v1 +kind: VTCluster +metadata: + name: example +spec: + storage: + replicaCount: 3 + maintenanceInsertNodeIDs: [2] + maintenanceSelectNodeIDs: [2] # the pod with ordinal 2 gets no traffic and is left alone by the Operator +``` + +When the node is back, remove the ID from both fields before making other changes. +The Operator then applies the `storage` changes deferred during maintenance to the pod, +and waits for it to become ready before adding it back to `insert` and `select`, so an unhealthy pod never receives traffic. + +**Limitations:** + +- Spec changes are deferred only with the default `rollingUpdateStrategy: OnDelete`, where the Operator updates `storage` pods itself. + With `RollingUpdate`, Kubernetes rolls out all pods, including the pod under maintenance, using `OrderedReady` pod management, + which is also used when `minReadySeconds` is set. In this mode, while the pod under maintenance isn't ready, Kubernetes doesn't roll out updates, + scale the StatefulSet or recreate pods with a higher ordinal, so reconciliation that changes `storage` pods can block until the pod is ready again. +- With `storage.podDisruptionBudget.maxUnavailable: 1`, the pod under maintenance already uses the disruption budget, + so any `storage` change that requires pod restarts fails until the pod is ready again. + ## Resource management You can specify resources for each `VTCluster` resource components in the `spec` section of the `VTCluster` CRD. diff --git a/internal/controller/operator/factory/reconcile/statefulset.go b/internal/controller/operator/factory/reconcile/statefulset.go index 723dece076..bcf98a4fc4 100644 --- a/internal/controller/operator/factory/reconcile/statefulset.go +++ b/internal/controller/operator/factory/reconcile/statefulset.go @@ -65,12 +65,11 @@ func waitForStatefulSetReady(ctx context.Context, rclient client.Client, newObj if targetReplicas == nil { return true, nil } - if *targetReplicas != existingObj.Status.UpdatedReplicas { - return false, nil - } if ignoreNotReadyPodOrdinals.Len() == 0 { - return *targetReplicas == existingObj.Status.ReadyReplicas, nil + return *targetReplicas == existingObj.Status.UpdatedReplicas && *targetReplicas == existingObj.Status.ReadyReplicas, nil } + // status.updatedReplicas doesn't count terminating pods, e.g. a pod stuck terminating on a shut down node, + // so check each pod, which is not excluded by ignoreNotReadyPodOrdinals, individually var podList corev1.PodList if err := rclient.List(ctx, &podList, &client.ListOptions{ Namespace: newObj.Namespace, @@ -80,6 +79,11 @@ func waitForStatefulSetReady(ctx context.Context, rclient client.Client, newObj } readyOrdinals := sets.New[int32]() for _, pod := range podList.Items { + // a pod counts only if it isn't terminating and already runs the latest revision, + // so a pod that is ready but not yet updated keeps the wait going until k8s replaces it + if !pod.DeletionTimestamp.IsZero() || pod.Labels[podRevisionLabel] != existingObj.Status.UpdateRevision { + continue + } if PodIsReady(&pod, newObj.Spec.MinReadySeconds) { if ordinal, err := podOrdinal(pod.Name); err == nil { readyOrdinals.Insert(ordinal) @@ -195,7 +199,7 @@ func StatefulSet(ctx context.Context, rclient client.Client, newObj, prevObj *ap // check if pvcs need to resize if len(newObj.Spec.VolumeClaimTemplates) > 0 { - if err := updateSTSPVC(ctx, rclient, newObj, prevVCTs, existingObj.Status.Replicas); err != nil { + if err := updateSTSPVC(ctx, rclient, newObj, prevVCTs, existingObj.Status.Replicas, o.IgnoreNotReadyPodOrdinals); err != nil { return err } } diff --git a/internal/controller/operator/factory/reconcile/statefulset_pvc_expand.go b/internal/controller/operator/factory/reconcile/statefulset_pvc_expand.go index 9633f15473..cea6967aab 100644 --- a/internal/controller/operator/factory/reconcile/statefulset_pvc_expand.go +++ b/internal/controller/operator/factory/reconcile/statefulset_pvc_expand.go @@ -15,6 +15,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -83,7 +84,7 @@ func getPVCByName(vcts []corev1.PersistentVolumeClaim, name string) *corev1.Pers return nil } -func updateSTSPVC(ctx context.Context, rclient client.Client, sts *appsv1.StatefulSet, prevVCTs []corev1.PersistentVolumeClaim, liveReplicas int32) error { +func updateSTSPVC(ctx context.Context, rclient client.Client, sts *appsv1.StatefulSet, prevVCTs []corev1.PersistentVolumeClaim, liveReplicas int32, ignorePodOrdinals sets.Set[int32]) error { // fast path if sts.Spec.Replicas != nil && *sts.Spec.Replicas == 0 { return nil @@ -129,6 +130,18 @@ func updateSTSPVC(ctx context.Context, rclient client.Client, sts *appsv1.Statef l.Info(fmt.Sprintf("possible BUG, cannot find target PVC=%s in new statefulset by claim name=%s", pvc.Name, stsClaimName)) continue } + // defer changes for pvcs of pods excluded by maintenance node IDs, the same as their pod updates. + // a resize usually completes only after the kubelet grows the filesystem on the node where the volume is mounted, + // so it cannot complete while the pod is down, and waiting for it would fail the reconcile before pod updates. + // + // changes are applied once the pod is removed from maintenance, since each reconcile compares pvcs with the claim templates. + if ordinal, err := podOrdinal(pvc.Name); err == nil && ignorePodOrdinals.Has(ordinal) { + if pvc.Spec.Resources.Requests.Storage().Cmp(*stsClaim.Spec.Resources.Requests.Storage()) != 0 || + ptr.Deref(pvc.Spec.VolumeAttributesClassName, "") != ptr.Deref(stsClaim.Spec.VolumeAttributesClassName, "") { + l.Info(fmt.Sprintf("deferring update for PVC=%s, excluded by maintenance node IDs", pvc.Name)) + } + continue + } prevVCT := getPVCByName(prevVCTs, stsClaimName) nsnPvc := types.NamespacedName{Name: pvc.Name, Namespace: pvc.Namespace} // update PVC size and metadata if it's needed, retrying on conflict diff --git a/internal/controller/operator/factory/reconcile/statefulset_pvc_expand_test.go b/internal/controller/operator/factory/reconcile/statefulset_pvc_expand_test.go index 98c1f712b5..1f42b1ec44 100644 --- a/internal/controller/operator/factory/reconcile/statefulset_pvc_expand_test.go +++ b/internal/controller/operator/factory/reconcile/statefulset_pvc_expand_test.go @@ -14,6 +14,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -160,6 +161,7 @@ func Test_updateSTSPVC(t *testing.T) { actions []k8stools.ClientAction k8sMinorVersion uint64 liveReplicas *int32 + ignoreOrdinals sets.Set[int32] } f := func(o opts) { t.Helper() @@ -181,7 +183,7 @@ func Test_updateSTSPVC(t *testing.T) { cl.Actions = nil } synctest.Test(t, func(t *testing.T) { - err := updateSTSPVC(ctx, cl, o.sts, o.prevVCTs, liveReplicas) + err := updateSTSPVC(ctx, cl, o.sts, o.prevVCTs, liveReplicas, o.ignoreOrdinals) if o.wantErrText != "" { assert.ErrorContains(t, err, o.wantErrText) } else { @@ -1751,4 +1753,101 @@ func Test_updateSTSPVC(t *testing.T) { }, }, }) + + // PVC of a pod excluded by maintenance node IDs is not updated and not waited for + maintenancePVCNSN := types.NamespacedName{Name: "vmselect-cachedir-vmselect-1", Namespace: "default"} + f(opts{ + sts: buildSTS(func(sts *appsv1.StatefulSet) { + sts.Spec.VolumeClaimTemplates = []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "vmselect-cachedir", + Labels: map[string]string{"app": "vmselect"}, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("15Gi"), + }, + }, + }, + }, + } + }), + ignoreOrdinals: sets.New[int32](1), + preRun: func(c client.Client) { + for _, nsn := range []types.NamespacedName{pvc1NSN, maintenancePVCNSN} { + assert.NoError(t, c.Create(context.TODO(), &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsn.Name, + Namespace: nsn.Namespace, + Labels: map[string]string{"app": "vmselect"}, + Annotations: map[string]string{ + "operator.victoriametrics.com/pvc-allow-volume-expansion": "true", + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + }, + }, + })) + } + }, + actions: []k8stools.ClientAction{ + {Verb: "Get", Kind: "PersistentVolumeClaim", Resource: pvc1NSN}, + {Verb: "Update", Kind: "PersistentVolumeClaim", Resource: pvc1NSN}, + {Verb: "Get", Kind: "PersistentVolumeClaim", Resource: pvc1NSN}, + }, + expected: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: pvc1NSN.Name, + Namespace: pvc1NSN.Namespace, + Labels: map[string]string{"app": "vmselect"}, + Annotations: map[string]string{ + "operator.victoriametrics.com/pvc-allow-volume-expansion": "true", + }, + ResourceVersion: "4", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("15Gi"), + }, + }, + }, + Status: corev1.PersistentVolumeClaimStatus{ + Capacity: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("15Gi"), + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: maintenancePVCNSN.Name, + Namespace: maintenancePVCNSN.Namespace, + Labels: map[string]string{"app": "vmselect"}, + Annotations: map[string]string{ + "operator.victoriametrics.com/pvc-allow-volume-expansion": "true", + }, + ResourceVersion: "2", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + }, + }, + Status: corev1.PersistentVolumeClaimStatus{ + Capacity: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + }, + }, + }, + }) } diff --git a/internal/controller/operator/factory/reconcile/statefulset_test.go b/internal/controller/operator/factory/reconcile/statefulset_test.go index 82cfbb466e..acaec3b99e 100644 --- a/internal/controller/operator/factory/reconcile/statefulset_test.go +++ b/internal/controller/operator/factory/reconcile/statefulset_test.go @@ -1501,12 +1501,12 @@ func TestStatefulsetReconcile(t *testing.T) { }) // RollingUpdate strategy: excluded pod ordinal doesn't count against readiness - mkPod := func(idx int, ready bool) *corev1.Pod { + mkPod := func(idx int, ready bool, fns ...func(*corev1.Pod)) *corev1.Pod { p := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("test-1-%d", idx), Namespace: "default", - Labels: map[string]string{"label": "value"}, + Labels: map[string]string{"label": "value", podRevisionLabel: "rev1"}, }, } if ready { @@ -1515,20 +1515,28 @@ func TestStatefulsetReconcile(t *testing.T) { Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}, } } + for _, fn := range fns { + fn(p) + } return p } - f(opts{ - new: getSts(func(s *appsv1.StatefulSet) { + mkRollingSts := func(updatedReplicas, readyReplicas int32) *appsv1.StatefulSet { + return getSts(func(s *appsv1.StatefulSet) { s.Spec.UpdateStrategy.Type = appsv1.RollingUpdateStatefulSetStrategyType - }), + s.Status.Replicas = 4 + s.Status.ReadyReplicas = readyReplicas + s.Status.UpdatedReplicas = updatedReplicas + s.Status.UpdateRevision = "rev1" + }) + } + newRollingSts := getSts(func(s *appsv1.StatefulSet) { + s.Spec.UpdateStrategy.Type = appsv1.RollingUpdateStatefulSetStrategyType + }) + f(opts{ + new: newRollingSts, prev: getSts(), predefinedObjects: []runtime.Object{ - getSts(func(s *appsv1.StatefulSet) { - s.Spec.UpdateStrategy.Type = appsv1.RollingUpdateStatefulSetStrategyType - s.Status.Replicas = 4 - s.Status.ReadyReplicas = 3 - s.Status.UpdatedReplicas = 4 - }), + mkRollingSts(4, 3), mkPod(0, true), mkPod(1, true), mkPod(2, true), @@ -1543,19 +1551,36 @@ func TestStatefulsetReconcile(t *testing.T) { }, }) - // RollingUpdate strategy: an unready non-maintenance pod still blocks readiness + // RollingUpdate strategy: excluded pod stuck terminating, e.g. at a shut down node, + // isn't counted at status.updatedReplicas, but must not block readiness f(opts{ - new: getSts(func(s *appsv1.StatefulSet) { - s.Spec.UpdateStrategy.Type = appsv1.RollingUpdateStatefulSetStrategyType - }), + new: newRollingSts, prev: getSts(), predefinedObjects: []runtime.Object{ - getSts(func(s *appsv1.StatefulSet) { - s.Spec.UpdateStrategy.Type = appsv1.RollingUpdateStatefulSetStrategyType - s.Status.Replicas = 4 - s.Status.ReadyReplicas = 3 - s.Status.UpdatedReplicas = 4 + mkRollingSts(3, 3), + mkPod(0, true), + mkPod(1, true), + mkPod(2, true), + mkPod(3, true, func(p *corev1.Pod) { + p.DeletionTimestamp = ptr.To(metav1.Now()) + p.Finalizers = []string{"test"} }), + }, + actions: []k8stools.ClientAction{ + {Verb: "Get", Kind: "StatefulSet", Resource: nn}, + {Verb: "Get", Kind: "StatefulSet", Resource: nn}, + }, + o: &StatefulSetOpts{ + IgnoreNotReadyPodOrdinals: sets.New[int32](3), + }, + }) + + // RollingUpdate strategy: an unready non-maintenance pod still blocks readiness + f(opts{ + new: newRollingSts, + prev: getSts(), + predefinedObjects: []runtime.Object{ + mkRollingSts(4, 3), mkPod(0, true), mkPod(1, false), mkPod(2, true), @@ -1566,6 +1591,25 @@ func TestStatefulsetReconcile(t *testing.T) { }, wantErr: true, }) + + // RollingUpdate strategy: a ready non-maintenance pod at the previous revision still blocks readiness + f(opts{ + new: newRollingSts, + prev: getSts(), + predefinedObjects: []runtime.Object{ + mkRollingSts(3, 4), + mkPod(0, true, func(p *corev1.Pod) { + p.Labels[podRevisionLabel] = "rev0" + }), + mkPod(1, true), + mkPod(2, true), + mkPod(3, true), + }, + o: &StatefulSetOpts{ + IgnoreNotReadyPodOrdinals: sets.New[int32](3), + }, + wantErr: true, + }) } func TestStatefulsetReconcile_ExistingObjNotStaleAfterNotFoundRetry(t *testing.T) { diff --git a/internal/controller/operator/factory/vmcluster/vmcluster_test.go b/internal/controller/operator/factory/vmcluster/vmcluster_test.go index e917d6a838..0974964f5a 100644 --- a/internal/controller/operator/factory/vmcluster/vmcluster_test.go +++ b/internal/controller/operator/factory/vmcluster/vmcluster_test.go @@ -101,11 +101,11 @@ func TestCreateOrUpdate(t *testing.T) { }, predefinedObjects: []runtime.Object{ &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-0", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-0", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-1", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-1", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, }, @@ -234,43 +234,43 @@ func TestCreateOrUpdate(t *testing.T) { }, predefinedObjects: []runtime.Object{ &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-0", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-0", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-1", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-1", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-2", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-2", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-3", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-3", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-4", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-4", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-5", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-5", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-6", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-6", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-7", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-7", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-8", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-8", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-9", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-9", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, }, @@ -304,43 +304,43 @@ func TestCreateOrUpdate(t *testing.T) { }, predefinedObjects: []runtime.Object{ &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-0", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-0", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-1", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-1", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-2", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-2", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-3", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-3", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-4", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-4", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-5", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-5", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-6", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-6", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-7", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-7", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-8", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-8", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-9", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-9", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}}, }, }, From 76b52135ed72546ca7bfac41682e1bc3fc7f0d51 Mon Sep 17 00:00:00 2001 From: Hui Wang Date: Wed, 21 Oct 2026 12:41:50 +0800 Subject: [PATCH 3/5] not using storage node maintenance mode with automatic vmstorage discovery --- api/operator/v1beta1/vmcluster_types.go | 8 +++++ api/operator/v1beta1/vmcluster_types_test.go | 38 ++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/api/operator/v1beta1/vmcluster_types.go b/api/operator/v1beta1/vmcluster_types.go index 33c0cc2ae8..51be57687b 100644 --- a/api/operator/v1beta1/vmcluster_types.go +++ b/api/operator/v1beta1/vmcluster_types.go @@ -947,6 +947,14 @@ func (cr *VMCluster) StorageNodeIDsUnderMaintenance() sets.Set[int32] { if cr.Spec.VMStorage == nil { return nil } + // with discovery enabled vminsert or vmselect find all vmstorage pods via DNS SRV records and ignore maintenance node ids, + // so the listed pods still receive traffic. The webhook rejects such configuration, but it can be disabled. + if cr.Spec.VMInsert != nil && cr.Spec.VMInsert.Discovery.OrDefault(cr.Spec.Discovery).enabled() { + return nil + } + if cr.Spec.VMSelect != nil && cr.Spec.VMSelect.Discovery.OrDefault(cr.Spec.Discovery).enabled() { + return nil + } insertNodes := sets.New(cr.Spec.VMStorage.MaintenanceInsertNodeIDs...) selectNodes := sets.New(cr.Spec.VMStorage.MaintenanceSelectNodeIDs...) return insertNodes.Intersection(selectNodes) diff --git a/api/operator/v1beta1/vmcluster_types_test.go b/api/operator/v1beta1/vmcluster_types_test.go index fccb2ee054..0e64706fc3 100644 --- a/api/operator/v1beta1/vmcluster_types_test.go +++ b/api/operator/v1beta1/vmcluster_types_test.go @@ -151,6 +151,44 @@ func TestVMCluster_StorageNodeIDsUnderMaintenance(t *testing.T) { }, }, sets.New[int32]()) + // global discovery enabled, maintenance node ids don't affect routing + f(&VMCluster{ + Spec: VMClusterSpec{ + Discovery: &VMClusterDiscovery{Enabled: true}, + VMInsert: &VMInsert{}, + VMSelect: &VMSelect{}, + VMStorage: &VMStorage{ + MaintenanceSelectNodeIDs: []int32{3}, + MaintenanceInsertNodeIDs: []int32{3}, + }, + }, + }, nil) + + // vmselect discovery enabled only + f(&VMCluster{ + Spec: VMClusterSpec{ + VMInsert: &VMInsert{}, + VMSelect: &VMSelect{Discovery: &VMClusterDiscovery{Enabled: true}}, + VMStorage: &VMStorage{ + MaintenanceSelectNodeIDs: []int32{3}, + MaintenanceInsertNodeIDs: []int32{3}, + }, + }, + }, nil) + + // global discovery disabled by vminsert and vmselect overrides + f(&VMCluster{ + Spec: VMClusterSpec{ + Discovery: &VMClusterDiscovery{Enabled: true}, + VMInsert: &VMInsert{Discovery: &VMClusterDiscovery{Enabled: false}}, + VMSelect: &VMSelect{Discovery: &VMClusterDiscovery{Enabled: false}}, + VMStorage: &VMStorage{ + MaintenanceSelectNodeIDs: []int32{3}, + MaintenanceInsertNodeIDs: []int32{3}, + }, + }, + }, sets.New[int32](3)) + // no VMStorage f(&VMCluster{}, nil) } From 35ed612064c63f9ee7f37879275dfebd8da100b9 Mon Sep 17 00:00:00 2001 From: Hui Wang Date: Wed, 21 Oct 2026 12:48:37 +0800 Subject: [PATCH 4/5] fix lint --- .../controller/operator/factory/reconcile/statefulset.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/controller/operator/factory/reconcile/statefulset.go b/internal/controller/operator/factory/reconcile/statefulset.go index bcf98a4fc4..9482b4dcad 100644 --- a/internal/controller/operator/factory/reconcile/statefulset.go +++ b/internal/controller/operator/factory/reconcile/statefulset.go @@ -66,7 +66,10 @@ func waitForStatefulSetReady(ctx context.Context, rclient client.Client, newObj return true, nil } if ignoreNotReadyPodOrdinals.Len() == 0 { - return *targetReplicas == existingObj.Status.UpdatedReplicas && *targetReplicas == existingObj.Status.ReadyReplicas, nil + if *targetReplicas != existingObj.Status.UpdatedReplicas { + return false, nil + } + return *targetReplicas == existingObj.Status.ReadyReplicas, nil } // status.updatedReplicas doesn't count terminating pods, e.g. a pod stuck terminating on a shut down node, // so check each pod, which is not excluded by ignoreNotReadyPodOrdinals, individually From 18e45069afa31345fe76798a2a84656088e28122 Mon Sep 17 00:00:00 2001 From: Hui Wang Date: Wed, 21 Oct 2026 13:19:27 +0800 Subject: [PATCH 5/5] apply review comments --- api/operator/v1/vlcluster_types.go | 13 ++++++- api/operator/v1/vtcluster_types.go | 13 ++++++- api/operator/v1beta1/vmcluster_types.go | 13 ++++++- config/crd/overlay/crd.yaml | 39 ++++++++++++++++--- docs/CHANGELOG.md | 2 +- docs/api.md | 12 +++--- docs/resources/vlcluster.md | 6 +-- docs/resources/vmcluster.md | 6 +-- docs/resources/vtcluster.md | 6 +-- .../factory/reconcile/statefulset_test.go | 16 ++++---- 10 files changed, 91 insertions(+), 35 deletions(-) diff --git a/api/operator/v1/vlcluster_types.go b/api/operator/v1/vlcluster_types.go index feb0492510..33ae841dd6 100644 --- a/api/operator/v1/vlcluster_types.go +++ b/api/operator/v1/vlcluster_types.go @@ -649,11 +649,20 @@ type VLStorage struct { // MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. // lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3]. // Useful at storage expanding, when you want to rebalance some data at cluster. + // + // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic. + // With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready + // and defers changes to the pod and its volumes until the id is removed from either list. + // With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready. + // See https://docs.victoriametrics.com/operator/resources/vlcluster/#pods-excluded-from-all-routing // +optional MaintenanceInsertNodeIDs []int32 `json:"maintenanceInsertNodeIDs,omitempty"` // MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. - // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic, so the operator doesn't wait for its pod to become ready - // and doesn't apply spec changes to the pod and its volumes until the id is removed from either list. + // + // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic. + // With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready + // and defers changes to the pod and its volumes until the id is removed from either list. + // With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready. // See https://docs.victoriametrics.com/operator/resources/vlcluster/#pods-excluded-from-all-routing // +optional MaintenanceSelectNodeIDs []int32 `json:"maintenanceSelectNodeIDs,omitempty"` diff --git a/api/operator/v1/vtcluster_types.go b/api/operator/v1/vtcluster_types.go index 9c24c3876d..304e638458 100644 --- a/api/operator/v1/vtcluster_types.go +++ b/api/operator/v1/vtcluster_types.go @@ -512,11 +512,20 @@ type VTStorage struct { // MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. // lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3]. // Useful at storage expanding, when you want to rebalance some data at cluster. + // + // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic. + // With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready + // and defers changes to the pod and its volumes until the id is removed from either list. + // With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready. + // See https://docs.victoriametrics.com/operator/resources/vtcluster/#pods-excluded-from-all-routing // +optional MaintenanceInsertNodeIDs []int32 `json:"maintenanceInsertNodeIDs,omitempty"` // MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. - // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic, so the operator doesn't wait for its pod to become ready - // and doesn't apply spec changes to the pod and its volumes until the id is removed from either list. + // + // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic. + // With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready + // and defers changes to the pod and its volumes until the id is removed from either list. + // With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready. // See https://docs.victoriametrics.com/operator/resources/vtcluster/#pods-excluded-from-all-routing // +optional MaintenanceSelectNodeIDs []int32 `json:"maintenanceSelectNodeIDs,omitempty"` diff --git a/api/operator/v1beta1/vmcluster_types.go b/api/operator/v1beta1/vmcluster_types.go index 51be57687b..6f4cfd66cc 100644 --- a/api/operator/v1beta1/vmcluster_types.go +++ b/api/operator/v1beta1/vmcluster_types.go @@ -589,11 +589,20 @@ type VMStorage struct { // MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. // lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3]. // Useful at storage expanding, when you want to rebalance some data at cluster. + // + // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic. + // With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready + // and defers changes to the pod and its volumes until the id is removed from either list. + // With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready. + // See https://docs.victoriametrics.com/operator/resources/vmcluster/#pods-excluded-from-all-routing // +optional MaintenanceInsertNodeIDs []int32 `json:"maintenanceInsertNodeIDs,omitempty"` // MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. - // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic, so the operator doesn't wait for its pod to become ready - // and doesn't apply spec changes to the pod and its volumes until the id is removed from either list. + // + // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic. + // With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready + // and defers changes to the pod and its volumes until the id is removed from either list. + // With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready. // See https://docs.victoriametrics.com/operator/resources/vmcluster/#pods-excluded-from-all-routing // +optional MaintenanceSelectNodeIDs []int32 `json:"maintenanceSelectNodeIDs,omitempty"` diff --git a/config/crd/overlay/crd.yaml b/config/crd/overlay/crd.yaml index a695c08243..8d21dbf76d 100644 --- a/config/crd/overlay/crd.yaml +++ b/config/crd/overlay/crd.yaml @@ -7249,6 +7249,12 @@ spec: MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3]. Useful at storage expanding, when you want to rebalance some data at cluster. + + A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic. + With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready + and defers changes to the pod and its volumes until the id is removed from either list. + With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready. + See https://docs.victoriametrics.com/operator/resources/vlcluster/#pods-excluded-from-all-routing items: format: int32 type: integer @@ -7256,8 +7262,11 @@ spec: maintenanceSelectNodeIDs: description: |- MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. - A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic, so the operator doesn't wait for its pod to become ready - and doesn't apply spec changes to the pod and its volumes until the id is removed from either list. + + A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic. + With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready + and defers changes to the pod and its volumes until the id is removed from either list. + With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready. See https://docs.victoriametrics.com/operator/resources/vlcluster/#pods-excluded-from-all-routing items: format: int32 @@ -49347,6 +49356,12 @@ spec: MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3]. Useful at storage expanding, when you want to rebalance some data at cluster. + + A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic. + With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready + and defers changes to the pod and its volumes until the id is removed from either list. + With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready. + See https://docs.victoriametrics.com/operator/resources/vmcluster/#pods-excluded-from-all-routing items: format: int32 type: integer @@ -49354,8 +49369,11 @@ spec: maintenanceSelectNodeIDs: description: |- MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. - A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic, so the operator doesn't wait for its pod to become ready - and doesn't apply spec changes to the pod and its volumes until the id is removed from either list. + + A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic. + With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready + and defers changes to the pod and its volumes until the id is removed from either list. + With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready. See https://docs.victoriametrics.com/operator/resources/vmcluster/#pods-excluded-from-all-routing items: format: int32 @@ -93101,6 +93119,12 @@ spec: MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3]. Useful at storage expanding, when you want to rebalance some data at cluster. + + A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic. + With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready + and defers changes to the pod and its volumes until the id is removed from either list. + With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready. + See https://docs.victoriametrics.com/operator/resources/vtcluster/#pods-excluded-from-all-routing items: format: int32 type: integer @@ -93108,8 +93132,11 @@ spec: maintenanceSelectNodeIDs: description: |- MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. - A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic, so the operator doesn't wait for its pod to become ready - and doesn't apply spec changes to the pod and its volumes until the id is removed from either list. + + A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic. + With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready + and defers changes to the pod and its volumes until the id is removed from either list. + With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready. See https://docs.victoriametrics.com/operator/resources/vtcluster/#pods-excluded-from-all-routing items: format: int32 diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 43ff82979b..8dd393cfdf 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -19,7 +19,7 @@ aliases: * FEATURE: [vmuser](https://docs.victoriametrics.com/operator/resources/vmuser/), [vmauth](https://docs.victoriametrics.com/operator/resources/vmauth/): allow `VTAgent` in `targetRef.crd.kind`, so `vmauth` can route to a `VTAgent` by name instead of a static URL. The CRDs must be updated to use it. See [#2687](https://github.com/VictoriaMetrics/operator/issues/2687). * FEATURE: [vmanomaly](https://docs.victoriametrics.com/operator/resources/vmanomaly/), [vmanomalyconfig](https://docs.victoriametrics.com/operator/resources/vmanomalyconfig/): support the experimental `peer_outlier` model and its peer-group parameters in managed configuration, available since vmanomaly [v1.31.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1310). See [#2714](https://github.com/VictoriaMetrics/operator/pull/2714). -* FEATURE: [vmcluster](https://docs.victoriametrics.com/operator/resources/vmcluster/), [vlcluster](https://docs.victoriametrics.com/operator/resources/vlcluster/), [vtcluster](https://docs.victoriametrics.com/operator/resources/vtcluster/): leave storage pods listed in both `maintenanceInsertNodeIDs` and `maintenanceSelectNodeIDs` alone during reconciliation: do not wait for them to become ready, and defer spec changes to them and their volumes until they leave maintenance. Previously, such a pod stuck `Pending` or `Terminating` (e.g. its node is under maintenance and its volume is bound to that node) failed every reconcile at the storage step, so changes to insert and select components were not applied until the pod came back. See [these docs](https://docs.victoriametrics.com/operator/resources/vmcluster/#pods-excluded-from-all-routing) and [#2665](https://github.com/VictoriaMetrics/operator/issues/2665). +* FEATURE: [vmcluster](https://docs.victoriametrics.com/operator/resources/vmcluster/), [vlcluster](https://docs.victoriametrics.com/operator/resources/vlcluster/), [vtcluster](https://docs.victoriametrics.com/operator/resources/vtcluster/): with the default `rollingUpdateStrategy: OnDelete`, do not wait for storage pods listed in both `maintenanceInsertNodeIDs` and `maintenanceSelectNodeIDs` to become ready, and defer spec changes to these pods and their volumes until they leave maintenance. With `RollingUpdate`, Kubernetes still blocks pod updates while such a pod isn't ready. Previously, such a pod stuck `Pending` or `Terminating` (e.g. while its node is under maintenance and its volume is bound to that node) made every reconcile fail at the storage step, so changes to the insert and select components were not applied until the pod came back. See [these docs](https://docs.victoriametrics.com/operator/resources/vmcluster/#pods-excluded-from-all-routing) and [#2665](https://github.com/VictoriaMetrics/operator/issues/2665). * BUGFIX: [vmsingle](https://docs.victoriametrics.com/operator/resources/vmsingle/), [vmcluster](https://docs.victoriametrics.com/operator/resources/vmcluster/), [vlsingle](https://docs.victoriametrics.com/operator/resources/vlsingle/), [vlcluster](https://docs.victoriametrics.com/operator/resources/vlcluster/), [vtsingle](https://docs.victoriametrics.com/operator/resources/vtsingle/), [vtcluster](https://docs.victoriametrics.com/operator/resources/vtcluster/): accept the months unit `M` in `retentionPeriod` and `futureRetention`. Previously, the CRD rejected values such as `3M`, even though VictoriaMetrics, VictoriaLogs and VictoriaTraces accept them. The CRDs must be updated to get this fix. See [#2693](https://github.com/VictoriaMetrics/operator/issues/2693). * BUGFIX: [vmanomaly](https://docs.victoriametrics.com/operator/resources/vmanomaly/): allow `spec.configRawYaml` and `spec.configSecret` without `reader.queries` when selected [VMAnomalyConfig](https://docs.victoriametrics.com/operator/resources/vmanomalyconfig/) objects provide the queries. Previously, such a config failed reconciliation with `reader.queries must be provided via configRawYaml or configSecret`, so queries could not come from `VMAnomalyConfig` objects alone. A config with no queries from any source is still rejected. This bug was introduced in [v0.69.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.69.0) with `VMAnomalyConfig`. See [#2685](https://github.com/VictoriaMetrics/operator/pull/2685). diff --git a/docs/api.md b/docs/api.md index c092f67561..acc5f0ef56 100644 --- a/docs/api.md +++ b/docs/api.md @@ -583,8 +583,8 @@ Appears in: [VLClusterSpec (v1)](#v1-vlclusterspec) | logIngestedRows#
_boolean_ | _(Required)_
Whether to log all the ingested log entries; this can be useful for debugging of data ingestion; see https://docs.victoriametrics.com/victorialogs/data-ingestion/ | | logLevel#
_string_ | _(Optional)_
LogLevel for VLStorage to be configured with. | | logNewStreams#
_boolean_ | _(Required)_
LogNewStreams Whether to log creation of new streams; this can be useful for debugging of high cardinality issues with log streams; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields | -| maintenanceInsertNodeIDs#
_integer array_ | _(Optional)_
MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
Useful at storage expanding, when you want to rebalance some data at cluster. | -| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic, so the operator doesn't wait for its pod to become ready
and doesn't apply spec changes to the pod and its volumes until the id is removed from either list.
See https://docs.victoriametrics.com/operator/resources/vlcluster/#pods-excluded-from-all-routing | +| maintenanceInsertNodeIDs#
_integer array_ | _(Optional)_
MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
Useful at storage expanding, when you want to rebalance some data at cluster.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
and defers changes to the pod and its volumes until the id is removed from either list.
With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
See https://docs.victoriametrics.com/operator/resources/vlcluster/#pods-excluded-from-all-routing | +| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
and defers changes to the pod and its volumes until the id is removed from either list.
With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
See https://docs.victoriametrics.com/operator/resources/vlcluster/#pods-excluded-from-all-routing | | minReadySeconds#
_integer_ | _(Optional)_
MinReadySeconds defines a minimum number of seconds to wait before starting update next pod
if previous in healthy state
Has no effect for VLogs and VMSingle | | networkPolicy#
_[EmbeddedNetworkPolicy (v1beta1)](#v1beta1-embeddednetworkpolicy)_ | _(Optional)_
NetworkPolicy defines network access rules for pods created by this CR. | | nodeSelector#
_object (keys:string, values:string)_ | _(Optional)_
NodeSelector Define which Nodes the Pods are scheduled on. | @@ -1279,8 +1279,8 @@ Appears in: [VTClusterSpec (v1)](#v1-vtclusterspec) | logIngestedRows#
_boolean_ | _(Required)_
Whether to log all the ingested log entries; this can be useful for debugging of data ingestion
see https://docs.victoriametrics.com/victoriatraces/#configure-victoriatraces | | logLevel#
_string_ | _(Optional)_
LogLevel for VTStorage to be configured with. | | logNewStreams#
_boolean_ | _(Required)_
LogNewStreams Whether to log creation of new streams; this can be useful for debugging of high cardinality issues with log streams
see https://docs.victoriametrics.com/victoriatraces/#configure-victoriatraces | -| maintenanceInsertNodeIDs#
_integer array_ | _(Optional)_
MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
Useful at storage expanding, when you want to rebalance some data at cluster. | -| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic, so the operator doesn't wait for its pod to become ready
and doesn't apply spec changes to the pod and its volumes until the id is removed from either list.
See https://docs.victoriametrics.com/operator/resources/vtcluster/#pods-excluded-from-all-routing | +| maintenanceInsertNodeIDs#
_integer array_ | _(Optional)_
MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
Useful at storage expanding, when you want to rebalance some data at cluster.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
and defers changes to the pod and its volumes until the id is removed from either list.
With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
See https://docs.victoriametrics.com/operator/resources/vtcluster/#pods-excluded-from-all-routing | +| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
and defers changes to the pod and its volumes until the id is removed from either list.
With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
See https://docs.victoriametrics.com/operator/resources/vtcluster/#pods-excluded-from-all-routing | | minReadySeconds#
_integer_ | _(Optional)_
MinReadySeconds defines a minimum number of seconds to wait before starting update next pod
if previous in healthy state
Has no effect for VLogs and VMSingle | | networkPolicy#
_[EmbeddedNetworkPolicy (v1beta1)](#v1beta1-embeddednetworkpolicy)_ | _(Optional)_
NetworkPolicy defines network access rules for pods created by this CR. | | nodeSelector#
_object (keys:string, values:string)_ | _(Optional)_
NodeSelector Define which Nodes the Pods are scheduled on. | @@ -5754,8 +5754,8 @@ Appears in: [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec) | livenessProbe#
_[Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#probe-v1-core)_ | _(Optional)_
LivenessProbe that will be added to CR pod | | logFormat#
_string_ | _(Optional)_
LogFormat for VMStorage to be configured with.
default or json | | logLevel#
_string_ | _(Optional)_
LogLevel for VMStorage to be configured with. | -| maintenanceInsertNodeIDs#
_integer array_ | _(Optional)_
MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
Useful at storage expanding, when you want to rebalance some data at cluster. | -| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic, so the operator doesn't wait for its pod to become ready
and doesn't apply spec changes to the pod and its volumes until the id is removed from either list.
See https://docs.victoriametrics.com/operator/resources/vmcluster/#pods-excluded-from-all-routing | +| maintenanceInsertNodeIDs#
_integer array_ | _(Optional)_
MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
Useful at storage expanding, when you want to rebalance some data at cluster.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
and defers changes to the pod and its volumes until the id is removed from either list.
With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
See https://docs.victoriametrics.com/operator/resources/vmcluster/#pods-excluded-from-all-routing | +| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
and defers changes to the pod and its volumes until the id is removed from either list.
With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
See https://docs.victoriametrics.com/operator/resources/vmcluster/#pods-excluded-from-all-routing | | minReadySeconds#
_integer_ | _(Optional)_
MinReadySeconds defines a minimum number of seconds to wait before starting update next pod
if previous in healthy state
Has no effect for VLogs and VMSingle | | networkPolicy#
_[EmbeddedNetworkPolicy (v1beta1)](#v1beta1-embeddednetworkpolicy)_ | _(Optional)_
NetworkPolicy defines network access rules for pods created by this CR. | | nodeSelector#
_object (keys:string, values:string)_ | _(Optional)_
NodeSelector Define which Nodes the Pods are scheduled on. | diff --git a/docs/resources/vlcluster.md b/docs/resources/vlcluster.md index a8df3b5dcf..2e7cb31049 100644 --- a/docs/resources/vlcluster.md +++ b/docs/resources/vlcluster.md @@ -233,12 +233,12 @@ and waits for it to become ready before adding it back to `vlinsert` and `vlsele **Limitations:** -- Spec changes are deferred only with the default `rollingUpdateStrategy: OnDelete`, where the Operator updates `vlstorage` pods itself. +- Pod spec changes are deferred only with the default `rollingUpdateStrategy: OnDelete`, where the Operator updates `vlstorage` pods itself. With `RollingUpdate`, Kubernetes rolls out all pods, including the pod under maintenance, using `OrderedReady` pod management, which is also used when `minReadySeconds` is set. In this mode, while the pod under maintenance isn't ready, Kubernetes doesn't roll out updates, scale the StatefulSet or recreate pods with a higher ordinal, so reconciliation that changes `vlstorage` pods can block until the pod is ready again. -- With `vlstorage.podDisruptionBudget.maxUnavailable: 1`, the pod under maintenance already uses the disruption budget, - so any `vlstorage` change that requires pod restarts fails until the pod is ready again. +- With `vlstorage.podDisruptionBudget.maxUnavailable: 1`, a pod under maintenance that isn't ready uses up the disruption budget, + so the Operator can't evict other `vlstorage` pods, and changes that require restarting them fail until the pod is ready again. ## Resource management diff --git a/docs/resources/vmcluster.md b/docs/resources/vmcluster.md index 5b8ae0ec41..057adf0670 100644 --- a/docs/resources/vmcluster.md +++ b/docs/resources/vmcluster.md @@ -472,12 +472,12 @@ and waits for it to become ready before adding it back to `vminsert` and `vmsele **Limitations:** -- Spec changes are deferred only with the default `rollingUpdateStrategy: OnDelete`, where the Operator updates `vmstorage` pods itself. +- Pod spec changes are deferred only with the default `rollingUpdateStrategy: OnDelete`, where the Operator updates `vmstorage` pods itself. With `RollingUpdate`, Kubernetes rolls out all pods, including the pod under maintenance, using `OrderedReady` pod management, which is also used when `minReadySeconds` is set. In this mode, while the pod under maintenance isn't ready, Kubernetes doesn't roll out updates, scale the StatefulSet or recreate pods with a higher ordinal, so reconciliation that changes `vmstorage` pods can block until the pod is ready again. -- With `vmstorage.podDisruptionBudget.maxUnavailable: 1`, the pod under maintenance already uses the disruption budget, - so any `vmstorage` change that requires pod restarts fails until the pod is ready again. +- With `vmstorage.podDisruptionBudget.maxUnavailable: 1`, a pod under maintenance that isn't ready uses up the disruption budget, + so the Operator can't evict other `vmstorage` pods, and changes that require restarting them fail until the pod is ready again. ## Resource management diff --git a/docs/resources/vtcluster.md b/docs/resources/vtcluster.md index c2d11de549..7d3c1cb327 100644 --- a/docs/resources/vtcluster.md +++ b/docs/resources/vtcluster.md @@ -259,12 +259,12 @@ and waits for it to become ready before adding it back to `insert` and `select`, **Limitations:** -- Spec changes are deferred only with the default `rollingUpdateStrategy: OnDelete`, where the Operator updates `storage` pods itself. +- Pod spec changes are deferred only with the default `rollingUpdateStrategy: OnDelete`, where the Operator updates `storage` pods itself. With `RollingUpdate`, Kubernetes rolls out all pods, including the pod under maintenance, using `OrderedReady` pod management, which is also used when `minReadySeconds` is set. In this mode, while the pod under maintenance isn't ready, Kubernetes doesn't roll out updates, scale the StatefulSet or recreate pods with a higher ordinal, so reconciliation that changes `storage` pods can block until the pod is ready again. -- With `storage.podDisruptionBudget.maxUnavailable: 1`, the pod under maintenance already uses the disruption budget, - so any `storage` change that requires pod restarts fails until the pod is ready again. +- With `storage.podDisruptionBudget.maxUnavailable: 1`, a pod under maintenance that isn't ready uses up the disruption budget, + so the Operator can't evict other `storage` pods, and changes that require restarting them fail until the pod is ready again. ## Resource management diff --git a/internal/controller/operator/factory/reconcile/statefulset_test.go b/internal/controller/operator/factory/reconcile/statefulset_test.go index acaec3b99e..6b7637e5d0 100644 --- a/internal/controller/operator/factory/reconcile/statefulset_test.go +++ b/internal/controller/operator/factory/reconcile/statefulset_test.go @@ -1529,11 +1529,13 @@ func TestStatefulsetReconcile(t *testing.T) { s.Status.UpdateRevision = "rev1" }) } - newRollingSts := getSts(func(s *appsv1.StatefulSet) { - s.Spec.UpdateStrategy.Type = appsv1.RollingUpdateStatefulSetStrategyType - }) + newRollingSts := func() *appsv1.StatefulSet { + return getSts(func(s *appsv1.StatefulSet) { + s.Spec.UpdateStrategy.Type = appsv1.RollingUpdateStatefulSetStrategyType + }) + } f(opts{ - new: newRollingSts, + new: newRollingSts(), prev: getSts(), predefinedObjects: []runtime.Object{ mkRollingSts(4, 3), @@ -1554,7 +1556,7 @@ func TestStatefulsetReconcile(t *testing.T) { // RollingUpdate strategy: excluded pod stuck terminating, e.g. at a shut down node, // isn't counted at status.updatedReplicas, but must not block readiness f(opts{ - new: newRollingSts, + new: newRollingSts(), prev: getSts(), predefinedObjects: []runtime.Object{ mkRollingSts(3, 3), @@ -1577,7 +1579,7 @@ func TestStatefulsetReconcile(t *testing.T) { // RollingUpdate strategy: an unready non-maintenance pod still blocks readiness f(opts{ - new: newRollingSts, + new: newRollingSts(), prev: getSts(), predefinedObjects: []runtime.Object{ mkRollingSts(4, 3), @@ -1594,7 +1596,7 @@ func TestStatefulsetReconcile(t *testing.T) { // RollingUpdate strategy: a ready non-maintenance pod at the previous revision still blocks readiness f(opts{ - new: newRollingSts, + new: newRollingSts(), prev: getSts(), predefinedObjects: []runtime.Object{ mkRollingSts(3, 4),