diff --git a/api/operator/v1/cluster_types_test.go b/api/operator/v1/cluster_types_test.go
index 9bca6afe18..5c22883c44 100644
--- a/api/operator/v1/cluster_types_test.go
+++ b/api/operator/v1/cluster_types_test.go
@@ -4,6 +4,7 @@ import (
"testing"
"github.com/stretchr/testify/assert"
+ "k8s.io/apimachinery/pkg/util/sets"
"k8s.io/utils/ptr"
vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1"
@@ -79,6 +80,42 @@ func TestVLCluster_AvailableStorageNodeIDs(t *testing.T) {
}, vmv1beta1.ClusterComponentSelect, []int32{0, 1, 2})
}
+func TestVTCluster_StorageNodeIDsUnderMaintenance(t *testing.T) {
+ f := func(cr *VTCluster, want sets.Set[int32]) {
+ t.Helper()
+ assert.Equal(t, want, cr.StorageNodeIDsUnderMaintenance())
+ }
+
+ f(&VTCluster{
+ Spec: VTClusterSpec{
+ Storage: &VTStorage{
+ MaintenanceSelectNodeIDs: []int32{1, 3},
+ MaintenanceInsertNodeIDs: []int32{0, 3},
+ },
+ },
+ }, sets.New[int32](3))
+
+ f(&VTCluster{}, nil)
+}
+
+func TestVLCluster_StorageNodeIDsUnderMaintenance(t *testing.T) {
+ f := func(cr *VLCluster, want sets.Set[int32]) {
+ t.Helper()
+ assert.Equal(t, want, cr.StorageNodeIDsUnderMaintenance())
+ }
+
+ f(&VLCluster{
+ Spec: VLClusterSpec{
+ VLStorage: &VLStorage{
+ MaintenanceSelectNodeIDs: []int32{1, 3},
+ MaintenanceInsertNodeIDs: []int32{0, 3},
+ },
+ },
+ }, sets.New[int32](3))
+
+ f(&VLCluster{}, nil)
+}
+
func TestVLSingle_PrefixedName(t *testing.T) {
f := func(name string, omit bool, want string) {
t.Helper()
diff --git a/api/operator/v1/vlcluster_types.go b/api/operator/v1/vlcluster_types.go
index 67d9f94c2f..33ae841dd6 100644
--- a/api/operator/v1/vlcluster_types.go
+++ b/api/operator/v1/vlcluster_types.go
@@ -649,9 +649,21 @@ type VLStorage struct {
// MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
// lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
// Useful at storage expanding, when you want to rebalance some data at cluster.
+ //
+ // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
+ // With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
+ // and defers changes to the pod and its volumes until the id is removed from either list.
+ // With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
+ // See https://docs.victoriametrics.com/operator/resources/vlcluster/#pods-excluded-from-all-routing
// +optional
MaintenanceInsertNodeIDs []int32 `json:"maintenanceInsertNodeIDs,omitempty"`
// MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
+ //
+ // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
+ // With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
+ // and defers changes to the pod and its volumes until the id is removed from either list.
+ // With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
+ // See https://docs.victoriametrics.com/operator/resources/vlcluster/#pods-excluded-from-all-routing
// +optional
MaintenanceSelectNodeIDs []int32 `json:"maintenanceSelectNodeIDs,omitempty"`
@@ -902,6 +914,17 @@ func (cr *VLCluster) AvailableStorageNodeIDs(kind vmv1beta1.ClusterComponent) []
return result
}
+// StorageNodeIDsUnderMaintenance returns the vlstorage pod ordinals excluded from both
+// insert and select routing via MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs.
+func (cr *VLCluster) StorageNodeIDsUnderMaintenance() sets.Set[int32] {
+ if cr.Spec.VLStorage == nil {
+ return nil
+ }
+ insertNodes := sets.New(cr.Spec.VLStorage.MaintenanceInsertNodeIDs...)
+ selectNodes := sets.New(cr.Spec.VLStorage.MaintenanceSelectNodeIDs...)
+ return insertNodes.Intersection(selectNodes)
+}
+
// LastSpecUpdated compares spec with last applied spec stored, replaces old spec and returns true if it's updated
func (cr *VLCluster) LastSpecUpdated() bool {
updated := cr.Status.LastAppliedSpec == nil || !equality.Semantic.DeepEqual(&cr.Spec, cr.Status.LastAppliedSpec)
diff --git a/api/operator/v1/vtcluster_types.go b/api/operator/v1/vtcluster_types.go
index 5efa0a9014..304e638458 100644
--- a/api/operator/v1/vtcluster_types.go
+++ b/api/operator/v1/vtcluster_types.go
@@ -512,9 +512,21 @@ type VTStorage struct {
// MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
// lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
// Useful at storage expanding, when you want to rebalance some data at cluster.
+ //
+ // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
+ // With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
+ // and defers changes to the pod and its volumes until the id is removed from either list.
+ // With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
+ // See https://docs.victoriametrics.com/operator/resources/vtcluster/#pods-excluded-from-all-routing
// +optional
MaintenanceInsertNodeIDs []int32 `json:"maintenanceInsertNodeIDs,omitempty"`
// MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
+ //
+ // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
+ // With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
+ // and defers changes to the pod and its volumes until the id is removed from either list.
+ // With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
+ // See https://docs.victoriametrics.com/operator/resources/vtcluster/#pods-excluded-from-all-routing
// +optional
MaintenanceSelectNodeIDs []int32 `json:"maintenanceSelectNodeIDs,omitempty"`
@@ -770,6 +782,17 @@ func (cr *VTCluster) AvailableStorageNodeIDs(kind vmv1beta1.ClusterComponent) []
return result
}
+// StorageNodeIDsUnderMaintenance returns the storage pod ordinals excluded from both
+// insert and select routing via MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs.
+func (cr *VTCluster) StorageNodeIDsUnderMaintenance() sets.Set[int32] {
+ if cr.Spec.Storage == nil {
+ return nil
+ }
+ insertNodes := sets.New(cr.Spec.Storage.MaintenanceInsertNodeIDs...)
+ selectNodes := sets.New(cr.Spec.Storage.MaintenanceSelectNodeIDs...)
+ return insertNodes.Intersection(selectNodes)
+}
+
// LastSpecUpdated compares spec with last applied spec stored, replaces old spec and returns true if it's updated
func (cr *VTCluster) LastSpecUpdated() bool {
updated := cr.Status.LastAppliedSpec == nil || !equality.Semantic.DeepEqual(&cr.Spec, cr.Status.LastAppliedSpec)
diff --git a/api/operator/v1beta1/vmcluster_types.go b/api/operator/v1beta1/vmcluster_types.go
index 55184a062b..6f4cfd66cc 100644
--- a/api/operator/v1beta1/vmcluster_types.go
+++ b/api/operator/v1beta1/vmcluster_types.go
@@ -589,9 +589,21 @@ type VMStorage struct {
// MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
// lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
// Useful at storage expanding, when you want to rebalance some data at cluster.
+ //
+ // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
+ // With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
+ // and defers changes to the pod and its volumes until the id is removed from either list.
+ // With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
+ // See https://docs.victoriametrics.com/operator/resources/vmcluster/#pods-excluded-from-all-routing
// +optional
MaintenanceInsertNodeIDs []int32 `json:"maintenanceInsertNodeIDs,omitempty"`
// MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
+ //
+ // A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
+ // With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
+ // and defers changes to the pod and its volumes until the id is removed from either list.
+ // With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
+ // See https://docs.victoriametrics.com/operator/resources/vmcluster/#pods-excluded-from-all-routing
// +optional
MaintenanceSelectNodeIDs []int32 `json:"maintenanceSelectNodeIDs,omitempty"`
@@ -938,6 +950,25 @@ func (cr *VMCluster) AvailableStorageNodeIDs(kind ClusterComponent) []int32 {
return result
}
+// StorageNodeIDsUnderMaintenance returns the vmstorage pod ordinals excluded from both
+// insert and select routing via MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs.
+func (cr *VMCluster) StorageNodeIDsUnderMaintenance() sets.Set[int32] {
+ if cr.Spec.VMStorage == nil {
+ return nil
+ }
+ // with discovery enabled vminsert or vmselect find all vmstorage pods via DNS SRV records and ignore maintenance node ids,
+ // so the listed pods still receive traffic. The webhook rejects such configuration, but it can be disabled.
+ if cr.Spec.VMInsert != nil && cr.Spec.VMInsert.Discovery.OrDefault(cr.Spec.Discovery).enabled() {
+ return nil
+ }
+ if cr.Spec.VMSelect != nil && cr.Spec.VMSelect.Discovery.OrDefault(cr.Spec.Discovery).enabled() {
+ return nil
+ }
+ insertNodes := sets.New(cr.Spec.VMStorage.MaintenanceInsertNodeIDs...)
+ selectNodes := sets.New(cr.Spec.VMStorage.MaintenanceSelectNodeIDs...)
+ return insertNodes.Intersection(selectNodes)
+}
+
// FinalLabels adds cluster labels to the base labels and filters by prefix if needed
func (cr *VMCluster) FinalLabels(kind ClusterComponent) map[string]string {
v := AddClusterLabels(cr.SelectorLabels(kind), "vm")
diff --git a/api/operator/v1beta1/vmcluster_types_test.go b/api/operator/v1beta1/vmcluster_types_test.go
index dba5b9517a..0e64706fc3 100644
--- a/api/operator/v1beta1/vmcluster_types_test.go
+++ b/api/operator/v1beta1/vmcluster_types_test.go
@@ -6,6 +6,7 @@ import (
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/util/sets"
"k8s.io/utils/ptr"
)
@@ -124,6 +125,74 @@ func TestVMCluster_AvailableStorageNodeIDs(t *testing.T) {
}, ClusterComponentSelect, []int32{0, 1, 2})
}
+func TestVMCluster_StorageNodeIDsUnderMaintenance(t *testing.T) {
+ f := func(cr *VMCluster, want sets.Set[int32]) {
+ t.Helper()
+ assert.Equal(t, want, cr.StorageNodeIDsUnderMaintenance())
+ }
+
+ // node 3 is excluded from both insert and select routing
+ f(&VMCluster{
+ Spec: VMClusterSpec{
+ VMStorage: &VMStorage{
+ MaintenanceSelectNodeIDs: []int32{1, 3},
+ MaintenanceInsertNodeIDs: []int32{0, 3},
+ },
+ },
+ }, sets.New[int32](3))
+
+ // no overlap between the two lists
+ f(&VMCluster{
+ Spec: VMClusterSpec{
+ VMStorage: &VMStorage{
+ MaintenanceSelectNodeIDs: []int32{1},
+ MaintenanceInsertNodeIDs: []int32{0},
+ },
+ },
+ }, sets.New[int32]())
+
+ // global discovery enabled, maintenance node ids don't affect routing
+ f(&VMCluster{
+ Spec: VMClusterSpec{
+ Discovery: &VMClusterDiscovery{Enabled: true},
+ VMInsert: &VMInsert{},
+ VMSelect: &VMSelect{},
+ VMStorage: &VMStorage{
+ MaintenanceSelectNodeIDs: []int32{3},
+ MaintenanceInsertNodeIDs: []int32{3},
+ },
+ },
+ }, nil)
+
+ // vmselect discovery enabled only
+ f(&VMCluster{
+ Spec: VMClusterSpec{
+ VMInsert: &VMInsert{},
+ VMSelect: &VMSelect{Discovery: &VMClusterDiscovery{Enabled: true}},
+ VMStorage: &VMStorage{
+ MaintenanceSelectNodeIDs: []int32{3},
+ MaintenanceInsertNodeIDs: []int32{3},
+ },
+ },
+ }, nil)
+
+ // global discovery disabled by vminsert and vmselect overrides
+ f(&VMCluster{
+ Spec: VMClusterSpec{
+ Discovery: &VMClusterDiscovery{Enabled: true},
+ VMInsert: &VMInsert{Discovery: &VMClusterDiscovery{Enabled: false}},
+ VMSelect: &VMSelect{Discovery: &VMClusterDiscovery{Enabled: false}},
+ VMStorage: &VMStorage{
+ MaintenanceSelectNodeIDs: []int32{3},
+ MaintenanceInsertNodeIDs: []int32{3},
+ },
+ },
+ }, sets.New[int32](3))
+
+ // no VMStorage
+ f(&VMCluster{}, nil)
+}
+
func TestVMCluster_Validate(t *testing.T) {
f := func(spec VMClusterSpec, wantErr bool) {
t.Helper()
diff --git a/config/crd/overlay/crd.yaml b/config/crd/overlay/crd.yaml
index 6be0b26dd3..8d21dbf76d 100644
--- a/config/crd/overlay/crd.yaml
+++ b/config/crd/overlay/crd.yaml
@@ -7249,14 +7249,25 @@ spec:
MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
Useful at storage expanding, when you want to rebalance some data at cluster.
+
+ A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
+ With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
+ and defers changes to the pod and its volumes until the id is removed from either list.
+ With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
+ See https://docs.victoriametrics.com/operator/resources/vlcluster/#pods-excluded-from-all-routing
items:
format: int32
type: integer
type: array
maintenanceSelectNodeIDs:
- description: MaintenanceSelectNodeIDs - excludes given node ids
- from select requests routing, must contain pod suffixes - for
- pod-0, id will be 0 and etc.
+ description: |-
+ MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
+
+ A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
+ With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
+ and defers changes to the pod and its volumes until the id is removed from either list.
+ With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
+ See https://docs.victoriametrics.com/operator/resources/vlcluster/#pods-excluded-from-all-routing
items:
format: int32
type: integer
@@ -49345,14 +49356,25 @@ spec:
MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
Useful at storage expanding, when you want to rebalance some data at cluster.
+
+ A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
+ With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
+ and defers changes to the pod and its volumes until the id is removed from either list.
+ With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
+ See https://docs.victoriametrics.com/operator/resources/vmcluster/#pods-excluded-from-all-routing
items:
format: int32
type: integer
type: array
maintenanceSelectNodeIDs:
- description: MaintenanceSelectNodeIDs - excludes given node ids
- from select requests routing, must contain pod suffixes - for
- pod-0, id will be 0 and etc.
+ description: |-
+ MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
+
+ A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
+ With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
+ and defers changes to the pod and its volumes until the id is removed from either list.
+ With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
+ See https://docs.victoriametrics.com/operator/resources/vmcluster/#pods-excluded-from-all-routing
items:
format: int32
type: integer
@@ -93097,14 +93119,25 @@ spec:
MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
Useful at storage expanding, when you want to rebalance some data at cluster.
+
+ A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
+ With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
+ and defers changes to the pod and its volumes until the id is removed from either list.
+ With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
+ See https://docs.victoriametrics.com/operator/resources/vtcluster/#pods-excluded-from-all-routing
items:
format: int32
type: integer
type: array
maintenanceSelectNodeIDs:
- description: MaintenanceSelectNodeIDs - excludes given node ids
- from select requests routing, must contain pod suffixes - for
- pod-0, id will be 0 and etc.
+ description: |-
+ MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
+
+ A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
+ With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
+ and defers changes to the pod and its volumes until the id is removed from either list.
+ With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
+ See https://docs.victoriametrics.com/operator/resources/vtcluster/#pods-excluded-from-all-routing
items:
format: int32
type: integer
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 907bd8702b..8dd393cfdf 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -19,6 +19,7 @@ aliases:
* FEATURE: [vmuser](https://docs.victoriametrics.com/operator/resources/vmuser/), [vmauth](https://docs.victoriametrics.com/operator/resources/vmauth/): allow `VTAgent` in `targetRef.crd.kind`, so `vmauth` can route to a `VTAgent` by name instead of a static URL. The CRDs must be updated to use it. See [#2687](https://github.com/VictoriaMetrics/operator/issues/2687).
* FEATURE: [vmanomaly](https://docs.victoriametrics.com/operator/resources/vmanomaly/), [vmanomalyconfig](https://docs.victoriametrics.com/operator/resources/vmanomalyconfig/): support the experimental `peer_outlier` model and its peer-group parameters in managed configuration, available since vmanomaly [v1.31.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1310). See [#2714](https://github.com/VictoriaMetrics/operator/pull/2714).
+* FEATURE: [vmcluster](https://docs.victoriametrics.com/operator/resources/vmcluster/), [vlcluster](https://docs.victoriametrics.com/operator/resources/vlcluster/), [vtcluster](https://docs.victoriametrics.com/operator/resources/vtcluster/): with the default `rollingUpdateStrategy: OnDelete`, do not wait for storage pods listed in both `maintenanceInsertNodeIDs` and `maintenanceSelectNodeIDs` to become ready, and defer spec changes to these pods and their volumes until they leave maintenance. With `RollingUpdate`, Kubernetes still blocks pod updates while such a pod isn't ready. Previously, such a pod stuck `Pending` or `Terminating` (e.g. while its node is under maintenance and its volume is bound to that node) made every reconcile fail at the storage step, so changes to the insert and select components were not applied until the pod came back. See [these docs](https://docs.victoriametrics.com/operator/resources/vmcluster/#pods-excluded-from-all-routing) and [#2665](https://github.com/VictoriaMetrics/operator/issues/2665).
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/operator/resources/vmsingle/), [vmcluster](https://docs.victoriametrics.com/operator/resources/vmcluster/), [vlsingle](https://docs.victoriametrics.com/operator/resources/vlsingle/), [vlcluster](https://docs.victoriametrics.com/operator/resources/vlcluster/), [vtsingle](https://docs.victoriametrics.com/operator/resources/vtsingle/), [vtcluster](https://docs.victoriametrics.com/operator/resources/vtcluster/): accept the months unit `M` in `retentionPeriod` and `futureRetention`. Previously, the CRD rejected values such as `3M`, even though VictoriaMetrics, VictoriaLogs and VictoriaTraces accept them. The CRDs must be updated to get this fix. See [#2693](https://github.com/VictoriaMetrics/operator/issues/2693).
* BUGFIX: [vmanomaly](https://docs.victoriametrics.com/operator/resources/vmanomaly/): allow `spec.configRawYaml` and `spec.configSecret` without `reader.queries` when selected [VMAnomalyConfig](https://docs.victoriametrics.com/operator/resources/vmanomalyconfig/) objects provide the queries. Previously, such a config failed reconciliation with `reader.queries must be provided via configRawYaml or configSecret`, so queries could not come from `VMAnomalyConfig` objects alone. A config with no queries from any source is still rejected. This bug was introduced in [v0.69.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.69.0) with `VMAnomalyConfig`. See [#2685](https://github.com/VictoriaMetrics/operator/pull/2685).
diff --git a/docs/api.md b/docs/api.md
index cb3a36914d..acc5f0ef56 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -583,8 +583,8 @@ Appears in: [VLClusterSpec (v1)](#v1-vlclusterspec)
| logIngestedRows#
_boolean_ | _(Required)_
Whether to log all the ingested log entries; this can be useful for debugging of data ingestion; see https://docs.victoriametrics.com/victorialogs/data-ingestion/ |
| logLevel#
_string_ | _(Optional)_
LogLevel for VLStorage to be configured with. |
| logNewStreams#
_boolean_ | _(Required)_
LogNewStreams Whether to log creation of new streams; this can be useful for debugging of high cardinality issues with log streams; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields |
-| maintenanceInsertNodeIDs#
_integer array_ | _(Optional)_
MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
Useful at storage expanding, when you want to rebalance some data at cluster. |
-| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. |
+| maintenanceInsertNodeIDs#
_integer array_ | _(Optional)_
MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
Useful at storage expanding, when you want to rebalance some data at cluster.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
and defers changes to the pod and its volumes until the id is removed from either list.
With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
See https://docs.victoriametrics.com/operator/resources/vlcluster/#pods-excluded-from-all-routing |
+| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
and defers changes to the pod and its volumes until the id is removed from either list.
With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
See https://docs.victoriametrics.com/operator/resources/vlcluster/#pods-excluded-from-all-routing |
| minReadySeconds#
_integer_ | _(Optional)_
MinReadySeconds defines a minimum number of seconds to wait before starting update next pod
if previous in healthy state
Has no effect for VLogs and VMSingle |
| networkPolicy#
_[EmbeddedNetworkPolicy (v1beta1)](#v1beta1-embeddednetworkpolicy)_ | _(Optional)_
NetworkPolicy defines network access rules for pods created by this CR. |
| nodeSelector#
_object (keys:string, values:string)_ | _(Optional)_
NodeSelector Define which Nodes the Pods are scheduled on. |
@@ -1279,8 +1279,8 @@ Appears in: [VTClusterSpec (v1)](#v1-vtclusterspec)
| logIngestedRows#
_boolean_ | _(Required)_
Whether to log all the ingested log entries; this can be useful for debugging of data ingestion
see https://docs.victoriametrics.com/victoriatraces/#configure-victoriatraces |
| logLevel#
_string_ | _(Optional)_
LogLevel for VTStorage to be configured with. |
| logNewStreams#
_boolean_ | _(Required)_
LogNewStreams Whether to log creation of new streams; this can be useful for debugging of high cardinality issues with log streams
see https://docs.victoriametrics.com/victoriatraces/#configure-victoriatraces |
-| maintenanceInsertNodeIDs#
_integer array_ | _(Optional)_
MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
Useful at storage expanding, when you want to rebalance some data at cluster. |
-| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. |
+| maintenanceInsertNodeIDs#
_integer array_ | _(Optional)_
MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
Useful at storage expanding, when you want to rebalance some data at cluster.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
and defers changes to the pod and its volumes until the id is removed from either list.
With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
See https://docs.victoriametrics.com/operator/resources/vtcluster/#pods-excluded-from-all-routing |
+| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
and defers changes to the pod and its volumes until the id is removed from either list.
With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
See https://docs.victoriametrics.com/operator/resources/vtcluster/#pods-excluded-from-all-routing |
| minReadySeconds#
_integer_ | _(Optional)_
MinReadySeconds defines a minimum number of seconds to wait before starting update next pod
if previous in healthy state
Has no effect for VLogs and VMSingle |
| networkPolicy#
_[EmbeddedNetworkPolicy (v1beta1)](#v1beta1-embeddednetworkpolicy)_ | _(Optional)_
NetworkPolicy defines network access rules for pods created by this CR. |
| nodeSelector#
_object (keys:string, values:string)_ | _(Optional)_
NodeSelector Define which Nodes the Pods are scheduled on. |
@@ -5754,8 +5754,8 @@ Appears in: [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec)
| livenessProbe#
_[Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#probe-v1-core)_ | _(Optional)_
LivenessProbe that will be added to CR pod |
| logFormat#
_string_ | _(Optional)_
LogFormat for VMStorage to be configured with.
default or json |
| logLevel#
_string_ | _(Optional)_
LogLevel for VMStorage to be configured with. |
-| maintenanceInsertNodeIDs#
_integer array_ | _(Optional)_
MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
Useful at storage expanding, when you want to rebalance some data at cluster. |
-| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc. |
+| maintenanceInsertNodeIDs#
_integer array_ | _(Optional)_
MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
Useful at storage expanding, when you want to rebalance some data at cluster.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
and defers changes to the pod and its volumes until the id is removed from either list.
With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
See https://docs.victoriametrics.com/operator/resources/vmcluster/#pods-excluded-from-all-routing |
+| maintenanceSelectNodeIDs#
_integer array_ | _(Optional)_
MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
and defers changes to the pod and its volumes until the id is removed from either list.
With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
See https://docs.victoriametrics.com/operator/resources/vmcluster/#pods-excluded-from-all-routing |
| minReadySeconds#
_integer_ | _(Optional)_
MinReadySeconds defines a minimum number of seconds to wait before starting update next pod
if previous in healthy state
Has no effect for VLogs and VMSingle |
| networkPolicy#
_[EmbeddedNetworkPolicy (v1beta1)](#v1beta1-embeddednetworkpolicy)_ | _(Optional)_
NetworkPolicy defines network access rules for pods created by this CR. |
| nodeSelector#
_object (keys:string, values:string)_ | _(Optional)_
NodeSelector Define which Nodes the Pods are scheduled on. |
diff --git a/docs/resources/vlcluster.md b/docs/resources/vlcluster.md
index bce71d7c20..2e7cb31049 100644
--- a/docs/resources/vlcluster.md
+++ b/docs/resources/vlcluster.md
@@ -190,6 +190,56 @@ spec:
maxSurge: 1
```
+## Storage node maintenance mode
+
+`vlstorage.maintenanceInsertNodeIDs` and `vlstorage.maintenanceSelectNodeIDs` exclude the listed `vlstorage` pods from `vlinsert` and `vlselect` routing.
+Each ID is a pod ordinal, the numeric suffix of the pod name, e.g. `2` for the pod ending with `-2`.
+Use them when a `vlstorage` pod is expected to be unavailable for a while,
+for example when its Kubernetes node is under maintenance and the pod can't move, since its volume is bound to that node.
+
+Excluding a pod from routing affects the cluster:
+
+- with the pod in `maintenanceSelectNodeIDs`, logs stored on this pod are missing from query responses,
+ and `vlselect` doesn't mark them as partial, since it doesn't know about the pod;
+- with the pod in `maintenanceInsertNodeIDs`, `vlinsert` spreads incoming logs across the remaining pods,
+ so they take over the excluded pod's share of writes.
+
+### Pods excluded from all routing
+
+Starting from [v0.75.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.75.1),
+the Operator leaves a pod listed in **both** fields alone during reconciliation, since it receives no reads and no writes:
+
+- it doesn't wait for the pod to become ready, so a pod stuck `Pending` or `Terminating` no longer blocks
+ reconciliation of `vlinsert` and `vlselect`;
+- it doesn't apply `vlstorage` spec changes to the pod or its volumes, e.g. a new image or a bigger volume size.
+
+A pod listed in only one of the two fields still receives the other kind of traffic, so the Operator handles it as usual.
+
+```yaml
+apiVersion: operator.victoriametrics.com/v1
+kind: VLCluster
+metadata:
+ name: example
+spec:
+ vlstorage:
+ replicaCount: 3
+ maintenanceInsertNodeIDs: [2]
+ maintenanceSelectNodeIDs: [2] # the pod with ordinal 2 gets no traffic and is left alone by the Operator
+```
+
+When the node is back, remove the ID from both fields before making other changes.
+The Operator then applies the `vlstorage` changes deferred during maintenance to the pod,
+and waits for it to become ready before adding it back to `vlinsert` and `vlselect`, so an unhealthy pod never receives traffic.
+
+**Limitations:**
+
+- Pod spec changes are deferred only with the default `rollingUpdateStrategy: OnDelete`, where the Operator updates `vlstorage` pods itself.
+ With `RollingUpdate`, Kubernetes rolls out all pods, including the pod under maintenance, using `OrderedReady` pod management,
+ which is also used when `minReadySeconds` is set. In this mode, while the pod under maintenance isn't ready, Kubernetes doesn't roll out updates,
+ scale the StatefulSet or recreate pods with a higher ordinal, so reconciliation that changes `vlstorage` pods can block until the pod is ready again.
+- With `vlstorage.podDisruptionBudget.maxUnavailable: 1`, a pod under maintenance that isn't ready uses up the disruption budget,
+ so the Operator can't evict other `vlstorage` pods, and changes that require restarting them fail until the pod is ready again.
+
## Resource management
You can specify resources for each `VLCluster` resource components in the `spec` section of the `VLCluster` CRD.
diff --git a/docs/resources/vmcluster.md b/docs/resources/vmcluster.md
index 6d623af04d..057adf0670 100644
--- a/docs/resources/vmcluster.md
+++ b/docs/resources/vmcluster.md
@@ -426,6 +426,59 @@ spec:
maxSurge: 1
```
+## Storage node maintenance mode
+
+`vmstorage.maintenanceInsertNodeIDs` and `vmstorage.maintenanceSelectNodeIDs` exclude the listed `vmstorage` pods from `vminsert` and `vmselect` routing.
+Each ID is a pod ordinal, the numeric suffix of the pod name, e.g. `2` for the pod ending with `-2`.
+Use them when a `vmstorage` pod is expected to be unavailable for a while,
+for example when its Kubernetes node is under maintenance and the pod can't move, since its volume is bound to that node.
+
+Excluding a pod from routing affects the cluster:
+
+- with the pod in `maintenanceSelectNodeIDs`, data stored only on this pod is missing from query responses,
+ and `vmselect` doesn't mark them as partial, since it doesn't know about the pod.
+ Use [replicationFactor](#high-availability) > 1 to keep such data available;
+- with the pod in `maintenanceInsertNodeIDs`, `vminsert` spreads incoming series across the remaining pods,
+ so they take over the excluded pod's share of writes, and some series move to other pods, which increases churn there.
+
+These fields cannot be combined with [automatic vmstorage discovery](#automatic-vmstorage-discovery).
+
+### Pods excluded from all routing
+
+Starting from [v0.75.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.75.1),
+the Operator leaves a pod listed in **both** fields alone during reconciliation, since it receives no reads and no writes:
+
+- it doesn't wait for the pod to become ready, so a pod stuck `Pending` or `Terminating` no longer blocks
+ reconciliation of `vminsert` and `vmselect`;
+- it doesn't apply `vmstorage` spec changes to the pod or its volumes, e.g. a new image or a bigger volume size.
+
+A pod listed in only one of the two fields still receives the other kind of traffic, so the Operator handles it as usual.
+
+```yaml
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMCluster
+metadata:
+ name: example
+spec:
+ vmstorage:
+ replicaCount: 3
+ maintenanceInsertNodeIDs: [2]
+ maintenanceSelectNodeIDs: [2] # the pod with ordinal 2 gets no traffic and is left alone by the Operator
+```
+
+When the node is back, remove the ID from both fields before making other changes.
+The Operator then applies the `vmstorage` changes deferred during maintenance to the pod,
+and waits for it to become ready before adding it back to `vminsert` and `vmselect`, so an unhealthy pod never receives traffic.
+
+**Limitations:**
+
+- Pod spec changes are deferred only with the default `rollingUpdateStrategy: OnDelete`, where the Operator updates `vmstorage` pods itself.
+ With `RollingUpdate`, Kubernetes rolls out all pods, including the pod under maintenance, using `OrderedReady` pod management,
+ which is also used when `minReadySeconds` is set. In this mode, while the pod under maintenance isn't ready, Kubernetes doesn't roll out updates,
+ scale the StatefulSet or recreate pods with a higher ordinal, so reconciliation that changes `vmstorage` pods can block until the pod is ready again.
+- With `vmstorage.podDisruptionBudget.maxUnavailable: 1`, a pod under maintenance that isn't ready uses up the disruption budget,
+ so the Operator can't evict other `vmstorage` pods, and changes that require restarting them fail until the pod is ready again.
+
## Resource management
You can specify resources for each component of `VMCluster` resource in the `spec` section of the `VMCluster` CRD.
diff --git a/docs/resources/vtcluster.md b/docs/resources/vtcluster.md
index 2c653a0a4f..7d3c1cb327 100644
--- a/docs/resources/vtcluster.md
+++ b/docs/resources/vtcluster.md
@@ -216,6 +216,56 @@ spec:
maxSurge: 1
```
+## Storage node maintenance mode
+
+`storage.maintenanceInsertNodeIDs` and `storage.maintenanceSelectNodeIDs` exclude the listed `storage` pods from `insert` and `select` routing.
+Each ID is a pod ordinal, the numeric suffix of the pod name, e.g. `2` for the pod ending with `-2`.
+Use them when a `storage` pod is expected to be unavailable for a while,
+for example when its Kubernetes node is under maintenance and the pod can't move, since its volume is bound to that node.
+
+Excluding a pod from routing affects the cluster:
+
+- with the pod in `maintenanceSelectNodeIDs`, traces stored on this pod are missing from query responses,
+ and `select` doesn't mark them as partial, since it doesn't know about the pod;
+- with the pod in `maintenanceInsertNodeIDs`, `insert` spreads incoming spans across the remaining pods,
+ so they take over the excluded pod's share of writes.
+
+### Pods excluded from all routing
+
+Starting from [v0.75.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.75.1),
+the Operator leaves a pod listed in **both** fields alone during reconciliation, since it receives no reads and no writes:
+
+- it doesn't wait for the pod to become ready, so a pod stuck `Pending` or `Terminating` no longer blocks
+ reconciliation of `insert` and `select`;
+- it doesn't apply `storage` spec changes to the pod or its volumes, e.g. a new image or a bigger volume size.
+
+A pod listed in only one of the two fields still receives the other kind of traffic, so the Operator handles it as usual.
+
+```yaml
+apiVersion: operator.victoriametrics.com/v1
+kind: VTCluster
+metadata:
+ name: example
+spec:
+ storage:
+ replicaCount: 3
+ maintenanceInsertNodeIDs: [2]
+ maintenanceSelectNodeIDs: [2] # the pod with ordinal 2 gets no traffic and is left alone by the Operator
+```
+
+When the node is back, remove the ID from both fields before making other changes.
+The Operator then applies the `storage` changes deferred during maintenance to the pod,
+and waits for it to become ready before adding it back to `insert` and `select`, so an unhealthy pod never receives traffic.
+
+**Limitations:**
+
+- Pod spec changes are deferred only with the default `rollingUpdateStrategy: OnDelete`, where the Operator updates `storage` pods itself.
+ With `RollingUpdate`, Kubernetes rolls out all pods, including the pod under maintenance, using `OrderedReady` pod management,
+ which is also used when `minReadySeconds` is set. In this mode, while the pod under maintenance isn't ready, Kubernetes doesn't roll out updates,
+ scale the StatefulSet or recreate pods with a higher ordinal, so reconciliation that changes `storage` pods can block until the pod is ready again.
+- With `storage.podDisruptionBudget.maxUnavailable: 1`, a pod under maintenance that isn't ready uses up the disruption budget,
+ so the Operator can't evict other `storage` pods, and changes that require restarting them fail until the pod is ready again.
+
## Resource management
You can specify resources for each `VTCluster` resource components in the `spec` section of the `VTCluster` CRD.
diff --git a/internal/controller/operator/factory/reconcile/statefulset.go b/internal/controller/operator/factory/reconcile/statefulset.go
index bc3aaab490..9482b4dcad 100644
--- a/internal/controller/operator/factory/reconcile/statefulset.go
+++ b/internal/controller/operator/factory/reconcile/statefulset.go
@@ -35,12 +35,13 @@ const podRevisionLabel = "controller-revision-hash"
// StatefulSetOpts options for StatefulSet update
// HPA and UpdateReplicaCount optional
type StatefulSetOpts struct {
- SelectorLabels map[string]string
- PatchSpec func(existingSpec, newSpec *appsv1.StatefulSetSpec)
- UpdateBehavior *vmv1beta1.StatefulSetUpdateStrategyBehavior
+ SelectorLabels map[string]string
+ PatchSpec func(existingSpec, newSpec *appsv1.StatefulSetSpec)
+ UpdateBehavior *vmv1beta1.StatefulSetUpdateStrategyBehavior
+ IgnoreNotReadyPodOrdinals sets.Set[int32]
}
-func waitForStatefulSetReady(ctx context.Context, rclient client.Client, newObj *appsv1.StatefulSet) error {
+func waitForStatefulSetReady(ctx context.Context, rclient client.Client, newObj *appsv1.StatefulSet, ignoreNotReadyPodOrdinals sets.Set[int32]) error {
err := wait.PollUntilContextTimeout(ctx, podWaitReadyInterval, appWaitReadyTimeout, true, func(ctx context.Context) (done bool, err error) {
var existingObj appsv1.StatefulSet
if err := rclient.Get(ctx, types.NamespacedName{Namespace: newObj.Namespace, Name: newObj.Name}, &existingObj); err != nil {
@@ -64,8 +65,41 @@ func waitForStatefulSetReady(ctx context.Context, rclient client.Client, newObj
if targetReplicas == nil {
return true, nil
}
- if *targetReplicas != existingObj.Status.ReadyReplicas || *targetReplicas != existingObj.Status.UpdatedReplicas {
- return false, nil
+ if ignoreNotReadyPodOrdinals.Len() == 0 {
+ if *targetReplicas != existingObj.Status.UpdatedReplicas {
+ return false, nil
+ }
+ return *targetReplicas == existingObj.Status.ReadyReplicas, nil
+ }
+ // status.updatedReplicas doesn't count terminating pods, e.g. a pod stuck terminating on a shut down node,
+ // so check each pod, which is not excluded by ignoreNotReadyPodOrdinals, individually
+ var podList corev1.PodList
+ if err := rclient.List(ctx, &podList, &client.ListOptions{
+ Namespace: newObj.Namespace,
+ LabelSelector: labels.SelectorFromSet(newObj.Spec.Selector.MatchLabels),
+ }); err != nil {
+ return false, err
+ }
+ readyOrdinals := sets.New[int32]()
+ for _, pod := range podList.Items {
+ // a pod counts only if it isn't terminating and already runs the latest revision,
+ // so a pod that is ready but not yet updated keeps the wait going until k8s replaces it
+ if !pod.DeletionTimestamp.IsZero() || pod.Labels[podRevisionLabel] != existingObj.Status.UpdateRevision {
+ continue
+ }
+ if PodIsReady(&pod, newObj.Spec.MinReadySeconds) {
+ if ordinal, err := podOrdinal(pod.Name); err == nil {
+ readyOrdinals.Insert(ordinal)
+ }
+ }
+ }
+ for i := int32(0); i < *targetReplicas; i++ {
+ if ignoreNotReadyPodOrdinals.Has(i) {
+ continue
+ }
+ if !readyOrdinals.Has(i) {
+ return false, nil
+ }
}
return true, nil
})
@@ -168,7 +202,7 @@ func StatefulSet(ctx context.Context, rclient client.Client, newObj, prevObj *ap
// check if pvcs need to resize
if len(newObj.Spec.VolumeClaimTemplates) > 0 {
- if err := updateSTSPVC(ctx, rclient, newObj, prevVCTs, existingObj.Status.Replicas); err != nil {
+ if err := updateSTSPVC(ctx, rclient, newObj, prevVCTs, existingObj.Status.Replicas, o.IgnoreNotReadyPodOrdinals); err != nil {
return err
}
}
@@ -177,9 +211,10 @@ func StatefulSet(ctx context.Context, rclient client.Client, newObj, prevObj *ap
switch updateStrategy {
case appsv1.OnDeleteStatefulSetStrategyType:
opts := rollingUpdateOpts{
- recreate: recreatePod,
- selector: o.SelectorLabels,
- maxUnavailable: 1,
+ recreate: recreatePod,
+ selector: o.SelectorLabels,
+ maxUnavailable: 1,
+ ignoreNotReadyOrdinals: o.IgnoreNotReadyPodOrdinals,
}
if o.UpdateBehavior != nil {
if o.UpdateBehavior.MaxUnavailable.String() == "100%" {
@@ -199,7 +234,7 @@ func StatefulSet(ctx context.Context, rclient client.Client, newObj, prevObj *ap
if hpaManaged {
newObj.Spec.Replicas = nil
}
- if err := waitForStatefulSetReady(ctx, rclient, newObj); err != nil {
+ if err := waitForStatefulSetReady(ctx, rclient, newObj, o.IgnoreNotReadyPodOrdinals); err != nil {
return fmt.Errorf("cannot ensure that statefulset is ready with strategy=%q: %w", updateStrategy, err)
}
}
@@ -229,10 +264,11 @@ func getLatestStsState(ctx context.Context, rclient client.Client, targetSTS typ
}
type rollingUpdateOpts struct {
- recreate bool
- maxUnavailable int
- selector map[string]string
- delete bool
+ recreate bool
+ maxUnavailable int
+ selector map[string]string
+ delete bool
+ ignoreNotReadyOrdinals sets.Set[int32]
}
// patchSTSCurrentRevision patches statefulset status.currentRevision to match status.updateRevision
@@ -326,10 +362,27 @@ func performRollingUpdateOnSts(ctx context.Context, rclient client.Client, obj *
return patchSTSCurrentRevision(ctx, rclient, nsn, stsVersion, int32(neededPodCount))
}
+ // defer updates for pods excluded by maintenance node IDs
+ var deferredUpdate bool
+ filteredPodsForUpdate := podsForUpdate[:0]
+ for _, pod := range podsForUpdate {
+ if ordinal, err := podOrdinal(pod.Name); err == nil && o.ignoreNotReadyOrdinals.Has(ordinal) {
+ l.Info(fmt.Sprintf("deferring update for pod %s, excluded by maintenance node IDs", pod.Name))
+ deferredUpdate = true
+ continue
+ }
+ filteredPodsForUpdate = append(filteredPodsForUpdate, pod)
+ }
+ podsForUpdate = filteredPodsForUpdate
+
l.Info(fmt.Sprintf("discovered already updated pods=%d, pods needed to be update=%d", len(updatedPods), len(podsForUpdate)))
// check updated, by not ready pods
for _, pod := range updatedPods {
+ if ordinal, err := podOrdinal(pod.Name); err == nil && o.ignoreNotReadyOrdinals.Has(ordinal) {
+ l.Info(fmt.Sprintf("skipping not-ready check for pod %s, excluded by maintenance node IDs", pod.Name))
+ continue
+ }
l.Info(fmt.Sprintf("checking ready status for already updated pod %s to revision version=%q", pod.Name, stsVersion))
podNsn := types.NamespacedName{Namespace: obj.Namespace, Name: pod.Name}
if err := waitForPodReady(ctx, rclient, podNsn, stsVersion, sts.Spec.MinReadySeconds); err != nil {
@@ -395,6 +448,11 @@ func performRollingUpdateOnSts(ctx context.Context, rclient client.Client, obj *
}
}
+ if deferredUpdate {
+ l.Info("statefulset update is incomplete: some pods were deferred due to maintenance node IDs")
+ return nil
+ }
+
l.Info(fmt.Sprintf("finished statefulset update from revision=%q to revision=%q", sts.Status.CurrentRevision, stsVersion))
return patchSTSCurrentRevision(ctx, rclient, nsn, stsVersion, int32(neededPodCount))
@@ -495,6 +553,18 @@ func podStatusesToError(origin error, pod *corev1.Pod) error {
return fmt.Errorf("%s: %w", msg, origin)
}
+func podOrdinal(name string) (int32, error) {
+ n := strings.LastIndexByte(name, '-')
+ if n <= 0 {
+ return 0, fmt.Errorf("cannot find - at the pod name: %s", name)
+ }
+ id, err := strconv.ParseInt(name[n+1:], 10, 32)
+ if err != nil {
+ return 0, fmt.Errorf("cannot parse pod id number: %s from name: %s", name[n+1:], name)
+ }
+ return int32(id), nil
+}
+
func sortStsPodsByID(src []corev1.Pod) error {
var firstParseError error
sort.Slice(src, func(i, j int) bool {
diff --git a/internal/controller/operator/factory/reconcile/statefulset_pvc_expand.go b/internal/controller/operator/factory/reconcile/statefulset_pvc_expand.go
index 9633f15473..cea6967aab 100644
--- a/internal/controller/operator/factory/reconcile/statefulset_pvc_expand.go
+++ b/internal/controller/operator/factory/reconcile/statefulset_pvc_expand.go
@@ -15,6 +15,7 @@ import (
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
+ "k8s.io/apimachinery/pkg/util/sets"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -83,7 +84,7 @@ func getPVCByName(vcts []corev1.PersistentVolumeClaim, name string) *corev1.Pers
return nil
}
-func updateSTSPVC(ctx context.Context, rclient client.Client, sts *appsv1.StatefulSet, prevVCTs []corev1.PersistentVolumeClaim, liveReplicas int32) error {
+func updateSTSPVC(ctx context.Context, rclient client.Client, sts *appsv1.StatefulSet, prevVCTs []corev1.PersistentVolumeClaim, liveReplicas int32, ignorePodOrdinals sets.Set[int32]) error {
// fast path
if sts.Spec.Replicas != nil && *sts.Spec.Replicas == 0 {
return nil
@@ -129,6 +130,18 @@ func updateSTSPVC(ctx context.Context, rclient client.Client, sts *appsv1.Statef
l.Info(fmt.Sprintf("possible BUG, cannot find target PVC=%s in new statefulset by claim name=%s", pvc.Name, stsClaimName))
continue
}
+ // defer changes for pvcs of pods excluded by maintenance node IDs, the same as their pod updates.
+ // a resize usually completes only after the kubelet grows the filesystem on the node where the volume is mounted,
+ // so it cannot complete while the pod is down, and waiting for it would fail the reconcile before pod updates.
+ //
+ // changes are applied once the pod is removed from maintenance, since each reconcile compares pvcs with the claim templates.
+ if ordinal, err := podOrdinal(pvc.Name); err == nil && ignorePodOrdinals.Has(ordinal) {
+ if pvc.Spec.Resources.Requests.Storage().Cmp(*stsClaim.Spec.Resources.Requests.Storage()) != 0 ||
+ ptr.Deref(pvc.Spec.VolumeAttributesClassName, "") != ptr.Deref(stsClaim.Spec.VolumeAttributesClassName, "") {
+ l.Info(fmt.Sprintf("deferring update for PVC=%s, excluded by maintenance node IDs", pvc.Name))
+ }
+ continue
+ }
prevVCT := getPVCByName(prevVCTs, stsClaimName)
nsnPvc := types.NamespacedName{Name: pvc.Name, Namespace: pvc.Namespace}
// update PVC size and metadata if it's needed, retrying on conflict
diff --git a/internal/controller/operator/factory/reconcile/statefulset_pvc_expand_test.go b/internal/controller/operator/factory/reconcile/statefulset_pvc_expand_test.go
index 98c1f712b5..1f42b1ec44 100644
--- a/internal/controller/operator/factory/reconcile/statefulset_pvc_expand_test.go
+++ b/internal/controller/operator/factory/reconcile/statefulset_pvc_expand_test.go
@@ -14,6 +14,7 @@ import (
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
+ "k8s.io/apimachinery/pkg/util/sets"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -160,6 +161,7 @@ func Test_updateSTSPVC(t *testing.T) {
actions []k8stools.ClientAction
k8sMinorVersion uint64
liveReplicas *int32
+ ignoreOrdinals sets.Set[int32]
}
f := func(o opts) {
t.Helper()
@@ -181,7 +183,7 @@ func Test_updateSTSPVC(t *testing.T) {
cl.Actions = nil
}
synctest.Test(t, func(t *testing.T) {
- err := updateSTSPVC(ctx, cl, o.sts, o.prevVCTs, liveReplicas)
+ err := updateSTSPVC(ctx, cl, o.sts, o.prevVCTs, liveReplicas, o.ignoreOrdinals)
if o.wantErrText != "" {
assert.ErrorContains(t, err, o.wantErrText)
} else {
@@ -1751,4 +1753,101 @@ func Test_updateSTSPVC(t *testing.T) {
},
},
})
+
+ // PVC of a pod excluded by maintenance node IDs is not updated and not waited for
+ maintenancePVCNSN := types.NamespacedName{Name: "vmselect-cachedir-vmselect-1", Namespace: "default"}
+ f(opts{
+ sts: buildSTS(func(sts *appsv1.StatefulSet) {
+ sts.Spec.VolumeClaimTemplates = []corev1.PersistentVolumeClaim{
+ {
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "vmselect-cachedir",
+ Labels: map[string]string{"app": "vmselect"},
+ },
+ Spec: corev1.PersistentVolumeClaimSpec{
+ Resources: corev1.VolumeResourceRequirements{
+ Requests: map[corev1.ResourceName]resource.Quantity{
+ corev1.ResourceStorage: resource.MustParse("15Gi"),
+ },
+ },
+ },
+ },
+ }
+ }),
+ ignoreOrdinals: sets.New[int32](1),
+ preRun: func(c client.Client) {
+ for _, nsn := range []types.NamespacedName{pvc1NSN, maintenancePVCNSN} {
+ assert.NoError(t, c.Create(context.TODO(), &corev1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: nsn.Name,
+ Namespace: nsn.Namespace,
+ Labels: map[string]string{"app": "vmselect"},
+ Annotations: map[string]string{
+ "operator.victoriametrics.com/pvc-allow-volume-expansion": "true",
+ },
+ },
+ Spec: corev1.PersistentVolumeClaimSpec{
+ Resources: corev1.VolumeResourceRequirements{
+ Requests: map[corev1.ResourceName]resource.Quantity{
+ corev1.ResourceStorage: resource.MustParse("10Gi"),
+ },
+ },
+ },
+ }))
+ }
+ },
+ actions: []k8stools.ClientAction{
+ {Verb: "Get", Kind: "PersistentVolumeClaim", Resource: pvc1NSN},
+ {Verb: "Update", Kind: "PersistentVolumeClaim", Resource: pvc1NSN},
+ {Verb: "Get", Kind: "PersistentVolumeClaim", Resource: pvc1NSN},
+ },
+ expected: []corev1.PersistentVolumeClaim{
+ {
+ ObjectMeta: metav1.ObjectMeta{
+ Name: pvc1NSN.Name,
+ Namespace: pvc1NSN.Namespace,
+ Labels: map[string]string{"app": "vmselect"},
+ Annotations: map[string]string{
+ "operator.victoriametrics.com/pvc-allow-volume-expansion": "true",
+ },
+ ResourceVersion: "4",
+ },
+ Spec: corev1.PersistentVolumeClaimSpec{
+ Resources: corev1.VolumeResourceRequirements{
+ Requests: map[corev1.ResourceName]resource.Quantity{
+ corev1.ResourceStorage: resource.MustParse("15Gi"),
+ },
+ },
+ },
+ Status: corev1.PersistentVolumeClaimStatus{
+ Capacity: map[corev1.ResourceName]resource.Quantity{
+ corev1.ResourceStorage: resource.MustParse("15Gi"),
+ },
+ },
+ },
+ {
+ ObjectMeta: metav1.ObjectMeta{
+ Name: maintenancePVCNSN.Name,
+ Namespace: maintenancePVCNSN.Namespace,
+ Labels: map[string]string{"app": "vmselect"},
+ Annotations: map[string]string{
+ "operator.victoriametrics.com/pvc-allow-volume-expansion": "true",
+ },
+ ResourceVersion: "2",
+ },
+ Spec: corev1.PersistentVolumeClaimSpec{
+ Resources: corev1.VolumeResourceRequirements{
+ Requests: map[corev1.ResourceName]resource.Quantity{
+ corev1.ResourceStorage: resource.MustParse("10Gi"),
+ },
+ },
+ },
+ Status: corev1.PersistentVolumeClaimStatus{
+ Capacity: map[corev1.ResourceName]resource.Quantity{
+ corev1.ResourceStorage: resource.MustParse("10Gi"),
+ },
+ },
+ },
+ },
+ })
}
diff --git a/internal/controller/operator/factory/reconcile/statefulset_test.go b/internal/controller/operator/factory/reconcile/statefulset_test.go
index 1b3b17ea01..6b7637e5d0 100644
--- a/internal/controller/operator/factory/reconcile/statefulset_test.go
+++ b/internal/controller/operator/factory/reconcile/statefulset_test.go
@@ -18,6 +18,7 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
+ "k8s.io/apimachinery/pkg/util/sets"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
@@ -803,6 +804,141 @@ func Test_performRollingUpdateOnSts(t *testing.T) {
},
wantErr: true,
})
+
+ // not-ready pod at current revision times out
+ f(opts{
+ sts: &appsv1.StatefulSet{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "vmstorage-sts",
+ Namespace: "default",
+ },
+ Status: appsv1.StatefulSetStatus{
+ CurrentRevision: "rev1",
+ UpdateRevision: "rev1",
+ },
+ },
+ opts: rollingUpdateOpts{
+ selector: map[string]string{"app": "vmstorage"},
+ maxUnavailable: 1,
+ },
+ predefinedObjects: []runtime.Object{
+ &appsv1.StatefulSet{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "vmstorage-sts",
+ Namespace: "default",
+ Labels: map[string]string{"app": "vmstorage"},
+ },
+ Spec: appsv1.StatefulSetSpec{Replicas: ptr.To(int32(1))},
+ Status: appsv1.StatefulSetStatus{
+ CurrentRevision: "rev1",
+ UpdateRevision: "rev1",
+ },
+ },
+ &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "vmstorage-sts-0",
+ Namespace: "default",
+ Labels: map[string]string{"app": "vmstorage", podRevisionLabel: "rev1"},
+ OwnerReferences: []metav1.OwnerReference{{
+ Kind: "StatefulSet",
+ }},
+ },
+ Status: corev1.PodStatus{},
+ },
+ },
+ wantErr: true,
+ })
+
+ // not-ready pod excluded via ignoreNotReadyOrdinals does not block the update
+ f(opts{
+ sts: &appsv1.StatefulSet{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "vmstorage-sts",
+ Namespace: "default",
+ },
+ Status: appsv1.StatefulSetStatus{
+ CurrentRevision: "rev1",
+ UpdateRevision: "rev1",
+ },
+ },
+ opts: rollingUpdateOpts{
+ selector: map[string]string{"app": "vmstorage"},
+ maxUnavailable: 1,
+ ignoreNotReadyOrdinals: sets.New[int32](0),
+ },
+ predefinedObjects: []runtime.Object{
+ &appsv1.StatefulSet{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "vmstorage-sts",
+ Namespace: "default",
+ Labels: map[string]string{"app": "vmstorage"},
+ },
+ Spec: appsv1.StatefulSetSpec{Replicas: ptr.To(int32(1))},
+ Status: appsv1.StatefulSetStatus{
+ CurrentRevision: "rev1",
+ UpdateRevision: "rev1",
+ },
+ },
+ &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "vmstorage-sts-0",
+ Namespace: "default",
+ Labels: map[string]string{"app": "vmstorage", podRevisionLabel: "rev1"},
+ OwnerReferences: []metav1.OwnerReference{{
+ Kind: "StatefulSet",
+ }},
+ },
+ Status: corev1.PodStatus{},
+ },
+ },
+ actions: map[string][]string{},
+ })
+
+ // a pod on the old revision excluded via ignoreNotReadyOrdinals is deferred, not
+ // evicted and waited on
+ f(opts{
+ sts: &appsv1.StatefulSet{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "vmstorage-sts",
+ Namespace: "default",
+ },
+ Status: appsv1.StatefulSetStatus{
+ CurrentRevision: "rev0",
+ UpdateRevision: "rev1",
+ },
+ },
+ opts: rollingUpdateOpts{
+ selector: map[string]string{"app": "vmstorage"},
+ maxUnavailable: 1,
+ ignoreNotReadyOrdinals: sets.New[int32](0),
+ },
+ predefinedObjects: []runtime.Object{
+ &appsv1.StatefulSet{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "vmstorage-sts",
+ Namespace: "default",
+ Labels: map[string]string{"app": "vmstorage"},
+ },
+ Spec: appsv1.StatefulSetSpec{Replicas: ptr.To(int32(1))},
+ Status: appsv1.StatefulSetStatus{
+ CurrentRevision: "rev0",
+ UpdateRevision: "rev1",
+ },
+ },
+ &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "vmstorage-sts-0",
+ Namespace: "default",
+ Labels: map[string]string{"app": "vmstorage", podRevisionLabel: "rev0"},
+ OwnerReferences: []metav1.OwnerReference{{
+ Kind: "StatefulSet",
+ }},
+ },
+ Status: corev1.PodStatus{},
+ },
+ },
+ actions: map[string][]string{},
+ })
}
func TestSortPodsByID(t *testing.T) {
@@ -1363,6 +1499,119 @@ func TestStatefulsetReconcile(t *testing.T) {
assert.Equal(t, ptr.To[int32](6), s.Spec.Replicas)
},
})
+
+ // RollingUpdate strategy: excluded pod ordinal doesn't count against readiness
+ mkPod := func(idx int, ready bool, fns ...func(*corev1.Pod)) *corev1.Pod {
+ p := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: fmt.Sprintf("test-1-%d", idx),
+ Namespace: "default",
+ Labels: map[string]string{"label": "value", podRevisionLabel: "rev1"},
+ },
+ }
+ if ready {
+ p.Status = corev1.PodStatus{
+ Phase: corev1.PodRunning,
+ Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}},
+ }
+ }
+ for _, fn := range fns {
+ fn(p)
+ }
+ return p
+ }
+ mkRollingSts := func(updatedReplicas, readyReplicas int32) *appsv1.StatefulSet {
+ return getSts(func(s *appsv1.StatefulSet) {
+ s.Spec.UpdateStrategy.Type = appsv1.RollingUpdateStatefulSetStrategyType
+ s.Status.Replicas = 4
+ s.Status.ReadyReplicas = readyReplicas
+ s.Status.UpdatedReplicas = updatedReplicas
+ s.Status.UpdateRevision = "rev1"
+ })
+ }
+ newRollingSts := func() *appsv1.StatefulSet {
+ return getSts(func(s *appsv1.StatefulSet) {
+ s.Spec.UpdateStrategy.Type = appsv1.RollingUpdateStatefulSetStrategyType
+ })
+ }
+ f(opts{
+ new: newRollingSts(),
+ prev: getSts(),
+ predefinedObjects: []runtime.Object{
+ mkRollingSts(4, 3),
+ mkPod(0, true),
+ mkPod(1, true),
+ mkPod(2, true),
+ mkPod(3, false),
+ },
+ actions: []k8stools.ClientAction{
+ {Verb: "Get", Kind: "StatefulSet", Resource: nn},
+ {Verb: "Get", Kind: "StatefulSet", Resource: nn},
+ },
+ o: &StatefulSetOpts{
+ IgnoreNotReadyPodOrdinals: sets.New[int32](3),
+ },
+ })
+
+ // RollingUpdate strategy: excluded pod stuck terminating, e.g. at a shut down node,
+ // isn't counted at status.updatedReplicas, but must not block readiness
+ f(opts{
+ new: newRollingSts(),
+ prev: getSts(),
+ predefinedObjects: []runtime.Object{
+ mkRollingSts(3, 3),
+ mkPod(0, true),
+ mkPod(1, true),
+ mkPod(2, true),
+ mkPod(3, true, func(p *corev1.Pod) {
+ p.DeletionTimestamp = ptr.To(metav1.Now())
+ p.Finalizers = []string{"test"}
+ }),
+ },
+ actions: []k8stools.ClientAction{
+ {Verb: "Get", Kind: "StatefulSet", Resource: nn},
+ {Verb: "Get", Kind: "StatefulSet", Resource: nn},
+ },
+ o: &StatefulSetOpts{
+ IgnoreNotReadyPodOrdinals: sets.New[int32](3),
+ },
+ })
+
+ // RollingUpdate strategy: an unready non-maintenance pod still blocks readiness
+ f(opts{
+ new: newRollingSts(),
+ prev: getSts(),
+ predefinedObjects: []runtime.Object{
+ mkRollingSts(4, 3),
+ mkPod(0, true),
+ mkPod(1, false),
+ mkPod(2, true),
+ mkPod(3, true),
+ },
+ o: &StatefulSetOpts{
+ IgnoreNotReadyPodOrdinals: sets.New[int32](3),
+ },
+ wantErr: true,
+ })
+
+ // RollingUpdate strategy: a ready non-maintenance pod at the previous revision still blocks readiness
+ f(opts{
+ new: newRollingSts(),
+ prev: getSts(),
+ predefinedObjects: []runtime.Object{
+ mkRollingSts(3, 4),
+ mkPod(0, true, func(p *corev1.Pod) {
+ p.Labels[podRevisionLabel] = "rev0"
+ }),
+ mkPod(1, true),
+ mkPod(2, true),
+ mkPod(3, true),
+ },
+ o: &StatefulSetOpts{
+ IgnoreNotReadyPodOrdinals: sets.New[int32](3),
+ },
+ wantErr: true,
+ })
}
func TestStatefulsetReconcile_ExistingObjNotStaleAfterNotFoundRetry(t *testing.T) {
diff --git a/internal/controller/operator/factory/vlcluster/vlstorage.go b/internal/controller/operator/factory/vlcluster/vlstorage.go
index 016eb4237b..6842a7c352 100644
--- a/internal/controller/operator/factory/vlcluster/vlstorage.go
+++ b/internal/controller/operator/factory/vlcluster/vlstorage.go
@@ -189,6 +189,7 @@ func createOrUpdateVLStorageSTS(ctx context.Context, rclient client.Client, cr,
cr.Spec.VLStorage.ReplicaCount = existingSpec.Replicas
}
},
+ IgnoreNotReadyPodOrdinals: cr.StorageNodeIDsUnderMaintenance(),
}
owner := cr.AsOwner()
return reconcile.StatefulSet(ctx, rclient, newSts, prevSts, &owner, &o)
diff --git a/internal/controller/operator/factory/vmcluster/vmcluster.go b/internal/controller/operator/factory/vmcluster/vmcluster.go
index b72d3b1ef7..37ca976ca9 100644
--- a/internal/controller/operator/factory/vmcluster/vmcluster.go
+++ b/internal/controller/operator/factory/vmcluster/vmcluster.go
@@ -458,6 +458,7 @@ func createOrUpdateVMStorage(ctx context.Context, rclient client.Client, cr, pre
cr.Spec.VMStorage.ReplicaCount = existingSpec.Replicas
}
},
+ IgnoreNotReadyPodOrdinals: cr.StorageNodeIDsUnderMaintenance(),
}
owner := cr.AsOwner()
return reconcile.StatefulSet(ctx, rclient, newSts, prevSts, &owner, &o)
diff --git a/internal/controller/operator/factory/vmcluster/vmcluster_test.go b/internal/controller/operator/factory/vmcluster/vmcluster_test.go
index 33f3d57dea..0974964f5a 100644
--- a/internal/controller/operator/factory/vmcluster/vmcluster_test.go
+++ b/internal/controller/operator/factory/vmcluster/vmcluster_test.go
@@ -101,12 +101,12 @@ func TestCreateOrUpdate(t *testing.T) {
},
predefinedObjects: []runtime.Object{
&corev1.Pod{
- ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-0", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-0", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
&corev1.Pod{
- ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-1", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-1", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
},
})
@@ -197,11 +197,11 @@ func TestCreateOrUpdate(t *testing.T) {
predefinedObjects: []runtime.Object{
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "select-0", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmselect", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "select-1", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmselect", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
},
})
@@ -234,44 +234,44 @@ func TestCreateOrUpdate(t *testing.T) {
},
predefinedObjects: []runtime.Object{
&corev1.Pod{
- ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-0", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-0", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
&corev1.Pod{
- ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-1", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-1", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
&corev1.Pod{
- ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-2", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-2", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
&corev1.Pod{
- ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-3", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-3", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
&corev1.Pod{
- ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-4", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-4", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
&corev1.Pod{
- ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-5", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-5", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
&corev1.Pod{
- ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-6", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-6", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
&corev1.Pod{
- ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-7", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-7", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
&corev1.Pod{
- ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-8", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-8", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
&corev1.Pod{
- ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-9", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-9", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
},
})
@@ -304,44 +304,44 @@ func TestCreateOrUpdate(t *testing.T) {
},
predefinedObjects: []runtime.Object{
&corev1.Pod{
- ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-0", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-0", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
&corev1.Pod{
- ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-1", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-1", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
&corev1.Pod{
- ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-2", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-2", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
&corev1.Pod{
- ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-3", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-3", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
&corev1.Pod{
- ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-4", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-4", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
&corev1.Pod{
- ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-5", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-5", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
&corev1.Pod{
- ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-6", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-6", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
&corev1.Pod{
- ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-7", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-7", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
&corev1.Pod{
- ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-8", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-8", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
&corev1.Pod{
- ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-9", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator"}},
- Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "true"}}},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "storage-9", Labels: map[string]string{"app.kubernetes.io/component": "monitoring", "app.kubernetes.io/name": "vmstorage", "app.kubernetes.io/instance": "cluster-1", "managed-by": "vm-operator", "controller-revision-hash": "v1"}},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: "True"}}},
},
},
})
diff --git a/internal/controller/operator/factory/vtcluster/storage.go b/internal/controller/operator/factory/vtcluster/storage.go
index 91009e6d99..49472c4ff3 100644
--- a/internal/controller/operator/factory/vtcluster/storage.go
+++ b/internal/controller/operator/factory/vtcluster/storage.go
@@ -186,6 +186,7 @@ func createOrUpdateVTStorageSTS(ctx context.Context, rclient client.Client, cr,
cr.Spec.Storage.ReplicaCount = existingSpec.Replicas
}
},
+ IgnoreNotReadyPodOrdinals: cr.StorageNodeIDsUnderMaintenance(),
}
owner := cr.AsOwner()
return reconcile.StatefulSet(ctx, rclient, newSts, prevSts, &owner, &o)