diff --git a/internal/controller/operator/factory/reconcile/pvc.go b/internal/controller/operator/factory/reconcile/pvc.go index 0d42b1a0d..81f7d5b65 100644 --- a/internal/controller/operator/factory/reconcile/pvc.go +++ b/internal/controller/operator/factory/reconcile/pvc.go @@ -8,6 +8,7 @@ import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/logger" @@ -20,24 +21,58 @@ import ( // in case of deletion timestamp > 0 does nothing // user must manually remove finalizer if needed func PersistentVolumeClaim(ctx context.Context, rclient client.Client, newObj, prevObj *corev1.PersistentVolumeClaim, owner *metav1.OwnerReference) error { - l := logger.WithContext(ctx) - var existingObj corev1.PersistentVolumeClaim nsn := types.NamespacedName{Namespace: newObj.Namespace, Name: newObj.Name} - if err := rclient.Get(ctx, nsn, &existingObj); err != nil { - if k8serrors.IsNotFound(err) { - l.Info(fmt.Sprintf("creating new PVC=%s", nsn.String())) - if err := rclient.Create(ctx, newObj); err != nil { - return fmt.Errorf("cannot create new PVC=%s: %w", nsn.String(), err) + var existingObj corev1.PersistentVolumeClaim + err := retryOnConflict(func() error { + if err := rclient.Get(ctx, nsn, &existingObj); err != nil { + if k8serrors.IsNotFound(err) { + logger.WithContext(ctx).Info(fmt.Sprintf("creating new PVC=%s", nsn.String())) + return rclient.Create(ctx, newObj) } + return fmt.Errorf("cannot get existing PVC=%s: %w", nsn.String(), err) + } + if !existingObj.DeletionTimestamp.IsZero() { + logger.WithContext(ctx).Info(fmt.Sprintf("PVC=%s has non zero DeletionTimestamp, skip update."+ + " To fix this, make backup for this pvc, delete pvc finalizers and restore from backup.", nsn.String())) return nil } - return fmt.Errorf("cannot get existing PVC=%s: %w", nsn.String(), err) + return updatePVC(ctx, rclient, &existingObj, newObj, prevObj, owner) + }) + if err != nil { + return err } - if !existingObj.DeletionTimestamp.IsZero() { - l.Info(fmt.Sprintf("PVC=%s has non zero DeletionTimestamp, skip update."+ - " To fix this, make backup for this pvc, delete pvc finalizers and restore from backup.", nsn.String())) - return nil + var generation int64 + switch { + case !newObj.CreationTimestamp.IsZero(): + generation = newObj.Generation + case !existingObj.CreationTimestamp.IsZero(): + generation = existingObj.Generation } + return waitForPVCBound(ctx, rclient, nsn, generation) +} - return updatePVC(ctx, rclient, &existingObj, newObj, prevObj, owner) +func waitForPVCBound(ctx context.Context, rclient client.Client, nsn types.NamespacedName, generation int64) error { + var pvc corev1.PersistentVolumeClaim + return wait.PollUntilContextTimeout(ctx, pvcWaitBoundIntervalCheck, pvcWaitReadyTimeout, true, func(ctx context.Context) (done bool, err error) { + if err := rclient.Get(ctx, nsn, &pvc); err != nil { + if k8serrors.IsNotFound(err) { + return false, nil + } + return false, fmt.Errorf("cannot get PVC=%s: %w", nsn.String(), err) + } + if !pvc.DeletionTimestamp.IsZero() { + return true, fmt.Errorf("cannot wait for PVC=%s, which is being terminated", nsn.String()) + } + if generation > pvc.Generation { + return false, nil + } + switch pvc.Status.Phase { + case corev1.ClaimBound: + return true, nil + case corev1.ClaimPending: + return false, nil + default: + return false, fmt.Errorf("failed to wait for PVC=%s, which in %s phase", nsn.String(), pvc.Status.Phase) + } + }) } diff --git a/internal/controller/operator/factory/reconcile/reconcile.go b/internal/controller/operator/factory/reconcile/reconcile.go index bd39a4b67..2bdd27a27 100644 --- a/internal/controller/operator/factory/reconcile/reconcile.go +++ b/internal/controller/operator/factory/reconcile/reconcile.go @@ -23,6 +23,8 @@ import ( ) var ( + pvcWaitBoundIntervalCheck = 1 * time.Second + pvcWaitReadyTimeout = 5 * time.Second podWaitReadyIntervalCheck = 50 * time.Millisecond appWaitReadyDeadline = 5 * time.Second podWaitReadyTimeout = 5 * time.Second diff --git a/internal/controller/operator/factory/reconcile/statefulset_pvc_expand.go b/internal/controller/operator/factory/reconcile/statefulset_pvc_expand.go index a438b12cb..751c697d2 100644 --- a/internal/controller/operator/factory/reconcile/statefulset_pvc_expand.go +++ b/internal/controller/operator/factory/reconcile/statefulset_pvc_expand.go @@ -121,6 +121,9 @@ func updateSTSPVC(ctx context.Context, rclient client.Client, sts *appsv1.Statef if err := updatePVC(ctx, rclient, &pvc, &stsClaim, prevVCT, nil); err != nil { return err } + if err := waitForPVCBound(ctx, rclient, nsn, pvc.Generation); err != nil { + return err + } } return nil }