Resize volume by changing pvc size if enabled in config. (#958)

* Try to resize pvc if resizing pv has failed

* added config option to switch between storage resize strategies

* changes according to requests

* Update pkg/controller/operator_config.go

Co-authored-by: Felix Kunde <felix-kunde@gmx.de>

* enable_storage_resize documented

added examples to the default configuration and helm value files

* enable_storage_resize renamed to volume_resize_mode, off by default

* volume_resize_mode renamed to storage_resize_mode

* Update pkg/apis/acid.zalan.do/v1/crds.go

* pkg/cluster/volumes.go updated

* Update docs/reference/operator_parameters.md

* Update manifests/postgresql-operator-default-configuration.yaml

* Update pkg/controller/operator_config.go

* Update pkg/util/config/config.go

* Update charts/postgres-operator/values-crd.yaml

* Update charts/postgres-operator/values.yaml

* Update docs/reference/operator_parameters.md

* added logging if no changes required

Co-authored-by: Felix Kunde <felix-kunde@gmx.de>

pkg/cluster/sync.go

@@ -57,16 +57,26 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
 		return err
 	}
 
-	// potentially enlarge volumes before changing the statefulset. By doing that
-	// in this order we make sure the operator is not stuck waiting for a pod that
-	// cannot start because it ran out of disk space.
-	// TODO: handle the case of the cluster that is downsized and enlarged again
-	// (there will be a volume from the old pod for which we can't act before the
-	//  the statefulset modification is concluded)
-	c.logger.Debugf("syncing persistent volumes")
-	if err = c.syncVolumes(); err != nil {
-		err = fmt.Errorf("could not sync persistent volumes: %v", err)
-		return err
+	if c.OpConfig.StorageResizeMode == "pvc" {
+		c.logger.Debugf("syncing persistent volume claims")
+		if err = c.syncVolumeClaims(); err != nil {
+			err = fmt.Errorf("could not sync persistent volume claims: %v", err)
+			return err
+		}
+	} else if c.OpConfig.StorageResizeMode == "ebs" {
+		// potentially enlarge volumes before changing the statefulset. By doing that
+		// in this order we make sure the operator is not stuck waiting for a pod that
+		// cannot start because it ran out of disk space.
+		// TODO: handle the case of the cluster that is downsized and enlarged again
+		// (there will be a volume from the old pod for which we can't act before the
+		//  the statefulset modification is concluded)
+		c.logger.Debugf("syncing persistent volumes")
+		if err = c.syncVolumes(); err != nil {
+			err = fmt.Errorf("could not sync persistent volumes: %v", err)
+			return err
+		}
+	} else {
+		c.logger.Infof("Storage resize is disabled (storage_resize_mode is off). Skipping volume sync.")
 	}
 
 	if err = c.enforceMinResourceLimits(&c.Spec); err != nil {
@@ -571,6 +581,27 @@ func (c *Cluster) syncRoles() (err error) {
 	return nil
 }
 
+// syncVolumeClaims reads all persistent volume claims and checks that their size matches the one declared in the statefulset.
+func (c *Cluster) syncVolumeClaims() error {
+	c.setProcessName("syncing volume claims")
+
+	act, err := c.volumeClaimsNeedResizing(c.Spec.Volume)
+	if err != nil {
+		return fmt.Errorf("could not compare size of the volume claims: %v", err)
+	}
+	if !act {
+		c.logger.Infof("volume claims don't require changes")
+		return nil
+	}
+	if err := c.resizeVolumeClaims(c.Spec.Volume); err != nil {
+		return fmt.Errorf("could not sync volume claims: %v", err)
+	}
+
+	c.logger.Infof("volume claims have been synced successfully")
+
+	return nil
+}
+
 // syncVolumes reads all persistent volumes and checks that their size matches the one declared in the statefulset.
 func (c *Cluster) syncVolumes() error {
 	c.setProcessName("syncing volumes")

pkg/cluster/volumes.go

@@ -52,6 +52,35 @@ func (c *Cluster) deletePersistentVolumeClaims() error {
 	return nil
 }
 
+func (c *Cluster) resizeVolumeClaims(newVolume acidv1.Volume) error {
+	c.logger.Debugln("resizing PVCs")
+	pvcs, err := c.listPersistentVolumeClaims()
+	if err != nil {
+		return err
+	}
+	newQuantity, err := resource.ParseQuantity(newVolume.Size)
+	if err != nil {
+		return fmt.Errorf("could not parse volume size: %v", err)
+	}
+	_, newSize, err := c.listVolumesWithManifestSize(newVolume)
+	for _, pvc := range pvcs {
+		volumeSize := quantityToGigabyte(pvc.Spec.Resources.Requests[v1.ResourceStorage])
+		if volumeSize >= newSize {
+			if volumeSize > newSize {
+				c.logger.Warningf("cannot shrink persistent volume")
+			}
+			continue
+		}
+		pvc.Spec.Resources.Requests[v1.ResourceStorage] = newQuantity
+		c.logger.Debugf("updating persistent volume claim definition for volume %q", pvc.Name)
+		if _, err := c.KubeClient.PersistentVolumeClaims(pvc.Namespace).Update(context.TODO(), &pvc, metav1.UpdateOptions{}); err != nil {
+			return fmt.Errorf("could not update persistent volume claim: %q", err)
+		}
+		c.logger.Debugf("successfully updated persistent volume claim %q", pvc.Name)
+	}
+	return nil
+}
+
 func (c *Cluster) listPersistentVolumes() ([]*v1.PersistentVolume, error) {
 	result := make([]*v1.PersistentVolume, 0)
 
@@ -150,7 +179,7 @@ func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.Volu
 			c.logger.Debugf("successfully updated persistent volume %q", pv.Name)
 		}
 		if !compatible {
-			c.logger.Warningf("volume %q is incompatible with all available resizing providers", pv.Name)
+			c.logger.Warningf("volume %q is incompatible with all available resizing providers, consider switching storage_resize_mode to pvc or off", pv.Name)
 			totalIncompatible++
 		}
 	}
@@ -160,6 +189,25 @@ func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.Volu
 	return nil
 }
 
+func (c *Cluster) volumeClaimsNeedResizing(newVolume acidv1.Volume) (bool, error) {
+	newSize, err := resource.ParseQuantity(newVolume.Size)
+	manifestSize := quantityToGigabyte(newSize)
+	if err != nil {
+		return false, fmt.Errorf("could not parse volume size from the manifest: %v", err)
+	}
+	pvcs, err := c.listPersistentVolumeClaims()
+	if err != nil {
+		return false, fmt.Errorf("could not receive persistent volume claims: %v", err)
+	}
+	for _, pvc := range pvcs {
+		currentSize := quantityToGigabyte(pvc.Spec.Resources.Requests[v1.ResourceStorage])
+		if currentSize != manifestSize {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
 func (c *Cluster) volumesNeedResizing(newVolume acidv1.Volume) (bool, error) {
 	vols, manifestSize, err := c.listVolumesWithManifestSize(newVolume)
 	if err != nil {