Support EBS gp2 to gp3 migration on sync for below 1tb volumes (#1242)

* initial commit for gp3 migration.

* Default volume migration done.

* Added Gomock and one test case with mock.

* Dep update.

* more changes for code gen.

* push fake package.

* Rename var.

* Changes to Makefile and return value.

* Macke mocks phony due to overlap in foldername.

* Learning as one goes. Initialize map.

* Wrong toggle.

* Expect modify call.

* Fix mapping of ids in test.

* Fix volume id.

* volume ids.

* Fixing test setup. Late night...

* create all pvs.

* Fix test case config.

* store volumes and compare.

* More logs.

* Logging of migration action.

* Ensure to log errors.

* Log warning if modify failed, e.g. due to ebs volume state.

* Add more output.

* Skip local e2e tests.

* Reflect k8s volume id in test data. Extract aws volume id from k8s value.

* Finalizing ebs migration.

* More logs. describe fails.

* Fix non existing fields in gp2 discovery.

* Remove nothing to do flag for migration.

* Final commit for migration.

* add new options to all places

Co-authored-by: Felix Kunde <felix-kunde@gmx.de>

pkg/cluster/cluster.go

@@ -25,6 +25,7 @@ import (
 	"github.com/zalando/postgres-operator/pkg/util/patroni"
 	"github.com/zalando/postgres-operator/pkg/util/teams"
 	"github.com/zalando/postgres-operator/pkg/util/users"
+	"github.com/zalando/postgres-operator/pkg/util/volumes"
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	policybeta1 "k8s.io/api/policy/v1beta1"
@@ -89,7 +90,10 @@ type Cluster struct {
 	processMu        sync.RWMutex // protects the current operation for reporting, no need to hold the master mutex
 	specMu           sync.RWMutex // protects the spec for reporting, no need to hold the master mutex
 	ConnectionPooler map[PostgresRole]*ConnectionPoolerObjects
+	EBSVolumes       map[string]volumes.VolumeProperties
+	VolumeResizer    volumes.VolumeResizer
 }
+
 type compareStatefulsetResult struct {
 	match         bool
 	replace       bool
@@ -134,6 +138,13 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres
 	cluster.oauthTokenGetter = newSecretOauthTokenGetter(&kubeClient, cfg.OpConfig.OAuthTokenSecretName)
 	cluster.patroni = patroni.New(cluster.logger)
 	cluster.eventRecorder = eventRecorder
+
+	cluster.EBSVolumes = make(map[string]volumes.VolumeProperties)
+	if cfg.OpConfig.StorageResizeMode != "pvc" || cfg.OpConfig.EnableEBSGp3Migration {
+		cluster.VolumeResizer = &volumes.EBSVolumeResizer{AWSRegion: cfg.OpConfig.AWSRegion}
+
+	}
+
 	return cluster
 }
 

pkg/cluster/sync.go

@@ -11,7 +11,6 @@ import (
 	"github.com/zalando/postgres-operator/pkg/util"
 	"github.com/zalando/postgres-operator/pkg/util/constants"
 	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
-	"github.com/zalando/postgres-operator/pkg/util/volumes"
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1beta1 "k8s.io/api/batch/v1beta1"
 	v1 "k8s.io/api/core/v1"
@@ -55,7 +54,23 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
 	}
 
 	c.logger.Debugf("syncing volumes using %q storage resize mode", c.OpConfig.StorageResizeMode)
-	if c.OpConfig.StorageResizeMode == "pvc" {
+
+	if c.OpConfig.EnableEBSGp3Migration {
+		err = c.executeEBSMigration()
+		if nil != err {
+			return err
+		}
+	}
+
+	if c.OpConfig.StorageResizeMode == "mixed" {
+		// mixed op uses AWS API to adjust size,throughput,iops and calls pvc chance for file system resize
+
+		// resize pvc to adjust filesystem size until better K8s support
+		if err = c.syncVolumeClaims(); err != nil {
+			err = fmt.Errorf("could not sync persistent volume claims: %v", err)
+			return err
+		}
+	} else if c.OpConfig.StorageResizeMode == "pvc" {
 		if err = c.syncVolumeClaims(); err != nil {
 			err = fmt.Errorf("could not sync persistent volume claims: %v", err)
 			return err
@@ -599,7 +614,8 @@ func (c *Cluster) syncVolumes() error {
 	if !act {
 		return nil
 	}
-	if err := c.resizeVolumes(c.Spec.Volume, []volumes.VolumeResizer{&volumes.EBSVolumeResizer{AWSRegion: c.OpConfig.AWSRegion}}); err != nil {
+
+	if err := c.resizeVolumes(); err != nil {
 		return fmt.Errorf("could not sync volumes: %v", err)
 	}
 

pkg/cluster/volumes.go

@@ -15,7 +15,6 @@ import (
 	"github.com/zalando/postgres-operator/pkg/util"
 	"github.com/zalando/postgres-operator/pkg/util/constants"
 	"github.com/zalando/postgres-operator/pkg/util/filesystems"
-	"github.com/zalando/postgres-operator/pkg/util/volumes"
 )
 
 func (c *Cluster) listPersistentVolumeClaims() ([]v1.PersistentVolumeClaim, error) {
@@ -119,19 +118,26 @@ func (c *Cluster) listPersistentVolumes() ([]*v1.PersistentVolume, error) {
 }
 
 // resizeVolumes resize persistent volumes compatible with the given resizer interface
-func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.VolumeResizer) error {
-	c.setProcessName("resizing volumes")
+func (c *Cluster) resizeVolumes() error {
+	if c.VolumeResizer == nil {
+		return fmt.Errorf("no volume resizer set for EBS volume handling")
+	}
 
+	c.setProcessName("resizing EBS volumes")
+
+	resizer := c.VolumeResizer
 	var totalIncompatible int
 
-	newQuantity, err := resource.ParseQuantity(newVolume.Size)
+	newQuantity, err := resource.ParseQuantity(c.Spec.Volume.Size)
 	if err != nil {
 		return fmt.Errorf("could not parse volume size: %v", err)
 	}
-	pvs, newSize, err := c.listVolumesWithManifestSize(newVolume)
+
+	pvs, newSize, err := c.listVolumesWithManifestSize(c.Spec.Volume)
 	if err != nil {
 		return fmt.Errorf("could not list persistent volumes: %v", err)
 	}
+
 	for _, pv := range pvs {
 		volumeSize := quantityToGigabyte(pv.Spec.Capacity[v1.ResourceStorage])
 		if volumeSize >= newSize {
@@ -141,43 +147,43 @@ func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.Volu
 			continue
 		}
 		compatible := false
-		for _, resizer := range resizers {
-			if !resizer.VolumeBelongsToProvider(pv) {
-				continue
-			}
-			compatible = true
-			if !resizer.IsConnectedToProvider() {
-				err := resizer.ConnectToProvider()
-				if err != nil {
-					return fmt.Errorf("could not connect to the volume provider: %v", err)
-				}
-				defer func() {
-					if err := resizer.DisconnectFromProvider(); err != nil {
-						c.logger.Errorf("%v", err)
-					}
-				}()
-			}
-			awsVolumeID, err := resizer.GetProviderVolumeID(pv)
-			if err != nil {
-				return err
-			}
-			c.logger.Debugf("updating persistent volume %q to %d", pv.Name, newSize)
-			if err := resizer.ResizeVolume(awsVolumeID, newSize); err != nil {
-				return fmt.Errorf("could not resize EBS volume %q: %v", awsVolumeID, err)
-			}
-			c.logger.Debugf("resizing the filesystem on the volume %q", pv.Name)
-			podName := getPodNameFromPersistentVolume(pv)
-			if err := c.resizePostgresFilesystem(podName, []filesystems.FilesystemResizer{&filesystems.Ext234Resize{}}); err != nil {
-				return fmt.Errorf("could not resize the filesystem on pod %q: %v", podName, err)
-			}
-			c.logger.Debugf("filesystem resize successful on volume %q", pv.Name)
-			pv.Spec.Capacity[v1.ResourceStorage] = newQuantity
-			c.logger.Debugf("updating persistent volume definition for volume %q", pv.Name)
-			if _, err := c.KubeClient.PersistentVolumes().Update(context.TODO(), pv, metav1.UpdateOptions{}); err != nil {
-				return fmt.Errorf("could not update persistent volume: %q", err)
-			}
-			c.logger.Debugf("successfully updated persistent volume %q", pv.Name)
+
+		if !resizer.VolumeBelongsToProvider(pv) {
+			continue
 		}
+		compatible = true
+		if !resizer.IsConnectedToProvider() {
+			err := resizer.ConnectToProvider()
+			if err != nil {
+				return fmt.Errorf("could not connect to the volume provider: %v", err)
+			}
+			defer func() {
+				if err := resizer.DisconnectFromProvider(); err != nil {
+					c.logger.Errorf("%v", err)
+				}
+			}()
+		}
+		awsVolumeID, err := resizer.GetProviderVolumeID(pv)
+		if err != nil {
+			return err
+		}
+		c.logger.Debugf("updating persistent volume %q to %d", pv.Name, newSize)
+		if err := resizer.ResizeVolume(awsVolumeID, newSize); err != nil {
+			return fmt.Errorf("could not resize EBS volume %q: %v", awsVolumeID, err)
+		}
+		c.logger.Debugf("resizing the filesystem on the volume %q", pv.Name)
+		podName := getPodNameFromPersistentVolume(pv)
+		if err := c.resizePostgresFilesystem(podName, []filesystems.FilesystemResizer{&filesystems.Ext234Resize{}}); err != nil {
+			return fmt.Errorf("could not resize the filesystem on pod %q: %v", podName, err)
+		}
+		c.logger.Debugf("filesystem resize successful on volume %q", pv.Name)
+		pv.Spec.Capacity[v1.ResourceStorage] = newQuantity
+		c.logger.Debugf("updating persistent volume definition for volume %q", pv.Name)
+		if _, err := c.KubeClient.PersistentVolumes().Update(context.TODO(), pv, metav1.UpdateOptions{}); err != nil {
+			return fmt.Errorf("could not update persistent volume: %q", err)
+		}
+		c.logger.Debugf("successfully updated persistent volume %q", pv.Name)
+
 		if !compatible {
 			c.logger.Warningf("volume %q is incompatible with all available resizing providers, consider switching storage_resize_mode to pvc or off", pv.Name)
 			totalIncompatible++
@@ -245,3 +251,61 @@ func getPodNameFromPersistentVolume(pv *v1.PersistentVolume) *spec.NamespacedNam
 func quantityToGigabyte(q resource.Quantity) int64 {
 	return q.ScaledValue(0) / (1 * constants.Gigabyte)
 }
+
+func (c *Cluster) executeEBSMigration() error {
+	if !c.OpConfig.EnableEBSGp3Migration {
+		return nil
+	}
+	c.logger.Infof("starting EBS gp2 to gp3 migration")
+
+	pvs, _, err := c.listVolumesWithManifestSize(c.Spec.Volume)
+	if err != nil {
+		return fmt.Errorf("could not list persistent volumes: %v", err)
+	}
+	c.logger.Debugf("found %d volumes, size of known volumes %d", len(pvs), len(c.EBSVolumes))
+
+	volumeIds := []string{}
+	var volumeID string
+	for _, pv := range pvs {
+		volumeID, err = c.VolumeResizer.ExtractVolumeID(pv.Spec.AWSElasticBlockStore.VolumeID)
+		if err != nil {
+			continue
+		}
+
+		volumeIds = append(volumeIds, volumeID)
+	}
+
+	if len(volumeIds) == len(c.EBSVolumes) {
+		hasGp2 := false
+		for _, v := range c.EBSVolumes {
+			if v.VolumeType == "gp2" {
+				hasGp2 = true
+			}
+		}
+
+		if !hasGp2 {
+			c.logger.Infof("no EBS gp2 volumes left to migrate")
+			return nil
+		}
+	}
+
+	awsVolumes, err := c.VolumeResizer.DescribeVolumes(volumeIds)
+	if nil != err {
+		return err
+	}
+
+	for _, volume := range awsVolumes {
+		if volume.VolumeType == "gp2" && volume.Size < c.OpConfig.EnableEBSGp3MigrationMaxSize {
+			c.logger.Infof("modifying EBS volume %s to type gp3 migration (%d)", volume.VolumeID, volume.Size)
+			err = c.VolumeResizer.ModifyVolume(volume.VolumeID, "gp3", volume.Size, 3000, 125)
+			if nil != err {
+				c.logger.Warningf("modifying volume %s failed: %v", volume.VolumeID, err)
+			}
+		} else {
+			c.logger.Debugf("skipping EBS volume %s to type gp3 migration (%d)", volume.VolumeID, volume.Size)
+		}
+		c.EBSVolumes[volume.VolumeID] = volume
+	}
+
+	return nil
+}

pkg/cluster/volumes_test.go

@@ -1,6 +1,7 @@
 package cluster
 
 import (
+	"fmt"
 	"testing"
 
 	"context"
@@ -10,11 +11,14 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 
+	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/assert"
+	"github.com/zalando/postgres-operator/mocks"
 	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	"github.com/zalando/postgres-operator/pkg/util/config"
 	"github.com/zalando/postgres-operator/pkg/util/constants"
 	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
+	"github.com/zalando/postgres-operator/pkg/util/volumes"
 	"k8s.io/client-go/kubernetes/fake"
 )
 
@@ -23,6 +27,8 @@ func NewFakeKubernetesClient() (k8sutil.KubernetesClient, *fake.Clientset) {
 
 	return k8sutil.KubernetesClient{
 		PersistentVolumeClaimsGetter: clientSet.CoreV1(),
+		PersistentVolumesGetter:      clientSet.CoreV1(),
+		PodsGetter:                   clientSet.CoreV1(),
 	}, clientSet
 }
 
@@ -33,6 +39,9 @@ func TestResizeVolumeClaim(t *testing.T) {
 	namespace := "default"
 	newVolumeSize := "2Gi"
 
+	storage1Gi, err := resource.ParseQuantity("1Gi")
+	assert.NoError(t, err)
+
 	// new cluster with pvc storage resize mode and configured labels
 	var cluster = New(
 		Config{
@@ -51,55 +60,9 @@ func TestResizeVolumeClaim(t *testing.T) {
 	filterLabels := cluster.labelsSet(false)
 
 	// define and create PVCs for 1Gi volumes
-	storage1Gi, err := resource.ParseQuantity("1Gi")
-	assert.NoError(t, err)
-
-	pvcList := &v1.PersistentVolumeClaimList{
-		Items: []v1.PersistentVolumeClaim{
-			{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      constants.DataVolumeName + "-" + clusterName + "-0",
-					Namespace: namespace,
-					Labels:    filterLabels,
-				},
-				Spec: v1.PersistentVolumeClaimSpec{
-					Resources: v1.ResourceRequirements{
-						Requests: v1.ResourceList{
-							v1.ResourceStorage: storage1Gi,
-						},
-					},
-				},
-			},
-			{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      constants.DataVolumeName + "-" + clusterName + "-1",
-					Namespace: namespace,
-					Labels:    filterLabels,
-				},
-				Spec: v1.PersistentVolumeClaimSpec{
-					Resources: v1.ResourceRequirements{
-						Requests: v1.ResourceList{
-							v1.ResourceStorage: storage1Gi,
-						},
-					},
-				},
-			},
-			{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      constants.DataVolumeName + "-" + clusterName + "-2-0",
-					Namespace: namespace,
-					Labels:    labels.Set{},
-				},
-				Spec: v1.PersistentVolumeClaimSpec{
-					Resources: v1.ResourceRequirements{
-						Requests: v1.ResourceList{
-							v1.ResourceStorage: storage1Gi,
-						},
-					},
-				},
-			},
-		},
-	}
+	pvcList := CreatePVCs(namespace, clusterName, filterLabels, 2, "1Gi")
+	// add another PVC with different cluster name
+	pvcList.Items = append(pvcList.Items, CreatePVCs(namespace, clusterName+"-2", labels.Set{}, 1, "1Gi").Items[0])
 
 	for _, pvc := range pvcList.Items {
 		cluster.KubeClient.PersistentVolumeClaims(namespace).Create(context.TODO(), &pvc, metav1.CreateOptions{})
@@ -169,3 +132,132 @@ func TestQuantityToGigabyte(t *testing.T) {
 		}
 	}
 }
+
+func CreatePVCs(namespace string, clusterName string, labels labels.Set, n int, size string) v1.PersistentVolumeClaimList {
+	// define and create PVCs for 1Gi volumes
+	storage1Gi, _ := resource.ParseQuantity(size)
+	pvcList := v1.PersistentVolumeClaimList{
+		Items: []v1.PersistentVolumeClaim{},
+	}
+
+	for i := 0; i < n; i++ {
+		pvc := v1.PersistentVolumeClaim{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      fmt.Sprintf("%s-%s-%d", constants.DataVolumeName, clusterName, i),
+				Namespace: namespace,
+				Labels:    labels,
+			},
+			Spec: v1.PersistentVolumeClaimSpec{
+				Resources: v1.ResourceRequirements{
+					Requests: v1.ResourceList{
+						v1.ResourceStorage: storage1Gi,
+					},
+				},
+				VolumeName: fmt.Sprintf("persistent-volume-%d", i),
+			},
+		}
+		pvcList.Items = append(pvcList.Items, pvc)
+	}
+
+	return pvcList
+}
+
+func TestMigrateEBS(t *testing.T) {
+	client, _ := NewFakeKubernetesClient()
+	clusterName := "acid-test-cluster"
+	namespace := "default"
+
+	// new cluster with pvc storage resize mode and configured labels
+	var cluster = New(
+		Config{
+			OpConfig: config.Config{
+				Resources: config.Resources{
+					ClusterLabels:    map[string]string{"application": "spilo"},
+					ClusterNameLabel: "cluster-name",
+				},
+				StorageResizeMode:            "pvc",
+				EnableEBSGp3Migration:        true,
+				EnableEBSGp3MigrationMaxSize: 1000,
+			},
+		}, client, acidv1.Postgresql{}, logger, eventRecorder)
+	cluster.Spec.Volume.Size = "1Gi"
+
+	// set metadata, so that labels will get correct values
+	cluster.Name = clusterName
+	cluster.Namespace = namespace
+	filterLabels := cluster.labelsSet(false)
+
+	pvcList := CreatePVCs(namespace, clusterName, filterLabels, 2, "1Gi")
+
+	ps := v1.PersistentVolumeSpec{}
+	ps.AWSElasticBlockStore = &v1.AWSElasticBlockStoreVolumeSource{}
+	ps.AWSElasticBlockStore.VolumeID = "aws://eu-central-1b/ebs-volume-1"
+
+	ps2 := v1.PersistentVolumeSpec{}
+	ps2.AWSElasticBlockStore = &v1.AWSElasticBlockStoreVolumeSource{}
+	ps2.AWSElasticBlockStore.VolumeID = "aws://eu-central-1b/ebs-volume-2"
+
+	pvList := &v1.PersistentVolumeList{
+		Items: []v1.PersistentVolume{
+			{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "persistent-volume-0",
+				},
+				Spec: ps,
+			},
+			{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "persistent-volume-1",
+				},
+				Spec: ps2,
+			},
+		},
+	}
+
+	for _, pvc := range pvcList.Items {
+		cluster.KubeClient.PersistentVolumeClaims(namespace).Create(context.TODO(), &pvc, metav1.CreateOptions{})
+	}
+
+	for _, pv := range pvList.Items {
+		cluster.KubeClient.PersistentVolumes().Create(context.TODO(), &pv, metav1.CreateOptions{})
+	}
+
+	pod := v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   clusterName + "-0",
+			Labels: filterLabels,
+		},
+		Spec: v1.PodSpec{},
+	}
+
+	cluster.KubeClient.Pods(namespace).Create(context.TODO(), &pod, metav1.CreateOptions{})
+
+	pod = v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   clusterName + "-1",
+			Labels: filterLabels,
+		},
+		Spec: v1.PodSpec{},
+	}
+
+	cluster.KubeClient.Pods(namespace).Create(context.TODO(), &pod, metav1.CreateOptions{})
+
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	resizer := mocks.NewMockVolumeResizer(ctrl)
+
+	resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-1")).Return("ebs-volume-1", nil)
+	resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-2")).Return("ebs-volume-2", nil)
+
+	resizer.EXPECT().DescribeVolumes(gomock.Eq([]string{"ebs-volume-1", "ebs-volume-2"})).Return(
+		[]volumes.VolumeProperties{
+			{VolumeID: "ebs-volume-1", VolumeType: "gp2", Size: 100},
+			{VolumeID: "ebs-volume-2", VolumeType: "gp3", Size: 100}}, nil)
+
+	// expect only gp2 volume to be modified
+	resizer.EXPECT().ModifyVolume(gomock.Eq("ebs-volume-1"), gomock.Eq("gp3"), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
+
+	cluster.VolumeResizer = resizer
+	cluster.executeEBSMigration()
+}