public_git/postgres-operator
1
0
Fork 0
mirror of https://github.com/zalando/postgres-operator.git synced 2026-09-30 14:47:34 +02:00
Code Issues Projects Releases Wiki Activity

Support EBS gp2 to gp3 migration on sync for below 1tb volumes (#1242)

Browse Source 
* initial commit for gp3 migration.

* Default volume migration done.

* Added Gomock and one test case with mock.

* Dep update.

* more changes for code gen.

* push fake package.

* Rename var.

* Changes to Makefile and return value.

* Macke mocks phony due to overlap in foldername.

* Learning as one goes. Initialize map.

* Wrong toggle.

* Expect modify call.

* Fix mapping of ids in test.

* Fix volume id.

* volume ids.

* Fixing test setup. Late night...

* create all pvs.

* Fix test case config.

* store volumes and compare.

* More logs.

* Logging of migration action.

* Ensure to log errors.

* Log warning if modify failed, e.g. due to ebs volume state.

* Add more output.

* Skip local e2e tests.

* Reflect k8s volume id in test data. Extract aws volume id from k8s value.

* Finalizing ebs migration.

* More logs. describe fails.

* Fix non existing fields in gp2 discovery.

* Remove nothing to do flag for migration.

* Final commit for migration.

* add new options to all places

Co-authored-by: Felix Kunde <felix-kunde@gmx.de>
This commit is contained in:
Jan Mussler
2020-12-11 15:52:32 +01:00
committed by GitHub
co-authored by Felix Kunde
parent b9ef88f842
commit 549f71bb49
27 changed files with 568 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/util/config/config.go
+2
View File
@@ -163,6 +163,8 @@ type Config struct {
GCPCredentials string `name:"gcp_credentials"`
AdditionalSecretMount string `name:"additional_secret_mount"`
AdditionalSecretMountPath string `name:"additional_secret_mount_path" default:"/meta/credentials"`
EnableEBSGp3Migration bool `name:"enable_ebs_gp3_migration" default:"false"`
EnableEBSGp3MigrationMaxSize int64 `name:"enable_ebs_gp3_migration_max_size" default:"1000"`
DebugLogging bool `name:"debug_logging" default:"true"`
EnableDBAccess bool `name:"enable_database_access" default:"true"`
EnableTeamsAPI bool `name:"enable_teams_api" default:"true"`
pkg/util/volumes/ebs.go
+105 -20
View File
@@ -7,7 +7,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"github.com/zalando/postgres-operator/pkg/util/constants"
"github.com/zalando/postgres-operator/pkg/util/retryutil"
@@ -20,42 +20,80 @@ type EBSVolumeResizer struct {
}
// ConnectToProvider connects to AWS.
func (c *EBSVolumeResizer) ConnectToProvider() error {
sess, err := session.NewSession(&aws.Config{Region: aws.String(c.AWSRegion)})
func (r *EBSVolumeResizer) ConnectToProvider() error {
sess, err := session.NewSession(&aws.Config{Region: aws.String(r.AWSRegion)})
if err != nil {
return fmt.Errorf("could not establish AWS session: %v", err)
}
c.connection = ec2.New(sess)
r.connection = ec2.New(sess)
return nil
}
// IsConnectedToProvider checks if AWS connection is established.
func (c *EBSVolumeResizer) IsConnectedToProvider() bool {
return c.connection != nil
func (r *EBSVolumeResizer) IsConnectedToProvider() bool {
return r.connection != nil
}
// VolumeBelongsToProvider checks if the given persistent volume is backed by EBS.
func (c *EBSVolumeResizer) VolumeBelongsToProvider(pv *v1.PersistentVolume) bool {
func (r *EBSVolumeResizer) VolumeBelongsToProvider(pv *v1.PersistentVolume) bool {
return pv.Spec.AWSElasticBlockStore != nil && pv.Annotations[constants.VolumeStorateProvisionerAnnotation] == constants.EBSProvisioner
}
// GetProviderVolumeID converts aws://eu-central-1b/vol-00f93d4827217c629 to vol-00f93d4827217c629 for EBS volumes
func (c *EBSVolumeResizer) GetProviderVolumeID(pv *v1.PersistentVolume) (string, error) {
volumeID := pv.Spec.AWSElasticBlockStore.VolumeID
if volumeID == "" {
return "", fmt.Errorf("volume id is empty for volume %q", pv.Name)
}
// ExtractVolumeID extracts volumeID
func (r *EBSVolumeResizer) ExtractVolumeID(volumeID string) (string, error) {
idx := strings.LastIndex(volumeID, constants.EBSVolumeIDStart) + 1
if idx == 0 {
return "", fmt.Errorf("malfored EBS volume id %q", volumeID)
return "", fmt.Errorf("malformed EBS volume id %q", volumeID)
}
return volumeID[idx:], nil
}
// GetProviderVolumeID converts aws://eu-central-1b/vol-00f93d4827217c629 to vol-00f93d4827217c629 for EBS volumes
func (r *EBSVolumeResizer) GetProviderVolumeID(pv *v1.PersistentVolume) (string, error) {
volumeID := pv.Spec.AWSElasticBlockStore.VolumeID
if volumeID == "" {
return "", fmt.Errorf("got empty volume id for volume %v", pv)
}
return r.ExtractVolumeID(volumeID)
}
// DescribeVolumes ...
func (r *EBSVolumeResizer) DescribeVolumes(volumeIds []string) ([]VolumeProperties, error) {
if !r.IsConnectedToProvider() {
err := r.ConnectToProvider()
if err != nil {
return nil, err
}
}
volumeOutput, err := r.connection.DescribeVolumes(&ec2.DescribeVolumesInput{VolumeIds: aws.StringSlice((volumeIds))})
if err != nil {
return nil, err
}
p := []VolumeProperties{}
if nil == volumeOutput.Volumes {
return p, nil
}
for _, v := range volumeOutput.Volumes {
if *v.VolumeType == "gp3" {
p = append(p, VolumeProperties{VolumeID: *v.VolumeId, Size: *v.Size, VolumeType: *v.VolumeType, Iops: *v.Iops, Throughput: *v.Throughput})
} else if *v.VolumeType == "gp2" {
p = append(p, VolumeProperties{VolumeID: *v.VolumeId, Size: *v.Size, VolumeType: *v.VolumeType})
} else {
return nil, fmt.Errorf("Discovered unexpected volume type %s %s", *v.VolumeId, *v.VolumeType)
}
}
return p, nil
}
// ResizeVolume actually calls AWS API to resize the EBS volume if necessary.
func (c *EBSVolumeResizer) ResizeVolume(volumeID string, newSize int64) error {
func (r *EBSVolumeResizer) ResizeVolume(volumeID string, newSize int64) error {
/* first check if the volume is already of a requested size */
volumeOutput, err := c.connection.DescribeVolumes(&ec2.DescribeVolumesInput{VolumeIds: []*string{&volumeID}})
volumeOutput, err := r.connection.DescribeVolumes(&ec2.DescribeVolumesInput{VolumeIds: []*string{&volumeID}})
if err != nil {
return fmt.Errorf("could not get information about the volume: %v", err)
}
@@ -68,7 +106,7 @@ func (c *EBSVolumeResizer) ResizeVolume(volumeID string, newSize int64) error {
return nil
}
input := ec2.ModifyVolumeInput{Size: &newSize, VolumeId: &volumeID}
output, err := c.connection.ModifyVolume(&input)
output, err := r.connection.ModifyVolume(&input)
if err != nil {
return fmt.Errorf("could not modify persistent volume: %v", err)
}
@@ -87,7 +125,54 @@ func (c *EBSVolumeResizer) ResizeVolume(volumeID string, newSize int64) error {
in := ec2.DescribeVolumesModificationsInput{VolumeIds: []*string{&volumeID}}
return retryutil.Retry(constants.EBSVolumeResizeWaitInterval, constants.EBSVolumeResizeWaitTimeout,
func() (bool, error) {
out, err := c.connection.DescribeVolumesModifications(&in)
out, err := r.connection.DescribeVolumesModifications(&in)
if err != nil {
return false, fmt.Errorf("could not describe volume modification: %v", err)
}
if len(out.VolumesModifications) != 1 {
return false, fmt.Errorf("describe volume modification didn't return one record for volume %q", volumeID)
}
if *out.VolumesModifications[0].VolumeId != volumeID {
return false, fmt.Errorf("non-matching volume id when describing modifications: %q is different from %q",
*out.VolumesModifications[0].VolumeId, volumeID)
}
return *out.VolumesModifications[0].ModificationState != constants.EBSVolumeStateModifying, nil
})
}
// ModifyVolume Modify EBS volume
func (r *EBSVolumeResizer) ModifyVolume(volumeID string, newType string, newSize int64, iops int64, throughput int64) error {
/* first check if the volume is already of a requested size */
volumeOutput, err := r.connection.DescribeVolumes(&ec2.DescribeVolumesInput{VolumeIds: []*string{&volumeID}})
if err != nil {
return fmt.Errorf("could not get information about the volume: %v", err)
}
vol := volumeOutput.Volumes[0]
if *vol.VolumeId != volumeID {
return fmt.Errorf("describe volume %q returned information about a non-matching volume %q", volumeID, *vol.VolumeId)
}
input := ec2.ModifyVolumeInput{Size: &newSize, VolumeId: &volumeID, VolumeType: &newType, Iops: &iops, Throughput: &throughput}
output, err := r.connection.ModifyVolume(&input)
if err != nil {
return fmt.Errorf("could not modify persistent volume: %v", err)
}
state := *output.VolumeModification.ModificationState
if state == constants.EBSVolumeStateFailed {
return fmt.Errorf("could not modify persistent volume %q: modification state failed", volumeID)
}
if state == "" {
return fmt.Errorf("received empty modification status")
}
if state == constants.EBSVolumeStateOptimizing || state == constants.EBSVolumeStateCompleted {
return nil
}
// wait until the volume reaches the "optimizing" or "completed" state
in := ec2.DescribeVolumesModificationsInput{VolumeIds: []*string{&volumeID}}
return retryutil.Retry(constants.EBSVolumeResizeWaitInterval, constants.EBSVolumeResizeWaitTimeout,
func() (bool, error) {
out, err := r.connection.DescribeVolumesModifications(&in)
if err != nil {
return false, fmt.Errorf("could not describe volume modification: %v", err)
}
@@ -103,7 +188,7 @@ func (c *EBSVolumeResizer) ResizeVolume(volumeID string, newSize int64) error {
}
// DisconnectFromProvider closes connection to the EC2 instance
func (c *EBSVolumeResizer) DisconnectFromProvider() error {
c.connection = nil
func (r *EBSVolumeResizer) DisconnectFromProvider() error {
r.connection = nil
return nil
}
pkg/util/volumes/volumes.go
+15 -3
View File
@@ -1,8 +1,17 @@
package volumes
import (
"k8s.io/api/core/v1"
)
//go:generate mockgen -package mocks -destination=$PWD/mocks/$GOFILE -source=$GOFILE -build_flags=-mod=vendor
import v1 "k8s.io/api/core/v1"
// VolumeProperties ...
type VolumeProperties struct {
VolumeID string
VolumeType string
Size int64
Iops int64
Throughput int64
}
// VolumeResizer defines the set of methods used to implememnt provider-specific resizing of persistent volumes.
type VolumeResizer interface {
@@ -10,6 +19,9 @@ type VolumeResizer interface {
IsConnectedToProvider() bool
VolumeBelongsToProvider(pv *v1.PersistentVolume) bool
GetProviderVolumeID(pv *v1.PersistentVolume) (string, error)
ExtractVolumeID(volumeID string) (string, error)
ResizeVolume(providerVolumeID string, newSize int64) error
ModifyVolume(providerVolumeID string, newType string, newSize int64, iops int64, throughput int64) error
DisconnectFromProvider() error
DescribeVolumes(providerVolumesID []string) ([]VolumeProperties, error)
}