initial commit for gp3 migration.

This commit is contained in:
Jan Mußler 2020-12-07 14:46:59 +01:00
parent dc9a5b1e61
commit 6bb78c8de2
9 changed files with 149 additions and 58 deletions

7
go.mod
View File

@ -1,16 +1,17 @@
module github.com/zalando/postgres-operator
go 1.14
go 1.15
require (
github.com/aws/aws-sdk-go v1.35.15
github.com/aws/aws-sdk-go v1.36.2
github.com/lib/pq v1.8.0
github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d
github.com/r3labs/diff v1.1.0
github.com/sirupsen/logrus v1.7.0
github.com/stretchr/testify v1.5.1
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
golang.org/x/tools v0.0.0-20201121010211-780cb80bd7fb // indirect
golang.org/x/mod v0.4.0 // indirect
golang.org/x/tools v0.0.0-20201206230334-368bee879bfd // indirect
gopkg.in/yaml.v2 v2.2.8
k8s.io/api v0.19.3
k8s.io/apiextensions-apiserver v0.19.3

13
go.sum
View File

@ -44,8 +44,8 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/aws/aws-sdk-go v1.35.15 h1:JdQNM8hJe+9N9xP53S54NDmX8GCaZn8CCJ4LBHfom4U=
github.com/aws/aws-sdk-go v1.35.15/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
github.com/aws/aws-sdk-go v1.36.2 h1:UAeFPct+jHqWM+tgiqDrC9/sfbWj6wkcvpsJ+zdcsvA=
github.com/aws/aws-sdk-go v1.36.2/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
@ -403,6 +403,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8=
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -423,12 +425,13 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -505,8 +508,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20201121010211-780cb80bd7fb h1:z5+u0pkAUPUWd3taoTialQ2JAMo4Wo1Z3L25U4ZV9r0=
golang.org/x/tools v0.0.0-20201121010211-780cb80bd7fb/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201206230334-368bee879bfd h1:EqFvKLTxjH6gEy2baWxX2AgJwZkBIDIcZFYcoYlI9RA=
golang.org/x/tools v0.0.0-20201206230334-368bee879bfd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=

View File

@ -115,8 +115,11 @@ type Volume struct {
Size string `json:"size"`
StorageClass string `json:"storageClass"`
SubPath string `json:"subPath,omitempty"`
Iops *int64 `json:"iops,omitempty"`
Throughput *int64 `json:"throughput,omitempty"`
}
// AdditionalVolume specs additional optional volumes for statefulset
type AdditionalVolume struct {
Name string `json:"name"`
MountPath string `json:"mountPath"`
@ -156,11 +159,12 @@ type Patroni struct {
SynchronousModeStrict bool `json:"synchronous_mode_strict"`
}
//StandbyCluster
// StandbyDescription contains s3 wal path
type StandbyDescription struct {
S3WalPath string `json:"s3_wal_path,omitempty"`
}
// TLSDescription specs TLS properties
type TLSDescription struct {
SecretName string `json:"secretName,omitempty"`
CertificateFile string `json:"certificateFile,omitempty"`
@ -198,7 +202,7 @@ type PostgresStatus struct {
PostgresClusterStatus string `json:"PostgresClusterStatus"`
}
// Options for connection pooler
// ConnectionPooler Options for connection pooler
//
// TODO: prepared snippets of configuration, one can choose via type, e.g.
// pgbouncer-large (with higher resources) or odyssey-small (with smaller

View File

@ -25,6 +25,7 @@ import (
"github.com/zalando/postgres-operator/pkg/util/patroni"
"github.com/zalando/postgres-operator/pkg/util/teams"
"github.com/zalando/postgres-operator/pkg/util/users"
"github.com/zalando/postgres-operator/pkg/util/volumes"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
policybeta1 "k8s.io/api/policy/v1beta1"
@ -64,6 +65,14 @@ type kubeResources struct {
//PVCs are treated separately
}
type EBSVolume struct {
volumeId string
volumeType string
size int64
iops int32
throughput int32
}
// Cluster describes postgresql cluster
type Cluster struct {
kubeResources
@ -89,7 +98,10 @@ type Cluster struct {
processMu sync.RWMutex // protects the current operation for reporting, no need to hold the master mutex
specMu sync.RWMutex // protects the spec for reporting, no need to hold the master mutex
ConnectionPooler map[PostgresRole]*ConnectionPoolerObjects
EBSVolumes map[string]EBSVolume
VolumeResizer volumes.VolumeResizer
}
type compareStatefulsetResult struct {
match bool
replace bool
@ -134,6 +146,11 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres
cluster.oauthTokenGetter = newSecretOauthTokenGetter(&kubeClient, cfg.OpConfig.OAuthTokenSecretName)
cluster.patroni = patroni.New(cluster.logger)
cluster.eventRecorder = eventRecorder
if cfg.OpConfig.StorageResizeMode != "pvc" {
cluster.VolumeResizer = &volumes.EBSVolumeResizer{AWSRegion: cfg.OpConfig.AWSRegion}
}
return cluster
}

View File

@ -11,7 +11,6 @@ import (
"github.com/zalando/postgres-operator/pkg/util"
"github.com/zalando/postgres-operator/pkg/util/constants"
"github.com/zalando/postgres-operator/pkg/util/k8sutil"
"github.com/zalando/postgres-operator/pkg/util/volumes"
appsv1 "k8s.io/api/apps/v1"
batchv1beta1 "k8s.io/api/batch/v1beta1"
v1 "k8s.io/api/core/v1"
@ -55,7 +54,15 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
}
c.logger.Debugf("syncing volumes using %q storage resize mode", c.OpConfig.StorageResizeMode)
if c.OpConfig.StorageResizeMode == "pvc" {
if c.OpConfig.StorageResizeMode == "mixed" {
// resize pvc to adjust filesystem size until better K8s support
if err = c.syncVolumeClaims(); err != nil {
err = fmt.Errorf("could not sync persistent volume claims: %v", err)
return err
}
} else if c.OpConfig.StorageResizeMode == "pvc" {
if err = c.syncVolumeClaims(); err != nil {
err = fmt.Errorf("could not sync persistent volume claims: %v", err)
return err
@ -599,7 +606,8 @@ func (c *Cluster) syncVolumes() error {
if !act {
return nil
}
if err := c.resizeVolumes(c.Spec.Volume, []volumes.VolumeResizer{&volumes.EBSVolumeResizer{AWSRegion: c.OpConfig.AWSRegion}}); err != nil {
if err := c.resizeVolumes(); err != nil {
return fmt.Errorf("could not sync volumes: %v", err)
}

View File

@ -15,7 +15,6 @@ import (
"github.com/zalando/postgres-operator/pkg/util"
"github.com/zalando/postgres-operator/pkg/util/constants"
"github.com/zalando/postgres-operator/pkg/util/filesystems"
"github.com/zalando/postgres-operator/pkg/util/volumes"
)
func (c *Cluster) listPersistentVolumeClaims() ([]v1.PersistentVolumeClaim, error) {
@ -119,19 +118,26 @@ func (c *Cluster) listPersistentVolumes() ([]*v1.PersistentVolume, error) {
}
// resizeVolumes resize persistent volumes compatible with the given resizer interface
func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.VolumeResizer) error {
c.setProcessName("resizing volumes")
func (c *Cluster) resizeVolumes() error {
if c.VolumeResizer == nil {
return fmt.Errorf("no volume resizer set for EBS volume handling")
}
c.setProcessName("resizing EBS volumes")
resizer := c.VolumeResizer
var totalIncompatible int
newQuantity, err := resource.ParseQuantity(newVolume.Size)
newQuantity, err := resource.ParseQuantity(c.Spec.Volume.Size)
if err != nil {
return fmt.Errorf("could not parse volume size: %v", err)
}
pvs, newSize, err := c.listVolumesWithManifestSize(newVolume)
pvs, newSize, err := c.listVolumesWithManifestSize(c.Spec.Volume)
if err != nil {
return fmt.Errorf("could not list persistent volumes: %v", err)
}
for _, pv := range pvs {
volumeSize := quantityToGigabyte(pv.Spec.Capacity[v1.ResourceStorage])
if volumeSize >= newSize {
@ -141,43 +147,43 @@ func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.Volu
continue
}
compatible := false
for _, resizer := range resizers {
if !resizer.VolumeBelongsToProvider(pv) {
continue
}
compatible = true
if !resizer.IsConnectedToProvider() {
err := resizer.ConnectToProvider()
if err != nil {
return fmt.Errorf("could not connect to the volume provider: %v", err)
}
defer func() {
if err := resizer.DisconnectFromProvider(); err != nil {
c.logger.Errorf("%v", err)
}
}()
}
awsVolumeID, err := resizer.GetProviderVolumeID(pv)
if err != nil {
return err
}
c.logger.Debugf("updating persistent volume %q to %d", pv.Name, newSize)
if err := resizer.ResizeVolume(awsVolumeID, newSize); err != nil {
return fmt.Errorf("could not resize EBS volume %q: %v", awsVolumeID, err)
}
c.logger.Debugf("resizing the filesystem on the volume %q", pv.Name)
podName := getPodNameFromPersistentVolume(pv)
if err := c.resizePostgresFilesystem(podName, []filesystems.FilesystemResizer{&filesystems.Ext234Resize{}}); err != nil {
return fmt.Errorf("could not resize the filesystem on pod %q: %v", podName, err)
}
c.logger.Debugf("filesystem resize successful on volume %q", pv.Name)
pv.Spec.Capacity[v1.ResourceStorage] = newQuantity
c.logger.Debugf("updating persistent volume definition for volume %q", pv.Name)
if _, err := c.KubeClient.PersistentVolumes().Update(context.TODO(), pv, metav1.UpdateOptions{}); err != nil {
return fmt.Errorf("could not update persistent volume: %q", err)
}
c.logger.Debugf("successfully updated persistent volume %q", pv.Name)
if !resizer.VolumeBelongsToProvider(pv) {
continue
}
compatible = true
if !resizer.IsConnectedToProvider() {
err := resizer.ConnectToProvider()
if err != nil {
return fmt.Errorf("could not connect to the volume provider: %v", err)
}
defer func() {
if err := resizer.DisconnectFromProvider(); err != nil {
c.logger.Errorf("%v", err)
}
}()
}
awsVolumeID, err := resizer.GetProviderVolumeID(pv)
if err != nil {
return err
}
c.logger.Debugf("updating persistent volume %q to %d", pv.Name, newSize)
if err := resizer.ResizeVolume(awsVolumeID, newSize); err != nil {
return fmt.Errorf("could not resize EBS volume %q: %v", awsVolumeID, err)
}
c.logger.Debugf("resizing the filesystem on the volume %q", pv.Name)
podName := getPodNameFromPersistentVolume(pv)
if err := c.resizePostgresFilesystem(podName, []filesystems.FilesystemResizer{&filesystems.Ext234Resize{}}); err != nil {
return fmt.Errorf("could not resize the filesystem on pod %q: %v", podName, err)
}
c.logger.Debugf("filesystem resize successful on volume %q", pv.Name)
pv.Spec.Capacity[v1.ResourceStorage] = newQuantity
c.logger.Debugf("updating persistent volume definition for volume %q", pv.Name)
if _, err := c.KubeClient.PersistentVolumes().Update(context.TODO(), pv, metav1.UpdateOptions{}); err != nil {
return fmt.Errorf("could not update persistent volume: %q", err)
}
c.logger.Debugf("successfully updated persistent volume %q", pv.Name)
if !compatible {
c.logger.Warningf("volume %q is incompatible with all available resizing providers, consider switching storage_resize_mode to pvc or off", pv.Name)
totalIncompatible++

View File

@ -198,6 +198,8 @@ type Config struct {
SetMemoryRequestToLimit bool `name:"set_memory_request_to_limit" default:"false"`
EnableLazySpiloUpgrade bool `name:"enable_lazy_spilo_upgrade" default:"false"`
EnablePgVersionEnvVar bool `name:"enable_pgversion_env_var" default:"false"`
EnableEBSGp3Migration bool `name:"enable_ebs_gp3_migration" default:"false"`
EnableEBSGp3MaxSize int32 `name:"enable_ebs_gp3_max_size" default:"1000"`
}
// MustMarshal marshals the config or panics

View File

@ -7,7 +7,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"github.com/zalando/postgres-operator/pkg/util/constants"
"github.com/zalando/postgres-operator/pkg/util/retryutil"
@ -102,6 +102,57 @@ func (c *EBSVolumeResizer) ResizeVolume(volumeID string, newSize int64) error {
})
}
// ModifyVolume Modify EBS volume
func (c *EBSVolumeResizer) ModifyVolume(volumeID string, newType string, newSize int64, iops int64, throughput int64) error {
/* first check if the volume is already of a requested size */
volumeOutput, err := c.connection.DescribeVolumes(&ec2.DescribeVolumesInput{VolumeIds: []*string{&volumeID}})
if err != nil {
return fmt.Errorf("could not get information about the volume: %v", err)
}
vol := volumeOutput.Volumes[0]
if *vol.VolumeId != volumeID {
return fmt.Errorf("describe volume %q returned information about a non-matching volume %q", volumeID, *vol.VolumeId)
}
if *vol.Size == newSize {
// nothing to do
return nil
}
input := ec2.ModifyVolumeInput{Size: &newSize, VolumeId: &volumeID, VolumeType: &newType, Iops: &iops, Throughput: &throughput}
output, err := c.connection.ModifyVolume(&input)
if err != nil {
return fmt.Errorf("could not modify persistent volume: %v", err)
}
state := *output.VolumeModification.ModificationState
if state == constants.EBSVolumeStateFailed {
return fmt.Errorf("could not modify persistent volume %q: modification state failed", volumeID)
}
if state == "" {
return fmt.Errorf("received empty modification status")
}
if state == constants.EBSVolumeStateOptimizing || state == constants.EBSVolumeStateCompleted {
return nil
}
// wait until the volume reaches the "optimizing" or "completed" state
in := ec2.DescribeVolumesModificationsInput{VolumeIds: []*string{&volumeID}}
return retryutil.Retry(constants.EBSVolumeResizeWaitInterval, constants.EBSVolumeResizeWaitTimeout,
func() (bool, error) {
out, err := c.connection.DescribeVolumesModifications(&in)
if err != nil {
return false, fmt.Errorf("could not describe volume modification: %v", err)
}
if len(out.VolumesModifications) != 1 {
return false, fmt.Errorf("describe volume modification didn't return one record for volume %q", volumeID)
}
if *out.VolumesModifications[0].VolumeId != volumeID {
return false, fmt.Errorf("non-matching volume id when describing modifications: %q is different from %q",
*out.VolumesModifications[0].VolumeId, volumeID)
}
return *out.VolumesModifications[0].ModificationState != constants.EBSVolumeStateModifying, nil
})
}
// DisconnectFromProvider closes connection to the EC2 instance
func (c *EBSVolumeResizer) DisconnectFromProvider() error {
c.connection = nil

View File

@ -1,8 +1,6 @@
package volumes
import (
"k8s.io/api/core/v1"
)
import v1 "k8s.io/api/core/v1"
// VolumeResizer defines the set of methods used to implememnt provider-specific resizing of persistent volumes.
type VolumeResizer interface {
@ -11,5 +9,6 @@ type VolumeResizer interface {
VolumeBelongsToProvider(pv *v1.PersistentVolume) bool
GetProviderVolumeID(pv *v1.PersistentVolume) (string, error)
ResizeVolume(providerVolumeID string, newSize int64) error
ModifyVolume(providerVolumeID string, newType string, newSize int64, iops int64, throughput int64) error
DisconnectFromProvider() error
}