Implements EBS volume resizing #35.

In order to support volumes different from EBS and filesystems other than EXT2/3/4 the respective code parts were implemented as interfaces. Adding the new resize for the volume or the filesystem will require implementing the interface, but no other changes in the cluster code itself.

Volume resizing first changes the EBS and the filesystem, and only afterwards is reflected in the Kubernetes "PersistentVolume" object. This is done deliberately to be able to check if the volume needs resizing by peeking at the Size of the PersistentVolume structure. We recheck, nevertheless, in the EBSVolumeResizer, whether the actual EBS volume size doesn't match the spec, since call to the AWS ModifyVolume is counted against the resize limit of once every 6 hours, even for those calls that shouldn't result in an actual resize (i.e. when the size matches the one for the running volume).

As a collateral, split the constants into multiple files, move the volume code into a separate file and fix minor issues related to the error reporting.

pkg/util/constants/annotations.go

@@ -0,0 +1,9 @@
+package constants
+
+const (
+	ZalandoDNSNameAnnotation           = "external-dns.alpha.kubernetes.io/hostname"
+	ElbTimeoutAnnotationName           = "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout"
+	ElbTimeoutAnnotationValue          = "3600"
+	KubeIAmAnnotation                  = "iam.amazonaws.com/role"
+	VolumeStorateProvisionerAnnotation = "pv.kubernetes.io/provisioned-by"
+)

pkg/util/constants/aws.go

@@ -0,0 +1,16 @@
+package constants
+
+import "time"
+
+const (
+	AWS_REGION       = "eu-central-1"
+	EBSVolumeIDStart = "/vol-"
+	EBSProvisioner   = "kubernetes.io/aws-ebs"
+	//https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_VolumeModification.html
+	EBSVolumeStateModifying     = "modifying"
+	EBSVolumeStateOptimizing    = "optimizing"
+	EBSVolumeStateFailed        = "failed"
+	EBSVolumeStateCompleted     = "completed"
+	EBSVolumeResizeWaitInterval = 2 * time.Second
+	EBSVolumeResizeWaitTimeout  = 30 * time.Second
+)

pkg/util/constants/constants.go

@@ -1,35 +0,0 @@
-package constants
-
-import "time"
-
-const (
-	TPRName                     = "postgresql"
-	TPRVendor                   = "acid.zalan.do"
-	TPRDescription              = "Managed PostgreSQL clusters"
-	TPRApiVersion               = "v1"
-	ListClustersURITemplate     = "/apis/" + TPRVendor + "/" + TPRApiVersion + "/namespaces/%s/" + ResourceName       // Namespace
-	WatchClustersURITemplate    = "/apis/" + TPRVendor + "/" + TPRApiVersion + "/watch/namespaces/%s/" + ResourceName // Namespace
-	K8sVersion                  = "v1"
-	K8sAPIPath                  = "/api"
-	DataVolumeName              = "pgdata"
-	PasswordLength              = 64
-	UserSecretTemplate          = "%s.%s.credentials." + TPRName + "." + TPRVendor // Username, ClusterName
-	ZalandoDNSNameAnnotation    = "external-dns.alpha.kubernetes.io/hostname"
-	ElbTimeoutAnnotationName    = "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout"
-	ElbTimeoutAnnotationValue   = "3600"
-	KubeIAmAnnotation           = "iam.amazonaws.com/role"
-	ResourceName                = TPRName + "s"
-	PodRoleMaster               = "master"
-	PodRoleReplica              = "replica"
-	SuperuserKeyName            = "superuser"
-	ReplicationUserKeyName      = "replication"
-	StatefulsetDeletionInterval = 1 * time.Second
-	StatefulsetDeletionTimeout  = 30 * time.Second
-
-	RoleFlagSuperuser  = "SUPERUSER"
-	RoleFlagInherit    = "INHERIT"
-	RoleFlagLogin      = "LOGIN"
-	RoleFlagNoLogin    = "NOLOGIN"
-	RoleFlagCreateRole = "CREATEROLE"
-	RoleFlagCreateDB   = "CREATEDB"
-)

pkg/util/constants/kubernetes.go

@@ -0,0 +1,12 @@
+package constants
+
+import "time"
+
+const (
+	ListClustersURITemplate     = "/apis/" + TPRVendor + "/" + TPRApiVersion + "/namespaces/%s/" + ResourceName       // Namespace
+	WatchClustersURITemplate    = "/apis/" + TPRVendor + "/" + TPRApiVersion + "/watch/namespaces/%s/" + ResourceName // Namespace
+	K8sVersion                  = "v1"
+	K8sAPIPath                  = "/api"
+	StatefulsetDeletionInterval = 1 * time.Second
+	StatefulsetDeletionTimeout  = 30 * time.Second
+)

pkg/util/constants/postgresql.go

@@ -0,0 +1,9 @@
+package constants
+
+const (
+	DataVolumeName    = "pgdata"
+	PodRoleMaster     = "master"
+	PodRoleReplica    = "replica"
+	PostgresDataMount = "/home/postgres/pgdata"
+	PostgresDataPath  = PostgresDataMount + "/pgroot"
+)

pkg/util/constants/roles.go

@@ -0,0 +1,14 @@
+package constants
+
+const (
+	PasswordLength         = 64
+	UserSecretTemplate     = "%s.%s.credentials." + TPRName + "." + TPRVendor // Username, ClusterName
+	SuperuserKeyName       = "superuser"
+	ReplicationUserKeyName = "replication"
+	RoleFlagSuperuser      = "SUPERUSER"
+	RoleFlagInherit        = "INHERIT"
+	RoleFlagLogin          = "LOGIN"
+	RoleFlagNoLogin        = "NOLOGIN"
+	RoleFlagCreateRole     = "CREATEROLE"
+	RoleFlagCreateDB       = "CREATEDB"
+)

pkg/util/constants/thirdpartyresource.go

@@ -0,0 +1,9 @@
+package constants
+
+const (
+	TPRName        = "postgresql"
+	TPRVendor      = "acid.zalan.do"
+	TPRDescription = "Managed PostgreSQL clusters"
+	TPRApiVersion  = "v1"
+	ResourceName   = TPRName + "s"
+)

pkg/util/constants/units.go

@@ -0,0 +1,5 @@
+package constants
+
+const (
+	Gigabyte = 1073741824
+)

pkg/util/filesystems/ext234.go

@@ -0,0 +1,38 @@
+package filesystems
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+var (
+	ext2fsSuccessRegexp = regexp.MustCompile(`The filesystem on [/a-z0-9]+ is now \d+ \(\d+\w+\) blocks long.`)
+)
+
+const (
+	EXT2      = "ext2"
+	EXT3      = "ext3"
+	EXT4      = "ext4"
+	resize2fs = "resize2fs"
+)
+
+type Ext234Resize struct {
+}
+
+func (c *Ext234Resize) CanResizeFilesystem(fstype string) bool {
+	return fstype == EXT2 || fstype == EXT3 || fstype == EXT4
+}
+
+func (c *Ext234Resize) ResizeFilesystem(deviceName string, commandExecutor func(cmd string) (out string, err error)) error {
+	command := fmt.Sprintf("%s %s 2>&1", resize2fs, deviceName)
+	out, err := commandExecutor(command)
+	if err != nil {
+		return err
+	}
+	if strings.Contains(out, "Nothing to do") ||
+		(strings.Contains(out, "on-line resizing required") && ext2fsSuccessRegexp.MatchString(out)) {
+		return nil
+	}
+	return fmt.Errorf("unrecognized output: %s, assuming error", out)
+}

pkg/util/filesystems/filesystems.go

@@ -0,0 +1,6 @@
+package filesystems
+
+type FilesystemResizer interface {
+	CanResizeFilesystem(fstype string) bool
+	ResizeFilesystem(deviceName string, commandExecutor func(string) (out string, err error)) error
+}

pkg/util/teams/teams_test.go

@@ -144,7 +144,7 @@ func TestInfo(t *testing.T) {
 	for _, tc := range teamsAPItc {
 		func() {
 			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				if r.Header.Get("Authorization") != "Bearer " + token {
+				if r.Header.Get("Authorization") != "Bearer "+token {
 					t.Errorf("Authorization token is wrong or not provided")
 				}
 				w.WriteHeader(tc.inCode)

pkg/util/volumes/ebs.go

@@ -0,0 +1,101 @@
+package volumes
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
+
+	"github.com/zalando-incubator/postgres-operator/pkg/util/constants"
+	"github.com/zalando-incubator/postgres-operator/pkg/util/retryutil"
+	"k8s.io/client-go/pkg/api/v1"
+)
+
+type EBSVolumeResizer struct {
+	connection *ec2.EC2
+}
+
+func (c *EBSVolumeResizer) ConnectToProvider() error {
+	sess, err := session.NewSession(&aws.Config{Region: aws.String(constants.AWS_REGION)})
+	if err != nil {
+		return fmt.Errorf("could not establish AWS session: %v", err)
+	}
+	c.connection = ec2.New(sess)
+	return nil
+}
+
+func (c *EBSVolumeResizer) IsConnectedToProvider() bool {
+	return c.connection != nil
+}
+
+func (c *EBSVolumeResizer) VolumeBelongsToProvider(pv *v1.PersistentVolume) bool {
+	return pv.Spec.AWSElasticBlockStore != nil && pv.Annotations[constants.VolumeStorateProvisionerAnnotation] == constants.EBSProvisioner
+}
+
+// GetProviderVolumeID converts aws://eu-central-1b/vol-00f93d4827217c629 to vol-00f93d4827217c629 for EBS volumes
+func (c *EBSVolumeResizer) GetProviderVolumeID(pv *v1.PersistentVolume) (string, error) {
+	volumeID := pv.Spec.AWSElasticBlockStore.VolumeID
+	if volumeID == "" {
+		return "", fmt.Errorf("volume id is empty for volume %s", pv.Name)
+	}
+	idx := strings.LastIndex(volumeID, constants.EBSVolumeIDStart) + 1
+	if idx == 0 {
+		return "", fmt.Errorf("malfored EBS volume id %s", volumeID)
+	}
+	return volumeID[idx:], nil
+}
+
+func (c *EBSVolumeResizer) ResizeVolume(volumeId string, newSize int64) error {
+	/* first check if the volume is already of a requested size */
+	volumeOutput, err := c.connection.DescribeVolumes(&ec2.DescribeVolumesInput{VolumeIds: []*string{&volumeId}})
+	if err != nil {
+		return fmt.Errorf("could not get information about the volume: %v", err)
+	}
+	vol := volumeOutput.Volumes[0]
+	if *vol.VolumeId != volumeId {
+		return fmt.Errorf("describe volume %s returned information about a non-matching volume %s", volumeId, *vol.VolumeId)
+	}
+	if *vol.Size == newSize {
+		// nothing to do
+		return nil
+	}
+	input := ec2.ModifyVolumeInput{Size: &newSize, VolumeId: &volumeId}
+	output, err := c.connection.ModifyVolume(&input)
+	if err != nil {
+		return fmt.Errorf("could not modify persistent volume: %v", err)
+	}
+
+	state := *output.VolumeModification.ModificationState
+	if state == constants.EBSVolumeStateFailed {
+		return fmt.Errorf("could not modify persistent volume %s: modification state failed", volumeId)
+	}
+	if state == "" {
+		return fmt.Errorf("received empty modification status")
+	}
+	if state == constants.EBSVolumeStateOptimizing || state == constants.EBSVolumeStateCompleted {
+		return nil
+	}
+	// wait until the volume reaches the "optimizing" or "completed" state
+	in := ec2.DescribeVolumesModificationsInput{VolumeIds: []*string{&volumeId}}
+	return retryutil.Retry(constants.EBSVolumeResizeWaitInterval, constants.EBSVolumeResizeWaitTimeout,
+		func() (bool, error) {
+			out, err := c.connection.DescribeVolumesModifications(&in)
+			if err != nil {
+				return false, fmt.Errorf("could not describe volume modification: %v", err)
+			}
+			if len(out.VolumesModifications) != 1 {
+				return false, fmt.Errorf("describe volume modification didn't return one record for volume \"%s\"", volumeId)
+			}
+			if *out.VolumesModifications[0].VolumeId != volumeId {
+				return false, fmt.Errorf("non-matching volume id when describing modifications: \"%s\" is different from \"%s\"")
+			}
+			return *out.VolumesModifications[0].ModificationState != constants.EBSVolumeStateModifying, nil
+		})
+}
+
+func (c *EBSVolumeResizer) DisconnectFromProvider() error {
+	c.connection = nil
+	return nil
+}

pkg/util/volumes/volumes.go

@@ -0,0 +1,14 @@
+package volumes
+
+import (
+	"k8s.io/client-go/pkg/api/v1"
+)
+
+type VolumeResizer interface {
+	ConnectToProvider() error
+	IsConnectedToProvider() bool
+	VolumeBelongsToProvider(pv *v1.PersistentVolume) bool
+	GetProviderVolumeID(pv *v1.PersistentVolume) (string, error)
+	ResizeVolume(providerVolumeId string, newSize int64) error
+	DisconnectFromProvider() error
+}