Merge branch 'master' into feature/tests

pkg/util/constants/annotations.go

@@ -0,0 +1,9 @@
+package constants
+
+const (
+	ZalandoDNSNameAnnotation           = "external-dns.alpha.kubernetes.io/hostname"
+	ElbTimeoutAnnotationName           = "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout"
+	ElbTimeoutAnnotationValue          = "3600"
+	KubeIAmAnnotation                  = "iam.amazonaws.com/role"
+	VolumeStorateProvisionerAnnotation = "pv.kubernetes.io/provisioned-by"
+)

pkg/util/constants/aws.go

@@ -0,0 +1,16 @@
+package constants
+
+import "time"
+
+const (
+	AWS_REGION       = "eu-central-1"
+	EBSVolumeIDStart = "/vol-"
+	EBSProvisioner   = "kubernetes.io/aws-ebs"
+	//https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_VolumeModification.html
+	EBSVolumeStateModifying     = "modifying"
+	EBSVolumeStateOptimizing    = "optimizing"
+	EBSVolumeStateFailed        = "failed"
+	EBSVolumeStateCompleted     = "completed"
+	EBSVolumeResizeWaitInterval = 2 * time.Second
+	EBSVolumeResizeWaitTimeout  = 30 * time.Second
+)

pkg/util/constants/constants.go

@@ -1,35 +0,0 @@
-package constants
-
-import "time"
-
-const (
-	TPRName                     = "postgresql"
-	TPRVendor                   = "acid.zalan.do"
-	TPRDescription              = "Managed PostgreSQL clusters"
-	TPRApiVersion               = "v1"
-	ListClustersURITemplate     = "/apis/" + TPRVendor + "/" + TPRApiVersion + "/namespaces/%s/" + ResourceName       // Namespace
-	WatchClustersURITemplate    = "/apis/" + TPRVendor + "/" + TPRApiVersion + "/watch/namespaces/%s/" + ResourceName // Namespace
-	K8sVersion                  = "v1"
-	K8sAPIPath                  = "/api"
-	DataVolumeName              = "pgdata"
-	PasswordLength              = 64
-	UserSecretTemplate          = "%s.%s.credentials." + TPRName + "." + TPRVendor // Username, ClusterName
-	ZalandoDNSNameAnnotation    = "external-dns.alpha.kubernetes.io/hostname"
-	ElbTimeoutAnnotationName    = "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout"
-	ElbTimeoutAnnotationValue   = "3600"
-	KubeIAmAnnotation           = "iam.amazonaws.com/role"
-	ResourceName                = TPRName + "s"
-	PodRoleMaster               = "master"
-	PodRoleReplica              = "replica"
-	SuperuserKeyName            = "superuser"
-	ReplicationUserKeyName      = "replication"
-	StatefulsetDeletionInterval = 1 * time.Second
-	StatefulsetDeletionTimeout  = 30 * time.Second
-
-	RoleFlagSuperuser  = "SUPERUSER"
-	RoleFlagInherit    = "INHERIT"
-	RoleFlagLogin      = "LOGIN"
-	RoleFlagNoLogin    = "NOLOGIN"
-	RoleFlagCreateRole = "CREATEROLE"
-	RoleFlagCreateDB   = "CREATEDB"
-)

pkg/util/constants/kubernetes.go

@@ -0,0 +1,12 @@
+package constants
+
+import "time"
+
+const (
+	ListClustersURITemplate     = "/apis/" + TPRVendor + "/" + TPRApiVersion + "/namespaces/%s/" + ResourceName       // Namespace
+	WatchClustersURITemplate    = "/apis/" + TPRVendor + "/" + TPRApiVersion + "/watch/namespaces/%s/" + ResourceName // Namespace
+	K8sVersion                  = "v1"
+	K8sAPIPath                  = "/api"
+	StatefulsetDeletionInterval = 1 * time.Second
+	StatefulsetDeletionTimeout  = 30 * time.Second
+)

pkg/util/constants/postgresql.go

@@ -0,0 +1,9 @@
+package constants
+
+const (
+	DataVolumeName    = "pgdata"
+	PodRoleMaster     = "master"
+	PodRoleReplica    = "replica"
+	PostgresDataMount = "/home/postgres/pgdata"
+	PostgresDataPath  = PostgresDataMount + "/pgroot"
+)

pkg/util/constants/roles.go

@@ -0,0 +1,14 @@
+package constants
+
+const (
+	PasswordLength         = 64
+	UserSecretTemplate     = "%s.%s.credentials." + TPRName + "." + TPRVendor // Username, ClusterName
+	SuperuserKeyName       = "superuser"
+	ReplicationUserKeyName = "replication"
+	RoleFlagSuperuser      = "SUPERUSER"
+	RoleFlagInherit        = "INHERIT"
+	RoleFlagLogin          = "LOGIN"
+	RoleFlagNoLogin        = "NOLOGIN"
+	RoleFlagCreateRole     = "CREATEROLE"
+	RoleFlagCreateDB       = "CREATEDB"
+)

pkg/util/constants/thirdpartyresource.go

@@ -0,0 +1,9 @@
+package constants
+
+const (
+	TPRName        = "postgresql"
+	TPRVendor      = "acid.zalan.do"
+	TPRDescription = "Managed PostgreSQL clusters"
+	TPRApiVersion  = "v1"
+	ResourceName   = TPRName + "s"
+)

pkg/util/constants/units.go

@@ -0,0 +1,5 @@
+package constants
+
+const (
+	Gigabyte = 1073741824
+)

pkg/util/filesystems/ext234.go

@@ -0,0 +1,38 @@
+package filesystems
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+var (
+	ext2fsSuccessRegexp = regexp.MustCompile(`The filesystem on [/a-z0-9]+ is now \d+ \(\d+\w+\) blocks long.`)
+)
+
+const (
+	EXT2      = "ext2"
+	EXT3      = "ext3"
+	EXT4      = "ext4"
+	resize2fs = "resize2fs"
+)
+
+type Ext234Resize struct {
+}
+
+func (c *Ext234Resize) CanResizeFilesystem(fstype string) bool {
+	return fstype == EXT2 || fstype == EXT3 || fstype == EXT4
+}
+
+func (c *Ext234Resize) ResizeFilesystem(deviceName string, commandExecutor func(cmd string) (out string, err error)) error {
+	command := fmt.Sprintf("%s %s 2>&1", resize2fs, deviceName)
+	out, err := commandExecutor(command)
+	if err != nil {
+		return err
+	}
+	if strings.Contains(out, "Nothing to do") ||
+		(strings.Contains(out, "on-line resizing required") && ext2fsSuccessRegexp.MatchString(out)) {
+		return nil
+	}
+	return fmt.Errorf("unrecognized output: %s, assuming error", out)
+}

pkg/util/filesystems/filesystems.go

@@ -0,0 +1,6 @@
+package filesystems
+
+type FilesystemResizer interface {
+	CanResizeFilesystem(fstype string) bool
+	ResizeFilesystem(deviceName string, commandExecutor func(string) (out string, err error)) error
+}

pkg/util/teams/teams_test.go

@@ -144,7 +144,7 @@ func TestInfo(t *testing.T) {
 	for _, tc := range teamsAPItc {
 		func() {
 			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				if r.Header.Get("Authorization") != "Bearer " + token {
+				if r.Header.Get("Authorization") != "Bearer "+token {
 					t.Errorf("Authorization token is wrong or not provided")
 				}
 				w.WriteHeader(tc.inCode)

pkg/util/volumes/ebs.go

@@ -0,0 +1,101 @@
+package volumes
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
+
+	"github.com/zalando-incubator/postgres-operator/pkg/util/constants"
+	"github.com/zalando-incubator/postgres-operator/pkg/util/retryutil"
+	"k8s.io/client-go/pkg/api/v1"
+)
+
+type EBSVolumeResizer struct {
+	connection *ec2.EC2
+}
+
+func (c *EBSVolumeResizer) ConnectToProvider() error {
+	sess, err := session.NewSession(&aws.Config{Region: aws.String(constants.AWS_REGION)})
+	if err != nil {
+		return fmt.Errorf("could not establish AWS session: %v", err)
+	}
+	c.connection = ec2.New(sess)
+	return nil
+}
+
+func (c *EBSVolumeResizer) IsConnectedToProvider() bool {
+	return c.connection != nil
+}
+
+func (c *EBSVolumeResizer) VolumeBelongsToProvider(pv *v1.PersistentVolume) bool {
+	return pv.Spec.AWSElasticBlockStore != nil && pv.Annotations[constants.VolumeStorateProvisionerAnnotation] == constants.EBSProvisioner
+}
+
+// GetProviderVolumeID converts aws://eu-central-1b/vol-00f93d4827217c629 to vol-00f93d4827217c629 for EBS volumes
+func (c *EBSVolumeResizer) GetProviderVolumeID(pv *v1.PersistentVolume) (string, error) {
+	volumeID := pv.Spec.AWSElasticBlockStore.VolumeID
+	if volumeID == "" {
+		return "", fmt.Errorf("volume id is empty for volume %s", pv.Name)
+	}
+	idx := strings.LastIndex(volumeID, constants.EBSVolumeIDStart) + 1
+	if idx == 0 {
+		return "", fmt.Errorf("malfored EBS volume id %s", volumeID)
+	}
+	return volumeID[idx:], nil
+}
+
+func (c *EBSVolumeResizer) ResizeVolume(volumeId string, newSize int64) error {
+	/* first check if the volume is already of a requested size */
+	volumeOutput, err := c.connection.DescribeVolumes(&ec2.DescribeVolumesInput{VolumeIds: []*string{&volumeId}})
+	if err != nil {
+		return fmt.Errorf("could not get information about the volume: %v", err)
+	}
+	vol := volumeOutput.Volumes[0]
+	if *vol.VolumeId != volumeId {
+		return fmt.Errorf("describe volume %s returned information about a non-matching volume %s", volumeId, *vol.VolumeId)
+	}
+	if *vol.Size == newSize {
+		// nothing to do
+		return nil
+	}
+	input := ec2.ModifyVolumeInput{Size: &newSize, VolumeId: &volumeId}
+	output, err := c.connection.ModifyVolume(&input)
+	if err != nil {
+		return fmt.Errorf("could not modify persistent volume: %v", err)
+	}
+
+	state := *output.VolumeModification.ModificationState
+	if state == constants.EBSVolumeStateFailed {
+		return fmt.Errorf("could not modify persistent volume %s: modification state failed", volumeId)
+	}
+	if state == "" {
+		return fmt.Errorf("received empty modification status")
+	}
+	if state == constants.EBSVolumeStateOptimizing || state == constants.EBSVolumeStateCompleted {
+		return nil
+	}
+	// wait until the volume reaches the "optimizing" or "completed" state
+	in := ec2.DescribeVolumesModificationsInput{VolumeIds: []*string{&volumeId}}
+	return retryutil.Retry(constants.EBSVolumeResizeWaitInterval, constants.EBSVolumeResizeWaitTimeout,
+		func() (bool, error) {
+			out, err := c.connection.DescribeVolumesModifications(&in)
+			if err != nil {
+				return false, fmt.Errorf("could not describe volume modification: %v", err)
+			}
+			if len(out.VolumesModifications) != 1 {
+				return false, fmt.Errorf("describe volume modification didn't return one record for volume \"%s\"", volumeId)
+			}
+			if *out.VolumesModifications[0].VolumeId != volumeId {
+				return false, fmt.Errorf("non-matching volume id when describing modifications: \"%s\" is different from \"%s\"")
+			}
+			return *out.VolumesModifications[0].ModificationState != constants.EBSVolumeStateModifying, nil
+		})
+}
+
+func (c *EBSVolumeResizer) DisconnectFromProvider() error {
+	c.connection = nil
+	return nil
+}

pkg/util/volumes/volumes.go

@@ -0,0 +1,14 @@
+package volumes
+
+import (
+	"k8s.io/client-go/pkg/api/v1"
+)
+
+type VolumeResizer interface {
+	ConnectToProvider() error
+	IsConnectedToProvider() bool
+	VolumeBelongsToProvider(pv *v1.PersistentVolume) bool
+	GetProviderVolumeID(pv *v1.PersistentVolume) (string, error)
+	ResizeVolume(providerVolumeId string, newSize int64) error
+	DisconnectFromProvider() error
+}