public_git/postgres-operator
1
0
Fork 0
mirror of https://github.com/zalando/postgres-operator.git synced 2026-10-18 19:02:52 +02:00
Code Issues Projects Releases Wiki Activity

Implements EBS volume resizing #35.

Browse Source 
In order to support volumes different from EBS and filesystems other than EXT2/3/4 the respective code parts were implemented as interfaces. Adding the new resize for the volume or the filesystem will require implementing the interface, but no other changes in the cluster code itself.

Volume resizing first changes the EBS and the filesystem, and only afterwards is reflected in the Kubernetes "PersistentVolume" object. This is done deliberately to be able to check if the volume needs resizing by peeking at the Size of the PersistentVolume structure. We recheck, nevertheless, in the EBSVolumeResizer, whether the actual EBS volume size doesn't match the spec, since call to the AWS ModifyVolume is counted against the resize limit of once every 6 hours, even for those calls that shouldn't result in an actual resize (i.e. when the size matches the one for the running volume).

As a collateral, split the constants into multiple files, move the volume code into a separate file and fix minor issues related to the error reporting.
This commit is contained in:
Oleksii Kliukin
2017-06-06 13:53:27 +02:00
committed by GitHub
parent 1fb05212a9
commit 7b0ca31bfb
25 changed files with 554 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/cluster/cluster.go
+44 -33
View File
@@ -26,6 +26,7 @@ import (
"github.com/zalando-incubator/postgres-operator/pkg/util/k8sutil"
"github.com/zalando-incubator/postgres-operator/pkg/util/teams"
"github.com/zalando-incubator/postgres-operator/pkg/util/users"
"github.com/zalando-incubator/postgres-operator/pkg/util/volumes"
)
var (
@@ -37,6 +38,7 @@ var (
type Config struct {
KubeClient *kubernetes.Clientset //TODO: move clients to the better place?
RestClient *rest.RESTClient
RestConfig *rest.Config
TeamsAPIClient *teams.API
OpConfig config.Config
InfrastructureRoles map[string]spec.PgUser // inherited from the controller
@@ -68,6 +70,13 @@ type Cluster struct {
podEventsQueue *cache.FIFO
}
type compareStatefulsetResult struct {
match bool
replace bool
rollingUpdate bool
reasons []string
}
func New(cfg Config, pgSpec spec.Postgresql, logger *logrus.Entry) *Cluster {
lg := logger.WithField("pkg", "cluster").WithField("cluster-name", pgSpec.Metadata.Name)
kubeResources := kubeResources{Secrets: make(map[types.UID]*v1.Secret)}
@@ -244,20 +253,24 @@ func (c *Cluster) sameVolumeWith(volume spec.Volume) (match bool, reason string)
return
}
func (c *Cluster) compareStatefulSetWith(statefulSet *v1beta1.StatefulSet) (match, needsReplace, needsRollUpdate bool, reason string) {
func (c *Cluster) compareStatefulSetWith(statefulSet *v1beta1.StatefulSet) *compareStatefulsetResult {
reasons := make([]string, 0)
var match, needsRollUpdate, needsReplace bool
match = true
//TODO: improve me
if *c.Statefulset.Spec.Replicas != *statefulSet.Spec.Replicas {
match = false
reason = "new statefulset's number of replicas doesn't match the current one"
reasons = append(reasons, "new statefulset's number of replicas doesn't match the current one")
}
if len(c.Statefulset.Spec.Template.Spec.Containers) != len(statefulSet.Spec.Template.Spec.Containers) {
needsRollUpdate = true
reason = "new statefulset's container specification doesn't match the current one"
reasons = append(reasons, "new statefulset's container specification doesn't match the current one")
}
if len(c.Statefulset.Spec.Template.Spec.Containers) == 0 {
c.logger.Warnf("statefulset '%s' has no container", util.NameFromMeta(c.Statefulset.ObjectMeta))
return
return &compareStatefulsetResult{}
}
// In the comparisons below, the needsReplace and needsRollUpdate flags are never reset, since checks fall through
// and the combined effect of all the changes should be applied.
@@ -267,48 +280,44 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *v1beta1.StatefulSet) (matc
if c.Statefulset.Spec.Template.Spec.ServiceAccountName != statefulSet.Spec.Template.Spec.ServiceAccountName {
needsReplace = true
needsRollUpdate = true
reason = "new statefulset's serviceAccountName service asccount name doesn't match the current one"
reasons = append(reasons, "new statefulset's serviceAccountName service asccount name doesn't match the current one")
}
if *c.Statefulset.Spec.Template.Spec.TerminationGracePeriodSeconds != *statefulSet.Spec.Template.Spec.TerminationGracePeriodSeconds {
needsReplace = true
needsRollUpdate = true
reason = "new statefulset's terminationGracePeriodSeconds doesn't match the current one"
reasons = append(reasons, "new statefulset's terminationGracePeriodSeconds doesn't match the current one")
}
// Some generated fields like creationTimestamp make it not possible to use DeepCompare on Spec.Template.ObjectMeta
if !reflect.DeepEqual(c.Statefulset.Spec.Template.Labels, statefulSet.Spec.Template.Labels) {
needsReplace = true
needsRollUpdate = true
reason = "new statefulset's metadata labels doesn't match the current one"
reasons = append(reasons, "new statefulset's metadata labels doesn't match the current one")
}
if !reflect.DeepEqual(c.Statefulset.Spec.Template.Annotations, statefulSet.Spec.Template.Annotations) {
needsRollUpdate = true
needsReplace = true
reason = "new statefulset's metadata annotations doesn't match the current one"
reasons = append(reasons, "new statefulset's metadata annotations doesn't match the current one")
}
if len(c.Statefulset.Spec.VolumeClaimTemplates) != len(statefulSet.Spec.VolumeClaimTemplates) {
needsReplace = true
needsRollUpdate = true
reason = "new statefulset's volumeClaimTemplates contains different number of volumes to the old one"
reasons = append(reasons, "new statefulset's volumeClaimTemplates contains different number of volumes to the old one")
}
for i := 0; i < len(c.Statefulset.Spec.VolumeClaimTemplates); i++ {
name := c.Statefulset.Spec.VolumeClaimTemplates[i].Name
// Some generated fields like creationTimestamp make it not possible to use DeepCompare on ObjectMeta
if name != statefulSet.Spec.VolumeClaimTemplates[i].Name {
needsReplace = true
needsRollUpdate = true
reason = fmt.Sprintf("new statefulset's name for volume %d doesn't match the current one", i)
reasons = append(reasons, fmt.Sprintf("new statefulset's name for volume %d doesn't match the current one", i))
continue
}
if !reflect.DeepEqual(c.Statefulset.Spec.VolumeClaimTemplates[i].Annotations, statefulSet.Spec.VolumeClaimTemplates[i].Annotations) {
needsReplace = true
needsRollUpdate = true
reason = fmt.Sprintf("new statefulset's annotations for volume %s doesn't match the current one", name)
reasons = append(reasons, fmt.Sprintf("new statefulset's annotations for volume %s doesn't match the current one", name))
}
if !reflect.DeepEqual(c.Statefulset.Spec.VolumeClaimTemplates[i].Spec, statefulSet.Spec.VolumeClaimTemplates[i].Spec) {
name := c.Statefulset.Spec.VolumeClaimTemplates[i].Name
needsReplace = true
needsRollUpdate = true
reason = fmt.Sprintf("new statefulset's volumeClaimTemplates specification for volume %s doesn't match the current one", name)
reasons = append(reasons, fmt.Sprintf("new statefulset's volumeClaimTemplates specification for volume %s doesn't match the current one", name))
}
}
@@ -316,28 +325,27 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *v1beta1.StatefulSet) (matc
container2 := statefulSet.Spec.Template.Spec.Containers[0]
if container1.Image != container2.Image {
needsRollUpdate = true
reason = "new statefulset's container image doesn't match the current one"
reasons = append(reasons, "new statefulset's container image doesn't match the current one")
}
if !reflect.DeepEqual(container1.Ports, container2.Ports) {
needsRollUpdate = true
reason = "new statefulset's container ports don't match the current one"
reasons = append(reasons, "new statefulset's container ports don't match the current one")
}
if !compareResources(&container1.Resources, &container2.Resources) {
needsRollUpdate = true
reason = "new statefulset's container resources don't match the current ones"
reasons = append(reasons, "new statefulset's container resources don't match the current ones")
}
if !reflect.DeepEqual(container1.Env, container2.Env) {
needsRollUpdate = true
reason = "new statefulset's container environment doesn't match the current one"
reasons = append(reasons, "new statefulset's container environment doesn't match the current one")
}
if needsRollUpdate || needsReplace {
match = false
}
return
return &compareStatefulsetResult{match: match, reasons: reasons, rollingUpdate: needsRollUpdate, replace: needsReplace}
}
func compareResources(a *v1.ResourceRequirements, b *v1.ResourceRequirements) (equal bool) {
@@ -387,22 +395,16 @@ func (c *Cluster) Update(newSpec *spec.Postgresql) error {
c.logger.Infof("service '%s' has been updated", util.NameFromMeta(c.Service.ObjectMeta))
}
if match, reason := c.sameVolumeWith(newSpec.Spec.Volume); !match {
c.logVolumeChanges(c.Spec.Volume, newSpec.Spec.Volume, reason)
//TODO: update PVC
}
newStatefulSet, err := c.genStatefulSet(newSpec.Spec)
if err != nil {
return fmt.Errorf("could not generate statefulset: %v", err)
}
cmp := c.compareStatefulSetWith(newStatefulSet)
sameSS, needsReplace, rollingUpdate, reason := c.compareStatefulSetWith(newStatefulSet)
if !sameSS {
c.logStatefulSetChanges(c.Statefulset, newStatefulSet, true, reason)
if !cmp.match {
c.logStatefulSetChanges(c.Statefulset, newStatefulSet, true, cmp.reasons)
//TODO: mind the case of updating allowedSourceRanges
if !needsReplace {
if !cmp.replace {
if err := c.updateStatefulSet(newStatefulSet); err != nil {
c.setStatus(spec.ClusterStatusUpdateFailed)
return fmt.Errorf("could not upate statefulset: %v", err)
@@ -423,7 +425,7 @@ func (c *Cluster) Update(newSpec *spec.Postgresql) error {
//TODO: rewrite pg version in tpr spec
}
if rollingUpdate {
if cmp.rollingUpdate {
c.logger.Infof("Rolling update is needed")
// TODO: wait for actual streaming to the replica
if err := c.recreatePods(); err != nil {
@@ -432,6 +434,15 @@ func (c *Cluster) Update(newSpec *spec.Postgresql) error {
}
c.logger.Infof("Rolling update has been finished")
}
if match, reason := c.sameVolumeWith(newSpec.Spec.Volume); !match {
c.logVolumeChanges(c.Spec.Volume, newSpec.Spec.Volume, reason)
if err := c.resizeVolumes(newSpec.Spec.Volume, []volumes.VolumeResizer{&volumes.EBSVolumeResizer{}}); err != nil {
return fmt.Errorf("Could not update volumes: %v", err)
}
c.logger.Infof("volumes have been updated successfully")
}
c.setStatus(spec.ClusterStatusRunning)
return nil