Merge branch 'rolling_updates_with_statefulset_annotations' of github.com:zalando-incubator/postgres-operator into rolling_updates_with_statefulset_annotations

2026-10-22 02:49:32 +02:00 · 2018-05-08 14:51:10 +02:00
parent e25f3b9a18 f41a42f922
commit 332dab5237
7 changed files with 153 additions and 95 deletions
@@ -63,38 +63,9 @@ func (c *Cluster) listResources() error {
 	return nil
 }

-func setRollingUpdateFlag(sset *v1beta1.StatefulSet, val bool) {
-	anno := sset.GetAnnotations()
-	fmt.Printf("rolling upgrade flag has been set to %t", val)
-	if anno == nil {
-		anno = make(map[string]string)
-	}
-	anno[RollingUpdateStatefulsetAnnotationKey] = strconv.FormatBool(val)
-	sset.SetAnnotations(anno)
-}
-
-func getRollingUpdateFlag(sset *v1beta1.StatefulSet, defaultValue bool) (flag bool) {
-	anno := sset.GetAnnotations()
-	flag = defaultValue
-
-	stringFlag, exists := anno[RollingUpdateStatefulsetAnnotationKey]
-	if exists {
-		var err error
-		if flag, err = strconv.ParseBool(stringFlag); err != nil {
-			fmt.Printf("error when parsing %s annotation for the statefulset %s: expected boolean value, got %s\n",
-				RollingUpdateStatefulsetAnnotationKey,
-				types.NamespacedName{sset.Namespace, sset.Name},
-				stringFlag)
-			flag = defaultValue
-		}
-	}
-	return flag
-}
-
-func (c *Cluster) createStatefulSet(pendingRollingUpgrade bool) (*v1beta1.StatefulSet, error) {
+func (c *Cluster) createStatefulSet() (*v1beta1.StatefulSet, error) {
 	c.setProcessName("creating statefulset")
 	statefulSetSpec, err := c.generateStatefulSet(&c.Spec)
-	setRollingUpdateFlag(statefulSetSpec, pendingRollingUpgrade)
 	if err != nil {
 		return nil, fmt.Errorf("could not generate statefulset: %v", err)
 	}
@@ -161,7 +132,96 @@ func (c *Cluster) preScaleDown(newStatefulSet *v1beta1.StatefulSet) error {
 	return nil
 }

-func (c *Cluster) updateStatefulSet(newStatefulSet *v1beta1.StatefulSet, includeAnnotations bool) error {
+// setRollingUpdateFlagForStatefulSet sets the indicator or the rolling upgrade requirement
+// in the StatefulSet annotation.
+func (c *Cluster) setRollingUpdateFlagForStatefulSet(sset *v1beta1.StatefulSet, val bool) {
+	anno := sset.GetAnnotations()
+	c.logger.Debugf("rolling upgrade flag has been set to %t", val)
+	if anno == nil {
+		anno = make(map[string]string)
+	}
+	anno[RollingUpdateStatefulsetAnnotationKey] = strconv.FormatBool(val)
+	sset.SetAnnotations(anno)
+}
+
+// applyRollingUpdateFlagforStatefulSet sets the rolling update flag for the cluster's StatefulSet
+// and applies that setting to the actual running cluster.
+func (c *Cluster) applyRollingUpdateFlagforStatefulSet(val bool) error {
+	c.setRollingUpdateFlagForStatefulSet(c.Statefulset, val)
+	sset, err := c.updateStatefulSetAnnotations(c.Statefulset.GetAnnotations())
+	if err != nil {
+		return err
+	}
+	c.Statefulset = sset
+	return nil
+}
+
+// getRollingUpdateFlagFromStatefulSet returns the value of the rollingUpdate flag from the passed
+// StatefulSet, reverting to the default value in case of errors
+func (c *Cluster) getRollingUpdateFlagFromStatefulSet(sset *v1beta1.StatefulSet, defaultValue bool) (flag bool) {
+	anno := sset.GetAnnotations()
+	flag = defaultValue
+
+	stringFlag, exists := anno[RollingUpdateStatefulsetAnnotationKey]
+	if exists {
+		var err error
+		if flag, err = strconv.ParseBool(stringFlag); err != nil {
+			c.logger.Warnf("error when parsing %q annotation for the statefulset %q: expected boolean value, got %q\n",
+				RollingUpdateStatefulsetAnnotationKey,
+				types.NamespacedName{sset.Namespace, sset.Name},
+				stringFlag)
+			flag = defaultValue
+		}
+	}
+	return flag
+}
+
+// mergeRollingUpdateFlagUsingCache return the value of the rollingUpdate flag from the passed
+// statefulset, however, the value can be cleared if there is a cached flag in the cluster that
+// is set to false (the disrepancy could be a result of a failed StatefulSet update).s
+func (c *Cluster) mergeRollingUpdateFlagUsingCache(runningStatefulSet *v1beta1.StatefulSet) bool {
+	var (
+		cachedStatefulsetExists, clearRollingUpdateFromCache, podsRollingUpdateRequired bool
+	)
+
+	if c.Statefulset != nil {
+		// if we reset the rolling update flag in the statefulset structure in memory but didn't manage to update
+		// the actual object in Kubernetes for some reason we want to avoid doing an unnecessary update by relying
+		// on the 'cached' in-memory flag.
+		cachedStatefulsetExists = true
+		clearRollingUpdateFromCache = !c.getRollingUpdateFlagFromStatefulSet(c.Statefulset, true)
+		c.logger.Debugf("cached StatefulSet value exists, rollingUpdate flag is %t", clearRollingUpdateFromCache)
+	}
+
+	if podsRollingUpdateRequired = c.getRollingUpdateFlagFromStatefulSet(runningStatefulSet, false); podsRollingUpdateRequired {
+		if cachedStatefulsetExists && clearRollingUpdateFromCache {
+			c.logger.Infof("clearing the rolling update flag based on the cached information")
+			podsRollingUpdateRequired = false
+		} else {
+			c.logger.Infof("found a statefulset with an unfinished pods rolling update")
+
+		}
+	}
+	return podsRollingUpdateRequired
+}
+
+func (c *Cluster) updateStatefulSetAnnotations(annotations map[string]string) (*v1beta1.StatefulSet, error) {
+	c.logger.Debugf("updating statefulset annotations")
+	patchData, err := metaAnnotationsPatch(annotations)
+	if err != nil {
+		return nil, fmt.Errorf("could not form patch for the statefulset metadata: %v", err)
+	}
+	result, err := c.KubeClient.StatefulSets(c.Statefulset.Namespace).Patch(
+		c.Statefulset.Name,
+		types.MergePatchType,
+		[]byte(patchData), "")
+	if err != nil {
+		return nil, fmt.Errorf("could not patch statefulset annotations %q: %v", patchData, err)
+	}
+	return result, nil
+
+}
+func (c *Cluster) updateStatefulSet(newStatefulSet *v1beta1.StatefulSet) error {
 	c.setProcessName("updating statefulset")
 	if c.Statefulset == nil {
 		return fmt.Errorf("there is no statefulset in the cluster")
@@ -188,14 +248,11 @@ func (c *Cluster) updateStatefulSet(newStatefulSet *v1beta1.StatefulSet, include
 	if err != nil {
 		return fmt.Errorf("could not patch statefulset spec %q: %v", statefulSetName, err)
 	}
-	if includeAnnotations && newStatefulSet.Annotations != nil {
-		patchData := metadataAnnotationsPatch(newStatefulSet.Annotations)
-		statefulSet, err = c.KubeClient.StatefulSets(c.Statefulset.Namespace).Patch(
-			c.Statefulset.Name,
-			types.StrategicMergePatchType,
-			[]byte(patchData), "")
+
+	if newStatefulSet.Annotations != nil {
+		statefulSet, err = c.updateStatefulSetAnnotations(newStatefulSet.Annotations)
 		if err != nil {
-			return fmt.Errorf("could not patch statefulset annotations %q: %v", patchData, err)
+			return err
 		}
 	}

@@ -342,16 +399,19 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error
 		return nil
 	}

+	// update the service annotation in order to propagate ELB notation.
 	if len(newService.ObjectMeta.Annotations) > 0 {
-		annotationsPatchData := metadataAnnotationsPatch(newService.ObjectMeta.Annotations)
+		if annotationsPatchData, err := metaAnnotationsPatch(newService.ObjectMeta.Annotations); err == nil {
+			_, err = c.KubeClient.Services(serviceName.Namespace).Patch(
+				serviceName.Name,
+				types.MergePatchType,
+				[]byte(annotationsPatchData), "")

-		_, err := c.KubeClient.Services(serviceName.Namespace).Patch(
-			serviceName.Name,
-			types.StrategicMergePatchType,
-			[]byte(annotationsPatchData), "")
-
-		if err != nil {
-			return fmt.Errorf("could not replace annotations for the service %q: %v", serviceName, err)
+			if err != nil {
+				return fmt.Errorf("could not replace annotations for the service %q: %v", serviceName, err)
+			}
+		} else {
+			return fmt.Errorf("could not form patch for the service metadata: %v", err)
 		}
 	}

@@ -360,6 +420,7 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error
 		return fmt.Errorf("could not form patch for the service %q: %v", serviceName, err)
 	}

+	// update the service spec
 	svc, err := c.KubeClient.Services(serviceName.Namespace).Patch(
 		serviceName.Name,
 		types.MergePatchType,