add annotation to ignore resources thresholds (#3030)

* add annotation to ignore resources thresholds
* add test case when annotation key is set but value is not true

charts/postgres-operator/crds/operatorconfigurations.yaml

@@ -96,6 +96,8 @@ spec:
                 default: ""
               ignore_instance_limits_annotation_key:
                 type: string
+              ignore_resources_limits_annotation_key:
+                type: string
               kubernetes_use_configmaps:
                 type: boolean
                 default: false

charts/postgres-operator/values.yaml

@@ -43,6 +43,9 @@ configGeneral:
   # key name for annotation to ignore globally configured instance limits
   # ignore_instance_limits_annotation_key: ""
 
+  # key name for annotation to ignore globally configured resources thresholds
+  # ignore_resources_limits_annotation_key: ""
+
   # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s)
   # kubernetes_use_configmaps: false
 

docs/reference/operator_parameters.md

@@ -163,7 +163,15 @@ Those are top-level keys, containing both leaf keys and groups.
   for some clusters it might be required to scale beyond the limits that can be
   configured with `min_instances` and `max_instances` options. You can define
   an annotation key that can be used as a toggle in cluster manifests to ignore
-  globally configured instance limits. The default is empty.
+  globally configured instance limits. The value must be `"true"` to be
+  effective. The default is empty which means the feature is disabled.
+
+* **ignore_resources_limits_annotation_key**
+  for some clusters it might be required to request resources beyond the globally
+  configured thresholds for maximum requests and minimum limits. You can define
+  an annotation key that can be used as a toggle in cluster manifests to ignore
+  the thresholds. The value must be `"true"` to be effective. The default is empty
+  which means the feature is disabled.
 
 * **resync_period**
   period between consecutive sync requests. The default is `30m`.

manifests/configmap.yaml

@@ -75,6 +75,7 @@ data:
   # infrastructure_roles_secret_name: "postgresql-infrastructure-roles"
   # infrastructure_roles_secrets: "secretname:monitoring-roles,userkey:user,passwordkey:password,rolekey:inrole"
   # ignore_instance_limits_annotation_key: ""
+  # ignore_resources_limits_annotation_key: ""
   # inherited_annotations: owned-by
   # inherited_labels: application,environment
   # kube_iam_role: ""

manifests/operatorconfiguration.crd.yaml

@@ -94,6 +94,8 @@ spec:
                 default: ""
               ignore_instance_limits_annotation_key:
                 type: string
+              ignore_resources_limits_annotation_key:
+                type: string
               kubernetes_use_configmaps:
                 type: boolean
                 default: false

manifests/postgresql-operator-default-configuration.yaml

@@ -14,6 +14,7 @@ configuration:
   enable_team_id_clustername_prefix: false
   etcd_host: ""
   # ignore_instance_limits_annotation_key: ""
+  # ignore_resources_limits_annotation_key: ""
   # kubernetes_use_configmaps: false
   max_instances: -1
   min_instances: -1

pkg/apis/acid.zalan.do/v1/crds.go

@@ -122,6 +122,9 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
 					"ignore_instance_limits_annotation_key": {
 						Type: "string",
 					},
+					"ignore_resources_limits_annotation_key": {
+						Type: "string",
+					},
 					"kubernetes_use_configmaps": {
 						Type: "boolean",
 					},

pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

@@ -288,6 +288,8 @@ type OperatorConfigurationData struct {
 	MinInstances                      int32  `json:"min_instances,omitempty"`
 	MaxInstances                      int32  `json:"max_instances,omitempty"`
 	IgnoreInstanceLimitsAnnotationKey string `json:"ignore_instance_limits_annotation_key,omitempty"`
+
+	IgnoreResourcesLimitsAnnotationKey string `json:"ignore_resources_limits_annotation_key,omitempty"`
 }
 
 // Duration shortens this frequently used name

pkg/cluster/k8sres.go

@@ -313,6 +313,14 @@ func (c *Cluster) generateResourceRequirements(
 	specLimits := acidv1.ResourceDescription{}
 	result := v1.ResourceRequirements{}
 
+	enforceThresholds := true
+	resourcesLimitAnnotationKey := c.OpConfig.IgnoreResourcesLimitsAnnotationKey
+	if resourcesLimitAnnotationKey != "" {
+		if value, exists := c.ObjectMeta.Annotations[resourcesLimitAnnotationKey]; exists && value == "true" {
+			enforceThresholds = false
+		}
+	}
+
 	if resources != nil {
 		specRequests = resources.ResourceRequests
 		specLimits = resources.ResourceLimits
@@ -329,7 +337,7 @@ func (c *Cluster) generateResourceRequirements(
 	}
 
 	// enforce minimum cpu and memory limits for Postgres containers only
-	if containerName == constants.PostgresContainerName {
+	if containerName == constants.PostgresContainerName && enforceThresholds {
 		if err = c.enforceMinResourceLimits(&result); err != nil {
 			return nil, fmt.Errorf("could not enforce minimum resource limits: %v", err)
 		}
@@ -344,7 +352,7 @@ func (c *Cluster) generateResourceRequirements(
 	}
 
 	// enforce maximum cpu and memory requests for Postgres containers only
-	if containerName == constants.PostgresContainerName {
+	if containerName == constants.PostgresContainerName && enforceThresholds {
 		if err = c.enforceMaxResourceRequests(&result); err != nil {
 			return nil, fmt.Errorf("could not enforce maximum resource requests: %v", err)
 		}

pkg/cluster/k8sres_test.go

@@ -3130,6 +3130,9 @@ func TestGenerateResourceRequirements(t *testing.T) {
 		PodRoleLabel:         "spilo-role",
 	}
 
+	configWithEnabledIgnoreResourcesLimits := configResources
+	configWithEnabledIgnoreResourcesLimits.IgnoreResourcesLimitsAnnotationKey = "zalando.org/ignore-resources-limits"
+
 	tests := []struct {
 		subTest           string
 		config            config.Config
@@ -3465,14 +3468,15 @@ func TestGenerateResourceRequirements(t *testing.T) {
 		{
 			subTest: "test enforcing min cpu and memory limit",
 			config: config.Config{
-				Resources:               configResources,
+				Resources:               configWithEnabledIgnoreResourcesLimits,
 				PodManagementPolicy:     "ordered_ready",
 				SetMemoryRequestToLimit: false,
 			},
 			pgSpec: acidv1.Postgresql{
 				ObjectMeta: metav1.ObjectMeta{
-					Name:      clusterName,
+					Name:        clusterName,
-					Namespace: namespace,
+					Namespace:   namespace,
+					Annotations: map[string]string{"zalando.org/ignore-resources-limits": "false"},
 				},
 				Spec: acidv1.PostgresSpec{
 					Resources: &acidv1.Resources{
@@ -3490,6 +3494,35 @@ func TestGenerateResourceRequirements(t *testing.T) {
 				ResourceLimits:   acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("250m"), Memory: k8sutil.StringToPointer("250Mi")},
 			},
 		},
+		{
+			subTest: "ingnore min cpu and memory limit threshold",
+			config: config.Config{
+				Resources:               configWithEnabledIgnoreResourcesLimits,
+				PodManagementPolicy:     "ordered_ready",
+				SetMemoryRequestToLimit: false,
+			},
+			pgSpec: acidv1.Postgresql{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:        clusterName,
+					Namespace:   namespace,
+					Annotations: map[string]string{"zalando.org/ignore-resources-limits": "true"},
+				},
+				Spec: acidv1.PostgresSpec{
+					Resources: &acidv1.Resources{
+						ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("100m"), Memory: k8sutil.StringToPointer("100Mi")},
+						ResourceLimits:   acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("200m"), Memory: k8sutil.StringToPointer("200Mi")},
+					},
+					TeamID: "acid",
+					Volume: acidv1.Volume{
+						Size: "1G",
+					},
+				},
+			},
+			expectedResources: acidv1.Resources{
+				ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("100m"), Memory: k8sutil.StringToPointer("100Mi")},
+				ResourceLimits:   acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("200m"), Memory: k8sutil.StringToPointer("200Mi")},
+			},
+		},
 		{
 			subTest: "test min cpu and memory limit are not enforced on sidecar",
 			config: config.Config{
@@ -3527,14 +3560,15 @@ func TestGenerateResourceRequirements(t *testing.T) {
 		{
 			subTest: "test enforcing max cpu and memory requests",
 			config: config.Config{
-				Resources:               configResources,
+				Resources:               configWithEnabledIgnoreResourcesLimits,
 				PodManagementPolicy:     "ordered_ready",
 				SetMemoryRequestToLimit: false,
 			},
 			pgSpec: acidv1.Postgresql{
 				ObjectMeta: metav1.ObjectMeta{
-					Name:      clusterName,
+					Name:        clusterName,
-					Namespace: namespace,
+					Namespace:   namespace,
+					Annotations: map[string]string{"zalando.org/ignore-resources-limits": "yes"},
 				},
 				Spec: acidv1.PostgresSpec{
 					Resources: &acidv1.Resources{
@@ -3552,6 +3586,35 @@ func TestGenerateResourceRequirements(t *testing.T) {
 				ResourceLimits:   acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("2"), Memory: k8sutil.StringToPointer("4Gi")},
 			},
 		},
+		{
+			subTest: "ignore max cpu and memory requests limit",
+			config: config.Config{
+				Resources:               configWithEnabledIgnoreResourcesLimits,
+				PodManagementPolicy:     "ordered_ready",
+				SetMemoryRequestToLimit: false,
+			},
+			pgSpec: acidv1.Postgresql{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:        clusterName,
+					Namespace:   namespace,
+					Annotations: map[string]string{"zalando.org/ignore-resources-limits": "true"},
+				},
+				Spec: acidv1.PostgresSpec{
+					Resources: &acidv1.Resources{
+						ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("2Gi")},
+						ResourceLimits:   acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("2"), Memory: k8sutil.StringToPointer("4Gi")},
+					},
+					TeamID: "acid",
+					Volume: acidv1.Volume{
+						Size: "1G",
+					},
+				},
+			},
+			expectedResources: acidv1.Resources{
+				ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("2Gi")},
+				ResourceLimits:   acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("2"), Memory: k8sutil.StringToPointer("4Gi")},
+			},
+		},
 		{
 			subTest: "test SetMemoryRequestToLimit flag but raise only until max memory request",
 			config: config.Config{

pkg/controller/operator_config.go

@@ -44,6 +44,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	result.MinInstances = fromCRD.MinInstances
 	result.MaxInstances = fromCRD.MaxInstances
 	result.IgnoreInstanceLimitsAnnotationKey = fromCRD.IgnoreInstanceLimitsAnnotationKey
+	result.IgnoreResourcesLimitsAnnotationKey = fromCRD.IgnoreResourcesLimitsAnnotationKey
 	result.ResyncPeriod = util.CoalesceDuration(time.Duration(fromCRD.ResyncPeriod), "30m")
 	result.RepairPeriod = util.CoalesceDuration(time.Duration(fromCRD.RepairPeriod), "5m")
 	result.SetMemoryRequestToLimit = fromCRD.SetMemoryRequestToLimit

pkg/util/config/config.go

@@ -66,6 +66,8 @@ type Resources struct {
 	MaxInstances                      int32  `name:"max_instances" default:"-1"`
 	MinInstances                      int32  `name:"min_instances" default:"-1"`
 	IgnoreInstanceLimitsAnnotationKey string `name:"ignore_instance_limits_annotation_key"`
+
+	IgnoreResourcesLimitsAnnotationKey string `name:"ignore_resources_limits_annotation_key"`
 }
 
 type InfrastructureRole struct {