public_git/postgres-operator
1
0
Fork 0
mirror of https://github.com/zalando/postgres-operator.git synced 2026-09-30 12:54:25 +02:00
Code Issues Projects Releases Wiki Activity

make minimum limits boundaries configurable (#808)

Browse Source 
* make minimum limits boundaries configurable
* add e2e test
This commit is contained in:
Felix Kunde
2020-02-03 11:43:18 +01:00
committed by GitHub
parent fddaf0fb73
commit 1f0312a014
17 changed files with 175 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/apis/acid.zalan.do/v1/crds.go
+8
View File
@@ -810,6 +810,14 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation
Type: "string",
Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$",
},
"min_cpu_limit": {
Type: "string",
Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$",
},
"min_memory_limit": {
Type: "string",
Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$",
},
},
},
"timeouts": {
pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
+2
View File
@@ -79,6 +79,8 @@ type PostgresPodResourcesDefaults struct {
DefaultMemoryRequest string `json:"default_memory_request,omitempty"`
DefaultCPULimit string `json:"default_cpu_limit,omitempty"`
DefaultMemoryLimit string `json:"default_memory_limit,omitempty"`
MinCPULimit string `json:"min_cpu_limit,omitempty"`
MinMemoryLimit string `json:"min_memory_limit,omitempty"`
}
// OperatorTimeouts defines the timeout of ResourceCheck, PodWait, ReadyWait
pkg/cluster/cluster.go
+24 -32
View File
@@ -227,8 +227,8 @@ func (c *Cluster) Create() error {
c.setStatus(acidv1.ClusterStatusCreating)
if err = c.validateResources(&c.Spec); err != nil {
return fmt.Errorf("insufficient resource limits specified: %v", err)
if err = c.enforceMinResourceLimits(&c.Spec); err != nil {
return fmt.Errorf("could not enforce minimum resource limits: %v", err)
}
for _, role := range []PostgresRole{Master, Replica} {
@@ -495,38 +495,38 @@ func compareResourcesAssumeFirstNotNil(a *v1.ResourceRequirements, b *v1.Resourc
}
func (c *Cluster) validateResources(spec *acidv1.PostgresSpec) error {
// setting limits too low can cause unnecessary evictions / OOM kills
const (
cpuMinLimit = "256m"
memoryMinLimit = "256Mi"
)
func (c *Cluster) enforceMinResourceLimits(spec *acidv1.PostgresSpec) error {
var (
isSmaller bool
err error
)
// setting limits too low can cause unnecessary evictions / OOM kills
minCPULimit := c.OpConfig.MinCPULimit
minMemoryLimit := c.OpConfig.MinMemoryLimit
cpuLimit := spec.Resources.ResourceLimits.CPU
if cpuLimit != "" {
isSmaller, err = util.IsSmallerQuantity(cpuLimit, cpuMinLimit)
isSmaller, err = util.IsSmallerQuantity(cpuLimit, minCPULimit)
if err != nil {
return fmt.Errorf("error validating CPU limit: %v", err)
return fmt.Errorf("could not compare defined CPU limit %s with configured minimum value %s: %v", cpuLimit, minCPULimit, err)
}
if isSmaller {
return fmt.Errorf("defined CPU limit %s is below required minimum %s to properly run postgresql resource", cpuLimit, cpuMinLimit)
c.logger.Warningf("defined CPU limit %s is below required minimum %s and will be set to it", cpuLimit, minCPULimit)
spec.Resources.ResourceLimits.CPU = minCPULimit
}
}
memoryLimit := spec.Resources.ResourceLimits.Memory
if memoryLimit != "" {
isSmaller, err = util.IsSmallerQuantity(memoryLimit, memoryMinLimit)
isSmaller, err = util.IsSmallerQuantity(memoryLimit, minMemoryLimit)
if err != nil {
return fmt.Errorf("error validating memory limit: %v", err)
return fmt.Errorf("could not compare defined memory limit %s with configured minimum value %s: %v", memoryLimit, minMemoryLimit, err)
}
if isSmaller {
return fmt.Errorf("defined memory limit %s is below required minimum %s to properly run postgresql resource", memoryLimit, memoryMinLimit)
c.logger.Warningf("defined memory limit %s is below required minimum %s and will be set to it", memoryLimit, minMemoryLimit)
spec.Resources.ResourceLimits.Memory = minMemoryLimit
}
}
@@ -543,7 +543,6 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
c.mu.Lock()
defer c.mu.Unlock()
oldStatus := c.Status
c.setStatus(acidv1.ClusterStatusUpdating)
c.setSpec(newSpec)
@@ -555,22 +554,6 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
}
}()
if err := c.validateResources(&newSpec.Spec); err != nil {
err = fmt.Errorf("insufficient resource limits specified: %v", err)
// cancel update only when (already too low) pod resources were edited
// if cluster was successfully running before the update, continue but log a warning
isCPULimitSmaller, err2 := util.IsSmallerQuantity(newSpec.Spec.Resources.ResourceLimits.CPU, oldSpec.Spec.Resources.ResourceLimits.CPU)
isMemoryLimitSmaller, err3 := util.IsSmallerQuantity(newSpec.Spec.Resources.ResourceLimits.Memory, oldSpec.Spec.Resources.ResourceLimits.Memory)
if oldStatus.Running() && !isCPULimitSmaller && !isMemoryLimitSmaller && err2 == nil && err3 == nil {
c.logger.Warning(err)
} else {
updateFailed = true
return err
}
}
if oldSpec.Spec.PgVersion != newSpec.Spec.PgVersion { // PG versions comparison
c.logger.Warningf("postgresql version change(%q -> %q) has no effect", oldSpec.Spec.PgVersion, newSpec.Spec.PgVersion)
//we need that hack to generate statefulset with the old version
@@ -616,6 +599,12 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
// Statefulset
func() {
if err := c.enforceMinResourceLimits(&c.Spec); err != nil {
c.logger.Errorf("could not sync resources: %v", err)
updateFailed = true
return
}
oldSs, err := c.generateStatefulSet(&oldSpec.Spec)
if err != nil {
c.logger.Errorf("could not generate old statefulset spec: %v", err)
@@ -623,6 +612,9 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
return
}
// update newSpec to for latter comparison with oldSpec
c.enforceMinResourceLimits(&newSpec.Spec)
newSs, err := c.generateStatefulSet(&newSpec.Spec)
if err != nil {
c.logger.Errorf("could not generate new statefulset spec: %v", err)
pkg/cluster/sync.go
+5 -11
View File
@@ -23,7 +23,6 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
c.mu.Lock()
defer c.mu.Unlock()
oldStatus := c.Status
c.setSpec(newSpec)
defer func() {
@@ -35,16 +34,6 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
}
}()
if err = c.validateResources(&c.Spec); err != nil {
err = fmt.Errorf("insufficient resource limits specified: %v", err)
if oldStatus.Running() {
c.logger.Warning(err)
err = nil
} else {
return err
}
}
if err = c.initUsers(); err != nil {
err = fmt.Errorf("could not init users: %v", err)
return err
@@ -76,6 +65,11 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
return err
}
if err = c.enforceMinResourceLimits(&c.Spec); err != nil {
err = fmt.Errorf("could not enforce minimum resource limits: %v", err)
return err
}
c.logger.Debugf("syncing statefulsets")
if err = c.syncStatefulSet(); err != nil {
if !k8sutil.ResourceAlreadyExists(err) {
pkg/controller/operator_config.go
+2
View File
@@ -75,6 +75,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
result.DefaultMemoryRequest = fromCRD.PostgresPodResources.DefaultMemoryRequest
result.DefaultCPULimit = fromCRD.PostgresPodResources.DefaultCPULimit
result.DefaultMemoryLimit = fromCRD.PostgresPodResources.DefaultMemoryLimit
result.MinCPULimit = fromCRD.PostgresPodResources.MinCPULimit
result.MinMemoryLimit = fromCRD.PostgresPodResources.MinMemoryLimit
// timeout config
result.ResourceCheckInterval = time.Duration(fromCRD.Timeouts.ResourceCheckInterval)
pkg/util/config/config.go
+5 -3
View File
@@ -37,8 +37,10 @@ type Resources struct {
PodToleration map[string]string `name:"toleration" default:""`
DefaultCPURequest string `name:"default_cpu_request" default:"100m"`
DefaultMemoryRequest string `name:"default_memory_request" default:"100Mi"`
DefaultCPULimit string `name:"default_cpu_limit" default:"3"`
DefaultMemoryLimit string `name:"default_memory_limit" default:"1Gi"`
DefaultCPULimit string `name:"default_cpu_limit" default:"1"`
DefaultMemoryLimit string `name:"default_memory_limit" default:"500Mi"`
MinCPULimit string `name:"min_cpu_limit" default:"250m"`
MinMemoryLimit string `name:"min_memory_limit" default:"250Mi"`
PodEnvironmentConfigMap string `name:"pod_environment_configmap" default:""`
NodeReadinessLabel map[string]string `name:"node_readiness_label" default:""`
MaxInstances int32 `name:"max_instances" default:"-1"`
@@ -66,7 +68,7 @@ type Scalyr struct {
ScalyrCPURequest string `name:"scalyr_cpu_request" default:"100m"`
ScalyrMemoryRequest string `name:"scalyr_memory_request" default:"50Mi"`
ScalyrCPULimit string `name:"scalyr_cpu_limit" default:"1"`
ScalyrMemoryLimit string `name:"scalyr_memory_limit" default:"1Gi"`
ScalyrMemoryLimit string `name:"scalyr_memory_limit" default:"500Mi"`
}
// LogicalBackup defines configuration for logical backup