Enforce minimum cpu and memory limits (#731)

* add validation for PG resources and volume size
* check resource requests also on UPDATE and SYNC + update docs
* if cluster was running don't error on sync
This commit is contained in:
Felix Kunde
2019-12-12 16:43:55 +01:00
committed by GitHub
parent 0628439256
commit cd110aabf4
8 changed files with 117 additions and 18 deletions
+59
View File
@@ -227,6 +227,10 @@ func (c *Cluster) Create() error {
c.setStatus(acidv1.ClusterStatusCreating)
if err = c.validateResources(&c.Spec); err != nil {
return fmt.Errorf("insufficient resource limits specified: %v", err)
}
for _, role := range []PostgresRole{Master, Replica} {
if c.Endpoints[role] != nil {
@@ -491,6 +495,44 @@ func compareResourcesAssumeFirstNotNil(a *v1.ResourceRequirements, b *v1.Resourc
}
func (c *Cluster) validateResources(spec *acidv1.PostgresSpec) error {
// setting limits too low can cause unnecessary evictions / OOM kills
const (
cpuMinLimit = "256m"
memoryMinLimit = "256Mi"
)
var (
isSmaller bool
err error
)
cpuLimit := spec.Resources.ResourceLimits.CPU
if cpuLimit != "" {
isSmaller, err = util.IsSmallerQuantity(cpuLimit, cpuMinLimit)
if err != nil {
return fmt.Errorf("error validating CPU limit: %v", err)
}
if isSmaller {
return fmt.Errorf("defined CPU limit %s is below required minimum %s to properly run postgresql resource", cpuLimit, cpuMinLimit)
}
}
memoryLimit := spec.Resources.ResourceLimits.Memory
if memoryLimit != "" {
isSmaller, err = util.IsSmallerQuantity(memoryLimit, memoryMinLimit)
if err != nil {
return fmt.Errorf("error validating memory limit: %v", err)
}
if isSmaller {
return fmt.Errorf("defined memory limit %s is below required minimum %s to properly run postgresql resource", memoryLimit, memoryMinLimit)
}
}
return nil
}
// Update changes Kubernetes objects according to the new specification. Unlike the sync case, the missing object
// (i.e. service) is treated as an error
// logical backup cron jobs are an exception: a user-initiated Update can enable a logical backup job
@@ -501,6 +543,7 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
c.mu.Lock()
defer c.mu.Unlock()
oldStatus := c.Status
c.setStatus(acidv1.ClusterStatusUpdating)
c.setSpec(newSpec)
@@ -512,6 +555,22 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
}
}()
if err := c.validateResources(&newSpec.Spec); err != nil {
err = fmt.Errorf("insufficient resource limits specified: %v", err)
// cancel update only when (already too low) pod resources were edited
// if cluster was successfully running before the update, continue but log a warning
isCPULimitSmaller, err2 := util.IsSmallerQuantity(newSpec.Spec.Resources.ResourceLimits.CPU, oldSpec.Spec.Resources.ResourceLimits.CPU)
isMemoryLimitSmaller, err3 := util.IsSmallerQuantity(newSpec.Spec.Resources.ResourceLimits.Memory, oldSpec.Spec.Resources.ResourceLimits.Memory)
if oldStatus.Running() && !isCPULimitSmaller && !isMemoryLimitSmaller && err2 == nil && err3 == nil {
c.logger.Warning(err)
} else {
updateFailed = true
return err
}
}
if oldSpec.Spec.PgVersion != newSpec.Spec.PgVersion { // PG versions comparison
c.logger.Warningf("postgresql version change(%q -> %q) has no effect", oldSpec.Spec.PgVersion, newSpec.Spec.PgVersion)
//we need that hack to generate statefulset with the old version
+2 -2
View File
@@ -741,7 +741,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
limit = c.OpConfig.DefaultMemoryLimit
}
isSmaller, err := util.RequestIsSmallerThanLimit(request, limit)
isSmaller, err := util.IsSmallerQuantity(request, limit)
if err != nil {
return nil, err
}
@@ -768,7 +768,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
limit = c.OpConfig.DefaultMemoryLimit
}
isSmaller, err := util.RequestIsSmallerThanLimit(sidecarRequest, sidecarLimit)
isSmaller, err := util.IsSmallerQuantity(sidecarRequest, sidecarLimit)
if err != nil {
return nil, err
}
+11
View File
@@ -23,6 +23,7 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
c.mu.Lock()
defer c.mu.Unlock()
oldStatus := c.Status
c.setSpec(newSpec)
defer func() {
@@ -34,6 +35,16 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
}
}()
if err = c.validateResources(&c.Spec); err != nil {
err = fmt.Errorf("insufficient resource limits specified: %v", err)
if oldStatus.Running() {
c.logger.Warning(err)
err = nil
} else {
return err
}
}
if err = c.initUsers(); err != nil {
err = fmt.Errorf("could not init users: %v", err)
return err