mirror of
https://github.com/zalando/postgres-operator.git
synced 2026-10-03 04:41:51 +02:00
Enforce minimum cpu and memory limits (#731)
* add validation for PG resources and volume size * check resource requests also on UPDATE and SYNC + update docs * if cluster was running don't error on sync
This commit is contained in:
@@ -227,6 +227,10 @@ func (c *Cluster) Create() error {
|
||||
|
||||
c.setStatus(acidv1.ClusterStatusCreating)
|
||||
|
||||
if err = c.validateResources(&c.Spec); err != nil {
|
||||
return fmt.Errorf("insufficient resource limits specified: %v", err)
|
||||
}
|
||||
|
||||
for _, role := range []PostgresRole{Master, Replica} {
|
||||
|
||||
if c.Endpoints[role] != nil {
|
||||
@@ -491,6 +495,44 @@ func compareResourcesAssumeFirstNotNil(a *v1.ResourceRequirements, b *v1.Resourc
|
||||
|
||||
}
|
||||
|
||||
func (c *Cluster) validateResources(spec *acidv1.PostgresSpec) error {
|
||||
|
||||
// setting limits too low can cause unnecessary evictions / OOM kills
|
||||
const (
|
||||
cpuMinLimit = "256m"
|
||||
memoryMinLimit = "256Mi"
|
||||
)
|
||||
|
||||
var (
|
||||
isSmaller bool
|
||||
err error
|
||||
)
|
||||
|
||||
cpuLimit := spec.Resources.ResourceLimits.CPU
|
||||
if cpuLimit != "" {
|
||||
isSmaller, err = util.IsSmallerQuantity(cpuLimit, cpuMinLimit)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error validating CPU limit: %v", err)
|
||||
}
|
||||
if isSmaller {
|
||||
return fmt.Errorf("defined CPU limit %s is below required minimum %s to properly run postgresql resource", cpuLimit, cpuMinLimit)
|
||||
}
|
||||
}
|
||||
|
||||
memoryLimit := spec.Resources.ResourceLimits.Memory
|
||||
if memoryLimit != "" {
|
||||
isSmaller, err = util.IsSmallerQuantity(memoryLimit, memoryMinLimit)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error validating memory limit: %v", err)
|
||||
}
|
||||
if isSmaller {
|
||||
return fmt.Errorf("defined memory limit %s is below required minimum %s to properly run postgresql resource", memoryLimit, memoryMinLimit)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Update changes Kubernetes objects according to the new specification. Unlike the sync case, the missing object
|
||||
// (i.e. service) is treated as an error
|
||||
// logical backup cron jobs are an exception: a user-initiated Update can enable a logical backup job
|
||||
@@ -501,6 +543,7 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
oldStatus := c.Status
|
||||
c.setStatus(acidv1.ClusterStatusUpdating)
|
||||
c.setSpec(newSpec)
|
||||
|
||||
@@ -512,6 +555,22 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
|
||||
}
|
||||
}()
|
||||
|
||||
if err := c.validateResources(&newSpec.Spec); err != nil {
|
||||
err = fmt.Errorf("insufficient resource limits specified: %v", err)
|
||||
|
||||
// cancel update only when (already too low) pod resources were edited
|
||||
// if cluster was successfully running before the update, continue but log a warning
|
||||
isCPULimitSmaller, err2 := util.IsSmallerQuantity(newSpec.Spec.Resources.ResourceLimits.CPU, oldSpec.Spec.Resources.ResourceLimits.CPU)
|
||||
isMemoryLimitSmaller, err3 := util.IsSmallerQuantity(newSpec.Spec.Resources.ResourceLimits.Memory, oldSpec.Spec.Resources.ResourceLimits.Memory)
|
||||
|
||||
if oldStatus.Running() && !isCPULimitSmaller && !isMemoryLimitSmaller && err2 == nil && err3 == nil {
|
||||
c.logger.Warning(err)
|
||||
} else {
|
||||
updateFailed = true
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if oldSpec.Spec.PgVersion != newSpec.Spec.PgVersion { // PG versions comparison
|
||||
c.logger.Warningf("postgresql version change(%q -> %q) has no effect", oldSpec.Spec.PgVersion, newSpec.Spec.PgVersion)
|
||||
//we need that hack to generate statefulset with the old version
|
||||
|
||||
@@ -741,7 +741,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
|
||||
limit = c.OpConfig.DefaultMemoryLimit
|
||||
}
|
||||
|
||||
isSmaller, err := util.RequestIsSmallerThanLimit(request, limit)
|
||||
isSmaller, err := util.IsSmallerQuantity(request, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -768,7 +768,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
|
||||
limit = c.OpConfig.DefaultMemoryLimit
|
||||
}
|
||||
|
||||
isSmaller, err := util.RequestIsSmallerThanLimit(sidecarRequest, sidecarLimit)
|
||||
isSmaller, err := util.IsSmallerQuantity(sidecarRequest, sidecarLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
oldStatus := c.Status
|
||||
c.setSpec(newSpec)
|
||||
|
||||
defer func() {
|
||||
@@ -34,6 +35,16 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
|
||||
}
|
||||
}()
|
||||
|
||||
if err = c.validateResources(&c.Spec); err != nil {
|
||||
err = fmt.Errorf("insufficient resource limits specified: %v", err)
|
||||
if oldStatus.Running() {
|
||||
c.logger.Warning(err)
|
||||
err = nil
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err = c.initUsers(); err != nil {
|
||||
err = fmt.Errorf("could not init users: %v", err)
|
||||
return err
|
||||
|
||||
Reference in New Issue
Block a user