public_git/postgres-operator
1
0
Fork 0
mirror of https://github.com/zalando/postgres-operator.git synced 2026-09-30 15:46:44 +02:00
Code Issues Projects Releases Wiki Activity

Enforce minimum cpu and memory limits (#731)

Browse Source 
* add validation for PG resources and volume size
* check resource requests also on UPDATE and SYNC + update docs
* if cluster was running don't error on sync
This commit is contained in:
Felix Kunde
2019-12-12 16:43:55 +01:00
committed by GitHub
parent 0628439256
commit cd110aabf4
8 changed files with 117 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/cluster/cluster.go
+59
View File
@@ -227,6 +227,10 @@ func (c *Cluster) Create() error {
c.setStatus(acidv1.ClusterStatusCreating)
if err = c.validateResources(&c.Spec); err != nil {
return fmt.Errorf("insufficient resource limits specified: %v", err)
}
for _, role := range []PostgresRole{Master, Replica} {
if c.Endpoints[role] != nil {
@@ -491,6 +495,44 @@ func compareResourcesAssumeFirstNotNil(a *v1.ResourceRequirements, b *v1.Resourc
}
func (c *Cluster) validateResources(spec *acidv1.PostgresSpec) error {
// setting limits too low can cause unnecessary evictions / OOM kills
const (
cpuMinLimit = "256m"
memoryMinLimit = "256Mi"
)
var (
isSmaller bool
err error
)
cpuLimit := spec.Resources.ResourceLimits.CPU
if cpuLimit != "" {
isSmaller, err = util.IsSmallerQuantity(cpuLimit, cpuMinLimit)
if err != nil {
return fmt.Errorf("error validating CPU limit: %v", err)
}
if isSmaller {
return fmt.Errorf("defined CPU limit %s is below required minimum %s to properly run postgresql resource", cpuLimit, cpuMinLimit)
}
}
memoryLimit := spec.Resources.ResourceLimits.Memory
if memoryLimit != "" {
isSmaller, err = util.IsSmallerQuantity(memoryLimit, memoryMinLimit)
if err != nil {
return fmt.Errorf("error validating memory limit: %v", err)
}
if isSmaller {
return fmt.Errorf("defined memory limit %s is below required minimum %s to properly run postgresql resource", memoryLimit, memoryMinLimit)
}
}
return nil
}
// Update changes Kubernetes objects according to the new specification. Unlike the sync case, the missing object
// (i.e. service) is treated as an error
// logical backup cron jobs are an exception: a user-initiated Update can enable a logical backup job
@@ -501,6 +543,7 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
c.mu.Lock()
defer c.mu.Unlock()
oldStatus := c.Status
c.setStatus(acidv1.ClusterStatusUpdating)
c.setSpec(newSpec)
@@ -512,6 +555,22 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
}
}()
if err := c.validateResources(&newSpec.Spec); err != nil {
err = fmt.Errorf("insufficient resource limits specified: %v", err)
// cancel update only when (already too low) pod resources were edited
// if cluster was successfully running before the update, continue but log a warning
isCPULimitSmaller, err2 := util.IsSmallerQuantity(newSpec.Spec.Resources.ResourceLimits.CPU, oldSpec.Spec.Resources.ResourceLimits.CPU)
isMemoryLimitSmaller, err3 := util.IsSmallerQuantity(newSpec.Spec.Resources.ResourceLimits.Memory, oldSpec.Spec.Resources.ResourceLimits.Memory)
if oldStatus.Running() && !isCPULimitSmaller && !isMemoryLimitSmaller && err2 == nil && err3 == nil {
c.logger.Warning(err)
} else {
updateFailed = true
return err
}
}
if oldSpec.Spec.PgVersion != newSpec.Spec.PgVersion { // PG versions comparison
c.logger.Warningf("postgresql version change(%q -> %q) has no effect", oldSpec.Spec.PgVersion, newSpec.Spec.PgVersion)
//we need that hack to generate statefulset with the old version
pkg/cluster/k8sres.go
+2 -2
View File
@@ -741,7 +741,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
limit = c.OpConfig.DefaultMemoryLimit
}
isSmaller, err := util.RequestIsSmallerThanLimit(request, limit)
isSmaller, err := util.IsSmallerQuantity(request, limit)
if err != nil {
return nil, err
}
@@ -768,7 +768,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
limit = c.OpConfig.DefaultMemoryLimit
}
isSmaller, err := util.RequestIsSmallerThanLimit(sidecarRequest, sidecarLimit)
isSmaller, err := util.IsSmallerQuantity(sidecarRequest, sidecarLimit)
if err != nil {
return nil, err
}
pkg/cluster/sync.go
+11
View File
@@ -23,6 +23,7 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
c.mu.Lock()
defer c.mu.Unlock()
oldStatus := c.Status
c.setSpec(newSpec)
defer func() {
@@ -34,6 +35,16 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
}
}()
if err = c.validateResources(&c.Spec); err != nil {
err = fmt.Errorf("insufficient resource limits specified: %v", err)
if oldStatus.Running() {
c.logger.Warning(err)
err = nil
} else {
return err
}
}
if err = c.initUsers(); err != nil {
err = fmt.Errorf("could not init users: %v", err)
return err
pkg/controller/controller.go
+2 -2
View File
@@ -111,7 +111,7 @@ func (c *Controller) initOperatorConfig() {
if c.opConfig.SetMemoryRequestToLimit {
isSmaller, err := util.RequestIsSmallerThanLimit(c.opConfig.DefaultMemoryRequest, c.opConfig.DefaultMemoryLimit)
isSmaller, err := util.IsSmallerQuantity(c.opConfig.DefaultMemoryRequest, c.opConfig.DefaultMemoryLimit)
if err != nil {
panic(err)
}
@@ -120,7 +120,7 @@ func (c *Controller) initOperatorConfig() {
c.opConfig.DefaultMemoryRequest = c.opConfig.DefaultMemoryLimit
}
isSmaller, err = util.RequestIsSmallerThanLimit(c.opConfig.ScalyrMemoryRequest, c.opConfig.ScalyrMemoryLimit)
isSmaller, err = util.IsSmallerQuantity(c.opConfig.ScalyrMemoryRequest, c.opConfig.ScalyrMemoryLimit)
if err != nil {
panic(err)
}
pkg/util/util.go
+4 -4
View File
@@ -141,17 +141,17 @@ func Coalesce(val, defaultVal string) string {
return val
}
// RequestIsSmallerThanLimit : ...
func RequestIsSmallerThanLimit(requestStr, limitStr string) (bool, error) {
// IsSmallerQuantity : checks if first resource is of a smaller quantity than the second
func IsSmallerQuantity(requestStr, limitStr string) (bool, error) {
request, err := resource.ParseQuantity(requestStr)
if err != nil {
return false, fmt.Errorf("could not parse memory request %v : %v", requestStr, err)
return false, fmt.Errorf("could not parse request %v : %v", requestStr, err)
}
limit, err2 := resource.ParseQuantity(limitStr)
if err2 != nil {
return false, fmt.Errorf("could not parse memory limit %v : %v", limitStr, err2)
return false, fmt.Errorf("could not parse limit %v : %v", limitStr, err2)
}
return request.Cmp(limit) == -1, nil
pkg/util/util_test.go
+6 -6
View File
@@ -69,7 +69,7 @@ var substringMatch = []struct {
{regexp.MustCompile(`aaaa (\d+) bbbb`), "aaaa 123 bbbb", nil},
}
var requestIsSmallerThanLimitTests = []struct {
var requestIsSmallerQuantityTests = []struct {
request string
limit string
out bool
@@ -155,14 +155,14 @@ func TestMapContains(t *testing.T) {
}
}
func TestRequestIsSmallerThanLimit(t *testing.T) {
for _, tt := range requestIsSmallerThanLimitTests {
res, err := RequestIsSmallerThanLimit(tt.request, tt.limit)
func TestIsSmallerQuantity(t *testing.T) {
for _, tt := range requestIsSmallerQuantityTests {
res, err := IsSmallerQuantity(tt.request, tt.limit)
if err != nil {
t.Errorf("RequestIsSmallerThanLimit returned unexpected error: %#v", err)
t.Errorf("IsSmallerQuantity returned unexpected error: %#v", err)
}
if res != tt.out {
t.Errorf("RequestIsSmallerThanLimit expected: %#v, got: %#v", tt.out, res)
t.Errorf("IsSmallerQuantity expected: %#v, got: %#v", tt.out, res)
}
}
}