public_git/postgres-operator
1
0
Fork 0
mirror of https://github.com/zalando/postgres-operator.git synced 2026-10-09 08:26:21 +02:00
Code Issues Projects Releases Wiki Activity

merge with master

Browse Source
This commit is contained in:
Felix Kunde
2021-01-26 17:11:58 +01:00
parent c3c413f0d1 43168ca622
commit 6deac03f32
190 changed files with 21094 additions and 3588 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/cluster/cluster.go
+392 -113
View File
@@ -3,25 +3,21 @@ package cluster
// Postgres CustomResourceDefinition object i.e. Spilo
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"reflect"
"regexp"
"strings"
"sync"
"time"
"github.com/sirupsen/logrus"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
policybeta1 "k8s.io/api/policy/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
"github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme"
"github.com/zalando/postgres-operator/pkg/spec"
pgteams "github.com/zalando/postgres-operator/pkg/teams"
"github.com/zalando/postgres-operator/pkg/util"
"github.com/zalando/postgres-operator/pkg/util/config"
"github.com/zalando/postgres-operator/pkg/util/constants"
@@ -29,7 +25,17 @@ import (
"github.com/zalando/postgres-operator/pkg/util/patroni"
"github.com/zalando/postgres-operator/pkg/util/teams"
"github.com/zalando/postgres-operator/pkg/util/users"
"github.com/zalando/postgres-operator/pkg/util/volumes"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
policybeta1 "k8s.io/api/policy/v1beta1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/record"
"k8s.io/client-go/tools/reference"
)
var (
@@ -43,6 +49,7 @@ var (
type Config struct {
OpConfig config.Config
RestConfig *rest.Config
PgTeamMap pgteams.PostgresTeamMap
InfrastructureRoles map[string]spec.PgUser // inherited from the controller
PodServiceAccount *v1.ServiceAccount
PodServiceAccountRole *rbacv1.Role
@@ -65,6 +72,7 @@ type Cluster struct {
acidv1.Postgresql
Config
logger *logrus.Entry
eventRecorder record.EventRecorder
patroni patroni.Interface
pgUsers map[string]spec.PgUser
systemUsers map[string]spec.PgUser
@@ -73,7 +81,7 @@ type Cluster struct {
pgDb *sql.DB
mu sync.Mutex
userSyncStrategy spec.UserSyncer
deleteOptions *metav1.DeleteOptions
deleteOptions metav1.DeleteOptions
podEventsQueue *cache.FIFO
teamsAPIClient teams.Interface
@@ -82,7 +90,9 @@ type Cluster struct {
currentProcess Process
processMu sync.RWMutex // protects the current operation for reporting, no need to hold the master mutex
specMu sync.RWMutex // protects the spec for reporting, no need to hold the master mutex
ConnectionPooler map[PostgresRole]*ConnectionPoolerObjects
EBSVolumes map[string]volumes.VolumeProperties
VolumeResizer volumes.VolumeResizer
}
type compareStatefulsetResult struct {
@@ -93,7 +103,7 @@ type compareStatefulsetResult struct {
}
// New creates a new cluster. This function should be called from a controller.
func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgresql, logger *logrus.Entry) *Cluster {
func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgresql, logger *logrus.Entry, eventRecorder record.EventRecorder) *Cluster {
deletePropagationPolicy := metav1.DeletePropagationOrphan
podEventsQueue := cache.NewFIFO(func(obj interface{}) (string, error) {
@@ -104,6 +114,10 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres
return fmt.Sprintf("%s-%s", e.PodName, e.ResourceVersion), nil
})
passwordEncryption, ok := pgSpec.Spec.PostgresqlParam.Parameters["password_encryption"]
if !ok {
passwordEncryption = "md5"
}
cluster := &Cluster{
Config: cfg,
@@ -115,8 +129,8 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres
Secrets: make(map[types.UID]*v1.Secret),
Services: make(map[PostgresRole]*v1.Service),
Endpoints: make(map[PostgresRole]*v1.Endpoints)},
userSyncStrategy: users.DefaultUserSyncStrategy{},
deleteOptions: &metav1.DeleteOptions{PropagationPolicy: &deletePropagationPolicy},
userSyncStrategy: users.DefaultUserSyncStrategy{PasswordEncryption: passwordEncryption},
deleteOptions: metav1.DeleteOptions{PropagationPolicy: &deletePropagationPolicy},
podEventsQueue: podEventsQueue,
KubeClient: kubeClient,
}
@@ -124,6 +138,13 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres
cluster.teamsAPIClient = teams.NewTeamsAPI(cfg.OpConfig.TeamsAPIUrl, logger)
cluster.oauthTokenGetter = newSecretOauthTokenGetter(&kubeClient, cfg.OpConfig.OAuthTokenSecretName)
cluster.patroni = patroni.New(cluster.logger)
cluster.eventRecorder = eventRecorder
cluster.EBSVolumes = make(map[string]volumes.VolumeProperties)
if cfg.OpConfig.StorageResizeMode != "pvc" || cfg.OpConfig.EnableEBSGp3Migration {
cluster.VolumeResizer = &volumes.EBSVolumeResizer{AWSRegion: cfg.OpConfig.AWSRegion}
}
return cluster
}
@@ -150,31 +171,14 @@ func (c *Cluster) setProcessName(procName string, args ...interface{}) {
}
}
// SetStatus of Postgres cluster
// TODO: eventually switch to updateStatus() for kubernetes 1.11 and above
func (c *Cluster) setStatus(status string) {
var pgStatus acidv1.PostgresStatus
pgStatus.PostgresClusterStatus = status
patch, err := json.Marshal(struct {
PgStatus interface{} `json:"status"`
}{&pgStatus})
// GetReference of Postgres CR object
// i.e. required to emit events to this resource
func (c *Cluster) GetReference() *v1.ObjectReference {
ref, err := reference.GetReference(scheme.Scheme, &c.Postgresql)
if err != nil {
c.logger.Errorf("could not marshal status: %v", err)
c.logger.Errorf("could not get reference for Postgresql CR %v/%v: %v", c.Postgresql.Namespace, c.Postgresql.Name, err)
}
// we cannot do a full scale update here without fetching the previous manifest (as the resourceVersion may differ),
// however, we could do patch without it. In the future, once /status subresource is there (starting Kubernetes 1.11)
// we should take advantage of it.
newspec, err := c.KubeClient.AcidV1ClientSet.AcidV1().Postgresqls(c.clusterNamespace()).Patch(c.Name, types.MergePatchType, patch, "status")
if err != nil {
c.logger.Errorf("could not update status: %v", err)
// return as newspec is empty, see PR654
return
}
// update the spec, maintaining the new resourceVersion.
c.setSpec(newspec)
return ref
}
func (c *Cluster) isNewCluster() bool {
@@ -185,7 +189,8 @@ func (c *Cluster) isNewCluster() bool {
func (c *Cluster) initUsers() error {
c.setProcessName("initializing users")
// clear our the previous state of the cluster users (in case we are running a sync).
// clear our the previous state of the cluster users (in case we are
// running a sync).
c.systemUsers = map[string]spec.PgUser{}
c.pgUsers = map[string]spec.PgUser{}
@@ -195,6 +200,10 @@ func (c *Cluster) initUsers() error {
return fmt.Errorf("could not init infrastructure roles: %v", err)
}
if err := c.initPreparedDatabaseRoles(); err != nil {
return fmt.Errorf("could not init default users: %v", err)
}
if err := c.initRobotUsers(); err != nil {
return fmt.Errorf("could not init robot users: %v", err)
}
@@ -220,13 +229,14 @@ func (c *Cluster) Create() error {
defer func() {
if err == nil {
c.setStatus(acidv1.ClusterStatusRunning) //TODO: are you sure it's running?
c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusRunning) //TODO: are you sure it's running?
} else {
c.setStatus(acidv1.ClusterStatusAddFailed)
c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusAddFailed)
}
}()
c.setStatus(acidv1.ClusterStatusCreating)
c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusCreating)
c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Create", "Started creation of new cluster resources")
if err = c.enforceMinResourceLimits(&c.Spec); err != nil {
return fmt.Errorf("could not enforce minimum resource limits: %v", err)
@@ -239,12 +249,13 @@ func (c *Cluster) Create() error {
}
if role == Master {
// replica endpoint will be created by the replica service. Master endpoint needs to be created by us,
// since the corresponding master service doesn't define any selectors.
// since the corresponding master service does not define any selectors.
ep, err = c.createEndpoint(role)
if err != nil {
return fmt.Errorf("could not create %s endpoint: %v", role, err)
}
c.logger.Infof("endpoint %q has been successfully created", util.NameFromMeta(ep.ObjectMeta))
c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Endpoints", "Endpoint %q has been successfully created", util.NameFromMeta(ep.ObjectMeta))
}
if c.Services[role] != nil {
@@ -255,6 +266,7 @@ func (c *Cluster) Create() error {
return fmt.Errorf("could not create %s service: %v", role, err)
}
c.logger.Infof("%s service %q has been successfully created", role, util.NameFromMeta(service.ObjectMeta))
c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Services", "The service %q for role %s has been successfully created", util.NameFromMeta(service.ObjectMeta), role)
}
if err = c.initUsers(); err != nil {
@@ -266,6 +278,7 @@ func (c *Cluster) Create() error {
return fmt.Errorf("could not create secrets: %v", err)
}
c.logger.Infof("secrets have been successfully created")
c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Secrets", "The secrets have been successfully created")
if c.PodDisruptionBudget != nil {
return fmt.Errorf("pod disruption budget already exists in the cluster")
@@ -284,6 +297,7 @@ func (c *Cluster) Create() error {
return fmt.Errorf("could not create statefulset: %v", err)
}
c.logger.Infof("statefulset %q has been successfully created", util.NameFromMeta(ss.ObjectMeta))
c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "StatefulSet", "Statefulset %q has been successfully created", util.NameFromMeta(ss.ObjectMeta))
c.logger.Info("waiting for the cluster being ready")
@@ -292,9 +306,12 @@ func (c *Cluster) Create() error {
return err
}
c.logger.Infof("pods are ready")
c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "StatefulSet", "Pods are ready")
// create database objects unless we are running without pods or disabled that feature explicitly
// create database objects unless we are running without pods or disabled
// that feature explicitly
if !(c.databaseAccessDisabled() || c.getNumberOfInstances(&c.Spec) <= 0 || c.Spec.StandbyCluster != nil) {
c.logger.Infof("Create roles")
if err = c.createRoles(); err != nil {
return fmt.Errorf("could not create users: %v", err)
}
@@ -303,6 +320,9 @@ func (c *Cluster) Create() error {
if err = c.syncDatabases(); err != nil {
return fmt.Errorf("could not sync databases: %v", err)
}
if err = c.syncPreparedDatabases(); err != nil {
return fmt.Errorf("could not sync prepared databases: %v", err)
}
c.logger.Infof("databases have been successfully created")
}
@@ -317,6 +337,14 @@ func (c *Cluster) Create() error {
c.logger.Errorf("could not list resources: %v", err)
}
// Create connection pooler deployment and services if necessary. Since we
// need to perform some operations with the database itself (e.g. install
// lookup function), do it as the last step, when everything is available.
//
// Do not consider connection pooler as a strict requirement, and if
// something fails, report warning
c.createConnectionPooler(c.installLookupFunction)
return nil
}
@@ -328,11 +356,11 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa
//TODO: improve me
if *c.Statefulset.Spec.Replicas != *statefulSet.Spec.Replicas {
match = false
reasons = append(reasons, "new statefulset's number of replicas doesn't match the current one")
reasons = append(reasons, "new statefulset's number of replicas does not match the current one")
}
if !reflect.DeepEqual(c.Statefulset.Annotations, statefulSet.Annotations) {
match = false
reasons = append(reasons, "new statefulset's annotations doesn't match the current one")
reasons = append(reasons, "new statefulset's annotations does not match the current one")
}
needsRollUpdate, reasons = c.compareContainers("initContainers", c.Statefulset.Spec.Template.Spec.InitContainers, statefulSet.Spec.Template.Spec.InitContainers, needsRollUpdate, reasons)
@@ -349,24 +377,24 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa
if c.Statefulset.Spec.Template.Spec.ServiceAccountName != statefulSet.Spec.Template.Spec.ServiceAccountName {
needsReplace = true
needsRollUpdate = true
reasons = append(reasons, "new statefulset's serviceAccountName service account name doesn't match the current one")
reasons = append(reasons, "new statefulset's serviceAccountName service account name does not match the current one")
}
if *c.Statefulset.Spec.Template.Spec.TerminationGracePeriodSeconds != *statefulSet.Spec.Template.Spec.TerminationGracePeriodSeconds {
needsReplace = true
needsRollUpdate = true
reasons = append(reasons, "new statefulset's terminationGracePeriodSeconds doesn't match the current one")
reasons = append(reasons, "new statefulset's terminationGracePeriodSeconds does not match the current one")
}
if !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.Affinity, statefulSet.Spec.Template.Spec.Affinity) {
needsReplace = true
needsRollUpdate = true
reasons = append(reasons, "new statefulset's pod affinity doesn't match the current one")
reasons = append(reasons, "new statefulset's pod affinity does not match the current one")
}
// Some generated fields like creationTimestamp make it not possible to use DeepCompare on Spec.Template.ObjectMeta
if !reflect.DeepEqual(c.Statefulset.Spec.Template.Labels, statefulSet.Spec.Template.Labels) {
needsReplace = true
needsRollUpdate = true
reasons = append(reasons, "new statefulset's metadata labels doesn't match the current one")
reasons = append(reasons, "new statefulset's metadata labels does not match the current one")
}
if (c.Statefulset.Spec.Selector != nil) && (statefulSet.Spec.Selector != nil) {
if !reflect.DeepEqual(c.Statefulset.Spec.Selector.MatchLabels, statefulSet.Spec.Selector.MatchLabels) {
@@ -377,7 +405,7 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa
return &compareStatefulsetResult{}
}
needsReplace = true
reasons = append(reasons, "new statefulset's selector doesn't match the current one")
reasons = append(reasons, "new statefulset's selector does not match the current one")
}
}
@@ -385,7 +413,13 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa
match = false
needsReplace = true
needsRollUpdate = true
reasons = append(reasons, "new statefulset's pod template metadata annotations doesn't match the current one")
reasons = append(reasons, "new statefulset's pod template metadata annotations does not match the current one")
}
if !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.SecurityContext, statefulSet.Spec.Template.Spec.SecurityContext) {
match = false
needsReplace = true
needsRollUpdate = true
reasons = append(reasons, "new statefulset's pod template security context in spec does not match the current one")
}
if len(c.Statefulset.Spec.VolumeClaimTemplates) != len(statefulSet.Spec.VolumeClaimTemplates) {
needsReplace = true
@@ -396,20 +430,36 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa
// Some generated fields like creationTimestamp make it not possible to use DeepCompare on ObjectMeta
if name != statefulSet.Spec.VolumeClaimTemplates[i].Name {
needsReplace = true
reasons = append(reasons, fmt.Sprintf("new statefulset's name for volume %d doesn't match the current one", i))
reasons = append(reasons, fmt.Sprintf("new statefulset's name for volume %d does not match the current one", i))
continue
}
if !reflect.DeepEqual(c.Statefulset.Spec.VolumeClaimTemplates[i].Annotations, statefulSet.Spec.VolumeClaimTemplates[i].Annotations) {
needsReplace = true
reasons = append(reasons, fmt.Sprintf("new statefulset's annotations for volume %q doesn't match the current one", name))
reasons = append(reasons, fmt.Sprintf("new statefulset's annotations for volume %q does not match the current one", name))
}
if !reflect.DeepEqual(c.Statefulset.Spec.VolumeClaimTemplates[i].Spec, statefulSet.Spec.VolumeClaimTemplates[i].Spec) {
name := c.Statefulset.Spec.VolumeClaimTemplates[i].Name
needsReplace = true
reasons = append(reasons, fmt.Sprintf("new statefulset's volumeClaimTemplates specification for volume %q doesn't match the current one", name))
reasons = append(reasons, fmt.Sprintf("new statefulset's volumeClaimTemplates specification for volume %q does not match the current one", name))
}
}
// we assume any change in priority happens by rolling out a new priority class
// changing the priority value in an existing class is not supproted
if c.Statefulset.Spec.Template.Spec.PriorityClassName != statefulSet.Spec.Template.Spec.PriorityClassName {
match = false
needsReplace = true
needsRollUpdate = true
reasons = append(reasons, "new statefulset's pod priority class in spec does not match the current one")
}
// lazy Spilo update: modify the image in the statefulset itself but let its pods run with the old image
// until they are re-created for other reasons, for example node rotation
if c.OpConfig.EnableLazySpiloUpgrade && !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.Containers[0].Image, statefulSet.Spec.Template.Spec.Containers[0].Image) {
needsReplace = true
reasons = append(reasons, "lazy Spilo update: new statefulset's pod image does not match the current one")
}
if needsRollUpdate || needsReplace {
match = false
}
@@ -439,18 +489,23 @@ func (c *Cluster) compareContainers(description string, setA, setB []v1.Containe
}
checks := []containerCheck{
newCheck("new statefulset %s's %s (index %d) name doesn't match the current one",
newCheck("new statefulset %s's %s (index %d) name does not match the current one",
func(a, b v1.Container) bool { return a.Name != b.Name }),
newCheck("new statefulset %s's %s (index %d) image doesn't match the current one",
func(a, b v1.Container) bool { return a.Image != b.Image }),
newCheck("new statefulset %s's %s (index %d) ports don't match the current one",
newCheck("new statefulset %s's %s (index %d) ports do not match the current one",
func(a, b v1.Container) bool { return !reflect.DeepEqual(a.Ports, b.Ports) }),
newCheck("new statefulset %s's %s (index %d) resources don't match the current ones",
newCheck("new statefulset %s's %s (index %d) resources do not match the current ones",
func(a, b v1.Container) bool { return !compareResources(&a.Resources, &b.Resources) }),
newCheck("new statefulset %s's %s (index %d) environment doesn't match the current one",
newCheck("new statefulset %s's %s (index %d) environment does not match the current one",
func(a, b v1.Container) bool { return !reflect.DeepEqual(a.Env, b.Env) }),
newCheck("new statefulset %s's %s (index %d) environment sources don't match the current one",
newCheck("new statefulset %s's %s (index %d) environment sources do not match the current one",
func(a, b v1.Container) bool { return !reflect.DeepEqual(a.EnvFrom, b.EnvFrom) }),
newCheck("new statefulset %s's %s (index %d) security context does not match the current one",
func(a, b v1.Container) bool { return !reflect.DeepEqual(a.SecurityContext, b.SecurityContext) }),
}
if !c.OpConfig.EnableLazySpiloUpgrade {
checks = append(checks, newCheck("new statefulset %s's %s (index %d) image does not match the current one",
func(a, b v1.Container) bool { return a.Image != b.Image }))
}
for index, containerA := range setA {
@@ -514,7 +569,8 @@ func (c *Cluster) enforceMinResourceLimits(spec *acidv1.PostgresSpec) error {
return fmt.Errorf("could not compare defined CPU limit %s with configured minimum value %s: %v", cpuLimit, minCPULimit, err)
}
if isSmaller {
c.logger.Warningf("defined CPU limit %s is below required minimum %s and will be set to it", cpuLimit, minCPULimit)
c.logger.Warningf("defined CPU limit %s is below required minimum %s and will be increased", cpuLimit, minCPULimit)
c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", "defined CPU limit %s is below required minimum %s and will be set to it", cpuLimit, minCPULimit)
spec.Resources.ResourceLimits.CPU = minCPULimit
}
}
@@ -526,7 +582,8 @@ func (c *Cluster) enforceMinResourceLimits(spec *acidv1.PostgresSpec) error {
return fmt.Errorf("could not compare defined memory limit %s with configured minimum value %s: %v", memoryLimit, minMemoryLimit, err)
}
if isSmaller {
c.logger.Warningf("defined memory limit %s is below required minimum %s and will be set to it", memoryLimit, minMemoryLimit)
c.logger.Warningf("defined memory limit %s is below required minimum %s and will be increased", memoryLimit, minMemoryLimit)
c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", "defined memory limit %s is below required minimum %s and will be set to it", memoryLimit, minMemoryLimit)
spec.Resources.ResourceLimits.Memory = minMemoryLimit
}
}
@@ -540,38 +597,53 @@ func (c *Cluster) enforceMinResourceLimits(spec *acidv1.PostgresSpec) error {
// for a cluster that had no such job before. In this case a missing job is not an error.
func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
updateFailed := false
syncStatetfulSet := false
c.mu.Lock()
defer c.mu.Unlock()
c.setStatus(acidv1.ClusterStatusUpdating)
c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusUpdating)
c.setSpec(newSpec)
defer func() {
if updateFailed {
c.setStatus(acidv1.ClusterStatusUpdateFailed)
c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusUpdateFailed)
} else {
c.setStatus(acidv1.ClusterStatusRunning)
c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusRunning)
}
}()
if oldSpec.Spec.PgVersion != newSpec.Spec.PgVersion { // PG versions comparison
c.logger.Warningf("postgresql version change(%q -> %q) has no effect", oldSpec.Spec.PgVersion, newSpec.Spec.PgVersion)
//we need that hack to generate statefulset with the old version
newSpec.Spec.PgVersion = oldSpec.Spec.PgVersion
logNiceDiff(c.logger, oldSpec, newSpec)
if oldSpec.Spec.PostgresqlParam.PgVersion > newSpec.Spec.PostgresqlParam.PgVersion {
c.logger.Warningf("postgresql version change(%q -> %q) has no effect",
oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion)
c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "PostgreSQL", "postgresql version change(%q -> %q) has no effect",
oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion)
// we need that hack to generate statefulset with the old version
newSpec.Spec.PostgresqlParam.PgVersion = oldSpec.Spec.PostgresqlParam.PgVersion
} else if oldSpec.Spec.PostgresqlParam.PgVersion < newSpec.Spec.PostgresqlParam.PgVersion {
c.logger.Infof("postgresql version increased (%q -> %q), major version upgrade can be done manually after StatefulSet Sync",
oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion)
syncStatetfulSet = true
}
// Service
if !reflect.DeepEqual(c.generateService(Master, &oldSpec.Spec), c.generateService(Master, &newSpec.Spec)) ||
!reflect.DeepEqual(c.generateService(Replica, &oldSpec.Spec), c.generateService(Replica, &newSpec.Spec)) {
c.logger.Debugf("syncing services")
if err := c.syncServices(); err != nil {
c.logger.Errorf("could not sync services: %v", err)
updateFailed = true
}
}
if !reflect.DeepEqual(oldSpec.Spec.Users, newSpec.Spec.Users) {
// connection pooler needs one system user created, which is done in
// initUsers. Check if it needs to be called.
sameUsers := reflect.DeepEqual(oldSpec.Spec.Users, newSpec.Spec.Users) &&
reflect.DeepEqual(oldSpec.Spec.PreparedDatabases, newSpec.Spec.PreparedDatabases)
needConnectionPooler := needMasterConnectionPoolerWorker(&newSpec.Spec) ||
needReplicaConnectionPoolerWorker(&newSpec.Spec)
if !sameUsers || needConnectionPooler {
c.logger.Debugf("syncing secrets")
if err := c.initUsers(); err != nil {
c.logger.Errorf("could not init users: %v", err)
@@ -588,14 +660,10 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
}
// Volume
if oldSpec.Spec.Size != newSpec.Spec.Size {
c.logger.Debugf("syncing persistent volumes")
c.logVolumeChanges(oldSpec.Spec.Volume, newSpec.Spec.Volume)
if err := c.syncVolumes(); err != nil {
c.logger.Errorf("could not sync persistent volumes: %v", err)
updateFailed = true
}
if c.OpConfig.StorageResizeMode != "off" {
c.syncVolumes()
} else {
c.logger.Infof("Storage resize is disabled (storage_resize_mode is off). Skipping volume sync.")
}
// Statefulset
@@ -622,9 +690,9 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
updateFailed = true
return
}
if !reflect.DeepEqual(oldSs, newSs) {
if syncStatetfulSet || !reflect.DeepEqual(oldSs, newSs) || !reflect.DeepEqual(oldSpec.Annotations, newSpec.Annotations) {
c.logger.Debugf("syncing statefulsets")
syncStatetfulSet = false
// TODO: avoid generating the StatefulSet object twice by passing it to syncStatefulSet
if err := c.syncStatefulSet(); err != nil {
c.logger.Errorf("could not sync statefulsets: %v", err)
@@ -686,18 +754,51 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
c.logger.Errorf("could not sync roles: %v", err)
updateFailed = true
}
if !reflect.DeepEqual(oldSpec.Spec.Databases, newSpec.Spec.Databases) {
if !reflect.DeepEqual(oldSpec.Spec.Databases, newSpec.Spec.Databases) ||
!reflect.DeepEqual(oldSpec.Spec.PreparedDatabases, newSpec.Spec.PreparedDatabases) {
c.logger.Infof("syncing databases")
if err := c.syncDatabases(); err != nil {
c.logger.Errorf("could not sync databases: %v", err)
updateFailed = true
}
}
if !reflect.DeepEqual(oldSpec.Spec.PreparedDatabases, newSpec.Spec.PreparedDatabases) {
c.logger.Infof("syncing prepared databases")
if err := c.syncPreparedDatabases(); err != nil {
c.logger.Errorf("could not sync prepared databases: %v", err)
updateFailed = true
}
}
}
// Sync connection pooler. Before actually doing sync reset lookup
// installation flag, since manifest updates could add another db which we
// need to process. In the future we may want to do this more careful and
// check which databases we need to process, but even repeating the whole
// installation process should be good enough.
if _, err := c.syncConnectionPooler(oldSpec, newSpec, c.installLookupFunction); err != nil {
c.logger.Errorf("could not sync connection pooler: %v", err)
updateFailed = true
}
return nil
}
func syncResources(a, b *v1.ResourceRequirements) bool {
for _, res := range []v1.ResourceName{
v1.ResourceCPU,
v1.ResourceMemory,
} {
if !a.Limits[res].Equal(b.Limits[res]) ||
!a.Requests[res].Equal(b.Requests[res]) {
return true
}
}
return false
}
// Delete deletes the cluster and cleans up all objects associated with it (including statefulsets).
// The deletion order here is somewhat significant, because Patroni, when running with the Kubernetes
// DCS, reuses the master's endpoint to store the leader related metadata. If we remove the endpoint
@@ -706,6 +807,7 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
func (c *Cluster) Delete() {
c.mu.Lock()
defer c.mu.Unlock()
c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Delete", "Started deletion of new cluster resources")
// delete the backup job before the stateful set of the cluster to prevent connections to non-existing pods
// deleting the cron job also removes pods and batch jobs it created
@@ -717,14 +819,8 @@ func (c *Cluster) Delete() {
c.logger.Warningf("could not delete statefulset: %v", err)
}
for _, obj := range c.Secrets {
if doDelete, user := c.shouldDeleteSecret(obj); !doDelete {
c.logger.Warningf("not removing secret %q for the system user %q", obj.GetName(), user)
continue
}
if err := c.deleteSecret(obj); err != nil {
c.logger.Warningf("could not delete secret: %v", err)
}
if err := c.deleteSecrets(); err != nil {
c.logger.Warningf("could not delete secrets: %v", err)
}
if err := c.deletePodDisruptionBudget(); err != nil {
@@ -733,8 +829,10 @@ func (c *Cluster) Delete() {
for _, role := range []PostgresRole{Master, Replica} {
if err := c.deleteEndpoint(role); err != nil {
c.logger.Warningf("could not delete %s endpoint: %v", role, err)
if !c.patroniKubernetesUseConfigMaps() {
if err := c.deleteEndpoint(role); err != nil {
c.logger.Warningf("could not delete %s endpoint: %v", role, err)
}
}
if err := c.deleteService(role); err != nil {
@@ -746,6 +844,15 @@ func (c *Cluster) Delete() {
c.logger.Warningf("could not remove leftover patroni objects; %v", err)
}
// Delete connection pooler objects anyway, even if it's not mentioned in the
// manifest, just to not keep orphaned components in case if something went
// wrong
for _, role := range [2]PostgresRole{Master, Replica} {
if err := c.deleteConnectionPooler(role); err != nil {
c.logger.Warningf("could not remove connection pooler: %v", err)
}
}
}
//NeedsRepair returns true if the cluster should be included in the repair scan (based on its in-memory status).
@@ -812,6 +919,151 @@ func (c *Cluster) initSystemUsers() {
Name: c.OpConfig.ReplicationUsername,
Password: util.RandomPassword(constants.PasswordLength),
}
// Connection pooler user is an exception, if requested it's going to be
// created by operator as a normal pgUser
if needConnectionPooler(&c.Spec) {
// initialize empty connection pooler if not done yet
if c.Spec.ConnectionPooler == nil {
c.Spec.ConnectionPooler = &acidv1.ConnectionPooler{}
}
// Using superuser as pooler user is not a good idea. First of all it's
// not going to be synced correctly with the current implementation,
// and second it's a bad practice.
username := c.OpConfig.ConnectionPooler.User
isSuperUser := c.Spec.ConnectionPooler.User == c.OpConfig.SuperUsername
isProtectedUser := c.shouldAvoidProtectedOrSystemRole(
c.Spec.ConnectionPooler.User, "connection pool role")
if !isSuperUser && !isProtectedUser {
username = util.Coalesce(
c.Spec.ConnectionPooler.User,
c.OpConfig.ConnectionPooler.User)
}
// connection pooler application should be able to login with this role
connectionPoolerUser := spec.PgUser{
Origin: spec.RoleConnectionPooler,
Name: username,
Flags: []string{constants.RoleFlagLogin},
Password: util.RandomPassword(constants.PasswordLength),
}
if _, exists := c.pgUsers[username]; !exists {
c.pgUsers[username] = connectionPoolerUser
}
if _, exists := c.systemUsers[constants.ConnectionPoolerUserKeyName]; !exists {
c.systemUsers[constants.ConnectionPoolerUserKeyName] = connectionPoolerUser
}
}
}
func (c *Cluster) initPreparedDatabaseRoles() error {
if c.Spec.PreparedDatabases != nil && len(c.Spec.PreparedDatabases) == 0 { // TODO: add option to disable creating such a default DB
c.Spec.PreparedDatabases = map[string]acidv1.PreparedDatabase{strings.Replace(c.Name, "-", "_", -1): {}}
}
// create maps with default roles/users as keys and their membership as values
defaultRoles := map[string]string{
constants.OwnerRoleNameSuffix: "",
constants.ReaderRoleNameSuffix: "",
constants.WriterRoleNameSuffix: constants.ReaderRoleNameSuffix,
}
defaultUsers := map[string]string{
constants.OwnerRoleNameSuffix + constants.UserRoleNameSuffix: constants.OwnerRoleNameSuffix,
constants.ReaderRoleNameSuffix + constants.UserRoleNameSuffix: constants.ReaderRoleNameSuffix,
constants.WriterRoleNameSuffix + constants.UserRoleNameSuffix: constants.WriterRoleNameSuffix,
}
for preparedDbName, preparedDB := range c.Spec.PreparedDatabases {
// get list of prepared schemas to set in search_path
preparedSchemas := preparedDB.PreparedSchemas
if len(preparedDB.PreparedSchemas) == 0 {
preparedSchemas = map[string]acidv1.PreparedSchema{"data": {DefaultRoles: util.True()}}
}
var searchPath strings.Builder
searchPath.WriteString(constants.DefaultSearchPath)
for preparedSchemaName := range preparedSchemas {
searchPath.WriteString(", " + preparedSchemaName)
}
// default roles per database
if err := c.initDefaultRoles(defaultRoles, "admin", preparedDbName, searchPath.String()); err != nil {
return fmt.Errorf("could not initialize default roles for database %s: %v", preparedDbName, err)
}
if preparedDB.DefaultUsers {
if err := c.initDefaultRoles(defaultUsers, "admin", preparedDbName, searchPath.String()); err != nil {
return fmt.Errorf("could not initialize default roles for database %s: %v", preparedDbName, err)
}
}
// default roles per database schema
for preparedSchemaName, preparedSchema := range preparedSchemas {
if preparedSchema.DefaultRoles == nil || *preparedSchema.DefaultRoles {
if err := c.initDefaultRoles(defaultRoles,
preparedDbName+constants.OwnerRoleNameSuffix,
preparedDbName+"_"+preparedSchemaName,
constants.DefaultSearchPath+", "+preparedSchemaName); err != nil {
return fmt.Errorf("could not initialize default roles for database schema %s: %v", preparedSchemaName, err)
}
if preparedSchema.DefaultUsers {
if err := c.initDefaultRoles(defaultUsers,
preparedDbName+constants.OwnerRoleNameSuffix,
preparedDbName+"_"+preparedSchemaName,
constants.DefaultSearchPath+", "+preparedSchemaName); err != nil {
return fmt.Errorf("could not initialize default users for database schema %s: %v", preparedSchemaName, err)
}
}
}
}
}
return nil
}
func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix string, searchPath string) error {
for defaultRole, inherits := range defaultRoles {
roleName := prefix + defaultRole
flags := []string{constants.RoleFlagNoLogin}
if defaultRole[len(defaultRole)-5:] == constants.UserRoleNameSuffix {
flags = []string{constants.RoleFlagLogin}
}
memberOf := make([]string, 0)
if inherits != "" {
memberOf = append(memberOf, prefix+inherits)
}
adminRole := ""
if strings.Contains(defaultRole, constants.OwnerRoleNameSuffix) {
adminRole = admin
} else {
adminRole = prefix + constants.OwnerRoleNameSuffix
}
newRole := spec.PgUser{
Origin: spec.RoleOriginBootstrap,
Name: roleName,
Password: util.RandomPassword(constants.PasswordLength),
Flags: flags,
MemberOf: memberOf,
Parameters: map[string]string{"search_path": searchPath},
AdminRole: adminRole,
}
if currentRole, present := c.pgUsers[roleName]; present {
c.pgUsers[roleName] = c.resolveNameConflict(&currentRole, &newRole)
} else {
c.pgUsers[roleName] = newRole
}
}
return nil
}
func (c *Cluster) initRobotUsers() error {
@@ -861,7 +1113,7 @@ func (c *Cluster) initTeamMembers(teamID string, isPostgresSuperuserTeam bool) e
if c.shouldAvoidProtectedOrSystemRole(username, "API role") {
continue
}
if c.OpConfig.EnableTeamSuperuser || isPostgresSuperuserTeam {
if (c.OpConfig.EnableTeamSuperuser && teamID == c.Spec.TeamID) || isPostgresSuperuserTeam {
flags = append(flags, constants.RoleFlagSuperuser)
} else {
if c.OpConfig.TeamAdminRole != "" {
@@ -890,17 +1142,38 @@ func (c *Cluster) initTeamMembers(teamID string, isPostgresSuperuserTeam bool) e
func (c *Cluster) initHumanUsers() error {
var clusterIsOwnedBySuperuserTeam bool
superuserTeams := []string{}
if c.OpConfig.EnablePostgresTeamCRDSuperusers {
superuserTeams = c.PgTeamMap.GetAdditionalSuperuserTeams(c.Spec.TeamID, true)
}
for _, postgresSuperuserTeam := range c.OpConfig.PostgresSuperuserTeams {
err := c.initTeamMembers(postgresSuperuserTeam, true)
if err != nil {
return fmt.Errorf("Cannot create a team %q of Postgres superusers: %v", postgresSuperuserTeam, err)
if !(util.SliceContains(superuserTeams, postgresSuperuserTeam)) {
superuserTeams = append(superuserTeams, postgresSuperuserTeam)
}
if postgresSuperuserTeam == c.Spec.TeamID {
}
for _, superuserTeam := range superuserTeams {
err := c.initTeamMembers(superuserTeam, true)
if err != nil {
return fmt.Errorf("Cannot initialize members for team %q of Postgres superusers: %v", superuserTeam, err)
}
if superuserTeam == c.Spec.TeamID {
clusterIsOwnedBySuperuserTeam = true
}
}
additionalTeams := c.PgTeamMap.GetAdditionalTeams(c.Spec.TeamID, true)
for _, additionalTeam := range additionalTeams {
if !(util.SliceContains(superuserTeams, additionalTeam)) {
err := c.initTeamMembers(additionalTeam, false)
if err != nil {
return fmt.Errorf("Cannot initialize members for additional team %q for cluster owned by %q: %v", additionalTeam, c.Spec.TeamID, err)
}
}
}
if clusterIsOwnedBySuperuserTeam {
c.logger.Infof("Team %q owning the cluster is also a team of superusers. Created superuser roles for its members instead of admin roles.", c.Spec.TeamID)
return nil
@@ -908,7 +1181,7 @@ func (c *Cluster) initHumanUsers() error {
err := c.initTeamMembers(c.Spec.TeamID, false)
if err != nil {
return fmt.Errorf("Cannot create a team %q of admins owning the PG cluster: %v", c.Spec.TeamID, err)
return fmt.Errorf("Cannot initialize members for team %q who owns the Postgres cluster: %v", c.Spec.TeamID, err)
}
return nil
@@ -996,6 +1269,7 @@ func (c *Cluster) Switchover(curMaster *v1.Pod, candidate spec.NamespacedName) e
var err error
c.logger.Debugf("switching over from %q to %q", curMaster.Name, candidate)
c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Switchover", "Switching over from %q to %q", curMaster.Name, candidate)
var wg sync.WaitGroup
@@ -1022,6 +1296,7 @@ func (c *Cluster) Switchover(curMaster *v1.Pod, candidate spec.NamespacedName) e
if err = c.patroni.Switchover(curMaster, candidate.Name); err == nil {
c.logger.Debugf("successfully switched over from %q to %q", curMaster.Name, candidate)
c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Switchover", "Successfully switched over from %q to %q", curMaster.Name, candidate)
if err = <-podLabelErr; err != nil {
err = fmt.Errorf("could not get master pod label: %v", err)
}
@@ -1037,6 +1312,7 @@ func (c *Cluster) Switchover(curMaster *v1.Pod, candidate spec.NamespacedName) e
// close the label waiting channel no sooner than the waiting goroutine terminates.
close(podLabelErr)
c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Switchover", "Switchover from %q to %q FAILED: %v", curMaster.Name, candidate, err)
return err
}
@@ -1051,11 +1327,6 @@ func (c *Cluster) Unlock() {
c.mu.Unlock()
}
func (c *Cluster) shouldDeleteSecret(secret *v1.Secret) (delete bool, userName string) {
secretUser := string(secret.Data["username"])
return (secretUser != c.OpConfig.ReplicationUsername && secretUser != c.OpConfig.SuperUsername), secretUser
}
type simpleActionWithResult func() error
type clusterObjectGet func(name string) (spec.NamespacedName, error)
@@ -1064,11 +1335,19 @@ type clusterObjectDelete func(name string) error
func (c *Cluster) deletePatroniClusterObjects() error {
// TODO: figure out how to remove leftover patroni objects in other cases
var actionsList []simpleActionWithResult
if !c.patroniUsesKubernetes() {
c.logger.Infof("not cleaning up Etcd Patroni objects on cluster delete")
}
c.logger.Debugf("removing leftover Patroni objects (endpoints, services and configmaps)")
for _, deleter := range []simpleActionWithResult{c.deletePatroniClusterEndpoints, c.deletePatroniClusterServices, c.deletePatroniClusterConfigMaps} {
if !c.patroniKubernetesUseConfigMaps() {
actionsList = append(actionsList, c.deletePatroniClusterEndpoints)
}
actionsList = append(actionsList, c.deletePatroniClusterServices, c.deletePatroniClusterConfigMaps)
c.logger.Debugf("removing leftover Patroni objects (endpoints / services and configmaps)")
for _, deleter := range actionsList {
if err := deleter(); err != nil {
return err
}
@@ -1088,7 +1367,7 @@ func (c *Cluster) deleteClusterObject(
objType, namespacedName)
if err = del(name); err != nil {
return fmt.Errorf("could not Patroni delete cluster object %q with name %q: %v",
return fmt.Errorf("could not delete Patroni cluster object %q with name %q: %v",
objType, namespacedName, err)
}
@@ -1102,12 +1381,12 @@ func (c *Cluster) deleteClusterObject(
func (c *Cluster) deletePatroniClusterServices() error {
get := func(name string) (spec.NamespacedName, error) {
svc, err := c.KubeClient.Services(c.Namespace).Get(name, metav1.GetOptions{})
svc, err := c.KubeClient.Services(c.Namespace).Get(context.TODO(), name, metav1.GetOptions{})
return util.NameFromMeta(svc.ObjectMeta), err
}
deleteServiceFn := func(name string) error {
return c.KubeClient.Services(c.Namespace).Delete(name, c.deleteOptions)
return c.KubeClient.Services(c.Namespace).Delete(context.TODO(), name, c.deleteOptions)
}
return c.deleteClusterObject(get, deleteServiceFn, "service")
@@ -1115,12 +1394,12 @@ func (c *Cluster) deletePatroniClusterServices() error {
func (c *Cluster) deletePatroniClusterEndpoints() error {
get := func(name string) (spec.NamespacedName, error) {
ep, err := c.KubeClient.Endpoints(c.Namespace).Get(name, metav1.GetOptions{})
ep, err := c.KubeClient.Endpoints(c.Namespace).Get(context.TODO(), name, metav1.GetOptions{})
return util.NameFromMeta(ep.ObjectMeta), err
}
deleteEndpointFn := func(name string) error {
return c.KubeClient.Endpoints(c.Namespace).Delete(name, c.deleteOptions)
return c.KubeClient.Endpoints(c.Namespace).Delete(context.TODO(), name, c.deleteOptions)
}
return c.deleteClusterObject(get, deleteEndpointFn, "endpoint")
@@ -1128,12 +1407,12 @@ func (c *Cluster) deletePatroniClusterEndpoints() error {
func (c *Cluster) deletePatroniClusterConfigMaps() error {
get := func(name string) (spec.NamespacedName, error) {
cm, err := c.KubeClient.ConfigMaps(c.Namespace).Get(name, metav1.GetOptions{})
cm, err := c.KubeClient.ConfigMaps(c.Namespace).Get(context.TODO(), name, metav1.GetOptions{})
return util.NameFromMeta(cm.ObjectMeta), err
}
deleteConfigMapFn := func(name string) error {
return c.KubeClient.ConfigMaps(c.Namespace).Delete(name, c.deleteOptions)
return c.KubeClient.ConfigMaps(c.Namespace).Delete(context.TODO(), name, c.deleteOptions)
}
return c.deleteClusterObject(get, deleteConfigMapFn, "configmap")
pkg/cluster/cluster_test.go
+174 -33
View File
@@ -9,9 +9,11 @@ import (
acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
"github.com/zalando/postgres-operator/pkg/spec"
"github.com/zalando/postgres-operator/pkg/util/config"
"github.com/zalando/postgres-operator/pkg/util/constants"
"github.com/zalando/postgres-operator/pkg/util/k8sutil"
"github.com/zalando/postgres-operator/pkg/util/teams"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/record"
)
const (
@@ -20,21 +22,53 @@ const (
)
var logger = logrus.New().WithField("test", "cluster")
var eventRecorder = record.NewFakeRecorder(1)
var cl = New(
Config{
OpConfig: config.Config{
ProtectedRoles: []string{"admin"},
PodManagementPolicy: "ordered_ready",
ProtectedRoles: []string{"admin"},
Auth: config.Auth{
SuperUsername: superUserName,
ReplicationUsername: replicationUserName,
},
Resources: config.Resources{
DownscalerAnnotations: []string{"downscaler/*"},
},
},
},
k8sutil.NewMockKubernetesClient(),
acidv1.Postgresql{},
acidv1.Postgresql{ObjectMeta: metav1.ObjectMeta{Name: "acid-test", Namespace: "test", Annotations: map[string]string{"downscaler/downtime_replicas": "0"}}},
logger,
eventRecorder,
)
func TestStatefulSetAnnotations(t *testing.T) {
testName := "CheckStatefulsetAnnotations"
spec := acidv1.PostgresSpec{
TeamID: "myapp", NumberOfInstances: 1,
Resources: acidv1.Resources{
ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"},
ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"},
},
Volume: acidv1.Volume{
Size: "1G",
},
}
ss, err := cl.generateStatefulSet(&spec)
if err != nil {
t.Errorf("in %s no statefulset created %v", testName, err)
}
if ss != nil {
annotation := ss.ObjectMeta.GetAnnotations()
if _, ok := annotation["downscaler/downtime_replicas"]; !ok {
t.Errorf("in %s respective annotation not found on sts", testName)
}
}
}
func TestInitRobotUsers(t *testing.T) {
testName := "TestInitRobotUsers"
tests := []struct {
@@ -299,36 +333,6 @@ func TestInitHumanUsersWithSuperuserTeams(t *testing.T) {
}
}
func TestShouldDeleteSecret(t *testing.T) {
testName := "TestShouldDeleteSecret"
tests := []struct {
secret *v1.Secret
outcome bool
}{
{
secret: &v1.Secret{Data: map[string][]byte{"username": []byte("foobar")}},
outcome: true,
},
{
secret: &v1.Secret{Data: map[string][]byte{"username": []byte(superUserName)}},
outcome: false,
},
{
secret: &v1.Secret{Data: map[string][]byte{"username": []byte(replicationUserName)}},
outcome: false,
},
}
for _, tt := range tests {
if outcome, username := cl.shouldDeleteSecret(tt.secret); outcome != tt.outcome {
t.Errorf("%s expects the check for deletion of the username %q secret to return %t, got %t",
testName, username, tt.outcome, outcome)
}
}
}
func TestPodAnnotations(t *testing.T) {
testName := "TestPodAnnotations"
tests := []struct {
@@ -704,3 +708,140 @@ func TestServiceAnnotations(t *testing.T) {
})
}
}
func TestInitSystemUsers(t *testing.T) {
testName := "Test system users initialization"
// default cluster without connection pooler
cl.initSystemUsers()
if _, exist := cl.systemUsers[constants.ConnectionPoolerUserKeyName]; exist {
t.Errorf("%s, connection pooler user is present", testName)
}
// cluster with connection pooler
cl.Spec.EnableConnectionPooler = boolToPointer(true)
cl.initSystemUsers()
if _, exist := cl.systemUsers[constants.ConnectionPoolerUserKeyName]; !exist {
t.Errorf("%s, connection pooler user is not present", testName)
}
// superuser is not allowed as connection pool user
cl.Spec.ConnectionPooler = &acidv1.ConnectionPooler{
User: "postgres",
}
cl.OpConfig.SuperUsername = "postgres"
cl.OpConfig.ConnectionPooler.User = "pooler"
cl.initSystemUsers()
if _, exist := cl.pgUsers["pooler"]; !exist {
t.Errorf("%s, Superuser is not allowed to be a connection pool user", testName)
}
// neither protected users are
delete(cl.pgUsers, "pooler")
cl.Spec.ConnectionPooler = &acidv1.ConnectionPooler{
User: "admin",
}
cl.OpConfig.ProtectedRoles = []string{"admin"}
cl.initSystemUsers()
if _, exist := cl.pgUsers["pooler"]; !exist {
t.Errorf("%s, Protected user are not allowed to be a connection pool user", testName)
}
delete(cl.pgUsers, "pooler")
cl.Spec.ConnectionPooler = &acidv1.ConnectionPooler{
User: "standby",
}
cl.initSystemUsers()
if _, exist := cl.pgUsers["pooler"]; !exist {
t.Errorf("%s, System users are not allowed to be a connection pool user", testName)
}
}
func TestPreparedDatabases(t *testing.T) {
testName := "TestDefaultPreparedDatabase"
cl.Spec.PreparedDatabases = map[string]acidv1.PreparedDatabase{}
cl.initPreparedDatabaseRoles()
for _, role := range []string{"acid_test_owner", "acid_test_reader", "acid_test_writer",
"acid_test_data_owner", "acid_test_data_reader", "acid_test_data_writer"} {
if _, exist := cl.pgUsers[role]; !exist {
t.Errorf("%s, default role %q for prepared database not present", testName, role)
}
}
testName = "TestPreparedDatabaseWithSchema"
cl.Spec.PreparedDatabases = map[string]acidv1.PreparedDatabase{
"foo": {
DefaultUsers: true,
PreparedSchemas: map[string]acidv1.PreparedSchema{
"bar": {
DefaultUsers: true,
},
},
},
}
cl.initPreparedDatabaseRoles()
for _, role := range []string{
"foo_owner", "foo_reader", "foo_writer",
"foo_owner_user", "foo_reader_user", "foo_writer_user",
"foo_bar_owner", "foo_bar_reader", "foo_bar_writer",
"foo_bar_owner_user", "foo_bar_reader_user", "foo_bar_writer_user"} {
if _, exist := cl.pgUsers[role]; !exist {
t.Errorf("%s, default role %q for prepared database not present", testName, role)
}
}
roleTests := []struct {
subTest string
role string
memberOf string
admin string
}{
{
subTest: "Test admin role of owner",
role: "foo_owner",
memberOf: "",
admin: "admin",
},
{
subTest: "Test writer is a member of reader",
role: "foo_writer",
memberOf: "foo_reader",
admin: "foo_owner",
},
{
subTest: "Test reader LOGIN role",
role: "foo_reader_user",
memberOf: "foo_reader",
admin: "foo_owner",
},
{
subTest: "Test schema owner",
role: "foo_bar_owner",
memberOf: "",
admin: "foo_owner",
},
{
subTest: "Test schema writer LOGIN role",
role: "foo_bar_writer_user",
memberOf: "foo_bar_writer",
admin: "foo_bar_owner",
},
}
for _, tt := range roleTests {
user := cl.pgUsers[tt.role]
if (tt.memberOf == "" && len(user.MemberOf) > 0) || (tt.memberOf != "" && user.MemberOf[0] != tt.memberOf) {
t.Errorf("%s, incorrect membership for default role %q. Expected %q, got %q", tt.subTest, tt.role, tt.memberOf, user.MemberOf[0])
}
if user.AdminRole != tt.admin {
t.Errorf("%s, incorrect admin role for default role %q. Expected %q, got %q", tt.subTest, tt.role, tt.admin, user.AdminRole)
}
}
}
pkg/cluster/connection_pooler.go
+939
View File
@@ -0,0 +1,939 @@
package cluster
import (
"context"
"fmt"
"strings"
"github.com/r3labs/diff"
"github.com/sirupsen/logrus"
acidzalando "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do"
acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"github.com/zalando/postgres-operator/pkg/util"
"github.com/zalando/postgres-operator/pkg/util/config"
"github.com/zalando/postgres-operator/pkg/util/constants"
"github.com/zalando/postgres-operator/pkg/util/k8sutil"
)
// ConnectionPoolerObjects K8s objects that are belong to connection pooler
type ConnectionPoolerObjects struct {
Deployment *appsv1.Deployment
Service *v1.Service
Name string
ClusterName string
Namespace string
Role PostgresRole
// It could happen that a connection pooler was enabled, but the operator
// was not able to properly process a corresponding event or was restarted.
// In this case we will miss missing/require situation and a lookup function
// will not be installed. To avoid synchronizing it all the time to prevent
// this, we can remember the result in memory at least until the next
// restart.
LookupFunction bool
// Careful with referencing cluster.spec this object pointer changes
// during runtime and lifetime of cluster
}
func (c *Cluster) connectionPoolerName(role PostgresRole) string {
name := c.Name + "-pooler"
if role == Replica {
name = name + "-repl"
}
return name
}
// isConnectionPoolerEnabled
func needConnectionPooler(spec *acidv1.PostgresSpec) bool {
return needMasterConnectionPoolerWorker(spec) ||
needReplicaConnectionPoolerWorker(spec)
}
func needMasterConnectionPooler(spec *acidv1.PostgresSpec) bool {
return needMasterConnectionPoolerWorker(spec)
}
func needMasterConnectionPoolerWorker(spec *acidv1.PostgresSpec) bool {
return (nil != spec.EnableConnectionPooler && *spec.EnableConnectionPooler) ||
(spec.ConnectionPooler != nil && spec.EnableConnectionPooler == nil)
}
func needReplicaConnectionPooler(spec *acidv1.PostgresSpec) bool {
return needReplicaConnectionPoolerWorker(spec)
}
func needReplicaConnectionPoolerWorker(spec *acidv1.PostgresSpec) bool {
return spec.EnableReplicaConnectionPooler != nil &&
*spec.EnableReplicaConnectionPooler
}
// Return connection pooler labels selector, which should from one point of view
// inherit most of the labels from the cluster itself, but at the same time
// have e.g. different `application` label, so that recreatePod operation will
// not interfere with it (it lists all the pods via labels, and if there would
// be no difference, it will recreate also pooler pods).
func (c *Cluster) connectionPoolerLabels(role PostgresRole, addExtraLabels bool) *metav1.LabelSelector {
poolerLabels := c.labelsSet(addExtraLabels)
// TODO should be config values
poolerLabels["application"] = "db-connection-pooler"
poolerLabels["connection-pooler"] = c.connectionPoolerName(role)
if addExtraLabels {
extraLabels := map[string]string{}
extraLabels[c.OpConfig.PodRoleLabel] = string(role)
poolerLabels = labels.Merge(poolerLabels, extraLabels)
}
return &metav1.LabelSelector{
MatchLabels: poolerLabels,
MatchExpressions: nil,
}
}
// Prepare the database for connection pooler to be used, i.e. install lookup
// function (do it first, because it should be fast and if it didn't succeed,
// it doesn't makes sense to create more K8S objects. At this moment we assume
// that necessary connection pooler user exists.
//
// After that create all the objects for connection pooler, namely a deployment
// with a chosen pooler and a service to expose it.
// have connectionpooler name in the cp object to have it immutable name
// add these cp related functions to a new cp file
// opConfig, cluster, and database name
func (c *Cluster) createConnectionPooler(LookupFunction InstallFunction) (SyncReason, error) {
var reason SyncReason
c.setProcessName("creating connection pooler")
//this is essentially sync with nil as oldSpec
if reason, err := c.syncConnectionPooler(nil, &c.Postgresql, LookupFunction); err != nil {
return reason, err
}
return reason, nil
}
//
// Generate pool size related environment variables.
//
// MAX_DB_CONN would specify the global maximum for connections to a target
// database.
//
// MAX_CLIENT_CONN is not configurable at the moment, just set it high enough.
//
// DEFAULT_SIZE is a pool size per db/user (having in mind the use case when
// most of the queries coming through a connection pooler are from the same
// user to the same db). In case if we want to spin up more connection pooler
// instances, take this into account and maintain the same number of
// connections.
//
// MIN_SIZE is a pool's minimal size, to prevent situation when sudden workload
// have to wait for spinning up a new connections.
//
// RESERVE_SIZE is how many additional connections to allow for a pooler.
func (c *Cluster) getConnectionPoolerEnvVars() []v1.EnvVar {
spec := &c.Spec
effectiveMode := util.Coalesce(
spec.ConnectionPooler.Mode,
c.OpConfig.ConnectionPooler.Mode)
numberOfInstances := spec.ConnectionPooler.NumberOfInstances
if numberOfInstances == nil {
numberOfInstances = util.CoalesceInt32(
c.OpConfig.ConnectionPooler.NumberOfInstances,
k8sutil.Int32ToPointer(1))
}
effectiveMaxDBConn := util.CoalesceInt32(
spec.ConnectionPooler.MaxDBConnections,
c.OpConfig.ConnectionPooler.MaxDBConnections)
if effectiveMaxDBConn == nil {
effectiveMaxDBConn = k8sutil.Int32ToPointer(
constants.ConnectionPoolerMaxDBConnections)
}
maxDBConn := *effectiveMaxDBConn / *numberOfInstances
defaultSize := maxDBConn / 2
minSize := defaultSize / 2
reserveSize := minSize
return []v1.EnvVar{
{
Name: "CONNECTION_POOLER_PORT",
Value: fmt.Sprint(pgPort),
},
{
Name: "CONNECTION_POOLER_MODE",
Value: effectiveMode,
},
{
Name: "CONNECTION_POOLER_DEFAULT_SIZE",
Value: fmt.Sprint(defaultSize),
},
{
Name: "CONNECTION_POOLER_MIN_SIZE",
Value: fmt.Sprint(minSize),
},
{
Name: "CONNECTION_POOLER_RESERVE_SIZE",
Value: fmt.Sprint(reserveSize),
},
{
Name: "CONNECTION_POOLER_MAX_CLIENT_CONN",
Value: fmt.Sprint(constants.ConnectionPoolerMaxClientConnections),
},
{
Name: "CONNECTION_POOLER_MAX_DB_CONN",
Value: fmt.Sprint(maxDBConn),
},
}
}
func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) (
*v1.PodTemplateSpec, error) {
spec := &c.Spec
gracePeriod := int64(c.OpConfig.PodTerminateGracePeriod.Seconds())
resources, err := generateResourceRequirements(
spec.ConnectionPooler.Resources,
makeDefaultConnectionPoolerResources(&c.OpConfig))
effectiveDockerImage := util.Coalesce(
spec.ConnectionPooler.DockerImage,
c.OpConfig.ConnectionPooler.Image)
effectiveSchema := util.Coalesce(
spec.ConnectionPooler.Schema,
c.OpConfig.ConnectionPooler.Schema)
if err != nil {
return nil, fmt.Errorf("could not generate resource requirements: %v", err)
}
secretSelector := func(key string) *v1.SecretKeySelector {
effectiveUser := util.Coalesce(
spec.ConnectionPooler.User,
c.OpConfig.ConnectionPooler.User)
return &v1.SecretKeySelector{
LocalObjectReference: v1.LocalObjectReference{
Name: c.credentialSecretName(effectiveUser),
},
Key: key,
}
}
envVars := []v1.EnvVar{
{
Name: "PGHOST",
Value: c.serviceAddress(role),
},
{
Name: "PGPORT",
Value: c.servicePort(role),
},
{
Name: "PGUSER",
ValueFrom: &v1.EnvVarSource{
SecretKeyRef: secretSelector("username"),
},
},
// the convention is to use the same schema name as
// connection pooler username
{
Name: "PGSCHEMA",
Value: effectiveSchema,
},
{
Name: "PGPASSWORD",
ValueFrom: &v1.EnvVarSource{
SecretKeyRef: secretSelector("password"),
},
},
}
envVars = append(envVars, c.getConnectionPoolerEnvVars()...)
poolerContainer := v1.Container{
Name: connectionPoolerContainer,
Image: effectiveDockerImage,
ImagePullPolicy: v1.PullIfNotPresent,
Resources: *resources,
Ports: []v1.ContainerPort{
{
ContainerPort: pgPort,
Protocol: v1.ProtocolTCP,
},
},
Env: envVars,
ReadinessProbe: &v1.Probe{
Handler: v1.Handler{
TCPSocket: &v1.TCPSocketAction{
Port: intstr.IntOrString{IntVal: pgPort},
},
},
},
SecurityContext: &v1.SecurityContext{
AllowPrivilegeEscalation: util.False(),
},
}
podTemplate := &v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: c.connectionPoolerLabels(role, true).MatchLabels,
Namespace: c.Namespace,
Annotations: c.annotationsSet(c.generatePodAnnotations(spec)),
},
Spec: v1.PodSpec{
TerminationGracePeriodSeconds: &gracePeriod,
Containers: []v1.Container{poolerContainer},
// TODO: add tolerations to scheduler pooler on the same node
// as database
//Tolerations: *tolerationsSpec,
},
}
return podTemplate, nil
}
func (c *Cluster) generateConnectionPoolerDeployment(connectionPooler *ConnectionPoolerObjects) (
*appsv1.Deployment, error) {
spec := &c.Spec
// there are two ways to enable connection pooler, either to specify a
// connectionPooler section or enableConnectionPooler. In the second case
// spec.connectionPooler will be nil, so to make it easier to calculate
// default values, initialize it to an empty structure. It could be done
// anywhere, but here is the earliest common entry point between sync and
// create code, so init here.
if spec.ConnectionPooler == nil {
spec.ConnectionPooler = &acidv1.ConnectionPooler{}
}
podTemplate, err := c.generateConnectionPoolerPodTemplate(connectionPooler.Role)
numberOfInstances := spec.ConnectionPooler.NumberOfInstances
if numberOfInstances == nil {
numberOfInstances = util.CoalesceInt32(
c.OpConfig.ConnectionPooler.NumberOfInstances,
k8sutil.Int32ToPointer(1))
}
if *numberOfInstances < constants.ConnectionPoolerMinInstances {
msg := "Adjusted number of connection pooler instances from %d to %d"
c.logger.Warningf(msg, *numberOfInstances, constants.ConnectionPoolerMinInstances)
*numberOfInstances = constants.ConnectionPoolerMinInstances
}
if err != nil {
return nil, err
}
deployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: connectionPooler.Name,
Namespace: connectionPooler.Namespace,
Labels: c.connectionPoolerLabels(connectionPooler.Role, true).MatchLabels,
Annotations: c.AnnotationsToPropagate(c.annotationsSet(nil)),
// make StatefulSet object its owner to represent the dependency.
// By itself StatefulSet is being deleted with "Orphaned"
// propagation policy, which means that it's deletion will not
// clean up this deployment, but there is a hope that this object
// will be garbage collected if something went wrong and operator
// didn't deleted it.
OwnerReferences: c.ownerReferences(),
},
Spec: appsv1.DeploymentSpec{
Replicas: numberOfInstances,
Selector: c.connectionPoolerLabels(connectionPooler.Role, false),
Template: *podTemplate,
},
}
return deployment, nil
}
func (c *Cluster) generateConnectionPoolerService(connectionPooler *ConnectionPoolerObjects) *v1.Service {
spec := &c.Spec
// there are two ways to enable connection pooler, either to specify a
// connectionPooler section or enableConnectionPooler. In the second case
// spec.connectionPooler will be nil, so to make it easier to calculate
// default values, initialize it to an empty structure. It could be done
// anywhere, but here is the earliest common entry point between sync and
// create code, so init here.
if spec.ConnectionPooler == nil {
spec.ConnectionPooler = &acidv1.ConnectionPooler{}
}
serviceSpec := v1.ServiceSpec{
Ports: []v1.ServicePort{
{
Name: connectionPooler.Name,
Port: pgPort,
TargetPort: intstr.IntOrString{StrVal: c.servicePort(connectionPooler.Role)},
},
},
Type: v1.ServiceTypeClusterIP,
Selector: map[string]string{
"connection-pooler": c.connectionPoolerName(connectionPooler.Role),
},
}
service := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: connectionPooler.Name,
Namespace: connectionPooler.Namespace,
Labels: c.connectionPoolerLabels(connectionPooler.Role, false).MatchLabels,
Annotations: c.annotationsSet(c.generateServiceAnnotations(connectionPooler.Role, spec)),
// make StatefulSet object its owner to represent the dependency.
// By itself StatefulSet is being deleted with "Orphaned"
// propagation policy, which means that it's deletion will not
// clean up this service, but there is a hope that this object will
// be garbage collected if something went wrong and operator didn't
// deleted it.
OwnerReferences: c.ownerReferences(),
},
Spec: serviceSpec,
}
return service
}
//delete connection pooler
func (c *Cluster) deleteConnectionPooler(role PostgresRole) (err error) {
c.logger.Infof("deleting connection pooler spilo-role=%s", role)
// Lack of connection pooler objects is not a fatal error, just log it if
// it was present before in the manifest
if c.ConnectionPooler[role] == nil || role == "" {
c.logger.Debugf("no connection pooler to delete")
return nil
}
// Clean up the deployment object. If deployment resource we've remembered
// is somehow empty, try to delete based on what would we generate
var deployment *appsv1.Deployment
deployment = c.ConnectionPooler[role].Deployment
policy := metav1.DeletePropagationForeground
options := metav1.DeleteOptions{PropagationPolicy: &policy}
if deployment != nil {
// set delete propagation policy to foreground, so that replica set will be
// also deleted.
err = c.KubeClient.
Deployments(c.Namespace).
Delete(context.TODO(), deployment.Name, options)
if k8sutil.ResourceNotFound(err) {
c.logger.Debugf("connection pooler deployment was already deleted")
} else if err != nil {
return fmt.Errorf("could not delete connection pooler deployment: %v", err)
}
c.logger.Infof("connection pooler deployment %s has been deleted for role %s", deployment.Name, role)
}
// Repeat the same for the service object
var service *v1.Service
service = c.ConnectionPooler[role].Service
if service == nil {
c.logger.Debugf("no connection pooler service object to delete")
} else {
err = c.KubeClient.
Services(c.Namespace).
Delete(context.TODO(), service.Name, options)
if k8sutil.ResourceNotFound(err) {
c.logger.Debugf("connection pooler service was already deleted")
} else if err != nil {
return fmt.Errorf("could not delete connection pooler service: %v", err)
}
c.logger.Infof("connection pooler service %s has been deleted for role %s", service.Name, role)
}
c.ConnectionPooler[role].Deployment = nil
c.ConnectionPooler[role].Service = nil
return nil
}
//delete connection pooler
func (c *Cluster) deleteConnectionPoolerSecret() (err error) {
// Repeat the same for the secret object
secretName := c.credentialSecretName(c.OpConfig.ConnectionPooler.User)
secret, err := c.KubeClient.
Secrets(c.Namespace).
Get(context.TODO(), secretName, metav1.GetOptions{})
if err != nil {
c.logger.Debugf("could not get connection pooler secret %s: %v", secretName, err)
} else {
if err = c.deleteSecret(secret.UID, *secret); err != nil {
return fmt.Errorf("could not delete pooler secret: %v", err)
}
}
return nil
}
// Perform actual patching of a connection pooler deployment, assuming that all
// the check were already done before.
func updateConnectionPoolerDeployment(KubeClient k8sutil.KubernetesClient, newDeployment *appsv1.Deployment) (*appsv1.Deployment, error) {
if newDeployment == nil {
return nil, fmt.Errorf("there is no connection pooler in the cluster")
}
patchData, err := specPatch(newDeployment.Spec)
if err != nil {
return nil, fmt.Errorf("could not form patch for the connection pooler deployment: %v", err)
}
// An update probably requires RetryOnConflict, but since only one operator
// worker at one time will try to update it chances of conflicts are
// minimal.
deployment, err := KubeClient.
Deployments(newDeployment.Namespace).Patch(
context.TODO(),
newDeployment.Name,
types.MergePatchType,
patchData,
metav1.PatchOptions{},
"")
if err != nil {
return nil, fmt.Errorf("could not patch connection pooler deployment: %v", err)
}
return deployment, nil
}
//updateConnectionPoolerAnnotations updates the annotations of connection pooler deployment
func updateConnectionPoolerAnnotations(KubeClient k8sutil.KubernetesClient, deployment *appsv1.Deployment, annotations map[string]string) (*appsv1.Deployment, error) {
patchData, err := metaAnnotationsPatch(annotations)
if err != nil {
return nil, fmt.Errorf("could not form patch for the connection pooler deployment metadata: %v", err)
}
result, err := KubeClient.Deployments(deployment.Namespace).Patch(
context.TODO(),
deployment.Name,
types.MergePatchType,
[]byte(patchData),
metav1.PatchOptions{},
"")
if err != nil {
return nil, fmt.Errorf("could not patch connection pooler annotations %q: %v", patchData, err)
}
return result, nil
}
// Test if two connection pooler configuration needs to be synced. For simplicity
// compare not the actual K8S objects, but the configuration itself and request
// sync if there is any difference.
func needSyncConnectionPoolerSpecs(oldSpec, newSpec *acidv1.ConnectionPooler, logger *logrus.Entry) (sync bool, reasons []string) {
reasons = []string{}
sync = false
changelog, err := diff.Diff(oldSpec, newSpec)
if err != nil {
logger.Infof("cannot get diff, do not do anything, %+v", err)
return false, reasons
}
if len(changelog) > 0 {
sync = true
}
for _, change := range changelog {
msg := fmt.Sprintf("%s %+v from '%+v' to '%+v'",
change.Type, change.Path, change.From, change.To)
reasons = append(reasons, msg)
}
return sync, reasons
}
// Check if we need to synchronize connection pooler deployment due to new
// defaults, that are different from what we see in the DeploymentSpec
func needSyncConnectionPoolerDefaults(Config *Config, spec *acidv1.ConnectionPooler, deployment *appsv1.Deployment) (sync bool, reasons []string) {
reasons = []string{}
sync = false
config := Config.OpConfig.ConnectionPooler
podTemplate := deployment.Spec.Template
poolerContainer := podTemplate.Spec.Containers[constants.ConnectionPoolerContainer]
if spec == nil {
spec = &acidv1.ConnectionPooler{}
}
if spec.NumberOfInstances == nil &&
*deployment.Spec.Replicas != *config.NumberOfInstances {
sync = true
msg := fmt.Sprintf("NumberOfInstances is different (having %d, required %d)",
*deployment.Spec.Replicas, *config.NumberOfInstances)
reasons = append(reasons, msg)
}
if spec.DockerImage == "" &&
poolerContainer.Image != config.Image {
sync = true
msg := fmt.Sprintf("DockerImage is different (having %s, required %s)",
poolerContainer.Image, config.Image)
reasons = append(reasons, msg)
}
expectedResources, err := generateResourceRequirements(spec.Resources,
makeDefaultConnectionPoolerResources(&Config.OpConfig))
// An error to generate expected resources means something is not quite
// right, but for the purpose of robustness do not panic here, just report
// and ignore resources comparison (in the worst case there will be no
// updates for new resource values).
if err == nil && syncResources(&poolerContainer.Resources, expectedResources) {
sync = true
msg := fmt.Sprintf("Resources are different (having %+v, required %+v)",
poolerContainer.Resources, expectedResources)
reasons = append(reasons, msg)
}
if err != nil {
return false, reasons
}
for _, env := range poolerContainer.Env {
if spec.User == "" && env.Name == "PGUSER" {
ref := env.ValueFrom.SecretKeyRef.LocalObjectReference
secretName := Config.OpConfig.SecretNameTemplate.Format(
"username", strings.Replace(config.User, "_", "-", -1),
"cluster", deployment.ClusterName,
"tprkind", acidv1.PostgresCRDResourceKind,
"tprgroup", acidzalando.GroupName)
if ref.Name != secretName {
sync = true
msg := fmt.Sprintf("pooler user is different (having %s, required %s)",
ref.Name, config.User)
reasons = append(reasons, msg)
}
}
if spec.Schema == "" && env.Name == "PGSCHEMA" && env.Value != config.Schema {
sync = true
msg := fmt.Sprintf("pooler schema is different (having %s, required %s)",
env.Value, config.Schema)
reasons = append(reasons, msg)
}
}
return sync, reasons
}
// Generate default resource section for connection pooler deployment, to be
// used if nothing custom is specified in the manifest
func makeDefaultConnectionPoolerResources(config *config.Config) acidv1.Resources {
defaultRequests := acidv1.ResourceDescription{
CPU: config.ConnectionPooler.ConnectionPoolerDefaultCPURequest,
Memory: config.ConnectionPooler.ConnectionPoolerDefaultMemoryRequest,
}
defaultLimits := acidv1.ResourceDescription{
CPU: config.ConnectionPooler.ConnectionPoolerDefaultCPULimit,
Memory: config.ConnectionPooler.ConnectionPoolerDefaultMemoryLimit,
}
return acidv1.Resources{
ResourceRequests: defaultRequests,
ResourceLimits: defaultLimits,
}
}
func logPoolerEssentials(log *logrus.Entry, oldSpec, newSpec *acidv1.Postgresql) {
var v []string
var input []*bool
if oldSpec == nil {
input = []*bool{nil, nil, newSpec.Spec.EnableConnectionPooler, newSpec.Spec.EnableReplicaConnectionPooler}
} else {
input = []*bool{oldSpec.Spec.EnableConnectionPooler, oldSpec.Spec.EnableReplicaConnectionPooler, newSpec.Spec.EnableConnectionPooler, newSpec.Spec.EnableReplicaConnectionPooler}
}
for _, b := range input {
if b == nil {
v = append(v, "nil")
} else {
v = append(v, fmt.Sprintf("%v", *b))
}
}
log.Debugf("syncing connection pooler from (%v, %v) to (%v, %v)", v[0], v[1], v[2], v[3])
}
func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, LookupFunction InstallFunction) (SyncReason, error) {
var reason SyncReason
var err error
var newNeedConnectionPooler, oldNeedConnectionPooler bool
oldNeedConnectionPooler = false
if oldSpec == nil {
oldSpec = &acidv1.Postgresql{
Spec: acidv1.PostgresSpec{
ConnectionPooler: &acidv1.ConnectionPooler{},
},
}
}
needSync, _ := needSyncConnectionPoolerSpecs(oldSpec.Spec.ConnectionPooler, newSpec.Spec.ConnectionPooler, c.logger)
masterChanges, err := diff.Diff(oldSpec.Spec.EnableConnectionPooler, newSpec.Spec.EnableConnectionPooler)
if err != nil {
c.logger.Error("Error in getting diff of master connection pooler changes")
}
replicaChanges, err := diff.Diff(oldSpec.Spec.EnableReplicaConnectionPooler, newSpec.Spec.EnableReplicaConnectionPooler)
if err != nil {
c.logger.Error("Error in getting diff of replica connection pooler changes")
}
// skip pooler sync only
// 1. if there is no diff in spec, AND
// 2. if connection pooler is already there and is also required as per newSpec
//
// Handling the case when connectionPooler is not there but it is required
// as per spec, hence do not skip syncing in that case, even though there
// is no diff in specs
if (!needSync && len(masterChanges) <= 0 && len(replicaChanges) <= 0) &&
(c.ConnectionPooler != nil && (needConnectionPooler(&newSpec.Spec))) {
c.logger.Debugln("syncing pooler is not required")
return nil, nil
}
logPoolerEssentials(c.logger, oldSpec, newSpec)
// Check and perform the sync requirements for each of the roles.
for _, role := range [2]PostgresRole{Master, Replica} {
if role == Master {
newNeedConnectionPooler = needMasterConnectionPoolerWorker(&newSpec.Spec)
if oldSpec != nil {
oldNeedConnectionPooler = needMasterConnectionPoolerWorker(&oldSpec.Spec)
}
} else {
newNeedConnectionPooler = needReplicaConnectionPoolerWorker(&newSpec.Spec)
if oldSpec != nil {
oldNeedConnectionPooler = needReplicaConnectionPoolerWorker(&oldSpec.Spec)
}
}
// if the call is via createConnectionPooler, then it is required to initialize
// the structure
if c.ConnectionPooler == nil {
c.ConnectionPooler = map[PostgresRole]*ConnectionPoolerObjects{}
}
if c.ConnectionPooler[role] == nil {
c.ConnectionPooler[role] = &ConnectionPoolerObjects{
Deployment: nil,
Service: nil,
Name: c.connectionPoolerName(role),
ClusterName: c.ClusterName,
Namespace: c.Namespace,
LookupFunction: false,
Role: role,
}
}
if newNeedConnectionPooler {
// Try to sync in any case. If we didn't needed connection pooler before,
// it means we want to create it. If it was already present, still sync
// since it could happen that there is no difference in specs, and all
// the resources are remembered, but the deployment was manually deleted
// in between
// in this case also do not forget to install lookup function as for
// creating cluster
if !oldNeedConnectionPooler || !c.ConnectionPooler[role].LookupFunction {
newConnectionPooler := newSpec.Spec.ConnectionPooler
specSchema := ""
specUser := ""
if newConnectionPooler != nil {
specSchema = newConnectionPooler.Schema
specUser = newConnectionPooler.User
}
schema := util.Coalesce(
specSchema,
c.OpConfig.ConnectionPooler.Schema)
user := util.Coalesce(
specUser,
c.OpConfig.ConnectionPooler.User)
if err = LookupFunction(schema, user, role); err != nil {
return NoSync, err
}
}
if reason, err = c.syncConnectionPoolerWorker(oldSpec, newSpec, role); err != nil {
c.logger.Errorf("could not sync connection pooler: %v", err)
return reason, err
}
} else {
// delete and cleanup resources if they are already detected
if c.ConnectionPooler[role] != nil &&
(c.ConnectionPooler[role].Deployment != nil ||
c.ConnectionPooler[role].Service != nil) {
if err = c.deleteConnectionPooler(role); err != nil {
c.logger.Warningf("could not remove connection pooler: %v", err)
}
}
}
}
if !needMasterConnectionPoolerWorker(&newSpec.Spec) &&
!needReplicaConnectionPoolerWorker(&newSpec.Spec) {
if err = c.deleteConnectionPoolerSecret(); err != nil {
c.logger.Warningf("could not remove connection pooler secret: %v", err)
}
}
return reason, nil
}
// Synchronize connection pooler resources. Effectively we're interested only in
// synchronizing the corresponding deployment, but in case of deployment or
// service is missing, create it. After checking, also remember an object for
// the future references.
func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql, role PostgresRole) (
SyncReason, error) {
deployment, err := c.KubeClient.
Deployments(c.Namespace).
Get(context.TODO(), c.connectionPoolerName(role), metav1.GetOptions{})
if err != nil && k8sutil.ResourceNotFound(err) {
msg := "deployment %s for connection pooler synchronization is not found, create it"
c.logger.Warningf(msg, c.connectionPoolerName(role))
deploymentSpec, err := c.generateConnectionPoolerDeployment(c.ConnectionPooler[role])
if err != nil {
msg = "could not generate deployment for connection pooler: %v"
return NoSync, fmt.Errorf(msg, err)
}
deployment, err := c.KubeClient.
Deployments(deploymentSpec.Namespace).
Create(context.TODO(), deploymentSpec, metav1.CreateOptions{})
if err != nil {
return NoSync, err
}
c.ConnectionPooler[role].Deployment = deployment
} else if err != nil {
msg := "could not get connection pooler deployment to sync: %v"
return NoSync, fmt.Errorf(msg, err)
} else {
c.ConnectionPooler[role].Deployment = deployment
// actual synchronization
var oldConnectionPooler *acidv1.ConnectionPooler
if oldSpec != nil {
oldConnectionPooler = oldSpec.Spec.ConnectionPooler
}
newConnectionPooler := newSpec.Spec.ConnectionPooler
// sync implementation below assumes that both old and new specs are
// not nil, but it can happen. To avoid any confusion like updating a
// deployment because the specification changed from nil to an empty
// struct (that was initialized somewhere before) replace any nil with
// an empty spec.
if oldConnectionPooler == nil {
oldConnectionPooler = &acidv1.ConnectionPooler{}
}
if newConnectionPooler == nil {
newConnectionPooler = &acidv1.ConnectionPooler{}
}
c.logger.Infof("old: %+v, new %+v", oldConnectionPooler, newConnectionPooler)
var specSync bool
var specReason []string
if oldSpec != nil {
specSync, specReason = needSyncConnectionPoolerSpecs(oldConnectionPooler, newConnectionPooler, c.logger)
}
defaultsSync, defaultsReason := needSyncConnectionPoolerDefaults(&c.Config, newConnectionPooler, deployment)
reason := append(specReason, defaultsReason...)
if specSync || defaultsSync {
c.logger.Infof("Update connection pooler deployment %s, reason: %+v",
c.connectionPoolerName(role), reason)
newDeploymentSpec, err := c.generateConnectionPoolerDeployment(c.ConnectionPooler[role])
if err != nil {
msg := "could not generate deployment for connection pooler: %v"
return reason, fmt.Errorf(msg, err)
}
deployment, err := updateConnectionPoolerDeployment(c.KubeClient,
newDeploymentSpec)
if err != nil {
return reason, err
}
c.ConnectionPooler[role].Deployment = deployment
}
}
newAnnotations := c.AnnotationsToPropagate(c.annotationsSet(c.ConnectionPooler[role].Deployment.Annotations))
if newAnnotations != nil {
deployment, err = updateConnectionPoolerAnnotations(c.KubeClient, c.ConnectionPooler[role].Deployment, newAnnotations)
if err != nil {
return nil, err
}
c.ConnectionPooler[role].Deployment = deployment
}
service, err := c.KubeClient.
Services(c.Namespace).
Get(context.TODO(), c.connectionPoolerName(role), metav1.GetOptions{})
if err != nil && k8sutil.ResourceNotFound(err) {
msg := "Service %s for connection pooler synchronization is not found, create it"
c.logger.Warningf(msg, c.connectionPoolerName(role))
serviceSpec := c.generateConnectionPoolerService(c.ConnectionPooler[role])
service, err := c.KubeClient.
Services(serviceSpec.Namespace).
Create(context.TODO(), serviceSpec, metav1.CreateOptions{})
if err != nil {
return NoSync, err
}
c.ConnectionPooler[role].Service = service
} else if err != nil {
msg := "could not get connection pooler service to sync: %v"
return NoSync, fmt.Errorf(msg, err)
} else {
// Service updates are not supported and probably not that useful anyway
c.ConnectionPooler[role].Service = service
}
return NoSync, nil
}
pkg/cluster/connection_pooler_new_test.go
+45
View File
@@ -0,0 +1,45 @@
package cluster
import (
"testing"
"context"
appsv1 "k8s.io/api/apps/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/kubernetes/fake"
)
func TestFakeClient(t *testing.T) {
clientSet := fake.NewSimpleClientset()
namespace := "default"
l := labels.Set(map[string]string{
"application": "spilo",
})
deployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "my-deployment1",
Namespace: namespace,
Labels: l,
},
}
clientSet.AppsV1().Deployments(namespace).Create(context.TODO(), deployment, metav1.CreateOptions{})
deployment2, _ := clientSet.AppsV1().Deployments(namespace).Get(context.TODO(), "my-deployment1", metav1.GetOptions{})
if deployment.ObjectMeta.Name != deployment2.ObjectMeta.Name {
t.Errorf("Deployments are not equal")
}
deployments, _ := clientSet.AppsV1().Deployments(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: "application=spilo"})
if len(deployments.Items) != 1 {
t.Errorf("Label search does not work")
}
}
pkg/cluster/connection_pooler_test.go
+1008
View File
File diff suppressed because it is too large Load Diff
pkg/cluster/database.go
+359 -26
View File
@@ -1,10 +1,12 @@
package cluster
import (
"bytes"
"database/sql"
"fmt"
"net"
"strings"
"text/template"
"time"
"github.com/lib/pq"
@@ -25,16 +27,66 @@ const (
WHERE a.rolname = ANY($1)
ORDER BY 1;`
getDatabasesSQL = `SELECT datname, pg_get_userbyid(datdba) AS owner FROM pg_database;`
createDatabaseSQL = `CREATE DATABASE "%s" OWNER "%s";`
alterDatabaseOwnerSQL = `ALTER DATABASE "%s" OWNER TO "%s";`
getDatabasesSQL = `SELECT datname, pg_get_userbyid(datdba) AS owner FROM pg_database;`
getSchemasSQL = `SELECT n.nspname AS dbschema FROM pg_catalog.pg_namespace n
WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema' ORDER BY 1`
getExtensionsSQL = `SELECT e.extname, n.nspname FROM pg_catalog.pg_extension e
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = e.extnamespace ORDER BY 1;`
createDatabaseSQL = `CREATE DATABASE "%s" OWNER "%s";`
createDatabaseSchemaSQL = `SET ROLE TO "%s"; CREATE SCHEMA IF NOT EXISTS "%s" AUTHORIZATION "%s"`
alterDatabaseOwnerSQL = `ALTER DATABASE "%s" OWNER TO "%s";`
createExtensionSQL = `CREATE EXTENSION IF NOT EXISTS "%s" SCHEMA "%s"`
alterExtensionSQL = `ALTER EXTENSION "%s" SET SCHEMA "%s"`
globalDefaultPrivilegesSQL = `SET ROLE TO "%s";
ALTER DEFAULT PRIVILEGES GRANT USAGE ON SCHEMAS TO "%s","%s";
ALTER DEFAULT PRIVILEGES GRANT SELECT ON TABLES TO "%s";
ALTER DEFAULT PRIVILEGES GRANT SELECT ON SEQUENCES TO "%s";
ALTER DEFAULT PRIVILEGES GRANT INSERT, UPDATE, DELETE ON TABLES TO "%s";
ALTER DEFAULT PRIVILEGES GRANT USAGE, UPDATE ON SEQUENCES TO "%s";
ALTER DEFAULT PRIVILEGES GRANT EXECUTE ON FUNCTIONS TO "%s","%s";
ALTER DEFAULT PRIVILEGES GRANT USAGE ON TYPES TO "%s","%s";`
schemaDefaultPrivilegesSQL = `SET ROLE TO "%s";
GRANT USAGE ON SCHEMA "%s" TO "%s","%s";
ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT SELECT ON TABLES TO "%s";
ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT SELECT ON SEQUENCES TO "%s";
ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT INSERT, UPDATE, DELETE ON TABLES TO "%s";
ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT USAGE, UPDATE ON SEQUENCES TO "%s";
ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT EXECUTE ON FUNCTIONS TO "%s","%s";
ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT USAGE ON TYPES TO "%s","%s";`
connectionPoolerLookup = `
CREATE SCHEMA IF NOT EXISTS {{.pooler_schema}};
CREATE OR REPLACE FUNCTION {{.pooler_schema}}.user_lookup(
in i_username text, out uname text, out phash text)
RETURNS record AS $$
BEGIN
SELECT usename, passwd FROM pg_catalog.pg_shadow
WHERE usename = i_username INTO uname, phash;
RETURN;
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;
REVOKE ALL ON FUNCTION {{.pooler_schema}}.user_lookup(text)
FROM public, {{.pooler_user}};
GRANT EXECUTE ON FUNCTION {{.pooler_schema}}.user_lookup(text)
TO {{.pooler_user}};
GRANT USAGE ON SCHEMA {{.pooler_schema}} TO {{.pooler_user}};
`
)
func (c *Cluster) pgConnectionString() string {
func (c *Cluster) pgConnectionString(dbname string) string {
password := c.systemUsers[constants.SuperuserKeyName].Password
return fmt.Sprintf("host='%s' dbname=postgres sslmode=require user='%s' password='%s' connect_timeout='%d'",
if dbname == "" {
dbname = "postgres"
}
return fmt.Sprintf("host='%s' dbname='%s' sslmode=require user='%s' password='%s' connect_timeout='%d'",
fmt.Sprintf("%s.%s.svc.%s", c.Name, c.Namespace, c.OpConfig.ClusterDomain),
dbname,
c.systemUsers[constants.SuperuserKeyName].Name,
strings.Replace(password, "$", "\\$", -1),
constants.PostgresConnectTimeout/time.Second)
@@ -49,13 +101,22 @@ func (c *Cluster) databaseAccessDisabled() bool {
}
func (c *Cluster) initDbConn() error {
c.setProcessName("initializing db connection")
if c.pgDb != nil {
return nil
}
return c.initDbConnWithName("")
}
// Worker function for connection initialization. This function does not check
// if the connection is already open, if it is then it will be overwritten.
// Callers need to make sure no connection is open, otherwise we could leak
// connections
func (c *Cluster) initDbConnWithName(dbname string) error {
c.setProcessName("initializing db connection")
var conn *sql.DB
connstring := c.pgConnectionString()
connstring := c.pgConnectionString(dbname)
finalerr := retryutil.Retry(constants.PostgresConnectTimeout, constants.PostgresConnectRetryTimeout,
func() (bool, error) {
@@ -70,12 +131,12 @@ func (c *Cluster) initDbConn() error {
}
if _, ok := err.(*net.OpError); ok {
c.logger.Errorf("could not connect to PostgreSQL database: %v", err)
c.logger.Warningf("could not connect to Postgres database: %v", err)
return false, nil
}
if err2 := conn.Close(); err2 != nil {
c.logger.Errorf("error when closing PostgreSQL connection after another error: %v", err)
c.logger.Errorf("error when closing Postgres connection after another error: %v", err)
return false, err2
}
@@ -89,13 +150,23 @@ func (c *Cluster) initDbConn() error {
conn.SetMaxOpenConns(1)
conn.SetMaxIdleConns(-1)
if c.pgDb != nil {
msg := "closing an existing connection before opening a new one to %s"
c.logger.Warningf(msg, dbname)
c.closeDbConn()
}
c.pgDb = conn
return nil
}
func (c *Cluster) connectionIsClosed() bool {
return c.pgDb == nil
}
func (c *Cluster) closeDbConn() (err error) {
c.setProcessName("closing db connection")
c.setProcessName("closing database connection")
if c.pgDb != nil {
c.logger.Debug("closing database connection")
if err = c.pgDb.Close(); err != nil {
@@ -110,7 +181,7 @@ func (c *Cluster) closeDbConn() (err error) {
}
func (c *Cluster) readPgUsersFromDatabase(userNames []string) (users spec.PgUserMap, err error) {
c.setProcessName("reading users from the db")
c.setProcessName("reading users from the database")
var rows *sql.Rows
users = make(spec.PgUserMap)
if rows, err = c.pgDb.Query(getUserSQL, pq.Array(userNames)); err != nil {
@@ -187,43 +258,141 @@ func (c *Cluster) getDatabases() (dbs map[string]string, err error) {
}
// executeCreateDatabase creates new database with the given owner.
// The caller is responsible for openinging and closing the database connection.
func (c *Cluster) executeCreateDatabase(datname, owner string) error {
return c.execCreateOrAlterDatabase(datname, owner, createDatabaseSQL,
// The caller is responsible for opening and closing the database connection.
func (c *Cluster) executeCreateDatabase(databaseName, owner string) error {
return c.execCreateOrAlterDatabase(databaseName, owner, createDatabaseSQL,
"creating database", "create database")
}
// executeCreateDatabase changes the owner of the given database.
// The caller is responsible for openinging and closing the database connection.
func (c *Cluster) executeAlterDatabaseOwner(datname string, owner string) error {
return c.execCreateOrAlterDatabase(datname, owner, alterDatabaseOwnerSQL,
// executeAlterDatabaseOwner changes the owner of the given database.
// The caller is responsible for opening and closing the database connection.
func (c *Cluster) executeAlterDatabaseOwner(databaseName string, owner string) error {
return c.execCreateOrAlterDatabase(databaseName, owner, alterDatabaseOwnerSQL,
"changing owner for database", "alter database owner")
}
func (c *Cluster) execCreateOrAlterDatabase(datname, owner, statement, doing, operation string) error {
if !c.databaseNameOwnerValid(datname, owner) {
func (c *Cluster) execCreateOrAlterDatabase(databaseName, owner, statement, doing, operation string) error {
if !c.databaseNameOwnerValid(databaseName, owner) {
return nil
}
c.logger.Infof("%s %q owner %q", doing, datname, owner)
if _, err := c.pgDb.Exec(fmt.Sprintf(statement, datname, owner)); err != nil {
c.logger.Infof("%s %q owner %q", doing, databaseName, owner)
if _, err := c.pgDb.Exec(fmt.Sprintf(statement, databaseName, owner)); err != nil {
return fmt.Errorf("could not execute %s: %v", operation, err)
}
return nil
}
func (c *Cluster) databaseNameOwnerValid(datname, owner string) bool {
func (c *Cluster) databaseNameOwnerValid(databaseName, owner string) bool {
if _, ok := c.pgUsers[owner]; !ok {
c.logger.Infof("skipping creation of the %q database, user %q does not exist", datname, owner)
c.logger.Infof("skipping creation of the %q database, user %q does not exist", databaseName, owner)
return false
}
if !databaseNameRegexp.MatchString(datname) {
c.logger.Infof("database %q has invalid name", datname)
if !databaseNameRegexp.MatchString(databaseName) {
c.logger.Infof("database %q has invalid name", databaseName)
return false
}
return true
}
// getSchemas returns the list of current database schemas
// The caller is responsible for opening and closing the database connection
func (c *Cluster) getSchemas() (schemas []string, err error) {
var (
rows *sql.Rows
dbschemas []string
)
if rows, err = c.pgDb.Query(getSchemasSQL); err != nil {
return nil, fmt.Errorf("could not query database schemas: %v", err)
}
defer func() {
if err2 := rows.Close(); err2 != nil {
if err != nil {
err = fmt.Errorf("error when closing query cursor: %v, previous error: %v", err2, err)
} else {
err = fmt.Errorf("error when closing query cursor: %v", err2)
}
}
}()
for rows.Next() {
var dbschema string
if err = rows.Scan(&dbschema); err != nil {
return nil, fmt.Errorf("error when processing row: %v", err)
}
dbschemas = append(dbschemas, dbschema)
}
return dbschemas, err
}
// executeCreateDatabaseSchema creates new database schema with the given owner.
// The caller is responsible for opening and closing the database connection.
func (c *Cluster) executeCreateDatabaseSchema(databaseName, schemaName, dbOwner string, schemaOwner string) error {
return c.execCreateDatabaseSchema(databaseName, schemaName, dbOwner, schemaOwner, createDatabaseSchemaSQL,
"creating database schema", "create database schema")
}
func (c *Cluster) execCreateDatabaseSchema(databaseName, schemaName, dbOwner, schemaOwner, statement, doing, operation string) error {
if !c.databaseSchemaNameValid(schemaName) {
return nil
}
c.logger.Infof("%s %q owner %q", doing, schemaName, schemaOwner)
if _, err := c.pgDb.Exec(fmt.Sprintf(statement, dbOwner, schemaName, schemaOwner)); err != nil {
return fmt.Errorf("could not execute %s: %v", operation, err)
}
// set default privileges for schema
c.execAlterSchemaDefaultPrivileges(schemaName, schemaOwner, databaseName)
if schemaOwner != dbOwner {
c.execAlterSchemaDefaultPrivileges(schemaName, dbOwner, databaseName+"_"+schemaName)
c.execAlterSchemaDefaultPrivileges(schemaName, schemaOwner, databaseName+"_"+schemaName)
}
return nil
}
func (c *Cluster) databaseSchemaNameValid(schemaName string) bool {
if !databaseNameRegexp.MatchString(schemaName) {
c.logger.Infof("database schema %q has invalid name", schemaName)
return false
}
return true
}
func (c *Cluster) execAlterSchemaDefaultPrivileges(schemaName, owner, rolePrefix string) error {
if _, err := c.pgDb.Exec(fmt.Sprintf(schemaDefaultPrivilegesSQL, owner,
schemaName, rolePrefix+constants.ReaderRoleNameSuffix, rolePrefix+constants.WriterRoleNameSuffix, // schema
schemaName, rolePrefix+constants.ReaderRoleNameSuffix, // tables
schemaName, rolePrefix+constants.ReaderRoleNameSuffix, // sequences
schemaName, rolePrefix+constants.WriterRoleNameSuffix, // tables
schemaName, rolePrefix+constants.WriterRoleNameSuffix, // sequences
schemaName, rolePrefix+constants.ReaderRoleNameSuffix, rolePrefix+constants.WriterRoleNameSuffix, // types
schemaName, rolePrefix+constants.ReaderRoleNameSuffix, rolePrefix+constants.WriterRoleNameSuffix)); err != nil { // functions
return fmt.Errorf("could not alter default privileges for database schema %s: %v", schemaName, err)
}
return nil
}
func (c *Cluster) execAlterGlobalDefaultPrivileges(owner, rolePrefix string) error {
if _, err := c.pgDb.Exec(fmt.Sprintf(globalDefaultPrivilegesSQL, owner,
rolePrefix+constants.WriterRoleNameSuffix, rolePrefix+constants.ReaderRoleNameSuffix, // schemas
rolePrefix+constants.ReaderRoleNameSuffix, // tables
rolePrefix+constants.ReaderRoleNameSuffix, // sequences
rolePrefix+constants.WriterRoleNameSuffix, // tables
rolePrefix+constants.WriterRoleNameSuffix, // sequences
rolePrefix+constants.ReaderRoleNameSuffix, rolePrefix+constants.WriterRoleNameSuffix, // types
rolePrefix+constants.ReaderRoleNameSuffix, rolePrefix+constants.WriterRoleNameSuffix)); err != nil { // functions
return fmt.Errorf("could not alter default privileges for database %s: %v", rolePrefix, err)
}
return nil
}
func makeUserFlags(rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin bool) (result []string) {
if rolsuper {
result = append(result, constants.RoleFlagSuperuser)
@@ -243,3 +412,167 @@ func makeUserFlags(rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin
return result
}
// getExtension returns the list of current database extensions
// The caller is responsible for opening and closing the database connection
func (c *Cluster) getExtensions() (dbExtensions map[string]string, err error) {
var (
rows *sql.Rows
)
if rows, err = c.pgDb.Query(getExtensionsSQL); err != nil {
return nil, fmt.Errorf("could not query database extensions: %v", err)
}
defer func() {
if err2 := rows.Close(); err2 != nil {
if err != nil {
err = fmt.Errorf("error when closing query cursor: %v, previous error: %v", err2, err)
} else {
err = fmt.Errorf("error when closing query cursor: %v", err2)
}
}
}()
dbExtensions = make(map[string]string)
for rows.Next() {
var extension, schema string
if err = rows.Scan(&extension, &schema); err != nil {
return nil, fmt.Errorf("error when processing row: %v", err)
}
dbExtensions[extension] = schema
}
return dbExtensions, err
}
// executeCreateExtension creates new extension in the given schema.
// The caller is responsible for opening and closing the database connection.
func (c *Cluster) executeCreateExtension(extName, schemaName string) error {
return c.execCreateOrAlterExtension(extName, schemaName, createExtensionSQL,
"creating extension", "create extension")
}
// executeAlterExtension changes the schema of the given extension.
// The caller is responsible for opening and closing the database connection.
func (c *Cluster) executeAlterExtension(extName, schemaName string) error {
return c.execCreateOrAlterExtension(extName, schemaName, alterExtensionSQL,
"changing schema for extension", "alter extension schema")
}
func (c *Cluster) execCreateOrAlterExtension(extName, schemaName, statement, doing, operation string) error {
c.logger.Infof("%s %q schema %q", doing, extName, schemaName)
if _, err := c.pgDb.Exec(fmt.Sprintf(statement, extName, schemaName)); err != nil {
return fmt.Errorf("could not execute %s: %v", operation, err)
}
return nil
}
// Creates a connection pool credentials lookup function in every database to
// perform remote authentication.
func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string, role PostgresRole) error {
var stmtBytes bytes.Buffer
c.logger.Info("Installing lookup function")
// Open a new connection if not yet done. This connection will be used only
// to get the list of databases, not for the actuall installation.
if err := c.initDbConn(); err != nil {
return fmt.Errorf("could not init database connection")
}
defer func() {
if c.connectionIsClosed() {
return
}
if err := c.closeDbConn(); err != nil {
c.logger.Errorf("could not close database connection: %v", err)
}
}()
// List of databases we failed to process. At the moment it function just
// like a flag to retry on the next sync, but in the future we may want to
// retry only necessary parts, so let's keep the list.
failedDatabases := []string{}
currentDatabases, err := c.getDatabases()
if err != nil {
msg := "could not get databases to install pooler lookup function: %v"
return fmt.Errorf(msg, err)
}
// We've got the list of target databases, now close this connection to
// open a new one to every each of them.
if err := c.closeDbConn(); err != nil {
c.logger.Errorf("could not close database connection: %v", err)
}
templater := template.Must(template.New("sql").Parse(connectionPoolerLookup))
params := TemplateParams{
"pooler_schema": poolerSchema,
"pooler_user": poolerUser,
}
if err := templater.Execute(&stmtBytes, params); err != nil {
msg := "could not prepare sql statement %+v: %v"
return fmt.Errorf(msg, params, err)
}
for dbname := range currentDatabases {
if dbname == "template0" || dbname == "template1" {
continue
}
c.logger.Infof("install pooler lookup function into database '%s'", dbname)
// golang sql will do retries couple of times if pq driver reports
// connections issues (driver.ErrBadConn), but since our query is
// idempotent, we can retry in a view of other errors (e.g. due to
// failover a db is temporary in a read-only mode or so) to make sure
// it was applied.
execErr := retryutil.Retry(
constants.PostgresConnectTimeout,
constants.PostgresConnectRetryTimeout,
func() (bool, error) {
// At this moment we are not connected to any database
if err := c.initDbConnWithName(dbname); err != nil {
msg := "could not init database connection to %s"
return false, fmt.Errorf(msg, dbname)
}
defer func() {
if err := c.closeDbConn(); err != nil {
msg := "could not close database connection: %v"
c.logger.Errorf(msg, err)
}
}()
if _, err = c.pgDb.Exec(stmtBytes.String()); err != nil {
msg := fmt.Errorf("could not execute sql statement %s: %v",
stmtBytes.String(), err)
return false, msg
}
return true, nil
})
if execErr != nil {
c.logger.Errorf("could not execute after retries %s: %v",
stmtBytes.String(), err)
// process other databases
failedDatabases = append(failedDatabases, dbname)
continue
}
c.logger.Infof("pooler lookup function installed into %s", dbname)
}
if len(failedDatabases) == 0 {
c.ConnectionPooler[role].LookupFunction = true
}
return nil
}
pkg/cluster/exec.go
+3 -2
View File
@@ -2,10 +2,11 @@ package cluster
import (
"bytes"
"context"
"fmt"
"strings"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/remotecommand"
@@ -23,7 +24,7 @@ func (c *Cluster) ExecCommand(podName *spec.NamespacedName, command ...string) (
execErr bytes.Buffer
)
pod, err := c.KubeClient.Pods(podName.Namespace).Get(podName.Name, metav1.GetOptions{})
pod, err := c.KubeClient.Pods(podName.Namespace).Get(context.TODO(), podName.Name, metav1.GetOptions{})
if err != nil {
return "", fmt.Errorf("could not get pod info: %v", err)
}
pkg/cluster/k8sres.go
+679 -226
View File
File diff suppressed because it is too large Load Diff
pkg/cluster/k8sres_test.go
+1055 -17
View File
File diff suppressed because it is too large Load Diff
pkg/cluster/pod.go
+62 -9
View File
@@ -1,8 +1,10 @@
package cluster
import (
"context"
"fmt"
"math/rand"
"time"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
@@ -10,6 +12,7 @@ import (
"github.com/zalando/postgres-operator/pkg/spec"
"github.com/zalando/postgres-operator/pkg/util"
"github.com/zalando/postgres-operator/pkg/util/retryutil"
)
func (c *Cluster) listPods() ([]v1.Pod, error) {
@@ -17,7 +20,7 @@ func (c *Cluster) listPods() ([]v1.Pod, error) {
LabelSelector: c.labelsSet(false).String(),
}
pods, err := c.KubeClient.Pods(c.Namespace).List(listOptions)
pods, err := c.KubeClient.Pods(c.Namespace).List(context.TODO(), listOptions)
if err != nil {
return nil, fmt.Errorf("could not get list of pods: %v", err)
}
@@ -30,7 +33,7 @@ func (c *Cluster) getRolePods(role PostgresRole) ([]v1.Pod, error) {
LabelSelector: c.roleLabelsSet(false, role).String(),
}
pods, err := c.KubeClient.Pods(c.Namespace).List(listOptions)
pods, err := c.KubeClient.Pods(c.Namespace).List(context.TODO(), listOptions)
if err != nil {
return nil, fmt.Errorf("could not get list of pods: %v", err)
}
@@ -73,7 +76,7 @@ func (c *Cluster) deletePod(podName spec.NamespacedName) error {
ch := c.registerPodSubscriber(podName)
defer c.unregisterPodSubscriber(podName)
if err := c.KubeClient.Pods(podName.Namespace).Delete(podName.Name, c.deleteOptions); err != nil {
if err := c.KubeClient.Pods(podName.Namespace).Delete(context.TODO(), podName.Name, c.deleteOptions); err != nil {
return err
}
@@ -183,7 +186,7 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error {
eol bool
)
oldMaster, err := c.KubeClient.Pods(podName.Namespace).Get(podName.Name, metav1.GetOptions{})
oldMaster, err := c.KubeClient.Pods(podName.Namespace).Get(context.TODO(), podName.Name, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("could not get pod: %v", err)
@@ -206,7 +209,9 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error {
// we must have a statefulset in the cluster for the migration to work
if c.Statefulset == nil {
var sset *appsv1.StatefulSet
if sset, err = c.KubeClient.StatefulSets(c.Namespace).Get(c.statefulSetName(),
if sset, err = c.KubeClient.StatefulSets(c.Namespace).Get(
context.TODO(),
c.statefulSetName(),
metav1.GetOptions{}); err != nil {
return fmt.Errorf("could not retrieve cluster statefulset: %v", err)
}
@@ -247,7 +252,7 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error {
// MigrateReplicaPod recreates pod on a new node
func (c *Cluster) MigrateReplicaPod(podName spec.NamespacedName, fromNodeName string) error {
replicaPod, err := c.KubeClient.Pods(podName.Namespace).Get(podName.Name, metav1.GetOptions{})
replicaPod, err := c.KubeClient.Pods(podName.Namespace).Get(context.TODO(), podName.Name, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("could not get pod: %v", err)
}
@@ -276,7 +281,7 @@ func (c *Cluster) recreatePod(podName spec.NamespacedName) (*v1.Pod, error) {
defer c.unregisterPodSubscriber(podName)
stopChan := make(chan struct{})
if err := c.KubeClient.Pods(podName.Namespace).Delete(podName.Name, c.deleteOptions); err != nil {
if err := c.KubeClient.Pods(podName.Namespace).Delete(context.TODO(), podName.Name, c.deleteOptions); err != nil {
return nil, fmt.Errorf("could not delete pod: %v", err)
}
@@ -291,6 +296,50 @@ func (c *Cluster) recreatePod(podName spec.NamespacedName) (*v1.Pod, error) {
return pod, nil
}
func (c *Cluster) isSafeToRecreatePods(pods *v1.PodList) bool {
/*
Operator should not re-create pods if there is at least one replica being bootstrapped
because Patroni might use other replicas to take basebackup from (see Patroni's "clonefrom" tag).
XXX operator cannot forbid replica re-init, so we might still fail if re-init is started
after this check succeeds but before a pod is re-created
*/
for _, pod := range pods.Items {
c.logger.Debugf("name=%s phase=%s ip=%s", pod.Name, pod.Status.Phase, pod.Status.PodIP)
}
for _, pod := range pods.Items {
var state string
err := retryutil.Retry(1*time.Second, 5*time.Second,
func() (bool, error) {
var err error
state, err = c.patroni.GetPatroniMemberState(&pod)
if err != nil {
return false, err
}
return true, nil
},
)
if err != nil {
c.logger.Errorf("failed to get Patroni state for pod: %s", err)
return false
} else if state == "creating replica" {
c.logger.Warningf("cannot re-create replica %s: it is currently being initialized", pod.Name)
return false
}
}
return true
}
func (c *Cluster) recreatePods() error {
c.setProcessName("starting to recreate pods")
ls := c.labelsSet(false)
@@ -300,12 +349,16 @@ func (c *Cluster) recreatePods() error {
LabelSelector: ls.String(),
}
pods, err := c.KubeClient.Pods(namespace).List(listOptions)
pods, err := c.KubeClient.Pods(namespace).List(context.TODO(), listOptions)
if err != nil {
return fmt.Errorf("could not get the list of pods: %v", err)
}
c.logger.Infof("there are %d pods in the cluster to recreate", len(pods.Items))
if !c.isSafeToRecreatePods(pods) {
return fmt.Errorf("postpone pod recreation until next Sync: recreation is unsafe because pods are being initialized")
}
var (
masterPod, newMasterPod, newPod *v1.Pod
)
@@ -349,7 +402,7 @@ func (c *Cluster) recreatePods() error {
}
func (c *Cluster) podIsEndOfLife(pod *v1.Pod) (bool, error) {
node, err := c.KubeClient.Nodes().Get(pod.Spec.NodeName, metav1.GetOptions{})
node, err := c.KubeClient.Nodes().Get(context.TODO(), pod.Spec.NodeName, metav1.GetOptions{})
if err != nil {
return false, err
}
pkg/cluster/resources.go
+72 -38
View File
@@ -1,6 +1,7 @@
package cluster
import (
"context"
"fmt"
"strconv"
"strings"
@@ -80,7 +81,10 @@ func (c *Cluster) createStatefulSet() (*appsv1.StatefulSet, error) {
if err != nil {
return nil, fmt.Errorf("could not generate statefulset: %v", err)
}
statefulSet, err := c.KubeClient.StatefulSets(statefulSetSpec.Namespace).Create(statefulSetSpec)
statefulSet, err := c.KubeClient.StatefulSets(statefulSetSpec.Namespace).Create(
context.TODO(),
statefulSetSpec,
metav1.CreateOptions{})
if err != nil {
return nil, err
}
@@ -125,7 +129,7 @@ func (c *Cluster) preScaleDown(newStatefulSet *appsv1.StatefulSet) error {
}
podName := fmt.Sprintf("%s-0", c.Statefulset.Name)
masterCandidatePod, err := c.KubeClient.Pods(c.clusterNamespace()).Get(podName, metav1.GetOptions{})
masterCandidatePod, err := c.KubeClient.Pods(c.clusterNamespace()).Get(context.TODO(), podName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("could not get master candidate pod: %v", err)
}
@@ -145,7 +149,7 @@ func (c *Cluster) preScaleDown(newStatefulSet *appsv1.StatefulSet) error {
// setRollingUpdateFlagForStatefulSet sets the indicator or the rolling update requirement
// in the StatefulSet annotation.
func (c *Cluster) setRollingUpdateFlagForStatefulSet(sset *appsv1.StatefulSet, val bool) {
func (c *Cluster) setRollingUpdateFlagForStatefulSet(sset *appsv1.StatefulSet, val bool, msg string) {
anno := sset.GetAnnotations()
if anno == nil {
anno = make(map[string]string)
@@ -153,13 +157,13 @@ func (c *Cluster) setRollingUpdateFlagForStatefulSet(sset *appsv1.StatefulSet, v
anno[rollingUpdateStatefulsetAnnotationKey] = strconv.FormatBool(val)
sset.SetAnnotations(anno)
c.logger.Debugf("statefulset's rolling update annotation has been set to %t", val)
c.logger.Debugf("set statefulset's rolling update annotation to %t: caller/reason %s", val, msg)
}
// applyRollingUpdateFlagforStatefulSet sets the rolling update flag for the cluster's StatefulSet
// and applies that setting to the actual running cluster.
func (c *Cluster) applyRollingUpdateFlagforStatefulSet(val bool) error {
c.setRollingUpdateFlagForStatefulSet(c.Statefulset, val)
c.setRollingUpdateFlagForStatefulSet(c.Statefulset, val, "applyRollingUpdateFlag")
sset, err := c.updateStatefulSetAnnotations(c.Statefulset.GetAnnotations())
if err != nil {
return err
@@ -211,22 +215,24 @@ func (c *Cluster) mergeRollingUpdateFlagUsingCache(runningStatefulSet *appsv1.St
podsRollingUpdateRequired = false
} else {
c.logger.Infof("found a statefulset with an unfinished rolling update of the pods")
}
}
return podsRollingUpdateRequired
}
func (c *Cluster) updateStatefulSetAnnotations(annotations map[string]string) (*appsv1.StatefulSet, error) {
c.logger.Debugf("updating statefulset annotations")
c.logger.Debugf("patching statefulset annotations")
patchData, err := metaAnnotationsPatch(annotations)
if err != nil {
return nil, fmt.Errorf("could not form patch for the statefulset metadata: %v", err)
}
result, err := c.KubeClient.StatefulSets(c.Statefulset.Namespace).Patch(
context.TODO(),
c.Statefulset.Name,
types.MergePatchType,
[]byte(patchData), "")
[]byte(patchData),
metav1.PatchOptions{},
"")
if err != nil {
return nil, fmt.Errorf("could not patch statefulset annotations %q: %v", patchData, err)
}
@@ -254,9 +260,12 @@ func (c *Cluster) updateStatefulSet(newStatefulSet *appsv1.StatefulSet) error {
}
statefulSet, err := c.KubeClient.StatefulSets(c.Statefulset.Namespace).Patch(
context.TODO(),
c.Statefulset.Name,
types.MergePatchType,
patchData, "")
patchData,
metav1.PatchOptions{},
"")
if err != nil {
return fmt.Errorf("could not patch statefulset spec %q: %v", statefulSetName, err)
}
@@ -288,7 +297,7 @@ func (c *Cluster) replaceStatefulSet(newStatefulSet *appsv1.StatefulSet) error {
oldStatefulset := c.Statefulset
options := metav1.DeleteOptions{PropagationPolicy: &deletePropagationPolicy}
err := c.KubeClient.StatefulSets(oldStatefulset.Namespace).Delete(oldStatefulset.Name, &options)
err := c.KubeClient.StatefulSets(oldStatefulset.Namespace).Delete(context.TODO(), oldStatefulset.Name, options)
if err != nil {
return fmt.Errorf("could not delete statefulset %q: %v", statefulSetName, err)
}
@@ -299,7 +308,7 @@ func (c *Cluster) replaceStatefulSet(newStatefulSet *appsv1.StatefulSet) error {
err = retryutil.Retry(c.OpConfig.ResourceCheckInterval, c.OpConfig.ResourceCheckTimeout,
func() (bool, error) {
_, err2 := c.KubeClient.StatefulSets(oldStatefulset.Namespace).Get(oldStatefulset.Name, metav1.GetOptions{})
_, err2 := c.KubeClient.StatefulSets(oldStatefulset.Namespace).Get(context.TODO(), oldStatefulset.Name, metav1.GetOptions{})
if err2 == nil {
return false, nil
}
@@ -313,7 +322,7 @@ func (c *Cluster) replaceStatefulSet(newStatefulSet *appsv1.StatefulSet) error {
}
// create the new statefulset with the desired spec. It would take over the remaining pods.
createdStatefulset, err := c.KubeClient.StatefulSets(newStatefulSet.Namespace).Create(newStatefulSet)
createdStatefulset, err := c.KubeClient.StatefulSets(newStatefulSet.Namespace).Create(context.TODO(), newStatefulSet, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("could not create statefulset %q: %v", statefulSetName, err)
}
@@ -334,7 +343,7 @@ func (c *Cluster) deleteStatefulSet() error {
return fmt.Errorf("there is no statefulset in the cluster")
}
err := c.KubeClient.StatefulSets(c.Statefulset.Namespace).Delete(c.Statefulset.Name, c.deleteOptions)
err := c.KubeClient.StatefulSets(c.Statefulset.Namespace).Delete(context.TODO(), c.Statefulset.Name, c.deleteOptions)
if err != nil {
return err
}
@@ -356,7 +365,7 @@ func (c *Cluster) createService(role PostgresRole) (*v1.Service, error) {
c.setProcessName("creating %v service", role)
serviceSpec := c.generateService(role, &c.Spec)
service, err := c.KubeClient.Services(serviceSpec.Namespace).Create(serviceSpec)
service, err := c.KubeClient.Services(serviceSpec.Namespace).Create(context.TODO(), serviceSpec, metav1.CreateOptions{})
if err != nil {
return nil, err
}
@@ -383,9 +392,12 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error
if len(newService.ObjectMeta.Annotations) > 0 {
if annotationsPatchData, err := metaAnnotationsPatch(newService.ObjectMeta.Annotations); err == nil {
_, err = c.KubeClient.Services(serviceName.Namespace).Patch(
context.TODO(),
serviceName.Name,
types.MergePatchType,
[]byte(annotationsPatchData), "")
[]byte(annotationsPatchData),
metav1.PatchOptions{},
"")
if err != nil {
return fmt.Errorf("could not replace annotations for the service %q: %v", serviceName, err)
@@ -402,7 +414,7 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error
if newServiceType == "ClusterIP" && newServiceType != oldServiceType {
newService.ResourceVersion = c.Services[role].ResourceVersion
newService.Spec.ClusterIP = c.Services[role].Spec.ClusterIP
svc, err = c.KubeClient.Services(serviceName.Namespace).Update(newService)
svc, err = c.KubeClient.Services(serviceName.Namespace).Update(context.TODO(), newService, metav1.UpdateOptions{})
if err != nil {
return fmt.Errorf("could not update service %q: %v", serviceName, err)
}
@@ -413,9 +425,7 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error
}
svc, err = c.KubeClient.Services(serviceName.Namespace).Patch(
serviceName.Name,
types.MergePatchType,
patchData, "")
context.TODO(), serviceName.Name, types.MergePatchType, patchData, metav1.PatchOptions{}, "")
if err != nil {
return fmt.Errorf("could not patch service %q: %v", serviceName, err)
}
@@ -434,7 +444,7 @@ func (c *Cluster) deleteService(role PostgresRole) error {
return nil
}
if err := c.KubeClient.Services(service.Namespace).Delete(service.Name, c.deleteOptions); err != nil {
if err := c.KubeClient.Services(service.Namespace).Delete(context.TODO(), service.Name, c.deleteOptions); err != nil {
return err
}
@@ -458,7 +468,7 @@ func (c *Cluster) createEndpoint(role PostgresRole) (*v1.Endpoints, error) {
}
endpointsSpec := c.generateEndpoint(role, subsets)
endpoints, err := c.KubeClient.Endpoints(endpointsSpec.Namespace).Create(endpointsSpec)
endpoints, err := c.KubeClient.Endpoints(endpointsSpec.Namespace).Create(context.TODO(), endpointsSpec, metav1.CreateOptions{})
if err != nil {
return nil, fmt.Errorf("could not create %s endpoint: %v", role, err)
}
@@ -500,7 +510,7 @@ func (c *Cluster) createPodDisruptionBudget() (*policybeta1.PodDisruptionBudget,
podDisruptionBudgetSpec := c.generatePodDisruptionBudget()
podDisruptionBudget, err := c.KubeClient.
PodDisruptionBudgets(podDisruptionBudgetSpec.Namespace).
Create(podDisruptionBudgetSpec)
Create(context.TODO(), podDisruptionBudgetSpec, metav1.CreateOptions{})
if err != nil {
return nil, err
@@ -521,7 +531,7 @@ func (c *Cluster) updatePodDisruptionBudget(pdb *policybeta1.PodDisruptionBudget
newPdb, err := c.KubeClient.
PodDisruptionBudgets(pdb.Namespace).
Create(pdb)
Create(context.TODO(), pdb, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("could not create pod disruption budget: %v", err)
}
@@ -539,7 +549,7 @@ func (c *Cluster) deletePodDisruptionBudget() error {
pdbName := util.NameFromMeta(c.PodDisruptionBudget.ObjectMeta)
err := c.KubeClient.
PodDisruptionBudgets(c.PodDisruptionBudget.Namespace).
Delete(c.PodDisruptionBudget.Name, c.deleteOptions)
Delete(context.TODO(), c.PodDisruptionBudget.Name, c.deleteOptions)
if err != nil {
return fmt.Errorf("could not delete pod disruption budget: %v", err)
}
@@ -548,7 +558,7 @@ func (c *Cluster) deletePodDisruptionBudget() error {
err = retryutil.Retry(c.OpConfig.ResourceCheckInterval, c.OpConfig.ResourceCheckTimeout,
func() (bool, error) {
_, err2 := c.KubeClient.PodDisruptionBudgets(pdbName.Namespace).Get(pdbName.Name, metav1.GetOptions{})
_, err2 := c.KubeClient.PodDisruptionBudgets(pdbName.Namespace).Get(context.TODO(), pdbName.Name, metav1.GetOptions{})
if err2 == nil {
return false, nil
}
@@ -571,7 +581,8 @@ func (c *Cluster) deleteEndpoint(role PostgresRole) error {
return fmt.Errorf("there is no %s endpoint in the cluster", role)
}
if err := c.KubeClient.Endpoints(c.Endpoints[role].Namespace).Delete(c.Endpoints[role].Name, c.deleteOptions); err != nil {
if err := c.KubeClient.Endpoints(c.Endpoints[role].Namespace).Delete(
context.TODO(), c.Endpoints[role].Name, c.deleteOptions); err != nil {
return fmt.Errorf("could not delete endpoint: %v", err)
}
@@ -582,17 +593,37 @@ func (c *Cluster) deleteEndpoint(role PostgresRole) error {
return nil
}
func (c *Cluster) deleteSecret(secret *v1.Secret) error {
c.setProcessName("deleting secret %q", util.NameFromMeta(secret.ObjectMeta))
c.logger.Debugf("deleting secret %q", util.NameFromMeta(secret.ObjectMeta))
err := c.KubeClient.Secrets(secret.Namespace).Delete(secret.Name, c.deleteOptions)
if err != nil {
return err
func (c *Cluster) deleteSecrets() error {
c.setProcessName("deleting secrets")
var errors []string
errorCount := 0
for uid, secret := range c.Secrets {
err := c.deleteSecret(uid, *secret)
if err != nil {
errors = append(errors, fmt.Sprintf("%v", err))
errorCount++
}
}
c.logger.Infof("secret %q has been deleted", util.NameFromMeta(secret.ObjectMeta))
delete(c.Secrets, secret.UID)
return err
if errorCount > 0 {
return fmt.Errorf("could not delete all secrets: %v", errors)
}
return nil
}
func (c *Cluster) deleteSecret(uid types.UID, secret v1.Secret) error {
c.setProcessName("deleting secret")
secretName := util.NameFromMeta(secret.ObjectMeta)
c.logger.Debugf("deleting secret %q", secretName)
err := c.KubeClient.Secrets(secret.Namespace).Delete(context.TODO(), secret.Name, c.deleteOptions)
if err != nil {
return fmt.Errorf("could not delete secret %q: %v", secretName, err)
}
c.logger.Infof("secret %q has been deleted", secretName)
c.Secrets[uid] = nil
return nil
}
func (c *Cluster) createRoles() (err error) {
@@ -610,7 +641,7 @@ func (c *Cluster) createLogicalBackupJob() (err error) {
}
c.logger.Debugf("Generated cronJobSpec: %v", logicalBackupJobSpec)
_, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Create(logicalBackupJobSpec)
_, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Create(context.TODO(), logicalBackupJobSpec, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("could not create k8s cron job: %v", err)
}
@@ -628,9 +659,12 @@ func (c *Cluster) patchLogicalBackupJob(newJob *batchv1beta1.CronJob) error {
// update the backup job spec
_, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Patch(
context.TODO(),
c.getLogicalBackupJobName(),
types.MergePatchType,
patchData, "")
patchData,
metav1.PatchOptions{},
"")
if err != nil {
return fmt.Errorf("could not patch logical backup job: %v", err)
}
@@ -642,7 +676,7 @@ func (c *Cluster) deleteLogicalBackupJob() error {
c.logger.Info("removing the logical backup job")
return c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Delete(c.getLogicalBackupJobName(), c.deleteOptions)
return c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Delete(context.TODO(), c.getLogicalBackupJobName(), c.deleteOptions)
}
// GetServiceMaster returns cluster's kubernetes master Service
pkg/cluster/sync.go
+261 -70
View File
@@ -1,19 +1,21 @@
package cluster
import (
"context"
"fmt"
batchv1beta1 "k8s.io/api/batch/v1beta1"
v1 "k8s.io/api/core/v1"
policybeta1 "k8s.io/api/policy/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"regexp"
"strings"
acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
"github.com/zalando/postgres-operator/pkg/spec"
"github.com/zalando/postgres-operator/pkg/util"
"github.com/zalando/postgres-operator/pkg/util/constants"
"github.com/zalando/postgres-operator/pkg/util/k8sutil"
"github.com/zalando/postgres-operator/pkg/util/volumes"
appsv1 "k8s.io/api/apps/v1"
batchv1beta1 "k8s.io/api/batch/v1beta1"
v1 "k8s.io/api/core/v1"
policybeta1 "k8s.io/api/policy/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// Sync syncs the cluster, making sure the actual Kubernetes objects correspond to what is defined in the manifest.
@@ -23,14 +25,15 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
c.mu.Lock()
defer c.mu.Unlock()
oldSpec := c.Postgresql
c.setSpec(newSpec)
defer func() {
if err != nil {
c.logger.Warningf("error while syncing cluster state: %v", err)
c.setStatus(acidv1.ClusterStatusSyncFailed)
c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusSyncFailed)
} else if !c.Status.Running() {
c.setStatus(acidv1.ClusterStatusRunning)
c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusRunning)
}
}()
@@ -39,29 +42,25 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
return err
}
c.logger.Debugf("syncing secrets")
//TODO: mind the secrets of the deleted/new users
if err = c.syncSecrets(); err != nil {
err = fmt.Errorf("could not sync secrets: %v", err)
return err
}
c.logger.Debugf("syncing services")
if err = c.syncServices(); err != nil {
err = fmt.Errorf("could not sync services: %v", err)
return err
}
// potentially enlarge volumes before changing the statefulset. By doing that
// in this order we make sure the operator is not stuck waiting for a pod that
// cannot start because it ran out of disk space.
// TODO: handle the case of the cluster that is downsized and enlarged again
// (there will be a volume from the old pod for which we can't act before the
// the statefulset modification is concluded)
c.logger.Debugf("syncing persistent volumes")
if c.OpConfig.EnableEBSGp3Migration {
err = c.executeEBSMigration()
if nil != err {
return err
}
}
if err = c.syncVolumes(); err != nil {
err = fmt.Errorf("could not sync persistent volumes: %v", err)
return err
}
@@ -106,6 +105,16 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
err = fmt.Errorf("could not sync databases: %v", err)
return err
}
c.logger.Debugf("syncing prepared databases with schemas")
if err = c.syncPreparedDatabases(); err != nil {
err = fmt.Errorf("could not sync prepared database: %v", err)
return err
}
}
// sync connection pooler
if _, err = c.syncConnectionPooler(&oldSpec, newSpec, c.installLookupFunction); err != nil {
return fmt.Errorf("could not sync connection pooler: %v", err)
}
return err
@@ -115,10 +124,11 @@ func (c *Cluster) syncServices() error {
for _, role := range []PostgresRole{Master, Replica} {
c.logger.Debugf("syncing %s service", role)
if err := c.syncEndpoint(role); err != nil {
return fmt.Errorf("could not sync %s endpoint: %v", role, err)
if !c.patroniKubernetesUseConfigMaps() {
if err := c.syncEndpoint(role); err != nil {
return fmt.Errorf("could not sync %s endpoint: %v", role, err)
}
}
if err := c.syncService(role); err != nil {
return fmt.Errorf("could not sync %s service: %v", role, err)
}
@@ -134,7 +144,7 @@ func (c *Cluster) syncService(role PostgresRole) error {
)
c.setProcessName("syncing %s service", role)
if svc, err = c.KubeClient.Services(c.Namespace).Get(c.serviceName(role), metav1.GetOptions{}); err == nil {
if svc, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.serviceName(role), metav1.GetOptions{}); err == nil {
c.Services[role] = svc
desiredSvc := c.generateService(role, &c.Spec)
if match, reason := k8sutil.SameService(svc, desiredSvc); !match {
@@ -160,7 +170,7 @@ func (c *Cluster) syncService(role PostgresRole) error {
return fmt.Errorf("could not create missing %s service: %v", role, err)
}
c.logger.Infof("%s service %q already exists", role, util.NameFromMeta(svc.ObjectMeta))
if svc, err = c.KubeClient.Services(c.Namespace).Get(c.serviceName(role), metav1.GetOptions{}); err != nil {
if svc, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.serviceName(role), metav1.GetOptions{}); err != nil {
return fmt.Errorf("could not fetch existing %s service: %v", role, err)
}
}
@@ -175,7 +185,7 @@ func (c *Cluster) syncEndpoint(role PostgresRole) error {
)
c.setProcessName("syncing %s endpoint", role)
if ep, err = c.KubeClient.Endpoints(c.Namespace).Get(c.endpointName(role), metav1.GetOptions{}); err == nil {
if ep, err = c.KubeClient.Endpoints(c.Namespace).Get(context.TODO(), c.endpointName(role), metav1.GetOptions{}); err == nil {
// TODO: No syncing of endpoints here, is this covered completely by updateService?
c.Endpoints[role] = ep
return nil
@@ -194,7 +204,7 @@ func (c *Cluster) syncEndpoint(role PostgresRole) error {
return fmt.Errorf("could not create missing %s endpoint: %v", role, err)
}
c.logger.Infof("%s endpoint %q already exists", role, util.NameFromMeta(ep.ObjectMeta))
if ep, err = c.KubeClient.Endpoints(c.Namespace).Get(c.endpointName(role), metav1.GetOptions{}); err != nil {
if ep, err = c.KubeClient.Endpoints(c.Namespace).Get(context.TODO(), c.endpointName(role), metav1.GetOptions{}); err != nil {
return fmt.Errorf("could not fetch existing %s endpoint: %v", role, err)
}
}
@@ -207,7 +217,7 @@ func (c *Cluster) syncPodDisruptionBudget(isUpdate bool) error {
pdb *policybeta1.PodDisruptionBudget
err error
)
if pdb, err = c.KubeClient.PodDisruptionBudgets(c.Namespace).Get(c.podDisruptionBudgetName(), metav1.GetOptions{}); err == nil {
if pdb, err = c.KubeClient.PodDisruptionBudgets(c.Namespace).Get(context.TODO(), c.podDisruptionBudgetName(), metav1.GetOptions{}); err == nil {
c.PodDisruptionBudget = pdb
newPDB := c.generatePodDisruptionBudget()
if match, reason := k8sutil.SamePDB(pdb, newPDB); !match {
@@ -233,7 +243,7 @@ func (c *Cluster) syncPodDisruptionBudget(isUpdate bool) error {
return fmt.Errorf("could not create pod disruption budget: %v", err)
}
c.logger.Infof("pod disruption budget %q already exists", util.NameFromMeta(pdb.ObjectMeta))
if pdb, err = c.KubeClient.PodDisruptionBudgets(c.Namespace).Get(c.podDisruptionBudgetName(), metav1.GetOptions{}); err != nil {
if pdb, err = c.KubeClient.PodDisruptionBudgets(c.Namespace).Get(context.TODO(), c.podDisruptionBudgetName(), metav1.GetOptions{}); err != nil {
return fmt.Errorf("could not fetch existing %q pod disruption budget", util.NameFromMeta(pdb.ObjectMeta))
}
}
@@ -244,12 +254,34 @@ func (c *Cluster) syncPodDisruptionBudget(isUpdate bool) error {
return nil
}
func (c *Cluster) mustUpdatePodsAfterLazyUpdate(desiredSset *appsv1.StatefulSet) (bool, error) {
pods, err := c.listPods()
if err != nil {
return false, fmt.Errorf("could not list pods of the statefulset: %v", err)
}
for _, pod := range pods {
effectivePodImage := pod.Spec.Containers[0].Image
ssImage := desiredSset.Spec.Template.Spec.Containers[0].Image
if ssImage != effectivePodImage {
c.logger.Infof("not all pods were re-started when the lazy upgrade was enabled; forcing the rolling upgrade now")
return true, nil
}
}
return false, nil
}
func (c *Cluster) syncStatefulSet() error {
var (
podsRollingUpdateRequired bool
)
// NB: Be careful to consider the codepath that acts on podsRollingUpdateRequired before returning early.
sset, err := c.KubeClient.StatefulSets(c.Namespace).Get(c.statefulSetName(), metav1.GetOptions{})
sset, err := c.KubeClient.StatefulSets(c.Namespace).Get(context.TODO(), c.statefulSetName(), metav1.GetOptions{})
if err != nil {
if !k8sutil.ResourceNotFound(err) {
return fmt.Errorf("could not get statefulset: %v", err)
@@ -289,13 +321,13 @@ func (c *Cluster) syncStatefulSet() error {
if err != nil {
return fmt.Errorf("could not generate statefulset: %v", err)
}
c.setRollingUpdateFlagForStatefulSet(desiredSS, podsRollingUpdateRequired)
c.setRollingUpdateFlagForStatefulSet(desiredSS, podsRollingUpdateRequired, "from cache")
cmp := c.compareStatefulSetWith(desiredSS)
if !cmp.match {
if cmp.rollingUpdate && !podsRollingUpdateRequired {
podsRollingUpdateRequired = true
c.setRollingUpdateFlagForStatefulSet(desiredSS, podsRollingUpdateRequired)
c.setRollingUpdateFlagForStatefulSet(desiredSS, podsRollingUpdateRequired, "statefulset changes")
}
c.logStatefulSetChanges(c.Statefulset, desiredSS, false, cmp.reasons)
@@ -310,6 +342,21 @@ func (c *Cluster) syncStatefulSet() error {
}
}
}
c.updateStatefulSetAnnotations(c.AnnotationsToPropagate(c.annotationsSet(c.Statefulset.Annotations)))
if !podsRollingUpdateRequired && !c.OpConfig.EnableLazySpiloUpgrade {
// even if desired and actual statefulsets match
// there still may be not up-to-date pods on condition
// (a) the lazy update was just disabled
// and
// (b) some of the pods were not restarted when the lazy update was still in place
podsRollingUpdateRequired, err = c.mustUpdatePodsAfterLazyUpdate(desiredSS)
if err != nil {
return fmt.Errorf("could not list pods of the statefulset: %v", err)
}
}
}
// Apply special PostgreSQL parameters that can only be set via the Patroni API.
@@ -323,10 +370,12 @@ func (c *Cluster) syncStatefulSet() error {
// statefulset or those that got their configuration from the outdated statefulset)
if podsRollingUpdateRequired {
c.logger.Debugln("performing rolling update")
c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", "Performing rolling update")
if err := c.recreatePods(); err != nil {
return fmt.Errorf("could not recreate pods: %v", err)
}
c.logger.Infof("pods have been recreated")
c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", "Rolling update done - pods have been recreated")
if err := c.applyRollingUpdateFlagforStatefulSet(false); err != nil {
c.logger.Warningf("could not clear rolling update for the statefulset: %v", err)
}
@@ -334,6 +383,38 @@ func (c *Cluster) syncStatefulSet() error {
return nil
}
// AnnotationsToPropagate get the annotations to update if required
// based on the annotations in postgres CRD
func (c *Cluster) AnnotationsToPropagate(annotations map[string]string) map[string]string {
if annotations == nil {
annotations = make(map[string]string)
}
pgCRDAnnotations := c.ObjectMeta.Annotations
if pgCRDAnnotations != nil {
for _, anno := range c.OpConfig.DownscalerAnnotations {
for k, v := range pgCRDAnnotations {
matched, err := regexp.MatchString(anno, k)
if err != nil {
c.logger.Errorf("annotations matching issue: %v", err)
return nil
}
if matched {
annotations[k] = v
}
}
}
}
if len(annotations) > 0 {
return annotations
}
return nil
}
// checkAndSetGlobalPostgreSQLConfiguration checks whether cluster-wide API parameters
// (like max_connections) has changed and if necessary sets it via the Patroni API
func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration() error {
@@ -382,25 +463,27 @@ func (c *Cluster) syncSecrets() error {
err error
secret *v1.Secret
)
c.logger.Info("syncing secrets")
c.setProcessName("syncing secrets")
secrets := c.generateUserSecrets()
for secretUsername, secretSpec := range secrets {
if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Create(secretSpec); err == nil {
if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Create(context.TODO(), secretSpec, metav1.CreateOptions{}); err == nil {
c.Secrets[secret.UID] = secret
c.logger.Debugf("created new secret %q, uid: %q", util.NameFromMeta(secret.ObjectMeta), secret.UID)
continue
}
if k8sutil.ResourceAlreadyExists(err) {
var userMap map[string]spec.PgUser
if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Get(secretSpec.Name, metav1.GetOptions{}); err != nil {
if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Get(context.TODO(), secretSpec.Name, metav1.GetOptions{}); err != nil {
return fmt.Errorf("could not get current secret: %v", err)
}
if secretUsername != string(secret.Data["username"]) {
c.logger.Warningf("secret %q does not contain the role %q", secretSpec.Name, secretUsername)
c.logger.Errorf("secret %s does not contain the role %q", secretSpec.Name, secretUsername)
continue
}
c.logger.Debugf("secret %q already exists, fetching its password", util.NameFromMeta(secret.ObjectMeta))
c.Secrets[secret.UID] = secret
c.logger.Debugf("secret %s already exists, fetching its password", util.NameFromMeta(secret.ObjectMeta))
if secretUsername == c.systemUsers[constants.SuperuserKeyName].Name {
secretUsername = constants.SuperuserKeyName
userMap = c.systemUsers
@@ -412,9 +495,11 @@ func (c *Cluster) syncSecrets() error {
}
pwdUser := userMap[secretUsername]
// if this secret belongs to the infrastructure role and the password has changed - replace it in the secret
if pwdUser.Password != string(secret.Data["password"]) && pwdUser.Origin == spec.RoleOriginInfrastructure {
if pwdUser.Password != string(secret.Data["password"]) &&
pwdUser.Origin == spec.RoleOriginInfrastructure {
c.logger.Debugf("updating the secret %q from the infrastructure roles", secretSpec.Name)
if _, err = c.KubeClient.Secrets(secretSpec.Namespace).Update(secretSpec); err != nil {
if _, err = c.KubeClient.Secrets(secretSpec.Namespace).Update(context.TODO(), secretSpec, metav1.UpdateOptions{}); err != nil {
return fmt.Errorf("could not update infrastructure role secret for role %q: %v", secretUsername, err)
}
} else {
@@ -456,6 +541,16 @@ func (c *Cluster) syncRoles() (err error) {
for _, u := range c.pgUsers {
userNames = append(userNames, u.Name)
}
if needMasterConnectionPooler(&c.Spec) || needReplicaConnectionPooler(&c.Spec) {
connectionPoolerUser := c.systemUsers[constants.ConnectionPoolerUserKeyName]
userNames = append(userNames, connectionPoolerUser.Name)
if _, exists := c.pgUsers[connectionPoolerUser.Name]; !exists {
c.pgUsers[connectionPoolerUser.Name] = connectionPoolerUser
}
}
dbUsers, err = c.readPgUsersFromDatabase(userNames)
if err != nil {
return fmt.Errorf("error getting users from the database: %v", err)
@@ -469,31 +564,12 @@ func (c *Cluster) syncRoles() (err error) {
return nil
}
// syncVolumes reads all persistent volumes and checks that their size matches the one declared in the statefulset.
func (c *Cluster) syncVolumes() error {
c.setProcessName("syncing volumes")
act, err := c.volumesNeedResizing(c.Spec.Volume)
if err != nil {
return fmt.Errorf("could not compare size of the volumes: %v", err)
}
if !act {
return nil
}
if err := c.resizeVolumes(c.Spec.Volume, []volumes.VolumeResizer{&volumes.EBSVolumeResizer{AWSRegion: c.OpConfig.AWSRegion}}); err != nil {
return fmt.Errorf("could not sync volumes: %v", err)
}
c.logger.Infof("volumes have been synced successfully")
return nil
}
func (c *Cluster) syncDatabases() error {
c.setProcessName("syncing databases")
createDatabases := make(map[string]string)
alterOwnerDatabases := make(map[string]string)
preparedDatabases := make([]string, 0)
if err := c.initDbConn(); err != nil {
return fmt.Errorf("could not init database connection")
@@ -509,12 +585,24 @@ func (c *Cluster) syncDatabases() error {
return fmt.Errorf("could not get current databases: %v", err)
}
for datname, newOwner := range c.Spec.Databases {
currentOwner, exists := currentDatabases[datname]
// if no prepared databases are specified create a database named like the cluster
if c.Spec.PreparedDatabases != nil && len(c.Spec.PreparedDatabases) == 0 { // TODO: add option to disable creating such a default DB
c.Spec.PreparedDatabases = map[string]acidv1.PreparedDatabase{strings.Replace(c.Name, "-", "_", -1): {}}
}
for preparedDatabaseName := range c.Spec.PreparedDatabases {
_, exists := currentDatabases[preparedDatabaseName]
if !exists {
createDatabases[datname] = newOwner
createDatabases[preparedDatabaseName] = preparedDatabaseName + constants.OwnerRoleNameSuffix
preparedDatabases = append(preparedDatabases, preparedDatabaseName)
}
}
for databaseName, newOwner := range c.Spec.Databases {
currentOwner, exists := currentDatabases[databaseName]
if !exists {
createDatabases[databaseName] = newOwner
} else if currentOwner != newOwner {
alterOwnerDatabases[datname] = newOwner
alterOwnerDatabases[databaseName] = newOwner
}
}
@@ -522,13 +610,116 @@ func (c *Cluster) syncDatabases() error {
return nil
}
for datname, owner := range createDatabases {
if err = c.executeCreateDatabase(datname, owner); err != nil {
for databaseName, owner := range createDatabases {
if err = c.executeCreateDatabase(databaseName, owner); err != nil {
return err
}
}
for datname, owner := range alterOwnerDatabases {
if err = c.executeAlterDatabaseOwner(datname, owner); err != nil {
for databaseName, owner := range alterOwnerDatabases {
if err = c.executeAlterDatabaseOwner(databaseName, owner); err != nil {
return err
}
}
// set default privileges for prepared database
for _, preparedDatabase := range preparedDatabases {
if err = c.execAlterGlobalDefaultPrivileges(preparedDatabase+constants.OwnerRoleNameSuffix, preparedDatabase); err != nil {
return err
}
}
return nil
}
func (c *Cluster) syncPreparedDatabases() error {
c.setProcessName("syncing prepared databases")
for preparedDbName, preparedDB := range c.Spec.PreparedDatabases {
if err := c.initDbConnWithName(preparedDbName); err != nil {
return fmt.Errorf("could not init connection to database %s: %v", preparedDbName, err)
}
c.logger.Debugf("syncing prepared database %q", preparedDbName)
// now, prepare defined schemas
preparedSchemas := preparedDB.PreparedSchemas
if len(preparedDB.PreparedSchemas) == 0 {
preparedSchemas = map[string]acidv1.PreparedSchema{"data": {DefaultRoles: util.True()}}
}
if err := c.syncPreparedSchemas(preparedDbName, preparedSchemas); err != nil {
return err
}
// install extensions
if err := c.syncExtensions(preparedDB.Extensions); err != nil {
return err
}
if err := c.closeDbConn(); err != nil {
c.logger.Errorf("could not close database connection: %v", err)
}
}
return nil
}
func (c *Cluster) syncPreparedSchemas(databaseName string, preparedSchemas map[string]acidv1.PreparedSchema) error {
c.setProcessName("syncing prepared schemas")
currentSchemas, err := c.getSchemas()
if err != nil {
return fmt.Errorf("could not get current schemas: %v", err)
}
var schemas []string
for schema := range preparedSchemas {
schemas = append(schemas, schema)
}
if createPreparedSchemas, equal := util.SubstractStringSlices(schemas, currentSchemas); !equal {
for _, schemaName := range createPreparedSchemas {
owner := constants.OwnerRoleNameSuffix
dbOwner := databaseName + owner
if preparedSchemas[schemaName].DefaultRoles == nil || *preparedSchemas[schemaName].DefaultRoles {
owner = databaseName + "_" + schemaName + owner
} else {
owner = dbOwner
}
if err = c.executeCreateDatabaseSchema(databaseName, schemaName, dbOwner, owner); err != nil {
return err
}
}
}
return nil
}
func (c *Cluster) syncExtensions(extensions map[string]string) error {
c.setProcessName("syncing database extensions")
createExtensions := make(map[string]string)
alterExtensions := make(map[string]string)
currentExtensions, err := c.getExtensions()
if err != nil {
return fmt.Errorf("could not get current database extensions: %v", err)
}
for extName, newSchema := range extensions {
currentSchema, exists := currentExtensions[extName]
if !exists {
createExtensions[extName] = newSchema
} else if currentSchema != newSchema {
alterExtensions[extName] = newSchema
}
}
for extName, schema := range createExtensions {
if err = c.executeCreateExtension(extName, schema); err != nil {
return err
}
}
for extName, schema := range alterExtensions {
if err = c.executeAlterExtension(extName, schema); err != nil {
return err
}
}
@@ -547,14 +738,14 @@ func (c *Cluster) syncLogicalBackupJob() error {
// sync the job if it exists
jobName := c.getLogicalBackupJobName()
if job, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Get(jobName, metav1.GetOptions{}); err == nil {
if job, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Get(context.TODO(), jobName, metav1.GetOptions{}); err == nil {
desiredJob, err = c.generateLogicalBackupJob()
if err != nil {
return fmt.Errorf("could not generate the desired logical backup job state: %v", err)
}
if match, reason := k8sutil.SameLogicalBackupJob(job, desiredJob); !match {
c.logger.Infof("logical job %q is not in the desired state and needs to be updated",
c.logger.Infof("logical job %s is not in the desired state and needs to be updated",
c.getLogicalBackupJobName(),
)
if reason != "" {
@@ -575,13 +766,13 @@ func (c *Cluster) syncLogicalBackupJob() error {
c.logger.Info("could not find the cluster's logical backup job")
if err = c.createLogicalBackupJob(); err == nil {
c.logger.Infof("created missing logical backup job %q", jobName)
c.logger.Infof("created missing logical backup job %s", jobName)
} else {
if !k8sutil.ResourceAlreadyExists(err) {
return fmt.Errorf("could not create missing logical backup job: %v", err)
}
c.logger.Infof("logical backup job %q already exists", jobName)
if _, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Get(jobName, metav1.GetOptions{}); err != nil {
c.logger.Infof("logical backup job %s already exists", jobName)
if _, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Get(context.TODO(), jobName, metav1.GetOptions{}); err != nil {
return fmt.Errorf("could not fetch existing logical backup job: %v", err)
}
}
pkg/cluster/types.go
+9
View File
@@ -69,3 +69,12 @@ type ClusterStatus struct {
Spec acidv1.PostgresSpec
Error error
}
type TemplateParams map[string]interface{}
type InstallFunction func(schema string, user string, role PostgresRole) error
type SyncReason []string
// no sync happened, empty value
var NoSync SyncReason = []string{}
pkg/cluster/util.go
+116 -23
View File
@@ -2,6 +2,7 @@ package cluster
import (
"bytes"
"context"
"encoding/gob"
"encoding/json"
"fmt"
@@ -17,12 +18,14 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"github.com/sirupsen/logrus"
acidzalando "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do"
acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
"github.com/zalando/postgres-operator/pkg/spec"
"github.com/zalando/postgres-operator/pkg/util"
"github.com/zalando/postgres-operator/pkg/util/constants"
"github.com/zalando/postgres-operator/pkg/util/k8sutil"
"github.com/zalando/postgres-operator/pkg/util/nicediff"
"github.com/zalando/postgres-operator/pkg/util/retryutil"
)
@@ -47,7 +50,7 @@ func (g *SecretOauthTokenGetter) getOAuthToken() (string, error) {
// Temporary getting postgresql-operator secret from the NamespaceDefault
credentialsSecret, err := g.kubeClient.
Secrets(g.OAuthTokenSecretName.Namespace).
Get(g.OAuthTokenSecretName.Name, metav1.GetOptions{})
Get(context.TODO(), g.OAuthTokenSecretName.Name, metav1.GetOptions{})
if err != nil {
return "", fmt.Errorf("could not get credentials secret: %v", err)
@@ -165,40 +168,59 @@ func (c *Cluster) logPDBChanges(old, new *policybeta1.PodDisruptionBudget, isUpd
)
}
c.logger.Debugf("diff\n%s\n", util.PrettyDiff(old.Spec, new.Spec))
logNiceDiff(c.logger, old.Spec, new.Spec)
}
func logNiceDiff(log *logrus.Entry, old, new interface{}) {
o, erro := json.MarshalIndent(old, "", " ")
n, errn := json.MarshalIndent(new, "", " ")
if erro != nil || errn != nil {
panic("could not marshal API objects, should not happen")
}
nice := nicediff.Diff(string(o), string(n), true)
for _, s := range strings.Split(nice, "\n") {
// " is not needed in the value to understand
log.Debugf(strings.ReplaceAll(s, "\"", ""))
}
}
func (c *Cluster) logStatefulSetChanges(old, new *appsv1.StatefulSet, isUpdate bool, reasons []string) {
if isUpdate {
c.logger.Infof("statefulset %q has been changed", util.NameFromMeta(old.ObjectMeta))
c.logger.Infof("statefulset %s has been changed", util.NameFromMeta(old.ObjectMeta))
} else {
c.logger.Infof("statefulset %q is not in the desired state and needs to be updated",
c.logger.Infof("statefulset %s is not in the desired state and needs to be updated",
util.NameFromMeta(old.ObjectMeta),
)
}
logNiceDiff(c.logger, old.Spec, new.Spec)
if !reflect.DeepEqual(old.Annotations, new.Annotations) {
c.logger.Debugf("metadata.annotation diff\n%s\n", util.PrettyDiff(old.Annotations, new.Annotations))
c.logger.Debugf("metadata.annotation are different")
logNiceDiff(c.logger, old.Annotations, new.Annotations)
}
c.logger.Debugf("spec diff between old and new statefulsets: \n%s\n", util.PrettyDiff(old.Spec, new.Spec))
if len(reasons) > 0 {
for _, reason := range reasons {
c.logger.Infof("reason: %q", reason)
c.logger.Infof("reason: %s", reason)
}
}
}
func (c *Cluster) logServiceChanges(role PostgresRole, old, new *v1.Service, isUpdate bool, reason string) {
if isUpdate {
c.logger.Infof("%s service %q has been changed",
c.logger.Infof("%s service %s has been changed",
role, util.NameFromMeta(old.ObjectMeta),
)
} else {
c.logger.Infof("%s service %q is not in the desired state and needs to be updated",
c.logger.Infof("%s service %s is not in the desired state and needs to be updated",
role, util.NameFromMeta(old.ObjectMeta),
)
}
c.logger.Debugf("diff\n%s\n", util.PrettyDiff(old.Spec, new.Spec))
logNiceDiff(c.logger, old.Spec, new.Spec)
if reason != "" {
c.logger.Infof("reason: %s", reason)
@@ -207,7 +229,7 @@ func (c *Cluster) logServiceChanges(role PostgresRole, old, new *v1.Service, isU
func (c *Cluster) logVolumeChanges(old, new acidv1.Volume) {
c.logger.Infof("volume specification has been changed")
c.logger.Debugf("diff\n%s\n", util.PrettyDiff(old, new))
logNiceDiff(c.logger, old, new)
}
func (c *Cluster) getTeamMembers(teamID string) ([]string, error) {
@@ -216,24 +238,64 @@ func (c *Cluster) getTeamMembers(teamID string) ([]string, error) {
return nil, fmt.Errorf("no teamId specified")
}
c.logger.Debugf("fetching possible additional team members for team %q", teamID)
members := []string{}
additionalMembers := c.PgTeamMap[teamID].AdditionalMembers
for _, member := range additionalMembers {
members = append(members, member)
}
if !c.OpConfig.EnableTeamsAPI {
c.logger.Debugf("team API is disabled, returning empty list of members for team %q", teamID)
return []string{}, nil
c.logger.Debugf("team API is disabled, only returning %d members for team %q", len(members), teamID)
return members, nil
}
token, err := c.oauthTokenGetter.getOAuthToken()
if err != nil {
c.logger.Warnf("could not get oauth token to authenticate to team service API, returning empty list of team members: %v", err)
return []string{}, nil
c.logger.Warnf("could not get oauth token to authenticate to team service API, only returning %d members for team %q: %v", len(members), teamID, err)
return members, nil
}
teamInfo, err := c.teamsAPIClient.TeamInfo(teamID, token)
if err != nil {
c.logger.Warnf("could not get team info for team %q, returning empty list of team members: %v", teamID, err)
return []string{}, nil
c.logger.Warnf("could not get team info for team %q, only returning %d members: %v", teamID, len(members), err)
return members, nil
}
return teamInfo.Members, nil
for _, member := range teamInfo.Members {
if !(util.SliceContains(members, member)) {
members = append(members, member)
}
}
return members, nil
}
// Returns annotations to be passed to child objects
func (c *Cluster) annotationsSet(annotations map[string]string) map[string]string {
if annotations == nil {
annotations = make(map[string]string)
}
pgCRDAnnotations := c.ObjectMeta.Annotations
// allow to inherit certain labels from the 'postgres' object
if pgCRDAnnotations != nil {
for k, v := range pgCRDAnnotations {
for _, match := range c.OpConfig.InheritedAnnotations {
if k == match {
annotations[k] = v
}
}
}
}
if len(annotations) > 0 {
return annotations
}
return nil
}
func (c *Cluster) waitForPodLabel(podEvents chan PodEvent, stopChan chan struct{}, role *PostgresRole) (*v1.Pod, error) {
@@ -278,7 +340,7 @@ func (c *Cluster) waitStatefulsetReady() error {
listOptions := metav1.ListOptions{
LabelSelector: c.labelsSet(false).String(),
}
ss, err := c.KubeClient.StatefulSets(c.Namespace).List(listOptions)
ss, err := c.KubeClient.StatefulSets(c.Namespace).List(context.TODO(), listOptions)
if err != nil {
return false, err
}
@@ -313,7 +375,7 @@ func (c *Cluster) _waitPodLabelsReady(anyReplica bool) error {
}
podsNumber = 1
if !anyReplica {
pods, err := c.KubeClient.Pods(namespace).List(listOptions)
pods, err := c.KubeClient.Pods(namespace).List(context.TODO(), listOptions)
if err != nil {
return err
}
@@ -327,7 +389,7 @@ func (c *Cluster) _waitPodLabelsReady(anyReplica bool) error {
func() (bool, error) {
masterCount := 0
if !anyReplica {
masterPods, err2 := c.KubeClient.Pods(namespace).List(masterListOption)
masterPods, err2 := c.KubeClient.Pods(namespace).List(context.TODO(), masterListOption)
if err2 != nil {
return false, err2
}
@@ -337,7 +399,7 @@ func (c *Cluster) _waitPodLabelsReady(anyReplica bool) error {
}
masterCount = len(masterPods.Items)
}
replicaPods, err2 := c.KubeClient.Pods(namespace).List(replicaListOption)
replicaPods, err2 := c.KubeClient.Pods(namespace).List(context.TODO(), replicaListOption)
if err2 != nil {
return false, err2
}
@@ -408,7 +470,10 @@ func (c *Cluster) labelsSet(shouldAddExtraLabels bool) labels.Set {
}
func (c *Cluster) labelsSelector() *metav1.LabelSelector {
return &metav1.LabelSelector{MatchLabels: c.labelsSet(false), MatchExpressions: nil}
return &metav1.LabelSelector{
MatchLabels: c.labelsSet(false),
MatchExpressions: nil,
}
}
func (c *Cluster) roleLabelsSet(shouldAddExtraLabels bool, role PostgresRole) labels.Set {
@@ -483,3 +548,31 @@ func (c *Cluster) GetSpec() (*acidv1.Postgresql, error) {
func (c *Cluster) patroniUsesKubernetes() bool {
return c.OpConfig.EtcdHost == ""
}
func (c *Cluster) patroniKubernetesUseConfigMaps() bool {
if !c.patroniUsesKubernetes() {
return false
}
// otherwise, follow the operator configuration
return c.OpConfig.KubernetesUseConfigMaps
}
// Earlier arguments take priority
func mergeContainers(containers ...[]v1.Container) ([]v1.Container, []string) {
containerNameTaken := map[string]bool{}
result := make([]v1.Container, 0)
conflicts := make([]string, 0)
for _, containerArray := range containers {
for _, container := range containerArray {
if _, taken := containerNameTaken[container.Name]; taken {
conflicts = append(conflicts, container.Name)
} else {
containerNameTaken[container.Name] = true
result = append(result, container)
}
}
}
return result, conflicts
}
pkg/cluster/util_test.go
+141
View File
@@ -0,0 +1,141 @@
package cluster
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
fakeacidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake"
"github.com/zalando/postgres-operator/pkg/util"
"github.com/zalando/postgres-operator/pkg/util/config"
"github.com/zalando/postgres-operator/pkg/util/k8sutil"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
k8sFake "k8s.io/client-go/kubernetes/fake"
)
func newFakeK8sAnnotationsClient() (k8sutil.KubernetesClient, *k8sFake.Clientset) {
clientSet := k8sFake.NewSimpleClientset()
acidClientSet := fakeacidv1.NewSimpleClientset()
return k8sutil.KubernetesClient{
PodDisruptionBudgetsGetter: clientSet.PolicyV1beta1(),
ServicesGetter: clientSet.CoreV1(),
StatefulSetsGetter: clientSet.AppsV1(),
PostgresqlsGetter: acidClientSet.AcidV1(),
}, clientSet
}
func TestInheritedAnnotations(t *testing.T) {
testName := "test inheriting annotations from manifest"
client, _ := newFakeK8sAnnotationsClient()
clusterName := "acid-test-cluster"
namespace := "default"
annotationValue := "acid"
role := Master
pg := acidv1.Postgresql{
ObjectMeta: metav1.ObjectMeta{
Name: clusterName,
Annotations: map[string]string{
"owned-by": annotationValue,
},
},
Spec: acidv1.PostgresSpec{
EnableReplicaConnectionPooler: boolToPointer(true),
Volume: acidv1.Volume{
Size: "1Gi",
},
},
}
var cluster = New(
Config{
OpConfig: config.Config{
ConnectionPooler: config.ConnectionPooler{
ConnectionPoolerDefaultCPURequest: "100m",
ConnectionPoolerDefaultCPULimit: "100m",
ConnectionPoolerDefaultMemoryRequest: "100Mi",
ConnectionPoolerDefaultMemoryLimit: "100Mi",
NumberOfInstances: int32ToPointer(1),
},
PodManagementPolicy: "ordered_ready",
Resources: config.Resources{
ClusterLabels: map[string]string{"application": "spilo"},
ClusterNameLabel: "cluster-name",
DefaultCPURequest: "300m",
DefaultCPULimit: "300m",
DefaultMemoryRequest: "300Mi",
DefaultMemoryLimit: "300Mi",
InheritedAnnotations: []string{"owned-by"},
PodRoleLabel: "spilo-role",
},
},
}, client, pg, logger, eventRecorder)
cluster.Name = clusterName
cluster.Namespace = namespace
// test annotationsSet function
inheritedAnnotations := cluster.annotationsSet(nil)
listOptions := metav1.ListOptions{
LabelSelector: cluster.labelsSet(false).String(),
}
// check statefulset annotations
_, err := cluster.createStatefulSet()
assert.NoError(t, err)
stsList, err := client.StatefulSets(namespace).List(context.TODO(), listOptions)
assert.NoError(t, err)
for _, sts := range stsList.Items {
if !(util.MapContains(sts.ObjectMeta.Annotations, inheritedAnnotations)) {
t.Errorf("%s: StatefulSet %v not inherited annotations %#v, got %#v", testName, sts.ObjectMeta.Name, inheritedAnnotations, sts.ObjectMeta.Annotations)
}
// pod template
if !(util.MapContains(sts.Spec.Template.ObjectMeta.Annotations, inheritedAnnotations)) {
t.Errorf("%s: pod template %v not inherited annotations %#v, got %#v", testName, sts.ObjectMeta.Name, inheritedAnnotations, sts.ObjectMeta.Annotations)
}
// pvc template
if util.MapContains(sts.Spec.VolumeClaimTemplates[0].Annotations, inheritedAnnotations) {
t.Errorf("%s: PVC template %v not expected to have inherited annotations %#v, got %#v", testName, sts.ObjectMeta.Name, inheritedAnnotations, sts.ObjectMeta.Annotations)
}
}
// check service annotations
cluster.createService(Master)
svcList, err := client.Services(namespace).List(context.TODO(), listOptions)
assert.NoError(t, err)
for _, svc := range svcList.Items {
if !(util.MapContains(svc.ObjectMeta.Annotations, inheritedAnnotations)) {
t.Errorf("%s: Service %v not inherited annotations %#v, got %#v", testName, svc.ObjectMeta.Name, inheritedAnnotations, svc.ObjectMeta.Annotations)
}
}
// check pod disruption budget annotations
cluster.createPodDisruptionBudget()
pdbList, err := client.PodDisruptionBudgets(namespace).List(context.TODO(), listOptions)
assert.NoError(t, err)
for _, pdb := range pdbList.Items {
if !(util.MapContains(pdb.ObjectMeta.Annotations, inheritedAnnotations)) {
t.Errorf("%s: Pod Disruption Budget %v not inherited annotations %#v, got %#v", testName, pdb.ObjectMeta.Name, inheritedAnnotations, pdb.ObjectMeta.Annotations)
}
}
// check pooler deployment annotations
cluster.ConnectionPooler = map[PostgresRole]*ConnectionPoolerObjects{}
cluster.ConnectionPooler[role] = &ConnectionPoolerObjects{
Name: cluster.connectionPoolerName(role),
ClusterName: cluster.ClusterName,
Namespace: cluster.Namespace,
Role: role,
}
deploy, err := cluster.generateConnectionPoolerDeployment(cluster.ConnectionPooler[role])
assert.NoError(t, err)
if !(util.MapContains(deploy.ObjectMeta.Annotations, inheritedAnnotations)) {
t.Errorf("%s: Deployment %v not inherited annotations %#v, got %#v", testName, deploy.ObjectMeta.Name, inheritedAnnotations, deploy.ObjectMeta.Annotations)
}
}
pkg/cluster/volumes.go
+368 -59
View File
@@ -1,14 +1,16 @@
package cluster
import (
"context"
"fmt"
"strconv"
"strings"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/aws/aws-sdk-go/aws"
acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
"github.com/zalando/postgres-operator/pkg/spec"
"github.com/zalando/postgres-operator/pkg/util"
@@ -17,13 +19,213 @@ import (
"github.com/zalando/postgres-operator/pkg/util/volumes"
)
func (c *Cluster) syncVolumes() error {
c.logger.Debugf("syncing volumes using %q storage resize mode", c.OpConfig.StorageResizeMode)
var err error
// check quantity string once, and do not bother with it anymore anywhere else
_, err = resource.ParseQuantity(c.Spec.Volume.Size)
if err != nil {
return fmt.Errorf("could not parse volume size from the manifest: %v", err)
}
if c.OpConfig.StorageResizeMode == "mixed" {
// mixed op uses AWS API to adjust size, throughput, iops, and calls pvc change for file system resize
// in case of errors we proceed to let K8s do its work, favoring disk space increase of other adjustments
err = c.populateVolumeMetaData()
if err != nil {
c.logger.Errorf("populating EBS meta data failed, skipping potential adjustements: %v", err)
} else {
err = c.syncUnderlyingEBSVolume()
if err != nil {
c.logger.Errorf("errors occured during EBS volume adjustments: %v", err)
}
}
// resize pvc to adjust filesystem size until better K8s support
if err = c.syncVolumeClaims(); err != nil {
err = fmt.Errorf("could not sync persistent volume claims: %v", err)
return err
}
} else if c.OpConfig.StorageResizeMode == "pvc" {
if err = c.syncVolumeClaims(); err != nil {
err = fmt.Errorf("could not sync persistent volume claims: %v", err)
return err
}
} else if c.OpConfig.StorageResizeMode == "ebs" {
// potentially enlarge volumes before changing the statefulset. By doing that
// in this order we make sure the operator is not stuck waiting for a pod that
// cannot start because it ran out of disk space.
// TODO: handle the case of the cluster that is downsized and enlarged again
// (there will be a volume from the old pod for which we can't act before the
// the statefulset modification is concluded)
if err = c.syncEbsVolumes(); err != nil {
err = fmt.Errorf("could not sync persistent volumes: %v", err)
return err
}
} else {
c.logger.Infof("Storage resize is disabled (storage_resize_mode is off). Skipping volume sync.")
}
return nil
}
func (c *Cluster) syncUnderlyingEBSVolume() error {
c.logger.Infof("starting to sync EBS volumes: type, iops, throughput, and size")
var err error
targetValue := c.Spec.Volume
newSize, err := resource.ParseQuantity(targetValue.Size)
targetSize := quantityToGigabyte(newSize)
awsGp3 := aws.String("gp3")
awsIo2 := aws.String("io2")
errors := []string{}
for _, volume := range c.EBSVolumes {
var modifyIops *int64
var modifyThroughput *int64
var modifySize *int64
var modifyType *string
if targetValue.Iops != nil {
if volume.Iops != *targetValue.Iops {
modifyIops = targetValue.Iops
}
}
if targetValue.Throughput != nil {
if volume.Throughput != *targetValue.Throughput {
modifyThroughput = targetValue.Throughput
}
}
if targetSize > volume.Size {
modifySize = &targetSize
}
if modifyIops != nil || modifyThroughput != nil || modifySize != nil {
if modifyIops != nil || modifyThroughput != nil {
// we default to gp3 if iops and throughput are configured
modifyType = awsGp3
if targetValue.VolumeType == "io2" {
modifyType = awsIo2
}
} else if targetValue.VolumeType == "gp3" && volume.VolumeType != "gp3" {
modifyType = awsGp3
} else {
// do not touch type
modifyType = nil
}
err = c.VolumeResizer.ModifyVolume(volume.VolumeID, modifyType, modifySize, modifyIops, modifyThroughput)
if err != nil {
errors = append(errors, fmt.Sprintf("modify volume failed: volume=%s size=%d iops=%d throughput=%d", volume.VolumeID, volume.Size, volume.Iops, volume.Throughput))
}
}
}
if len(errors) > 0 {
for _, s := range errors {
c.logger.Warningf(s)
}
// c.logger.Errorf("failed to modify %d of %d volumes", len(c.EBSVolumes), len(errors))
}
return nil
}
func (c *Cluster) populateVolumeMetaData() error {
c.logger.Infof("starting reading ebs meta data")
pvs, err := c.listPersistentVolumes()
if err != nil {
return fmt.Errorf("could not list persistent volumes: %v", err)
}
c.logger.Debugf("found %d volumes, size of known volumes %d", len(pvs), len(c.EBSVolumes))
volumeIds := []string{}
var volumeID string
for _, pv := range pvs {
volumeID, err = c.VolumeResizer.ExtractVolumeID(pv.Spec.AWSElasticBlockStore.VolumeID)
if err != nil {
continue
}
volumeIds = append(volumeIds, volumeID)
}
currentVolumes, err := c.VolumeResizer.DescribeVolumes(volumeIds)
if nil != err {
return err
}
if len(currentVolumes) != len(c.EBSVolumes) {
c.logger.Debugf("number of ebs volumes (%d) discovered differs from already known volumes (%d)", len(currentVolumes), len(c.EBSVolumes))
}
// reset map, operator is not responsible for dangling ebs volumes
c.EBSVolumes = make(map[string]volumes.VolumeProperties)
for _, volume := range currentVolumes {
c.EBSVolumes[volume.VolumeID] = volume
}
return nil
}
// syncVolumeClaims reads all persistent volume claims and checks that their size matches the one declared in the statefulset.
func (c *Cluster) syncVolumeClaims() error {
c.setProcessName("syncing volume claims")
needsResizing, err := c.volumeClaimsNeedResizing(c.Spec.Volume)
if err != nil {
return fmt.Errorf("could not compare size of the volume claims: %v", err)
}
if !needsResizing {
c.logger.Infof("volume claims do not require changes")
return nil
}
if err := c.resizeVolumeClaims(c.Spec.Volume); err != nil {
return fmt.Errorf("could not sync volume claims: %v", err)
}
c.logger.Infof("volume claims have been synced successfully")
return nil
}
// syncVolumes reads all persistent volumes and checks that their size matches the one declared in the statefulset.
func (c *Cluster) syncEbsVolumes() error {
c.setProcessName("syncing EBS and Claims volumes")
act, err := c.volumesNeedResizing()
if err != nil {
return fmt.Errorf("could not compare size of the volumes: %v", err)
}
if !act {
return nil
}
if err := c.resizeVolumes(); err != nil {
return fmt.Errorf("could not sync volumes: %v", err)
}
c.logger.Infof("volumes have been synced successfully")
return nil
}
func (c *Cluster) listPersistentVolumeClaims() ([]v1.PersistentVolumeClaim, error) {
ns := c.Namespace
listOptions := metav1.ListOptions{
LabelSelector: c.labelsSet(false).String(),
}
pvcs, err := c.KubeClient.PersistentVolumeClaims(ns).List(listOptions)
pvcs, err := c.KubeClient.PersistentVolumeClaims(ns).List(context.TODO(), listOptions)
if err != nil {
return nil, fmt.Errorf("could not list of PersistentVolumeClaims: %v", err)
}
@@ -38,7 +240,7 @@ func (c *Cluster) deletePersistentVolumeClaims() error {
}
for _, pvc := range pvcs {
c.logger.Debugf("deleting PVC %q", util.NameFromMeta(pvc.ObjectMeta))
if err := c.KubeClient.PersistentVolumeClaims(pvc.Namespace).Delete(pvc.Name, c.deleteOptions); err != nil {
if err := c.KubeClient.PersistentVolumeClaims(pvc.Namespace).Delete(context.TODO(), pvc.Name, c.deleteOptions); err != nil {
c.logger.Warningf("could not delete PersistentVolumeClaim: %v", err)
}
}
@@ -51,6 +253,35 @@ func (c *Cluster) deletePersistentVolumeClaims() error {
return nil
}
func (c *Cluster) resizeVolumeClaims(newVolume acidv1.Volume) error {
c.logger.Debugln("resizing PVCs")
pvcs, err := c.listPersistentVolumeClaims()
if err != nil {
return err
}
newQuantity, err := resource.ParseQuantity(newVolume.Size)
if err != nil {
return fmt.Errorf("could not parse volume size: %v", err)
}
newSize := quantityToGigabyte(newQuantity)
for _, pvc := range pvcs {
volumeSize := quantityToGigabyte(pvc.Spec.Resources.Requests[v1.ResourceStorage])
if volumeSize >= newSize {
if volumeSize > newSize {
c.logger.Warningf("cannot shrink persistent volume")
}
continue
}
pvc.Spec.Resources.Requests[v1.ResourceStorage] = newQuantity
c.logger.Debugf("updating persistent volume claim definition for volume %q", pvc.Name)
if _, err := c.KubeClient.PersistentVolumeClaims(pvc.Namespace).Update(context.TODO(), &pvc, metav1.UpdateOptions{}); err != nil {
return fmt.Errorf("could not update persistent volume claim: %q", err)
}
c.logger.Debugf("successfully updated persistent volume claim %q", pvc.Name)
}
return nil
}
func (c *Cluster) listPersistentVolumes() ([]*v1.PersistentVolume, error) {
result := make([]*v1.PersistentVolume, 0)
@@ -78,7 +309,7 @@ func (c *Cluster) listPersistentVolumes() ([]*v1.PersistentVolume, error) {
continue
}
}
pv, err := c.KubeClient.PersistentVolumes().Get(pvc.Spec.VolumeName, metav1.GetOptions{})
pv, err := c.KubeClient.PersistentVolumes().Get(context.TODO(), pvc.Spec.VolumeName, metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("could not get PersistentVolume: %v", err)
}
@@ -89,19 +320,27 @@ func (c *Cluster) listPersistentVolumes() ([]*v1.PersistentVolume, error) {
}
// resizeVolumes resize persistent volumes compatible with the given resizer interface
func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.VolumeResizer) error {
c.setProcessName("resizing volumes")
func (c *Cluster) resizeVolumes() error {
if c.VolumeResizer == nil {
return fmt.Errorf("no volume resizer set for EBS volume handling")
}
var totalIncompatible int
c.setProcessName("resizing EBS volumes")
newQuantity, err := resource.ParseQuantity(newVolume.Size)
newQuantity, err := resource.ParseQuantity(c.Spec.Volume.Size)
if err != nil {
return fmt.Errorf("could not parse volume size: %v", err)
}
pvs, newSize, err := c.listVolumesWithManifestSize(newVolume)
newSize := quantityToGigabyte(newQuantity)
resizer := c.VolumeResizer
var totalIncompatible int
pvs, err := c.listPersistentVolumes()
if err != nil {
return fmt.Errorf("could not list persistent volumes: %v", err)
}
for _, pv := range pvs {
volumeSize := quantityToGigabyte(pv.Spec.Capacity[v1.ResourceStorage])
if volumeSize >= newSize {
@@ -111,45 +350,45 @@ func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.Volu
continue
}
compatible := false
for _, resizer := range resizers {
if !resizer.VolumeBelongsToProvider(pv) {
continue
}
compatible = true
if !resizer.IsConnectedToProvider() {
err := resizer.ConnectToProvider()
if err != nil {
return fmt.Errorf("could not connect to the volume provider: %v", err)
}
defer func() {
if err := resizer.DisconnectFromProvider(); err != nil {
c.logger.Errorf("%v", err)
}
}()
}
awsVolumeID, err := resizer.GetProviderVolumeID(pv)
if err != nil {
return err
}
c.logger.Debugf("updating persistent volume %q to %d", pv.Name, newSize)
if err := resizer.ResizeVolume(awsVolumeID, newSize); err != nil {
return fmt.Errorf("could not resize EBS volume %q: %v", awsVolumeID, err)
}
c.logger.Debugf("resizing the filesystem on the volume %q", pv.Name)
podName := getPodNameFromPersistentVolume(pv)
if err := c.resizePostgresFilesystem(podName, []filesystems.FilesystemResizer{&filesystems.Ext234Resize{}}); err != nil {
return fmt.Errorf("could not resize the filesystem on pod %q: %v", podName, err)
}
c.logger.Debugf("filesystem resize successful on volume %q", pv.Name)
pv.Spec.Capacity[v1.ResourceStorage] = newQuantity
c.logger.Debugf("updating persistent volume definition for volume %q", pv.Name)
if _, err := c.KubeClient.PersistentVolumes().Update(pv); err != nil {
return fmt.Errorf("could not update persistent volume: %q", err)
}
c.logger.Debugf("successfully updated persistent volume %q", pv.Name)
if !resizer.VolumeBelongsToProvider(pv) {
continue
}
compatible = true
if !resizer.IsConnectedToProvider() {
err := resizer.ConnectToProvider()
if err != nil {
return fmt.Errorf("could not connect to the volume provider: %v", err)
}
defer func() {
if err := resizer.DisconnectFromProvider(); err != nil {
c.logger.Errorf("%v", err)
}
}()
}
awsVolumeID, err := resizer.GetProviderVolumeID(pv)
if err != nil {
return err
}
c.logger.Debugf("updating persistent volume %q to %d", pv.Name, newSize)
if err := resizer.ResizeVolume(awsVolumeID, newSize); err != nil {
return fmt.Errorf("could not resize EBS volume %q: %v", awsVolumeID, err)
}
c.logger.Debugf("resizing the filesystem on the volume %q", pv.Name)
podName := getPodNameFromPersistentVolume(pv)
if err := c.resizePostgresFilesystem(podName, []filesystems.FilesystemResizer{&filesystems.Ext234Resize{}}); err != nil {
return fmt.Errorf("could not resize the filesystem on pod %q: %v", podName, err)
}
c.logger.Debugf("filesystem resize successful on volume %q", pv.Name)
pv.Spec.Capacity[v1.ResourceStorage] = newQuantity
c.logger.Debugf("updating persistent volume definition for volume %q", pv.Name)
if _, err := c.KubeClient.PersistentVolumes().Update(context.TODO(), pv, metav1.UpdateOptions{}); err != nil {
return fmt.Errorf("could not update persistent volume: %q", err)
}
c.logger.Debugf("successfully updated persistent volume %q", pv.Name)
if !compatible {
c.logger.Warningf("volume %q is incompatible with all available resizing providers", pv.Name)
c.logger.Warningf("volume %q is incompatible with all available resizing providers, consider switching storage_resize_mode to pvc or off", pv.Name)
totalIncompatible++
}
}
@@ -159,13 +398,18 @@ func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.Volu
return nil
}
func (c *Cluster) volumesNeedResizing(newVolume acidv1.Volume) (bool, error) {
vols, manifestSize, err := c.listVolumesWithManifestSize(newVolume)
func (c *Cluster) volumeClaimsNeedResizing(newVolume acidv1.Volume) (bool, error) {
newSize, err := resource.ParseQuantity(newVolume.Size)
manifestSize := quantityToGigabyte(newSize)
if err != nil {
return false, err
return false, fmt.Errorf("could not parse volume size from the manifest: %v", err)
}
for _, pv := range vols {
currentSize := quantityToGigabyte(pv.Spec.Capacity[v1.ResourceStorage])
pvcs, err := c.listPersistentVolumeClaims()
if err != nil {
return false, fmt.Errorf("could not receive persistent volume claims: %v", err)
}
for _, pvc := range pvcs {
currentSize := quantityToGigabyte(pvc.Spec.Resources.Requests[v1.ResourceStorage])
if currentSize != manifestSize {
return true, nil
}
@@ -173,17 +417,21 @@ func (c *Cluster) volumesNeedResizing(newVolume acidv1.Volume) (bool, error) {
return false, nil
}
func (c *Cluster) listVolumesWithManifestSize(newVolume acidv1.Volume) ([]*v1.PersistentVolume, int64, error) {
newSize, err := resource.ParseQuantity(newVolume.Size)
if err != nil {
return nil, 0, fmt.Errorf("could not parse volume size from the manifest: %v", err)
}
manifestSize := quantityToGigabyte(newSize)
func (c *Cluster) volumesNeedResizing() (bool, error) {
newQuantity, _ := resource.ParseQuantity(c.Spec.Volume.Size)
newSize := quantityToGigabyte(newQuantity)
vols, err := c.listPersistentVolumes()
if err != nil {
return nil, 0, fmt.Errorf("could not list persistent volumes: %v", err)
return false, err
}
return vols, manifestSize, nil
for _, pv := range vols {
currentSize := quantityToGigabyte(pv.Spec.Capacity[v1.ResourceStorage])
if currentSize != newSize {
return true, nil
}
}
return false, nil
}
// getPodNameFromPersistentVolume returns a pod name that it extracts from the volume claim ref.
@@ -196,3 +444,64 @@ func getPodNameFromPersistentVolume(pv *v1.PersistentVolume) *spec.NamespacedNam
func quantityToGigabyte(q resource.Quantity) int64 {
return q.ScaledValue(0) / (1 * constants.Gigabyte)
}
func (c *Cluster) executeEBSMigration() error {
if !c.OpConfig.EnableEBSGp3Migration {
return nil
}
c.logger.Infof("starting EBS gp2 to gp3 migration")
pvs, err := c.listPersistentVolumes()
if err != nil {
return fmt.Errorf("could not list persistent volumes: %v", err)
}
c.logger.Debugf("found %d volumes, size of known volumes %d", len(pvs), len(c.EBSVolumes))
volumeIds := []string{}
var volumeID string
for _, pv := range pvs {
volumeID, err = c.VolumeResizer.ExtractVolumeID(pv.Spec.AWSElasticBlockStore.VolumeID)
if err != nil {
continue
}
volumeIds = append(volumeIds, volumeID)
}
if len(volumeIds) == len(c.EBSVolumes) {
hasGp2 := false
for _, v := range c.EBSVolumes {
if v.VolumeType == "gp2" {
hasGp2 = true
}
}
if !hasGp2 {
c.logger.Infof("no EBS gp2 volumes left to migrate")
return nil
}
}
awsVolumes, err := c.VolumeResizer.DescribeVolumes(volumeIds)
if nil != err {
return err
}
var i3000 int64 = 3000
var i125 int64 = 125
for _, volume := range awsVolumes {
if volume.VolumeType == "gp2" && volume.Size < c.OpConfig.EnableEBSGp3MigrationMaxSize {
c.logger.Infof("modifying EBS volume %s to type gp3 migration (%d)", volume.VolumeID, volume.Size)
err = c.VolumeResizer.ModifyVolume(volume.VolumeID, aws.String("gp3"), &volume.Size, &i3000, &i125)
if nil != err {
c.logger.Warningf("modifying volume %s failed: %v", volume.VolumeID, err)
}
} else {
c.logger.Debugf("skipping EBS volume %s to type gp3 migration (%d)", volume.VolumeID, volume.Size)
}
c.EBSVolumes[volume.VolumeID] = volume
}
return nil
}
pkg/cluster/volumes_test.go
+464
View File
@@ -0,0 +1,464 @@
package cluster
import (
"fmt"
"testing"
"context"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"github.com/aws/aws-sdk-go/aws"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
"github.com/zalando/postgres-operator/mocks"
acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
"github.com/zalando/postgres-operator/pkg/util/config"
"github.com/zalando/postgres-operator/pkg/util/constants"
"github.com/zalando/postgres-operator/pkg/util/k8sutil"
"github.com/zalando/postgres-operator/pkg/util/volumes"
"k8s.io/client-go/kubernetes/fake"
)
func newFakeK8sPVCclient() (k8sutil.KubernetesClient, *fake.Clientset) {
clientSet := fake.NewSimpleClientset()
return k8sutil.KubernetesClient{
PersistentVolumeClaimsGetter: clientSet.CoreV1(),
PersistentVolumesGetter: clientSet.CoreV1(),
PodsGetter: clientSet.CoreV1(),
}, clientSet
}
func TestResizeVolumeClaim(t *testing.T) {
testName := "test resizing of persistent volume claims"
client, _ := newFakeK8sPVCclient()
clusterName := "acid-test-cluster"
namespace := "default"
newVolumeSize := "2Gi"
storage1Gi, err := resource.ParseQuantity("1Gi")
assert.NoError(t, err)
// new cluster with pvc storage resize mode and configured labels
var cluster = New(
Config{
OpConfig: config.Config{
Resources: config.Resources{
ClusterLabels: map[string]string{"application": "spilo"},
ClusterNameLabel: "cluster-name",
},
StorageResizeMode: "pvc",
},
}, client, acidv1.Postgresql{}, logger, eventRecorder)
// set metadata, so that labels will get correct values
cluster.Name = clusterName
cluster.Namespace = namespace
filterLabels := cluster.labelsSet(false)
// define and create PVCs for 1Gi volumes
pvcList := CreatePVCs(namespace, clusterName, filterLabels, 2, "1Gi")
// add another PVC with different cluster name
pvcList.Items = append(pvcList.Items, CreatePVCs(namespace, clusterName+"-2", labels.Set{}, 1, "1Gi").Items[0])
for _, pvc := range pvcList.Items {
cluster.KubeClient.PersistentVolumeClaims(namespace).Create(context.TODO(), &pvc, metav1.CreateOptions{})
}
// test resizing
cluster.resizeVolumeClaims(acidv1.Volume{Size: newVolumeSize})
pvcs, err := cluster.listPersistentVolumeClaims()
assert.NoError(t, err)
// check if listPersistentVolumeClaims returns only the PVCs matching the filter
if len(pvcs) != len(pvcList.Items)-1 {
t.Errorf("%s: could not find all PVCs, got %v, expected %v", testName, len(pvcs), len(pvcList.Items)-1)
}
// check if PVCs were correctly resized
for _, pvc := range pvcs {
newStorageSize := quantityToGigabyte(pvc.Spec.Resources.Requests[v1.ResourceStorage])
expectedQuantity, err := resource.ParseQuantity(newVolumeSize)
assert.NoError(t, err)
expectedSize := quantityToGigabyte(expectedQuantity)
if newStorageSize != expectedSize {
t.Errorf("%s: resizing failed, got %v, expected %v", testName, newStorageSize, expectedSize)
}
}
// check if other PVC was not resized
pvc2, err := cluster.KubeClient.PersistentVolumeClaims(namespace).Get(context.TODO(), constants.DataVolumeName+"-"+clusterName+"-2-0", metav1.GetOptions{})
assert.NoError(t, err)
unchangedSize := quantityToGigabyte(pvc2.Spec.Resources.Requests[v1.ResourceStorage])
expectedSize := quantityToGigabyte(storage1Gi)
if unchangedSize != expectedSize {
t.Errorf("%s: volume size changed, got %v, expected %v", testName, unchangedSize, expectedSize)
}
}
func TestQuantityToGigabyte(t *testing.T) {
tests := []struct {
name string
quantityStr string
expected int64
}{
{
"test with 1Gi",
"1Gi",
1,
},
{
"test with float",
"1.5Gi",
int64(1),
},
{
"test with 1000Mi",
"1000Mi",
int64(0),
},
}
for _, tt := range tests {
quantity, err := resource.ParseQuantity(tt.quantityStr)
assert.NoError(t, err)
gigabyte := quantityToGigabyte(quantity)
if gigabyte != tt.expected {
t.Errorf("%s: got %v, expected %v", tt.name, gigabyte, tt.expected)
}
}
}
func CreatePVCs(namespace string, clusterName string, labels labels.Set, n int, size string) v1.PersistentVolumeClaimList {
// define and create PVCs for 1Gi volumes
storage1Gi, _ := resource.ParseQuantity(size)
pvcList := v1.PersistentVolumeClaimList{
Items: []v1.PersistentVolumeClaim{},
}
for i := 0; i < n; i++ {
pvc := v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-%s-%d", constants.DataVolumeName, clusterName, i),
Namespace: namespace,
Labels: labels,
},
Spec: v1.PersistentVolumeClaimSpec{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceStorage: storage1Gi,
},
},
VolumeName: fmt.Sprintf("persistent-volume-%d", i),
},
}
pvcList.Items = append(pvcList.Items, pvc)
}
return pvcList
}
func TestMigrateEBS(t *testing.T) {
client, _ := newFakeK8sPVCclient()
clusterName := "acid-test-cluster"
namespace := "default"
// new cluster with pvc storage resize mode and configured labels
var cluster = New(
Config{
OpConfig: config.Config{
Resources: config.Resources{
ClusterLabels: map[string]string{"application": "spilo"},
ClusterNameLabel: "cluster-name",
},
StorageResizeMode: "pvc",
EnableEBSGp3Migration: true,
EnableEBSGp3MigrationMaxSize: 1000,
},
}, client, acidv1.Postgresql{}, logger, eventRecorder)
cluster.Spec.Volume.Size = "1Gi"
// set metadata, so that labels will get correct values
cluster.Name = clusterName
cluster.Namespace = namespace
filterLabels := cluster.labelsSet(false)
testVolumes := []testVolume{
{
size: 100,
},
{
size: 100,
},
}
initTestVolumesAndPods(cluster.KubeClient, namespace, clusterName, filterLabels, testVolumes)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
resizer := mocks.NewMockVolumeResizer(ctrl)
resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-1")).Return("ebs-volume-1", nil)
resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-2")).Return("ebs-volume-2", nil)
resizer.EXPECT().DescribeVolumes(gomock.Eq([]string{"ebs-volume-1", "ebs-volume-2"})).Return(
[]volumes.VolumeProperties{
{VolumeID: "ebs-volume-1", VolumeType: "gp2", Size: 100},
{VolumeID: "ebs-volume-2", VolumeType: "gp3", Size: 100}}, nil)
// expect only gp2 volume to be modified
resizer.EXPECT().ModifyVolume(gomock.Eq("ebs-volume-1"), gomock.Eq(aws.String("gp3")), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
cluster.VolumeResizer = resizer
cluster.executeEBSMigration()
}
type testVolume struct {
iops int64
throughtput int64
size int64
volType string
}
func initTestVolumesAndPods(client k8sutil.KubernetesClient, namespace, clustername string, labels labels.Set, volumes []testVolume) {
i := 0
for _, v := range volumes {
storage1Gi, _ := resource.ParseQuantity(fmt.Sprintf("%d", v.size))
ps := v1.PersistentVolumeSpec{}
ps.AWSElasticBlockStore = &v1.AWSElasticBlockStoreVolumeSource{}
ps.AWSElasticBlockStore.VolumeID = fmt.Sprintf("aws://eu-central-1b/ebs-volume-%d", i+1)
pv := v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("persistent-volume-%d", i),
},
Spec: ps,
}
client.PersistentVolumes().Create(context.TODO(), &pv, metav1.CreateOptions{})
pvc := v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-%s-%d", constants.DataVolumeName, clustername, i),
Namespace: namespace,
Labels: labels,
},
Spec: v1.PersistentVolumeClaimSpec{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceStorage: storage1Gi,
},
},
VolumeName: fmt.Sprintf("persistent-volume-%d", i),
},
}
client.PersistentVolumeClaims(namespace).Create(context.TODO(), &pvc, metav1.CreateOptions{})
pod := v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-%d", clustername, i),
Labels: labels,
},
Spec: v1.PodSpec{},
}
client.Pods(namespace).Create(context.TODO(), &pod, metav1.CreateOptions{})
i = i + 1
}
}
func TestMigrateGp3Support(t *testing.T) {
client, _ := newFakeK8sPVCclient()
clusterName := "acid-test-cluster"
namespace := "default"
// new cluster with pvc storage resize mode and configured labels
var cluster = New(
Config{
OpConfig: config.Config{
Resources: config.Resources{
ClusterLabels: map[string]string{"application": "spilo"},
ClusterNameLabel: "cluster-name",
},
StorageResizeMode: "mixed",
EnableEBSGp3Migration: false,
EnableEBSGp3MigrationMaxSize: 1000,
},
}, client, acidv1.Postgresql{}, logger, eventRecorder)
cluster.Spec.Volume.Size = "150Gi"
cluster.Spec.Volume.Iops = aws.Int64(6000)
cluster.Spec.Volume.Throughput = aws.Int64(275)
// set metadata, so that labels will get correct values
cluster.Name = clusterName
cluster.Namespace = namespace
filterLabels := cluster.labelsSet(false)
testVolumes := []testVolume{
{
size: 100,
},
{
size: 100,
},
{
size: 100,
},
}
initTestVolumesAndPods(cluster.KubeClient, namespace, clusterName, filterLabels, testVolumes)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
resizer := mocks.NewMockVolumeResizer(ctrl)
resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-1")).Return("ebs-volume-1", nil)
resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-2")).Return("ebs-volume-2", nil)
resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-3")).Return("ebs-volume-3", nil)
resizer.EXPECT().DescribeVolumes(gomock.Eq([]string{"ebs-volume-1", "ebs-volume-2", "ebs-volume-3"})).Return(
[]volumes.VolumeProperties{
{VolumeID: "ebs-volume-1", VolumeType: "gp3", Size: 100, Iops: 3000},
{VolumeID: "ebs-volume-2", VolumeType: "gp3", Size: 105, Iops: 4000},
{VolumeID: "ebs-volume-3", VolumeType: "gp3", Size: 151, Iops: 6000, Throughput: 275}}, nil)
// expect only gp2 volume to be modified
resizer.EXPECT().ModifyVolume(gomock.Eq("ebs-volume-1"), gomock.Eq(aws.String("gp3")), gomock.Eq(aws.Int64(150)), gomock.Eq(aws.Int64(6000)), gomock.Eq(aws.Int64(275))).Return(nil)
resizer.EXPECT().ModifyVolume(gomock.Eq("ebs-volume-2"), gomock.Eq(aws.String("gp3")), gomock.Eq(aws.Int64(150)), gomock.Eq(aws.Int64(6000)), gomock.Eq(aws.Int64(275))).Return(nil)
// resizer.EXPECT().ModifyVolume(gomock.Eq("ebs-volume-3"), gomock.Eq(aws.String("gp3")), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
cluster.VolumeResizer = resizer
cluster.syncVolumes()
}
func TestManualGp2Gp3Support(t *testing.T) {
client, _ := newFakeK8sPVCclient()
clusterName := "acid-test-cluster"
namespace := "default"
// new cluster with pvc storage resize mode and configured labels
var cluster = New(
Config{
OpConfig: config.Config{
Resources: config.Resources{
ClusterLabels: map[string]string{"application": "spilo"},
ClusterNameLabel: "cluster-name",
},
StorageResizeMode: "mixed",
EnableEBSGp3Migration: false,
EnableEBSGp3MigrationMaxSize: 1000,
},
}, client, acidv1.Postgresql{}, logger, eventRecorder)
cluster.Spec.Volume.Size = "150Gi"
cluster.Spec.Volume.Iops = aws.Int64(6000)
cluster.Spec.Volume.Throughput = aws.Int64(275)
// set metadata, so that labels will get correct values
cluster.Name = clusterName
cluster.Namespace = namespace
filterLabels := cluster.labelsSet(false)
testVolumes := []testVolume{
{
size: 100,
},
{
size: 100,
},
}
initTestVolumesAndPods(cluster.KubeClient, namespace, clusterName, filterLabels, testVolumes)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
resizer := mocks.NewMockVolumeResizer(ctrl)
resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-1")).Return("ebs-volume-1", nil)
resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-2")).Return("ebs-volume-2", nil)
resizer.EXPECT().DescribeVolumes(gomock.Eq([]string{"ebs-volume-1", "ebs-volume-2"})).Return(
[]volumes.VolumeProperties{
{VolumeID: "ebs-volume-1", VolumeType: "gp2", Size: 150, Iops: 3000},
{VolumeID: "ebs-volume-2", VolumeType: "gp2", Size: 150, Iops: 4000},
}, nil)
// expect only gp2 volume to be modified
resizer.EXPECT().ModifyVolume(gomock.Eq("ebs-volume-1"), gomock.Eq(aws.String("gp3")), gomock.Nil(), gomock.Eq(aws.Int64(6000)), gomock.Eq(aws.Int64(275))).Return(nil)
resizer.EXPECT().ModifyVolume(gomock.Eq("ebs-volume-2"), gomock.Eq(aws.String("gp3")), gomock.Nil(), gomock.Eq(aws.Int64(6000)), gomock.Eq(aws.Int64(275))).Return(nil)
cluster.VolumeResizer = resizer
cluster.syncVolumes()
}
func TestDontTouchType(t *testing.T) {
client, _ := newFakeK8sPVCclient()
clusterName := "acid-test-cluster"
namespace := "default"
// new cluster with pvc storage resize mode and configured labels
var cluster = New(
Config{
OpConfig: config.Config{
Resources: config.Resources{
ClusterLabels: map[string]string{"application": "spilo"},
ClusterNameLabel: "cluster-name",
},
StorageResizeMode: "mixed",
EnableEBSGp3Migration: false,
EnableEBSGp3MigrationMaxSize: 1000,
},
}, client, acidv1.Postgresql{}, logger, eventRecorder)
cluster.Spec.Volume.Size = "177Gi"
// set metadata, so that labels will get correct values
cluster.Name = clusterName
cluster.Namespace = namespace
filterLabels := cluster.labelsSet(false)
testVolumes := []testVolume{
{
size: 150,
},
{
size: 150,
},
}
initTestVolumesAndPods(cluster.KubeClient, namespace, clusterName, filterLabels, testVolumes)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
resizer := mocks.NewMockVolumeResizer(ctrl)
resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-1")).Return("ebs-volume-1", nil)
resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-2")).Return("ebs-volume-2", nil)
resizer.EXPECT().DescribeVolumes(gomock.Eq([]string{"ebs-volume-1", "ebs-volume-2"})).Return(
[]volumes.VolumeProperties{
{VolumeID: "ebs-volume-1", VolumeType: "gp2", Size: 150, Iops: 3000},
{VolumeID: "ebs-volume-2", VolumeType: "gp2", Size: 150, Iops: 4000},
}, nil)
// expect only gp2 volume to be modified
resizer.EXPECT().ModifyVolume(gomock.Eq("ebs-volume-1"), gomock.Nil(), gomock.Eq(aws.Int64(177)), gomock.Nil(), gomock.Nil()).Return(nil)
resizer.EXPECT().ModifyVolume(gomock.Eq("ebs-volume-2"), gomock.Nil(), gomock.Eq(aws.Int64(177)), gomock.Nil(), gomock.Nil()).Return(nil)
cluster.VolumeResizer = resizer
cluster.syncVolumes()
}