public_git
/
postgres-operator
mirror of https://github.com/zalando/postgres-operator.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

allow in place pw rotation of system users (#1953) 

* allow in place pw rotation of system users
* block postgres user from rotation
* mark pooler pods for replacement
* adding podsGetter where pooler is synced in unit tests
* move rotation code in extra function
This commit is contained in:
Felix Kunde 2022-08-18 14:14:31 +02:00 committed by GitHub
parent 88a2931550
commit b2642fa2fc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 192 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
e2e/tests/test_e2e.py
View File

@ -1486,6 +1486,7 @@ class EndToEndTestCase(unittest.TestCase):
raise
@timeout_decorator.timeout(TEST_TIMEOUT_SEC)
@unittest.skip("Skipping this test until fixed")
def test_rolling_update_label_timeout(self):
'''
Simulate case when replica does not receive label in time and rolling update does not finish

12
pkg/cluster/cluster.go
View File

@ -1112,17 +1112,13 @@ func (c *Cluster) initSystemUsers() {
// connection pooler application should be able to login with this role
connectionPoolerUser := spec.PgUser{
Origin: spec.RoleConnectionPooler,
Origin: spec.RoleOriginConnectionPooler,
Name: username,
Namespace: c.Namespace,
Flags: []string{constants.RoleFlagLogin},
Password: util.RandomPassword(constants.PasswordLength),
}
if _, exists := c.pgUsers[username]; !exists {
c.pgUsers[username] = connectionPoolerUser
}
if _, exists := c.systemUsers[constants.ConnectionPoolerUserKeyName]; !exists {
c.systemUsers[constants.ConnectionPoolerUserKeyName] = connectionPoolerUser
}
@ -1133,15 +1129,15 @@ func (c *Cluster) initSystemUsers() {
if len(c.Spec.Streams) > 0 {
username := constants.EventStreamSourceSlotPrefix + constants.UserRoleNameSuffix
streamUser := spec.PgUser{
Origin: spec.RoleConnectionPooler,
Origin: spec.RoleOriginStream,
Name: username,
Namespace: c.Namespace,
Flags: []string{constants.RoleFlagLogin, constants.RoleFlagReplication},
Password: util.RandomPassword(constants.PasswordLength),
}
if _, exists := c.pgUsers[username]; !exists {
c.pgUsers[username] = streamUser
if _, exists := c.systemUsers[username]; !exists {
c.systemUsers[username] = streamUser
}
}
}

10
pkg/cluster/cluster_test.go
View File

@ -780,29 +780,29 @@ func TestInitSystemUsers(t *testing.T) {
cl.OpConfig.ConnectionPooler.User = "pooler"
cl.initSystemUsers()
if _, exist := cl.pgUsers["pooler"]; !exist {
if _, exist := cl.systemUsers["pooler"]; !exist {
t.Errorf("%s, Superuser is not allowed to be a connection pool user", testName)
}
// neither protected users are
delete(cl.pgUsers, "pooler")
delete(cl.systemUsers, "pooler")
cl.Spec.ConnectionPooler = &acidv1.ConnectionPooler{
User: "admin",
}
cl.OpConfig.ProtectedRoles = []string{"admin"}
cl.initSystemUsers()
if _, exist := cl.pgUsers["pooler"]; !exist {
if _, exist := cl.systemUsers["pooler"]; !exist {
t.Errorf("%s, Protected user are not allowed to be a connection pool user", testName)
}
delete(cl.pgUsers, "pooler")
delete(cl.systemUsers, "pooler")
cl.Spec.ConnectionPooler = &acidv1.ConnectionPooler{
User: "standby",
}
cl.initSystemUsers()
if _, exist := cl.pgUsers["pooler"]; !exist {
if _, exist := cl.systemUsers["pooler"]; !exist {
t.Errorf("%s, System users are not allowed to be a connection pool user", testName)
}
}

58
pkg/cluster/connection_pooler.go
View File

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"strings"
"time"
"github.com/r3labs/diff"
"github.com/sirupsen/logrus"
@ -20,6 +21,7 @@ import (
"github.com/zalando/postgres-operator/pkg/util/config"
"github.com/zalando/postgres-operator/pkg/util/constants"
"github.com/zalando/postgres-operator/pkg/util/k8sutil"
"github.com/zalando/postgres-operator/pkg/util/retryutil"
)
// ConnectionPoolerObjects K8s objects that are belong to connection pooler
@ -73,27 +75,36 @@ func needReplicaConnectionPoolerWorker(spec *acidv1.PostgresSpec) bool {
*spec.EnableReplicaConnectionPooler
}
// when listing pooler k8s objects
func (c *Cluster) poolerLabelsSet(addExtraLabels bool) labels.Set {
poolerLabels := c.labelsSet(addExtraLabels)
// TODO should be config values
poolerLabels["application"] = "db-connection-pooler"
return poolerLabels
}
// Return connection pooler labels selector, which should from one point of view
// inherit most of the labels from the cluster itself, but at the same time
// have e.g. different `application` label, so that recreatePod operation will
// not interfere with it (it lists all the pods via labels, and if there would
// be no difference, it will recreate also pooler pods).
func (c *Cluster) connectionPoolerLabels(role PostgresRole, addExtraLabels bool) *metav1.LabelSelector {
poolerLabels := c.labelsSet(addExtraLabels)
poolerLabelsSet := c.poolerLabelsSet(addExtraLabels)
// TODO should be config values
poolerLabels["application"] = "db-connection-pooler"
poolerLabels["connection-pooler"] = c.connectionPoolerName(role)
poolerLabelsSet["connection-pooler"] = c.connectionPoolerName(role)
if addExtraLabels {
extraLabels := map[string]string{}
extraLabels[c.OpConfig.PodRoleLabel] = string(role)
poolerLabels = labels.Merge(poolerLabels, extraLabels)
poolerLabelsSet = labels.Merge(poolerLabelsSet, extraLabels)
}
return &metav1.LabelSelector{
MatchLabels: poolerLabels,
MatchLabels: poolerLabelsSet,
MatchExpressions: nil,
}
}
@ -442,6 +453,14 @@ func (c *Cluster) shouldCreateLoadBalancerForPoolerService(role PostgresRole, sp
}
}
func (c *Cluster) listPoolerPods(listOptions metav1.ListOptions) ([]v1.Pod, error) {
pods, err := c.KubeClient.Pods(c.Namespace).List(context.TODO(), listOptions)
if err != nil {
return nil, fmt.Errorf("could not get list of pooler pods: %v", err)
}
return pods.Items, nil
}
//delete connection pooler
func (c *Cluster) deleteConnectionPooler(role PostgresRole) (err error) {
c.logger.Infof("deleting connection pooler spilo-role=%s", role)
@ -820,6 +839,7 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql
var (
deployment *appsv1.Deployment
newDeployment *appsv1.Deployment
pods []v1.Pod
service *v1.Service
newService *v1.Service
err error
@ -909,6 +929,34 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql
c.ConnectionPooler[role].Deployment = deployment
}
// check if pooler pods must be replaced due to secret update
listOptions := metav1.ListOptions{
LabelSelector: labels.Set(c.connectionPoolerLabels(role, true).MatchLabels).String(),
}
pods, err = c.listPoolerPods(listOptions)
if err != nil {
return nil, err
}
for i, pod := range pods {
if c.getRollingUpdateFlagFromPod(&pod) {
podName := util.NameFromMeta(pods[i].ObjectMeta)
err := retryutil.Retry(1*time.Second, 5*time.Second,
func() (bool, error) {
err2 := c.KubeClient.Pods(podName.Namespace).Delete(
context.TODO(),
podName.Name,
c.deleteOptions)
if err2 != nil {
return false, err2
}
return true, nil
})
if err != nil {
return nil, fmt.Errorf("could not delete pooler pod: %v", err)
}
}
}
if service, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.connectionPoolerName(role), metav1.GetOptions{}); err == nil {
c.ConnectionPooler[role].Service = service
desiredSvc := c.generateConnectionPoolerService(c.ConnectionPooler[role])

2
pkg/cluster/connection_pooler_test.go
View File

@ -263,6 +263,7 @@ func TestConnectionPoolerCreateDeletion(t *testing.T) {
client := k8sutil.KubernetesClient{
StatefulSetsGetter: clientSet.AppsV1(),
ServicesGetter: clientSet.CoreV1(),
PodsGetter: clientSet.CoreV1(),
DeploymentsGetter: clientSet.AppsV1(),
PostgresqlsGetter: acidClientSet.AcidV1(),
SecretsGetter: clientSet.CoreV1(),
@ -372,6 +373,7 @@ func TestConnectionPoolerSync(t *testing.T) {
client := k8sutil.KubernetesClient{
StatefulSetsGetter: clientSet.AppsV1(),
ServicesGetter: clientSet.CoreV1(),
PodsGetter: clientSet.CoreV1(),
DeploymentsGetter: clientSet.AppsV1(),
PostgresqlsGetter: acidClientSet.AcidV1(),
SecretsGetter: clientSet.CoreV1(),

1
pkg/cluster/k8sres_test.go
View File

@ -2233,6 +2233,7 @@ func newLBFakeClient() (k8sutil.KubernetesClient, *fake.Clientset) {
return k8sutil.KubernetesClient{
DeploymentsGetter: clientSet.AppsV1(),
PodsGetter: clientSet.CoreV1(),
ServicesGetter: clientSet.CoreV1(),
}, clientSet
}

177
pkg/cluster/sync.go
View File

@ -694,12 +694,10 @@ func (c *Cluster) updateSecret(
retentionUsers *[]string,
currentTime time.Time) error {
var (
secret *v1.Secret
err error
updateSecret bool
updateSecretMsg string
nextRotationDate time.Time
nextRotationDateStr string
secret *v1.Secret
err error
updateSecret bool
updateSecretMsg string
)
// get the secret first
@ -717,6 +715,12 @@ func (c *Cluster) updateSecret(
} else if secretUsername == c.systemUsers[constants.ReplicationUserKeyName].Name {
userKey = constants.ReplicationUserKeyName
userMap = c.systemUsers
} else if secretUsername == constants.ConnectionPoolerUserName {
userKey = constants.ConnectionPoolerUserName
userMap = c.systemUsers
} else if secretUsername == constants.EventStreamSourceSlotPrefix+constants.UserRoleNameSuffix {
userKey = constants.EventStreamSourceSlotPrefix + constants.UserRoleNameSuffix
userMap = c.systemUsers
} else {
userKey = secretUsername
userMap = c.pgUsers
@ -725,46 +729,22 @@ func (c *Cluster) updateSecret(
secretName := util.NameFromMeta(secret.ObjectMeta)
// if password rotation is enabled update password and username if rotation interval has been passed
if (c.OpConfig.EnablePasswordRotation && !pwdUser.IsDbOwner &&
pwdUser.Origin != spec.RoleOriginInfrastructure && pwdUser.Origin != spec.RoleOriginSystem) ||
util.SliceContains(c.Spec.UsersWithSecretRotation, secretUsername) ||
util.SliceContains(c.Spec.UsersWithInPlaceSecretRotation, secretUsername) {
// rotation can be enabled globally or via the manifest (excluding the Postgres superuser)
rotationEnabledInManifest := secretUsername != constants.SuperuserKeyName &&
(util.SliceContains(c.Spec.UsersWithSecretRotation, secretUsername) ||
util.SliceContains(c.Spec.UsersWithInPlaceSecretRotation, secretUsername))
// initialize password rotation setting first rotation date
nextRotationDateStr = string(secret.Data["nextRotation"])
if nextRotationDate, err = time.ParseInLocation(time.RFC3339, nextRotationDateStr, currentTime.UTC().Location()); err != nil {
nextRotationDate, nextRotationDateStr = c.getNextRotationDate(currentTime)
secret.Data["nextRotation"] = []byte(nextRotationDateStr)
updateSecret = true
updateSecretMsg = fmt.Sprintf("rotation date not found in secret %q. Setting it to %s", secretName, nextRotationDateStr)
// globally enabled rotation is only allowed for manifest and bootstrapped roles
allowedRoleTypes := []spec.RoleOrigin{spec.RoleOriginManifest, spec.RoleOriginBootstrap}
rotationAllowed := !pwdUser.IsDbOwner && util.SliceContains(allowedRoleTypes, pwdUser.Origin)
if (c.OpConfig.EnablePasswordRotation && rotationAllowed) || rotationEnabledInManifest {
updateSecretMsg, err = c.rotatePasswordInSecret(secret, pwdUser, secretUsername, currentTime, rotationUsers, retentionUsers)
if err != nil {
c.logger.Warnf("password rotation failed for user %s: %v", secretUsername, err)
}
// check if next rotation can happen sooner
// if rotation interval has been decreased
currentRotationDate, nextRotationDateStr := c.getNextRotationDate(currentTime)
if nextRotationDate.After(currentRotationDate) {
nextRotationDate = currentRotationDate
}
// update password and next rotation date if configured interval has passed
if currentTime.After(nextRotationDate) {
// create rotation user if role is not listed for in-place password update
if !util.SliceContains(c.Spec.UsersWithInPlaceSecretRotation, secretUsername) {
rotationUser := pwdUser
newRotationUsername := secretUsername + currentTime.Format("060102")
rotationUser.Name = newRotationUsername
rotationUser.MemberOf = []string{secretUsername}
(*rotationUsers)[newRotationUsername] = rotationUser
secret.Data["username"] = []byte(newRotationUsername)
// whenever there is a rotation, check if old rotation users can be deleted
*retentionUsers = append(*retentionUsers, secretUsername)
}
secret.Data["password"] = []byte(util.RandomPassword(constants.PasswordLength))
secret.Data["nextRotation"] = []byte(nextRotationDateStr)
if updateSecretMsg != "" {
updateSecret = true
updateSecretMsg = fmt.Sprintf("updating secret %q due to password rotation - next rotation date: %s", secretName, nextRotationDateStr)
}
} else {
// username might not match if password rotation has been disabled again
@ -792,7 +772,7 @@ func (c *Cluster) updateSecret(
if updateSecret {
c.logger.Debugln(updateSecretMsg)
if _, err = c.KubeClient.Secrets(secret.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{}); err != nil {
return fmt.Errorf("could not update secret %q: %v", secretName, err)
return fmt.Errorf("could not update secret %s: %v", secretName, err)
}
c.Secrets[secret.UID] = secret
}
@ -800,11 +780,101 @@ func (c *Cluster) updateSecret(
return nil
}
func (c *Cluster) rotatePasswordInSecret(
secret *v1.Secret,
secretPgUser spec.PgUser,
secretUsername string,
currentTime time.Time,
rotationUsers *spec.PgUserMap,
retentionUsers *[]string) (string, error) {
var (
err error
nextRotationDate time.Time
nextRotationDateStr string
updateSecretMsg string
)
secretName := util.NameFromMeta(secret.ObjectMeta)
// initialize password rotation setting first rotation date
nextRotationDateStr = string(secret.Data["nextRotation"])
if nextRotationDate, err = time.ParseInLocation(time.RFC3339, nextRotationDateStr, currentTime.UTC().Location()); err != nil {
nextRotationDate, nextRotationDateStr = c.getNextRotationDate(currentTime)
secret.Data["nextRotation"] = []byte(nextRotationDateStr)
updateSecretMsg = fmt.Sprintf("rotation date not found in secret %s. Setting it to %s", secretName, nextRotationDateStr)
}
// check if next rotation can happen sooner
// if rotation interval has been decreased
currentRotationDate, nextRotationDateStr := c.getNextRotationDate(currentTime)
if nextRotationDate.After(currentRotationDate) {
nextRotationDate = currentRotationDate
}
// update password and next rotation date if configured interval has passed
if currentTime.After(nextRotationDate) {
// create rotation user if role is not listed for in-place password update
if !util.SliceContains(c.Spec.UsersWithInPlaceSecretRotation, secretUsername) {
rotationUser := secretPgUser
newRotationUsername := secretUsername + currentTime.Format("060102")
rotationUser.Name = newRotationUsername
rotationUser.MemberOf = []string{secretUsername}
(*rotationUsers)[newRotationUsername] = rotationUser
secret.Data["username"] = []byte(newRotationUsername)
// whenever there is a rotation, check if old rotation users can be deleted
*retentionUsers = append(*retentionUsers, secretUsername)
} else {
// when passwords of system users are rotated in place, pods have to be replaced
if secretPgUser.Origin == spec.RoleOriginSystem {
pods, err := c.listPods()
if err != nil {
return "", fmt.Errorf("could not list pods of the statefulset: %v", err)
}
for _, pod := range pods {
if err = c.markRollingUpdateFlagForPod(&pod,
fmt.Sprintf("replace pod due to password rotation of system user %s", secretUsername)); err != nil {
c.logger.Warnf("marking pod for rolling update due to password rotation failed: %v", err)
}
}
}
// when password of connection pooler is rotated in place, pooler pods have to be replaced
if secretPgUser.Origin == spec.RoleOriginConnectionPooler {
listOptions := metav1.ListOptions{
LabelSelector: c.poolerLabelsSet(true).String(),
}
poolerPods, err := c.listPoolerPods(listOptions)
if err != nil {
return "", fmt.Errorf("could not list pods of the pooler deployment: %v", err)
}
for _, poolerPod := range poolerPods {
if err = c.markRollingUpdateFlagForPod(&poolerPod,
fmt.Sprintf("replace pooler pod due to password rotation of pooler user %s", secretUsername)); err != nil {
c.logger.Warnf("marking pooler pod for rolling update due to password rotation failed: %v", err)
}
}
}
// when password of stream user is rotated in place, it should trigger rolling update in FES deployment
if secretPgUser.Origin == spec.RoleOriginStream {
c.logger.Warnf("secret of stream user %s changed", constants.EventStreamSourceSlotPrefix+constants.UserRoleNameSuffix)
}
}
secret.Data["password"] = []byte(util.RandomPassword(constants.PasswordLength))
secret.Data["nextRotation"] = []byte(nextRotationDateStr)
updateSecretMsg = fmt.Sprintf("updating secret %s due to password rotation - next rotation date: %s", secretName, nextRotationDateStr)
}
return updateSecretMsg, nil
}
func (c *Cluster) syncRoles() (err error) {
c.setProcessName("syncing roles")
var (
dbUsers spec.PgUserMap
newUsers spec.PgUserMap
userNames []string
)
@ -825,6 +895,7 @@ func (c *Cluster) syncRoles() (err error) {
// mapping between original role name and with deletion suffix
deletedUsers := map[string]string{}
newUsers = make(map[string]spec.PgUser)
// create list of database roles to query
for _, u := range c.pgUsers {
@ -845,15 +916,13 @@ func (c *Cluster) syncRoles() (err error) {
}
}
// add pooler user to list of pgUsers, too
// to check if the pooler user exists or has to be created
if needMasterConnectionPooler(&c.Spec) || needReplicaConnectionPooler(&c.Spec) {
connectionPoolerUser := c.systemUsers[constants.ConnectionPoolerUserKeyName]
userNames = append(userNames, connectionPoolerUser.Name)
if _, exists := c.pgUsers[connectionPoolerUser.Name]; !exists {
c.pgUsers[connectionPoolerUser.Name] = connectionPoolerUser
}
// copy map for ProduceSyncRequests to include also system users
for userName, pgUser := range c.pgUsers {
newUsers[userName] = pgUser
}
for _, systemUser := range c.systemUsers {
userNames = append(userNames, systemUser.Name)
newUsers[systemUser.Name] = systemUser
}
dbUsers, err = c.readPgUsersFromDatabase(userNames)
@ -871,7 +940,7 @@ func (c *Cluster) syncRoles() (err error) {
}
}
pgSyncRequests := c.userSyncStrategy.ProduceSyncRequests(dbUsers, c.pgUsers)
pgSyncRequests := c.userSyncStrategy.ProduceSyncRequests(dbUsers, newUsers)
if err = c.userSyncStrategy.ExecuteSyncRequests(pgSyncRequests, c.pgDb); err != nil {
return fmt.Errorf("error executing sync statements: %v", err)
}

5
pkg/spec/types.go
View File

@ -32,7 +32,8 @@ const (
RoleOriginTeamsAPI
RoleOriginSystem
RoleOriginBootstrap
RoleConnectionPooler
RoleOriginConnectionPooler
RoleOriginStream
)
type syncUserOperation int
@ -194,7 +195,7 @@ func (r RoleOrigin) String() string {
return "system role"
case RoleOriginBootstrap:
return "bootstrapped role"
case RoleConnectionPooler:
case RoleOriginConnectionPooler:
return "connection pooler role"
default:
panic(fmt.Sprintf("bogus role origin value %d", r))