Password rotation in secrets (#1749)

* password rotation in K8s secrets
* add db connection to syncSecrets
* add user retention
* add e2e test
* cleanup on username mismatch if rotation was switched off
* add unit test for syncSecrets + new updateSecret func

pkg/apis/acid.zalan.do/v1/crds.go

@@ -833,6 +833,24 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 							},
 						},
 					},
+					"usersWithSecretRotation": {
+						Type:     "array",
+						Nullable: true,
+						Items: &apiextv1.JSONSchemaPropsOrArray{
+							Schema: &apiextv1.JSONSchemaProps{
+								Type: "string",
+							},
+						},
+					},
+					"usersWithInPlaceSecretRotation": {
+						Type:     "array",
+						Nullable: true,
+						Items: &apiextv1.JSONSchemaPropsOrArray{
+							Schema: &apiextv1.JSONSchemaProps{
+								Type: "string",
+							},
+						},
+					},
 					"volume": {
 						Type:     "object",
 						Required: []string{"size"},

pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

@@ -37,8 +37,11 @@ type OperatorConfigurationList struct {
 
 // PostgresUsersConfiguration defines the system users of Postgres.
 type PostgresUsersConfiguration struct {
-	SuperUsername       string `json:"super_username,omitempty"`
-	ReplicationUsername string `json:"replication_username,omitempty"`
+	SuperUsername                 string `json:"super_username,omitempty"`
+	ReplicationUsername           string `json:"replication_username,omitempty"`
+	EnablePasswordRotation        bool   `json:"enable_password_rotation,omitempty"`
+	PasswordRotationInterval      uint32 `json:"password_rotation_interval,omitempty"`
+	PasswordRotationUserRetention uint32 `json:"password_rotation_user_retention,omitempty"`
 }
 
 // MajorVersionUpgradeConfiguration defines how to execute major version upgrades of Postgres.

pkg/apis/acid.zalan.do/v1/postgresql_type.go

@@ -53,8 +53,11 @@ type PostgresSpec struct {
 	// load balancers' source ranges are the same for master and replica services
 	AllowedSourceRanges []string `json:"allowedSourceRanges"`
 
+	Users                          map[string]UserFlags `json:"users,omitempty"`
+	UsersWithSecretRotation        []string             `json:"usersWithSecretRotation,omitempty"`
+	UsersWithInPlaceSecretRotation []string             `json:"usersWithInPlaceSecretRotation,omitempty"`
+
 	NumberOfInstances     int32                       `json:"numberOfInstances"`
-	Users                 map[string]UserFlags        `json:"users,omitempty"`
 	MaintenanceWindows    []MaintenanceWindow         `json:"maintenanceWindows,omitempty"`
 	Clone                 *CloneDescription           `json:"clone,omitempty"`
 	ClusterName           string                      `json:"-"`

pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go

@@ -641,6 +641,16 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) {
 			(*out)[key] = outVal
 		}
 	}
+	if in.UsersWithSecretRotation != nil {
+		in, out := &in.UsersWithSecretRotation, &out.UsersWithSecretRotation
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.UsersWithInPlaceSecretRotation != nil {
+		in, out := &in.UsersWithInPlaceSecretRotation, &out.UsersWithInPlaceSecretRotation
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	if in.MaintenanceWindows != nil {
 		in, out := &in.MaintenanceWindows, &out.MaintenanceWindows
 		*out = make([]MaintenanceWindow, len(*in))

pkg/cluster/cluster.go

@@ -711,13 +711,18 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
 		}
 	}
 
-	// connection pooler needs one system user created, which is done in
-	// initUsers. Check if it needs to be called.
+	// check if users need to be synced
 	sameUsers := reflect.DeepEqual(oldSpec.Spec.Users, newSpec.Spec.Users) &&
 		reflect.DeepEqual(oldSpec.Spec.PreparedDatabases, newSpec.Spec.PreparedDatabases)
+	sameRotatedUsers := reflect.DeepEqual(oldSpec.Spec.UsersWithSecretRotation, newSpec.Spec.UsersWithSecretRotation) &&
+		reflect.DeepEqual(oldSpec.Spec.UsersWithInPlaceSecretRotation, newSpec.Spec.UsersWithInPlaceSecretRotation)
+
+	// connection pooler needs one system user created, which is done in
+	// initUsers. Check if it needs to be called.
 	needConnectionPooler := needMasterConnectionPoolerWorker(&newSpec.Spec) ||
 		needReplicaConnectionPoolerWorker(&newSpec.Spec)
-	if !sameUsers || needConnectionPooler {
+
+	if !sameUsers || !sameRotatedUsers || needConnectionPooler {
 		c.logger.Debugf("initialize users")
 		if err := c.initUsers(); err != nil {
 			c.logger.Errorf("could not init users: %v", err)
@@ -1001,6 +1006,7 @@ func (c *Cluster) initSystemUsers() {
 		Origin:    spec.RoleOriginSystem,
 		Name:      c.OpConfig.ReplicationUsername,
 		Namespace: c.Namespace,
+		Flags:     []string{constants.RoleFlagLogin},
 		Password:  util.RandomPassword(constants.PasswordLength),
 	}
 
@@ -1113,7 +1119,6 @@ func (c *Cluster) initPreparedDatabaseRoles() error {
 func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix, searchPath, secretNamespace string) error {
 
 	for defaultRole, inherits := range defaultRoles {
-
 		namespace := c.Namespace
 		//if namespaced secrets are allowed
 		if secretNamespace != "" {
@@ -1136,8 +1141,10 @@ func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix
 		}
 
 		adminRole := ""
+		isOwner := false
 		if strings.Contains(defaultRole, constants.OwnerRoleNameSuffix) {
 			adminRole = admin
+			isOwner = true
 		} else {
 			adminRole = prefix + constants.OwnerRoleNameSuffix
 		}
@@ -1151,6 +1158,7 @@ func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix
 			MemberOf:   memberOf,
 			Parameters: map[string]string{"search_path": searchPath},
 			AdminRole:  adminRole,
+			IsDbOwner:  isOwner,
 		}
 		if currentRole, present := c.pgUsers[roleName]; present {
 			c.pgUsers[roleName] = c.resolveNameConflict(&currentRole, &newRole)
@@ -1172,6 +1180,14 @@ func (c *Cluster) initRobotUsers() error {
 		}
 		namespace := c.Namespace
 
+		// check if role is specified as database owner
+		isOwner := false
+		for _, owner := range c.Spec.Databases {
+			if username == owner {
+				isOwner = true
+			}
+		}
+
 		//if namespaced secrets are allowed
 		if c.Config.OpConfig.EnableCrossNamespaceSecret {
 			if strings.Contains(username, ".") {
@@ -1196,6 +1212,7 @@ func (c *Cluster) initRobotUsers() error {
 			Password:  util.RandomPassword(constants.PasswordLength),
 			Flags:     flags,
 			AdminRole: adminRole,
+			IsDbOwner: isOwner,
 		}
 		if currentRole, present := c.pgUsers[username]; present {
 			c.pgUsers[username] = c.resolveNameConflict(&currentRole, &newRole)

pkg/cluster/database.go

@@ -14,18 +14,25 @@ import (
 	"github.com/zalando/postgres-operator/pkg/spec"
 	"github.com/zalando/postgres-operator/pkg/util/constants"
 	"github.com/zalando/postgres-operator/pkg/util/retryutil"
+	"github.com/zalando/postgres-operator/pkg/util/users"
 )
 
 const (
 	getUserSQL = `SELECT a.rolname, COALESCE(a.rolpassword, ''), a.rolsuper, a.rolinherit,
-	        a.rolcreaterole, a.rolcreatedb, a.rolcanlogin, s.setconfig,
-	        ARRAY(SELECT b.rolname
-	              FROM pg_catalog.pg_auth_members m
-	              JOIN pg_catalog.pg_authid b ON (m.roleid = b.oid)
-	             WHERE m.member = a.oid) as memberof
-	 FROM pg_catalog.pg_authid a LEFT JOIN pg_db_role_setting s ON (a.oid = s.setrole AND s.setdatabase = 0::oid)
-	 WHERE a.rolname = ANY($1)
-	 ORDER BY 1;`
+	       a.rolcreaterole, a.rolcreatedb, a.rolcanlogin, s.setconfig,
+	       ARRAY(SELECT b.rolname
+	             FROM pg_catalog.pg_auth_members m
+	             JOIN pg_catalog.pg_authid b ON (m.roleid = b.oid)
+	            WHERE m.member = a.oid) as memberof
+	FROM pg_catalog.pg_authid a LEFT JOIN pg_db_role_setting s ON (a.oid = s.setrole AND s.setdatabase = 0::oid)
+	WHERE a.rolname = ANY($1)
+	ORDER BY 1;`
+
+	getUsersForRetention = `SELECT r.rolname, right(r.rolname, 6) AS roldatesuffix
+	        FROM pg_roles r
+	        JOIN unnest($1::text[]) AS u(name) ON r.rolname LIKE u.name || '%'
+			AND right(r.rolname, 6) ~ '^[0-9\.]+$'
+			ORDER BY 1;`
 
 	getDatabasesSQL = `SELECT datname, pg_get_userbyid(datdba) AS owner FROM pg_database;`
 	getSchemasSQL   = `SELECT n.nspname AS dbschema FROM pg_catalog.pg_namespace n
@@ -227,6 +234,65 @@ func (c *Cluster) readPgUsersFromDatabase(userNames []string) (users spec.PgUser
 	return users, nil
 }
 
+func findUsersFromRotation(rotatedUsers []string, db *sql.DB) (map[string]string, error) {
+	extraUsers := make(map[string]string, 0)
+	rows, err := db.Query(getUsersForRetention, pq.Array(rotatedUsers))
+	if err != nil {
+		return nil, fmt.Errorf("query failed: %v", err)
+	}
+	defer func() {
+		if err2 := rows.Close(); err2 != nil {
+			err = fmt.Errorf("error when closing query cursor: %v", err2)
+		}
+	}()
+
+	for rows.Next() {
+		var (
+			rolname, roldatesuffix string
+		)
+		err := rows.Scan(&rolname, &roldatesuffix)
+		if err != nil {
+			return nil, fmt.Errorf("error when processing rows of deprecated users: %v", err)
+		}
+		extraUsers[rolname] = roldatesuffix
+	}
+
+	return extraUsers, nil
+}
+
+func (c *Cluster) cleanupRotatedUsers(rotatedUsers []string, db *sql.DB) error {
+	c.setProcessName("checking for rotated users to remove from the database due to configured retention")
+	extraUsers, err := findUsersFromRotation(rotatedUsers, db)
+	if err != nil {
+		return fmt.Errorf("error when querying for deprecated users from password rotation: %v", err)
+	}
+
+	// make sure user retention policy aligns with rotation interval
+	retenionDays := c.OpConfig.PasswordRotationUserRetention
+	if retenionDays < 2*c.OpConfig.PasswordRotationInterval {
+		retenionDays = 2 * c.OpConfig.PasswordRotationInterval
+		c.logger.Warnf("user retention days too few compared to rotation interval %d - setting it to %d", c.OpConfig.PasswordRotationInterval, retenionDays)
+	}
+	retentionDate := time.Now().AddDate(0, 0, int(retenionDays)*-1)
+
+	for rotatedUser, dateSuffix := range extraUsers {
+		userCreationDate, err := time.Parse("060102", dateSuffix)
+		if err != nil {
+			c.logger.Errorf("could not parse creation date suffix of user %q: %v", rotatedUser, err)
+			continue
+		}
+		if retentionDate.After(userCreationDate) {
+			c.logger.Infof("dropping user %q due to configured days in password_rotation_user_retention", rotatedUser)
+			if err = users.DropPgUser(rotatedUser, db); err != nil {
+				c.logger.Errorf("could not drop role %q: %v", rotatedUser, err)
+				continue
+			}
+		}
+	}
+
+	return nil
+}
+
 // getDatabases returns the map of current databases with owners
 // The caller is responsible for opening and closing the database connection
 func (c *Cluster) getDatabases() (dbs map[string]string, err error) {

pkg/cluster/sync.go

@@ -611,60 +611,180 @@ func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, patroniC
 	return requiresMasterRestart, nil
 }
 
+func (c *Cluster) getNextRotationDate(currentDate time.Time) (time.Time, string) {
+	nextRotationDate := currentDate.AddDate(0, 0, int(c.OpConfig.PasswordRotationInterval))
+	return nextRotationDate, nextRotationDate.Format("2006-01-02 15:04:05")
+}
+
 func (c *Cluster) syncSecrets() error {
-	var (
-		err    error
-		secret *v1.Secret
-	)
+
 	c.logger.Info("syncing secrets")
 	c.setProcessName("syncing secrets")
-	secrets := c.generateUserSecrets()
+	generatedSecrets := c.generateUserSecrets()
+	rotationUsers := make(spec.PgUserMap)
+	retentionUsers := make([]string, 0)
+	currentTime := time.Now()
 
-	for secretUsername, secretSpec := range secrets {
-		if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Create(context.TODO(), secretSpec, metav1.CreateOptions{}); err == nil {
+	for secretUsername, generatedSecret := range generatedSecrets {
+		secret, err := c.KubeClient.Secrets(generatedSecret.Namespace).Create(context.TODO(), generatedSecret, metav1.CreateOptions{})
+		if err == nil {
 			c.Secrets[secret.UID] = secret
-			c.logger.Debugf("created new secret %s, namespace: %s, uid: %s", util.NameFromMeta(secret.ObjectMeta), secretSpec.Namespace, secret.UID)
+			c.logger.Debugf("created new secret %s, namespace: %s, uid: %s", util.NameFromMeta(secret.ObjectMeta), generatedSecret.Namespace, secret.UID)
 			continue
 		}
 		if k8sutil.ResourceAlreadyExists(err) {
-			var userMap map[string]spec.PgUser
-			if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Get(context.TODO(), secretSpec.Name, metav1.GetOptions{}); err != nil {
-				return fmt.Errorf("could not get current secret: %v", err)
-			}
-			if secretUsername != string(secret.Data["username"]) {
-				c.logger.Errorf("secret %s does not contain the role %s", secretSpec.Name, secretUsername)
-				continue
-			}
-			c.Secrets[secret.UID] = secret
-			c.logger.Debugf("secret %s already exists, fetching its password", util.NameFromMeta(secret.ObjectMeta))
-			if secretUsername == c.systemUsers[constants.SuperuserKeyName].Name {
-				secretUsername = constants.SuperuserKeyName
-				userMap = c.systemUsers
-			} else if secretUsername == c.systemUsers[constants.ReplicationUserKeyName].Name {
-				secretUsername = constants.ReplicationUserKeyName
-				userMap = c.systemUsers
-			} else {
-				userMap = c.pgUsers
-			}
-			pwdUser := userMap[secretUsername]
-			// if this secret belongs to the infrastructure role and the password has changed - replace it in the secret
-			if pwdUser.Password != string(secret.Data["password"]) &&
-				pwdUser.Origin == spec.RoleOriginInfrastructure {
-
-				c.logger.Debugf("updating the secret %s from the infrastructure roles", secretSpec.Name)
-				if _, err = c.KubeClient.Secrets(secretSpec.Namespace).Update(context.TODO(), secretSpec, metav1.UpdateOptions{}); err != nil {
-					return fmt.Errorf("could not update infrastructure role secret for role %q: %v", secretUsername, err)
-				}
-			} else {
-				// for non-infrastructure role - update the role with the password from the secret
-				pwdUser.Password = string(secret.Data["password"])
-				userMap[secretUsername] = pwdUser
+			if err = c.updateSecret(secretUsername, generatedSecret, &rotationUsers, &retentionUsers, currentTime); err != nil {
+				c.logger.Warningf("syncing secret %s failed: %v", util.NameFromMeta(secret.ObjectMeta), err)
 			}
 		} else {
-			return fmt.Errorf("could not create secret for user %s: in namespace %s: %v", secretUsername, secretSpec.Namespace, err)
+			return fmt.Errorf("could not create secret for user %s: in namespace %s: %v", secretUsername, generatedSecret.Namespace, err)
 		}
 	}
 
+	// add new user with date suffix and use it in the secret of the original user
+	if len(rotationUsers) > 0 {
+		err := c.initDbConn()
+		if err != nil {
+			return fmt.Errorf("could not init db connection: %v", err)
+		}
+		pgSyncRequests := c.userSyncStrategy.ProduceSyncRequests(spec.PgUserMap{}, rotationUsers)
+		if err = c.userSyncStrategy.ExecuteSyncRequests(pgSyncRequests, c.pgDb); err != nil {
+			return fmt.Errorf("error creating database roles for password rotation: %v", err)
+		}
+		if err := c.closeDbConn(); err != nil {
+			c.logger.Errorf("could not close database connection after creating users for password rotation: %v", err)
+		}
+	}
+
+	// remove rotation users that exceed the retention interval
+	if len(retentionUsers) > 0 {
+		err := c.initDbConn()
+		if err != nil {
+			return fmt.Errorf("could not init db connection: %v", err)
+		}
+		if err = c.cleanupRotatedUsers(retentionUsers, c.pgDb); err != nil {
+			return fmt.Errorf("error removing users exceeding configured retention interval: %v", err)
+		}
+		if err := c.closeDbConn(); err != nil {
+			c.logger.Errorf("could not close database connection after removing users exceeding configured retention interval: %v", err)
+		}
+	}
+
+	return nil
+}
+
+func (c *Cluster) updateSecret(
+	secretUsername string,
+	generatedSecret *v1.Secret,
+	rotationUsers *spec.PgUserMap,
+	retentionUsers *[]string,
+	currentTime time.Time) error {
+	var (
+		secret              *v1.Secret
+		err                 error
+		updateSecret        bool
+		updateSecretMsg     string
+		nextRotationDate    time.Time
+		nextRotationDateStr string
+	)
+
+	// get the secret first
+	if secret, err = c.KubeClient.Secrets(generatedSecret.Namespace).Get(context.TODO(), generatedSecret.Name, metav1.GetOptions{}); err != nil {
+		return fmt.Errorf("could not get current secret: %v", err)
+	}
+	c.Secrets[secret.UID] = secret
+
+	// fetch user map to update later
+	var userMap map[string]spec.PgUser
+	var userKey string
+	if secretUsername == c.systemUsers[constants.SuperuserKeyName].Name {
+		userKey = constants.SuperuserKeyName
+		userMap = c.systemUsers
+	} else if secretUsername == c.systemUsers[constants.ReplicationUserKeyName].Name {
+		userKey = constants.ReplicationUserKeyName
+		userMap = c.systemUsers
+	} else {
+		userKey = secretUsername
+		userMap = c.pgUsers
+	}
+	pwdUser := userMap[userKey]
+	secretName := util.NameFromMeta(secret.ObjectMeta)
+
+	// if password rotation is enabled update password and username if rotation interval has been passed
+	if (c.OpConfig.EnablePasswordRotation && !pwdUser.IsDbOwner &&
+		pwdUser.Origin != spec.RoleOriginInfrastructure && pwdUser.Origin != spec.RoleOriginSystem) ||
+		util.SliceContains(c.Spec.UsersWithSecretRotation, secretUsername) ||
+		util.SliceContains(c.Spec.UsersWithInPlaceSecretRotation, secretUsername) {
+
+		// initialize password rotation setting first rotation date
+		nextRotationDateStr = string(secret.Data["nextRotation"])
+		if nextRotationDate, err = time.ParseInLocation("2006-01-02 15:04:05", nextRotationDateStr, time.Local); err != nil {
+			nextRotationDate, nextRotationDateStr = c.getNextRotationDate(currentTime)
+			secret.Data["nextRotation"] = []byte(nextRotationDateStr)
+			updateSecret = true
+			updateSecretMsg = fmt.Sprintf("rotation date not found in secret %q. Setting it to %s", secretName, nextRotationDateStr)
+		}
+
+		// check if next rotation can happen sooner
+		// if rotation interval has been decreased
+		currentRotationDate, _ := c.getNextRotationDate(currentTime)
+		if nextRotationDate.After(currentRotationDate) {
+			nextRotationDate = currentRotationDate
+		}
+
+		// update password and next rotation date if configured interval has passed
+		if currentTime.After(nextRotationDate) {
+			// create rotation user if role is not listed for in-place password update
+			if !util.SliceContains(c.Spec.UsersWithInPlaceSecretRotation, secretUsername) {
+				rotationUser := pwdUser
+				newRotationUsername := secretUsername + currentTime.Format("060102")
+				rotationUser.Name = newRotationUsername
+				rotationUser.MemberOf = []string{secretUsername}
+				(*rotationUsers)[newRotationUsername] = rotationUser
+				secret.Data["username"] = []byte(newRotationUsername)
+
+				// whenever there is a rotation, check if old rotation users can be deleted
+				*retentionUsers = append(*retentionUsers, secretUsername)
+			}
+			secret.Data["password"] = []byte(util.RandomPassword(constants.PasswordLength))
+
+			_, nextRotationDateStr = c.getNextRotationDate(nextRotationDate)
+			secret.Data["nextRotation"] = []byte(nextRotationDateStr)
+
+			updateSecret = true
+			updateSecretMsg = fmt.Sprintf("updating secret %q due to password rotation - next rotation date: %s", secretName, nextRotationDateStr)
+		}
+	} else {
+		// username might not match if password rotation has been disabled again
+		if secretUsername != string(secret.Data["username"]) {
+			*retentionUsers = append(*retentionUsers, secretUsername)
+			secret.Data["username"] = []byte(secretUsername)
+			secret.Data["password"] = []byte(util.RandomPassword(constants.PasswordLength))
+			secret.Data["nextRotation"] = []byte{}
+			updateSecret = true
+			updateSecretMsg = fmt.Sprintf("secret %s does not contain the role %s - updating username and resetting password", secretName, secretUsername)
+		}
+	}
+
+	// if this secret belongs to the infrastructure role and the password has changed - replace it in the secret
+	if pwdUser.Password != string(secret.Data["password"]) && pwdUser.Origin == spec.RoleOriginInfrastructure {
+		secret = generatedSecret
+		updateSecret = true
+		updateSecretMsg = fmt.Sprintf("updating the secret %s from the infrastructure roles", secretName)
+	} else {
+		// for non-infrastructure role - update the role with the password from the secret
+		pwdUser.Password = string(secret.Data["password"])
+		userMap[userKey] = pwdUser
+	}
+
+	if updateSecret {
+		c.logger.Debugln(updateSecretMsg)
+		if _, err = c.KubeClient.Secrets(secret.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{}); err != nil {
+			return fmt.Errorf("could not update secret %q: %v", secretName, err)
+		}
+		c.Secrets[secret.UID] = secret
+	}
+
 	return nil
 }
 

pkg/cluster/sync_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/zalando/postgres-operator/mocks"
 	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	fakeacidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake"
+	"github.com/zalando/postgres-operator/pkg/spec"
 	"github.com/zalando/postgres-operator/pkg/util/config"
 	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
 	"github.com/zalando/postgres-operator/pkg/util/patroni"
@@ -26,6 +27,8 @@ import (
 )
 
 var patroniLogger = logrus.New().WithField("test", "patroni")
+var acidClientSet = fakeacidv1.NewSimpleClientset()
+var clientSet = fake.NewSimpleClientset()
 
 func newMockPod(ip string) *v1.Pod {
 	return &v1.Pod{
@@ -36,9 +39,6 @@ func newMockPod(ip string) *v1.Pod {
 }
 
 func newFakeK8sSyncClient() (k8sutil.KubernetesClient, *fake.Clientset) {
-	acidClientSet := fakeacidv1.NewSimpleClientset()
-	clientSet := fake.NewSimpleClientset()
-
 	return k8sutil.KubernetesClient{
 		PodsGetter:         clientSet.CoreV1(),
 		PostgresqlsGetter:  acidClientSet.AcidV1(),
@@ -46,6 +46,12 @@ func newFakeK8sSyncClient() (k8sutil.KubernetesClient, *fake.Clientset) {
 	}, clientSet
 }
 
+func newFakeK8sSyncSecretsClient() (k8sutil.KubernetesClient, *fake.Clientset) {
+	return k8sutil.KubernetesClient{
+		SecretsGetter: clientSet.CoreV1(),
+	}, clientSet
+}
+
 func TestSyncStatefulSetsAnnotations(t *testing.T) {
 	testName := "test syncing statefulsets annotations"
 	client, _ := newFakeK8sSyncClient()
@@ -257,3 +263,72 @@ func TestCheckAndSetGlobalPostgreSQLConfiguration(t *testing.T) {
 		}
 	}
 }
+
+func TestUpdateSecret(t *testing.T) {
+	testName := "test syncing secrets"
+	client, _ := newFakeK8sSyncSecretsClient()
+
+	clusterName := "acid-test-cluster"
+	namespace := "default"
+	username := "foo"
+	secretTemplate := config.StringTemplate("{username}.{cluster}.credentials")
+	rotationUsers := make(spec.PgUserMap)
+	retentionUsers := make([]string, 0)
+	yesterday := time.Now().AddDate(0, 0, -1)
+
+	// new cluster with pvc storage resize mode and configured labels
+	var cluster = New(
+		Config{
+			OpConfig: config.Config{
+				Auth: config.Auth{
+					SecretNameTemplate:            secretTemplate,
+					EnablePasswordRotation:        true,
+					PasswordRotationInterval:      1,
+					PasswordRotationUserRetention: 3,
+				},
+				Resources: config.Resources{
+					ClusterLabels:    map[string]string{"application": "spilo"},
+					ClusterNameLabel: "cluster-name",
+				},
+			},
+		}, client, acidv1.Postgresql{}, logger, eventRecorder)
+
+	cluster.Name = clusterName
+	cluster.Namespace = namespace
+	cluster.pgUsers = map[string]spec.PgUser{}
+	cluster.Spec.Users = map[string]acidv1.UserFlags{username: {}}
+	cluster.initRobotUsers()
+
+	// create a secret for user foo
+	cluster.syncSecrets()
+
+	secret, err := cluster.KubeClient.Secrets(namespace).Get(context.TODO(), secretTemplate.Format("username", username, "cluster", clusterName), metav1.GetOptions{})
+	assert.NoError(t, err)
+	generatedSecret := cluster.Secrets[secret.UID]
+
+	// now update the secret setting next rotation date (yesterday + interval)
+	cluster.updateSecret(username, generatedSecret, &rotationUsers, &retentionUsers, yesterday)
+	updatedSecret, err := cluster.KubeClient.Secrets(namespace).Get(context.TODO(), secretTemplate.Format("username", username, "cluster", clusterName), metav1.GetOptions{})
+	assert.NoError(t, err)
+
+	nextRotation := string(updatedSecret.Data["nextRotation"])
+	_, nextRotationDate := cluster.getNextRotationDate(yesterday)
+	if nextRotation != nextRotationDate {
+		t.Errorf("%s: updated secret does not contain correct rotation date: expected %s, got %s", testName, nextRotationDate, nextRotation)
+	}
+
+	// update secret again but use current time to trigger rotation
+	cluster.updateSecret(username, generatedSecret, &rotationUsers, &retentionUsers, time.Now())
+	updatedSecret, err = cluster.KubeClient.Secrets(namespace).Get(context.TODO(), secretTemplate.Format("username", username, "cluster", clusterName), metav1.GetOptions{})
+	assert.NoError(t, err)
+
+	if len(rotationUsers) != 1 && len(retentionUsers) != 1 {
+		t.Errorf("%s: unexpected number of users to rotate - expected only foo, found %d", testName, len(rotationUsers))
+	}
+
+	secretUsername := string(updatedSecret.Data["username"])
+	rotatedUsername := username + time.Now().Format("060102")
+	if secretUsername != rotatedUsername {
+		t.Errorf("%s: updated secret does not contain correct username: expected %s, got %s", testName, rotatedUsername, secretUsername)
+	}
+}

pkg/controller/operator_config.go

@@ -54,6 +54,9 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	// user config
 	result.SuperUsername = util.Coalesce(fromCRD.PostgresUsersConfiguration.SuperUsername, "postgres")
 	result.ReplicationUsername = util.Coalesce(fromCRD.PostgresUsersConfiguration.ReplicationUsername, "standby")
+	result.EnablePasswordRotation = fromCRD.PostgresUsersConfiguration.EnablePasswordRotation
+	result.PasswordRotationInterval = util.CoalesceUInt32(fromCRD.PostgresUsersConfiguration.PasswordRotationInterval, 90)
+	result.PasswordRotationUserRetention = util.CoalesceUInt32(fromCRD.PostgresUsersConfiguration.DeepCopy().PasswordRotationUserRetention, 180)
 
 	// major version upgrade config
 	result.MajorVersionUpgradeMode = util.Coalesce(fromCRD.MajorVersionUpgrade.MajorVersionUpgradeMode, "off")

pkg/spec/types.go

@@ -55,6 +55,7 @@ type PgUser struct {
 	MemberOf   []string          `yaml:"inrole"`
 	Parameters map[string]string `yaml:"db_parameters"`
 	AdminRole  string            `yaml:"admin_role"`
+	IsDbOwner  bool              `yaml:"is_db_owner"`
 	Deleted    bool              `yaml:"deleted"`
 }
 

pkg/util/config/config.go

@@ -100,6 +100,9 @@ type Auth struct {
 	InfrastructureRolesDefs       string                `name:"infrastructure_roles_secrets"`
 	SuperUsername                 string                `name:"super_username" default:"postgres"`
 	ReplicationUsername           string                `name:"replication_username" default:"standby"`
+	EnablePasswordRotation        bool                  `name:"enable_password_rotation" default:"false"`
+	PasswordRotationInterval      uint32                `name:"password_rotation_interval" default:"90"`
+	PasswordRotationUserRetention uint32                `name:"password_rotation_user_retention" default:"180"`
 }
 
 // Scalyr holds the configuration for the Scalyr Agent sidecar for log shipping:

pkg/util/users/users.go

@@ -18,6 +18,7 @@ const (
 	alterUserRenameSQL   = `ALTER ROLE "%s" RENAME TO "%s%s"`
 	alterRoleResetAllSQL = `ALTER ROLE "%s" RESET ALL`
 	alterRoleSetSQL      = `ALTER ROLE "%s" SET %s TO %s`
+	dropUserSQL          = `SET LOCAL synchronous_commit = 'local'; DROP ROLE "%s";`
 	grantToUserSQL       = `GRANT %s TO "%s"`
 	doBlockStmt          = `SET LOCAL synchronous_commit = 'local'; DO $$ BEGIN %s; END;$$;`
 	passwordTemplate     = "ENCRYPTED PASSWORD '%s'"
@@ -288,3 +289,13 @@ func quoteParameterValue(name, val string) string {
 	}
 	return fmt.Sprintf(`'%s'`, strings.Trim(val, " "))
 }
+
+// DropPgUser to remove user created by the operator e.g. for password rotation
+func DropPgUser(user string, db *sql.DB) error {
+	query := fmt.Sprintf(dropUserSQL, user)
+	if _, err := db.Exec(query); err != nil { // TODO: Try several times
+		return err
+	}
+
+	return nil
+}