add user retention

charts/postgres-operator/crds/operatorconfigurations.yaml

@@ -128,6 +128,9 @@ spec:
                   password_rotation_interval:
                     type: integer
                     default: 90
+                  password_rotation_user_retention:
+                    type: integer
+                    default: 180
                   replication_username:
                      type: string
                      default: standby

docs/administrator.md

@@ -293,6 +293,62 @@ that are aggregated into the K8s [default roles](https://kubernetes.io/docs/refe
 
 For Helm deployments setting `rbac.createAggregateClusterRoles: true` adds these clusterroles to the deployment.
 
+## Password rotation in K8s secrets
+
+The operator regularly updates credentials in the K8s secrets if the
+`enable_password_rotation` option is set to `true` in the configuration.
+It happens only for LOGIN roles with an associated secret (manifest roles,
+default user from `preparedDatabases`, system users). Furthermore, there
+the following roles are excluded:
+
+1. Infrastructure role secrets since rotation should happen by the infrastructure.
+2. Team API roles that connect via OAuth2 and JWT token. Rotation should be provided by the infrastructure + there is even no secret for these roles
+3. Database owners and members of owners, since ownership can not be inherited.
+
+The interval of days can be set with `password_rotation_interval` (default
+`90` = 90 days, minimum 1). On each rotation the user name and password values
+are replaced in the secret. They belong to a newly created user named after
+the original role plus rotation date in YYMMDD format. All priviliges are
+inherited meaning that migration scripts continue to apply grants/revokes
+against the original role. The timestamp of the next rotation is written to
+the secret as well.
+
+Pods still using the previous secret values in memory continue to connect to
+the database since the password of the corresponding user is not replaced.
+However, a retention policy can be configured for created roles by password
+rotation with `password_rotation_user_retention`. The operator will ensure
+that this period is at least twice as long as the configured rotation
+interval, hence the default of `180` = 180 days.
+
+### Password rotation for single roles
+
+From the configuration, password rotation is enabled for all secrets with the
+mentioned exceptions. If you wish to first test rotation for a single user (or
+just have it enabled only for a few secrets) you can specify it in the cluster
+manifest. The rotation and retention intervals can only be configured globally.
+
+```
+spec:
+  usersWithSecretRotation: "foo_user,bar_reader_user"
+```
+
+### Password replacement without extra roles
+
+For some use cases where the secret is only used rarely - think of a `flyway`
+user running a migration script on pod start - we do not need to create extra
+database roles but can replace only the password in the K8s secret. This type
+of rotation cannot be configured globally but specified in the cluster
+manifest:
+
+```
+spec:
+  usersWithInPlaceSecretRotation: "flyway,bar_owner_user"
+```
+
+This would be the recommended option to enable rotation in secrets of database
+owners, But only if they are not used as application users for regular read
+and write operation.
+
 ## Use taints and tolerations for dedicated PostgreSQL nodes
 
 To ensure Postgres pods are running on nodes without any other application pods,

docs/reference/operator_parameters.md

@@ -174,6 +174,28 @@ under the `users` key.
   Postgres username used for replication between instances. The default is
   `standby`.
 
+* **enable_password_rotation**
+  For all LOGIN roles that are not database owners the Operator can rotate
+  credentials in the corresponding K8s secrets by replacing the username and
+  password. This means, new users will be added on each rotation inheriting
+  all priviliges from the original roles. The rotation date in the YYMMDD is
+  appended to the new user names. The timestamp of the next rotation is
+  written to the secret. The default is `false`.
+
+* **password_rotation_interval**
+  If password rotation is enabled (either from config or cluster manifest) the
+  interval can be configured with this parameter. The measure is in days which
+  means daily rotation (`1`) is the most frequent interval possible.
+  Default is `90`.
+
+* **password_rotation_user_retention**
+  To avoid an ever growing amount of new users due to password rotation the
+  operator will remove the created users again after a certain amount of days
+  has passed. The number can be configured with this parameter. However, the
+  operator will check that the retention policy is at least twice as long as
+  the rotation interval and update to this minimum in case it is not.
+  Default is `180`.
+
 ## Major version upgrades
 
 Parameters configuring automatic major version upgrades. In a

manifests/complete-postgres-manifest.yaml

@@ -16,6 +16,9 @@ spec:
     zalando:
     - superuser
     - createdb
+    foo_user: []
+#  usersWithSecretRotation: "foo_user"
+#  usersWithInPlaceSecretRotation: "flyway,bar_owner_user"
   enableMasterLoadBalancer: false
   enableReplicaLoadBalancer: false
   enableConnectionPooler: false # enable/disable connection pooler deployment

manifests/configmap.yaml

@@ -93,6 +93,7 @@ data:
   #  https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees
   # pam_role_name: zalandos
   # password_rotation_interval: "90"
+  # password_rotation_user_retention: "180"
   pdb_name_format: "postgres-{cluster}-pdb"
   # pod_antiaffinity_topology_key: "kubernetes.io/hostname"
   pod_deletion_wait_timeout: 10m

manifests/operatorconfiguration.crd.yaml

@@ -126,6 +126,9 @@ spec:
                   password_rotation_interval:
                     type: integer
                     default: 90
+                  password_rotation_user_retention:
+                    type: integer
+                    default: 180
                   replication_username:
                      type: string
                      default: standby

manifests/postgresql-operator-default-configuration.yaml

@@ -27,6 +27,7 @@ configuration:
   users:
     enable_password_rotation: false
     password_rotation_interval: 90
+    password_rotation_user_retention: 180
     replication_username: standby
     super_username: postgres
   major_version_upgrade:

pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

@@ -37,10 +37,11 @@ type OperatorConfigurationList struct {
 
 // PostgresUsersConfiguration defines the system users of Postgres.
 type PostgresUsersConfiguration struct {
-	SuperUsername            string `json:"super_username,omitempty"`
+	SuperUsername                 string `json:"super_username,omitempty"`
-	ReplicationUsername      string `json:"replication_username,omitempty"`
+	ReplicationUsername           string `json:"replication_username,omitempty"`
-	EnablePasswordRotation   bool   `json:"enable_password_rotation,omitempty"`
+	EnablePasswordRotation        bool   `json:"enable_password_rotation,omitempty"`
-	PasswordRotationInterval uint32 `json:"password_rotation_interval,omitempty"`
+	PasswordRotationInterval      uint32 `json:"password_rotation_interval,omitempty"`
+	PasswordRotationUserRetention uint32 `json:"password_rotation_user_retention,omitempty"`
 }
 
 // MajorVersionUpgradeConfiguration defines how to execute major version upgrades of Postgres.

pkg/apis/acid.zalan.do/v1/postgresql_type.go

@@ -53,8 +53,11 @@ type PostgresSpec struct {
 	// load balancers' source ranges are the same for master and replica services
 	AllowedSourceRanges []string `json:"allowedSourceRanges"`
 
+	Users                          map[string]UserFlags `json:"users,omitempty"`
+	UsersWithSecretRotation        []string             `json:"usersWithSecretRotation,omitempty"`
+	UsersWithInPlaceSecretRotation []string             `json:"usersWithInPlaceSecretRotation,omitempty"`
+
 	NumberOfInstances     int32                       `json:"numberOfInstances"`
-	Users                 map[string]UserFlags        `json:"users,omitempty"`
 	MaintenanceWindows    []MaintenanceWindow         `json:"maintenanceWindows,omitempty"`
 	Clone                 *CloneDescription           `json:"clone,omitempty"`
 	ClusterName           string                      `json:"-"`

pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go

@@ -641,6 +641,16 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) {
 			(*out)[key] = outVal
 		}
 	}
+	if in.UsersWithSecretRotation != nil {
+		in, out := &in.UsersWithSecretRotation, &out.UsersWithSecretRotation
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.UsersWithInPlaceSecretRotation != nil {
+		in, out := &in.UsersWithInPlaceSecretRotation, &out.UsersWithInPlaceSecretRotation
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	if in.MaintenanceWindows != nil {
 		in, out := &in.MaintenanceWindows, &out.MaintenanceWindows
 		*out = make([]MaintenanceWindow, len(*in))

pkg/cluster/cluster.go

@@ -1001,6 +1001,7 @@ func (c *Cluster) initSystemUsers() {
 		Origin:    spec.RoleOriginSystem,
 		Name:      c.OpConfig.ReplicationUsername,
 		Namespace: c.Namespace,
+		Flags:     []string{constants.RoleFlagLogin},
 		Password:  util.RandomPassword(constants.PasswordLength),
 	}
 

pkg/cluster/database.go

@@ -14,34 +14,11 @@ import (
 	"github.com/zalando/postgres-operator/pkg/spec"
 	"github.com/zalando/postgres-operator/pkg/util/constants"
 	"github.com/zalando/postgres-operator/pkg/util/retryutil"
+	"github.com/zalando/postgres-operator/pkg/util/users"
 )
 
 const (
-	getUserSQL = `WITH dbowners AS (
+	getUserSQL = `SELECT a.rolname, COALESCE(a.rolpassword, ''), a.rolsuper, a.rolinherit,
-		SELECT DISTINCT pg_catalog.pg_get_userbyid(datdba) AS owner
-		  FROM pg_database
-		 WHERE datname NOT IN ('postgres', 'template0', 'template1')
-	  ), roles AS (
-		SELECT a.rolname, COALESCE(a.rolpassword, '') AS rolpassword, a.rolsuper, a.rolinherit,
-			   a.rolcreaterole, a.rolcreatedb, a.rolcanlogin, s.setconfig,
-			   ARRAY(SELECT b.rolname
-					   FROM pg_catalog.pg_auth_members m
-					   JOIN pg_catalog.pg_authid b ON (m.roleid = b.oid)
-					  WHERE m.member = a.oid) as memberof
-		  FROM pg_catalog.pg_authid a
-		  LEFT JOIN pg_catalog.pg_db_role_setting s ON (a.oid = s.setrole AND s.setdatabase = 0::oid)
-		 WHERE a.rolname = ANY($1)
-		 ORDER BY 1
-	  )
-	  SELECT r.rolname, r.rolpassword, r.rolsuper, r.rolinherit,
-			 r.rolcreaterole, r.rolcreatedb, r.rolcanlogin, r.setconfig,
-			 r.memberof,
-			 o.owner IS NOT NULL OR r.rolname LIKE '%_owner' AS is_owner
-		FROM roles r
-		LEFT JOIN dbowners o ON o.owner = r.rolname OR o.owner = ANY (r.memberof)
-	   ORDER BY 1;`
-
-	/*`SELECT a.rolname, COALESCE(a.rolpassword, ''), a.rolsuper, a.rolinherit,
 	       a.rolcreaterole, a.rolcreatedb, a.rolcanlogin, s.setconfig,
 	       ARRAY(SELECT b.rolname
 	             FROM pg_catalog.pg_auth_members m
@@ -49,7 +26,13 @@ const (
 	            WHERE m.member = a.oid) as memberof
 	FROM pg_catalog.pg_authid a LEFT JOIN pg_db_role_setting s ON (a.oid = s.setrole AND s.setdatabase = 0::oid)
 	WHERE a.rolname = ANY($1)
-	ORDER BY 1;`*/
+	ORDER BY 1;`
+
+	getUsersForRetention = `SELECT r.rolname, right(r.rolname, 6) AS roldatesuffix
+	        FROM pg_roles r
+	        JOIN unnest($1::text[]) AS u(name) ON r.rolname LIKE u.name || '%'
+			AND right(r.rolname, 6) ~ '^[0-9\.]+$'
+			ORDER BY 1;`
 
 	getDatabasesSQL = `SELECT datname, pg_get_userbyid(datdba) AS owner FROM pg_database;`
 	getSchemasSQL   = `SELECT n.nspname AS dbschema FROM pg_catalog.pg_namespace n
@@ -222,10 +205,10 @@ func (c *Cluster) readPgUsersFromDatabase(userNames []string) (users spec.PgUser
 			rolname, rolpassword                                          string
 			rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin bool
 			roloptions, memberof                                          []string
-			rolowner, roldeleted                                          bool
+			roldeleted                                                    bool
 		)
 		err := rows.Scan(&rolname, &rolpassword, &rolsuper, &rolinherit,
-			&rolcreaterole, &rolcreatedb, &rolcanlogin, pq.Array(&roloptions), pq.Array(&memberof), &rolowner)
+			&rolcreaterole, &rolcreatedb, &rolcanlogin, pq.Array(&roloptions), pq.Array(&memberof))
 		if err != nil {
 			return nil, fmt.Errorf("error when processing user rows: %v", err)
 		}
@@ -245,12 +228,70 @@ func (c *Cluster) readPgUsersFromDatabase(userNames []string) (users spec.PgUser
 			roldeleted = true
 		}
 
-		users[rolname] = spec.PgUser{Name: rolname, Password: rolpassword, Flags: flags, MemberOf: memberof, Parameters: parameters, IsDbOwner: rolowner, Deleted: roldeleted}
+		users[rolname] = spec.PgUser{Name: rolname, Password: rolpassword, Flags: flags, MemberOf: memberof, Parameters: parameters, Deleted: roldeleted}
 	}
 
 	return users, nil
 }
 
+func findUsersFromRotation(rotatedUsers []string, db *sql.DB) (map[string]string, error) {
+	extraUsers := make(map[string]string, 0)
+	rows, err := db.Query(getUsersForRetention, pq.Array(rotatedUsers))
+	if err != nil {
+		return nil, fmt.Errorf("query failed: %v", err)
+	}
+	defer func() {
+		if err2 := rows.Close(); err2 != nil {
+			err = fmt.Errorf("error when closing query cursor: %v", err2)
+		}
+	}()
+
+	for rows.Next() {
+		var (
+			rolname, roldatesuffix string
+		)
+		err := rows.Scan(&rolname, &roldatesuffix)
+		if err != nil {
+			return nil, fmt.Errorf("error when processing rows of deprecated users: %v", err)
+		}
+		extraUsers[rolname] = roldatesuffix
+	}
+
+	return extraUsers, nil
+}
+
+func (c *Cluster) cleanupRotatedUsers(rotatedUsers []string, db *sql.DB) error {
+	c.setProcessName("checking for rotated users to remove from the database due to configured retention")
+	extraUsers, err := findUsersFromRotation(rotatedUsers, db)
+	if err != nil {
+		return fmt.Errorf("error when querying for deprecated users from password rotation: %v", err)
+	}
+
+	for rotatedUser, dateSuffix := range extraUsers {
+		// make sure user retention policy aligns with rotation interval
+		retenionDays := c.OpConfig.PasswordRotationUserRetention
+		if retenionDays < 2*c.OpConfig.PasswordRotationInterval {
+			retenionDays = 2 * c.OpConfig.PasswordRotationInterval
+			c.logger.Warnf("user retention days too few compared to rotation interval %d - setting it to %d", c.OpConfig.PasswordRotationInterval, retenionDays)
+		}
+		retentionDate := time.Now().AddDate(0, 0, int(retenionDays)*-1)
+		userCreationDate, err := time.Parse("060102", dateSuffix)
+		if err != nil {
+			c.logger.Errorf("could not parse creation date suffix of user %q: %v", rotatedUser, err)
+			continue
+		}
+		if retentionDate.After(userCreationDate) {
+			c.logger.Infof("dropping user %q due to configured days in password_rotation_user_retention", rotatedUser)
+			if err = users.DropPgUser(rotatedUser, db); err != nil {
+				c.logger.Errorf("could not drop role %q: %v", rotatedUser, err)
+				continue
+			}
+		}
+	}
+
+	return nil
+}
+
 // getDatabases returns the map of current databases with owners
 // The caller is responsible for opening and closing the database connection
 func (c *Cluster) getDatabases() (dbs map[string]string, err error) {

pkg/cluster/sync.go

@@ -627,6 +627,7 @@ func (c *Cluster) syncSecrets() error {
 	c.setProcessName("syncing secrets")
 	secrets := c.generateUserSecrets()
 	rotationUsers := make(spec.PgUserMap)
+	retentionUsers := make([]string, 0)
 
 	for secretUsername, secretSpec := range secrets {
 		if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Create(context.TODO(), secretSpec, metav1.CreateOptions{}); err == nil {
@@ -672,7 +673,8 @@ func (c *Cluster) syncSecrets() error {
 			}
 
 			// if password rotation is enabled update password and username if rotation interval has been passed
-			if c.OpConfig.EnablePasswordRotation && pwdUser.Origin != spec.RoleOriginInfrastructure && !pwdUser.IsDbOwner { // || c.Spec.InPlacePasswordRotation[secretUsername] {
+			if (c.OpConfig.EnablePasswordRotation && pwdUser.Origin != spec.RoleOriginInfrastructure && !pwdUser.IsDbOwner) ||
+				util.SliceContains(c.Spec.UsersWithSecretRotation, secretUsername) || util.SliceContains(c.Spec.UsersWithInPlaceSecretRotation, secretUsername) {
 				currentTime := time.Now()
 
 				// initialize password rotation setting first rotation date
@@ -688,13 +690,14 @@ func (c *Cluster) syncSecrets() error {
 				}
 
 				if currentTime.After(nextRotationDate) {
-					//if !c.Spec.InPlacePasswordRotation[secretUsername] {
+					if !util.SliceContains(c.Spec.UsersWithInPlaceSecretRotation, secretUsername) {
-					newRotationUsername := pwdUser.Name + "_" + currentTime.Format("060102")
+						retentionUsers = append(retentionUsers, pwdUser.Name)
-					pwdUser.MemberOf = []string{pwdUser.Name}
+						newRotationUsername := pwdUser.Name + currentTime.Format("060102")
-					pwdUser.Name = newRotationUsername
+						pwdUser.MemberOf = []string{pwdUser.Name}
-					rotationUsers[newRotationUsername] = pwdUser
+						pwdUser.Name = newRotationUsername
-					secret.Data["username"] = []byte(newRotationUsername)
+						rotationUsers[newRotationUsername] = pwdUser
-					//}
+						secret.Data["username"] = []byte(newRotationUsername)
+					}
 					secret.Data["password"] = []byte(util.RandomPassword(constants.PasswordLength))
 
 					_, nextRotationDateStr = c.getNextRotationDate(nextRotationDate)
@@ -722,14 +725,13 @@ func (c *Cluster) syncSecrets() error {
 		}
 		pgSyncRequests := c.userSyncStrategy.ProduceSyncRequests(spec.PgUserMap{}, rotationUsers)
 		if err = c.userSyncStrategy.ExecuteSyncRequests(pgSyncRequests, c.pgDb); err != nil {
-			return fmt.Errorf("error executing sync statements: %v", err)
+			return fmt.Errorf("error creating database roles for password rotation: %v", err)
 		}
-		if err2 := c.closeDbConn(); err2 != nil {
+		if err = c.cleanupRotatedUsers(retentionUsers, c.pgDb); err != nil {
-			if err == nil {
+			return fmt.Errorf("error creating database roles for password rotation: %v", err)
-				return fmt.Errorf("could not close database connection: %v", err2)
+		}
-			} else {
+		if err := c.closeDbConn(); err != nil {
-				return fmt.Errorf("could not close database connection: %v (prior error: %v)", err2, err)
+			c.logger.Errorf("could not close database connection during secret rotation: %v", err)
-			}
 		}
 	}
 

pkg/util/config/config.go

@@ -101,6 +101,7 @@ type Auth struct {
 	ReplicationUsername           string                `name:"replication_username" default:"standby"`
 	EnablePasswordRotation        bool                  `name:"enable_password_rotation" default:"false"`
 	PasswordRotationInterval      uint32                `name:"password_rotation_interval" default:"90"`
+	PasswordRotationUserRetention uint32                `name:"password_rotation_user_retention" default:"180"`
 }
 
 // Scalyr holds the configuration for the Scalyr Agent sidecar for log shipping:

pkg/util/users/users.go

@@ -18,6 +18,7 @@ const (
 	alterUserRenameSQL   = `ALTER ROLE "%s" RENAME TO "%s%s"`
 	alterRoleResetAllSQL = `ALTER ROLE "%s" RESET ALL`
 	alterRoleSetSQL      = `ALTER ROLE "%s" SET %s TO %s`
+	dropUserSQL          = `SET LOCAL synchronous_commit = 'local'; DROP ROLE "%s";`
 	grantToUserSQL       = `GRANT %s TO "%s"`
 	doBlockStmt          = `SET LOCAL synchronous_commit = 'local'; DO $$ BEGIN %s; END;$$;`
 	passwordTemplate     = "ENCRYPTED PASSWORD '%s'"
@@ -288,3 +289,13 @@ func quoteParameterValue(name, val string) string {
 	}
 	return fmt.Sprintf(`'%s'`, strings.Trim(val, " "))
 }
+
+// DropPgUser to remove user created by the operator e.g. for password rotation
+func DropPgUser(user string, db *sql.DB) error {
+	query := fmt.Sprintf(dropUserSQL, user)
+	if _, err := db.Exec(query); err != nil { // TODO: Try several times
+		return err
+	}
+
+	return nil
+}