password rotation in K8s secrets

add db connection to syncSecrets

charts/postgres-operator/crds/operatorconfigurations.yaml

@@ -126,8 +126,8 @@ spec:
                     type: boolean
                     default: false
                   password_rotation_interval:
-                    type: string
+                    type: integer
-                    default: "90d"
+                    default: 90
                   replication_username:
                      type: string
                      default: standby

manifests/configmap.yaml

@@ -44,7 +44,7 @@ data:
   # enable_init_containers: "true"
   # enable_lazy_spilo_upgrade: "false"
   enable_master_load_balancer: "false"
-  enable_password_rotation: "true"
+  enable_password_rotation: "false"
   enable_pgversion_env_var: "true"
   # enable_pod_antiaffinity: "false"
   # enable_pod_disruption_budget: "true"
@@ -92,7 +92,7 @@ data:
   # pam_configuration: |
   #  https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees
   # pam_role_name: zalandos
-  password_rotation_interval: 10m
+  # password_rotation_interval: "90"
   pdb_name_format: "postgres-{cluster}-pdb"
   # pod_antiaffinity_topology_key: "kubernetes.io/hostname"
   pod_deletion_wait_timeout: 10m

manifests/operatorconfiguration.crd.yaml

@@ -124,8 +124,8 @@ spec:
                     type: boolean
                     default: false
                   password_rotation_interval:
-                    type: string
+                    type: integer
-                    default: "90d"
+                    default: 90
                   replication_username:
                      type: string
                      default: standby

manifests/postgres-operator.yaml

@@ -19,7 +19,7 @@ spec:
       serviceAccountName: postgres-operator
       containers:
       - name: postgres-operator
-        image: registry.opensource.zalan.do/acid/postgres-operator:v1.7.1-16-gfe340192-dirty
+        image: registry.opensource.zalan.do/acid/postgres-operator:v1.7.1
         imagePullPolicy: IfNotPresent
         resources:
           requests:

manifests/postgresql-operator-default-configuration.yaml

@@ -26,7 +26,7 @@ configuration:
   workers: 8
   users:
     enable_password_rotation: false
-    password_rotation_interval: 90d
+    password_rotation_interval: 90
     replication_username: standby
     super_username: postgres
   major_version_upgrade:

pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

@@ -37,10 +37,10 @@ type OperatorConfigurationList struct {
 
 // PostgresUsersConfiguration defines the system users of Postgres.
 type PostgresUsersConfiguration struct {
-	SuperUsername            string   `json:"super_username,omitempty"`
+	SuperUsername            string `json:"super_username,omitempty"`
-	ReplicationUsername      string   `json:"replication_username,omitempty"`
+	ReplicationUsername      string `json:"replication_username,omitempty"`
-	EnablePasswordRotation   bool     `json:"enable_password_rotation,omitempty"`
+	EnablePasswordRotation   bool   `json:"enable_password_rotation,omitempty"`
-	PasswordRotationInterval Duration `json:"password_rotation_interval,omitempty"`
+	PasswordRotationInterval uint32 `json:"password_rotation_interval,omitempty"`
 }
 
 // MajorVersionUpgradeConfiguration defines how to execute major version upgrades of Postgres.

pkg/cluster/cluster.go

@@ -995,6 +995,7 @@ func (c *Cluster) initSystemUsers() {
 		Name:      c.OpConfig.SuperUsername,
 		Namespace: c.Namespace,
 		Password:  util.RandomPassword(constants.PasswordLength),
+		IsDbOwner: true,
 	}
 	c.systemUsers[constants.ReplicationUserKeyName] = spec.PgUser{
 		Origin:    spec.RoleOriginSystem,
@@ -1112,7 +1113,6 @@ func (c *Cluster) initPreparedDatabaseRoles() error {
 func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix, searchPath, secretNamespace string) error {
 
 	for defaultRole, inherits := range defaultRoles {
-
 		namespace := c.Namespace
 		//if namespaced secrets are allowed
 		if secretNamespace != "" {
@@ -1135,8 +1135,10 @@ func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix
 		}
 
 		adminRole := ""
+		isOwner := false
 		if strings.Contains(defaultRole, constants.OwnerRoleNameSuffix) {
 			adminRole = admin
+			isOwner = true
 		} else {
 			adminRole = prefix + constants.OwnerRoleNameSuffix
 		}
@@ -1150,6 +1152,7 @@ func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix
 			MemberOf:   memberOf,
 			Parameters: map[string]string{"search_path": searchPath},
 			AdminRole:  adminRole,
+			IsDbOwner:  isOwner,
 		}
 		if currentRole, present := c.pgUsers[roleName]; present {
 			c.pgUsers[roleName] = c.resolveNameConflict(&currentRole, &newRole)
@@ -1171,6 +1174,14 @@ func (c *Cluster) initRobotUsers() error {
 		}
 		namespace := c.Namespace
 
+		// check if role is specified as database owner
+		isOwner := false
+		for _, owner := range c.Spec.Databases {
+			if username == owner {
+				isOwner = true
+			}
+		}
+
 		//if namespaced secrets are allowed
 		if c.Config.OpConfig.EnableCrossNamespaceSecret {
 			if strings.Contains(username, ".") {
@@ -1195,6 +1206,7 @@ func (c *Cluster) initRobotUsers() error {
 			Password:  util.RandomPassword(constants.PasswordLength),
 			Flags:     flags,
 			AdminRole: adminRole,
+			IsDbOwner: isOwner,
 		}
 		if currentRole, present := c.pgUsers[username]; present {
 			c.pgUsers[username] = c.resolveNameConflict(&currentRole, &newRole)

pkg/cluster/database.go

@@ -245,7 +245,7 @@ func (c *Cluster) readPgUsersFromDatabase(userNames []string) (users spec.PgUser
 			roldeleted = true
 		}
 
-		users[rolname] = spec.PgUser{Name: rolname, Password: rolpassword, Flags: flags, MemberOf: memberof, Parameters: parameters, IsOwner: rolowner, Deleted: roldeleted}
+		users[rolname] = spec.PgUser{Name: rolname, Password: rolpassword, Flags: flags, MemberOf: memberof, Parameters: parameters, IsDbOwner: rolowner, Deleted: roldeleted}
 	}
 
 	return users, nil

pkg/cluster/sync.go

@@ -611,15 +611,22 @@ func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, patroniC
 	return requiresMasterRestart, nil
 }
 
+func (c *Cluster) getNextRotationDate(currentDate time.Time) (time.Time, string) {
+	nextRotationDate := currentDate.AddDate(0, 0, int(c.OpConfig.PasswordRotationInterval))
+	return nextRotationDate, nextRotationDate.Format("2006-01-02 15:04:05")
+}
+
 func (c *Cluster) syncSecrets() error {
 	var (
-		err              error
+		err                 error
-		secret           *v1.Secret
+		secret              *v1.Secret
-		nextRotationDate time.Time
+		nextRotationDate    time.Time
+		nextRotationDateStr string
 	)
 	c.logger.Info("syncing secrets")
 	c.setProcessName("syncing secrets")
 	secrets := c.generateUserSecrets()
+	rotationUsers := make(spec.PgUserMap)
 
 	for secretUsername, secretSpec := range secrets {
 		if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Create(context.TODO(), secretSpec, metav1.CreateOptions{}); err == nil {
@@ -632,11 +639,11 @@ func (c *Cluster) syncSecrets() error {
 			if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Get(context.TODO(), secretSpec.Name, metav1.GetOptions{}); err != nil {
 				return fmt.Errorf("could not get current secret: %v", err)
 			}
-			username := string(secret.Data["username"])
+			/*username := string(secret.Data["username"])
 			if secretUsername != username {
 				c.logger.Errorf("secret %s does not contain the role %s", secretSpec.Name, secretUsername)
 				continue
-			}
+			}*/
 
 			c.logger.Debugf("secret %s already exists, fetching its password", util.NameFromMeta(secret.ObjectMeta))
 			if secretUsername == c.systemUsers[constants.SuperuserKeyName].Name {
@@ -650,36 +657,6 @@ func (c *Cluster) syncSecrets() error {
 			}
 			pwdUser := userMap[secretUsername]
 
-			// if password rotation is enabled update password and username if rotation interval has been passed
-			if c.OpConfig.EnablePasswordRotation && pwdUser.Origin != spec.RoleOriginInfrastructure && !pwdUser.IsOwner { // || c.Spec.InPlacePasswordRotation[secretUsername] {
-				err = json.Unmarshal(secret.Data["nextRotation"], &nextRotationDate)
-				if err != nil {
-					c.logger.Warningf("could not read rotation date of secret %s", secretSpec.Name)
-					nextRotationDate = time.Now()
-				}
-
-				currentTime := time.Now()
-				if currentTime.After(nextRotationDate) {
-					//if !c.Spec.InPlacePasswordRotation[secretUsername] {
-					newRotationUsername := secretUsername + "_" + currentTime.Format("060102")
-					pwdUser.Name = newRotationUsername
-					pwdUser.MemberOf = []string{secretUsername}
-					pgSyncRequests := c.userSyncStrategy.ProduceSyncRequests(spec.PgUserMap{}, map[string]spec.PgUser{newRotationUsername: pwdUser})
-					if err = c.userSyncStrategy.ExecuteSyncRequests(pgSyncRequests, c.pgDb); err != nil {
-						return fmt.Errorf("error executing sync statements: %v", err)
-					}
-					secret.Data["username"] = []byte(newRotationUsername)
-					//}
-					secret.Data["password"] = []byte(util.RandomPassword(constants.PasswordLength))
-					secret.Data["nextRotation"] = []byte(currentTime.Add(c.OpConfig.PasswordRotationInterval).Format("2006-01-02 15:04:05"))
-
-					c.logger.Debugf("updating the secret %s due to password rotation", secretSpec.Name)
-					if _, err = c.KubeClient.Secrets(secretSpec.Namespace).Update(context.TODO(), secretSpec, metav1.UpdateOptions{}); err != nil {
-						return fmt.Errorf("could not update secret %q: %v", secretUsername, err)
-					}
-				}
-			}
-
 			c.Secrets[secret.UID] = secret
 
 			// if this secret belongs to the infrastructure role and the password has changed - replace it in the secret
@@ -693,11 +670,69 @@ func (c *Cluster) syncSecrets() error {
 				pwdUser.Password = string(secret.Data["password"])
 				userMap[secretUsername] = pwdUser
 			}
+
+			// if password rotation is enabled update password and username if rotation interval has been passed
+			if c.OpConfig.EnablePasswordRotation && pwdUser.Origin != spec.RoleOriginInfrastructure && !pwdUser.IsDbOwner { // || c.Spec.InPlacePasswordRotation[secretUsername] {
+				currentTime := time.Now()
+
+				// initialize password rotation setting first rotation date
+				nextRotationDateStr = string(secret.Data["nextRotation"])
+				if nextRotationDate, err = time.Parse("2006-01-02 15:04:05", nextRotationDateStr); err != nil {
+					nextRotationDate, nextRotationDateStr = c.getNextRotationDate(currentTime)
+					c.logger.Warningf("rotation date not found in secret %q. Setting it to %s", secretSpec.Name, nextRotationDateStr)
+					secret.Data["nextRotation"] = []byte(nextRotationDateStr)
+					if _, err = c.KubeClient.Secrets(secretSpec.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{}); err != nil {
+						c.logger.Warningf("could not update secret %q: %v", secretSpec.Name, err)
+						continue
+					}
+				}
+
+				if currentTime.After(nextRotationDate) {
+					//if !c.Spec.InPlacePasswordRotation[secretUsername] {
+					newRotationUsername := pwdUser.Name + "_" + currentTime.Format("060102")
+					pwdUser.MemberOf = []string{pwdUser.Name}
+					pwdUser.Name = newRotationUsername
+					rotationUsers[newRotationUsername] = pwdUser
+					secret.Data["username"] = []byte(newRotationUsername)
+					//}
+					secret.Data["password"] = []byte(util.RandomPassword(constants.PasswordLength))
+
+					_, nextRotationDateStr = c.getNextRotationDate(nextRotationDate)
+					secret.Data["nextRotation"] = []byte(nextRotationDateStr)
+
+					c.logger.Debugf("updating secret %q due to password rotation - next rotation date: %s", secretSpec.Name, nextRotationDateStr)
+					if _, err = c.KubeClient.Secrets(secretSpec.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{}); err != nil {
+						c.logger.Warningf("could not update secret %q: %v", secretSpec.Name, err)
+						continue
+					}
+				}
+
+				c.Secrets[secret.UID] = secret
+			}
 		} else {
 			return fmt.Errorf("could not create secret for user %s: in namespace %s: %v", secretUsername, secretSpec.Namespace, err)
 		}
 	}
 
+	// add new user with date suffix and use it in the secret of the original user
+	if len(rotationUsers) > 0 {
+		err = c.initDbConn()
+		if err != nil {
+			return fmt.Errorf("could not init db connection: %v", err)
+		}
+		pgSyncRequests := c.userSyncStrategy.ProduceSyncRequests(spec.PgUserMap{}, rotationUsers)
+		if err = c.userSyncStrategy.ExecuteSyncRequests(pgSyncRequests, c.pgDb); err != nil {
+			return fmt.Errorf("error executing sync statements: %v", err)
+		}
+		if err2 := c.closeDbConn(); err2 != nil {
+			if err == nil {
+				return fmt.Errorf("could not close database connection: %v", err2)
+			} else {
+				return fmt.Errorf("could not close database connection: %v (prior error: %v)", err2, err)
+			}
+		}
+	}
+
 	return nil
 }
 

pkg/controller/operator_config.go

@@ -55,7 +55,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	result.SuperUsername = util.Coalesce(fromCRD.PostgresUsersConfiguration.SuperUsername, "postgres")
 	result.ReplicationUsername = util.Coalesce(fromCRD.PostgresUsersConfiguration.ReplicationUsername, "standby")
 	result.EnablePasswordRotation = fromCRD.PostgresUsersConfiguration.EnablePasswordRotation
-	result.PasswordRotationInterval = util.CoalesceDuration(time.Duration(fromCRD.PostgresUsersConfiguration.PasswordRotationInterval), "90d")
+	result.PasswordRotationInterval = util.CoalesceUInt32(fromCRD.PostgresUsersConfiguration.PasswordRotationInterval, 90)
 
 	// major version upgrade config
 	result.MajorVersionUpgradeMode = util.Coalesce(fromCRD.MajorVersionUpgrade.MajorVersionUpgradeMode, "off")

pkg/spec/types.go

@@ -55,7 +55,7 @@ type PgUser struct {
 	MemberOf   []string          `yaml:"inrole"`
 	Parameters map[string]string `yaml:"db_parameters"`
 	AdminRole  string            `yaml:"admin_role"`
-	IsOwner    bool              `yaml:"is_owner"`
+	IsDbOwner  bool              `yaml:"is_db_owner"`
 	Deleted    bool              `yaml:"deleted"`
 }
 

pkg/util/config/config.go

@@ -100,7 +100,7 @@ type Auth struct {
 	SuperUsername                 string                `name:"super_username" default:"postgres"`
 	ReplicationUsername           string                `name:"replication_username" default:"standby"`
 	EnablePasswordRotation        bool                  `name:"enable_password_rotation" default:"false"`
-	PasswordRotationInterval      time.Duration         `name:"password_rotation_interval" default:"90d"`
+	PasswordRotationInterval      uint32                `name:"password_rotation_interval" default:"90"`
 }
 
 // Scalyr holds the configuration for the Scalyr Agent sidecar for log shipping: