Resize volume by changing pvc size if enabled in config. (#958)
* Try to resize pvc if resizing pv has failed * added config option to switch between storage resize strategies * changes according to requests * Update pkg/controller/operator_config.go Co-authored-by: Felix Kunde <felix-kunde@gmx.de> * enable_storage_resize documented added examples to the default configuration and helm value files * enable_storage_resize renamed to volume_resize_mode, off by default * volume_resize_mode renamed to storage_resize_mode * Update pkg/apis/acid.zalan.do/v1/crds.go * pkg/cluster/volumes.go updated * Update docs/reference/operator_parameters.md * Update manifests/postgresql-operator-default-configuration.yaml * Update pkg/controller/operator_config.go * Update pkg/util/config/config.go * Update charts/postgres-operator/values-crd.yaml * Update charts/postgres-operator/values.yaml * Update docs/reference/operator_parameters.md * added logging if no changes required Co-authored-by: Felix Kunde <felix-kunde@gmx.de>
This commit is contained in:
Igor Yanchenko
GitHub
parent
6869c2cf1b
commit
88735a798a
|
|
@ -124,6 +124,8 @@ configKubernetes:
|
|||
|
||||
# whether the Spilo container should run in privileged mode
|
||||
spilo_privileged: false
|
||||
# storage resize strategy, available options are: ebs, pvc, off
|
||||
storage_resize_mode: ebs
|
||||
# operator watches for postgres objects in the given namespace
|
||||
watched_namespace: "*" # listen to all namespaces
|
||||
|
||||
|
|
|
|||
|
|
@ -115,6 +115,8 @@ configKubernetes:
|
|||
|
||||
# whether the Spilo container should run in privileged mode
|
||||
spilo_privileged: "false"
|
||||
# storage resize strategy, available options are: ebs, pvc, off
|
||||
storage_resize_mode: ebs
|
||||
# operator watches for postgres objects in the given namespace
|
||||
watched_namespace: "*" # listen to all namespaces
|
||||
|
||||
|
|
|
|||
|
|
@ -333,6 +333,12 @@ configuration they are grouped under the `kubernetes` key.
|
|||
of stateful sets of PG clusters. The default is `ordered_ready`, the second
|
||||
possible value is `parallel`.
|
||||
|
||||
* **storage_resize_mode**
|
||||
defines how operator handels the difference between requested volume size and
|
||||
actual size. Available options are: ebs - tries to resize EBS volume, pvc -
|
||||
changes PVC definition, off - disables resize of the volumes. Default is "ebs".
|
||||
When using OpenShift please use one of the other available options.
|
||||
|
||||
## Kubernetes resource requests
|
||||
|
||||
This group allows you to configure resource requests for the Postgres pods.
|
||||
|
|
|
|||
|
|
@ -97,6 +97,7 @@ data:
|
|||
# set_memory_request_to_limit: "false"
|
||||
# spilo_fsgroup: 103
|
||||
spilo_privileged: "false"
|
||||
# storage_resize_mode: "off"
|
||||
super_username: postgres
|
||||
# team_admin_role: "admin"
|
||||
# team_api_role_configuration: "log_statement:all"
|
||||
|
|
|
|||
|
|
@ -168,6 +168,12 @@ spec:
|
|||
type: integer
|
||||
spilo_privileged:
|
||||
type: boolean
|
||||
storage_resize_mode:
|
||||
type: string
|
||||
enum:
|
||||
- "ebs"
|
||||
- "pvc"
|
||||
- "off"
|
||||
toleration:
|
||||
type: object
|
||||
additionalProperties:
|
||||
|
|
|
|||
|
|
@ -59,6 +59,7 @@ configuration:
|
|||
secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}"
|
||||
# spilo_fsgroup: 103
|
||||
spilo_privileged: false
|
||||
storage_resize_mode: ebs
|
||||
# toleration: {}
|
||||
# watched_namespace: ""
|
||||
postgres_pod_resources:
|
||||
|
|
|
|||
|
|
@ -980,6 +980,20 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation
|
|||
"spilo_privileged": {
|
||||
Type: "boolean",
|
||||
},
|
||||
"storage_resize_mode": {
|
||||
Type: "string",
|
||||
Enum: []apiextv1beta1.JSON{
|
||||
{
|
||||
Raw: []byte(`"ebs"`),
|
||||
},
|
||||
{
|
||||
Raw: []byte(`"pvc"`),
|
||||
},
|
||||
{
|
||||
Raw: []byte(`"off"`),
|
||||
},
|
||||
},
|
||||
},
|
||||
"toleration": {
|
||||
Type: "object",
|
||||
AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ type KubernetesMetaConfiguration struct {
|
|||
WatchedNamespace string `json:"watched_namespace,omitempty"`
|
||||
PDBNameFormat config.StringTemplate `json:"pdb_name_format,omitempty"`
|
||||
EnablePodDisruptionBudget *bool `json:"enable_pod_disruption_budget,omitempty"`
|
||||
StorageResizeMode string `json:"storage_resize_mode,omitempty"`
|
||||
EnableInitContainers *bool `json:"enable_init_containers,omitempty"`
|
||||
EnableSidecars *bool `json:"enable_sidecars,omitempty"`
|
||||
SecretNameTemplate config.StringTemplate `json:"secret_name_template,omitempty"`
|
||||
|
|
|
|||
|
|
@ -57,6 +57,13 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
|
|||
return err
|
||||
}
|
||||
|
||||
if c.OpConfig.StorageResizeMode == "pvc" {
|
||||
c.logger.Debugf("syncing persistent volume claims")
|
||||
if err = c.syncVolumeClaims(); err != nil {
|
||||
err = fmt.Errorf("could not sync persistent volume claims: %v", err)
|
||||
return err
|
||||
}
|
||||
} else if c.OpConfig.StorageResizeMode == "ebs" {
|
||||
// potentially enlarge volumes before changing the statefulset. By doing that
|
||||
// in this order we make sure the operator is not stuck waiting for a pod that
|
||||
// cannot start because it ran out of disk space.
|
||||
|
|
@ -68,6 +75,9 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
|
|||
err = fmt.Errorf("could not sync persistent volumes: %v", err)
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
c.logger.Infof("Storage resize is disabled (storage_resize_mode is off). Skipping volume sync.")
|
||||
}
|
||||
|
||||
if err = c.enforceMinResourceLimits(&c.Spec); err != nil {
|
||||
err = fmt.Errorf("could not enforce minimum resource limits: %v", err)
|
||||
|
|
@ -571,6 +581,27 @@ func (c *Cluster) syncRoles() (err error) {
|
|||
return nil
|
||||
}
|
||||
|
||||
// syncVolumeClaims reads all persistent volume claims and checks that their size matches the one declared in the statefulset.
|
||||
func (c *Cluster) syncVolumeClaims() error {
|
||||
c.setProcessName("syncing volume claims")
|
||||
|
||||
act, err := c.volumeClaimsNeedResizing(c.Spec.Volume)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not compare size of the volume claims: %v", err)
|
||||
}
|
||||
if !act {
|
||||
c.logger.Infof("volume claims don't require changes")
|
||||
return nil
|
||||
}
|
||||
if err := c.resizeVolumeClaims(c.Spec.Volume); err != nil {
|
||||
return fmt.Errorf("could not sync volume claims: %v", err)
|
||||
}
|
||||
|
||||
c.logger.Infof("volume claims have been synced successfully")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// syncVolumes reads all persistent volumes and checks that their size matches the one declared in the statefulset.
|
||||
func (c *Cluster) syncVolumes() error {
|
||||
c.setProcessName("syncing volumes")
|
||||
|
|
|
|||
|
|
@ -52,6 +52,35 @@ func (c *Cluster) deletePersistentVolumeClaims() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) resizeVolumeClaims(newVolume acidv1.Volume) error {
|
||||
c.logger.Debugln("resizing PVCs")
|
||||
pvcs, err := c.listPersistentVolumeClaims()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newQuantity, err := resource.ParseQuantity(newVolume.Size)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not parse volume size: %v", err)
|
||||
}
|
||||
_, newSize, err := c.listVolumesWithManifestSize(newVolume)
|
||||
for _, pvc := range pvcs {
|
||||
volumeSize := quantityToGigabyte(pvc.Spec.Resources.Requests[v1.ResourceStorage])
|
||||
if volumeSize >= newSize {
|
||||
if volumeSize > newSize {
|
||||
c.logger.Warningf("cannot shrink persistent volume")
|
||||
}
|
||||
continue
|
||||
}
|
||||
pvc.Spec.Resources.Requests[v1.ResourceStorage] = newQuantity
|
||||
c.logger.Debugf("updating persistent volume claim definition for volume %q", pvc.Name)
|
||||
if _, err := c.KubeClient.PersistentVolumeClaims(pvc.Namespace).Update(context.TODO(), &pvc, metav1.UpdateOptions{}); err != nil {
|
||||
return fmt.Errorf("could not update persistent volume claim: %q", err)
|
||||
}
|
||||
c.logger.Debugf("successfully updated persistent volume claim %q", pvc.Name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) listPersistentVolumes() ([]*v1.PersistentVolume, error) {
|
||||
result := make([]*v1.PersistentVolume, 0)
|
||||
|
||||
|
|
@ -150,7 +179,7 @@ func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.Volu
|
|||
c.logger.Debugf("successfully updated persistent volume %q", pv.Name)
|
||||
}
|
||||
if !compatible {
|
||||
c.logger.Warningf("volume %q is incompatible with all available resizing providers", pv.Name)
|
||||
c.logger.Warningf("volume %q is incompatible with all available resizing providers, consider switching storage_resize_mode to pvc or off", pv.Name)
|
||||
totalIncompatible++
|
||||
}
|
||||
}
|
||||
|
|
@ -160,6 +189,25 @@ func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.Volu
|
|||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) volumeClaimsNeedResizing(newVolume acidv1.Volume) (bool, error) {
|
||||
newSize, err := resource.ParseQuantity(newVolume.Size)
|
||||
manifestSize := quantityToGigabyte(newSize)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("could not parse volume size from the manifest: %v", err)
|
||||
}
|
||||
pvcs, err := c.listPersistentVolumeClaims()
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("could not receive persistent volume claims: %v", err)
|
||||
}
|
||||
for _, pvc := range pvcs {
|
||||
currentSize := quantityToGigabyte(pvc.Spec.Resources.Requests[v1.ResourceStorage])
|
||||
if currentSize != manifestSize {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) volumesNeedResizing(newVolume acidv1.Volume) (bool, error) {
|
||||
vols, manifestSize, err := c.listVolumesWithManifestSize(newVolume)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -65,6 +65,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
|
|||
result.WatchedNamespace = fromCRD.Kubernetes.WatchedNamespace
|
||||
result.PDBNameFormat = fromCRD.Kubernetes.PDBNameFormat
|
||||
result.EnablePodDisruptionBudget = util.CoalesceBool(fromCRD.Kubernetes.EnablePodDisruptionBudget, util.True())
|
||||
result.StorageResizeMode = util.Coalesce(fromCRD.Kubernetes.StorageResizeMode, "ebs")
|
||||
result.EnableInitContainers = util.CoalesceBool(fromCRD.Kubernetes.EnableInitContainers, util.True())
|
||||
result.EnableSidecars = util.CoalesceBool(fromCRD.Kubernetes.EnableSidecars, util.True())
|
||||
result.SecretNameTemplate = fromCRD.Kubernetes.SecretNameTemplate
|
||||
|
|
|
|||
|
|
@ -142,6 +142,7 @@ type Config struct {
|
|||
CustomPodAnnotations map[string]string `name:"custom_pod_annotations"`
|
||||
EnablePodAntiAffinity bool `name:"enable_pod_antiaffinity" default:"false"`
|
||||
PodAntiAffinityTopologyKey string `name:"pod_antiaffinity_topology_key" default:"kubernetes.io/hostname"`
|
||||
StorageResizeMode string `name:"storage_resize_mode" default:"ebs"`
|
||||
// deprecated and kept for backward compatibility
|
||||
EnableLoadBalancer *bool `name:"enable_load_balancer"`
|
||||
MasterDNSNameFormat StringTemplate `name:"master_dns_name_format" default:"{cluster}.{team}.{hostedzone}"`
|
||||
|
|
|
|||
Loading…
Reference in New Issue