Resize volume by changing pvc size if enabled in config. (zalando#958)

* Try to resize pvc if resizing pv has failed

* added config option to switch between storage resize strategies

* changes according to requests

* Update pkg/controller/operator_config.go

Co-authored-by: Felix Kunde <[email protected]>

* enable_storage_resize documented

added examples to the default configuration and helm value files

* enable_storage_resize renamed to volume_resize_mode, off by default

* volume_resize_mode renamed to storage_resize_mode

* Update pkg/apis/acid.zalan.do/v1/crds.go

* pkg/cluster/volumes.go updated

* Update docs/reference/operator_parameters.md

* Update manifests/postgresql-operator-default-configuration.yaml

* Update pkg/controller/operator_config.go

* Update pkg/util/config/config.go

* Update charts/postgres-operator/values-crd.yaml

* Update charts/postgres-operator/values.yaml

* Update docs/reference/operator_parameters.md

* added logging if no changes required

Co-authored-by: Felix Kunde <[email protected]>

Author: yanchenko-igor

charts/postgres-operator/values-crd.yaml

@@ -124,6 +124,8 @@ configKubernetes:
 
   # whether the Spilo container should run in privileged mode
   spilo_privileged: false
+  # storage resize strategy, available options are: ebs, pvc, off
+  storage_resize_mode: ebs
   # operator watches for postgres objects in the given namespace
   watched_namespace: "*"  # listen to all namespaces
 

charts/postgres-operator/values.yaml

@@ -115,6 +115,8 @@ configKubernetes:
 
   # whether the Spilo container should run in privileged mode
   spilo_privileged: "false"
+  # storage resize strategy, available options are: ebs, pvc, off
+  storage_resize_mode: ebs
   # operator watches for postgres objects in the given namespace
   watched_namespace: "*"  # listen to all namespaces
 

docs/reference/operator_parameters.md

@@ -333,6 +333,12 @@ configuration they are grouped under the `kubernetes` key.
   of stateful sets of PG clusters. The default is `ordered_ready`, the second
   possible value is `parallel`.
 
+* **storage_resize_mode**
+  defines how operator handels the difference between requested volume size and
+  actual size. Available options are: ebs - tries to resize EBS volume, pvc -
+  changes PVC definition, off - disables resize of the volumes. Default is "ebs".
+  When using OpenShift please use one of the other available options.
+
 ## Kubernetes resource requests
 
 This group allows you to configure resource requests for the Postgres pods.

manifests/configmap.yaml

@@ -97,6 +97,7 @@ data:
   # set_memory_request_to_limit: "false"
   # spilo_fsgroup: 103
   spilo_privileged: "false"
+  # storage_resize_mode: "off"
   super_username: postgres
   # team_admin_role: "admin"
   # team_api_role_configuration: "log_statement:all"

manifests/operatorconfiguration.crd.yaml

@@ -168,6 +168,12 @@ spec:
                   type: integer
                 spilo_privileged:
                   type: boolean
+                storage_resize_mode:
+                  type: string
+                  enum:
+                    - "ebs"
+                    - "pvc"
+                    - "off"
                 toleration:
                   type: object
                   additionalProperties:

manifests/postgresql-operator-default-configuration.yaml

@@ -59,6 +59,7 @@ configuration:
     secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}"
     # spilo_fsgroup: 103
     spilo_privileged: false
+    storage_resize_mode: ebs
     # toleration: {}
     # watched_namespace: ""
   postgres_pod_resources:

pkg/apis/acid.zalan.do/v1/crds.go

@@ -980,6 +980,20 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation
 							"spilo_privileged": {
 								Type: "boolean",
 							},
+                                                      "storage_resize_mode": {
+								Type: "string",
+								Enum: []apiextv1beta1.JSON{
+									{
+										Raw: []byte(`"ebs"`),
+									},
+									{
+										Raw: []byte(`"pvc"`),
+									},
+									{
+										Raw: []byte(`"off"`),
+									},
+								},
+							},
 							"toleration": {
 								Type: "object",
 								AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{

pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

@@ -53,6 +53,7 @@ type KubernetesMetaConfiguration struct {
 	WatchedNamespace                       string                `json:"watched_namespace,omitempty"`
 	PDBNameFormat                          config.StringTemplate `json:"pdb_name_format,omitempty"`
 	EnablePodDisruptionBudget              *bool                 `json:"enable_pod_disruption_budget,omitempty"`
+	StorageResizeMode                      string                `json:"storage_resize_mode,omitempty"`
 	EnableInitContainers                   *bool                 `json:"enable_init_containers,omitempty"`
 	EnableSidecars                         *bool                 `json:"enable_sidecars,omitempty"`
 	SecretNameTemplate                     config.StringTemplate `json:"secret_name_template,omitempty"`

pkg/cluster/sync.go

@@ -57,16 +57,26 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
 		return err
 	}
 
-	// potentially enlarge volumes before changing the statefulset. By doing that
-	// in this order we make sure the operator is not stuck waiting for a pod that
-	// cannot start because it ran out of disk space.
-	// TODO: handle the case of the cluster that is downsized and enlarged again
-	// (there will be a volume from the old pod for which we can't act before the
-	//  the statefulset modification is concluded)
-	c.logger.Debugf("syncing persistent volumes")
-	if err = c.syncVolumes(); err != nil {
-		err = fmt.Errorf("could not sync persistent volumes: %v", err)
-		return err
+	if c.OpConfig.StorageResizeMode == "pvc" {
+		c.logger.Debugf("syncing persistent volume claims")
+		if err = c.syncVolumeClaims(); err != nil {
+			err = fmt.Errorf("could not sync persistent volume claims: %v", err)
+			return err
+		}
+	} else if c.OpConfig.StorageResizeMode == "ebs" {
+		// potentially enlarge volumes before changing the statefulset. By doing that
+		// in this order we make sure the operator is not stuck waiting for a pod that
+		// cannot start because it ran out of disk space.
+		// TODO: handle the case of the cluster that is downsized and enlarged again
+		// (there will be a volume from the old pod for which we can't act before the
+		//  the statefulset modification is concluded)
+		c.logger.Debugf("syncing persistent volumes")
+		if err = c.syncVolumes(); err != nil {
+			err = fmt.Errorf("could not sync persistent volumes: %v", err)
+			return err
+		}
+	} else {
+		c.logger.Infof("Storage resize is disabled (storage_resize_mode is off). Skipping volume sync.")
 	}
 
 	if err = c.enforceMinResourceLimits(&c.Spec); err != nil {
@@ -571,6 +581,27 @@ func (c *Cluster) syncRoles() (err error) {
 	return nil
 }
 
+// syncVolumeClaims reads all persistent volume claims and checks that their size matches the one declared in the statefulset.
+func (c *Cluster) syncVolumeClaims() error {
+	c.setProcessName("syncing volume claims")
+
+	act, err := c.volumeClaimsNeedResizing(c.Spec.Volume)
+	if err != nil {
+		return fmt.Errorf("could not compare size of the volume claims: %v", err)
+	}
+	if !act {
+		c.logger.Infof("volume claims don't require changes")
+		return nil
+	}
+	if err := c.resizeVolumeClaims(c.Spec.Volume); err != nil {
+		return fmt.Errorf("could not sync volume claims: %v", err)
+	}
+
+	c.logger.Infof("volume claims have been synced successfully")
+
+	return nil
+}
+
 // syncVolumes reads all persistent volumes and checks that their size matches the one declared in the statefulset.
 func (c *Cluster) syncVolumes() error {
 	c.setProcessName("syncing volumes")

pkg/cluster/volumes.go

@@ -52,6 +52,35 @@ func (c *Cluster) deletePersistentVolumeClaims() error {
 	return nil
 }
 
+func (c *Cluster) resizeVolumeClaims(newVolume acidv1.Volume) error {
+	c.logger.Debugln("resizing PVCs")
+	pvcs, err := c.listPersistentVolumeClaims()
+	if err != nil {
+		return err
+	}
+	newQuantity, err := resource.ParseQuantity(newVolume.Size)
+	if err != nil {
+		return fmt.Errorf("could not parse volume size: %v", err)
+	}
+	_, newSize, err := c.listVolumesWithManifestSize(newVolume)
+	for _, pvc := range pvcs {
+		volumeSize := quantityToGigabyte(pvc.Spec.Resources.Requests[v1.ResourceStorage])
+		if volumeSize >= newSize {
+			if volumeSize > newSize {
+				c.logger.Warningf("cannot shrink persistent volume")
+			}
+			continue
+		}
+		pvc.Spec.Resources.Requests[v1.ResourceStorage] = newQuantity
+		c.logger.Debugf("updating persistent volume claim definition for volume %q", pvc.Name)
+		if _, err := c.KubeClient.PersistentVolumeClaims(pvc.Namespace).Update(context.TODO(), &pvc, metav1.UpdateOptions{}); err != nil {
+			return fmt.Errorf("could not update persistent volume claim: %q", err)
+		}
+		c.logger.Debugf("successfully updated persistent volume claim %q", pvc.Name)
+	}
+	return nil
+}
+
 func (c *Cluster) listPersistentVolumes() ([]*v1.PersistentVolume, error) {
 	result := make([]*v1.PersistentVolume, 0)
 
@@ -150,7 +179,7 @@ func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.Volu
 			c.logger.Debugf("successfully updated persistent volume %q", pv.Name)
 		}
 		if !compatible {
-			c.logger.Warningf("volume %q is incompatible with all available resizing providers", pv.Name)
+			c.logger.Warningf("volume %q is incompatible with all available resizing providers, consider switching storage_resize_mode to pvc or off", pv.Name)
 			totalIncompatible++
 		}
 	}
@@ -160,6 +189,25 @@ func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.Volu
 	return nil
 }
 
+func (c *Cluster) volumeClaimsNeedResizing(newVolume acidv1.Volume) (bool, error) {
+	newSize, err := resource.ParseQuantity(newVolume.Size)
+	manifestSize := quantityToGigabyte(newSize)
+	if err != nil {
+		return false, fmt.Errorf("could not parse volume size from the manifest: %v", err)
+	}
+	pvcs, err := c.listPersistentVolumeClaims()
+	if err != nil {
+		return false, fmt.Errorf("could not receive persistent volume claims: %v", err)
+	}
+	for _, pvc := range pvcs {
+		currentSize := quantityToGigabyte(pvc.Spec.Resources.Requests[v1.ResourceStorage])
+		if currentSize != manifestSize {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
 func (c *Cluster) volumesNeedResizing(newVolume acidv1.Volume) (bool, error) {
 	vols, manifestSize, err := c.listVolumesWithManifestSize(newVolume)
 	if err != nil {