make minimum limits boundaries configurable (zalando#808)

* make minimum limits boundaries configurable
* add e2e test

Author: FxKu

charts/postgres-operator/crds/operatorconfigurations.yaml

@@ -179,6 +179,12 @@ spec:
                 default_memory_request:
                   type: string
                   pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
+                min_cpu_limit:
+                  type: string
+                  pattern: '^(\d+m|\d+(\.\d{1,3})?)$'
+                min_memory_limit:
+                  type: string
+                  pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
             timeouts:
               type: object
               properties:

charts/postgres-operator/values-crd.yaml

@@ -115,13 +115,17 @@ configKubernetes:
 # configure resource requests for the Postgres pods
 configPostgresPodResources:
   # CPU limits for the postgres containers
-  default_cpu_limit: "3"
-  # cpu request value for the postgres containers
+  default_cpu_limit: "1"
+  # CPU request value for the postgres containers
   default_cpu_request: 100m
   # memory limits for the postgres containers
-  default_memory_limit: 1Gi
+  default_memory_limit: 500Mi
   # memory request value for the postgres containers
   default_memory_request: 100Mi
+  # hard CPU minimum required to properly run a Postgres cluster
+  min_cpu_limit: 250m
+  # hard memory minimum required to properly run a Postgres cluster
+  min_memory_limit: 250Mi
 
 # timeouts related to some operator actions
 configTimeouts:
@@ -251,7 +255,7 @@ configScalyr:
   # CPU rquest value for the Scalyr sidecar
   scalyr_cpu_request: 100m
   # Memory limit value for the Scalyr sidecar
-  scalyr_memory_limit: 1Gi
+  scalyr_memory_limit: 500Mi
   # Memory request value for the Scalyr sidecar
   scalyr_memory_request: 50Mi
 
@@ -272,13 +276,13 @@ serviceAccount:
 
 priorityClassName: ""
 
-resources: {}
-  # limits:
-  #   cpu: 100m
-  #   memory: 300Mi
-  # requests:
-  #   cpu: 100m
-  #   memory: 300Mi
+resources:
+  limits:
+    cpu: 500m
+    memory: 500Mi
+  requests:
+    cpu: 100m
+    memory: 250Mi
 
 # Affinity for pod assignment
 # Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity

charts/postgres-operator/values.yaml

@@ -108,13 +108,17 @@ configKubernetes:
 # configure resource requests for the Postgres pods
 configPostgresPodResources:
   # CPU limits for the postgres containers
-  default_cpu_limit: "3"
-  # cpu request value for the postgres containers
+  default_cpu_limit: "1"
+  # CPU request value for the postgres containers
   default_cpu_request: 100m
   # memory limits for the postgres containers
-  default_memory_limit: 1Gi
+  default_memory_limit: 500Mi
   # memory request value for the postgres containers
   default_memory_request: 100Mi
+  # hard CPU minimum required to properly run a Postgres cluster
+  min_cpu_limit: 250m
+  # hard memory minimum required to properly run a Postgres cluster
+  min_memory_limit: 250Mi
 
 # timeouts related to some operator actions
 configTimeouts:
@@ -248,13 +252,13 @@ serviceAccount:
 
 priorityClassName: ""
 
-resources: {}
-  # limits:
-  #   cpu: 100m
-  #   memory: 300Mi
-  # requests:
-  #   cpu: 100m
-  #   memory: 300Mi
+resources:
+  limits:
+    cpu: 500m
+    memory: 500Mi
+  requests:
+    cpu: 100m
+    memory: 250Mi
 
 # Affinity for pod assignment
 # Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity

docs/reference/operator_parameters.md

@@ -318,11 +318,19 @@ CRD-based configuration.
 
 * **default_cpu_limit**
   CPU limits for the Postgres containers, unless overridden by cluster-specific
-  settings. The default is `3`.
+  settings. The default is `1`.
 
 * **default_memory_limit**
   memory limits for the Postgres containers, unless overridden by cluster-specific
-  settings. The default is `1Gi`.
+  settings. The default is `500Mi`.
+
+* **min_cpu_limit**
+  hard CPU minimum what we consider to be required to properly run Postgres
+  clusters with Patroni on Kubernetes. The default is `250m`.
+
+* **min_memory_limit**
+  hard memory minimum what we consider to be required to properly run Postgres
+  clusters with Patroni on Kubernetes. The default is `250Mi`.
 
 ## Operator timeouts
 
@@ -579,4 +587,4 @@ scalyr sidecar. In the CRD-based configuration they are grouped under the
   CPU limit value for the Scalyr sidecar. The default is `1`.
 
 * **scalyr_memory_limit**
-  Memory limit value for the Scalyr sidecar. The default is `1Gi`.
+  Memory limit value for the Scalyr sidecar. The default is `500Mi`.

docs/user.md

@@ -232,11 +232,11 @@ spec:
       memory: 300Mi
 ```
 
-The minimum limit to properly run the `postgresql` resource is `256m` for `cpu`
-and `256Mi` for `memory`. If a lower value is set in the manifest the operator
-will cancel ADD or UPDATE events on this resource with an error. If no
-resources are defined in the manifest the operator will obtain the configured
-[default requests](reference/operator_parameters.md#kubernetes-resource-requests).
+The minimum limits to properly run the `postgresql` resource are configured to
+`250m` for `cpu` and `250Mi` for `memory`. If a lower value is set in the
+manifest the operator will raise the limits to the configured minimum values.
+If no resources are defined in the manifest they will be obtained from the
+configured [default requests](reference/operator_parameters.md#kubernetes-resource-requests).
 
 ## Use taints and tolerations for dedicated PostgreSQL nodes
 

e2e/tests/test_e2e.py

@@ -58,6 +58,57 @@ def setUpClass(cls):
         k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml")
         k8s.wait_for_pod_start('spilo-role=master')
 
+    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
+    def test_min_resource_limits(self):
+        '''
+        Lower resource limits below configured minimum and let operator fix it
+        '''
+        k8s = self.k8s
+        cluster_label = 'version=acid-minimal-cluster'
+        _, failover_targets = k8s.get_pg_nodes(cluster_label)
+
+        # configure minimum boundaries for CPU and memory limits
+        minCPULimit = '250m'
+        minMemoryLimit = '250Mi'
+        patch_min_resource_limits = {
+            "data": {
+                "min_cpu_limit": minCPULimit,
+                "min_memory_limit": minMemoryLimit
+            }
+        }
+        k8s.update_config(patch_min_resource_limits)
+
+        # lower resource limits below minimum
+        pg_patch_resources = {
+            "spec": {
+                "resources": {
+                    "requests": {
+                        "cpu": "10m",
+                        "memory": "50Mi"
+                    },
+                    "limits": {
+                        "cpu": "200m",
+                        "memory": "200Mi"
+                    }
+                }
+            }
+        }
+        k8s.api.custom_objects_api.patch_namespaced_custom_object(
+            "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources)
+        k8s.wait_for_master_failover(failover_targets)
+
+        pods = k8s.api.core_v1.list_namespaced_pod(
+            'default', label_selector='spilo-role=master,' + cluster_label).items
+        self.assert_master_is_unique()
+        masterPod = pods[0]
+
+        self.assertEqual(masterPod.spec.containers[0].resources.limits['cpu'], minCPULimit,
+                         "Expected CPU limit {}, found {}"
+                         .format(minCPULimit, masterPod.spec.containers[0].resources.limits['cpu']))
+        self.assertEqual(masterPod.spec.containers[0].resources.limits['memory'], minMemoryLimit,
+                         "Expected memory limit {}, found {}"
+                         .format(minMemoryLimit, masterPod.spec.containers[0].resources.limits['memory']))
+
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_multi_namespace_support(self):
         '''
@@ -76,10 +127,9 @@ def test_multi_namespace_support(self):
 
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_scaling(self):
-        """
+        '''
            Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime.
-        """
-
+        '''
         k8s = self.k8s
         labels = "version=acid-minimal-cluster"
 
@@ -93,9 +143,9 @@ def test_scaling(self):
 
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_taint_based_eviction(self):
-        """
+        '''
            Add taint "postgres=:NoExecute" to node with master. This must cause a failover.
-        """
+        '''
         k8s = self.k8s
         cluster_label = 'version=acid-minimal-cluster'
 
@@ -145,15 +195,15 @@ def test_taint_based_eviction(self):
 
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_logical_backup_cron_job(self):
-        """
+        '''
         Ensure we can (a) create the cron job at user request for a specific PG cluster
                       (b) update the cluster-wide image for the logical backup pod
                       (c) delete the job at user request
 
         Limitations:
         (a) Does not run the actual batch job because there is no S3 mock to upload backups to
         (b) Assumes 'acid-minimal-cluster' exists as defined in setUp
-        """
+        '''
 
         k8s = self.k8s
 
@@ -208,10 +258,10 @@ def test_logical_backup_cron_job(self):
                          "Expected 0 logical backup jobs, found {}".format(len(jobs)))
 
     def assert_master_is_unique(self, namespace='default', version="acid-minimal-cluster"):
-        """
+        '''
            Check that there is a single pod in the k8s cluster with the label "spilo-role=master"
            To be called manually after operations that affect pods
-        """
+        '''
 
         k8s = self.k8s
         labels = 'spilo-role=master,version=' + version

manifests/complete-postgres-manifest.yaml

@@ -42,8 +42,8 @@ spec:
       cpu: 10m
       memory: 100Mi
     limits:
-      cpu: 300m
-      memory: 300Mi
+      cpu: 500m
+      memory: 500Mi
   patroni:
     initdb:
       encoding: "UTF8"

manifests/configmap.yaml

@@ -15,9 +15,9 @@ data:
   # custom_pod_annotations: "keya:valuea,keyb:valueb"
   db_hosted_zone: db.example.com
   debug_logging: "true"
-  # default_cpu_limit: "3"
+  # default_cpu_limit: "1"
   # default_cpu_request: 100m
-  # default_memory_limit: 1Gi
+  # default_memory_limit: 500Mi
   # default_memory_request: 100Mi
   docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p16
   # enable_admin_role_for_users: "true"
@@ -48,6 +48,8 @@ data:
   # master_pod_move_timeout: 10m
   # max_instances: "-1"
   # min_instances: "-1"
+  # min_cpu_limit: 250m
+  # min_memory_limit: 250Mi
   # node_readiness_label: ""
   # oauth_token_secret_name: postgresql-operator
   # pam_configuration: |

manifests/operatorconfiguration.crd.yaml

@@ -155,6 +155,12 @@ spec:
                 default_memory_request:
                   type: string
                   pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
+                min_cpu_limit:
+                  type: string
+                  pattern: '^(\d+m|\d+(\.\d{1,3})?)$'
+                min_memory_limit:
+                  type: string
+                  pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
             timeouts:
               type: object
               properties:

manifests/postgres-operator.yaml

@@ -19,10 +19,10 @@ spec:
         imagePullPolicy: IfNotPresent
         resources:
           requests:
-            cpu: 500m
+            cpu: 100m
             memory: 250Mi
           limits:
-            cpu: 2000m
+            cpu: 500m
             memory: 500Mi
         securityContext:
           runAsUser: 1000