make Postgres 14 available (zalando#1636)

FxKu · web-flow · commit ab25fb29b7bc · 2021-10-12T12:00:59.000+02:00
* make Postgres 14 available
* don't be too hard to 9.5
* bump Spilo image and more docs updates
* update e2e test upgrading to 14
diff --git a/README.md b/README.md
@@ -29,13 +29,13 @@ pipelines with no access to Kubernetes API directly, promoting infrastructure as
 
 ### PostgreSQL features
 
-* Supports PostgreSQL 13, starting from 9.6+
+* Supports PostgreSQL 14, starting from 9.6+
 * Streaming replication cluster via Patroni
 * Point-In-Time-Recovery with
 [pg_basebackup](https://www.postgresql.org/docs/11/app-pgbasebackup.html) /
 [WAL-E](https://github.com/wal-e/wal-e) via [Spilo](https://github.com/zalando/spilo)
 * Preload libraries: [bg_mon](https://github.com/CyberDem0n/bg_mon),
-[pg_stat_statements](https://www.postgresql.org/docs/9.4/pgstatstatements.html),
+[pg_stat_statements](https://www.postgresql.org/docs/14/pgstatstatements.html),
 [pgextwlist](https://github.com/dimitri/pgextwlist),
 [pg_auth_mon](https://github.com/RafiaSabih/pg_auth_mon)
 * Incl. popular Postgres extensions such as
@@ -53,20 +53,15 @@ pipelines with no access to Kubernetes API directly, promoting infrastructure as
 The Postgres Operator has been developed at Zalando and is being used in
 production for over three years.
 
-## Notes on Postgres 13 support
-
-If you are new to the operator, you can skip this and just start using the Postgres operator as is, Postgres 13 is ready to go.
-
-The Postgres operator supports Postgres 13 with the new Spilo Image that includes also the recent Patroni version to support PG13 settings.
-More work on optimizing restarts and rolling upgrades is pending.
+## Using Spilo 12 images or lower
 
 If you are already using the Postgres operator in older version with a Spilo 12 Docker image you need to be aware of the changes for the backup path.
-We introduce the major version into the backup path to smoothen the [major version upgrade](docs/administrator.md#minor-and-major-version-upgrade) that is now supported manually.
+We introduce the major version into the backup path to smoothen the [major version upgrade](docs/administrator.md#minor-and-major-version-upgrade) that is now supported.
 
 The new operator configuration can set a compatibility flag *enable_spilo_wal_path_compat* to make Spilo look for wal segments in the current path but also old format paths.
 This comes at potential performance costs and should be disabled after a few days.
 
-The newest Spilo 13 image is: `registry.opensource.zalan.do/acid/spilo-13:2.1-p1`
+The newest Spilo image is: `registry.opensource.zalan.do/acid/spilo-14:2.1-p2`
 
 The last Spilo 12 image is: `registry.opensource.zalan.do/acid/spilo-12:1.6-p5`
 
diff --git a/charts/postgres-operator-ui/templates/deployment.yaml b/charts/postgres-operator-ui/templates/deployment.yaml
@@ -71,6 +71,7 @@ spec:
                   "resources_visible": true,
                   "users_visible": true,
                   "postgresql_versions": [
+                    "14",
                     "13",
                     "12",
                     "11"
diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml
@@ -65,7 +65,7 @@ spec:
             properties:
               docker_image:
                 type: string
-                default: "registry.opensource.zalan.do/acid/spilo-13:2.1-p1"
+                default: "registry.opensource.zalan.do/acid/spilo-14:2.1-p2"
               enable_crd_validation:
                 type: boolean
                 default: true
@@ -135,10 +135,10 @@ spec:
                     default: "off"
                   minimal_major_version:
                     type: string
-                    default: "9.5"
+                    default: "9.6"
                   target_major_version:
                     type: string
-                    default: "13"
+                    default: "14"
               kubernetes:
                 type: object
                 properties:
diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml
@@ -362,14 +362,13 @@ spec:
                   version:
                     type: string
                     enum:
-                      - "9.3"
-                      - "9.4"
                       - "9.5"
                       - "9.6"
                       - "10"
                       - "11"
                       - "12"
                       - "13"
+                      - "14"
                   parameters:
                     type: object
                     additionalProperties:
diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml
@@ -28,14 +28,14 @@ configGeneral:
   enable_pgversion_env_var: true
   # start any new database pod without limitations on shm memory
   enable_shm_volume: true
-  # enables backwards compatible path between Spilo 12 and Spilo 13 images
+  # enables backwards compatible path between Spilo 12 and Spilo 13+ images
   enable_spilo_wal_path_compat: false
   # etcd connection string for Patroni. Empty uses K8s-native DCS.
   etcd_host: ""
   # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s)
   # kubernetes_use_configmaps: false
   # Spilo docker image
-  docker_image: registry.opensource.zalan.do/acid/spilo-13:2.1-p1
+  docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p2
   # min number of instances in Postgres cluster. -1 = no limit
   min_instances: -1
   # max number of instances in Postgres cluster. -1 = no limit
@@ -65,9 +65,9 @@ configMajorVersionUpgrade:
   # "off": no upgrade, "manual": manifest triggers action, "full": minimal version violation triggers too
   major_version_upgrade_mode: "off"
   # minimal Postgres major version that will not automatically be upgraded
-  minimal_major_version: "9.5"
+  minimal_major_version: "9.6"
   # target Postgres major version when upgrading clusters automatically
-  target_major_version: "13"
+  target_major_version: "14"
 
 configKubernetes:
   # list of additional capabilities for postgres container
diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md
@@ -83,7 +83,7 @@ Those are top-level keys, containing both leaf keys and groups.
   With newer versions of Spilo, it is preferable to use `PGVERSION` pod environment variable instead of the setting `postgresql.bin_dir` in the `SPILO_CONFIGURATION` env variable. When this option is true, the operator sets `PGVERSION` and omits `postgresql.bin_dir` from  `SPILO_CONFIGURATION`. When false, the `postgresql.bin_dir` is set. This setting takes precedence over `PGVERSION`; see PR 222 in Spilo. The default is `true`.
 
 * **enable_spilo_wal_path_compat**
-  enables backwards compatible path between Spilo 12 and Spilo 13 images. The default is `false`.
+  enables backwards compatible path between Spilo 12 and Spilo 13+ images. The default is `false`.
 
 * **etcd_host**
   Etcd connection string for Patroni defined as `host:port`. Not required when
@@ -186,12 +186,12 @@ CRD-configuration, they are grouped under the `major_version_upgrade` key.
 
 * **minimal_major_version**
   The minimal Postgres major version that will not automatically be upgraded
-  when `major_version_upgrade_mode` is set to `"full"`. The default is `"9.5"`.
+  when `major_version_upgrade_mode` is set to `"full"`. The default is `"9.6"`.
 
 * **target_major_version**
   The target Postgres major version when upgrading clusters automatically
   which violate the configured allowed `minimal_major_version` when
-  `major_version_upgrade_mode` is set to `"full"`. The default is `"13"`.
+  `major_version_upgrade_mode` is set to `"full"`. The default is `"14"`.
 
 ## Kubernetes resources
 
diff --git a/docs/user.md b/docs/user.md
@@ -30,7 +30,7 @@ spec:
   databases:
     foo: zalando
   postgresql:
-    version: "13"
+    version: "14"
 ```
 
 Once you cloned the Postgres Operator [repository](https://github.com/zalando/postgres-operator)
@@ -702,7 +702,7 @@ spec:
 ## In-place major version upgrade
 
 Starting with Spilo 13, operator supports in-place major version upgrade to a
-higher major version (e.g. from PG 10 to PG 12). To trigger the upgrade,
+higher major version (e.g. from PG 10 to PG 13). To trigger the upgrade,
 simply increase the version in the manifest. It is your responsibility to test
 your applications against the new version before the upgrade; downgrading is
 not supported. The easiest way to do so is to try the upgrade on the cloned
diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py
@@ -769,7 +769,7 @@ def test_major_version_upgrade(self):
         pg_patch_version = {
             "spec": {
                 "postgres": {
-                    "version": "13"
+                    "version": "14"
                 }
             }
         }
@@ -778,12 +778,12 @@ def test_major_version_upgrade(self):
 
         self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
 
-        def check_version_13():
+        def check_version_14():
             p = k8s.get_patroni_state("acid-upgrade-test-0")
             version = p["server_version"][0:2]
             return version
 
-        self.evantuallyEqual(check_version_13, "13", "Version was not upgrade to 13")
+        self.evantuallyEqual(check_version_14, "14", "Version was not upgrade to 14")
 
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_min_resource_limits(self):
diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml
@@ -9,7 +9,7 @@ metadata:
 #    "delete-date": "2020-08-31"  # can only be deleted on that day if "delete-date "key is configured
 #    "delete-clustername": "acid-test-cluster"  # can only be deleted when name matches if "delete-clustername" key is configured
 spec:
-  dockerImage: registry.opensource.zalan.do/acid/spilo-13:2.1-p1
+  dockerImage: registry.opensource.zalan.do/acid/spilo-14:2.1-p2
   teamId: "acid"
   numberOfInstances: 2
   users:  # Application/Robot users
@@ -36,7 +36,7 @@ spec:
           defaultRoles: true
           defaultUsers: false
   postgresql:
-    version: "13"
+    version: "14"
     parameters:  # Expert section
       shared_buffers: "32MB"
       max_connections: "10"
diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml
@@ -32,7 +32,7 @@ data:
   # default_memory_request: 100Mi
   # delete_annotation_date_key: delete-date
   # delete_annotation_name_key: delete-clustername
-  docker_image: registry.opensource.zalan.do/acid/spilo-13:2.1-p1
+  docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p2
   # downscaler_annotations: "deployment-time,downscaler/*"
   # enable_admin_role_for_users: "true"
   # enable_crd_validation: "true"
@@ -83,7 +83,7 @@ data:
   # min_instances: "-1"
   # min_cpu_limit: 250m
   # min_memory_limit: 250Mi
-  # minimal_major_version: "9.5"
+  # minimal_major_version: "9.6"
   # node_readiness_label: ""
   # oauth_token_secret_name: postgresql-operator
   # pam_configuration: |
@@ -124,7 +124,7 @@ data:
   spilo_privileged: "false"
   storage_resize_mode: "pvc"
   super_username: postgres
-  # target_major_version: "13"
+  # target_major_version: "14"
   # team_admin_role: "admin"
   # team_api_role_configuration: "log_statement:all"
   # teams_api_url: http://fake-teams-api.default.svc.cluster.local
diff --git a/manifests/minimal-postgres-manifest.yaml b/manifests/minimal-postgres-manifest.yaml
@@ -18,4 +18,4 @@ spec:
   preparedDatabases:
     bar: {}
   postgresql:
-    version: "13"
+    version: "14"
diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml
@@ -61,7 +61,7 @@ spec:
             properties:
               docker_image:
                 type: string
-                default: "registry.opensource.zalan.do/acid/spilo-13:2.1-p1"
+                default: "registry.opensource.zalan.do/acid/spilo-14:2.1-p2"
               enable_crd_validation:
                 type: boolean
                 default: true
@@ -131,10 +131,10 @@ spec:
                     default: "off"
                   minimal_major_version:
                     type: string
-                    default: "9.5"
+                    default: "9.6"
                   target_major_version:
                     type: string
-                    default: "13"
+                    default: "14"
               kubernetes:
                 type: object
                 properties:
diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml
@@ -3,7 +3,7 @@ kind: OperatorConfiguration
 metadata:
   name: postgresql-operator-default-configuration
 configuration:
-  docker_image: registry.opensource.zalan.do/acid/spilo-13:2.1-p1
+  docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p2
   # enable_crd_validation: true
   # enable_lazy_spilo_upgrade: false
   enable_pgversion_env_var: true
@@ -28,8 +28,8 @@ configuration:
     super_username: postgres
   major_version_upgrade:
     major_version_upgrade_mode: "off"
-    minimal_major_version: "9.5"
-    target_major_version: "13"
+    minimal_major_version: "9.6"
+    target_major_version: "14"
   kubernetes:
     # additional_pod_capabilities:
     # - "SYS_NICE"
diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml
@@ -358,14 +358,13 @@ spec:
                   version:
                     type: string
                     enum:
-                      - "9.3"
-                      - "9.4"
                       - "9.5"
                       - "9.6"
                       - "10"
                       - "11"
                       - "12"
                       - "13"
+                      - "14"
                   parameters:
                     type: object
                     additionalProperties:
diff --git a/manifests/standby-manifest.yaml b/manifests/standby-manifest.yaml
@@ -9,7 +9,7 @@ spec:
     size: 1Gi
   numberOfInstances: 1
   postgresql:
-    version: "13"
+    version: "14"
 # Make this a standby cluster and provide the s3 bucket path of source cluster for continuous streaming.
   standby:
     s3_wal_path: "s3://path/to/bucket/containing/wal/of/source/cluster/"
diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go
@@ -504,12 +504,6 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 							"version": {
 								Type: "string",
 								Enum: []apiextv1.JSON{
-									{
-										Raw: []byte(`"9.3"`),
-									},
-									{
-										Raw: []byte(`"9.4"`),
-									},
 									{
 										Raw: []byte(`"9.5"`),
 									},
@@ -528,6 +522,9 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 									{
 										Raw: []byte(`"13"`),
 									},
+									{
+										Raw: []byte(`"14"`),
+									},
 								},
 							},
 							"parameters": {
diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
@@ -44,8 +44,8 @@ type PostgresUsersConfiguration struct {
 // MajorVersionUpgradeConfiguration defines how to execute major version upgrades of Postgres.
 type MajorVersionUpgradeConfiguration struct {
 	MajorVersionUpgradeMode string `json:"major_version_upgrade_mode" default:"off"` // off - no actions, manual - manifest triggers action, full - manifest and minimal version violation trigger upgrade
-	MinimalMajorVersion     string `json:"minimal_major_version" default:"9.5"`
-	TargetMajorVersion      string `json:"target_major_version" default:"13"`
+	MinimalMajorVersion     string `json:"minimal_major_version" default:"9.6"`
+	TargetMajorVersion      string `json:"target_major_version" default:"14"`
 }
 
 // KubernetesMetaConfiguration defines k8s conf required for all Postgres clusters and the operator itself
diff --git a/pkg/cluster/majorversionupgrade.go b/pkg/cluster/majorversionupgrade.go
@@ -15,6 +15,7 @@ var VersionMap = map[string]int{
 	"11":  110000,
 	"12":  120000,
 	"13":  130000,
+	"14":  140000,
 }
 
 // IsBiggerPostgresVersion Compare two Postgres version numbers
@@ -33,7 +34,7 @@ func (c *Cluster) GetDesiredMajorVersionAsInt() int {
 func (c *Cluster) GetDesiredMajorVersion() string {
 
 	if c.Config.OpConfig.MajorVersionUpgradeMode == "full" {
-		// current is 9.5, minimal is 11 allowing 11 to 13 clusters, everything below is upgraded
+		// e.g. current is 9.6, minimal is 11 allowing 11 to 14 clusters, everything below is upgraded
 		if IsBiggerPostgresVersion(c.Spec.PgVersion, c.Config.OpConfig.MinimalMajorVersion) {
 			c.logger.Infof("overwriting configured major version %s to %s", c.Spec.PgVersion, c.Config.OpConfig.TargetMajorVersion)
 			return c.Config.OpConfig.TargetMajorVersion
diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go
@@ -39,7 +39,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	result.EnableSpiloWalPathCompat = fromCRD.EnableSpiloWalPathCompat
 	result.EtcdHost = fromCRD.EtcdHost
 	result.KubernetesUseConfigMaps = fromCRD.KubernetesUseConfigMaps
-	result.DockerImage = util.Coalesce(fromCRD.DockerImage, "registry.opensource.zalan.do/acid/spilo-13:2.1-p1")
+	result.DockerImage = util.Coalesce(fromCRD.DockerImage, "registry.opensource.zalan.do/acid/spilo-14:2.1-p2")
 	result.Workers = util.CoalesceUInt32(fromCRD.Workers, 8)
 	result.MinInstances = fromCRD.MinInstances
 	result.MaxInstances = fromCRD.MaxInstances
@@ -56,8 +56,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 
 	// major version upgrade config
 	result.MajorVersionUpgradeMode = util.Coalesce(fromCRD.MajorVersionUpgrade.MajorVersionUpgradeMode, "off")
-	result.MinimalMajorVersion = util.Coalesce(fromCRD.MajorVersionUpgrade.MinimalMajorVersion, "9.5")
-	result.TargetMajorVersion = util.Coalesce(fromCRD.MajorVersionUpgrade.TargetMajorVersion, "13")
+	result.MinimalMajorVersion = util.Coalesce(fromCRD.MajorVersionUpgrade.MinimalMajorVersion, "9.6")
+	result.TargetMajorVersion = util.Coalesce(fromCRD.MajorVersionUpgrade.TargetMajorVersion, "14")
 
 	// kubernetes config
 	result.CustomPodAnnotations = fromCRD.Kubernetes.CustomPodAnnotations
diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go
@@ -152,7 +152,7 @@ type Config struct {
 	WatchedNamespace        string            `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to'
 	KubernetesUseConfigMaps bool              `name:"kubernetes_use_configmaps" default:"false"`
 	EtcdHost                string            `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS
-	DockerImage             string            `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-13:2.1-p1"`
+	DockerImage             string            `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-14:2.1-p2"`
 	SidecarImages           map[string]string `name:"sidecar_docker_images"` // deprecated in favour of SidecarContainers
 	SidecarContainers       []v1.Container    `name:"sidecars"`
 	PodServiceAccountName   string            `name:"pod_service_account_name" default:"postgres-pod"`
@@ -212,8 +212,8 @@ type Config struct {
 	EnablePgVersionEnvVar                  bool              `name:"enable_pgversion_env_var" default:"true"`
 	EnableSpiloWalPathCompat               bool              `name:"enable_spilo_wal_path_compat" default:"false"`
 	MajorVersionUpgradeMode                string            `name:"major_version_upgrade_mode" default:"off"`
-	MinimalMajorVersion                    string            `name:"minimal_major_version" default:"9.5"`
-	TargetMajorVersion                     string            `name:"target_major_version" default:"13"`
+	MinimalMajorVersion                    string            `name:"minimal_major_version" default:"9.6"`
+	TargetMajorVersion                     string            `name:"target_major_version" default:"14"`
 }
 
 // MustMarshal marshals the config or panics
diff --git a/ui/manifests/deployment.yaml b/ui/manifests/deployment.yaml
@@ -66,6 +66,7 @@ spec:
                   "cost_core": 0.0575,
                   "cost_memory": 0.014375,
                   "postgresql_versions": [
+                    "14",
                     "13",
                     "12",
                     "11"
diff --git a/ui/operator_ui/spiloutils.py b/ui/operator_ui/spiloutils.py
diff --git a/ui/run_local.sh b/ui/run_local.sh

Original file line number	Diff line number	Diff line change
`@@ -769,7 +769,7 @@ def test_major_version_upgrade(self):`
`769`	`769`	`pg_patch_version = {`
`770`	`770`	`"spec": {`
`771`	`771`	`"postgres": {`
`772`		`- "version": "13"`
	`772`	`+ "version": "14"`
`773`	`773`	`}`
`774`	`774`	`}`
`775`	`775`	`}`
`@@ -778,12 +778,12 @@ def test_major_version_upgrade(self):`
`778`	`778`
`779`	`779`	`self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")`
`780`	`780`
`781`		`- def check_version_13():`
	`781`	`+ def check_version_14():`
`782`	`782`	`p = k8s.get_patroni_state("acid-upgrade-test-0")`
`783`	`783`	`version = p["server_version"][0:2]`
`784`	`784`	`return version`
`785`	`785`
`786`		`- self.evantuallyEqual(check_version_13, "13", "Version was not upgrade to 13")`
	`786`	`+ self.evantuallyEqual(check_version_14, "14", "Version was not upgrade to 14")`
`787`	`787`
`788`	`788`	`@timeout_decorator.timeout(TEST_TIMEOUT_SEC)`
`789`	`789`	`def test_min_resource_limits(self):`