Fix S3 backup list

.gitignore

@@ -7,6 +7,8 @@
 _obj
 _test
 _manifests
+_tmp
+github.com
 
 # Architecture specific extensions/prefixes
 *.[568vq]
@@ -26,6 +28,7 @@ _testmain.go
 /vendor/
 /build/
 /docker/build/
+/github.com/
 .idea
 
 scm-source.json

Makefile

@@ -97,4 +97,4 @@ test:
 	GO111MODULE=on go test ./...
 
 e2e: docker # build operator image to be tested
-	cd e2e; make tools test clean
+	cd e2e; make tools e2etest clean

README.md

@@ -17,8 +17,9 @@ pipelines with no access to Kubernetes directly.
 
 * Rolling updates on Postgres cluster changes
 * Volume resize without Pod restarts
+* Database connection pooler
 * Cloning Postgres clusters
-* Logical Backups to S3 Bucket
+* Logical backups to S3 Bucket
 * Standby cluster from S3 WAL archive
 * Configurable for non-cloud environments
 * UI to create and edit Postgres cluster manifests

charts/postgres-operator-ui/templates/service.yaml

@@ -11,6 +11,9 @@ spec:
   ports:
     - port: {{ .Values.service.port }}
       targetPort: 8081
+      {{- if and (eq .Values.service.type "NodePort") .Values.service.nodePort }}
+      nodePort: {{ .Values.service.nodePort }}
+      {{- end }}
       protocol: TCP
   selector:
     app.kubernetes.io/instance: {{ .Release.Name }}

charts/postgres-operator-ui/values.yaml

@@ -42,6 +42,9 @@ envs:
 service:
   type: "ClusterIP"
   port: "8080"
+  # If the type of the service is NodePort a port can be specified using the nodePort field
+  # If the nodePort field is not specified, or if it has no value, then a random port is used
+  # notePort: 32521
 
 # configure UI ingress. If needed: "enabled: true"
 ingress:

charts/postgres-operator/crds/operatorconfigurations.yaml

@@ -62,10 +62,14 @@ spec:
               type: string
             enable_crd_validation:
               type: boolean
+            enable_lazy_spilo_upgrade:
+              type: boolean
             enable_shm_volume:
               type: boolean
             etcd_host:
               type: string
+            kubernetes_use_configmaps:
+              type: boolean
             max_instances:
               type: integer
               minimum: -1  # -1 = disabled
@@ -82,6 +86,12 @@ spec:
               type: object
               additionalProperties:
                 type: string
+            sidecars:
+              type: array
+              nullable: true
+              items:
+                type: object
+                additionalProperties: true
             workers:
               type: integer
               minimum: 1
@@ -297,7 +307,7 @@ spec:
                   type: integer
                 ring_log_lines:
                   type: integer
-            scalyr:
+            scalyr:  # deprecated
               type: object
               properties:
                 scalyr_api_key:
@@ -318,44 +328,44 @@ spec:
                   pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
                 scalyr_server_url:
                   type: string
-            connection_pool:
+            connection_pooler:
               type: object
               properties:
-                connection_pool_schema:
+                connection_pooler_schema:
                   type: string
                   #default: "pooler"
-                connection_pool_user:
+                connection_pooler_user:
                   type: string
                   #default: "pooler"
-                connection_pool_image:
+                connection_pooler_image:
                   type: string
                   #default: "registry.opensource.zalan.do/acid/pgbouncer"
-                connection_pool_max_db_connections:
+                connection_pooler_max_db_connections:
                   type: integer
                   #default: 60
-                connection_pool_mode:
+                connection_pooler_mode:
                   type: string
                   enum:
                     - "session"
                     - "transaction"
                   #default: "transaction"
-                connection_pool_number_of_instances:
+                connection_pooler_number_of_instances:
                   type: integer
                   minimum: 2
                   #default: 2
-                connection_pool_default_cpu_limit:
+                connection_pooler_default_cpu_limit:
                   type: string
                   pattern: '^(\d+m|\d+(\.\d{1,3})?)$'
                   #default: "1"
-                connection_pool_default_cpu_request:
+                connection_pooler_default_cpu_request:
                   type: string
                   pattern: '^(\d+m|\d+(\.\d{1,3})?)$'
                   #default: "500m"
-                connection_pool_default_memory_limit:
+                connection_pooler_default_memory_limit:
                   type: string
                   pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
                   #default: "100Mi"
-                connection_pool_default_memory_request:
+                connection_pooler_default_memory_request:
                   type: string
                   pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
                   #default: "100Mi"

charts/postgres-operator/crds/postgresqls.yaml

@@ -74,6 +74,28 @@ spec:
             - teamId
             - postgresql
           properties:
+            additionalVolumes:
+              type: array
+              items:
+                type: object
+                required:
+                  - name
+                  - mountPath
+                  - volumeSource
+                properties:
+                  name:
+                    type: string
+                  mountPath:
+                    type: string
+                  targetContainers:
+                    type: array
+                    nullable: true
+                    items:
+                      type: string
+                  volumeSource:
+                    type: object
+                  subPath:
+                    type: string
             allowedSourceRanges:
               type: array
               nullable: true
@@ -106,7 +128,7 @@ spec:
                 uid:
                   format: uuid
                   type: string
-            connectionPool:
+            connectionPooler:
               type: object
               properties:
                 dockerImage:
@@ -162,7 +184,7 @@ spec:
               # Note: usernames specified here as database owners must be declared in the users key of the spec key.
             dockerImage:
               type: string
-            enableConnectionPool:
+            enableConnectionPooler:
               type: boolean
             enableLogicalBackup:
               type: boolean
@@ -218,6 +240,10 @@ spec:
                   type: integer
                 retry_timeout:
                   type: integer
+                synchronous_mode:
+                  type: boolean
+                synchronous_mode_strict:
+                  type: boolean
                 maximum_lag_on_failover:
                   type: integer
             podAnnotations:
@@ -247,6 +273,26 @@ spec:
                   type: object
                   additionalProperties:
                     type: string
+            preparedDatabases:
+              type: object
+              additionalProperties:
+                type: object
+                properties:
+                  defaultUsers:
+                    type: boolean
+                  extensions:
+                    type: object
+                    additionalProperties:
+                      type: string
+                  schemas:
+                    type: object
+                    additionalProperties:
+                      type: object
+                      properties:
+                        defaultUsers:
+                          type: boolean
+                        defaultRoles:
+                          type: boolean
             replicaLoadBalancer:  # deprecated
               type: boolean
             resources:
@@ -338,6 +384,21 @@ spec:
                   type: string
             teamId:
               type: string
+            tls:
+              type: object
+              required:
+                - secretName
+              properties:
+                secretName:
+                  type: string
+                certificateFile:
+                  type: string
+                privateKeyFile:
+                  type: string
+                caFile:
+                  type: string
+                caSecretName:
+                  type: string
             tolerations:
               type: array
               items:

charts/postgres-operator/templates/clusterrole.yaml

@@ -42,6 +42,18 @@ rules:
   - configmaps
   verbs:
   - get
+# to send events to the CRs
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
+  - watch
 # to manage endpoints which are also used by Patroni
 - apiGroups:
   - ""

charts/postgres-operator/templates/configmap.yaml

@@ -20,5 +20,5 @@ data:
 {{ toYaml .Values.configDebug | indent 2 }}
 {{ toYaml .Values.configLoggingRestApi | indent 2 }}
 {{ toYaml .Values.configTeamsApi | indent 2 }}
-{{ toYaml .Values.configConnectionPool | indent 2 }}
+{{ toYaml .Values.configConnectionPooler | indent 2 }}
 {{- end }}

charts/postgres-operator/templates/operatorconfiguration.yaml

@@ -32,8 +32,6 @@ configuration:
 {{ toYaml .Values.configTeamsApi | indent 4 }}
   logging_rest_api:
 {{ toYaml .Values.configLoggingRestApi | indent 4 }}
-  scalyr:
-{{ toYaml .Values.configScalyr | indent 4 }}
-  connection_pool:
-{{ toYaml .Values.configConnectionPool | indent 4 }}
+  connection_pooler:
+{{ toYaml .Values.configConnectionPooler | indent 4 }}
 {{- end }}

charts/postgres-operator/values-crd.yaml

@@ -19,12 +19,16 @@ configTarget: "OperatorConfigurationCRD"
 configGeneral:
   # choose if deployment creates/updates CRDs with OpenAPIV3Validation
   enable_crd_validation: true
+  # update only the statefulsets without immediately doing the rolling update
+  enable_lazy_spilo_upgrade: false
   # start any new database pod without limitations on shm memory
   enable_shm_volume: true
   # etcd connection string for Patroni. Empty uses K8s-native DCS.
   etcd_host: ""
+  # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s)
+  # kubernetes_use_configmaps: false
   # Spilo docker image
-  docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p2
+  docker_image: registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p115
   # max number of instances in Postgres cluster. -1 = no limit
   min_instances: -1
   # min number of instances in Postgres cluster. -1 = no limit
@@ -250,48 +254,32 @@ configTeamsApi:
   # URL of the Teams API service
   # teams_api_url: http://fake-teams-api.default.svc.cluster.local
 
-# Scalyr is a log management tool that Zalando uses as a sidecar
-configScalyr:
-  # API key for the Scalyr sidecar
-  # scalyr_api_key: ""
-
-  # Docker image for the Scalyr sidecar
-  # scalyr_image: ""
-
-  # CPU limit value for the Scalyr sidecar
-  scalyr_cpu_limit: "1"
-  # CPU rquest value for the Scalyr sidecar
-  scalyr_cpu_request: 100m
-  # Memory limit value for the Scalyr sidecar
-  scalyr_memory_limit: 500Mi
-  # Memory request value for the Scalyr sidecar
-  scalyr_memory_request: 50Mi
-
-configConnectionPool:
+configConnectionPooler:
   # db schema to install lookup function into
-  connection_pool_schema: "pooler"
+  connection_pooler_schema: "pooler"
   # db user for pooler to use
-  connection_pool_user: "pooler"
+  connection_pooler_user: "pooler"
   # docker image
-  connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer"
+  connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer"
   # max db connections the pooler should hold
-  connection_pool_max_db_connections: 60
+  connection_pooler_max_db_connections: "60"
   # default pooling mode
-  connection_pool_mode: "transaction"
+  connection_pooler_mode: "transaction"
   # number of pooler instances
-  connection_pool_number_of_instances: 2
+  connection_pooler_number_of_instances: "2"
   # default resources
-  connection_pool_default_cpu_request: 500m
-  connection_pool_default_memory_request: 100Mi
-  connection_pool_default_cpu_limit: "1"
-  connection_pool_default_memory_limit: 100Mi
+  connection_pooler_default_cpu_request: 500m
+  connection_pooler_default_memory_request: 100Mi
+  connection_pooler_default_cpu_limit: "1"
+  connection_pooler_default_memory_limit: 100Mi
 
 rbac:
   # Specifies whether RBAC resources should be created
   create: true
 
 crd:
   # Specifies whether custom resource definitions should be created
+  # When using helm3, this is ignored; instead use "--skip-crds" to skip.
   create: true
 
 serviceAccount: