Support standby replication from GS (GCS) (zalando#1446)

* Add support for manual gs_wal_path in standby
* Remove separate standby version configuration
* Remove setting standby path via cluster/uid/version
Picking up the version doesn't work reliably without making changes to
Spilo. It's clearer to just specify the full S3/GS bucket path.

Co-authored-by: Felix Kunde <[email protected]>

Author: jamesmcdonald

charts/postgres-operator/crds/postgresqls.yaml

@@ -464,11 +464,11 @@ spec:
                 type: integer
               standby:
                 type: object
-                required:
-                  - s3_wal_path
                 properties:
                   s3_wal_path:
                     type: string
+                  gs_wal_path:
+                    type: string
               teamId:
                 type: string
               tls:

docs/reference/cluster_manifest.md

@@ -366,12 +366,16 @@ under the `clone` top-level key and do not affect the already running cluster.
 ## Standby cluster
 
 On startup, an existing `standby` top-level key creates a standby Postgres
-cluster streaming from a remote location. So far only streaming from a S3 WAL
-archive is supported.
+cluster streaming from a remote location. So far streaming from S3 and GCS WAL
+archives is supported.
 
 * **s3_wal_path**
   the url to S3 bucket containing the WAL archive of the remote primary.
-  Required when the `standby` section is present.
+  Optional, but `s3_wal_path`  or `gs_wal_path` is required.
+
+* **gs_wal_path**
+  the url to GS bucket containing the WAL archive of the remote primary.
+  Optional, but `s3_wal_path`  or `gs_wal_path` is required.
 
 ## Volume properties
 

docs/user.md

@@ -798,15 +798,21 @@ different location than its source database. Unlike cloning, the PostgreSQL
 version between source and target cluster has to be the same.
 
 To start a cluster as standby, add the following `standby` section in the YAML
-file and specify the S3 bucket path. An empty path will result in an error and
-no statefulset will be created.
+file. Specify the S3/GS bucket path. Omitting both settings will result in an error
+and no statefulset will be created.
 
 ```yaml
 spec:
   standby:
     s3_wal_path: "s3://<bucketname>/spilo/<source_db_cluster>/<UID>/wal/<PGVERSION>"
 ```
 
+```yaml
+spec:
+  standby:
+    gs_wal_path: "gs://<bucketname>/spilo/<source_db_cluster>/<UID>/wal/<PGVERSION>"
+```
+
 At the moment, the operator only allows to stream from the WAL archive of the
 master. Thus, it is recommended to deploy standby clusters with only [one pod](https://github.com/zalando/postgres-operator/blob/master/manifests/standby-manifest.yaml#L10).
 You can raise the instance count when detaching. Note, that the same pod role

pkg/apis/acid.zalan.do/v1/crds.go

@@ -649,12 +649,14 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 						Type: "integer",
 					},
 					"standby": {
-						Type:     "object",
-						Required: []string{"s3_wal_path"},
+						Type: "object",
 						Properties: map[string]apiextv1.JSONSchemaProps{
 							"s3_wal_path": {
 								Type: "string",
 							},
+							"gs_wal_path": {
+								Type: "string",
+							},
 						},
 					},
 					"teamId": {

pkg/apis/acid.zalan.do/v1/postgresql_type.go

@@ -166,6 +166,7 @@ type Patroni struct {
 // StandbyDescription contains s3 wal path
 type StandbyDescription struct {
 	S3WalPath string `json:"s3_wal_path,omitempty"`
+	GSWalPath string `json:"gs_wal_path,omitempty"`
 }
 
 // TLSDescription specs TLS properties

pkg/cluster/k8sres.go

@@ -1058,8 +1058,9 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 	sort.Slice(customPodEnvVarsList,
 		func(i, j int) bool { return customPodEnvVarsList[i].Name < customPodEnvVarsList[j].Name })
 
-	if spec.StandbyCluster != nil && spec.StandbyCluster.S3WalPath == "" {
-		return nil, fmt.Errorf("s3_wal_path is empty for standby cluster")
+	if spec.StandbyCluster != nil && spec.StandbyCluster.S3WalPath == "" &&
+		spec.StandbyCluster.GSWalPath == "" {
+		return nil, fmt.Errorf("one of s3_wal_path or gs_wal_path must be set for standby cluster")
 	}
 
 	// backward compatible check for InitContainers
@@ -1874,17 +1875,36 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription)
 func (c *Cluster) generateStandbyEnvironment(description *acidv1.StandbyDescription) []v1.EnvVar {
 	result := make([]v1.EnvVar, 0)
 
-	if description.S3WalPath == "" {
+	if description.S3WalPath == "" && description.GSWalPath == "" {
 		return nil
 	}
-	// standby with S3, find out the bucket to setup standby
-	msg := "Standby from S3 bucket using custom parsed S3WalPath from the manifest %s "
-	c.logger.Infof(msg, description.S3WalPath)
 
-	result = append(result, v1.EnvVar{
-		Name:  "STANDBY_WALE_S3_PREFIX",
-		Value: description.S3WalPath,
-	})
+	if description.S3WalPath != "" {
+		// standby with S3, find out the bucket to setup standby
+		msg := "Standby from S3 bucket using custom parsed S3WalPath from the manifest %s "
+		c.logger.Infof(msg, description.S3WalPath)
+
+		result = append(result, v1.EnvVar{
+			Name:  "STANDBY_WALE_S3_PREFIX",
+			Value: description.S3WalPath,
+		})
+	} else if description.GSWalPath != "" {
+		msg := "Standby from GS bucket using custom parsed GSWalPath from the manifest %s "
+		c.logger.Infof(msg, description.GSWalPath)
+
+		envs := []v1.EnvVar{
+			{
+				Name:  "STANDBY_WALE_GS_PREFIX",
+				Value: description.GSWalPath,
+			},
+			{
+				Name:  "STANDBY_GOOGLE_APPLICATION_CREDENTIALS",
+				Value: c.OpConfig.GCPCredentials,
+			},
+		}
+		result = append(result, envs...)
+
+	}
 
 	result = append(result, v1.EnvVar{Name: "STANDBY_METHOD", Value: "STANDBY_WITH_WALE"})
 	result = append(result, v1.EnvVar{Name: "STANDBY_WAL_BUCKET_SCOPE_PREFIX", Value: ""})