Add logical backup for azure (zalando#2052)

* Add logical backup for azure

Author: stgrace

charts/postgres-operator/crds/operatorconfigurations.yaml

@@ -498,6 +498,12 @@ spec:
                     type: string
                     pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$'
                     default: "30 00 * * *"
+                  logical_backup_azure_storage_account_name:
+                    type: string
+                  logical_backup_azure_storage_container:
+                    type: string
+                  logical_backup_azure_storage_account_key:
+                    type: string
               debug:
                 type: object
                 properties:

docker/logical-backup/Dockerfile

@@ -15,6 +15,7 @@ RUN apt-get update     \
         gnupg \
         gcc \
         libffi-dev \
+    && curl -sL https://aka.ms/InstallAzureCLIDeb | bash \
     && pip3 install --upgrade pip \
     && pip3 install --no-cache-dir awscli --upgrade \
     && pip3 install --no-cache-dir gsutil --upgrade \

docker/logical-backup/dump.sh

@@ -40,6 +40,12 @@ function compress {
     pigz
 }
 
+function az_upload {
+    PATH_TO_BACKUP=$LOGICAL_BACKUP_S3_BUCKET"/spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz
+
+    az storage blob upload --file "$1" --account-name "$LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_NAME" --account-key "$LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_KEY" -c "$LOGICAL_BACKUP_AZURE_STORAGE_CONTAINER" -n "$PATH_TO_BACKUP"
+}
+
 function aws_delete_objects {
     args=(
       "--bucket=$LOGICAL_BACKUP_S3_BUCKET"
@@ -120,7 +126,7 @@ function upload {
         "gcs")
             gcs_upload
             ;;
-        *)
+        "aws")
             aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF))
             aws_delete_outdated
             ;;
@@ -174,8 +180,13 @@ for search in "${search_strategy[@]}"; do
 done
 
 set -x
-dump | compress | upload
-[[ ${PIPESTATUS[0]} != 0 || ${PIPESTATUS[1]} != 0 || ${PIPESTATUS[2]} != 0 ]] && (( ERRORCOUNT += 1 ))
-set +x
+if [ "$LOGICAL_BACKUP_PROVIDER" == "az" ]; then
+    dump | compress > /tmp/azure-backup.sql.gz
+    az_upload /tmp/azure-backup.sql.gz
+else
+    dump | compress | upload
+    [[ ${PIPESTATUS[0]} != 0 || ${PIPESTATUS[1]} != 0 || ${PIPESTATUS[2]} != 0 ]] && (( ERRORCOUNT += 1 ))
+    set +x
 
-exit $ERRORCOUNT
+    exit $ERRORCOUNT
+fi

docs/reference/operator_parameters.md

@@ -765,6 +765,15 @@ grouped under the `logical_backup` key.
   [reference schedule format](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule)
   into account. Default: "30 00 \* \* \*"
 
+* **logical_backup_azure_storage_account_name**
+  Storage account name used to upload logical backups to when using Azure. Default: ""
+
+* **logical_backup_azure_storage_container**
+  Storage container used to upload logical backups to when using Azure. Default: ""
+
+* **logical_backup_azure_storage_account_key**
+  Storage account key used to authenticate with Azure when uploading logical backups. Default: ""
+
 ## Debugging the operator
 
 Options to aid debugging of the operator itself. Grouped under the `debug` key.

manifests/configmap.yaml

@@ -87,6 +87,9 @@ data:
   logical_backup_s3_sse: "AES256"
   # logical_backup_s3_retention_time: ""
   logical_backup_schedule: "30 00 * * *"
+  # logical_backup_azure_storage_account_name: ""
+  # logical_backup_azure_storage_container: ""
+  # logical_backup_azure_storage_account_key: ""
   major_version_upgrade_mode: "manual"
   # major_version_upgrade_team_allow_list: ""
   master_dns_name_format: "{cluster}.{namespace}.{hostedzone}"

manifests/operatorconfiguration.crd.yaml

@@ -496,6 +496,12 @@ spec:
                     type: string
                     pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$'
                     default: "30 00 * * *"
+                  logical_backup_azure_storage_account_name:
+                    type: string
+                  logical_backup_azure_storage_container:
+                    type: string
+                  logical_backup_azure_storage_account_key:
+                    type: string
               debug:
                 type: object
                 properties:

manifests/postgresql-operator-default-configuration.yaml

@@ -163,6 +163,9 @@ configuration:
     logical_backup_s3_sse: "AES256"
     # logical_backup_s3_retention_time: ""
     logical_backup_schedule: "30 00 * * *"
+    # logical_backup_azure_storage_account_name: ""
+    # logical_backup_azure_storage_container: ""
+    # logical_backup_azure_storage_account_key: ""
   debug:
     debug_logging: true
     enable_database_access: true

pkg/apis/acid.zalan.do/v1/crds.go

@@ -1655,6 +1655,15 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
 								Type:    "string",
 								Pattern: "^(\\d+|\\*)(/\\d+)?(\\s+(\\d+|\\*)(/\\d+)?){4}$",
 							},
+							"logical_backup_azure_storage_account_name": {
+								Type: "string",
+							},
+							"logical_backup_azure_storage_container": {
+								Type: "string",
+							},
+							"logical_backup_azure_storage_account_key": {
+								Type: "string",
+							},
 						},
 					},
 					"debug": {

pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

@@ -227,6 +227,9 @@ type OperatorLogicalBackupConfiguration struct {
 	RetentionTime                string `json:"logical_backup_s3_retention_time,omitempty"`
 	GoogleApplicationCredentials string `json:"logical_backup_google_application_credentials,omitempty"`
 	JobPrefix                    string `json:"logical_backup_job_prefix,omitempty"`
+	AzureStorageAccountName      string `json:"logical_backup_azure_storage_account_name,omitempty"`
+	AzureStorageContainer        string `json:"logical_backup_azure_storage_container,omitempty"`
+	AzureStorageAccountKey       string `json:"logical_backup_azure_storage_account_key,omitempty"`
 }
 
 // PatroniConfiguration defines configuration for Patroni

pkg/cluster/k8sres.go

@@ -2262,6 +2262,18 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
 			Name:  "LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS",
 			Value: c.OpConfig.LogicalBackup.LogicalBackupGoogleApplicationCredentials,
 		},
+		{
+			Name:  "LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_NAME",
+			Value: c.OpConfig.LogicalBackup.LogicalBackupAzureStorageAccountName,
+		},
+		{
+			Name:  "LOGICAL_BACKUP_AZURE_STORAGE_CONTAINER",
+			Value: c.OpConfig.LogicalBackup.LogicalBackupAzureStorageContainer,
+		},
+		{
+			Name:  "LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_KEY",
+			Value: c.OpConfig.LogicalBackup.LogicalBackupAzureStorageAccountKey,
+		},
 		// Postgres env vars
 		{
 			Name:  "PG_VERSION",