Logical backup retention time (zalando#1337)

* Add optional logical backup retention time
* Set defaults for potentially unbound variables, so that the script will work with older operator versions
* Document retention time parameter for logical backups
* Add retention time parameter to resources and charts

Co-authored-by: Felix Kunde <[email protected]>

Author: stoewer

charts/postgres-operator/crds/operatorconfigurations.yaml

@@ -450,6 +450,8 @@ spec:
                     type: string
                   logical_backup_s3_sse:
                     type: string
+                  logical_backup_s3_retention_time:
+                    type: string
                   logical_backup_schedule:
                     type: string
                     pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$'

charts/postgres-operator/values.yaml

@@ -310,6 +310,8 @@ configLogicalBackup:
   logical_backup_s3_secret_access_key: ""
   # S3 server side encryption
   logical_backup_s3_sse: "AES256"
+  # S3 retention time for stored backups for example "2 week" or "7 days"
+  logical_backup_s3_retention_time: ""
   # backup schedule in the cron format
   logical_backup_schedule: "30 00 * * *"
 

docker/logical-backup/dump.sh

@@ -15,6 +15,9 @@ TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
 K8S_API_URL=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1
 CERT=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
 
+LOGICAL_BACKUP_PROVIDER=${LOGICAL_BACKUP_PROVIDER:="s3"}
+LOGICAL_BACKUP_S3_RETENTION_TIME=${LOGICAL_BACKUP_S3_RETENTION_TIME:=""}
+
 function estimate_size {
     "$PG_BIN"/psql -tqAc "${ALL_DB_SIZE_QUERY}"
 }
@@ -28,6 +31,57 @@ function compress {
     pigz
 }
 
+function aws_delete_objects {
+    args=(
+      "--bucket=$LOGICAL_BACKUP_S3_BUCKET"
+    )
+
+    [[ ! -z "$LOGICAL_BACKUP_S3_ENDPOINT" ]] && args+=("--endpoint-url=$LOGICAL_BACKUP_S3_ENDPOINT")
+    [[ ! -z "$LOGICAL_BACKUP_S3_REGION" ]] && args+=("--region=$LOGICAL_BACKUP_S3_REGION")
+
+    aws s3api delete-objects "${args[@]}" --delete Objects=["$(printf {Key=%q}, "$@")"],Quiet=true
+}
+export -f aws_delete_objects
+
+function aws_delete_outdated {
+      if [[ -z "$LOGICAL_BACKUP_S3_RETENTION_TIME" ]] ; then
+          echo "no retention time configured: skip cleanup of outdated backups"
+          return 0
+      fi
+
+      # define cutoff date for outdated backups (day precision)
+      cutoff_date=$(date -d "$LOGICAL_BACKUP_S3_RETENTION_TIME ago" +%F)
+
+      # mimic bucket setup from Spilo
+      prefix="spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"
+
+      args=(
+        "--no-paginate"
+        "--output=text"
+        "--prefix=$prefix"
+        "--bucket=$LOGICAL_BACKUP_S3_BUCKET"
+      )
+
+      [[ ! -z "$LOGICAL_BACKUP_S3_ENDPOINT" ]] && args+=("--endpoint-url=$LOGICAL_BACKUP_S3_ENDPOINT")
+      [[ ! -z "$LOGICAL_BACKUP_S3_REGION" ]] && args+=("--region=$LOGICAL_BACKUP_S3_REGION")
+
+      # list objects older than the cutoff date
+      aws s3api list-objects "${args[@]}" --query="Contents[?LastModified<='$cutoff_date'].[Key]" > /tmp/outdated-backups
+
+      # spare the last backup
+      sed -i '$d' /tmp/outdated-backups
+
+      count=$(wc -l < /tmp/outdated-backups)
+      if [[ $count == 0 ]] ; then
+        echo "no outdated backups to delete"
+        return 0
+      fi
+      echo "deleting $count outdated backups created before $cutoff_date"
+
+      # deleted outdated files in batches with 100 at a time
+      tr '\n' '\0'  < /tmp/outdated-backups | xargs -0 -P1 -n100 bash -c 'aws_delete_objects "$@"' _
+}
+
 function aws_upload {
     declare -r EXPECTED_SIZE="$1"
 
@@ -59,6 +113,7 @@ function upload {
             ;;
         *)
             aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF))
+            aws_delete_outdated
             ;;
     esac
 }

docs/reference/operator_parameters.md

@@ -676,6 +676,11 @@ grouped under the `logical_backup` key.
   Specify server side encryption that S3 storage is using. If empty string
   is specified, no argument will be passed to `aws s3` command. Default: "AES256".
 
+* **logical_backup_s3_retention_time**
+  Specify a retention time for logical backups stored in S3. Backups older than the specified retention 
+  time will be deleted after a new backup was uploaded. If empty, all backups will be kept. Example values are
+  "3 days", "2 weeks", or "1 month". The default is empty.
+
 * **logical_backup_schedule**
   Backup schedule in the cron format. Please take the
   [reference schedule format](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule)

manifests/configmap.yaml

@@ -77,6 +77,7 @@ data:
   # logical_backup_s3_endpoint: ""
   # logical_backup_s3_secret_access_key: ""
   logical_backup_s3_sse: "AES256"
+  # logical_backup_s3_retention_time: ""
   logical_backup_schedule: "30 00 * * *"
   major_version_upgrade_mode: "manual"
   # major_version_upgrade_team_allow_list: ""

manifests/operatorconfiguration.crd.yaml

@@ -448,6 +448,8 @@ spec:
                     type: string
                   logical_backup_s3_sse:
                     type: string
+                  logical_backup_s3_retention_time:
+                    type: string
                   logical_backup_schedule:
                     type: string
                     pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$'

manifests/postgresql-operator-default-configuration.yaml

@@ -146,6 +146,7 @@ configuration:
     # logical_backup_s3_region: ""
     # logical_backup_s3_secret_access_key: ""
     logical_backup_s3_sse: "AES256"
+    # logical_backup_s3_retention_time: ""
     logical_backup_schedule: "30 00 * * *"
   debug:
     debug_logging: true

pkg/apis/acid.zalan.do/v1/crds.go

@@ -1556,6 +1556,9 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
 							"logical_backup_s3_sse": {
 								Type: "string",
 							},
+							"logical_backup_s3_retention_time": {
+								Type: "string",
+							},
 							"logical_backup_schedule": {
 								Type:    "string",
 								Pattern: "^(\\d+|\\*)(/\\d+)?(\\s+(\\d+|\\*)(/\\d+)?){4}$",

pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

@@ -213,6 +213,7 @@ type OperatorLogicalBackupConfiguration struct {
 	S3AccessKeyID                string `json:"logical_backup_s3_access_key_id,omitempty"`
 	S3SecretAccessKey            string `json:"logical_backup_s3_secret_access_key,omitempty"`
 	S3SSE                        string `json:"logical_backup_s3_sse,omitempty"`
+	RetentionTime                string `json:"logical_backup_s3_retention_time,omitempty"`
 	GoogleApplicationCredentials string `json:"logical_backup_google_application_credentials,omitempty"`
 	JobPrefix                    string `json:"logical_backup_job_prefix,omitempty"`
 }

pkg/cluster/k8sres.go

@@ -2134,6 +2134,10 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
 			Name:  "LOGICAL_BACKUP_S3_SSE",
 			Value: c.OpConfig.LogicalBackup.LogicalBackupS3SSE,
 		},
+		{
+			Name:  "LOGICAL_BACKUP_S3_RETENTION_TIME",
+			Value: c.OpConfig.LogicalBackup.LogicalBackupS3RetentionTime,
+		},
 		{
 			Name:  "LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX",
 			Value: getBucketScopeSuffix(string(c.Postgresql.GetUID())),