Skip to content

Commit 5f87384

Browse files
ermajnsdudoladov
ermajn
authored and
sdudoladov
committed
Passing endpoint, access and secret key to logical-backup container (zalando#628)
* Added possibility to add custom annotations to LoadBalancer service. * Added parameters for custom endpoint, access and secret key for logical backup. * Modified dump.sh so it knows how to handle new features. Configurable S3 SSE
1 parent 2ce602f commit 5f87384

File tree

10 files changed

+86
-18
lines changed

10 files changed

+86
-18
lines changed

‎charts/postgres-operator/values-crd.yaml‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -184,12 +184,20 @@ configAwsOrGcp:
184184

185185
# configure K8s cron job managed by the operator
186186
configLogicalBackup:
187-
# backup schedule in the cron format
188-
logical_backup_schedule: "30 00 * * *"
189187
# image for pods of the logical backup job (example runs pg_dumpall)
190188
logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup"
189+
# S3 Access Key ID
190+
logical_backup_s3_access_key_id: ""
191191
# S3 bucket to store backup results
192192
logical_backup_s3_bucket: "my-bucket-url"
193+
# S3 endpoint url when not using AWS
194+
logical_backup_s3_endpoint: ""
195+
# S3 Secret Access Key
196+
logical_backup_s3_secret_access_key: ""
197+
# S3 server side encription
198+
logical_backup_s3_sse: "AES256"
199+
# backup schedule in the cron format
200+
logical_backup_schedule: "30 00 * * *"
193201

194202
# automate creation of human users with teams API service
195203
configTeamsApi:

‎charts/postgres-operator/values.yaml‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -178,12 +178,20 @@ configAwsOrGcp:
178178

179179
# configure K8s cron job managed by the operator
180180
configLogicalBackup:
181-
# backup schedule in the cron format
182-
logical_backup_schedule: "30 00 * * *"
183181
# image for pods of the logical backup job (example runs pg_dumpall)
184182
logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup"
183+
# S3 Access Key ID
184+
logical_backup_s3_access_key_id: ""
185185
# S3 bucket to store backup results
186186
logical_backup_s3_bucket: "my-bucket-url"
187+
# S3 endpoint url when not using AWS
188+
logical_backup_s3_endpoint: ""
189+
# S3 Secret Access Key
190+
logical_backup_s3_secret_access_key: ""
191+
# S3 server side encription
192+
logical_backup_s3_sse: "AES256"
193+
# backup schedule in the cron format
194+
logical_backup_schedule: "30 00 * * *"
187195

188196
# automate creation of human users with teams API service
189197
configTeamsApi:

‎docker/logical-backup/dump.sh‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,13 @@ function aws_upload {
3838
# NB: $LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX already contains the leading "/" when set by the Postgres Operator
3939
PATH_TO_BACKUP=s3://$LOGICAL_BACKUP_S3_BUCKET"/spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz
4040

41-
if [ -z "$EXPECTED_SIZE" ]; then
42-
aws s3 cp - "$PATH_TO_BACKUP" --debug --sse="AES256"
43-
else
44-
aws s3 cp - "$PATH_TO_BACKUP" --debug --expected-size "$EXPECTED_SIZE" --sse="AES256"
45-
fi;
41+
args=()
42+
43+
[[ ! -z "$EXPECTED_SIZE" ]] && args+=("--expected-size=$EXPECTED_SIZE")
44+
[[ ! -z "$LOGICAL_BACKUP_S3_ENDPOINT" ]] && args+=("--endpoint-url=\"$LOGICAL_BACKUP_S3_ENDPOINT\"")
45+
[[ ! "$LOGICAL_BACKUP_S3_SSE" == "" ]] && args+=("--sse=\"$LOGICAL_BACKUP_S3_SSE\"")
46+
47+
aws s3 cp - "$PATH_TO_BACKUP" "${args[@]//\'/}" --debug
4648
}
4749

4850
function get_pods {

‎docs/reference/operator_parameters.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -441,6 +441,19 @@ grouped under the `logical_backup` key.
441441
S3 bucket to store backup results. The bucket has to be present and
442442
accessible by Postgres pods. Default: empty.
443443

444+
* **logical_backup_s3_endpoint**
445+
When using non-AWS S3 storage, endpoint can be set as a ENV variable.
446+
447+
* **logical_backup_s3_sse**
448+
Specify server side encription that S3 storage is using. If empty string
449+
is specified, no argument will be passed to `aws s3` command. Default: "AES256".
450+
451+
* **logical_backup_s3_access_key_id**
452+
When set, value will be in AWS_ACCESS_KEY_ID env variable. The Default is empty.
453+
454+
* **logical_backup_s3_secret_access_key**
455+
When set, value will be in AWS_SECRET_ACCESS_KEY env variable. The Default is empty.
456+
444457
## Debugging the operator
445458

446459
Options to aid debugging of the operator itself. Grouped under the `debug` key.

‎manifests/configmap.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,11 @@ data:
3535
# kube_iam_role: ""
3636
# log_s3_bucket: ""
3737
# logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup"
38+
# logical_backup_s3_access_key_id: ""
3839
# logical_backup_s3_bucket: "my-bucket-url"
40+
# logical_backup_s3_endpoint: ""
41+
# logical_backup_s3_secret_access_key: ""
42+
# logical_backup_s3_sse: "AES256"
3943
# logical_backup_schedule: "30 00 * * *"
4044
master_dns_name_format: "{cluster}.{team}.staging.{hostedzone}"
4145
# master_pod_move_timeout: 10m

‎manifests/postgresql-operator-default-configuration.yaml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,9 +74,14 @@ configuration:
7474
# log_s3_bucket: ""
7575
# wal_s3_bucket: ""
7676
logical_backup:
77-
logical_backup_schedule: "30 00 * * *"
77+
log_s3_bucket: ""
7878
logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup"
79+
logical_backup_s3_access_key_id: ""
7980
logical_backup_s3_bucket: "my-bucket-url"
81+
logical_backup_s3_endpoint: ""
82+
logical_backup_s3_secret_access_key: ""
83+
logical_backup_s3_sse: "AES256"
84+
logical_backup_schedule: "30 00 * * *"
8085
debug:
8186
debug_logging: true
8287
enable_database_access: true

‎pkg/apis/acid.zalan.do/v1/operator_configuration_type.go‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -184,8 +184,13 @@ type OperatorConfigurationUsers struct {
184184
//Duration shortens this frequently used name
185185
type Duration time.Duration
186186

187+
// OperatorLogicalBackupConfiguration defines configration for logical backup
187188
type OperatorLogicalBackupConfiguration struct {
188-
Schedule string `json:"logical_backup_schedule,omitempty"`
189-
DockerImage string `json:"logical_backup_docker_image,omitempty"`
190-
S3Bucket string `json:"logical_backup_s3_bucket,omitempty"`
189+
Schedule string `json:"logical_backup_schedule,omitempty"`
190+
DockerImage string `json:"logical_backup_docker_image,omitempty"`
191+
S3Bucket string `json:"logical_backup_s3_bucket,omitempty"`
192+
S3Endpoint string `json:"logical_backup_s3_endpoint,omitempty"`
193+
S3AccessKeyID string `json:"logical_backup_s3_access_key_id,omitempty"`
194+
S3SecretAccessKey string `json:"logical_backup_s3_secret_access_key,omitempty"`
195+
S3SSE string `json:"logical_backup_s3_sse,omitempty"`
191196
}

‎pkg/cluster/k8sres.go‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1575,6 +1575,14 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
15751575
Name: "LOGICAL_BACKUP_S3_BUCKET",
15761576
Value: c.OpConfig.LogicalBackup.LogicalBackupS3Bucket,
15771577
},
1578+
{
1579+
Name: "LOGICAL_BACKUP_S3_ENDPOINT",
1580+
Value: c.OpConfig.LogicalBackup.LogicalBackupS3Endpoint,
1581+
},
1582+
{
1583+
Name: "LOGICAL_BACKUP_S3_SSE",
1584+
Value: c.OpConfig.LogicalBackup.LogicalBackupS3SSE,
1585+
},
15781586
{
15791587
Name: "LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX",
15801588
Value: getBucketScopeSuffix(string(c.Postgresql.GetUID())),
@@ -1613,8 +1621,15 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
16131621
},
16141622
}
16151623

1616-
c.logger.Debugf("Generated logical backup env vars %v", envVars)
1624+
if c.OpConfig.LogicalBackup.LogicalBackupS3AccessKeyID != "" {
1625+
envVars = append(envVars, v1.EnvVar{Name: "AWS_ACCESS_KEY_ID", Value: c.OpConfig.LogicalBackup.LogicalBackupS3AccessKeyID})
1626+
}
1627+
1628+
if c.OpConfig.LogicalBackup.LogicalBackupS3SecretAccessKey != "" {
1629+
envVars = append(envVars, v1.EnvVar{Name: "AWS_SECRET_ACCESS_KEY", Value: c.OpConfig.LogicalBackup.LogicalBackupS3SecretAccessKey})
1630+
}
16171631

1632+
c.logger.Debugf("Generated logical backup env vars %v", envVars)
16181633
return envVars
16191634
}
16201635

‎pkg/controller/operator_config.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,10 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
101101
result.LogicalBackupSchedule = fromCRD.LogicalBackup.Schedule
102102
result.LogicalBackupDockerImage = fromCRD.LogicalBackup.DockerImage
103103
result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket
104+
result.LogicalBackupS3Endpoint = fromCRD.LogicalBackup.S3Endpoint
105+
result.LogicalBackupS3AccessKeyID = fromCRD.LogicalBackup.S3AccessKeyID
106+
result.LogicalBackupS3SecretAccessKey = fromCRD.LogicalBackup.S3SecretAccessKey
107+
result.LogicalBackupS3SSE = fromCRD.LogicalBackup.S3SSE
104108

105109
// debug config
106110
result.DebugLogging = fromCRD.OperatorDebug.DebugLogging

‎pkg/util/config/config.go‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -68,11 +68,15 @@ type Scalyr struct {
6868
ScalyrMemoryLimit string `name:"scalyr_memory_limit" default:"1Gi"`
6969
}
7070

71-
// LogicalBackup
71+
// LogicalBackup defines configration for logical backup
7272
type LogicalBackup struct {
73-
LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"`
74-
LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup"`
75-
LogicalBackupS3Bucket string `name:"logical_backup_s3_bucket" default:""`
73+
LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"`
74+
LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup"`
75+
LogicalBackupS3Bucket string `name:"logical_backup_s3_bucket" default:""`
76+
LogicalBackupS3Endpoint string `name:"logical_backup_s3_endpoint" default:""`
77+
LogicalBackupS3AccessKeyID string `name:"logical_backup_s3_access_key_id" default:""`
78+
LogicalBackupS3SecretAccessKey string `name:"logical_backup_s3_secret_access_key" default:""`
79+
LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:"AES256"`
7680
}
7781

7882
// Config describes operator config

0 commit comments

Comments
 (0)