[UI] Remove deprecated WAL-E library and enable WAL-G backup support in UI backend (zalando#2915)

Author: idanovinda

charts/postgres-operator-ui/values.yaml

@@ -62,8 +62,6 @@ podAnnotations:
 extraEnvs:
   []
   # Exemple of settings to make snapshot view working in the ui when using AWS
-  # - name: WALE_S3_ENDPOINT
-  #   value: https+path://s3.us-east-1.amazonaws.com:443
   # - name: SPILO_S3_BACKUP_PREFIX
   #   value: spilo/
   # - name: AWS_ACCESS_KEY_ID
@@ -83,8 +81,6 @@ extraEnvs:
   #       key: AWS_DEFAULT_REGION
   # - name: SPILO_S3_BACKUP_BUCKET
   #   value: <s3 bucket used by the operator>
-  # - name: "USE_AWS_INSTANCE_PROFILE"
-  #   value: "true"
 
 # configure UI service
 service:

ui/manifests/deployment.yaml

@@ -81,8 +81,6 @@ spec:
                   ]
                 }
             # Exemple of settings to make snapshot view working in the ui when using AWS
-            # - name: WALE_S3_ENDPOINT
-            #   value: https+path://s3.us-east-1.amazonaws.com:443
             # - name: SPILO_S3_BACKUP_PREFIX
             #   value: spilo/
             # - name: AWS_ACCESS_KEY_ID
@@ -102,5 +100,3 @@ spec:
             #       key: AWS_DEFAULT_REGION
             # - name: SPILO_S3_BACKUP_BUCKET
             #   value: <s3 bucket used by the operator>
-            # - name: "USE_AWS_INSTANCE_PROFILE"
-            #   value: "true"

ui/operator_ui/main.py

@@ -95,14 +95,6 @@
 DEFAULT_CPU = getenv('DEFAULT_CPU', '10m')
 DEFAULT_CPU_LIMIT = getenv('DEFAULT_CPU_LIMIT', '300m')
 
-WALE_S3_ENDPOINT = getenv(
-    'WALE_S3_ENDPOINT',
-    'https+path://s3.eu-central-1.amazonaws.com:443',
-)
-
-USE_AWS_INSTANCE_PROFILE = (
-    getenv('USE_AWS_INSTANCE_PROFILE', 'false').lower() != 'false'
-)
 
 AWS_ENDPOINT = getenv('AWS_ENDPOINT')
 
@@ -784,8 +776,6 @@ def get_versions(pg_cluster: str):
             bucket=SPILO_S3_BACKUP_BUCKET,
             pg_cluster=pg_cluster,
             prefix=SPILO_S3_BACKUP_PREFIX,
-            s3_endpoint=WALE_S3_ENDPOINT,
-            use_aws_instance_profile=USE_AWS_INSTANCE_PROFILE,
         ),
     )
 
@@ -797,9 +787,8 @@ def get_basebackups(pg_cluster: str, uid: str):
             bucket=SPILO_S3_BACKUP_BUCKET,
             pg_cluster=pg_cluster,
             prefix=SPILO_S3_BACKUP_PREFIX,
-            s3_endpoint=WALE_S3_ENDPOINT,
             uid=uid,
-            use_aws_instance_profile=USE_AWS_INSTANCE_PROFILE,
+            postgresql_versions=OPERATOR_UI_CONFIG.get('postgresql_versions', DEFAULT_UI_CONFIG['postgresql_versions']),
         ),
     )
 
@@ -991,8 +980,6 @@ def main(port, debug, clusters: list):
     logger.info(f'Superuser team: {SUPERUSER_TEAM}')
     logger.info(f'Target namespace: {TARGET_NAMESPACE}')
     logger.info(f'Teamservice URL: {TEAM_SERVICE_URL}')
-    logger.info(f'Use AWS instance_profile: {USE_AWS_INSTANCE_PROFILE}')
-    logger.info(f'WAL-E S3 endpoint: {WALE_S3_ENDPOINT}')
     logger.info(f'AWS S3 endpoint: {AWS_ENDPOINT}')
 
     if TARGET_NAMESPACE is None:

ui/operator_ui/spiloutils.py

@@ -6,9 +6,8 @@
 from requests import Session
 from urllib.parse import urljoin
 from uuid import UUID
-from wal_e.cmd import configure_backup_cxt
 
-from .utils import Attrs, defaulting, these
+from .utils import defaulting, these
 from operator_ui.adapters.logger import logger
 
 session = Session()
@@ -284,10 +283,8 @@ def read_stored_clusters(bucket, prefix, delimiter='/'):
 def read_versions(
     pg_cluster,
     bucket,
-    s3_endpoint,
     prefix,
     delimiter='/',
-    use_aws_instance_profile=False,
 ):
     return [
         'base' if uid == 'wal' else uid
@@ -305,35 +302,72 @@ def read_versions(
         if uid == 'wal' or defaulting(lambda: UUID(uid))
     ]
 
-BACKUP_VERSION_PREFIXES = ['', '10/', '11/', '12/', '13/', '14/', '15/', '16/', '17/']
+def lsn_to_wal_segment_stop(finish_lsn, start_segment, wal_segment_size=16 * 1024 * 1024):
+    timeline = int(start_segment[:8], 16)
+    log_id = finish_lsn >> 32
+    seg_id = (finish_lsn & 0xFFFFFFFF) // wal_segment_size
+    return f"{timeline:08X}{log_id:08X}{seg_id:08X}"
+
+def lsn_to_offset_hex(lsn, wal_segment_size=16 * 1024 * 1024):
+    return f"{lsn % wal_segment_size:08X}"
 
 def read_basebackups(
     pg_cluster,
     uid,
     bucket,
-    s3_endpoint,
     prefix,
-    delimiter='/',
-    use_aws_instance_profile=False,
+    postgresql_versions,
 ):
-    environ['WALE_S3_ENDPOINT'] = s3_endpoint
     suffix = '' if uid == 'base' else '/' + uid
     backups = []
 
-    for vp in BACKUP_VERSION_PREFIXES:
-
-        backups = backups + [
-            {
-                key: value
-                for key, value in basebackup.__dict__.items()
-                if isinstance(value, str) or isinstance(value, int)
-            }
-            for basebackup in Attrs.call(
-                f=configure_backup_cxt,
-                aws_instance_profile=use_aws_instance_profile,
-                s3_prefix=f's3://{bucket}/{prefix}{pg_cluster}{suffix}/wal/{vp}',
-            )._backup_list(detail=True)
-        ]
+    for vp in postgresql_versions:
+        backup_prefix = f'{prefix}{pg_cluster}{suffix}/wal/{vp}/basebackups_005/'
+        logger.info(f"{bucket}/{backup_prefix}")
+
+        paginator = client('s3').get_paginator('list_objects_v2')
+        pages = paginator.paginate(Bucket=bucket, Prefix=backup_prefix)
+
+        for page in pages:
+            for obj in page.get("Contents", []):
+                key = obj["Key"]
+                if not key.endswith("backup_stop_sentinel.json"):
+                    continue
+
+                response = client('s3').get_object(Bucket=bucket, Key=key)
+                backup_info = loads(response["Body"].read().decode("utf-8"))
+                last_modified = response["LastModified"].astimezone(timezone.utc).isoformat()
+
+                backup_name = key.split("/")[-1].replace("_backup_stop_sentinel.json", "")
+                start_seg, start_offset = backup_name.split("_")[1], backup_name.split("_")[-1] if "_" in backup_name else None
+
+                if "LSN" in backup_info and "FinishLSN" in backup_info:
+                    # WAL-G
+                    lsn = backup_info["LSN"]
+                    finish_lsn = backup_info["FinishLSN"]
+                    backups.append({
+                        "expanded_size_bytes": backup_info.get("UncompressedSize"),
+                        "last_modified": last_modified,
+                        "name": backup_name,
+                        "wal_segment_backup_start": start_seg,
+                        "wal_segment_backup_stop": lsn_to_wal_segment_stop(finish_lsn, start_seg),
+                        "wal_segment_offset_backup_start": lsn_to_offset_hex(lsn),
+                        "wal_segment_offset_backup_stop": lsn_to_offset_hex(finish_lsn),
+                    })
+                elif "wal_segment_backup_stop" in backup_info:
+                    # WAL-E
+                    stop_seg = backup_info["wal_segment_backup_stop"]
+                    stop_offset = backup_info["wal_segment_offset_backup_stop"]
+
+                    backups.append({
+                        "expanded_size_bytes": backup_info.get("expanded_size_bytes"),
+                        "last_modified": last_modified,
+                        "name": backup_name,
+                        "wal_segment_backup_start": start_seg,
+                        "wal_segment_backup_stop": stop_seg,
+                        "wal_segment_offset_backup_start": start_offset,
+                        "wal_segment_offset_backup_stop": stop_offset,
+                    })
 
     return backups
 

ui/requirements.txt

@@ -11,5 +11,4 @@ kubernetes==11.0.0
 python-json-logger==2.0.7
 requests==2.32.2
 stups-tokens>=1.1.19
-wal_e==1.1.1
 werkzeug==3.0.6