public_git
/
postgres-operator
mirror of https://github.com/zalando/postgres-operator.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

[UI] Remove deprecated WAL-E library and enable WAL-G backup support in UI backend (#2915)

This commit is contained in:
Ida Novindasari 2025-05-20 16:31:26 +02:00 committed by GitHub
parent 68c4b49636
commit ccb52c094d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 57 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
charts/postgres-operator-ui/values.yaml
View File

@ -62,8 +62,6 @@ podAnnotations:
extraEnvs:
[]
# Exemple of settings to make snapshot view working in the ui when using AWS
# - name: WALE_S3_ENDPOINT
# value: https+path://s3.us-east-1.amazonaws.com:443
# - name: SPILO_S3_BACKUP_PREFIX
# value: spilo/
# - name: AWS_ACCESS_KEY_ID
@ -83,8 +81,6 @@ extraEnvs:
# key: AWS_DEFAULT_REGION
# - name: SPILO_S3_BACKUP_BUCKET
# value: <s3 bucket used by the operator>
# - name: "USE_AWS_INSTANCE_PROFILE"
# value: "true"
# configure UI service
service:

4
ui/manifests/deployment.yaml
View File

@ -81,8 +81,6 @@ spec:
]
}
# Exemple of settings to make snapshot view working in the ui when using AWS
# - name: WALE_S3_ENDPOINT
# value: https+path://s3.us-east-1.amazonaws.com:443
# - name: SPILO_S3_BACKUP_PREFIX
# value: spilo/
# - name: AWS_ACCESS_KEY_ID
@ -102,5 +100,3 @@ spec:
# key: AWS_DEFAULT_REGION
# - name: SPILO_S3_BACKUP_BUCKET
# value: <s3 bucket used by the operator>
# - name: "USE_AWS_INSTANCE_PROFILE"
# value: "true"

15
ui/operator_ui/main.py
View File

@ -95,14 +95,6 @@ DEFAULT_MEMORY_LIMIT = getenv('DEFAULT_MEMORY_LIMIT', '300Mi')
DEFAULT_CPU = getenv('DEFAULT_CPU', '10m')
DEFAULT_CPU_LIMIT = getenv('DEFAULT_CPU_LIMIT', '300m')
WALE_S3_ENDPOINT = getenv(
'WALE_S3_ENDPOINT',
'https+path://s3.eu-central-1.amazonaws.com:443',
)
USE_AWS_INSTANCE_PROFILE = (
getenv('USE_AWS_INSTANCE_PROFILE', 'false').lower() != 'false'
)
AWS_ENDPOINT = getenv('AWS_ENDPOINT')
@ -784,8 +776,6 @@ def get_versions(pg_cluster: str):
bucket=SPILO_S3_BACKUP_BUCKET,
pg_cluster=pg_cluster,
prefix=SPILO_S3_BACKUP_PREFIX,
s3_endpoint=WALE_S3_ENDPOINT,
use_aws_instance_profile=USE_AWS_INSTANCE_PROFILE,
),
)
@ -797,9 +787,8 @@ def get_basebackups(pg_cluster: str, uid: str):
bucket=SPILO_S3_BACKUP_BUCKET,
pg_cluster=pg_cluster,
prefix=SPILO_S3_BACKUP_PREFIX,
s3_endpoint=WALE_S3_ENDPOINT,
uid=uid,
use_aws_instance_profile=USE_AWS_INSTANCE_PROFILE,
postgresql_versions=OPERATOR_UI_CONFIG.get('postgresql_versions', DEFAULT_UI_CONFIG['postgresql_versions']),
),
)
@ -991,8 +980,6 @@ def main(port, debug, clusters: list):
logger.info(f'Superuser team: {SUPERUSER_TEAM}')
logger.info(f'Target namespace: {TARGET_NAMESPACE}')
logger.info(f'Teamservice URL: {TEAM_SERVICE_URL}')
logger.info(f'Use AWS instance_profile: {USE_AWS_INSTANCE_PROFILE}')
logger.info(f'WAL-E S3 endpoint: {WALE_S3_ENDPOINT}')
logger.info(f'AWS S3 endpoint: {AWS_ENDPOINT}')
if TARGET_NAMESPACE is None:

78
ui/operator_ui/spiloutils.py
View File

@ -6,9 +6,8 @@ from os import environ, getenv
from requests import Session
from urllib.parse import urljoin
from uuid import UUID
from wal_e.cmd import configure_backup_cxt
from .utils import Attrs, defaulting, these
from .utils import defaulting, these
from operator_ui.adapters.logger import logger
session = Session()
@ -284,10 +283,8 @@ def read_stored_clusters(bucket, prefix, delimiter='/'):
def read_versions(
pg_cluster,
bucket,
s3_endpoint,
prefix,
delimiter='/',
use_aws_instance_profile=False,
):
return [
'base' if uid == 'wal' else uid
@ -305,35 +302,72 @@ def read_versions(
if uid == 'wal' or defaulting(lambda: UUID(uid))
]
BACKUP_VERSION_PREFIXES = ['', '10/', '11/', '12/', '13/', '14/', '15/', '16/', '17/']
def lsn_to_wal_segment_stop(finish_lsn, start_segment, wal_segment_size=16 * 1024 * 1024):
timeline = int(start_segment[:8], 16)
log_id = finish_lsn >> 32
seg_id = (finish_lsn & 0xFFFFFFFF) // wal_segment_size
return f"{timeline:08X}{log_id:08X}{seg_id:08X}"
def lsn_to_offset_hex(lsn, wal_segment_size=16 * 1024 * 1024):
return f"{lsn % wal_segment_size:08X}"
def read_basebackups(
pg_cluster,
uid,
bucket,
s3_endpoint,
prefix,
delimiter='/',
use_aws_instance_profile=False,
postgresql_versions,
):
environ['WALE_S3_ENDPOINT'] = s3_endpoint
suffix = '' if uid == 'base' else '/' + uid
backups = []
for vp in BACKUP_VERSION_PREFIXES:
for vp in postgresql_versions:
backup_prefix = f'{prefix}{pg_cluster}{suffix}/wal/{vp}/basebackups_005/'
logger.info(f"{bucket}/{backup_prefix}")
backups = backups + [
{
key: value
for key, value in basebackup.__dict__.items()
if isinstance(value, str) or isinstance(value, int)
}
for basebackup in Attrs.call(
f=configure_backup_cxt,
aws_instance_profile=use_aws_instance_profile,
s3_prefix=f's3://{bucket}/{prefix}{pg_cluster}{suffix}/wal/{vp}',
)._backup_list(detail=True)
]
paginator = client('s3').get_paginator('list_objects_v2')
pages = paginator.paginate(Bucket=bucket, Prefix=backup_prefix)
for page in pages:
for obj in page.get("Contents", []):
key = obj["Key"]
if not key.endswith("backup_stop_sentinel.json"):
continue
response = client('s3').get_object(Bucket=bucket, Key=key)
backup_info = loads(response["Body"].read().decode("utf-8"))
last_modified = response["LastModified"].astimezone(timezone.utc).isoformat()
backup_name = key.split("/")[-1].replace("_backup_stop_sentinel.json", "")
start_seg, start_offset = backup_name.split("_")[1], backup_name.split("_")[-1] if "_" in backup_name else None
if "LSN" in backup_info and "FinishLSN" in backup_info:
# WAL-G
lsn = backup_info["LSN"]
finish_lsn = backup_info["FinishLSN"]
backups.append({
"expanded_size_bytes": backup_info.get("UncompressedSize"),
"last_modified": last_modified,
"name": backup_name,
"wal_segment_backup_start": start_seg,
"wal_segment_backup_stop": lsn_to_wal_segment_stop(finish_lsn, start_seg),
"wal_segment_offset_backup_start": lsn_to_offset_hex(lsn),
"wal_segment_offset_backup_stop": lsn_to_offset_hex(finish_lsn),
})
elif "wal_segment_backup_stop" in backup_info:
# WAL-E
stop_seg = backup_info["wal_segment_backup_stop"]
stop_offset = backup_info["wal_segment_offset_backup_stop"]
backups.append({
"expanded_size_bytes": backup_info.get("expanded_size_bytes"),
"last_modified": last_modified,
"name": backup_name,
"wal_segment_backup_start": start_seg,
"wal_segment_backup_stop": stop_seg,
"wal_segment_offset_backup_start": start_offset,
"wal_segment_offset_backup_stop": stop_offset,
})
return backups

1
ui/requirements.txt
View File

@ -11,5 +11,4 @@ kubernetes==11.0.0
python-json-logger==2.0.7
requests==2.32.2
stups-tokens>=1.1.19
wal_e==1.1.1
werkzeug==3.0.6