Sample logical backup image.
Based on the earlier work by Dmitry Dolgov @erthalion
This commit is contained in:
		
							parent
							
								
									24d412a562
								
							
						
					
					
						commit
						5c93825eda
					
				|  | @ -0,0 +1,33 @@ | |||
| FROM ubuntu:18.04 | ||||
| LABEL maintainer="Team ACID @ Zalando <team-acid@zalando.de>" | ||||
| 
 | ||||
| SHELL ["/bin/bash", "-o", "pipefail", "-c"] | ||||
| RUN apt-get update     \ | ||||
|     && apt-get install --no-install-recommends -y \ | ||||
|         apt-utils=1.6.10 \ | ||||
|         ca-certificates=20180409 \ | ||||
|         lsb-release=9.20170808ubuntu1 \ | ||||
|         pigz=2.4-1 \ | ||||
|         python3-pip=9.0.1-2.3~ubuntu1 \ | ||||
|         python3-setuptools=39.0.1-2 \ | ||||
|         curl=7.58.0-2ubuntu3.7 \ | ||||
|         jq=1.5+dfsg-2 \ | ||||
|         gnupg=2.2.4-1ubuntu1.2 \ | ||||
|     && pip3 install --no-cache-dir awscli==1.14.44 --upgrade \ | ||||
|     && echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ | ||||
|     && cat /etc/apt/sources.list.d/pgdg.list \ | ||||
|     && curl --silent https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ | ||||
|     && apt-get update \ | ||||
|     && apt-get install --no-install-recommends -y  \ | ||||
|         postgresql-client-11=11.3-1.pgdg18.04+1    \ | ||||
|         postgresql-client-10=10.8-1.pgdg18.04+1    \ | ||||
|         postgresql-client-9.6=9.6.13-1.pgdg18.04+1 \ | ||||
|         postgresql-client-9.5=9.5.17-1.pgdg18.04+1 \ | ||||
|     && apt-get clean \ | ||||
|     && rm -rf /var/lib/apt/lists/* | ||||
| 
 | ||||
| COPY dump.sh /dump.sh | ||||
| 
 | ||||
| ENV PG_DIR=/usr/lib/postgresql/ | ||||
| 
 | ||||
| ENTRYPOINT ["/dump.sh"] | ||||
|  | @ -0,0 +1,85 @@ | |||
| #! /usr/bin/env bash | ||||
| set -ex | ||||
| 
 | ||||
| ALL_DB_SIZE_QUERY="select sum(pg_database_size(datname)::numeric) from pg_database;" | ||||
| PG_BIN=$PG_DIR/$PG_VERSION/bin | ||||
| DUMP_SIZE_COEFF=5 | ||||
| 
 | ||||
| TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) | ||||
| K8S_API_URL=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1 | ||||
| CERT=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt | ||||
| 
 | ||||
| function estimate_size { | ||||
|     "$PG_BIN"/psql -tqAc "${ALL_DB_SIZE_QUERY}" | ||||
| } | ||||
| 
 | ||||
| function dump { | ||||
|     "$PG_BIN"/pg_dumpall "$PG_EXTRA_OPTIONS" | ||||
| } | ||||
| 
 | ||||
| function compress { | ||||
|     pigz | ||||
| } | ||||
| 
 | ||||
| function aws_upload { | ||||
|     declare -r EXPECTED_SIZE="$1" | ||||
| 
 | ||||
|     # mimic bucket setup from Spilo | ||||
|     # to keep logical backups at the same path as WAL | ||||
|     # NB: $LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX already contains the leading "/" when set by the Postgres operator | ||||
|     PATH_TO_BACKUP=s3://$LOGICAL_BACKUP_S3_BUCKET"/spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz | ||||
| 
 | ||||
|     if [ -z "$EXPECTED_SIZE" ]; then | ||||
|         aws s3 cp - "$PATH_TO_BACKUP" --debug --sse="AES256" | ||||
|     else | ||||
|         aws s3 cp - "$PATH_TO_BACKUP" --debug --expected-size "$EXPECTED_SIZE" --sse="AES256" | ||||
|     fi; | ||||
| } | ||||
| 
 | ||||
| function get_pods { | ||||
|     declare -r SELECTOR="$1" | ||||
| 
 | ||||
|     curl "${K8S_API_URL}/pods?$SELECTOR"        \ | ||||
|         --cacert $CERT                          \ | ||||
|         -H "Authorization: Bearer ${TOKEN}" | jq .items[].status.podIP -r | ||||
| } | ||||
| 
 | ||||
| function get_current_pod { | ||||
|     curl "${K8S_API_URL}/pods?fieldSelector=metadata.name%3D${HOSTNAME}" \ | ||||
|         --cacert $CERT                                                   \ | ||||
|         -H "Authorization: Bearer ${TOKEN}" | ||||
| } | ||||
| 
 | ||||
| declare -a search_strategy=( | ||||
|     list_all_replica_pods_current_node | ||||
|     list_all_replica_pods_any_node | ||||
|     get_master_pod | ||||
| ) | ||||
| 
 | ||||
| function list_all_replica_pods_current_node { | ||||
|     get_pods "labelSelector=version%3D${SCOPE},spilo-role%3Dreplica&fieldSelector=spec.nodeName%3D${CURRENT_NODENAME}" | head -n 1 | ||||
| } | ||||
| 
 | ||||
| function list_all_replica_pods_any_node { | ||||
|     get_pods "labelSelector=version%3D${SCOPE},spilo-role%3Dreplica" | head -n 1 | ||||
| } | ||||
| 
 | ||||
| function get_master_pod { | ||||
|     get_pods "labelSelector=version%3D${SCOPE},spilo-role%3Dmaster" | head -n 1 | ||||
| } | ||||
| 
 | ||||
| CURRENT_NODENAME=$(get_current_pod | jq .items[].spec.nodeName --raw-output) | ||||
| export CURRENT_NODENAME | ||||
| 
 | ||||
| for search in "${search_strategy[@]}"; do | ||||
| 
 | ||||
|     PGHOST=$(eval "$search") | ||||
|     export PGHOST | ||||
| 
 | ||||
|     if [ -n "$PGHOST" ]; then | ||||
|         break | ||||
|     fi | ||||
| 
 | ||||
| done | ||||
| 
 | ||||
| dump | compress | aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF)) | ||||
		Loading…
	
		Reference in New Issue