public_git
/
postgres-operator
mirror of https://github.com/zalando/postgres-operator.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into version-16-pg13

This commit is contained in:
Felix Kunde 2020-12-16 11:38:53 +01:00
parent 48d1caf727 5076e669cb
commit e4757670a3
12 changed files with 88 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
charts/postgres-operator/crds/postgresqls.yaml
View File

@ -128,7 +128,7 @@ spec:
type: string
timestamp:
type: string
pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$'
pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$'
# The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC
# Example: 1996-12-19T16:39:57-08:00
# Note: this field requires a timezone

22
charts/postgres-operator/templates/clusterrole-postgres-pod.yaml
View File

@ -10,6 +10,27 @@ metadata:
app.kubernetes.io/instance: {{ .Release.Name }}
rules:
# Patroni needs to watch and manage endpoints
{{- if toString .Values.configGeneral.kubernetes_use_configmaps | eq "true" }}
- apiGroups:
- ""
resources:
- configmaps
verbs:
- create
- delete
- deletecollection
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
- endpoints
verbs:
- get
{{- else }}
- apiGroups:
- ""
resources:
@ -23,6 +44,7 @@ rules:
- patch
- update
- watch
{{- end }}
# Patroni needs to watch pods
- apiGroups:
- ""

6
charts/postgres-operator/templates/clusterrole.yaml
View File

@ -63,11 +63,15 @@ rules:
- patch
- update
- watch
# to manage endpoints which are also used by Patroni
# to manage endpoints/configmaps which are also used by Patroni
- apiGroups:
- ""
resources:
{{- if toString .Values.configGeneral.kubernetes_use_configmaps | eq "true" }}
- configmaps
{{- else }}
- endpoints
{{- end }}
verbs:
- create
- delete

3
docker/logical-backup/Dockerfile
View File

@ -13,7 +13,10 @@ RUN apt-get update \
curl \
jq \
gnupg \
gcc \
libffi-dev \
&& pip3 install --no-cache-dir awscli --upgrade \
&& pip3 install --no-cache-dir gsutil --upgrade \
&& echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
&& cat /etc/apt/sources.list.d/pgdg.list \
&& curl --silent https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \

19
docker/logical-backup/dump.sh
View File

@ -46,6 +46,23 @@ function aws_upload {
aws s3 cp - "$PATH_TO_BACKUP" "${args[@]//\'/}"
}
function gcs_upload {
PATH_TO_BACKUP=gs://$LOGICAL_BACKUP_S3_BUCKET"/spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz
gsutil -o Credentials:gs_service_key_file=$LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS cp - "$PATH_TO_BACKUP"
}
function upload {
case $LOGICAL_BACKUP_PROVIDER in
"gcs")
gcs_upload
;;
*)
aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF))
;;
esac
}
function get_pods {
declare -r SELECTOR="$1"
@ -93,7 +110,7 @@ for search in "${search_strategy[@]}"; do
done
set -x
dump | compress | aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF))
dump | compress | upload
[[ ${PIPESTATUS[0]} != 0 || ${PIPESTATUS[1]} != 0 || ${PIPESTATUS[2]} != 0 ]] && (( ERRORCOUNT += 1 ))
set +x

7
docs/reference/operator_parameters.md
View File

@ -563,6 +563,10 @@ grouped under the `logical_backup` key.
The default image is the same image built with the Zalando-internal CI
pipeline. Default: "registry.opensource.zalan.do/acid/logical-backup"
* **logical_backup_provider**
Specifies the storage provider to which the backup should be uploaded (`s3` or `gcs`).
Default: "s3"
* **logical_backup_s3_bucket**
S3 bucket to store backup results. The bucket has to be present and
accessible by Postgres pods. Default: empty.
@ -583,6 +587,9 @@ grouped under the `logical_backup` key.
* **logical_backup_s3_secret_access_key**
When set, value will be in AWS_SECRET_ACCESS_KEY env variable. The Default is empty.
* **logical_backup_google_application_credentials**
Specifies the path of the google cloud service account json file. Default is empty.
## Debugging the operator
Options to aid debugging of the operator itself. Grouped under the `debug` key.

2
manifests/postgresql.crd.yaml
View File

@ -124,7 +124,7 @@ spec:
type: string
timestamp:
type: string
pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$'
pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$'
# The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC
# Example: 1996-12-19T16:39:57-08:00
# Note: this field requires a timezone

2
pkg/apis/acid.zalan.do/v1/crds.go
View File

@ -202,7 +202,7 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
"timestamp": {
Type: "string",
Description: "Date-time format that specifies a timezone as an offset relative to UTC e.g. 1996-12-19T16:39:57-08:00",
Pattern: "^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$",
Pattern: "^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\\.[0-9]+)?(([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$",
},
"uid": {
Type: "string",

2
pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
View File

@ -188,12 +188,14 @@ type ConnectionPoolerConfiguration struct {
type OperatorLogicalBackupConfiguration struct {
Schedule string `json:"logical_backup_schedule,omitempty"`
DockerImage string `json:"logical_backup_docker_image,omitempty"`
BackupProvider string `json:"logical_backup_provider,omitempty"`
S3Bucket string `json:"logical_backup_s3_bucket,omitempty"`
S3Region string `json:"logical_backup_s3_region,omitempty"`
S3Endpoint string `json:"logical_backup_s3_endpoint,omitempty"`
S3AccessKeyID string `json:"logical_backup_s3_access_key_id,omitempty"`
S3SecretAccessKey string `json:"logical_backup_s3_secret_access_key,omitempty"`
S3SSE string `json:"logical_backup_s3_sse,omitempty"`
GoogleApplicationCredentials string `json:"logical_backup_google_application_credentials,omitempty"`
}
// OperatorConfigurationData defines the operation config

8
pkg/cluster/k8sres.go
View File

@ -1988,6 +1988,10 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
},
},
// Bucket env vars
{
Name: "LOGICAL_BACKUP_PROVIDER",
Value: c.OpConfig.LogicalBackup.LogicalBackupProvider,
},
{
Name: "LOGICAL_BACKUP_S3_BUCKET",
Value: c.OpConfig.LogicalBackup.LogicalBackupS3Bucket,
@ -2008,6 +2012,10 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
Name: "LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX",
Value: getBucketScopeSuffix(string(c.Postgresql.GetUID())),
},
{
Name: "LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS",
Value: c.OpConfig.LogicalBackup.LogicalBackupGoogleApplicationCredentials,
},
// Postgres env vars
{
Name: "PG_VERSION",

2
pkg/controller/operator_config.go
View File

@ -146,12 +146,14 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
// logical backup config
result.LogicalBackupSchedule = util.Coalesce(fromCRD.LogicalBackup.Schedule, "30 00 * * *")
result.LogicalBackupDockerImage = util.Coalesce(fromCRD.LogicalBackup.DockerImage, "registry.opensource.zalan.do/acid/logical-backup")
result.LogicalBackupProvider = util.Coalesce(fromCRD.LogicalBackup.BackupProvider, "s3")
result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket
result.LogicalBackupS3Region = fromCRD.LogicalBackup.S3Region
result.LogicalBackupS3Endpoint = fromCRD.LogicalBackup.S3Endpoint
result.LogicalBackupS3AccessKeyID = fromCRD.LogicalBackup.S3AccessKeyID
result.LogicalBackupS3SecretAccessKey = fromCRD.LogicalBackup.S3SecretAccessKey
result.LogicalBackupS3SSE = fromCRD.LogicalBackup.S3SSE
result.LogicalBackupGoogleApplicationCredentials = fromCRD.LogicalBackup.GoogleApplicationCredentials
// debug config
result.DebugLogging = fromCRD.OperatorDebug.DebugLogging

2
pkg/util/config/config.go
View File

@ -113,12 +113,14 @@ type Scalyr struct {
type LogicalBackup struct {
LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"`
LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup"`
LogicalBackupProvider string `name:"logical_backup_provider" default:"s3"`
LogicalBackupS3Bucket string `name:"logical_backup_s3_bucket" default:""`
LogicalBackupS3Region string `name:"logical_backup_s3_region" default:""`
LogicalBackupS3Endpoint string `name:"logical_backup_s3_endpoint" default:""`
LogicalBackupS3AccessKeyID string `name:"logical_backup_s3_access_key_id" default:""`
LogicalBackupS3SecretAccessKey string `name:"logical_backup_s3_secret_access_key" default:""`
LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:""`
LogicalBackupGoogleApplicationCredentials string `name:"logical_backup_google_application_credentials" default:""`
}
// Operator options for connection pooler