make bucket prefix for logical backup configurable (#2609)

* make bucket prefix for logical backup configurable
* include container comparison in logical backup diff
* add unit test and update description for compareContainers
* don't rely on users putting / in the config - reflect other comments from review
This commit is contained in:
Felix Kunde 2024-04-23 14:24:04 +02:00 committed by GitHub
parent 6ddafadc09
commit 83878fe447
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 165 additions and 65 deletions

View File

@ -81,7 +81,7 @@ jobs:
- name: Build and push multiarch logical-backup image to ghcr - name: Build and push multiarch logical-backup image to ghcr
uses: docker/build-push-action@v3 uses: docker/build-push-action@v3
with: with:
context: docker/logical-backup context: logical-backup
push: true push: true
build-args: BASE_IMAGE=ubuntu:22.04 build-args: BASE_IMAGE=ubuntu:22.04
tags: "${{ steps.image_lb.outputs.BACKUP_IMAGE }}" tags: "${{ steps.image_lb.outputs.BACKUP_IMAGE }}"

View File

@ -528,6 +528,8 @@ spec:
type: string type: string
logical_backup_s3_bucket: logical_backup_s3_bucket:
type: string type: string
logical_backup_s3_bucket_prefix:
type: string
logical_backup_s3_endpoint: logical_backup_s3_endpoint:
type: string type: string
logical_backup_s3_region: logical_backup_s3_region:

View File

@ -372,6 +372,8 @@ configLogicalBackup:
logical_backup_s3_access_key_id: "" logical_backup_s3_access_key_id: ""
# S3 bucket to store backup results # S3 bucket to store backup results
logical_backup_s3_bucket: "my-bucket-url" logical_backup_s3_bucket: "my-bucket-url"
# S3 bucket prefix to use
logical_backup_s3_bucket_prefix: "spilo"
# S3 region of bucket # S3 region of bucket
logical_backup_s3_region: "" logical_backup_s3_region: ""
# S3 endpoint url when not using AWS # S3 endpoint url when not using AWS

View File

@ -90,7 +90,7 @@ pipeline:
commands: commands:
- desc: Build image - desc: Build image
cmd: | cmd: |
cd docker/logical-backup cd logical-backup
export TAG=$(git describe --tags --always --dirty) export TAG=$(git describe --tags --always --dirty)
IMAGE="registry-write.opensource.zalan.do/acid/logical-backup" IMAGE="registry-write.opensource.zalan.do/acid/logical-backup"
docker build --rm -t "$IMAGE:$TAG$CDP_TAG" . docker build --rm -t "$IMAGE:$TAG$CDP_TAG" .

View File

@ -1283,7 +1283,7 @@ but only snapshots of your data. In its current state, see logical backups as a
way to quickly create SQL dumps that you can easily restore in an empty test way to quickly create SQL dumps that you can easily restore in an empty test
cluster. cluster.
2. The [example image](https://github.com/zalando/postgres-operator/blob/master/docker/logical-backup/Dockerfile) implements the backup 2. The [example image](https://github.com/zalando/postgres-operator/blob/master/logical-backup/Dockerfile) implements the backup
via `pg_dumpall` and upload of compressed and encrypted results to an S3 bucket. via `pg_dumpall` and upload of compressed and encrypted results to an S3 bucket.
`pg_dumpall` requires a `superuser` access to a DB and runs on the replica when `pg_dumpall` requires a `superuser` access to a DB and runs on the replica when
possible. possible.

View File

@ -813,9 +813,9 @@ grouped under the `logical_backup` key.
default values from `postgres_pod_resources` will be used. default values from `postgres_pod_resources` will be used.
* **logical_backup_docker_image** * **logical_backup_docker_image**
An image for pods of the logical backup job. The [example image](https://github.com/zalando/postgres-operator/blob/master/docker/logical-backup/Dockerfile) An image for pods of the logical backup job. The [example image](https://github.com/zalando/postgres-operator/blob/master/logical-backup/Dockerfile)
runs `pg_dumpall` on a replica if possible and uploads compressed results to runs `pg_dumpall` on a replica if possible and uploads compressed results to
an S3 bucket under the key `/spilo/pg_cluster_name/cluster_k8s_uuid/logical_backups`. an S3 bucket under the key `/<configured-s3-bucket-prefix>/<pg_cluster_name>/<cluster_k8s_uuid>/logical_backups`.
The default image is the same image built with the Zalando-internal CI The default image is the same image built with the Zalando-internal CI
pipeline. Default: "registry.opensource.zalan.do/acid/logical-backup:v1.11.0" pipeline. Default: "registry.opensource.zalan.do/acid/logical-backup:v1.11.0"
@ -845,6 +845,9 @@ grouped under the `logical_backup` key.
S3 bucket to store backup results. The bucket has to be present and S3 bucket to store backup results. The bucket has to be present and
accessible by Postgres pods. Default: empty. accessible by Postgres pods. Default: empty.
* **logical_backup_s3_bucket_prefix**
S3 bucket prefix to use in configured bucket. Default: "spilo"
* **logical_backup_s3_endpoint** * **logical_backup_s3_endpoint**
When using non-AWS S3 storage, endpoint can be set as a ENV variable. The default is empty. When using non-AWS S3 storage, endpoint can be set as a ENV variable. The default is empty.

View File

@ -45,7 +45,7 @@ function compress {
} }
function az_upload { function az_upload {
PATH_TO_BACKUP=$LOGICAL_BACKUP_S3_BUCKET"/spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz PATH_TO_BACKUP=$LOGICAL_BACKUP_S3_BUCKET"/"$LOGICAL_BACKUP_S3_BUCKET_PREFIX"/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz
az storage blob upload --file "$1" --account-name "$LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_NAME" --account-key "$LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_KEY" -c "$LOGICAL_BACKUP_AZURE_STORAGE_CONTAINER" -n "$PATH_TO_BACKUP" az storage blob upload --file "$1" --account-name "$LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_NAME" --account-key "$LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_KEY" -c "$LOGICAL_BACKUP_AZURE_STORAGE_CONTAINER" -n "$PATH_TO_BACKUP"
} }
@ -72,7 +72,7 @@ function aws_delete_outdated {
cutoff_date=$(date -d "$LOGICAL_BACKUP_S3_RETENTION_TIME ago" +%F) cutoff_date=$(date -d "$LOGICAL_BACKUP_S3_RETENTION_TIME ago" +%F)
# mimic bucket setup from Spilo # mimic bucket setup from Spilo
prefix="spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/" prefix=$LOGICAL_BACKUP_S3_BUCKET_PREFIX"/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"
args=( args=(
"--no-paginate" "--no-paginate"
@ -107,7 +107,7 @@ function aws_upload {
# mimic bucket setup from Spilo # mimic bucket setup from Spilo
# to keep logical backups at the same path as WAL # to keep logical backups at the same path as WAL
# NB: $LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX already contains the leading "/" when set by the Postgres Operator # NB: $LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX already contains the leading "/" when set by the Postgres Operator
PATH_TO_BACKUP=s3://$LOGICAL_BACKUP_S3_BUCKET"/spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz PATH_TO_BACKUP=s3://$LOGICAL_BACKUP_S3_BUCKET"/"$LOGICAL_BACKUP_S3_BUCKET_PREFIX"/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz
args=() args=()
@ -120,7 +120,7 @@ function aws_upload {
} }
function gcs_upload { function gcs_upload {
PATH_TO_BACKUP=gs://$LOGICAL_BACKUP_S3_BUCKET"/spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz PATH_TO_BACKUP=gs://$LOGICAL_BACKUP_S3_BUCKET"/"$LOGICAL_BACKUP_S3_BUCKET_PREFIX"/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz
gsutil -o Credentials:gs_service_key_file=$LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS cp - "$PATH_TO_BACKUP" gsutil -o Credentials:gs_service_key_file=$LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS cp - "$PATH_TO_BACKUP"
} }

View File

@ -90,6 +90,7 @@ data:
logical_backup_provider: "s3" logical_backup_provider: "s3"
# logical_backup_s3_access_key_id: "" # logical_backup_s3_access_key_id: ""
logical_backup_s3_bucket: "my-bucket-url" logical_backup_s3_bucket: "my-bucket-url"
# logical_backup_s3_bucket_prefix: "spilo"
# logical_backup_s3_region: "" # logical_backup_s3_region: ""
# logical_backup_s3_endpoint: "" # logical_backup_s3_endpoint: ""
# logical_backup_s3_secret_access_key: "" # logical_backup_s3_secret_access_key: ""

View File

@ -526,6 +526,8 @@ spec:
type: string type: string
logical_backup_s3_bucket: logical_backup_s3_bucket:
type: string type: string
logical_backup_s3_bucket_prefix:
type: string
logical_backup_s3_endpoint: logical_backup_s3_endpoint:
type: string type: string
logical_backup_s3_region: logical_backup_s3_region:

View File

@ -172,6 +172,7 @@ configuration:
logical_backup_provider: "s3" logical_backup_provider: "s3"
# logical_backup_s3_access_key_id: "" # logical_backup_s3_access_key_id: ""
logical_backup_s3_bucket: "my-bucket-url" logical_backup_s3_bucket: "my-bucket-url"
# logical_backup_s3_bucket_prefix: "spilo"
# logical_backup_s3_endpoint: "" # logical_backup_s3_endpoint: ""
# logical_backup_s3_region: "" # logical_backup_s3_region: ""
# logical_backup_s3_secret_access_key: "" # logical_backup_s3_secret_access_key: ""

View File

@ -1762,6 +1762,9 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
"logical_backup_s3_bucket": { "logical_backup_s3_bucket": {
Type: "string", Type: "string",
}, },
"logical_backup_s3_bucket_prefix": {
Type: "string",
},
"logical_backup_s3_endpoint": { "logical_backup_s3_endpoint": {
Type: "string", Type: "string",
}, },

View File

@ -228,6 +228,7 @@ type OperatorLogicalBackupConfiguration struct {
AzureStorageContainer string `json:"logical_backup_azure_storage_container,omitempty"` AzureStorageContainer string `json:"logical_backup_azure_storage_container,omitempty"`
AzureStorageAccountKey string `json:"logical_backup_azure_storage_account_key,omitempty"` AzureStorageAccountKey string `json:"logical_backup_azure_storage_account_key,omitempty"`
S3Bucket string `json:"logical_backup_s3_bucket,omitempty"` S3Bucket string `json:"logical_backup_s3_bucket,omitempty"`
S3BucketPrefix string `json:"logical_backup_s3_bucket_prefix,omitempty"`
S3Region string `json:"logical_backup_s3_region,omitempty"` S3Region string `json:"logical_backup_s3_region,omitempty"`
S3Endpoint string `json:"logical_backup_s3_endpoint,omitempty"` S3Endpoint string `json:"logical_backup_s3_endpoint,omitempty"`
S3AccessKeyID string `json:"logical_backup_s3_access_key_id,omitempty"` S3AccessKeyID string `json:"logical_backup_s3_access_key_id,omitempty"`

View File

@ -28,6 +28,7 @@ import (
"github.com/zalando/postgres-operator/pkg/util/users" "github.com/zalando/postgres-operator/pkg/util/users"
"github.com/zalando/postgres-operator/pkg/util/volumes" "github.com/zalando/postgres-operator/pkg/util/volumes"
appsv1 "k8s.io/api/apps/v1" appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
policyv1 "k8s.io/api/policy/v1" policyv1 "k8s.io/api/policy/v1"
rbacv1 "k8s.io/api/rbac/v1" rbacv1 "k8s.io/api/rbac/v1"
@ -438,8 +439,8 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa
reasons = append(reasons, "new statefulset's persistent volume claim retention policy do not match") reasons = append(reasons, "new statefulset's persistent volume claim retention policy do not match")
} }
needsRollUpdate, reasons = c.compareContainers("initContainers", c.Statefulset.Spec.Template.Spec.InitContainers, statefulSet.Spec.Template.Spec.InitContainers, needsRollUpdate, reasons) needsRollUpdate, reasons = c.compareContainers("statefulset initContainers", c.Statefulset.Spec.Template.Spec.InitContainers, statefulSet.Spec.Template.Spec.InitContainers, needsRollUpdate, reasons)
needsRollUpdate, reasons = c.compareContainers("containers", c.Statefulset.Spec.Template.Spec.Containers, statefulSet.Spec.Template.Spec.Containers, needsRollUpdate, reasons) needsRollUpdate, reasons = c.compareContainers("statefulset containers", c.Statefulset.Spec.Template.Spec.Containers, statefulSet.Spec.Template.Spec.Containers, needsRollUpdate, reasons)
if len(c.Statefulset.Spec.Template.Spec.Containers) == 0 { if len(c.Statefulset.Spec.Template.Spec.Containers) == 0 {
c.logger.Warningf("statefulset %q has no container", util.NameFromMeta(c.Statefulset.ObjectMeta)) c.logger.Warningf("statefulset %q has no container", util.NameFromMeta(c.Statefulset.ObjectMeta))
@ -571,30 +572,30 @@ func newCheck(msg string, cond containerCondition) containerCheck {
func (c *Cluster) compareContainers(description string, setA, setB []v1.Container, needsRollUpdate bool, reasons []string) (bool, []string) { func (c *Cluster) compareContainers(description string, setA, setB []v1.Container, needsRollUpdate bool, reasons []string) (bool, []string) {
if len(setA) != len(setB) { if len(setA) != len(setB) {
return true, append(reasons, fmt.Sprintf("new statefulset %s's length does not match the current ones", description)) return true, append(reasons, fmt.Sprintf("new %s's length does not match the current ones", description))
} }
checks := []containerCheck{ checks := []containerCheck{
newCheck("new statefulset %s's %s (index %d) name does not match the current one", newCheck("new %s's %s (index %d) name does not match the current one",
func(a, b v1.Container) bool { return a.Name != b.Name }), func(a, b v1.Container) bool { return a.Name != b.Name }),
newCheck("new statefulset %s's %s (index %d) readiness probe does not match the current one", newCheck("new %s's %s (index %d) readiness probe does not match the current one",
func(a, b v1.Container) bool { return !reflect.DeepEqual(a.ReadinessProbe, b.ReadinessProbe) }), func(a, b v1.Container) bool { return !reflect.DeepEqual(a.ReadinessProbe, b.ReadinessProbe) }),
newCheck("new statefulset %s's %s (index %d) ports do not match the current one", newCheck("new %s's %s (index %d) ports do not match the current one",
func(a, b v1.Container) bool { return !comparePorts(a.Ports, b.Ports) }), func(a, b v1.Container) bool { return !comparePorts(a.Ports, b.Ports) }),
newCheck("new statefulset %s's %s (index %d) resources do not match the current ones", newCheck("new %s's %s (index %d) resources do not match the current ones",
func(a, b v1.Container) bool { return !compareResources(&a.Resources, &b.Resources) }), func(a, b v1.Container) bool { return !compareResources(&a.Resources, &b.Resources) }),
newCheck("new statefulset %s's %s (index %d) environment does not match the current one", newCheck("new %s's %s (index %d) environment does not match the current one",
func(a, b v1.Container) bool { return !compareEnv(a.Env, b.Env) }), func(a, b v1.Container) bool { return !compareEnv(a.Env, b.Env) }),
newCheck("new statefulset %s's %s (index %d) environment sources do not match the current one", newCheck("new %s's %s (index %d) environment sources do not match the current one",
func(a, b v1.Container) bool { return !reflect.DeepEqual(a.EnvFrom, b.EnvFrom) }), func(a, b v1.Container) bool { return !reflect.DeepEqual(a.EnvFrom, b.EnvFrom) }),
newCheck("new statefulset %s's %s (index %d) security context does not match the current one", newCheck("new %s's %s (index %d) security context does not match the current one",
func(a, b v1.Container) bool { return !reflect.DeepEqual(a.SecurityContext, b.SecurityContext) }), func(a, b v1.Container) bool { return !reflect.DeepEqual(a.SecurityContext, b.SecurityContext) }),
newCheck("new statefulset %s's %s (index %d) volume mounts do not match the current one", newCheck("new %s's %s (index %d) volume mounts do not match the current one",
func(a, b v1.Container) bool { return !reflect.DeepEqual(a.VolumeMounts, b.VolumeMounts) }), func(a, b v1.Container) bool { return !reflect.DeepEqual(a.VolumeMounts, b.VolumeMounts) }),
} }
if !c.OpConfig.EnableLazySpiloUpgrade { if !c.OpConfig.EnableLazySpiloUpgrade {
checks = append(checks, newCheck("new statefulset %s's %s (index %d) image does not match the current one", checks = append(checks, newCheck("new %s's %s (index %d) image does not match the current one",
func(a, b v1.Container) bool { return a.Image != b.Image })) func(a, b v1.Container) bool { return a.Image != b.Image }))
} }
@ -786,6 +787,47 @@ func (c *Cluster) compareServices(old, new *v1.Service) (bool, string) {
return true, "" return true, ""
} }
func (c *Cluster) compareLogicalBackupJob(cur, new *batchv1.CronJob) (match bool, reason string) {
if cur.Spec.Schedule != new.Spec.Schedule {
return false, fmt.Sprintf("new job's schedule %q does not match the current one %q",
new.Spec.Schedule, cur.Spec.Schedule)
}
newImage := new.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Image
curImage := cur.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Image
if newImage != curImage {
return false, fmt.Sprintf("new job's image %q does not match the current one %q",
newImage, curImage)
}
newPgVersion := getPgVersion(new)
curPgVersion := getPgVersion(cur)
if newPgVersion != curPgVersion {
return false, fmt.Sprintf("new job's env PG_VERSION %q does not match the current one %q",
newPgVersion, curPgVersion)
}
needsReplace := false
reasons := make([]string, 0)
needsReplace, reasons = c.compareContainers("cronjob container", cur.Spec.JobTemplate.Spec.Template.Spec.Containers, new.Spec.JobTemplate.Spec.Template.Spec.Containers, needsReplace, reasons)
if needsReplace {
return false, fmt.Sprintf("logical backup container specs do not match: %v", strings.Join(reasons, `', '`))
}
return true, ""
}
func getPgVersion(cronJob *batchv1.CronJob) string {
envs := cronJob.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Env
for _, env := range envs {
if env.Name == "PG_VERSION" {
return env.Value
}
}
return ""
}
// addFinalizer patches the postgresql CR to add finalizer // addFinalizer patches the postgresql CR to add finalizer
func (c *Cluster) addFinalizer() error { func (c *Cluster) addFinalizer() error {
if c.hasFinalizer() { if c.hasFinalizer() {

View File

@ -19,6 +19,7 @@ import (
"github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/constants"
"github.com/zalando/postgres-operator/pkg/util/k8sutil" "github.com/zalando/postgres-operator/pkg/util/k8sutil"
"github.com/zalando/postgres-operator/pkg/util/teams" "github.com/zalando/postgres-operator/pkg/util/teams"
batchv1 "k8s.io/api/batch/v1"
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/kubernetes/fake"
@ -1649,16 +1650,92 @@ func TestCompareServices(t *testing.T) {
if match && !tt.match { if match && !tt.match {
t.Logf("match=%v current=%v, old=%v reason=%s", match, tt.current.Annotations, tt.new.Annotations, reason) t.Logf("match=%v current=%v, old=%v reason=%s", match, tt.current.Annotations, tt.new.Annotations, reason)
t.Errorf("%s - expected services to do not match: %q and %q", t.Name(), tt.current, tt.new) t.Errorf("%s - expected services to do not match: %q and %q", t.Name(), tt.current, tt.new)
return
} }
if !match && tt.match { if !match && tt.match {
t.Errorf("%s - expected services to be the same: %q and %q", t.Name(), tt.current, tt.new) t.Errorf("%s - expected services to be the same: %q and %q", t.Name(), tt.current, tt.new)
return
} }
if !match && !tt.match { if !match && !tt.match {
if !strings.HasPrefix(reason, tt.reason) { if !strings.HasPrefix(reason, tt.reason) {
t.Errorf("%s - expected reason prefix %s, found %s", t.Name(), tt.reason, reason) t.Errorf("%s - expected reason prefix %s, found %s", t.Name(), tt.reason, reason)
return }
}
})
}
}
func newCronJob(image, schedule string, vars []v1.EnvVar) *batchv1.CronJob {
cron := &batchv1.CronJob{
Spec: batchv1.CronJobSpec{
Schedule: schedule,
JobTemplate: batchv1.JobTemplateSpec{
Spec: batchv1.JobSpec{
Template: v1.PodTemplateSpec{
Spec: v1.PodSpec{
Containers: []v1.Container{
v1.Container{
Name: "logical-backup",
Image: image,
Env: vars,
},
},
},
},
},
},
},
}
return cron
}
func TestCompareLogicalBackupJob(t *testing.T) {
img1 := "registry.opensource.zalan.do/acid/logical-backup:v1.0"
img2 := "registry.opensource.zalan.do/acid/logical-backup:v2.0"
tests := []struct {
about string
current *batchv1.CronJob
new *batchv1.CronJob
match bool
reason string
}{
{
about: "two equal cronjobs",
current: newCronJob(img1, "0 0 * * *", []v1.EnvVar{}),
new: newCronJob(img1, "0 0 * * *", []v1.EnvVar{}),
match: true,
},
{
about: "two cronjobs with different image",
current: newCronJob(img1, "0 0 * * *", []v1.EnvVar{}),
new: newCronJob(img2, "0 0 * * *", []v1.EnvVar{}),
match: false,
reason: fmt.Sprintf("new job's image %q does not match the current one %q", img2, img1),
},
{
about: "two cronjobs with different schedule",
current: newCronJob(img1, "0 0 * * *", []v1.EnvVar{}),
new: newCronJob(img1, "0 * * * *", []v1.EnvVar{}),
match: false,
reason: fmt.Sprintf("new job's schedule %q does not match the current one %q", "0 * * * *", "0 0 * * *"),
},
{
about: "two cronjobs with different environment variables",
current: newCronJob(img1, "0 0 * * *", []v1.EnvVar{{Name: "LOGICAL_BACKUP_S3_BUCKET_PREFIX", Value: "spilo"}}),
new: newCronJob(img1, "0 0 * * *", []v1.EnvVar{{Name: "LOGICAL_BACKUP_S3_BUCKET_PREFIX", Value: "logical-backup"}}),
match: false,
reason: "logical backup container specs do not match: new cronjob container's logical-backup (index 0) environment does not match the current one",
},
}
for _, tt := range tests {
t.Run(tt.about, func(t *testing.T) {
match, reason := cl.compareLogicalBackupJob(tt.current, tt.new)
if match != tt.match {
t.Errorf("%s - unexpected match result %t when comparing cronjobs %q and %q", t.Name(), match, tt.current, tt.new)
} else {
if !strings.HasPrefix(reason, tt.reason) {
t.Errorf("%s - expected reason prefix %s, found %s", t.Name(), tt.reason, reason)
} }
} }
}) })

View File

@ -2403,6 +2403,10 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
Name: "LOGICAL_BACKUP_S3_RETENTION_TIME", Name: "LOGICAL_BACKUP_S3_RETENTION_TIME",
Value: c.OpConfig.LogicalBackup.LogicalBackupS3RetentionTime, Value: c.OpConfig.LogicalBackup.LogicalBackupS3RetentionTime,
}, },
{
Name: "LOGICAL_BACKUP_S3_BUCKET_PREFIX",
Value: c.OpConfig.LogicalBackup.LogicalBackupS3BucketPrefix,
},
{ {
Name: "LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX", Name: "LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX",
Value: getBucketScopeSuffix(string(c.Postgresql.GetUID())), Value: getBucketScopeSuffix(string(c.Postgresql.GetUID())),

View File

@ -1367,7 +1367,7 @@ func (c *Cluster) syncLogicalBackupJob() error {
if err != nil { if err != nil {
return fmt.Errorf("could not generate the desired logical backup job state: %v", err) return fmt.Errorf("could not generate the desired logical backup job state: %v", err)
} }
if match, reason := k8sutil.SameLogicalBackupJob(job, desiredJob); !match { if match, reason := c.compareLogicalBackupJob(job, desiredJob); !match {
c.logger.Infof("logical job %s is not in the desired state and needs to be updated", c.logger.Infof("logical job %s is not in the desired state and needs to be updated",
c.getLogicalBackupJobName(), c.getLogicalBackupJobName(),
) )

View File

@ -184,6 +184,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
result.LogicalBackupAzureStorageAccountKey = fromCRD.LogicalBackup.AzureStorageAccountKey result.LogicalBackupAzureStorageAccountKey = fromCRD.LogicalBackup.AzureStorageAccountKey
result.LogicalBackupAzureStorageContainer = fromCRD.LogicalBackup.AzureStorageContainer result.LogicalBackupAzureStorageContainer = fromCRD.LogicalBackup.AzureStorageContainer
result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket
result.LogicalBackupS3BucketPrefix = util.Coalesce(fromCRD.LogicalBackup.S3BucketPrefix, "spilo")
result.LogicalBackupS3Region = fromCRD.LogicalBackup.S3Region result.LogicalBackupS3Region = fromCRD.LogicalBackup.S3Region
result.LogicalBackupS3Endpoint = fromCRD.LogicalBackup.S3Endpoint result.LogicalBackupS3Endpoint = fromCRD.LogicalBackup.S3Endpoint
result.LogicalBackupS3AccessKeyID = fromCRD.LogicalBackup.S3AccessKeyID result.LogicalBackupS3AccessKeyID = fromCRD.LogicalBackup.S3AccessKeyID

View File

@ -132,6 +132,7 @@ type LogicalBackup struct {
LogicalBackupAzureStorageContainer string `name:"logical_backup_azure_storage_container" default:""` LogicalBackupAzureStorageContainer string `name:"logical_backup_azure_storage_container" default:""`
LogicalBackupAzureStorageAccountKey string `name:"logical_backup_azure_storage_account_key" default:""` LogicalBackupAzureStorageAccountKey string `name:"logical_backup_azure_storage_account_key" default:""`
LogicalBackupS3Bucket string `name:"logical_backup_s3_bucket" default:""` LogicalBackupS3Bucket string `name:"logical_backup_s3_bucket" default:""`
LogicalBackupS3BucketPrefix string `name:"logical_backup_s3_bucket_prefix" default:"spilo"`
LogicalBackupS3Region string `name:"logical_backup_s3_region" default:""` LogicalBackupS3Region string `name:"logical_backup_s3_region" default:""`
LogicalBackupS3Endpoint string `name:"logical_backup_s3_endpoint" default:""` LogicalBackupS3Endpoint string `name:"logical_backup_s3_endpoint" default:""`
LogicalBackupS3AccessKeyID string `name:"logical_backup_s3_access_key_id" default:""` LogicalBackupS3AccessKeyID string `name:"logical_backup_s3_access_key_id" default:""`

View File

@ -8,7 +8,6 @@ import (
b64 "encoding/base64" b64 "encoding/base64"
"encoding/json" "encoding/json"
batchv1 "k8s.io/api/batch/v1"
clientbatchv1 "k8s.io/client-go/kubernetes/typed/batch/v1" clientbatchv1 "k8s.io/client-go/kubernetes/typed/batch/v1"
apiacidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" apiacidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
@ -254,45 +253,6 @@ func SamePDB(cur, new *apipolicyv1.PodDisruptionBudget) (match bool, reason stri
return return
} }
func getJobImage(cronJob *batchv1.CronJob) string {
return cronJob.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Image
}
func getPgVersion(cronJob *batchv1.CronJob) string {
envs := cronJob.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Env
for _, env := range envs {
if env.Name == "PG_VERSION" {
return env.Value
}
}
return ""
}
// SameLogicalBackupJob compares Specs of logical backup cron jobs
func SameLogicalBackupJob(cur, new *batchv1.CronJob) (match bool, reason string) {
if cur.Spec.Schedule != new.Spec.Schedule {
return false, fmt.Sprintf("new job's schedule %q does not match the current one %q",
new.Spec.Schedule, cur.Spec.Schedule)
}
newImage := getJobImage(new)
curImage := getJobImage(cur)
if newImage != curImage {
return false, fmt.Sprintf("new job's image %q does not match the current one %q",
newImage, curImage)
}
newPgVersion := getPgVersion(new)
curPgVersion := getPgVersion(cur)
if newPgVersion != curPgVersion {
return false, fmt.Sprintf("new job's env PG_VERSION %q does not match the current one %q",
newPgVersion, curPgVersion)
}
return true, ""
}
func (c *mockSecret) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.Secret, error) { func (c *mockSecret) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.Secret, error) {
oldFormatSecret := &v1.Secret{} oldFormatSecret := &v1.Secret{}
oldFormatSecret.Name = "testcluster" oldFormatSecret.Name = "testcluster"