From 29cec0ceda6dba6e761c217137f8fa9c1c90cb8c Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 5 Jan 2023 15:19:36 +0100 Subject: [PATCH] configurable resources for logical backup pod template (#710) * new config options to specify resources for logical backup jobs * bug in logical backup script for s3 dumps * define enum for logical_backup_provider * changed order of logical backup azure options * fix unit test for stream comparison --- .../crds/operatorconfigurations.yaml | 28 +++- charts/postgres-operator/values.yaml | 13 +- docker/logical-backup/dump.sh | 2 +- docs/administrator.md | 4 + docs/reference/operator_parameters.md | 29 ++-- manifests/configmap.yaml | 10 +- manifests/operatorconfiguration.crd.yaml | 28 +++- ...gresql-operator-default-configuration.yaml | 10 +- pkg/apis/acid.zalan.do/v1/crds.go | 45 ++++-- .../v1/operator_configuration_type.go | 10 +- pkg/cluster/k8sres.go | 24 +++- pkg/cluster/k8sres_test.go | 134 +++++++++++++++++- pkg/cluster/streams_test.go | 3 +- pkg/controller/operator_config.go | 10 +- pkg/util/config/config.go | 10 +- 15 files changed, 305 insertions(+), 55 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index b36c58b74..e10296659 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -469,6 +469,18 @@ spec: logical_backup: type: object properties: + logical_backup_azure_storage_account_name: + type: string + logical_backup_azure_storage_container: + type: string + logical_backup_azure_storage_account_key: + type: string + logical_backup_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + logical_backup_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' logical_backup_docker_image: type: string default: "registry.opensource.zalan.do/acid/logical-backup:v1.8.2" @@ -477,8 +489,18 @@ spec: logical_backup_job_prefix: type: string default: "logical-backup-" + logical_backup_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + logical_backup_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' logical_backup_provider: type: string + enum: + - "az" + - "gcs" + - "s3" default: "s3" logical_backup_s3_access_key_id: type: string @@ -498,12 +520,6 @@ spec: type: string pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' default: "30 00 * * *" - logical_backup_azure_storage_account_name: - type: string - logical_backup_azure_storage_container: - type: string - logical_backup_azure_storage_account_key: - type: string debug: type: object properties: diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 36c09c3ee..7766519ab 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -332,6 +332,17 @@ configAwsOrGcp: # configure K8s cron job managed by the operator configLogicalBackup: + # Azure Storage Account specs to store backup results + # logical_backup_azure_storage_account_name: "" + # logical_backup_azure_storage_container: "" + # logical_backup_azure_storage_account_key: "" + + # resources for logical backup pod, if empty configPostgresPodResources will be used + # logical_backup_cpu_limit: "" + # logical_backup_cpu_request: "" + # logical_backup_memory_limit: "" + # logical_backup_memory_request: "" + # image for pods of the logical backup job (example runs pg_dumpall) logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.8.0" # path of google cloud service account json file @@ -339,7 +350,7 @@ configLogicalBackup: # prefix for the backup job name logical_backup_job_prefix: "logical-backup-" - # storage provider - either "s3" or "gcs" + # storage provider - either "s3", "gcs" or "az" logical_backup_provider: "s3" # S3 Access Key ID logical_backup_s3_access_key_id: "" diff --git a/docker/logical-backup/dump.sh b/docker/logical-backup/dump.sh index 0aa723ab5..178577ced 100755 --- a/docker/logical-backup/dump.sh +++ b/docker/logical-backup/dump.sh @@ -126,7 +126,7 @@ function upload { "gcs") gcs_upload ;; - "aws") + "s3") aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF)) aws_delete_outdated ;; diff --git a/docs/administrator.md b/docs/administrator.md index 8fde634c4..5e1ade60c 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -1201,6 +1201,10 @@ of the backup cron job. `cronjobs` resource from the `batch` API group for the operator service account. See [example RBAC](https://github.com/zalando/postgres-operator/blob/master/manifests/operator-service-account-rbac.yaml) +7. Resources of the pod template in the cron job can be configured. When left +empty [default values of spilo pods](reference/operator_parameters.md#kubernetes-resource-requests) +will be used. + ## Sidecars for Postgres clusters A list of sidecars is added to each cluster created by the operator. The default diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index dd80ce569..981ba1ab6 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -718,6 +718,13 @@ These parameters configure a K8s cron job managed by the operator to produce Postgres logical backups. In the CRD-based configuration those parameters are grouped under the `logical_backup` key. +* **logical_backup_cpu_limit** + **logical_backup_cpu_request** + **logical_backup_memory_limit** + **logical_backup_memory_request** + Resource configuration for pod template in logical backup cron job. If empty + default values from `postgres_pod_resources` will be used. + * **logical_backup_docker_image** An image for pods of the logical backup job. The [example image](https://github.com/zalando/postgres-operator/blob/master/docker/logical-backup/Dockerfile) runs `pg_dumpall` on a replica if possible and uploads compressed results to @@ -732,8 +739,17 @@ grouped under the `logical_backup` key. The prefix to be prepended to the name of a k8s CronJob running the backups. Beware the prefix counts towards the name length restrictions imposed by k8s. Empty string is a legitimate value. Operator does not do the actual renaming: It simply creates the job with the new prefix. You will have to delete the old cron job manually. Default: "logical-backup-". * **logical_backup_provider** - Specifies the storage provider to which the backup should be uploaded (`s3` or `gcs`). - Default: "s3" + Specifies the storage provider to which the backup should be uploaded + (`s3`, `gcs` or `az`). Default: "s3" + +* **logical_backup_azure_storage_account_name** + Storage account name used to upload logical backups to when using Azure. Default: "" + +* **logical_backup_azure_storage_container** + Storage container used to upload logical backups to when using Azure. Default: "" + +* **logical_backup_azure_storage_account_key** + Storage account key used to authenticate with Azure when uploading logical backups. Default: "" * **logical_backup_s3_access_key_id** When set, value will be in AWS_ACCESS_KEY_ID env variable. The Default is empty. @@ -765,15 +781,6 @@ grouped under the `logical_backup` key. [reference schedule format](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule) into account. Default: "30 00 \* \* \*" -* **logical_backup_azure_storage_account_name** - Storage account name used to upload logical backups to when using Azure. Default: "" - -* **logical_backup_azure_storage_container** - Storage container used to upload logical backups to when using Azure. Default: "" - -* **logical_backup_azure_storage_account_key** - Storage account key used to authenticate with Azure when uploading logical backups. Default: "" - ## Debugging the operator Options to aid debugging of the operator itself. Grouped under the `debug` key. diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 15816c693..1b8c6a42f 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -75,9 +75,16 @@ data: # kube_iam_role: "" # kubernetes_use_configmaps: "false" # log_s3_bucket: "" + # logical_backup_azure_storage_account_name: "" + # logical_backup_azure_storage_container: "" + # logical_backup_azure_storage_account_key: "" + # logical_backup_cpu_limit: "" + # logical_backup_cpu_request: "" logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.8.2" # logical_backup_google_application_credentials: "" logical_backup_job_prefix: "logical-backup-" + # logical_backup_memory_limit: "" + # logical_backup_memory_request: "" logical_backup_provider: "s3" # logical_backup_s3_access_key_id: "" logical_backup_s3_bucket: "my-bucket-url" @@ -87,9 +94,6 @@ data: logical_backup_s3_sse: "AES256" # logical_backup_s3_retention_time: "" logical_backup_schedule: "30 00 * * *" - # logical_backup_azure_storage_account_name: "" - # logical_backup_azure_storage_container: "" - # logical_backup_azure_storage_account_key: "" major_version_upgrade_mode: "manual" # major_version_upgrade_team_allow_list: "" master_dns_name_format: "{cluster}.{namespace}.{hostedzone}" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index df6343852..14c0106db 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -467,6 +467,18 @@ spec: logical_backup: type: object properties: + logical_backup_azure_storage_account_name: + type: string + logical_backup_azure_storage_container: + type: string + logical_backup_azure_storage_account_key: + type: string + logical_backup_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + logical_backup_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' logical_backup_docker_image: type: string default: "registry.opensource.zalan.do/acid/logical-backup:v1.8.2" @@ -475,8 +487,18 @@ spec: logical_backup_job_prefix: type: string default: "logical-backup-" + logical_backup_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + logical_backup_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' logical_backup_provider: type: string + enum: + - "az" + - "gcs" + - "s3" default: "s3" logical_backup_s3_access_key_id: type: string @@ -496,12 +518,6 @@ spec: type: string pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' default: "30 00 * * *" - logical_backup_azure_storage_account_name: - type: string - logical_backup_azure_storage_container: - type: string - logical_backup_azure_storage_account_key: - type: string debug: type: object properties: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 36d009bbf..b0e751d1b 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -151,6 +151,13 @@ configuration: # wal_gs_bucket: "" # wal_s3_bucket: "" logical_backup: + # logical_backup_azure_storage_account_name: "" + # logical_backup_azure_storage_container: "" + # logical_backup_azure_storage_account_key: "" + # logical_backup_cpu_limit: "" + # logical_backup_cpu_request: "" + # logical_backup_memory_limit: "" + # logical_backup_memory_request: "" logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.8.2" # logical_backup_google_application_credentials: "" logical_backup_job_prefix: "logical-backup-" @@ -163,9 +170,6 @@ configuration: logical_backup_s3_sse: "AES256" # logical_backup_s3_retention_time: "" logical_backup_schedule: "30 00 * * *" - # logical_backup_azure_storage_account_name: "" - # logical_backup_azure_storage_container: "" - # logical_backup_azure_storage_account_key: "" debug: debug_logging: true enable_database_access: true diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index e0cb148eb..22d82a201 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -1618,6 +1618,23 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ "logical_backup": { Type: "object", Properties: map[string]apiextv1.JSONSchemaProps{ + "logical_backup_azure_storage_account_name": { + Type: "string", + }, + "logical_backup_azure_storage_container": { + Type: "string", + }, + "logical_backup_azure_storage_account_key": { + Type: "string", + }, + "logical_backup_cpu_limit": { + Type: "string", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + }, + "logical_backup_cpu_request": { + Type: "string", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + }, "logical_backup_docker_image": { Type: "string", }, @@ -1627,8 +1644,27 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ "logical_backup_job_prefix": { Type: "string", }, + "logical_backup_memory_limit": { + Type: "string", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + "logical_backup_memory_request": { + Type: "string", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, "logical_backup_provider": { Type: "string", + Enum: []apiextv1.JSON{ + { + Raw: []byte(`"az"`), + }, + { + Raw: []byte(`"gcs"`), + }, + { + Raw: []byte(`"s3"`), + }, + }, }, "logical_backup_s3_access_key_id": { Type: "string", @@ -1655,15 +1691,6 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "string", Pattern: "^(\\d+|\\*)(/\\d+)?(\\s+(\\d+|\\*)(/\\d+)?){4}$", }, - "logical_backup_azure_storage_account_name": { - Type: "string", - }, - "logical_backup_azure_storage_container": { - Type: "string", - }, - "logical_backup_azure_storage_account_key": { - Type: "string", - }, }, }, "debug": { diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index c57dff0cb..c62f1ad31 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -218,6 +218,9 @@ type OperatorLogicalBackupConfiguration struct { Schedule string `json:"logical_backup_schedule,omitempty"` DockerImage string `json:"logical_backup_docker_image,omitempty"` BackupProvider string `json:"logical_backup_provider,omitempty"` + AzureStorageAccountName string `json:"logical_backup_azure_storage_account_name,omitempty"` + AzureStorageContainer string `json:"logical_backup_azure_storage_container,omitempty"` + AzureStorageAccountKey string `json:"logical_backup_azure_storage_account_key,omitempty"` S3Bucket string `json:"logical_backup_s3_bucket,omitempty"` S3Region string `json:"logical_backup_s3_region,omitempty"` S3Endpoint string `json:"logical_backup_s3_endpoint,omitempty"` @@ -227,9 +230,10 @@ type OperatorLogicalBackupConfiguration struct { RetentionTime string `json:"logical_backup_s3_retention_time,omitempty"` GoogleApplicationCredentials string `json:"logical_backup_google_application_credentials,omitempty"` JobPrefix string `json:"logical_backup_job_prefix,omitempty"` - AzureStorageAccountName string `json:"logical_backup_azure_storage_account_name,omitempty"` - AzureStorageContainer string `json:"logical_backup_azure_storage_container,omitempty"` - AzureStorageAccountKey string `json:"logical_backup_azure_storage_account_key,omitempty"` + CPURequest string `json:"logical_backup_cpu_request,omitempty"` + MemoryRequest string `json:"logical_backup_memory_request,omitempty"` + CPULimit string `json:"logical_backup_cpu_limit,omitempty"` + MemoryLimit string `json:"logical_backup_memory_limit,omitempty"` } // PatroniConfiguration defines configuration for Patroni diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 963814941..4ef61f8b9 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -139,6 +139,23 @@ func makeDefaultResources(config *config.Config) acidv1.Resources { } } +func makeLogicalBackupResources(config *config.Config) acidv1.Resources { + + logicalBackupResourceRequests := acidv1.ResourceDescription{ + CPU: config.LogicalBackup.LogicalBackupCPURequest, + Memory: config.LogicalBackup.LogicalBackupMemoryRequest, + } + logicalBackupResourceLimits := acidv1.ResourceDescription{ + CPU: config.LogicalBackup.LogicalBackupCPULimit, + Memory: config.LogicalBackup.LogicalBackupMemoryLimit, + } + + return acidv1.Resources{ + ResourceRequests: logicalBackupResourceRequests, + ResourceLimits: logicalBackupResourceLimits, + } +} + func (c *Cluster) enforceMinResourceLimits(resources *v1.ResourceRequirements) error { var ( isSmaller bool @@ -2107,9 +2124,12 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1.CronJob, error) { c.logger.Debug("Generating logical backup pod template") - // allocate for the backup pod the same amount of resources as for normal DB pods + // allocate configured resources for logical backup pod + logicalBackupResources := makeLogicalBackupResources(&c.OpConfig) + // if not defined only default resources from spilo pods are used resourceRequirements, err = c.generateResourceRequirements( - c.Spec.Resources, makeDefaultResources(&c.OpConfig), logicalBackupContainerName) + &logicalBackupResources, makeDefaultResources(&c.OpConfig), logicalBackupContainerName) + if err != nil { return nil, fmt.Errorf("could not generate resource requirements for logical backup pods: %v", err) } diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 175f6fd7f..226e5ced5 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -2638,7 +2638,6 @@ func TestGenerateResourceRequirements(t *testing.T) { clusterName := "acid-test-cluster" namespace := "default" clusterNameLabel := "cluster-name" - roleLabel := "spilo-role" sidecarName := "postgres-exporter" // enforceMinResourceLimits will be called 2 twice emitting 4 events (2x cpu, 2x memory raise) @@ -2657,7 +2656,7 @@ func TestGenerateResourceRequirements(t *testing.T) { DefaultMemoryLimit: "500Mi", MaxMemoryRequest: "1Gi", MinMemoryLimit: "250Mi", - PodRoleLabel: roleLabel, + PodRoleLabel: "spilo-role", } tests := []struct { @@ -3008,6 +3007,137 @@ func TestGenerateResourceRequirements(t *testing.T) { } } +func TestGenerateLogicalBackupJob(t *testing.T) { + clusterName := "acid-test-cluster" + configResources := config.Resources{ + DefaultCPURequest: "100m", + DefaultCPULimit: "1", + DefaultMemoryRequest: "100Mi", + DefaultMemoryLimit: "500Mi", + } + + tests := []struct { + subTest string + config config.Config + specSchedule string + expectedSchedule string + expectedJobName string + expectedResources acidv1.Resources + }{ + { + subTest: "test generation of logical backup pod resources when not configured", + config: config.Config{ + LogicalBackup: config.LogicalBackup{ + LogicalBackupJobPrefix: "logical-backup-", + LogicalBackupSchedule: "30 00 * * *", + }, + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + specSchedule: "", + expectedSchedule: "30 00 * * *", + expectedJobName: "logical-backup-acid-test-cluster", + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "100m", Memory: "100Mi"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "500Mi"}, + }, + }, + { + subTest: "test generation of logical backup pod resources when configured", + config: config.Config{ + LogicalBackup: config.LogicalBackup{ + LogicalBackupCPURequest: "10m", + LogicalBackupCPULimit: "300m", + LogicalBackupMemoryRequest: "50Mi", + LogicalBackupMemoryLimit: "300Mi", + LogicalBackupJobPrefix: "lb-", + LogicalBackupSchedule: "30 00 * * *", + }, + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + specSchedule: "30 00 * * 7", + expectedSchedule: "30 00 * * 7", + expectedJobName: "lb-acid-test-cluster", + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "10m", Memory: "50Mi"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "300m", Memory: "300Mi"}, + }, + }, + { + subTest: "test generation of logical backup pod resources when partly configured", + config: config.Config{ + LogicalBackup: config.LogicalBackup{ + LogicalBackupCPURequest: "50m", + LogicalBackupCPULimit: "250m", + LogicalBackupJobPrefix: "", + LogicalBackupSchedule: "30 00 * * *", + }, + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + specSchedule: "", + expectedSchedule: "30 00 * * *", + expectedJobName: "acid-test-cluster", + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "50m", Memory: "100Mi"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "250m", Memory: "500Mi"}, + }, + }, + { + subTest: "test generation of logical backup pod resources with SetMemoryRequestToLimit enabled", + config: config.Config{ + LogicalBackup: config.LogicalBackup{ + LogicalBackupMemoryRequest: "80Mi", + LogicalBackupMemoryLimit: "200Mi", + LogicalBackupJobPrefix: "test-long-prefix-so-name-must-be-trimmed-", + LogicalBackupSchedule: "30 00 * * *", + }, + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: true, + }, + specSchedule: "", + expectedSchedule: "30 00 * * *", + expectedJobName: "test-long-prefix-so-name-must-be-trimmed-acid-test-c", + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "100m", Memory: "200Mi"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "200Mi"}, + }, + }, + } + + for _, tt := range tests { + var cluster = New( + Config{ + OpConfig: tt.config, + }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger, eventRecorder) + + cluster.ObjectMeta.Name = clusterName + cluster.Spec.LogicalBackupSchedule = tt.specSchedule + cronJob, err := cluster.generateLogicalBackupJob() + assert.NoError(t, err) + + if cronJob.Spec.Schedule != tt.expectedSchedule { + t.Errorf("%s - %s: expected schedule %s, got %s", t.Name(), tt.subTest, tt.expectedSchedule, cronJob.Spec.Schedule) + } + + if cronJob.Name != tt.expectedJobName { + t.Errorf("%s - %s: expected job name %s, got %s", t.Name(), tt.subTest, tt.expectedJobName, cronJob.Name) + } + + containers := cronJob.Spec.JobTemplate.Spec.Template.Spec.Containers + clusterResources, err := parseResourceRequirements(containers[0].Resources) + assert.NoError(t, err) + if !reflect.DeepEqual(tt.expectedResources, clusterResources) { + t.Errorf("%s - %s: expected resources %#v, got %#v", t.Name(), tt.subTest, tt.expectedResources, clusterResources) + } + } +} + func TestGenerateCapabilities(t *testing.T) { tests := []struct { subTest string diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go index 1e59e0f86..00f18b7a2 100644 --- a/pkg/cluster/streams_test.go +++ b/pkg/cluster/streams_test.go @@ -2,7 +2,6 @@ package cluster import ( "fmt" - "reflect" "strings" "context" @@ -389,7 +388,7 @@ func TestUpdateFabricEventStream(t *testing.T) { assert.NoError(t, err) result := cluster.generateFabricEventStream(appId) - if !reflect.DeepEqual(result.Spec.EventStreams, streams.Items[0].Spec.EventStreams) { + if match, _ := sameStreams(streams.Items[0].Spec.EventStreams, result.Spec.EventStreams); !match { t.Errorf("Malformed FabricEventStream, expected %#v, got %#v", streams.Items[0], result) } } diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index fc4aba73e..00a7ea001 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -174,6 +174,9 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.LogicalBackupSchedule = util.Coalesce(fromCRD.LogicalBackup.Schedule, "30 00 * * *") result.LogicalBackupDockerImage = util.Coalesce(fromCRD.LogicalBackup.DockerImage, "registry.opensource.zalan.do/acid/logical-backup:v1.8.2") result.LogicalBackupProvider = util.Coalesce(fromCRD.LogicalBackup.BackupProvider, "s3") + result.LogicalBackupAzureStorageAccountName = fromCRD.LogicalBackup.AzureStorageAccountName + result.LogicalBackupAzureStorageAccountKey = fromCRD.LogicalBackup.AzureStorageAccountKey + result.LogicalBackupAzureStorageContainer = fromCRD.LogicalBackup.AzureStorageContainer result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket result.LogicalBackupS3Region = fromCRD.LogicalBackup.S3Region result.LogicalBackupS3Endpoint = fromCRD.LogicalBackup.S3Endpoint @@ -183,9 +186,10 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.LogicalBackupS3RetentionTime = fromCRD.LogicalBackup.RetentionTime result.LogicalBackupGoogleApplicationCredentials = fromCRD.LogicalBackup.GoogleApplicationCredentials result.LogicalBackupJobPrefix = util.Coalesce(fromCRD.LogicalBackup.JobPrefix, "logical-backup-") - result.LogicalBackupAzureStorageAccountName = fromCRD.LogicalBackup.AzureStorageAccountName - result.LogicalBackupAzureStorageAccountKey = fromCRD.LogicalBackup.AzureStorageAccountKey - result.LogicalBackupAzureStorageContainer = fromCRD.LogicalBackup.AzureStorageContainer + result.LogicalBackupCPURequest = fromCRD.LogicalBackup.CPURequest + result.LogicalBackupMemoryRequest = fromCRD.LogicalBackup.MemoryRequest + result.LogicalBackupCPULimit = fromCRD.LogicalBackup.CPULimit + result.LogicalBackupMemoryLimit = fromCRD.LogicalBackup.MemoryLimit // debug config result.DebugLogging = fromCRD.OperatorDebug.DebugLogging diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 30e557447..612b7a242 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -128,6 +128,9 @@ type LogicalBackup struct { LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"` LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup:v1.8.2"` LogicalBackupProvider string `name:"logical_backup_provider" default:"s3"` + LogicalBackupAzureStorageAccountName string `name:"logical_backup_azure_storage_account_name" default:""` + LogicalBackupAzureStorageContainer string `name:"logical_backup_azure_storage_container" default:""` + LogicalBackupAzureStorageAccountKey string `name:"logical_backup_azure_storage_account_key" default:""` LogicalBackupS3Bucket string `name:"logical_backup_s3_bucket" default:""` LogicalBackupS3Region string `name:"logical_backup_s3_region" default:""` LogicalBackupS3Endpoint string `name:"logical_backup_s3_endpoint" default:""` @@ -137,9 +140,10 @@ type LogicalBackup struct { LogicalBackupS3RetentionTime string `name:"logical_backup_s3_retention_time" default:""` LogicalBackupGoogleApplicationCredentials string `name:"logical_backup_google_application_credentials" default:""` LogicalBackupJobPrefix string `name:"logical_backup_job_prefix" default:"logical-backup-"` - LogicalBackupAzureStorageAccountName string `name:"logical_backup_azure_storage_account_name" default:""` - LogicalBackupAzureStorageContainer string `name:"logical_backup_azure_storage_container" default:""` - LogicalBackupAzureStorageAccountKey string `name:"logical_backup_azure_storage_account_key" default:""` + LogicalBackupCPURequest string `name:"logical_backup_cpu_request"` + LogicalBackupMemoryRequest string `name:"logical_backup_memory_request"` + LogicalBackupCPULimit string `name:"logical_backup_cpu_limit"` + LogicalBackupMemoryLimit string `name:"logical_backup_memory_limit"` } // Operator options for connection pooler