public_git/postgres-operator
1
0
Fork 0
mirror of https://github.com/zalando/postgres-operator.git synced 2026-10-06 00:41:34 +02:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into patroni-4-integration

Browse Source
This commit is contained in:
idanovinda
2025-01-03 13:51:08 +01:00
parent aa896edd36 6035fdd58e
commit efff3f8c56
70 changed files with 769 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/apis/acid.zalan.do/v1/crds.go
+5 -4
View File
@@ -595,9 +595,6 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
"version": {
Type: "string",
Enum: []apiextv1.JSON{
{
Raw: []byte(`"12"`),
},
{
Raw: []byte(`"13"`),
},
@@ -610,6 +607,9 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
{
Raw: []byte(`"16"`),
},
{
Raw: []byte(`"17"`),
},
},
},
"parameters": {
@@ -1164,7 +1164,8 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
Type: "boolean",
},
"enable_spilo_wal_path_compat": {
Type: "boolean",
Type: "boolean",
Description: "deprecated",
},
"enable_team_id_clustername_prefix": {
Type: "boolean",
pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
+2 -2
View File
@@ -49,8 +49,8 @@ type PostgresUsersConfiguration struct {
type MajorVersionUpgradeConfiguration struct {
MajorVersionUpgradeMode string `json:"major_version_upgrade_mode" default:"manual"` // off - no actions, manual - manifest triggers action, full - manifest and minimal version violation trigger upgrade
MajorVersionUpgradeTeamAllowList []string `json:"major_version_upgrade_team_allow_list,omitempty"`
MinimalMajorVersion string `json:"minimal_major_version" default:"12"`
TargetMajorVersion string `json:"target_major_version" default:"16"`
MinimalMajorVersion string `json:"minimal_major_version" default:"13"`
TargetMajorVersion string `json:"target_major_version" default:"17"`
}
// KubernetesMetaConfiguration defines k8s conf required for all Postgres clusters and the operator itself
pkg/apis/acid.zalan.do/v1/postgresql_type.go
+4
View File
@@ -220,6 +220,7 @@ type Sidecar struct {
DockerImage string `json:"image,omitempty"`
Ports []v1.ContainerPort `json:"ports,omitempty"`
Env []v1.EnvVar `json:"env,omitempty"`
Command []string `json:"command,omitempty"`
}
// UserFlags defines flags (such as superuser, nologin) that could be assigned to individual users
@@ -258,6 +259,8 @@ type Stream struct {
Tables map[string]StreamTable `json:"tables"`
Filter map[string]*string `json:"filter,omitempty"`
BatchSize *uint32 `json:"batchSize,omitempty"`
CPU *string `json:"cpu,omitempty"`
Memory *string `json:"memory,omitempty"`
EnableRecovery *bool `json:"enableRecovery,omitempty"`
}
@@ -265,6 +268,7 @@ type Stream struct {
type StreamTable struct {
EventType string `json:"eventType"`
RecoveryEventType string `json:"recoveryEventType,omitempty"`
IgnoreRecovery *bool `json:"ignoreRecovery,omitempty"`
IdColumn *string `json:"idColumn,omitempty"`
PayloadColumn *string `json:"payloadColumn,omitempty"`
}
pkg/apis/acid.zalan.do/v1/util_test.go
+5 -5
View File
@@ -219,7 +219,7 @@ var unmarshalCluster = []struct {
"127.0.0.1/32"
],
"postgresql": {
"version": "16",
"version": "17",
"parameters": {
"shared_buffers": "32MB",
"max_connections": "10",
@@ -279,7 +279,7 @@ var unmarshalCluster = []struct {
},
Spec: PostgresSpec{
PostgresqlParam: PostgresqlParam{
PgVersion: "16",
PgVersion: "17",
Parameters: map[string]string{
"shared_buffers": "32MB",
"max_connections": "10",
@@ -339,7 +339,7 @@ var unmarshalCluster = []struct {
},
Error: "",
},
marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"16","parameters":{"log_statement":"all","max_connections":"10","shared_buffers":"32MB"}},"pod_priority_class_name":"spilo-pod-priority","volume":{"size":"5Gi","storageClass":"SSD", "subPath": "subdir"},"enableShmVolume":false,"patroni":{"initdb":{"data-checksums":"true","encoding":"UTF8","locale":"en_US.UTF-8"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"],"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}},"resources":{"requests":{"cpu":"10m","memory":"50Mi"},"limits":{"cpu":"300m","memory":"3000Mi"}},"teamId":"acid","allowedSourceRanges":["127.0.0.1/32"],"numberOfInstances":2,"users":{"zalando":["superuser","createdb"]},"maintenanceWindows":["Mon:01:00-06:00","Sat:00:00-04:00","05:00-05:15"],"clone":{"cluster":"acid-batman"}},"status":{"PostgresClusterStatus":""}}`),
marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"17","parameters":{"log_statement":"all","max_connections":"10","shared_buffers":"32MB"}},"pod_priority_class_name":"spilo-pod-priority","volume":{"size":"5Gi","storageClass":"SSD", "subPath": "subdir"},"enableShmVolume":false,"patroni":{"initdb":{"data-checksums":"true","encoding":"UTF8","locale":"en_US.UTF-8"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"],"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}},"resources":{"requests":{"cpu":"10m","memory":"50Mi"},"limits":{"cpu":"300m","memory":"3000Mi"}},"teamId":"acid","allowedSourceRanges":["127.0.0.1/32"],"numberOfInstances":2,"users":{"zalando":["superuser","createdb"]},"maintenanceWindows":["Mon:01:00-06:00","Sat:00:00-04:00","05:00-05:15"],"clone":{"cluster":"acid-batman"}},"status":{"PostgresClusterStatus":""}}`),
err: nil},
{
about: "example with clone",
@@ -404,7 +404,7 @@ var postgresqlList = []struct {
out PostgresqlList
err error
}{
{"expect success", []byte(`{"apiVersion":"v1","items":[{"apiVersion":"acid.zalan.do/v1","kind":"Postgresql","metadata":{"labels":{"team":"acid"},"name":"acid-testcluster42","namespace":"default","resourceVersion":"30446957","selfLink":"/apis/acid.zalan.do/v1/namespaces/default/postgresqls/acid-testcluster42","uid":"857cd208-33dc-11e7-b20a-0699041e4b03"},"spec":{"allowedSourceRanges":["185.85.220.0/22"],"numberOfInstances":1,"postgresql":{"version":"16"},"teamId":"acid","volume":{"size":"10Gi"}},"status":{"PostgresClusterStatus":"Running"}}],"kind":"List","metadata":{},"resourceVersion":"","selfLink":""}`),
{"expect success", []byte(`{"apiVersion":"v1","items":[{"apiVersion":"acid.zalan.do/v1","kind":"Postgresql","metadata":{"labels":{"team":"acid"},"name":"acid-testcluster42","namespace":"default","resourceVersion":"30446957","selfLink":"/apis/acid.zalan.do/v1/namespaces/default/postgresqls/acid-testcluster42","uid":"857cd208-33dc-11e7-b20a-0699041e4b03"},"spec":{"allowedSourceRanges":["185.85.220.0/22"],"numberOfInstances":1,"postgresql":{"version":"17"},"teamId":"acid","volume":{"size":"10Gi"}},"status":{"PostgresClusterStatus":"Running"}}],"kind":"List","metadata":{},"resourceVersion":"","selfLink":""}`),
PostgresqlList{
TypeMeta: metav1.TypeMeta{
Kind: "List",
@@ -425,7 +425,7 @@ var postgresqlList = []struct {
},
Spec: PostgresSpec{
ClusterName: "testcluster42",
PostgresqlParam: PostgresqlParam{PgVersion: "16"},
PostgresqlParam: PostgresqlParam{PgVersion: "17"},
Volume: Volume{Size: "10Gi"},
TeamID: "acid",
AllowedSourceRanges: []string{"185.85.220.0/22"},
pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
+20
View File
@@ -1277,6 +1277,11 @@ func (in *Sidecar) DeepCopyInto(out *Sidecar) {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.Command != nil {
in, out := &in.Command, &out.Command
*out = make([]string, len(*in))
copy(*out, *in)
}
return
}
@@ -1336,6 +1341,16 @@ func (in *Stream) DeepCopyInto(out *Stream) {
*out = new(uint32)
**out = **in
}
if in.CPU != nil {
in, out := &in.CPU, &out.CPU
*out = new(string)
**out = **in
}
if in.Memory != nil {
in, out := &in.Memory, &out.Memory
*out = new(string)
**out = **in
}
if in.EnableRecovery != nil {
in, out := &in.EnableRecovery, &out.EnableRecovery
*out = new(bool)
@@ -1357,6 +1372,11 @@ func (in *Stream) DeepCopy() *Stream {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *StreamTable) DeepCopyInto(out *StreamTable) {
*out = *in
if in.IgnoreRecovery != nil {
in, out := &in.IgnoreRecovery, &out.IgnoreRecovery
*out = new(bool)
**out = **in
}
if in.IdColumn != nil {
in, out := &in.IdColumn, &out.IdColumn
*out = new(string)
pkg/cluster/k8sres.go
+1
View File
@@ -1230,6 +1230,7 @@ func getSidecarContainer(sidecar acidv1.Sidecar, index int, resources *v1.Resour
Resources: *resources,
Env: sidecar.Env,
Ports: sidecar.Ports,
Command: sidecar.Command,
}
}
pkg/cluster/k8sres_test.go
+15 -15
View File
@@ -72,18 +72,18 @@ func TestGenerateSpiloJSONConfiguration(t *testing.T) {
}{
{
subtest: "Patroni default configuration",
pgParam: &acidv1.PostgresqlParam{PgVersion: "16"},
pgParam: &acidv1.PostgresqlParam{PgVersion: "17"},
patroni: &acidv1.Patroni{},
opConfig: &config.Config{
Auth: config.Auth{
PamRoleName: "zalandos",
},
},
result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/16/bin"},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"}],"dcs":{}}}`,
result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/17/bin"},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"}],"dcs":{}}}`,
},
{
subtest: "Patroni configured",
pgParam: &acidv1.PostgresqlParam{PgVersion: "16"},
pgParam: &acidv1.PostgresqlParam{PgVersion: "17"},
patroni: &acidv1.Patroni{
InitDB: map[string]string{
"encoding": "UTF8",
@@ -102,38 +102,38 @@ func TestGenerateSpiloJSONConfiguration(t *testing.T) {
FailsafeMode: util.True(),
},
opConfig: &config.Config{},
result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/16/bin","pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"synchronous_mode":true,"synchronous_mode_strict":true,"synchronous_node_count":1,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}},"failsafe_mode":true}}}`,
result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/17/bin","pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"synchronous_mode":true,"synchronous_mode_strict":true,"synchronous_node_count":1,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}},"failsafe_mode":true}}}`,
},
{
subtest: "Patroni failsafe_mode configured globally",
pgParam: &acidv1.PostgresqlParam{PgVersion: "16"},
pgParam: &acidv1.PostgresqlParam{PgVersion: "17"},
patroni: &acidv1.Patroni{},
opConfig: &config.Config{
EnablePatroniFailsafeMode: util.True(),
},
result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/16/bin"},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"}],"dcs":{"failsafe_mode":true}}}`,
result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/17/bin"},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"}],"dcs":{"failsafe_mode":true}}}`,
},
{
subtest: "Patroni failsafe_mode configured globally, disabled for cluster",
pgParam: &acidv1.PostgresqlParam{PgVersion: "16"},
pgParam: &acidv1.PostgresqlParam{PgVersion: "17"},
patroni: &acidv1.Patroni{
FailsafeMode: util.False(),
},
opConfig: &config.Config{
EnablePatroniFailsafeMode: util.True(),
},
result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/16/bin"},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"}],"dcs":{"failsafe_mode":false}}}`,
result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/17/bin"},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"}],"dcs":{"failsafe_mode":false}}}`,
},
{
subtest: "Patroni failsafe_mode disabled globally, configured for cluster",
pgParam: &acidv1.PostgresqlParam{PgVersion: "16"},
pgParam: &acidv1.PostgresqlParam{PgVersion: "17"},
patroni: &acidv1.Patroni{
FailsafeMode: util.True(),
},
opConfig: &config.Config{
EnablePatroniFailsafeMode: util.False(),
},
result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/16/bin"},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"}],"dcs":{"failsafe_mode":true}}}`,
result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/17/bin"},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"}],"dcs":{"failsafe_mode":true}}}`,
},
}
for _, tt := range tests {
@@ -164,15 +164,15 @@ func TestExtractPgVersionFromBinPath(t *testing.T) {
},
{
subTest: "test current bin path against hard coded template",
binPath: "/usr/lib/postgresql/16/bin",
binPath: "/usr/lib/postgresql/17/bin",
template: pgBinariesLocationTemplate,
expected: "16",
expected: "17",
},
{
subTest: "test alternative bin path against a matching template",
binPath: "/usr/pgsql-16/bin",
binPath: "/usr/pgsql-17/bin",
template: "/usr/pgsql-%v/bin",
expected: "16",
expected: "17",
},
}
@@ -2148,7 +2148,7 @@ func TestSidecars(t *testing.T) {
spec = acidv1.PostgresSpec{
PostgresqlParam: acidv1.PostgresqlParam{
PgVersion: "16",
PgVersion: "17",
Parameters: map[string]string{
"max_connections": "100",
},
pkg/cluster/majorversionupgrade.go
+2 -1
View File
@@ -21,6 +21,7 @@ var VersionMap = map[string]int{
"14": 140000,
"15": 150000,
"16": 160000,
"17": 170000,
}
const (
@@ -44,7 +45,7 @@ func (c *Cluster) GetDesiredMajorVersionAsInt() int {
func (c *Cluster) GetDesiredMajorVersion() string {
if c.Config.OpConfig.MajorVersionUpgradeMode == "full" {
// e.g. current is 12, minimal is 12 allowing 12 to 16 clusters, everything below is upgraded
// e.g. current is 13, minimal is 13 allowing 13 to 17 clusters, everything below is upgraded
if IsBiggerPostgresVersion(c.Spec.PgVersion, c.Config.OpConfig.MinimalMajorVersion) {
c.logger.Infof("overwriting configured major version %s to %s", c.Spec.PgVersion, c.Config.OpConfig.TargetMajorVersion)
return c.Config.OpConfig.TargetMajorVersion
pkg/cluster/streams.go
+60 -15
View File
@@ -178,16 +178,25 @@ func (c *Cluster) syncPublication(dbName string, databaseSlotsList map[string]za
func (c *Cluster) generateFabricEventStream(appId string) *zalandov1.FabricEventStream {
eventStreams := make([]zalandov1.EventStream, 0)
resourceAnnotations := map[string]string{}
var err, err2 error
for _, stream := range c.Spec.Streams {
if stream.ApplicationId != appId {
continue
}
err = setResourceAnnotation(&resourceAnnotations, stream.CPU, constants.EventStreamCpuAnnotationKey)
err2 = setResourceAnnotation(&resourceAnnotations, stream.Memory, constants.EventStreamMemoryAnnotationKey)
if err != nil || err2 != nil {
c.logger.Warningf("could not set resource annotation for event stream: %v", err)
}
for tableName, table := range stream.Tables {
streamSource := c.getEventStreamSource(stream, tableName, table.IdColumn)
streamFlow := getEventStreamFlow(table.PayloadColumn)
streamSink := getEventStreamSink(stream, table.EventType)
streamRecovery := getEventStreamRecovery(stream, table.RecoveryEventType, table.EventType)
streamRecovery := getEventStreamRecovery(stream, table.RecoveryEventType, table.EventType, table.IgnoreRecovery)
eventStreams = append(eventStreams, zalandov1.EventStream{
EventStreamFlow: streamFlow,
@@ -207,7 +216,7 @@ func (c *Cluster) generateFabricEventStream(appId string) *zalandov1.FabricEvent
Name: fmt.Sprintf("%s-%s", c.Name, strings.ToLower(util.RandomPassword(5))),
Namespace: c.Namespace,
Labels: c.labelsSet(true),
Annotations: c.AnnotationsToPropagate(c.annotationsSet(nil)),
Annotations: c.AnnotationsToPropagate(c.annotationsSet(resourceAnnotations)),
OwnerReferences: c.ownerReferences(),
},
Spec: zalandov1.FabricEventStreamSpec{
@@ -217,6 +226,27 @@ func (c *Cluster) generateFabricEventStream(appId string) *zalandov1.FabricEvent
}
}
func setResourceAnnotation(annotations *map[string]string, resource *string, key string) error {
var (
isSmaller bool
err error
)
if resource != nil {
currentValue, exists := (*annotations)[key]
if exists {
isSmaller, err = util.IsSmallerQuantity(currentValue, *resource)
if err != nil {
return fmt.Errorf("could not compare resource in %q annotation: %v", key, err)
}
}
if isSmaller || !exists {
(*annotations)[key] = *resource
}
}
return nil
}
func (c *Cluster) getEventStreamSource(stream acidv1.Stream, tableName string, idColumn *string) zalandov1.EventStreamSource {
table, schema := getTableSchema(tableName)
streamFilter := stream.Filter[tableName]
@@ -247,7 +277,7 @@ func getEventStreamSink(stream acidv1.Stream, eventType string) zalandov1.EventS
}
}
func getEventStreamRecovery(stream acidv1.Stream, recoveryEventType, eventType string) zalandov1.EventStreamRecovery {
func getEventStreamRecovery(stream acidv1.Stream, recoveryEventType, eventType string, ignoreRecovery *bool) zalandov1.EventStreamRecovery {
if (stream.EnableRecovery != nil && !*stream.EnableRecovery) ||
(stream.EnableRecovery == nil && recoveryEventType == "") {
return zalandov1.EventStreamRecovery{
@@ -255,6 +285,12 @@ func getEventStreamRecovery(stream acidv1.Stream, recoveryEventType, eventType s
}
}
if ignoreRecovery != nil && *ignoreRecovery {
return zalandov1.EventStreamRecovery{
Type: constants.EventStreamRecoveryIgnoreType,
}
}
if stream.EnableRecovery != nil && *stream.EnableRecovery && recoveryEventType == "" {
recoveryEventType = fmt.Sprintf("%s-%s", eventType, constants.EventStreamRecoverySuffix)
}
@@ -442,7 +478,9 @@ func (c *Cluster) syncStream(appId string) error {
c.setProcessName("syncing stream with applicationId %s", appId)
c.logger.Debugf("syncing stream with applicationId %s", appId)
listOptions := metav1.ListOptions{LabelSelector: c.labelsSet(true).String()}
listOptions := metav1.ListOptions{
LabelSelector: c.labelsSet(false).String(),
}
streams, err = c.KubeClient.FabricEventStreams(c.Namespace).List(context.TODO(), listOptions)
if err != nil {
return fmt.Errorf("could not list of FabricEventStreams for applicationId %s: %v", appId, err)
@@ -453,15 +491,6 @@ func (c *Cluster) syncStream(appId string) error {
if stream.Spec.ApplicationId != appId {
continue
}
if streamExists {
c.logger.Warningf("more than one event stream with applicationId %s found, delete it", appId)
if err = c.KubeClient.FabricEventStreams(stream.ObjectMeta.Namespace).Delete(context.TODO(), stream.ObjectMeta.Name, metav1.DeleteOptions{}); err != nil {
c.logger.Errorf("could not delete event stream %q with applicationId %s: %v", stream.ObjectMeta.Name, appId, err)
} else {
c.logger.Infof("redundant event stream %q with applicationId %s has been successfully deleted", stream.ObjectMeta.Name, appId)
}
continue
}
streamExists = true
desiredStreams := c.generateFabricEventStream(appId)
if !reflect.DeepEqual(stream.ObjectMeta.OwnerReferences, desiredStreams.ObjectMeta.OwnerReferences) {
@@ -476,7 +505,8 @@ func (c *Cluster) syncStream(appId string) error {
}
if match, reason := c.compareStreams(&stream, desiredStreams); !match {
c.logger.Infof("updating event streams with applicationId %s: %s", appId, reason)
desiredStreams.ObjectMeta = stream.ObjectMeta
// make sure to keep the old name with randomly generated suffix
desiredStreams.ObjectMeta.Name = stream.ObjectMeta.Name
updatedStream, err := c.updateStreams(desiredStreams)
if err != nil {
return fmt.Errorf("failed updating event streams %s with applicationId %s: %v", stream.Name, appId, err)
@@ -484,6 +514,7 @@ func (c *Cluster) syncStream(appId string) error {
c.Streams[appId] = updatedStream
c.logger.Infof("event streams %q with applicationId %s have been successfully updated", updatedStream.Name, appId)
}
break
}
if !streamExists {
@@ -501,15 +532,29 @@ func (c *Cluster) syncStream(appId string) error {
func (c *Cluster) compareStreams(curEventStreams, newEventStreams *zalandov1.FabricEventStream) (match bool, reason string) {
reasons := make([]string, 0)
desiredAnnotations := make(map[string]string)
match = true
// stream operator can add extra annotations so incl. current annotations in desired annotations
desiredAnnotations := c.annotationsSet(curEventStreams.Annotations)
for curKey, curValue := range curEventStreams.Annotations {
if _, exists := desiredAnnotations[curKey]; !exists {
desiredAnnotations[curKey] = curValue
}
}
// add/or override annotations if cpu and memory values were changed
for newKey, newValue := range newEventStreams.Annotations {
desiredAnnotations[newKey] = newValue
}
if changed, reason := c.compareAnnotations(curEventStreams.ObjectMeta.Annotations, desiredAnnotations); changed {
match = false
reasons = append(reasons, fmt.Sprintf("new streams annotations do not match: %s", reason))
}
if !reflect.DeepEqual(curEventStreams.ObjectMeta.Labels, newEventStreams.ObjectMeta.Labels) {
match = false
reasons = append(reasons, "new streams labels do not match the current ones")
}
if changed, reason := sameEventStreams(curEventStreams.Spec.EventStreams, newEventStreams.Spec.EventStreams); !changed {
match = false
reasons = append(reasons, fmt.Sprintf("new streams EventStreams array does not match : %s", reason))
pkg/cluster/streams_test.go
+196 -59
View File
@@ -65,12 +65,17 @@ var (
EventType: "stream-type-b",
RecoveryEventType: "stream-type-b-dlq",
},
"data.foofoobar": {
EventType: "stream-type-c",
IgnoreRecovery: util.True(),
},
},
EnableRecovery: util.True(),
Filter: map[string]*string{
"data.bar": k8sutil.StringToPointer("[?(@.source.txId > 500 && @.source.lsn > 123456)]"),
},
BatchSize: k8sutil.UInt32ToPointer(uint32(100)),
CPU: k8sutil.StringToPointer("250m"),
},
},
TeamID: "acid",
@@ -88,9 +93,12 @@ var (
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-12345", clusterName),
Namespace: namespace,
Annotations: map[string]string{
constants.EventStreamCpuAnnotationKey: "250m",
},
Labels: map[string]string{
"application": "spilo",
"cluster-name": fmt.Sprintf("%s-2", clusterName),
"cluster-name": clusterName,
"team": "acid",
},
OwnerReferences: []metav1.OwnerReference{
@@ -180,6 +188,37 @@ var (
Type: constants.EventStreamSourcePGType,
},
},
{
EventStreamFlow: zalandov1.EventStreamFlow{
Type: constants.EventStreamFlowPgGenericType,
},
EventStreamRecovery: zalandov1.EventStreamRecovery{
Type: constants.EventStreamRecoveryIgnoreType,
},
EventStreamSink: zalandov1.EventStreamSink{
EventType: "stream-type-c",
MaxBatchSize: k8sutil.UInt32ToPointer(uint32(100)),
Type: constants.EventStreamSinkNakadiType,
},
EventStreamSource: zalandov1.EventStreamSource{
Connection: zalandov1.Connection{
DBAuth: zalandov1.DBAuth{
Name: fmt.Sprintf("fes-user.%s.credentials.postgresql.acid.zalan.do", clusterName),
PasswordKey: "password",
Type: constants.EventStreamSourceAuthType,
UserKey: "username",
},
Url: fmt.Sprintf("jdbc:postgresql://%s.%s/foo?user=%s&ssl=true&sslmode=require", clusterName, namespace, fesUser),
SlotName: slotName,
PluginType: constants.EventStreamSourcePluginType,
},
Schema: "data",
EventStreamTable: zalandov1.EventStreamTable{
Name: "foofoobar",
},
Type: constants.EventStreamSourcePGType,
},
},
},
},
}
@@ -449,7 +488,7 @@ func TestGenerateFabricEventStream(t *testing.T) {
}
listOptions := metav1.ListOptions{
LabelSelector: cluster.labelsSet(true).String(),
LabelSelector: cluster.labelsSet(false).String(),
}
streams, err := cluster.KubeClient.FabricEventStreams(namespace).List(context.TODO(), listOptions)
assert.NoError(t, err)
@@ -488,20 +527,20 @@ func newFabricEventStream(streams []zalandov1.EventStream, annotations map[strin
}
func TestSyncStreams(t *testing.T) {
pg.Name = fmt.Sprintf("%s-2", pg.Name)
newClusterName := fmt.Sprintf("%s-2", pg.Name)
pg.Name = newClusterName
var cluster = New(
Config{
OpConfig: config.Config{
PodManagementPolicy: "ordered_ready",
Resources: config.Resources{
ClusterLabels: map[string]string{"application": "spilo"},
ClusterNameLabel: "cluster-name",
DefaultCPURequest: "300m",
DefaultCPULimit: "300m",
DefaultMemoryRequest: "300Mi",
DefaultMemoryLimit: "300Mi",
EnableOwnerReferences: util.True(),
PodRoleLabel: "spilo-role",
ClusterLabels: map[string]string{"application": "spilo"},
ClusterNameLabel: "cluster-name",
DefaultCPURequest: "300m",
DefaultCPULimit: "300m",
DefaultMemoryRequest: "300Mi",
DefaultMemoryLimit: "300Mi",
PodRoleLabel: "spilo-role",
},
},
}, client, pg, logger, eventRecorder)
@@ -514,39 +553,23 @@ func TestSyncStreams(t *testing.T) {
err = cluster.syncStream(appId)
assert.NoError(t, err)
// create a second stream with same spec but with different name
createdStream, err := cluster.KubeClient.FabricEventStreams(namespace).Create(
context.TODO(), fes, metav1.CreateOptions{})
assert.NoError(t, err)
assert.Equal(t, createdStream.Spec.ApplicationId, appId)
// check that two streams exist
listOptions := metav1.ListOptions{
LabelSelector: cluster.labelsSet(true).String(),
}
streams, err := cluster.KubeClient.FabricEventStreams(namespace).List(context.TODO(), listOptions)
assert.NoError(t, err)
assert.Equalf(t, 2, len(streams.Items), "unexpected number of streams found: got %d, but expected only 2", len(streams.Items))
// sync the stream which should remove the redundant stream
// sync the stream again
err = cluster.syncStream(appId)
assert.NoError(t, err)
// check that only one stream remains after sync
streams, err = cluster.KubeClient.FabricEventStreams(namespace).List(context.TODO(), listOptions)
listOptions := metav1.ListOptions{
LabelSelector: cluster.labelsSet(false).String(),
}
streams, err := cluster.KubeClient.FabricEventStreams(namespace).List(context.TODO(), listOptions)
assert.NoError(t, err)
assert.Equalf(t, 1, len(streams.Items), "unexpected number of streams found: got %d, but expected only 1", len(streams.Items))
// check owner references
if !reflect.DeepEqual(streams.Items[0].OwnerReferences, cluster.ownerReferences()) {
t.Errorf("unexpected owner references, expected %#v, got %#v", cluster.ownerReferences(), streams.Items[0].OwnerReferences)
}
}
func TestSameStreams(t *testing.T) {
testName := "TestSameStreams"
annotationsA := map[string]string{"owned-by": "acid"}
annotationsB := map[string]string{"owned-by": "foo"}
annotationsA := map[string]string{constants.EventStreamMemoryAnnotationKey: "500Mi"}
annotationsB := map[string]string{constants.EventStreamMemoryAnnotationKey: "1Gi"}
stream1 := zalandov1.EventStream{
EventStreamFlow: zalandov1.EventStreamFlow{},
@@ -615,42 +638,49 @@ func TestSameStreams(t *testing.T) {
streamsA: newFabricEventStream([]zalandov1.EventStream{stream1}, nil),
streamsB: newFabricEventStream([]zalandov1.EventStream{stream1, stream2}, nil),
match: false,
reason: "number of defined streams is different",
reason: "new streams EventStreams array does not match : number of defined streams is different",
},
{
subTest: "different number of streams",
streamsA: newFabricEventStream([]zalandov1.EventStream{stream1}, nil),
streamsB: newFabricEventStream([]zalandov1.EventStream{stream1, stream2}, nil),
match: false,
reason: "number of defined streams is different",
reason: "new streams EventStreams array does not match : number of defined streams is different",
},
{
subTest: "event stream specs differ",
streamsA: newFabricEventStream([]zalandov1.EventStream{stream1, stream2}, nil),
streamsB: fes,
match: false,
reason: "number of defined streams is different",
reason: "new streams annotations do not match: Added \"fes.zalando.org/FES_CPU\" with value \"250m\"., new streams labels do not match the current ones, new streams EventStreams array does not match : number of defined streams is different",
},
{
subTest: "event stream recovery specs differ",
streamsA: newFabricEventStream([]zalandov1.EventStream{stream2}, nil),
streamsB: newFabricEventStream([]zalandov1.EventStream{stream3}, nil),
match: false,
reason: "event stream specs differ",
reason: "new streams EventStreams array does not match : event stream specs differ",
},
{
subTest: "event stream with new annotations",
streamsA: newFabricEventStream([]zalandov1.EventStream{stream2}, nil),
streamsB: newFabricEventStream([]zalandov1.EventStream{stream2}, annotationsA),
match: false,
reason: "new streams annotations do not match: Added \"fes.zalando.org/FES_MEMORY\" with value \"500Mi\".",
},
{
subTest: "event stream annotations differ",
streamsA: newFabricEventStream([]zalandov1.EventStream{stream2}, annotationsA),
streamsA: newFabricEventStream([]zalandov1.EventStream{stream3}, annotationsA),
streamsB: newFabricEventStream([]zalandov1.EventStream{stream3}, annotationsB),
match: false,
reason: "event stream specs differ",
reason: "new streams annotations do not match: \"fes.zalando.org/FES_MEMORY\" changed from \"500Mi\" to \"1Gi\".",
},
}
for _, tt := range tests {
streamsMatch, matchReason := cluster.compareStreams(tt.streamsA, tt.streamsB)
if streamsMatch != tt.match {
t.Errorf("%s %s: unexpected match result when comparing streams: got %s, epxected %s",
if streamsMatch != tt.match || matchReason != tt.reason {
t.Errorf("%s %s: unexpected match result when comparing streams: got %s, expected %s",
testName, tt.subTest, matchReason, tt.reason)
}
}
@@ -658,6 +688,105 @@ func TestSameStreams(t *testing.T) {
func TestUpdateStreams(t *testing.T) {
pg.Name = fmt.Sprintf("%s-3", pg.Name)
var cluster = New(
Config{
OpConfig: config.Config{
PodManagementPolicy: "ordered_ready",
Resources: config.Resources{
ClusterLabels: map[string]string{"application": "spilo"},
ClusterNameLabel: "cluster-name",
DefaultCPURequest: "300m",
DefaultCPULimit: "300m",
DefaultMemoryRequest: "300Mi",
DefaultMemoryLimit: "300Mi",
EnableOwnerReferences: util.True(),
PodRoleLabel: "spilo-role",
},
},
}, client, pg, logger, eventRecorder)
_, err := cluster.KubeClient.Postgresqls(namespace).Create(
context.TODO(), &pg, metav1.CreateOptions{})
assert.NoError(t, err)
// create stream with different owner reference
fes.ObjectMeta.Name = fmt.Sprintf("%s-12345", pg.Name)
fes.ObjectMeta.Labels["cluster-name"] = pg.Name
createdStream, err := cluster.KubeClient.FabricEventStreams(namespace).Create(
context.TODO(), fes, metav1.CreateOptions{})
assert.NoError(t, err)
assert.Equal(t, createdStream.Spec.ApplicationId, appId)
// sync the stream which should update the owner reference
err = cluster.syncStream(appId)
assert.NoError(t, err)
// check that only one stream exists after sync
listOptions := metav1.ListOptions{
LabelSelector: cluster.labelsSet(true).String(),
}
streams, err := cluster.KubeClient.FabricEventStreams(namespace).List(context.TODO(), listOptions)
assert.NoError(t, err)
assert.Equalf(t, 1, len(streams.Items), "unexpected number of streams found: got %d, but expected only 1", len(streams.Items))
// compare owner references
if !reflect.DeepEqual(streams.Items[0].OwnerReferences, cluster.ownerReferences()) {
t.Errorf("unexpected owner references, expected %#v, got %#v", cluster.ownerReferences(), streams.Items[0].OwnerReferences)
}
// change specs of streams and patch CRD
for i, stream := range pg.Spec.Streams {
if stream.ApplicationId == appId {
streamTable := stream.Tables["data.bar"]
streamTable.EventType = "stream-type-c"
stream.Tables["data.bar"] = streamTable
stream.BatchSize = k8sutil.UInt32ToPointer(uint32(250))
pg.Spec.Streams[i] = stream
}
}
// compare stream returned from API with expected stream
streams = patchPostgresqlStreams(t, cluster, &pg.Spec, listOptions)
result := cluster.generateFabricEventStream(appId)
if match, _ := cluster.compareStreams(&streams.Items[0], result); !match {
t.Errorf("Malformed FabricEventStream after updating manifest, expected %#v, got %#v", streams.Items[0], result)
}
// disable recovery
for idx, stream := range pg.Spec.Streams {
if stream.ApplicationId == appId {
stream.EnableRecovery = util.False()
pg.Spec.Streams[idx] = stream
}
}
streams = patchPostgresqlStreams(t, cluster, &pg.Spec, listOptions)
result = cluster.generateFabricEventStream(appId)
if match, _ := cluster.compareStreams(&streams.Items[0], result); !match {
t.Errorf("Malformed FabricEventStream after disabling event recovery, expected %#v, got %#v", streams.Items[0], result)
}
}
func patchPostgresqlStreams(t *testing.T, cluster *Cluster, pgSpec *acidv1.PostgresSpec, listOptions metav1.ListOptions) (streams *zalandov1.FabricEventStreamList) {
patchData, err := specPatch(pgSpec)
assert.NoError(t, err)
pgPatched, err := cluster.KubeClient.Postgresqls(namespace).Patch(
context.TODO(), cluster.Name, types.MergePatchType, patchData, metav1.PatchOptions{}, "spec")
assert.NoError(t, err)
cluster.Postgresql.Spec = pgPatched.Spec
err = cluster.syncStream(appId)
assert.NoError(t, err)
streams, err = cluster.KubeClient.FabricEventStreams(namespace).List(context.TODO(), listOptions)
assert.NoError(t, err)
return streams
}
func TestDeleteStreams(t *testing.T) {
pg.Name = fmt.Sprintf("%s-4", pg.Name)
var cluster = New(
Config{
OpConfig: config.Config{
@@ -695,7 +824,7 @@ func TestUpdateStreams(t *testing.T) {
// compare stream returned from API with expected stream
listOptions := metav1.ListOptions{
LabelSelector: cluster.labelsSet(true).String(),
LabelSelector: cluster.labelsSet(false).String(),
}
streams := patchPostgresqlStreams(t, cluster, &pg.Spec, listOptions)
result := cluster.generateFabricEventStream(appId)
@@ -703,6 +832,14 @@ func TestUpdateStreams(t *testing.T) {
t.Errorf("Malformed FabricEventStream after updating manifest, expected %#v, got %#v", streams.Items[0], result)
}
// change teamId and check that stream is updated
pg.Spec.TeamID = "new-team"
streams = patchPostgresqlStreams(t, cluster, &pg.Spec, listOptions)
result = cluster.generateFabricEventStream(appId)
if match, _ := cluster.compareStreams(&streams.Items[0], result); !match {
t.Errorf("Malformed FabricEventStream after updating teamId, expected %#v, got %#v", streams.Items[0].ObjectMeta.Labels, result.ObjectMeta.Labels)
}
// disable recovery
for idx, stream := range pg.Spec.Streams {
if stream.ApplicationId == appId {
@@ -717,9 +854,6 @@ func TestUpdateStreams(t *testing.T) {
t.Errorf("Malformed FabricEventStream after disabling event recovery, expected %#v, got %#v", streams.Items[0], result)
}
mockClient := k8sutil.NewMockKubernetesClient()
cluster.KubeClient.CustomResourceDefinitionsGetter = mockClient.CustomResourceDefinitionsGetter
// remove streams from manifest
pg.Spec.Streams = nil
pgUpdated, err := cluster.KubeClient.Postgresqls(namespace).Update(
@@ -729,26 +863,29 @@ func TestUpdateStreams(t *testing.T) {
appIds := getDistinctApplicationIds(pgUpdated.Spec.Streams)
cluster.cleanupRemovedStreams(appIds)
// check that streams have been deleted
streams, err = cluster.KubeClient.FabricEventStreams(namespace).List(context.TODO(), listOptions)
if len(streams.Items) > 0 || err != nil {
t.Errorf("stream resource has not been removed or unexpected error %v", err)
}
}
assert.NoError(t, err)
assert.Equalf(t, 0, len(streams.Items), "unexpected number of streams found: got %d, but expected none", len(streams.Items))
func patchPostgresqlStreams(t *testing.T, cluster *Cluster, pgSpec *acidv1.PostgresSpec, listOptions metav1.ListOptions) (streams *zalandov1.FabricEventStreamList) {
patchData, err := specPatch(pgSpec)
// create stream to test deleteStreams code
fes.ObjectMeta.Name = fmt.Sprintf("%s-12345", pg.Name)
fes.ObjectMeta.Labels["cluster-name"] = pg.Name
_, err = cluster.KubeClient.FabricEventStreams(namespace).Create(
context.TODO(), fes, metav1.CreateOptions{})
assert.NoError(t, err)
pgPatched, err := cluster.KubeClient.Postgresqls(namespace).Patch(
context.TODO(), cluster.Name, types.MergePatchType, patchData, metav1.PatchOptions{}, "spec")
assert.NoError(t, err)
cluster.Postgresql.Spec = pgPatched.Spec
// sync it once to cluster struct
err = cluster.syncStream(appId)
assert.NoError(t, err)
// we need a mock client because deleteStreams checks for CRD existance
mockClient := k8sutil.NewMockKubernetesClient()
cluster.KubeClient.CustomResourceDefinitionsGetter = mockClient.CustomResourceDefinitionsGetter
cluster.deleteStreams()
// check that streams have been deleted
streams, err = cluster.KubeClient.FabricEventStreams(namespace).List(context.TODO(), listOptions)
assert.NoError(t, err)
return streams
assert.Equalf(t, 0, len(streams.Items), "unexpected number of streams found: got %d, but expected none", len(streams.Items))
}
pkg/cluster/volumes.go
+1 -1
View File
@@ -151,7 +151,7 @@ func (c *Cluster) populateVolumeMetaData() error {
volumeIds := []string{}
var volumeID string
for _, pv := range pvs {
volumeID, err = c.VolumeResizer.ExtractVolumeID(pv.Spec.AWSElasticBlockStore.VolumeID)
volumeID, err = c.VolumeResizer.GetProviderVolumeID(pv)
if err != nil {
continue
}
pkg/cluster/volumes_test.go
+24
View File
@@ -216,6 +216,12 @@ func TestMigrateEBS(t *testing.T) {
resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-1")).Return("ebs-volume-1", nil)
resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-2")).Return("ebs-volume-2", nil)
resizer.EXPECT().GetProviderVolumeID(gomock.Any()).
DoAndReturn(func(pv *v1.PersistentVolume) (string, error) {
return resizer.ExtractVolumeID(pv.Spec.AWSElasticBlockStore.VolumeID)
}).
Times(2)
resizer.EXPECT().DescribeVolumes(gomock.Eq([]string{"ebs-volume-1", "ebs-volume-2"})).Return(
[]volumes.VolumeProperties{
{VolumeID: "ebs-volume-1", VolumeType: "gp2", Size: 100},
@@ -322,6 +328,12 @@ func TestMigrateGp3Support(t *testing.T) {
resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-2")).Return("ebs-volume-2", nil)
resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-3")).Return("ebs-volume-3", nil)
resizer.EXPECT().GetProviderVolumeID(gomock.Any()).
DoAndReturn(func(pv *v1.PersistentVolume) (string, error) {
return resizer.ExtractVolumeID(pv.Spec.AWSElasticBlockStore.VolumeID)
}).
Times(3)
resizer.EXPECT().DescribeVolumes(gomock.Eq([]string{"ebs-volume-1", "ebs-volume-2", "ebs-volume-3"})).Return(
[]volumes.VolumeProperties{
{VolumeID: "ebs-volume-1", VolumeType: "gp3", Size: 100, Iops: 3000},
@@ -377,6 +389,12 @@ func TestManualGp2Gp3Support(t *testing.T) {
resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-1")).Return("ebs-volume-1", nil)
resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-2")).Return("ebs-volume-2", nil)
resizer.EXPECT().GetProviderVolumeID(gomock.Any()).
DoAndReturn(func(pv *v1.PersistentVolume) (string, error) {
return resizer.ExtractVolumeID(pv.Spec.AWSElasticBlockStore.VolumeID)
}).
Times(2)
resizer.EXPECT().DescribeVolumes(gomock.Eq([]string{"ebs-volume-1", "ebs-volume-2"})).Return(
[]volumes.VolumeProperties{
{VolumeID: "ebs-volume-1", VolumeType: "gp2", Size: 150, Iops: 3000},
@@ -436,6 +454,12 @@ func TestDontTouchType(t *testing.T) {
resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-1")).Return("ebs-volume-1", nil)
resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-2")).Return("ebs-volume-2", nil)
resizer.EXPECT().GetProviderVolumeID(gomock.Any()).
DoAndReturn(func(pv *v1.PersistentVolume) (string, error) {
return resizer.ExtractVolumeID(pv.Spec.AWSElasticBlockStore.VolumeID)
}).
Times(2)
resizer.EXPECT().DescribeVolumes(gomock.Eq([]string{"ebs-volume-1", "ebs-volume-2"})).Return(
[]volumes.VolumeProperties{
{VolumeID: "ebs-volume-1", VolumeType: "gp2", Size: 150, Iops: 3000},
pkg/controller/operator_config.go
+4 -4
View File
@@ -39,7 +39,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
result.EnableTeamIdClusternamePrefix = fromCRD.EnableTeamIdClusternamePrefix
result.EtcdHost = fromCRD.EtcdHost
result.KubernetesUseConfigMaps = fromCRD.KubernetesUseConfigMaps
result.DockerImage = util.Coalesce(fromCRD.DockerImage, "ghcr.io/zalando/spilo-16:3.3-p1")
result.DockerImage = util.Coalesce(fromCRD.DockerImage, "ghcr.io/zalando/spilo-17:4.0-p2")
result.Workers = util.CoalesceUInt32(fromCRD.Workers, 8)
result.MinInstances = fromCRD.MinInstances
result.MaxInstances = fromCRD.MaxInstances
@@ -62,8 +62,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
// major version upgrade config
result.MajorVersionUpgradeMode = util.Coalesce(fromCRD.MajorVersionUpgrade.MajorVersionUpgradeMode, "manual")
result.MajorVersionUpgradeTeamAllowList = fromCRD.MajorVersionUpgrade.MajorVersionUpgradeTeamAllowList
result.MinimalMajorVersion = util.Coalesce(fromCRD.MajorVersionUpgrade.MinimalMajorVersion, "12")
result.TargetMajorVersion = util.Coalesce(fromCRD.MajorVersionUpgrade.TargetMajorVersion, "16")
result.MinimalMajorVersion = util.Coalesce(fromCRD.MajorVersionUpgrade.MinimalMajorVersion, "13")
result.TargetMajorVersion = util.Coalesce(fromCRD.MajorVersionUpgrade.TargetMajorVersion, "17")
// kubernetes config
result.EnableOwnerReferences = util.CoalesceBool(fromCRD.Kubernetes.EnableOwnerReferences, util.False())
@@ -181,7 +181,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
// logical backup config
result.LogicalBackupSchedule = util.Coalesce(fromCRD.LogicalBackup.Schedule, "30 00 * * *")
result.LogicalBackupDockerImage = util.Coalesce(fromCRD.LogicalBackup.DockerImage, "ghcr.io/zalando/postgres-operator/logical-backup:v1.13.0")
result.LogicalBackupDockerImage = util.Coalesce(fromCRD.LogicalBackup.DockerImage, "ghcr.io/zalando/postgres-operator/logical-backup:v1.14.0")
result.LogicalBackupProvider = util.Coalesce(fromCRD.LogicalBackup.BackupProvider, "s3")
result.LogicalBackupAzureStorageAccountName = fromCRD.LogicalBackup.AzureStorageAccountName
result.LogicalBackupAzureStorageAccountKey = fromCRD.LogicalBackup.AzureStorageAccountKey
pkg/spec/types.go
+3
View File
@@ -122,6 +122,9 @@ type ControllerConfig struct {
IgnoredAnnotations []string
EnableJsonLogging bool
KubeQPS int
KubeBurst int
}
// cached value for the GetOperatorNamespace
pkg/util/config/config.go
+4 -4
View File
@@ -128,7 +128,7 @@ type Scalyr struct {
// LogicalBackup defines configuration for logical backup
type LogicalBackup struct {
LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"`
LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"ghcr.io/zalando/postgres-operator/logical-backup:v1.13.0"`
LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"ghcr.io/zalando/postgres-operator/logical-backup:v1.14.0"`
LogicalBackupProvider string `name:"logical_backup_provider" default:"s3"`
LogicalBackupAzureStorageAccountName string `name:"logical_backup_azure_storage_account_name" default:""`
LogicalBackupAzureStorageContainer string `name:"logical_backup_azure_storage_container" default:""`
@@ -176,7 +176,7 @@ type Config struct {
WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to'
KubernetesUseConfigMaps bool `name:"kubernetes_use_configmaps" default:"false"`
EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS
DockerImage string `name:"docker_image" default:"ghcr.io/zalando/spilo-16:3.3-p1"`
DockerImage string `name:"docker_image" default:"ghcr.io/zalando/spilo-17:4.0-p2"`
SidecarImages map[string]string `name:"sidecar_docker_images"` // deprecated in favour of SidecarContainers
SidecarContainers []v1.Container `name:"sidecars"`
PodServiceAccountName string `name:"pod_service_account_name" default:"postgres-pod"`
@@ -247,8 +247,8 @@ type Config struct {
EnableTeamIdClusternamePrefix bool `name:"enable_team_id_clustername_prefix" default:"false"`
MajorVersionUpgradeMode string `name:"major_version_upgrade_mode" default:"manual"`
MajorVersionUpgradeTeamAllowList []string `name:"major_version_upgrade_team_allow_list" default:""`
MinimalMajorVersion string `name:"minimal_major_version" default:"12"`
TargetMajorVersion string `name:"target_major_version" default:"16"`
MinimalMajorVersion string `name:"minimal_major_version" default:"13"`
TargetMajorVersion string `name:"target_major_version" default:"17"`
PatroniAPICheckInterval time.Duration `name:"patroni_api_check_interval" default:"1s"`
PatroniAPICheckTimeout time.Duration `name:"patroni_api_check_timeout" default:"5s"`
EnablePatroniFailsafeMode *bool `name:"enable_patroni_failsafe_mode" default:"false"`
pkg/util/constants/aws.go
+1
View File
@@ -7,6 +7,7 @@ const (
// EBS related constants
EBSVolumeIDStart = "/vol-"
EBSProvisioner = "kubernetes.io/aws-ebs"
EBSDriver = "ebs.csi.aws.com"
//https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_VolumeModification.html
EBSVolumeStateModifying = "modifying"
EBSVolumeStateOptimizing = "optimizing"
pkg/util/constants/streams.go
+15 -12
View File
@@ -2,16 +2,19 @@ package constants
// PostgreSQL specific constants
const (
EventStreamCRDApiVersion = "zalando.org/v1"
EventStreamCRDKind = "FabricEventStream"
EventStreamCRDName = "fabriceventstreams.zalando.org"
EventStreamSourcePGType = "PostgresLogicalReplication"
EventStreamSourceSlotPrefix = "fes"
EventStreamSourcePluginType = "pgoutput"
EventStreamSourceAuthType = "DatabaseAuthenticationSecret"
EventStreamFlowPgGenericType = "PostgresWalToGenericNakadiEvent"
EventStreamSinkNakadiType = "Nakadi"
EventStreamRecoveryNoneType = "None"
EventStreamRecoveryDLQType = "DeadLetter"
EventStreamRecoverySuffix = "dead-letter-queue"
EventStreamCRDApiVersion = "zalando.org/v1"
EventStreamCRDKind = "FabricEventStream"
EventStreamCRDName = "fabriceventstreams.zalando.org"
EventStreamSourcePGType = "PostgresLogicalReplication"
EventStreamSourceSlotPrefix = "fes"
EventStreamSourcePluginType = "pgoutput"
EventStreamSourceAuthType = "DatabaseAuthenticationSecret"
EventStreamFlowPgGenericType = "PostgresWalToGenericNakadiEvent"
EventStreamSinkNakadiType = "Nakadi"
EventStreamRecoveryDLQType = "DeadLetter"
EventStreamRecoveryIgnoreType = "Ignore"
EventStreamRecoveryNoneType = "None"
EventStreamRecoverySuffix = "dead-letter-queue"
EventStreamCpuAnnotationKey = "fes.zalando.org/FES_CPU"
EventStreamMemoryAnnotationKey = "fes.zalando.org/FES_MEMORY"
)
pkg/util/volumes/ebs.go
+8 -2
View File
@@ -36,7 +36,8 @@ func (r *EBSVolumeResizer) IsConnectedToProvider() bool {
// VolumeBelongsToProvider checks if the given persistent volume is backed by EBS.
func (r *EBSVolumeResizer) VolumeBelongsToProvider(pv *v1.PersistentVolume) bool {
return pv.Spec.AWSElasticBlockStore != nil && pv.Annotations[constants.VolumeStorateProvisionerAnnotation] == constants.EBSProvisioner
return (pv.Spec.AWSElasticBlockStore != nil && pv.Annotations[constants.VolumeStorateProvisionerAnnotation] == constants.EBSProvisioner) ||
(pv.Spec.CSI != nil && pv.Spec.CSI.Driver == constants.EBSDriver)
}
// ExtractVolumeID extracts volumeID from "aws://eu-central-1a/vol-075ddfc4a127d0bd4"
@@ -54,7 +55,12 @@ func (r *EBSVolumeResizer) ExtractVolumeID(volumeID string) (string, error) {
// GetProviderVolumeID converts aws://eu-central-1b/vol-00f93d4827217c629 to vol-00f93d4827217c629 for EBS volumes
func (r *EBSVolumeResizer) GetProviderVolumeID(pv *v1.PersistentVolume) (string, error) {
volumeID := pv.Spec.AWSElasticBlockStore.VolumeID
var volumeID string = ""
if pv.Spec.CSI != nil {
volumeID = pv.Spec.CSI.VolumeHandle
} else if pv.Spec.AWSElasticBlockStore != nil {
volumeID = pv.Spec.AWSElasticBlockStore.VolumeID
}
if volumeID == "" {
return "", fmt.Errorf("got empty volume id for volume %v", pv)
}
pkg/util/volumes/ebs_test.go
+123
View File
@@ -0,0 +1,123 @@
package volumes
import (
"fmt"
"testing"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestGetProviderVolumeID(t *testing.T) {
tests := []struct {
name string
pv *v1.PersistentVolume
expected string
err error
}{
{
name: "CSI volume handle",
pv: &v1.PersistentVolume{
Spec: v1.PersistentVolumeSpec{
PersistentVolumeSource: v1.PersistentVolumeSource{
CSI: &v1.CSIPersistentVolumeSource{
VolumeHandle: "vol-075ddfc4a127d0bd5",
},
},
},
},
expected: "vol-075ddfc4a127d0bd5",
err: nil,
},
{
name: "AWS EBS volume handle",
pv: &v1.PersistentVolume{
Spec: v1.PersistentVolumeSpec{
PersistentVolumeSource: v1.PersistentVolumeSource{
AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{
VolumeID: "aws://eu-central-1a/vol-075ddfc4a127d0bd4",
},
},
},
},
expected: "vol-075ddfc4a127d0bd4",
err: nil,
},
{
name: "Empty volume handle",
pv: &v1.PersistentVolume{
Spec: v1.PersistentVolumeSpec{},
},
expected: "",
err: fmt.Errorf("got empty volume id for volume %v", &v1.PersistentVolume{}),
},
}
resizer := EBSVolumeResizer{}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
volumeID, err := resizer.GetProviderVolumeID(tt.pv)
if volumeID != tt.expected || (err != nil && err.Error() != tt.err.Error()) {
t.Errorf("expected %v, got %v, expected err %v, got %v", tt.expected, volumeID, tt.err, err)
}
})
}
}
func TestVolumeBelongsToProvider(t *testing.T) {
tests := []struct {
name string
pv *v1.PersistentVolume
expected bool
}{
{
name: "CSI volume handle",
pv: &v1.PersistentVolume{
Spec: v1.PersistentVolumeSpec{
PersistentVolumeSource: v1.PersistentVolumeSource{
CSI: &v1.CSIPersistentVolumeSource{
Driver: "ebs.csi.aws.com",
VolumeHandle: "vol-075ddfc4a127d0bd5",
},
},
},
},
expected: true,
},
{
name: "AWS EBS volume handle",
pv: &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string {
"pv.kubernetes.io/provisioned-by": "kubernetes.io/aws-ebs",
},
},
Spec: v1.PersistentVolumeSpec{
PersistentVolumeSource: v1.PersistentVolumeSource{
AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{
VolumeID: "aws://eu-central-1a/vol-075ddfc4a127d0bd4",
},
},
},
},
expected: true,
},
{
name: "Empty volume source",
pv: &v1.PersistentVolume{
Spec: v1.PersistentVolumeSpec{},
},
expected: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
resizer := EBSVolumeResizer{}
isProvider := resizer.VolumeBelongsToProvider(tt.pv)
if isProvider != tt.expected {
t.Errorf("expected %v, got %v", tt.expected, isProvider)
}
})
}
}