public_git/postgres-operator
1
0
Fork 0
mirror of https://github.com/zalando/postgres-operator.git synced 2026-09-30 21:30:28 +02:00
Code Issues Projects Releases Wiki Activity

add resource annotation and ignore recovery type (#2817)

Browse Source 
* add resource annotation and ignore recovery type
* Update docs/reference/cluster_manifest.md

---------

Co-authored-by: Ida Novindasari <idanovinda@gmail.com>
This commit is contained in:
Felix Kunde
2024-12-16 18:17:19 +01:00
committed by GitHub
co-authored by Ida Novindasari
parent 301462c415
commit 80ef38f7f0
8 changed files with 150 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/apis/acid.zalan.do/v1/postgresql_type.go
+3
View File
@@ -258,6 +258,8 @@ type Stream struct {
Tables map[string]StreamTable `json:"tables"`
Filter map[string]*string `json:"filter,omitempty"`
BatchSize *uint32 `json:"batchSize,omitempty"`
CPU *string `json:"cpu,omitempty"`
Memory *string `json:"memory,omitempty"`
EnableRecovery *bool `json:"enableRecovery,omitempty"`
}
@@ -265,6 +267,7 @@ type Stream struct {
type StreamTable struct {
EventType string `json:"eventType"`
RecoveryEventType string `json:"recoveryEventType,omitempty"`
IgnoreRecovery *bool `json:"ignoreRecovery,omitempty"`
IdColumn *string `json:"idColumn,omitempty"`
PayloadColumn *string `json:"payloadColumn,omitempty"`
}
pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
+15
View File
@@ -1336,6 +1336,16 @@ func (in *Stream) DeepCopyInto(out *Stream) {
*out = new(uint32)
**out = **in
}
if in.CPU != nil {
in, out := &in.CPU, &out.CPU
*out = new(string)
**out = **in
}
if in.Memory != nil {
in, out := &in.Memory, &out.Memory
*out = new(string)
**out = **in
}
if in.EnableRecovery != nil {
in, out := &in.EnableRecovery, &out.EnableRecovery
*out = new(bool)
@@ -1357,6 +1367,11 @@ func (in *Stream) DeepCopy() *Stream {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *StreamTable) DeepCopyInto(out *StreamTable) {
*out = *in
if in.IgnoreRecovery != nil {
in, out := &in.IgnoreRecovery, &out.IgnoreRecovery
*out = new(bool)
**out = **in
}
if in.IdColumn != nil {
in, out := &in.IdColumn, &out.IdColumn
*out = new(string)
pkg/cluster/streams.go
+28 -3
View File
@@ -178,16 +178,35 @@ func (c *Cluster) syncPublication(dbName string, databaseSlotsList map[string]za
func (c *Cluster) generateFabricEventStream(appId string) *zalandov1.FabricEventStream {
eventStreams := make([]zalandov1.EventStream, 0)
resourceAnnotations := map[string]string{}
for _, stream := range c.Spec.Streams {
if stream.ApplicationId != appId {
continue
}
if stream.CPU != nil {
cpu, exists := resourceAnnotations[constants.EventStreamCpuAnnotationKey]
if exists {
isSmaller, _ := util.IsSmallerQuantity(cpu, *stream.CPU)
if isSmaller {
resourceAnnotations[constants.EventStreamCpuAnnotationKey] = *stream.CPU
}
}
}
if stream.Memory != nil {
memory, exists := resourceAnnotations[constants.EventStreamMemoryAnnotationKey]
if exists {
isSmaller, _ := util.IsSmallerQuantity(memory, *stream.Memory)
if isSmaller {
resourceAnnotations[constants.EventStreamMemoryAnnotationKey] = *stream.Memory
}
}
}
for tableName, table := range stream.Tables {
streamSource := c.getEventStreamSource(stream, tableName, table.IdColumn)
streamFlow := getEventStreamFlow(table.PayloadColumn)
streamSink := getEventStreamSink(stream, table.EventType)
streamRecovery := getEventStreamRecovery(stream, table.RecoveryEventType, table.EventType)
streamRecovery := getEventStreamRecovery(stream, table.RecoveryEventType, table.EventType, table.IgnoreRecovery)
eventStreams = append(eventStreams, zalandov1.EventStream{
EventStreamFlow: streamFlow,
@@ -207,7 +226,7 @@ func (c *Cluster) generateFabricEventStream(appId string) *zalandov1.FabricEvent
Name: fmt.Sprintf("%s-%s", c.Name, strings.ToLower(util.RandomPassword(5))),
Namespace: c.Namespace,
Labels: c.labelsSet(true),
Annotations: c.AnnotationsToPropagate(c.annotationsSet(nil)),
Annotations: c.AnnotationsToPropagate(c.annotationsSet(resourceAnnotations)),
OwnerReferences: c.ownerReferences(),
},
Spec: zalandov1.FabricEventStreamSpec{
@@ -247,7 +266,7 @@ func getEventStreamSink(stream acidv1.Stream, eventType string) zalandov1.EventS
}
}
func getEventStreamRecovery(stream acidv1.Stream, recoveryEventType, eventType string) zalandov1.EventStreamRecovery {
func getEventStreamRecovery(stream acidv1.Stream, recoveryEventType, eventType string, ignoreRecovery *bool) zalandov1.EventStreamRecovery {
if (stream.EnableRecovery != nil && !*stream.EnableRecovery) ||
(stream.EnableRecovery == nil && recoveryEventType == "") {
return zalandov1.EventStreamRecovery{
@@ -255,6 +274,12 @@ func getEventStreamRecovery(stream acidv1.Stream, recoveryEventType, eventType s
}
}
if ignoreRecovery != nil && *ignoreRecovery {
return zalandov1.EventStreamRecovery{
Type: constants.EventStreamRecoveryIgnoreType,
}
}
if stream.EnableRecovery != nil && *stream.EnableRecovery && recoveryEventType == "" {
recoveryEventType = fmt.Sprintf("%s-%s", eventType, constants.EventStreamRecoverySuffix)
}
pkg/cluster/streams_test.go
+50 -2
View File
@@ -65,12 +65,18 @@ var (
EventType: "stream-type-b",
RecoveryEventType: "stream-type-b-dlq",
},
"data.foofoobar": {
EventType: "stream-type-c",
IgnoreRecovery: util.True(),
},
},
EnableRecovery: util.True(),
Filter: map[string]*string{
"data.bar": k8sutil.StringToPointer("[?(@.source.txId > 500 && @.source.lsn > 123456)]"),
},
BatchSize: k8sutil.UInt32ToPointer(uint32(100)),
CPU: k8sutil.StringToPointer("250m"),
Memory: k8sutil.StringToPointer("500Mi"),
},
},
TeamID: "acid",
@@ -88,6 +94,10 @@ var (
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-12345", clusterName),
Namespace: namespace,
Annotations: map[string]string{
constants.EventStreamCpuAnnotationKey: "250m",
constants.EventStreamMemoryAnnotationKey: "500Mi",
},
Labels: map[string]string{
"application": "spilo",
"cluster-name": clusterName,
@@ -180,6 +190,37 @@ var (
Type: constants.EventStreamSourcePGType,
},
},
{
EventStreamFlow: zalandov1.EventStreamFlow{
Type: constants.EventStreamFlowPgGenericType,
},
EventStreamRecovery: zalandov1.EventStreamRecovery{
Type: constants.EventStreamRecoveryIgnoreType,
},
EventStreamSink: zalandov1.EventStreamSink{
EventType: "stream-type-c",
MaxBatchSize: k8sutil.UInt32ToPointer(uint32(100)),
Type: constants.EventStreamSinkNakadiType,
},
EventStreamSource: zalandov1.EventStreamSource{
Connection: zalandov1.Connection{
DBAuth: zalandov1.DBAuth{
Name: fmt.Sprintf("fes-user.%s.credentials.postgresql.acid.zalan.do", clusterName),
PasswordKey: "password",
Type: constants.EventStreamSourceAuthType,
UserKey: "username",
},
Url: fmt.Sprintf("jdbc:postgresql://%s.%s/foo?user=%s&ssl=true&sslmode=require", clusterName, namespace, fesUser),
SlotName: slotName,
PluginType: constants.EventStreamSourcePluginType,
},
Schema: "data",
EventStreamTable: zalandov1.EventStreamTable{
Name: "foofoobar",
},
Type: constants.EventStreamSourcePGType,
},
},
},
},
}
@@ -528,8 +569,8 @@ func TestSyncStreams(t *testing.T) {
func TestSameStreams(t *testing.T) {
testName := "TestSameStreams"
annotationsA := map[string]string{"owned-by": "acid"}
annotationsB := map[string]string{"owned-by": "foo"}
annotationsA := map[string]string{constants.EventStreamMemoryAnnotationKey: "500Mi"}
annotationsB := map[string]string{constants.EventStreamMemoryAnnotationKey: "1Gi"}
stream1 := zalandov1.EventStream{
EventStreamFlow: zalandov1.EventStreamFlow{},
@@ -621,6 +662,13 @@ func TestSameStreams(t *testing.T) {
match: false,
reason: "event stream specs differ",
},
{
subTest: "event stream annotations differ",
streamsA: newFabricEventStream([]zalandov1.EventStream{stream2}, nil),
streamsB: newFabricEventStream([]zalandov1.EventStream{stream3}, annotationsA),
match: false,
reason: "event stream specs differ",
},
{
subTest: "event stream annotations differ",
streamsA: newFabricEventStream([]zalandov1.EventStream{stream2}, annotationsA),
pkg/util/constants/streams.go
+15 -12
View File
@@ -2,16 +2,19 @@ package constants
// PostgreSQL specific constants
const (
EventStreamCRDApiVersion = "zalando.org/v1"
EventStreamCRDKind = "FabricEventStream"
EventStreamCRDName = "fabriceventstreams.zalando.org"
EventStreamSourcePGType = "PostgresLogicalReplication"
EventStreamSourceSlotPrefix = "fes"
EventStreamSourcePluginType = "pgoutput"
EventStreamSourceAuthType = "DatabaseAuthenticationSecret"
EventStreamFlowPgGenericType = "PostgresWalToGenericNakadiEvent"
EventStreamSinkNakadiType = "Nakadi"
EventStreamRecoveryNoneType = "None"
EventStreamRecoveryDLQType = "DeadLetter"
EventStreamRecoverySuffix = "dead-letter-queue"
EventStreamCRDApiVersion = "zalando.org/v1"
EventStreamCRDKind = "FabricEventStream"
EventStreamCRDName = "fabriceventstreams.zalando.org"
EventStreamSourcePGType = "PostgresLogicalReplication"
EventStreamSourceSlotPrefix = "fes"
EventStreamSourcePluginType = "pgoutput"
EventStreamSourceAuthType = "DatabaseAuthenticationSecret"
EventStreamFlowPgGenericType = "PostgresWalToGenericNakadiEvent"
EventStreamSinkNakadiType = "Nakadi"
EventStreamRecoveryDLQType = "DeadLetter"
EventStreamRecoveryIgnoreType = "Ignore"
EventStreamRecoveryNoneType = "None"
EventStreamRecoverySuffix = "dead-letter-queue"
EventStreamCpuAnnotationKey = "fes.zalando.org/FES_CPU"
EventStreamMemoryAnnotationKey = "fes.zalando.org/FES_MEMORY"
)