public_git/postgres-operator
1
0
Fork 0
mirror of https://github.com/zalando/postgres-operator.git synced 2026-10-18 05:35:07 +02:00
Code Issues Projects Releases Wiki Activity

add support for recovery section in event streams (#2421)

Browse Source
This commit is contained in:
Felix Kunde
2023-09-19 17:15:50 +02:00
committed by GitHub
parent 102a22e486
commit e03fdaaa51
7 changed files with 133 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/cluster/streams.go
+29 -4
View File
@@ -145,11 +145,13 @@ func (c *Cluster) generateFabricEventStream(appId string) *zalandov1.FabricEvent
streamSource := c.getEventStreamSource(stream, tableName, table.IdColumn)
streamFlow := getEventStreamFlow(stream, table.PayloadColumn)
streamSink := getEventStreamSink(stream, table.EventType)
streamRecovery := getEventStreamRecovery(stream, table.RecoveryEventType, table.EventType)
eventStreams = append(eventStreams, zalandov1.EventStream{
EventStreamFlow: streamFlow,
EventStreamSink: streamSink,
EventStreamSource: streamSource})
EventStreamFlow: streamFlow,
EventStreamRecovery: streamRecovery,
EventStreamSink: streamSink,
EventStreamSource: streamSource})
}
}
@@ -204,6 +206,28 @@ func getEventStreamSink(stream acidv1.Stream, eventType string) zalandov1.EventS
}
}
func getEventStreamRecovery(stream acidv1.Stream, recoveryEventType, eventType string) zalandov1.EventStreamRecovery {
if (stream.EnableRecovery != nil && !*stream.EnableRecovery) ||
(stream.EnableRecovery == nil && recoveryEventType == "") {
return zalandov1.EventStreamRecovery{
Type: constants.EventStreamRecoveryNoneType,
}
}
if stream.EnableRecovery != nil && *stream.EnableRecovery && recoveryEventType == "" {
recoveryEventType = fmt.Sprintf("%s-%s", eventType, constants.EventStreamRecoverySuffix)
}
return zalandov1.EventStreamRecovery{
Type: constants.EventStreamRecoveryDLQType,
Sink: &zalandov1.EventStreamSink{
Type: constants.EventStreamSinkNakadiType,
EventType: recoveryEventType,
MaxBatchSize: stream.BatchSize,
},
}
}
func getTableSchema(fullTableName string) (tableName, schemaName string) {
schemaName = "public"
tableName = fullTableName
@@ -381,7 +405,8 @@ func sameStreams(curEventStreams, newEventStreams []zalandov1.EventStream) (matc
for _, curStream := range curEventStreams {
if reflect.DeepEqual(newStream.EventStreamSource, curStream.EventStreamSource) &&
reflect.DeepEqual(newStream.EventStreamFlow, curStream.EventStreamFlow) &&
reflect.DeepEqual(newStream.EventStreamSink, curStream.EventStreamSink) {
reflect.DeepEqual(newStream.EventStreamSink, curStream.EventStreamSink) &&
reflect.DeepEqual(newStream.EventStreamRecovery, curStream.EventStreamRecovery) {
match = true
break
}
pkg/cluster/streams_test.go
+23 -3
View File
@@ -65,9 +65,11 @@ var (
PayloadColumn: k8sutil.StringToPointer("b_payload"),
},
"data.foobar": acidv1.StreamTable{
EventType: "stream-type-b",
EventType: "stream-type-b",
RecoveryEventType: "stream-type-b-dlq",
},
},
EnableRecovery: util.True(),
Filter: map[string]*string{
"data.bar": k8sutil.StringToPointer("[?(@.source.txId > 500 && @.source.lsn > 123456)]"),
},
@@ -106,6 +108,14 @@ var (
PayloadColumn: k8sutil.StringToPointer("b_payload"),
Type: constants.EventStreamFlowPgGenericType,
},
EventStreamRecovery: zalandov1.EventStreamRecovery{
Type: constants.EventStreamRecoveryDLQType,
Sink: &zalandov1.EventStreamSink{
EventType: fmt.Sprintf("%s-%s", "stream-type-a", constants.EventStreamRecoverySuffix),
MaxBatchSize: k8sutil.UInt32ToPointer(uint32(100)),
Type: constants.EventStreamSinkNakadiType,
},
},
EventStreamSink: zalandov1.EventStreamSink{
EventType: "stream-type-a",
MaxBatchSize: k8sutil.UInt32ToPointer(uint32(100)),
@@ -136,6 +146,14 @@ var (
EventStreamFlow: zalandov1.EventStreamFlow{
Type: constants.EventStreamFlowPgGenericType,
},
EventStreamRecovery: zalandov1.EventStreamRecovery{
Type: constants.EventStreamRecoveryDLQType,
Sink: &zalandov1.EventStreamSink{
EventType: "stream-type-b-dlq",
MaxBatchSize: k8sutil.UInt32ToPointer(uint32(100)),
Type: constants.EventStreamSinkNakadiType,
},
},
EventStreamSink: zalandov1.EventStreamSink{
EventType: "stream-type-b",
MaxBatchSize: k8sutil.UInt32ToPointer(uint32(100)),
@@ -251,7 +269,8 @@ func TestSameStreams(t *testing.T) {
testName := "TestSameStreams"
stream1 := zalandov1.EventStream{
EventStreamFlow: zalandov1.EventStreamFlow{},
EventStreamFlow: zalandov1.EventStreamFlow{},
EventStreamRecovery: zalandov1.EventStreamRecovery{},
EventStreamSink: zalandov1.EventStreamSink{
EventType: "stream-type-a",
},
@@ -263,7 +282,8 @@ func TestSameStreams(t *testing.T) {
}
stream2 := zalandov1.EventStream{
EventStreamFlow: zalandov1.EventStreamFlow{},
EventStreamFlow: zalandov1.EventStreamFlow{},
EventStreamRecovery: zalandov1.EventStreamRecovery{},
EventStreamSink: zalandov1.EventStreamSink{
EventType: "stream-type-b",
},