public_git
/
postgres-operator
mirror of https://github.com/zalando/postgres-operator.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

re-add 3rd type, remove callHome flow

This commit is contained in:
Felix Kunde 2021-08-27 13:00:38 +02:00
parent 7a84c0852c
commit d2addd3b17
9 changed files with 68 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
charts/postgres-operator/crds/postgresqls.yaml
View File

@ -490,6 +490,8 @@ spec:
type: string
sqsArn:
type: string
sqsFifo:
type: boolean
tables:
type: object
additionalProperties:
@ -499,6 +501,7 @@ spec:
enum:
- "nakadi"
- "sqs"
- "wal"
teamId:
type: string
tls:

7
manifests/complete-postgres-manifest.yaml
View File

@ -198,10 +198,17 @@ spec:
# tables:
# ta: event_type_a
# tb: event_type_b
# - type: wal
# batchSize: 100
# database: foo
# tables:
# public.tx: event_type_a
# public.ty: event_type_b
# - type: sqs
# database: foo
# tables:
# ta: ""
# tb: ""
# sqsArn: arn:aws:sqs:eu-central-1:111122223333
# sqsFifo: true
# queueName: foo-queue

3
manifests/postgresql.crd.yaml
View File

@ -486,6 +486,8 @@ spec:
type: string
sqsArn:
type: string
sqsFifo:
type: boolean
tables:
type: object
additionalProperties:
@ -495,6 +497,7 @@ spec:
enum:
- "nakadi"
- "sqs"
- "wal"
teamId:
type: string
tls:

6
pkg/apis/acid.zalan.do/v1/crds.go
View File

@ -684,6 +684,9 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
"sqsArn": {
Type: "string",
},
"sqsFifo": {
Type: "boolean",
},
"tables": {
Type: "object",
AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{
@ -701,6 +704,9 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
{
Raw: []byte(`"sqs"`),
},
{
Raw: []byte(`"wal"`),
},
},
},
},

1
pkg/apis/acid.zalan.do/v1/postgresql_type.go
View File

@ -233,5 +233,6 @@ type Stream struct {
Filter map[string]string `json:"filter,omitempty"`
BatchSize uint32 `json:"batchSize,omitempty"`
SqsArn string `json:"sqsArn,omitempty"`
SqsFifo bool `json:"sqsFifo,omitempty"`
QueueName string `json:"queueName,omitempty"`
}

2
pkg/apis/zalando.org/v1alpha1/fabriceventstream.go
View File

@ -45,6 +45,7 @@ type EventStreamFlow struct {
DataOpColumn string `json:"dataOpColumn,omitempty"`
MetadataColumn string `json:"metadataColumn,omitempty"`
DataColumn string `json:"dataColumn,omitempty"`
PayloadColumn string `json:"payloadColumn,omitempty"`
CallHomeIdColumn string `json:"callHomeIdColumn,omitempty"`
CallHomeUrl string `json:"callHomeUrl,omitempty"`
}
@ -55,6 +56,7 @@ type EventStreamSink struct {
EventType string `json:"eventType,omitempty"`
MaxBatchSize uint32 `json:"maxBatchSize,omitempty"`
QueueName string `json:"queueName,omitempty"`
QueueUrl string `json:"queueUrl,omitempty"`
}
// EventStreamSource defines the source of the event stream and connection for FES operator

38
pkg/cluster/streams.go
View File

@ -142,10 +142,11 @@ func (c *Cluster) getEventStreamSource(stream acidv1.Stream, table, eventType st
Filter: streamFilter,
Connection: c.getStreamConnection(stream.Database, constants.EventStreamSourceSlotPrefix+constants.UserRoleNameSuffix),
}
case "sqs":
case "default":
return zalandov1alpha1.EventStreamSource{
Type: constants.EventStreamSourcePGType,
EventStreamTable: getSqsTable(table),
Schema: schema,
EventStreamTable: getSourceTable(table),
Connection: c.getStreamConnection(stream.Database, constants.EventStreamSourceSlotPrefix+constants.UserRoleNameSuffix),
}
}
@ -161,12 +162,13 @@ func getEventStreamFlow(stream acidv1.Stream) zalandov1alpha1.EventStreamFlow {
DataTypeColumn: constants.EventStreamFlowDataTypeColumn,
DataOpColumn: constants.EventStreamFlowDataOpColumn,
MetadataColumn: constants.EventStreamFlowMetadataColumn,
DataColumn: constants.EventStreamFlowDataColumn}
case "sqs":
DataColumn: constants.EventStreamFlowDataColumn,
}
case "default":
return zalandov1alpha1.EventStreamFlow{
Type: constants.EventStreamFlowPgApiType,
CallHomeIdColumn: "id",
CallHomeUrl: stream.SqsArn}
Type: constants.EventStreamFlowPgGenericType,
PayloadColumn: constants.EventStreamFlowPayloadColumn,
}
}
return zalandov1alpha1.EventStreamFlow{}
@ -174,15 +176,23 @@ func getEventStreamFlow(stream acidv1.Stream) zalandov1alpha1.EventStreamFlow {
func getEventStreamSink(stream acidv1.Stream, eventType string) zalandov1alpha1.EventStreamSink {
switch stream.StreamType {
case "nakadi":
case "sqs":
sqsSinkType := constants.EventStreamSinkSqsStandardType
if stream.SqsFifo {
sqsSinkType = constants.EventStreamSinkSqsFifoType
}
return zalandov1alpha1.EventStreamSink{
Type: sqsSinkType,
QueueName: stream.QueueName,
QueueUrl: stream.SqsArn,
MaxBatchSize: stream.BatchSize,
}
case "default":
return zalandov1alpha1.EventStreamSink{
Type: constants.EventStreamSinkNakadiType,
EventType: eventType,
MaxBatchSize: stream.BatchSize}
case "sqs":
return zalandov1alpha1.EventStreamSink{
Type: constants.EventStreamSinkSqsType,
QueueName: stream.QueueName}
MaxBatchSize: stream.BatchSize,
}
}
return zalandov1alpha1.EventStreamSink{}
@ -206,7 +216,7 @@ func getOutboxTable(tableName, eventType string) zalandov1alpha1.EventStreamTabl
}
}
func getSqsTable(tableName string) zalandov1alpha1.EventStreamTable {
func getSourceTable(tableName string) zalandov1alpha1.EventStreamTable {
return zalandov1alpha1.EventStreamTable{
Name: outboxTableNameTemplate.Format("table", tableName),
}

8
pkg/cluster/streams_test.go
View File

@ -54,6 +54,14 @@ var (
},
BatchSize: uint32(100),
},
{
StreamType: "wal",
Database: "foo",
Tables: map[string]string{
"bar": "stream_type_a",
},
BatchSize: uint32(100),
},
{
StreamType: "sqs",
Database: "foo",

25
pkg/util/constants/streams.go
View File

@ -2,15 +2,18 @@ package constants
// PostgreSQL specific constants
const (
EventStreamSourcePGType = "PostgresLogicalReplication"
EventStreamSourceSlotPrefix = "fes_"
EventStreamSourceAuthType = "DatabaseAuthenticationSecret"
EventStreamFlowPgNakadiType = "PostgresWalToNakadiDataEvent"
EventStreamFlowPgApiType = "PostgresWalToApiCallHomeEvent"
EventStreamFlowDataTypeColumn = "data_type"
EventStreamFlowDataOpColumn = "data_op"
EventStreamFlowMetadataColumn = "metadata"
EventStreamFlowDataColumn = "data"
EventStreamSinkNakadiType = "Nakadi"
EventStreamSinkSqsType = "Sqs"
EventStreamSourcePGType = "PostgresLogicalReplication"
EventStreamSourceSlotPrefix = "fes_"
EventStreamSourceAuthType = "DatabaseAuthenticationSecret"
EventStreamFlowPgNakadiType = "PostgresWalToNakadiDataEvent"
EventStreamFlowPgGenericType = "PostgresWalToGenericNakadiEvent"
EventStreamFlowPgApiType = "PostgresWalToApiCallHomeEvent"
EventStreamFlowDataTypeColumn = "data_type"
EventStreamFlowDataOpColumn = "data_op"
EventStreamFlowMetadataColumn = "metadata"
EventStreamFlowDataColumn = "data"
EventStreamFlowPayloadColumn = "payload"
EventStreamSinkNakadiType = "Nakadi"
EventStreamSinkSqsStandardType = "SqsStandard"
EventStreamSinkSqsFifoType = "SqsFifo"
)