re-add 3rd type, remove callHome flow

charts/postgres-operator/crds/postgresqls.yaml

@@ -490,6 +490,8 @@ spec:
                       type: string
                     sqsArn:
                       type: string
+                    sqsFifo:
+                      type: boolean
                     tables:
                       type: object
                       additionalProperties:
@@ -499,6 +501,7 @@ spec:
                       enum:
                         - "nakadi"
                         - "sqs"
+                        - "wal"
               teamId:
                 type: string
               tls:

manifests/complete-postgres-manifest.yaml

@@ -198,10 +198,17 @@ spec:
 #    tables:
 #       ta: event_type_a
 #       tb: event_type_b
+#  - type: wal
+#    batchSize: 100
+#    database: foo
+#    tables:
+#       public.tx: event_type_a
+#       public.ty: event_type_b
 #  - type: sqs
 #    database: foo
 #    tables:
 #       ta: ""
 #       tb: ""
 #    sqsArn: arn:aws:sqs:eu-central-1:111122223333
+#    sqsFifo: true
 #    queueName: foo-queue

manifests/postgresql.crd.yaml

@@ -486,6 +486,8 @@ spec:
                       type: string
                     sqsArn:
                       type: string
+                    sqsFifo:
+                      type: boolean
                     tables:
                       type: object
                       additionalProperties:
@@ -495,6 +497,7 @@ spec:
                       enum:
                         - "nakadi"
                         - "sqs"
+                        - "wal"
               teamId:
                 type: string
               tls:

pkg/apis/acid.zalan.do/v1/crds.go

@@ -684,6 +684,9 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 									"sqsArn": {
 										Type: "string",
 									},
+									"sqsFifo": {
+										Type: "boolean",
+									},
 									"tables": {
 										Type: "object",
 										AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{
@@ -701,6 +704,9 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 											{
 												Raw: []byte(`"sqs"`),
 											},
+											{
+												Raw: []byte(`"wal"`),
+											},
 										},
 									},
 								},

pkg/apis/acid.zalan.do/v1/postgresql_type.go

@@ -233,5 +233,6 @@ type Stream struct {
 	Filter     map[string]string `json:"filter,omitempty"`
 	BatchSize  uint32            `json:"batchSize,omitempty"`
 	SqsArn     string            `json:"sqsArn,omitempty"`
+	SqsFifo    bool              `json:"sqsFifo,omitempty"`
 	QueueName  string            `json:"queueName,omitempty"`
 }

pkg/apis/zalando.org/v1alpha1/fabriceventstream.go

@@ -45,6 +45,7 @@ type EventStreamFlow struct {
 	DataOpColumn     string `json:"dataOpColumn,omitempty"`
 	MetadataColumn   string `json:"metadataColumn,omitempty"`
 	DataColumn       string `json:"dataColumn,omitempty"`
+	PayloadColumn    string `json:"payloadColumn,omitempty"`
 	CallHomeIdColumn string `json:"callHomeIdColumn,omitempty"`
 	CallHomeUrl      string `json:"callHomeUrl,omitempty"`
 }
@@ -55,6 +56,7 @@ type EventStreamSink struct {
 	EventType    string `json:"eventType,omitempty"`
 	MaxBatchSize uint32 `json:"maxBatchSize,omitempty"`
 	QueueName    string `json:"queueName,omitempty"`
+	QueueUrl     string `json:"queueUrl,omitempty"`
 }
 
 // EventStreamSource defines the source of the event stream and connection for FES operator

pkg/cluster/streams.go

@@ -142,10 +142,11 @@ func (c *Cluster) getEventStreamSource(stream acidv1.Stream, table, eventType st
 			Filter:           streamFilter,
 			Connection:       c.getStreamConnection(stream.Database, constants.EventStreamSourceSlotPrefix+constants.UserRoleNameSuffix),
 		}
-	case "sqs":
+	case "default":
 		return zalandov1alpha1.EventStreamSource{
 			Type:             constants.EventStreamSourcePGType,
-			EventStreamTable: getSqsTable(table),
+			Schema:           schema,
+			EventStreamTable: getSourceTable(table),
 			Connection:       c.getStreamConnection(stream.Database, constants.EventStreamSourceSlotPrefix+constants.UserRoleNameSuffix),
 		}
 	}
@@ -161,12 +162,13 @@ func getEventStreamFlow(stream acidv1.Stream) zalandov1alpha1.EventStreamFlow {
 			DataTypeColumn: constants.EventStreamFlowDataTypeColumn,
 			DataOpColumn:   constants.EventStreamFlowDataOpColumn,
 			MetadataColumn: constants.EventStreamFlowMetadataColumn,
-			DataColumn:     constants.EventStreamFlowDataColumn}
+			DataColumn:     constants.EventStreamFlowDataColumn,
-	case "sqs":
+		}
+	case "default":
 		return zalandov1alpha1.EventStreamFlow{
-			Type:             constants.EventStreamFlowPgApiType,
+			Type:          constants.EventStreamFlowPgGenericType,
-			CallHomeIdColumn: "id",
+			PayloadColumn: constants.EventStreamFlowPayloadColumn,
-			CallHomeUrl:      stream.SqsArn}
+		}
 	}
 
 	return zalandov1alpha1.EventStreamFlow{}
@@ -174,15 +176,23 @@ func getEventStreamFlow(stream acidv1.Stream) zalandov1alpha1.EventStreamFlow {
 
 func getEventStreamSink(stream acidv1.Stream, eventType string) zalandov1alpha1.EventStreamSink {
 	switch stream.StreamType {
-	case "nakadi":
+	case "sqs":
+		sqsSinkType := constants.EventStreamSinkSqsStandardType
+		if stream.SqsFifo {
+			sqsSinkType = constants.EventStreamSinkSqsFifoType
+		}
+		return zalandov1alpha1.EventStreamSink{
+			Type:         sqsSinkType,
+			QueueName:    stream.QueueName,
+			QueueUrl:     stream.SqsArn,
+			MaxBatchSize: stream.BatchSize,
+		}
+	case "default":
 		return zalandov1alpha1.EventStreamSink{
 			Type:         constants.EventStreamSinkNakadiType,
 			EventType:    eventType,
-			MaxBatchSize: stream.BatchSize}
+			MaxBatchSize: stream.BatchSize,
-	case "sqs":
+		}
-		return zalandov1alpha1.EventStreamSink{
-			Type:      constants.EventStreamSinkSqsType,
-			QueueName: stream.QueueName}
 	}
 
 	return zalandov1alpha1.EventStreamSink{}
@@ -206,7 +216,7 @@ func getOutboxTable(tableName, eventType string) zalandov1alpha1.EventStreamTabl
 	}
 }
 
-func getSqsTable(tableName string) zalandov1alpha1.EventStreamTable {
+func getSourceTable(tableName string) zalandov1alpha1.EventStreamTable {
 	return zalandov1alpha1.EventStreamTable{
 		Name: outboxTableNameTemplate.Format("table", tableName),
 	}

pkg/cluster/streams_test.go

@@ -54,6 +54,14 @@ var (
 					},
 					BatchSize: uint32(100),
 				},
+				{
+					StreamType: "wal",
+					Database:   "foo",
+					Tables: map[string]string{
+						"bar": "stream_type_a",
+					},
+					BatchSize: uint32(100),
+				},
 				{
 					StreamType: "sqs",
 					Database:   "foo",

pkg/util/constants/streams.go

@@ -2,15 +2,18 @@ package constants
 
 // PostgreSQL specific constants
 const (
-	EventStreamSourcePGType       = "PostgresLogicalReplication"
+	EventStreamSourcePGType        = "PostgresLogicalReplication"
-	EventStreamSourceSlotPrefix   = "fes_"
+	EventStreamSourceSlotPrefix    = "fes_"
-	EventStreamSourceAuthType     = "DatabaseAuthenticationSecret"
+	EventStreamSourceAuthType      = "DatabaseAuthenticationSecret"
-	EventStreamFlowPgNakadiType   = "PostgresWalToNakadiDataEvent"
+	EventStreamFlowPgNakadiType    = "PostgresWalToNakadiDataEvent"
-	EventStreamFlowPgApiType      = "PostgresWalToApiCallHomeEvent"
+	EventStreamFlowPgGenericType   = "PostgresWalToGenericNakadiEvent"
-	EventStreamFlowDataTypeColumn = "data_type"
+	EventStreamFlowPgApiType       = "PostgresWalToApiCallHomeEvent"
-	EventStreamFlowDataOpColumn   = "data_op"
+	EventStreamFlowDataTypeColumn  = "data_type"
-	EventStreamFlowMetadataColumn = "metadata"
+	EventStreamFlowDataOpColumn    = "data_op"
-	EventStreamFlowDataColumn     = "data"
+	EventStreamFlowMetadataColumn  = "metadata"
-	EventStreamSinkNakadiType     = "Nakadi"
+	EventStreamFlowDataColumn      = "data"
-	EventStreamSinkSqsType        = "Sqs"
+	EventStreamFlowPayloadColumn   = "payload"
+	EventStreamSinkNakadiType      = "Nakadi"
+	EventStreamSinkSqsStandardType = "SqsStandard"
+	EventStreamSinkSqsFifoType     = "SqsFifo"
 )