create CDC event stream CRD (#1570)

* provide event stream API
* check manifest settings for logical decoding before creating streams
* operator updates Postgres config and creates replication user
* name FES like the Postgres cluster
* add delete case and fix updating streams + update unit test
* check if fes CRD exists before syncing
* existing slot must use the same plugin
* make id and payload columns configurable
* sync streams only when they are defined in manifest
* introduce applicationId for separate stream CRDs
* add FES to RBAC in chart
* disable streams in chart
* switch to pgoutput plugin and let operator create publications
* reflect code review and additional refactoring

Co-authored-by: Paŭlo Ebermann <paul.ebermann@zalando.de>

pkg/apis/acid.zalan.do/v1/crds.go

@@ -716,6 +716,54 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 							},
 						},
 					},
+					"streams": {
+						Type: "array",
+						Items: &apiextv1.JSONSchemaPropsOrArray{
+							Schema: &apiextv1.JSONSchemaProps{
+								Type:     "object",
+								Required: []string{"applicationId", "database", "tables"},
+								Properties: map[string]apiextv1.JSONSchemaProps{
+									"applicationId": {
+										Type: "string",
+									},
+									"batchSize": {
+										Type: "integer",
+									},
+									"database": {
+										Type: "string",
+									},
+									"filter": {
+										Type: "object",
+										AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{
+											Schema: &apiextv1.JSONSchemaProps{
+												Type: "string",
+											},
+										},
+									},
+									"tables": {
+										Type: "object",
+										AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{
+											Schema: &apiextv1.JSONSchemaProps{
+												Type:     "object",
+												Required: []string{"eventType"},
+												Properties: map[string]apiextv1.JSONSchemaProps{
+													"eventType": {
+														Type: "string",
+													},
+													"idColumn": {
+														Type: "string",
+													},
+													"payloadColumn": {
+														Type: "string",
+													},
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
 					"teamId": {
 						Type: "string",
 					},

pkg/apis/acid.zalan.do/v1/postgresql_type.go

@@ -77,6 +77,7 @@ type PostgresSpec struct {
 	ServiceAnnotations    map[string]string           `json:"serviceAnnotations,omitempty"`
 	TLS                   *TLSDescription             `json:"tls,omitempty"`
 	AdditionalVolumes     []AdditionalVolume          `json:"additionalVolumes,omitempty"`
+	Streams               []Stream                    `json:"streams,omitempty"`
 
 	// deprecated json tags
 	InitContainersOld       []v1.Container `json:"init_containers,omitempty"`
@@ -231,3 +232,17 @@ type ConnectionPooler struct {
 
 	Resources `json:"resources,omitempty"`
 }
+
+type Stream struct {
+	ApplicationId string                 `json:"applicationId"`
+	Database      string                 `json:"database"`
+	Tables        map[string]StreamTable `json:"tables"`
+	Filter        map[string]string      `json:"filter,omitempty"`
+	BatchSize     uint32                 `json:"batchSize,omitempty"`
+}
+
+type StreamTable struct {
+	EventType     string `json:"eventType"`
+	IdColumn      string `json:"idColumn,omitempty" defaults:"id"`
+	PayloadColumn string `json:"payloadColumn,omitempty" defaults:"payload"`
+}

pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go

@@ -749,6 +749,13 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.Streams != nil {
+		in, out := &in.Streams, &out.Streams
+		*out = make([]Stream, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	if in.InitContainersOld != nil {
 		in, out := &in.InitContainersOld, &out.InitContainersOld
 		*out = make([]corev1.Container, len(*in))
@@ -1152,6 +1159,52 @@ func (in *StandbyDescription) DeepCopy() *StandbyDescription {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Stream) DeepCopyInto(out *Stream) {
+	*out = *in
+	if in.Tables != nil {
+		in, out := &in.Tables, &out.Tables
+		*out = make(map[string]StreamTable, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.Filter != nil {
+		in, out := &in.Filter, &out.Filter
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Stream.
+func (in *Stream) DeepCopy() *Stream {
+	if in == nil {
+		return nil
+	}
+	out := new(Stream)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *StreamTable) DeepCopyInto(out *StreamTable) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StreamTable.
+func (in *StreamTable) DeepCopy() *StreamTable {
+	if in == nil {
+		return nil
+	}
+	out := new(StreamTable)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TLSDescription) DeepCopyInto(out *TLSDescription) {
 	*out = *in

pkg/apis/zalando.org/register.go

@@ -0,0 +1,6 @@
+package zalando
+
+const (
+	// GroupName is the group name for the operator CRDs
+	GroupName = "zalando.org"
+)

pkg/apis/zalando.org/v1/fabriceventstream.go

@@ -0,0 +1,84 @@
+package v1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// +genclient
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// FabricEventStream defines FabricEventStream Custom Resource Definition Object.
+type FabricEventStream struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec FabricEventStreamSpec `json:"spec"`
+}
+
+// FabricEventStreamSpec defines the specification for the FabricEventStream TPR.
+type FabricEventStreamSpec struct {
+	ApplicationId string        `json:"applicationId"`
+	EventStreams  []EventStream `json:"eventStreams"`
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// FabricEventStreamList defines a list of FabricEventStreams .
+type FabricEventStreamList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata"`
+
+	Items []FabricEventStream `json:"items"`
+}
+
+// EventStream defines the source, flow and sink of the event stream
+type EventStream struct {
+	EventStreamFlow   EventStreamFlow   `json:"flow"`
+	EventStreamSink   EventStreamSink   `json:"sink"`
+	EventStreamSource EventStreamSource `json:"source"`
+}
+
+// EventStreamFlow defines the flow characteristics of the event stream
+type EventStreamFlow struct {
+	Type          string `json:"type"`
+	PayloadColumn string `json:"payloadColumn,omitempty" defaults:"payload"`
+}
+
+// EventStreamSink defines the target of the event stream
+type EventStreamSink struct {
+	Type         string `json:"type"`
+	EventType    string `json:"eventType,omitempty"`
+	MaxBatchSize uint32 `json:"maxBatchSize,omitempty"`
+}
+
+// EventStreamSource defines the source of the event stream and connection for FES operator
+type EventStreamSource struct {
+	Type             string           `json:"type"`
+	Schema           string           `json:"schema,omitempty" defaults:"public"`
+	EventStreamTable EventStreamTable `json:"table"`
+	Filter           string           `json:"filter,omitempty"`
+	Connection       Connection       `json:"jdbcConnection"`
+}
+
+// EventStreamTable defines the name and ID column to be used for streaming
+type EventStreamTable struct {
+	Name     string `json:"name"`
+	IDColumn string `json:"idColumn,omitempty" defaults:"id"`
+}
+
+// Connection to be used for allowing the FES operator to connect to a database
+type Connection struct {
+	Url             string `json:"jdbcUrl"`
+	SlotName        string `json:"slotName"`
+	PluginType      string `json:"pluginType,omitempty" defaults:"wal2json"`
+	PublicationName string `json:"publicationName,omitempty"`
+	DBAuth          DBAuth `json:"databaseAuthentication"`
+}
+
+// DBAuth specifies the credentials to be used for connecting with the database
+type DBAuth struct {
+	Type        string `json:"type"`
+	Name        string `json:"name,omitempty"`
+	UserKey     string `json:"userKey,omitempty"`
+	PasswordKey string `json:"passwordKey,omitempty"`
+}

pkg/apis/zalando.org/v1/register.go

@@ -0,0 +1,44 @@
+package v1
+
+import (
+	"github.com/zalando/postgres-operator/pkg/apis/zalando.org"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/kubernetes/scheme"
+)
+
+// APIVersion of the `fabriceventstream` CRD
+const (
+	APIVersion = "v1"
+)
+
+var (
+	schemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
+	AddToScheme   = schemeBuilder.AddToScheme
+)
+
+func init() {
+	err := AddToScheme(scheme.Scheme)
+	if err != nil {
+		panic(err)
+	}
+}
+
+// SchemeGroupVersion is the group version used to register these objects.
+var SchemeGroupVersion = schema.GroupVersion{Group: zalando.GroupName, Version: APIVersion}
+
+// Resource takes an unqualified resource and returns a Group-qualified GroupResource.
+func Resource(resource string) schema.GroupResource {
+	return SchemeGroupVersion.WithResource(resource).GroupResource()
+}
+
+// addKnownTypes adds the set of types defined in this package to the supplied scheme.
+func addKnownTypes(scheme *runtime.Scheme) error {
+	scheme.AddKnownTypes(SchemeGroupVersion,
+		&FabricEventStream{},
+		&FabricEventStreamList{},
+	)
+	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
+	return nil
+}

pkg/apis/zalando.org/v1/zz_generated.deepcopy.go

@@ -0,0 +1,227 @@
+// +build !ignore_autogenerated
+
+/*
+Copyright 2021 Compose, Zalando SE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+// Code generated by deepcopy-gen. DO NOT EDIT.
+
+package v1
+
+import (
+	runtime "k8s.io/apimachinery/pkg/runtime"
+)
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Connection) DeepCopyInto(out *Connection) {
+	*out = *in
+	return
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Connection) DeepCopy() *Connection {
+	if in == nil {
+		return nil
+	}
+	out := new(Connection)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DBAuth) DeepCopyInto(out *DBAuth) {
+	*out = *in
+	return
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DBAuth) DeepCopy() *DBAuth {
+	if in == nil {
+		return nil
+	}
+	out := new(DBAuth)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EventStream) DeepCopyInto(out *EventStream) {
+	*out = *in
+	return
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EventStream) DeepCopy() *EventStream {
+	if in == nil {
+		return nil
+	}
+	out := new(EventStream)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EventStreamFlow) DeepCopyInto(out *EventStreamFlow) {
+	*out = *in
+	return
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EventStreamFlow) DeepCopy() *EventStreamFlow {
+	if in == nil {
+		return nil
+	}
+	out := new(EventStreamFlow)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EventStreamSink) DeepCopyInto(out *EventStreamSink) {
+	*out = *in
+	return
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EventStreamSink) DeepCopy() *EventStreamSink {
+	if in == nil {
+		return nil
+	}
+	out := new(EventStreamSink)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EventStreamSource) DeepCopyInto(out *EventStreamSource) {
+	*out = *in
+	return
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EventStreamSource) DeepCopy() *EventStreamSource {
+	if in == nil {
+		return nil
+	}
+	out := new(EventStreamSource)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EventStreamTable) DeepCopyInto(out *EventStreamTable) {
+	*out = *in
+	return
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EventStreamTable) DeepCopy() *EventStreamTable {
+	if in == nil {
+		return nil
+	}
+	out := new(EventStreamTable)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *FabricEventStreamSpec) DeepCopyInto(out *FabricEventStreamSpec) {
+	*out = *in
+	if in.EventStreams != nil {
+		in, out := &in.EventStreams, &out.EventStreams
+		*out = make([]EventStream, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FabricEventStreamSpec.
+func (in *FabricEventStreamSpec) DeepCopy() *FabricEventStreamSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(FabricEventStreamSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *FabricEventStream) DeepCopyInto(out *FabricEventStream) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FabricEventStream.
+func (in *FabricEventStream) DeepCopy() *FabricEventStream {
+	if in == nil {
+		return nil
+	}
+	out := new(FabricEventStream)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *FabricEventStream) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *FabricEventStreamList) DeepCopyInto(out *FabricEventStreamList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]FabricEventStream, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FabricEventStreamList.
+func (in *FabricEventStreamList) DeepCopy() *FabricEventStreamList {
+	if in == nil {
+		return nil
+	}
+	out := new(FabricEventStreamList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *FabricEventStreamList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}