remove sqs and a few bugs

pkg/cluster/cluster.go

@@ -361,8 +361,8 @@ func (c *Cluster) Create() error {
 	// something fails, report warning
 	c.createConnectionPooler(c.installLookupFunction)
 
-	if len(c.Spec.Streams) > 0 {
-		c.syncStreams()
+	if err = c.syncStreams(); err != nil {
+		return fmt.Errorf("could not create streams: %v", err)
 	}
 
 	return nil
@@ -1060,7 +1060,7 @@ func (c *Cluster) initSystemUsers() {
 	// replication users for event streams are another exception
 	// the operator will create one replication user for all streams
 	if len(c.Spec.Streams) > 0 {
-		username := constants.EventStreamSourceSlotPrefix + constants.UserRoleNameSuffix
+		username := constants.EventStreamSourceSlotPrefix + "user"
 		streamUser := spec.PgUser{
 			Origin:    spec.RoleConnectionPooler,
 			Name:      username,

pkg/cluster/streams.go

@@ -56,14 +56,13 @@ func (c *Cluster) syncPostgresConfig() error {
 	desiredPostgresConfig := make(map[string]interface{})
 	slots := make(map[string]map[string]string)
 
-	c.logger.Debugf("setting wal level to 'logical' in postgres configuration")
+	// if streams are defined wal_level must be switched to logical and slots have to be defined
 	desiredPostgresConfig["postgresql"] = map[string]interface{}{patroniPGParametersParameterName: map[string]string{"wal_level": "logical"}}
 
 	for _, stream := range c.Spec.Streams {
 		slotName := c.getLogicalReplicationSlot(stream.Database)
 
 		if slotName == "" {
-			c.logger.Debugf("creating logical replication slot %q in database %q", constants.EventStreamSourceSlotPrefix+stream.Database, stream.Database)
 			slot := map[string]string{
 				"database": stream.Database,
 				"plugin":   "wal2json",
@@ -74,6 +73,10 @@ func (c *Cluster) syncPostgresConfig() error {
 	}
 
 	if len(slots) > 0 {
+		c.logger.Debugf("setting wal level to 'logical' in Postgres configuration")
+		for slotName, slot := range slots {
+			c.logger.Debugf("creating logical replication slot %q in database %q", slotName, slot["database"])
+		}
 		desiredPostgresConfig["slots"] = slots
 	} else {
 		return nil
@@ -175,27 +178,11 @@ func getEventStreamFlow(stream acidv1.Stream) zalandov1alpha1.EventStreamFlow {
 }
 
 func getEventStreamSink(stream acidv1.Stream, eventType string) zalandov1alpha1.EventStreamSink {
-	switch stream.StreamType {
-	case "sqs":
-		sqsSinkType := constants.EventStreamSinkSqsStandardType
-		if stream.SqsFifo {
-			sqsSinkType = constants.EventStreamSinkSqsFifoType
-		}
-		return zalandov1alpha1.EventStreamSink{
-			Type:         sqsSinkType,
-			QueueName:    stream.QueueName,
-			QueueUrl:     stream.SqsArn,
-			MaxBatchSize: stream.BatchSize,
-		}
-	case "default":
-		return zalandov1alpha1.EventStreamSink{
-			Type:         constants.EventStreamSinkNakadiType,
-			EventType:    eventType,
-			MaxBatchSize: stream.BatchSize,
-		}
+	return zalandov1alpha1.EventStreamSink{
+		Type:         constants.EventStreamSinkNakadiType,
+		EventType:    eventType,
+		MaxBatchSize: stream.BatchSize,
 	}
-
-	return zalandov1alpha1.EventStreamSink{}
 }
 
 func getTableSchema(fullTableName string) (tableName, schemaName string) {

pkg/cluster/streams_test.go

@@ -62,15 +62,6 @@ var (
 					},
 					BatchSize: uint32(100),
 				},
-				{
-					StreamType: "sqs",
-					Database:   "foo",
-					Tables: map[string]string{
-						"bar": "stream_type_a",
-					},
-					SqsArn:    "arn:aws:sqs:eu-central-1:111122223333",
-					QueueName: "foo-queue",
-				},
 			},
 			Users: map[string]acidv1.UserFlags{
 				"foo_user": []string{"replication"},

pkg/cluster/sync.go

@@ -72,7 +72,7 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
 		return err
 	}
 
-	c.logger.Debugf("syncing statefulsets")
+	c.logger.Debug("syncing statefulsets")
 	if err = c.syncStatefulSet(); err != nil {
 		if !k8sutil.ResourceAlreadyExists(err) {
 			err = fmt.Errorf("could not sync statefulsets: %v", err)
@@ -98,17 +98,17 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
 
 	// create database objects unless we are running without pods or disabled that feature explicitly
 	if !(c.databaseAccessDisabled() || c.getNumberOfInstances(&newSpec.Spec) <= 0 || c.Spec.StandbyCluster != nil) {
-		c.logger.Debugf("syncing roles")
+		c.logger.Debug("syncing roles")
 		if err = c.syncRoles(); err != nil {
 			err = fmt.Errorf("could not sync roles: %v", err)
 			return err
 		}
-		c.logger.Debugf("syncing databases")
+		c.logger.Debug("syncing databases")
 		if err = c.syncDatabases(); err != nil {
 			err = fmt.Errorf("could not sync databases: %v", err)
 			return err
 		}
-		c.logger.Debugf("syncing prepared databases with schemas")
+		c.logger.Debug("syncing prepared databases with schemas")
 		if err = c.syncPreparedDatabases(); err != nil {
 			err = fmt.Errorf("could not sync prepared database: %v", err)
 			return err
@@ -120,6 +120,12 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
 		return fmt.Errorf("could not sync connection pooler: %v", err)
 	}
 
+	c.logger.Debug("syncing streams")
+	if err = c.syncStreams(); err != nil {
+		err = fmt.Errorf("could not sync streams: %v", err)
+		return err
+	}
+
 	// Major version upgrade must only run after success of all earlier operations, must remain last item in sync
 	if err := c.majorVersionUpgrade(); err != nil {
 		c.logger.Errorf("major version upgrade failed: %v", err)