provide event stream API

completed codegen manually provide update and compare code for fes add db check for specified databases and tables add unit test
2026-10-05 18:47:36 +02:00 · 2021-07-27 19:48:34 +02:00
parent 2057ad8191
commit 1bd12961fd
33 changed files with 1779 additions and 14 deletions
@@ -361,6 +361,10 @@ func (c *Cluster) Create() error {
 	// something fails, report warning
 	c.createConnectionPooler(c.installLookupFunction)

+	if len(c.Spec.Streams) > 0 {
+		c.createStreams()
+	}
+
 	return nil
 }

@@ -855,6 +859,11 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
 		updateFailed = true
 	}

+	if err := c.syncStreams(); err != nil {
+		c.logger.Errorf("could not sync streams: %v", err)
+		updateFailed = true
+	}
+
 	if !updateFailed {
 		// Major version upgrade must only fire after success of earlier operations and should stay last
 		if err := c.majorVersionUpgrade(); err != nil {
@@ -33,6 +33,8 @@ const (
 	getExtensionsSQL = `SELECT e.extname, n.nspname FROM pg_catalog.pg_extension e
 	        LEFT JOIN pg_catalog.pg_namespace n ON n.oid = e.extnamespace ORDER BY 1;`

+	tableExistsSQL = `SELECT TRUE FROM pg_tables WHERE tablename = $1 AND schemaname = $2;`
+
 	createDatabaseSQL       = `CREATE DATABASE "%s" OWNER "%s";`
 	createDatabaseSchemaSQL = `SET ROLE TO "%s"; CREATE SCHEMA IF NOT EXISTS "%s" AUTHORIZATION "%s"`
 	alterDatabaseOwnerSQL   = `ALTER DATABASE "%s" OWNER TO "%s";`
@@ -506,6 +508,42 @@ func (c *Cluster) execCreateOrAlterExtension(extName, schemaName, statement, doi
 	return nil
 }

+// getExtension returns the list of current database extensions
+// The caller is responsible for opening and closing the database connection
+func (c *Cluster) tableExists(tableName, schemaName string) (bool, error) {
+	var (
+		rows   *sql.Rows
+		exists bool
+		err    error
+	)
+
+	if rows, err = c.pgDb.Query(tableExistsSQL, tableName, schemaName); err != nil {
+		return false, fmt.Errorf("could not check table for existence: %v", err)
+	}
+
+	defer func() {
+		if err2 := rows.Close(); err2 != nil {
+			if err != nil {
+				err = fmt.Errorf("error when closing query cursor: %v, previous error: %v", err2, err)
+			} else {
+				err = fmt.Errorf("error when closing query cursor: %v", err2)
+			}
+		}
+	}()
+
+	for rows.Next() {
+		if err = rows.Scan(&exists); err != nil {
+			return false, fmt.Errorf("error when processing row: %v", err)
+		}
+	}
+
+	if exists {
+		return true, nil
+	} else {
+		return false, fmt.Errorf("table %s not found", schemaName+"."+tableName)
+	}
+}
+
 // Creates a connection pool credentials lookup function in every database to
 // perform remote authentication.
 func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string, role PostgresRole) error {
@@ -0,0 +1,196 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"strings"
+
+	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
+	zalandov1alpha1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1alpha1"
+	"github.com/zalando/postgres-operator/pkg/util/config"
+	"github.com/zalando/postgres-operator/pkg/util/constants"
+	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var outboxTableNameTemplate config.StringTemplate = "{table}_{eventtype}_outbox"
+
+func (c *Cluster) createStreams() error {
+	c.setProcessName("creating streams")
+
+	fes := c.generateFabricEventStream()
+	_, err := c.KubeClient.FabricEventStreamsGetter.FabricEventStreams(c.Namespace).Create(context.TODO(), fes, metav1.CreateOptions{})
+	if err != nil {
+		return fmt.Errorf("could not create event stream custom resource: %v", err)
+	}
+
+	return nil
+}
+
+func (c *Cluster) syncStreamDbResources() error {
+
+	for _, stream := range c.Spec.Streams {
+		if err := c.initDbConnWithName(stream.Database); err != nil {
+			return fmt.Errorf("could not init connection to database %s specified for event stream: %v", stream.Database, err)
+		}
+
+		for table, eventType := range stream.Tables {
+			tableName, schemaName := getTableSchema(table)
+			if exists, err := c.tableExists(tableName, schemaName); !exists {
+				return fmt.Errorf("could not find table %s specified for event stream: %v", table, err)
+			}
+			// check if outbox table exists and if not, create it
+			outboxTable := outboxTableNameTemplate.Format("table", tableName, "eventtype", eventType)
+			if exists, err := c.tableExists(outboxTable, schemaName); !exists {
+				return fmt.Errorf("could not find outbox table %s specified for event stream: %v", outboxTable, err)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (c *Cluster) generateFabricEventStream() *zalandov1alpha1.FabricEventStream {
+	eventStreams := make([]zalandov1alpha1.EventStream, 0)
+
+	for _, stream := range c.Spec.Streams {
+		for table, eventType := range stream.Tables {
+			streamSource := c.getEventStreamSource(stream, table, eventType)
+			streamFlow := getEventStreamFlow(stream)
+			streamSink := getEventStreamSink(stream, eventType)
+
+			eventStreams = append(eventStreams, zalandov1alpha1.EventStream{
+				EventStreamFlow:   streamFlow,
+				EventStreamSink:   streamSink,
+				EventStreamSource: streamSource})
+		}
+	}
+
+	return &zalandov1alpha1.FabricEventStream{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        c.Name + constants.FESsuffix,
+			Namespace:   c.Namespace,
+			Annotations: c.AnnotationsToPropagate(c.annotationsSet(nil)),
+		},
+		Spec: zalandov1alpha1.FabricEventStreamSpec{
+			ApplicationId: "",
+			EventStreams:  eventStreams,
+		},
+	}
+}
+
+func (c *Cluster) getEventStreamSource(stream acidv1.Stream, table, eventType string) zalandov1alpha1.EventStreamSource {
+	streamFilter := stream.Filter[table]
+	_, schema := getTableSchema(table)
+	return zalandov1alpha1.EventStreamSource{
+		Type:             constants.EventStreamSourcePGType,
+		Schema:           schema,
+		EventStreamTable: getOutboxTable(table, eventType),
+		Filter:           streamFilter,
+		Connection:       c.getStreamConnection(stream.Database, stream.User),
+	}
+}
+
+func getEventStreamFlow(stream acidv1.Stream) zalandov1alpha1.EventStreamFlow {
+	switch stream.Type {
+	case "nakadi":
+		return zalandov1alpha1.EventStreamFlow{
+			Type:           constants.EventStreamFlowPgNakadiType,
+			DataTypeColumn: constants.EventStreamFlowDataTypeColumn,
+			DataOpColumn:   constants.EventStreamFlowDataOpColumn,
+			MetadataColumn: constants.EventStreamFlowMetadataColumn,
+			DataColumn:     constants.EventStreamFlowDataColumn}
+	case "sqs":
+		return zalandov1alpha1.EventStreamFlow{
+			Type:             constants.EventStreamFlowPgApiType,
+			CallHomeIdColumn: "id",
+			CallHomeUrl:      stream.SqsArn}
+	}
+
+	return zalandov1alpha1.EventStreamFlow{}
+}
+
+func getEventStreamSink(stream acidv1.Stream, eventType string) zalandov1alpha1.EventStreamSink {
+	switch stream.Type {
+	case "nakadi":
+		return zalandov1alpha1.EventStreamSink{
+			Type:         constants.EventStreamSinkNakadiType,
+			EventType:    eventType,
+			MaxBatchSize: stream.BatchSize}
+	case "sqs":
+		return zalandov1alpha1.EventStreamSink{
+			Type:      constants.EventStreamSinkSqsType,
+			QueueName: stream.QueueName}
+	}
+
+	return zalandov1alpha1.EventStreamSink{}
+}
+
+func getTableSchema(fullTableName string) (tableName, schemaName string) {
+	schemaName = "public"
+	tableName = fullTableName
+	if strings.Contains(fullTableName, ".") {
+		schemaName = strings.Split(fullTableName, ".")[0]
+		tableName = strings.Split(fullTableName, ".")[1]
+	}
+
+	return tableName, schemaName
+}
+
+func getOutboxTable(tableName, eventType string) zalandov1alpha1.EventStreamTable {
+	return zalandov1alpha1.EventStreamTable{
+		Name:     outboxTableNameTemplate.Format("table", tableName, "eventtype", eventType),
+		IDColumn: "id",
+	}
+}
+
+func (c *Cluster) getStreamConnection(database, user string) zalandov1alpha1.Connection {
+	return zalandov1alpha1.Connection{
+		Url:      fmt.Sprintf("jdbc:postgresql://%s.%s/%s?user=%s&ssl=true&sslmode=require", c.Name, c.Namespace, database, user),
+		SlotName: constants.EventStreamSourceSlotName,
+		DBAuth: zalandov1alpha1.DBAuth{
+			Type:        constants.EventStreamSourceAuthType,
+			Name:        c.credentialSecretNameForCluster(user, c.ClusterName),
+			UserKey:     "username",
+			PasswordKey: "password",
+		},
+	}
+}
+
+func (c *Cluster) syncStreams() error {
+
+	c.setProcessName("syncing streams")
+
+	effectiveStreams, err := c.KubeClient.FabricEventStreamsGetter.FabricEventStreams(c.Namespace).Get(context.TODO(), c.Name+constants.FESsuffix, metav1.GetOptions{})
+	if err != nil {
+		if !k8sutil.ResourceNotFound(err) {
+			return fmt.Errorf("error during reading of event streams: %v", err)
+		}
+
+		c.logger.Infof("event streams do not exist")
+		err := c.createStreams()
+		if err != nil {
+			return fmt.Errorf("could not create missing streams: %v", err)
+		}
+	} else {
+		c.syncStreamDbResources()
+		desiredStreams := c.generateFabricEventStream()
+		if reflect.DeepEqual(effectiveStreams.Spec, desiredStreams.Spec) {
+			c.updateStreams(desiredStreams)
+		}
+	}
+
+	return nil
+}
+
+func (c *Cluster) updateStreams(newEventStreams *zalandov1alpha1.FabricEventStream) error {
+	c.setProcessName("updating event streams")
+
+	_, err := c.KubeClient.FabricEventStreamsGetter.FabricEventStreams(c.Namespace).Update(context.TODO(), newEventStreams, metav1.UpdateOptions{})
+	if err != nil {
+		return fmt.Errorf("could not update event stream custom resource: %v", err)
+	}
+
+	return nil
+}
@@ -0,0 +1,105 @@
+package cluster
+
+import (
+	"reflect"
+
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
+	fakezalandov1alpha1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake"
+	"github.com/zalando/postgres-operator/pkg/util/config"
+	"github.com/zalando/postgres-operator/pkg/util/constants"
+	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/fake"
+)
+
+func newFakeK8sStreamClient() (k8sutil.KubernetesClient, *fake.Clientset) {
+	zalandoClientSet := fakezalandov1alpha1.NewSimpleClientset()
+	clientSet := fake.NewSimpleClientset()
+
+	return k8sutil.KubernetesClient{
+		FabricEventStreamsGetter: zalandoClientSet.ZalandoV1alpha1(),
+	}, clientSet
+}
+
+func TestGenerateFabricEventStream(t *testing.T) {
+	client, _ := newFakeK8sStreamClient()
+	clusterName := "acid-test-cluster"
+	namespace := "default"
+
+	pg := acidv1.Postgresql{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      clusterName,
+			Namespace: namespace,
+		},
+		Spec: acidv1.PostgresSpec{
+			Databases: map[string]string{
+				"foo": "foo_user",
+			},
+			Streams: []acidv1.Stream{
+				{
+					Type:     "nakadi",
+					Database: "foo",
+					Tables: map[string]string{
+						"bar": "stream_type_a",
+					},
+					BatchSize: uint32(100),
+					User:      "foo_user",
+				},
+				{
+					Type:     "wal",
+					Database: "foo",
+					Tables: map[string]string{
+						"bar": "stream_type_a",
+					},
+					BatchSize: uint32(100),
+					User:      "zalando",
+				},
+				{
+					Type:      "sqs",
+					Database:  "foo",
+					SqsArn:    "arn:aws:sqs:eu-central-1:111122223333",
+					QueueName: "foo-queue",
+					User:      "foo_user",
+				},
+			},
+			Users: map[string]acidv1.UserFlags{
+				"foo_user": {},
+				"zalando":  {},
+			},
+			Volume: acidv1.Volume{
+				Size: "1Gi",
+			},
+		},
+	}
+
+	var cluster = New(
+		Config{
+			OpConfig: config.Config{
+				PodManagementPolicy: "ordered_ready",
+				Resources: config.Resources{
+					ClusterLabels:        map[string]string{"application": "spilo"},
+					ClusterNameLabel:     "cluster-name",
+					DefaultCPURequest:    "300m",
+					DefaultCPULimit:      "300m",
+					DefaultMemoryRequest: "300Mi",
+					DefaultMemoryLimit:   "300Mi",
+					PodRoleLabel:         "spilo-role",
+				},
+			},
+		}, client, pg, logger, eventRecorder)
+
+	cluster.syncStreams()
+
+	streamCRD, err := cluster.KubeClient.FabricEventStreams(namespace).Get(context.TODO(), cluster.Name+constants.FESsuffix, metav1.GetOptions{})
+	assert.NoError(t, err)
+
+	result := cluster.generateFabricEventStream()
+	if !reflect.DeepEqual(result, streamCRD) {
+		t.Errorf("Malformed FabricEventStream, expected %#v, got %#v", streamCRD, result)
+	}
+}