public_git/postgres-operator
1
0
Fork 0
mirror of https://github.com/zalando/postgres-operator.git synced 2026-09-30 14:01:46 +02:00
Code Issues Projects Releases Wiki Activity

create CDC event stream CRD (#1570)

Browse Source 
* provide event stream API
* check manifest settings for logical decoding before creating streams
* operator updates Postgres config and creates replication user
* name FES like the Postgres cluster
* add delete case and fix updating streams + update unit test
* check if fes CRD exists before syncing
* existing slot must use the same plugin
* make id and payload columns configurable
* sync streams only when they are defined in manifest
* introduce applicationId for separate stream CRDs
* add FES to RBAC in chart
* disable streams in chart
* switch to pgoutput plugin and let operator create publications
* reflect code review and additional refactoring

Co-authored-by: Paŭlo Ebermann <paul.ebermann@zalando.de>
This commit is contained in:
Felix Kunde
2022-02-28 10:09:42 +01:00
committed by GitHub
co-authored by Paŭlo Ebermann
parent 8b404fd049
commit d8a159ef1a
42 changed files with 2330 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/util/constants/streams.go
+14
View File
@@ -0,0 +1,14 @@
package constants
// PostgreSQL specific constants
const (
EventStreamCRDApiVersion = "zalando.org/v1"
EventStreamCRDKind = "FabricEventStream"
EventStreamCRDName = "fabriceventstreams.zalando.org"
EventStreamSourcePGType = "PostgresLogicalReplication"
EventStreamSourceSlotPrefix = "fes"
EventStreamSourcePluginType = "pgoutput"
EventStreamSourceAuthType = "DatabaseAuthenticationSecret"
EventStreamFlowPgGenericType = "PostgresWalToGenericNakadiEvent"
EventStreamSinkNakadiType = "Nakadi"
)
pkg/util/k8sutil/k8sutil.go
+12 -4
View File
@@ -12,8 +12,9 @@ import (
clientbatchv1beta1 "k8s.io/client-go/kubernetes/typed/batch/v1beta1"
apiacidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
acidv1client "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned"
zalandoclient "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned"
acidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1"
zalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1"
"github.com/zalando/postgres-operator/pkg/spec"
apiappsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
@@ -58,9 +59,11 @@ type KubernetesClient struct {
acidv1.OperatorConfigurationsGetter
acidv1.PostgresTeamsGetter
acidv1.PostgresqlsGetter
zalandov1.FabricEventStreamsGetter
RESTClient rest.Interface
AcidV1ClientSet *acidv1client.Clientset
RESTClient rest.Interface
AcidV1ClientSet *zalandoclient.Clientset
Zalandov1ClientSet *zalandoclient.Clientset
}
type mockSecret struct {
@@ -158,14 +161,19 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) {
kubeClient.CustomResourceDefinitionsGetter = apiextClient.ApiextensionsV1()
kubeClient.AcidV1ClientSet = acidv1client.NewForConfigOrDie(cfg)
kubeClient.AcidV1ClientSet = zalandoclient.NewForConfigOrDie(cfg)
if err != nil {
return kubeClient, fmt.Errorf("could not create acid.zalan.do clientset: %v", err)
}
kubeClient.Zalandov1ClientSet = zalandoclient.NewForConfigOrDie(cfg)
if err != nil {
return kubeClient, fmt.Errorf("could not create zalando.org clientset: %v", err)
}
kubeClient.OperatorConfigurationsGetter = kubeClient.AcidV1ClientSet.AcidV1()
kubeClient.PostgresTeamsGetter = kubeClient.AcidV1ClientSet.AcidV1()
kubeClient.PostgresqlsGetter = kubeClient.AcidV1ClientSet.AcidV1()
kubeClient.FabricEventStreamsGetter = kubeClient.Zalandov1ClientSet.ZalandoV1()
return kubeClient, nil
}
pkg/util/patroni/patroni_test.go
+3 -3
View File
@@ -185,7 +185,7 @@ func TestGetConfig(t *testing.T) {
Slots: map[string]map[string]string{
"cdc": {
"database": "foo",
"plugin": "wal2json",
"plugin": "pgoutput",
"type": "logical",
},
},
@@ -218,7 +218,7 @@ func TestGetConfig(t *testing.T) {
"wal_log_hints": "on",
}
configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 100, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "hot_standby", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "wal2json", "type": "logical"}}, "ttl": 30}`
configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 100, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "hot_standby", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "pgoutput", "type": "logical"}}, "ttl": 30}`
r := ioutil.NopCloser(bytes.NewReader([]byte(configJson)))
response := http.Response{
@@ -253,7 +253,7 @@ func TestSetPostgresParameters(t *testing.T) {
"wal_level": "logical",
}
configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 50, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "logical", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "wal2json", "type": "logical"}}, "ttl": 30}`
configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 50, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "logical", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "pgoutput", "type": "logical"}}, "ttl": 30}`
r := ioutil.NopCloser(bytes.NewReader([]byte(configJson)))
response := http.Response{