public_git
/
postgres-operator
mirror of https://github.com/zalando/postgres-operator.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix sync of publications

This commit is contained in:
Felix Kunde 2022-01-26 23:15:28 +01:00
parent 2630d8eff3
commit 3f0ed26828
9 changed files with 28 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
manifests/complete-postgres-manifest.yaml
View File

@ -111,7 +111,7 @@ spec:
# permanent_logical_1: # permanent_logical_1:
# type: logical # type: logical
# database: foo # database: foo
# plugin: pgoutput # plugin: wal2json
ttl: 30 ttl: 30
loop_wait: &loop_wait 10 loop_wait: &loop_wait 10
retry_timeout: 10 retry_timeout: 10

6
pkg/apis/acid.zalan.do/v1/util_test.go
View File

@ -252,7 +252,7 @@ var unmarshalCluster = []struct {
"permanent_logical_1" : { "permanent_logical_1" : {
"type" : "logical", "type" : "logical",
"database" : "foo", "database" : "foo",
"plugin" : "pgoutput" "plugin" : "wal2json"
} }
} }
}, },
@ -298,7 +298,7 @@ var unmarshalCluster = []struct {
LoopWait: 10, LoopWait: 10,
RetryTimeout: 10, RetryTimeout: 10,
MaximumLagOnFailover: 33554432, MaximumLagOnFailover: 33554432,
Slots: map[string]map[string]string{"permanent_logical_1": {"type": "logical", "database": "foo", "plugin": "pgoutput"}}, Slots: map[string]map[string]string{"permanent_logical_1": {"type": "logical", "database": "foo", "plugin": "wal2json"}},
}, },
Resources: Resources{ Resources: Resources{
ResourceRequests: ResourceDescription{CPU: "10m", Memory: "50Mi"}, ResourceRequests: ResourceDescription{CPU: "10m", Memory: "50Mi"},
@ -334,7 +334,7 @@ var unmarshalCluster = []struct {
}, },
Error: "", Error: "",
}, },
marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"9.6","parameters":{"log_statement":"all","max_connections":"10","shared_buffers":"32MB"}},"pod_priority_class_name":"spilo-pod-priority","volume":{"size":"5Gi","storageClass":"SSD", "subPath": "subdir"},"enableShmVolume":false,"patroni":{"initdb":{"data-checksums":"true","encoding":"UTF8","locale":"en_US.UTF-8"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"],"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}},"resources":{"requests":{"cpu":"10m","memory":"50Mi"},"limits":{"cpu":"300m","memory":"3000Mi"}},"teamId":"acid","allowedSourceRanges":["127.0.0.1/32"],"numberOfInstances":2,"users":{"zalando":["superuser","createdb"]},"maintenanceWindows":["Mon:01:00-06:00","Sat:00:00-04:00","05:00-05:15"],"clone":{"cluster":"acid-batman"}},"status":{"PostgresClusterStatus":""}}`), marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"9.6","parameters":{"log_statement":"all","max_connections":"10","shared_buffers":"32MB"}},"pod_priority_class_name":"spilo-pod-priority","volume":{"size":"5Gi","storageClass":"SSD", "subPath": "subdir"},"enableShmVolume":false,"patroni":{"initdb":{"data-checksums":"true","encoding":"UTF8","locale":"en_US.UTF-8"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"],"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"slots":{"permanent_logical_1":{"database":"foo","plugin":"wal2json","type":"logical"}}},"resources":{"requests":{"cpu":"10m","memory":"50Mi"},"limits":{"cpu":"300m","memory":"3000Mi"}},"teamId":"acid","allowedSourceRanges":["127.0.0.1/32"],"numberOfInstances":2,"users":{"zalando":["superuser","createdb"]},"maintenanceWindows":["Mon:01:00-06:00","Sat:00:00-04:00","05:00-05:15"],"clone":{"cluster":"acid-batman"}},"status":{"PostgresClusterStatus":""}}`),
err: nil}, err: nil},
{ {
about: "example with teamId set in input", about: "example with teamId set in input",

3
pkg/apis/zalando.org/v1/fabriceventstream.go
View File

@ -70,7 +70,8 @@ type EventStreamTable struct {
type Connection struct { type Connection struct {
Url string `json:"jdbcUrl"` Url string `json:"jdbcUrl"`
SlotName string `json:"slotName"` SlotName string `json:"slotName"`
PublicationName string `json:"publicationName"` PluginType string `json:"pluginType,omitempty" defaults:"wal2json"`
PublicationName string `json:"publicationName,omitempty"`
DBAuth DBAuth `json:"databaseAuthentication"` DBAuth DBAuth `json:"databaseAuthentication"`
} }

5
pkg/cluster/database.go
View File

@ -518,7 +518,6 @@ func (c *Cluster) execCreateOrAlterExtension(extName, schemaName, statement, doi
func (c *Cluster) getPublications() (publications map[string]string, err error) { func (c *Cluster) getPublications() (publications map[string]string, err error) {
var ( var (
rows *sql.Rows rows *sql.Rows
dbPublications map[string]string
) )
if rows, err = c.pgDb.Query(getPublicationsSQL); err != nil { if rows, err = c.pgDb.Query(getPublicationsSQL); err != nil {
@ -535,6 +534,8 @@ func (c *Cluster) getPublications() (publications map[string]string, err error)
} }
}() }()
dbPublications := make(map[string]string)
for rows.Next() { for rows.Next() {
var ( var (
dbPublication string dbPublication string
@ -566,7 +567,7 @@ func (c *Cluster) executeAlterPublication(pubName, tableList string) error {
func (c *Cluster) execCreateOrAlterPublication(pubName, tableList, statement, doing, operation string) error { func (c *Cluster) execCreateOrAlterPublication(pubName, tableList, statement, doing, operation string) error {
c.logger.Infof("%s %q table list %q", doing, pubName, tableList) c.logger.Infof("%s %q with table list %q", doing, pubName, tableList)
if _, err := c.pgDb.Exec(fmt.Sprintf(statement, pubName, tableList)); err != nil { if _, err := c.pgDb.Exec(fmt.Sprintf(statement, pubName, tableList)); err != nil {
return fmt.Errorf("could not execute %s: %v", operation, err) return fmt.Errorf("could not execute %s: %v", operation, err)
} }

4
pkg/cluster/k8sres_test.go
View File

@ -91,11 +91,11 @@ func TestGenerateSpiloJSONConfiguration(t *testing.T) {
MaximumLagOnFailover: 33554432, MaximumLagOnFailover: 33554432,
SynchronousMode: true, SynchronousMode: true,
SynchronousModeStrict: true, SynchronousModeStrict: true,
Slots: map[string]map[string]string{"permanent_logical_1": {"type": "logical", "database": "foo", "plugin": "pgoutput"}}, Slots: map[string]map[string]string{"permanent_logical_1": {"type": "logical", "database": "foo", "plugin": "wal2json"}},
}, },
role: "zalandos", role: "zalandos",
opConfig: config.Config{}, opConfig: config.Config{},
result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/11/bin","pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"zalandos":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"synchronous_mode":true,"synchronous_mode_strict":true,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}}}}`, result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/11/bin","pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"zalandos":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"synchronous_mode":true,"synchronous_mode_strict":true,"slots":{"permanent_logical_1":{"database":"foo","plugin":"wal2json","type":"logical"}}}}}`,
}, },
} }
for _, tt := range tests { for _, tt := range tests {

16
pkg/cluster/streams.go
View File

@ -75,7 +75,6 @@ func gatherApplicationIds(streams []acidv1.Stream) []string {
} }
func (c *Cluster) syncPostgresConfig() error { func (c *Cluster) syncPostgresConfig() error {
slots := make(map[string]map[string]string) slots := make(map[string]map[string]string)
publications := make(map[string]map[string]acidv1.StreamTable) publications := make(map[string]map[string]acidv1.StreamTable)
createPublications := make(map[string]string) createPublications := make(map[string]string)
@ -92,10 +91,11 @@ func (c *Cluster) syncPostgresConfig() error {
slots = desiredPatroniConfig.Slots slots = desiredPatroniConfig.Slots
} }
// define extra logical slots for Patroni config
for _, stream := range c.Spec.Streams { for _, stream := range c.Spec.Streams {
slot := map[string]string{ slot := map[string]string{
"database": stream.Database, "database": stream.Database,
"plugin": "pgoutput", "plugin": constants.EventStreamSourcePluginType,
"type": "logical", "type": "logical",
} }
slotName := getSlotName(stream.Database, stream.ApplicationId) slotName := getSlotName(stream.Database, stream.ApplicationId)
@ -114,18 +114,15 @@ func (c *Cluster) syncPostgresConfig() error {
} }
if len(slots) > 0 { if len(slots) > 0 {
c.logger.Debugf("setting wal level to 'logical' in Postgres configuration to allow for decoding changes")
for slotName, slot := range slots {
c.logger.Debugf("creating logical replication slot %q in database %q", slotName, slot["database"])
}
desiredPatroniConfig.Slots = slots desiredPatroniConfig.Slots = slots
} else { } else {
return nil return nil
} }
// if streams are defined wal_level must be switched to logical and slots have to be defined // if streams are defined wal_level must be switched to logical
desiredPgParameters := map[string]string{"wal_level": "logical"} desiredPgParameters := map[string]string{"wal_level": "logical"}
// apply config changes in pods
pods, err := c.listPods() pods, err := c.listPods()
if err != nil || len(pods) == 0 { if err != nil || len(pods) == 0 {
c.logger.Warningf("could not list pods of the statefulset: %v", err) c.logger.Warningf("could not list pods of the statefulset: %v", err)
@ -146,6 +143,7 @@ func (c *Cluster) syncPostgresConfig() error {
} }
// next, create publications to each created slot // next, create publications to each created slot
c.logger.Debug("syncing database publications")
for publication, tables := range publications { for publication, tables := range publications {
// but first check for existing publications // but first check for existing publications
dbName := slots[publication]["database"] dbName := slots[publication]["database"]
@ -162,7 +160,7 @@ func (c *Cluster) syncPostgresConfig() error {
i := 0 i := 0
for t := range tables { for t := range tables {
tableName, schemaName := getTableSchema(t) tableName, schemaName := getTableSchema(t)
tableNames[i] = fmt.Sprintf("%q.%q", schemaName, tableName) tableNames[i] = fmt.Sprintf("%s.%s", schemaName, tableName)
i++ i++
} }
sort.Strings(tableNames) sort.Strings(tableNames)
@ -288,7 +286,7 @@ func (c *Cluster) getStreamConnection(database, user, appId string) zalandov1.Co
return zalandov1.Connection{ return zalandov1.Connection{
Url: fmt.Sprintf("jdbc:postgresql://%s.%s/%s?user=%s&ssl=true&sslmode=require", c.Name, c.Namespace, database, user), Url: fmt.Sprintf("jdbc:postgresql://%s.%s/%s?user=%s&ssl=true&sslmode=require", c.Name, c.Namespace, database, user),
SlotName: getSlotName(database, appId), SlotName: getSlotName(database, appId),
PublicationName: getSlotName(database, appId), PluginType: constants.EventStreamSourcePluginType,
DBAuth: zalandov1.DBAuth{ DBAuth: zalandov1.DBAuth{
Type: constants.EventStreamSourceAuthType, Type: constants.EventStreamSourceAuthType,
Name: c.credentialSecretNameForCluster(user, c.Name), Name: c.credentialSecretNameForCluster(user, c.Name),

2
pkg/cluster/streams_test.go
View File

@ -121,7 +121,7 @@ var (
}, },
Url: fmt.Sprintf("jdbc:postgresql://%s.%s/foo?user=%s&ssl=true&sslmode=require", clusterName, namespace, fesUser), Url: fmt.Sprintf("jdbc:postgresql://%s.%s/foo?user=%s&ssl=true&sslmode=require", clusterName, namespace, fesUser),
SlotName: slotName, SlotName: slotName,
PublicationName: slotName, PluginType: constants.EventStreamSourcePluginType,
}, },
Schema: "data", Schema: "data",
EventStreamTable: v1.EventStreamTable{ EventStreamTable: v1.EventStreamTable{

1
pkg/util/constants/streams.go
View File

@ -6,6 +6,7 @@ const (
EventStreamSourceCRDName = "fabriceventstreams.zalando.org" EventStreamSourceCRDName = "fabriceventstreams.zalando.org"
EventStreamSourcePGType = "PostgresLogicalReplication" EventStreamSourcePGType = "PostgresLogicalReplication"
EventStreamSourceSlotPrefix = "fes" EventStreamSourceSlotPrefix = "fes"
EventStreamSourcePluginType = "pgoutput"
EventStreamSourceAuthType = "DatabaseAuthenticationSecret" EventStreamSourceAuthType = "DatabaseAuthenticationSecret"
EventStreamFlowPgGenericType = "PostgresWalToGenericNakadiEvent" EventStreamFlowPgGenericType = "PostgresWalToGenericNakadiEvent"
EventStreamSinkNakadiType = "Nakadi" EventStreamSinkNakadiType = "Nakadi"

6
pkg/util/patroni/patroni_test.go
View File

@ -185,7 +185,7 @@ func TestGetConfig(t *testing.T) {
Slots: map[string]map[string]string{ Slots: map[string]map[string]string{
"cdc": { "cdc": {
"database": "foo", "database": "foo",
"plugin": "pgoutput", "plugin": "wal2json",
"type": "logical", "type": "logical",
}, },
}, },
@ -218,7 +218,7 @@ func TestGetConfig(t *testing.T) {
"wal_log_hints": "on", "wal_log_hints": "on",
} }
configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 100, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "hot_standby", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "pgoutput", "type": "logical"}}, "ttl": 30}` configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 100, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "hot_standby", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "wal2json", "type": "logical"}}, "ttl": 30}`
r := ioutil.NopCloser(bytes.NewReader([]byte(configJson))) r := ioutil.NopCloser(bytes.NewReader([]byte(configJson)))
response := http.Response{ response := http.Response{
@ -253,7 +253,7 @@ func TestSetPostgresParameters(t *testing.T) {
"wal_level": "logical", "wal_level": "logical",
} }
configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 50, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "logical", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "pgoutput", "type": "logical"}}, "ttl": 30}` configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 50, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "logical", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "wal2json", "type": "logical"}}, "ttl": 30}`
r := ioutil.NopCloser(bytes.NewReader([]byte(configJson))) r := ioutil.NopCloser(bytes.NewReader([]byte(configJson)))
response := http.Response{ response := http.Response{