public_git/postgres-operator
1
0
Fork 0
mirror of https://github.com/zalando/postgres-operator.git synced 2026-10-02 20:22:00 +02:00
Code Issues Projects Releases Wiki Activity

standby cluster that streams from a remote primary (#1830)

Browse Source 
* add the possibility to create a standby cluster that streams from a remote primary
* extending unit tests
* add more docs and e2e test

Co-authored-by: machine424 <ayoubmrini424@gmail.com>
This commit is contained in:
Felix Kunde
2022-04-04 15:41:11 +02:00
committed by GitHub
co-authored by machine424
parent 2dfb11ad4c
commit 0dc370f15d
12 changed files with 303 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/apis/acid.zalan.do/v1/crds.go
+11
View File
@@ -714,6 +714,17 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
"gs_wal_path": {
Type: "string",
},
"standby_host": {
Type: "string",
},
"standby_port": {
Type: "string",
},
},
OneOf: []apiextv1.JSONSchemaProps{
apiextv1.JSONSchemaProps{Required: []string{"s3_wal_path"}},
apiextv1.JSONSchemaProps{Required: []string{"gs_wal_path"}},
apiextv1.JSONSchemaProps{Required: []string{"standby_host"}},
},
},
"streams": {
pkg/apis/acid.zalan.do/v1/postgresql_type.go
+5 -3
View File
@@ -170,10 +170,12 @@ type Patroni struct {
SynchronousNodeCount uint32 `json:"synchronous_node_count,omitempty" defaults:"1"`
}
// StandbyDescription contains s3 wal path
// StandbyDescription contains remote primary config or s3/gs wal path
type StandbyDescription struct {
S3WalPath string `json:"s3_wal_path,omitempty"`
GSWalPath string `json:"gs_wal_path,omitempty"`
S3WalPath string `json:"s3_wal_path,omitempty"`
GSWalPath string `json:"gs_wal_path,omitempty"`
StandbyHost string `json:"standby_host,omitempty"`
StandbyPort string `json:"standby_port,omitempty"`
}
// TLSDescription specs TLS properties
pkg/cluster/k8sres.go
+43 -34
View File
@@ -763,7 +763,12 @@ func (c *Cluster) generatePodTemplate(
}
// generatePodEnvVars generates environment variables for the Spilo Pod
func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration string, cloneDescription *acidv1.CloneDescription, standbyDescription *acidv1.StandbyDescription, customPodEnvVarsList []v1.EnvVar) []v1.EnvVar {
func (c *Cluster) generateSpiloPodEnvVars(
uid types.UID,
spiloConfiguration string,
cloneDescription *acidv1.CloneDescription,
standbyDescription *acidv1.StandbyDescription,
customPodEnvVarsList []v1.EnvVar) []v1.EnvVar {
envVars := []v1.EnvVar{
{
Name: "SCOPE",
@@ -1128,11 +1133,6 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
sort.Slice(customPodEnvVarsList,
func(i, j int) bool { return customPodEnvVarsList[i].Name < customPodEnvVarsList[j].Name })
if spec.StandbyCluster != nil && spec.StandbyCluster.S3WalPath == "" &&
spec.StandbyCluster.GSWalPath == "" {
return nil, fmt.Errorf("one of s3_wal_path or gs_wal_path must be set for standby cluster")
}
// backward compatible check for InitContainers
if spec.InitContainersOld != nil {
msg := "manifest parameter init_containers is deprecated."
@@ -1944,40 +1944,49 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription)
func (c *Cluster) generateStandbyEnvironment(description *acidv1.StandbyDescription) []v1.EnvVar {
result := make([]v1.EnvVar, 0)
if description.S3WalPath == "" && description.GSWalPath == "" {
return nil
}
if description.S3WalPath != "" {
// standby with S3, find out the bucket to setup standby
msg := "standby from S3 bucket using custom parsed S3WalPath from the manifest %s "
c.logger.Infof(msg, description.S3WalPath)
if description.StandbyHost != "" {
// standby from remote primary
result = append(result, v1.EnvVar{
Name: "STANDBY_WALE_S3_PREFIX",
Value: description.S3WalPath,
Name: "STANDBY_HOST",
Value: description.StandbyHost,
})
} else if description.GSWalPath != "" {
msg := "standby from GS bucket using custom parsed GSWalPath from the manifest %s "
c.logger.Infof(msg, description.GSWalPath)
envs := []v1.EnvVar{
{
Name: "STANDBY_WALE_GS_PREFIX",
Value: description.GSWalPath,
},
{
Name: "STANDBY_GOOGLE_APPLICATION_CREDENTIALS",
Value: c.OpConfig.GCPCredentials,
},
if description.StandbyPort != "" {
result = append(result, v1.EnvVar{
Name: "STANDBY_PORT",
Value: description.StandbyPort,
})
}
result = append(result, envs...)
} else {
if description.S3WalPath != "" {
// standby with S3, find out the bucket to setup standby
msg := "Standby from S3 bucket using custom parsed S3WalPath from the manifest %s "
c.logger.Infof(msg, description.S3WalPath)
result = append(result, v1.EnvVar{
Name: "STANDBY_WALE_S3_PREFIX",
Value: description.S3WalPath,
})
} else if description.GSWalPath != "" {
msg := "Standby from GS bucket using custom parsed GSWalPath from the manifest %s "
c.logger.Infof(msg, description.GSWalPath)
envs := []v1.EnvVar{
{
Name: "STANDBY_WALE_GS_PREFIX",
Value: description.GSWalPath,
},
{
Name: "STANDBY_GOOGLE_APPLICATION_CREDENTIALS",
Value: c.OpConfig.GCPCredentials,
},
}
result = append(result, envs...)
}
result = append(result, v1.EnvVar{Name: "STANDBY_METHOD", Value: "STANDBY_WITH_WALE"})
result = append(result, v1.EnvVar{Name: "STANDBY_WAL_BUCKET_SCOPE_PREFIX", Value: ""})
}
result = append(result, v1.EnvVar{Name: "STANDBY_METHOD", Value: "STANDBY_WITH_WALE"})
result = append(result, v1.EnvVar{Name: "STANDBY_WAL_BUCKET_SCOPE_PREFIX", Value: ""})
return result
}
pkg/cluster/k8sres_test.go
+111
View File
@@ -531,6 +531,117 @@ func TestCloneEnv(t *testing.T) {
}
}
func TestStandbyEnv(t *testing.T) {
testName := "TestStandbyEnv"
tests := []struct {
subTest string
standbyOpts *acidv1.StandbyDescription
env v1.EnvVar
envPos int
envLen int
}{
{
subTest: "from custom s3 path",
standbyOpts: &acidv1.StandbyDescription{
S3WalPath: "s3://some/path/",
},
env: v1.EnvVar{
Name: "STANDBY_WALE_S3_PREFIX",
Value: "s3://some/path/",
},
envPos: 0,
envLen: 3,
},
{
subTest: "from custom gs path",
standbyOpts: &acidv1.StandbyDescription{
GSWalPath: "gs://some/path/",
},
env: v1.EnvVar{
Name: "STANDBY_GOOGLE_APPLICATION_CREDENTIALS",
Value: "",
},
envPos: 1,
envLen: 4,
},
{
subTest: "ignore gs path if s3 is set",
standbyOpts: &acidv1.StandbyDescription{
S3WalPath: "s3://some/path/",
GSWalPath: "gs://some/path/",
},
env: v1.EnvVar{
Name: "STANDBY_METHOD",
Value: "STANDBY_WITH_WALE",
},
envPos: 1,
envLen: 3,
},
{
subTest: "from remote primary",
standbyOpts: &acidv1.StandbyDescription{
StandbyHost: "remote-primary",
},
env: v1.EnvVar{
Name: "STANDBY_HOST",
Value: "remote-primary",
},
envPos: 0,
envLen: 1,
},
{
subTest: "from remote primary with port",
standbyOpts: &acidv1.StandbyDescription{
StandbyHost: "remote-primary",
StandbyPort: "9876",
},
env: v1.EnvVar{
Name: "STANDBY_PORT",
Value: "9876",
},
envPos: 1,
envLen: 2,
},
{
subTest: "from remote primary - ignore WAL path",
standbyOpts: &acidv1.StandbyDescription{
GSWalPath: "gs://some/path/",
StandbyHost: "remote-primary",
},
env: v1.EnvVar{
Name: "STANDBY_HOST",
Value: "remote-primary",
},
envPos: 0,
envLen: 1,
},
}
var cluster = New(
Config{}, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder)
for _, tt := range tests {
envs := cluster.generateStandbyEnvironment(tt.standbyOpts)
env := envs[tt.envPos]
if env.Name != tt.env.Name {
t.Errorf("%s %s: Expected env name %s, have %s instead",
testName, tt.subTest, tt.env.Name, env.Name)
}
if env.Value != tt.env.Value {
t.Errorf("%s %s: Expected env value %s, have %s instead",
testName, tt.subTest, tt.env.Value, env.Value)
}
if len(envs) != tt.envLen {
t.Errorf("%s %s: Expected number of env variables %d, have %d instead",
testName, tt.subTest, tt.envLen, len(envs))
}
}
}
func TestExtractPgVersionFromBinPath(t *testing.T) {
testName := "TestExtractPgVersionFromBinPath"
tests := []struct {