public_git/postgres-operator
1
0
Fork 0
mirror of https://github.com/zalando/postgres-operator.git synced 2026-10-07 17:39:34 +02:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into feature/actions

Browse Source
This commit is contained in:
erthalion
2018-08-30 16:51:28 +02:00
parent 332db3bbda 25fa45fd58
commit 87eee1ed72
91 changed files with 5275 additions and 1561 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/util/config/config.go
+26 -20
View File
@@ -14,7 +14,8 @@ import (
type CRD struct {
ReadyWaitInterval time.Duration `name:"ready_wait_interval" default:"4s"`
ReadyWaitTimeout time.Duration `name:"ready_wait_timeout" default:"30s"`
ResyncPeriod time.Duration `name:"resync_period" default:"5m"`
ResyncPeriod time.Duration `name:"resync_period" default:"30m"`
RepairPeriod time.Duration `name:"repair_period" default:"5m"`
}
// Resources describes kubernetes resource specific configuration parameters
@@ -24,6 +25,7 @@ type Resources struct {
PodLabelWaitTimeout time.Duration `name:"pod_label_wait_timeout" default:"10m"`
PodDeletionWaitTimeout time.Duration `name:"pod_deletion_wait_timeout" default:"10m"`
PodTerminateGracePeriod time.Duration `name:"pod_terminate_grace_period" default:"5m"`
PodPriorityClassName string `name:"pod_priority_class_name"`
ClusterLabels map[string]string `name:"cluster_labels" default:"application:spilo"`
ClusterNameLabel string `name:"cluster_name_label" default:"cluster-name"`
PodRoleLabel string `name:"pod_role_label" default:"spilo-role"`
@@ -40,7 +42,7 @@ type Resources struct {
// Auth describes authentication specific configuration parameters
type Auth struct {
SecretNameTemplate stringTemplate `name:"secret_name_template" default:"{username}.{cluster}.credentials.{tprkind}.{tprgroup}"`
SecretNameTemplate StringTemplate `name:"secret_name_template" default:"{username}.{cluster}.credentials.{tprkind}.{tprgroup}"`
PamRoleName string `name:"pam_role_name" default:"zalandos"`
PamConfiguration string `name:"pam_configuration" default:"https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees"`
TeamsAPIUrl string `name:"teams_api_url" default:"https://teams.example.com/api/"`
@@ -68,29 +70,32 @@ type Config struct {
Auth
Scalyr
WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to'
EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use k8s as a DCS
DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-cdp-10:1.4-p8"`
WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to'
EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use k8s as a DCS
DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-cdp-10:1.4-p8"`
Sidecars map[string]string `name:"sidecar_docker_images"`
// default name `operator` enables backward compatibility with the older ServiceAccountName field
PodServiceAccountName string `name:"pod_service_account_name" default:"operator"`
// value of this string must be valid JSON or YAML; see initPodServiceAccount
PodServiceAccountDefinition string `name:"pod_service_account_definition" default:""`
DbHostedZone string `name:"db_hosted_zone" default:"db.example.com"`
WALES3Bucket string `name:"wal_s3_bucket"`
LogS3Bucket string `name:"log_s3_bucket"`
KubeIAMRole string `name:"kube_iam_role"`
DebugLogging bool `name:"debug_logging" default:"true"`
EnableDBAccess bool `name:"enable_database_access" default:"true"`
EnableTeamsAPI bool `name:"enable_teams_api" default:"true"`
EnableTeamSuperuser bool `name:"enable_team_superuser" default:"false"`
TeamAdminRole string `name:"team_admin_role" default:"admin"`
EnableMasterLoadBalancer bool `name:"enable_master_load_balancer" default:"true"`
EnableReplicaLoadBalancer bool `name:"enable_replica_load_balancer" default:"false"`
PodServiceAccountDefinition string `name:"pod_service_account_definition" default:""`
PodServiceAccountRoleBindingDefinition string `name:"pod_service_account_role_binding_definition" default:""`
DbHostedZone string `name:"db_hosted_zone" default:"db.example.com"`
AWSRegion string `name:"aws_region" default:"eu-central-1"`
WALES3Bucket string `name:"wal_s3_bucket"`
LogS3Bucket string `name:"log_s3_bucket"`
KubeIAMRole string `name:"kube_iam_role"`
DebugLogging bool `name:"debug_logging" default:"true"`
EnableDBAccess bool `name:"enable_database_access" default:"true"`
EnableTeamsAPI bool `name:"enable_teams_api" default:"true"`
EnableTeamSuperuser bool `name:"enable_team_superuser" default:"false"`
TeamAdminRole string `name:"team_admin_role" default:"admin"`
EnableMasterLoadBalancer bool `name:"enable_master_load_balancer" default:"true"`
EnableReplicaLoadBalancer bool `name:"enable_replica_load_balancer" default:"false"`
// deprecated and kept for backward compatibility
EnableLoadBalancer *bool `name:"enable_load_balancer"`
MasterDNSNameFormat stringTemplate `name:"master_dns_name_format" default:"{cluster}.{team}.{hostedzone}"`
ReplicaDNSNameFormat stringTemplate `name:"replica_dns_name_format" default:"{cluster}-repl.{team}.{hostedzone}"`
PDBNameFormat stringTemplate `name:"pdb_name_format" default:"postgres-{cluster}-pdb"`
MasterDNSNameFormat StringTemplate `name:"master_dns_name_format" default:"{cluster}.{team}.{hostedzone}"`
ReplicaDNSNameFormat StringTemplate `name:"replica_dns_name_format" default:"{cluster}-repl.{team}.{hostedzone}"`
PDBNameFormat StringTemplate `name:"pdb_name_format" default:"postgres-{cluster}-pdb"`
Workers uint32 `name:"workers" default:"4"`
APIPort int `name:"api_port" default:"8080"`
RingLogLines int `name:"ring_log_lines" default:"100"`
@@ -98,6 +103,7 @@ type Config struct {
TeamAPIRoleConfiguration map[string]string `name:"team_api_role_configuration" default:"log_statement:all"`
PodTerminateGracePeriod time.Duration `name:"pod_terminate_grace_period" default:"5m"`
ProtectedRoles []string `name:"protected_role_names" default:"admin"`
PostgresSuperuserTeams []string `name:"postgres_superuser_teams" default:""`
}
// MustMarshal marshals the config or panics
pkg/util/config/util.go
+19 -20
View File
@@ -19,7 +19,7 @@ type fieldInfo struct {
Field reflect.Value
}
type stringTemplate string
type StringTemplate string
func decoderFrom(field reflect.Value) (d decoder) {
// it may be impossible for a struct field to fail this check
@@ -172,10 +172,9 @@ func processField(value string, field reflect.Value) error {
type parserState int
const (
Plain parserState = iota
DoubleQuoted
SingleQuoted
Escape
plain parserState = iota
doubleQuoted
singleQuoted
)
// Split the pair candidates by commas not located inside open quotes
@@ -183,7 +182,7 @@ const (
// expect to find them inside the map values for our use cases
func getMapPairsFromString(value string) (pairs []string, err error) {
pairs = make([]string, 0)
state := Plain
state := plain
var start, quote int
for i, ch := range strings.Split(value, "") {
@@ -191,29 +190,29 @@ func getMapPairsFromString(value string) (pairs []string, err error) {
fmt.Printf("Parser warning: ecape character '\\' have no effect on quotes inside the configuration value %s\n", value)
}
if ch == `"` {
if state == Plain {
state = DoubleQuoted
if state == plain {
state = doubleQuoted
quote = i
} else if state == DoubleQuoted {
state = Plain
} else if state == doubleQuoted {
state = plain
quote = 0
}
}
if ch == "'" {
if state == Plain {
state = SingleQuoted
if state == plain {
state = singleQuoted
quote = i
} else if state == SingleQuoted {
state = Plain
} else if state == singleQuoted {
state = plain
quote = 0
}
}
if ch == "," && state == Plain {
if ch == "," && state == plain {
pairs = append(pairs, strings.Trim(value[start:i], " \t"))
start = i + 1
}
}
if state != Plain {
if state != plain {
err = fmt.Errorf("unmatched quote starting at position %d", quote+1)
pairs = nil
} else {
@@ -222,13 +221,13 @@ func getMapPairsFromString(value string) (pairs []string, err error) {
return
}
func (f *stringTemplate) Decode(value string) error {
*f = stringTemplate(value)
func (f *StringTemplate) Decode(value string) error {
*f = StringTemplate(value)
return nil
}
func (f *stringTemplate) Format(a ...string) string {
func (f *StringTemplate) Format(a ...string) string {
res := string(*f)
for i := 0; i < len(a); i += 2 {
@@ -238,6 +237,6 @@ func (f *stringTemplate) Format(a ...string) string {
return res
}
func (f stringTemplate) MarshalJSON() ([]byte, error) {
func (f StringTemplate) MarshalJSON() ([]byte, error) {
return json.Marshal(string(f))
}
pkg/util/constants/aws.go
-2
View File
@@ -4,8 +4,6 @@ import "time"
// AWS specific constants used by other modules
const (
// default region for AWS. TODO: move it to the operator configuration
AWSRegion = "eu-central-1"
// EBS related constants
EBSVolumeIDStart = "/vol-"
EBSProvisioner = "kubernetes.io/aws-ebs"
pkg/util/constants/crd.go
-10
View File
@@ -1,10 +0,0 @@
package constants
// Different properties of the PostgreSQL Custom Resource Definition
const (
CRDKind = "postgresql"
CRDResource = "postgresqls"
CRDShort = "pg"
CRDGroup = "acid.zalan.do"
CRDApiVersion = "v1"
)
pkg/util/k8sutil/k8sutil.go
+11 -24
View File
@@ -2,24 +2,22 @@ package k8sutil
import (
"fmt"
"reflect"
"github.com/zalando-incubator/postgres-operator/pkg/util/constants"
"k8s.io/api/core/v1"
policybeta1 "k8s.io/api/policy/v1beta1"
apiextclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
apiextbeta1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/typed/apps/v1beta1"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
policyv1beta1 "k8s.io/client-go/kubernetes/typed/policy/v1beta1"
"k8s.io/client-go/pkg/api"
"k8s.io/client-go/pkg/api/v1"
policybeta1 "k8s.io/client-go/pkg/apis/policy/v1beta1"
rbacv1beta1 "k8s.io/client-go/kubernetes/typed/rbac/v1beta1"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"reflect"
"github.com/zalando-incubator/postgres-operator/pkg/util/constants"
acidv1client "github.com/zalando-incubator/postgres-operator/pkg/generated/clientset/versioned"
)
// KubernetesClient describes getters for Kubernetes objects
@@ -35,11 +33,12 @@ type KubernetesClient struct {
v1core.NamespacesGetter
v1core.ServiceAccountsGetter
v1beta1.StatefulSetsGetter
rbacv1beta1.RoleBindingsGetter
policyv1beta1.PodDisruptionBudgetsGetter
apiextbeta1.CustomResourceDefinitionsGetter
RESTClient rest.Interface
CRDREST rest.Interface
RESTClient rest.Interface
AcidV1ClientSet *acidv1client.Clientset
}
// RestConfig creates REST config
@@ -83,20 +82,7 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) {
kubeClient.StatefulSetsGetter = client.AppsV1beta1()
kubeClient.PodDisruptionBudgetsGetter = client.PolicyV1beta1()
kubeClient.RESTClient = client.CoreV1().RESTClient()
cfg2 := *cfg
cfg2.GroupVersion = &schema.GroupVersion{
Group: constants.CRDGroup,
Version: constants.CRDApiVersion,
}
cfg2.APIPath = constants.K8sAPIPath
cfg2.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: api.Codecs}
crd, err := rest.RESTClientFor(&cfg2)
if err != nil {
return kubeClient, fmt.Errorf("could not get rest client: %v", err)
}
kubeClient.CRDREST = crd
kubeClient.RoleBindingsGetter = client.RbacV1beta1()
apiextClient, err := apiextclient.NewForConfig(cfg)
if err != nil {
@@ -104,6 +90,7 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) {
}
kubeClient.CustomResourceDefinitionsGetter = apiextClient.ApiextensionsV1beta1()
kubeClient.AcidV1ClientSet = acidv1client.NewForConfigOrDie(cfg)
return kubeClient, nil
}
pkg/util/patroni/patroni.go
+14 -6
View File
@@ -9,7 +9,7 @@ import (
"time"
"github.com/Sirupsen/logrus"
"k8s.io/client-go/pkg/api/v1"
"k8s.io/api/core/v1"
)
const (
@@ -47,7 +47,7 @@ func apiURL(masterPod *v1.Pod) string {
return fmt.Sprintf("http://%s:%d", masterPod.Status.PodIP, apiPort)
}
func (p *Patroni) httpPostOrPatch(method string, url string, body *bytes.Buffer) error {
func (p *Patroni) httpPostOrPatch(method string, url string, body *bytes.Buffer) (err error) {
request, err := http.NewRequest(method, url, body)
if err != nil {
return fmt.Errorf("could not create request: %v", err)
@@ -59,7 +59,16 @@ func (p *Patroni) httpPostOrPatch(method string, url string, body *bytes.Buffer)
if err != nil {
return fmt.Errorf("could not make request: %v", err)
}
defer resp.Body.Close()
defer func() {
if err2 := resp.Body.Close(); err2 != nil {
if err != nil {
err = fmt.Errorf("could not close request: %v, prior error: %v", err2, err)
} else {
err = fmt.Errorf("could not close request: %v", err2)
}
return
}
}()
if resp.StatusCode != http.StatusOK {
bodyBytes, err := ioutil.ReadAll(resp.Body)
@@ -80,12 +89,11 @@ func (p *Patroni) Switchover(master *v1.Pod, candidate string) error {
return fmt.Errorf("could not encode json: %v", err)
}
return p.httpPostOrPatch(http.MethodPost, apiURL(master)+failoverPath, buf)
return nil
}
//TODO: add an option call /patroni to check if it is necessary to restart the server
// SetPostgresParameters sets Postgres options via Patroni patch API call.
//SetPostgresParameters sets Postgres options via Patroni patch API call.
func (p *Patroni) SetPostgresParameters(server *v1.Pod, parameters map[string]string) error {
buf := &bytes.Buffer{}
err := json.NewEncoder(buf).Encode(map[string]map[string]interface{}{"postgresql": {"parameters": parameters}})
pkg/util/teams/teams.go
+10 -19
View File
@@ -76,17 +76,15 @@ func (t *API) TeamInfo(teamID, token string) (tm *Team, err error) {
t.logger.Debugf("request url: %s", url)
req, err = http.NewRequest("GET", url, nil)
if err != nil {
return
return nil, err
}
req.Header.Add("Authorization", "Bearer "+token)
resp, err = t.httpClient.Do(req)
if err != nil {
return
if resp, err = t.httpClient.Do(req); err != nil {
return nil, err
}
defer func() {
closeErr := resp.Body.Close()
if closeErr != nil {
if closeErr := resp.Body.Close(); closeErr != nil {
err = fmt.Errorf("error when closing response: %v", closeErr)
}
}()
@@ -95,27 +93,20 @@ func (t *API) TeamInfo(teamID, token string) (tm *Team, err error) {
d := json.NewDecoder(resp.Body)
err = d.Decode(&raw)
if err != nil {
err = fmt.Errorf("team API query failed with status code %d and malformed response: %v", resp.StatusCode, err)
return
return nil, fmt.Errorf("team API query failed with status code %d and malformed response: %v", resp.StatusCode, err)
}
if errMessage, ok := raw["error"]; ok {
err = fmt.Errorf("team API query failed with status code %d and message: '%v'", resp.StatusCode, string(errMessage))
return
return nil, fmt.Errorf("team API query failed with status code %d and message: '%v'", resp.StatusCode, string(errMessage))
}
err = fmt.Errorf("team API query failed with status code %d", resp.StatusCode)
return
return nil, fmt.Errorf("team API query failed with status code %d", resp.StatusCode)
}
tm = &Team{}
d := json.NewDecoder(resp.Body)
err = d.Decode(tm)
if err != nil {
err = fmt.Errorf("could not parse team API response: %v", err)
tm = nil
return
if err = d.Decode(tm); err != nil {
return nil, fmt.Errorf("could not parse team API response: %v", err)
}
return
return tm, nil
}
pkg/util/users/users.go
+17 -21
View File
@@ -30,8 +30,9 @@ type DefaultUserSyncStrategy struct {
// ProduceSyncRequests figures out the types of changes that need to happen with the given users.
func (strategy DefaultUserSyncStrategy) ProduceSyncRequests(dbUsers spec.PgUserMap,
newUsers spec.PgUserMap) (reqs []spec.PgSyncUserRequest) {
newUsers spec.PgUserMap) []spec.PgSyncUserRequest {
var reqs []spec.PgSyncUserRequest
// No existing roles are deleted or stripped of role memebership/flags
for name, newUser := range newUsers {
dbUser, exists := dbUsers[name]
@@ -66,7 +67,7 @@ func (strategy DefaultUserSyncStrategy) ProduceSyncRequests(dbUsers spec.PgUserM
}
}
return
return reqs
}
// ExecuteSyncRequests makes actual database changes from the requests passed in its arguments.
@@ -102,7 +103,7 @@ func (strategy DefaultUserSyncStrategy) alterPgUserSet(user spec.PgUser, db *sql
return
}
func (strategy DefaultUserSyncStrategy) createPgUser(user spec.PgUser, db *sql.DB) (err error) {
func (strategy DefaultUserSyncStrategy) createPgUser(user spec.PgUser, db *sql.DB) error {
var userFlags []string
var userPassword string
@@ -120,16 +121,14 @@ func (strategy DefaultUserSyncStrategy) createPgUser(user spec.PgUser, db *sql.D
}
query := fmt.Sprintf(createUserSQL, user.Name, strings.Join(userFlags, " "), userPassword)
_, err = db.Exec(query) // TODO: Try several times
if err != nil {
err = fmt.Errorf("dB error: %v, query: %s", err, query)
return
if _, err := db.Exec(query); err != nil { // TODO: Try several times
return fmt.Errorf("dB error: %v, query: %s", err, query)
}
return
return nil
}
func (strategy DefaultUserSyncStrategy) alterPgUser(user spec.PgUser, db *sql.DB) (err error) {
func (strategy DefaultUserSyncStrategy) alterPgUser(user spec.PgUser, db *sql.DB) error {
var resultStmt []string
if user.Password != "" || len(user.Flags) > 0 {
@@ -140,19 +139,16 @@ func (strategy DefaultUserSyncStrategy) alterPgUser(user spec.PgUser, db *sql.DB
grantStmt := produceGrantStmt(user)
resultStmt = append(resultStmt, grantStmt)
}
if len(resultStmt) == 0 {
return nil
if len(resultStmt) > 0 {
query := fmt.Sprintf(doBlockStmt, strings.Join(resultStmt, ";"))
if _, err := db.Exec(query); err != nil { // TODO: Try several times
return fmt.Errorf("dB error: %v query %s", err, query)
}
}
query := fmt.Sprintf(doBlockStmt, strings.Join(resultStmt, ";"))
_, err = db.Exec(query) // TODO: Try several times
if err != nil {
err = fmt.Errorf("dB error: %v query %s", err, query)
return
}
return
return nil
}
func produceAlterStmt(user spec.PgUser) string {
@@ -205,7 +201,7 @@ func quoteParameterValue(name, val string) string {
// containing spaces (but something more complex, like double quotes inside double quotes or spaces
// in the schema name would break the parsing code in the operator.)
if start == '\'' && end == '\'' {
return fmt.Sprintf("%s", val[1:len(val)-1])
return val[1 : len(val)-1]
}
return val
pkg/util/util.go
+3 -2
View File
@@ -1,7 +1,7 @@
package util
import (
"crypto/md5"
"crypto/md5" // #nosec we need it to for PostgreSQL md5 passwords
"encoding/hex"
"math/rand"
"regexp"
@@ -48,7 +48,7 @@ func PGUserPassword(user spec.PgUser) string {
// Avoid processing already encrypted or empty passwords
return user.Password
}
s := md5.Sum([]byte(user.Password + user.Name))
s := md5.Sum([]byte(user.Password + user.Name)) // #nosec, using md5 since PostgreSQL uses it for hashing passwords.
return md5prefix + hex.EncodeToString(s[:])
}
@@ -120,6 +120,7 @@ func MapContains(haystack, needle map[string]string) bool {
return true
}
// Coalesce returns the first argument if it is not null, otherwise the second one.
func Coalesce(val, defaultVal string) string {
if val == "" {
return defaultVal
pkg/util/volumes/ebs.go
+3 -2
View File
@@ -7,7 +7,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"k8s.io/client-go/pkg/api/v1"
"k8s.io/api/core/v1"
"github.com/zalando-incubator/postgres-operator/pkg/util/constants"
"github.com/zalando-incubator/postgres-operator/pkg/util/retryutil"
@@ -16,11 +16,12 @@ import (
// EBSVolumeResizer implements volume resizing interface for AWS EBS volumes.
type EBSVolumeResizer struct {
connection *ec2.EC2
AWSRegion string
}
// ConnectToProvider connects to AWS.
func (c *EBSVolumeResizer) ConnectToProvider() error {
sess, err := session.NewSession(&aws.Config{Region: aws.String(constants.AWSRegion)})
sess, err := session.NewSession(&aws.Config{Region: aws.String(c.AWSRegion)})
if err != nil {
return fmt.Errorf("could not establish AWS session: %v", err)
}
pkg/util/volumes/volumes.go
+1 -1
View File
@@ -1,7 +1,7 @@
package volumes
import (
"k8s.io/client-go/pkg/api/v1"
"k8s.io/api/core/v1"
)
// VolumeResizer defines the set of methods used to implememnt provider-specific resizing of persistent volumes.