public_git/postgres-operator
1
0
Fork 0
mirror of https://github.com/zalando/postgres-operator.git synced 2026-10-04 23:41:31 +02:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into lazy-update

Browse Source
This commit is contained in:
Felix Kunde
2020-03-27 09:41:52 +01:00
committed by GitHub
parent d1830f3ab3 ba9cf68650
commit 8f88fe7338
58 changed files with 3696 additions and 222 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/util/config/config.go
+46 -25
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"github.com/zalando/postgres-operator/pkg/spec"
"github.com/zalando/postgres-operator/pkg/util/constants"
)
// CRD describes CustomResourceDefinition specific configuration parameters
@@ -21,31 +22,31 @@ type CRD struct {
// Resources describes kubernetes resource specific configuration parameters
type Resources struct {
ResourceCheckInterval time.Duration `name:"resource_check_interval" default:"3s"`
ResourceCheckTimeout time.Duration `name:"resource_check_timeout" default:"10m"`
PodLabelWaitTimeout time.Duration `name:"pod_label_wait_timeout" default:"10m"`
PodDeletionWaitTimeout time.Duration `name:"pod_deletion_wait_timeout" default:"10m"`
PodTerminateGracePeriod time.Duration `name:"pod_terminate_grace_period" default:"5m"`
SpiloFSGroup *int64 `name:"spilo_fsgroup"`
PodPriorityClassName string `name:"pod_priority_class_name"`
ClusterDomain string `name:"cluster_domain" default:"cluster.local"`
SpiloPrivileged bool `name:"spilo_privileged" default:"false"`
ClusterLabels map[string]string `name:"cluster_labels" default:"application:spilo"`
InheritedLabels []string `name:"inherited_labels" default:""`
ClusterNameLabel string `name:"cluster_name_label" default:"cluster-name"`
PodRoleLabel string `name:"pod_role_label" default:"spilo-role"`
PodToleration map[string]string `name:"toleration" default:""`
DefaultCPURequest string `name:"default_cpu_request" default:"100m"`
DefaultMemoryRequest string `name:"default_memory_request" default:"100Mi"`
DefaultCPULimit string `name:"default_cpu_limit" default:"1"`
DefaultMemoryLimit string `name:"default_memory_limit" default:"500Mi"`
MinCPULimit string `name:"min_cpu_limit" default:"250m"`
MinMemoryLimit string `name:"min_memory_limit" default:"250Mi"`
PodEnvironmentConfigMap string `name:"pod_environment_configmap" default:""`
NodeReadinessLabel map[string]string `name:"node_readiness_label" default:""`
MaxInstances int32 `name:"max_instances" default:"-1"`
MinInstances int32 `name:"min_instances" default:"-1"`
ShmVolume *bool `name:"enable_shm_volume" default:"true"`
ResourceCheckInterval time.Duration `name:"resource_check_interval" default:"3s"`
ResourceCheckTimeout time.Duration `name:"resource_check_timeout" default:"10m"`
PodLabelWaitTimeout time.Duration `name:"pod_label_wait_timeout" default:"10m"`
PodDeletionWaitTimeout time.Duration `name:"pod_deletion_wait_timeout" default:"10m"`
PodTerminateGracePeriod time.Duration `name:"pod_terminate_grace_period" default:"5m"`
SpiloFSGroup *int64 `name:"spilo_fsgroup"`
PodPriorityClassName string `name:"pod_priority_class_name"`
ClusterDomain string `name:"cluster_domain" default:"cluster.local"`
SpiloPrivileged bool `name:"spilo_privileged" default:"false"`
ClusterLabels map[string]string `name:"cluster_labels" default:"application:spilo"`
InheritedLabels []string `name:"inherited_labels" default:""`
ClusterNameLabel string `name:"cluster_name_label" default:"cluster-name"`
PodRoleLabel string `name:"pod_role_label" default:"spilo-role"`
PodToleration map[string]string `name:"toleration" default:""`
DefaultCPURequest string `name:"default_cpu_request" default:"100m"`
DefaultMemoryRequest string `name:"default_memory_request" default:"100Mi"`
DefaultCPULimit string `name:"default_cpu_limit" default:"1"`
DefaultMemoryLimit string `name:"default_memory_limit" default:"500Mi"`
MinCPULimit string `name:"min_cpu_limit" default:"250m"`
MinMemoryLimit string `name:"min_memory_limit" default:"250Mi"`
PodEnvironmentConfigMap spec.NamespacedName `name:"pod_environment_configmap"`
NodeReadinessLabel map[string]string `name:"node_readiness_label" default:""`
MaxInstances int32 `name:"max_instances" default:"-1"`
MinInstances int32 `name:"min_instances" default:"-1"`
ShmVolume *bool `name:"enable_shm_volume" default:"true"`
}
// Auth describes authentication specific configuration parameters
@@ -83,6 +84,20 @@ type LogicalBackup struct {
LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:"AES256"`
}
// Operator options for connection pooler
type ConnectionPool struct {
NumberOfInstances *int32 `name:"connection_pool_number_of_instances" default:"2"`
Schema string `name:"connection_pool_schema" default:"pooler"`
User string `name:"connection_pool_user" default:"pooler"`
Image string `name:"connection_pool_image" default:"registry.opensource.zalan.do/acid/pgbouncer"`
Mode string `name:"connection_pool_mode" default:"transaction"`
MaxDBConnections *int32 `name:"connection_pool_max_db_connections" default:"60"`
ConnPoolDefaultCPURequest string `name:"connection_pool_default_cpu_request" default:"500m"`
ConnPoolDefaultMemoryRequest string `name:"connection_pool_default_memory_request" default:"100Mi"`
ConnPoolDefaultCPULimit string `name:"connection_pool_default_cpu_limit" default:"1"`
ConnPoolDefaultMemoryLimit string `name:"connection_pool_default_memory_limit" default:"100Mi"`
}
// Config describes operator config
type Config struct {
CRD
@@ -90,6 +105,7 @@ type Config struct {
Auth
Scalyr
LogicalBackup
ConnectionPool
WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to'
EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS
@@ -197,5 +213,10 @@ func validate(cfg *Config) (err error) {
if cfg.Workers == 0 {
err = fmt.Errorf("number of workers should be higher than 0")
}
if *cfg.ConnectionPool.NumberOfInstances < constants.ConnPoolMinInstances {
msg := "number of connection pool instances should be higher than %d"
err = fmt.Errorf(msg, constants.ConnPoolMinInstances)
}
return
}
pkg/util/constants/annotations.go
+1
View File
@@ -7,4 +7,5 @@ const (
ElbTimeoutAnnotationValue = "3600"
KubeIAmAnnotation = "iam.amazonaws.com/role"
VolumeStorateProvisionerAnnotation = "pv.kubernetes.io/provisioned-by"
PostgresqlControllerAnnotationKey = "acid.zalan.do/controller"
)
pkg/util/constants/pooler.go
+18
View File
@@ -0,0 +1,18 @@
package constants
// Connection pool specific constants
const (
ConnectionPoolUserName = "pooler"
ConnectionPoolSchemaName = "pooler"
ConnectionPoolDefaultType = "pgbouncer"
ConnectionPoolDefaultMode = "transaction"
ConnectionPoolDefaultCpuRequest = "500m"
ConnectionPoolDefaultCpuLimit = "1"
ConnectionPoolDefaultMemoryRequest = "100Mi"
ConnectionPoolDefaultMemoryLimit = "100Mi"
ConnPoolContainer = 0
ConnPoolMaxDBConnections = 60
ConnPoolMaxClientConnections = 10000
ConnPoolMinInstances = 2
)
pkg/util/constants/roles.go
+12 -11
View File
@@ -2,15 +2,16 @@ package constants
// Roles specific constants
const (
PasswordLength = 64
SuperuserKeyName = "superuser"
ReplicationUserKeyName = "replication"
RoleFlagSuperuser = "SUPERUSER"
RoleFlagInherit = "INHERIT"
RoleFlagLogin = "LOGIN"
RoleFlagNoLogin = "NOLOGIN"
RoleFlagCreateRole = "CREATEROLE"
RoleFlagCreateDB = "CREATEDB"
RoleFlagReplication = "REPLICATION"
RoleFlagByPassRLS = "BYPASSRLS"
PasswordLength = 64
SuperuserKeyName = "superuser"
ConnectionPoolUserKeyName = "pooler"
ReplicationUserKeyName = "replication"
RoleFlagSuperuser = "SUPERUSER"
RoleFlagInherit = "INHERIT"
RoleFlagLogin = "LOGIN"
RoleFlagNoLogin = "NOLOGIN"
RoleFlagCreateRole = "CREATEROLE"
RoleFlagCreateDB = "CREATEDB"
RoleFlagReplication = "REPLICATION"
RoleFlagByPassRLS = "BYPASSRLS"
)
pkg/util/k8sutil/k8sutil.go
+166 -4
View File
@@ -9,11 +9,13 @@ import (
batchv1beta1 "k8s.io/api/batch/v1beta1"
clientbatchv1beta1 "k8s.io/client-go/kubernetes/typed/batch/v1beta1"
apiappsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
policybeta1 "k8s.io/api/policy/v1beta1"
apiextclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
apiextbeta1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
appsv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
@@ -26,6 +28,10 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func Int32ToPointer(value int32) *int32 {
return &value
}
// KubernetesClient describes getters for Kubernetes objects
type KubernetesClient struct {
corev1.SecretsGetter
@@ -39,6 +45,7 @@ type KubernetesClient struct {
corev1.NamespacesGetter
corev1.ServiceAccountsGetter
appsv1.StatefulSetsGetter
appsv1.DeploymentsGetter
rbacv1.RoleBindingsGetter
policyv1beta1.PodDisruptionBudgetsGetter
apiextbeta1.CustomResourceDefinitionsGetter
@@ -55,6 +62,34 @@ type mockSecret struct {
type MockSecretGetter struct {
}
type mockDeployment struct {
appsv1.DeploymentInterface
}
type mockDeploymentNotExist struct {
appsv1.DeploymentInterface
}
type MockDeploymentGetter struct {
}
type MockDeploymentNotExistGetter struct {
}
type mockService struct {
corev1.ServiceInterface
}
type mockServiceNotExist struct {
corev1.ServiceInterface
}
type MockServiceGetter struct {
}
type MockServiceNotExistGetter struct {
}
type mockConfigMap struct {
corev1.ConfigMapInterface
}
@@ -101,6 +136,7 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) {
kubeClient.NodesGetter = client.CoreV1()
kubeClient.NamespacesGetter = client.CoreV1()
kubeClient.StatefulSetsGetter = client.AppsV1()
kubeClient.DeploymentsGetter = client.AppsV1()
kubeClient.PodDisruptionBudgetsGetter = client.PolicyV1beta1()
kubeClient.RESTClient = client.CoreV1().RESTClient()
kubeClient.RoleBindingsGetter = client.RbacV1()
@@ -230,19 +266,145 @@ func (c *mockConfigMap) Get(name string, options metav1.GetOptions) (*v1.ConfigM
}
// Secrets to be mocked
func (c *MockSecretGetter) Secrets(namespace string) corev1.SecretInterface {
func (mock *MockSecretGetter) Secrets(namespace string) corev1.SecretInterface {
return &mockSecret{}
}
// ConfigMaps to be mocked
func (c *MockConfigMapsGetter) ConfigMaps(namespace string) corev1.ConfigMapInterface {
func (mock *MockConfigMapsGetter) ConfigMaps(namespace string) corev1.ConfigMapInterface {
return &mockConfigMap{}
}
func (mock *MockDeploymentGetter) Deployments(namespace string) appsv1.DeploymentInterface {
return &mockDeployment{}
}
func (mock *MockDeploymentNotExistGetter) Deployments(namespace string) appsv1.DeploymentInterface {
return &mockDeploymentNotExist{}
}
func (mock *mockDeployment) Create(*apiappsv1.Deployment) (*apiappsv1.Deployment, error) {
return &apiappsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "test-deployment",
},
Spec: apiappsv1.DeploymentSpec{
Replicas: Int32ToPointer(1),
},
}, nil
}
func (mock *mockDeployment) Delete(name string, opts *metav1.DeleteOptions) error {
return nil
}
func (mock *mockDeployment) Get(name string, opts metav1.GetOptions) (*apiappsv1.Deployment, error) {
return &apiappsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "test-deployment",
},
Spec: apiappsv1.DeploymentSpec{
Replicas: Int32ToPointer(1),
Template: v1.PodTemplateSpec{
Spec: v1.PodSpec{
Containers: []v1.Container{
v1.Container{
Image: "pooler:1.0",
},
},
},
},
},
}, nil
}
func (mock *mockDeployment) Patch(name string, t types.PatchType, data []byte, subres ...string) (*apiappsv1.Deployment, error) {
return &apiappsv1.Deployment{
Spec: apiappsv1.DeploymentSpec{
Replicas: Int32ToPointer(2),
},
ObjectMeta: metav1.ObjectMeta{
Name: "test-deployment",
},
}, nil
}
func (mock *mockDeploymentNotExist) Get(name string, opts metav1.GetOptions) (*apiappsv1.Deployment, error) {
return nil, &apierrors.StatusError{
ErrStatus: metav1.Status{
Reason: metav1.StatusReasonNotFound,
},
}
}
func (mock *mockDeploymentNotExist) Create(*apiappsv1.Deployment) (*apiappsv1.Deployment, error) {
return &apiappsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "test-deployment",
},
Spec: apiappsv1.DeploymentSpec{
Replicas: Int32ToPointer(1),
},
}, nil
}
func (mock *MockServiceGetter) Services(namespace string) corev1.ServiceInterface {
return &mockService{}
}
func (mock *MockServiceNotExistGetter) Services(namespace string) corev1.ServiceInterface {
return &mockServiceNotExist{}
}
func (mock *mockService) Create(*v1.Service) (*v1.Service, error) {
return &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test-service",
},
}, nil
}
func (mock *mockService) Delete(name string, opts *metav1.DeleteOptions) error {
return nil
}
func (mock *mockService) Get(name string, opts metav1.GetOptions) (*v1.Service, error) {
return &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test-service",
},
}, nil
}
func (mock *mockServiceNotExist) Create(*v1.Service) (*v1.Service, error) {
return &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test-service",
},
}, nil
}
func (mock *mockServiceNotExist) Get(name string, opts metav1.GetOptions) (*v1.Service, error) {
return nil, &apierrors.StatusError{
ErrStatus: metav1.Status{
Reason: metav1.StatusReasonNotFound,
},
}
}
// NewMockKubernetesClient for other tests
func NewMockKubernetesClient() KubernetesClient {
return KubernetesClient{
SecretsGetter: &MockSecretGetter{},
ConfigMapsGetter: &MockConfigMapsGetter{},
SecretsGetter: &MockSecretGetter{},
ConfigMapsGetter: &MockConfigMapsGetter{},
DeploymentsGetter: &MockDeploymentGetter{},
ServicesGetter: &MockServiceGetter{},
}
}
func ClientMissingObjects() KubernetesClient {
return KubernetesClient{
DeploymentsGetter: &MockDeploymentNotExistGetter{},
ServicesGetter: &MockServiceNotExistGetter{},
}
}
pkg/util/util.go
+43 -3
View File
@@ -2,8 +2,10 @@ package util
import (
"crypto/md5" // #nosec we need it to for PostgreSQL md5 passwords
cryptoRand "crypto/rand"
"encoding/hex"
"fmt"
"math/big"
"math/rand"
"regexp"
"strings"
@@ -37,13 +39,17 @@ func False() *bool {
return &b
}
// RandomPassword generates random alphanumeric password of a given length.
// RandomPassword generates a secure, random alphanumeric password of a given length.
func RandomPassword(n int) string {
b := make([]byte, n)
for i := range b {
b[i] = passwordChars[rand.Intn(len(passwordChars))]
maxN := big.NewInt(int64(len(passwordChars)))
if n, err := cryptoRand.Int(cryptoRand.Reader, maxN); err != nil {
panic(fmt.Errorf("Unable to generate secure, random password: %v", err))
} else {
b[i] = passwordChars[n.Int64()]
}
}
return string(b)
}
@@ -141,6 +147,40 @@ func Coalesce(val, defaultVal string) string {
return val
}
// Yeah, golang
func CoalesceInt32(val, defaultVal *int32) *int32 {
if val == nil {
return defaultVal
}
return val
}
// Test if any of the values is nil
func testNil(values ...*int32) bool {
for _, v := range values {
if v == nil {
return true
}
}
return false
}
// Return maximum of two integers provided via pointers. If one value is not
// defined, return the other one. If both are not defined, result is also
// undefined, caller needs to check for that.
func MaxInt32(a, b *int32) *int32 {
if testNil(a, b) {
return nil
}
if *a > *b {
return a
}
return b
}
// IsSmallerQuantity : checks if first resource is of a smaller quantity than the second
func IsSmallerQuantity(requestStr, limitStr string) (bool, error) {