mirror of
https://github.com/zalando/postgres-operator.git
synced 2026-09-30 14:47:34 +02:00
Pod Rolling update
introduce Pod events channel; add parsing of the MaintenanceWindows section; skip deleting Etcd key on cluster delete; use external etcd host; watch for tpr/pods in the namespace of the operator pod only;
This commit is contained in:
Murat Kabilov
+224
-296
@@ -3,32 +3,32 @@ package cluster
|
||||
// Postgres ThirdPartyResource object i.e. Spilo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
etcdclient "github.com/coreos/etcd/client"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/pkg/api"
|
||||
"k8s.io/client-go/pkg/api/v1"
|
||||
"k8s.io/client-go/pkg/labels"
|
||||
"k8s.io/client-go/pkg/apis/apps/v1beta1"
|
||||
"k8s.io/client-go/pkg/types"
|
||||
"k8s.io/client-go/rest"
|
||||
|
||||
"github.bus.zalan.do/acid/postgres-operator/pkg/spec"
|
||||
"github.bus.zalan.do/acid/postgres-operator/pkg/util"
|
||||
"github.bus.zalan.do/acid/postgres-operator/pkg/util/constants"
|
||||
"github.bus.zalan.do/acid/postgres-operator/pkg/util/retryutil"
|
||||
"github.bus.zalan.do/acid/postgres-operator/pkg/util/k8sutil"
|
||||
"github.bus.zalan.do/acid/postgres-operator/pkg/util/resources"
|
||||
"github.bus.zalan.do/acid/postgres-operator/pkg/util/teams"
|
||||
)
|
||||
|
||||
var (
|
||||
superuserName = "postgres"
|
||||
replicationUsername = "replication"
|
||||
|
||||
alphaNumericRegexp = regexp.MustCompile("^[a-zA-Z0-9]*$")
|
||||
alphaNumericRegexp = regexp.MustCompile("^[a-zA-Z][a-zA-Z0-9]*$")
|
||||
)
|
||||
|
||||
//TODO: remove struct duplication
|
||||
@@ -40,351 +40,279 @@ type Config struct {
|
||||
TeamsAPIClient *teams.TeamsAPI
|
||||
}
|
||||
|
||||
type pgUser struct {
|
||||
name string
|
||||
password string
|
||||
flags []string
|
||||
type KubeResources struct {
|
||||
Services map[types.UID]*v1.Service
|
||||
Endpoints map[types.UID]*v1.Endpoints
|
||||
Secrets map[types.UID]*v1.Secret
|
||||
Statefulsets map[types.UID]*v1beta1.StatefulSet
|
||||
//Pods are treated separately
|
||||
}
|
||||
|
||||
type Cluster struct {
|
||||
config Config
|
||||
logger *logrus.Entry
|
||||
etcdHost string
|
||||
dockerImage string
|
||||
cluster *spec.Postgresql
|
||||
pgUsers map[string]pgUser
|
||||
|
||||
pgDb *sql.DB
|
||||
mu sync.Mutex
|
||||
KubeResources
|
||||
spec.Postgresql
|
||||
config Config
|
||||
logger *logrus.Entry
|
||||
etcdHost string
|
||||
dockerImage string
|
||||
pgUsers map[string]spec.PgUser
|
||||
podEvents chan spec.PodEvent
|
||||
podSubscribers map[spec.PodName]chan spec.PodEvent
|
||||
pgDb *sql.DB
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func New(cfg Config, spec *spec.Postgresql) *Cluster {
|
||||
lg := logrus.WithField("pkg", "cluster").WithField("cluster-name", spec.Metadata.Name)
|
||||
func New(cfg Config, pgSpec spec.Postgresql) *Cluster {
|
||||
lg := logrus.WithField("pkg", "cluster").WithField("cluster-name", pgSpec.Metadata.Name)
|
||||
kubeResources := KubeResources{
|
||||
Services: make(map[types.UID]*v1.Service),
|
||||
Endpoints: make(map[types.UID]*v1.Endpoints),
|
||||
Secrets: make(map[types.UID]*v1.Secret),
|
||||
Statefulsets: make(map[types.UID]*v1beta1.StatefulSet),
|
||||
}
|
||||
|
||||
cluster := &Cluster{
|
||||
config: cfg,
|
||||
cluster: spec,
|
||||
logger: lg,
|
||||
etcdHost: constants.EtcdHost,
|
||||
dockerImage: constants.SpiloImage,
|
||||
config: cfg,
|
||||
Postgresql: pgSpec,
|
||||
logger: lg,
|
||||
etcdHost: constants.EtcdHost,
|
||||
dockerImage: constants.SpiloImage,
|
||||
pgUsers: make(map[string]spec.PgUser),
|
||||
podEvents: make(chan spec.PodEvent),
|
||||
podSubscribers: make(map[spec.PodName]chan spec.PodEvent),
|
||||
KubeResources: kubeResources,
|
||||
}
|
||||
cluster.init()
|
||||
|
||||
return cluster
|
||||
}
|
||||
|
||||
func (c *Cluster) getReadonlyToken() (string, error) {
|
||||
// for some reason PlatformCredentialsSet creates secrets only in the default namespace
|
||||
credentialsSecret, err := c.config.KubeClient.Secrets(v1.NamespaceDefault).Get("postgresql-operator")
|
||||
func (c *Cluster) ClusterName() spec.ClusterName {
|
||||
return spec.ClusterName{
|
||||
Name: c.Metadata.Name,
|
||||
Namespace: c.Metadata.Namespace,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) Run(stopCh <-chan struct{}) {
|
||||
go c.podEventsDispatcher(stopCh)
|
||||
|
||||
<-stopCh
|
||||
}
|
||||
|
||||
func (c *Cluster) NeedsRollingUpdate(otherSpec *spec.Postgresql) bool {
|
||||
//TODO: add more checks
|
||||
if c.Spec.Version != otherSpec.Spec.Version {
|
||||
return true
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(c.Spec.Resources, otherSpec.Spec.Resources) {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *Cluster) MustSetStatus(status spec.PostgresStatus) {
|
||||
b, err := json.Marshal(status)
|
||||
if err != nil {
|
||||
c.logger.Fatalf("Can't marshal status: %s", err)
|
||||
}
|
||||
request := []byte(fmt.Sprintf(`{"status": %s}`, string(b))) //TODO: Look into/wait for k8s go client methods
|
||||
|
||||
_, err = c.config.RestClient.Patch(api.MergePatchType).
|
||||
RequestURI(c.Metadata.GetSelfLink()).
|
||||
Body(request).
|
||||
DoRaw()
|
||||
|
||||
if k8sutil.ResourceNotFound(err) {
|
||||
c.logger.Warningf("Can't set status for the non-existing cluster")
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("Can't get credentials secret: %s", err)
|
||||
c.logger.Fatalf("Can't set status for cluster '%s': %s", c.ClusterName(), err)
|
||||
}
|
||||
data := credentialsSecret.Data
|
||||
|
||||
if string(data["read-only-token-type"]) != "Bearer" {
|
||||
return "", fmt.Errorf("Wrong token type: %s", data["read-only-token-type"])
|
||||
}
|
||||
|
||||
return string(data["read-only-token-secret"]), nil
|
||||
|
||||
}
|
||||
|
||||
func (c *Cluster) getTeamMembers() ([]string, error) {
|
||||
token, err := c.getReadonlyToken()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Can't get oauth token: %s", err)
|
||||
}
|
||||
|
||||
c.config.TeamsAPIClient.OauthToken = token
|
||||
teamInfo, err := c.config.TeamsAPIClient.TeamInfo((*c.cluster.Spec).TeamId)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Can't get team info: %s", err)
|
||||
}
|
||||
|
||||
return teamInfo.Members, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) labelsSet() labels.Set {
|
||||
return labels.Set{
|
||||
"application": "spilo",
|
||||
"spilo-cluster": (*c.cluster).Metadata.Name,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) credentialSecretName(username string) string {
|
||||
return fmt.Sprintf(
|
||||
"%s.%s.credentials.%s.%s",
|
||||
username,
|
||||
(*c.cluster).Metadata.Name,
|
||||
constants.TPRName,
|
||||
constants.TPRVendor)
|
||||
}
|
||||
|
||||
func isValidUsername(username string) bool {
|
||||
return alphaNumericRegexp.MatchString(username)
|
||||
}
|
||||
|
||||
func normalizeUserFlags(userFlags []string) (flags []string, err error) {
|
||||
uniqueFlags := make(map[string]bool)
|
||||
|
||||
for _, flag := range userFlags {
|
||||
if !alphaNumericRegexp.MatchString(flag) {
|
||||
err = fmt.Errorf("User flag '%s' is not alphanumeric", flag)
|
||||
return
|
||||
} else {
|
||||
flag = strings.ToUpper(flag)
|
||||
if _, ok := uniqueFlags[flag]; !ok {
|
||||
uniqueFlags[flag] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
flags = []string{}
|
||||
for k := range uniqueFlags {
|
||||
flags = append(flags, k)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (c *Cluster) init() {
|
||||
users := (*c.cluster.Spec).Users
|
||||
c.pgUsers = make(map[string]pgUser, len(users)+2) // + [superuser and replication]
|
||||
|
||||
c.pgUsers[superuserName] = pgUser{
|
||||
name: superuserName,
|
||||
password: util.RandomPassword(constants.PasswordLength),
|
||||
}
|
||||
|
||||
c.pgUsers[replicationUsername] = pgUser{
|
||||
name: replicationUsername,
|
||||
password: util.RandomPassword(constants.PasswordLength),
|
||||
}
|
||||
|
||||
for username, userFlags := range users {
|
||||
if !isValidUsername(username) {
|
||||
c.logger.Warningf("Invalid username: '%s'", username)
|
||||
continue
|
||||
}
|
||||
|
||||
flags, err := normalizeUserFlags(userFlags)
|
||||
if err != nil {
|
||||
c.logger.Warningf("Invalid flags for user '%s': %s", username, err)
|
||||
}
|
||||
|
||||
c.pgUsers[username] = pgUser{
|
||||
name: username,
|
||||
password: util.RandomPassword(constants.PasswordLength),
|
||||
flags: flags,
|
||||
}
|
||||
}
|
||||
|
||||
teamMembers, err := c.getTeamMembers()
|
||||
if err != nil {
|
||||
c.logger.Errorf("Can't get list of team members: %s", err)
|
||||
} else {
|
||||
for _, username := range teamMembers {
|
||||
c.pgUsers[username] = pgUser{name: username}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) waitPodDelete() error {
|
||||
ls := c.labelsSet()
|
||||
|
||||
listOptions := v1.ListOptions{
|
||||
LabelSelector: ls.String(),
|
||||
}
|
||||
return retryutil.Retry(
|
||||
constants.ResourceCheckInterval, int(constants.ResourceCheckTimeout/constants.ResourceCheckInterval),
|
||||
func() (bool, error) {
|
||||
pods, err := c.config.KubeClient.Pods((*c.cluster).Metadata.Namespace).List(listOptions)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
return len(pods.Items) == 0, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (c *Cluster) waitStatefulsetReady() error {
|
||||
return retryutil.Retry(constants.ResourceCheckInterval, int(constants.ResourceCheckTimeout/constants.ResourceCheckInterval),
|
||||
func() (bool, error) {
|
||||
listOptions := v1.ListOptions{
|
||||
LabelSelector: c.labelsSet().String(),
|
||||
}
|
||||
ss, err := c.config.KubeClient.StatefulSets((*c.cluster).Metadata.Namespace).List(listOptions)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if len(ss.Items) != 1 {
|
||||
return false, fmt.Errorf("StatefulSet is not found")
|
||||
}
|
||||
|
||||
return *ss.Items[0].Spec.Replicas == ss.Items[0].Status.Replicas, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (c *Cluster) waitPodLabelsReady() error {
|
||||
ls := c.labelsSet()
|
||||
namespace := (*c.cluster).Metadata.Namespace
|
||||
|
||||
listOptions := v1.ListOptions{
|
||||
LabelSelector: ls.String(),
|
||||
}
|
||||
masterListOption := v1.ListOptions{
|
||||
LabelSelector: labels.Merge(ls, labels.Set{"spilo-role": "master"}).String(),
|
||||
}
|
||||
replicaListOption := v1.ListOptions{
|
||||
LabelSelector: labels.Merge(ls, labels.Set{"spilo-role": "replica"}).String(),
|
||||
}
|
||||
pods, err := c.config.KubeClient.Pods(namespace).List(listOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
podsNumber := len(pods.Items)
|
||||
|
||||
return retryutil.Retry(
|
||||
constants.ResourceCheckInterval, int(constants.ResourceCheckTimeout/constants.ResourceCheckInterval),
|
||||
func() (bool, error) {
|
||||
masterPods, err := c.config.KubeClient.Pods(namespace).List(masterListOption)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
replicaPods, err := c.config.KubeClient.Pods(namespace).List(replicaListOption)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if len(masterPods.Items) > 1 {
|
||||
return false, fmt.Errorf("Too many masters")
|
||||
}
|
||||
|
||||
return len(masterPods.Items)+len(replicaPods.Items) == podsNumber, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (c *Cluster) Create() error {
|
||||
c.createEndpoint()
|
||||
c.createService()
|
||||
c.applySecrets()
|
||||
c.createStatefulSet()
|
||||
//TODO: service will create endpoint implicitly
|
||||
ep, err := c.createEndpoint()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't create endpoint: %s", err)
|
||||
}
|
||||
c.logger.Infof("Endpoint '%s' has been successfully created", util.NameFromMeta(ep.ObjectMeta))
|
||||
|
||||
service, err := c.createService()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't create service: %s", err)
|
||||
} else {
|
||||
c.logger.Infof("Service '%s' has been successfully created", util.NameFromMeta(service.ObjectMeta))
|
||||
}
|
||||
|
||||
c.initSystemUsers()
|
||||
err = c.initRobotUsers()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't init robot users: %s", err)
|
||||
}
|
||||
|
||||
err = c.initHumanUsers()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't init human users: %s", err)
|
||||
}
|
||||
|
||||
err = c.applySecrets()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't create secrets: %s", err)
|
||||
} else {
|
||||
c.logger.Infof("Secrets have been successfully created")
|
||||
}
|
||||
|
||||
ss, err := c.createStatefulSet()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't create StatefulSet: %s", err)
|
||||
} else {
|
||||
c.logger.Infof("StatefulSet '%s' has been successfully created", util.NameFromMeta(ss.ObjectMeta))
|
||||
}
|
||||
|
||||
c.logger.Info("Waiting for cluster being ready")
|
||||
err := c.waitClusterReady()
|
||||
err = c.waitClusterReady()
|
||||
if err != nil {
|
||||
c.logger.Errorf("Failed to create cluster: %s", err)
|
||||
return err
|
||||
}
|
||||
c.logger.Info("Cluster is ready")
|
||||
|
||||
err = c.initDbConn()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to init db connection: %s", err)
|
||||
return fmt.Errorf("Can't init db connection: %s", err)
|
||||
}
|
||||
|
||||
c.createUsers()
|
||||
err = c.createUsers()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't create users: %s", err)
|
||||
} else {
|
||||
c.logger.Infof("Users have been successfully created")
|
||||
}
|
||||
|
||||
c.ListResources()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) waitClusterReady() error {
|
||||
// TODO: wait for the first Pod only
|
||||
err := c.waitStatefulsetReady()
|
||||
func (c *Cluster) Update(newSpec *spec.Postgresql, rollingUpdate bool) error {
|
||||
nSpec := newSpec.Spec
|
||||
clusterName := c.ClusterName()
|
||||
resourceList := resources.ResourceList(nSpec.Resources)
|
||||
template := resources.PodTemplate(clusterName, resourceList, c.dockerImage, nSpec.Version, c.etcdHost)
|
||||
statefulSet := resources.StatefulSet(clusterName, template, nSpec.NumberOfInstances)
|
||||
|
||||
//TODO: mind the case of updating allowedSourceRanges
|
||||
err := c.updateStatefulSet(statefulSet)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Statuful set error: %s", err)
|
||||
return fmt.Errorf("Can't upate cluster: %s", err)
|
||||
}
|
||||
|
||||
// TODO: wait only for master
|
||||
err = c.waitPodLabelsReady()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Pod labels error: %s", err)
|
||||
if rollingUpdate {
|
||||
err = c.recreatePods()
|
||||
// TODO: wait for actual streaming to the replica
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't recreate pods: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) Delete() error {
|
||||
clusterName := (*c.cluster).Metadata.Name
|
||||
namespace := (*c.cluster).Metadata.Namespace
|
||||
orphanDependents := false
|
||||
deleteOptions := &v1.DeleteOptions{
|
||||
OrphanDependents: &orphanDependents,
|
||||
}
|
||||
|
||||
listOptions := v1.ListOptions{
|
||||
LabelSelector: c.labelsSet().String(),
|
||||
}
|
||||
|
||||
kubeClient := c.config.KubeClient
|
||||
|
||||
podList, err := kubeClient.Pods(namespace).List(listOptions)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't get list of pods: %s", err)
|
||||
}
|
||||
|
||||
err = kubeClient.StatefulSets(namespace).Delete(clusterName, deleteOptions)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't delete statefulset: %s", err)
|
||||
}
|
||||
c.logger.Infof("Statefulset '%s' has been deleted", util.FullObjectName(namespace, clusterName))
|
||||
|
||||
for _, pod := range podList.Items {
|
||||
err = kubeClient.Pods(namespace).Delete(pod.Name, deleteOptions)
|
||||
for _, obj := range c.Statefulsets {
|
||||
err := c.deleteStatefulSet(obj)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error while deleting pod '%s': %s", util.FullObjectName(pod.Namespace, pod.Name), err)
|
||||
c.logger.Errorf("Can't delete StatefulSet: %s", err)
|
||||
} else {
|
||||
c.logger.Infof("StatefulSet '%s' has been deleted", util.NameFromMeta(obj.ObjectMeta))
|
||||
}
|
||||
|
||||
c.logger.Infof("Pod '%s' has been deleted", util.FullObjectName(pod.Namespace, pod.Name))
|
||||
}
|
||||
|
||||
serviceList, err := kubeClient.Services(namespace).List(listOptions)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't get list of the services: %s", err)
|
||||
}
|
||||
|
||||
for _, service := range serviceList.Items {
|
||||
err = kubeClient.Services(namespace).Delete(service.Name, deleteOptions)
|
||||
for _, obj := range c.Secrets {
|
||||
err := c.deleteSecret(obj)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't delete service '%s': %s", util.FullObjectName(service.Namespace, service.Name), err)
|
||||
c.logger.Errorf("Can't delete secret: %s", err)
|
||||
} else {
|
||||
c.logger.Infof("Secret '%s' has been deleted", util.NameFromMeta(obj.ObjectMeta))
|
||||
}
|
||||
|
||||
c.logger.Infof("Service '%s' has been deleted", util.FullObjectName(service.Namespace, service.Name))
|
||||
}
|
||||
|
||||
secretsList, err := kubeClient.Secrets(namespace).List(listOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, secret := range secretsList.Items {
|
||||
err = kubeClient.Secrets(namespace).Delete(secret.Name, deleteOptions)
|
||||
for _, obj := range c.Endpoints {
|
||||
err := c.deleteEndpoint(obj)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't delete secret '%s': %s", util.FullObjectName(secret.Namespace, secret.Name), err)
|
||||
c.logger.Errorf("Can't delete endpoint: %s", err)
|
||||
} else {
|
||||
c.logger.Infof("Endpoint '%s' has been deleted", util.NameFromMeta(obj.ObjectMeta))
|
||||
}
|
||||
|
||||
c.logger.Infof("Secret '%s' has been deleted", util.FullObjectName(secret.Namespace, secret.Name))
|
||||
}
|
||||
|
||||
c.waitPodDelete()
|
||||
|
||||
etcdKey := fmt.Sprintf("/service/%s", clusterName)
|
||||
|
||||
resp, err := c.config.EtcdClient.Delete(context.Background(),
|
||||
etcdKey,
|
||||
&etcdclient.DeleteOptions{Recursive: true})
|
||||
for _, obj := range c.Services {
|
||||
err := c.deleteService(obj)
|
||||
if err != nil {
|
||||
c.logger.Errorf("Can't delete service: %s", err)
|
||||
} else {
|
||||
c.logger.Infof("Service '%s' has been deleted", util.NameFromMeta(obj.ObjectMeta))
|
||||
}
|
||||
}
|
||||
|
||||
err := c.deletePods()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't delete etcd key: %s", err)
|
||||
return fmt.Errorf("Can't delete pods: %s", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) ReceivePodEvent(event spec.PodEvent) {
|
||||
c.podEvents <- event
|
||||
}
|
||||
|
||||
func (c *Cluster) initSystemUsers() {
|
||||
c.pgUsers[constants.SuperuserName] = spec.PgUser{
|
||||
Name: constants.SuperuserName,
|
||||
Password: util.RandomPassword(constants.PasswordLength),
|
||||
}
|
||||
|
||||
c.pgUsers[constants.ReplicationUsername] = spec.PgUser{
|
||||
Name: constants.ReplicationUsername,
|
||||
Password: util.RandomPassword(constants.PasswordLength),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) initRobotUsers() error {
|
||||
for username, userFlags := range c.Spec.Users {
|
||||
if !isValidUsername(username) {
|
||||
return fmt.Errorf("Invalid username: '%s'", username)
|
||||
}
|
||||
|
||||
flags, err := normalizeUserFlags(userFlags)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Invalid flags for user '%s': %s", username, err)
|
||||
}
|
||||
|
||||
c.pgUsers[username] = spec.PgUser{
|
||||
Name: username,
|
||||
Password: util.RandomPassword(constants.PasswordLength),
|
||||
Flags: flags,
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) initHumanUsers() error {
|
||||
teamMembers, err := c.getTeamMembers()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't get list of team members: %s", err)
|
||||
} else {
|
||||
for _, username := range teamMembers {
|
||||
c.pgUsers[username] = spec.PgUser{Name: username}
|
||||
}
|
||||
}
|
||||
|
||||
if resp == nil {
|
||||
c.logger.Warningf("No response from etcd cluster")
|
||||
}
|
||||
|
||||
c.logger.Infof("Etcd key '%s' has been deleted", etcdKey)
|
||||
|
||||
//TODO: Ensure objects are deleted
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
+46
-4
@@ -3,18 +3,23 @@ package cluster
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
"strings"
|
||||
|
||||
"github.bus.zalan.do/acid/postgres-operator/pkg/spec"
|
||||
"github.bus.zalan.do/acid/postgres-operator/pkg/util/constants"
|
||||
)
|
||||
|
||||
var createUserSQL = `SET LOCAL synchronous_commit = 'local'; CREATE ROLE "%s" %s PASSWORD %s;`
|
||||
|
||||
func (c *Cluster) pgConnectionString() string {
|
||||
hostname := fmt.Sprintf("%s.%s.svc.cluster.local", (*c.cluster).Metadata.Name, (*c.cluster).Metadata.Namespace)
|
||||
password := c.pgUsers[superuserName].password
|
||||
hostname := fmt.Sprintf("%s.%s.svc.cluster.local", c.Metadata.Name, c.Metadata.Namespace)
|
||||
password := c.pgUsers[constants.SuperuserName].Password
|
||||
|
||||
return fmt.Sprintf("host='%s' dbname=postgres sslmode=require user='%s' password='%s'",
|
||||
hostname,
|
||||
superuserName,
|
||||
constants.SuperuserName,
|
||||
strings.Replace(password, "$", "\\$", -1))
|
||||
}
|
||||
|
||||
@@ -39,3 +44,40 @@ func (c *Cluster) initDbConn() error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) createPgUser(user spec.PgUser) (isHuman bool, err error) {
|
||||
var flags []string = user.Flags
|
||||
|
||||
if user.Password == "" {
|
||||
isHuman = true
|
||||
flags = append(flags, fmt.Sprintf("IN ROLE \"%s\"", constants.PamRoleName))
|
||||
} else {
|
||||
isHuman = false
|
||||
}
|
||||
|
||||
addLoginFlag := true
|
||||
for _, v := range flags {
|
||||
if v == "NOLOGIN" {
|
||||
addLoginFlag = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if addLoginFlag {
|
||||
flags = append(flags, "LOGIN")
|
||||
}
|
||||
|
||||
userFlags := strings.Join(flags, " ")
|
||||
userPassword := fmt.Sprintf("'%s'", user.Password)
|
||||
if user.Password == "" {
|
||||
userPassword = "NULL"
|
||||
}
|
||||
query := fmt.Sprintf(createUserSQL, user.Name, userFlags, userPassword)
|
||||
|
||||
_, err = c.pgDb.Query(query) // TODO: Try several times
|
||||
if err != nil {
|
||||
err = fmt.Errorf("DB error: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"k8s.io/client-go/pkg/api/v1"
|
||||
|
||||
"github.bus.zalan.do/acid/postgres-operator/pkg/spec"
|
||||
"github.bus.zalan.do/acid/postgres-operator/pkg/util"
|
||||
)
|
||||
|
||||
func (c *Cluster) clusterPods() ([]v1.Pod, error) {
|
||||
ns := c.Metadata.Namespace
|
||||
listOptions := v1.ListOptions{
|
||||
LabelSelector: c.labelsSet().String(),
|
||||
}
|
||||
|
||||
pods, err := c.config.KubeClient.Pods(ns).List(listOptions)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Can't get list of pods: %s", err)
|
||||
}
|
||||
|
||||
return pods.Items, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) deletePods() error {
|
||||
pods, err := c.clusterPods()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, obj := range pods {
|
||||
err := c.deletePod(&obj)
|
||||
if err != nil {
|
||||
c.logger.Errorf("Can't delete pod: %s", err)
|
||||
} else {
|
||||
c.logger.Infof("Pod '%s' has been deleted", util.NameFromMeta(obj.ObjectMeta))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) deletePod(pod *v1.Pod) error {
|
||||
podName := spec.PodName{
|
||||
Namespace: pod.Namespace,
|
||||
Name: pod.Name,
|
||||
}
|
||||
|
||||
ch := make(chan spec.PodEvent)
|
||||
if _, ok := c.podSubscribers[podName]; ok {
|
||||
panic("Pod '" + podName.String() + "' is already subscribed")
|
||||
}
|
||||
c.podSubscribers[podName] = ch
|
||||
defer func() {
|
||||
close(ch)
|
||||
delete(c.podSubscribers, podName)
|
||||
}()
|
||||
|
||||
err := c.config.KubeClient.Pods(pod.Namespace).Delete(pod.Name, deleteOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = c.waitForPodDeletion(ch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) recreatePod(pod v1.Pod, spiloRole string) error {
|
||||
podName := spec.PodName{
|
||||
Namespace: pod.Namespace,
|
||||
Name: pod.Name,
|
||||
}
|
||||
|
||||
orphanDependents := false
|
||||
deleteOptions := &v1.DeleteOptions{
|
||||
OrphanDependents: &orphanDependents,
|
||||
}
|
||||
|
||||
ch := make(chan spec.PodEvent)
|
||||
if _, ok := c.podSubscribers[podName]; ok {
|
||||
panic("Pod '" + podName.String() + "' is already subscribed")
|
||||
}
|
||||
c.podSubscribers[podName] = ch
|
||||
defer func() {
|
||||
close(ch)
|
||||
delete(c.podSubscribers, podName)
|
||||
}()
|
||||
|
||||
err := c.config.KubeClient.Pods(pod.Namespace).Delete(pod.Name, deleteOptions)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't delete pod: %s", err)
|
||||
}
|
||||
err = c.waitForPodDeletion(ch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = c.waitForPodLabel(ch, spiloRole)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.logger.Infof("Pod '%s' is ready", podName)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) podEventsDispatcher(stopCh <-chan struct{}) {
|
||||
c.logger.Infof("Watching '%s' cluster", c.ClusterName())
|
||||
for {
|
||||
select {
|
||||
case event := <-c.podEvents:
|
||||
if subscriber, ok := c.podSubscribers[event.PodName]; ok {
|
||||
c.logger.Debugf("New event for '%s' pod", event.PodName)
|
||||
go func() { subscriber <- event }() //TODO: is it a right way to do nonblocking send to the channel?
|
||||
} else {
|
||||
c.logger.Debugf("Skipping event for an unwatched pod '%s'", event.PodName)
|
||||
}
|
||||
case <-stopCh:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) recreatePods() error {
|
||||
ls := c.labelsSet()
|
||||
namespace := c.Metadata.Namespace
|
||||
|
||||
listOptions := v1.ListOptions{
|
||||
LabelSelector: ls.String(),
|
||||
}
|
||||
pods, err := c.config.KubeClient.Pods(namespace).List(listOptions)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't get list of pods: %s", err)
|
||||
} else {
|
||||
c.logger.Infof("There are %d pods in the cluster to recreate", len(pods.Items))
|
||||
}
|
||||
|
||||
var masterPod v1.Pod
|
||||
for _, pod := range pods.Items {
|
||||
role, ok := pod.Labels["spilo-role"]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
if role == "master" {
|
||||
masterPod = pod
|
||||
continue
|
||||
}
|
||||
|
||||
err = c.recreatePod(pod, "replica")
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't recreate replica pod '%s': %s", util.NameFromMeta(pod.ObjectMeta), err)
|
||||
}
|
||||
}
|
||||
|
||||
//TODO: do manual failover
|
||||
//TODO: specify master, leave new master empty
|
||||
c.logger.Infof("Recreating master pod '%s'", util.NameFromMeta(masterPod.ObjectMeta))
|
||||
err = c.recreatePod(masterPod, "replica")
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't recreate master pod '%s': %s", util.NameFromMeta(masterPod.ObjectMeta), err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
+231
-303
@@ -2,330 +2,258 @@ package cluster
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"k8s.io/client-go/pkg/api/resource"
|
||||
"k8s.io/client-go/pkg/api/v1"
|
||||
"k8s.io/client-go/pkg/apis/apps/v1beta1"
|
||||
"k8s.io/client-go/pkg/util/intstr"
|
||||
|
||||
"github.bus.zalan.do/acid/postgres-operator/pkg/util"
|
||||
"github.bus.zalan.do/acid/postgres-operator/pkg/util/constants"
|
||||
"github.bus.zalan.do/acid/postgres-operator/pkg/util/k8sutil"
|
||||
"github.bus.zalan.do/acid/postgres-operator/pkg/util/resources"
|
||||
)
|
||||
|
||||
var createUserSQL = `DO $$
|
||||
BEGIN
|
||||
SET LOCAL synchronous_commit = 'local';
|
||||
CREATE ROLE "%s" %s PASSWORD %s;
|
||||
END;
|
||||
$$`
|
||||
|
||||
func (c *Cluster) createStatefulSet() {
|
||||
meta := (*c.cluster).Metadata
|
||||
|
||||
envVars := []v1.EnvVar{
|
||||
{
|
||||
Name: "SCOPE",
|
||||
Value: meta.Name,
|
||||
},
|
||||
{
|
||||
Name: "PGROOT",
|
||||
Value: "/home/postgres/pgdata/pgroot",
|
||||
},
|
||||
{
|
||||
Name: "ETCD_HOST",
|
||||
Value: c.etcdHost,
|
||||
},
|
||||
{
|
||||
Name: "POD_IP",
|
||||
ValueFrom: &v1.EnvVarSource{
|
||||
FieldRef: &v1.ObjectFieldSelector{
|
||||
APIVersion: "v1",
|
||||
FieldPath: "status.podIP",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "POD_NAMESPACE",
|
||||
ValueFrom: &v1.EnvVarSource{
|
||||
FieldRef: &v1.ObjectFieldSelector{
|
||||
APIVersion: "v1",
|
||||
FieldPath: "metadata.namespace",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "PGPASSWORD_SUPERUSER",
|
||||
ValueFrom: &v1.EnvVarSource{
|
||||
SecretKeyRef: &v1.SecretKeySelector{
|
||||
LocalObjectReference: v1.LocalObjectReference{
|
||||
Name: c.credentialSecretName(superuserName),
|
||||
},
|
||||
Key: "password",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "PGPASSWORD_STANDBY",
|
||||
ValueFrom: &v1.EnvVarSource{
|
||||
SecretKeyRef: &v1.SecretKeySelector{
|
||||
LocalObjectReference: v1.LocalObjectReference{
|
||||
Name: c.credentialSecretName(replicationUsername),
|
||||
},
|
||||
Key: "password",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "PAM_OAUTH2", //TODO: get from the operator tpr spec
|
||||
Value: constants.PamConfiguration, //space before uid is obligatory
|
||||
},
|
||||
{
|
||||
Name: "SPILO_CONFIGURATION", //TODO: get from the operator tpr spec
|
||||
Value: fmt.Sprintf(`
|
||||
postgresql:
|
||||
bin_dir: /usr/lib/postgresql/%s/bin
|
||||
bootstrap:
|
||||
initdb:
|
||||
- auth-host: md5
|
||||
- auth-local: trust
|
||||
users:
|
||||
%s:
|
||||
password: NULL
|
||||
options:
|
||||
- createdb
|
||||
- nologin
|
||||
pg_hba:
|
||||
- hostnossl all all all reject
|
||||
- hostssl all +%s all pam
|
||||
- hostssl all all all md5`, (*c.cluster.Spec).Version, constants.PamRoleName, constants.PamRoleName),
|
||||
},
|
||||
}
|
||||
|
||||
resourceList := v1.ResourceList{}
|
||||
|
||||
if cpu := (*c.cluster).Spec.Resources.Cpu; cpu != "" {
|
||||
resourceList[v1.ResourceCPU] = resource.MustParse(cpu)
|
||||
}
|
||||
|
||||
if memory := (*c.cluster).Spec.Resources.Memory; memory != "" {
|
||||
resourceList[v1.ResourceMemory] = resource.MustParse(memory)
|
||||
}
|
||||
|
||||
container := v1.Container{
|
||||
Name: meta.Name,
|
||||
Image: c.dockerImage,
|
||||
ImagePullPolicy: v1.PullAlways,
|
||||
Resources: v1.ResourceRequirements{
|
||||
Requests: resourceList,
|
||||
},
|
||||
Ports: []v1.ContainerPort{
|
||||
{
|
||||
ContainerPort: 8008,
|
||||
Protocol: v1.ProtocolTCP,
|
||||
},
|
||||
{
|
||||
ContainerPort: 5432,
|
||||
Protocol: v1.ProtocolTCP,
|
||||
},
|
||||
},
|
||||
VolumeMounts: []v1.VolumeMount{
|
||||
{
|
||||
Name: "pgdata",
|
||||
MountPath: "/home/postgres/pgdata", //TODO: fetch from manifesto
|
||||
},
|
||||
},
|
||||
Env: envVars,
|
||||
}
|
||||
|
||||
terminateGracePeriodSeconds := int64(30)
|
||||
|
||||
podSpec := v1.PodSpec{
|
||||
TerminationGracePeriodSeconds: &terminateGracePeriodSeconds,
|
||||
Volumes: []v1.Volume{
|
||||
{
|
||||
Name: "pgdata",
|
||||
VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}},
|
||||
},
|
||||
},
|
||||
Containers: []v1.Container{container},
|
||||
}
|
||||
|
||||
template := v1.PodTemplateSpec{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Labels: c.labelsSet(),
|
||||
Namespace: meta.Namespace,
|
||||
Annotations: map[string]string{"pod.alpha.kubernetes.io/initialized": "true"},
|
||||
},
|
||||
Spec: podSpec,
|
||||
}
|
||||
|
||||
statefulSet := &v1beta1.StatefulSet{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: meta.Name,
|
||||
Namespace: meta.Namespace,
|
||||
Labels: c.labelsSet(),
|
||||
},
|
||||
Spec: v1beta1.StatefulSetSpec{
|
||||
Replicas: &c.cluster.Spec.NumberOfInstances,
|
||||
ServiceName: meta.Name,
|
||||
Template: template,
|
||||
},
|
||||
}
|
||||
|
||||
_, err := c.config.KubeClient.StatefulSets(meta.Namespace).Create(statefulSet)
|
||||
if err != nil {
|
||||
c.logger.Errorf("Can't create statefulset: %s", err)
|
||||
} else {
|
||||
c.logger.Infof("Statefulset has been created: '%s'", util.FullObjectNameFromMeta(statefulSet.ObjectMeta))
|
||||
}
|
||||
var orphanDependents = false
|
||||
var deleteOptions = &v1.DeleteOptions{
|
||||
OrphanDependents: &orphanDependents,
|
||||
}
|
||||
|
||||
func (c *Cluster) applySecrets() {
|
||||
var err error
|
||||
namespace := (*c.cluster).Metadata.Namespace
|
||||
for username, pgUser := range c.pgUsers {
|
||||
//Skip users with no password i.e. human users (they'll be authenticated using pam)
|
||||
if pgUser.password == "" {
|
||||
func (c *Cluster) LoadResources() error {
|
||||
ns := c.Metadata.Namespace
|
||||
listOptions := v1.ListOptions{
|
||||
LabelSelector: c.labelsSet().String(),
|
||||
}
|
||||
|
||||
services, err := c.config.KubeClient.Services(ns).List(listOptions)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't get list of services: %s", err)
|
||||
}
|
||||
for i, service := range services.Items {
|
||||
if _, ok := c.Services[service.UID]; ok {
|
||||
continue
|
||||
}
|
||||
secret := v1.Secret{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: c.credentialSecretName(username),
|
||||
Namespace: namespace,
|
||||
Labels: c.labelsSet(),
|
||||
},
|
||||
Type: v1.SecretTypeOpaque,
|
||||
Data: map[string][]byte{
|
||||
"username": []byte(pgUser.name),
|
||||
"password": []byte(pgUser.password),
|
||||
},
|
||||
}
|
||||
_, err = c.config.KubeClient.Secrets(namespace).Create(&secret)
|
||||
if k8sutil.IsKubernetesResourceAlreadyExistError(err) {
|
||||
c.logger.Infof("Skipping update of '%s'", secret.Name)
|
||||
|
||||
curSecrets, err := c.config.KubeClient.Secrets(namespace).Get(c.credentialSecretName(username))
|
||||
if err != nil {
|
||||
c.logger.Errorf("Can't get current secret: %s", err)
|
||||
}
|
||||
user := pgUser
|
||||
user.password = string(curSecrets.Data["password"])
|
||||
c.pgUsers[username] = user
|
||||
c.logger.Infof("Password fetched for user '%s' from the secrets", username)
|
||||
|
||||
continue
|
||||
} else {
|
||||
if err != nil {
|
||||
c.logger.Errorf("Error while creating secret: %s", err)
|
||||
} else {
|
||||
c.logger.Infof("Secret created: '%s'", util.FullObjectNameFromMeta(secret.ObjectMeta))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) createService() {
|
||||
meta := (*c.cluster).Metadata
|
||||
|
||||
_, err := c.config.KubeClient.Services(meta.Namespace).Get(meta.Name)
|
||||
if !k8sutil.ResourceNotFound(err) {
|
||||
c.logger.Infof("Service '%s' already exists", meta.Name)
|
||||
return
|
||||
c.Services[service.UID] = &services.Items[i]
|
||||
}
|
||||
|
||||
service := v1.Service{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: meta.Name,
|
||||
Namespace: meta.Namespace,
|
||||
Labels: c.labelsSet(),
|
||||
},
|
||||
Spec: v1.ServiceSpec{
|
||||
Type: v1.ServiceTypeLoadBalancer,
|
||||
Ports: []v1.ServicePort{{Port: 5432, TargetPort: intstr.IntOrString{IntVal: 5432}}},
|
||||
LoadBalancerSourceRanges: (*c.cluster).Spec.AllowedSourceRanges,
|
||||
},
|
||||
}
|
||||
|
||||
_, err = c.config.KubeClient.Services(meta.Namespace).Create(&service)
|
||||
endpoints, err := c.config.KubeClient.Endpoints(ns).List(listOptions)
|
||||
if err != nil {
|
||||
c.logger.Errorf("Error while creating service: %+v", err)
|
||||
} else {
|
||||
c.logger.Infof("Service created: '%s'", util.FullObjectNameFromMeta(service.ObjectMeta))
|
||||
return fmt.Errorf("Can't get list of endpoints: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) createEndpoint() {
|
||||
meta := (*c.cluster).Metadata
|
||||
|
||||
_, err := c.config.KubeClient.Endpoints(meta.Namespace).Get(meta.Name)
|
||||
if !k8sutil.ResourceNotFound(err) {
|
||||
c.logger.Infof("Endpoint '%s' already exists", meta.Name)
|
||||
return
|
||||
}
|
||||
|
||||
endpoint := v1.Endpoints{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: meta.Name,
|
||||
Namespace: meta.Namespace,
|
||||
Labels: c.labelsSet(),
|
||||
},
|
||||
}
|
||||
|
||||
_, err = c.config.KubeClient.Endpoints(meta.Namespace).Create(&endpoint)
|
||||
if err != nil {
|
||||
c.logger.Errorf("Error while creating endpoint: %+v", err)
|
||||
} else {
|
||||
c.logger.Infof("Endpoint created: %s", endpoint.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) createUser(user pgUser) {
|
||||
var userType string
|
||||
var flags []string = user.flags
|
||||
|
||||
if user.password == "" {
|
||||
userType = "human"
|
||||
flags = append(flags, fmt.Sprintf("IN ROLE \"%s\"", constants.PamRoleName))
|
||||
} else {
|
||||
userType = "app"
|
||||
}
|
||||
|
||||
addLoginFlag := true
|
||||
for _, v := range flags {
|
||||
if v == "NOLOGIN" {
|
||||
addLoginFlag = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if addLoginFlag {
|
||||
flags = append(flags, "LOGIN")
|
||||
}
|
||||
|
||||
userFlags := strings.Join(flags, " ")
|
||||
userPassword := fmt.Sprintf("'%s'", user.password)
|
||||
if user.password == "" {
|
||||
userPassword = "NULL"
|
||||
}
|
||||
query := fmt.Sprintf(createUserSQL, user.name, userFlags, userPassword)
|
||||
|
||||
_, err := c.pgDb.Query(query)
|
||||
if err != nil {
|
||||
c.logger.Errorf("Can't create %s user '%s': %s", user.name, err)
|
||||
} else {
|
||||
c.logger.Infof("Created %s user '%s' with %s flags", userType, user.name, flags)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) createUsers() error {
|
||||
for username, user := range c.pgUsers {
|
||||
if username == superuserName || username == replicationUsername {
|
||||
for i, endpoint := range endpoints.Items {
|
||||
if _, ok := c.Endpoints[endpoint.UID]; ok {
|
||||
continue
|
||||
}
|
||||
c.Endpoints[endpoint.UID] = &endpoints.Items[i]
|
||||
c.logger.Debugf("Endpoint loaded, uid: %s", endpoint.UID)
|
||||
}
|
||||
|
||||
c.createUser(user)
|
||||
secrets, err := c.config.KubeClient.Secrets(ns).List(listOptions)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't get list of secrets: %s", err)
|
||||
}
|
||||
for i, secret := range secrets.Items {
|
||||
if _, ok := c.Secrets[secret.UID]; ok {
|
||||
continue
|
||||
}
|
||||
c.Secrets[secret.UID] = &secrets.Items[i]
|
||||
c.logger.Debugf("Secret loaded, uid: %s", secret.UID)
|
||||
}
|
||||
|
||||
statefulSets, err := c.config.KubeClient.StatefulSets(ns).List(listOptions)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't get list of stateful sets: %s", err)
|
||||
}
|
||||
for i, statefulSet := range statefulSets.Items {
|
||||
if _, ok := c.Statefulsets[statefulSet.UID]; ok {
|
||||
continue
|
||||
}
|
||||
c.Statefulsets[statefulSet.UID] = &statefulSets.Items[i]
|
||||
c.logger.Debugf("StatefulSet loaded, uid: %s", statefulSet.UID)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) ListResources() error {
|
||||
for _, obj := range c.Statefulsets {
|
||||
c.logger.Infof("StatefulSet: %s", util.NameFromMeta(obj.ObjectMeta))
|
||||
}
|
||||
|
||||
for _, obj := range c.Secrets {
|
||||
c.logger.Infof("Secret: %s", util.NameFromMeta(obj.ObjectMeta))
|
||||
}
|
||||
|
||||
for _, obj := range c.Endpoints {
|
||||
c.logger.Infof("Endpoint: %s", util.NameFromMeta(obj.ObjectMeta))
|
||||
}
|
||||
|
||||
for _, obj := range c.Services {
|
||||
c.logger.Infof("Service: %s", util.NameFromMeta(obj.ObjectMeta))
|
||||
}
|
||||
|
||||
pods, err := c.clusterPods()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't get pods: %s", err)
|
||||
}
|
||||
|
||||
for _, obj := range pods {
|
||||
c.logger.Infof("Pod: %s", util.NameFromMeta(obj.ObjectMeta))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) createStatefulSet() (*v1beta1.StatefulSet, error) {
|
||||
cSpec := c.Spec
|
||||
clusterName := c.ClusterName()
|
||||
resourceList := resources.ResourceList(cSpec.Resources)
|
||||
template := resources.PodTemplate(clusterName, resourceList, c.dockerImage, cSpec.Version, c.etcdHost)
|
||||
statefulSet := resources.StatefulSet(clusterName, template, cSpec.NumberOfInstances)
|
||||
|
||||
statefulSet, err := c.config.KubeClient.StatefulSets(statefulSet.Namespace).Create(statefulSet)
|
||||
if k8sutil.ResourceAlreadyExists(err) {
|
||||
return nil, fmt.Errorf("StatefulSet '%s' already exists", util.NameFromMeta(statefulSet.ObjectMeta))
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.Statefulsets[statefulSet.UID] = statefulSet
|
||||
c.logger.Debugf("Created new StatefulSet, uid: %s", statefulSet.UID)
|
||||
|
||||
return statefulSet, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) updateStatefulSet(statefulSet *v1beta1.StatefulSet) error {
|
||||
statefulSet, err := c.config.KubeClient.StatefulSets(statefulSet.Namespace).Update(statefulSet)
|
||||
if err != nil {
|
||||
c.Statefulsets[statefulSet.UID] = statefulSet
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Cluster) deleteStatefulSet(statefulSet *v1beta1.StatefulSet) error {
|
||||
err := c.config.KubeClient.
|
||||
StatefulSets(statefulSet.Namespace).
|
||||
Delete(statefulSet.Name, deleteOptions)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
delete(c.Statefulsets, statefulSet.UID)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) createEndpoint() (*v1.Endpoints, error) {
|
||||
endpoint := resources.Endpoint(c.ClusterName())
|
||||
|
||||
endpoint, err := c.config.KubeClient.Endpoints(endpoint.Namespace).Create(endpoint)
|
||||
if k8sutil.ResourceAlreadyExists(err) {
|
||||
return nil, fmt.Errorf("Endpoint '%s' already exists", util.NameFromMeta(endpoint.ObjectMeta))
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.Endpoints[endpoint.UID] = endpoint
|
||||
c.logger.Debugf("Created new endpoint, uid: %s", endpoint.UID)
|
||||
|
||||
return endpoint, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) deleteEndpoint(endpoint *v1.Endpoints) error {
|
||||
err := c.config.KubeClient.Endpoints(endpoint.Namespace).Delete(endpoint.Name, deleteOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
delete(c.Endpoints, endpoint.UID)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) createService() (*v1.Service, error) {
|
||||
service := resources.Service(c.ClusterName(), c.Spec.AllowedSourceRanges)
|
||||
|
||||
service, err := c.config.KubeClient.Services(service.Namespace).Create(service)
|
||||
if k8sutil.ResourceAlreadyExists(err) {
|
||||
return nil, fmt.Errorf("Service '%s' already exists", util.NameFromMeta(service.ObjectMeta))
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.Services[service.UID] = service
|
||||
c.logger.Debugf("Created new service, uid: %s", service.UID)
|
||||
|
||||
return service, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) deleteService(service *v1.Service) error {
|
||||
err := c.config.KubeClient.Services(service.Namespace).Delete(service.Name, deleteOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
delete(c.Services, service.UID)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) createUsers() error {
|
||||
for username, user := range c.pgUsers {
|
||||
if username == constants.SuperuserName || username == constants.ReplicationUsername {
|
||||
continue
|
||||
}
|
||||
|
||||
isHuman, err := c.createPgUser(user)
|
||||
var userType string
|
||||
if isHuman {
|
||||
userType = "human"
|
||||
} else {
|
||||
userType = "robot"
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't create %s user '%s': %s", userType, username, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) applySecrets() error {
|
||||
secrets, err := resources.UserSecrets(c.ClusterName(), c.pgUsers)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't get user secrets")
|
||||
}
|
||||
|
||||
for username, secret := range secrets {
|
||||
secret, err := c.config.KubeClient.Secrets(secret.Namespace).Create(secret)
|
||||
if k8sutil.ResourceAlreadyExists(err) {
|
||||
curSecrets, err := c.config.KubeClient.Secrets(secret.Namespace).Get(secret.Name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't get current secret: %s", err)
|
||||
}
|
||||
pwdUser := c.pgUsers[username]
|
||||
pwdUser.Password = string(curSecrets.Data["password"])
|
||||
c.pgUsers[username] = pwdUser
|
||||
|
||||
continue
|
||||
} else {
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't create secret for user '%s': %s", username, err)
|
||||
}
|
||||
c.Secrets[secret.UID] = secret
|
||||
c.logger.Debugf("Created new secret, uid: %s", secret.UID)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) deleteSecret(secret *v1.Secret) error {
|
||||
err := c.config.KubeClient.Secrets(secret.Namespace).Delete(secret.Name, deleteOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
delete(c.Secrets, secret.UID)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
etcdclient "github.com/coreos/etcd/client"
|
||||
"k8s.io/client-go/pkg/api/v1"
|
||||
"k8s.io/client-go/pkg/labels"
|
||||
|
||||
"github.bus.zalan.do/acid/postgres-operator/pkg/spec"
|
||||
"github.bus.zalan.do/acid/postgres-operator/pkg/util/constants"
|
||||
"github.bus.zalan.do/acid/postgres-operator/pkg/util/retryutil"
|
||||
)
|
||||
|
||||
func isValidUsername(username string) bool {
|
||||
return alphaNumericRegexp.MatchString(username)
|
||||
}
|
||||
|
||||
func normalizeUserFlags(userFlags []string) (flags []string, err error) {
|
||||
uniqueFlags := make(map[string]bool)
|
||||
|
||||
for _, flag := range userFlags {
|
||||
if !alphaNumericRegexp.MatchString(flag) {
|
||||
err = fmt.Errorf("User flag '%s' is not alphanumeric", flag)
|
||||
return
|
||||
} else {
|
||||
flag = strings.ToUpper(flag)
|
||||
if _, ok := uniqueFlags[flag]; !ok {
|
||||
uniqueFlags[flag] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
flags = []string{}
|
||||
for k := range uniqueFlags {
|
||||
flags = append(flags, k)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (c *Cluster) getTeamMembers() ([]string, error) {
|
||||
teamInfo, err := c.config.TeamsAPIClient.TeamInfo(c.Spec.TeamId)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Can't get team info: %s", err)
|
||||
}
|
||||
|
||||
return teamInfo.Members, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) waitForPodLabel(podEvents chan spec.PodEvent, spiloRole string) error {
|
||||
for {
|
||||
select {
|
||||
case podEvent := <-podEvents:
|
||||
podLabels := podEvent.CurPod.Labels
|
||||
c.logger.Debugf("Pod has following labels: %+v", podLabels)
|
||||
val, ok := podLabels["spilo-role"]
|
||||
if ok && val == spiloRole {
|
||||
return nil
|
||||
}
|
||||
case <-time.After(constants.PodLabelWaitTimeout):
|
||||
return fmt.Errorf("Pod label wait timeout")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) waitForPodDeletion(podEvents chan spec.PodEvent) error {
|
||||
for {
|
||||
select {
|
||||
case podEvent := <-podEvents:
|
||||
if podEvent.EventType == spec.PodEventDelete {
|
||||
return nil
|
||||
}
|
||||
case <-time.After(constants.PodDeletionWaitTimeout):
|
||||
return fmt.Errorf("Pod deletion wait timeout")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) waitStatefulsetReady() error {
|
||||
return retryutil.Retry(constants.ResourceCheckInterval, int(constants.ResourceCheckTimeout/constants.ResourceCheckInterval),
|
||||
func() (bool, error) {
|
||||
listOptions := v1.ListOptions{
|
||||
LabelSelector: c.labelsSet().String(),
|
||||
}
|
||||
ss, err := c.config.KubeClient.StatefulSets(c.Metadata.Namespace).List(listOptions)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if len(ss.Items) != 1 {
|
||||
return false, fmt.Errorf("StatefulSet is not found")
|
||||
}
|
||||
|
||||
return *ss.Items[0].Spec.Replicas == ss.Items[0].Status.Replicas, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (c *Cluster) waitPodLabelsReady() error {
|
||||
ls := c.labelsSet()
|
||||
namespace := c.Metadata.Namespace
|
||||
|
||||
listOptions := v1.ListOptions{
|
||||
LabelSelector: ls.String(),
|
||||
}
|
||||
masterListOption := v1.ListOptions{
|
||||
LabelSelector: labels.Merge(ls, labels.Set{"spilo-role": "master"}).String(),
|
||||
}
|
||||
replicaListOption := v1.ListOptions{
|
||||
LabelSelector: labels.Merge(ls, labels.Set{"spilo-role": "replica"}).String(),
|
||||
}
|
||||
pods, err := c.config.KubeClient.Pods(namespace).List(listOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
podsNumber := len(pods.Items)
|
||||
|
||||
return retryutil.Retry(
|
||||
constants.ResourceCheckInterval, int(constants.ResourceCheckTimeout/constants.ResourceCheckInterval),
|
||||
func() (bool, error) {
|
||||
masterPods, err := c.config.KubeClient.Pods(namespace).List(masterListOption)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
replicaPods, err := c.config.KubeClient.Pods(namespace).List(replicaListOption)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if len(masterPods.Items) > 1 {
|
||||
return false, fmt.Errorf("Too many masters")
|
||||
}
|
||||
|
||||
return len(masterPods.Items)+len(replicaPods.Items) == podsNumber, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (c *Cluster) waitClusterReady() error {
|
||||
// TODO: wait for the first Pod only
|
||||
err := c.waitStatefulsetReady()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Statuful set error: %s", err)
|
||||
}
|
||||
|
||||
// TODO: wait only for master
|
||||
err = c.waitPodLabelsReady()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Pod labels error: %s", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) labelsSet() labels.Set {
|
||||
return labels.Set{
|
||||
"application": "spilo",
|
||||
"spilo-cluster": c.Metadata.Name,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) credentialSecretName(username string) string {
|
||||
return fmt.Sprintf(constants.UserSecretTemplate,
|
||||
username,
|
||||
c.Metadata.Name,
|
||||
constants.TPRName,
|
||||
constants.TPRVendor)
|
||||
}
|
||||
|
||||
func (c *Cluster) deleteEtcdKey() error {
|
||||
etcdKey := fmt.Sprintf("/service/%s", c.Metadata.Name)
|
||||
|
||||
//TODO: retry multiple times
|
||||
resp, err := c.config.EtcdClient.Delete(context.Background(),
|
||||
etcdKey,
|
||||
&etcdclient.DeleteOptions{Recursive: true})
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't delete etcd key: %s", err)
|
||||
}
|
||||
|
||||
if resp == nil {
|
||||
return fmt.Errorf("No response from etcd cluster")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user