postgres-operator/pkg/cluster/cluster.go

package cluster

// Postgres ThirdPartyResource object i.e. Spilo

import (
	"context"
	"database/sql"
	"fmt"
	"regexp"
	"strings"
	"sync"

	"github.com/Sirupsen/logrus"
	etcdclient "github.com/coreos/etcd/client"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/pkg/labels"
	"k8s.io/client-go/rest"

	"github.bus.zalan.do/acid/postgres-operator/pkg/spec"
	"github.bus.zalan.do/acid/postgres-operator/pkg/util"
	"github.bus.zalan.do/acid/postgres-operator/pkg/util/constants"
	"github.bus.zalan.do/acid/postgres-operator/pkg/util/retryutil"
	"github.bus.zalan.do/acid/postgres-operator/pkg/util/teams"
)

var (
	superuserName       = "postgres"
	replicationUsername = "replication"

	alphaNumericRegexp = regexp.MustCompile("^[a-zA-Z0-9]*$")
)

//TODO: remove struct duplication
type Config struct {
	Namespace      string
	KubeClient     *kubernetes.Clientset //TODO: move clients to the better place?
	RestClient     *rest.RESTClient
	EtcdClient     etcdclient.KeysAPI
	TeamsAPIClient *teams.TeamsAPI
}

type pgUser struct {
	name     string
	password string
	flags    []string
}

type Cluster struct {
	config      Config
	logger      *logrus.Entry
	etcdHost    string
	dockerImage string
	cluster     *spec.Postgresql
	pgUsers     map[string]pgUser

	pgDb *sql.DB
	mu   sync.Mutex
}

func New(cfg Config, spec *spec.Postgresql) *Cluster {
	lg := logrus.WithField("pkg", "cluster").WithField("cluster-name", spec.Metadata.Name)

	cluster := &Cluster{
		config:      cfg,
		cluster:     spec,
		logger:      lg,
		etcdHost:    constants.EtcdHost,
		dockerImage: constants.SpiloImage,
	}
	cluster.init()

	return cluster
}

func (c *Cluster) getReadonlyToken() (string, error) {
	credentialsSecret, err := c.config.KubeClient.Secrets(c.config.Namespace).Get("postgresql-operator")

	if err != nil {
		return "", fmt.Errorf("Can't get credentials secret: %s", err)
	}
	data := credentialsSecret.Data

	if string(data["read-only-token-type"]) != "Bearer" {
		return "", fmt.Errorf("Wrong token type: %s", data["read-only-token-type"])
	}

	return string(data["read-only-token-secret"]), nil

}

func (c *Cluster) getTeamMembers() ([]string, error) {
	token, err := c.getReadonlyToken()
	if err != nil {
		return nil, fmt.Errorf("Can't get oauth token: %s", err)
	}

	c.config.TeamsAPIClient.OauthToken = token
	teamInfo, err := c.config.TeamsAPIClient.TeamInfo((*c.cluster.Spec).TeamId)
	if err != nil {
		return nil, fmt.Errorf("Can't get team info: %s", err)
	}

	return teamInfo.Members, nil
}

func (c *Cluster) labelsSet() labels.Set {
	return labels.Set{
		"application":   "spilo",
		"spilo-cluster": (*c.cluster).Metadata.Name,
	}
}

func (c *Cluster) credentialSecretName(username string) string {
	return fmt.Sprintf(
		"%s.%s.credentials.%s.%s",
		username,
		(*c.cluster).Metadata.Name,
		constants.TPRName,
		constants.TPRVendor)
}

func isValidUsername(username string) bool {
	return alphaNumericRegexp.MatchString(username)
}

func normalizeUserFlags(userFlags []string) (flags []string, err error) {
	uniqueFlags := make(map[string]bool)

	for _, flag := range userFlags {
		if !alphaNumericRegexp.MatchString(flag) {
			err = fmt.Errorf("User flag '%s' is not alphanumeric", flag)
			return
		} else {
			flag = strings.ToUpper(flag)
			if _, ok := uniqueFlags[flag]; !ok {
				uniqueFlags[flag] = true
			}
		}
	}

	flags = []string{}
	for k := range uniqueFlags {
		flags = append(flags, k)
	}

	return
}

func (c *Cluster) init() {
	users := (*c.cluster.Spec).Users
	c.pgUsers = make(map[string]pgUser, len(users)+2) // + [superuser and replication]

	c.pgUsers[superuserName] = pgUser{
		name:     superuserName,
		password: util.RandomPassword(constants.PasswordLength),
	}

	c.pgUsers[replicationUsername] = pgUser{
		name:     replicationUsername,
		password: util.RandomPassword(constants.PasswordLength),
	}

	for username, userFlags := range users {
		if !isValidUsername(username) {
			c.logger.Warningf("Invalid username: '%s'", username)
			continue
		}

		flags, err := normalizeUserFlags(userFlags)
		if err != nil {
			c.logger.Warningf("Invalid flags for user '%s': %s", username, err)
		}

		c.pgUsers[username] = pgUser{
			name:     username,
			password: util.RandomPassword(constants.PasswordLength),
			flags:    flags,
		}
	}

	teamMembers, err := c.getTeamMembers()
	if err != nil {
		c.logger.Errorf("Can't get list of team members: %s", err)
	} else {
		for _, username := range teamMembers {
			c.pgUsers[username] = pgUser{name: username}
		}
	}
}

func (c *Cluster) waitPodDelete() error {
	ls := c.labelsSet()

	listOptions := v1.ListOptions{
		LabelSelector: ls.String(),
	}
	return retryutil.Retry(
		constants.ResourceCheckInterval, int(constants.ResourceCheckTimeout/constants.ResourceCheckInterval),
		func() (bool, error) {
			pods, err := c.config.KubeClient.Pods(c.config.Namespace).List(listOptions)
			if err != nil {
				return false, err
			}

			return len(pods.Items) == 0, nil
		})
}

func (c *Cluster) waitStatefulsetReady() error {
	return retryutil.Retry(constants.ResourceCheckInterval, int(constants.ResourceCheckTimeout/constants.ResourceCheckInterval),
		func() (bool, error) {
			listOptions := v1.ListOptions{
				LabelSelector: c.labelsSet().String(),
			}
			ss, err := c.config.KubeClient.StatefulSets(c.config.Namespace).List(listOptions)
			if err != nil {
				return false, err
			}

			if len(ss.Items) != 1 {
				return false, fmt.Errorf("StatefulSet is not found")
			}

			return *ss.Items[0].Spec.Replicas == ss.Items[0].Status.Replicas, nil
		})
}

func (c *Cluster) waitPodLabelsReady() error {
	ls := c.labelsSet()

	listOptions := v1.ListOptions{
		LabelSelector: ls.String(),
	}
	masterListOption := v1.ListOptions{
		LabelSelector: labels.Merge(ls, labels.Set{"spilo-role": "master"}).String(),
	}
	replicaListOption := v1.ListOptions{
		LabelSelector: labels.Merge(ls, labels.Set{"spilo-role": "replica"}).String(),
	}
	pods, err := c.config.KubeClient.Pods(c.config.Namespace).List(listOptions)
	if err != nil {
		return err
	}
	podsNumber := len(pods.Items)

	return retryutil.Retry(
		constants.ResourceCheckInterval, int(constants.ResourceCheckTimeout/constants.ResourceCheckInterval),
		func() (bool, error) {
			masterPods, err := c.config.KubeClient.Pods(c.config.Namespace).List(masterListOption)
			if err != nil {
				return false, err
			}
			replicaPods, err := c.config.KubeClient.Pods(c.config.Namespace).List(replicaListOption)
			if err != nil {
				return false, err
			}
			if len(masterPods.Items) > 1 {
				return false, fmt.Errorf("Too many masters")
			}

			return len(masterPods.Items)+len(replicaPods.Items) == podsNumber, nil
		})
}

func (c *Cluster) Create() error {
	c.createEndpoint()
	c.createService()
	c.applySecrets()
	c.createStatefulSet()

	c.logger.Info("Waiting for cluster being ready")
	err := c.waitClusterReady()
	if err != nil {
		c.logger.Errorf("Failed to create cluster: %s", err)
		return err
	}
	c.logger.Info("Cluster is ready")

	err = c.initDbConn()
	if err != nil {
		return fmt.Errorf("Failed to init db connection: %s", err)
	}

	c.createUsers()

	return nil
}

func (c *Cluster) waitClusterReady() error {
	// TODO: wait for the first Pod only
	err := c.waitStatefulsetReady()
	if err != nil {
		return fmt.Errorf("Statuful set error: %s", err)
	}

	// TODO: wait only for master
	err = c.waitPodLabelsReady()
	if err != nil {
		return fmt.Errorf("Pod labels error: %s", err)
	}

	return nil
}

func (c *Cluster) Delete() error {
	clusterName := (*c.cluster).Metadata.Name
	namespace := c.config.Namespace
	orphanDependents := false
	deleteOptions := &v1.DeleteOptions{
		OrphanDependents: &orphanDependents,
	}

	listOptions := v1.ListOptions{
		LabelSelector: c.labelsSet().String(),
	}

	kubeClient := c.config.KubeClient

	podList, err := kubeClient.Pods(namespace).List(listOptions)
	if err != nil {
		return fmt.Errorf("Can't get list of pods: %s", err)
	}

	err = kubeClient.StatefulSets(namespace).Delete(clusterName, deleteOptions)
	if err != nil {
		return fmt.Errorf("Can't delete statefulset: %s", err)
	}
	c.logger.Infof("Statefulset '%s' has been deleted", util.FullObjectName(namespace, clusterName))

	for _, pod := range podList.Items {
		err = kubeClient.Pods(namespace).Delete(pod.Name, deleteOptions)
		if err != nil {
			return fmt.Errorf("Error while deleting pod '%s': %s", util.FullObjectName(pod.Namespace, pod.Name), err)
		}

		c.logger.Infof("Pod '%s' has been deleted", util.FullObjectName(pod.Namespace, pod.Name))
	}

	serviceList, err := kubeClient.Services(namespace).List(listOptions)
	if err != nil {
		return fmt.Errorf("Can't get list of the services: %s", err)
	}

	for _, service := range serviceList.Items {
		err = kubeClient.Services(namespace).Delete(service.Name, deleteOptions)
		if err != nil {
			return fmt.Errorf("Can't delete service '%s': %s", util.FullObjectName(service.Namespace, service.Name), err)
		}

		c.logger.Infof("Service '%s' has been deleted", util.FullObjectName(service.Namespace, service.Name))
	}

	secretsList, err := kubeClient.Secrets(namespace).List(listOptions)
	if err != nil {
		return err
	}
	for _, secret := range secretsList.Items {
		err = kubeClient.Secrets(namespace).Delete(secret.Name, deleteOptions)
		if err != nil {
			return fmt.Errorf("Can't delete secret '%s': %s", util.FullObjectName(secret.Namespace, secret.Name), err)
		}

		c.logger.Infof("Secret '%s' has been deleted", util.FullObjectName(secret.Namespace, secret.Name))
	}

	c.waitPodDelete()

	etcdKey := fmt.Sprintf("/service/%s", clusterName)

	resp, err := c.config.EtcdClient.Delete(context.Background(),
		etcdKey,
		&etcdclient.DeleteOptions{Recursive: true})

	if err != nil {
		return fmt.Errorf("Can't delete etcd key: %s", err)
	}

	if resp == nil {
		c.logger.Warningf("No response from etcd cluster")
	}

	c.logger.Infof("Etcd key '%s' has been deleted", etcdKey)

	//TODO: Ensure objects are deleted

	return nil
}