Pod Rolling update

introduce Pod events channel;
add parsing of the MaintenanceWindows section;
skip deleting Etcd key on cluster delete;
use external etcd host;
watch for tpr/pods in the namespace of the operator pod only;
This commit is contained in:
Murat Kabilov
2017-05-12 11:41:25 +02:00
parent 2b8956bd33
commit ae77fa15e8
23 changed files with 1763 additions and 790 deletions
+49 -118
View File
@@ -1,23 +1,18 @@
package controller
import (
"fmt"
"sync"
"github.com/Sirupsen/logrus"
etcdclient "github.com/coreos/etcd/client"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/pkg/api/v1"
v1beta1extensions "k8s.io/client-go/pkg/apis/extensions/v1beta1"
"k8s.io/client-go/pkg/fields"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
"github.bus.zalan.do/acid/postgres-operator/pkg/cluster"
"github.bus.zalan.do/acid/postgres-operator/pkg/spec"
"github.bus.zalan.do/acid/postgres-operator/pkg/util"
"github.bus.zalan.do/acid/postgres-operator/pkg/util/constants"
"github.bus.zalan.do/acid/postgres-operator/pkg/util/k8sutil"
"github.bus.zalan.do/acid/postgres-operator/pkg/util/teams"
)
@@ -32,25 +27,23 @@ type Config struct {
type Controller struct {
config Config
logger *logrus.Entry
events chan *Event
clusters map[string]*cluster.Cluster
stopChMap map[string]chan struct{}
clusters map[spec.ClusterName]*cluster.Cluster
stopChMap map[spec.ClusterName]chan struct{}
waitCluster sync.WaitGroup
postgresqlInformer cache.SharedIndexInformer
}
podInformer cache.SharedIndexInformer
type Event struct {
Type string
Object *spec.Postgresql
podCh chan spec.PodEvent
}
func New(cfg *Config) *Controller {
return &Controller{
config: *cfg,
logger: logrus.WithField("pkg", "controller"),
clusters: make(map[string]*cluster.Cluster),
stopChMap: make(map[string]chan struct{}),
clusters: make(map[spec.ClusterName]*cluster.Cluster),
stopChMap: make(map[spec.ClusterName]chan struct{}),
podCh: make(chan spec.PodEvent),
}
}
@@ -65,127 +58,65 @@ func (c *Controller) Run(stopCh <-chan struct{}, wg *sync.WaitGroup) {
return
}
go c.watchTpr(stopCh)
c.logger.Infof("'%s' namespace will be watched", c.config.PodNamespace)
go c.runInformers(stopCh)
c.logger.Info("Started working in background")
}
func (c *Controller) watchTpr(stopCh <-chan struct{}) {
go c.postgresqlInformer.Run(stopCh)
<-stopCh
}
func (c *Controller) createTPR() error {
TPRName := fmt.Sprintf("%s.%s", constants.TPRName, constants.TPRVendor)
tpr := &v1beta1extensions.ThirdPartyResource{
ObjectMeta: v1.ObjectMeta{
Name: TPRName,
//PodNamespace: c.config.PodNamespace, //ThirdPartyResources are cluster-wide
},
Versions: []v1beta1extensions.APIVersion{
{Name: constants.TPRApiVersion},
},
Description: constants.TPRDescription,
}
_, err := c.config.KubeClient.ExtensionsV1beta1().ThirdPartyResources().Create(tpr)
if err != nil {
if !k8sutil.IsKubernetesResourceAlreadyExistError(err) {
return err
} else {
c.logger.Infof("ThirdPartyResource '%s' is already registered", TPRName)
}
} else {
c.logger.Infof("ThirdPartyResource '%s' has been registered", TPRName)
}
restClient := c.config.RestClient
return k8sutil.WaitTPRReady(restClient, constants.TPRReadyWaitInterval, constants.TPRReadyWaitTimeout, c.config.PodNamespace)
}
func (c *Controller) makeClusterConfig() cluster.Config {
return cluster.Config{
ControllerNamespace: c.config.PodNamespace,
KubeClient: c.config.KubeClient,
RestClient: c.config.RestClient,
EtcdClient: c.config.EtcdClient,
TeamsAPIClient: c.config.TeamsAPIClient,
}
}
func (c *Controller) initController() {
err := c.createTPR()
if err != nil {
c.logger.Fatalf("Can't register ThirdPartyResource: %s", err)
}
token, err := c.getOAuthToken()
if err != nil {
c.logger.Errorf("Can't get OAuth token: %s", err)
} else {
c.config.TeamsAPIClient.OAuthToken = token
}
// Postgresqls
clusterLw := &cache.ListWatch{
ListFunc: c.clusterListFunc,
WatchFunc: c.clusterWatchFunc,
}
c.postgresqlInformer = cache.NewSharedIndexInformer(
cache.NewListWatchFromClient(c.config.RestClient, constants.ResourceName, v1.NamespaceAll, fields.Everything()),
clusterLw,
&spec.Postgresql{},
constants.ResyncPeriod,
constants.ResyncPeriodTPR,
cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
c.postgresqlInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: c.clusterAdd,
UpdateFunc: c.clusterUpdate,
DeleteFunc: c.clusterDelete,
AddFunc: c.postgresqlAdd,
UpdateFunc: c.postgresqlUpdate,
DeleteFunc: c.postgresqlDelete,
})
// Pods
podLw := &cache.ListWatch{
ListFunc: c.podListFunc,
WatchFunc: c.podWatchFunc,
}
c.podInformer = cache.NewSharedIndexInformer(
podLw,
&v1.Pod{},
constants.ResyncPeriodPod,
cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
c.podInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: c.podAdd,
UpdateFunc: c.podUpdate,
DeleteFunc: c.podDelete,
})
}
func (c *Controller) clusterAdd(obj interface{}) {
pg := obj.(*spec.Postgresql)
func (c *Controller) runInformers(stopCh <-chan struct{}) {
go c.postgresqlInformer.Run(stopCh)
go c.podInformer.Run(stopCh)
go c.podEventsDispatcher(stopCh)
//TODO: why do we need to have this check
if pg.Spec == nil {
return
}
clusterName := (*pg).Metadata.Name
cl := cluster.New(c.makeClusterConfig(), pg)
err := cl.Create()
if err != nil {
c.logger.Errorf("Can't create cluster: %s", err)
return
}
c.stopChMap[clusterName] = make(chan struct{})
c.clusters[clusterName] = cl
c.logger.Infof("Postgresql cluster '%s' has been created", util.FullObjectNameFromMeta((*pg).Metadata))
}
func (c *Controller) clusterUpdate(prev, cur interface{}) {
pgPrev := prev.(*spec.Postgresql)
pgCur := cur.(*spec.Postgresql)
if pgPrev.Spec == nil || pgCur.Spec == nil {
return
}
if pgPrev.Metadata.ResourceVersion == pgCur.Metadata.ResourceVersion {
return
}
c.logger.Infof("Update: %+v -> %+v", *pgPrev, *pgCur)
}
func (c *Controller) clusterDelete(obj interface{}) {
pg := obj.(*spec.Postgresql)
if pg.Spec == nil {
return
}
clusterName := (*pg).Metadata.Name
cluster := cluster.New(c.makeClusterConfig(), pg)
err := cluster.Delete()
if err != nil {
c.logger.Errorf("Can't delete cluster '%s': %s", util.FullObjectNameFromMeta((*pg).Metadata), err)
return
}
close(c.stopChMap[clusterName])
delete(c.clusters, clusterName)
c.logger.Infof("Cluster has been deleted: '%s'", util.FullObjectNameFromMeta((*pg).Metadata))
<-stopCh
}
+2 -1
View File
@@ -2,9 +2,10 @@ package controller
import (
"fmt"
"time"
"github.bus.zalan.do/acid/postgres-operator/pkg/util/constants"
etcdclient "github.com/coreos/etcd/client"
"time"
)
func (c *Controller) initEtcdClient() error {
+134
View File
@@ -0,0 +1,134 @@
package controller
import (
"k8s.io/client-go/pkg/api"
"k8s.io/client-go/pkg/api/v1"
"k8s.io/client-go/pkg/runtime"
"k8s.io/client-go/pkg/watch"
"github.bus.zalan.do/acid/postgres-operator/pkg/spec"
"github.bus.zalan.do/acid/postgres-operator/pkg/util"
)
func (c *Controller) podListFunc(options api.ListOptions) (runtime.Object, error) {
var labelSelector string
var fieldSelector string
if options.LabelSelector != nil {
labelSelector = options.LabelSelector.String()
}
if options.FieldSelector != nil {
fieldSelector = options.FieldSelector.String()
}
opts := v1.ListOptions{
LabelSelector: labelSelector,
FieldSelector: fieldSelector,
Watch: options.Watch,
ResourceVersion: options.ResourceVersion,
TimeoutSeconds: options.TimeoutSeconds,
}
return c.config.KubeClient.CoreV1().Pods(c.config.PodNamespace).List(opts)
}
func (c *Controller) podWatchFunc(options api.ListOptions) (watch.Interface, error) {
var labelSelector string
var fieldSelector string
if options.LabelSelector != nil {
labelSelector = options.LabelSelector.String()
}
if options.FieldSelector != nil {
fieldSelector = options.FieldSelector.String()
}
opts := v1.ListOptions{
LabelSelector: labelSelector,
FieldSelector: fieldSelector,
Watch: options.Watch,
ResourceVersion: options.ResourceVersion,
TimeoutSeconds: options.TimeoutSeconds,
}
return c.config.KubeClient.CoreV1Client.Pods(c.config.PodNamespace).Watch(opts)
}
func PodNameFromMeta(meta v1.ObjectMeta) spec.PodName {
return spec.PodName{
Namespace: meta.Namespace,
Name: meta.Name,
}
}
func (c *Controller) podAdd(obj interface{}) {
pod, ok := obj.(*v1.Pod)
if !ok {
return
}
podEvent := spec.PodEvent{
ClusterName: util.PodClusterName(pod),
PodName: PodNameFromMeta(pod.ObjectMeta),
CurPod: pod,
EventType: spec.PodEventAdd,
}
c.podCh <- podEvent
}
func (c *Controller) podUpdate(prev, cur interface{}) {
prevPod, ok := prev.(*v1.Pod)
if !ok {
return
}
curPod, ok := cur.(*v1.Pod)
if !ok {
return
}
podEvent := spec.PodEvent{
ClusterName: util.PodClusterName(curPod),
PodName: PodNameFromMeta(curPod.ObjectMeta),
PrevPod: prevPod,
CurPod: curPod,
EventType: spec.PodEventUpdate,
}
c.podCh <- podEvent
}
func (c *Controller) podDelete(obj interface{}) {
pod, ok := obj.(*v1.Pod)
if !ok {
return
}
podEvent := spec.PodEvent{
ClusterName: util.PodClusterName(pod),
PodName: PodNameFromMeta(pod.ObjectMeta),
CurPod: pod,
EventType: spec.PodEventDelete,
}
c.podCh <- podEvent
}
func (c *Controller) podEventsDispatcher(stopCh <-chan struct{}) {
c.logger.Infof("Watching all pod events")
for {
select {
case event := <-c.podCh:
if subscriber, ok := c.clusters[event.ClusterName]; ok {
c.logger.Debugf("Sending %s event of pod '%s' to the '%s' cluster channel", event.EventType, event.PodName, event.ClusterName)
go subscriber.ReceivePodEvent(event)
} else {
c.logger.Debugf("Skipping pods unrelated to clusters: %s", event.PodName)
}
case <-stopCh:
return
}
}
}
+189
View File
@@ -0,0 +1,189 @@
package controller
import (
"fmt"
"reflect"
"k8s.io/client-go/pkg/api"
"k8s.io/client-go/pkg/api/meta"
"k8s.io/client-go/pkg/fields"
"k8s.io/client-go/pkg/runtime"
"k8s.io/client-go/pkg/watch"
"github.bus.zalan.do/acid/postgres-operator/pkg/cluster"
"github.bus.zalan.do/acid/postgres-operator/pkg/spec"
"github.bus.zalan.do/acid/postgres-operator/pkg/util"
"github.bus.zalan.do/acid/postgres-operator/pkg/util/constants"
)
func (c *Controller) clusterListFunc(options api.ListOptions) (runtime.Object, error) {
c.logger.Info("Getting list of currently running clusters")
object, err := c.config.RestClient.Get().
Namespace(c.config.PodNamespace).
Resource(constants.ResourceName).
VersionedParams(&options, api.ParameterCodec).
FieldsSelectorParam(fields.Everything()).
Do().
Get()
if err != nil {
return nil, fmt.Errorf("Can't get list of postgresql objects: %s", err)
}
objList, err := meta.ExtractList(object)
if err != nil {
return nil, fmt.Errorf("Can't extract list of postgresql objects: %s", err)
}
clusterConfig := c.makeClusterConfig()
for _, obj := range objList {
pg, ok := obj.(*spec.Postgresql)
if !ok {
return nil, fmt.Errorf("Can't cast object to postgresql")
}
clusterName := spec.ClusterName{
Namespace: pg.Metadata.Namespace,
Name: pg.Metadata.Name,
}
cl := cluster.New(clusterConfig, *pg)
stopCh := make(chan struct{})
c.stopChMap[clusterName] = stopCh
c.clusters[clusterName] = cl
cl.LoadResources()
cl.ListResources()
go cl.Run(stopCh)
}
if len(c.clusters) > 0 {
c.logger.Infof("There are %d clusters currently running", len(c.clusters))
} else {
c.logger.Infof("No clusters running")
}
return object, err
}
func (c *Controller) clusterWatchFunc(options api.ListOptions) (watch.Interface, error) {
return c.config.RestClient.Get().
Prefix("watch").
Namespace(c.config.PodNamespace).
Resource(constants.ResourceName).
VersionedParams(&options, api.ParameterCodec).
FieldsSelectorParam(fields.Everything()).
Watch()
}
func (c *Controller) postgresqlAdd(obj interface{}) {
pg, ok := obj.(*spec.Postgresql)
if !ok {
c.logger.Errorf("Can't cast to postgresql spec")
return
}
clusterName := spec.ClusterName{
Namespace: pg.Metadata.Namespace,
Name: pg.Metadata.Name,
}
_, ok = c.clusters[clusterName]
if ok {
c.logger.Infof("Cluster '%s' already exists", clusterName)
return
}
c.logger.Infof("Creation of a new Postgresql cluster '%s' started", clusterName)
cl := cluster.New(c.makeClusterConfig(), *pg)
cl.MustSetStatus(spec.ClusterStatusCreating)
err := cl.Create()
if err != nil {
c.logger.Errorf("Can't create cluster: %s", err)
cl.MustSetStatus(spec.ClusterStatusAddFailed)
return
}
cl.MustSetStatus(spec.ClusterStatusRunning) //TODO: are you sure it's running?
stopCh := make(chan struct{})
c.stopChMap[clusterName] = stopCh
c.clusters[clusterName] = cl
go cl.Run(stopCh)
c.logger.Infof("Postgresql cluster '%s' has been created", clusterName)
}
func (c *Controller) postgresqlUpdate(prev, cur interface{}) {
pgPrev, ok := prev.(*spec.Postgresql)
if !ok {
c.logger.Errorf("Can't cast to postgresql spec")
}
pgNew, ok := cur.(*spec.Postgresql)
if !ok {
c.logger.Errorf("Can't cast to postgresql spec")
}
clusterName := spec.ClusterName{
Namespace: pgNew.Metadata.Namespace,
Name: pgNew.Metadata.Name,
}
//TODO: Do not update cluster which is currently creating
if pgPrev.Metadata.ResourceVersion == pgNew.Metadata.ResourceVersion {
c.logger.Debugf("Skipping update with no resource version change")
return
}
pgCluster := c.clusters[clusterName] // current
if reflect.DeepEqual(pgPrev.Spec, pgNew.Spec) {
c.logger.Infof("Skipping update with no spec change")
return
}
c.logger.Infof("Cluster update: %s(version: %s) -> %s(version: %s)",
util.NameFromMeta(pgPrev.Metadata), pgPrev.Metadata.ResourceVersion,
util.NameFromMeta(pgNew.Metadata), pgNew.Metadata.ResourceVersion)
rollingUpdate := pgCluster.NeedsRollingUpdate(pgNew)
if rollingUpdate {
c.logger.Infof("Pods need to be recreated")
}
pgCluster.MustSetStatus(spec.ClusterStatusUpdating)
err := pgCluster.Update(pgNew, rollingUpdate)
if err != nil {
pgCluster.MustSetStatus(spec.ClusterStatusUpdateFailed)
c.logger.Errorf("Can't update cluster: %s", err)
} else {
c.logger.Infof("Cluster has been updated")
}
}
func (c *Controller) postgresqlDelete(obj interface{}) {
pgCur, ok := obj.(*spec.Postgresql)
if !ok {
c.logger.Errorf("Can't cast to postgresql spec")
return
}
clusterName := spec.ClusterName{
Namespace: pgCur.Metadata.Namespace,
Name: pgCur.Metadata.Name,
}
pgCluster, ok := c.clusters[clusterName]
if !ok {
c.logger.Errorf("Unknown cluster: %s", clusterName)
return
}
c.logger.Infof("Cluster delete: %s", util.NameFromMeta(pgCur.Metadata))
err := pgCluster.Delete()
if err != nil {
c.logger.Errorf("Can't delete cluster '%s': %s", clusterName, err)
return
}
close(c.stopChMap[clusterName])
delete(c.clusters, clusterName)
c.logger.Infof("Cluster '%s' has been successfully deleted", clusterName)
}
+57
View File
@@ -0,0 +1,57 @@
package controller
import (
"fmt"
"github.bus.zalan.do/acid/postgres-operator/pkg/cluster"
"github.bus.zalan.do/acid/postgres-operator/pkg/util/constants"
"github.bus.zalan.do/acid/postgres-operator/pkg/util/k8sutil"
"github.bus.zalan.do/acid/postgres-operator/pkg/util/resources"
"k8s.io/client-go/pkg/api"
)
func (c *Controller) makeClusterConfig() cluster.Config {
return cluster.Config{
ControllerNamespace: c.config.PodNamespace,
KubeClient: c.config.KubeClient,
RestClient: c.config.RestClient,
EtcdClient: c.config.EtcdClient,
TeamsAPIClient: c.config.TeamsAPIClient,
}
}
func (c *Controller) getOAuthToken() (string, error) {
// Temporary getting postgresql-operator secret from the NamespaceDefault
credentialsSecret, err := c.config.KubeClient.Secrets(api.NamespaceDefault).Get(constants.OAuthTokenSecretName)
if err != nil {
return "", fmt.Errorf("Can't get credentials secret: %s", err)
}
data := credentialsSecret.Data
if string(data["read-only-token-type"]) != "Bearer" {
return "", fmt.Errorf("Wrong token type: %s", data["read-only-token-type"])
}
return string(data["read-only-token-secret"]), nil
}
func (c *Controller) createTPR() error {
TPRName := fmt.Sprintf("%s.%s", constants.TPRName, constants.TPRVendor)
tpr := resources.ThirdPartyResource(TPRName)
_, err := c.config.KubeClient.ExtensionsV1beta1().ThirdPartyResources().Create(tpr)
if err != nil {
if !k8sutil.ResourceAlreadyExists(err) {
return err
} else {
c.logger.Infof("ThirdPartyResource '%s' is already registered", TPRName)
}
} else {
c.logger.Infof("ThirdPartyResource '%s' has been registered", TPRName)
}
restClient := c.config.RestClient
return k8sutil.WaitTPRReady(restClient, constants.TPRReadyWaitInterval, constants.TPRReadyWaitTimeout, c.config.PodNamespace)
}