Merge branch 'feature/refactor-tpr' into fix/graceful-shutdown

# Conflicts:
#	pkg/controller/controller.go
This commit is contained in:
Murat Kabilov
2017-07-25 10:38:33 +02:00
28 changed files with 509 additions and 322 deletions
+83 -42
View File
@@ -5,7 +5,7 @@ import (
"sync"
"github.com/Sirupsen/logrus"
"k8s.io/client-go/kubernetes"
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/pkg/api/v1"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
@@ -14,21 +14,26 @@ import (
"github.com/zalando-incubator/postgres-operator/pkg/spec"
"github.com/zalando-incubator/postgres-operator/pkg/util/config"
"github.com/zalando-incubator/postgres-operator/pkg/util/constants"
"github.com/zalando-incubator/postgres-operator/pkg/util/teams"
"github.com/zalando-incubator/postgres-operator/pkg/util/k8sutil"
)
type Config struct {
RestConfig *rest.Config
KubeClient *kubernetes.Clientset
RestClient *rest.RESTClient
TeamsAPIClient *teams.API
InfrastructureRoles map[string]spec.PgUser
NoDatabaseAccess bool
NoTeamsAPI bool
ConfigMapName spec.NamespacedName
Namespace string
}
type Controller struct {
Config
config Config
opConfig *config.Config
logger *logrus.Entry
logger *logrus.Entry
KubeClient k8sutil.KubernetesClient
RestClient rest.Interface
clustersMu sync.RWMutex
clusters map[spec.NamespacedName]*cluster.Cluster
@@ -38,23 +43,16 @@ type Controller struct {
podInformer cache.SharedIndexInformer
podCh chan spec.PodEvent
clusterEventQueues []*cache.FIFO
clusterEventQueues []*cache.FIFO
lastClusterSyncTime int64
}
func New(controllerConfig *Config, operatorConfig *config.Config) *Controller {
func NewController(controllerConfig *Config) *Controller {
logger := logrus.New()
if operatorConfig.DebugLogging {
logger.Level = logrus.DebugLevel
}
controllerConfig.TeamsAPIClient = teams.NewTeamsAPI(operatorConfig.TeamsAPIUrl, logger)
return &Controller{
Config: *controllerConfig,
opConfig: operatorConfig,
config: *controllerConfig,
opConfig: &config.Config{},
logger: logger.WithField("pkg", "controller"),
clusters: make(map[spec.NamespacedName]*cluster.Cluster),
stopChs: make(map[spec.NamespacedName]chan struct{}),
@@ -62,45 +60,76 @@ func New(controllerConfig *Config, operatorConfig *config.Config) *Controller {
}
}
func (c *Controller) Run(stopCh <-chan struct{}, wg *sync.WaitGroup) {
func (c *Controller) initClients() {
client, err := k8sutil.ClientSet(c.config.RestConfig)
if err != nil {
c.logger.Fatalf("couldn't create client: %v", err)
}
c.KubeClient = k8sutil.NewFromKubernetesInterface(client)
c.initController()
c.RestClient, err = k8sutil.KubernetesRestClient(*c.config.RestConfig)
if err != nil {
c.logger.Fatalf("couldn't create rest client: %v", err)
}
}
wg.Add(4)
go c.runPodInformer(stopCh, wg)
go c.runPostgresqlInformer(stopCh, wg)
go c.podEventsDispatcher(stopCh, wg)
go c.clusterResync(stopCh, wg)
func (c *Controller) initOperatorConfig() {
configMapData := make(map[string]string)
for i := range c.clusterEventQueues {
wg.Add(1)
go c.processClusterEventsQueue(stopCh, i, wg)
if c.config.ConfigMapName != (spec.NamespacedName{}) {
configMap, err := c.KubeClient.ConfigMaps(c.config.ConfigMapName.Namespace).
Get(c.config.ConfigMapName.Name, meta_v1.GetOptions{})
if err != nil {
panic(err)
}
configMapData = configMap.Data
} else {
c.logger.Infoln("No ConfigMap specified. Loading default values")
}
c.logger.Info("Started working in background")
if configMapData["namespace"] == "" { // Namespace in ConfigMap has priority over env var
configMapData["namespace"] = c.config.Namespace
}
if c.config.NoDatabaseAccess {
configMapData["enable_database_access"] = "false"
}
if c.config.NoTeamsAPI {
configMapData["enable_teams_api"] = "false"
}
c.opConfig = config.NewFromMap(configMapData)
}
func (c *Controller) initController() {
c.initClients()
c.initOperatorConfig()
c.logger.Infof("Config: %s", c.opConfig.MustMarshal())
if c.opConfig.DebugLogging {
c.logger.Level = logrus.DebugLevel
}
if err := c.createTPR(); err != nil {
c.logger.Fatalf("could not register ThirdPartyResource: %v", err)
}
if infraRoles, err := c.getInfrastructureRoles(); err != nil {
if infraRoles, err := c.getInfrastructureRoles(&c.opConfig.InfrastructureRolesSecretName); err != nil {
c.logger.Warningf("could not get infrastructure roles: %v", err)
} else {
c.InfrastructureRoles = infraRoles
c.config.InfrastructureRoles = infraRoles
}
// Postgresqls
clusterLw := &cache.ListWatch{
ListFunc: c.clusterListFunc,
WatchFunc: c.clusterWatchFunc,
}
c.postgresqlInformer = cache.NewSharedIndexInformer(
clusterLw,
&cache.ListWatch{
ListFunc: c.clusterListFunc,
WatchFunc: c.clusterWatchFunc,
},
&spec.Postgresql{},
constants.QueueResyncPeriodTPR,
cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
cache.Indexers{})
c.postgresqlInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: c.postgresqlAdd,
@@ -139,14 +168,26 @@ func (c *Controller) initController() {
}
}
func (c *Controller) runPodInformer(stopCh <-chan struct{}, wg *sync.WaitGroup) {
func (c *Controller) Run(stopCh <-chan struct{}, wg *sync.WaitGroup) {
defer wg.Done()
wg.Add(1)
c.podInformer.Run(stopCh)
c.initController()
go c.runInformers(stopCh)
for i := range c.clusterEventQueues {
go c.processClusterEventsQueue(i)
}
c.logger.Info("Started working in background")
}
func (c *Controller) runPostgresqlInformer(stopCh <-chan struct{}, wg *sync.WaitGroup) {
defer wg.Done()
func (c *Controller) runInformers(stopCh <-chan struct{}) {
go c.postgresqlInformer.Run(stopCh)
go c.podInformer.Run(stopCh)
go c.podEventsDispatcher(stopCh)
go c.clusterResync(stopCh)
c.postgresqlInformer.Run(stopCh)
<-stopCh
}
+3 -3
View File
@@ -24,7 +24,7 @@ func (c *Controller) podListFunc(options meta_v1.ListOptions) (runtime.Object, e
TimeoutSeconds: options.TimeoutSeconds,
}
return c.KubeClient.CoreV1().Pods(c.opConfig.Namespace).List(opts)
return c.KubeClient.Pods(c.opConfig.Namespace).List(opts)
}
func (c *Controller) podWatchFunc(options meta_v1.ListOptions) (watch.Interface, error) {
@@ -39,7 +39,7 @@ func (c *Controller) podWatchFunc(options meta_v1.ListOptions) (watch.Interface,
TimeoutSeconds: options.TimeoutSeconds,
}
return c.KubeClient.CoreV1Client.Pods(c.opConfig.Namespace).Watch(opts)
return c.KubeClient.Pods(c.opConfig.Namespace).Watch(opts)
}
func (c *Controller) podAdd(obj interface{}) {
@@ -111,7 +111,7 @@ func (c *Controller) podEventsDispatcher(stopCh <-chan struct{}, wg *sync.WaitGr
c.clustersMu.RUnlock()
if ok {
c.logger.Debugf("Sending %s event of pod '%s' to the '%s' cluster channel", event.EventType, event.PodName, event.ClusterName)
c.logger.Debugf("Sending %q event of pod %q to the %q cluster channel", event.EventType, event.PodName, event.ClusterName)
cluster.ReceivePodEvent(event)
}
case <-stopCh:
+72 -50
View File
@@ -1,19 +1,17 @@
package controller
import (
"encoding/json"
"fmt"
"reflect"
"sync"
"sync/atomic"
"time"
"k8s.io/apimachinery/pkg/api/meta"
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/pkg/api"
"k8s.io/client-go/tools/cache"
"github.com/zalando-incubator/postgres-operator/pkg/cluster"
@@ -37,44 +35,35 @@ func (c *Controller) clusterResync(stopCh <-chan struct{}, wg *sync.WaitGroup) {
}
func (c *Controller) clusterListFunc(options meta_v1.ListOptions) (runtime.Object, error) {
c.logger.Info("Getting list of currently running clusters")
var list spec.PostgresqlList
var activeClustersCnt, failedClustersCnt int
req := c.RestClient.Get().
RequestURI(fmt.Sprintf(constants.ListClustersURITemplate, c.opConfig.Namespace)).
VersionedParams(&options, api.ParameterCodec).
FieldsSelectorParam(fields.Everything())
object, err := req.Do().Get()
req := c.RestClient.
Get().
Namespace(c.opConfig.Namespace).
Resource(constants.ResourceName).
VersionedParams(&options, meta_v1.ParameterCodec)
b, err := req.DoRaw()
if err != nil {
return nil, fmt.Errorf("could not get list of postgresql objects: %v", err)
}
objList, err := meta.ExtractList(object)
if err != nil {
return nil, fmt.Errorf("could not extract list of postgresql objects: %v", err)
return nil, err
}
err = json.Unmarshal(b, &list)
if time.Now().Unix()-atomic.LoadInt64(&c.lastClusterSyncTime) <= int64(c.opConfig.ResyncPeriod.Seconds()) {
c.logger.Debugln("skipping resync of clusters")
return object, err
return &list, err
}
var activeClustersCnt, failedClustersCnt int
for _, obj := range objList {
pg, ok := obj.(*spec.Postgresql)
if !ok {
return nil, fmt.Errorf("could not cast object to postgresql")
}
for _, pg := range list.Items {
if pg.Error != nil {
failedClustersCnt++
continue
}
c.queueClusterEvent(nil, pg, spec.EventSync)
c.queueClusterEvent(nil, &pg, spec.EventSync)
activeClustersCnt++
}
if len(objList) > 0 {
if len(list.Items) > 0 {
if failedClustersCnt > 0 && activeClustersCnt == 0 {
c.logger.Infof("There are no clusters running. %d are in the failed state", failedClustersCnt)
} else if failedClustersCnt == 0 && activeClustersCnt > 0 {
@@ -88,15 +77,48 @@ func (c *Controller) clusterListFunc(options meta_v1.ListOptions) (runtime.Objec
atomic.StoreInt64(&c.lastClusterSyncTime, time.Now().Unix())
return object, err
return &list, err
}
type tprDecoder struct {
dec *json.Decoder
close func() error
}
func (d *tprDecoder) Close() {
d.close()
}
func (d *tprDecoder) Decode() (action watch.EventType, object runtime.Object, err error) {
var e struct {
Type watch.EventType
Object spec.Postgresql
}
if err := d.dec.Decode(&e); err != nil {
return watch.Error, nil, err
}
return e.Type, &e.Object, nil
}
func (c *Controller) clusterWatchFunc(options meta_v1.ListOptions) (watch.Interface, error) {
req := c.RestClient.Get().
RequestURI(fmt.Sprintf(constants.WatchClustersURITemplate, c.opConfig.Namespace)).
VersionedParams(&options, api.ParameterCodec).
FieldsSelectorParam(fields.Everything())
return req.Watch()
options.Watch = true
r, err := c.RestClient.
Get().
Namespace(c.opConfig.Namespace).
Resource(constants.ResourceName).
VersionedParams(&options, meta_v1.ParameterCodec).
FieldsSelectorParam(nil).
Stream()
if err != nil {
return nil, err
}
return watch.NewStreamWatcher(&tprDecoder{
dec: json.NewDecoder(r),
close: r.Close,
}), nil
}
func (c *Controller) processEvent(obj interface{}) error {
@@ -121,14 +143,14 @@ func (c *Controller) processEvent(obj interface{}) error {
switch event.EventType {
case spec.EventAdd:
if clusterFound {
logger.Debugf("Cluster '%s' already exists", clusterName)
logger.Debugf("Cluster %q already exists", clusterName)
return nil
}
logger.Infof("Creation of the '%s' cluster started", clusterName)
logger.Infof("Creation of the %q cluster started", clusterName)
stopCh := make(chan struct{})
cl = cluster.New(c.makeClusterConfig(), *event.NewSpec, logger)
cl = cluster.New(c.makeClusterConfig(), c.KubeClient, *event.NewSpec, logger)
cl.Run(stopCh)
c.clustersMu.Lock()
@@ -143,31 +165,31 @@ func (c *Controller) processEvent(obj interface{}) error {
return nil
}
logger.Infof("Cluster '%s' has been created", clusterName)
logger.Infof("Cluster %q has been created", clusterName)
case spec.EventUpdate:
logger.Infof("Update of the '%s' cluster started", clusterName)
logger.Infof("Update of the %q cluster started", clusterName)
if !clusterFound {
logger.Warnf("Cluster '%s' does not exist", clusterName)
logger.Warnf("Cluster %q does not exist", clusterName)
return nil
}
if err := cl.Update(event.NewSpec); err != nil {
cl.Error = fmt.Errorf("could not update cluster: %s", err)
cl.Error = fmt.Errorf("could not update cluster: %v", err)
logger.Errorf("%v", cl.Error)
return nil
}
cl.Error = nil
logger.Infof("Cluster '%s' has been updated", clusterName)
logger.Infof("Cluster %q has been updated", clusterName)
case spec.EventDelete:
logger.Infof("Deletion of the '%s' cluster started", clusterName)
logger.Infof("Deletion of the %q cluster started", clusterName)
if !clusterFound {
logger.Errorf("Unknown cluster: %s", clusterName)
logger.Errorf("Unknown cluster: %q", clusterName)
return nil
}
if err := cl.Delete(); err != nil {
logger.Errorf("could not delete cluster '%s': %s", clusterName, err)
logger.Errorf("could not delete cluster %q: %v", clusterName, err)
return nil
}
close(c.stopChs[clusterName])
@@ -177,14 +199,14 @@ func (c *Controller) processEvent(obj interface{}) error {
delete(c.stopChs, clusterName)
c.clustersMu.Unlock()
logger.Infof("Cluster '%s' has been deleted", clusterName)
logger.Infof("Cluster %q has been deleted", clusterName)
case spec.EventSync:
logger.Infof("Syncing of the '%s' cluster started", clusterName)
logger.Infof("Syncing of the %q cluster started", clusterName)
// no race condition because a cluster is always processed by single worker
if !clusterFound {
stopCh := make(chan struct{})
cl = cluster.New(c.makeClusterConfig(), *event.NewSpec, logger)
cl = cluster.New(c.makeClusterConfig(), c.KubeClient, *event.NewSpec, logger)
cl.Run(stopCh)
c.clustersMu.Lock()
@@ -194,13 +216,13 @@ func (c *Controller) processEvent(obj interface{}) error {
}
if err := cl.Sync(); err != nil {
cl.Error = fmt.Errorf("could not sync cluster '%s': %v", clusterName, err)
cl.Error = fmt.Errorf("could not sync cluster %q: %v", clusterName, err)
logger.Errorf("%v", cl.Error)
return nil
}
cl.Error = nil
logger.Infof("Cluster '%s' has been synced", clusterName)
logger.Infof("Cluster %q has been synced", clusterName)
}
return nil
@@ -241,7 +263,7 @@ func (c *Controller) queueClusterEvent(old, new *spec.Postgresql, eventType spec
}
if clusterError != nil && eventType != spec.EventDelete {
c.logger.Debugf("Skipping %s event for invalid cluster %s (reason: %v)", eventType, clusterName, clusterError)
c.logger.Debugf("Skipping %q event for invalid cluster %q (reason: %v)", eventType, clusterName, clusterError)
return
}
@@ -258,7 +280,7 @@ func (c *Controller) queueClusterEvent(old, new *spec.Postgresql, eventType spec
if err := c.clusterEventQueues[workerID].Add(clusterEvent); err != nil {
c.logger.WithField("worker", workerID).Errorf("error when queueing cluster event: %v", clusterEvent)
}
c.logger.WithField("worker", workerID).Infof("%s of the '%s' cluster has been queued", eventType, clusterName)
c.logger.WithField("worker", workerID).Infof("%q of the %q cluster has been queued", eventType, clusterName)
}
func (c *Controller) postgresqlAdd(obj interface{}) {
+12 -16
View File
@@ -17,15 +17,12 @@ import (
func (c *Controller) makeClusterConfig() cluster.Config {
infrastructureRoles := make(map[string]spec.PgUser)
for k, v := range c.InfrastructureRoles {
for k, v := range c.config.InfrastructureRoles {
infrastructureRoles[k] = v
}
return cluster.Config{
KubeClient: c.KubeClient,
RestClient: c.RestClient,
RestConfig: c.RestConfig,
TeamsAPIClient: c.TeamsAPIClient,
RestConfig: c.config.RestConfig,
OpConfig: config.Copy(c.opConfig),
InfrastructureRoles: infrastructureRoles,
}
@@ -49,33 +46,32 @@ func (c *Controller) clusterWorkerID(clusterName spec.NamespacedName) uint32 {
}
func (c *Controller) createTPR() error {
TPRName := fmt.Sprintf("%s.%s", constants.TPRName, constants.TPRVendor)
tpr := thirdPartyResource(TPRName)
tpr := thirdPartyResource(constants.TPRName)
_, err := c.KubeClient.ExtensionsV1beta1().ThirdPartyResources().Create(tpr)
_, err := c.KubeClient.ThirdPartyResources().Create(tpr)
if err != nil {
if !k8sutil.ResourceAlreadyExists(err) {
return err
}
c.logger.Infof("ThirdPartyResource '%s' is already registered", TPRName)
c.logger.Infof("ThirdPartyResource %q is already registered", constants.TPRName)
} else {
c.logger.Infof("ThirdPartyResource '%s' has been registered", TPRName)
c.logger.Infof("ThirdPartyResource %q' has been registered", constants.TPRName)
}
return k8sutil.WaitTPRReady(c.RestClient, c.opConfig.TPR.ReadyWaitInterval, c.opConfig.TPR.ReadyWaitTimeout, c.opConfig.Namespace)
}
func (c *Controller) getInfrastructureRoles() (result map[string]spec.PgUser, err error) {
if c.opConfig.InfrastructureRolesSecretName == (spec.NamespacedName{}) {
func (c *Controller) getInfrastructureRoles(rolesSecret *spec.NamespacedName) (result map[string]spec.PgUser, err error) {
if *rolesSecret == (spec.NamespacedName{}) {
// we don't have infrastructure roles defined, bail out
return nil, nil
}
infraRolesSecret, err := c.KubeClient.
Secrets(c.opConfig.InfrastructureRolesSecretName.Namespace).
Get(c.opConfig.InfrastructureRolesSecretName.Name, meta_v1.GetOptions{})
Secrets(rolesSecret.Namespace).
Get(rolesSecret.Name, meta_v1.GetOptions{})
if err != nil {
c.logger.Debugf("Infrastructure roles secret name: %s", c.opConfig.InfrastructureRolesSecretName)
c.logger.Debugf("Infrastructure roles secret name: %q", *rolesSecret)
return nil, fmt.Errorf("could not get infrastructure roles secret: %v", err)
}
@@ -103,7 +99,7 @@ Users:
case "inrole":
t.MemberOf = append(t.MemberOf, s)
default:
c.logger.Warnf("Unknown key %s", p)
c.logger.Warnf("Unknown key %q", p)
}
}
}
+154
View File
@@ -0,0 +1,154 @@
package controller
import (
"fmt"
"reflect"
"testing"
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/pkg/api/v1"
"github.com/zalando-incubator/postgres-operator/pkg/spec"
"github.com/zalando-incubator/postgres-operator/pkg/util/k8sutil"
)
const (
testInfrastructureRolesSecretName = "infrastructureroles-test"
)
type mockSecret struct {
v1core.SecretInterface
}
func (c *mockSecret) Get(name string, options meta_v1.GetOptions) (*v1.Secret, error) {
if name != testInfrastructureRolesSecretName {
return nil, fmt.Errorf("NotFound")
}
secret := &v1.Secret{}
secret.Name = mockController.opConfig.ClusterNameLabel
secret.Data = map[string][]byte{
"user1": []byte("testrole"),
"password1": []byte("testpassword"),
"inrole1": []byte("testinrole"),
}
return secret, nil
}
type MockSecretGetter struct {
}
func (c *MockSecretGetter) Secrets(namespace string) v1core.SecretInterface {
return &mockSecret{}
}
func newMockKubernetesClient() k8sutil.KubernetesClient {
return k8sutil.KubernetesClient{SecretsGetter: &MockSecretGetter{}}
}
func newMockController() *Controller {
controller := NewController(&Config{})
controller.opConfig.ClusterNameLabel = "cluster-name"
controller.opConfig.InfrastructureRolesSecretName =
spec.NamespacedName{v1.NamespaceDefault, testInfrastructureRolesSecretName}
controller.opConfig.Workers = 4
controller.KubeClient = newMockKubernetesClient()
return controller
}
var mockController = newMockController()
func TestPodClusterName(t *testing.T) {
var testTable = []struct {
in *v1.Pod
expected spec.NamespacedName
}{
{
&v1.Pod{},
spec.NamespacedName{},
},
{
&v1.Pod{
ObjectMeta: meta_v1.ObjectMeta{
Namespace: v1.NamespaceDefault,
Labels: map[string]string{
mockController.opConfig.ClusterNameLabel: "testcluster",
},
},
},
spec.NamespacedName{v1.NamespaceDefault, "testcluster"},
},
}
for _, test := range testTable {
resp := mockController.podClusterName(test.in)
if resp != test.expected {
t.Errorf("expected response %v does not match the actual %v", test.expected, resp)
}
}
}
func TestClusterWorkerID(t *testing.T) {
var testTable = []struct {
in spec.NamespacedName
expected uint32
}{
{
in: spec.NamespacedName{"foo", "bar"},
expected: 2,
},
{
in: spec.NamespacedName{"default", "testcluster"},
expected: 3,
},
}
for _, test := range testTable {
resp := mockController.clusterWorkerID(test.in)
if resp != test.expected {
t.Errorf("expected response %v does not match the actual %v", test.expected, resp)
}
}
}
func TestGetInfrastructureRoles(t *testing.T) {
var testTable = []struct {
secretName spec.NamespacedName
expectedRoles map[string]spec.PgUser
expectedError error
}{
{
spec.NamespacedName{},
nil,
nil,
},
{
spec.NamespacedName{v1.NamespaceDefault, "null"},
nil,
fmt.Errorf(`could not get infrastructure roles secret: NotFound`),
},
{
spec.NamespacedName{v1.NamespaceDefault, testInfrastructureRolesSecretName},
map[string]spec.PgUser{
"testrole": {
"testrole",
"testpassword",
nil,
[]string{"testinrole"},
},
},
nil,
},
}
for _, test := range testTable {
roles, err := mockController.getInfrastructureRoles(&test.secretName)
if err != test.expectedError {
if err != nil && test.expectedError != nil && err.Error() == test.expectedError.Error() {
continue
}
t.Errorf("expected error '%v' does not match the actual error '%v'", test.expectedError, err)
}
if !reflect.DeepEqual(roles, test.expectedRoles) {
t.Errorf("expected roles output %v does not match the actual %v", test.expectedRoles, roles)
}
}
}