package cluster import ( "fmt" "k8s.io/client-go/pkg/api" "k8s.io/client-go/pkg/api/v1" "k8s.io/client-go/pkg/apis/apps/v1beta1" "github.com/zalando-incubator/postgres-operator/pkg/spec" "github.com/zalando-incubator/postgres-operator/pkg/util" "github.com/zalando-incubator/postgres-operator/pkg/util/constants" "github.com/zalando-incubator/postgres-operator/pkg/util/k8sutil" "github.com/zalando-incubator/postgres-operator/pkg/util/retryutil" ) func (c *Cluster) loadResources() error { ns := c.Metadata.Namespace listOptions := v1.ListOptions{ LabelSelector: c.labelsSet().String(), } services, err := c.KubeClient.Services(ns).List(listOptions) if err != nil { return fmt.Errorf("could not get list of services: %v", err) } if len(services.Items) > 2 { return fmt.Errorf("too many(%d) services for a cluster", len(services.Items)) } for i, svc := range services.Items { switch PostgresRole(svc.Labels[c.PodRoleLabel]) { case Replica: c.Service[Replica] = &services.Items[i] default: c.Service[Master] = &services.Items[i] } } endpoints, err := c.KubeClient.Endpoints(ns).List(listOptions) if err != nil { return fmt.Errorf("could not get list of endpoints: %v", err) } if len(endpoints.Items) > 2 { return fmt.Errorf("too many(%d) endpoints for a cluster", len(endpoints.Items)) } for i, ep := range endpoints.Items { if ep.Labels[c.PodRoleLabel] != string(Replica) { c.Endpoint = &endpoints.Items[i] break } } secrets, err := c.KubeClient.Secrets(ns).List(listOptions) if err != nil { return fmt.Errorf("could not get list of secrets: %v", err) } for i, secret := range secrets.Items { if _, ok := c.Secrets[secret.UID]; ok { continue } c.Secrets[secret.UID] = &secrets.Items[i] c.logger.Debugf("secret loaded, uid: %s", secret.UID) } statefulSets, err := c.KubeClient.StatefulSets(ns).List(listOptions) if err != nil { return fmt.Errorf("could not get list of statefulsets: %v", err) } if len(statefulSets.Items) > 1 { return fmt.Errorf("too many(%d) statefulsets for a cluster", len(statefulSets.Items)) } if len(statefulSets.Items) == 1 { c.Statefulset = &statefulSets.Items[0] } return nil } func (c *Cluster) listResources() error { if c.Statefulset != nil { c.logger.Infof("Found statefulset: %s (uid: %s)", util.NameFromMeta(c.Statefulset.ObjectMeta), c.Statefulset.UID) } for _, obj := range c.Secrets { c.logger.Infof("Found secret: %s (uid: %s)", util.NameFromMeta(obj.ObjectMeta), obj.UID) } if c.Endpoint != nil { c.logger.Infof("Found endpoint: %s (uid: %s)", util.NameFromMeta(c.Endpoint.ObjectMeta), c.Endpoint.UID) } for role, service := range c.Service { c.logger.Infof("Found %s service: %s (uid: %s)", role, util.NameFromMeta(service.ObjectMeta), service.UID) } pods, err := c.listPods() if err != nil { return fmt.Errorf("could not get the list of pods: %v", err) } for _, obj := range pods { c.logger.Infof("Found pod: %s (uid: %s)", util.NameFromMeta(obj.ObjectMeta), obj.UID) } pvcs, err := c.listPersistentVolumeClaims() if err != nil { return fmt.Errorf("could not get the list of PVCs: %v", err) } for _, obj := range pvcs { c.logger.Infof("Found PVC: %s (uid: %s)", util.NameFromMeta(obj.ObjectMeta), obj.UID) } return nil } func (c *Cluster) createStatefulSet() (*v1beta1.StatefulSet, error) { if c.Statefulset != nil { return nil, fmt.Errorf("statefulset already exists in the cluster") } statefulSetSpec, err := c.generateStatefulSet(c.Spec) if err != nil { return nil, fmt.Errorf("could not generate statefulset: %v", err) } statefulSet, err := c.KubeClient.StatefulSets(statefulSetSpec.Namespace).Create(statefulSetSpec) if err != nil { return nil, err } c.Statefulset = statefulSet c.logger.Debugf("Created new statefulset '%s', uid: %s", util.NameFromMeta(statefulSet.ObjectMeta), statefulSet.UID) return statefulSet, nil } func (c *Cluster) updateStatefulSet(newStatefulSet *v1beta1.StatefulSet) error { if c.Statefulset == nil { return fmt.Errorf("there is no statefulset in the cluster") } statefulSetName := util.NameFromMeta(c.Statefulset.ObjectMeta) c.logger.Debugf("Updating statefulset") patchData, err := specPatch(newStatefulSet.Spec) if err != nil { return fmt.Errorf("could not form patch for the statefulset '%s': %v", statefulSetName, err) } statefulSet, err := c.KubeClient.StatefulSets(c.Statefulset.Namespace).Patch( c.Statefulset.Name, api.MergePatchType, patchData, "") if err != nil { return fmt.Errorf("could not patch statefulset '%s': %v", statefulSetName, err) } c.Statefulset = statefulSet return nil } // replaceStatefulSet deletes an old StatefulSet and creates the new using spec in the PostgreSQL TPR. func (c *Cluster) replaceStatefulSet(newStatefulSet *v1beta1.StatefulSet) error { if c.Statefulset == nil { return fmt.Errorf("there is no statefulset in the cluster") } statefulSetName := util.NameFromMeta(c.Statefulset.ObjectMeta) c.logger.Debugf("Replacing statefulset") // Delete the current statefulset without deleting the pods orphanDepencies := true oldStatefulset := c.Statefulset options := v1.DeleteOptions{OrphanDependents: &orphanDepencies} if err := c.KubeClient.StatefulSets(oldStatefulset.Namespace).Delete(oldStatefulset.Name, &options); err != nil { return fmt.Errorf("could not delete statefulset '%s': %v", statefulSetName, err) } // make sure we clear the stored statefulset status if the subsequent create fails. c.Statefulset = nil // wait until the statefulset is truly deleted c.logger.Debugf("Waiting for the statefulset to be deleted") err := retryutil.Retry(constants.StatefulsetDeletionInterval, constants.StatefulsetDeletionTimeout, func() (bool, error) { _, err := c.KubeClient.StatefulSets(oldStatefulset.Namespace).Get(oldStatefulset.Name) return err != nil, nil }) if err != nil { return fmt.Errorf("could not delete statefulset: %v", err) } // create the new statefulset with the desired spec. It would take over the remaining pods. createdStatefulset, err := c.KubeClient.StatefulSets(newStatefulSet.Namespace).Create(newStatefulSet) if err != nil { return fmt.Errorf("could not create statefulset '%s': %v", statefulSetName, err) } // check that all the previous replicas were picked up. if newStatefulSet.Spec.Replicas == oldStatefulset.Spec.Replicas && createdStatefulset.Status.Replicas != oldStatefulset.Status.Replicas { c.logger.Warnf("Number of pods for the old and updated Statefulsets is not identical") } c.Statefulset = createdStatefulset return nil } func (c *Cluster) deleteStatefulSet() error { c.logger.Debugln("Deleting statefulset") if c.Statefulset == nil { return fmt.Errorf("there is no statefulset in the cluster") } err := c.KubeClient.StatefulSets(c.Statefulset.Namespace).Delete(c.Statefulset.Name, c.deleteOptions) if err != nil { return err } c.logger.Infof("statefulset '%s' has been deleted", util.NameFromMeta(c.Statefulset.ObjectMeta)) c.Statefulset = nil if err := c.deletePods(); err != nil { return fmt.Errorf("could not delete pods: %v", err) } if err := c.deletePersistenVolumeClaims(); err != nil { return fmt.Errorf("could not delete PersistentVolumeClaims: %v", err) } return nil } func (c *Cluster) createService(role PostgresRole) (*v1.Service, error) { if c.Service[role] != nil { return nil, fmt.Errorf("service already exists in the cluster") } serviceSpec := c.generateService(role, &c.Spec) service, err := c.KubeClient.Services(serviceSpec.Namespace).Create(serviceSpec) if err != nil { return nil, err } c.Service[role] = service return service, nil } func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error { if c.Service[role] == nil { return fmt.Errorf("there is no service in the cluster") } serviceName := util.NameFromMeta(c.Service[role].ObjectMeta) endpointName := util.NameFromMeta(c.Endpoint.ObjectMeta) // TODO: check if it possible to change the service type with a patch in future versions of Kubernetes if newService.Spec.Type != c.Service[role].Spec.Type { // service type has changed, need to replace the service completely. // we cannot use just pach the current service, since it may contain attributes incompatible with the new type. var ( currentEndpoint *v1.Endpoints err error ) if role == Master { // for the master service we need to re-create the endpoint as well. Get the up-to-date version of // the addresses stored in it before the service is deleted (deletion of the service removes the endpooint) currentEndpoint, err = c.KubeClient.Endpoints(c.Service[role].Namespace).Get(c.Service[role].Name) if err != nil { return fmt.Errorf("could not get current cluster endpoints: %v", err) } } err = c.KubeClient.Services(c.Service[role].Namespace).Delete(c.Service[role].Name, c.deleteOptions) if err != nil { return fmt.Errorf("could not delete service '%s': '%v'", serviceName, err) } c.Endpoint = nil svc, err := c.KubeClient.Services(newService.Namespace).Create(newService) if err != nil { return fmt.Errorf("could not create service '%s': '%v'", serviceName, err) } c.Service[role] = svc if role == Master { // create the new endpoint using the addresses obtained from the previous one endpointSpec := c.generateMasterEndpoints(currentEndpoint.Subsets) ep, err := c.KubeClient.Endpoints(c.Service[role].Namespace).Create(endpointSpec) if err != nil { return fmt.Errorf("could not create endpoint '%s': '%v'", endpointName, err) } c.Endpoint = ep } return nil } if len(newService.ObjectMeta.Annotations) > 0 { annotationsPatchData := metadataAnnotationsPatch(newService.ObjectMeta.Annotations) _, err := c.KubeClient.Services(c.Service[role].Namespace).Patch( c.Service[role].Name, api.StrategicMergePatchType, []byte(annotationsPatchData), "") if err != nil { return fmt.Errorf("could not replace annotations for the service '%s': %v", serviceName, err) } } patchData, err := specPatch(newService.Spec) if err != nil { return fmt.Errorf("could not form patch for the service '%s': %v", serviceName, err) } svc, err := c.KubeClient.Services(c.Service[role].Namespace).Patch( c.Service[role].Name, api.MergePatchType, patchData, "") if err != nil { return fmt.Errorf("could not patch service '%s': %v", serviceName, err) } c.Service[role] = svc return nil } func (c *Cluster) deleteService(role PostgresRole) error { c.logger.Debugf("Deleting service %s", role) if c.Service[role] == nil { return fmt.Errorf("There is no %s service in the cluster", role) } service := c.Service[role] err := c.KubeClient.Services(service.Namespace).Delete(service.Name, c.deleteOptions) if err != nil { return err } c.logger.Infof("%s service '%s' has been deleted", role, util.NameFromMeta(service.ObjectMeta)) c.Service[role] = nil return nil } func (c *Cluster) createEndpoint() (*v1.Endpoints, error) { if c.Endpoint != nil { return nil, fmt.Errorf("endpoint already exists in the cluster") } endpointsSpec := c.generateMasterEndpoints(nil) endpoints, err := c.KubeClient.Endpoints(endpointsSpec.Namespace).Create(endpointsSpec) if err != nil { return nil, err } c.Endpoint = endpoints return endpoints, nil } func (c *Cluster) deleteEndpoint() error { c.logger.Debugln("Deleting endpoint") if c.Endpoint == nil { return fmt.Errorf("there is no endpoint in the cluster") } err := c.KubeClient.Endpoints(c.Endpoint.Namespace).Delete(c.Endpoint.Name, c.deleteOptions) if err != nil { return err } c.logger.Infof("endpoint '%s' has been deleted", util.NameFromMeta(c.Endpoint.ObjectMeta)) c.Endpoint = nil return nil } func (c *Cluster) applySecrets() error { secrets := c.generateUserSecrets() for secretUsername, secretSpec := range secrets { secret, err := c.KubeClient.Secrets(secretSpec.Namespace).Create(secretSpec) if k8sutil.ResourceAlreadyExists(err) { var userMap map[string]spec.PgUser curSecret, err := c.KubeClient.Secrets(secretSpec.Namespace).Get(secretSpec.Name) if err != nil { return fmt.Errorf("could not get current secret: %v", err) } c.logger.Debugf("secret '%s' already exists, fetching it's password", util.NameFromMeta(curSecret.ObjectMeta)) if secretUsername == c.systemUsers[constants.SuperuserKeyName].Name { secretUsername = constants.SuperuserKeyName userMap = c.systemUsers } else if secretUsername == c.systemUsers[constants.ReplicationUserKeyName].Name { secretUsername = constants.ReplicationUserKeyName userMap = c.systemUsers } else { userMap = c.pgUsers } pwdUser := userMap[secretUsername] pwdUser.Password = string(curSecret.Data["password"]) userMap[secretUsername] = pwdUser continue } else { if err != nil { return fmt.Errorf("could not create secret for user '%s': %v", secretUsername, err) } c.Secrets[secret.UID] = secret c.logger.Debugf("Created new secret '%s', uid: %s", util.NameFromMeta(secret.ObjectMeta), secret.UID) } } return nil } func (c *Cluster) deleteSecret(secret *v1.Secret) error { c.logger.Debugf("Deleting secret '%s'", util.NameFromMeta(secret.ObjectMeta)) err := c.KubeClient.Secrets(secret.Namespace).Delete(secret.Name, c.deleteOptions) if err != nil { return err } c.logger.Infof("secret '%s' has been deleted", util.NameFromMeta(secret.ObjectMeta)) delete(c.Secrets, secret.UID) return err } func (c *Cluster) createUsers() (err error) { // TODO: figure out what to do with duplicate names (humans and robots) among pgUsers reqs := c.userSyncStrategy.ProduceSyncRequests(nil, c.pgUsers) err = c.userSyncStrategy.ExecuteSyncRequests(reqs, c.pgDb) return err }