Merge branch 'master' into bump-v1.8.1

This commit is contained in:
Felix Kunde 2022-05-19 11:34:56 +02:00
commit 6aa3ce7693
16 changed files with 238 additions and 97 deletions

View File

@ -1,2 +1,2 @@
# global owners # global owners
* @sdudoladov @Jan-M @CyberDem0n @FxKu @jopadi * @sdudoladov @Jan-M @CyberDem0n @FxKu @jopadi @idanovinda

View File

@ -1,4 +1,5 @@
Sergey Dudoladov <sergey.dudoladov@zalando.de> Sergey Dudoladov <sergey.dudoladov@zalando.de>
Felix Kunde <felix.kunde@zalando.de> Felix Kunde <felix.kunde@zalando.de>
Jan Mussler <jan.mussler@zalando.de> Jan Mussler <jan.mussler@zalando.de>
Jociele Padilha <jociele.padilha@zalando.de> Jociele Padilha <jociele.padilha@zalando.de>
Ida Novindasari <ida.novindasari@zalando.de>

View File

@ -1032,12 +1032,20 @@ func (c *Cluster) processPodEvent(obj interface{}) error {
return fmt.Errorf("could not cast to PodEvent") return fmt.Errorf("could not cast to PodEvent")
} }
// can only take lock when (un)registerPodSubscriber is finshed
c.podSubscribersMu.RLock() c.podSubscribersMu.RLock()
subscriber, ok := c.podSubscribers[spec.NamespacedName(event.PodName)] subscriber, ok := c.podSubscribers[spec.NamespacedName(event.PodName)]
c.podSubscribersMu.RUnlock()
if ok { if ok {
subscriber <- event select {
case subscriber <- event:
default:
// ending up here when there is no receiver on the channel (i.e. waitForPodLabel finished)
// avoids blocking channel: https://gobyexample.com/non-blocking-channel-operations
}
} }
// hold lock for the time of processing the event to avoid race condition
// with unregisterPodSubscriber closing the channel (see #1876)
c.podSubscribersMu.RUnlock()
return nil return nil
} }
@ -1501,34 +1509,16 @@ func (c *Cluster) Switchover(curMaster *v1.Pod, candidate spec.NamespacedName) e
var err error var err error
c.logger.Debugf("switching over from %q to %q", curMaster.Name, candidate) c.logger.Debugf("switching over from %q to %q", curMaster.Name, candidate)
c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Switchover", "Switching over from %q to %q", curMaster.Name, candidate) c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Switchover", "Switching over from %q to %q", curMaster.Name, candidate)
var wg sync.WaitGroup
podLabelErr := make(chan error)
stopCh := make(chan struct{}) stopCh := make(chan struct{})
ch := c.registerPodSubscriber(candidate)
wg.Add(1) defer c.unregisterPodSubscriber(candidate)
defer close(stopCh)
go func() {
defer wg.Done()
ch := c.registerPodSubscriber(candidate)
defer c.unregisterPodSubscriber(candidate)
role := Master
select {
case <-stopCh:
case podLabelErr <- func() (err2 error) {
_, err2 = c.waitForPodLabel(ch, stopCh, &role)
return
}():
}
}()
if err = c.patroni.Switchover(curMaster, candidate.Name); err == nil { if err = c.patroni.Switchover(curMaster, candidate.Name); err == nil {
c.logger.Debugf("successfully switched over from %q to %q", curMaster.Name, candidate) c.logger.Debugf("successfully switched over from %q to %q", curMaster.Name, candidate)
c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Switchover", "Successfully switched over from %q to %q", curMaster.Name, candidate) c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Switchover", "Successfully switched over from %q to %q", curMaster.Name, candidate)
if err = <-podLabelErr; err != nil { _, err = c.waitForPodLabel(ch, stopCh, nil)
if err != nil {
err = fmt.Errorf("could not get master pod label: %v", err) err = fmt.Errorf("could not get master pod label: %v", err)
} }
} else { } else {
@ -1536,14 +1526,6 @@ func (c *Cluster) Switchover(curMaster *v1.Pod, candidate spec.NamespacedName) e
c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Switchover", "Switchover from %q to %q FAILED: %v", curMaster.Name, candidate, err) c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Switchover", "Switchover from %q to %q FAILED: %v", curMaster.Name, candidate, err)
} }
// signal the role label waiting goroutine to close the shop and go home
close(stopCh)
// wait until the goroutine terminates, since unregisterPodSubscriber
// must be called before the outer return; otherwise we risk subscribing to the same pod twice.
wg.Wait()
// close the label waiting channel no sooner than the waiting goroutine terminates.
close(podLabelErr)
return err return err
} }

View File

@ -940,7 +940,6 @@ func (c *Cluster) generateSpiloPodEnvVars(
func appendEnvVars(envs []v1.EnvVar, appEnv ...v1.EnvVar) []v1.EnvVar { func appendEnvVars(envs []v1.EnvVar, appEnv ...v1.EnvVar) []v1.EnvVar {
collectedEnvs := envs collectedEnvs := envs
for _, env := range appEnv { for _, env := range appEnv {
env.Name = strings.ToUpper(env.Name)
if !isEnvVarPresent(collectedEnvs, env.Name) { if !isEnvVarPresent(collectedEnvs, env.Name) {
collectedEnvs = append(collectedEnvs, env) collectedEnvs = append(collectedEnvs, env)
} }
@ -950,7 +949,7 @@ func appendEnvVars(envs []v1.EnvVar, appEnv ...v1.EnvVar) []v1.EnvVar {
func isEnvVarPresent(envs []v1.EnvVar, key string) bool { func isEnvVarPresent(envs []v1.EnvVar, key string) bool {
for _, env := range envs { for _, env := range envs {
if env.Name == key { if strings.EqualFold(env.Name, key) {
return true return true
} }
} }

View File

@ -504,7 +504,7 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) {
expectedS3BucketConfigMap := []ExpectedValue{ expectedS3BucketConfigMap := []ExpectedValue{
{ {
envIndex: 17, envIndex: 17,
envVarConstant: "WAL_S3_BUCKET", envVarConstant: "wal_s3_bucket",
envVarValue: "global-s3-bucket-configmap", envVarValue: "global-s3-bucket-configmap",
}, },
} }
@ -518,7 +518,7 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) {
expectedCustomVariableSecret := []ExpectedValue{ expectedCustomVariableSecret := []ExpectedValue{
{ {
envIndex: 16, envIndex: 16,
envVarConstant: "CUSTOM_VARIABLE", envVarConstant: "custom_variable",
envVarValueRef: &v1.EnvVarSource{ envVarValueRef: &v1.EnvVarSource{
SecretKeyRef: &v1.SecretKeySelector{ SecretKeyRef: &v1.SecretKeySelector{
LocalObjectReference: v1.LocalObjectReference{ LocalObjectReference: v1.LocalObjectReference{
@ -532,7 +532,7 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) {
expectedCustomVariableConfigMap := []ExpectedValue{ expectedCustomVariableConfigMap := []ExpectedValue{
{ {
envIndex: 16, envIndex: 16,
envVarConstant: "CUSTOM_VARIABLE", envVarConstant: "custom_variable",
envVarValue: "configmap-test", envVarValue: "configmap-test",
}, },
} }
@ -573,14 +573,14 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) {
}, },
{ {
envIndex: 20, envIndex: 20,
envVarConstant: "CLONE_AWS_ENDPOINT", envVarConstant: "clone_aws_endpoint",
envVarValue: "s3.eu-west-1.amazonaws.com", envVarValue: "s3.eu-west-1.amazonaws.com",
}, },
} }
expectedCloneEnvSecret := []ExpectedValue{ expectedCloneEnvSecret := []ExpectedValue{
{ {
envIndex: 20, envIndex: 20,
envVarConstant: "CLONE_AWS_ACCESS_KEY_ID", envVarConstant: "clone_aws_access_key_id",
envVarValueRef: &v1.EnvVarSource{ envVarValueRef: &v1.EnvVarSource{
SecretKeyRef: &v1.SecretKeySelector{ SecretKeyRef: &v1.SecretKeySelector{
LocalObjectReference: v1.LocalObjectReference{ LocalObjectReference: v1.LocalObjectReference{
@ -599,7 +599,7 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) {
}, },
{ {
envIndex: 20, envIndex: 20,
envVarConstant: "STANDBY_GOOGLE_APPLICATION_CREDENTIALS", envVarConstant: "standby_google_application_credentials",
envVarValueRef: &v1.EnvVarSource{ envVarValueRef: &v1.EnvVarSource{
SecretKeyRef: &v1.SecretKeySelector{ SecretKeyRef: &v1.SecretKeySelector{
LocalObjectReference: v1.LocalObjectReference{ LocalObjectReference: v1.LocalObjectReference{

View File

@ -67,7 +67,7 @@ func (c *Cluster) markRollingUpdateFlagForPod(pod *v1.Pod, msg string) error {
return fmt.Errorf("could not form patch for pod's rolling update flag: %v", err) return fmt.Errorf("could not form patch for pod's rolling update flag: %v", err)
} }
err = retryutil.Retry(c.OpConfig.PatroniAPICheckInterval, c.OpConfig.PatroniAPICheckTimeout, err = retryutil.Retry(1*time.Second, 5*time.Second,
func() (bool, error) { func() (bool, error) {
_, err2 := c.KubeClient.Pods(pod.Namespace).Patch( _, err2 := c.KubeClient.Pods(pod.Namespace).Patch(
context.TODO(), context.TODO(),
@ -151,12 +151,13 @@ func (c *Cluster) unregisterPodSubscriber(podName spec.NamespacedName) {
c.podSubscribersMu.Lock() c.podSubscribersMu.Lock()
defer c.podSubscribersMu.Unlock() defer c.podSubscribersMu.Unlock()
if _, ok := c.podSubscribers[podName]; !ok { ch, ok := c.podSubscribers[podName]
if !ok {
panic("subscriber for pod '" + podName.String() + "' is not found") panic("subscriber for pod '" + podName.String() + "' is not found")
} }
close(c.podSubscribers[podName])
delete(c.podSubscribers, podName) delete(c.podSubscribers, podName)
close(ch)
} }
func (c *Cluster) registerPodSubscriber(podName spec.NamespacedName) chan PodEvent { func (c *Cluster) registerPodSubscriber(podName spec.NamespacedName) chan PodEvent {
@ -399,11 +400,12 @@ func (c *Cluster) getPatroniMemberData(pod *v1.Pod) (patroni.MemberData, error)
} }
func (c *Cluster) recreatePod(podName spec.NamespacedName) (*v1.Pod, error) { func (c *Cluster) recreatePod(podName spec.NamespacedName) (*v1.Pod, error) {
stopCh := make(chan struct{})
ch := c.registerPodSubscriber(podName) ch := c.registerPodSubscriber(podName)
defer c.unregisterPodSubscriber(podName) defer c.unregisterPodSubscriber(podName)
stopChan := make(chan struct{}) defer close(stopCh)
err := retryutil.Retry(c.OpConfig.PatroniAPICheckInterval, c.OpConfig.PatroniAPICheckTimeout, err := retryutil.Retry(1*time.Second, 5*time.Second,
func() (bool, error) { func() (bool, error) {
err2 := c.KubeClient.Pods(podName.Namespace).Delete( err2 := c.KubeClient.Pods(podName.Namespace).Delete(
context.TODO(), context.TODO(),
@ -421,7 +423,7 @@ func (c *Cluster) recreatePod(podName spec.NamespacedName) (*v1.Pod, error) {
if err := c.waitForPodDeletion(ch); err != nil { if err := c.waitForPodDeletion(ch); err != nil {
return nil, err return nil, err
} }
pod, err := c.waitForPodLabel(ch, stopChan, nil) pod, err := c.waitForPodLabel(ch, stopCh, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -446,7 +448,7 @@ func (c *Cluster) recreatePods(pods []v1.Pod, switchoverCandidates []spec.Namesp
continue continue
} }
podName := util.NameFromMeta(pod.ObjectMeta) podName := util.NameFromMeta(pods[i].ObjectMeta)
newPod, err := c.recreatePod(podName) newPod, err := c.recreatePod(podName)
if err != nil { if err != nil {
return fmt.Errorf("could not recreate replica pod %q: %v", util.NameFromMeta(pod.ObjectMeta), err) return fmt.Errorf("could not recreate replica pod %q: %v", util.NameFromMeta(pod.ObjectMeta), err)
@ -520,13 +522,13 @@ func (c *Cluster) getSwitchoverCandidate(master *v1.Pod) (spec.NamespacedName, e
// if sync_standby replicas were found assume synchronous_mode is enabled and ignore other candidates list // if sync_standby replicas were found assume synchronous_mode is enabled and ignore other candidates list
if len(syncCandidates) > 0 { if len(syncCandidates) > 0 {
sort.Slice(syncCandidates, func(i, j int) bool { sort.Slice(syncCandidates, func(i, j int) bool {
return util.IntFromIntStr(syncCandidates[i].Lag) < util.IntFromIntStr(syncCandidates[j].Lag) return syncCandidates[i].Lag < syncCandidates[j].Lag
}) })
return spec.NamespacedName{Namespace: master.Namespace, Name: syncCandidates[0].Name}, nil return spec.NamespacedName{Namespace: master.Namespace, Name: syncCandidates[0].Name}, nil
} }
if len(candidates) > 0 { if len(candidates) > 0 {
sort.Slice(candidates, func(i, j int) bool { sort.Slice(candidates, func(i, j int) bool {
return util.IntFromIntStr(candidates[i].Lag) < util.IntFromIntStr(candidates[j].Lag) return candidates[i].Lag < candidates[j].Lag
}) })
return spec.NamespacedName{Namespace: master.Namespace, Name: candidates[0].Name}, nil return spec.NamespacedName{Namespace: master.Namespace, Name: candidates[0].Name}, nil
} }

View File

@ -316,7 +316,7 @@ func (c *Cluster) annotationsSet(annotations map[string]string) map[string]strin
return nil return nil
} }
func (c *Cluster) waitForPodLabel(podEvents chan PodEvent, stopChan chan struct{}, role *PostgresRole) (*v1.Pod, error) { func (c *Cluster) waitForPodLabel(podEvents chan PodEvent, stopCh chan struct{}, role *PostgresRole) (*v1.Pod, error) {
timeout := time.After(c.OpConfig.PodLabelWaitTimeout) timeout := time.After(c.OpConfig.PodLabelWaitTimeout)
for { for {
select { select {
@ -332,7 +332,7 @@ func (c *Cluster) waitForPodLabel(podEvents chan PodEvent, stopChan chan struct{
} }
case <-timeout: case <-timeout:
return nil, fmt.Errorf("pod label wait timeout") return nil, fmt.Errorf("pod label wait timeout")
case <-stopChan: case <-stopCh:
return nil, fmt.Errorf("pod label wait cancelled") return nil, fmt.Errorf("pod label wait cancelled")
} }
} }

View File

@ -451,7 +451,7 @@ func (c *Controller) Run(stopCh <-chan struct{}, wg *sync.WaitGroup) {
panic("could not acquire initial list of clusters") panic("could not acquire initial list of clusters")
} }
wg.Add(5) wg.Add(5 + util.Bool2Int(c.opConfig.EnablePostgresTeamCRD))
go c.runPodInformer(stopCh, wg) go c.runPodInformer(stopCh, wg)
go c.runPostgresqlInformer(stopCh, wg) go c.runPostgresqlInformer(stopCh, wg)
go c.clusterResync(stopCh, wg) go c.clusterResync(stopCh, wg)

View File

@ -225,7 +225,7 @@ func (c *Controller) processEvent(event ClusterEvent) {
switch event.EventType { switch event.EventType {
case EventAdd: case EventAdd:
if clusterFound { if clusterFound {
lg.Infof("recieved add event for already existing Postgres cluster") lg.Infof("received add event for already existing Postgres cluster")
return return
} }

View File

@ -5,6 +5,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"math"
"net" "net"
"net/http" "net/http"
"strconv" "strconv"
@ -16,7 +17,6 @@ import (
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/intstr"
) )
const ( const (
@ -185,11 +185,27 @@ type ClusterMembers struct {
// ClusterMember cluster member data from Patroni API // ClusterMember cluster member data from Patroni API
type ClusterMember struct { type ClusterMember struct {
Name string `json:"name"` Name string `json:"name"`
Role string `json:"role"` Role string `json:"role"`
State string `json:"state"` State string `json:"state"`
Timeline int `json:"timeline"` Timeline int `json:"timeline"`
Lag intstr.IntOrString `json:"lag,omitempty"` Lag ReplicationLag `json:"lag,omitempty"`
}
type ReplicationLag uint64
// UnmarshalJSON converts member lag (can be int or string) into uint64
func (rl *ReplicationLag) UnmarshalJSON(data []byte) error {
var lagUInt64 uint64
if data[0] == '"' {
*rl = math.MaxUint64
return nil
}
if err := json.Unmarshal(data, &lagUInt64); err != nil {
return err
}
*rl = ReplicationLag(lagUInt64)
return nil
} }
// MemberDataPatroni child element // MemberDataPatroni child element

View File

@ -5,6 +5,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"math"
"net/http" "net/http"
"reflect" "reflect"
"testing" "testing"
@ -15,7 +16,6 @@ import (
acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/intstr"
) )
var logger = logrus.New().WithField("test", "patroni") var logger = logrus.New().WithField("test", "patroni")
@ -101,16 +101,27 @@ func TestGetClusterMembers(t *testing.T) {
Role: "sync_standby", Role: "sync_standby",
State: "running", State: "running",
Timeline: 1, Timeline: 1,
Lag: intstr.IntOrString{IntVal: 0}, Lag: 0,
}, { }, {
Name: "acid-test-cluster-2", Name: "acid-test-cluster-2",
Role: "replica", Role: "replica",
State: "running", State: "running",
Timeline: 1, Timeline: 1,
Lag: intstr.IntOrString{Type: 1, StrVal: "unknown"}, Lag: math.MaxUint64,
}, {
Name: "acid-test-cluster-3",
Role: "replica",
State: "running",
Timeline: 1,
Lag: 3000000000,
}} }}
json := `{"members": [{"name": "acid-test-cluster-0", "role": "leader", "state": "running", "api_url": "http://192.168.100.1:8008/patroni", "host": "192.168.100.1", "port": 5432, "timeline": 1}, {"name": "acid-test-cluster-1", "role": "sync_standby", "state": "running", "api_url": "http://192.168.100.2:8008/patroni", "host": "192.168.100.2", "port": 5432, "timeline": 1, "lag": 0}, {"name": "acid-test-cluster-2", "role": "replica", "state": "running", "api_url": "http://192.168.100.3:8008/patroni", "host": "192.168.100.3", "port": 5432, "timeline": 1, "lag": "unknown"}]}` json := `{"members": [
{"name": "acid-test-cluster-0", "role": "leader", "state": "running", "api_url": "http://192.168.100.1:8008/patroni", "host": "192.168.100.1", "port": 5432, "timeline": 1},
{"name": "acid-test-cluster-1", "role": "sync_standby", "state": "running", "api_url": "http://192.168.100.2:8008/patroni", "host": "192.168.100.2", "port": 5432, "timeline": 1, "lag": 0},
{"name": "acid-test-cluster-2", "role": "replica", "state": "running", "api_url": "http://192.168.100.3:8008/patroni", "host": "192.168.100.3", "port": 5432, "timeline": 1, "lag": "unknown"},
{"name": "acid-test-cluster-3", "role": "replica", "state": "running", "api_url": "http://192.168.100.3:8008/patroni", "host": "192.168.100.3", "port": 5432, "timeline": 1, "lag": 3000000000}
]}`
r := ioutil.NopCloser(bytes.NewReader([]byte(json))) r := ioutil.NopCloser(bytes.NewReader([]byte(json)))
response := http.Response{ response := http.Response{

View File

@ -8,7 +8,6 @@ import (
"encoding/base64" "encoding/base64"
"encoding/hex" "encoding/hex"
"fmt" "fmt"
"math"
"math/big" "math/big"
"math/rand" "math/rand"
"reflect" "reflect"
@ -324,18 +323,18 @@ func testNil(values ...*int32) bool {
return false return false
} }
// Convert int to IntOrString type // ToIntStr converts int to IntOrString type
func ToIntStr(val int) *intstr.IntOrString { func ToIntStr(val int) *intstr.IntOrString {
b := intstr.FromInt(val) b := intstr.FromInt(val)
return &b return &b
} }
// Get int from IntOrString and return max int if string // Bool2Int converts bool to int
func IntFromIntStr(intOrStr intstr.IntOrString) int { func Bool2Int(flag bool) int {
if intOrStr.Type == 1 { if flag {
return math.MaxInt return 1
} }
return intOrStr.IntValue() return 0
} }
// MaxInt32 : Return maximum of two integers provided via pointers. If one value // MaxInt32 : Return maximum of two integers provided via pointers. If one value

View File

@ -51,7 +51,23 @@ postgresqls
th(style='width: 140px') CPU th(style='width: 140px') CPU
th(style='width: 130px') Memory th(style='width: 130px') Memory
th(style='width: 100px') Size th(style='width: 100px') Size
th(style='width: 120px') Cost/Month th(style='width: 100px') IOPS
th(style='width: 100px') Throughput
th(style='width: 120px')
.tooltip(style='width: 120px')
| Cost/Month
.tooltiptext
strong Cost = MAX(CPU, Memory) + rest
br
| 1 CPU core : 42.09$
br
| 1GB memory: 10.5225$
br
| 1GB volume: 0.0952$
br
| IOPS (-3000 baseline): 0.006$
br
| Throughput (-125 baseline): 0.0476$
th(stlye='width: 120px') th(stlye='width: 120px')
tbody tbody
@ -69,6 +85,8 @@ postgresqls
td { cpu } / { cpu_limit } td { cpu } / { cpu_limit }
td { memory } / { memory_limit } td { memory } / { memory_limit }
td { volume_size } td { volume_size }
td { iops }
td { throughput }
td { calcCosts(nodes, cpu, memory, volume_size, iops, throughput) }$ td { calcCosts(nodes, cpu, memory, volume_size, iops, throughput) }$
td td
@ -132,7 +150,23 @@ postgresqls
th(style='width: 140px') CPU th(style='width: 140px') CPU
th(style='width: 130px') Memory th(style='width: 130px') Memory
th(style='width: 100px') Size th(style='width: 100px') Size
th(style='width: 120px') Cost/Month th(style='width: 100px') IOPS
th(style='width: 100px') Throughput
th(style='width: 120px')
.tooltip(style='width: 120px')
| Cost/Month
.tooltiptext
strong Cost = MAX(CPU, Memory) + rest
br
| 1 CPU core : 42.09$
br
| 1GB memory: 10.5225$
br
| 1GB volume: 0.0952$
br
| IOPS (-3000 baseline): 0.006$
br
| Throughput (-125 baseline): 0.0476$
th(stlye='width: 120px') th(stlye='width: 120px')
tbody tbody
@ -152,6 +186,8 @@ postgresqls
td { cpu } / { cpu_limit } td { cpu } / { cpu_limit }
td { memory } / { memory_limit } td { memory } / { memory_limit }
td { volume_size } td { volume_size }
td { iops }
td { throughput }
td { calcCosts(nodes, cpu, memory, volume_size, iops, throughput) }$ td { calcCosts(nodes, cpu, memory, volume_size, iops, throughput) }$
td td
@ -229,28 +265,44 @@ postgresqls
const calcCosts = this.calcCosts = (nodes, cpu, memory, disk, iops, throughput) => { const calcCosts = this.calcCosts = (nodes, cpu, memory, disk, iops, throughput) => {
podcount = Math.max(nodes, opts.config.min_pods) podcount = Math.max(nodes, opts.config.min_pods)
corecost = toCores(cpu) * opts.config.cost_core corecost = toCores(cpu) * opts.config.cost_core * 30.5 * 24
memorycost = toMemory(memory) * opts.config.cost_memory memorycost = toMemory(memory) * opts.config.cost_memory * 30.5 * 24
diskcost = toDisk(disk) * opts.config.cost_ebs diskcost = toDisk(disk) * opts.config.cost_ebs
iopscost = 0 iopscost = 0
if (iops !== undefined && iops > 3000) { if (iops !== undefined && iops > opts.config.free_iops) {
iopscost = (iops - 3000) * opts.config.cost_iops if (iops > opts.config.limit_iops) {
iops = opts.config.limit_iops
}
iopscost = (iops - opts.config.free_iops) * opts.config.cost_iops
} }
throughputcost = 0 throughputcost = 0
if (throughput !== undefined && throughput > 125) { if (throughput !== undefined && throughput > opts.config.free_throughput) {
throughputcost = (throughput - 125) * opts.config.cost_throughput if (throughput > opts.config.limit_throughput) {
throughput = opts.config.limit_throughput
}
throughputcost = (throughput - opts.config.free_throughput) * opts.config.cost_throughput
} }
costs = podcount * (corecost + memorycost + diskcost + iopscost + throughputcost) costs = podcount * (Math.max(corecost, memorycost) + diskcost + iopscost + throughputcost)
return costs.toFixed(2) return costs.toFixed(2)
} }
const toDisk = this.toDisk = value => { const toDisk = this.toDisk = value => {
if(value.endsWith("Gi")) { if(value.endsWith("Mi")) {
value = value.substring(0, value.length-2)
value = Number(value) / 1000.
return value
}
else if(value.endsWith("Gi")) {
value = value.substring(0, value.length-2) value = value.substring(0, value.length-2)
value = Number(value) value = Number(value)
return value return value
} }
else if(value.endsWith("Ti")) {
value = value.substring(0, value.length-2)
value = Number(value) * 1000
return value
}
return value return value
} }

View File

@ -67,6 +67,10 @@ spec:
"cost_throughput": 0.0476, "cost_throughput": 0.0476,
"cost_core": 0.0575, "cost_core": 0.0575,
"cost_memory": 0.014375, "cost_memory": 0.014375,
"free_iops": 3000,
"free_throughput": 125,
"limit_iops": 16000,
"limit_throughput": 1000,
"postgresql_versions": [ "postgresql_versions": [
"14", "14",
"13", "13",

View File

@ -82,12 +82,16 @@ OPERATOR_CLUSTER_NAME_LABEL = getenv('OPERATOR_CLUSTER_NAME_LABEL', 'cluster-nam
OPERATOR_UI_CONFIG = getenv('OPERATOR_UI_CONFIG', '{}') OPERATOR_UI_CONFIG = getenv('OPERATOR_UI_CONFIG', '{}')
OPERATOR_UI_MAINTENANCE_CHECK = getenv('OPERATOR_UI_MAINTENANCE_CHECK', '{}') OPERATOR_UI_MAINTENANCE_CHECK = getenv('OPERATOR_UI_MAINTENANCE_CHECK', '{}')
READ_ONLY_MODE = getenv('READ_ONLY_MODE', False) in [True, 'true'] READ_ONLY_MODE = getenv('READ_ONLY_MODE', False) in [True, 'true']
RESOURCES_VISIBLE = getenv('RESOURCES_VISIBLE', True)
SPILO_S3_BACKUP_PREFIX = getenv('SPILO_S3_BACKUP_PREFIX', 'spilo/') SPILO_S3_BACKUP_PREFIX = getenv('SPILO_S3_BACKUP_PREFIX', 'spilo/')
SUPERUSER_TEAM = getenv('SUPERUSER_TEAM', 'acid') SUPERUSER_TEAM = getenv('SUPERUSER_TEAM', 'acid')
TARGET_NAMESPACE = getenv('TARGET_NAMESPACE') TARGET_NAMESPACE = getenv('TARGET_NAMESPACE')
GOOGLE_ANALYTICS = getenv('GOOGLE_ANALYTICS', False) GOOGLE_ANALYTICS = getenv('GOOGLE_ANALYTICS', False)
MIN_PODS= getenv('MIN_PODS', 2) MIN_PODS= getenv('MIN_PODS', 2)
RESOURCES_VISIBLE = getenv('RESOURCES_VISIBLE', True)
CUSTOM_MESSAGE_RED = getenv('CUSTOM_MESSAGE_RED', '')
APPLICATION_DEPLOYMENT_DOCS = getenv('APPLICATION_DEPLOYMENT_DOCS', '')
CONNECTION_DOCS = getenv('CONNECTION_DOCS', '')
# storage pricing, i.e. https://aws.amazon.com/ebs/pricing/ (e.g. Europe - Franfurt) # storage pricing, i.e. https://aws.amazon.com/ebs/pricing/ (e.g. Europe - Franfurt)
COST_EBS = float(getenv('COST_EBS', 0.0952)) # GB per month COST_EBS = float(getenv('COST_EBS', 0.0952)) # GB per month
@ -95,8 +99,19 @@ COST_IOPS = float(getenv('COST_IOPS', 0.006)) # IOPS per month above 3000 basel
COST_THROUGHPUT = float(getenv('COST_THROUGHPUT', 0.0476)) # MB/s per month above 125 MB/s baseline COST_THROUGHPUT = float(getenv('COST_THROUGHPUT', 0.0476)) # MB/s per month above 125 MB/s baseline
# compute costs, i.e. https://www.ec2instances.info/?region=eu-central-1&selected=m5.2xlarge # compute costs, i.e. https://www.ec2instances.info/?region=eu-central-1&selected=m5.2xlarge
COST_CORE = 30.5 * 24 * float(getenv('COST_CORE', 0.0575)) # Core per hour m5.2xlarge / 8. COST_CORE = float(getenv('COST_CORE', 0.0575)) # Core per hour m5.2xlarge / 8.
COST_MEMORY = 30.5 * 24 * float(getenv('COST_MEMORY', 0.014375)) # Memory GB m5.2xlarge / 32. COST_MEMORY = float(getenv('COST_MEMORY', 0.014375)) # Memory GB m5.2xlarge / 32.
# maximum and limitation of IOPS and throughput
FREE_IOPS = float(getenv('FREE_IOPS', 3000))
LIMIT_IOPS = float(getenv('LIMIT_IOPS', 16000))
FREE_THROUGHPUT = float(getenv('FREE_THROUGHPUT', 125))
LIMIT_THROUGHPUT = float(getenv('LIMIT_THROUGHPUT', 1000))
# get the default value of core and memory
DEFAULT_MEMORY = getenv('DEFAULT_MEMORY', '300Mi')
DEFAULT_MEMORY_LIMIT = getenv('DEFAULT_MEMORY_LIMIT', '300Mi')
DEFAULT_CPU = getenv('DEFAULT_CPU', '10m')
DEFAULT_CPU_LIMIT = getenv('DEFAULT_CPU_LIMIT', '300m')
WALE_S3_ENDPOINT = getenv( WALE_S3_ENDPOINT = getenv(
'WALE_S3_ENDPOINT', 'WALE_S3_ENDPOINT',
@ -304,29 +319,34 @@ DEFAULT_UI_CONFIG = {
'nat_gateways_visible': True, 'nat_gateways_visible': True,
'users_visible': True, 'users_visible': True,
'databases_visible': True, 'databases_visible': True,
'resources_visible': True, 'resources_visible': RESOURCES_VISIBLE,
'postgresql_versions': ['11','12','13'], 'postgresql_versions': ['11','12','13','14'],
'dns_format_string': '{0}.{1}.{2}', 'dns_format_string': '{0}.{1}.{2}',
'pgui_link': '', 'pgui_link': '',
'static_network_whitelist': {}, 'static_network_whitelist': {},
'read_only_mode': READ_ONLY_MODE,
'superuser_team': SUPERUSER_TEAM,
'target_namespace': TARGET_NAMESPACE,
'connection_docs': CONNECTION_DOCS,
'application_deployment_docs': APPLICATION_DEPLOYMENT_DOCS,
'cost_ebs': COST_EBS, 'cost_ebs': COST_EBS,
'cost_iops': COST_IOPS, 'cost_iops': COST_IOPS,
'cost_throughput': COST_THROUGHPUT, 'cost_throughput': COST_THROUGHPUT,
'cost_core': COST_CORE, 'cost_core': COST_CORE,
'cost_memory': COST_MEMORY, 'cost_memory': COST_MEMORY,
'min_pods': MIN_PODS 'min_pods': MIN_PODS,
'free_iops': FREE_IOPS,
'free_throughput': FREE_THROUGHPUT,
'limit_iops': LIMIT_IOPS,
'limit_throughput': LIMIT_THROUGHPUT
} }
@app.route('/config') @app.route('/config')
@authorize @authorize
def get_config(): def get_config():
config = loads(OPERATOR_UI_CONFIG) or DEFAULT_UI_CONFIG config = DEFAULT_UI_CONFIG.copy()
config['read_only_mode'] = READ_ONLY_MODE config.update(loads(OPERATOR_UI_CONFIG))
config['resources_visible'] = RESOURCES_VISIBLE
config['superuser_team'] = SUPERUSER_TEAM
config['target_namespace'] = TARGET_NAMESPACE
config['min_pods'] = MIN_PODS
config['namespaces'] = ( config['namespaces'] = (
[TARGET_NAMESPACE] [TARGET_NAMESPACE]
@ -961,11 +981,13 @@ def get_operator_get_logs(worker: int):
@app.route('/operator/clusters/<namespace>/<cluster>/logs') @app.route('/operator/clusters/<namespace>/<cluster>/logs')
@authorize @authorize
def get_operator_get_logs_per_cluster(namespace: str, cluster: str): def get_operator_get_logs_per_cluster(namespace: str, cluster: str):
team, cluster_name = cluster.split('-', 1)
# team id might contain hyphens, try to find correct team name
user_teams = get_teams_for_user(session.get('user_name', '')) user_teams = get_teams_for_user(session.get('user_name', ''))
for user_team in user_teams: for user_team in user_teams:
if cluster.find(user_team) == 0: if cluster.find(user_team + '-') == 0:
team = cluster[:len(user_team)] team = cluster[:len(user_team)]
cluster_name = cluster[len(user_team)+1:] cluster_name = cluster[len(user_team + '-'):]
break break
return proxy_operator(f'/clusters/{team}/{namespace}/{cluster_name}/logs/') return proxy_operator(f'/clusters/{team}/{namespace}/{cluster_name}/logs/')

View File

@ -64,3 +64,56 @@ label {
td { td {
vertical-align: middle !important; vertical-align: middle !important;
} }
.tooltip {
position: relative;
display: inline-block;
opacity: 1;
font-size: 14px;
font-weight: bold;
}
.tooltip:after {
content: '?';
display: inline-block;
font-family: sans-serif;
font-weight: bold;
text-align: center;
width: 16px;
height: 16px;
font-size: 12px;
line-height: 16px;
border-radius: 12px;
padding: 0px;
color: white;
background: black;
border: 1px solid black;
}
.tooltip .tooltiptext {
visibility: hidden;
width: 250px;
background-color: white;
color: #000;
text-align: justify;
border-radius: 6px;
padding: 10px 10px;
position: absolute;
z-index: 1;
bottom: 150%;
left: 50%;
margin-left: -120px;
border: 1px solid black;
font-weight: normal;
}
.tooltip .tooltiptext::after {
content: "";
position: absolute;
top: 100%;
left: 50%;
margin-left: -5px;
border-width: 5px;
border-style: solid;
border-color: black transparent transparent transparent;
}
.tooltip:hover .tooltiptext {
visibility: visible;
}