new toggle to use internal ELB by default

charts/postgres-operator/crds/operatorconfigurations.yaml

@@ -225,6 +225,8 @@ spec:
                   type: string
                 enable_master_load_balancer:
                   type: boolean
+                enable_public_load_balancer:
+                  type: boolean
                 enable_replica_load_balancer:
                   type: boolean
                 master_dns_name_format:

charts/postgres-operator/crds/postgresqls.yaml

@@ -190,6 +190,8 @@ spec:
               type: boolean
             enableMasterLoadBalancer:
               type: boolean
+            enablePublicLoadBalancer:
+              type: boolean
             enableReplicaLoadBalancer:
               type: boolean
             enableShmVolume:

charts/postgres-operator/values-crd.yaml

@@ -159,15 +159,17 @@ configTimeouts:
 
 # configure behavior of load balancers
 configLoadBalancer:
-  # DNS zone for cluster DNS name when load balancer is configured for cluster
-  db_hosted_zone: db.example.com
   # annotations to apply to service when load balancing is enabled
   # custom_service_annotations:
   #   keyx: valuez
   #   keya: valuea
 
+  # DNS zone for cluster DNS name when load balancer is configured for cluster
+  db_hosted_zone: db.example.com
   # toggles service type load balancer pointing to the master pod of the cluster
   enable_master_load_balancer: false
+  # toggles load balancer annotation service.beta.kubernetes.io/aws-load-balancer-internal
+  enable_public_load_balancer: false
   # toggles service type load balancer pointing to the replica pod of the cluster
   enable_replica_load_balancer: false
   # defines the DNS name string template for the master load balancer cluster

charts/postgres-operator/values.yaml

@@ -150,13 +150,15 @@ configTimeouts:
 
 # configure behavior of load balancers
 configLoadBalancer:
-  # DNS zone for cluster DNS name when load balancer is configured for cluster
-  db_hosted_zone: db.example.com
   # annotations to apply to service when load balancing is enabled
   # custom_service_annotations: "keyx:valuez,keya:valuea"
 
+  # DNS zone for cluster DNS name when load balancer is configured for cluster
+  db_hosted_zone: db.example.com
   # toggles service type load balancer pointing to the master pod of the cluster
   enable_master_load_balancer: "false"
+  # toggles load balancer annotation service.beta.kubernetes.io/aws-load-balancer-internal
+  enable_public_load_balancer: "false"
   # toggles service type load balancer pointing to the replica pod of the cluster
   enable_replica_load_balancer: "false"
   # defines the DNS name string template for the master load balancer cluster

docs/administrator.md

@@ -400,6 +400,11 @@ services:
   config parameter `custom_service_annotations` or the  cluster parameter
   `serviceAnnotations`.
 
+If `enable_public_load_balancer` is disabled (default) another annotation is
+added to indicate the usage of an internal ELB:
+
+- `service.beta.kubernetes.io/aws-load-balancer-internal: "true"`
+
 To limit the range of IP addresses that can reach a load balancer, specify the
 desired ranges in the `allowedSourceRanges` field (applies to both master and
 replica load balancers). To prevent exposing load balancers to the entire

docs/reference/cluster_manifest.md

@@ -82,6 +82,11 @@ These parameters are grouped directly under  the `spec` key in the manifest.
   `enable_replica_load_balancer` parameter) to define whether to enable the
   load balancer pointing to the Postgres standby instances. Optional.
 
+* **enablePublicLoadBalancer**
+  boolean flag to override the operator defaults (set by the
+  `enable_public_load_balancer` parameter) to define whether to use an internal
+  or public ELB. Optional.
+
 * **allowedSourceRanges**
   when one or more load balancers are enabled for the cluster, this parameter
   defines the comma-separated range of IP networks (in CIDR-notation). The
@@ -231,10 +236,10 @@ explanation of `ttl` and `loop_wait` parameters.
 
 * **synchronous_mode**
   Patroni `synchronous_mode` parameter value. The default is set to `false`. Optional.
-  
+
 * **synchronous_mode_strict**
   Patroni `synchronous_mode_strict` parameter value. Can be used in addition to `synchronous_mode`. The default is set to `false`. Optional.
-  
+
 ## Postgres container resources
 
 Those parameters define [CPU and memory requests and limits](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/)

docs/reference/operator_parameters.md

@@ -411,13 +411,17 @@ In the CRD-based configuration they are grouped under the `load_balancer` key.
 
 * **enable_master_load_balancer**
   toggles service type load balancer pointing to the master pod of the cluster.
-  Can be overridden by individual cluster settings. The default is `true`.
+  Can be overridden by individual cluster settings. The default is `false`.
 
 * **enable_replica_load_balancer**
   toggles service type load balancer pointing to the replica pod of the
   cluster.  Can be overridden by individual cluster settings. The default is
   `false`.
 
+* **enable_public_load_balancer**
+  when using load balancer services they can be internal or public which can be
+  toggled with this option. The default is `false`.  
+
 * **custom_service_annotations**
   This key/value map provides a list of annotations that get attached to each
   service of a cluster created by the operator. If the annotation key is also

manifests/complete-postgres-manifest.yaml

@@ -16,6 +16,7 @@ spec:
     - createdb
   enableMasterLoadBalancer: false
   enableReplicaLoadBalancer: false
+  enablePublicLoadBalancer: false
 #  enableConnectionPooler: true  # not needed when connectionPooler section is present (see below)
   allowedSourceRanges:  # load balancers' source ranges for both master and replica services
   - 127.0.0.1/32

manifests/configmap.yaml

@@ -39,6 +39,7 @@ data:
   enable_master_load_balancer: "false"
   # enable_pod_antiaffinity: "false"
   # enable_pod_disruption_budget: "true"
+  enable_public_load_balancer: "false"
   enable_replica_load_balancer: "false"
   # enable_shm_volume: "true"
   # enable_sidecars: "true"

manifests/operatorconfiguration.crd.yaml

@@ -201,6 +201,8 @@ spec:
                   type: string
                 enable_master_load_balancer:
                   type: boolean
+                enable_public_load_balancer:
+                  type: boolean
                 enable_replica_load_balancer:
                   type: boolean
                 master_dns_name_format:

manifests/postgresql-operator-default-configuration.yaml

@@ -76,12 +76,13 @@ configuration:
     resource_check_interval: 3s
     resource_check_timeout: 10m
   load_balancer:
-    # db_hosted_zone: ""
-    enable_master_load_balancer: false
-    enable_replica_load_balancer: false
     # custom_service_annotations:
     #   keyx: valuex
     #   keyy: valuey
+    # db_hosted_zone: ""
+    enable_master_load_balancer: false
+    enable_public_load_balancer: false
+    enable_replica_load_balancer: false
     master_dns_name_format: "{cluster}.{team}.{hostedzone}"
     replica_dns_name_format: "{cluster}-repl.{team}.{hostedzone}"
   aws_or_gcp:

manifests/postgresql.crd.yaml

@@ -154,6 +154,8 @@ spec:
               type: boolean
             enableMasterLoadBalancer:
               type: boolean
+            enablePublicLoadBalancer:
+              type: boolean
             enableReplicaLoadBalancer:
               type: boolean
             enableShmVolume:

pkg/apis/acid.zalan.do/v1/crds.go

@@ -268,6 +268,9 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{
 					"enableMasterLoadBalancer": {
 						Type: "boolean",
 					},
+					"enablePublicLoadBalancer": {
+						Type: "boolean",
+					},
 					"enableReplicaLoadBalancer": {
 						Type: "boolean",
 					},
@@ -1062,6 +1065,9 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation
 							"enable_master_load_balancer": {
 								Type: "boolean",
 							},
+							"enable_public_load_balancer": {
+								Type: "boolean",
+							},
 							"enable_replica_load_balancer": {
 								Type: "boolean",
 							},

pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

@@ -100,6 +100,7 @@ type OperatorTimeouts struct {
 type LoadBalancerConfiguration struct {
 	DbHostedZone              string                `json:"db_hosted_zone,omitempty"`
 	EnableMasterLoadBalancer  bool                  `json:"enable_master_load_balancer,omitempty"`
+	EnablePublicLoadBalancer  bool                  `json:"enable_public_load_balancer,omitempty"`
 	EnableReplicaLoadBalancer bool                  `json:"enable_replica_load_balancer,omitempty"`
 	CustomServiceAnnotations  map[string]string     `json:"custom_service_annotations,omitempty"`
 	MasterDNSNameFormat       config.StringTemplate `json:"master_dns_name_format,omitempty"`

pkg/apis/acid.zalan.do/v1/postgresql_type.go

@@ -40,6 +40,7 @@ type PostgresSpec struct {
 	// vars that enable load balancers are pointers because it is important to know if any of them is omitted from the Postgres manifest
 	// in that case the var evaluates to nil and the value is taken from the operator config
 	EnableMasterLoadBalancer  *bool `json:"enableMasterLoadBalancer,omitempty"`
+	EnablePublicLoadBalancer  *bool `json:"enablePublicLoadBalancer,omitempty"`
 	EnableReplicaLoadBalancer *bool `json:"enableReplicaLoadBalancer,omitempty"`
 
 	// deprecated load balancer settings maintained for backward compatibility

pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go

@@ -525,6 +525,11 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.EnablePublicLoadBalancer != nil {
+		in, out := &in.EnablePublicLoadBalancer, &out.EnablePublicLoadBalancer
+		*out = new(bool)
+		**out = **in
+	}
 	if in.EnableReplicaLoadBalancer != nil {
 		in, out := &in.EnableReplicaLoadBalancer, &out.EnableReplicaLoadBalancer
 		*out = new(bool)

pkg/cluster/cluster_test.go

@@ -403,6 +403,8 @@ func TestServiceAnnotations(t *testing.T) {
 		enableMasterLoadBalancerOC    bool
 		enableReplicaLoadBalancerSpec *bool
 		enableReplicaLoadBalancerOC   bool
+		enablePublicLoadBalancerSpec  *bool
+		enablePublicLoadBalancerOC    bool
 		operatorAnnotations           map[string]string
 		clusterAnnotations            map[string]string
 		expect                        map[string]string
@@ -427,6 +429,7 @@ func TestServiceAnnotations(t *testing.T) {
 			expect: map[string]string{
 				"external-dns.alpha.kubernetes.io/hostname":                            "test.acid.db.example.com",
 				"service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600",
+				"service.beta.kubernetes.io/aws-load-balancer-internal":                "true",
 			},
 		},
 		{
@@ -447,6 +450,7 @@ func TestServiceAnnotations(t *testing.T) {
 			expect: map[string]string{
 				"external-dns.alpha.kubernetes.io/hostname":                            "test.acid.db.example.com",
 				"service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600",
+				"service.beta.kubernetes.io/aws-load-balancer-internal":                "true",
 			},
 		},
 		{
@@ -458,6 +462,7 @@ func TestServiceAnnotations(t *testing.T) {
 			expect: map[string]string{
 				"external-dns.alpha.kubernetes.io/hostname":                            "test.acid.db.example.com",
 				"service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600",
+				"service.beta.kubernetes.io/aws-load-balancer-internal":                "true",
 				"foo": "bar",
 			},
 		},
@@ -479,6 +484,7 @@ func TestServiceAnnotations(t *testing.T) {
 			expect: map[string]string{
 				"external-dns.alpha.kubernetes.io/hostname":                            "test.acid.db.example.com",
 				"service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600",
+				"service.beta.kubernetes.io/aws-load-balancer-internal":                "true",
 				"foo": "bar",
 			},
 		},
@@ -493,6 +499,7 @@ func TestServiceAnnotations(t *testing.T) {
 			expect: map[string]string{
 				"external-dns.alpha.kubernetes.io/hostname":                            "test.acid.db.example.com",
 				"service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "1800",
+				"service.beta.kubernetes.io/aws-load-balancer-internal":                "true",
 			},
 		},
 		{
@@ -506,6 +513,7 @@ func TestServiceAnnotations(t *testing.T) {
 			expect: map[string]string{
 				"external-dns.alpha.kubernetes.io/hostname":                            "test.acid.db.example.com",
 				"service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "1800",
+				"service.beta.kubernetes.io/aws-load-balancer-internal":                "true",
 			},
 		},
 		{
@@ -519,6 +527,7 @@ func TestServiceAnnotations(t *testing.T) {
 			expect: map[string]string{
 				"external-dns.alpha.kubernetes.io/hostname":                            "test.acid.db.example.com",
 				"service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600",
+				"service.beta.kubernetes.io/aws-load-balancer-internal":                "true",
 			},
 		},
 		{
@@ -532,6 +541,7 @@ func TestServiceAnnotations(t *testing.T) {
 			expect: map[string]string{
 				"external-dns.alpha.kubernetes.io/hostname":                            "test.acid.db.example.com",
 				"service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600",
+				"service.beta.kubernetes.io/aws-load-balancer-internal":                "true",
 			},
 		},
 		// REPLICA
@@ -554,6 +564,7 @@ func TestServiceAnnotations(t *testing.T) {
 			expect: map[string]string{
 				"external-dns.alpha.kubernetes.io/hostname":                            "test-repl.acid.db.example.com",
 				"service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600",
+				"service.beta.kubernetes.io/aws-load-balancer-internal":                "true",
 			},
 		},
 		{
@@ -574,6 +585,7 @@ func TestServiceAnnotations(t *testing.T) {
 			expect: map[string]string{
 				"external-dns.alpha.kubernetes.io/hostname":                            "test-repl.acid.db.example.com",
 				"service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600",
+				"service.beta.kubernetes.io/aws-load-balancer-internal":                "true",
 			},
 		},
 		{
@@ -585,6 +597,7 @@ func TestServiceAnnotations(t *testing.T) {
 			expect: map[string]string{
 				"external-dns.alpha.kubernetes.io/hostname":                            "test-repl.acid.db.example.com",
 				"service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600",
+				"service.beta.kubernetes.io/aws-load-balancer-internal":                "true",
 				"foo": "bar",
 			},
 		},
@@ -606,6 +619,7 @@ func TestServiceAnnotations(t *testing.T) {
 			expect: map[string]string{
 				"external-dns.alpha.kubernetes.io/hostname":                            "test-repl.acid.db.example.com",
 				"service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600",
+				"service.beta.kubernetes.io/aws-load-balancer-internal":                "true",
 				"foo": "bar",
 			},
 		},
@@ -620,6 +634,7 @@ func TestServiceAnnotations(t *testing.T) {
 			expect: map[string]string{
 				"external-dns.alpha.kubernetes.io/hostname":                            "test-repl.acid.db.example.com",
 				"service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "1800",
+				"service.beta.kubernetes.io/aws-load-balancer-internal":                "true",
 			},
 		},
 		{
@@ -633,6 +648,7 @@ func TestServiceAnnotations(t *testing.T) {
 			expect: map[string]string{
 				"external-dns.alpha.kubernetes.io/hostname":                            "test-repl.acid.db.example.com",
 				"service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "1800",
+				"service.beta.kubernetes.io/aws-load-balancer-internal":                "true",
 			},
 		},
 		{
@@ -646,6 +662,7 @@ func TestServiceAnnotations(t *testing.T) {
 			expect: map[string]string{
 				"external-dns.alpha.kubernetes.io/hostname":                            "test-repl.acid.db.example.com",
 				"service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600",
+				"service.beta.kubernetes.io/aws-load-balancer-internal":                "true",
 			},
 		},
 		{
@@ -659,6 +676,47 @@ func TestServiceAnnotations(t *testing.T) {
 			expect: map[string]string{
 				"external-dns.alpha.kubernetes.io/hostname":                            "test-repl.acid.db.example.com",
 				"service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600",
+				"service.beta.kubernetes.io/aws-load-balancer-internal":                "true",
+			},
+		},
+		//PUBLIC
+		{
+			about:                        "Public ELB disabled in spec and OperatorConfig",
+			role:                         "master",
+			enableMasterLoadBalancerSpec: &enabled,
+			enablePublicLoadBalancerSpec: &disabled,
+			enablePublicLoadBalancerOC:   false,
+			operatorAnnotations:          make(map[string]string),
+			clusterAnnotations:           make(map[string]string),
+			expect: map[string]string{
+				"external-dns.alpha.kubernetes.io/hostname":                            "test.acid.db.example.com",
+				"service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600",
+				"service.beta.kubernetes.io/aws-load-balancer-internal":                "true",
+			},
+		},
+		{
+			about:                        "Enable public ELB with EnablePublicLoadBalancer in spec but disabled in OperatorConfig",
+			role:                         "master",
+			enableMasterLoadBalancerSpec: &enabled,
+			enablePublicLoadBalancerSpec: &enabled,
+			enablePublicLoadBalancerOC:   false,
+			operatorAnnotations:          make(map[string]string),
+			clusterAnnotations:           make(map[string]string),
+			expect: map[string]string{
+				"external-dns.alpha.kubernetes.io/hostname":                            "test.acid.db.example.com",
+				"service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600",
+			},
+		},
+		{
+			about:                        "Enable public ELB with enable_public_load_balancer in OperatorConfig",
+			role:                         "master",
+			enableMasterLoadBalancerSpec: &enabled,
+			enablePublicLoadBalancerOC:   true,
+			operatorAnnotations:          make(map[string]string),
+			clusterAnnotations:           make(map[string]string),
+			expect: map[string]string{
+				"external-dns.alpha.kubernetes.io/hostname":                            "test.acid.db.example.com",
+				"service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600",
 			},
 		},
 		// COMMON
@@ -685,6 +743,7 @@ func TestServiceAnnotations(t *testing.T) {
 			cl.OpConfig.CustomServiceAnnotations = tt.operatorAnnotations
 			cl.OpConfig.EnableMasterLoadBalancer = tt.enableMasterLoadBalancerOC
 			cl.OpConfig.EnableReplicaLoadBalancer = tt.enableReplicaLoadBalancerOC
+			cl.OpConfig.EnablePublicLoadBalancer = tt.enablePublicLoadBalancerOC
 			cl.OpConfig.MasterDNSNameFormat = "{cluster}.{team}.{hostedzone}"
 			cl.OpConfig.ReplicaDNSNameFormat = "{cluster}-repl.{team}.{hostedzone}"
 			cl.OpConfig.DbHostedZone = "db.example.com"
@@ -694,6 +753,7 @@ func TestServiceAnnotations(t *testing.T) {
 			cl.Postgresql.Spec.ServiceAnnotations = tt.clusterAnnotations
 			cl.Postgresql.Spec.EnableMasterLoadBalancer = tt.enableMasterLoadBalancerSpec
 			cl.Postgresql.Spec.EnableReplicaLoadBalancer = tt.enableReplicaLoadBalancerSpec
+			cl.Postgresql.Spec.EnablePublicLoadBalancer = tt.enablePublicLoadBalancerSpec
 
 			got := cl.generateServiceAnnotations(tt.role, &cl.Postgresql.Spec)
 			if len(tt.expect) != len(got) {

pkg/cluster/k8sres.go

@@ -1525,6 +1525,17 @@ func (c *Cluster) shouldCreateLoadBalancerForService(role PostgresRole, spec *ac
 
 }
 
+func (c *Cluster) shouldUseInternalLoadBalancerForService(spec *acidv1.PostgresSpec) bool {
+
+	// if the value is explicitly set in a Postgresql manifest, follow this setting
+	if spec.EnablePublicLoadBalancer != nil {
+		return !(*spec.EnablePublicLoadBalancer)
+	}
+
+	// otherwise, follow the operator configuration
+	return !c.OpConfig.EnablePublicLoadBalancer
+}
+
 func (c *Cluster) generateService(role PostgresRole, spec *acidv1.PostgresSpec) *v1.Service {
 	serviceSpec := v1.ServiceSpec{
 		Ports: []v1.ServicePort{{Name: "postgresql", Port: 5432, TargetPort: intstr.IntOrString{IntVal: 5432}}},
@@ -1594,6 +1605,10 @@ func (c *Cluster) generateServiceAnnotations(role PostgresRole, spec *acidv1.Pos
 		}
 		// External DNS name annotation is not customizable
 		annotations[constants.ZalandoDNSNameAnnotation] = dnsName
+
+		if c.shouldUseInternalLoadBalancerForService(spec) {
+			annotations[constants.ElbInternal] = "true"
+		}
 	}
 
 	if len(annotations) == 0 {

pkg/controller/operator_config.go

@@ -101,6 +101,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	// load balancer config
 	result.DbHostedZone = fromCRD.LoadBalancer.DbHostedZone
 	result.EnableMasterLoadBalancer = fromCRD.LoadBalancer.EnableMasterLoadBalancer
+	result.EnablePublicLoadBalancer = fromCRD.LoadBalancer.EnablePublicLoadBalancer
 	result.EnableReplicaLoadBalancer = fromCRD.LoadBalancer.EnableReplicaLoadBalancer
 	result.CustomServiceAnnotations = fromCRD.LoadBalancer.CustomServiceAnnotations
 	result.MasterDNSNameFormat = fromCRD.LoadBalancer.MasterDNSNameFormat

pkg/util/config/config.go

@@ -134,7 +134,8 @@ type Config struct {
 	EnableTeamSuperuser                    bool              `name:"enable_team_superuser" default:"false"`
 	TeamAdminRole                          string            `name:"team_admin_role" default:"admin"`
 	EnableAdminRoleForUsers                bool              `name:"enable_admin_role_for_users" default:"true"`
-	EnableMasterLoadBalancer               bool              `name:"enable_master_load_balancer" default:"true"`
+	EnableMasterLoadBalancer               bool              `name:"enable_master_load_balancer" default:"false"`
+	EnablePublicLoadBalancer               bool              `name:"enable_public_load_balancer" default:"false"`
 	EnableReplicaLoadBalancer              bool              `name:"enable_replica_load_balancer" default:"false"`
 	CustomServiceAnnotations               map[string]string `name:"custom_service_annotations"`
 	CustomPodAnnotations                   map[string]string `name:"custom_pod_annotations"`

pkg/util/constants/annotations.go

@@ -5,6 +5,7 @@ const (
 	ZalandoDNSNameAnnotation           = "external-dns.alpha.kubernetes.io/hostname"
 	ElbTimeoutAnnotationName           = "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout"
 	ElbTimeoutAnnotationValue          = "3600"
+	ElbInternal                        = "service.beta.kubernetes.io/aws-load-balancer-internal"
 	KubeIAmAnnotation                  = "iam.amazonaws.com/role"
 	VolumeStorateProvisionerAnnotation = "pv.kubernetes.io/provisioned-by"
 	PostgresqlControllerAnnotationKey  = "acid.zalan.do/controller"