Fixes for the case of re-creating the cluster after deletion.

- make sure that the secrets for the system users (superuser, replication)
  are not deleted when the main cluster is. Therefore, we can re-create
  the cluster, potentially forcing Patroni to restore it from the backup
  and enable Patroni to connect, since it will use the old password, not
  the newly generated random one.

- when syncing users, always check whether they are already in the DB.
  Previously, we did this only for the sync cluster case, but the new
  cluster could be actually the one restored from the backup by Patroni,
  having all or some of the users already in place.

 - delete endponts last. Patroni uses the $clustername endpoint in order
   to store the leader related metadata. If we remove it before removing
   all pods, one of those pods running Patroni will re-create it and the
   next attempt to create the cluster with the same name will stuble on
   the existing endpoint.

 - Use db.Exec instead of db.Query for queries that expect no result.
   This also fixes the issue with the DB creation, since we didn't
   release an empty Row object it was not possible to create more than
   one database for a cluster.

pkg/cluster/cluster_test.go

@@ -7,14 +7,20 @@ import (
 	"github.com/zalando-incubator/postgres-operator/pkg/util/config"
 	"github.com/zalando-incubator/postgres-operator/pkg/util/k8sutil"
 	"github.com/zalando-incubator/postgres-operator/pkg/util/teams"
+	"k8s.io/client-go/pkg/api/v1"
 	"reflect"
 	"testing"
 )
 
+const (
+	superUserName       = "postgres"
+	replicationUserName = "standby"
+)
+
 var logger = logrus.New().WithField("test", "cluster")
 var cl = New(Config{OpConfig: config.Config{ProtectedRoles: []string{"admin"},
-	Auth: config.Auth{SuperUsername: "postgres",
-		ReplicationUsername: "standby"}}},
+	Auth: config.Auth{SuperUsername: superUserName,
+		ReplicationUsername: replicationUserName}}},
 	k8sutil.KubernetesClient{}, spec.Postgresql{}, logger)
 
 func TestInitRobotUsers(t *testing.T) {
@@ -52,7 +58,7 @@ func TestInitRobotUsers(t *testing.T) {
 				`conflicting user flags: "NOINHERIT" and "INHERIT"`),
 		},
 		{
-			manifestUsers: map[string]spec.UserFlags{"admin": {"superuser"}, "postgres": {"createdb"}},
+			manifestUsers: map[string]spec.UserFlags{"admin": {"superuser"}, superUserName: {"createdb"}},
 			infraRoles:    map[string]spec.PgUser{},
 			result:        map[string]spec.PgUser{},
 			err:           nil,
@@ -121,7 +127,7 @@ func TestInitHumanUsers(t *testing.T) {
 		},
 		{
 			existingRoles: map[string]spec.PgUser{},
-			teamRoles:     []string{"admin", "standby"},
+			teamRoles:     []string{"admin", replicationUserName},
 			result:        map[string]spec.PgUser{},
 		},
 	}
@@ -138,3 +144,33 @@ func TestInitHumanUsers(t *testing.T) {
 		}
 	}
 }
+
+func TestShouldDeleteSecret(t *testing.T) {
+	testName := "TestShouldDeleteSecret"
+
+	tests := []struct {
+		secret  *v1.Secret
+		outcome bool
+	}{
+		{
+			secret:  &v1.Secret{Data: map[string][]byte{"username": []byte("foobar")}},
+			outcome: true,
+		},
+		{
+			secret: &v1.Secret{Data: map[string][]byte{"username": []byte(superUserName)}},
+
+			outcome: false,
+		},
+		{
+			secret:  &v1.Secret{Data: map[string][]byte{"username": []byte(replicationUserName)}},
+			outcome: false,
+		},
+	}
+
+	for _, tt := range tests {
+		if outcome, username := cl.shouldDeleteSecret(tt.secret); outcome != tt.outcome {
+			t.Errorf("%s expects the check for deletion of the username %q secret to return %t, got %t",
+				testName, username, tt.outcome, outcome)
+		}
+	}
+}