Lookup function installation (#1171)

* Lookup function installation

Due to reusing a previous database connection without closing it, lookup
function installation process was skipping the first database in the
list, installing twice into postgres db instead. To prevent that, make
internal initDbConnWithName to overwrite a connection object, and return
the same object only from initDbConn, which is sort of public interface.

Another solution for this would be to modify initDbConnWithName to
return a connection object and then generate one temporary connection
for each db. It sound feasible but after one attempt it seems it
requires a bit more changes around (init, close connections) and
doesn't bring anything significantly better on the table. In case if
some future changes will prove this wrong, do not hesitate to refactor.

Change retry strategy to more insistive one, namely:

* retry on the next sync even if we failed to process one database and
install pooler appliance.

* perform the whole installation unconditionally on update, since the
list of target databases could be changed.

And for the sake of making it even more robust, also log the case when
operator decides to skip installation.

Extend connection pooler e2e test with verification that all dbs have
required schema installed.

e2e/exec.sh

@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-kubectl exec -it $1 -- sh -c "$2"
+kubectl exec -i $1 -- sh -c "$2"

e2e/tests/test_e2e.py

@@ -15,6 +15,14 @@ def to_selector(labels):
     return ",".join(["=".join(l) for l in labels.items()])
 
 
+def clean_list(values):
+    # value is not stripped bytes, strip and convert to a string
+    clean = lambda v: v.strip().decode()
+    notNone = lambda v: v
+
+    return list(filter(notNone, map(clean, values)))
+
+
 class EndToEndTestCase(unittest.TestCase):
     '''
     Test interaction of the operator with multiple K8s components.
@@ -140,6 +148,58 @@ class EndToEndTestCase(unittest.TestCase):
 
             k8s.wait_for_running_pods(pod_selector, 2)
 
+            # Verify that all the databases have pooler schema installed.
+            # Do this via psql, since otherwise we need to deal with
+            # credentials.
+            dbList = []
+
+            leader = k8s.get_cluster_leader_pod('acid-minimal-cluster')
+            dbListQuery = "select datname from pg_database"
+            schemasQuery = """
+                select schema_name
+                from information_schema.schemata
+                where schema_name = 'pooler'
+            """
+            exec_query = r"psql -tAq -c \"{}\" -d {}"
+
+            if leader:
+                try:
+                    q = exec_query.format(dbListQuery, "postgres")
+                    q = "su postgres -c \"{}\"".format(q)
+                    print('Get databases: {}'.format(q))
+                    result = k8s.exec_with_kubectl(leader.metadata.name, q)
+                    dbList = clean_list(result.stdout.split(b'\n'))
+                    print('dbList: {}, stdout: {}, stderr {}'.format(
+                        dbList, result.stdout, result.stderr
+                    ))
+                except Exception as ex:
+                    print('Could not get databases: {}'.format(ex))
+                    print('Stdout: {}'.format(result.stdout))
+                    print('Stderr: {}'.format(result.stderr))
+
+                for db in dbList:
+                    if db in ('template0', 'template1'):
+                        continue
+
+                    schemas = []
+                    try:
+                        q = exec_query.format(schemasQuery, db)
+                        q = "su postgres -c \"{}\"".format(q)
+                        print('Get schemas: {}'.format(q))
+                        result = k8s.exec_with_kubectl(leader.metadata.name, q)
+                        schemas = clean_list(result.stdout.split(b'\n'))
+                        print('schemas: {}, stdout: {}, stderr {}'.format(
+                            schemas, result.stdout, result.stderr
+                        ))
+                    except Exception as ex:
+                        print('Could not get databases: {}'.format(ex))
+                        print('Stdout: {}'.format(result.stdout))
+                        print('Stderr: {}'.format(result.stderr))
+
+                    self.assertNotEqual(len(schemas), 0)
+            else:
+                print('Could not find leader pod')
+
             # turn it off, keeping configuration section
             k8s.api.custom_objects_api.patch_namespaced_custom_object(
                 'acid.zalan.do', 'v1', 'default',
@@ -235,7 +295,10 @@ class EndToEndTestCase(unittest.TestCase):
             # operator configuration via API
             operator_pod = k8s.get_operator_pod()
             get_config_cmd = "wget --quiet -O - localhost:8080/config"
-            result = k8s.exec_with_kubectl(operator_pod.metadata.name, get_config_cmd)
+            result = k8s.exec_with_kubectl(
+                operator_pod.metadata.name,
+                get_config_cmd,
+            )
             roles_dict = (json.loads(result.stdout)
                           .get("controller", {})
                           .get("InfrastructureRoles"))
@@ -862,6 +925,19 @@ class K8s:
 
         return master_pod_node, replica_pod_nodes
 
+    def get_cluster_leader_pod(self, pg_cluster_name, namespace='default'):
+        labels = {
+            'application': 'spilo',
+            'cluster-name': pg_cluster_name,
+            'spilo-role': 'master',
+        }
+
+        pods = self.api.core_v1.list_namespaced_pod(
+                namespace, label_selector=to_selector(labels)).items
+
+        if pods:
+            return pods[0]
+
     def wait_for_operator_pod_start(self):
         self. wait_for_pod_start("name=postgres-operator")
         # HACK operator must register CRD and/or Sync existing PG clusters after start up