From 202f2de988ac451a962bf63603a73f93e20fa485 Mon Sep 17 00:00:00 2001
From: Murat Kabilov <murat@kabilov.com>
Date: Tue, 17 Oct 2017 17:03:50 +0200
Subject: [PATCH] Retry connecting to pg

---
 pkg/cluster/pg.go                | 55 +++++++++++++++++++++-----------
 pkg/util/constants/postgresql.go |  5 +++
 pkg/util/patroni/patroni.go      |  2 +-
 3 files changed, 43 insertions(+), 19 deletions(-)

diff --git a/pkg/cluster/pg.go b/pkg/cluster/pg.go
index 59eac0ef7..582877fd9 100644
--- a/pkg/cluster/pg.go
+++ b/pkg/cluster/pg.go
@@ -3,12 +3,14 @@ package cluster
 import (
 	"database/sql"
 	"fmt"
+	"net"
 	"strings"
 
 	"github.com/lib/pq"
 
 	"github.com/zalando-incubator/postgres-operator/pkg/spec"
 	"github.com/zalando-incubator/postgres-operator/pkg/util/constants"
+	"github.com/zalando-incubator/postgres-operator/pkg/util/retryutil"
 )
 
 const (
@@ -27,14 +29,13 @@ const (
 )
 
 func (c *Cluster) pgConnectionString() string {
-	hostname := fmt.Sprintf("%s.%s.svc.cluster.local", c.Name, c.Namespace)
-	username := c.systemUsers[constants.SuperuserKeyName].Name
 	password := c.systemUsers[constants.SuperuserKeyName].Password
 
-	return fmt.Sprintf("host='%s' dbname=postgres sslmode=require user='%s' password='%s'",
-		hostname,
-		username,
-		strings.Replace(password, "$", "\\$", -1))
+	return fmt.Sprintf("host='%s' dbname=postgres sslmode=require user='%s' password='%s' connect_timeout='%d'",
+		fmt.Sprintf("%s.%s.svc.cluster.local", c.Name, c.Namespace),
+		c.systemUsers[constants.SuperuserKeyName].Name,
+		strings.Replace(password, "$", "\\$", -1),
+		constants.PostgresConnectTimeout)
 }
 
 func (c *Cluster) databaseAccessDisabled() bool {
@@ -45,25 +46,43 @@ func (c *Cluster) databaseAccessDisabled() bool {
 	return !c.OpConfig.EnableDBAccess
 }
 
-func (c *Cluster) initDbConn() (err error) {
+func (c *Cluster) initDbConn() error {
 	c.setProcessName("initializing db connection")
-	if c.pgDb == nil {
-		conn, err := sql.Open("postgres", c.pgConnectionString())
-		if err != nil {
-			return err
-		}
-		c.logger.Debug("new database connection")
-		err = conn.Ping()
-		if err != nil {
+	if c.pgDb != nil {
+		return nil
+	}
+
+	conn, err := sql.Open("postgres", c.pgConnectionString())
+	if err != nil {
+		return err
+	}
+
+	c.logger.Debug("new database connection")
+	err = retryutil.Retry(0, constants.PostgresConnectRetryTimeout,
+		func() (bool, error) {
+			err := conn.Ping()
+			if err == nil {
+				return true, nil
+			}
+
 			if err2 := conn.Close(); err2 != nil {
 				c.logger.Errorf("error when closing PostgreSQL connection after another error: %v", err2)
+				return false, err2
 			}
-			return err
-		}
 
-		c.pgDb = conn
+			if _, ok := err.(*net.OpError); ok {
+				c.logger.Errorf("could not connect to PostgreSQL database: %v", err)
+				return false, nil
+			}
+			return false, err
+		})
+
+	if err != nil {
+		return fmt.Errorf("could not init db connection: %v", err)
 	}
 
+	c.pgDb = conn
+
 	return nil
 }
 
diff --git a/pkg/util/constants/postgresql.go b/pkg/util/constants/postgresql.go
index 2ff58fa1f..7556e8858 100644
--- a/pkg/util/constants/postgresql.go
+++ b/pkg/util/constants/postgresql.go
@@ -1,8 +1,13 @@
 package constants
 
+import "time"
+
 // PostgreSQL specific constants
 const (
 	DataVolumeName    = "pgdata"
 	PostgresDataMount = "/home/postgres/pgdata"
 	PostgresDataPath  = PostgresDataMount + "/pgroot"
+
+	PostgresConnectRetryTimeout = 2 * time.Minute
+	PostgresConnectTimeout      = 15 * time.Second
 )
diff --git a/pkg/util/patroni/patroni.go b/pkg/util/patroni/patroni.go
index 5789755aa..b27a20766 100644
--- a/pkg/util/patroni/patroni.go
+++ b/pkg/util/patroni/patroni.go
@@ -77,4 +77,4 @@ func (p *Patroni) Failover(master *v1.Pod, candidate string) error {
 	}
 
 	return nil
-}
\ No newline at end of file
+}