nearly there

.gitignore

@@ -11,7 +11,7 @@
 /unifi-poller.*.linux
 /unifi-poller.rb
 *.sha256
-/vendor
+#/vendor
 .DS_Store
 *~
 /package_build_*

pkg/poller/config.go

@@ -41,7 +41,6 @@ const ENVConfigPrefix = "UP"
 // UnifiPoller contains the application startup data, and auth info for UniFi & Influx.
 type UnifiPoller struct {
 	Influx     *influxunifi.InfluxUnifi
-	Unifi      *unifi.Unifi
 	Flag       *Flag
 	Config     *Config
 	LastCheck  time.Time
@@ -59,14 +58,14 @@ type Flag struct {
 // Controller represents the configuration for a UniFi Controller.
 // Each polled controller may have its own configuration.
 type Controller struct {
-	VerifySSL bool     `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"`
+	VerifySSL bool         `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"`
-	SaveIDS   bool     `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"`
+	SaveIDS   bool         `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"`
-	ReAuth    bool     `json:"reauthenticate" toml:"reauthenticate" xml:"reauthenticate" yaml:"reauthenticate"`
+	SaveSites bool         `json:"save_sites,omitempty" toml:"save_sites,omitempty" xml:"save_sites" yaml:"save_sites"`
-	SaveSites bool     `json:"save_sites,omitempty" toml:"save_sites,omitempty" xml:"save_sites" yaml:"save_sites"`
+	User      string       `json:"unifi_user,omitempty" toml:"unifi_user,omitempty" xml:"unifi_user" yaml:"unifi_user"`
-	User      string   `json:"unifi_user,omitempty" toml:"unifi_user,omitempty" xml:"unifi_user" yaml:"unifi_user"`
+	Pass      string       `json:"unifi_pass,omitempty" toml:"unifi_pass,omitempty" xml:"unifi_pass" yaml:"unifi_pass"`
-	Pass      string   `json:"unifi_pass,omitempty" toml:"unifi_pass,omitempty" xml:"unifi_pass" yaml:"unifi_pass"`
+	URL       string       `json:"unifi_url,omitempty" toml:"unifi_url,omitempty" xml:"unifi_url" yaml:"unifi_url"`
-	URL       string   `json:"unifi_url,omitempty" toml:"unifi_url,omitempty" xml:"unifi_url" yaml:"unifi_url"`
+	Sites     []string     `json:"sites,omitempty" toml:"sites,omitempty" xml:"sites" yaml:"sites"`
-	Sites     []string `json:"sites,omitempty" toml:"sites,omitempty" xml:"sites" yaml:"sites"`
+	Unifi     *unifi.Unifi `json:"-" toml:"-" xml:"-" yaml:"-"`
 }
 
 // Config represents the data needed to poll a controller and report to influxdb.
@@ -76,11 +75,7 @@ type Config struct {
 	Interval   config.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"`
 	Debug      bool            `json:"debug" toml:"debug" xml:"debug" yaml:"debug"`
 	Quiet      bool            `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet" yaml:"quiet"`
-	VerifySSL  bool            `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"`
-	SaveIDS    bool            `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"`
-	ReAuth     bool            `json:"reauthenticate" toml:"reauthenticate" xml:"reauthenticate" yaml:"reauthenticate"`
 	InfxBadSSL bool            `json:"influx_insecure_ssl" toml:"influx_insecure_ssl" xml:"influx_insecure_ssl" yaml:"influx_insecure_ssl"`
-	SaveSites  bool            `json:"save_sites,omitempty" toml:"save_sites,omitempty" xml:"save_sites" yaml:"save_sites"`
 	Mode       string          `json:"mode" toml:"mode" xml:"mode" yaml:"mode"`
 	HTTPListen string          `json:"http_listen" toml:"http_listen" xml:"http_listen" yaml:"http_listen"`
 	Namespace  string          `json:"namespace" toml:"namespace" xml:"namespace" yaml:"namespace"`
@@ -88,9 +83,5 @@ type Config struct {
 	InfluxUser string          `json:"influx_user,omitempty" toml:"influx_user,omitempty" xml:"influx_user" yaml:"influx_user"`
 	InfluxPass string          `json:"influx_pass,omitempty" toml:"influx_pass,omitempty" xml:"influx_pass" yaml:"influx_pass"`
 	InfluxDB   string          `json:"influx_db,omitempty" toml:"influx_db,omitempty" xml:"influx_db" yaml:"influx_db"`
-	UnifiUser  string          `json:"unifi_user,omitempty" toml:"unifi_user,omitempty" xml:"unifi_user" yaml:"unifi_user"`
-	UnifiPass  string          `json:"unifi_pass,omitempty" toml:"unifi_pass,omitempty" xml:"unifi_pass" yaml:"unifi_pass"`
-	UnifiBase  string          `json:"unifi_url,omitempty" toml:"unifi_url,omitempty" xml:"unifi_url" yaml:"unifi_url"`
-	Sites      []string        `json:"sites,omitempty" toml:"sites,omitempty" xml:"sites" yaml:"sites"`
 	Controller []Controller    `json:"controller,omitempty" toml:"controller,omitempty" xml:"controller" yaml:"controller"`
 }

pkg/poller/dumper.go

@@ -9,50 +9,53 @@ import (
 )
 
 // DumpJSONPayload prints raw json from the UniFi Controller.
+// This only works with controller 0 (first one) in the config.
 func (u *UnifiPoller) DumpJSONPayload() (err error) {
 	u.Config.Quiet = true
-	u.Unifi, err = unifi.NewUnifi(&unifi.Config{
+	config := u.Config.Controller[0]
-		User:      u.Config.UnifiUser,
+
-		Pass:      u.Config.UnifiPass,
+	config.Unifi, err = unifi.NewUnifi(&unifi.Config{
-		URL:       u.Config.UnifiBase,
+		User:      config.User,
-		VerifySSL: u.Config.VerifySSL,
+		Pass:      config.Pass,
+		URL:       config.URL,
+		VerifySSL: config.VerifySSL,
 	})
 	if err != nil {
 		return err
 	}
 
-	fmt.Fprintf(os.Stderr, "[INFO] Authenticated to UniFi Controller @ %v as user %v",
+	fmt.Fprintf(os.Stderr, "[INFO] Authenticated to UniFi Controller @ %v as user %v", config.URL, config.User)
-		u.Config.UnifiBase, u.Config.UnifiUser)
+
-	if err := u.CheckSites(); err != nil {
+	if err := u.CheckSites(config); err != nil {
 		return err
 	}
 
-	u.Unifi.ErrorLog = func(m string, v ...interface{}) {
+	config.Unifi.ErrorLog = func(m string, v ...interface{}) {
 		fmt.Fprintf(os.Stderr, "[ERROR] "+m, v...)
 	} // Log all errors to stderr.
 
-	switch sites, err := u.GetFilteredSites(); {
+	switch sites, err := u.GetFilteredSites(config); {
 	case err != nil:
 		return err
 	case StringInSlice(u.Flag.DumpJSON, []string{"d", "device", "devices"}):
-		return u.dumpSitesJSON(unifi.APIDevicePath, "Devices", sites)
+		return u.dumpSitesJSON(config, unifi.APIDevicePath, "Devices", sites)
 	case StringInSlice(u.Flag.DumpJSON, []string{"client", "clients", "c"}):
-		return u.dumpSitesJSON(unifi.APIClientPath, "Clients", sites)
+		return u.dumpSitesJSON(config, unifi.APIClientPath, "Clients", sites)
 	case strings.HasPrefix(u.Flag.DumpJSON, "other "):
 		apiPath := strings.SplitN(u.Flag.DumpJSON, " ", 2)[1]
 		_, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping Path '%s':\n", apiPath)
-		return u.PrintRawAPIJSON(apiPath)
+		return u.PrintRawAPIJSON(config, apiPath)
 	default:
 		return fmt.Errorf("must provide filter: devices, clients, other")
 	}
 }
 
-func (u *UnifiPoller) dumpSitesJSON(path, name string, sites unifi.Sites) error {
+func (u *UnifiPoller) dumpSitesJSON(c Controller, path, name string, sites unifi.Sites) error {
 	for _, s := range sites {
 		apiPath := fmt.Sprintf(path, s.Name)
 		_, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping %s: '%s' JSON for site: %s (%s):\n",
 			name, apiPath, s.Desc, s.Name)
-		if err := u.PrintRawAPIJSON(apiPath); err != nil {
+		if err := u.PrintRawAPIJSON(c, apiPath); err != nil {
 			return err
 		}
 	}
@@ -60,8 +63,8 @@ func (u *UnifiPoller) dumpSitesJSON(path, name string, sites unifi.Sites) error
 }
 
 // PrintRawAPIJSON prints the raw json for a user-provided path on a UniFi Controller.
-func (u *UnifiPoller) PrintRawAPIJSON(apiPath string) error {
+func (u *UnifiPoller) PrintRawAPIJSON(c Controller, apiPath string) error {
-	body, err := u.Unifi.GetJSON(apiPath)
+	body, err := c.Unifi.GetJSON(apiPath)
 	fmt.Println(string(body))
 	return err
 }

pkg/poller/influx.go

@@ -44,8 +44,6 @@ func (u *UnifiPoller) CollectAndProcess() error {
 		return err
 	}
 
-	u.AugmentMetrics(metrics)
-
 	report, err := u.Influx.ReportMetrics(metrics)
 	if err != nil {
 		return err
@@ -57,12 +55,7 @@ func (u *UnifiPoller) CollectAndProcess() error {
 
 // LogInfluxReport writes a log message after exporting to influxdb.
 func (u *UnifiPoller) LogInfluxReport(r *influxunifi.Report) {
-	idsMsg := ""
+	idsMsg := fmt.Sprintf("IDS Events: %d, ", len(r.Metrics.IDSList))
-
-	if u.Config.SaveIDS {
-		idsMsg = fmt.Sprintf("IDS Events: %d, ", len(r.Metrics.IDSList))
-	}
-
 	u.Logf("UniFi Metrics Recorded. Sites: %d, Clients: %d, "+
 		"UAP: %d, USG/UDM: %d, USW: %d, %sPoints: %d, Fields: %d, Errs: %d, Elapsed: %v",
 		len(r.Metrics.Sites), len(r.Metrics.Clients), len(r.Metrics.UAPs),

pkg/poller/prometheus.go

@@ -36,24 +36,7 @@ func (u *UnifiPoller) RunPrometheus() error {
 // HTTP at /metrics for prometheus collection.
 // This is run by Prometheus as CollectFn.
 func (u *UnifiPoller) ExportMetrics() (*metrics.Metrics, error) {
-	m, err := u.CollectMetrics()
+	return u.CollectMetrics()
-	if err != nil {
-		u.LogErrorf("collecting metrics: %v", err)
-		u.Logf("Re-authenticating to UniFi Controller")
-
-		if err := u.GetUnifi(); err != nil {
-			u.LogErrorf("re-authenticating: %v", err)
-			return nil, err
-		}
-
-		if m, err = u.CollectMetrics(); err != nil {
-			u.LogErrorf("collecting metrics: %v", err)
-			return nil, err
-		}
-	}
-
-	u.AugmentMetrics(m)
-	return m, nil
 }
 
 // LogExportReport is called after prometheus exports metrics.

pkg/poller/start.go

@@ -17,16 +17,18 @@ import (
 func New() *UnifiPoller {
 	return &UnifiPoller{
 		Config: &Config{
+			Controller: []Controller{{
+				Sites:     []string{"all"},
+				User:      defaultUnifiUser,
+				Pass:      "",
+				URL:       defaultUnifiURL,
+				SaveSites: true,
+			}},
 			InfluxURL:  defaultInfluxURL,
 			InfluxUser: defaultInfluxUser,
 			InfluxPass: defaultInfluxPass,
 			InfluxDB:   defaultInfluxDB,
-			UnifiUser:  defaultUnifiUser,
-			UnifiPass:  "",
-			UnifiBase:  defaultUnifiURL,
 			Interval:   config.Duration{Duration: defaultInterval},
-			Sites:      []string{"all"},
-			SaveSites:  true,
 			HTTPListen: defaultHTTPListen,
 			Namespace:  appName,
 		},
@@ -63,7 +65,7 @@ func (u *UnifiPoller) Start() error {
 	if _, err := config.ParseENV(u.Config, ENVConfigPrefix); err != nil {
 		return err
 	}
-	log.Println("START():", u.Config.Controller)
+	log.Println("START(): controller", u.Config.Controller)
 	if u.Flag.DumpJSON != "" {
 		return u.DumpJSONPayload()
 	}
@@ -72,8 +74,9 @@ func (u *UnifiPoller) Start() error {
 		log.SetFlags(log.Lshortfile | log.Lmicroseconds | log.Ldate)
 		u.LogDebugf("Debug Logging Enabled")
 	}
-	log.Println("sites", u.Config.Sites)
+
 	log.Printf("[INFO] UniFi Poller v%v Starting Up! PID: %d", Version, os.Getpid())
+
 	return u.Run()
 }
 
@@ -97,12 +100,14 @@ func (f *Flag) Parse(args []string) {
 // 2. Run the collector one time and report the metrics to influxdb. (lambda)
 // 3. Start a web server and wait for Prometheus to poll the application for metrics.
 func (u *UnifiPoller) Run() error {
-	switch err := u.GetUnifi(); err {
+	for _, c := range u.Config.Controller {
-	case nil:
+		switch err := u.GetUnifi(c); err {
-		u.Logf("Polling UniFi Controller at %s v%s as user %s. Sites: %v",
+		case nil:
-			u.Config.UnifiBase, u.Unifi.ServerVersion, u.Config.UnifiUser, u.Config.Sites)
+			u.Logf("Polling UniFi Controller at %s v%s as user %s. Sites: %v",
-	default:
+				c.URL, c.Unifi.ServerVersion, c.User, c.Sites)
-		u.LogErrorf("Controller Auth or Connection failed, but continuing to retry! %v", err)
+		default:
+			u.LogErrorf("Controller Auth or Connection failed, but continuing to retry! %s: %v", c.URL, err)
+		}
 	}
 
 	switch strings.ToLower(u.Config.Mode) {
@@ -130,11 +135,6 @@ func (u *UnifiPoller) PollController() {
 	for u.LastCheck = range ticker.C {
 		if err := u.CollectAndProcess(); err != nil {
 			u.LogErrorf("%v", err)
-
-			if u.Unifi != nil {
-				u.Unifi.CloseIdleConnections()
-				u.Unifi = nil // trigger re-auth in unifi.go.
-			}
 		}
 	}
 }

pkg/poller/unifi.go

@@ -10,43 +10,44 @@ import (
 )
 
 // GetUnifi returns a UniFi controller interface.
-func (u *UnifiPoller) GetUnifi() (err error) {
+func (u *UnifiPoller) GetUnifi(config Controller) error {
+	var err error
+
 	u.Lock()
 	defer u.Unlock()
 
-	if u.Unifi != nil {
+	if config.Unifi != nil {
-		u.Unifi.CloseIdleConnections()
+		config.Unifi.CloseIdleConnections()
 	}
 
 	// Create an authenticated session to the Unifi Controller.
-	u.Unifi, err = unifi.NewUnifi(&unifi.Config{
+	config.Unifi, err = unifi.NewUnifi(&unifi.Config{
-		User:      u.Config.UnifiUser,
+		User:      config.User,
-		Pass:      u.Config.UnifiPass,
+		Pass:      config.Pass,
-		URL:       u.Config.UnifiBase,
+		URL:       config.URL,
-		VerifySSL: u.Config.VerifySSL,
+		VerifySSL: config.VerifySSL,
 		ErrorLog:  u.LogErrorf, // Log all errors.
 		DebugLog:  u.LogDebugf, // Log debug messages.
 	})
 	if err != nil {
-		u.Unifi = nil
 		return fmt.Errorf("unifi controller: %v", err)
 	}
 
-	u.LogDebugf("Authenticated with controller successfully")
+	u.LogDebugf("Authenticated with controller successfully, %s", config.URL)
 
-	return u.CheckSites()
+	return u.CheckSites(config)
 }
 
 // CheckSites makes sure the list of provided sites exists on the controller.
 // This does not run in Lambda (run-once) mode.
-func (u *UnifiPoller) CheckSites() error {
+func (u *UnifiPoller) CheckSites(config Controller) error {
 	if strings.Contains(strings.ToLower(u.Config.Mode), "lambda") {
 		return nil // Skip this in lambda mode.
 	}
 
 	u.LogDebugf("Checking Controller Sites List")
 
-	sites, err := u.Unifi.GetSites()
+	sites, err := config.Unifi.GetSites()
 	if err != nil {
 		return err
 	}
@@ -58,13 +59,13 @@ func (u *UnifiPoller) CheckSites() error {
 	}
 	u.Logf("Found %d site(s) on controller: %v", len(msg), strings.Join(msg, ", "))
 
-	if StringInSlice("all", u.Config.Sites) {
+	if StringInSlice("all", config.Sites) {
-		u.Config.Sites = []string{"all"}
+		config.Sites = []string{"all"}
 		return nil
 	}
 
 FIRST:
-	for _, s := range u.Config.Sites {
+	for _, s := range config.Sites {
 		for _, site := range sites {
 			if s == site.Name {
 				continue FIRST
@@ -77,47 +78,90 @@ FIRST:
 }
 
 // CollectMetrics grabs all the measurements from a UniFi controller and returns them.
-func (u *UnifiPoller) CollectMetrics() (*metrics.Metrics, error) {
+func (u *UnifiPoller) CollectMetrics() (metrics *metrics.Metrics, err error) {
+	var errs []string
+
+	for _, c := range u.Config.Controller {
+		m, err := u.collectController(c)
+		if err != nil {
+			errs = append(errs, err.Error())
+			continue
+		}
+
+		if err != nil {
+			u.LogErrorf("collecting metrics from %s: %v", c.URL, err)
+			u.Logf("Re-authenticating to UniFi Controller: %s", c.URL)
+
+			if err := u.GetUnifi(c); err != nil {
+				u.LogErrorf("re-authenticating to %s: %v", c.URL, err)
+				errs = append(errs, err.Error())
+			} else if m, err = u.collectController(c); err != nil {
+				u.LogErrorf("collecting metrics from %s: %v", c.URL, err)
+				errs = append(errs, err.Error())
+			}
+		}
+
+		metrics.Sites = append(metrics.Sites, m.Sites...)
+		metrics.Clients = append(metrics.Clients, m.Clients...)
+		metrics.IDSList = append(metrics.IDSList, m.IDSList...)
+		metrics.UAPs = append(metrics.UAPs, m.UAPs...)
+		metrics.USGs = append(metrics.USGs, m.USGs...)
+		metrics.USWs = append(metrics.USWs, m.USWs...)
+		metrics.UDMs = append(metrics.UDMs, m.UDMs...)
+	}
+
+	if len(errs) > 0 {
+		err = fmt.Errorf(strings.Join(errs, ", "))
+	}
+
+	return
+}
+
+func (u *UnifiPoller) collectController(c Controller) (*metrics.Metrics, error) {
 	var err error
 
-	if u.Unifi == nil || u.Config.ReAuth {
+	if c.Unifi == nil {
 		// Some users need to re-auth every interval because the cookie times out.
 		// Sometimes we hit this path when the controller dies.
 		u.Logf("Re-authenticating to UniFi Controller")
-		if err := u.GetUnifi(); err != nil {
+
+		if err := u.GetUnifi(c); err != nil {
 			return nil, err
 		}
 	}
 
 	m := &metrics.Metrics{TS: u.LastCheck} // At this point, it's the Current Check.
+
 	// Get the sites we care about.
-	if m.Sites, err = u.GetFilteredSites(); err != nil {
+	if m.Sites, err = u.GetFilteredSites(c); err != nil {
 		return m, fmt.Errorf("unifi.GetSites(): %v", err)
 	}
 
-	if u.Config.SaveIDS {
+	if c.SaveIDS {
-		m.IDSList, err = u.Unifi.GetIDS(m.Sites, time.Now().Add(u.Config.Interval.Duration), time.Now())
+		m.IDSList, err = c.Unifi.GetIDS(m.Sites, time.Now().Add(u.Config.Interval.Duration), time.Now())
-		return m, fmt.Errorf("unifi.GetIDS(): %v", err)
+		if err != nil {
+			return m, fmt.Errorf("unifi.GetIDS(): %v", err)
+		}
 	}
 
 	// Get all the points.
-	if m.Clients, err = u.Unifi.GetClients(m.Sites); err != nil {
+	if m.Clients, err = c.Unifi.GetClients(m.Sites); err != nil {
 		return m, fmt.Errorf("unifi.GetClients(): %v", err)
 	}
 
-	if m.Devices, err = u.Unifi.GetDevices(m.Sites); err != nil {
+	if m.Devices, err = c.Unifi.GetDevices(m.Sites); err != nil {
 		return m, fmt.Errorf("unifi.GetDevices(): %v", err)
 	}
 
-	return m, nil
+	return u.augmentMetrics(c, m), nil
 }
 
-// AugmentMetrics is our middleware layer between collecting metrics and writing them.
+// augmentMetrics is our middleware layer between collecting metrics and writing them.
 // This is where we can manipuate the returned data or make arbitrary decisions.
 // This function currently adds parent device names to client metrics.
-func (u *UnifiPoller) AugmentMetrics(metrics *metrics.Metrics) {
+func (u *UnifiPoller) augmentMetrics(c Controller, metrics *metrics.Metrics) *metrics.Metrics {
 	if metrics == nil || metrics.Devices == nil || metrics.Clients == nil {
-		return
+		return metrics
 	}
 
 	devices := make(map[string]string)
@@ -150,27 +194,29 @@ func (u *UnifiPoller) AugmentMetrics(metrics *metrics.Metrics) {
 		metrics.Clients[i].RadioDescription = bssdIDs[metrics.Clients[i].Bssid] + metrics.Clients[i].RadioProto
 	}
 
-	if !u.Config.SaveSites {
+	if !c.SaveSites {
 		metrics.Sites = nil
 	}
+
+	return metrics
 }
 
 // GetFilteredSites returns a list of sites to fetch data for.
 // Omits requested but unconfigured sites. Grabs the full list from the
 // controller and returns the sites provided in the config file.
-func (u *UnifiPoller) GetFilteredSites() (unifi.Sites, error) {
+func (u *UnifiPoller) GetFilteredSites(config Controller) (unifi.Sites, error) {
 	var i int
 
-	sites, err := u.Unifi.GetSites()
+	sites, err := config.Unifi.GetSites()
 	if err != nil {
 		return nil, err
-	} else if len(u.Config.Sites) < 1 || StringInSlice("all", u.Config.Sites) {
+	} else if len(config.Sites) < 1 || StringInSlice("all", config.Sites) {
 		return sites, nil
 	}
 
 	for _, s := range sites {
 		// Only include valid sites in the request filter.
-		if StringInSlice(s.Name, u.Config.Sites) {
+		if StringInSlice(s.Name, config.Sites) {
 			sites[i] = s
 			i++
 		}