Merge pull request #36 from davidnewhall/dn2_debug

Add JSON Dump debug mode.

.travis.yml

@@ -14,7 +14,8 @@ before_install:
 - curl -sLo $GOPATH/bin/dep https://github.com/golang/dep/releases/download/v0.5.3/dep-darwin-amd64
 - chmod +x $GOPATH/bin/dep
   # download super-linter: golangci-lint
-- curl -sL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin latest
+- curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin latest
+#- curl -sL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin latest
 install:
 - dep ensure
 - rvm $brew_ruby do gem install --no-document fpm

cmd/unifi-poller/README.md

@@ -13,7 +13,7 @@ unifi-poller(1) -- Utility to poll UniFi Controller Metrics and store them in In
 
 ## OPTIONS
 
-`unifi-poller [-c <config-file>] [-h] [-v]`
+`unifi-poller [-c <config-file>] [-j <filter>] [-h] [-v]`
 
     -c, --config <config-file>
         Provide a configuration file (instead of the default).
@@ -21,6 +21,17 @@ unifi-poller(1) -- Utility to poll UniFi Controller Metrics and store them in In
     -v, --version
         Display version and exit.
 
+    -j, --dumpjson <filter>
+        This is a debug option; use this when you are missing data in your graphs,
+        and/or you want to inspect the raw data coming from the controller. The
+        filter only accepts two options: devices or clients. This will print a lot
+        of information. Recommend piping it into a file and/or into jq for better
+        visualization. This requires a valid config file that; one that contains
+        working authentication details for a Unifi Controller. This only dumps
+        data for sites listed in the config file. The application exits after
+        printing the JSON payload; it does not daemonize or report to InfluxDB
+        with this option.
+
     -h, --help
         Display usage and exit.
 

cmd/unifi-poller/config.go

@@ -1,6 +1,12 @@
 package main
 
-import "time"
+import (
+	"time"
+
+	"github.com/golift/unifi"
+	influx "github.com/influxdata/influxdb1-client/v2"
+	"github.com/spf13/pflag"
+)
 
 // Version is injected by the Makefile
 var Version = "development"
@@ -17,11 +23,27 @@ const (
 	defaultUnifURL  = "https://127.0.0.1:8443"
 )
 
+// Asset is used to give all devices and clients a common interface.
+type Asset interface {
+	Points() ([]*influx.Point, error)
+}
+
+// UnifiPoller contains the application startup data, and auth info for unifi & influx.
+type UnifiPoller struct {
+	ConfigFile string
+	DumpJSON   string
+	ShowVer    bool
+	Flag       *pflag.FlagSet
+	influx.Client
+	*unifi.Unifi
+	*Config
+}
+
 // Config represents the data needed to poll a controller and report to influxdb.
 type Config struct {
 	Interval   Dur      `json:"interval,_omitempty" toml:"interval,_omitempty" xml:"interval" yaml:"interval"`
 	Debug      bool     `json:"debug" toml:"debug" xml:"debug" yaml:"debug"`
-	Quiet      bool     `json:"quiet" toml:"quiet" xml:"quiet" yaml:"quiet"`
+	Quiet      bool     `json:"quiet,_omitempty" toml:"quiet,_omitempty" xml:"quiet" yaml:"quiet"`
 	VerifySSL  bool     `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"`
 	InfluxURL  string   `json:"influx_url,_omitempty" toml:"influx_url,_omitempty" xml:"influx_url" yaml:"influx_url"`
 	InfluxUser string   `json:"influx_user,_omitempty" toml:"influx_user,_omitempty" xml:"influx_user" yaml:"influx_user"`

cmd/unifi-poller/helpers.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"log"
+	"strings"
+)
+
+// hasErr checks a list of errors for a non-nil.
+func hasErr(errs []error) bool {
+	for _, err := range errs {
+		if err != nil {
+			return true
+		}
+	}
+	return false
+}
+
+// logErrors writes a slice of errors, with a prefix, to log-out.
+func logErrors(errs []error, prefix string) {
+	for _, err := range errs {
+		if err != nil {
+			log.Println("[ERROR]", prefix+":", err.Error())
+		}
+	}
+}
+
+// StringInSlice returns true if a string is in a slice.
+func StringInSlice(str string, slc []string) bool {
+	for _, s := range slc {
+		if strings.EqualFold(s, str) {
+			return true
+		}
+	}
+	return false
+}
+
+// Logf prints a log entry if quiet is false.
+func (c *Config) Logf(m string, v ...interface{}) {
+	if !c.Quiet {
+		log.Printf("[INFO] "+m, v...)
+	}
+}

cmd/unifi-poller/jsondebug.go

@@ -0,0 +1,80 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"github.com/golift/unifi"
+	"github.com/pkg/errors"
+)
+
+// DumpJSONPayload prints raw json from the Unifi Controller.
+func (u *UnifiPoller) DumpJSONPayload() (err error) {
+	u.Quiet = true
+	u.Unifi, err = unifi.NewUnifi(u.UnifiUser, u.UnifiPass, u.UnifiBase, u.VerifySSL)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintln(os.Stderr, "[INFO] Authenticated to Unifi Controller @", u.UnifiBase, "as user", u.UnifiUser)
+	if err := u.CheckSites(); err != nil {
+		return err
+	}
+	u.Unifi.ErrorLog = func(m string, v ...interface{}) {
+		fmt.Fprintf(os.Stderr, "[ERROR] "+m, v...)
+	} // Log all errors to stderr.
+
+	switch sites, err := u.filterSites(u.Sites); {
+	case err != nil:
+		return err
+	case StringInSlice(u.DumpJSON, []string{"d", "device", "devices"}):
+		return u.DumpDeviceJSON(sites)
+	case StringInSlice(u.DumpJSON, []string{"client", "clients", "c"}):
+		return u.DumpClientsJSON(sites)
+	default:
+		return errors.New("must provide filter: devices, clients")
+	}
+}
+
+// DumpClientsJSON prints the raw json for clients in a Unifi Controller.
+func (u *UnifiPoller) DumpClientsJSON(sites []unifi.Site) error {
+	for _, s := range sites {
+		path := fmt.Sprintf(unifi.ClientPath, s.Name)
+		if err := u.dumpJSON(path, "Client", s); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// DumpDeviceJSON prints the raw json for devices in a Unifi Controller.
+func (u *UnifiPoller) DumpDeviceJSON(sites []unifi.Site) error {
+	for _, s := range sites {
+		path := fmt.Sprintf(unifi.DevicePath, s.Name)
+		if err := u.dumpJSON(path, "Device", s); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (u *UnifiPoller) dumpJSON(path, what string, site unifi.Site) error {
+	req, err := u.UniReq(path, "")
+	if err != nil {
+		return err
+	}
+	resp, err := u.Do(req)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(os.Stderr, "[INFO] Dumping %s JSON for site %s (%s)\n", what, site.Desc, site.Name)
+	fmt.Println(string(body))
+	return nil
+}

cmd/unifi-poller/main.go

@@ -5,8 +5,6 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
-	"strings"
-	"time"
 
 	"github.com/golift/unifi"
 	influx "github.com/influxdata/influxdb1-client/v2"
@@ -15,108 +13,39 @@ import (
 	flag "github.com/spf13/pflag"
 )
 
-// Asset is used to give all devices and clients a common interface.
-type Asset interface {
-	Points() ([]*influx.Point, error)
-}
-
 func main() {
-	configFile := parseFlags()
+	u := &UnifiPoller{}
-	log.Println("Unifi-Poller Starting Up! PID:", os.Getpid())
+	if u.ParseFlags(os.Args[1:]); u.ShowVer {
-	config, err := GetConfig(configFile)
+		fmt.Printf("unifi-poller v%s\n", Version)
-	if err != nil {
+		return // don't run anything else.
-		flag.Usage()
-		log.Fatalf("[ERROR] config file '%v': %v", configFile, err)
 	}
-	if err := config.Run(); err != nil {
+	if err := u.GetConfig(); err != nil {
+		u.Flag.Usage()
+		log.Fatalf("[ERROR] config file '%v': %v", u.ConfigFile, err)
+	}
+	if err := u.Run(); err != nil {
 		log.Fatalln("[ERROR]", err)
 	}
 }
 
-// Run invokes all the application logic and routines.
+// ParseFlags runs the parser.
-func (c *Config) Run() error {
+func (u *UnifiPoller) ParseFlags(args []string) {
-	// Create an authenticated session to the Unifi Controller.
+	u.Flag = flag.NewFlagSet("unifi-poller", flag.ExitOnError)
-	controller, err := unifi.NewUnifi(c.UnifiUser, c.UnifiPass, c.UnifiBase, c.VerifySSL)
+	u.Flag.Usage = func() {
-	if err != nil {
-		return errors.Wrap(err, "unifi controller")
-	}
-	if !c.Quiet {
-		log.Println("Authenticated to Unifi Controller @", c.UnifiBase, "as user", c.UnifiUser)
-	}
-	if err := c.CheckSites(controller); err != nil {
-		return err
-	}
-	controller.ErrorLog = log.Printf // Log all errors.
-	if log.SetFlags(0); c.Debug {
-		log.Println("Debug Logging Enabled")
-		log.SetFlags(log.Lshortfile | log.Lmicroseconds | log.Ldate)
-		controller.DebugLog = log.Printf // Log debug messages.
-	}
-	infdb, err := influx.NewHTTPClient(influx.HTTPConfig{
-		Addr:     c.InfluxURL,
-		Username: c.InfluxUser,
-		Password: c.InfluxPass,
-	})
-	if err != nil {
-		return errors.Wrap(err, "influxdb")
-	}
-	if c.Quiet {
-		// Doing it this way allows debug error logs (line numbers, etc)
-		controller.DebugLog = nil
-	} else {
-		log.Println("Logging Unifi Metrics to InfluXDB @", c.InfluxURL, "as user", c.InfluxUser)
-		log.Printf("Polling Unifi Controller (sites %v), interval: %v", c.Sites, c.Interval.value)
-	}
-	c.PollUnifiController(controller, infdb)
-	return nil
-}
-
-func parseFlags() string {
-	flag.Usage = func() {
 		fmt.Println("Usage: unifi-poller [--config=filepath] [--version]")
-		flag.PrintDefaults()
+		u.Flag.PrintDefaults()
 	}
-	configFile := flag.StringP("config", "c", defaultConfFile, "Poller Config File (TOML Format)")
+	u.Flag.StringVarP(&u.DumpJSON, "dumpjson", "j", "",
-	version := flag.BoolP("version", "v", false, "Print the version and exit")
+		"This debug option prints the json payload for a device and exits.")
-	if flag.Parse(); *version {
+	u.Flag.StringVarP(&u.ConfigFile, "config", "c", defaultConfFile, "Poller Config File (TOML Format)")
-		fmt.Printf("unifi-poller v%s\n", Version)
+	u.Flag.BoolVarP(&u.ShowVer, "version", "v", false, "Print the version and exit")
-		os.Exit(0) // don't run anything else.
+	_ = u.Flag.Parse(args)
-	}
-	return *configFile
-}
-
-// CheckSites makes sure the list of provided sites exists on the controller.
-func (c *Config) CheckSites(controller *unifi.Unifi) error {
-	sites, err := controller.GetSites()
-	if err != nil {
-		return err
-	}
-	if !c.Quiet {
-		msg := []string{}
-		for _, site := range sites {
-			msg = append(msg, site.Name+" ("+site.Desc+")")
-		}
-		log.Printf("Found %d site(s) on controller: %v", len(msg), strings.Join(msg, ", "))
-	}
-	if StringInSlice("all", c.Sites) {
-		return nil
-	}
-FIRST:
-	for _, s := range c.Sites {
-		for _, site := range sites {
-			if s == site.Name {
-				continue FIRST
-			}
-		}
-		return errors.Errorf("configured site not found on controller: %v", s)
-	}
-	return nil
 }
 
 // GetConfig parses and returns our configuration data.
-func GetConfig(configFile string) (Config, error) {
+func (u *UnifiPoller) GetConfig() error {
 	// Preload our defaults.
-	config := Config{
+	u.Config = &Config{
 		InfluxURL:  defaultInfxURL,
 		InfluxUser: defaultInfxUser,
 		InfluxPass: defaultInfxPass,
@@ -127,131 +56,70 @@ func GetConfig(configFile string) (Config, error) {
 		Interval:   Dur{value: defaultInterval},
 		Sites:      []string{"default"},
 	}
-	if buf, err := ioutil.ReadFile(configFile); err != nil {
+	if buf, err := ioutil.ReadFile(u.ConfigFile); err != nil {
-		return config, err
+		return err
 		// This is where the defaults in the config variable are overwritten.
-	} else if err := toml.Unmarshal(buf, &config); err != nil {
+	} else if err := toml.Unmarshal(buf, u.Config); err != nil {
-		return config, err
+		return err
 	}
-	log.Println("Loaded Configuration:", configFile)
+	if u.DumpJSON != "" {
-	return config, nil
+		u.Quiet = true
+	}
+	u.Config.Logf("Loaded Configuration: %s", u.ConfigFile)
+	return nil
 }
 
-// PollUnifiController runs forever, polling and pushing.
+// Run invokes all the application logic and routines.
-func (c *Config) PollUnifiController(controller *unifi.Unifi, infdb influx.Client) {
+func (u *UnifiPoller) Run() (err error) {
-	log.Println("[INFO] Everything checks out! Beginning Poller Routine.")
+	if u.DumpJSON != "" {
-	ticker := time.NewTicker(c.Interval.value)
+		return u.DumpJSONPayload()
-
-	for range ticker.C {
-		sites, err := filterSites(controller, c.Sites)
-		if err != nil {
-			logErrors([]error{err}, "uni.GetSites()")
-		}
-		// Get all the points.
-		clients, err := controller.GetClients(sites)
-		if err != nil {
-			logErrors([]error{err}, "uni.GetClients()")
-		}
-		devices, err := controller.GetDevices(sites)
-		if err != nil {
-			logErrors([]error{err}, "uni.GetDevices()")
-		}
-		bp, err := influx.NewBatchPoints(influx.BatchPointsConfig{Database: c.InfluxDB})
-		if err != nil {
-			logErrors([]error{err}, "influx.NewBatchPoints")
-			continue
-		}
-		// Batch all the points.
-		if errs := batchPoints(devices, clients, bp); errs != nil && hasErr(errs) {
-			logErrors(errs, "asset.Points()")
-		}
-		if err := infdb.Write(bp); err != nil {
-			logErrors([]error{err}, "infdb.Write(bp)")
-		}
-		if !c.Quiet {
-			log.Printf("[INFO] Logged Unifi States. Sites: %d Clients: %d, Wireless APs: %d, Gateways: %d, Switches: %d",
-				len(sites), len(clients.UCLs), len(devices.UAPs), len(devices.USGs), len(devices.USWs))
-		}
 	}
+	if log.SetFlags(0); u.Debug {
+		log.SetFlags(log.Lshortfile | log.Lmicroseconds | log.Ldate)
+		log.Println("[DEBUG] Debug Logging Enabled")
+	}
+	log.Printf("[INFO] Unifi-Poller v%v Starting Up! PID: %d", Version, os.Getpid())
+
+	if err = u.GetUnifi(); err != nil {
+		return err
+	}
+	if err = u.GetInfluxDB(); err != nil {
+		return err
+	}
+	u.PollController()
+	return nil
 }
 
-// filterSites returns a list of sites to fetch data for.
+// GetInfluxDB returns an influxdb interface.
-// Omits requested but unconfigured sites.
+func (u *UnifiPoller) GetInfluxDB() (err error) {
-func filterSites(controller *unifi.Unifi, filter []string) ([]unifi.Site, error) {
+	u.Client, err = influx.NewHTTPClient(influx.HTTPConfig{
-	sites, err := controller.GetSites()
+		Addr:     u.InfluxURL,
+		Username: u.InfluxUser,
+		Password: u.InfluxPass,
+	})
 	if err != nil {
-		return nil, err
+		return errors.Wrap(err, "influxdb")
-	} else if len(filter) < 1 || StringInSlice("all", filter) {
-		return sites, nil
 	}
-	var i int
+	u.Logf("Logging Measurements to InfluxDB at %s as user %s", u.InfluxURL, u.InfluxUser)
-	for _, s := range sites {
+	return nil
-		// Only include valid sites in the request filter.
-		if StringInSlice(s.Name, filter) {
-			sites[i] = s
-			i++
-		}
-	}
-	return sites[:i], nil
 }
 
-// batchPoints combines all device and client data into influxdb data points.
+// GetUnifi returns a Unifi controller interface.
-func batchPoints(devices *unifi.Devices, clients *unifi.Clients, bp influx.BatchPoints) (errs []error) {
+func (u *UnifiPoller) GetUnifi() (err error) {
-	process := func(asset Asset) error {
+	// Create an authenticated session to the Unifi Controller.
-		if asset == nil {
+	u.Unifi, err = unifi.NewUnifi(u.UnifiUser, u.UnifiPass, u.UnifiBase, u.VerifySSL)
-			return nil
+	if err != nil {
-		}
+		return errors.Wrap(err, "unifi controller")
-		influxPoints, err := asset.Points()
-		if err != nil {
-			return err
-		}
-		bp.AddPoints(influxPoints)
-		return nil
 	}
-	if devices != nil {
+	u.Unifi.ErrorLog = log.Printf // Log all errors.
-		for _, asset := range devices.UAPs {
+	// Doing it this way allows debug error logs (line numbers, etc)
-			errs = append(errs, process(asset))
+	if u.Debug && !u.Quiet {
-		}
+		u.Unifi.DebugLog = log.Printf // Log debug messages.
-		for _, asset := range devices.USGs {
-			errs = append(errs, process(asset))
-		}
-		for _, asset := range devices.USWs {
-			errs = append(errs, process(asset))
-		}
 	}
-	if clients != nil {
+	u.Logf("Authenticated to Unifi Controller at %s as user %s", u.UnifiBase, u.UnifiUser)
-		for _, asset := range clients.UCLs {
+	if err = u.CheckSites(); err != nil {
-			errs = append(errs, process(asset))
+		return err
-		}
 	}
-	return
+	u.Logf("Polling Unifi Controller Sites: %v", u.Sites)
-}
+	return nil
-
-// hasErr checks a list of errors for a non-nil.
-func hasErr(errs []error) bool {
-	for _, err := range errs {
-		if err != nil {
-			return true
-		}
-	}
-	return false
-}
-
-// logErrors writes a slice of errors, with a prefix, to log-out.
-func logErrors(errs []error, prefix string) {
-	for _, err := range errs {
-		if err != nil {
-			log.Println("[ERROR]", prefix+":", err.Error())
-		}
-	}
-}
-
-// StringInSlice returns true if a string is in a slice.
-func StringInSlice(str string, slc []string) bool {
-	for _, s := range slc {
-		if strings.EqualFold(s, str) {
-			return true
-		}
-	}
-	return false
 }

cmd/unifi-poller/unifi.go

@@ -0,0 +1,137 @@
+package main
+
+import (
+	"log"
+	"strings"
+	"time"
+
+	"github.com/golift/unifi"
+	influx "github.com/influxdata/influxdb1-client/v2"
+	"github.com/pkg/errors"
+)
+
+// CheckSites makes sure the list of provided sites exists on the controller.
+func (u *UnifiPoller) CheckSites() error {
+	sites, err := u.GetSites()
+	if err != nil {
+		return err
+	}
+	msg := []string{}
+	for _, site := range sites {
+		msg = append(msg, site.Name+" ("+site.Desc+")")
+	}
+	u.Logf("Found %d site(s) on controller: %v", len(msg), strings.Join(msg, ", "))
+	if StringInSlice("all", u.Sites) {
+		return nil
+	}
+FIRST:
+	for _, s := range u.Sites {
+		for _, site := range sites {
+			if s == site.Name {
+				continue FIRST
+			}
+		}
+		return errors.Errorf("configured site not found on controller: %v", s)
+	}
+	return nil
+}
+
+// PollController runs forever, polling unifi, and pushing to influx.
+func (u *UnifiPoller) PollController() {
+	log.Println("[INFO] Everything checks out! Poller started, interval:", u.Interval.value)
+	ticker := time.NewTicker(u.Interval.value)
+
+	for range ticker.C {
+		// Get the sites we care about.
+		sites, err := u.filterSites(u.Sites)
+		if err != nil {
+			logErrors([]error{err}, "uni.GetSites()")
+		}
+		// Get all the points.
+		clients, err := u.GetClients(sites)
+		if err != nil {
+			logErrors([]error{err}, "uni.GetClients()")
+		}
+		devices, err := u.GetDevices(sites)
+		if err != nil {
+			logErrors([]error{err}, "uni.GetDevices()")
+		}
+		// Make a new Points Batcher.
+		bp, err := influx.NewBatchPoints(influx.BatchPointsConfig{Database: u.InfluxDB})
+		if err != nil {
+			logErrors([]error{err}, "influx.NewBatchPoints")
+			continue
+		}
+		// Batch (and send) all the points.
+		if errs := batchPoints(devices, clients, bp); errs != nil && hasErr(errs) {
+			logErrors(errs, "asset.Points()")
+		}
+		if err := u.Write(bp); err != nil {
+			logErrors([]error{err}, "infdb.Write(bp)")
+		}
+
+		// Talk about the data.
+		var fieldcount, pointcount int
+		for _, p := range bp.Points() {
+			pointcount++
+			i, _ := p.Fields()
+			fieldcount += len(i)
+		}
+		u.Logf("Unifi Measurements Recorded. Sites: %d Clients: %d, "+
+			"Wireless APs: %d, Gateways: %d, Switches: %d, Points: %d, Fields: %d",
+			len(sites), len(clients.UCLs),
+			len(devices.UAPs), len(devices.USGs), len(devices.USWs), pointcount, fieldcount)
+	}
+}
+
+// batchPoints combines all device and client data into influxdb data points.
+func batchPoints(devices *unifi.Devices, clients *unifi.Clients, bp influx.BatchPoints) (errs []error) {
+	process := func(asset Asset) error {
+		if asset == nil {
+			return nil
+		}
+		influxPoints, err := asset.Points()
+		if err != nil {
+			return err
+		}
+		bp.AddPoints(influxPoints)
+		return nil
+	}
+	if devices != nil {
+		for _, asset := range devices.UAPs {
+			errs = append(errs, process(asset))
+		}
+		for _, asset := range devices.USGs {
+			errs = append(errs, process(asset))
+		}
+		for _, asset := range devices.USWs {
+			errs = append(errs, process(asset))
+		}
+	}
+	if clients != nil {
+		for _, asset := range clients.UCLs {
+			errs = append(errs, process(asset))
+		}
+	}
+	return
+}
+
+// filterSites returns a list of sites to fetch data for.
+// Omits requested but unconfigured sites.
+func (u *UnifiPoller) filterSites(filter []string) ([]unifi.Site, error) {
+	sites, err := u.GetSites()
+	if err != nil {
+		return nil, err
+	} else if len(filter) < 1 || StringInSlice("all", filter) {
+		return sites, nil
+	}
+	var i int
+	for _, s := range sites {
+		// Only include valid sites in the request filter.
+		if StringInSlice(s.Name, filter) {
+			sites[i] = s
+			i++
+		}
+	}
+	return sites[:i], nil
+}