Merge pull request #50 from davidnewhall/dn2_fixes

Fix home-brew formula build. Better logging. Exit on error.

integrations/promunifi/Gopkg.lock

@@ -2,12 +2,12 @@
 
 
 [[projects]]
-  digest = "1:11e7c0f12739f8ddebb72d8555d5d336f2121fd5c7d9f5909763e918947a5232"
+  digest = "1:28ef1378055e34f154c8efcd8863a3e53a276c58cc7fc0d0a32d6b9eed6f6cfc"
   name = "github.com/golift/unifi"
   packages = ["."]
   pruneopts = "UT"
-  revision = "facbb7d0e5db951c7074504188fcfc13cca6d5b2"
-  version = "v2.1.2"
+  revision = "c610e15131f93950f7aa6e9c564a86d896b8b437"
+  version = "v2.1.3"
 
 [[projects]]
   branch = "master"

integrations/promunifi/Makefile

@@ -132,10 +132,10 @@ check_fpm:
 formula: $(BINARY).rb
 v$(VERSION).tar.gz.sha256:
 	# Calculate the SHA from the Github source file.
-	curl -sL $(URL)/archive/v$(VERSION).tar.gz | openssl dgst -sha256 | tee v$(VERSION).tar.gz.sha256
+	curl -sL $(URL)/archive/v$(VERSION).tar.gz | openssl dgst -r -sha256 | tee v$(VERSION).tar.gz.sha256
 $(BINARY).rb: v$(VERSION).tar.gz.sha256
 	# Creating formula from template using sed.
-	sed "s/{{Version}}/$(VERSION)/g;s/{{SHA256}}/$$(<v$(VERSION).tar.gz.sha256)/g;s/{{Desc}}/$(DESC)/g;s%{{URL}}%$(URL)%g" templates/$(BINARY).rb.tmpl | tee $(BINARY).rb
+	sed "s/{{Version}}/$(VERSION)/g;s/{{SHA256}}/`head -c64 v$(VERSION).tar.gz.sha256`/g;s/{{Desc}}/$(DESC)/g;s%{{URL}}%$(URL)%g" templates/$(BINARY).rb.tmpl | tee $(BINARY).rb
 
 # Extras
 

integrations/promunifi/cmd/unifi-poller/README.md

@@ -26,13 +26,15 @@ This daemon polls a Unifi controller at a short interval and stores the collecte
     -j, --dumpjson <filter>
         This is a debug option; use this when you are missing data in your graphs,
         and/or you want to inspect the raw data coming from the controller. The
-        filter only accepts two options: devices or clients. This will print a lot
-        of information. Recommend piping it into a file and/or into jq for better
-        visualization. This requires a valid config file that; one that contains
+        filter accepts three options: devices, clients, other. This will print a
+        lot of information. Recommend piping it into a file and/or into jq for
+        better visualization. This requires a valid config file that contains
         working authentication details for a Unifi Controller. This only dumps
         data for sites listed in the config file. The application exits after
         printing the JSON payload; it does not daemonize or report to InfluxDB
-        with this option.
+        with this option. The `other` option is special. This allows you request
+        any api path. It must be enclosed in quotes with the word other. Example:
+           unifi-poller -j "other /stat/admins"
 
     -h, --help
         Display usage and exit.
@@ -63,6 +65,18 @@ This daemon polls a Unifi controller at a short interval and stores the collecte
         errors will be logged. Using this with debug=true adds line numbers to
         any error logs.
 
+    `max_errors`     default: 0
+        If you restart the UniFI controller, the poller will lose access until
+        it is restarted. Specifying a number greater than -1 for max_errors will
+        cause the poller to exit when it reaches the error count specified.
+        This problematic condition can be triggered by InfluxDB having issues
+        too. Generally only 1 error per interval is created, but if more than one
+        backend is having issues > 1 error could be generated per interval. Once
+        the poller exits, it is expected that something will restart it
+        automatically so it gets back in line; something is usually systemd,
+        docker or launchd. The default setting of 0 will cause an exit after
+        just 1 error. Recommended values are 0-5.
+
     `influx_url`     default: http://127.0.0.1:8086
         This is the URL where the Influx web server is available.
 

integrations/promunifi/examples/up.conf.example

@@ -18,6 +18,12 @@
 # Recommend using debug with this setting for better error logging.
 #quiet = false
 
+# If the poller experiences an error from the Unifi Controller or from InfluxDB
+# it will exit. If you do not want it to exit, change max_errors to -1. You can
+# adjust the config to tolerate more errors by setting this to a higher value.
+# Recommend setting this between 0 and 5. See man page for more explanation.
+#max_errors = 0
+
 # InfluxDB does not require auth by default, so the user/password are probably unimportant.
 #influx_url = "http://127.0.0.1:8086"
 #influx_user = "unifi"

integrations/promunifi/init/launchd/com.github.davidnewhall.unifi-poller.plist

@@ -15,8 +15,8 @@
         <key>KeepAlive</key>
         <true/>
         <key>StandardErrorPath</key>
-        <string>/usr/local/var/log/unifi-poller/log</string>
+        <string>/usr/local/var/log/unifi-poller.log</string>
         <key>StandardOutPath</key>
-        <string>/usr/local/var/log/unifi-poller/log</string>
+        <string>/usr/local/var/log/unifi-poller.log</string>
     </dict>
 </plist>

integrations/promunifi/pkg/unifi-poller/config.go

@@ -35,6 +35,7 @@ type UnifiPoller struct {
 	DumpJSON   string
 	ShowVer    bool
 	Flag       *pflag.FlagSet
+	errorCount int
 	influx.Client
 	*unifi.Unifi
 	*Config
@@ -50,6 +51,7 @@ type Metrics struct {
 
 // Config represents the data needed to poll a controller and report to influxdb.
 type Config struct {
+	MaxErrors  int      `json:"max_errors,_omitempty" toml:"max_errors,_omitempty" xml:"max_errors" yaml:"max_errors"`
 	Interval   Dur      `json:"interval,_omitempty" toml:"interval,_omitempty" xml:"interval" yaml:"interval"`
 	Debug      bool     `json:"debug" toml:"debug" xml:"debug" yaml:"debug"`
 	Quiet      bool     `json:"quiet,_omitempty" toml:"quiet,_omitempty" xml:"quiet" yaml:"quiet"`

integrations/promunifi/pkg/unifi-poller/helpers.go

@@ -1,6 +1,7 @@
 package unifipoller
 
 import (
+	"fmt"
 	"log"
 	"strings"
 )
@@ -15,11 +16,13 @@ func hasErr(errs []error) bool {
 	return false
 }
 
-// logErrors writes a slice of errors, with a prefix, to log-out.
-func logErrors(errs []error, prefix string) {
+// LogErrors writes a slice of errors, with a prefix, to log-out.
+// It also increments the error counter.
+func (u *UnifiPoller) LogErrors(errs []error, prefix string) {
 	for _, err := range errs {
 		if err != nil {
-			log.Println("[ERROR]", prefix+":", err.Error())
+			u.errorCount++
+			_ = log.Output(2, fmt.Sprintf("[ERROR] (%v/%v) %v: %v", u.errorCount, u.MaxErrors, prefix, err))
 		}
 	}
 }
@@ -35,8 +38,20 @@ func StringInSlice(str string, slc []string) bool {
 }
 
 // Logf prints a log entry if quiet is false.
-func (c *Config) Logf(m string, v ...interface{}) {
-	if !c.Quiet {
-		log.Printf("[INFO] "+m, v...)
+func (u *UnifiPoller) Logf(m string, v ...interface{}) {
+	if !u.Quiet {
+		_ = log.Output(2, fmt.Sprintf("[INFO] "+m, v...))
 	}
 }
+
+// LogDebugf prints a debug log entry if debug is true and quite is false
+func (u *UnifiPoller) LogDebugf(m string, v ...interface{}) {
+	if u.Debug && !u.Quiet {
+		_ = log.Output(2, fmt.Sprintf("[DEBUG] "+m, v...))
+	}
+}
+
+// LogErrorf prints an error log entry.
+func (u *UnifiPoller) LogErrorf(m string, v ...interface{}) {
+	_ = log.Output(2, fmt.Sprintf("[ERROR] "+m, v...))
+}

integrations/promunifi/pkg/unifi-poller/jsondebug.go

@@ -2,7 +2,6 @@ package unifipoller
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"strings"
 
@@ -29,69 +28,32 @@ func (u *UnifiPoller) DumpJSONPayload() (err error) {
 	case err != nil:
 		return err
 	case StringInSlice(u.DumpJSON, []string{"d", "device", "devices"}):
-		return u.DumpDeviceJSON(sites)
+		return u.dumpSitesJSON(unifi.DevicePath, "Devices", sites)
 	case StringInSlice(u.DumpJSON, []string{"client", "clients", "c"}):
-		return u.DumpClientsJSON(sites)
+		return u.dumpSitesJSON(unifi.ClientPath, "Clients", sites)
 	case strings.HasPrefix(u.DumpJSON, "other "):
-		return u.DumpOtherJSON(sites)
+		apiPath := strings.SplitN(u.DumpJSON, " ", 2)[1]
+		_, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping Path '%s':\n", apiPath)
+		return u.PrintRawAPIJSON(apiPath)
 	default:
-		return errors.New("must provide filter: devices, clients")
+		return errors.New("must provide filter: devices, clients, other")
 	}
 }
 
-// DumpClientsJSON prints the raw json for clients in a Unifi Controller.
-func (u *UnifiPoller) DumpClientsJSON(sites []unifi.Site) error {
+func (u *UnifiPoller) dumpSitesJSON(path, name string, sites []unifi.Site) error {
 	for _, s := range sites {
-		path := fmt.Sprintf(unifi.ClientPath, s.Name)
-		if err := u.dumpJSON(path, "Client", s); err != nil {
+		apiPath := fmt.Sprintf(path, s.Name)
+		_, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping %s: '%s' JSON for site: %s (%s):\n", name, apiPath, s.Desc, s.Name)
+		if err := u.PrintRawAPIJSON(apiPath); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-// DumpDeviceJSON prints the raw json for devices in a Unifi Controller.
-func (u *UnifiPoller) DumpDeviceJSON(sites []unifi.Site) error {
-	for _, s := range sites {
-		path := fmt.Sprintf(unifi.DevicePath, s.Name)
-		if err := u.dumpJSON(path, "Device", s); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// DumpOtherJSON prints the raw json for a user-provided path in a Unifi Controller.
-func (u *UnifiPoller) DumpOtherJSON(sites []unifi.Site) error {
-	for _, s := range sites {
-		path := strings.SplitN(u.DumpJSON, " ", 2)[1]
-		if strings.Contains(path, "%s") {
-			path = fmt.Sprintf(path, s.Name)
-		}
-		if err := u.dumpJSON(path, "Other", s); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (u *UnifiPoller) dumpJSON(path, what string, site unifi.Site) error {
-	req, err := u.UniReq(path, "")
-	if err != nil {
-		return err
-	}
-	resp, err := u.Do(req)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		_ = resp.Body.Close()
-	}()
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-	fmt.Fprintf(os.Stderr, "[INFO] Dumping %s JSON for site %s (%s)\n", what, site.Desc, site.Name)
+// PrintRawAPIJSON prints the raw json for a user-provided path on a Unifi Controller.
+func (u *UnifiPoller) PrintRawAPIJSON(apiPath string) error {
+	body, err := u.GetJSON(apiPath)
 	fmt.Println(string(body))
-	return nil
+	return err
 }

integrations/promunifi/pkg/unifi-poller/poller.go

@@ -50,7 +50,7 @@ func (u *UnifiPoller) GetConfig() error {
 	if u.DumpJSON != "" {
 		u.Quiet = true
 	}
-	u.Config.Logf("Loaded Configuration: %s", u.ConfigFile)
+	u.Logf("Loaded Configuration: %s", u.ConfigFile)
 	return nil
 }
 
@@ -61,7 +61,7 @@ func (u *UnifiPoller) Run() (err error) {
 	}
 	if log.SetFlags(0); u.Debug {
 		log.SetFlags(log.Lshortfile | log.Lmicroseconds | log.Ldate)
-		log.Println("[DEBUG] Debug Logging Enabled")
+		u.LogDebugf("Debug Logging Enabled")
 	}
 	log.Printf("[INFO] Unifi-Poller v%v Starting Up! PID: %d", Version, os.Getpid())
 
@@ -71,8 +71,7 @@ func (u *UnifiPoller) Run() (err error) {
 	if err = u.GetInfluxDB(); err != nil {
 		return err
 	}
-	u.PollController()
-	return nil
+	return u.PollController()
 }
 
 // GetInfluxDB returns an influxdb interface.
@@ -96,11 +95,8 @@ func (u *UnifiPoller) GetUnifi() (err error) {
 	if err != nil {
 		return errors.Wrap(err, "unifi controller")
 	}
-	u.Unifi.ErrorLog = log.Printf // Log all errors.
-	// Doing it this way allows debug error logs (line numbers, etc)
-	if u.Debug && !u.Quiet {
-		u.Unifi.DebugLog = log.Printf // Log debug messages.
-	}
+	u.Unifi.ErrorLog = u.LogErrorf // Log all errors.
+	u.Unifi.DebugLog = u.LogDebugf // Log debug messages.
 	v, err := u.GetServer()
 	if err != nil {
 		v.ServerVersion = "unknown"

integrations/promunifi/pkg/unifi-poller/unifi.go

@@ -37,7 +37,7 @@ FIRST:
 }
 
 // PollController runs forever, polling unifi, and pushing to influx.
-func (u *UnifiPoller) PollController() {
+func (u *UnifiPoller) PollController() error {
 	log.Println("[INFO] Everything checks out! Poller started, interval:", u.Interval.value)
 	ticker := time.NewTicker(u.Interval.value)
 	var err error
@@ -45,28 +45,28 @@ func (u *UnifiPoller) PollController() {
 		m := &Metrics{}
 		// Get the sites we care about.
 		if m.Sites, err = u.GetFilteredSites(); err != nil {
-			logErrors([]error{err}, "uni.GetSites()")
+			u.LogErrors([]error{err}, "unifi.GetSites()")
 		}
 		// Get all the points.
 		if m.Clients, err = u.GetClients(m.Sites); err != nil {
-			logErrors([]error{err}, "uni.GetClients()")
+			u.LogErrors([]error{err}, "unifi.GetClients()")
 		}
 		if m.Devices, err = u.GetDevices(m.Sites); err != nil {
-			logErrors([]error{err}, "uni.GetDevices()")
+			u.LogErrors([]error{err}, "unifi.GetDevices()")
 		}
 
 		// Make a new Points Batcher.
 		m.BatchPoints, err = influx.NewBatchPoints(influx.BatchPointsConfig{Database: u.InfluxDB})
 		if err != nil {
-			logErrors([]error{err}, "influx.NewBatchPoints")
+			u.LogErrors([]error{err}, "influx.NewBatchPoints")
 			continue
 		}
 		// Batch (and send) all the points.
 		if errs := m.SendPoints(); errs != nil && hasErr(errs) {
-			logErrors(errs, "asset.Points()")
+			u.LogErrors(errs, "asset.Points()")
 		}
 		if err := u.Write(m.BatchPoints); err != nil {
-			logErrors([]error{err}, "infdb.Write(bp)")
+			u.LogErrors([]error{err}, "infdb.Write(bp)")
 		}
 
 		// Talk about the data.
@@ -79,7 +79,12 @@ func (u *UnifiPoller) PollController() {
 		u.Logf("Unifi Measurements Recorded. Sites: %d, Clients: %d, "+
 			"Wireless APs: %d, Gateways: %d, Switches: %d, Points: %d, Fields: %d",
 			len(m.Sites), len(m.Clients), len(m.UAPs), len(m.USGs), len(m.USWs), pointcount, fieldcount)
+
+		if u.MaxErrors >= 0 && u.errorCount > u.MaxErrors {
+			return errors.Errorf("reached maximum error count, stopping poller (%d > %d)", u.errorCount, u.MaxErrors)
+		}
 	}
+	return nil
 }
 
 // SendPoints combines all device and client data into influxdb data points.

integrations/promunifi/templates/unifi-poller.rb.tmpl

@@ -25,7 +25,7 @@ class UnifiPoller < Formula
       # If this fails, the user gets a nice big warning about write permissions on their
       # [/usr/local/]var/log folder. The alternative could be letting the app silently
       # fail to start when it cannot write logs. This is better. Fix perms; reinstall.
-      system "mkdir", "-p", "#{var}/log/unifi-poller"
+      system "touch", "#{var}/log/unifi-poller.log"
     end
   end
   def caveats
@@ -33,7 +33,7 @@ class UnifiPoller < Formula
   This application will not work until the config file has authentication
   information for a Unifi Controller and an Influx Database. Edit the config
   file at #{etc}/unifi-poller/up.conf then start the application with
-  brew services start unifi-poller ~ log file: #{var}/log/unifi-poller/log
+  brew services start unifi-poller ~ log file: #{var}/log/unifi-poller.log
   The manual explains the config file options: man unifi-poller
     EOS
   end
@@ -56,9 +56,9 @@ class UnifiPoller < Formula
            <key>KeepAlive</key>
            <true/>
            <key>StandardErrorPath</key>
-           <string>#{var}/log/unifi-poller/log</string>
+           <string>#{var}/log/unifi-poller.log</string>
            <key>StandardOutPath</key>
-           <string>#{var}/log/unifi-poller/log</string>
+           <string>#{var}/log/unifi-poller.log</string>
        </dict>
    </plist>
    EOS