From cef7815e2d939dc437dc8ebfca845d03b15b9838 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sat, 30 Nov 2019 03:42:17 -0800 Subject: [PATCH] allow more data reduction --- examples/up.conf.example | 12 ++++--- examples/up.json.example | 3 +- examples/up.xml.example | 78 ++++------------------------------------ examples/up.yaml.example | 53 +++------------------------ pkg/influxunifi/site.go | 27 ++++++-------- pkg/influxunifi/udm.go | 42 ++++++++++------------ pkg/influxunifi/usg.go | 24 +++++-------- pkg/influxunifi/usw.go | 19 +++++----- pkg/poller/config.go | 3 +- pkg/poller/influx.go | 2 +- pkg/poller/prometheus.go | 10 ++---- pkg/poller/start.go | 1 + pkg/poller/unifi.go | 5 ++- 13 files changed, 76 insertions(+), 203 deletions(-) diff --git a/examples/up.conf.example b/examples/up.conf.example index eb4fb0f2..c2e8cbbe 100644 --- a/examples/up.conf.example +++ b/examples/up.conf.example @@ -56,10 +56,6 @@ unifi_user = "influx" unifi_pass = "4BB9345C-2341-48D7-99F5-E01B583FF77F" unifi_url = "https://127.0.0.1:8443" -# Enable collection of Intrusion Detection System Data (InfluxDB only). -# Only useful if IDS or IPS are enabled on one of the sites. -collect_ids = false - # Some controllers or reverse proxy configurations do not allow cookies to be # re-user on every request (every interval). This setting provides a workaround # That causes the poller to re-auth (login) to the controller on every interval. @@ -67,6 +63,14 @@ collect_ids = false # a few more requests to your controller every interval. reauthenticate = false +# Enable collection of Intrusion Detection System Data (InfluxDB only). +# Only useful if IDS or IPS are enabled on one of the sites. +save_ids = false + +# Enable collection of site data. This data powers the Network Sites dashboard. +# It's not valuable to everyone and setting this to false will save resources. +save_sites = true + # If your UniFi controller has a valid SSL certificate (like lets encrypt), # you can enable this option to validate it. Otherwise, any SSL certificate is # valid. If you don't know if you have a valid SSL cert, then you don't have one. diff --git a/examples/up.json.example b/examples/up.json.example index e9961505..7842987f 100644 --- a/examples/up.json.example +++ b/examples/up.json.example @@ -13,7 +13,8 @@ "unifi_user": "influx", "unifi_pass": "", "unifi_url": "https://127.0.0.1:8443", - "collect_ids": false, + "save_ids": false, + "save_sites": true, "reauthenticate": false, "verify_ssl": false } diff --git a/examples/up.xml.example b/examples/up.xml.example index 325f12d4..e4523e3a 100644 --- a/examples/up.xml.example +++ b/examples/up.xml.example @@ -2,99 +2,33 @@ - + all + 60s - - 30s - - false - - false - influx - - 0.0.0.0:9130 - unifi unifi http://127.0.0.1:8086 unifi - false - influx https://127.0.0.1:8443 - - false - false - false + false + true + diff --git a/examples/up.yaml.example b/examples/up.yaml.example index 00e75fba..0eac5a24 100644 --- a/examples/up.yaml.example +++ b/examples/up.yaml.example @@ -1,73 +1,28 @@ +######################################################## # UniFi Poller primary configuration file. YAML FORMAT # -# provided values are defaults # +# provided values are defaults. See up.conf.example! # ######################################################## --- -# If the controller has more than one site, specify which sites to poll here. -# Set this to "default" to poll only the first site on the controller. -# A setting of "all" will poll all sites; this works if you only have 1 site too. sites: - all - -# The UniFi Controller only updates traffic stats about every 30 seconds. -# Setting this to something lower may lead to "zeros" in your data. -# If you're getting zeros now, set this to "1m" -# Only has effect if "mode" (below) is "influx" - other modes do not use interval. interval: "30s" -# Turns on line numbers, microsecond logging, and a per-device log. -# The default is false, but I personally leave this on at home (four devices). -# This may be noisy if you have a lot of devices. It adds one line per device. debug: false - -# Turns off per-interval logs. Only startup and error logs will be emitted. -# Recommend enabling debug with this setting for better error logging. quiet: false -# Which mode to run this application in. The default mode is "influx". Providing -# an invalid mode will also result in "influx". In this default mode the application -# runs as a daemon and polls the controller at the configured interval. -# -# There are two other options at this time: "influxlambda" and "prometheus" -# -# Mode "influxlambda" makes the application exit after collecting and reporting metrics -# to InfluxDB one time. This mode requires an external process like an AWS Lambda -# or a simple crontab to keep the timings accurate on UniFi Poller run intervals. -# -# Mode "prometheus" opens an HTTP server on port 9130 and exports the metrics at -# /metrics for polling collection by a prometheus server. This disables influxdb. -# IMPORTANT: The prometheus mode is still beta. -# Please help us test and provide your feedback on the github repo! mode: "influx" - -# This controls on which ip and port /metrics is exported when mode is "prometheus". -# This has no effect in other modes. Must contain a colon and port. http_listen: "0.0.0.0:9130" -# InfluxDB does not require auth by default, so the user/password are probably unimportant. influx_url: "http://127.0.0.1:8086" influx_user: "unifi" influx_pass: "unifi" -# Be sure to create this database. influx_db: "unifi" -# If your InfluxDB uses an invalid SSL cert, set this to true. influx_insecure_ssl: false -# Make a read-only user in the UniFi Admin Settings. unifi_user: "influx" unifi_pass: "" unifi_url: "https://127.0.0.1:8443" - -# Enable collection of Intrusion Detection System Data (InfluxDB only). -# Only useful if IDS or IPS are enabled on one of the sites. -collect_ids: false - -# Some controllers or reverse proxy configurations do not allow cookies to be -# re-user on every request (every interval). This setting provides a workaround -# That causes the poller to re-auth (login) to the controller on every interval. -# Only enable this if you get login errors after 30 seconds. This will generate -# a few more requests to your controller every interval. reauthenticate: false - -# If your UniFi controller has a valid SSL certificate, you can enable -# this option to validate it. Otherwise, any SSL certificate is valid. verify_ssl: false +save_ids: false +save_sites: true diff --git a/pkg/influxunifi/site.go b/pkg/influxunifi/site.go index 466a43ac..dbfdac38 100644 --- a/pkg/influxunifi/site.go +++ b/pkg/influxunifi/site.go @@ -1,7 +1,6 @@ package influxunifi import ( - "strings" "time" influx "github.com/influxdata/influxdb1-client/v2" @@ -14,20 +13,16 @@ func SitePoints(u *unifi.Site, now time.Time) ([]*influx.Point, error) { points := []*influx.Point{} for _, s := range u.Health { tags := map[string]string{ - "name": u.Name, - "site_name": u.SiteName, - "desc": u.Desc, - "status": s.Status, - "subsystem": s.Subsystem, - "wan_ip": s.WanIP, - "netmask": s.Netmask, - "gw_name": s.GwName, - "gw_mac": s.GwMac, - "gw_version": s.GwVersion, - "lan_ip": s.LanIP, - "nameservers": strings.Join(s.Nameservers, ","), - "gateways": strings.Join(s.Gateways, ","), - "num_new_alarms": u.NumNewAlarms.Txt, + "name": u.Name, + "site_name": u.SiteName, + "desc": u.Desc, + "status": s.Status, + "subsystem": s.Subsystem, + "wan_ip": s.WanIP, + "netmask": s.Netmask, + "gw_name": s.GwName, + "gw_mac": s.GwMac, + "lan_ip": s.LanIP, } fields := map[string]interface{}{ "num_user": s.NumUser.Val, @@ -61,8 +56,6 @@ func SitePoints(u *unifi.Site, now time.Time) ([]*influx.Point, error) { "remote_user_rx_packets": s.RemoteUserRxPackets.Val, "remote_user_tx_packets": s.RemoteUserTxPackets.Val, "num_new_alarms": u.NumNewAlarms.Val, - "nameservers": len(s.Nameservers), - "gateways": len(s.Gateways), } pt, err := influx.NewPoint("subsystems", tags, fields, time.Now()) if err != nil { diff --git a/pkg/influxunifi/udm.go b/pkg/influxunifi/udm.go index 54f9494f..f679c922 100644 --- a/pkg/influxunifi/udm.go +++ b/pkg/influxunifi/udm.go @@ -17,20 +17,15 @@ func UDMPoints(u *unifi.UDM, now time.Time) ([]*influx.Point, error) { u.Stat.Gw = &unifi.Gw{} } tags := map[string]string{ - "id": u.ID, - "mac": u.Mac, - "site_id": u.SiteID, - "site_name": u.SiteName, - "name": u.Name, - "cfgversion": u.Cfgversion, - "device_id": u.DeviceID, - "known_cfgversion": u.KnownCfgversion, - "model": u.Model, - "serial": u.Serial, - "type": u.Type, - "usg_caps": u.UsgCaps.Txt, - "wan1_up": u.Wan1.Up.Txt, - "wan2_up": u.Wan2.Up.Txt, + "ip": u.IP, + "mac": u.Mac, + "site_id": u.SiteID, + "site_name": u.SiteName, + "name": u.Name, + "cfgversion": u.Cfgversion, + "model": u.Model, + "serial": u.Serial, + "type": u.Type, } fields := map[string]interface{}{ "ip": u.IP, @@ -138,16 +133,15 @@ func UDMPoints(u *unifi.UDM, now time.Time) ([]*influx.Point, error) { } points := []*influx.Point{pt} tags = map[string]string{ - "id": u.ID, - "mac": u.Mac, - "site_id": u.SiteID, - "site_name": u.SiteName, - "name": u.Name, - "model": u.Model, - "serial": u.Serial, - "type": u.Type, - "has_fan": u.HasFan.Txt, - "has_temperature": u.HasTemperature.Txt, + "ip": u.IP, + "mac": u.Mac, + "site_id": u.SiteID, + "site_name": u.SiteName, + "name": u.Name, + "cfgversion": u.Cfgversion, + "model": u.Model, + "serial": u.Serial, + "type": u.Type, } fields = map[string]interface{}{ "fw_caps": u.FwCaps.Val, diff --git a/pkg/influxunifi/usg.go b/pkg/influxunifi/usg.go index a86f28b4..81ae87dc 100644 --- a/pkg/influxunifi/usg.go +++ b/pkg/influxunifi/usg.go @@ -15,21 +15,15 @@ func USGPoints(u *unifi.USG, now time.Time) ([]*influx.Point, error) { u.Stat.Gw = &unifi.Gw{} } tags := map[string]string{ - "id": u.ID, - "mac": u.Mac, - "device_type": u.Stat.O, - "site_id": u.SiteID, - "site_name": u.SiteName, - "name": u.Name, - "cfgversion": u.Cfgversion, - "device_id": u.DeviceID, - "known_cfgversion": u.KnownCfgversion, - "model": u.Model, - "serial": u.Serial, - "type": u.Type, - "usg_caps": u.UsgCaps.Txt, - "wan1_up": u.Wan1.Up.Txt, - "wan2_up": u.Wan2.Up.Txt, + "ip": u.IP, + "mac": u.Mac, + "site_id": u.SiteID, + "site_name": u.SiteName, + "name": u.Name, + "cfgversion": u.Cfgversion, + "model": u.Model, + "serial": u.Serial, + "type": u.Type, } fields := map[string]interface{}{ "ip": u.IP, diff --git a/pkg/influxunifi/usw.go b/pkg/influxunifi/usw.go index a627363a..7f392962 100644 --- a/pkg/influxunifi/usw.go +++ b/pkg/influxunifi/usw.go @@ -14,16 +14,15 @@ func USWPoints(u *unifi.USW, now time.Time) ([]*influx.Point, error) { u.Stat.Sw = &unifi.Sw{} } tags := map[string]string{ - "id": u.ID, - "mac": u.Mac, - "site_id": u.SiteID, - "site_name": u.SiteName, - "name": u.Name, - "model": u.Model, - "serial": u.Serial, - "type": u.Type, - "has_fan": u.HasFan.Txt, - "has_temperature": u.HasTemperature.Txt, + "ip": u.IP, + "mac": u.Mac, + "site_id": u.SiteID, + "site_name": u.SiteName, + "name": u.Name, + "cfgversion": u.Cfgversion, + "model": u.Model, + "serial": u.Serial, + "type": u.Type, } fields := map[string]interface{}{ "fw_caps": u.FwCaps.Val, diff --git a/pkg/poller/config.go b/pkg/poller/config.go index 743d3996..37ca9a90 100644 --- a/pkg/poller/config.go +++ b/pkg/poller/config.go @@ -65,9 +65,10 @@ type Config struct { Debug bool `json:"debug" toml:"debug" xml:"debug" yaml:"debug"` Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet" yaml:"quiet"` VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` - CollectIDS bool `json:"collect_ids" toml:"collect_ids" xml:"collect_ids" yaml:"collect_ids"` + SaveIDS bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` ReAuth bool `json:"reauthenticate" toml:"reauthenticate" xml:"reauthenticate" yaml:"reauthenticate"` InfxBadSSL bool `json:"influx_insecure_ssl" toml:"influx_insecure_ssl" xml:"influx_insecure_ssl" yaml:"influx_insecure_ssl"` + SaveSites bool `json:"save_sites,omitempty" toml:"save_sites,omitempty" xml:"save_sites" yaml:"save_sites"` Mode string `json:"mode" toml:"mode" xml:"mode" yaml:"mode"` HTTPListen string `json:"http_listen" toml:"http_listen" xml:"http_listen" yaml:"http_listen"` Namespace string `json:"namespace" toml:"namespace" xml:"namespace" yaml:"namespace"` diff --git a/pkg/poller/influx.go b/pkg/poller/influx.go index 1eb2bb83..9b2872d2 100644 --- a/pkg/poller/influx.go +++ b/pkg/poller/influx.go @@ -70,7 +70,7 @@ func (u *UnifiPoller) LogInfluxReport(m *influxunifi.Metrics) { fields += len(i) } idsMsg := "" - if u.Config.CollectIDS { + if u.Config.SaveIDS { idsMsg = fmt.Sprintf("IDS Events: %d, ", len(m.IDSList)) } u.Logf("UniFi Measurements Recorded. Sites: %d, Clients: %d, "+ diff --git a/pkg/poller/prometheus.go b/pkg/poller/prometheus.go index 88502dc9..45dd1176 100644 --- a/pkg/poller/prometheus.go +++ b/pkg/poller/prometheus.go @@ -1,7 +1,6 @@ package poller import ( - "fmt" "net/http" "strings" "time" @@ -54,16 +53,11 @@ func (u *UnifiPoller) ExportMetrics() (*metrics.Metrics, error) { // This is run by Prometheus as LoggingFn func (u *UnifiPoller) LogExportReport(report *promunifi.Report) { m := report.Metrics - idsMsg := "" - if u.Config.CollectIDS { - idsMsg = fmt.Sprintf(", IDS Events: %d, ", len(m.IDSList)) - } - u.Logf("UniFi Measurements Exported. Site: %d, Client: %d, "+ - "UAP: %d, USG/UDM: %d, USW: %d%s, Descs: %d, "+ + "UAP: %d, USG/UDM: %d, USW: %d, Descs: %d, "+ "Metrics: %d, Errs: %d, 0s: %d, Reqs/Total: %v / %v", len(m.Sites), len(m.Clients), len(m.UAPs), len(m.UDMs)+len(m.USGs), len(m.USWs), - idsMsg, report.Descs, report.Total, report.Errors, report.Zeros, + report.Descs, report.Total, report.Errors, report.Zeros, report.Fetch.Round(time.Millisecond/oneDecimalPoint), report.Elapsed.Round(time.Millisecond/oneDecimalPoint)) } diff --git a/pkg/poller/start.go b/pkg/poller/start.go index 11d70228..342d4d58 100644 --- a/pkg/poller/start.go +++ b/pkg/poller/start.go @@ -25,6 +25,7 @@ func New() *UnifiPoller { UnifiBase: defaultUnifiURL, Interval: Duration{defaultInterval}, Sites: []string{"all"}, + SaveSites: true, HTTPListen: defaultHTTPListen, Namespace: appName, }, Flag: &Flag{}, diff --git a/pkg/poller/unifi.go b/pkg/poller/unifi.go index c431f252..73708243 100644 --- a/pkg/poller/unifi.go +++ b/pkg/poller/unifi.go @@ -68,7 +68,7 @@ func (u *UnifiPoller) CollectMetrics() (*metrics.Metrics, error) { // Get the sites we care about. m.Sites, err = u.GetFilteredSites() u.LogError(err, "unifi.GetSites()") - if u.Config.CollectIDS { + if u.Config.SaveIDS { m.IDSList, err = u.Unifi.GetIDS(m.Sites, time.Now().Add(u.Config.Interval.Duration), time.Now()) u.LogError(err, "unifi.GetIDS()") } @@ -111,6 +111,9 @@ func (u *UnifiPoller) AugmentMetrics(metrics *metrics.Metrics) { metrics.Clients[i].GwName = devices[c.GwMac] metrics.Clients[i].RadioDescription = bssdIDs[metrics.Clients[i].Bssid] + metrics.Clients[i].RadioProto } + if !u.Config.SaveSites { + metrics.Sites = nil + } } // GetFilteredSites returns a list of sites to fetch data for.