public_git
/
unpoller_unpoller
mirror of https://github.com/unpoller/unpoller.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

allow more data reduction

This commit is contained in:
davidnewhall2 2019-11-30 03:42:17 -08:00
parent f9a4963b23
commit cef7815e2d
13 changed files with 76 additions and 203 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
examples/up.conf.example
View File

@ -56,10 +56,6 @@ unifi_user = "influx"
unifi_pass = "4BB9345C-2341-48D7-99F5-E01B583FF77F" unifi_pass = "4BB9345C-2341-48D7-99F5-E01B583FF77F"
unifi_url = "https://127.0.0.1:8443" unifi_url = "https://127.0.0.1:8443"
# Enable collection of Intrusion Detection System Data (InfluxDB only).
# Only useful if IDS or IPS are enabled on one of the sites.
collect_ids = false
# Some controllers or reverse proxy configurations do not allow cookies to be # Some controllers or reverse proxy configurations do not allow cookies to be
# re-user on every request (every interval). This setting provides a workaround # re-user on every request (every interval). This setting provides a workaround
# That causes the poller to re-auth (login) to the controller on every interval. # That causes the poller to re-auth (login) to the controller on every interval.
@ -67,6 +63,14 @@ collect_ids = false
# a few more requests to your controller every interval. # a few more requests to your controller every interval.
reauthenticate = false reauthenticate = false
# Enable collection of Intrusion Detection System Data (InfluxDB only).
# Only useful if IDS or IPS are enabled on one of the sites.
save_ids = false
# Enable collection of site data. This data powers the Network Sites dashboard.
# It's not valuable to everyone and setting this to false will save resources.
save_sites = true
# If your UniFi controller has a valid SSL certificate (like lets encrypt), # If your UniFi controller has a valid SSL certificate (like lets encrypt),
# you can enable this option to validate it. Otherwise, any SSL certificate is # you can enable this option to validate it. Otherwise, any SSL certificate is
# valid. If you don't know if you have a valid SSL cert, then you don't have one. # valid. If you don't know if you have a valid SSL cert, then you don't have one.

3
examples/up.json.example
View File

@ -13,7 +13,8 @@
"unifi_user": "influx", "unifi_user": "influx",
"unifi_pass": "", "unifi_pass": "",
"unifi_url": "https://127.0.0.1:8443", "unifi_url": "https://127.0.0.1:8443",
"collect_ids": false, "save_ids": false,
"save_sites": true,
"reauthenticate": false, "reauthenticate": false,
"verify_ssl": false "verify_ssl": false
} }

78
examples/up.xml.example
View File

@ -2,99 +2,33 @@
<!-- <!--
####################################################### #######################################################
# UniFi Poller primary configuration file. XML FORMAT # # UniFi Poller primary configuration file. XML FORMAT #
# provided values are defaults # # provided values are defaults. See up.conf.example! #
####################################################### #######################################################
--> -->
<unifi-poller> <unifi-poller>
<!--
# If the controller has more than one site, specify which sites to poll here.
# Set this to "default" to poll only the first site on the controller.
# A setting of "all" will poll all sites; this works if you only have 1 site too.
Add more sites by adding additional lines:
<sites>site1</sites>
<sites>site2</sites>
-->
<sites>all</sites> <sites>all</sites>
<interval>60s</interval>
<!--
# The UniFi Controller only updates traffic stats about every 30 seconds.
# Setting this to something lower may lead to "zeros" in your data.
# If you're getting zeros now, set this to "1m"
-->
<interval>30s</interval>
<!--
# Turns on line numbers, microsecond logging, and a per-device log.
# The default is false, but I personally leave this on at home (four devices).
# This may be noisy if you have a lot of devices. It adds one line per device.
-->
<debug>false</debug> <debug>false</debug>
<!--
# Turns off per-interval logs. Only startup and error logs will be emitted.
# Recommend enabling debug with this setting for better error logging.
-->
<quiet>false</quiet> <quiet>false</quiet>
<!--
# Which mode to run this application in. The default mode is "influx". Providing
# an invalid mode will also result in "influx". In this default mode the application
# runs as a daemon and polls the controller at the configured interval.
#
# There are two other options at this time: "influxlambda" and "prometheus"
#
# Mode "influxlambda" makes the application exit after collecting and reporting metrics
# to InfluxDB one time. This mode requires an external process like an AWS Lambda
# or a simple crontab to keep the timings accurate on UniFi Poller run intervals.
#
# Mode "prometheus" opens an HTTP server on port 9130 and exports the metrics at
# /metrics for polling collection by a prometheus server. This disables influxdb.
# IMPORTANT: The prometheus mode is still beta.
# Please help us test and provide your feedback on the github repo!
-->
<mode>influx</mode> <mode>influx</mode>
<!--
# This controls on which ip and port /metrics is exported when mode is "prometheus".
# This has no effect in other modes. Must contain a colon and port.
-->
<http_listen>0.0.0.0:9130</http_listen> <http_listen>0.0.0.0:9130</http_listen>
<!--
# InfluxDB does not require auth by default, so the user/password are probably unimportant.
-->
<influx_db>unifi</influx_db> <influx_db>unifi</influx_db>
<influx_pass>unifi</influx_pass> <influx_pass>unifi</influx_pass>
<influx_url>http://127.0.0.1:8086</influx_url> <influx_url>http://127.0.0.1:8086</influx_url>
<influx_user>unifi</influx_user> <influx_user>unifi</influx_user>
<!--
# If your InfluxDB uses an invalid SSL cert, set this to true.
-->
<influx_insecure_ssl>false</influx_insecure_ssl> <influx_insecure_ssl>false</influx_insecure_ssl>
<!--
# Make a read-only user in the UniFi Admin Settings.
-->
<unifi_user>influx</unifi_user> <unifi_user>influx</unifi_user>
<unifi_pass></unifi_pass> <unifi_pass></unifi_pass>
<unifi_url>https://127.0.0.1:8443</unifi_url> <unifi_url>https://127.0.0.1:8443</unifi_url>
<!--
# Enable collection of Intrusion Detection System Data (InfluxDB only).
# Only useful if IDS or IPS are enabled on one of the sites.
-->
<collect_ids>false</collect_ids>
<!--
# Some controllers or reverse proxy configurations do not allow cookies to be
# re-user on every request (every interval). This setting provides a workaround
# That causes the poller to re-auth (login) to the controller on every interval.
# Only enable this if you get login errors after 30 seconds. This will generate
# a few more requests to your controller every interval.
-->
<reauthenticate>false</reauthenticate> <reauthenticate>false</reauthenticate>
<!--
# If your UniFi controller has a valid SSL certificate, you can enable
# this option to validate it. Otherwise, any SSL certificate is valid.
-->
<verify_ssl>false</verify_ssl> <verify_ssl>false</verify_ssl>
<save_ids>false</save_ids>
<save_sites>true</save_sites>
</unifi-poller> </unifi-poller>

53
examples/up.yaml.example
View File

@ -1,73 +1,28 @@
########################################################
# UniFi Poller primary configuration file. YAML FORMAT # # UniFi Poller primary configuration file. YAML FORMAT #
# provided values are defaults # # provided values are defaults. See up.conf.example! #
######################################################## ########################################################
--- ---
# If the controller has more than one site, specify which sites to poll here.
# Set this to "default" to poll only the first site on the controller.
# A setting of "all" will poll all sites; this works if you only have 1 site too.
sites: sites:
- all - all
# The UniFi Controller only updates traffic stats about every 30 seconds.
# Setting this to something lower may lead to "zeros" in your data.
# If you're getting zeros now, set this to "1m"
# Only has effect if "mode" (below) is "influx" - other modes do not use interval.
interval: "30s" interval: "30s"
# Turns on line numbers, microsecond logging, and a per-device log.
# The default is false, but I personally leave this on at home (four devices).
# This may be noisy if you have a lot of devices. It adds one line per device.
debug: false debug: false
# Turns off per-interval logs. Only startup and error logs will be emitted.
# Recommend enabling debug with this setting for better error logging.
quiet: false quiet: false
# Which mode to run this application in. The default mode is "influx". Providing
# an invalid mode will also result in "influx". In this default mode the application
# runs as a daemon and polls the controller at the configured interval.
#
# There are two other options at this time: "influxlambda" and "prometheus"
#
# Mode "influxlambda" makes the application exit after collecting and reporting metrics
# to InfluxDB one time. This mode requires an external process like an AWS Lambda
# or a simple crontab to keep the timings accurate on UniFi Poller run intervals.
#
# Mode "prometheus" opens an HTTP server on port 9130 and exports the metrics at
# /metrics for polling collection by a prometheus server. This disables influxdb.
# IMPORTANT: The prometheus mode is still beta.
# Please help us test and provide your feedback on the github repo!
mode: "influx" mode: "influx"
# This controls on which ip and port /metrics is exported when mode is "prometheus".
# This has no effect in other modes. Must contain a colon and port.
http_listen: "0.0.0.0:9130" http_listen: "0.0.0.0:9130"
# InfluxDB does not require auth by default, so the user/password are probably unimportant.
influx_url: "http://127.0.0.1:8086" influx_url: "http://127.0.0.1:8086"
influx_user: "unifi" influx_user: "unifi"
influx_pass: "unifi" influx_pass: "unifi"
# Be sure to create this database.
influx_db: "unifi" influx_db: "unifi"
# If your InfluxDB uses an invalid SSL cert, set this to true.
influx_insecure_ssl: false influx_insecure_ssl: false
# Make a read-only user in the UniFi Admin Settings.
unifi_user: "influx" unifi_user: "influx"
unifi_pass: "" unifi_pass: ""
unifi_url: "https://127.0.0.1:8443" unifi_url: "https://127.0.0.1:8443"
# Enable collection of Intrusion Detection System Data (InfluxDB only).
# Only useful if IDS or IPS are enabled on one of the sites.
collect_ids: false
# Some controllers or reverse proxy configurations do not allow cookies to be
# re-user on every request (every interval). This setting provides a workaround
# That causes the poller to re-auth (login) to the controller on every interval.
# Only enable this if you get login errors after 30 seconds. This will generate
# a few more requests to your controller every interval.
reauthenticate: false reauthenticate: false
# If your UniFi controller has a valid SSL certificate, you can enable
# this option to validate it. Otherwise, any SSL certificate is valid.
verify_ssl: false verify_ssl: false
save_ids: false
save_sites: true

27
pkg/influxunifi/site.go
View File

@ -1,7 +1,6 @@
package influxunifi package influxunifi
import ( import (
"strings"
"time" "time"
influx "github.com/influxdata/influxdb1-client/v2" influx "github.com/influxdata/influxdb1-client/v2"
@ -14,20 +13,16 @@ func SitePoints(u *unifi.Site, now time.Time) ([]*influx.Point, error) {
points := []*influx.Point{} points := []*influx.Point{}
for _, s := range u.Health { for _, s := range u.Health {
tags := map[string]string{ tags := map[string]string{
"name": u.Name, "name": u.Name,
"site_name": u.SiteName, "site_name": u.SiteName,
"desc": u.Desc, "desc": u.Desc,
"status": s.Status, "status": s.Status,
"subsystem": s.Subsystem, "subsystem": s.Subsystem,
"wan_ip": s.WanIP, "wan_ip": s.WanIP,
"netmask": s.Netmask, "netmask": s.Netmask,
"gw_name": s.GwName, "gw_name": s.GwName,
"gw_mac": s.GwMac, "gw_mac": s.GwMac,
"gw_version": s.GwVersion, "lan_ip": s.LanIP,
"lan_ip": s.LanIP,
"nameservers": strings.Join(s.Nameservers, ","),
"gateways": strings.Join(s.Gateways, ","),
"num_new_alarms": u.NumNewAlarms.Txt,
} }
fields := map[string]interface{}{ fields := map[string]interface{}{
"num_user": s.NumUser.Val, "num_user": s.NumUser.Val,
@ -61,8 +56,6 @@ func SitePoints(u *unifi.Site, now time.Time) ([]*influx.Point, error) {
"remote_user_rx_packets": s.RemoteUserRxPackets.Val, "remote_user_rx_packets": s.RemoteUserRxPackets.Val,
"remote_user_tx_packets": s.RemoteUserTxPackets.Val, "remote_user_tx_packets": s.RemoteUserTxPackets.Val,
"num_new_alarms": u.NumNewAlarms.Val, "num_new_alarms": u.NumNewAlarms.Val,
"nameservers": len(s.Nameservers),
"gateways": len(s.Gateways),
} }
pt, err := influx.NewPoint("subsystems", tags, fields, time.Now()) pt, err := influx.NewPoint("subsystems", tags, fields, time.Now())
if err != nil { if err != nil {

42
pkg/influxunifi/udm.go
View File

@ -17,20 +17,15 @@ func UDMPoints(u *unifi.UDM, now time.Time) ([]*influx.Point, error) {
u.Stat.Gw = &unifi.Gw{} u.Stat.Gw = &unifi.Gw{}
} }
tags := map[string]string{ tags := map[string]string{
"id": u.ID, "ip": u.IP,
"mac": u.Mac, "mac": u.Mac,
"site_id": u.SiteID, "site_id": u.SiteID,
"site_name": u.SiteName, "site_name": u.SiteName,
"name": u.Name, "name": u.Name,
"cfgversion": u.Cfgversion, "cfgversion": u.Cfgversion,
"device_id": u.DeviceID, "model": u.Model,
"known_cfgversion": u.KnownCfgversion, "serial": u.Serial,
"model": u.Model, "type": u.Type,
"serial": u.Serial,
"type": u.Type,
"usg_caps": u.UsgCaps.Txt,
"wan1_up": u.Wan1.Up.Txt,
"wan2_up": u.Wan2.Up.Txt,
} }
fields := map[string]interface{}{ fields := map[string]interface{}{
"ip": u.IP, "ip": u.IP,
@ -138,16 +133,15 @@ func UDMPoints(u *unifi.UDM, now time.Time) ([]*influx.Point, error) {
} }
points := []*influx.Point{pt} points := []*influx.Point{pt}
tags = map[string]string{ tags = map[string]string{
"id": u.ID, "ip": u.IP,
"mac": u.Mac, "mac": u.Mac,
"site_id": u.SiteID, "site_id": u.SiteID,
"site_name": u.SiteName, "site_name": u.SiteName,
"name": u.Name, "name": u.Name,
"model": u.Model, "cfgversion": u.Cfgversion,
"serial": u.Serial, "model": u.Model,
"type": u.Type, "serial": u.Serial,
"has_fan": u.HasFan.Txt, "type": u.Type,
"has_temperature": u.HasTemperature.Txt,
} }
fields = map[string]interface{}{ fields = map[string]interface{}{
"fw_caps": u.FwCaps.Val, "fw_caps": u.FwCaps.Val,

24
pkg/influxunifi/usg.go
View File

@ -15,21 +15,15 @@ func USGPoints(u *unifi.USG, now time.Time) ([]*influx.Point, error) {
u.Stat.Gw = &unifi.Gw{} u.Stat.Gw = &unifi.Gw{}
} }
tags := map[string]string{ tags := map[string]string{
"id": u.ID, "ip": u.IP,
"mac": u.Mac, "mac": u.Mac,
"device_type": u.Stat.O, "site_id": u.SiteID,
"site_id": u.SiteID, "site_name": u.SiteName,
"site_name": u.SiteName, "name": u.Name,
"name": u.Name, "cfgversion": u.Cfgversion,
"cfgversion": u.Cfgversion, "model": u.Model,
"device_id": u.DeviceID, "serial": u.Serial,
"known_cfgversion": u.KnownCfgversion, "type": u.Type,
"model": u.Model,
"serial": u.Serial,
"type": u.Type,
"usg_caps": u.UsgCaps.Txt,
"wan1_up": u.Wan1.Up.Txt,
"wan2_up": u.Wan2.Up.Txt,
} }
fields := map[string]interface{}{ fields := map[string]interface{}{
"ip": u.IP, "ip": u.IP,

19
pkg/influxunifi/usw.go
View File

@ -14,16 +14,15 @@ func USWPoints(u *unifi.USW, now time.Time) ([]*influx.Point, error) {
u.Stat.Sw = &unifi.Sw{} u.Stat.Sw = &unifi.Sw{}
} }
tags := map[string]string{ tags := map[string]string{
"id": u.ID, "ip": u.IP,
"mac": u.Mac, "mac": u.Mac,
"site_id": u.SiteID, "site_id": u.SiteID,
"site_name": u.SiteName, "site_name": u.SiteName,
"name": u.Name, "name": u.Name,
"model": u.Model, "cfgversion": u.Cfgversion,
"serial": u.Serial, "model": u.Model,
"type": u.Type, "serial": u.Serial,
"has_fan": u.HasFan.Txt, "type": u.Type,
"has_temperature": u.HasTemperature.Txt,
} }
fields := map[string]interface{}{ fields := map[string]interface{}{
"fw_caps": u.FwCaps.Val, "fw_caps": u.FwCaps.Val,

3
pkg/poller/config.go
View File

@ -65,9 +65,10 @@ type Config struct {
Debug bool `json:"debug" toml:"debug" xml:"debug" yaml:"debug"` Debug bool `json:"debug" toml:"debug" xml:"debug" yaml:"debug"`
Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet" yaml:"quiet"` Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet" yaml:"quiet"`
VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"`
CollectIDS bool `json:"collect_ids" toml:"collect_ids" xml:"collect_ids" yaml:"collect_ids"` SaveIDS bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"`
ReAuth bool `json:"reauthenticate" toml:"reauthenticate" xml:"reauthenticate" yaml:"reauthenticate"` ReAuth bool `json:"reauthenticate" toml:"reauthenticate" xml:"reauthenticate" yaml:"reauthenticate"`
InfxBadSSL bool `json:"influx_insecure_ssl" toml:"influx_insecure_ssl" xml:"influx_insecure_ssl" yaml:"influx_insecure_ssl"` InfxBadSSL bool `json:"influx_insecure_ssl" toml:"influx_insecure_ssl" xml:"influx_insecure_ssl" yaml:"influx_insecure_ssl"`
SaveSites bool `json:"save_sites,omitempty" toml:"save_sites,omitempty" xml:"save_sites" yaml:"save_sites"`
Mode string `json:"mode" toml:"mode" xml:"mode" yaml:"mode"` Mode string `json:"mode" toml:"mode" xml:"mode" yaml:"mode"`
HTTPListen string `json:"http_listen" toml:"http_listen" xml:"http_listen" yaml:"http_listen"` HTTPListen string `json:"http_listen" toml:"http_listen" xml:"http_listen" yaml:"http_listen"`
Namespace string `json:"namespace" toml:"namespace" xml:"namespace" yaml:"namespace"` Namespace string `json:"namespace" toml:"namespace" xml:"namespace" yaml:"namespace"`

2
pkg/poller/influx.go
View File

@ -70,7 +70,7 @@ func (u *UnifiPoller) LogInfluxReport(m *influxunifi.Metrics) {
fields += len(i) fields += len(i)
} }
idsMsg := "" idsMsg := ""
if u.Config.CollectIDS { if u.Config.SaveIDS {
idsMsg = fmt.Sprintf("IDS Events: %d, ", len(m.IDSList)) idsMsg = fmt.Sprintf("IDS Events: %d, ", len(m.IDSList))
} }
u.Logf("UniFi Measurements Recorded. Sites: %d, Clients: %d, "+ u.Logf("UniFi Measurements Recorded. Sites: %d, Clients: %d, "+

10
pkg/poller/prometheus.go
View File

@ -1,7 +1,6 @@
package poller package poller
import ( import (
"fmt"
"net/http" "net/http"
"strings" "strings"
"time" "time"
@ -54,16 +53,11 @@ func (u *UnifiPoller) ExportMetrics() (*metrics.Metrics, error) {
// This is run by Prometheus as LoggingFn // This is run by Prometheus as LoggingFn
func (u *UnifiPoller) LogExportReport(report *promunifi.Report) { func (u *UnifiPoller) LogExportReport(report *promunifi.Report) {
m := report.Metrics m := report.Metrics
idsMsg := ""
if u.Config.CollectIDS {
idsMsg = fmt.Sprintf(", IDS Events: %d, ", len(m.IDSList))
}
u.Logf("UniFi Measurements Exported. Site: %d, Client: %d, "+ u.Logf("UniFi Measurements Exported. Site: %d, Client: %d, "+
"UAP: %d, USG/UDM: %d, USW: %d%s, Descs: %d, "+ "UAP: %d, USG/UDM: %d, USW: %d, Descs: %d, "+
"Metrics: %d, Errs: %d, 0s: %d, Reqs/Total: %v / %v", "Metrics: %d, Errs: %d, 0s: %d, Reqs/Total: %v / %v",
len(m.Sites), len(m.Clients), len(m.UAPs), len(m.UDMs)+len(m.USGs), len(m.USWs), len(m.Sites), len(m.Clients), len(m.UAPs), len(m.UDMs)+len(m.USGs), len(m.USWs),
idsMsg, report.Descs, report.Total, report.Errors, report.Zeros, report.Descs, report.Total, report.Errors, report.Zeros,
report.Fetch.Round(time.Millisecond/oneDecimalPoint), report.Fetch.Round(time.Millisecond/oneDecimalPoint),
report.Elapsed.Round(time.Millisecond/oneDecimalPoint)) report.Elapsed.Round(time.Millisecond/oneDecimalPoint))
} }

1
pkg/poller/start.go
View File

@ -25,6 +25,7 @@ func New() *UnifiPoller {
UnifiBase: defaultUnifiURL, UnifiBase: defaultUnifiURL,
Interval: Duration{defaultInterval}, Interval: Duration{defaultInterval},
Sites: []string{"all"}, Sites: []string{"all"},
SaveSites: true,
HTTPListen: defaultHTTPListen, HTTPListen: defaultHTTPListen,
Namespace: appName, Namespace: appName,
}, Flag: &Flag{}, }, Flag: &Flag{},

5
pkg/poller/unifi.go
View File

@ -68,7 +68,7 @@ func (u *UnifiPoller) CollectMetrics() (*metrics.Metrics, error) {
// Get the sites we care about. // Get the sites we care about.
m.Sites, err = u.GetFilteredSites() m.Sites, err = u.GetFilteredSites()
u.LogError(err, "unifi.GetSites()") u.LogError(err, "unifi.GetSites()")
if u.Config.CollectIDS { if u.Config.SaveIDS {
m.IDSList, err = u.Unifi.GetIDS(m.Sites, time.Now().Add(u.Config.Interval.Duration), time.Now()) m.IDSList, err = u.Unifi.GetIDS(m.Sites, time.Now().Add(u.Config.Interval.Duration), time.Now())
u.LogError(err, "unifi.GetIDS()") u.LogError(err, "unifi.GetIDS()")
} }
@ -111,6 +111,9 @@ func (u *UnifiPoller) AugmentMetrics(metrics *metrics.Metrics) {
metrics.Clients[i].GwName = devices[c.GwMac] metrics.Clients[i].GwName = devices[c.GwMac]
metrics.Clients[i].RadioDescription = bssdIDs[metrics.Clients[i].Bssid] + metrics.Clients[i].RadioProto metrics.Clients[i].RadioDescription = bssdIDs[metrics.Clients[i].Bssid] + metrics.Clients[i].RadioProto
} }
if !u.Config.SaveSites {
metrics.Sites = nil
}
} }
// GetFilteredSites returns a list of sites to fetch data for. // GetFilteredSites returns a list of sites to fetch data for.