diff --git a/Dockerfile b/Dockerfile index d1bc9e4b..ad2ed5aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM busybox:latest as builder +FROM busybox:latest AS builder # we have to do this hop because distroless is bare without common shell commands RUN mkdir -p /etc/unpoller diff --git a/examples/up.conf.example b/examples/up.conf.example index 1c1e3ce2..898416ce 100644 --- a/examples/up.conf.example +++ b/examples/up.conf.example @@ -242,6 +242,7 @@ # save_alarms = false # save_anomalies = false # save_dpi = false +# save_traffic = false # save_rogue = false # verify_ssl = false # ssl_cert_paths = [] diff --git a/go.mod b/go.mod index 088f8477..02db634d 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.25.5 require ( github.com/DataDog/datadog-go/v5 v5.8.2 + github.com/flaticols/countrycodes v0.0.2 github.com/gorilla/mux v1.8.1 github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 857e3399..47ceca39 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/flaticols/countrycodes v0.0.2 h1:vedxSqHwG3r7lwUK2bfGFWkVcFv7QuSCKFMkywI/rIE= +github.com/flaticols/countrycodes v0.0.2/go.mod h1:HCwEez5Z+nf062EOWMPqEh1uLb5QdZSTQmTrq4avBOA= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= diff --git a/pkg/inputunifi/collector.go b/pkg/inputunifi/collector.go index adfb7e24..45619ee6 100644 --- a/pkg/inputunifi/collector.go +++ b/pkg/inputunifi/collector.go @@ -11,6 +11,11 @@ import ( "github.com/unpoller/unpoller/pkg/poller" ) +const ( + history_seconds = 86400 + poll_duration = time.Second * history_seconds +) + var ErrScrapeFilterMatchFailed = fmt.Errorf("scrape filter match failed, and filter is not http URL") func (u *InputUnifi) isNill(c *Controller) bool { @@ -99,40 +104,153 @@ func (u *InputUnifi) pollController(c *Controller) (*poller.Metrics, error) { m := &Metrics{TS: time.Now(), Sites: sites} defer updateWeb(c, m) + // FIXME needs to be last poll time maybe + st := m.TS.Add(-1 * poll_duration) + tp := unifi.EpochMillisTimePeriod{StartEpochMillis: st.UnixMilli(), EndEpochMillis: m.TS.UnixMilli()} + if c.SaveRogue != nil && *c.SaveRogue { if m.RogueAPs, err = c.Unifi.GetRogueAPs(sites); err != nil { return nil, fmt.Errorf("unifi.GetRogueAPs(%s): %w", c.URL, err) } + u.LogDebugf("Found %d RogueAPs entries", len(m.RogueAPs)) } if c.SaveDPI != nil && *c.SaveDPI { if m.SitesDPI, err = c.Unifi.GetSiteDPI(sites); err != nil { return nil, fmt.Errorf("unifi.GetSiteDPI(%s): %w", c.URL, err) } + u.LogDebugf("Found %d SitesDPI entries", len(m.SitesDPI)) if m.ClientsDPI, err = c.Unifi.GetClientsDPI(sites); err != nil { return nil, fmt.Errorf("unifi.GetClientsDPI(%s): %w", c.URL, err) } + u.LogDebugf("Found %d ClientsDPI entries", len(m.ClientsDPI)) + } + + if c.SaveTraffic != nil && *c.SaveTraffic { + if m.CountryTraffic, err = c.Unifi.GetCountryTraffic(sites, &tp); err != nil { + return nil, fmt.Errorf("unifi.GetCountryTraffic(%s): %w", c.URL, err) + } + u.LogDebugf("Found %d CountryTraffic entries", len(m.CountryTraffic)) + } + + if c.SaveTraffic != nil && *c.SaveTraffic && c.SaveDPI != nil && *c.SaveDPI { + clientUsageByApp, err := c.Unifi.GetClientTraffic(sites, &tp, true) + if err != nil { + return nil, fmt.Errorf("unifi.GetClientTraffic(%s): %w", c.URL, err) + } + u.LogDebugf("Found %d ClientUsageByApp entries", len(clientUsageByApp)) + b4 := len(m.ClientsDPI) + u.convertToClientDPI(clientUsageByApp, m) + u.LogDebugf("Added %d ClientDPI entries for a total of %d", len(m.ClientsDPI)-b4, len(m.ClientsDPI)) } // Get all the points. if m.Clients, err = c.Unifi.GetClients(sites); err != nil { return nil, fmt.Errorf("unifi.GetClients(%s): %w", c.URL, err) } + u.LogDebugf("Found %d Clients entries", len(m.Clients)) if m.Devices, err = c.Unifi.GetDevices(sites); err != nil { return nil, fmt.Errorf("unifi.GetDevices(%s): %w", c.URL, err) } + u.LogDebugf("Found %d UBB, %d UXG, %d PDU, %d UCI, %d UAP %d USG %d USW %d UDM devices", + len(m.Devices.UBBs), len(m.Devices.UXGs), + len(m.Devices.PDUs), len(m.Devices.UCIs), + len(m.Devices.UAPs), len(m.Devices.USGs), + len(m.Devices.USWs), len(m.Devices.UDMs)) // Get speed test results for all WANs - if m.SpeedTests, err = c.Unifi.GetSpeedTests(sites, 86400); err != nil { + if m.SpeedTests, err = c.Unifi.GetSpeedTests(sites, history_seconds); err != nil { // Don't fail collection if speed tests fail - older controllers may not have this endpoint u.LogDebugf("unifi.GetSpeedTests(%s): %v (continuing)", c.URL, err) + } else { + u.LogDebugf("Found %d SpeedTests entries", len(m.SpeedTests)) } return u.augmentMetrics(c, m), nil } +// FIXME this would be better implemented on FlexInt itself +func (u *InputUnifi) intToFlexInt(i int) unifi.FlexInt { + return unifi.FlexInt{ + Val: float64(i), + Txt: fmt.Sprintf("%d", i), + } +} + +// FIXME this would be better implemented on FlexInt itself +func (u *InputUnifi) int64ToFlexInt(i int64) unifi.FlexInt { + return unifi.FlexInt{ + Val: float64(i), + Txt: fmt.Sprintf("%d", i), + } +} + +func (u *InputUnifi) convertToClientDPI(clientUsageByApp []*unifi.ClientUsageByApp, metrics *Metrics) { + for _, client := range clientUsageByApp { + byApp := make([]unifi.DPIData, 0) + byCat := make([]unifi.DPIData, 0) + type catCount struct { + BytesReceived int64 + BytesTransmitted int64 + } + byCatMap := make(map[int]catCount) + dpiClients := make([]*unifi.DPIClient, 0) + // TODO create cat table + for _, app := range client.UsageByApp { + dpiData := unifi.DPIData{ + App: u.intToFlexInt(app.Application), + Cat: u.intToFlexInt(app.Category), + Clients: dpiClients, + KnownClients: u.intToFlexInt(0), + RxBytes: u.int64ToFlexInt(app.BytesReceived), + RxPackets: u.int64ToFlexInt(0), // We don't have packets from Unifi Controller + TxBytes: u.int64ToFlexInt(app.BytesTransmitted), + TxPackets: u.int64ToFlexInt(0), // We don't have packets from Unifi Controller + } + cat, ok := byCatMap[app.Category] + if ok { + cat.BytesReceived += app.BytesReceived + cat.BytesTransmitted += app.BytesTransmitted + } else { + cat = catCount{ + BytesReceived: app.BytesReceived, + BytesTransmitted: app.BytesTransmitted, + } + byCatMap[app.Category] = cat + } + byApp = append(byApp, dpiData) + } + if len(byApp) <= 1 { + byCat = byApp + } else { + for category, cat := range byCatMap { + dpiData := unifi.DPIData{ + App: u.intToFlexInt(16777215), // Unknown + Cat: u.intToFlexInt(category), + Clients: dpiClients, + KnownClients: u.intToFlexInt(0), + RxBytes: u.int64ToFlexInt(cat.BytesReceived), + RxPackets: u.int64ToFlexInt(0), // We don't have packets from Unifi Controller + TxBytes: u.int64ToFlexInt(cat.BytesTransmitted), + TxPackets: u.int64ToFlexInt(0), // We don't have packets from Unifi Controller + } + byCat = append(byCat, dpiData) + } + } + dpiTable := unifi.DPITable{ + ByApp: byApp, + ByCat: byCat, + MAC: client.Client.Mac, + Name: client.Client.Name, + SiteName: client.TrafficSite.SiteName, + SourceName: client.TrafficSite.SourceName, + } + metrics.ClientsDPI = append(metrics.ClientsDPI, &dpiTable) + } +} + // augmentMetrics is our middleware layer between collecting metrics and writing them. // This is where we can manipuate the returned data or make arbitrary decisions. // This method currently adds parent device names to client metrics and hashes PII. @@ -191,6 +309,10 @@ func (u *InputUnifi) augmentMetrics(c *Controller, metrics *Metrics) *poller.Met m.SpeedTests = append(m.SpeedTests, speedTest) } + for _, traffic := range metrics.CountryTraffic { + m.CountryTraffic = append(m.CountryTraffic, traffic) + } + return m } diff --git a/pkg/inputunifi/input.go b/pkg/inputunifi/input.go index 52838321..078c2216 100644 --- a/pkg/inputunifi/input.go +++ b/pkg/inputunifi/input.go @@ -45,6 +45,7 @@ type Controller struct { ProtectThumbnails *bool `json:"protect_thumbnails" toml:"protect_thumbnails" xml:"protect_thumbnails" yaml:"protect_thumbnails"` SaveIDs *bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` SaveDPI *bool `json:"save_dpi" toml:"save_dpi" xml:"save_dpi" yaml:"save_dpi"` + SaveTraffic *bool `json:"save_traffic" toml:"save_traffic" xml:"save_traffic" yaml:"save_traffic"` SaveRogue *bool `json:"save_rogue" toml:"save_rogue" xml:"save_rogue" yaml:"save_rogue"` HashPII *bool `json:"hash_pii" toml:"hash_pii" xml:"hash_pii" yaml:"hash_pii"` DropPII *bool `json:"drop_pii" toml:"drop_pii" xml:"drop_pii" yaml:"drop_pii"` @@ -72,14 +73,15 @@ type Config struct { // Metrics is simply a useful container for everything. type Metrics struct { - TS time.Time - Sites []*unifi.Site - Clients []*unifi.Client - SitesDPI []*unifi.DPITable - ClientsDPI []*unifi.DPITable - RogueAPs []*unifi.RogueAP - SpeedTests []*unifi.SpeedTestResult - Devices *unifi.Devices + TS time.Time + Sites []*unifi.Site + Clients []*unifi.Client + SitesDPI []*unifi.DPITable + ClientsDPI []*unifi.DPITable + CountryTraffic []*unifi.UsageByCountry + RogueAPs []*unifi.RogueAP + SpeedTests []*unifi.SpeedTestResult + Devices *unifi.Devices } func init() { // nolint: gochecknoinits @@ -239,6 +241,10 @@ func (u *InputUnifi) setDefaults(c *Controller) { //nolint:cyclop c.SaveDPI = &f } + if c.SaveTraffic == nil { + c.SaveTraffic = &f + } + if c.SaveRogue == nil { c.SaveRogue = &f } @@ -271,6 +277,10 @@ func (u *InputUnifi) setDefaults(c *Controller) { //nolint:cyclop c.SaveAnomal = &f } + if c.SaveTraffic == nil { + c.SaveTraffic = &f + } + if c.URL == "" { c.URL = defaultURL } diff --git a/pkg/inputunifi/interface.go b/pkg/inputunifi/interface.go index 60e3be52..14ae2b1f 100644 --- a/pkg/inputunifi/interface.go +++ b/pkg/inputunifi/interface.go @@ -130,6 +130,7 @@ func (u *InputUnifi) logController(c *Controller) { u.Logf(" => Save Events %v / Save Syslog %v / Save IDs %v (logs)", *c.SaveEvents, *c.SaveSyslog, *c.SaveIDs) u.Logf(" => Save Alarms %v / Anomalies %v / Protect Logs %v (thumbnails: %v)", *c.SaveAlarms, *c.SaveAnomal, *c.SaveProtectLogs, *c.ProtectThumbnails) u.Logf(" => Save Rogue APs: %v", *c.SaveRogue) + u.Logf(" => Save Traffic %v", *c.SaveTraffic) } // Events allows you to pull only events (and IDs) from the UniFi Controller. diff --git a/pkg/inputunifi/updateweb.go b/pkg/inputunifi/updateweb.go index bed179ec..c124161c 100644 --- a/pkg/inputunifi/updateweb.go +++ b/pkg/inputunifi/updateweb.go @@ -51,6 +51,7 @@ func formatControllers(controllers []*Controller) []*Controller { HashPII: c.HashPII, DropPII: c.DropPII, SaveSites: c.SaveSites, + SaveTraffic: c.SaveTraffic, User: c.User, Pass: strconv.FormatBool(c.Pass != ""), APIKey: strconv.FormatBool(c.APIKey != ""), diff --git a/pkg/poller/config.go b/pkg/poller/config.go index 55806273..f520ab5a 100644 --- a/pkg/poller/config.go +++ b/pkg/poller/config.go @@ -33,10 +33,10 @@ func DefaultConfFile() string { case "netbsd": fallthrough case "openbsd": - return "/etc/unpoller/up.conf,/etc/unifi-poller/up.conf,/usr/local/etc/unifi-poller/up.conf" + return "/etc/unpoller/up.conf,/etc/unifi-poller/up.conf,/usr/local/etc/unifi-poller/up.conf,up.conf" default: // linux and everything else - return "/etc/unpoller/up.conf,/config/unifi-poller.conf,/etc/unifi-poller/up.conf" + return "/etc/unpoller/up.conf,/config/unifi-poller.conf,/etc/unifi-poller/up.conf,up.conf" } } @@ -79,14 +79,15 @@ type Flags struct { // Metrics is a type shared by the exporting and reporting packages. type Metrics struct { - TS time.Time - Sites []any - Clients []any - SitesDPI []any - ClientsDPI []any - Devices []any - RogueAPs []any - SpeedTests []any + TS time.Time + Sites []any + Clients []any + SitesDPI []any + ClientsDPI []any + Devices []any + RogueAPs []any + SpeedTests []any + CountryTraffic []any } // Events defines the type for log entries. diff --git a/pkg/poller/inputs.go b/pkg/poller/inputs.go index d9316695..28b3241b 100644 --- a/pkg/poller/inputs.go +++ b/pkg/poller/inputs.go @@ -268,7 +268,7 @@ func AppendMetrics(existing *Metrics, m *Metrics) *Metrics { existing.RogueAPs = append(existing.RogueAPs, m.RogueAPs...) existing.Clients = append(existing.Clients, m.Clients...) existing.Devices = append(existing.Devices, m.Devices...) - + existing.CountryTraffic = append(existing.CountryTraffic, m.CountryTraffic...) return existing } diff --git a/pkg/promunifi/collector.go b/pkg/promunifi/collector.go index 5a6c4c32..a4f310d0 100644 --- a/pkg/promunifi/collector.go +++ b/pkg/promunifi/collector.go @@ -35,16 +35,17 @@ const ( var ErrMetricFetchFailed = fmt.Errorf("metric fetch failed") type promUnifi struct { - *Config `json:"prometheus" toml:"prometheus" xml:"prometheus" yaml:"prometheus"` - Client *uclient - Device *unifiDevice - UAP *uap - USG *usg - USW *usw - PDU *pdu - Site *site - RogueAP *rogueap - SpeedTest *speedtest + *Config `json:"prometheus" toml:"prometheus" xml:"prometheus" yaml:"prometheus"` + Client *uclient + Device *unifiDevice + UAP *uap + USG *usg + USW *usw + PDU *pdu + Site *site + RogueAP *rogueap + SpeedTest *speedtest + CountryTraffic *ucountrytraffic // This interface is passed to the Collect() method. The Collect method uses // this interface to retrieve the latest UniFi measurements and export them. Collector poller.Collect @@ -203,6 +204,7 @@ func (u *promUnifi) Run(c poller.Collect) error { u.Site = descSite(u.Namespace + "_site_") u.RogueAP = descRogueAP(u.Namespace + "_rogueap_") u.SpeedTest = descSpeedTest(u.Namespace + "_speedtest_") + u.CountryTraffic = descCountryTraffic(u.Namespace + "_countrytraffic_") mux := http.NewServeMux() promver.Version = version.Version @@ -405,6 +407,10 @@ func (u *promUnifi) loopExports(r report) { u.exportClientDPI(r, c, appTotal, catTotal) } + for _, ct := range m.CountryTraffic { + u.exportCountryTraffic(r, ct) + } + u.exportClientDPItotals(r, appTotal, catTotal) } @@ -443,6 +449,8 @@ func (u *promUnifi) switchExport(r report, v any) { u.exportClient(r, v) case *unifi.SpeedTestResult: u.exportSpeedTest(r, v) + case *unifi.UsageByCountry: + u.exportCountryTraffic(r, v) default: u.LogErrorf("invalid type: %T", v) } diff --git a/pkg/promunifi/countries.go b/pkg/promunifi/countries.go new file mode 100644 index 00000000..3e9ec612 --- /dev/null +++ b/pkg/promunifi/countries.go @@ -0,0 +1,53 @@ +package promunifi + +import ( + "github.com/flaticols/countrycodes" + "github.com/prometheus/client_golang/prometheus" + "github.com/unpoller/unifi/v5" +) + +type ucountrytraffic struct { + RxBytes *prometheus.Desc + TxBytes *prometheus.Desc +} + +func descCountryTraffic(ns string) *ucountrytraffic { + labels := []string{ + "code", + "name", + "region", + "sub_region", + "site_name", + "source", + } + + return &ucountrytraffic{ + RxBytes: prometheus.NewDesc(ns+"receive_bytes_total", "Country Receive Bytes", labels, nil), + TxBytes: prometheus.NewDesc(ns+"transmit_bytes_total", "Country Transmit Bytes", labels, nil), + } +} + +func (u *promUnifi) exportCountryTraffic(r report, v any) { + s, ok := v.(*unifi.UsageByCountry) + if !ok { + u.LogErrorf("invalid type given to CountryTraffic: %T", v) + return + } + country, ok := countrycodes.GetByAlpha2(s.Country) + name := "Unknown" + region := "Unknown" + subRegion := "Unknown" + if ok { + name = country.Name + region = country.Region + subRegion = country.SubRegion + } + if s.Country == "GB" || s.Country == "UK" { + name = "United Kingdom" // Because the name is so long otherwise + } + labels := []string{s.Country, name, region, subRegion, s.TrafficSite.SiteName, s.TrafficSite.SourceName} + r.send([]*metric{ + {u.CountryTraffic.RxBytes, counter, s.BytesReceived, labels}, + {u.CountryTraffic.TxBytes, counter, s.BytesTransmitted, labels}, + }) +} diff --git a/pkg/promunifi/report.go b/pkg/promunifi/report.go index 8663ce12..52b879a6 100644 --- a/pkg/promunifi/report.go +++ b/pkg/promunifi/report.go @@ -50,10 +50,10 @@ func (r *Report) report(c poller.Logger, descs map[*prometheus.Desc]bool) { m := r.Metrics c.Logf("UniFi Measurements Exported. Site: %d, Client: %d, "+ - "UAP: %d, USG/UDM: %d, USW: %d, DPI Site/Client: %d/%d, Desc: %d, "+ + "UAP: %d, USG/UDM: %d, USW: %d, DPI Site/Client: %d/%d, Countries: %d, Desc: %d, "+ "Metric: %d, Bytes: %d, Err: %d, 0s: %d, Req/Total: %v / %v", len(m.Sites), len(m.Clients), r.UAP, r.UDM+r.USG+r.UXG, r.USW, len(m.SitesDPI), - len(m.ClientsDPI), len(descs), r.Total, r.Bytes, r.Errors, r.Zeros, + len(m.ClientsDPI), len(m.CountryTraffic), len(descs), r.Total, r.Bytes, r.Errors, r.Zeros, r.Fetch.Round(time.Millisecond/oneDecimalPoint), r.Elapsed.Round(time.Millisecond/oneDecimalPoint)) } diff --git a/pkg/unittest/dep.go b/pkg/unittest/dep.go index 86b4c5f6..b96f6888 100644 --- a/pkg/unittest/dep.go +++ b/pkg/unittest/dep.go @@ -20,15 +20,17 @@ func NewTestSetup(t *testing.T) *TestRig { testCollector := poller.NewTestCollector(t) enabled := true + disabled := false controller := inputunifi.Controller{ - SaveAnomal: &enabled, - SaveAlarms: &enabled, - SaveEvents: &enabled, - SaveIDs: &enabled, - SaveDPI: &enabled, - SaveRogue: &enabled, - SaveSites: &enabled, - URL: srv.Server.URL, + SaveAnomal: &enabled, + SaveAlarms: &enabled, + SaveEvents: &enabled, + SaveIDs: &enabled, + SaveDPI: &enabled, + SaveRogue: &enabled, + SaveSites: &enabled, + SaveTraffic: &disabled, + URL: srv.Server.URL, } in := &inputunifi.InputUnifi{ Logger: testCollector.Logger,