public_git
/
unpoller_unpoller
mirror of https://github.com/unpoller/unpoller.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge 9b62519bfe into bb4a4c823f

This commit is contained in:
aharper343 2025-12-24 05:30:58 +00:00 committed by GitHub
parent bb4a4c823f 9b62519bfe
commit cca686364b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 243 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -1,4 +1,4 @@
FROM busybox:latest as builder
FROM busybox:latest AS builder
# we have to do this hop because distroless is bare without common shell commands
RUN mkdir -p /etc/unpoller

1
examples/up.conf.example
View File

@ -242,6 +242,7 @@
# save_alarms = false
# save_anomalies = false
# save_dpi = false
# save_traffic = false
# save_rogue = false
# verify_ssl = false
# ssl_cert_paths = []

1
go.mod
View File

@ -4,6 +4,7 @@ go 1.25.5
require (
github.com/DataDog/datadog-go/v5 v5.8.2
github.com/flaticols/countrycodes v0.0.2
github.com/gorilla/mux v1.8.1
github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c
github.com/pkg/errors v0.9.1

2
go.sum
View File

@ -19,6 +19,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/flaticols/countrycodes v0.0.2 h1:vedxSqHwG3r7lwUK2bfGFWkVcFv7QuSCKFMkywI/rIE=
github.com/flaticols/countrycodes v0.0.2/go.mod h1:HCwEez5Z+nf062EOWMPqEh1uLb5QdZSTQmTrq4avBOA=
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=

124
pkg/inputunifi/collector.go
View File

@ -11,6 +11,11 @@ import (
"github.com/unpoller/unpoller/pkg/poller"
)
const (
history_seconds = 86400
poll_duration = time.Second * history_seconds
)
var ErrScrapeFilterMatchFailed = fmt.Errorf("scrape filter match failed, and filter is not http URL")
func (u *InputUnifi) isNill(c *Controller) bool {
@ -99,40 +104,153 @@ func (u *InputUnifi) pollController(c *Controller) (*poller.Metrics, error) {
m := &Metrics{TS: time.Now(), Sites: sites}
defer updateWeb(c, m)
// FIXME needs to be last poll time maybe
st := m.TS.Add(-1 * poll_duration)
tp := unifi.EpochMillisTimePeriod{StartEpochMillis: st.UnixMilli(), EndEpochMillis: m.TS.UnixMilli()}
if c.SaveRogue != nil && *c.SaveRogue {
if m.RogueAPs, err = c.Unifi.GetRogueAPs(sites); err != nil {
return nil, fmt.Errorf("unifi.GetRogueAPs(%s): %w", c.URL, err)
}
u.LogDebugf("Found %d RogueAPs entries", len(m.RogueAPs))
}
if c.SaveDPI != nil && *c.SaveDPI {
if m.SitesDPI, err = c.Unifi.GetSiteDPI(sites); err != nil {
return nil, fmt.Errorf("unifi.GetSiteDPI(%s): %w", c.URL, err)
}
u.LogDebugf("Found %d SitesDPI entries", len(m.SitesDPI))
if m.ClientsDPI, err = c.Unifi.GetClientsDPI(sites); err != nil {
return nil, fmt.Errorf("unifi.GetClientsDPI(%s): %w", c.URL, err)
}
u.LogDebugf("Found %d ClientsDPI entries", len(m.ClientsDPI))
}
if c.SaveTraffic != nil && *c.SaveTraffic {
if m.CountryTraffic, err = c.Unifi.GetCountryTraffic(sites, &tp); err != nil {
return nil, fmt.Errorf("unifi.GetCountryTraffic(%s): %w", c.URL, err)
}
u.LogDebugf("Found %d CountryTraffic entries", len(m.CountryTraffic))
}
if c.SaveTraffic != nil && *c.SaveTraffic && c.SaveDPI != nil && *c.SaveDPI {
clientUsageByApp, err := c.Unifi.GetClientTraffic(sites, &tp, true)
if err != nil {
return nil, fmt.Errorf("unifi.GetClientTraffic(%s): %w", c.URL, err)
}
u.LogDebugf("Found %d ClientUsageByApp entries", len(clientUsageByApp))
b4 := len(m.ClientsDPI)
u.convertToClientDPI(clientUsageByApp, m)
u.LogDebugf("Added %d ClientDPI entries for a total of %d", len(m.ClientsDPI)-b4, len(m.ClientsDPI))
}
// Get all the points.
if m.Clients, err = c.Unifi.GetClients(sites); err != nil {
return nil, fmt.Errorf("unifi.GetClients(%s): %w", c.URL, err)
}
u.LogDebugf("Found %d Clients entries", len(m.Clients))
if m.Devices, err = c.Unifi.GetDevices(sites); err != nil {
return nil, fmt.Errorf("unifi.GetDevices(%s): %w", c.URL, err)
}
u.LogDebugf("Found %d UBB, %d UXG, %d PDU, %d UCI, %d UAP %d USG %d USW %d UDM devices",
len(m.Devices.UBBs), len(m.Devices.UXGs),
len(m.Devices.PDUs), len(m.Devices.UCIs),
len(m.Devices.UAPs), len(m.Devices.USGs),
len(m.Devices.USWs), len(m.Devices.UDMs))
// Get speed test results for all WANs
if m.SpeedTests, err = c.Unifi.GetSpeedTests(sites, 86400); err != nil {
if m.SpeedTests, err = c.Unifi.GetSpeedTests(sites, history_seconds); err != nil {
// Don't fail collection if speed tests fail - older controllers may not have this endpoint
u.LogDebugf("unifi.GetSpeedTests(%s): %v (continuing)", c.URL, err)
} else {
u.LogDebugf("Found %d SpeedTests entries", len(m.SpeedTests))
}
return u.augmentMetrics(c, m), nil
}
// FIXME this would be better implemented on FlexInt itself
func (u *InputUnifi) intToFlexInt(i int) unifi.FlexInt {
return unifi.FlexInt{
Val: float64(i),
Txt: fmt.Sprintf("%d", i),
}
}
// FIXME this would be better implemented on FlexInt itself
func (u *InputUnifi) int64ToFlexInt(i int64) unifi.FlexInt {
return unifi.FlexInt{
Val: float64(i),
Txt: fmt.Sprintf("%d", i),
}
}
func (u *InputUnifi) convertToClientDPI(clientUsageByApp []*unifi.ClientUsageByApp, metrics *Metrics) {
for _, client := range clientUsageByApp {
byApp := make([]unifi.DPIData, 0)
byCat := make([]unifi.DPIData, 0)
type catCount struct {
BytesReceived int64
BytesTransmitted int64
}
byCatMap := make(map[int]catCount)
dpiClients := make([]*unifi.DPIClient, 0)
// TODO create cat table
for _, app := range client.UsageByApp {
dpiData := unifi.DPIData{
App: u.intToFlexInt(app.Application),
Cat: u.intToFlexInt(app.Category),
Clients: dpiClients,
KnownClients: u.intToFlexInt(0),
RxBytes: u.int64ToFlexInt(app.BytesReceived),
RxPackets: u.int64ToFlexInt(0), // We don't have packets from Unifi Controller
TxBytes: u.int64ToFlexInt(app.BytesTransmitted),
TxPackets: u.int64ToFlexInt(0), // We don't have packets from Unifi Controller
}
cat, ok := byCatMap[app.Category]
if ok {
cat.BytesReceived += app.BytesReceived
cat.BytesTransmitted += app.BytesTransmitted
} else {
cat = catCount{
BytesReceived: app.BytesReceived,
BytesTransmitted: app.BytesTransmitted,
}
byCatMap[app.Category] = cat
}
byApp = append(byApp, dpiData)
}
if len(byApp) <= 1 {
byCat = byApp
} else {
for category, cat := range byCatMap {
dpiData := unifi.DPIData{
App: u.intToFlexInt(16777215), // Unknown
Cat: u.intToFlexInt(category),
Clients: dpiClients,
KnownClients: u.intToFlexInt(0),
RxBytes: u.int64ToFlexInt(cat.BytesReceived),
RxPackets: u.int64ToFlexInt(0), // We don't have packets from Unifi Controller
TxBytes: u.int64ToFlexInt(cat.BytesTransmitted),
TxPackets: u.int64ToFlexInt(0), // We don't have packets from Unifi Controller
}
byCat = append(byCat, dpiData)
}
}
dpiTable := unifi.DPITable{
ByApp: byApp,
ByCat: byCat,
MAC: client.Client.Mac,
Name: client.Client.Name,
SiteName: client.TrafficSite.SiteName,
SourceName: client.TrafficSite.SourceName,
}
metrics.ClientsDPI = append(metrics.ClientsDPI, &dpiTable)
}
}
// augmentMetrics is our middleware layer between collecting metrics and writing them.
// This is where we can manipuate the returned data or make arbitrary decisions.
// This method currently adds parent device names to client metrics and hashes PII.
@ -191,6 +309,10 @@ func (u *InputUnifi) augmentMetrics(c *Controller, metrics *Metrics) *poller.Met
m.SpeedTests = append(m.SpeedTests, speedTest)
}
for _, traffic := range metrics.CountryTraffic {
m.CountryTraffic = append(m.CountryTraffic, traffic)
}
return m
}

26
pkg/inputunifi/input.go
View File

@ -45,6 +45,7 @@ type Controller struct {
ProtectThumbnails *bool `json:"protect_thumbnails" toml:"protect_thumbnails" xml:"protect_thumbnails" yaml:"protect_thumbnails"`
SaveIDs *bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"`
SaveDPI *bool `json:"save_dpi" toml:"save_dpi" xml:"save_dpi" yaml:"save_dpi"`
SaveTraffic *bool `json:"save_traffic" toml:"save_traffic" xml:"save_traffic" yaml:"save_traffic"`
SaveRogue *bool `json:"save_rogue" toml:"save_rogue" xml:"save_rogue" yaml:"save_rogue"`
HashPII *bool `json:"hash_pii" toml:"hash_pii" xml:"hash_pii" yaml:"hash_pii"`
DropPII *bool `json:"drop_pii" toml:"drop_pii" xml:"drop_pii" yaml:"drop_pii"`
@ -72,14 +73,15 @@ type Config struct {
// Metrics is simply a useful container for everything.
type Metrics struct {
TS time.Time
Sites []*unifi.Site
Clients []*unifi.Client
SitesDPI []*unifi.DPITable
ClientsDPI []*unifi.DPITable
RogueAPs []*unifi.RogueAP
SpeedTests []*unifi.SpeedTestResult
Devices *unifi.Devices
TS time.Time
Sites []*unifi.Site
Clients []*unifi.Client
SitesDPI []*unifi.DPITable
ClientsDPI []*unifi.DPITable
CountryTraffic []*unifi.UsageByCountry
RogueAPs []*unifi.RogueAP
SpeedTests []*unifi.SpeedTestResult
Devices *unifi.Devices
}
func init() { // nolint: gochecknoinits
@ -239,6 +241,10 @@ func (u *InputUnifi) setDefaults(c *Controller) { //nolint:cyclop
c.SaveDPI = &f
}
if c.SaveTraffic == nil {
c.SaveTraffic = &f
}
if c.SaveRogue == nil {
c.SaveRogue = &f
}
@ -271,6 +277,10 @@ func (u *InputUnifi) setDefaults(c *Controller) { //nolint:cyclop
c.SaveAnomal = &f
}
if c.SaveTraffic == nil {
c.SaveTraffic = &f
}
if c.URL == "" {
c.URL = defaultURL
}

1
pkg/inputunifi/interface.go
View File

@ -130,6 +130,7 @@ func (u *InputUnifi) logController(c *Controller) {
u.Logf(" => Save Events %v / Save Syslog %v / Save IDs %v (logs)", *c.SaveEvents, *c.SaveSyslog, *c.SaveIDs)
u.Logf(" => Save Alarms %v / Anomalies %v / Protect Logs %v (thumbnails: %v)", *c.SaveAlarms, *c.SaveAnomal, *c.SaveProtectLogs, *c.ProtectThumbnails)
u.Logf(" => Save Rogue APs: %v", *c.SaveRogue)
u.Logf(" => Save Traffic %v", *c.SaveTraffic)
}
// Events allows you to pull only events (and IDs) from the UniFi Controller.

1
pkg/inputunifi/updateweb.go
View File

@ -51,6 +51,7 @@ func formatControllers(controllers []*Controller) []*Controller {
HashPII: c.HashPII,
DropPII: c.DropPII,
SaveSites: c.SaveSites,
SaveTraffic: c.SaveTraffic,
User: c.User,
Pass: strconv.FormatBool(c.Pass != ""),
APIKey: strconv.FormatBool(c.APIKey != ""),

21
pkg/poller/config.go
View File

@ -33,10 +33,10 @@ func DefaultConfFile() string {
case "netbsd":
fallthrough
case "openbsd":
return "/etc/unpoller/up.conf,/etc/unifi-poller/up.conf,/usr/local/etc/unifi-poller/up.conf"
return "/etc/unpoller/up.conf,/etc/unifi-poller/up.conf,/usr/local/etc/unifi-poller/up.conf,up.conf"
default:
// linux and everything else
return "/etc/unpoller/up.conf,/config/unifi-poller.conf,/etc/unifi-poller/up.conf"
return "/etc/unpoller/up.conf,/config/unifi-poller.conf,/etc/unifi-poller/up.conf,up.conf"
}
}
@ -79,14 +79,15 @@ type Flags struct {
// Metrics is a type shared by the exporting and reporting packages.
type Metrics struct {
TS time.Time
Sites []any
Clients []any
SitesDPI []any
ClientsDPI []any
Devices []any
RogueAPs []any
SpeedTests []any
TS time.Time
Sites []any
Clients []any
SitesDPI []any
ClientsDPI []any
Devices []any
RogueAPs []any
SpeedTests []any
CountryTraffic []any
}
// Events defines the type for log entries.

2
pkg/poller/inputs.go
View File

@ -268,7 +268,7 @@ func AppendMetrics(existing *Metrics, m *Metrics) *Metrics {
existing.RogueAPs = append(existing.RogueAPs, m.RogueAPs...)
existing.Clients = append(existing.Clients, m.Clients...)
existing.Devices = append(existing.Devices, m.Devices...)
existing.CountryTraffic = append(existing.CountryTraffic, m.CountryTraffic...)
return existing
}

28
pkg/promunifi/collector.go
View File

@ -35,16 +35,17 @@ const (
var ErrMetricFetchFailed = fmt.Errorf("metric fetch failed")
type promUnifi struct {
*Config `json:"prometheus" toml:"prometheus" xml:"prometheus" yaml:"prometheus"`
Client *uclient
Device *unifiDevice
UAP *uap
USG *usg
USW *usw
PDU *pdu
Site *site
RogueAP *rogueap
SpeedTest *speedtest
*Config `json:"prometheus" toml:"prometheus" xml:"prometheus" yaml:"prometheus"`
Client *uclient
Device *unifiDevice
UAP *uap
USG *usg
USW *usw
PDU *pdu
Site *site
RogueAP *rogueap
SpeedTest *speedtest
CountryTraffic *ucountrytraffic
// This interface is passed to the Collect() method. The Collect method uses
// this interface to retrieve the latest UniFi measurements and export them.
Collector poller.Collect
@ -203,6 +204,7 @@ func (u *promUnifi) Run(c poller.Collect) error {
u.Site = descSite(u.Namespace + "_site_")
u.RogueAP = descRogueAP(u.Namespace + "_rogueap_")
u.SpeedTest = descSpeedTest(u.Namespace + "_speedtest_")
u.CountryTraffic = descCountryTraffic(u.Namespace + "_countrytraffic_")
mux := http.NewServeMux()
promver.Version = version.Version
@ -405,6 +407,10 @@ func (u *promUnifi) loopExports(r report) {
u.exportClientDPI(r, c, appTotal, catTotal)
}
for _, ct := range m.CountryTraffic {
u.exportCountryTraffic(r, ct)
}
u.exportClientDPItotals(r, appTotal, catTotal)
}
@ -443,6 +449,8 @@ func (u *promUnifi) switchExport(r report, v any) {
u.exportClient(r, v)
case *unifi.SpeedTestResult:
u.exportSpeedTest(r, v)
case *unifi.UsageByCountry:
u.exportCountryTraffic(r, v)
default:
u.LogErrorf("invalid type: %T", v)
}

53
pkg/promunifi/countries.go Normal file
View File

@ -0,0 +1,53 @@
package promunifi
import (
"github.com/flaticols/countrycodes"
"github.com/prometheus/client_golang/prometheus"
"github.com/unpoller/unifi/v5"
)
type ucountrytraffic struct {
RxBytes *prometheus.Desc
TxBytes *prometheus.Desc
}
func descCountryTraffic(ns string) *ucountrytraffic {
labels := []string{
"code",
"name",
"region",
"sub_region",
"site_name",
"source",
}
return &ucountrytraffic{
RxBytes: prometheus.NewDesc(ns+"receive_bytes_total", "Country Receive Bytes", labels, nil),
TxBytes: prometheus.NewDesc(ns+"transmit_bytes_total", "Country Transmit Bytes", labels, nil),
}
}
func (u *promUnifi) exportCountryTraffic(r report, v any) {
s, ok := v.(*unifi.UsageByCountry)
if !ok {
u.LogErrorf("invalid type given to CountryTraffic: %T", v)
return
}
country, ok := countrycodes.GetByAlpha2(s.Country)
name := "Unknown"
region := "Unknown"
subRegion := "Unknown"
if ok {
name = country.Name
region = country.Region
subRegion = country.SubRegion
}
if s.Country == "GB" || s.Country == "UK" {
name = "United Kingdom" // Because the name is so long otherwise
}
labels := []string{s.Country, name, region, subRegion, s.TrafficSite.SiteName, s.TrafficSite.SourceName}
r.send([]*metric{
{u.CountryTraffic.RxBytes, counter, s.BytesReceived, labels},
{u.CountryTraffic.TxBytes, counter, s.BytesTransmitted, labels},
})
}

4
pkg/promunifi/report.go
View File

@ -50,10 +50,10 @@ func (r *Report) report(c poller.Logger, descs map[*prometheus.Desc]bool) {
m := r.Metrics
c.Logf("UniFi Measurements Exported. Site: %d, Client: %d, "+
"UAP: %d, USG/UDM: %d, USW: %d, DPI Site/Client: %d/%d, Desc: %d, "+
"UAP: %d, USG/UDM: %d, USW: %d, DPI Site/Client: %d/%d, Countries: %d, Desc: %d, "+
"Metric: %d, Bytes: %d, Err: %d, 0s: %d, Req/Total: %v / %v",
len(m.Sites), len(m.Clients), r.UAP, r.UDM+r.USG+r.UXG, r.USW, len(m.SitesDPI),
len(m.ClientsDPI), len(descs), r.Total, r.Bytes, r.Errors, r.Zeros,
len(m.ClientsDPI), len(m.CountryTraffic), len(descs), r.Total, r.Bytes, r.Errors, r.Zeros,
r.Fetch.Round(time.Millisecond/oneDecimalPoint),
r.Elapsed.Round(time.Millisecond/oneDecimalPoint))
}

18
pkg/unittest/dep.go
View File

@ -20,15 +20,17 @@ func NewTestSetup(t *testing.T) *TestRig {
testCollector := poller.NewTestCollector(t)
enabled := true
disabled := false
controller := inputunifi.Controller{
SaveAnomal: &enabled,
SaveAlarms: &enabled,
SaveEvents: &enabled,
SaveIDs: &enabled,
SaveDPI: &enabled,
SaveRogue: &enabled,
SaveSites: &enabled,
URL: srv.Server.URL,
SaveAnomal: &enabled,
SaveAlarms: &enabled,
SaveEvents: &enabled,
SaveIDs: &enabled,
SaveDPI: &enabled,
SaveRogue: &enabled,
SaveSites: &enabled,
SaveTraffic: &disabled,
URL: srv.Server.URL,
}
in := &inputunifi.InputUnifi{
Logger: testCollector.Logger,