Merge pull request #1 from unifi-poller/dn2_dpi_totals

Add DPI totals

integrations/promunifi/.travis.yml

@@ -4,8 +4,6 @@ go:
 before_install:
   # download super-linter: golangci-lint
 - curl -sL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin latest
-install:
-- go mod download
 script:
-- golangci-lint run --enable-all -D gochecknoinits -D funlen -e gomnd
 - go test ./...
+- golangci-lint run --enable-all -D gochecknoinits -D funlen -D gomnd

integrations/promunifi/clients.go

@@ -72,17 +72,19 @@ func descClient(ns string) *uclient {
 	}
 }
 
-func (u *promUnifi) exportClientDPI(r report, s *unifi.DPITable) {
+func (u *promUnifi) exportClientDPI(r report, s *unifi.DPITable, appTotal, catTotal totalsDPImap) {
 	for _, dpi := range s.ByApp {
 		labelDPI := []string{s.Name, s.MAC, s.SiteName, s.SourceName,
 			unifi.DPICats.Get(dpi.Cat), unifi.DPIApps.GetApp(dpi.Cat, dpi.App)}
 
+		fillDPIMapTotals(appTotal, labelDPI[5], s.SourceName, s.SiteName, dpi)
+		fillDPIMapTotals(catTotal, labelDPI[4], s.SourceName, s.SiteName, dpi)
 		// log.Println(labelDPI, dpi.Cat, dpi.App, dpi.TxBytes, dpi.RxBytes, dpi.TxPackets, dpi.RxPackets)
 		r.send([]*metric{
-			{u.Client.DPITxPackets, gauge, dpi.TxPackets, labelDPI},
-			{u.Client.DPIRxPackets, gauge, dpi.RxPackets, labelDPI},
-			{u.Client.DPITxBytes, gauge, dpi.TxBytes, labelDPI},
-			{u.Client.DPIRxBytes, gauge, dpi.RxBytes, labelDPI},
+			{u.Client.DPITxPackets, counter, dpi.TxPackets, labelDPI},
+			{u.Client.DPIRxPackets, counter, dpi.RxPackets, labelDPI},
+			{u.Client.DPITxBytes, counter, dpi.TxBytes, labelDPI},
+			{u.Client.DPIRxBytes, counter, dpi.RxBytes, labelDPI},
 		})
 	}
 }
@@ -134,3 +136,84 @@ func (u *promUnifi) exportClient(r report, c *unifi.Client) {
 
 	r.send([]*metric{{u.Client.Uptime, gauge, c.Uptime, labelW}})
 }
+
+// totalsDPImap: controller, site, name (app/cat name), dpi
+type totalsDPImap map[string]map[string]map[string]unifi.DPIData
+
+// fillDPIMapTotals fills in totals for categories and applications. maybe clients too.
+// This allows less processing in InfluxDB to produce total transfer data per cat or app.
+func fillDPIMapTotals(m totalsDPImap, name, controller, site string, dpi unifi.DPIData) {
+	if _, ok := m[controller]; !ok {
+		m[controller] = make(map[string]map[string]unifi.DPIData)
+	}
+
+	if _, ok := m[controller][site]; !ok {
+		m[controller][site] = make(map[string]unifi.DPIData)
+	}
+
+	if _, ok := m[controller][site][name]; !ok {
+		m[controller][site][name] = dpi
+		return
+	}
+
+	oldDPI := m[controller][site][name]
+	oldDPI.TxPackets += dpi.TxPackets
+	oldDPI.RxPackets += dpi.RxPackets
+	oldDPI.TxBytes += dpi.TxBytes
+	oldDPI.RxBytes += dpi.RxBytes
+	m[controller][site][name] = oldDPI
+}
+
+func (u *promUnifi) exportClientDPItotals(r report, appTotal, catTotal totalsDPImap) {
+	type all []struct {
+		kind string
+		val  totalsDPImap
+	}
+	// This produces 7000+ metrics per site. Disabled for now.
+	if appTotal != nil {
+		appTotal = nil
+	}
+	// This can allow us to aggregate other data types later, like `name` or `mac`, or anything else unifi adds.
+	a := all{
+
+		{
+			kind: "application",
+			val:  appTotal,
+		},
+
+		{
+			kind: "category",
+			val:  catTotal,
+		},
+	}
+
+	for _, k := range a {
+		for controller, s := range k.val {
+			for site, c := range s {
+				for name, m := range c {
+					labelDPI := []string{"TOTAL", "TOTAL", site, controller, "TOTAL", "TOTAL"}
+
+					switch k.kind {
+					case "application":
+						labelDPI[5] = name
+					case "category":
+						labelDPI[4] = name
+					case "name":
+						labelDPI[0] = name
+					case "mac":
+						labelDPI[1] = name
+					}
+
+					m := []*metric{
+						{u.Client.DPITxPackets, counter, m.TxPackets, labelDPI},
+						{u.Client.DPIRxPackets, counter, m.RxPackets, labelDPI},
+						{u.Client.DPITxBytes, counter, m.TxBytes, labelDPI},
+						{u.Client.DPIRxBytes, counter, m.RxBytes, labelDPI},
+					}
+
+					r.send(m)
+				}
+			}
+		}
+	}
+}

integrations/promunifi/collector.go

@@ -307,9 +307,14 @@ func (u *promUnifi) loopExports(r report) {
 	go func() {
 		defer r.done()
 
+		appTotal := make(totalsDPImap)
+		catTotal := make(totalsDPImap)
+
 		for _, c := range m.ClientsDPI {
-			u.exportClientDPI(r, c)
+			u.exportClientDPI(r, c, appTotal, catTotal)
 		}
+
+		u.exportClientDPItotals(r, appTotal, catTotal)
 	}()
 
 	go func() {

integrations/promunifi/usg.go

@@ -61,9 +61,9 @@ func descUSG(ns string) *usg {
 		LanRxDropped:   prometheus.NewDesc(ns+"lan_receive_dropped_total", "LAN Receive Dropped Total", labels, nil),
 		LanTxPackets:   prometheus.NewDesc(ns+"lan_transmit_packets_total", "LAN Transmit Packets Total", labels, nil),
 		LanTxBytes:     prometheus.NewDesc(ns+"lan_transmit_bytes_total", "LAN Transmit Bytes Total", labels, nil),
-		Latency:        prometheus.NewDesc(ns+"speedtest_latency_seconds", "Speedtest Latency", labels, nil),
 		UplinkLatency:  prometheus.NewDesc(ns+"uplink_latency_seconds", "Uplink Latency", labels, nil),
 		UplinkSpeed:    prometheus.NewDesc(ns+"uplink_speed_mbps", "Uplink Speed", labels, nil),
+		Latency:        prometheus.NewDesc(ns+"speedtest_latency_seconds", "Speedtest Latency", labels, nil),
 		Runtime:        prometheus.NewDesc(ns+"speedtest_runtime_seconds", "Speedtest Run Time", labels, nil),
 		Rundate:        prometheus.NewDesc(ns+"speedtest_rundate_seconds", "Speedtest Run Date", labels, nil),
 		XputDownload:   prometheus.NewDesc(ns+"speedtest_download", "Speedtest Download Rate", labels, nil),