From c39da373857a2d621a0139741e62f01385b2e9bf Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sat, 20 Jun 2020 03:45:00 -0700 Subject: [PATCH] Add event output --- integrations/influxunifi/events.go | 203 +++++++++++++++++++++++++++ integrations/influxunifi/go.mod | 4 +- integrations/influxunifi/go.sum | 6 + integrations/influxunifi/ids.go | 42 ------ integrations/influxunifi/influxdb.go | 33 +++-- 5 files changed, 229 insertions(+), 59 deletions(-) create mode 100644 integrations/influxunifi/events.go delete mode 100644 integrations/influxunifi/ids.go diff --git a/integrations/influxunifi/events.go b/integrations/influxunifi/events.go new file mode 100644 index 00000000..d31f4f20 --- /dev/null +++ b/integrations/influxunifi/events.go @@ -0,0 +1,203 @@ +package influxunifi + +import ( + "github.com/unifi-poller/unifi" +) + +// batchIDS generates intrusion detection datapoints for InfluxDB. +func (u *InfluxUnifi) batchIDS(r report, i *unifi.IDS) { // nolint: funlen + fields := map[string]interface{}{ + /* + "site_id": i.SiteID, + "dstipASN": i.DstIPASN, + "dstipCountry": i.DstIPCountry, + "flow_id": i.FlowID, + "inner_alert_action": i.InnerAlertAction, + "inner_alert_category": i.InnerAlertCategory, + "inner_alert_signature": i.InnerAlertSignature, + "inner_alert_rev": i.InnerAlertRev, + "inner_alert_severity": i.InnerAlertSeverity, + "inner_alert_gid": i.InnerAlertGID, + "inner_alert_signature_id": i.InnerAlertSignatureID, + "srcipASN": i.SrcIPASN, + "srcipCountry": i.SrcIPCountry, + "unique_alertid": i.UniqueAlertID, + "usgipASN": i.UsgIPASN, + "usgipCountry": i.UsgIPCountry, + */ + "dest_port": i.DestPort, + "src_port": i.SrcPort, + "app_proto": i.AppProto, + "catname": i.Catname, + "dest_ip": i.DestIP, + "dst_mac": i.DstMAC, + "host": i.Host, + "key": i.Key, + "msg": i.Msg, + "proto": i.Proto, + "src_ip": i.SrcIP, + "src_mac": i.SrcMAC, + "usgip": i.USGIP, + "dstipGeo_asn": i.DestIPGeo.Asn, + "dstipGeo_latitude": i.DestIPGeo.Latitude, + "dstipGeo_longitude": i.DestIPGeo.Longitude, + "dstipGeo_city": i.DestIPGeo.City, + "dstipGeo_continent_code": i.DestIPGeo.ContinentCode, + "dstipGeo_country_code": i.DestIPGeo.CountryCode, + "dstipGeo_country_name": i.DestIPGeo.CountryName, + "dstipGeo_organization": i.DestIPGeo.Organization, + "srcipGeo_asn": i.SourceIPGeo.Asn, + "srcipGeo_latitude": i.SourceIPGeo.Latitude, + "srcipGeo_longitude": i.SourceIPGeo.Longitude, + "srcipGeo_city": i.SourceIPGeo.City, + "srcipGeo_continent_code": i.SourceIPGeo.ContinentCode, + "srcipGeo_country_code": i.SourceIPGeo.CountryCode, + "srcipGeo_country_name": i.SourceIPGeo.CountryName, + "srcipGeo_organization": i.SourceIPGeo.Organization, + } + + r.send(&metric{ + Table: "unifi_ids", + TS: i.Datetime, + Fields: cleanFields(fields), + Tags: cleanTags(map[string]string{ + "site_name": i.SiteName, + "source": i.SourceName, + "in_iface": i.InIface, + "event_type": i.EventType, + "subsystem": i.Subsystem, + "archived": i.Archived.Txt, + }), + }) +} + +// batchEvents generates events from UniFi for InfluxDB. +func (u *InfluxUnifi) batchEvent(r report, i *unifi.Event) { // nolint: funlen + fields := map[string]interface{}{ + /* + "site_id": i.SiteID, + "flow_id": i.FlowID, + "inner_alert_signature": i.InnerAlertSignature, + "inner_alert_gid": i.InnerAlertGID, + "inner_alert_rev": i.InnerAlertRev, + "inner_alert_severity": i.InnerAlertSeverity, + "inner_alert_signature_id": i.InnerAlertSignatureID, + "unique_alertid": i.UniqueAlertID, + "usgipASN": i.UsgIPASN, + "usgipCountry": i.UsgIPCountry, + "srcipASN": i.SrcIPASN, + "srcipCountry": i.SrcIPCountry, + */ + "dest_port": i.DestPort, + "src_port": i.SrcPort, + "bytes": i.Bytes, + "duration": i.Duration, + "admin": i.Admin, + "ap": i.Ap, + "ap_from": i.ApFrom, + "ap_name": i.ApName, + "ap_to": i.ApTo, + "app_proto": i.AppProto, + "catname": i.Catname, + "channel": i.Channel, + "channel_from": i.ChannelFrom, + "channel_to": i.ChannelTo, + "dest_ip": i.DestIP, + "dst_mac": i.DstMAC, + "guest": i.Guest, + "gw": i.Gw, + "gw_name": i.GwName, + "host": i.Host, + "hostname": i.Hostname, + "ip": i.IP, + "inner_alert_action": i.InnerAlertAction, + "inner_alert_category": i.InnerAlertCategory, + "key": i.Key, + "msg": i.Msg, + "network": i.Network, + "proto": i.Proto, + "radio": i.Radio, + "radio_from": i.RadioFrom, + "radio_to": i.RadioTo, + "src_ip": i.SrcIP, + "src_mac": i.SrcMAC, + "ssid": i.SSID, + "sw": i.Sw, + "sw_name": i.SwName, + "user": i.User, + "usgip": i.USGIP, + "dstipGeo_asn": i.DestIPGeo.Asn, + "dstipGeo_latitude": i.DestIPGeo.Latitude, + "dstipGeo_longitude": i.DestIPGeo.Longitude, + "dstipGeo_city": i.DestIPGeo.City, + "dstipGeo_continent_code": i.DestIPGeo.ContinentCode, + "dstipGeo_country_code": i.DestIPGeo.CountryCode, + "dstipGeo_country_name": i.DestIPGeo.CountryName, + "dstipGeo_organization": i.DestIPGeo.Organization, + "srcipGeo_asn": i.SourceIPGeo.Asn, + "srcipGeo_latitude": i.SourceIPGeo.Latitude, + "srcipGeo_longitude": i.SourceIPGeo.Longitude, + "srcipGeo_city": i.SourceIPGeo.City, + "srcipGeo_continent_code": i.SourceIPGeo.ContinentCode, + "srcipGeo_country_code": i.SourceIPGeo.CountryCode, + "srcipGeo_country_name": i.SourceIPGeo.CountryName, + "srcipGeo_organization": i.SourceIPGeo.Organization, + } + + r.send(&metric{ + TS: i.Datetime, + Table: "unifi_events", + Fields: cleanFields(fields), + Tags: cleanTags(map[string]string{ + "site_name": i.SiteName, + "source": i.SourceName, + "in_iface": i.InIface, + "event_type": i.EventType, + "subsystem": i.Subsystem, + "is_admin": i.IsAdmin.Txt, + "gw_name": i.GwName, // also field + "ap_name": i.ApName, // also field + "sw_name": i.SwName, // also field + "ssid": i.SSID, // also field + }), + }) +} + +// cleanTags removes any tag that is empty. +func cleanTags(tags map[string]string) map[string]string { + for i := range tags { + if tags[i] == "" { + delete(tags, i) + } + } + + return tags +} + +// cleanFields removes any field with a default (or empty) value. +func cleanFields(fields map[string]interface{}) map[string]interface{} { + for s := range fields { + switch v := fields[s].(type) { + case nil: + delete(fields, s) + case int, int64, float64: + if v == 0 { + delete(fields, s) + } + case unifi.FlexBool: + if v.Txt == "" { + delete(fields, s) + } + case unifi.FlexInt: + if v.Txt == "" { + delete(fields, s) + } + case string: + if v == "" { + delete(fields, s) + } + } + } + + return fields +} diff --git a/integrations/influxunifi/go.mod b/integrations/influxunifi/go.mod index 025d2aac..a08c177e 100644 --- a/integrations/influxunifi/go.mod +++ b/integrations/influxunifi/go.mod @@ -6,7 +6,7 @@ require ( github.com/influxdata/influxdb1-client v0.0.0-20200515024757-02f0bf5dbca3 github.com/pkg/errors v0.9.1 github.com/prometheus/procfs v0.1.2 // indirect - github.com/unifi-poller/poller v0.0.6 - github.com/unifi-poller/unifi v0.0.5 + github.com/unifi-poller/poller v0.0.8-0.20200619104117-a5e263a36ac9 + github.com/unifi-poller/unifi v0.0.5-0.20200620103801-b927287ea1cd golift.io/cnfg v0.0.5 ) diff --git a/integrations/influxunifi/go.sum b/integrations/influxunifi/go.sum index f3d9eaf4..986d7452 100644 --- a/integrations/influxunifi/go.sum +++ b/integrations/influxunifi/go.sum @@ -122,6 +122,8 @@ github.com/unifi-poller/poller v0.0.5 h1:qnofARTx0JveNc9PqGJmNUs7xsjwFqCWpS8pwDS github.com/unifi-poller/poller v0.0.5/go.mod h1:45TyAHk+xYF4KoFKaaZyjMmSdhbq2I1pLniYWfOXdHs= github.com/unifi-poller/poller v0.0.6 h1:KhOWUeYI029Nn/4NOIk/yblQ3tEd9QhS+u/8/S9ZoDs= github.com/unifi-poller/poller v0.0.6/go.mod h1:RkRJ4pAc2dAN8Xu9+VOumeE3BdN5QDQ3PC+jBx8hWW0= +github.com/unifi-poller/poller v0.0.8-0.20200619104117-a5e263a36ac9 h1:BYNOSvttR91a0F+ttG2ecD3DenVI9p9oi1W7m36zH98= +github.com/unifi-poller/poller v0.0.8-0.20200619104117-a5e263a36ac9/go.mod h1:q2tufLljemUR/blyhjIj+T0sdAOZo0kdlv3h79kEMgU= github.com/unifi-poller/unifi v0.0.1/go.mod h1:DagVD/I+VMnVUHmTT4Fi76lPI+DHbuMwwtMIzanwMxM= github.com/unifi-poller/unifi v0.0.2 h1:FRa6G+VcxOdvy0/u/QLCHGCRYF6EDcxVYij0dDRCKxg= github.com/unifi-poller/unifi v0.0.2/go.mod h1:DagVD/I+VMnVUHmTT4Fi76lPI+DHbuMwwtMIzanwMxM= @@ -135,6 +137,10 @@ github.com/unifi-poller/unifi v0.0.5-0.20200614034449-3f18c1be228b h1:bpQVeurqpl github.com/unifi-poller/unifi v0.0.5-0.20200614034449-3f18c1be228b/go.mod h1:L1kMRH2buZhB31vZnRC1im7Tk/4uD3ET4biwl2faYy8= github.com/unifi-poller/unifi v0.0.5-0.20200614034623-f4c1d18157c2 h1:iDOUe8UECHQoOyzWIarbGNVdwEth16iiUsw+1UgQ+sw= github.com/unifi-poller/unifi v0.0.5-0.20200614034623-f4c1d18157c2/go.mod h1:L1kMRH2buZhB31vZnRC1im7Tk/4uD3ET4biwl2faYy8= +github.com/unifi-poller/unifi v0.0.5-0.20200619092006-d24c776a42f5 h1:DI1Dtq+2CY1t0Woesz7CRkbNrUT5GSX0h8mDsP3pgcw= +github.com/unifi-poller/unifi v0.0.5-0.20200619092006-d24c776a42f5/go.mod h1:L1kMRH2buZhB31vZnRC1im7Tk/4uD3ET4biwl2faYy8= +github.com/unifi-poller/unifi v0.0.5-0.20200620103801-b927287ea1cd h1:jrwuiY1AdoPi+b+R8zjt/e8h8ZqULNB9izcyQnf3pSw= +github.com/unifi-poller/unifi v0.0.5-0.20200620103801-b927287ea1cd/go.mod h1:L1kMRH2buZhB31vZnRC1im7Tk/4uD3ET4biwl2faYy8= github.com/unifi-poller/unifi v0.0.5 h1:Izeun32YxcQOeKZUXY0Sy4ltKYFuYxWGcN9JS6xkIJU= github.com/unifi-poller/unifi v0.0.5/go.mod h1:L1kMRH2buZhB31vZnRC1im7Tk/4uD3ET4biwl2faYy8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= diff --git a/integrations/influxunifi/ids.go b/integrations/influxunifi/ids.go deleted file mode 100644 index f5e41d0b..00000000 --- a/integrations/influxunifi/ids.go +++ /dev/null @@ -1,42 +0,0 @@ -package influxunifi - -import ( - "github.com/unifi-poller/unifi" -) - -// batchIDS generates intrusion detection datapoints for InfluxDB. -// These points can be passed directly to influx. -func (u *InfluxUnifi) batchIDS(r report, i *unifi.IDS) { - tags := map[string]string{ - "site_name": i.SiteName, - "source": i.SourceName, - "in_iface": i.InIface, - "event_type": i.EventType, - "proto": i.Proto, - "app_proto": i.AppProto, - "usgip": i.Usgip, - "country_code": i.SrcipGeo.CountryCode, - "country_name": i.SrcipGeo.CountryName, - "region": i.SrcipGeo.Region, - "city": i.SrcipGeo.City, - "postal_code": i.SrcipGeo.PostalCode, - "srcipASN": i.SrcipASN, - "usgipASN": i.UsgipASN, - "alert_category": i.InnerAlertCategory, - "subsystem": i.Subsystem, - "catname": i.Catname, - } - fields := map[string]interface{}{ - "event_type": i.EventType, - "proto": i.Proto, - "app_proto": i.AppProto, - "usgip": i.Usgip, - "country_name": i.SrcipGeo.CountryName, - "city": i.SrcipGeo.City, - "postal_code": i.SrcipGeo.PostalCode, - "srcipASN": i.SrcipASN, - "usgipASN": i.UsgipASN, - } - - r.send(&metric{Table: "intrusion_detect", Tags: tags, Fields: fields}) -} diff --git a/integrations/influxunifi/influxdb.go b/integrations/influxunifi/influxdb.go index 03398931..50f51a4a 100644 --- a/integrations/influxunifi/influxdb.go +++ b/integrations/influxunifi/influxdb.go @@ -4,7 +4,6 @@ package influxunifi import ( "crypto/tls" - "fmt" "io/ioutil" "log" "strings" @@ -53,6 +52,7 @@ type metric struct { Table string Tags map[string]string Fields map[string]interface{} + TS time.Time } func init() { // nolint: gochecknoinits @@ -73,13 +73,10 @@ func (u *InfluxUnifi) PollController() { log.Printf("[INFO] Everything checks out! Poller started, InfluxDB interval: %v", interval) for u.LastCheck = range ticker.C { - metrics, ok, collectErr := u.Collector.Metrics() - if collectErr != nil { - u.Collector.LogErrorf("metric fetch for InfluxDB failed: %v", collectErr) - - if !ok { - continue - } + metrics, err := u.Collector.Metrics(nil) + if err != nil { + u.Collector.LogErrorf("metric fetch for InfluxDB failed: %v", err) + continue } report, err := u.ReportMetrics(metrics) @@ -89,7 +86,6 @@ func (u *InfluxUnifi) PollController() { continue } - report.error(collectErr) u.LogInfluxReport(report) } } @@ -194,7 +190,11 @@ func (u *InfluxUnifi) ReportMetrics(m *poller.Metrics) (*Report, error) { // collect runs in a go routine and batches all the points. func (u *InfluxUnifi) collect(r report, ch chan *metric) { for m := range ch { - pt, err := influx.NewPoint(m.Table, m.Tags, m.Fields, r.metrics().TS) + if m.TS.IsZero() { + m.TS = r.metrics().TS + } + + pt, err := influx.NewPoint(m.Table, m.Tags, m.Fields, m.TS) if err == nil { r.batch(m, pt) } @@ -234,6 +234,10 @@ func (u *InfluxUnifi) loopPoints(r report) { u.batchIDS(r, s) } + for _, s := range m.Events { + u.batchEvent(r, s) + } + u.loopDevicePoints(r) } @@ -264,11 +268,10 @@ func (u *InfluxUnifi) loopDevicePoints(r report) { // LogInfluxReport writes a log message after exporting to influxdb. func (u *InfluxUnifi) LogInfluxReport(r *Report) { m := r.Metrics - idsMsg := fmt.Sprintf("IDS Events: %d, ", len(m.IDSList)) - u.Collector.Logf("UniFi Metrics Recorded. Sites: %d, Clients: %d, "+ - "UAP: %d, USG/UDM: %d, USW: %d, %sPoints: %d, Fields: %d, Errs: %d, Elapsed: %v", - len(m.Sites), len(m.Clients), len(m.UAPs), - len(m.UDMs)+len(m.USGs), len(m.USWs), idsMsg, r.Total, + "UAP: %d, USG/UDM: %d, USW: %d, IDS/Events: %d/%d, Points: %d, "+ + "Fields: %d, Errs: %d, Elapsed: %v", + len(m.Sites), len(m.Clients), len(m.UAPs), len(m.UDMs)+len(m.USGs), + len(m.USWs), len(m.IDSList), len(m.Events), r.Total, r.Fields, len(r.Errors), r.Elapsed.Round(time.Millisecond)) }