From cf680fe64df6781707f6f41d1ee32f145c0fcab2 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Fri, 13 Dec 2019 14:17:28 -0800 Subject: [PATCH 01/41] not sure how far I got.. --- integrations/inputunifi/pkg/poller/config.go | 134 +++++++++++------- .../inputunifi/pkg/promunifi/clients.go | 2 + .../inputunifi/pkg/promunifi/collector.go | 6 + .../inputunifi/pkg/promunifi/report.go | 4 + integrations/inputunifi/pkg/promunifi/site.go | 3 +- integrations/inputunifi/pkg/promunifi/uap.go | 6 +- integrations/inputunifi/pkg/promunifi/udm.go | 4 + integrations/inputunifi/pkg/promunifi/usg.go | 3 + integrations/inputunifi/pkg/promunifi/usw.go | 8 ++ 9 files changed, 114 insertions(+), 56 deletions(-) diff --git a/integrations/inputunifi/pkg/poller/config.go b/integrations/inputunifi/pkg/poller/config.go index 1725b03a..bd811a04 100644 --- a/integrations/inputunifi/pkg/poller/config.go +++ b/integrations/inputunifi/pkg/poller/config.go @@ -14,7 +14,6 @@ import ( "fmt" "io/ioutil" "os" - "path" "reflect" "strconv" "strings" @@ -66,29 +65,44 @@ type Flag struct { *pflag.FlagSet } +// Controller represents the configuration for a UniFi Controller. +// Each polled controller may have its own configuration. +type Controller struct { + Interval Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` + VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` + SaveIDS bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` + ReAuth bool `json:"reauthenticate" toml:"reauthenticate" xml:"reauthenticate" yaml:"reauthenticate"` + SaveSites bool `json:"save_sites,omitempty" toml:"save_sites,omitempty" xml:"save_sites" yaml:"save_sites"` + User string `json:"unifi_user,omitempty" toml:"unifi_user,omitempty" xml:"unifi_user" yaml:"unifi_user"` + Pass string `json:"unifi_pass,omitempty" toml:"unifi_pass,omitempty" xml:"unifi_pass" yaml:"unifi_pass"` + URL string `json:"unifi_url,omitempty" toml:"unifi_url,omitempty" xml:"unifi_url" yaml:"unifi_url"` + Sites []string `json:"sites,omitempty" toml:"sites,omitempty" xml:"sites" yaml:"sites"` +} + // Config represents the data needed to poll a controller and report to influxdb. // This is all of the data stored in the config file. // Any with explicit defaults have omitempty on json and toml tags. type Config struct { - Interval Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` - Debug bool `json:"debug" toml:"debug" xml:"debug" yaml:"debug"` - Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet" yaml:"quiet"` - VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` - SaveIDS bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` - ReAuth bool `json:"reauthenticate" toml:"reauthenticate" xml:"reauthenticate" yaml:"reauthenticate"` - InfxBadSSL bool `json:"influx_insecure_ssl" toml:"influx_insecure_ssl" xml:"influx_insecure_ssl" yaml:"influx_insecure_ssl"` - SaveSites bool `json:"save_sites,omitempty" toml:"save_sites,omitempty" xml:"save_sites" yaml:"save_sites"` - Mode string `json:"mode" toml:"mode" xml:"mode" yaml:"mode"` - HTTPListen string `json:"http_listen" toml:"http_listen" xml:"http_listen" yaml:"http_listen"` - Namespace string `json:"namespace" toml:"namespace" xml:"namespace" yaml:"namespace"` - InfluxURL string `json:"influx_url,omitempty" toml:"influx_url,omitempty" xml:"influx_url" yaml:"influx_url"` - InfluxUser string `json:"influx_user,omitempty" toml:"influx_user,omitempty" xml:"influx_user" yaml:"influx_user"` - InfluxPass string `json:"influx_pass,omitempty" toml:"influx_pass,omitempty" xml:"influx_pass" yaml:"influx_pass"` - InfluxDB string `json:"influx_db,omitempty" toml:"influx_db,omitempty" xml:"influx_db" yaml:"influx_db"` - UnifiUser string `json:"unifi_user,omitempty" toml:"unifi_user,omitempty" xml:"unifi_user" yaml:"unifi_user"` - UnifiPass string `json:"unifi_pass,omitempty" toml:"unifi_pass,omitempty" xml:"unifi_pass" yaml:"unifi_pass"` - UnifiBase string `json:"unifi_url,omitempty" toml:"unifi_url,omitempty" xml:"unifi_url" yaml:"unifi_url"` - Sites []string `json:"sites,omitempty" toml:"sites,omitempty" xml:"sites" yaml:"sites"` + Interval Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` + Debug bool `json:"debug" toml:"debug" xml:"debug" yaml:"debug"` + Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet" yaml:"quiet"` + VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` + SaveIDS bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` + ReAuth bool `json:"reauthenticate" toml:"reauthenticate" xml:"reauthenticate" yaml:"reauthenticate"` + InfxBadSSL bool `json:"influx_insecure_ssl" toml:"influx_insecure_ssl" xml:"influx_insecure_ssl" yaml:"influx_insecure_ssl"` + SaveSites bool `json:"save_sites,omitempty" toml:"save_sites,omitempty" xml:"save_sites" yaml:"save_sites"` + Mode string `json:"mode" toml:"mode" xml:"mode" yaml:"mode"` + HTTPListen string `json:"http_listen" toml:"http_listen" xml:"http_listen" yaml:"http_listen"` + Namespace string `json:"namespace" toml:"namespace" xml:"namespace" yaml:"namespace"` + InfluxURL string `json:"influx_url,omitempty" toml:"influx_url,omitempty" xml:"influx_url" yaml:"influx_url"` + InfluxUser string `json:"influx_user,omitempty" toml:"influx_user,omitempty" xml:"influx_user" yaml:"influx_user"` + InfluxPass string `json:"influx_pass,omitempty" toml:"influx_pass,omitempty" xml:"influx_pass" yaml:"influx_pass"` + InfluxDB string `json:"influx_db,omitempty" toml:"influx_db,omitempty" xml:"influx_db" yaml:"influx_db"` + UnifiUser string `json:"unifi_user,omitempty" toml:"unifi_user,omitempty" xml:"unifi_user" yaml:"unifi_user"` + UnifiPass string `json:"unifi_pass,omitempty" toml:"unifi_pass,omitempty" xml:"unifi_pass" yaml:"unifi_pass"` + UnifiBase string `json:"unifi_url,omitempty" toml:"unifi_url,omitempty" xml:"unifi_url" yaml:"unifi_url"` + Sites []string `json:"sites,omitempty" toml:"sites,omitempty" xml:"sites" yaml:"sites"` + Controller []*Controller `json:"controller,omitempty" toml:"controller,omitempty" xml:"controller" yaml:"controller"` } // Duration is used to UnmarshalTOML into a time.Duration value. @@ -126,44 +140,58 @@ func (c *Config) ParseENV() error { for i := 0; i < t.NumField(); i++ { // Loop each Config struct member tag := t.Field(i).Tag.Get("json") // Get the ENV variable name from "json" struct tag tag = strings.Split(strings.ToUpper(tag), ",")[0] // Capitalize and remove ,omitempty suffix - env := os.Getenv(ENVConfigPrefix + tag) // Then pull value from OS. - if tag == "" || env == "" { // Skip if either are empty. + + env := os.Getenv(ENVConfigPrefix + tag) // Then pull value from OS. + if tag == "" || env == "" { // Skip if either are empty. continue } - // Reflect and update the u.Config struct member at position i. - switch field := reflect.ValueOf(c).Elem().Field(i); field.Type().String() { - // Handle each member type appropriately (differently). - case "string": - // This is a reflect package method to update a struct member by index. - field.SetString(env) - - case "int": - val, err := strconv.Atoi(env) - if err != nil { - return fmt.Errorf("%s: %v", tag, err) - } - field.Set(reflect.ValueOf(val)) - - case "[]string": - field.Set(reflect.ValueOf(strings.Split(env, ","))) - - case path.Base(t.PkgPath()) + ".Duration": - val, err := time.ParseDuration(env) - if err != nil { - return fmt.Errorf("%s: %v", tag, err) - } - field.Set(reflect.ValueOf(Duration{val})) - - case "bool": - val, err := strconv.ParseBool(env) - if err != nil { - return fmt.Errorf("%s: %v", tag, err) - } - field.SetBool(val) + if err := c.parseENV(reflect.ValueOf(c).Elem().Field(i), tag, env); err != nil { + return err } - // Add more types here if more types are added to the config struct. } return nil } + +func (c *Config) parseENV(field reflect.Value, tag, env string) error { + // Reflect and update the u.Config struct member at position i. + switch field.Type().String() { + // Handle each member type appropriately (differently). + case "string": + // This is a reflect package method to update a struct member by index. + field.SetString(env) + + case "int": + val, err := strconv.Atoi(env) + if err != nil { + return fmt.Errorf("%s: %v", tag, err) + } + field.Set(reflect.ValueOf(val)) + + case "[]string": + field.Set(reflect.ValueOf(strings.Split(env, ","))) + + case "poller.Duration": + val, err := time.ParseDuration(env) + if err != nil { + return fmt.Errorf("%s: %v", tag, err) + } + field.Set(reflect.ValueOf(Duration{val})) + + case "bool": + val, err := strconv.ParseBool(env) + if err != nil { + return fmt.Errorf("%s: %v", tag, err) + } + field.SetBool(val) + + case "poller.[]*Controller": + } + // Add more types here if more types are added to the config struct. + + return nil +} + +func (c *Config) parseSlice(field reflect.Value, tag, env string) error { +} diff --git a/integrations/inputunifi/pkg/promunifi/clients.go b/integrations/inputunifi/pkg/promunifi/clients.go index 3b48616a..fff79857 100644 --- a/integrations/inputunifi/pkg/promunifi/clients.go +++ b/integrations/inputunifi/pkg/promunifi/clients.go @@ -43,6 +43,7 @@ type uclient struct { func descClient(ns string) *uclient { labels := []string{"name", "mac", "site_name", "gw_name", "sw_name", "vlan", "ip", "oui", "network", "sw_port", "ap_name", "wired"} labelW := append([]string{"radio_name", "radio", "radio_proto", "channel", "essid", "bssid", "radio_desc"}, labels...) + return &uclient{ Anomalies: prometheus.NewDesc(ns+"anomalies", "Client Anomalies", labelW, nil), BytesR: prometheus.NewDesc(ns+"transfer_rate_bytes", "Client Data Rate", labelW, nil), @@ -115,6 +116,7 @@ func (u *promUnifi) exportClient(r report, c *unifi.Client) { {u.Client.BytesR, gauge, c.BytesR, labelW}, }) } + r.send([]*metric{{u.Client.Uptime, gauge, c.Uptime, labelW}}) /* needs more "looking into" {u.Client.DpiStatsApp, gauge, c.DpiStats.App, labels}, diff --git a/integrations/inputunifi/pkg/promunifi/collector.go b/integrations/inputunifi/pkg/promunifi/collector.go index aa14b1a8..5911d005 100644 --- a/integrations/inputunifi/pkg/promunifi/collector.go +++ b/integrations/inputunifi/pkg/promunifi/collector.go @@ -75,9 +75,11 @@ func NewUnifiCollector(opts UnifiCollectorCnfg) prometheus.Collector { if opts.CollectFn == nil { panic("nil collector function") } + if opts.Namespace = strings.Trim(opts.Namespace, "_") + "_"; opts.Namespace == "_" { opts.Namespace = "" } + return &promUnifi{ Config: opts, Client: descClient(opts.Namespace + "client_"), @@ -94,6 +96,7 @@ func NewUnifiCollector(opts UnifiCollectorCnfg) prometheus.Collector { func (u *promUnifi) Describe(ch chan<- *prometheus.Desc) { for _, f := range []interface{}{u.Client, u.Device, u.UAP, u.USG, u.USW, u.Site} { v := reflect.Indirect(reflect.ValueOf(f)) + // Loop each struct member and send it to the provided channel. for i := 0; i < v.NumField(); i++ { desc, ok := v.Field(i).Interface().(*prometheus.Desc) @@ -108,6 +111,7 @@ func (u *promUnifi) Describe(ch chan<- *prometheus.Desc) { // the current metrics (from another package) then exports them for prometheus. func (u *promUnifi) Collect(ch chan<- prometheus.Metric) { var err error + r := &Report{cf: u.Config, ch: make(chan []*metric, buffer), Start: time.Now()} defer r.close() @@ -116,6 +120,7 @@ func (u *promUnifi) Collect(ch chan<- prometheus.Metric) { return } r.Fetch = time.Since(r.Start) + if r.Metrics.Devices == nil { r.Metrics.Devices = &unifi.Devices{} } @@ -131,6 +136,7 @@ func (u *promUnifi) Collect(ch chan<- prometheus.Metric) { func (u *promUnifi) exportMetrics(r report, ch chan<- prometheus.Metric, ourChan chan []*metric) { descs := make(map[*prometheus.Desc]bool) // used as a counter defer r.report(descs) + for newMetrics := range ourChan { for _, m := range newMetrics { descs[m.Desc] = true diff --git a/integrations/inputunifi/pkg/promunifi/report.go b/integrations/inputunifi/pkg/promunifi/report.go index 7d40ea31..0ddf29d3 100644 --- a/integrations/inputunifi/pkg/promunifi/report.go +++ b/integrations/inputunifi/pkg/promunifi/report.go @@ -46,20 +46,24 @@ func (r *Report) report(descs map[*prometheus.Desc]bool) { if r.cf.LoggingFn == nil { return } + r.Descs = len(descs) r.cf.LoggingFn(r) } func (r *Report) export(m *metric, v float64) prometheus.Metric { r.Total++ + if v == 0 { r.Zeros++ } + return prometheus.MustNewConstMetric(m.Desc, m.ValueType, v, m.Labels...) } func (r *Report) error(ch chan<- prometheus.Metric, d *prometheus.Desc, v interface{}) { r.Errors++ + if r.cf.ReportErrors { ch <- prometheus.NewInvalidMetric(d, fmt.Errorf("error: %v", v)) } diff --git a/integrations/inputunifi/pkg/promunifi/site.go b/integrations/inputunifi/pkg/promunifi/site.go index c15db2c6..1f8999e9 100644 --- a/integrations/inputunifi/pkg/promunifi/site.go +++ b/integrations/inputunifi/pkg/promunifi/site.go @@ -66,8 +66,7 @@ func descSite(ns string) *site { func (u *promUnifi) exportSite(r report, s *unifi.Site) { for _, h := range s.Health { - labels := []string{h.Subsystem, h.Status, s.SiteName} - switch h.Subsystem { + switch labels := []string{h.Subsystem, h.Status, s.SiteName}; labels[0] { case "www": r.send([]*metric{ {u.Site.TxBytesR, gauge, h.TxBytesR, labels}, diff --git a/integrations/inputunifi/pkg/promunifi/uap.go b/integrations/inputunifi/pkg/promunifi/uap.go index 9998df30..53242159 100644 --- a/integrations/inputunifi/pkg/promunifi/uap.go +++ b/integrations/inputunifi/pkg/promunifi/uap.go @@ -83,6 +83,7 @@ func descUAP(ns string) *uap { labelA := []string{"stat", "site_name", "name"} // stat + labels[1:] labelV := []string{"vap_name", "bssid", "radio", "radio_name", "essid", "usage", "site_name", "name"} labelR := []string{"radio_name", "radio", "site_name", "name"} + return &uap{ // 3x each - stat table: total, guest, user ApWifiTxDropped: prometheus.NewDesc(ns+"stat_wifi_transmt_dropped_total", "Wifi Transmissions Dropped", labelA, nil), @@ -162,6 +163,7 @@ func (u *promUnifi) exportUAP(r report, d *unifi.UAP) { if !d.Adopted.Val || d.Locating.Val { return } + labels := []string{d.Type, d.SiteName, d.Name} infoLabels := []string{d.Version, d.Model, d.Serial, d.Mac, d.IP, d.ID, d.Bytes.Txt, d.Uptime.Txt} u.exportUAPstats(r, labels, d.Stat.Ap, d.BytesD, d.TxBytesD, d.RxBytesD, d.BytesR) @@ -181,6 +183,7 @@ func (u *promUnifi) exportUAPstats(r report, labels []string, ap *unifi.Ap, byte if ap == nil { return } + labelU := []string{"user", labels[1], labels[2]} labelG := []string{"guest", labels[1], labels[2]} r.send([]*metric{ @@ -229,8 +232,8 @@ func (u *promUnifi) exportVAPtable(r report, labels []string, vt unifi.VapTable) if !v.Up.Val { continue } - labelV := []string{v.Name, v.Bssid, v.Radio, v.RadioName, v.Essid, v.Usage, labels[1], labels[2]} + labelV := []string{v.Name, v.Bssid, v.Radio, v.RadioName, v.Essid, v.Usage, labels[1], labels[2]} r.send([]*metric{ {u.UAP.VAPCcq, gauge, float64(v.Ccq) / 1000.0, labelV}, {u.UAP.VAPMacFilterRejections, counter, v.MacFilterRejections, labelV}, @@ -294,6 +297,7 @@ func (u *promUnifi) exportRADtable(r report, labels []string, rt unifi.RadioTabl if t.Name != p.Name { continue } + r.send([]*metric{ {u.UAP.RadioTxPower, gauge, t.TxPower, labelR}, {u.UAP.RadioAstBeXmit, gauge, t.AstBeXmit, labelR}, diff --git a/integrations/inputunifi/pkg/promunifi/udm.go b/integrations/inputunifi/pkg/promunifi/udm.go index 3ce95117..34d7e66d 100644 --- a/integrations/inputunifi/pkg/promunifi/udm.go +++ b/integrations/inputunifi/pkg/promunifi/udm.go @@ -33,6 +33,7 @@ type unifiDevice struct { func descDevice(ns string) *unifiDevice { labels := []string{"type", "site_name", "name"} infoLabels := []string{"version", "model", "serial", "mac", "ip", "id", "bytes", "uptime"} + return &unifiDevice{ Info: prometheus.NewDesc(ns+"info", "Device Information", append(labels, infoLabels...), nil), Uptime: prometheus.NewDesc(ns+"uptime_seconds", "Device Uptime", labels, nil), @@ -63,6 +64,7 @@ func (u *promUnifi) exportUDM(r report, d *unifi.UDM) { if !d.Adopted.Val || d.Locating.Val { return } + labels := []string{d.Type, d.SiteName, d.Name} infoLabels := []string{d.Version, d.Model, d.Serial, d.Mac, d.IP, d.ID, d.Bytes.Txt, d.Uptime.Txt} // Shared data (all devices do this). @@ -80,6 +82,7 @@ func (u *promUnifi) exportUDM(r report, d *unifi.UDM) { {u.Device.Info, gauge, 1.0, append(labels, infoLabels...)}, {u.Device.Uptime, gauge, d.Uptime, labels}, }) + // Wireless Data - UDM (non-pro) only if d.Stat.Ap != nil && d.VapTable != nil { u.exportUAPstats(r, labels, d.Stat.Ap, d.BytesD, d.TxBytesD, d.RxBytesD, d.BytesR) @@ -103,6 +106,7 @@ func (u *promUnifi) exportSTAcount(r report, labels []string, stas ...unifi.Flex {u.Device.Counter, gauge, stas[0], append(labels, "user")}, {u.Device.Counter, gauge, stas[1], append(labels, "guest")}, }) + if len(stas) > 2 { r.send([]*metric{ {u.Device.Counter, gauge, stas[2], append(labels, "desktop")}, diff --git a/integrations/inputunifi/pkg/promunifi/usg.go b/integrations/inputunifi/pkg/promunifi/usg.go index 94a43f80..631ef358 100644 --- a/integrations/inputunifi/pkg/promunifi/usg.go +++ b/integrations/inputunifi/pkg/promunifi/usg.go @@ -72,6 +72,7 @@ func (u *promUnifi) exportUSG(r report, d *unifi.USG) { if !d.Adopted.Val || d.Locating.Val { return } + labels := []string{d.Type, d.SiteName, d.Name} infoLabels := []string{d.Version, d.Model, d.Serial, d.Mac, d.IP, d.ID, d.Bytes.Txt, d.Uptime.Txt} // Gateway System Data. @@ -91,6 +92,7 @@ func (u *promUnifi) exportUSGstats(r report, labels []string, gw *unifi.Gw, st u if gw == nil { return } + labelLan := []string{"lan", labels[1], labels[2]} labelWan := []string{"all", labels[1], labels[2]} r.send([]*metric{ @@ -115,6 +117,7 @@ func (u *promUnifi) exportWANPorts(r report, labels []string, wans ...unifi.Wan) if !wan.Up.Val { continue // only record UP interfaces. } + labelWan := []string{wan.Name, labels[1], labels[2]} r.send([]*metric{ {u.USG.WanRxPackets, counter, wan.RxPackets, labelWan}, diff --git a/integrations/inputunifi/pkg/promunifi/usw.go b/integrations/inputunifi/pkg/promunifi/usw.go index fd089ea7..e89506cb 100644 --- a/integrations/inputunifi/pkg/promunifi/usw.go +++ b/integrations/inputunifi/pkg/promunifi/usw.go @@ -49,6 +49,7 @@ func descUSW(ns string) *usw { pns := ns + "port_" labelS := []string{"site_name", "name"} labelP := []string{"port_id", "port_num", "port_name", "port_mac", "port_ip", "site_name", "name"} + return &usw{ // This data may be derivable by sum()ing the port data. SwRxPackets: prometheus.NewDesc(ns+"switch_receive_packets_total", "Switch Packets Received Total", labelS, nil), @@ -94,6 +95,7 @@ func (u *promUnifi) exportUSW(r report, d *unifi.USW) { if !d.Adopted.Val || d.Locating.Val { return } + labels := []string{d.Type, d.SiteName, d.Name} infoLabels := []string{d.Version, d.Model, d.Serial, d.Mac, d.IP, d.ID, d.Bytes.Txt, d.Uptime.Txt} u.exportUSWstats(r, labels, d.Stat.Sw) @@ -105,13 +107,16 @@ func (u *promUnifi) exportUSW(r report, d *unifi.USW) { {u.Device.Info, gauge, 1.0, append(labels, infoLabels...)}, {u.Device.Uptime, gauge, d.Uptime, labels}, }) + // Switch System Data. if d.HasTemperature.Val { r.send([]*metric{{u.Device.Temperature, gauge, d.GeneralTemperature, labels}}) } + if d.HasFan.Val { r.send([]*metric{{u.Device.FanLevel, gauge, d.FanLevel, labels}}) } + if d.TotalMaxPower.Txt != "" { r.send([]*metric{{u.Device.TotalMaxPower, gauge, d.TotalMaxPower, labels}}) } @@ -122,6 +127,7 @@ func (u *promUnifi) exportUSWstats(r report, labels []string, sw *unifi.Sw) { if sw == nil { return } + labelS := labels[1:] r.send([]*metric{ {u.USW.SwRxPackets, counter, sw.RxPackets, labelS}, @@ -150,8 +156,10 @@ func (u *promUnifi) exportPRTtable(r report, labels []string, pt []unifi.Port) { if !p.Up.Val || !p.Enable.Val { continue } + // Copy labels, and add four new ones. labelP := []string{labels[2] + " Port " + p.PortIdx.Txt, p.PortIdx.Txt, p.Name, p.Mac, p.IP, labels[1], labels[2]} + if p.PoeEnable.Val && p.PortPoe.Val { r.send([]*metric{ {u.USW.PoeCurrent, gauge, p.PoeCurrent, labelP}, From 4a5f1190a207899aa90907e936c792ee19d855c5 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sat, 14 Dec 2019 16:29:29 -0800 Subject: [PATCH 02/41] switch to config package --- integrations/inputunifi/pkg/poller/config.go | 145 +++---------------- integrations/inputunifi/pkg/poller/start.go | 11 +- integrations/inputunifi/pkg/poller/unifi.go | 3 +- 3 files changed, 29 insertions(+), 130 deletions(-) diff --git a/integrations/inputunifi/pkg/poller/config.go b/integrations/inputunifi/pkg/poller/config.go index bd811a04..38f645c9 100644 --- a/integrations/inputunifi/pkg/poller/config.go +++ b/integrations/inputunifi/pkg/poller/config.go @@ -9,22 +9,13 @@ package poller */ import ( - "encoding/json" - "encoding/xml" - "fmt" - "io/ioutil" - "os" - "reflect" - "strconv" - "strings" "sync" "time" - "github.com/BurntSushi/toml" "github.com/davidnewhall/unifi-poller/pkg/influxunifi" "github.com/spf13/pflag" + "golift.io/config" "golift.io/unifi" - yaml "gopkg.in/yaml.v2" ) // Version is injected by the Makefile @@ -45,7 +36,7 @@ const ( // ENVConfigPrefix is the prefix appended to an env variable tag // name before retrieving the value from the OS. -const ENVConfigPrefix = "UP_" +const ENVConfigPrefix = "UP" // UnifiPoller contains the application startup data, and auth info for UniFi & Influx. type UnifiPoller struct { @@ -68,7 +59,6 @@ type Flag struct { // Controller represents the configuration for a UniFi Controller. // Each polled controller may have its own configuration. type Controller struct { - Interval Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` SaveIDS bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` ReAuth bool `json:"reauthenticate" toml:"reauthenticate" xml:"reauthenticate" yaml:"reauthenticate"` @@ -83,115 +73,24 @@ type Controller struct { // This is all of the data stored in the config file. // Any with explicit defaults have omitempty on json and toml tags. type Config struct { - Interval Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` - Debug bool `json:"debug" toml:"debug" xml:"debug" yaml:"debug"` - Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet" yaml:"quiet"` - VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` - SaveIDS bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` - ReAuth bool `json:"reauthenticate" toml:"reauthenticate" xml:"reauthenticate" yaml:"reauthenticate"` - InfxBadSSL bool `json:"influx_insecure_ssl" toml:"influx_insecure_ssl" xml:"influx_insecure_ssl" yaml:"influx_insecure_ssl"` - SaveSites bool `json:"save_sites,omitempty" toml:"save_sites,omitempty" xml:"save_sites" yaml:"save_sites"` - Mode string `json:"mode" toml:"mode" xml:"mode" yaml:"mode"` - HTTPListen string `json:"http_listen" toml:"http_listen" xml:"http_listen" yaml:"http_listen"` - Namespace string `json:"namespace" toml:"namespace" xml:"namespace" yaml:"namespace"` - InfluxURL string `json:"influx_url,omitempty" toml:"influx_url,omitempty" xml:"influx_url" yaml:"influx_url"` - InfluxUser string `json:"influx_user,omitempty" toml:"influx_user,omitempty" xml:"influx_user" yaml:"influx_user"` - InfluxPass string `json:"influx_pass,omitempty" toml:"influx_pass,omitempty" xml:"influx_pass" yaml:"influx_pass"` - InfluxDB string `json:"influx_db,omitempty" toml:"influx_db,omitempty" xml:"influx_db" yaml:"influx_db"` - UnifiUser string `json:"unifi_user,omitempty" toml:"unifi_user,omitempty" xml:"unifi_user" yaml:"unifi_user"` - UnifiPass string `json:"unifi_pass,omitempty" toml:"unifi_pass,omitempty" xml:"unifi_pass" yaml:"unifi_pass"` - UnifiBase string `json:"unifi_url,omitempty" toml:"unifi_url,omitempty" xml:"unifi_url" yaml:"unifi_url"` - Sites []string `json:"sites,omitempty" toml:"sites,omitempty" xml:"sites" yaml:"sites"` - Controller []*Controller `json:"controller,omitempty" toml:"controller,omitempty" xml:"controller" yaml:"controller"` -} - -// Duration is used to UnmarshalTOML into a time.Duration value. -type Duration struct{ time.Duration } - -// UnmarshalText parses a duration type from a config file. -func (d *Duration) UnmarshalText(data []byte) (err error) { - d.Duration, err = time.ParseDuration(string(data)) - return -} - -// ParseFile parses and returns our configuration data. -func (c *Config) ParseFile(configFile string) error { - switch buf, err := ioutil.ReadFile(configFile); { - case err != nil: - return err - case strings.Contains(configFile, ".json"): - return json.Unmarshal(buf, c) - case strings.Contains(configFile, ".xml"): - return xml.Unmarshal(buf, c) - case strings.Contains(configFile, ".yaml"): - return yaml.Unmarshal(buf, c) - default: - return toml.Unmarshal(buf, c) - } -} - -// ParseENV copies environment variables into configuration values. -// This is useful for Docker users that find it easier to pass ENV variables -// than a specific configuration file. Uses reflection to find struct tags. -// This method uses the json struct tag member to match environment variables. -// Use a custom tag name by changing "json" below, but that's overkill for this app. -func (c *Config) ParseENV() error { - t := reflect.TypeOf(*c) // Get "types" from the Config struct. - for i := 0; i < t.NumField(); i++ { // Loop each Config struct member - tag := t.Field(i).Tag.Get("json") // Get the ENV variable name from "json" struct tag - tag = strings.Split(strings.ToUpper(tag), ",")[0] // Capitalize and remove ,omitempty suffix - - env := os.Getenv(ENVConfigPrefix + tag) // Then pull value from OS. - if tag == "" || env == "" { // Skip if either are empty. - continue - } - - if err := c.parseENV(reflect.ValueOf(c).Elem().Field(i), tag, env); err != nil { - return err - } - } - - return nil -} - -func (c *Config) parseENV(field reflect.Value, tag, env string) error { - // Reflect and update the u.Config struct member at position i. - switch field.Type().String() { - // Handle each member type appropriately (differently). - case "string": - // This is a reflect package method to update a struct member by index. - field.SetString(env) - - case "int": - val, err := strconv.Atoi(env) - if err != nil { - return fmt.Errorf("%s: %v", tag, err) - } - field.Set(reflect.ValueOf(val)) - - case "[]string": - field.Set(reflect.ValueOf(strings.Split(env, ","))) - - case "poller.Duration": - val, err := time.ParseDuration(env) - if err != nil { - return fmt.Errorf("%s: %v", tag, err) - } - field.Set(reflect.ValueOf(Duration{val})) - - case "bool": - val, err := strconv.ParseBool(env) - if err != nil { - return fmt.Errorf("%s: %v", tag, err) - } - field.SetBool(val) - - case "poller.[]*Controller": - } - // Add more types here if more types are added to the config struct. - - return nil -} - -func (c *Config) parseSlice(field reflect.Value, tag, env string) error { + Interval config.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` + Debug bool `json:"debug" toml:"debug" xml:"debug" yaml:"debug"` + Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet" yaml:"quiet"` + VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` + SaveIDS bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` + ReAuth bool `json:"reauthenticate" toml:"reauthenticate" xml:"reauthenticate" yaml:"reauthenticate"` + InfxBadSSL bool `json:"influx_insecure_ssl" toml:"influx_insecure_ssl" xml:"influx_insecure_ssl" yaml:"influx_insecure_ssl"` + SaveSites bool `json:"save_sites,omitempty" toml:"save_sites,omitempty" xml:"save_sites" yaml:"save_sites"` + Mode string `json:"mode" toml:"mode" xml:"mode" yaml:"mode"` + HTTPListen string `json:"http_listen" toml:"http_listen" xml:"http_listen" yaml:"http_listen"` + Namespace string `json:"namespace" toml:"namespace" xml:"namespace" yaml:"namespace"` + InfluxURL string `json:"influx_url,omitempty" toml:"influx_url,omitempty" xml:"influx_url" yaml:"influx_url"` + InfluxUser string `json:"influx_user,omitempty" toml:"influx_user,omitempty" xml:"influx_user" yaml:"influx_user"` + InfluxPass string `json:"influx_pass,omitempty" toml:"influx_pass,omitempty" xml:"influx_pass" yaml:"influx_pass"` + InfluxDB string `json:"influx_db,omitempty" toml:"influx_db,omitempty" xml:"influx_db" yaml:"influx_db"` + UnifiUser string `json:"unifi_user,omitempty" toml:"unifi_user,omitempty" xml:"unifi_user" yaml:"unifi_user"` + UnifiPass string `json:"unifi_pass,omitempty" toml:"unifi_pass,omitempty" xml:"unifi_pass" yaml:"unifi_pass"` + UnifiBase string `json:"unifi_url,omitempty" toml:"unifi_url,omitempty" xml:"unifi_url" yaml:"unifi_url"` + Sites []string `json:"sites,omitempty" toml:"sites,omitempty" xml:"sites" yaml:"sites"` + Controller []Controller `json:"controller,omitempty" toml:"controller,omitempty" xml:"controller" yaml:"controller"` } diff --git a/integrations/inputunifi/pkg/poller/start.go b/integrations/inputunifi/pkg/poller/start.go index 3f1a4d74..57d27720 100644 --- a/integrations/inputunifi/pkg/poller/start.go +++ b/integrations/inputunifi/pkg/poller/start.go @@ -9,6 +9,7 @@ import ( "time" "github.com/spf13/pflag" + "golift.io/config" ) // New returns a new poller struct preloaded with default values. @@ -23,7 +24,7 @@ func New() *UnifiPoller { UnifiUser: defaultUnifiUser, UnifiPass: "", UnifiBase: defaultUnifiURL, - Interval: Duration{defaultInterval}, + Interval: config.Duration{Duration: defaultInterval}, Sites: []string{"all"}, SaveSites: true, HTTPListen: defaultHTTPListen, @@ -53,16 +54,16 @@ func (u *UnifiPoller) Start() error { } // Parse config file. - if err := u.Config.ParseFile(u.Flag.ConfigFile); err != nil { + if err := config.ParseFile(u.Config, u.Flag.ConfigFile); err != nil { u.Flag.Usage() return err } // Update Config with ENV variable overrides. - if err := u.Config.ParseENV(); err != nil { + if _, err := config.ParseENV(u.Config, ENVConfigPrefix); err != nil { return err } - + log.Println("START():", u.Config.Controller) if u.Flag.DumpJSON != "" { return u.DumpJSONPayload() } @@ -71,7 +72,7 @@ func (u *UnifiPoller) Start() error { log.SetFlags(log.Lshortfile | log.Lmicroseconds | log.Ldate) u.LogDebugf("Debug Logging Enabled") } - + log.Println("sites", u.Config.Sites) log.Printf("[INFO] UniFi Poller v%v Starting Up! PID: %d", Version, os.Getpid()) return u.Run() } diff --git a/integrations/inputunifi/pkg/poller/unifi.go b/integrations/inputunifi/pkg/poller/unifi.go index beeacec5..c3eea421 100644 --- a/integrations/inputunifi/pkg/poller/unifi.go +++ b/integrations/inputunifi/pkg/poller/unifi.go @@ -70,8 +70,7 @@ FIRST: continue FIRST } } - // This is fine, it may get added later. - u.LogErrorf("configured site not found on controller: %v", s) + return fmt.Errorf("configured site not found on controller: %v", s) } return nil From b70fc7c9614cfb5f16f80ab91d983abc614425e9 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sat, 14 Dec 2019 18:10:35 -0800 Subject: [PATCH 03/41] nearly there --- integrations/inputunifi/.gitignore | 2 +- integrations/inputunifi/pkg/poller/config.go | 25 ++-- integrations/inputunifi/pkg/poller/dumper.go | 37 +++--- integrations/inputunifi/pkg/poller/influx.go | 9 +- .../inputunifi/pkg/poller/prometheus.go | 19 +-- integrations/inputunifi/pkg/poller/start.go | 36 +++--- integrations/inputunifi/pkg/poller/unifi.go | 114 ++++++++++++------ 7 files changed, 129 insertions(+), 113 deletions(-) diff --git a/integrations/inputunifi/.gitignore b/integrations/inputunifi/.gitignore index 118e4d74..f281b1b0 100644 --- a/integrations/inputunifi/.gitignore +++ b/integrations/inputunifi/.gitignore @@ -11,7 +11,7 @@ /unifi-poller.*.linux /unifi-poller.rb *.sha256 -/vendor +#/vendor .DS_Store *~ /package_build_* diff --git a/integrations/inputunifi/pkg/poller/config.go b/integrations/inputunifi/pkg/poller/config.go index 38f645c9..439b4e9a 100644 --- a/integrations/inputunifi/pkg/poller/config.go +++ b/integrations/inputunifi/pkg/poller/config.go @@ -41,7 +41,6 @@ const ENVConfigPrefix = "UP" // UnifiPoller contains the application startup data, and auth info for UniFi & Influx. type UnifiPoller struct { Influx *influxunifi.InfluxUnifi - Unifi *unifi.Unifi Flag *Flag Config *Config LastCheck time.Time @@ -59,14 +58,14 @@ type Flag struct { // Controller represents the configuration for a UniFi Controller. // Each polled controller may have its own configuration. type Controller struct { - VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` - SaveIDS bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` - ReAuth bool `json:"reauthenticate" toml:"reauthenticate" xml:"reauthenticate" yaml:"reauthenticate"` - SaveSites bool `json:"save_sites,omitempty" toml:"save_sites,omitempty" xml:"save_sites" yaml:"save_sites"` - User string `json:"unifi_user,omitempty" toml:"unifi_user,omitempty" xml:"unifi_user" yaml:"unifi_user"` - Pass string `json:"unifi_pass,omitempty" toml:"unifi_pass,omitempty" xml:"unifi_pass" yaml:"unifi_pass"` - URL string `json:"unifi_url,omitempty" toml:"unifi_url,omitempty" xml:"unifi_url" yaml:"unifi_url"` - Sites []string `json:"sites,omitempty" toml:"sites,omitempty" xml:"sites" yaml:"sites"` + VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` + SaveIDS bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` + SaveSites bool `json:"save_sites,omitempty" toml:"save_sites,omitempty" xml:"save_sites" yaml:"save_sites"` + User string `json:"unifi_user,omitempty" toml:"unifi_user,omitempty" xml:"unifi_user" yaml:"unifi_user"` + Pass string `json:"unifi_pass,omitempty" toml:"unifi_pass,omitempty" xml:"unifi_pass" yaml:"unifi_pass"` + URL string `json:"unifi_url,omitempty" toml:"unifi_url,omitempty" xml:"unifi_url" yaml:"unifi_url"` + Sites []string `json:"sites,omitempty" toml:"sites,omitempty" xml:"sites" yaml:"sites"` + Unifi *unifi.Unifi `json:"-" toml:"-" xml:"-" yaml:"-"` } // Config represents the data needed to poll a controller and report to influxdb. @@ -76,11 +75,7 @@ type Config struct { Interval config.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` Debug bool `json:"debug" toml:"debug" xml:"debug" yaml:"debug"` Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet" yaml:"quiet"` - VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` - SaveIDS bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` - ReAuth bool `json:"reauthenticate" toml:"reauthenticate" xml:"reauthenticate" yaml:"reauthenticate"` InfxBadSSL bool `json:"influx_insecure_ssl" toml:"influx_insecure_ssl" xml:"influx_insecure_ssl" yaml:"influx_insecure_ssl"` - SaveSites bool `json:"save_sites,omitempty" toml:"save_sites,omitempty" xml:"save_sites" yaml:"save_sites"` Mode string `json:"mode" toml:"mode" xml:"mode" yaml:"mode"` HTTPListen string `json:"http_listen" toml:"http_listen" xml:"http_listen" yaml:"http_listen"` Namespace string `json:"namespace" toml:"namespace" xml:"namespace" yaml:"namespace"` @@ -88,9 +83,5 @@ type Config struct { InfluxUser string `json:"influx_user,omitempty" toml:"influx_user,omitempty" xml:"influx_user" yaml:"influx_user"` InfluxPass string `json:"influx_pass,omitempty" toml:"influx_pass,omitempty" xml:"influx_pass" yaml:"influx_pass"` InfluxDB string `json:"influx_db,omitempty" toml:"influx_db,omitempty" xml:"influx_db" yaml:"influx_db"` - UnifiUser string `json:"unifi_user,omitempty" toml:"unifi_user,omitempty" xml:"unifi_user" yaml:"unifi_user"` - UnifiPass string `json:"unifi_pass,omitempty" toml:"unifi_pass,omitempty" xml:"unifi_pass" yaml:"unifi_pass"` - UnifiBase string `json:"unifi_url,omitempty" toml:"unifi_url,omitempty" xml:"unifi_url" yaml:"unifi_url"` - Sites []string `json:"sites,omitempty" toml:"sites,omitempty" xml:"sites" yaml:"sites"` Controller []Controller `json:"controller,omitempty" toml:"controller,omitempty" xml:"controller" yaml:"controller"` } diff --git a/integrations/inputunifi/pkg/poller/dumper.go b/integrations/inputunifi/pkg/poller/dumper.go index 09514e57..2843317c 100644 --- a/integrations/inputunifi/pkg/poller/dumper.go +++ b/integrations/inputunifi/pkg/poller/dumper.go @@ -9,50 +9,53 @@ import ( ) // DumpJSONPayload prints raw json from the UniFi Controller. +// This only works with controller 0 (first one) in the config. func (u *UnifiPoller) DumpJSONPayload() (err error) { u.Config.Quiet = true - u.Unifi, err = unifi.NewUnifi(&unifi.Config{ - User: u.Config.UnifiUser, - Pass: u.Config.UnifiPass, - URL: u.Config.UnifiBase, - VerifySSL: u.Config.VerifySSL, + config := u.Config.Controller[0] + + config.Unifi, err = unifi.NewUnifi(&unifi.Config{ + User: config.User, + Pass: config.Pass, + URL: config.URL, + VerifySSL: config.VerifySSL, }) if err != nil { return err } - fmt.Fprintf(os.Stderr, "[INFO] Authenticated to UniFi Controller @ %v as user %v", - u.Config.UnifiBase, u.Config.UnifiUser) - if err := u.CheckSites(); err != nil { + fmt.Fprintf(os.Stderr, "[INFO] Authenticated to UniFi Controller @ %v as user %v", config.URL, config.User) + + if err := u.CheckSites(config); err != nil { return err } - u.Unifi.ErrorLog = func(m string, v ...interface{}) { + config.Unifi.ErrorLog = func(m string, v ...interface{}) { fmt.Fprintf(os.Stderr, "[ERROR] "+m, v...) } // Log all errors to stderr. - switch sites, err := u.GetFilteredSites(); { + switch sites, err := u.GetFilteredSites(config); { case err != nil: return err case StringInSlice(u.Flag.DumpJSON, []string{"d", "device", "devices"}): - return u.dumpSitesJSON(unifi.APIDevicePath, "Devices", sites) + return u.dumpSitesJSON(config, unifi.APIDevicePath, "Devices", sites) case StringInSlice(u.Flag.DumpJSON, []string{"client", "clients", "c"}): - return u.dumpSitesJSON(unifi.APIClientPath, "Clients", sites) + return u.dumpSitesJSON(config, unifi.APIClientPath, "Clients", sites) case strings.HasPrefix(u.Flag.DumpJSON, "other "): apiPath := strings.SplitN(u.Flag.DumpJSON, " ", 2)[1] _, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping Path '%s':\n", apiPath) - return u.PrintRawAPIJSON(apiPath) + return u.PrintRawAPIJSON(config, apiPath) default: return fmt.Errorf("must provide filter: devices, clients, other") } } -func (u *UnifiPoller) dumpSitesJSON(path, name string, sites unifi.Sites) error { +func (u *UnifiPoller) dumpSitesJSON(c Controller, path, name string, sites unifi.Sites) error { for _, s := range sites { apiPath := fmt.Sprintf(path, s.Name) _, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping %s: '%s' JSON for site: %s (%s):\n", name, apiPath, s.Desc, s.Name) - if err := u.PrintRawAPIJSON(apiPath); err != nil { + if err := u.PrintRawAPIJSON(c, apiPath); err != nil { return err } } @@ -60,8 +63,8 @@ func (u *UnifiPoller) dumpSitesJSON(path, name string, sites unifi.Sites) error } // PrintRawAPIJSON prints the raw json for a user-provided path on a UniFi Controller. -func (u *UnifiPoller) PrintRawAPIJSON(apiPath string) error { - body, err := u.Unifi.GetJSON(apiPath) +func (u *UnifiPoller) PrintRawAPIJSON(c Controller, apiPath string) error { + body, err := c.Unifi.GetJSON(apiPath) fmt.Println(string(body)) return err } diff --git a/integrations/inputunifi/pkg/poller/influx.go b/integrations/inputunifi/pkg/poller/influx.go index ff2384e8..e9433460 100644 --- a/integrations/inputunifi/pkg/poller/influx.go +++ b/integrations/inputunifi/pkg/poller/influx.go @@ -44,8 +44,6 @@ func (u *UnifiPoller) CollectAndProcess() error { return err } - u.AugmentMetrics(metrics) - report, err := u.Influx.ReportMetrics(metrics) if err != nil { return err @@ -57,12 +55,7 @@ func (u *UnifiPoller) CollectAndProcess() error { // LogInfluxReport writes a log message after exporting to influxdb. func (u *UnifiPoller) LogInfluxReport(r *influxunifi.Report) { - idsMsg := "" - - if u.Config.SaveIDS { - idsMsg = fmt.Sprintf("IDS Events: %d, ", len(r.Metrics.IDSList)) - } - + idsMsg := fmt.Sprintf("IDS Events: %d, ", len(r.Metrics.IDSList)) u.Logf("UniFi Metrics Recorded. Sites: %d, Clients: %d, "+ "UAP: %d, USG/UDM: %d, USW: %d, %sPoints: %d, Fields: %d, Errs: %d, Elapsed: %v", len(r.Metrics.Sites), len(r.Metrics.Clients), len(r.Metrics.UAPs), diff --git a/integrations/inputunifi/pkg/poller/prometheus.go b/integrations/inputunifi/pkg/poller/prometheus.go index a0ffb818..13e4de49 100644 --- a/integrations/inputunifi/pkg/poller/prometheus.go +++ b/integrations/inputunifi/pkg/poller/prometheus.go @@ -36,24 +36,7 @@ func (u *UnifiPoller) RunPrometheus() error { // HTTP at /metrics for prometheus collection. // This is run by Prometheus as CollectFn. func (u *UnifiPoller) ExportMetrics() (*metrics.Metrics, error) { - m, err := u.CollectMetrics() - if err != nil { - u.LogErrorf("collecting metrics: %v", err) - u.Logf("Re-authenticating to UniFi Controller") - - if err := u.GetUnifi(); err != nil { - u.LogErrorf("re-authenticating: %v", err) - return nil, err - } - - if m, err = u.CollectMetrics(); err != nil { - u.LogErrorf("collecting metrics: %v", err) - return nil, err - } - } - - u.AugmentMetrics(m) - return m, nil + return u.CollectMetrics() } // LogExportReport is called after prometheus exports metrics. diff --git a/integrations/inputunifi/pkg/poller/start.go b/integrations/inputunifi/pkg/poller/start.go index 57d27720..a6d2fb33 100644 --- a/integrations/inputunifi/pkg/poller/start.go +++ b/integrations/inputunifi/pkg/poller/start.go @@ -17,16 +17,18 @@ import ( func New() *UnifiPoller { return &UnifiPoller{ Config: &Config{ + Controller: []Controller{{ + Sites: []string{"all"}, + User: defaultUnifiUser, + Pass: "", + URL: defaultUnifiURL, + SaveSites: true, + }}, InfluxURL: defaultInfluxURL, InfluxUser: defaultInfluxUser, InfluxPass: defaultInfluxPass, InfluxDB: defaultInfluxDB, - UnifiUser: defaultUnifiUser, - UnifiPass: "", - UnifiBase: defaultUnifiURL, Interval: config.Duration{Duration: defaultInterval}, - Sites: []string{"all"}, - SaveSites: true, HTTPListen: defaultHTTPListen, Namespace: appName, }, @@ -63,7 +65,7 @@ func (u *UnifiPoller) Start() error { if _, err := config.ParseENV(u.Config, ENVConfigPrefix); err != nil { return err } - log.Println("START():", u.Config.Controller) + log.Println("START(): controller", u.Config.Controller) if u.Flag.DumpJSON != "" { return u.DumpJSONPayload() } @@ -72,8 +74,9 @@ func (u *UnifiPoller) Start() error { log.SetFlags(log.Lshortfile | log.Lmicroseconds | log.Ldate) u.LogDebugf("Debug Logging Enabled") } - log.Println("sites", u.Config.Sites) + log.Printf("[INFO] UniFi Poller v%v Starting Up! PID: %d", Version, os.Getpid()) + return u.Run() } @@ -97,12 +100,14 @@ func (f *Flag) Parse(args []string) { // 2. Run the collector one time and report the metrics to influxdb. (lambda) // 3. Start a web server and wait for Prometheus to poll the application for metrics. func (u *UnifiPoller) Run() error { - switch err := u.GetUnifi(); err { - case nil: - u.Logf("Polling UniFi Controller at %s v%s as user %s. Sites: %v", - u.Config.UnifiBase, u.Unifi.ServerVersion, u.Config.UnifiUser, u.Config.Sites) - default: - u.LogErrorf("Controller Auth or Connection failed, but continuing to retry! %v", err) + for _, c := range u.Config.Controller { + switch err := u.GetUnifi(c); err { + case nil: + u.Logf("Polling UniFi Controller at %s v%s as user %s. Sites: %v", + c.URL, c.Unifi.ServerVersion, c.User, c.Sites) + default: + u.LogErrorf("Controller Auth or Connection failed, but continuing to retry! %s: %v", c.URL, err) + } } switch strings.ToLower(u.Config.Mode) { @@ -130,11 +135,6 @@ func (u *UnifiPoller) PollController() { for u.LastCheck = range ticker.C { if err := u.CollectAndProcess(); err != nil { u.LogErrorf("%v", err) - - if u.Unifi != nil { - u.Unifi.CloseIdleConnections() - u.Unifi = nil // trigger re-auth in unifi.go. - } } } } diff --git a/integrations/inputunifi/pkg/poller/unifi.go b/integrations/inputunifi/pkg/poller/unifi.go index c3eea421..1c0e64af 100644 --- a/integrations/inputunifi/pkg/poller/unifi.go +++ b/integrations/inputunifi/pkg/poller/unifi.go @@ -10,43 +10,44 @@ import ( ) // GetUnifi returns a UniFi controller interface. -func (u *UnifiPoller) GetUnifi() (err error) { +func (u *UnifiPoller) GetUnifi(config Controller) error { + var err error + u.Lock() defer u.Unlock() - if u.Unifi != nil { - u.Unifi.CloseIdleConnections() + if config.Unifi != nil { + config.Unifi.CloseIdleConnections() } // Create an authenticated session to the Unifi Controller. - u.Unifi, err = unifi.NewUnifi(&unifi.Config{ - User: u.Config.UnifiUser, - Pass: u.Config.UnifiPass, - URL: u.Config.UnifiBase, - VerifySSL: u.Config.VerifySSL, + config.Unifi, err = unifi.NewUnifi(&unifi.Config{ + User: config.User, + Pass: config.Pass, + URL: config.URL, + VerifySSL: config.VerifySSL, ErrorLog: u.LogErrorf, // Log all errors. DebugLog: u.LogDebugf, // Log debug messages. }) if err != nil { - u.Unifi = nil return fmt.Errorf("unifi controller: %v", err) } - u.LogDebugf("Authenticated with controller successfully") + u.LogDebugf("Authenticated with controller successfully, %s", config.URL) - return u.CheckSites() + return u.CheckSites(config) } // CheckSites makes sure the list of provided sites exists on the controller. // This does not run in Lambda (run-once) mode. -func (u *UnifiPoller) CheckSites() error { +func (u *UnifiPoller) CheckSites(config Controller) error { if strings.Contains(strings.ToLower(u.Config.Mode), "lambda") { return nil // Skip this in lambda mode. } u.LogDebugf("Checking Controller Sites List") - sites, err := u.Unifi.GetSites() + sites, err := config.Unifi.GetSites() if err != nil { return err } @@ -58,13 +59,13 @@ func (u *UnifiPoller) CheckSites() error { } u.Logf("Found %d site(s) on controller: %v", len(msg), strings.Join(msg, ", ")) - if StringInSlice("all", u.Config.Sites) { - u.Config.Sites = []string{"all"} + if StringInSlice("all", config.Sites) { + config.Sites = []string{"all"} return nil } FIRST: - for _, s := range u.Config.Sites { + for _, s := range config.Sites { for _, site := range sites { if s == site.Name { continue FIRST @@ -77,47 +78,90 @@ FIRST: } // CollectMetrics grabs all the measurements from a UniFi controller and returns them. -func (u *UnifiPoller) CollectMetrics() (*metrics.Metrics, error) { +func (u *UnifiPoller) CollectMetrics() (metrics *metrics.Metrics, err error) { + var errs []string + + for _, c := range u.Config.Controller { + m, err := u.collectController(c) + if err != nil { + errs = append(errs, err.Error()) + continue + } + + if err != nil { + u.LogErrorf("collecting metrics from %s: %v", c.URL, err) + u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) + + if err := u.GetUnifi(c); err != nil { + u.LogErrorf("re-authenticating to %s: %v", c.URL, err) + errs = append(errs, err.Error()) + } else if m, err = u.collectController(c); err != nil { + u.LogErrorf("collecting metrics from %s: %v", c.URL, err) + errs = append(errs, err.Error()) + } + } + + metrics.Sites = append(metrics.Sites, m.Sites...) + metrics.Clients = append(metrics.Clients, m.Clients...) + metrics.IDSList = append(metrics.IDSList, m.IDSList...) + metrics.UAPs = append(metrics.UAPs, m.UAPs...) + metrics.USGs = append(metrics.USGs, m.USGs...) + metrics.USWs = append(metrics.USWs, m.USWs...) + metrics.UDMs = append(metrics.UDMs, m.UDMs...) + } + + if len(errs) > 0 { + err = fmt.Errorf(strings.Join(errs, ", ")) + } + + return +} + +func (u *UnifiPoller) collectController(c Controller) (*metrics.Metrics, error) { var err error - if u.Unifi == nil || u.Config.ReAuth { + if c.Unifi == nil { // Some users need to re-auth every interval because the cookie times out. // Sometimes we hit this path when the controller dies. u.Logf("Re-authenticating to UniFi Controller") - if err := u.GetUnifi(); err != nil { + + if err := u.GetUnifi(c); err != nil { return nil, err } } m := &metrics.Metrics{TS: u.LastCheck} // At this point, it's the Current Check. + // Get the sites we care about. - if m.Sites, err = u.GetFilteredSites(); err != nil { + if m.Sites, err = u.GetFilteredSites(c); err != nil { return m, fmt.Errorf("unifi.GetSites(): %v", err) } - if u.Config.SaveIDS { - m.IDSList, err = u.Unifi.GetIDS(m.Sites, time.Now().Add(u.Config.Interval.Duration), time.Now()) - return m, fmt.Errorf("unifi.GetIDS(): %v", err) + if c.SaveIDS { + m.IDSList, err = c.Unifi.GetIDS(m.Sites, time.Now().Add(u.Config.Interval.Duration), time.Now()) + if err != nil { + return m, fmt.Errorf("unifi.GetIDS(): %v", err) + } } // Get all the points. - if m.Clients, err = u.Unifi.GetClients(m.Sites); err != nil { + if m.Clients, err = c.Unifi.GetClients(m.Sites); err != nil { return m, fmt.Errorf("unifi.GetClients(): %v", err) } - if m.Devices, err = u.Unifi.GetDevices(m.Sites); err != nil { + if m.Devices, err = c.Unifi.GetDevices(m.Sites); err != nil { return m, fmt.Errorf("unifi.GetDevices(): %v", err) } - return m, nil + return u.augmentMetrics(c, m), nil } -// AugmentMetrics is our middleware layer between collecting metrics and writing them. +// augmentMetrics is our middleware layer between collecting metrics and writing them. // This is where we can manipuate the returned data or make arbitrary decisions. // This function currently adds parent device names to client metrics. -func (u *UnifiPoller) AugmentMetrics(metrics *metrics.Metrics) { +func (u *UnifiPoller) augmentMetrics(c Controller, metrics *metrics.Metrics) *metrics.Metrics { if metrics == nil || metrics.Devices == nil || metrics.Clients == nil { - return + return metrics } devices := make(map[string]string) @@ -150,27 +194,29 @@ func (u *UnifiPoller) AugmentMetrics(metrics *metrics.Metrics) { metrics.Clients[i].RadioDescription = bssdIDs[metrics.Clients[i].Bssid] + metrics.Clients[i].RadioProto } - if !u.Config.SaveSites { + if !c.SaveSites { metrics.Sites = nil } + + return metrics } // GetFilteredSites returns a list of sites to fetch data for. // Omits requested but unconfigured sites. Grabs the full list from the // controller and returns the sites provided in the config file. -func (u *UnifiPoller) GetFilteredSites() (unifi.Sites, error) { +func (u *UnifiPoller) GetFilteredSites(config Controller) (unifi.Sites, error) { var i int - sites, err := u.Unifi.GetSites() + sites, err := config.Unifi.GetSites() if err != nil { return nil, err - } else if len(u.Config.Sites) < 1 || StringInSlice("all", u.Config.Sites) { + } else if len(config.Sites) < 1 || StringInSlice("all", config.Sites) { return sites, nil } for _, s := range sites { // Only include valid sites in the request filter. - if StringInSlice(s.Name, u.Config.Sites) { + if StringInSlice(s.Name, config.Sites) { sites[i] = s i++ } From 038d19e760d05617419a6b8720c71923ccc42aa9 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sat, 14 Dec 2019 18:31:13 -0800 Subject: [PATCH 04/41] should be good --- integrations/inputunifi/Gopkg.lock | 11 +++++++-- integrations/inputunifi/pkg/poller/config.go | 24 +++++++++--------- integrations/inputunifi/pkg/poller/dumper.go | 2 +- integrations/inputunifi/pkg/poller/start.go | 5 ++-- integrations/inputunifi/pkg/poller/unifi.go | 26 +++++++++----------- 5 files changed, 35 insertions(+), 33 deletions(-) diff --git a/integrations/inputunifi/Gopkg.lock b/integrations/inputunifi/Gopkg.lock index 3a105536..238a1b81 100644 --- a/integrations/inputunifi/Gopkg.lock +++ b/integrations/inputunifi/Gopkg.lock @@ -106,6 +106,14 @@ pruneopts = "UT" revision = "ac6580df4449443a05718fd7858c1f91ad5f8d20" +[[projects]] + branch = "master" + digest = "1:d54a8d89f95a4d2a5a24ce63cb1835ccdff337fde7776c87ceacb6fdbe4349ae" + name = "golift.io/config" + packages = ["."] + pruneopts = "UT" + revision = "fd8ffb02173aad2183e5555a03b1d1f909aca930" + [[projects]] digest = "1:2883cea734f2766f41ff9c9d4aefccccc53e3d44f5c8b08893b9c218cf666722" name = "golift.io/unifi" @@ -126,14 +134,13 @@ analyzer-name = "dep" analyzer-version = 1 input-imports = [ - "github.com/BurntSushi/toml", "github.com/influxdata/influxdb1-client/v2", "github.com/prometheus/client_golang/prometheus", "github.com/prometheus/client_golang/prometheus/promhttp", "github.com/prometheus/common/version", "github.com/spf13/pflag", + "golift.io/config", "golift.io/unifi", - "gopkg.in/yaml.v2", ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/integrations/inputunifi/pkg/poller/config.go b/integrations/inputunifi/pkg/poller/config.go index 439b4e9a..c77bdff6 100644 --- a/integrations/inputunifi/pkg/poller/config.go +++ b/integrations/inputunifi/pkg/poller/config.go @@ -72,16 +72,16 @@ type Controller struct { // This is all of the data stored in the config file. // Any with explicit defaults have omitempty on json and toml tags. type Config struct { - Interval config.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` - Debug bool `json:"debug" toml:"debug" xml:"debug" yaml:"debug"` - Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet" yaml:"quiet"` - InfxBadSSL bool `json:"influx_insecure_ssl" toml:"influx_insecure_ssl" xml:"influx_insecure_ssl" yaml:"influx_insecure_ssl"` - Mode string `json:"mode" toml:"mode" xml:"mode" yaml:"mode"` - HTTPListen string `json:"http_listen" toml:"http_listen" xml:"http_listen" yaml:"http_listen"` - Namespace string `json:"namespace" toml:"namespace" xml:"namespace" yaml:"namespace"` - InfluxURL string `json:"influx_url,omitempty" toml:"influx_url,omitempty" xml:"influx_url" yaml:"influx_url"` - InfluxUser string `json:"influx_user,omitempty" toml:"influx_user,omitempty" xml:"influx_user" yaml:"influx_user"` - InfluxPass string `json:"influx_pass,omitempty" toml:"influx_pass,omitempty" xml:"influx_pass" yaml:"influx_pass"` - InfluxDB string `json:"influx_db,omitempty" toml:"influx_db,omitempty" xml:"influx_db" yaml:"influx_db"` - Controller []Controller `json:"controller,omitempty" toml:"controller,omitempty" xml:"controller" yaml:"controller"` + Interval config.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` + Debug bool `json:"debug" toml:"debug" xml:"debug" yaml:"debug"` + Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet" yaml:"quiet"` + InfxBadSSL bool `json:"influx_insecure_ssl" toml:"influx_insecure_ssl" xml:"influx_insecure_ssl" yaml:"influx_insecure_ssl"` + Mode string `json:"mode" toml:"mode" xml:"mode" yaml:"mode"` + HTTPListen string `json:"http_listen" toml:"http_listen" xml:"http_listen" yaml:"http_listen"` + Namespace string `json:"namespace" toml:"namespace" xml:"namespace" yaml:"namespace"` + InfluxURL string `json:"influx_url,omitempty" toml:"influx_url,omitempty" xml:"influx_url" yaml:"influx_url"` + InfluxUser string `json:"influx_user,omitempty" toml:"influx_user,omitempty" xml:"influx_user" yaml:"influx_user"` + InfluxPass string `json:"influx_pass,omitempty" toml:"influx_pass,omitempty" xml:"influx_pass" yaml:"influx_pass"` + InfluxDB string `json:"influx_db,omitempty" toml:"influx_db,omitempty" xml:"influx_db" yaml:"influx_db"` + Controllers []Controller `json:"controller,omitempty" toml:"controller,omitempty" xml:"controller" yaml:"controller"` } diff --git a/integrations/inputunifi/pkg/poller/dumper.go b/integrations/inputunifi/pkg/poller/dumper.go index 2843317c..3151ae9c 100644 --- a/integrations/inputunifi/pkg/poller/dumper.go +++ b/integrations/inputunifi/pkg/poller/dumper.go @@ -12,7 +12,7 @@ import ( // This only works with controller 0 (first one) in the config. func (u *UnifiPoller) DumpJSONPayload() (err error) { u.Config.Quiet = true - config := u.Config.Controller[0] + config := u.Config.Controllers[0] config.Unifi, err = unifi.NewUnifi(&unifi.Config{ User: config.User, diff --git a/integrations/inputunifi/pkg/poller/start.go b/integrations/inputunifi/pkg/poller/start.go index a6d2fb33..54dc0731 100644 --- a/integrations/inputunifi/pkg/poller/start.go +++ b/integrations/inputunifi/pkg/poller/start.go @@ -17,7 +17,7 @@ import ( func New() *UnifiPoller { return &UnifiPoller{ Config: &Config{ - Controller: []Controller{{ + Controllers: []Controller{{ Sites: []string{"all"}, User: defaultUnifiUser, Pass: "", @@ -65,7 +65,6 @@ func (u *UnifiPoller) Start() error { if _, err := config.ParseENV(u.Config, ENVConfigPrefix); err != nil { return err } - log.Println("START(): controller", u.Config.Controller) if u.Flag.DumpJSON != "" { return u.DumpJSONPayload() } @@ -100,7 +99,7 @@ func (f *Flag) Parse(args []string) { // 2. Run the collector one time and report the metrics to influxdb. (lambda) // 3. Start a web server and wait for Prometheus to poll the application for metrics. func (u *UnifiPoller) Run() error { - for _, c := range u.Config.Controller { + for _, c := range u.Config.Controllers { switch err := u.GetUnifi(c); err { case nil: u.Logf("Polling UniFi Controller at %s v%s as user %s. Sites: %v", diff --git a/integrations/inputunifi/pkg/poller/unifi.go b/integrations/inputunifi/pkg/poller/unifi.go index 1c0e64af..c602293c 100644 --- a/integrations/inputunifi/pkg/poller/unifi.go +++ b/integrations/inputunifi/pkg/poller/unifi.go @@ -30,6 +30,7 @@ func (u *UnifiPoller) GetUnifi(config Controller) error { DebugLog: u.LogDebugf, // Log debug messages. }) if err != nil { + config.Unifi = nil return fmt.Errorf("unifi controller: %v", err) } @@ -81,13 +82,8 @@ FIRST: func (u *UnifiPoller) CollectMetrics() (metrics *metrics.Metrics, err error) { var errs []string - for _, c := range u.Config.Controller { + for _, c := range u.Config.Controllers { m, err := u.collectController(c) - if err != nil { - errs = append(errs, err.Error()) - continue - } - if err != nil { u.LogErrorf("collecting metrics from %s: %v", c.URL, err) u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) @@ -123,7 +119,7 @@ func (u *UnifiPoller) collectController(c Controller) (*metrics.Metrics, error) if c.Unifi == nil { // Some users need to re-auth every interval because the cookie times out. // Sometimes we hit this path when the controller dies. - u.Logf("Re-authenticating to UniFi Controller") + u.Logf("Re-authenticating to UniFi Controller: %v", c.URL) if err := u.GetUnifi(c); err != nil { return nil, err @@ -134,23 +130,23 @@ func (u *UnifiPoller) collectController(c Controller) (*metrics.Metrics, error) // Get the sites we care about. if m.Sites, err = u.GetFilteredSites(c); err != nil { - return m, fmt.Errorf("unifi.GetSites(): %v", err) + return m, fmt.Errorf("unifi.GetSites(%v): %v", c.URL, err) } if c.SaveIDS { m.IDSList, err = c.Unifi.GetIDS(m.Sites, time.Now().Add(u.Config.Interval.Duration), time.Now()) if err != nil { - return m, fmt.Errorf("unifi.GetIDS(): %v", err) + return m, fmt.Errorf("unifi.GetIDS(%v): %v", c.URL, err) } } // Get all the points. if m.Clients, err = c.Unifi.GetClients(m.Sites); err != nil { - return m, fmt.Errorf("unifi.GetClients(): %v", err) + return m, fmt.Errorf("unifi.GetClients(%v): %v", c.URL, err) } if m.Devices, err = c.Unifi.GetDevices(m.Sites); err != nil { - return m, fmt.Errorf("unifi.GetDevices(): %v", err) + return m, fmt.Errorf("unifi.GetDevices(%v): %v", c.URL, err) } return u.augmentMetrics(c, m), nil @@ -204,19 +200,19 @@ func (u *UnifiPoller) augmentMetrics(c Controller, metrics *metrics.Metrics) *me // GetFilteredSites returns a list of sites to fetch data for. // Omits requested but unconfigured sites. Grabs the full list from the // controller and returns the sites provided in the config file. -func (u *UnifiPoller) GetFilteredSites(config Controller) (unifi.Sites, error) { +func (u *UnifiPoller) GetFilteredSites(c Controller) (unifi.Sites, error) { var i int - sites, err := config.Unifi.GetSites() + sites, err := c.Unifi.GetSites() if err != nil { return nil, err - } else if len(config.Sites) < 1 || StringInSlice("all", config.Sites) { + } else if len(c.Sites) < 1 || StringInSlice("all", c.Sites) { return sites, nil } for _, s := range sites { // Only include valid sites in the request filter. - if StringInSlice(s.Name, config.Sites) { + if StringInSlice(s.Name, c.Sites) { sites[i] = s i++ } From 32038c5bac3ee2c39b0cc5777a97f6cca231ebb4 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sat, 14 Dec 2019 18:33:11 -0800 Subject: [PATCH 05/41] put gitignore back --- integrations/inputunifi/.gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/inputunifi/.gitignore b/integrations/inputunifi/.gitignore index f281b1b0..118e4d74 100644 --- a/integrations/inputunifi/.gitignore +++ b/integrations/inputunifi/.gitignore @@ -11,7 +11,7 @@ /unifi-poller.*.linux /unifi-poller.rb *.sha256 -#/vendor +/vendor .DS_Store *~ /package_build_* From fa185fb3399edc24ccf8dfba05975d900418c771 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sat, 14 Dec 2019 18:55:10 -0800 Subject: [PATCH 06/41] fixes --- integrations/inputunifi/pkg/poller/config.go | 6 +-- integrations/inputunifi/pkg/poller/unifi.go | 49 +++++++++++++------- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/integrations/inputunifi/pkg/poller/config.go b/integrations/inputunifi/pkg/poller/config.go index c77bdff6..b0266405 100644 --- a/integrations/inputunifi/pkg/poller/config.go +++ b/integrations/inputunifi/pkg/poller/config.go @@ -61,9 +61,9 @@ type Controller struct { VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` SaveIDS bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` SaveSites bool `json:"save_sites,omitempty" toml:"save_sites,omitempty" xml:"save_sites" yaml:"save_sites"` - User string `json:"unifi_user,omitempty" toml:"unifi_user,omitempty" xml:"unifi_user" yaml:"unifi_user"` - Pass string `json:"unifi_pass,omitempty" toml:"unifi_pass,omitempty" xml:"unifi_pass" yaml:"unifi_pass"` - URL string `json:"unifi_url,omitempty" toml:"unifi_url,omitempty" xml:"unifi_url" yaml:"unifi_url"` + User string `json:"user,omitempty" toml:"user,omitempty" xml:"user" yaml:"user"` + Pass string `json:"pass,omitempty" toml:"pass,omitempty" xml:"pass" yaml:"pass"` + URL string `json:"url,omitempty" toml:"url,omitempty" xml:"url" yaml:"url"` Sites []string `json:"sites,omitempty" toml:"sites,omitempty" xml:"sites" yaml:"sites"` Unifi *unifi.Unifi `json:"-" toml:"-" xml:"-" yaml:"-"` } diff --git a/integrations/inputunifi/pkg/poller/unifi.go b/integrations/inputunifi/pkg/poller/unifi.go index c602293c..03435a2b 100644 --- a/integrations/inputunifi/pkg/poller/unifi.go +++ b/integrations/inputunifi/pkg/poller/unifi.go @@ -10,45 +10,45 @@ import ( ) // GetUnifi returns a UniFi controller interface. -func (u *UnifiPoller) GetUnifi(config Controller) error { +func (u *UnifiPoller) GetUnifi(c Controller) error { var err error u.Lock() defer u.Unlock() - if config.Unifi != nil { - config.Unifi.CloseIdleConnections() + if c.Unifi != nil { + c.Unifi.CloseIdleConnections() } - // Create an authenticated session to the Unifi Controller. - config.Unifi, err = unifi.NewUnifi(&unifi.Config{ - User: config.User, - Pass: config.Pass, - URL: config.URL, - VerifySSL: config.VerifySSL, + c.Unifi, err = unifi.NewUnifi(&unifi.Config{ + User: c.User, + Pass: c.Pass, + URL: c.URL, + VerifySSL: c.VerifySSL, ErrorLog: u.LogErrorf, // Log all errors. DebugLog: u.LogDebugf, // Log debug messages. }) + if err != nil { - config.Unifi = nil + c.Unifi = nil return fmt.Errorf("unifi controller: %v", err) } - u.LogDebugf("Authenticated with controller successfully, %s", config.URL) + u.LogDebugf("Authenticated with controller successfully, %s", c.URL) - return u.CheckSites(config) + return u.CheckSites(c) } // CheckSites makes sure the list of provided sites exists on the controller. // This does not run in Lambda (run-once) mode. -func (u *UnifiPoller) CheckSites(config Controller) error { +func (u *UnifiPoller) CheckSites(c Controller) error { if strings.Contains(strings.ToLower(u.Config.Mode), "lambda") { return nil // Skip this in lambda mode. } u.LogDebugf("Checking Controller Sites List") - sites, err := config.Unifi.GetSites() + sites, err := c.Unifi.GetSites() if err != nil { return err } @@ -60,13 +60,13 @@ func (u *UnifiPoller) CheckSites(config Controller) error { } u.Logf("Found %d site(s) on controller: %v", len(msg), strings.Join(msg, ", ")) - if StringInSlice("all", config.Sites) { - config.Sites = []string{"all"} + if StringInSlice("all", c.Sites) { + c.Sites = []string{"all"} return nil } FIRST: - for _, s := range config.Sites { + for _, s := range c.Sites { for _, site := range sites { if s == site.Name { continue FIRST @@ -97,9 +97,22 @@ func (u *UnifiPoller) CollectMetrics() (metrics *metrics.Metrics, err error) { } } + if m == nil { + continue + } + metrics.Sites = append(metrics.Sites, m.Sites...) metrics.Clients = append(metrics.Clients, m.Clients...) metrics.IDSList = append(metrics.IDSList, m.IDSList...) + + if m.Devices == nil { + continue + } + + if metrics.Devices == nil { + metrics.Devices = &unifi.Devices{} + } + metrics.UAPs = append(metrics.UAPs, m.UAPs...) metrics.USGs = append(metrics.USGs, m.USGs...) metrics.USWs = append(metrics.USWs, m.USWs...) @@ -134,7 +147,7 @@ func (u *UnifiPoller) collectController(c Controller) (*metrics.Metrics, error) } if c.SaveIDS { - m.IDSList, err = c.Unifi.GetIDS(m.Sites, time.Now().Add(u.Config.Interval.Duration), time.Now()) + m.IDSList, err = c.Unifi.GetIDS(m.Sites, time.Now().Add(2*time.Minute), time.Now()) if err != nil { return m, fmt.Errorf("unifi.GetIDS(%v): %v", c.URL, err) } From c5755b999351f60ccd868a56d7d33468dc50818a Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sat, 14 Dec 2019 19:37:41 -0800 Subject: [PATCH 07/41] update docs --- integrations/inputunifi/examples/MANUAL.md | 34 ++++++++----------- .../inputunifi/examples/up.conf.example | 30 ++++++++-------- .../inputunifi/examples/up.json.example | 18 +++++----- .../inputunifi/examples/up.xml.example | 20 +++++------ .../inputunifi/examples/up.yaml.example | 19 ++++++----- integrations/inputunifi/pkg/poller/config.go | 1 + integrations/inputunifi/pkg/poller/start.go | 24 ++++++++----- 7 files changed, 76 insertions(+), 70 deletions(-) diff --git a/integrations/inputunifi/examples/MANUAL.md b/integrations/inputunifi/examples/MANUAL.md index 8df5e733..0b1a8860 100644 --- a/integrations/inputunifi/examples/MANUAL.md +++ b/integrations/inputunifi/examples/MANUAL.md @@ -65,14 +65,6 @@ is provided so the application can be easily adapted to any environment. `Config File Parameters` - sites default: ["all"] - This list of strings should represent the names of sites on the UniFi - controller that will be polled for data. Pass `all` in the list to - poll all sites. On startup, the application prints out all site names - found in the controller; they're cryptic, but they have the human-name - next to them. The cryptic names go into the config file `sites` list. - The controller's first site is not cryptic and is named `default`. - interval default: 30s How often to poll the controller for updated client and device data. The UniFi Controller only updates traffic stats about every 30-60 seconds. @@ -136,35 +128,39 @@ is provided so the application can be easily adapted to any environment. influx_insecure_ssl default: false Setting this to true will allow use of InfluxDB with an invalid SSL certificate. - unifi_url default: https://127.0.0.1:8443 + >>> CONTROLLER FIELDS FOLLOW - you may have multiple controllers: + + sites default: ["all"] + This list of strings should represent the names of sites on the UniFi + controller that will be polled for data. Pass `all` in the list to + poll all sites. On startup, the application prints out all site names + found in the controller; they're cryptic, but they have the human-name + next to them. The cryptic names go into the config file `sites` list. + The controller's first site is not cryptic and is named `default`. + + url default: https://127.0.0.1:8443 This is the URL where the UniFi Controller is available. - unifi_user default: influxdb + user default: influxdb Username used to authenticate with UniFi controller. This should be a special service account created on the control with read-only access. - unifi_user no default ENV: UNIFI_PASSWORD + user no default Password used to authenticate with UniFi controller. This can also be set in an environment variable instead of a configuration file. - save_ids default: false + save_ids default: false Setting this parameter to true will enable collection of Intrusion Detection System data. IDS and IPS are the same data set. This is off by default because most controllers do not have this enabled. It also creates a lot of new metrics from controllers with a lot of IDS entries. IDS data does not contain metrics, so this doesn't work with Prometheus. - save_sites + save_sites default: true Setting this parameter to false will disable saving Network Site data. This data populates the Sites dashboard, and this setting affects influx and prometheus. - reauthenticate default: false - Setting this parameter to true will make UniFi Poller send a new login - request on every interval. This generates a new cookie. Some controller - or reverse proxy configurations require this. Do not enable it unless - your configuration causes the poller to be logged out after some time. - verify_ssl default: false If your UniFi controller has a valid SSL certificate, you can enable this option to validate it. Otherwise, any SSL certificate is valid. diff --git a/integrations/inputunifi/examples/up.conf.example b/integrations/inputunifi/examples/up.conf.example index a403f5b7..31713a84 100644 --- a/integrations/inputunifi/examples/up.conf.example +++ b/integrations/inputunifi/examples/up.conf.example @@ -2,10 +2,6 @@ # commented lines are defaults, uncomment to change. # ######################################################## -# If the controller has more than one site, specify which sites to poll here. -# Set this to ["default"] to poll only the first site on the controller. -# A setting of ["all"] will poll all sites; this works if you only have 1 site too. -sites = ["all"] # The UniFi Controller only updates traffic stats about every 30 seconds. # Setting this to something lower may lead to "zeros" in your data. @@ -51,18 +47,22 @@ influx_db = "unifi" # If your InfluxDB uses an invalid SSL cert, set this to true. influx_insecure_ssl = false -# Make a read-only user in the UniFi Admin Settings. -unifi_user = "influx" -# You may also set env variable UNIFI_PASSWORD instead of putting this in the config. -unifi_pass = "4BB9345C-2341-48D7-99F5-E01B583FF77F" -unifi_url = "https://127.0.0.1:8443" +# You may repeat the following section to poll additional controllers. -# Some controllers or reverse proxy configurations do not allow cookies to be -# re-user on every request (every interval). This setting provides a workaround -# That causes the poller to re-auth (login) to the controller on every interval. -# Only enable this if you get login errors after 30 seconds. This will generate -# a few more requests to your controller every interval. -reauthenticate = false +[[controller]] +# Friendly name used in dashboards. +name = "" + +url = "https://127.0.0.1:8443" +# Make a read-only user in the UniFi Admin Settings. +user = "influx" +# You may also set env variable UNIFI_PASSWORD instead of putting this in the config. +pass = "4BB9345C-2341-48D7-99F5-E01B583FF77F" + +# If the controller has more than one site, specify which sites to poll here. +# Set this to ["default"] to poll only the first site on the controller. +# A setting of ["all"] will poll all sites; this works if you only have 1 site too. +sites = ["all"] # Enable collection of Intrusion Detection System Data (InfluxDB only). # Only useful if IDS or IPS are enabled on one of the sites. diff --git a/integrations/inputunifi/examples/up.json.example b/integrations/inputunifi/examples/up.json.example index 7842987f..07ecd509 100644 --- a/integrations/inputunifi/examples/up.json.example +++ b/integrations/inputunifi/examples/up.json.example @@ -1,5 +1,4 @@ { - "sites": ["all"], "interval": "30s", "debug": false, "quiet": false, @@ -10,11 +9,14 @@ "influx_pass": "unifi", "influx_db": "unifi", "influx_insecure_ssl": false, - "unifi_user": "influx", - "unifi_pass": "", - "unifi_url": "https://127.0.0.1:8443", - "save_ids": false, - "save_sites": true, - "reauthenticate": false, - "verify_ssl": false + "controller": [{ + "name": "", + "user": "influx", + "pass": "", + "url": "https://127.0.0.1:8443", + "sites": ["all"], + "save_ids": false, + "save_sites": true, + "verify_ssl": false + }] } diff --git a/integrations/inputunifi/examples/up.xml.example b/integrations/inputunifi/examples/up.xml.example index e4523e3a..551ff8b7 100644 --- a/integrations/inputunifi/examples/up.xml.example +++ b/integrations/inputunifi/examples/up.xml.example @@ -7,7 +7,6 @@ --> - all 60s false @@ -21,14 +20,13 @@ http://127.0.0.1:8086 unifi false - - influx - - https://127.0.0.1:8443 - false - false - - false - true - + + all + influx + + https://127.0.0.1:8443 + false + false + true + diff --git a/integrations/inputunifi/examples/up.yaml.example b/integrations/inputunifi/examples/up.yaml.example index 0eac5a24..b076feb4 100644 --- a/integrations/inputunifi/examples/up.yaml.example +++ b/integrations/inputunifi/examples/up.yaml.example @@ -3,8 +3,6 @@ # provided values are defaults. See up.conf.example! # ######################################################## --- -sites: - - all interval: "30s" debug: false @@ -19,10 +17,13 @@ influx_pass: "unifi" influx_db: "unifi" influx_insecure_ssl: false -unifi_user: "influx" -unifi_pass: "" -unifi_url: "https://127.0.0.1:8443" -reauthenticate: false -verify_ssl: false -save_ids: false -save_sites: true +controller: + - name: "" + user: "influx" + pass: "" + url: "https://127.0.0.1:8443" + sites: + - all + verify_ssl: false + save_ids: false + save_sites: true diff --git a/integrations/inputunifi/pkg/poller/config.go b/integrations/inputunifi/pkg/poller/config.go index b0266405..04d3f1a4 100644 --- a/integrations/inputunifi/pkg/poller/config.go +++ b/integrations/inputunifi/pkg/poller/config.go @@ -61,6 +61,7 @@ type Controller struct { VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` SaveIDS bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` SaveSites bool `json:"save_sites,omitempty" toml:"save_sites,omitempty" xml:"save_sites" yaml:"save_sites"` + Name string `json:"name" toml:"name" xml:"name,attr" yaml:"name"` User string `json:"user,omitempty" toml:"user,omitempty" xml:"user" yaml:"user"` Pass string `json:"pass,omitempty" toml:"pass,omitempty" xml:"pass" yaml:"pass"` URL string `json:"url,omitempty" toml:"url,omitempty" xml:"url" yaml:"url"` diff --git a/integrations/inputunifi/pkg/poller/start.go b/integrations/inputunifi/pkg/poller/start.go index 54dc0731..154f0395 100644 --- a/integrations/inputunifi/pkg/poller/start.go +++ b/integrations/inputunifi/pkg/poller/start.go @@ -17,13 +17,6 @@ import ( func New() *UnifiPoller { return &UnifiPoller{ Config: &Config{ - Controllers: []Controller{{ - Sites: []string{"all"}, - User: defaultUnifiUser, - Pass: "", - URL: defaultUnifiURL, - SaveSites: true, - }}, InfluxURL: defaultInfluxURL, InfluxUser: defaultInfluxUser, InfluxPass: defaultInfluxPass, @@ -65,6 +58,17 @@ func (u *UnifiPoller) Start() error { if _, err := config.ParseENV(u.Config, ENVConfigPrefix); err != nil { return err } + + if len(u.Config.Controllers) < 1 { + u.Config.Controllers = []Controller{{ + Sites: []string{"all"}, + User: defaultUnifiUser, + Pass: "", + URL: defaultUnifiURL, + SaveSites: true, + }} + } + if u.Flag.DumpJSON != "" { return u.DumpJSONPayload() } @@ -99,7 +103,11 @@ func (f *Flag) Parse(args []string) { // 2. Run the collector one time and report the metrics to influxdb. (lambda) // 3. Start a web server and wait for Prometheus to poll the application for metrics. func (u *UnifiPoller) Run() error { - for _, c := range u.Config.Controllers { + for i, c := range u.Config.Controllers { + if c.Name == "" { + u.Config.Controllers[i].Name = c.URL + } + switch err := u.GetUnifi(c); err { case nil: u.Logf("Polling UniFi Controller at %s v%s as user %s. Sites: %v", From 631f7cb720787e0a180f17e1eade7f2300aa7207 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sat, 14 Dec 2019 19:42:03 -0800 Subject: [PATCH 08/41] remove naked return --- integrations/inputunifi/pkg/poller/unifi.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/integrations/inputunifi/pkg/poller/unifi.go b/integrations/inputunifi/pkg/poller/unifi.go index 03435a2b..5ba11127 100644 --- a/integrations/inputunifi/pkg/poller/unifi.go +++ b/integrations/inputunifi/pkg/poller/unifi.go @@ -79,8 +79,9 @@ FIRST: } // CollectMetrics grabs all the measurements from a UniFi controller and returns them. -func (u *UnifiPoller) CollectMetrics() (metrics *metrics.Metrics, err error) { - var errs []string +func (u *UnifiPoller) CollectMetrics() (*metrics.Metrics, error) { + errs := []string{} + metrics := &metrics.Metrics{} for _, c := range u.Config.Controllers { m, err := u.collectController(c) @@ -119,11 +120,13 @@ func (u *UnifiPoller) CollectMetrics() (metrics *metrics.Metrics, err error) { metrics.UDMs = append(metrics.UDMs, m.UDMs...) } + var err error + if len(errs) > 0 { err = fmt.Errorf(strings.Join(errs, ", ")) } - return + return metrics, err } func (u *UnifiPoller) collectController(c Controller) (*metrics.Metrics, error) { From 34e2007aca275280149bbf1aa49bf0ae97caa42e Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sat, 14 Dec 2019 20:01:51 -0800 Subject: [PATCH 09/41] re-do re-auth --- integrations/inputunifi/pkg/poller/influx.go | 15 +++++++ integrations/inputunifi/pkg/poller/start.go | 14 ------- integrations/inputunifi/pkg/poller/unifi.go | 41 ++++++++++++-------- 3 files changed, 39 insertions(+), 31 deletions(-) diff --git a/integrations/inputunifi/pkg/poller/influx.go b/integrations/inputunifi/pkg/poller/influx.go index e9433460..52690c13 100644 --- a/integrations/inputunifi/pkg/poller/influx.go +++ b/integrations/inputunifi/pkg/poller/influx.go @@ -2,6 +2,7 @@ package poller import ( "fmt" + "log" "time" "github.com/davidnewhall/unifi-poller/pkg/influxunifi" @@ -29,6 +30,20 @@ func (u *UnifiPoller) GetInfluxDB() (err error) { return nil } +// PollController runs forever, polling UniFi and pushing to InfluxDB +// This is started by Run() or RunBoth() after everything checks out. +func (u *UnifiPoller) PollController() { + interval := u.Config.Interval.Round(time.Second) + log.Printf("[INFO] Everything checks out! Poller started, InfluxDB interval: %v", interval) + + ticker := time.NewTicker(interval) + for u.LastCheck = range ticker.C { + if err := u.CollectAndProcess(); err != nil { + u.LogErrorf("%v", err) + } + } +} + // CollectAndProcess collects measurements and then reports them to InfluxDB // Can be called once or in a ticker loop. This function and all the ones below // handle their own logging. An error is returned so the calling function may diff --git a/integrations/inputunifi/pkg/poller/start.go b/integrations/inputunifi/pkg/poller/start.go index 154f0395..2a81ac03 100644 --- a/integrations/inputunifi/pkg/poller/start.go +++ b/integrations/inputunifi/pkg/poller/start.go @@ -131,17 +131,3 @@ func (u *UnifiPoller) Run() error { return u.RunPrometheus() } } - -// PollController runs forever, polling UniFi and pushing to InfluxDB -// This is started by Run() or RunBoth() after everything checks out. -func (u *UnifiPoller) PollController() { - interval := u.Config.Interval.Round(time.Second) - log.Printf("[INFO] Everything checks out! Poller started, InfluxDB interval: %v", interval) - - ticker := time.NewTicker(interval) - for u.LastCheck = range ticker.C { - if err := u.CollectAndProcess(); err != nil { - u.LogErrorf("%v", err) - } - } -} diff --git a/integrations/inputunifi/pkg/poller/unifi.go b/integrations/inputunifi/pkg/poller/unifi.go index 5ba11127..0e598716 100644 --- a/integrations/inputunifi/pkg/poller/unifi.go +++ b/integrations/inputunifi/pkg/poller/unifi.go @@ -84,18 +84,9 @@ func (u *UnifiPoller) CollectMetrics() (*metrics.Metrics, error) { metrics := &metrics.Metrics{} for _, c := range u.Config.Controllers { - m, err := u.collectController(c) + m, err := u.checkAndPollController(c) if err != nil { - u.LogErrorf("collecting metrics from %s: %v", c.URL, err) - u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) - - if err := u.GetUnifi(c); err != nil { - u.LogErrorf("re-authenticating to %s: %v", c.URL, err) - errs = append(errs, err.Error()) - } else if m, err = u.collectController(c); err != nil { - u.LogErrorf("collecting metrics from %s: %v", c.URL, err) - errs = append(errs, err.Error()) - } + errs = append(errs, err.Error()) } if m == nil { @@ -129,19 +120,35 @@ func (u *UnifiPoller) CollectMetrics() (*metrics.Metrics, error) { return metrics, err } -func (u *UnifiPoller) collectController(c Controller) (*metrics.Metrics, error) { - var err error - +func (u *UnifiPoller) checkAndPollController(c Controller) (*metrics.Metrics, error) { if c.Unifi == nil { - // Some users need to re-auth every interval because the cookie times out. - // Sometimes we hit this path when the controller dies. - u.Logf("Re-authenticating to UniFi Controller: %v", c.URL) + u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) if err := u.GetUnifi(c); err != nil { + u.LogErrorf("re-authenticating to %s: %v", c.URL, err) return nil, err } } + m, err := u.collectController(c) + if err == nil { + return m, nil + } + + u.LogErrorf("collecting metrics %v", err) + u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) + + if err := u.GetUnifi(c); err != nil { + u.LogErrorf("re-authenticating to %s: %v", c.URL, err) + return nil, err + } + + return u.collectController(c) +} + +func (u *UnifiPoller) collectController(c Controller) (*metrics.Metrics, error) { + var err error + m := &metrics.Metrics{TS: u.LastCheck} // At this point, it's the Current Check. // Get the sites we care about. From b97afded1e7a1863e6179389a6329ac9b26d3a47 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sun, 15 Dec 2019 02:52:43 -0800 Subject: [PATCH 10/41] make output plugins call in to initialize --- integrations/inputunifi/.metadata.sh | 7 +- integrations/inputunifi/Makefile | 2 +- integrations/inputunifi/examples/MANUAL.md | 59 +------- .../inputunifi/examples/up.conf.example | 48 +++--- .../inputunifi/examples/up.json.example | 49 +++--- .../inputunifi/examples/up.xml.example | 29 ++-- .../inputunifi/examples/up.yaml.example | 25 ++-- integrations/inputunifi/main.go | 3 + .../inputunifi/pkg/influxunifi/metrics.go | 141 +++++++++++++++--- .../inputunifi/pkg/influxunifi/report.go | 8 +- .../inputunifi/pkg/metrics/metrics.go | 16 -- integrations/inputunifi/pkg/poller/config.go | 81 ++++++---- integrations/inputunifi/pkg/poller/influx.go | 79 ---------- integrations/inputunifi/pkg/poller/outputs.go | 72 +++++++++ .../inputunifi/pkg/poller/prometheus.go | 53 ------- integrations/inputunifi/pkg/poller/start.go | 48 ++---- integrations/inputunifi/pkg/poller/unifi.go | 20 +-- .../inputunifi/pkg/promunifi/collector.go | 140 ++++++++++------- .../inputunifi/pkg/promunifi/report.go | 27 ++-- 19 files changed, 461 insertions(+), 446 deletions(-) delete mode 100644 integrations/inputunifi/pkg/metrics/metrics.go delete mode 100644 integrations/inputunifi/pkg/poller/influx.go create mode 100644 integrations/inputunifi/pkg/poller/outputs.go delete mode 100644 integrations/inputunifi/pkg/poller/prometheus.go diff --git a/integrations/inputunifi/.metadata.sh b/integrations/inputunifi/.metadata.sh index 37506a59..11ba0838 100755 --- a/integrations/inputunifi/.metadata.sh +++ b/integrations/inputunifi/.metadata.sh @@ -32,11 +32,6 @@ SOURCE_URL="https://${IMPORT_PATH}" # Used for documentation links. URL="${SOURCE_URL}" -# This parameter is passed in as -X to go build. Used to override the Version variable in a package. -# This makes a path like github.com/user/hello-world/helloworld.Version=1.3.3 -# Name the Version-containing library the same as the github repo, without dashes. -VERSION_PATH="${IMPORT_PATH}/pkg/poller.Version" - # Dynamic. Recommend not changing. VVERSION=$(git describe --abbrev=0 --tags $(git rev-list --tags --max-count=1)) VERSION="$(echo $VVERSION | tr -d v | grep -E '^\S+$' || echo development)" @@ -51,4 +46,4 @@ COMMIT="$(git rev-parse --short HEAD || echo 0)" # This is a custom download path for homebrew formula. SOURCE_PATH=https://golift.io/${BINARY}/archive/v${VERSION}.tar.gz -export IMPORT_PATH SOURCE_URL URL VERSION_PATH VVERSION VERSION ITERATION DATE BRANCH COMMIT SOURCE_PATH +export IMPORT_PATH SOURCE_URL URL VVERSION VERSION ITERATION DATE BRANCH COMMIT SOURCE_PATH diff --git a/integrations/inputunifi/Makefile b/integrations/inputunifi/Makefile index 063fdafa..4cf96dc5 100644 --- a/integrations/inputunifi/Makefile +++ b/integrations/inputunifi/Makefile @@ -46,7 +46,7 @@ VERSION_LDFLAGS:= \ -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Branch=$(BRANCH) \ -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.BuildDate=$(DATE) \ -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Revision=$(COMMIT) \ - -X $(VERSION_PATH)=$(VERSION)-$(ITERATION) + -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Version=$(VERSION)-$(ITERATION) # Makefile targets follow. diff --git a/integrations/inputunifi/examples/MANUAL.md b/integrations/inputunifi/examples/MANUAL.md index 0b1a8860..012cd343 100644 --- a/integrations/inputunifi/examples/MANUAL.md +++ b/integrations/inputunifi/examples/MANUAL.md @@ -65,10 +65,10 @@ is provided so the application can be easily adapted to any environment. `Config File Parameters` - interval default: 30s - How often to poll the controller for updated client and device data. - The UniFi Controller only updates traffic stats about every 30-60 seconds. - Only works if "mode" (below) is "influx" - other modes do not use interval. +Additional parameters are added by output packages. Parameters can also be set +using environment variables. See the GitHub wiki for more information! + + >>> POLLER FIELDS FOLLOW - you may have multiple controllers: debug default: false This turns on time stamps and line numbers in logs, outputs a few extra @@ -79,56 +79,7 @@ is provided so the application can be easily adapted to any environment. errors will be logged. Using this with debug=true adds line numbers to any error logs. - mode default: "influx" - * Value: influx - This default mode runs this application as a daemon. It will poll - the controller at the configured interval and report measurements to - InfluxDB. Providing an invalid value will run in this default mode. - - * Value: influxlambda - Setting this value will invoke a run-once mode where the application - immediately polls the controller and reports the metrics to InfluxDB. - Then it exits. This mode is useful in an AWS Lambda or a crontab where - the execution timings are controlled. This mode may also be adapted - to run in other collector scripts and apps like telegraf or diamond. - This mode can also be combined with a "test database" in InfluxDB to - give yourself a "test config file" you may run ad-hoc to test changes. - - * Value: prometheus - In this mode the application opens an http interface and exports the - measurements at /metrics for collection by prometheus. Enabling this - mode disables InfluxDB usage entirely. - - * Value: both - Setting the mode to "both" will cause the InfluxDB poller routine to run - along with the Prometheus exporter. You can run both at the same time. - - http_listen default: 0.0.0.0:9130 - This option controls the IP and port the http listener uses when the - mode is set to prometheus. This setting has no effect when other modes - are in use. Metrics become available at the /metrics URI. - - influx_url default: http://127.0.0.1:8086 - This is the URL where the Influx web server is available. - - influx_user default: unifi - Username used to authenticate with InfluxDB. - - influx_pass default: unifi - Password used to authenticate with InfluxDB. - - influx_db default: unifi - Custom database created in InfluxDB to use with this application. - On first setup, log into InfluxDB and create access: - $ influx -host localhost -port 8086 - CREATE DATABASE unifi - CREATE USER unifi WITH PASSWORD 'unifi' WITH ALL PRIVILEGES - GRANT ALL ON unifi TO unifi - - influx_insecure_ssl default: false - Setting this to true will allow use of InfluxDB with an invalid SSL certificate. - - >>> CONTROLLER FIELDS FOLLOW - you may have multiple controllers: + >>> CONTROLLER FIELDS FOLLOW - you may have multiple controllers: sites default: ["all"] This list of strings should represent the names of sites on the UniFi diff --git a/integrations/inputunifi/examples/up.conf.example b/integrations/inputunifi/examples/up.conf.example index 31713a84..75c12941 100644 --- a/integrations/inputunifi/examples/up.conf.example +++ b/integrations/inputunifi/examples/up.conf.example @@ -3,11 +3,7 @@ ######################################################## -# The UniFi Controller only updates traffic stats about every 30 seconds. -# Setting this to something lower may lead to "zeros" in your data. -# If you're getting zeros now, set this to "1m" -interval = "30s" - +[poller] # Turns on line numbers, microsecond logging, and a per-device log. # The default is false, but I personally leave this on at home (four devices). # This may be noisy if you have a lot of devices. It adds one line per device. @@ -17,38 +13,34 @@ debug = false # Recommend enabling debug with this setting for better error logging. quiet = false -# Which mode to run this application in. The default mode is "influx". Providing -# an invalid mode will also result in "influx". In this default mode the application -# runs as a daemon and polls the controller at the configured interval. -# -# Other options: "influxlambda", "prometheus", "both" -# -# Mode "influxlambda" makes the application exit after collecting and reporting metrics -# to InfluxDB one time. This mode requires an external process like an AWS Lambda -# or a simple crontab to keep the timings accurate on UniFi Poller run intervals. -# -# Mode "prometheus" opens an HTTP server on port 9130 and exports the metrics at -# /metrics for polling collection by a prometheus server. This disables influxdb. -# -# Mode "both" runs the Prometheus HTTP server and InfluxDB poller interval at -# the same time. -mode = "influx" +#### OUTPUTS + +[prometheus] +disable = false # This controls on which ip and port /metrics is exported when mode is "prometheus". # This has no effect in other modes. Must contain a colon and port. http_listen = "0.0.0.0:9130" +report_errors = false +[influxdb] +disable = false # InfluxDB does not require auth by default, so the user/password are probably unimportant. -influx_url = "http://127.0.0.1:8086" -influx_user = "unifi" -influx_pass = "unifi" +url = "http://127.0.0.1:8086" +user = "unifi" +pass = "unifi" # Be sure to create this database. -influx_db = "unifi" -# If your InfluxDB uses an invalid SSL cert, set this to true. -influx_insecure_ssl = false +db = "unifi" +# If your InfluxDB uses a valid SSL cert, set this to true. +verify_ssl = false +# The UniFi Controller only updates traffic stats about every 30 seconds. +# Setting this to something lower may lead to "zeros" in your data. +# If you're getting zeros now, set this to "1m" +interval = "30s" + +#### INPUTS # You may repeat the following section to poll additional controllers. - [[controller]] # Friendly name used in dashboards. name = "" diff --git a/integrations/inputunifi/examples/up.json.example b/integrations/inputunifi/examples/up.json.example index 07ecd509..65e4d27e 100644 --- a/integrations/inputunifi/examples/up.json.example +++ b/integrations/inputunifi/examples/up.json.example @@ -1,22 +1,33 @@ { - "interval": "30s", - "debug": false, - "quiet": false, - "mode": "influx", - "http_listen": "0.0.0.0:9130", - "influx_url": "http://127.0.0.1:8086", - "influx_user": "unifi", - "influx_pass": "unifi", - "influx_db": "unifi", - "influx_insecure_ssl": false, - "controller": [{ - "name": "", - "user": "influx", - "pass": "", - "url": "https://127.0.0.1:8443", - "sites": ["all"], - "save_ids": false, - "save_sites": true, - "verify_ssl": false + "poller": { + "debug": false, + "quiet": false + }, + + "prometheus": { + "disable": false, + "http_listen": "0.0.0.0:9130", + "report_errors": false + }, + + "influxdb": { + "disable": false, + "url": "http://127.0.0.1:8086", + "user": "unifi", + "pass": "unifi", + "db": "unifi", + "verify_ssl": false, + "interval": "30s" + }, + + "controller": [{ + "name": "", + "user": "influx", + "pass": "", + "url": "https://127.0.0.1:8443", + "sites": ["all"], + "save_ids": false, + "save_sites": true, + "verify_ssl": false }] } diff --git a/integrations/inputunifi/examples/up.xml.example b/integrations/inputunifi/examples/up.xml.example index 551ff8b7..710c01ba 100644 --- a/integrations/inputunifi/examples/up.xml.example +++ b/integrations/inputunifi/examples/up.xml.example @@ -5,21 +5,23 @@ # provided values are defaults. See up.conf.example! # ####################################################### --> - + - 60s + + 0.0.0.0:9130 + false + - false - false + + 30s + http://127.0.0.1:8086 + unifi + unifi + unifi + false + - influx - 0.0.0.0:9130 - - unifi - unifi - http://127.0.0.1:8086 - unifi - false + all influx @@ -29,4 +31,5 @@ false true - + + diff --git a/integrations/inputunifi/examples/up.yaml.example b/integrations/inputunifi/examples/up.yaml.example index b076feb4..611c5fa8 100644 --- a/integrations/inputunifi/examples/up.yaml.example +++ b/integrations/inputunifi/examples/up.yaml.example @@ -3,19 +3,24 @@ # provided values are defaults. See up.conf.example! # ######################################################## --- -interval: "30s" -debug: false -quiet: false +poller: + debug: false + quiet: false -mode: "influx" -http_listen: "0.0.0.0:9130" +prometheus: + disable: false + http_listen: "0.0.0.0:9130" + report_errors: false -influx_url: "http://127.0.0.1:8086" -influx_user: "unifi" -influx_pass: "unifi" -influx_db: "unifi" -influx_insecure_ssl: false +influxdb: + disable: false + interval: "30s" + url: "http://127.0.0.1:8086" + user: "unifi" + pass: "unifi" + db: "unifi" + verify_ssl: false controller: - name: "" diff --git a/integrations/inputunifi/main.go b/integrations/inputunifi/main.go index 332bfd5d..be0fe13b 100644 --- a/integrations/inputunifi/main.go +++ b/integrations/inputunifi/main.go @@ -4,6 +4,9 @@ import ( "log" "github.com/davidnewhall/unifi-poller/pkg/poller" + // Enable output plugins! + _ "github.com/davidnewhall/unifi-poller/pkg/influxunifi" + _ "github.com/davidnewhall/unifi-poller/pkg/promunifi" ) // Keep it simple. diff --git a/integrations/inputunifi/pkg/influxunifi/metrics.go b/integrations/inputunifi/pkg/influxunifi/metrics.go index 90a95a31..6775ac06 100644 --- a/integrations/inputunifi/pkg/influxunifi/metrics.go +++ b/integrations/inputunifi/pkg/influxunifi/metrics.go @@ -1,29 +1,47 @@ -// Package influx provides the methods to turn UniFi measurements into influx +// Package influxunifi provides the methods to turn UniFi measurements into influx // data-points with appropriate tags and fields. package influxunifi import ( "crypto/tls" "fmt" + "log" "time" - "github.com/davidnewhall/unifi-poller/pkg/metrics" + "github.com/davidnewhall/unifi-poller/pkg/poller" influx "github.com/influxdata/influxdb1-client/v2" + conf "golift.io/config" +) + +const ( + defaultInterval = 30 * time.Second + defaultInfluxDB = "unifi" + defaultInfluxUser = "unifi" + defaultInfluxURL = "http://127.0.0.1:8086" ) // Config defines the data needed to store metrics in InfluxDB type Config struct { - Database string - URL string - User string - Pass string - BadSSL bool + Interval conf.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` + Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` + VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` + URL string `json:"url,omitempty" toml:"url,omitempty" xml:"url" yaml:"url"` + User string `json:"user,omitempty" toml:"user,omitempty" xml:"user" yaml:"user"` + Pass string `json:"pass,omitempty" toml:"pass,omitempty" xml:"pass" yaml:"pass"` + DB string `json:"db,omitempty" toml:"db,omitempty" xml:"db" yaml:"db"` +} + +// InfluxDB allows the data to be nested in the config file. +type InfluxDB struct { + Config Config `json:"influxdb" toml:"influxdb" xml:"influxdb" yaml:"influxdb"` } // InfluxUnifi is returned by New() after you provide a Config. type InfluxUnifi struct { - cf *Config - influx influx.Client + Collector poller.Collect + influx influx.Client + LastCheck time.Time + *InfluxDB } type metric struct { @@ -32,26 +50,101 @@ type metric struct { Fields map[string]interface{} } -// New returns an InfluxDB interface. -func New(c *Config) (*InfluxUnifi, error) { - i, err := influx.NewHTTPClient(influx.HTTPConfig{ - Addr: c.URL, - Username: c.User, - Password: c.Pass, - TLSConfig: &tls.Config{InsecureSkipVerify: c.BadSSL}, +func init() { + u := &InfluxUnifi{InfluxDB: &InfluxDB{}, LastCheck: time.Now()} + poller.NewOutput(&poller.Output{ + Name: "influxdb", + Config: u.InfluxDB, + Method: u.Run, }) - return &InfluxUnifi{cf: c, influx: i}, err +} + +// PollController runs forever, polling UniFi and pushing to InfluxDB +// This is started by Run() or RunBoth() after everything checks out. +func (u *InfluxUnifi) PollController() { + interval := u.Config.Interval.Round(time.Second) + log.Printf("[INFO] Everything checks out! Poller started, InfluxDB interval: %v", interval) + + ticker := time.NewTicker(interval) + for u.LastCheck = range ticker.C { + metrics, err := u.Collector.Metrics() + if err != nil { + u.Collector.LogErrorf("%v", err) + continue + } + + report, err := u.ReportMetrics(metrics) + if err != nil { + // XXX: reset and re-auth? not sure.. + u.Collector.LogErrorf("%v", err) + continue + } + + u.LogInfluxReport(report) + } +} + +// Run runs a ticker to poll the unifi server and update influxdb. +func (u *InfluxUnifi) Run(c poller.Collect) error { + var err error + + if u.Config.Disable { + return nil + } + + u.Collector = c + u.setConfigDefaults() + + u.influx, err = influx.NewHTTPClient(influx.HTTPConfig{ + Addr: u.Config.URL, + Username: u.Config.User, + Password: u.Config.Pass, + TLSConfig: &tls.Config{InsecureSkipVerify: !u.Config.VerifySSL}, + }) + if err != nil { + return err + } + + u.PollController() + + return nil +} + +func (u *InfluxUnifi) setConfigDefaults() { + if u.Config.URL == "" { + u.Config.URL = defaultInfluxURL + } + + if u.Config.User == "" { + u.Config.User = defaultInfluxUser + } + + if u.Config.Pass == "" { + u.Config.Pass = defaultInfluxUser + } + + if u.Config.DB == "" { + u.Config.DB = defaultInfluxDB + } + + if u.Config.Interval.Duration == 0 { + u.Config.Interval = conf.Duration{Duration: defaultInterval} + } else if u.Config.Interval.Duration < defaultInterval/2 { + u.Config.Interval = conf.Duration{Duration: defaultInterval / 2} + } + + u.Config.Interval = conf.Duration{Duration: u.Config.Interval.Duration.Round(time.Second)} } // ReportMetrics batches all device and client data into influxdb data points. // Call this after you've collected all the data you care about. // Returns an error if influxdb calls fail, otherwise returns a report. -func (u *InfluxUnifi) ReportMetrics(m *metrics.Metrics) (*Report, error) { +func (u *InfluxUnifi) ReportMetrics(m *poller.Metrics) (*Report, error) { r := &Report{Metrics: m, ch: make(chan *metric), Start: time.Now()} defer close(r.ch) // Make a new Influx Points Batcher. var err error - r.bp, err = influx.NewBatchPoints(influx.BatchPointsConfig{Database: u.cf.Database}) + r.bp, err = influx.NewBatchPoints(influx.BatchPointsConfig{Database: u.Config.DB}) if err != nil { return nil, fmt.Errorf("influx.NewBatchPoints: %v", err) } @@ -140,3 +233,13 @@ func (u *InfluxUnifi) loopPoints(r report) { } }() } + +// LogInfluxReport writes a log message after exporting to influxdb. +func (u *InfluxUnifi) LogInfluxReport(r *Report) { + idsMsg := fmt.Sprintf("IDS Events: %d, ", len(r.Metrics.IDSList)) + u.Collector.Logf("UniFi Metrics Recorded. Sites: %d, Clients: %d, "+ + "UAP: %d, USG/UDM: %d, USW: %d, %sPoints: %d, Fields: %d, Errs: %d, Elapsed: %v", + len(r.Metrics.Sites), len(r.Metrics.Clients), len(r.Metrics.UAPs), + len(r.Metrics.UDMs)+len(r.Metrics.USGs), len(r.Metrics.USWs), idsMsg, r.Total, + r.Fields, len(r.Errors), r.Elapsed.Round(time.Millisecond)) +} diff --git a/integrations/inputunifi/pkg/influxunifi/report.go b/integrations/inputunifi/pkg/influxunifi/report.go index 5d1e760d..3fdf77a9 100644 --- a/integrations/inputunifi/pkg/influxunifi/report.go +++ b/integrations/inputunifi/pkg/influxunifi/report.go @@ -4,13 +4,13 @@ import ( "sync" "time" - "github.com/davidnewhall/unifi-poller/pkg/metrics" + "github.com/davidnewhall/unifi-poller/pkg/poller" influx "github.com/influxdata/influxdb1-client/v2" ) // Report is returned to the calling procedure after everything is processed. type Report struct { - Metrics *metrics.Metrics + Metrics *poller.Metrics Errors []error Total int Fields int @@ -28,10 +28,10 @@ type report interface { send(m *metric) error(err error) batch(m *metric, pt *influx.Point) - metrics() *metrics.Metrics + metrics() *poller.Metrics } -func (r *Report) metrics() *metrics.Metrics { +func (r *Report) metrics() *poller.Metrics { return r.Metrics } diff --git a/integrations/inputunifi/pkg/metrics/metrics.go b/integrations/inputunifi/pkg/metrics/metrics.go deleted file mode 100644 index 2d38f54e..00000000 --- a/integrations/inputunifi/pkg/metrics/metrics.go +++ /dev/null @@ -1,16 +0,0 @@ -package metrics - -import ( - "time" - - "golift.io/unifi" -) - -// Metrics is a type shared by the exporting and reporting packages. -type Metrics struct { - TS time.Time - unifi.Sites - unifi.IDSList - unifi.Clients - *unifi.Devices -} diff --git a/integrations/inputunifi/pkg/poller/config.go b/integrations/inputunifi/pkg/poller/config.go index 04d3f1a4..0d6fe126 100644 --- a/integrations/inputunifi/pkg/poller/config.go +++ b/integrations/inputunifi/pkg/poller/config.go @@ -12,26 +12,17 @@ import ( "sync" "time" - "github.com/davidnewhall/unifi-poller/pkg/influxunifi" "github.com/spf13/pflag" "golift.io/config" "golift.io/unifi" ) -// Version is injected by the Makefile -var Version = "development" - +// App defaults in case they're missing from the config. const ( - // App defaults in case they're missing from the config. - appName = "unifi-poller" - defaultInterval = 30 * time.Second - defaultInfluxDB = "unifi" - defaultInfluxUser = "unifi" - defaultInfluxPass = "unifi" - defaultInfluxURL = "http://127.0.0.1:8086" - defaultUnifiUser = "influx" - defaultUnifiURL = "https://127.0.0.1:8443" - defaultHTTPListen = "0.0.0.0:9130" + // AppName is the name of the application. + AppName = "unifi-poller" + defaultUnifiUser = "influx" + defaultUnifiURL = "https://127.0.0.1:8443" ) // ENVConfigPrefix is the prefix appended to an env variable tag @@ -40,10 +31,8 @@ const ENVConfigPrefix = "UP" // UnifiPoller contains the application startup data, and auth info for UniFi & Influx. type UnifiPoller struct { - Influx *influxunifi.InfluxUnifi Flag *Flag Config *Config - LastCheck time.Time sync.Mutex // locks the Unifi struct member when re-authing to unifi. } @@ -55,6 +44,15 @@ type Flag struct { *pflag.FlagSet } +// Metrics is a type shared by the exporting and reporting packages. +type Metrics struct { + TS time.Time + unifi.Sites + unifi.IDSList + unifi.Clients + *unifi.Devices +} + // Controller represents the configuration for a UniFi Controller. // Each polled controller may have its own configuration. type Controller struct { @@ -73,16 +71,43 @@ type Controller struct { // This is all of the data stored in the config file. // Any with explicit defaults have omitempty on json and toml tags. type Config struct { - Interval config.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` - Debug bool `json:"debug" toml:"debug" xml:"debug" yaml:"debug"` - Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet" yaml:"quiet"` - InfxBadSSL bool `json:"influx_insecure_ssl" toml:"influx_insecure_ssl" xml:"influx_insecure_ssl" yaml:"influx_insecure_ssl"` - Mode string `json:"mode" toml:"mode" xml:"mode" yaml:"mode"` - HTTPListen string `json:"http_listen" toml:"http_listen" xml:"http_listen" yaml:"http_listen"` - Namespace string `json:"namespace" toml:"namespace" xml:"namespace" yaml:"namespace"` - InfluxURL string `json:"influx_url,omitempty" toml:"influx_url,omitempty" xml:"influx_url" yaml:"influx_url"` - InfluxUser string `json:"influx_user,omitempty" toml:"influx_user,omitempty" xml:"influx_user" yaml:"influx_user"` - InfluxPass string `json:"influx_pass,omitempty" toml:"influx_pass,omitempty" xml:"influx_pass" yaml:"influx_pass"` - InfluxDB string `json:"influx_db,omitempty" toml:"influx_db,omitempty" xml:"influx_db" yaml:"influx_db"` - Controllers []Controller `json:"controller,omitempty" toml:"controller,omitempty" xml:"controller" yaml:"controller"` + Poller `json:"poller" toml:"poller" xml:"poller" yaml:"poller"` + Controllers []Controller `json:"controller,omitempty" toml:"controller,omitempty" xml:"controller" yaml:"controller"` +} + +// Poller is the global config values. +type Poller struct { + Debug bool `json:"debug" toml:"debug" xml:"debug,attr" yaml:"debug"` + Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet,attr" yaml:"quiet"` +} + +// ParseConfigs parses the poller config and the config for each registered output plugin. +func (u *UnifiPoller) ParseConfigs() error { + // Parse config file. + if err := config.ParseFile(u.Config, u.Flag.ConfigFile); err != nil { + u.Flag.Usage() + return err + } + + // Update Config with ENV variable overrides. + if _, err := config.ParseENV(u.Config, ENVConfigPrefix); err != nil { + return err + } + + outputSync.Lock() + defer outputSync.Unlock() + + for _, o := range outputs { + // Parse config file for each output plugin. + if err := config.ParseFile(o.Config, u.Flag.ConfigFile); err != nil { + return err + } + + // Update Config for each output with ENV variable overrides. + if _, err := config.ParseENV(o.Config, ENVConfigPrefix); err != nil { + return err + } + } + + return nil } diff --git a/integrations/inputunifi/pkg/poller/influx.go b/integrations/inputunifi/pkg/poller/influx.go deleted file mode 100644 index 52690c13..00000000 --- a/integrations/inputunifi/pkg/poller/influx.go +++ /dev/null @@ -1,79 +0,0 @@ -package poller - -import ( - "fmt" - "log" - "time" - - "github.com/davidnewhall/unifi-poller/pkg/influxunifi" -) - -// GetInfluxDB returns an InfluxDB interface. -func (u *UnifiPoller) GetInfluxDB() (err error) { - if u.Influx != nil { - return nil - } - - u.Influx, err = influxunifi.New(&influxunifi.Config{ - Database: u.Config.InfluxDB, - User: u.Config.InfluxUser, - Pass: u.Config.InfluxPass, - BadSSL: u.Config.InfxBadSSL, - URL: u.Config.InfluxURL, - }) - if err != nil { - return fmt.Errorf("influxdb: %v", err) - } - - u.Logf("Logging Measurements to InfluxDB at %s as user %s", u.Config.InfluxURL, u.Config.InfluxUser) - - return nil -} - -// PollController runs forever, polling UniFi and pushing to InfluxDB -// This is started by Run() or RunBoth() after everything checks out. -func (u *UnifiPoller) PollController() { - interval := u.Config.Interval.Round(time.Second) - log.Printf("[INFO] Everything checks out! Poller started, InfluxDB interval: %v", interval) - - ticker := time.NewTicker(interval) - for u.LastCheck = range ticker.C { - if err := u.CollectAndProcess(); err != nil { - u.LogErrorf("%v", err) - } - } -} - -// CollectAndProcess collects measurements and then reports them to InfluxDB -// Can be called once or in a ticker loop. This function and all the ones below -// handle their own logging. An error is returned so the calling function may -// determine if there was a read or write error and act on it. This is currently -// called in two places in this library. One returns an error, one does not. -func (u *UnifiPoller) CollectAndProcess() error { - if err := u.GetInfluxDB(); err != nil { - return err - } - - metrics, err := u.CollectMetrics() - if err != nil { - return err - } - - report, err := u.Influx.ReportMetrics(metrics) - if err != nil { - return err - } - - u.LogInfluxReport(report) - return nil -} - -// LogInfluxReport writes a log message after exporting to influxdb. -func (u *UnifiPoller) LogInfluxReport(r *influxunifi.Report) { - idsMsg := fmt.Sprintf("IDS Events: %d, ", len(r.Metrics.IDSList)) - u.Logf("UniFi Metrics Recorded. Sites: %d, Clients: %d, "+ - "UAP: %d, USG/UDM: %d, USW: %d, %sPoints: %d, Fields: %d, Errs: %d, Elapsed: %v", - len(r.Metrics.Sites), len(r.Metrics.Clients), len(r.Metrics.UAPs), - len(r.Metrics.UDMs)+len(r.Metrics.USGs), len(r.Metrics.USWs), idsMsg, r.Total, - r.Fields, len(r.Errors), r.Elapsed.Round(time.Millisecond)) -} diff --git a/integrations/inputunifi/pkg/poller/outputs.go b/integrations/inputunifi/pkg/poller/outputs.go new file mode 100644 index 00000000..37f8fbb2 --- /dev/null +++ b/integrations/inputunifi/pkg/poller/outputs.go @@ -0,0 +1,72 @@ +package poller + +import ( + "fmt" + "sync" +) + +var ( + outputs []*Output + outputSync sync.Mutex +) + +// Collect is passed into output packages so they may collect metrics to output. +// Output packages must implement this interface. +type Collect interface { + Metrics() (*Metrics, error) + Logf(m string, v ...interface{}) + LogErrorf(m string, v ...interface{}) + LogDebugf(m string, v ...interface{}) +} + +// Output defines the output data for a metric exporter like influx or prometheus. +// Output packages should call NewOutput with this struct in init(). +type Output struct { + Name string + Config interface{} // Each config is passed into an unmarshaller later. + Method func(Collect) error // Called on startup for each configured output. +} + +// NewOutput should be called by each output package's init function. +func NewOutput(o *Output) { + outputSync.Lock() + defer outputSync.Unlock() + + if o == nil || o.Method == nil { + panic("nil output or method passed to poller.NewOutput") + } + + outputs = append(outputs, o) +} + +// InitializeOutputs runs all the configured output plugins. +// If none exist, or they all exit an error is returned. +func (u *UnifiPoller) InitializeOutputs() error { + v := make(chan error) + defer close(v) + + var count int + + for _, o := range outputs { + count++ + go func(o *Output) { + v <- o.Method(u) + }(o) + } + + if count < 1 { + return fmt.Errorf("no output plugins configured") + } + + for err := range v { + if err != nil { + return err + } + + if count--; count < 1 { + return fmt.Errorf("all output plugins have stopped") + } + } + + return nil +} diff --git a/integrations/inputunifi/pkg/poller/prometheus.go b/integrations/inputunifi/pkg/poller/prometheus.go deleted file mode 100644 index 13e4de49..00000000 --- a/integrations/inputunifi/pkg/poller/prometheus.go +++ /dev/null @@ -1,53 +0,0 @@ -package poller - -import ( - "net/http" - "strings" - "time" - - "github.com/davidnewhall/unifi-poller/pkg/metrics" - "github.com/davidnewhall/unifi-poller/pkg/promunifi" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/prometheus/common/version" -) - -const oneDecimalPoint = 10 - -// RunPrometheus starts the web server and registers the collector. -func (u *UnifiPoller) RunPrometheus() error { - u.Logf("Exporting Measurements for Prometheus at https://%s/metrics", u.Config.HTTPListen) - http.Handle("/metrics", promhttp.Handler()) - ns := strings.Replace(u.Config.Namespace, "-", "", -1) - prometheus.MustRegister(promunifi.NewUnifiCollector(promunifi.UnifiCollectorCnfg{ - Namespace: ns, - CollectFn: u.ExportMetrics, - LoggingFn: u.LogExportReport, - ReportErrors: true, // XXX: Does this need to be configurable? - })) - - version.Version = Version - prometheus.MustRegister(version.NewCollector(ns)) - - return http.ListenAndServe(u.Config.HTTPListen, nil) -} - -// ExportMetrics updates the internal metrics provided via -// HTTP at /metrics for prometheus collection. -// This is run by Prometheus as CollectFn. -func (u *UnifiPoller) ExportMetrics() (*metrics.Metrics, error) { - return u.CollectMetrics() -} - -// LogExportReport is called after prometheus exports metrics. -// This is run by Prometheus as LoggingFn -func (u *UnifiPoller) LogExportReport(report *promunifi.Report) { - m := report.Metrics - u.Logf("UniFi Measurements Exported. Site: %d, Client: %d, "+ - "UAP: %d, USG/UDM: %d, USW: %d, Descs: %d, "+ - "Metrics: %d, Errs: %d, 0s: %d, Reqs/Total: %v / %v", - len(m.Sites), len(m.Clients), len(m.UAPs), len(m.UDMs)+len(m.USGs), len(m.USWs), - report.Descs, report.Total, report.Errors, report.Zeros, - report.Fetch.Round(time.Millisecond/oneDecimalPoint), - report.Elapsed.Round(time.Millisecond/oneDecimalPoint)) -} diff --git a/integrations/inputunifi/pkg/poller/start.go b/integrations/inputunifi/pkg/poller/start.go index 2a81ac03..fa4213f5 100644 --- a/integrations/inputunifi/pkg/poller/start.go +++ b/integrations/inputunifi/pkg/poller/start.go @@ -5,26 +5,16 @@ import ( "fmt" "log" "os" - "strings" - "time" + "github.com/github/hub/version" "github.com/spf13/pflag" - "golift.io/config" ) // New returns a new poller struct preloaded with default values. // No need to call this if you call Start.c func New() *UnifiPoller { return &UnifiPoller{ - Config: &Config{ - InfluxURL: defaultInfluxURL, - InfluxUser: defaultInfluxUser, - InfluxPass: defaultInfluxPass, - InfluxDB: defaultInfluxDB, - Interval: config.Duration{Duration: defaultInterval}, - HTTPListen: defaultHTTPListen, - Namespace: appName, - }, + Config: &Config{}, Flag: &Flag{ ConfigFile: DefaultConfFile, }, @@ -40,7 +30,7 @@ func (u *UnifiPoller) Start() error { u.Flag.Parse(os.Args[1:]) if u.Flag.ShowVer { - fmt.Printf("%s v%s\n", appName, Version) + fmt.Printf("%s v%s\n", AppName, version.Version) return nil // don't run anything else w/ version request. } @@ -48,14 +38,8 @@ func (u *UnifiPoller) Start() error { u.Logf("Loading Configuration File: %s", u.Flag.ConfigFile) } - // Parse config file. - if err := config.ParseFile(u.Config, u.Flag.ConfigFile); err != nil { - u.Flag.Usage() - return err - } - - // Update Config with ENV variable overrides. - if _, err := config.ParseENV(u.Config, ENVConfigPrefix); err != nil { + // Parse config file and ENV variables. + if err := u.ParseConfigs(); err != nil { return err } @@ -78,16 +62,14 @@ func (u *UnifiPoller) Start() error { u.LogDebugf("Debug Logging Enabled") } - log.Printf("[INFO] UniFi Poller v%v Starting Up! PID: %d", Version, os.Getpid()) - return u.Run() } // Parse turns CLI arguments into data structures. Called by Start() on startup. func (f *Flag) Parse(args []string) { - f.FlagSet = pflag.NewFlagSet(appName, pflag.ExitOnError) + f.FlagSet = pflag.NewFlagSet(AppName, pflag.ExitOnError) f.Usage = func() { - fmt.Printf("Usage: %s [--config=/path/to/up.conf] [--version]", appName) + fmt.Printf("Usage: %s [--config=/path/to/up.conf] [--version]", AppName) f.PrintDefaults() } @@ -103,6 +85,8 @@ func (f *Flag) Parse(args []string) { // 2. Run the collector one time and report the metrics to influxdb. (lambda) // 3. Start a web server and wait for Prometheus to poll the application for metrics. func (u *UnifiPoller) Run() error { + log.Printf("[INFO] UniFi Poller v%v Starting Up! PID: %d", version.Version, os.Getpid()) + for i, c := range u.Config.Controllers { if c.Name == "" { u.Config.Controllers[i].Name = c.URL @@ -117,17 +101,5 @@ func (u *UnifiPoller) Run() error { } } - switch strings.ToLower(u.Config.Mode) { - default: - u.PollController() - return nil - case "influxlambda", "lambdainflux", "lambda_influx", "influx_lambda": - u.LastCheck = time.Now() - return u.CollectAndProcess() - case "both": - go u.PollController() - fallthrough - case "prometheus", "exporter": - return u.RunPrometheus() - } + return u.InitializeOutputs() } diff --git a/integrations/inputunifi/pkg/poller/unifi.go b/integrations/inputunifi/pkg/poller/unifi.go index 0e598716..516310a3 100644 --- a/integrations/inputunifi/pkg/poller/unifi.go +++ b/integrations/inputunifi/pkg/poller/unifi.go @@ -5,7 +5,6 @@ import ( "strings" "time" - "github.com/davidnewhall/unifi-poller/pkg/metrics" "golift.io/unifi" ) @@ -42,10 +41,6 @@ func (u *UnifiPoller) GetUnifi(c Controller) error { // CheckSites makes sure the list of provided sites exists on the controller. // This does not run in Lambda (run-once) mode. func (u *UnifiPoller) CheckSites(c Controller) error { - if strings.Contains(strings.ToLower(u.Config.Mode), "lambda") { - return nil // Skip this in lambda mode. - } - u.LogDebugf("Checking Controller Sites List") sites, err := c.Unifi.GetSites() @@ -58,6 +53,7 @@ func (u *UnifiPoller) CheckSites(c Controller) error { for _, site := range sites { msg = append(msg, site.Name+" ("+site.Desc+")") } + u.Logf("Found %d site(s) on controller: %v", len(msg), strings.Join(msg, ", ")) if StringInSlice("all", c.Sites) { @@ -78,10 +74,10 @@ FIRST: return nil } -// CollectMetrics grabs all the measurements from a UniFi controller and returns them. -func (u *UnifiPoller) CollectMetrics() (*metrics.Metrics, error) { +// Metrics grabs all the measurements from a UniFi controller and returns them. +func (u *UnifiPoller) Metrics() (*Metrics, error) { errs := []string{} - metrics := &metrics.Metrics{} + metrics := &Metrics{} for _, c := range u.Config.Controllers { m, err := u.checkAndPollController(c) @@ -120,7 +116,7 @@ func (u *UnifiPoller) CollectMetrics() (*metrics.Metrics, error) { return metrics, err } -func (u *UnifiPoller) checkAndPollController(c Controller) (*metrics.Metrics, error) { +func (u *UnifiPoller) checkAndPollController(c Controller) (*Metrics, error) { if c.Unifi == nil { u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) @@ -146,10 +142,10 @@ func (u *UnifiPoller) checkAndPollController(c Controller) (*metrics.Metrics, er return u.collectController(c) } -func (u *UnifiPoller) collectController(c Controller) (*metrics.Metrics, error) { +func (u *UnifiPoller) collectController(c Controller) (*Metrics, error) { var err error - m := &metrics.Metrics{TS: u.LastCheck} // At this point, it's the Current Check. + m := &Metrics{TS: time.Now()} // At this point, it's the Current Check. // Get the sites we care about. if m.Sites, err = u.GetFilteredSites(c); err != nil { @@ -178,7 +174,7 @@ func (u *UnifiPoller) collectController(c Controller) (*metrics.Metrics, error) // augmentMetrics is our middleware layer between collecting metrics and writing them. // This is where we can manipuate the returned data or make arbitrary decisions. // This function currently adds parent device names to client metrics. -func (u *UnifiPoller) augmentMetrics(c Controller, metrics *metrics.Metrics) *metrics.Metrics { +func (u *UnifiPoller) augmentMetrics(c Controller, metrics *Metrics) *Metrics { if metrics == nil || metrics.Devices == nil || metrics.Clients == nil { return metrics } diff --git a/integrations/inputunifi/pkg/promunifi/collector.go b/integrations/inputunifi/pkg/promunifi/collector.go index 5911d005..6da6db38 100644 --- a/integrations/inputunifi/pkg/promunifi/collector.go +++ b/integrations/inputunifi/pkg/promunifi/collector.go @@ -1,50 +1,59 @@ -// Package promunifi provides the bridge between unifi metrics and prometheus. +// Package promunifi provides the bridge between unifi-poller metrics and prometheus. package promunifi import ( "fmt" + "net/http" "reflect" "strings" "sync" "time" - "github.com/davidnewhall/unifi-poller/pkg/metrics" + "github.com/davidnewhall/unifi-poller/pkg/poller" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/version" "golift.io/unifi" ) -// channel buffer, fits at least one batch. -const buffer = 50 - -// simply fewer letters. -const counter = prometheus.CounterValue -const gauge = prometheus.GaugeValue - -// UnifiCollectorCnfg defines the data needed to collect and report UniFi Metrics. -type UnifiCollectorCnfg struct { - // If non-empty, each of the collected metrics is prefixed by the - // provided string and an underscore ("_"). - Namespace string - // If true, any error encountered during collection is reported as an - // invalid metric (see NewInvalidMetric). Otherwise, errors are ignored - // and the collected metrics will be incomplete. Possibly, no metrics - // will be collected at all. - ReportErrors bool - // This function is passed to the Collect() method. The Collect method runs - // this function to retrieve the latest UniFi measurements and export them. - CollectFn func() (*metrics.Metrics, error) - // Provide a logger function if you want to run a routine *after* prometheus checks in. - LoggingFn func(*Report) -} +const ( + // channel buffer, fits at least one batch. + buffer = 50 + defaultHTTPListen = "0.0.0.0:9130" + // simply fewer letters. + counter = prometheus.CounterValue + gauge = prometheus.GaugeValue +) type promUnifi struct { - Config UnifiCollectorCnfg + *Prometheus Client *uclient Device *unifiDevice UAP *uap USG *usg USW *usw Site *site + // This interface is passed to the Collect() method. The Collect method uses + // this interface to retrieve the latest UniFi measurements and export them. + Collector poller.Collect +} + +// Prometheus allows the data to be nested in the config file. +type Prometheus struct { + Config Config `json:"prometheus" toml:"prometheus" xml:"prometheus" yaml:"prometheus"` +} + +// Config is the input (config file) data used to initialize this output plugin. +type Config struct { + Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` + // If non-empty, each of the collected metrics is prefixed by the + // provided string and an underscore ("_"). + Namespace string `json:"namespace" toml:"namespace" xml:"namespace" yaml:"namespace"` + // If true, any error encountered during collection is reported as an + // invalid metric (see NewInvalidMetric). Otherwise, errors are ignored + // and the collected metrics will be incomplete. Possibly, no metrics + // will be collected at all. + ReportErrors bool `json:"report_errors" toml:"report_errors" xml:"report_errors" yaml:"report_errors"` + HTTPListen string `json:"http_listen" toml:"http_listen" xml:"http_listen" yaml:"http_listen"` } type metric struct { @@ -54,41 +63,64 @@ type metric struct { Labels []string } -// Report is passed into LoggingFn to log the export metrics to stdout (outside this package). +// Report accumulates counters that are printed to a log line. type Report struct { - Total int // Total count of metrics recorded. - Errors int // Total count of errors recording metrics. - Zeros int // Total count of metrics equal to zero. - Descs int // Total count of unique metrics descriptions. - Metrics *metrics.Metrics // Metrics collected and recorded. - Elapsed time.Duration // Duration elapsed collecting and exporting. - Fetch time.Duration // Duration elapsed making controller requests. - Start time.Time // Time collection began. + Total int // Total count of metrics recorded. + Errors int // Total count of errors recording metrics. + Zeros int // Total count of metrics equal to zero. + Metrics *poller.Metrics // Metrics collected and recorded. + Elapsed time.Duration // Duration elapsed collecting and exporting. + Fetch time.Duration // Duration elapsed making controller requests. + Start time.Time // Time collection began. ch chan []*metric wg sync.WaitGroup - cf UnifiCollectorCnfg + Config } -// NewUnifiCollector returns a prometheus collector that will export any available -// UniFi metrics. You must provide a collection function in the opts. -func NewUnifiCollector(opts UnifiCollectorCnfg) prometheus.Collector { - if opts.CollectFn == nil { - panic("nil collector function") +func init() { + u := &promUnifi{Prometheus: &Prometheus{}} + poller.NewOutput(&poller.Output{ + Name: "prometheus", + Config: u.Prometheus, + Method: u.Run, + }) +} + +// Run creates the collectors and starts the web server up. +// Should be run in a Go routine. Returns nil if not configured. +func (u *promUnifi) Run(c poller.Collect) error { + if u.Config.Disable { + return nil } - if opts.Namespace = strings.Trim(opts.Namespace, "_") + "_"; opts.Namespace == "_" { - opts.Namespace = "" + if u.Config.Namespace == "" { + u.Config.Namespace = strings.Replace(poller.AppName, "-", "", -1) } - return &promUnifi{ - Config: opts, - Client: descClient(opts.Namespace + "client_"), - Device: descDevice(opts.Namespace + "device_"), // stats for all device types. - UAP: descUAP(opts.Namespace + "device_"), - USG: descUSG(opts.Namespace + "device_"), - USW: descUSW(opts.Namespace + "device_"), - Site: descSite(opts.Namespace + "site_"), + if u.Config.HTTPListen == "" { + u.Config.HTTPListen = defaultHTTPListen } + + name := strings.Replace(u.Config.Namespace, "-", "_", -1) + + ns := name + if ns = strings.Trim(ns, "_") + "_"; ns == "_" { + ns = "" + } + + prometheus.MustRegister(version.NewCollector(name)) + prometheus.MustRegister(&promUnifi{ + Collector: c, + Client: descClient(ns + "client_"), + Device: descDevice(ns + "device_"), // stats for all device types. + UAP: descUAP(ns + "device_"), + USG: descUSG(ns + "device_"), + USW: descUSW(ns + "device_"), + Site: descSite(ns + "site_"), + }) + c.Logf("Exporting Measurements for Prometheus at https://%s/metrics, namespace: %s", u.Config.HTTPListen, u.Config.Namespace) + + return http.ListenAndServe(u.Config.HTTPListen, nil) } // Describe satisfies the prometheus Collector. This returns all of the @@ -112,10 +144,10 @@ func (u *promUnifi) Describe(ch chan<- *prometheus.Desc) { func (u *promUnifi) Collect(ch chan<- prometheus.Metric) { var err error - r := &Report{cf: u.Config, ch: make(chan []*metric, buffer), Start: time.Now()} + r := &Report{Config: u.Config, ch: make(chan []*metric, buffer), Start: time.Now()} defer r.close() - if r.Metrics, err = r.cf.CollectFn(); err != nil { + if r.Metrics, err = u.Collector.Metrics(); err != nil { r.error(ch, prometheus.NewInvalidDesc(fmt.Errorf("metric fetch failed")), err) return } @@ -135,7 +167,7 @@ func (u *promUnifi) Collect(ch chan<- prometheus.Metric) { // This is where our channels connects to the prometheus channel. func (u *promUnifi) exportMetrics(r report, ch chan<- prometheus.Metric, ourChan chan []*metric) { descs := make(map[*prometheus.Desc]bool) // used as a counter - defer r.report(descs) + defer r.report(u.Collector, descs) for newMetrics := range ourChan { for _, m := range newMetrics { diff --git a/integrations/inputunifi/pkg/promunifi/report.go b/integrations/inputunifi/pkg/promunifi/report.go index 0ddf29d3..9b6df74c 100644 --- a/integrations/inputunifi/pkg/promunifi/report.go +++ b/integrations/inputunifi/pkg/promunifi/report.go @@ -4,7 +4,7 @@ import ( "fmt" "time" - "github.com/davidnewhall/unifi-poller/pkg/metrics" + "github.com/davidnewhall/unifi-poller/pkg/poller" "github.com/prometheus/client_golang/prometheus" ) @@ -16,14 +16,15 @@ type report interface { add() done() send([]*metric) - metrics() *metrics.Metrics - report(descs map[*prometheus.Desc]bool) + metrics() *poller.Metrics + report(c poller.Collect, descs map[*prometheus.Desc]bool) export(m *metric, v float64) prometheus.Metric error(ch chan<- prometheus.Metric, d *prometheus.Desc, v interface{}) } // satisfy gomnd const one = 1 +const oneDecimalPoint = 10.0 func (r *Report) add() { r.wg.Add(one) @@ -38,17 +39,19 @@ func (r *Report) send(m []*metric) { r.ch <- m } -func (r *Report) metrics() *metrics.Metrics { +func (r *Report) metrics() *poller.Metrics { return r.Metrics } -func (r *Report) report(descs map[*prometheus.Desc]bool) { - if r.cf.LoggingFn == nil { - return - } - - r.Descs = len(descs) - r.cf.LoggingFn(r) +func (r *Report) report(c poller.Collect, descs map[*prometheus.Desc]bool) { + m := r.Metrics + c.Logf("UniFi Measurements Exported. Site: %d, Client: %d, "+ + "UAP: %d, USG/UDM: %d, USW: %d, Descs: %d, "+ + "Metrics: %d, Errs: %d, 0s: %d, Reqs/Total: %v / %v", + len(m.Sites), len(m.Clients), len(m.UAPs), len(m.UDMs)+len(m.USGs), len(m.USWs), + len(descs), r.Total, r.Errors, r.Zeros, + r.Fetch.Round(time.Millisecond/oneDecimalPoint), + r.Elapsed.Round(time.Millisecond/oneDecimalPoint)) } func (r *Report) export(m *metric, v float64) prometheus.Metric { @@ -64,7 +67,7 @@ func (r *Report) export(m *metric, v float64) prometheus.Metric { func (r *Report) error(ch chan<- prometheus.Metric, d *prometheus.Desc, v interface{}) { r.Errors++ - if r.cf.ReportErrors { + if r.Config.ReportErrors { ch <- prometheus.NewInvalidMetric(d, fmt.Errorf("error: %v", v)) } } From 3f1209cd8de741f7a61f0723c4ce85e1c92e672a Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sun, 15 Dec 2019 02:57:08 -0800 Subject: [PATCH 11/41] fix import --- integrations/inputunifi/pkg/poller/start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/inputunifi/pkg/poller/start.go b/integrations/inputunifi/pkg/poller/start.go index fa4213f5..06069583 100644 --- a/integrations/inputunifi/pkg/poller/start.go +++ b/integrations/inputunifi/pkg/poller/start.go @@ -6,7 +6,7 @@ import ( "log" "os" - "github.com/github/hub/version" + "github.com/prometheus/common/version" "github.com/spf13/pflag" ) From e9e6643ca330c810c83aebb1412f15e82c674046 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sun, 15 Dec 2019 02:57:53 -0800 Subject: [PATCH 12/41] update vendors --- integrations/inputunifi/Gopkg.lock | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/integrations/inputunifi/Gopkg.lock b/integrations/inputunifi/Gopkg.lock index 238a1b81..83212a0b 100644 --- a/integrations/inputunifi/Gopkg.lock +++ b/integrations/inputunifi/Gopkg.lock @@ -46,12 +46,11 @@ version = "v1.0.1" [[projects]] - digest = "1:eb04f69c8991e52eff33c428bd729e04208bf03235be88e4df0d88497c6861b9" + digest = "1:7097829edd12fd7211fca0d29496b44f94ef9e6d72f88fb64f3d7b06315818ad" name = "github.com/prometheus/client_golang" packages = [ "prometheus", "prometheus/internal", - "prometheus/promhttp", ] pruneopts = "UT" revision = "170205fb58decfd011f1550d4cfb737230d7ae4f" @@ -136,7 +135,6 @@ input-imports = [ "github.com/influxdata/influxdb1-client/v2", "github.com/prometheus/client_golang/prometheus", - "github.com/prometheus/client_golang/prometheus/promhttp", "github.com/prometheus/common/version", "github.com/spf13/pflag", "golift.io/config", From adb8a4fb527f93f37661869f5e628b5988263a11 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sun, 15 Dec 2019 03:27:34 -0800 Subject: [PATCH 13/41] trimming --- .../inputunifi/pkg/influxunifi/metrics.go | 34 ++++++++++++------- integrations/inputunifi/pkg/poller/config.go | 12 +++---- integrations/inputunifi/pkg/poller/dumper.go | 8 ++--- integrations/inputunifi/pkg/poller/outputs.go | 4 +-- integrations/inputunifi/pkg/poller/start.go | 22 +++++------- .../inputunifi/pkg/promunifi/collector.go | 27 ++++++++------- 6 files changed, 56 insertions(+), 51 deletions(-) diff --git a/integrations/inputunifi/pkg/influxunifi/metrics.go b/integrations/inputunifi/pkg/influxunifi/metrics.go index 6775ac06..e983edce 100644 --- a/integrations/inputunifi/pkg/influxunifi/metrics.go +++ b/integrations/inputunifi/pkg/influxunifi/metrics.go @@ -10,11 +10,12 @@ import ( "github.com/davidnewhall/unifi-poller/pkg/poller" influx "github.com/influxdata/influxdb1-client/v2" - conf "golift.io/config" + "golift.io/config" ) const ( defaultInterval = 30 * time.Second + minimumInterval = 10 * time.Second defaultInfluxDB = "unifi" defaultInfluxUser = "unifi" defaultInfluxURL = "http://127.0.0.1:8086" @@ -22,13 +23,13 @@ const ( // Config defines the data needed to store metrics in InfluxDB type Config struct { - Interval conf.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` - Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` - VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` - URL string `json:"url,omitempty" toml:"url,omitempty" xml:"url" yaml:"url"` - User string `json:"user,omitempty" toml:"user,omitempty" xml:"user" yaml:"user"` - Pass string `json:"pass,omitempty" toml:"pass,omitempty" xml:"pass" yaml:"pass"` - DB string `json:"db,omitempty" toml:"db,omitempty" xml:"db" yaml:"db"` + Interval config.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` + Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` + VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` + URL string `json:"url,omitempty" toml:"url,omitempty" xml:"url" yaml:"url"` + User string `json:"user,omitempty" toml:"user,omitempty" xml:"user" yaml:"user"` + Pass string `json:"pass,omitempty" toml:"pass,omitempty" xml:"pass" yaml:"pass"` + DB string `json:"db,omitempty" toml:"db,omitempty" xml:"db" yaml:"db"` } // InfluxDB allows the data to be nested in the config file. @@ -63,9 +64,9 @@ func init() { // This is started by Run() or RunBoth() after everything checks out. func (u *InfluxUnifi) PollController() { interval := u.Config.Interval.Round(time.Second) + ticker := time.NewTicker(interval) log.Printf("[INFO] Everything checks out! Poller started, InfluxDB interval: %v", interval) - ticker := time.NewTicker(interval) for u.LastCheck = range ticker.C { metrics, err := u.Collector.Metrics() if err != nil { @@ -128,12 +129,12 @@ func (u *InfluxUnifi) setConfigDefaults() { } if u.Config.Interval.Duration == 0 { - u.Config.Interval = conf.Duration{Duration: defaultInterval} - } else if u.Config.Interval.Duration < defaultInterval/2 { - u.Config.Interval = conf.Duration{Duration: defaultInterval / 2} + u.Config.Interval = config.Duration{Duration: defaultInterval} + } else if u.Config.Interval.Duration < minimumInterval { + u.Config.Interval = config.Duration{Duration: minimumInterval} } - u.Config.Interval = conf.Duration{Duration: u.Config.Interval.Duration.Round(time.Second)} + u.Config.Interval = config.Duration{Duration: u.Config.Interval.Duration.Round(time.Second)} } // ReportMetrics batches all device and client data into influxdb data points. @@ -179,6 +180,7 @@ func (u *InfluxUnifi) collect(r report, ch chan *metric) { // to the collect routine through the metric channel. func (u *InfluxUnifi) loopPoints(r report) { m := r.metrics() + r.add() go func() { defer r.done() @@ -186,6 +188,7 @@ func (u *InfluxUnifi) loopPoints(r report) { u.batchSite(r, s) } }() + r.add() go func() { defer r.done() @@ -193,6 +196,7 @@ func (u *InfluxUnifi) loopPoints(r report) { u.batchClient(r, s) } }() + r.add() go func() { defer r.done() @@ -200,6 +204,7 @@ func (u *InfluxUnifi) loopPoints(r report) { u.batchIDS(r, s) } }() + if m.Devices == nil { return } @@ -211,6 +216,7 @@ func (u *InfluxUnifi) loopPoints(r report) { u.batchUAP(r, s) } }() + r.add() go func() { defer r.done() @@ -218,6 +224,7 @@ func (u *InfluxUnifi) loopPoints(r report) { u.batchUSG(r, s) } }() + r.add() go func() { defer r.done() @@ -225,6 +232,7 @@ func (u *InfluxUnifi) loopPoints(r report) { u.batchUSW(r, s) } }() + r.add() go func() { defer r.done() diff --git a/integrations/inputunifi/pkg/poller/config.go b/integrations/inputunifi/pkg/poller/config.go index 0d6fe126..e77db20b 100644 --- a/integrations/inputunifi/pkg/poller/config.go +++ b/integrations/inputunifi/pkg/poller/config.go @@ -31,13 +31,13 @@ const ENVConfigPrefix = "UP" // UnifiPoller contains the application startup data, and auth info for UniFi & Influx. type UnifiPoller struct { - Flag *Flag + Flags *Flags Config *Config sync.Mutex // locks the Unifi struct member when re-authing to unifi. } -// Flag represents the CLI args available and their settings. -type Flag struct { +// Flags represents the CLI args available and their settings. +type Flags struct { ConfigFile string DumpJSON string ShowVer bool @@ -84,8 +84,8 @@ type Poller struct { // ParseConfigs parses the poller config and the config for each registered output plugin. func (u *UnifiPoller) ParseConfigs() error { // Parse config file. - if err := config.ParseFile(u.Config, u.Flag.ConfigFile); err != nil { - u.Flag.Usage() + if err := config.ParseFile(u.Config, u.Flags.ConfigFile); err != nil { + u.Flags.Usage() return err } @@ -99,7 +99,7 @@ func (u *UnifiPoller) ParseConfigs() error { for _, o := range outputs { // Parse config file for each output plugin. - if err := config.ParseFile(o.Config, u.Flag.ConfigFile); err != nil { + if err := config.ParseFile(o.Config, u.Flags.ConfigFile); err != nil { return err } diff --git a/integrations/inputunifi/pkg/poller/dumper.go b/integrations/inputunifi/pkg/poller/dumper.go index 3151ae9c..5892488b 100644 --- a/integrations/inputunifi/pkg/poller/dumper.go +++ b/integrations/inputunifi/pkg/poller/dumper.go @@ -37,12 +37,12 @@ func (u *UnifiPoller) DumpJSONPayload() (err error) { switch sites, err := u.GetFilteredSites(config); { case err != nil: return err - case StringInSlice(u.Flag.DumpJSON, []string{"d", "device", "devices"}): + case StringInSlice(u.Flags.DumpJSON, []string{"d", "device", "devices"}): return u.dumpSitesJSON(config, unifi.APIDevicePath, "Devices", sites) - case StringInSlice(u.Flag.DumpJSON, []string{"client", "clients", "c"}): + case StringInSlice(u.Flags.DumpJSON, []string{"client", "clients", "c"}): return u.dumpSitesJSON(config, unifi.APIClientPath, "Clients", sites) - case strings.HasPrefix(u.Flag.DumpJSON, "other "): - apiPath := strings.SplitN(u.Flag.DumpJSON, " ", 2)[1] + case strings.HasPrefix(u.Flags.DumpJSON, "other "): + apiPath := strings.SplitN(u.Flags.DumpJSON, " ", 2)[1] _, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping Path '%s':\n", apiPath) return u.PrintRawAPIJSON(config, apiPath) default: diff --git a/integrations/inputunifi/pkg/poller/outputs.go b/integrations/inputunifi/pkg/poller/outputs.go index 37f8fbb2..aaa9a47d 100644 --- a/integrations/inputunifi/pkg/poller/outputs.go +++ b/integrations/inputunifi/pkg/poller/outputs.go @@ -55,7 +55,7 @@ func (u *UnifiPoller) InitializeOutputs() error { } if count < 1 { - return fmt.Errorf("no output plugins configured") + return fmt.Errorf("no output plugins imported") } for err := range v { @@ -64,7 +64,7 @@ func (u *UnifiPoller) InitializeOutputs() error { } if count--; count < 1 { - return fmt.Errorf("all output plugins have stopped") + return fmt.Errorf("all output plugins have stopped, or none enabled") } } diff --git a/integrations/inputunifi/pkg/poller/start.go b/integrations/inputunifi/pkg/poller/start.go index 06069583..7893f32f 100644 --- a/integrations/inputunifi/pkg/poller/start.go +++ b/integrations/inputunifi/pkg/poller/start.go @@ -10,15 +10,9 @@ import ( "github.com/spf13/pflag" ) -// New returns a new poller struct preloaded with default values. -// No need to call this if you call Start.c +// New returns a new poller struct. func New() *UnifiPoller { - return &UnifiPoller{ - Config: &Config{}, - Flag: &Flag{ - ConfigFile: DefaultConfFile, - }, - } + return &UnifiPoller{Config: &Config{}, Flags: &Flags{}} } // Start begins the application from a CLI. @@ -27,15 +21,15 @@ func New() *UnifiPoller { func (u *UnifiPoller) Start() error { log.SetOutput(os.Stdout) log.SetFlags(log.LstdFlags) - u.Flag.Parse(os.Args[1:]) + u.Flags.Parse(os.Args[1:]) - if u.Flag.ShowVer { + if u.Flags.ShowVer { fmt.Printf("%s v%s\n", AppName, version.Version) return nil // don't run anything else w/ version request. } - if u.Flag.DumpJSON == "" { // do not print this when dumping JSON. - u.Logf("Loading Configuration File: %s", u.Flag.ConfigFile) + if u.Flags.DumpJSON == "" { // do not print this when dumping JSON. + u.Logf("Loading Configuration File: %s", u.Flags.ConfigFile) } // Parse config file and ENV variables. @@ -53,7 +47,7 @@ func (u *UnifiPoller) Start() error { }} } - if u.Flag.DumpJSON != "" { + if u.Flags.DumpJSON != "" { return u.DumpJSONPayload() } @@ -66,7 +60,7 @@ func (u *UnifiPoller) Start() error { } // Parse turns CLI arguments into data structures. Called by Start() on startup. -func (f *Flag) Parse(args []string) { +func (f *Flags) Parse(args []string) { f.FlagSet = pflag.NewFlagSet(AppName, pflag.ExitOnError) f.Usage = func() { fmt.Printf("Usage: %s [--config=/path/to/up.conf] [--version]", AppName) diff --git a/integrations/inputunifi/pkg/promunifi/collector.go b/integrations/inputunifi/pkg/promunifi/collector.go index 6da6db38..61c8db81 100644 --- a/integrations/inputunifi/pkg/promunifi/collector.go +++ b/integrations/inputunifi/pkg/promunifi/collector.go @@ -65,6 +65,7 @@ type metric struct { // Report accumulates counters that are printed to a log line. type Report struct { + Config Total int // Total count of metrics recorded. Errors int // Total count of errors recording metrics. Zeros int // Total count of metrics equal to zero. @@ -74,7 +75,6 @@ type Report struct { Start time.Time // Time collection began. ch chan []*metric wg sync.WaitGroup - Config } func init() { @@ -97,27 +97,28 @@ func (u *promUnifi) Run(c poller.Collect) error { u.Config.Namespace = strings.Replace(poller.AppName, "-", "", -1) } + u.Config.Namespace = strings.Replace(u.Config.Namespace, "-", "_", -1) + if u.Config.HTTPListen == "" { u.Config.HTTPListen = defaultHTTPListen } - name := strings.Replace(u.Config.Namespace, "-", "_", -1) + prometheus.MustRegister(version.NewCollector(u.Config.Namespace)) - ns := name - if ns = strings.Trim(ns, "_") + "_"; ns == "_" { - ns = "" + if u.Config.Namespace = strings.Trim(u.Config.Namespace, "_") + "_"; u.Config.Namespace == "_" { + u.Config.Namespace = "" } - prometheus.MustRegister(version.NewCollector(name)) prometheus.MustRegister(&promUnifi{ Collector: c, - Client: descClient(ns + "client_"), - Device: descDevice(ns + "device_"), // stats for all device types. - UAP: descUAP(ns + "device_"), - USG: descUSG(ns + "device_"), - USW: descUSW(ns + "device_"), - Site: descSite(ns + "site_"), + Client: descClient(u.Config.Namespace + "client_"), + Device: descDevice(u.Config.Namespace + "device_"), // stats for all device types. + UAP: descUAP(u.Config.Namespace + "device_"), + USG: descUSG(u.Config.Namespace + "device_"), + USW: descUSW(u.Config.Namespace + "device_"), + Site: descSite(u.Config.Namespace + "site_"), }) + c.Logf("Exporting Measurements for Prometheus at https://%s/metrics, namespace: %s", u.Config.HTTPListen, u.Config.Namespace) return http.ListenAndServe(u.Config.HTTPListen, nil) @@ -185,12 +186,14 @@ func (u *promUnifi) exportMetrics(r report, ch chan<- prometheus.Metric, ourChan r.error(ch, m.Desc, fmt.Sprintf("not a number: %v", m.Value)) } } + r.done() } } func (u *promUnifi) loopExports(r report) { m := r.metrics() + r.add() go func() { defer r.done() From e0607904f1126d65db21616da41ef5517e103e32 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sun, 15 Dec 2019 13:15:00 -0800 Subject: [PATCH 14/41] fix tests --- integrations/inputunifi/.metadata.sh | 2 +- integrations/inputunifi/pkg/poller/start.go | 2 +- integrations/inputunifi/pkg/promunifi/collector.go | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/integrations/inputunifi/.metadata.sh b/integrations/inputunifi/.metadata.sh index 11ba0838..c4b30be1 100755 --- a/integrations/inputunifi/.metadata.sh +++ b/integrations/inputunifi/.metadata.sh @@ -11,7 +11,7 @@ HBREPO="golift/homebrew-mugs" MAINT="David Newhall II " VENDOR="Go Lift " DESC="Polls a UniFi controller, exports metrics to InfluxDB and Prometheus" -GOLANGCI_LINT_ARGS="--enable-all -D gochecknoglobals -D dupl -D lll -D funlen -D wsl -e G402" +GOLANGCI_LINT_ARGS="--enable-all -D gochecknoglobals -D dupl -D lll -D funlen -D wsl -e G402 -D gochecknoinits" # Example must exist at examples/$CONFIG_FILE.example CONFIG_FILE="up.conf" LICENSE="MIT" diff --git a/integrations/inputunifi/pkg/poller/start.go b/integrations/inputunifi/pkg/poller/start.go index 7893f32f..cd293b2f 100644 --- a/integrations/inputunifi/pkg/poller/start.go +++ b/integrations/inputunifi/pkg/poller/start.go @@ -91,7 +91,7 @@ func (u *UnifiPoller) Run() error { u.Logf("Polling UniFi Controller at %s v%s as user %s. Sites: %v", c.URL, c.Unifi.ServerVersion, c.User, c.Sites) default: - u.LogErrorf("Controller Auth or Connection failed, but continuing to retry! %s: %v", c.URL, err) + u.LogErrorf("Controller Auth or Connection failed, but continuing to retry! %s: %v", c.Name, err) } } diff --git a/integrations/inputunifi/pkg/promunifi/collector.go b/integrations/inputunifi/pkg/promunifi/collector.go index 61c8db81..e999ac3f 100644 --- a/integrations/inputunifi/pkg/promunifi/collector.go +++ b/integrations/inputunifi/pkg/promunifi/collector.go @@ -44,16 +44,16 @@ type Prometheus struct { // Config is the input (config file) data used to initialize this output plugin. type Config struct { - Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` // If non-empty, each of the collected metrics is prefixed by the // provided string and an underscore ("_"). - Namespace string `json:"namespace" toml:"namespace" xml:"namespace" yaml:"namespace"` + Namespace string `json:"namespace" toml:"namespace" xml:"namespace" yaml:"namespace"` + HTTPListen string `json:"http_listen" toml:"http_listen" xml:"http_listen" yaml:"http_listen"` // If true, any error encountered during collection is reported as an // invalid metric (see NewInvalidMetric). Otherwise, errors are ignored // and the collected metrics will be incomplete. Possibly, no metrics // will be collected at all. - ReportErrors bool `json:"report_errors" toml:"report_errors" xml:"report_errors" yaml:"report_errors"` - HTTPListen string `json:"http_listen" toml:"http_listen" xml:"http_listen" yaml:"http_listen"` + ReportErrors bool `json:"report_errors" toml:"report_errors" xml:"report_errors" yaml:"report_errors"` + Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` } type metric struct { From 2032f9d6456d35a5c8ecb8a8339979fb19b0d0ea Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sun, 15 Dec 2019 20:56:42 -0800 Subject: [PATCH 15/41] convert input to plugin --- integrations/inputunifi/.metadata.sh | 2 +- integrations/inputunifi/main.go | 4 +- .../inputunifi/pkg/influxunifi/clients.go | 9 +- .../inputunifi/pkg/influxunifi/ids.go | 1 + .../influxunifi/{metrics.go => influxdb.go} | 36 ++- .../inputunifi/pkg/influxunifi/site.go | 1 + .../inputunifi/pkg/influxunifi/uap.go | 7 + .../inputunifi/pkg/influxunifi/udm.go | 12 +- .../inputunifi/pkg/influxunifi/usg.go | 38 +-- .../inputunifi/pkg/influxunifi/usw.go | 6 + .../inputunifi/pkg/inputunifi/collector.go | 138 ++++++++++ .../inputunifi/pkg/inputunifi/input.go | 78 ++++++ .../inputunifi/pkg/inputunifi/interface.go | 122 +++++++++ .../inputunifi/pkg/poller/build_macos.go | 2 +- .../inputunifi/pkg/poller/build_unix.go | 2 +- .../inputunifi/pkg/poller/build_windows.go | 2 +- integrations/inputunifi/pkg/poller/config.go | 74 +++--- integrations/inputunifi/pkg/poller/dumper.go | 87 ++++--- integrations/inputunifi/pkg/poller/inputs.go | 96 +++++++ .../pkg/poller/{helpers.go => logger.go} | 14 +- integrations/inputunifi/pkg/poller/outputs.go | 5 +- integrations/inputunifi/pkg/poller/start.go | 42 +-- integrations/inputunifi/pkg/poller/unifi.go | 241 ------------------ .../inputunifi/pkg/promunifi/clients.go | 48 ++-- .../inputunifi/pkg/promunifi/collector.go | 44 ++-- integrations/inputunifi/pkg/promunifi/site.go | 56 ++-- integrations/inputunifi/pkg/promunifi/uap.go | 139 +++++----- integrations/inputunifi/pkg/promunifi/usg.go | 4 + integrations/inputunifi/pkg/promunifi/usw.go | 73 +++--- 29 files changed, 797 insertions(+), 586 deletions(-) rename integrations/inputunifi/pkg/influxunifi/{metrics.go => influxdb.go} (97%) create mode 100644 integrations/inputunifi/pkg/inputunifi/collector.go create mode 100644 integrations/inputunifi/pkg/inputunifi/input.go create mode 100644 integrations/inputunifi/pkg/inputunifi/interface.go create mode 100644 integrations/inputunifi/pkg/poller/inputs.go rename integrations/inputunifi/pkg/poller/{helpers.go => logger.go} (75%) delete mode 100644 integrations/inputunifi/pkg/poller/unifi.go diff --git a/integrations/inputunifi/.metadata.sh b/integrations/inputunifi/.metadata.sh index c4b30be1..7a78171d 100755 --- a/integrations/inputunifi/.metadata.sh +++ b/integrations/inputunifi/.metadata.sh @@ -11,7 +11,7 @@ HBREPO="golift/homebrew-mugs" MAINT="David Newhall II " VENDOR="Go Lift " DESC="Polls a UniFi controller, exports metrics to InfluxDB and Prometheus" -GOLANGCI_LINT_ARGS="--enable-all -D gochecknoglobals -D dupl -D lll -D funlen -D wsl -e G402 -D gochecknoinits" +GOLANGCI_LINT_ARGS="--enable-all -D gochecknoglobals -D funlen -e G402 -D gochecknoinits" # Example must exist at examples/$CONFIG_FILE.example CONFIG_FILE="up.conf" LICENSE="MIT" diff --git a/integrations/inputunifi/main.go b/integrations/inputunifi/main.go index be0fe13b..24315b5d 100644 --- a/integrations/inputunifi/main.go +++ b/integrations/inputunifi/main.go @@ -4,7 +4,9 @@ import ( "log" "github.com/davidnewhall/unifi-poller/pkg/poller" - // Enable output plugins! + // Load input plugins! + _ "github.com/davidnewhall/unifi-poller/pkg/inputunifi" + // Load output plugins! _ "github.com/davidnewhall/unifi-poller/pkg/influxunifi" _ "github.com/davidnewhall/unifi-poller/pkg/promunifi" ) diff --git a/integrations/inputunifi/pkg/influxunifi/clients.go b/integrations/inputunifi/pkg/influxunifi/clients.go index 127ab7e2..b4c346dc 100644 --- a/integrations/inputunifi/pkg/influxunifi/clients.go +++ b/integrations/inputunifi/pkg/influxunifi/clients.go @@ -66,14 +66,7 @@ func (u *InfluxUnifi) batchClient(r report, s *unifi.Client) { "wired-tx_bytes": s.WiredTxBytes, "wired-tx_bytes-r": s.WiredTxBytesR, "wired-tx_packets": s.WiredTxPackets, - /* - "dpi_app": c.DpiStats.App.Val, - "dpi_cat": c.DpiStats.Cat.Val, - "dpi_rx_bytes": c.DpiStats.RxBytes.Val, - "dpi_rx_packets": c.DpiStats.RxPackets.Val, - "dpi_tx_bytes": c.DpiStats.TxBytes.Val, - "dpi_tx_packets": c.DpiStats.TxPackets.Val, - */ } + r.send(&metric{Table: "clients", Tags: tags, Fields: fields}) } diff --git a/integrations/inputunifi/pkg/influxunifi/ids.go b/integrations/inputunifi/pkg/influxunifi/ids.go index c7b8edba..ad0b855a 100644 --- a/integrations/inputunifi/pkg/influxunifi/ids.go +++ b/integrations/inputunifi/pkg/influxunifi/ids.go @@ -35,5 +35,6 @@ func (u *InfluxUnifi) batchIDS(r report, i *unifi.IDS) { "srcipASN": i.SrcipASN, "usgipASN": i.UsgipASN, } + r.send(&metric{Table: "intrusion_detect", Tags: tags, Fields: fields}) } diff --git a/integrations/inputunifi/pkg/influxunifi/metrics.go b/integrations/inputunifi/pkg/influxunifi/influxdb.go similarity index 97% rename from integrations/inputunifi/pkg/influxunifi/metrics.go rename to integrations/inputunifi/pkg/influxunifi/influxdb.go index e983edce..199ea594 100644 --- a/integrations/inputunifi/pkg/influxunifi/metrics.go +++ b/integrations/inputunifi/pkg/influxunifi/influxdb.go @@ -17,7 +17,7 @@ const ( defaultInterval = 30 * time.Second minimumInterval = 10 * time.Second defaultInfluxDB = "unifi" - defaultInfluxUser = "unifi" + defaultInfluxUser = "unifipoller" defaultInfluxURL = "http://127.0.0.1:8086" ) @@ -53,6 +53,7 @@ type metric struct { func init() { u := &InfluxUnifi{InfluxDB: &InfluxDB{}, LastCheck: time.Now()} + poller.NewOutput(&poller.Output{ Name: "influxdb", Config: u.InfluxDB, @@ -143,9 +144,12 @@ func (u *InfluxUnifi) setConfigDefaults() { func (u *InfluxUnifi) ReportMetrics(m *poller.Metrics) (*Report, error) { r := &Report{Metrics: m, ch: make(chan *metric), Start: time.Now()} defer close(r.ch) - // Make a new Influx Points Batcher. + var err error + + // Make a new Influx Points Batcher. r.bp, err = influx.NewBatchPoints(influx.BatchPointsConfig{Database: u.Config.DB}) + if err != nil { return nil, fmt.Errorf("influx.NewBatchPoints: %v", err) } @@ -159,7 +163,9 @@ func (u *InfluxUnifi) ReportMetrics(m *poller.Metrics) (*Report, error) { if err = u.influx.Write(r.bp); err != nil { return nil, fmt.Errorf("influxdb.Write(points): %v", err) } + r.Elapsed = time.Since(r.Start) + return r, nil } @@ -172,6 +178,7 @@ func (u *InfluxUnifi) collect(r report, ch chan *metric) { } else { r.batch(m, pt) } + r.done() } } @@ -182,24 +189,28 @@ func (u *InfluxUnifi) loopPoints(r report) { m := r.metrics() r.add() + r.add() + r.add() + go func() { defer r.done() + for _, s := range m.Sites { u.batchSite(r, s) } }() - r.add() go func() { defer r.done() + for _, s := range m.Clients { u.batchClient(r, s) } }() - r.add() go func() { defer r.done() + for _, s := range m.IDSList { u.batchIDS(r, s) } @@ -209,33 +220,44 @@ func (u *InfluxUnifi) loopPoints(r report) { return } + u.loopDevicePoints(r) +} + +func (u *InfluxUnifi) loopDevicePoints(r report) { + m := r.metrics() + r.add() + r.add() + r.add() + r.add() + go func() { defer r.done() + for _, s := range m.UAPs { u.batchUAP(r, s) } }() - r.add() go func() { defer r.done() + for _, s := range m.USGs { u.batchUSG(r, s) } }() - r.add() go func() { defer r.done() + for _, s := range m.USWs { u.batchUSW(r, s) } }() - r.add() go func() { defer r.done() + for _, s := range m.UDMs { u.batchUDM(r, s) } diff --git a/integrations/inputunifi/pkg/influxunifi/site.go b/integrations/inputunifi/pkg/influxunifi/site.go index 243d2acc..30e2ce37 100644 --- a/integrations/inputunifi/pkg/influxunifi/site.go +++ b/integrations/inputunifi/pkg/influxunifi/site.go @@ -51,6 +51,7 @@ func (u *InfluxUnifi) batchSite(r report, s *unifi.Site) { "remote_user_tx_packets": h.RemoteUserTxPackets.Val, "num_new_alarms": s.NumNewAlarms.Val, } + r.send(&metric{Table: "subsystems", Tags: tags, Fields: fields}) } } diff --git a/integrations/inputunifi/pkg/influxunifi/uap.go b/integrations/inputunifi/pkg/influxunifi/uap.go index 46c25c32..f47a11fc 100644 --- a/integrations/inputunifi/pkg/influxunifi/uap.go +++ b/integrations/inputunifi/pkg/influxunifi/uap.go @@ -10,6 +10,7 @@ func (u *InfluxUnifi) batchUAP(r report, s *unifi.UAP) { if !s.Adopted.Val || s.Locating.Val { return } + tags := map[string]string{ "mac": s.Mac, "site_name": s.SiteName, @@ -30,6 +31,7 @@ func (u *InfluxUnifi) batchUAP(r report, s *unifi.UAP) { fields["user-num_sta"] = int(s.UserNumSta.Val) fields["guest-num_sta"] = int(s.GuestNumSta.Val) fields["num_sta"] = s.NumSta.Val + r.send(&metric{Table: "uap", Tags: tags, Fields: fields}) u.processRadTable(r, tags, s.RadioTable, s.RadioTableStats) u.processVAPTable(r, tags, s.VapTable) @@ -39,6 +41,7 @@ func (u *InfluxUnifi) processUAPstats(ap *unifi.Ap) map[string]interface{} { if ap == nil { return map[string]interface{}{} } + // Accumulative Statistics. return map[string]interface{}{ "stat_user-rx_packets": ap.UserRxPackets.Val, @@ -135,6 +138,7 @@ func (u *InfluxUnifi) processVAPTable(r report, t map[string]string, vt unifi.Va "wifi_tx_latency_mov_total": s.WifiTxLatencyMov.Total.Val, "wifi_tx_latency_mov_cuont": s.WifiTxLatencyMov.TotalCount.Val, } + r.send(&metric{Table: "uap_vaps", Tags: tags, Fields: fields}) } } @@ -155,6 +159,7 @@ func (u *InfluxUnifi) processRadTable(r report, t map[string]string, rt unifi.Ra "nss": p.Nss.Val, "radio_caps": p.RadioCaps.Val, } + for _, t := range rts { if t.Name == p.Name { fields["ast_be_xmit"] = t.AstBeXmit.Val @@ -171,9 +176,11 @@ func (u *InfluxUnifi) processRadTable(r report, t map[string]string, rt unifi.Ra fields["tx_power"] = t.TxPower.Val fields["tx_retries"] = t.TxRetries.Val fields["user-num_sta"] = t.UserNumSta.Val + break } } + r.send(&metric{Table: "uap_radios", Tags: tags, Fields: fields}) } } diff --git a/integrations/inputunifi/pkg/influxunifi/udm.go b/integrations/inputunifi/pkg/influxunifi/udm.go index af42e5d0..20cea055 100644 --- a/integrations/inputunifi/pkg/influxunifi/udm.go +++ b/integrations/inputunifi/pkg/influxunifi/udm.go @@ -4,14 +4,16 @@ import ( "golift.io/unifi" ) -// Combines concatenates N maps. This will delete things if not used with caution. +// Combine concatenates N maps. This will delete things if not used with caution. func Combine(in ...map[string]interface{}) map[string]interface{} { out := make(map[string]interface{}) + for i := range in { for k := range in[i] { out[k] = in[i][k] } } + return out } @@ -36,6 +38,7 @@ func (u *InfluxUnifi) batchUDM(r report, s *unifi.UDM) { if !s.Adopted.Val || s.Locating.Val { return } + tags := map[string]string{ "mac": s.Mac, "site_name": s.SiteName, @@ -65,6 +68,7 @@ func (u *InfluxUnifi) batchUDM(r report, s *unifi.UDM) { "num_mobile": s.NumMobile.Val, }, ) + r.send(&metric{Table: "usg", Tags: tags, Fields: fields}) u.batchNetTable(r, tags, s.NetworkTable) u.batchUSGwans(r, tags, s.Wan1, s.Wan2) @@ -90,13 +94,14 @@ func (u *InfluxUnifi) batchUDM(r report, s *unifi.UDM) { "uptime": s.Uptime.Val, "state": s.State.Val, }) + r.send(&metric{Table: "usw", Tags: tags, Fields: fields}) u.batchPortTable(r, tags, s.PortTable) if s.Stat.Ap == nil { - return - // we're done now. the following code process UDM (non-pro) UAP data. + return // we're done now. the following code process UDM (non-pro) UAP data. } + tags = map[string]string{ "mac": s.Mac, "site_name": s.SiteName, @@ -117,6 +122,7 @@ func (u *InfluxUnifi) batchUDM(r report, s *unifi.UDM) { fields["user-num_sta"] = int(s.UserNumSta.Val) fields["guest-num_sta"] = int(s.GuestNumSta.Val) fields["num_sta"] = s.NumSta.Val + r.send(&metric{Table: "uap", Tags: tags, Fields: fields}) u.processRadTable(r, tags, *s.RadioTable, *s.RadioTableStats) u.processVAPTable(r, tags, *s.VapTable) diff --git a/integrations/inputunifi/pkg/influxunifi/usg.go b/integrations/inputunifi/pkg/influxunifi/usg.go index 221e0e40..36c9cdbd 100644 --- a/integrations/inputunifi/pkg/influxunifi/usg.go +++ b/integrations/inputunifi/pkg/influxunifi/usg.go @@ -10,6 +10,7 @@ func (u *InfluxUnifi) batchUSG(r report, s *unifi.USG) { if !s.Adopted.Val || s.Locating.Val { return } + tags := map[string]string{ "mac": s.Mac, "site_name": s.SiteName, @@ -39,44 +40,17 @@ func (u *InfluxUnifi) batchUSG(r report, s *unifi.USG) { "num_mobile": s.NumMobile.Val, }, ) + r.send(&metric{Table: "usg", Tags: tags, Fields: fields}) u.batchNetTable(r, tags, s.NetworkTable) u.batchUSGwans(r, tags, s.Wan1, s.Wan2) - - /* - for _, p := range s.PortTable { - t := map[string]string{ - "device_name": tags["name"], - "site_name": tags["site_name"], - "name": p.Name, - "ifname": p.Ifname, - "ip": p.IP, - "mac": p.Mac, - "up": p.Up.Txt, - "speed": p.Speed.Txt, - "full_duplex": p.FullDuplex.Txt, - "enable": p.Enable.Txt, - } - f := map[string]interface{}{ - "rx_bytes": p.RxBytes.Val, - "rx_dropped": p.RxDropped.Val, - "rx_errors": p.RxErrors.Val, - "rx_packets": p.RxBytes.Val, - "tx_bytes": p.TxBytes.Val, - "tx_dropped": p.TxDropped.Val, - "tx_errors": p.TxErrors.Val, - "tx_packets": p.TxPackets.Val, - "rx_multicast": p.RxMulticast.Val, - "dns_servers": strings.Join(p.DNS, ","), - } - r.send(&metric{Table: "usg_ports", Tags: t, Fields: f}) - } - */ } + func (u *InfluxUnifi) batchUSGstat(ss unifi.SpeedtestStatus, gw *unifi.Gw, ul unifi.Uplink) map[string]interface{} { if gw == nil { return map[string]interface{}{} } + return map[string]interface{}{ "uplink_latency": ul.Latency.Val, "uplink_speed": ul.Speed.Val, @@ -92,11 +66,13 @@ func (u *InfluxUnifi) batchUSGstat(ss unifi.SpeedtestStatus, gw *unifi.Gw, ul un "lan-rx_dropped": gw.LanRxDropped.Val, } } + func (u *InfluxUnifi) batchUSGwans(r report, tags map[string]string, wans ...unifi.Wan) { for _, wan := range wans { if !wan.Up.Val { continue } + tags := map[string]string{ "device_name": tags["name"], "site_name": tags["site_name"], @@ -129,6 +105,7 @@ func (u *InfluxUnifi) batchUSGwans(r report, tags map[string]string, wans ...uni "tx_broadcast": wan.TxBroadcast.Val, "tx_multicast": wan.TxMulticast.Val, } + r.send(&metric{Table: "usg_wan_ports", Tags: tags, Fields: fields}) } } @@ -154,6 +131,7 @@ func (u *InfluxUnifi) batchNetTable(r report, tags map[string]string, nt unifi.N "tx_bytes": p.TxBytes.Val, "tx_packets": p.TxPackets.Val, } + r.send(&metric{Table: "usg_networks", Tags: tags, Fields: fields}) } } diff --git a/integrations/inputunifi/pkg/influxunifi/usw.go b/integrations/inputunifi/pkg/influxunifi/usw.go index 7bc31c37..0a91a506 100644 --- a/integrations/inputunifi/pkg/influxunifi/usw.go +++ b/integrations/inputunifi/pkg/influxunifi/usw.go @@ -36,6 +36,7 @@ func (u *InfluxUnifi) batchUSW(r report, s *unifi.USW) { "state": s.State.Val, "user-num_sta": s.UserNumSta.Val, }) + r.send(&metric{Table: "usw", Tags: tags, Fields: fields}) u.batchPortTable(r, tags, s.PortTable) } @@ -44,6 +45,7 @@ func (u *InfluxUnifi) batchUSWstat(sw *unifi.Sw) map[string]interface{} { if sw == nil { return map[string]interface{}{} } + return map[string]interface{}{ "stat_bytes": sw.Bytes.Val, "stat_rx_bytes": sw.RxBytes.Val, @@ -59,11 +61,13 @@ func (u *InfluxUnifi) batchUSWstat(sw *unifi.Sw) map[string]interface{} { "stat_tx_retries": sw.TxRetries.Val, } } + func (u *InfluxUnifi) batchPortTable(r report, t map[string]string, pt []unifi.Port) { for _, p := range pt { if !p.Up.Val || !p.Enable.Val { continue // only record UP ports. } + tags := map[string]string{ "site_name": t["site_name"], "device_name": t["name"], @@ -96,11 +100,13 @@ func (u *InfluxUnifi) batchPortTable(r report, t map[string]string, pt []unifi.P "tx_multicast": p.TxMulticast.Val, "tx_packets": p.TxPackets.Val, } + if p.PoeEnable.Val && p.PortPoe.Val { fields["poe_current"] = p.PoeCurrent.Val fields["poe_power"] = p.PoePower.Val fields["poe_voltage"] = p.PoeVoltage.Val } + r.send(&metric{Table: "usw_ports", Tags: tags, Fields: fields}) } } diff --git a/integrations/inputunifi/pkg/inputunifi/collector.go b/integrations/inputunifi/pkg/inputunifi/collector.go new file mode 100644 index 00000000..20aeafac --- /dev/null +++ b/integrations/inputunifi/pkg/inputunifi/collector.go @@ -0,0 +1,138 @@ +package inputunifi + +import ( + "fmt" + "time" + + "github.com/davidnewhall/unifi-poller/pkg/poller" + "golift.io/unifi" +) + +func (u *InputUnifi) isNill(c Controller) bool { + u.Config.RLock() + defer u.Config.RUnlock() + + return c.Unifi == nil +} + +func (u *InputUnifi) collectController(c Controller) (*poller.Metrics, error) { + if u.isNill(c) { + u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) + + if err := u.getUnifi(c); err != nil { + return nil, fmt.Errorf("re-authenticating to %s: %v", c.Name, err) + } + } + + m, err := u.pollController(c) + if err == nil { + return m, nil + } + + return u.pollController(c) +} + +func (u *InputUnifi) pollController(c Controller) (*poller.Metrics, error) { + var err error + + u.Config.RLock() + defer u.Config.RUnlock() + + m := &poller.Metrics{TS: time.Now()} // At this point, it's the Current Check. + + // Get the sites we care about. + if m.Sites, err = u.getFilteredSites(c); err != nil { + return m, fmt.Errorf("unifi.GetSites(%v): %v", c.URL, err) + } + + if c.SaveIDS { + m.IDSList, err = c.Unifi.GetIDS(m.Sites, time.Now().Add(2*time.Minute), time.Now()) + if err != nil { + return m, fmt.Errorf("unifi.GetIDS(%v): %v", c.URL, err) + } + } + + // Get all the points. + if m.Clients, err = c.Unifi.GetClients(m.Sites); err != nil { + return m, fmt.Errorf("unifi.GetClients(%v): %v", c.URL, err) + } + + if m.Devices, err = c.Unifi.GetDevices(m.Sites); err != nil { + return m, fmt.Errorf("unifi.GetDevices(%v): %v", c.URL, err) + } + + return u.augmentMetrics(c, m), nil +} + +// augmentMetrics is our middleware layer between collecting metrics and writing them. +// This is where we can manipuate the returned data or make arbitrary decisions. +// This function currently adds parent device names to client metrics. +func (u *InputUnifi) augmentMetrics(c Controller, metrics *poller.Metrics) *poller.Metrics { + if metrics == nil || metrics.Devices == nil || metrics.Clients == nil { + return metrics + } + + devices := make(map[string]string) + bssdIDs := make(map[string]string) + + for _, r := range metrics.UAPs { + devices[r.Mac] = r.Name + + for _, v := range r.VapTable { + bssdIDs[v.Bssid] = fmt.Sprintf("%s %s %s:", r.Name, v.Radio, v.RadioName) + } + } + + for _, r := range metrics.USGs { + devices[r.Mac] = r.Name + } + + for _, r := range metrics.USWs { + devices[r.Mac] = r.Name + } + + for _, r := range metrics.UDMs { + devices[r.Mac] = r.Name + } + + // These come blank, so set them here. + for i, c := range metrics.Clients { + metrics.Clients[i].SwName = devices[c.SwMac] + metrics.Clients[i].ApName = devices[c.ApMac] + metrics.Clients[i].GwName = devices[c.GwMac] + metrics.Clients[i].RadioDescription = bssdIDs[metrics.Clients[i].Bssid] + metrics.Clients[i].RadioProto + } + + if !c.SaveSites { + metrics.Sites = nil + } + + return metrics +} + +// getFilteredSites returns a list of sites to fetch data for. +// Omits requested but unconfigured sites. Grabs the full list from the +// controller and returns the sites provided in the config file. +func (u *InputUnifi) getFilteredSites(c Controller) (unifi.Sites, error) { + u.Config.RLock() + defer u.Config.RUnlock() + + sites, err := c.Unifi.GetSites() + if err != nil { + return nil, err + } else if len(c.Sites) < 1 || poller.StringInSlice("all", c.Sites) { + return sites, nil + } + + var i int + + for _, s := range sites { + // Only include valid sites in the request filter. + if poller.StringInSlice(s.Name, c.Sites) { + sites[i] = s + i++ + } + } + + return sites[:i], nil +} diff --git a/integrations/inputunifi/pkg/inputunifi/input.go b/integrations/inputunifi/pkg/inputunifi/input.go new file mode 100644 index 00000000..f797d35b --- /dev/null +++ b/integrations/inputunifi/pkg/inputunifi/input.go @@ -0,0 +1,78 @@ +// Package inputunifi implements the poller.Input interface and bridges the gap between +// metrics from the unifi library, and the augments required to pump them into unifi-poller. +package inputunifi + +import ( + "fmt" + + "sync" + + "github.com/davidnewhall/unifi-poller/pkg/poller" + "golift.io/unifi" +) + +// InputUnifi contains the running data. +type InputUnifi struct { + Config Config `json:"unifi" toml:"unifi" xml:"unifi" yaml:"unifi"` + poller.Logger +} + +// Controller represents the configuration for a UniFi Controller. +// Each polled controller may have its own configuration. +type Controller struct { + VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` + SaveIDS bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` + SaveSites bool `json:"save_sites" toml:"save_sites" xml:"save_sites" yaml:"save_sites"` + Name string `json:"name" toml:"name" xml:"name,attr" yaml:"name"` + User string `json:"user" toml:"user" xml:"user" yaml:"user"` + Pass string `json:"pass" toml:"pass" xml:"pass" yaml:"pass"` + URL string `json:"url" toml:"url" xml:"url" yaml:"url"` + Sites []string `json:"sites,omitempty" toml:"sites,omitempty" xml:"sites" yaml:"sites"` + Unifi *unifi.Unifi `json:"-" toml:"-" xml:"-" yaml:"-"` +} + +// Config contains our configuration data +type Config struct { + sync.RWMutex // locks the Unifi struct member when re-authing to unifi. + Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` + Controllers []Controller `json:"controller" toml:"controller" xml:"controller" yaml:"controller"` +} + +func init() { + u := &InputUnifi{} + + poller.NewInput(&poller.InputPlugin{ + Input: u, // this library implements poller.Input interface for Metrics(). + Config: u, // Defines our config data interface. + }) +} + +// getUnifi (re-)authenticates to a unifi controller. +func (u *InputUnifi) getUnifi(c Controller) error { + var err error + + u.Config.Lock() + defer u.Config.Unlock() + + if c.Unifi != nil { + c.Unifi.CloseIdleConnections() + } + + // Create an authenticated session to the Unifi Controller. + c.Unifi, err = unifi.NewUnifi(&unifi.Config{ + User: c.User, + Pass: c.Pass, + URL: c.URL, + VerifySSL: c.VerifySSL, + ErrorLog: u.LogErrorf, // Log all errors. + DebugLog: u.LogDebugf, // Log debug messages. + }) + if err != nil { + c.Unifi = nil + return fmt.Errorf("unifi controller: %v", err) + } + + u.LogDebugf("Authenticated with controller successfully, %s", c.URL) + + return nil +} diff --git a/integrations/inputunifi/pkg/inputunifi/interface.go b/integrations/inputunifi/pkg/inputunifi/interface.go new file mode 100644 index 00000000..a2ef9765 --- /dev/null +++ b/integrations/inputunifi/pkg/inputunifi/interface.go @@ -0,0 +1,122 @@ +package inputunifi + +import ( + "fmt" + "strings" + + "github.com/davidnewhall/unifi-poller/pkg/poller" + "golift.io/unifi" +) + +// Metrics grabs all the measurements from a UniFi controller and returns them. +func (u *InputUnifi) Metrics() (*poller.Metrics, error) { + errs := []string{} + metrics := &poller.Metrics{} + + for _, c := range u.Config.Controllers { + m, err := u.collectController(c) + if err != nil { + errs = append(errs, err.Error()) + } + + if m == nil { + continue + } + + metrics.Sites = append(metrics.Sites, m.Sites...) + metrics.Clients = append(metrics.Clients, m.Clients...) + metrics.IDSList = append(metrics.IDSList, m.IDSList...) + + if m.Devices == nil { + continue + } + + if metrics.Devices == nil { + metrics.Devices = &unifi.Devices{} + } + + metrics.UAPs = append(metrics.UAPs, m.UAPs...) + metrics.USGs = append(metrics.USGs, m.USGs...) + metrics.USWs = append(metrics.USWs, m.USWs...) + metrics.UDMs = append(metrics.UDMs, m.UDMs...) + } + + if len(errs) > 0 { + return metrics, fmt.Errorf(strings.Join(errs, ", ")) + } + + return metrics, nil +} + +// Initialize gets called one time when starting up. +// Satisfies poller.Input interface. +func (u *InputUnifi) Initialize(l poller.Logger) error { + if u.Config.Disable { + l.Logf("unifi input disabled") + return nil + } + + if len(u.Config.Controllers) < 1 { + return fmt.Errorf("no unifi controllers defined for unifi input") + } + + u.Logger = l + + for i, c := range u.Config.Controllers { + if c.Name == "" { + u.Config.Controllers[i].Name = c.URL + } + + switch err := u.getUnifi(c); err { + case nil: + if err := u.checkSites(c); err != nil { + u.LogErrorf("checking sites on %s: %v", c.Name, err) + } + + u.Logf("Polling UniFi Controller at %s v%s as user %s. Sites: %v", + c.URL, c.Unifi.ServerVersion, c.User, c.Sites) + default: + u.LogErrorf("Controller Auth or Connection failed, but continuing to retry! %s: %v", c.Name, err) + } + } + + return nil +} + +// checkSites makes sure the list of provided sites exists on the controller. +// This only runs once during initialization. +func (u *InputUnifi) checkSites(c Controller) error { + u.Config.RLock() + defer u.Config.RUnlock() + u.LogDebugf("Checking Controller Sites List") + + sites, err := c.Unifi.GetSites() + if err != nil { + return err + } + + msg := []string{} + + for _, site := range sites { + msg = append(msg, site.Name+" ("+site.Desc+")") + } + + u.Logf("Found %d site(s) on controller: %v", len(msg), strings.Join(msg, ", ")) + + if poller.StringInSlice("all", c.Sites) { + c.Sites = []string{"all"} + return nil + } + +FIRST: + for _, s := range c.Sites { + for _, site := range sites { + if s == site.Name { + continue FIRST + } + } + return fmt.Errorf("configured site not found on controller: %v", s) + } + + return nil +} diff --git a/integrations/inputunifi/pkg/poller/build_macos.go b/integrations/inputunifi/pkg/poller/build_macos.go index 1ab32471..b3f37dbf 100644 --- a/integrations/inputunifi/pkg/poller/build_macos.go +++ b/integrations/inputunifi/pkg/poller/build_macos.go @@ -2,5 +2,5 @@ package poller -// DefaultConfFile is where to find config is --config is not prvided. +// DefaultConfFile is where to find config if --config is not prvided. const DefaultConfFile = "/usr/local/etc/unifi-poller/up.conf" diff --git a/integrations/inputunifi/pkg/poller/build_unix.go b/integrations/inputunifi/pkg/poller/build_unix.go index c1001ac9..c1f525a9 100644 --- a/integrations/inputunifi/pkg/poller/build_unix.go +++ b/integrations/inputunifi/pkg/poller/build_unix.go @@ -2,5 +2,5 @@ package poller -// DefaultConfFile is where to find config is --config is not prvided. +// DefaultConfFile is where to find config if --config is not prvided. const DefaultConfFile = "/etc/unifi-poller/up.conf" diff --git a/integrations/inputunifi/pkg/poller/build_windows.go b/integrations/inputunifi/pkg/poller/build_windows.go index 5c31504f..a74c76a8 100644 --- a/integrations/inputunifi/pkg/poller/build_windows.go +++ b/integrations/inputunifi/pkg/poller/build_windows.go @@ -2,5 +2,5 @@ package poller -// DefaultConfFile is where to find config is --config is not prvided. +// DefaultConfFile is where to find config if --config is not prvided. const DefaultConfFile = `C:\ProgramData\unifi-poller\up.conf` diff --git a/integrations/inputunifi/pkg/poller/config.go b/integrations/inputunifi/pkg/poller/config.go index e77db20b..03e69b81 100644 --- a/integrations/inputunifi/pkg/poller/config.go +++ b/integrations/inputunifi/pkg/poller/config.go @@ -9,7 +9,6 @@ package poller */ import ( - "sync" "time" "github.com/spf13/pflag" @@ -17,23 +16,17 @@ import ( "golift.io/unifi" ) -// App defaults in case they're missing from the config. const ( // AppName is the name of the application. - AppName = "unifi-poller" - defaultUnifiUser = "influx" - defaultUnifiURL = "https://127.0.0.1:8443" + AppName = "unifi-poller" + // ENVConfigPrefix is the prefix appended to an env variable tag name. + ENVConfigPrefix = "UP" ) -// ENVConfigPrefix is the prefix appended to an env variable tag -// name before retrieving the value from the OS. -const ENVConfigPrefix = "UP" - // UnifiPoller contains the application startup data, and auth info for UniFi & Influx. type UnifiPoller struct { - Flags *Flags - Config *Config - sync.Mutex // locks the Unifi struct member when re-authing to unifi. + Flags *Flags + *Config } // Flags represents the CLI args available and their settings. @@ -53,26 +46,9 @@ type Metrics struct { *unifi.Devices } -// Controller represents the configuration for a UniFi Controller. -// Each polled controller may have its own configuration. -type Controller struct { - VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` - SaveIDS bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` - SaveSites bool `json:"save_sites,omitempty" toml:"save_sites,omitempty" xml:"save_sites" yaml:"save_sites"` - Name string `json:"name" toml:"name" xml:"name,attr" yaml:"name"` - User string `json:"user,omitempty" toml:"user,omitempty" xml:"user" yaml:"user"` - Pass string `json:"pass,omitempty" toml:"pass,omitempty" xml:"pass" yaml:"pass"` - URL string `json:"url,omitempty" toml:"url,omitempty" xml:"url" yaml:"url"` - Sites []string `json:"sites,omitempty" toml:"sites,omitempty" xml:"sites" yaml:"sites"` - Unifi *unifi.Unifi `json:"-" toml:"-" xml:"-" yaml:"-"` -} - -// Config represents the data needed to poll a controller and report to influxdb. -// This is all of the data stored in the config file. -// Any with explicit defaults have omitempty on json and toml tags. +// Config represents the core library input data. type Config struct { - Poller `json:"poller" toml:"poller" xml:"poller" yaml:"poller"` - Controllers []Controller `json:"controller,omitempty" toml:"controller,omitempty" xml:"controller" yaml:"controller"` + Poller `json:"poller" toml:"poller" xml:"poller" yaml:"poller"` } // Poller is the global config values. @@ -83,31 +59,43 @@ type Poller struct { // ParseConfigs parses the poller config and the config for each registered output plugin. func (u *UnifiPoller) ParseConfigs() error { - // Parse config file. - if err := config.ParseFile(u.Config, u.Flags.ConfigFile); err != nil { - u.Flags.Usage() - return err - } - - // Update Config with ENV variable overrides. - if _, err := config.ParseENV(u.Config, ENVConfigPrefix); err != nil { + // Parse core config. + if err := u.ParseInterface(u.Config); err != nil { return err } + // Parse output plugin configs. outputSync.Lock() defer outputSync.Unlock() for _, o := range outputs { - // Parse config file for each output plugin. - if err := config.ParseFile(o.Config, u.Flags.ConfigFile); err != nil { + if err := u.ParseInterface(o.Config); err != nil { return err } + } - // Update Config for each output with ENV variable overrides. - if _, err := config.ParseENV(o.Config, ENVConfigPrefix); err != nil { + // Parse input plugin configs. + inputSync.Lock() + defer inputSync.Unlock() + + for _, i := range inputs { + if err := u.ParseInterface(i.Config); err != nil { return err } } return nil } + +// ParseInterface parses the config file and environment variables into the provided interface. +func (u *UnifiPoller) ParseInterface(i interface{}) error { + // Parse config file into provided interface. + if err := config.ParseFile(i, u.Flags.ConfigFile); err != nil { + return err + } + + // Parse environment variables into provided interface. + _, err := config.ParseENV(i, ENVConfigPrefix) + + return err +} diff --git a/integrations/inputunifi/pkg/poller/dumper.go b/integrations/inputunifi/pkg/poller/dumper.go index 5892488b..18abe901 100644 --- a/integrations/inputunifi/pkg/poller/dumper.go +++ b/integrations/inputunifi/pkg/poller/dumper.go @@ -1,55 +1,58 @@ package poller import ( - "fmt" - "os" "strings" - - "golift.io/unifi" ) // DumpJSONPayload prints raw json from the UniFi Controller. // This only works with controller 0 (first one) in the config. func (u *UnifiPoller) DumpJSONPayload() (err error) { - u.Config.Quiet = true - config := u.Config.Controllers[0] - - config.Unifi, err = unifi.NewUnifi(&unifi.Config{ - User: config.User, - Pass: config.Pass, - URL: config.URL, - VerifySSL: config.VerifySSL, - }) - if err != nil { - return err + if true { + return nil } + /* + u.Config.Quiet = true + config := u.Config.Controllers[0] - fmt.Fprintf(os.Stderr, "[INFO] Authenticated to UniFi Controller @ %v as user %v", config.URL, config.User) + config.Unifi, err = unifi.NewUnifi(&unifi.Config{ + User: config.User, + Pass: config.Pass, + URL: config.URL, + VerifySSL: config.VerifySSL, + }) + if err != nil { + return err + } - if err := u.CheckSites(config); err != nil { - return err - } + fmt.Fprintf(os.Stderr, "[INFO] Authenticated to UniFi Controller @ %v as user %v", config.URL, config.User) - config.Unifi.ErrorLog = func(m string, v ...interface{}) { - fmt.Fprintf(os.Stderr, "[ERROR] "+m, v...) - } // Log all errors to stderr. + if err := u.CheckSites(config); err != nil { + return err + } - switch sites, err := u.GetFilteredSites(config); { - case err != nil: - return err - case StringInSlice(u.Flags.DumpJSON, []string{"d", "device", "devices"}): - return u.dumpSitesJSON(config, unifi.APIDevicePath, "Devices", sites) - case StringInSlice(u.Flags.DumpJSON, []string{"client", "clients", "c"}): - return u.dumpSitesJSON(config, unifi.APIClientPath, "Clients", sites) - case strings.HasPrefix(u.Flags.DumpJSON, "other "): - apiPath := strings.SplitN(u.Flags.DumpJSON, " ", 2)[1] - _, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping Path '%s':\n", apiPath) - return u.PrintRawAPIJSON(config, apiPath) - default: - return fmt.Errorf("must provide filter: devices, clients, other") - } + config.Unifi.ErrorLog = func(m string, v ...interface{}) { + fmt.Fprintf(os.Stderr, "[ERROR] "+m, v...) + } // Log all errors to stderr. + + switch sites, err := u.GetFilteredSites(config); { + case err != nil: + return err + case StringInSlice(u.Flags.DumpJSON, []string{"d", "device", "devices"}): + return u.dumpSitesJSON(config, unifi.APIDevicePath, "Devices", sites) + case StringInSlice(u.Flags.DumpJSON, []string{"client", "clients", "c"}): + return u.dumpSitesJSON(config, unifi.APIClientPath, "Clients", sites) + case strings.HasPrefix(u.Flags.DumpJSON, "other "): + apiPath := strings.SplitN(u.Flags.DumpJSON, " ", 2)[1] + _, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping Path '%s':\n", apiPath) + return u.PrintRawAPIJSON(config, apiPath) + default: + return fmt.Errorf("must provide filter: devices, clients, other") + } + */ + return nil } +/* func (u *UnifiPoller) dumpSitesJSON(c Controller, path, name string, sites unifi.Sites) error { for _, s := range sites { apiPath := fmt.Sprintf(path, s.Name) @@ -68,3 +71,15 @@ func (u *UnifiPoller) PrintRawAPIJSON(c Controller, apiPath string) error { fmt.Println(string(body)) return err } +*/ + +// StringInSlice returns true if a string is in a slice. +func StringInSlice(str string, slice []string) bool { + for _, s := range slice { + if strings.EqualFold(s, str) { + return true + } + } + + return false +} diff --git a/integrations/inputunifi/pkg/poller/inputs.go b/integrations/inputunifi/pkg/poller/inputs.go new file mode 100644 index 00000000..c2140b01 --- /dev/null +++ b/integrations/inputunifi/pkg/poller/inputs.go @@ -0,0 +1,96 @@ +package poller + +import ( + "fmt" + "strings" + "sync" + + "golift.io/unifi" +) + +var ( + inputs []*InputPlugin + inputSync sync.Mutex +) + +// Input plugins must implement this interface. +type Input interface { + Initialize(Logger) error // Called once on startup to initialize the plugin. + Metrics() (*Metrics, error) // Called every time new metrics are requested. +} + +// InputPlugin describes an input plugin's consumable interface. +type InputPlugin struct { + Config interface{} // Each config is passed into an unmarshaller later. + Input +} + +// NewInput creates a metric input. This should be called by input plugins +// init() functions. +func NewInput(i *InputPlugin) { + inputSync.Lock() + defer inputSync.Unlock() + + if i == nil || i.Input == nil { + panic("nil output or method passed to poller.NewOutput") + } + + inputs = append(inputs, i) +} + +// InitializeInputs runs the passed-in initializer method for each input plugin. +func (u *UnifiPoller) InitializeInputs() error { + inputSync.Lock() + defer inputSync.Unlock() + + for _, input := range inputs { + // This must return, or the app locks up here. + if err := input.Initialize(u); err != nil { + return err + } + } + + return nil +} + +// Metrics aggregates all the measurements from all configured inputs and returns them. +func (u *UnifiPoller) Metrics() (*Metrics, error) { + errs := []string{} + metrics := &Metrics{} + + for _, input := range inputs { + m, err := input.Metrics() + if err != nil { + errs = append(errs, err.Error()) + } + + if m == nil { + continue + } + + metrics.Sites = append(metrics.Sites, m.Sites...) + metrics.Clients = append(metrics.Clients, m.Clients...) + metrics.IDSList = append(metrics.IDSList, m.IDSList...) + + if m.Devices == nil { + continue + } + + if metrics.Devices == nil { + metrics.Devices = &unifi.Devices{} + } + + metrics.UAPs = append(metrics.UAPs, m.UAPs...) + metrics.USGs = append(metrics.USGs, m.USGs...) + metrics.USWs = append(metrics.USWs, m.USWs...) + metrics.UDMs = append(metrics.UDMs, m.UDMs...) + } + + var err error + + if len(errs) > 0 { + err = fmt.Errorf(strings.Join(errs, ", ")) + } + + return metrics, err +} diff --git a/integrations/inputunifi/pkg/poller/helpers.go b/integrations/inputunifi/pkg/poller/logger.go similarity index 75% rename from integrations/inputunifi/pkg/poller/helpers.go rename to integrations/inputunifi/pkg/poller/logger.go index 92acd223..b498a9b5 100644 --- a/integrations/inputunifi/pkg/poller/helpers.go +++ b/integrations/inputunifi/pkg/poller/logger.go @@ -3,19 +3,15 @@ package poller import ( "fmt" "log" - "strings" ) const callDepth = 2 -// StringInSlice returns true if a string is in a slice. -func StringInSlice(str string, slice []string) bool { - for _, s := range slice { - if strings.EqualFold(s, str) { - return true - } - } - return false +// Logger is passed into input packages so they may write logs. +type Logger interface { + Logf(m string, v ...interface{}) + LogErrorf(m string, v ...interface{}) + LogDebugf(m string, v ...interface{}) } // Logf prints a log entry if quiet is false. diff --git a/integrations/inputunifi/pkg/poller/outputs.go b/integrations/inputunifi/pkg/poller/outputs.go index aaa9a47d..3bbcb72d 100644 --- a/integrations/inputunifi/pkg/poller/outputs.go +++ b/integrations/inputunifi/pkg/poller/outputs.go @@ -14,9 +14,7 @@ var ( // Output packages must implement this interface. type Collect interface { Metrics() (*Metrics, error) - Logf(m string, v ...interface{}) - LogErrorf(m string, v ...interface{}) - LogDebugf(m string, v ...interface{}) + Logger } // Output defines the output data for a metric exporter like influx or prometheus. @@ -49,6 +47,7 @@ func (u *UnifiPoller) InitializeOutputs() error { for _, o := range outputs { count++ + go func(o *Output) { v <- o.Method(u) }(o) diff --git a/integrations/inputunifi/pkg/poller/start.go b/integrations/inputunifi/pkg/poller/start.go index cd293b2f..9f6bedc8 100644 --- a/integrations/inputunifi/pkg/poller/start.go +++ b/integrations/inputunifi/pkg/poller/start.go @@ -37,25 +37,6 @@ func (u *UnifiPoller) Start() error { return err } - if len(u.Config.Controllers) < 1 { - u.Config.Controllers = []Controller{{ - Sites: []string{"all"}, - User: defaultUnifiUser, - Pass: "", - URL: defaultUnifiURL, - SaveSites: true, - }} - } - - if u.Flags.DumpJSON != "" { - return u.DumpJSONPayload() - } - - if u.Config.Debug { - log.SetFlags(log.Lshortfile | log.Lmicroseconds | log.Ldate) - u.LogDebugf("Debug Logging Enabled") - } - return u.Run() } @@ -79,20 +60,19 @@ func (f *Flags) Parse(args []string) { // 2. Run the collector one time and report the metrics to influxdb. (lambda) // 3. Start a web server and wait for Prometheus to poll the application for metrics. func (u *UnifiPoller) Run() error { + if u.Flags.DumpJSON != "" { + return u.DumpJSONPayload() + } + + if u.Config.Debug { + log.SetFlags(log.Lshortfile | log.Lmicroseconds | log.Ldate) + u.LogDebugf("Debug Logging Enabled") + } + log.Printf("[INFO] UniFi Poller v%v Starting Up! PID: %d", version.Version, os.Getpid()) - for i, c := range u.Config.Controllers { - if c.Name == "" { - u.Config.Controllers[i].Name = c.URL - } - - switch err := u.GetUnifi(c); err { - case nil: - u.Logf("Polling UniFi Controller at %s v%s as user %s. Sites: %v", - c.URL, c.Unifi.ServerVersion, c.User, c.Sites) - default: - u.LogErrorf("Controller Auth or Connection failed, but continuing to retry! %s: %v", c.Name, err) - } + if err := u.InitializeInputs(); err != nil { + return err } return u.InitializeOutputs() diff --git a/integrations/inputunifi/pkg/poller/unifi.go b/integrations/inputunifi/pkg/poller/unifi.go deleted file mode 100644 index 516310a3..00000000 --- a/integrations/inputunifi/pkg/poller/unifi.go +++ /dev/null @@ -1,241 +0,0 @@ -package poller - -import ( - "fmt" - "strings" - "time" - - "golift.io/unifi" -) - -// GetUnifi returns a UniFi controller interface. -func (u *UnifiPoller) GetUnifi(c Controller) error { - var err error - - u.Lock() - defer u.Unlock() - - if c.Unifi != nil { - c.Unifi.CloseIdleConnections() - } - // Create an authenticated session to the Unifi Controller. - c.Unifi, err = unifi.NewUnifi(&unifi.Config{ - User: c.User, - Pass: c.Pass, - URL: c.URL, - VerifySSL: c.VerifySSL, - ErrorLog: u.LogErrorf, // Log all errors. - DebugLog: u.LogDebugf, // Log debug messages. - }) - - if err != nil { - c.Unifi = nil - return fmt.Errorf("unifi controller: %v", err) - } - - u.LogDebugf("Authenticated with controller successfully, %s", c.URL) - - return u.CheckSites(c) -} - -// CheckSites makes sure the list of provided sites exists on the controller. -// This does not run in Lambda (run-once) mode. -func (u *UnifiPoller) CheckSites(c Controller) error { - u.LogDebugf("Checking Controller Sites List") - - sites, err := c.Unifi.GetSites() - if err != nil { - return err - } - - msg := []string{} - - for _, site := range sites { - msg = append(msg, site.Name+" ("+site.Desc+")") - } - - u.Logf("Found %d site(s) on controller: %v", len(msg), strings.Join(msg, ", ")) - - if StringInSlice("all", c.Sites) { - c.Sites = []string{"all"} - return nil - } - -FIRST: - for _, s := range c.Sites { - for _, site := range sites { - if s == site.Name { - continue FIRST - } - } - return fmt.Errorf("configured site not found on controller: %v", s) - } - - return nil -} - -// Metrics grabs all the measurements from a UniFi controller and returns them. -func (u *UnifiPoller) Metrics() (*Metrics, error) { - errs := []string{} - metrics := &Metrics{} - - for _, c := range u.Config.Controllers { - m, err := u.checkAndPollController(c) - if err != nil { - errs = append(errs, err.Error()) - } - - if m == nil { - continue - } - - metrics.Sites = append(metrics.Sites, m.Sites...) - metrics.Clients = append(metrics.Clients, m.Clients...) - metrics.IDSList = append(metrics.IDSList, m.IDSList...) - - if m.Devices == nil { - continue - } - - if metrics.Devices == nil { - metrics.Devices = &unifi.Devices{} - } - - metrics.UAPs = append(metrics.UAPs, m.UAPs...) - metrics.USGs = append(metrics.USGs, m.USGs...) - metrics.USWs = append(metrics.USWs, m.USWs...) - metrics.UDMs = append(metrics.UDMs, m.UDMs...) - } - - var err error - - if len(errs) > 0 { - err = fmt.Errorf(strings.Join(errs, ", ")) - } - - return metrics, err -} - -func (u *UnifiPoller) checkAndPollController(c Controller) (*Metrics, error) { - if c.Unifi == nil { - u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) - - if err := u.GetUnifi(c); err != nil { - u.LogErrorf("re-authenticating to %s: %v", c.URL, err) - return nil, err - } - } - - m, err := u.collectController(c) - if err == nil { - return m, nil - } - - u.LogErrorf("collecting metrics %v", err) - u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) - - if err := u.GetUnifi(c); err != nil { - u.LogErrorf("re-authenticating to %s: %v", c.URL, err) - return nil, err - } - - return u.collectController(c) -} - -func (u *UnifiPoller) collectController(c Controller) (*Metrics, error) { - var err error - - m := &Metrics{TS: time.Now()} // At this point, it's the Current Check. - - // Get the sites we care about. - if m.Sites, err = u.GetFilteredSites(c); err != nil { - return m, fmt.Errorf("unifi.GetSites(%v): %v", c.URL, err) - } - - if c.SaveIDS { - m.IDSList, err = c.Unifi.GetIDS(m.Sites, time.Now().Add(2*time.Minute), time.Now()) - if err != nil { - return m, fmt.Errorf("unifi.GetIDS(%v): %v", c.URL, err) - } - } - - // Get all the points. - if m.Clients, err = c.Unifi.GetClients(m.Sites); err != nil { - return m, fmt.Errorf("unifi.GetClients(%v): %v", c.URL, err) - } - - if m.Devices, err = c.Unifi.GetDevices(m.Sites); err != nil { - return m, fmt.Errorf("unifi.GetDevices(%v): %v", c.URL, err) - } - - return u.augmentMetrics(c, m), nil -} - -// augmentMetrics is our middleware layer between collecting metrics and writing them. -// This is where we can manipuate the returned data or make arbitrary decisions. -// This function currently adds parent device names to client metrics. -func (u *UnifiPoller) augmentMetrics(c Controller, metrics *Metrics) *Metrics { - if metrics == nil || metrics.Devices == nil || metrics.Clients == nil { - return metrics - } - - devices := make(map[string]string) - bssdIDs := make(map[string]string) - - for _, r := range metrics.UAPs { - devices[r.Mac] = r.Name - for _, v := range r.VapTable { - bssdIDs[v.Bssid] = fmt.Sprintf("%s %s %s:", r.Name, v.Radio, v.RadioName) - } - } - - for _, r := range metrics.USGs { - devices[r.Mac] = r.Name - } - - for _, r := range metrics.USWs { - devices[r.Mac] = r.Name - } - - for _, r := range metrics.UDMs { - devices[r.Mac] = r.Name - } - - // These come blank, so set them here. - for i, c := range metrics.Clients { - metrics.Clients[i].SwName = devices[c.SwMac] - metrics.Clients[i].ApName = devices[c.ApMac] - metrics.Clients[i].GwName = devices[c.GwMac] - metrics.Clients[i].RadioDescription = bssdIDs[metrics.Clients[i].Bssid] + metrics.Clients[i].RadioProto - } - - if !c.SaveSites { - metrics.Sites = nil - } - - return metrics -} - -// GetFilteredSites returns a list of sites to fetch data for. -// Omits requested but unconfigured sites. Grabs the full list from the -// controller and returns the sites provided in the config file. -func (u *UnifiPoller) GetFilteredSites(c Controller) (unifi.Sites, error) { - var i int - - sites, err := c.Unifi.GetSites() - if err != nil { - return nil, err - } else if len(c.Sites) < 1 || StringInSlice("all", c.Sites) { - return sites, nil - } - - for _, s := range sites { - // Only include valid sites in the request filter. - if StringInSlice(s.Name, c.Sites) { - sites[i] = s - i++ - } - } - - return sites[:i], nil -} diff --git a/integrations/inputunifi/pkg/promunifi/clients.go b/integrations/inputunifi/pkg/promunifi/clients.go index fff79857..0ffaf54c 100644 --- a/integrations/inputunifi/pkg/promunifi/clients.go +++ b/integrations/inputunifi/pkg/promunifi/clients.go @@ -41,7 +41,8 @@ type uclient struct { } func descClient(ns string) *uclient { - labels := []string{"name", "mac", "site_name", "gw_name", "sw_name", "vlan", "ip", "oui", "network", "sw_port", "ap_name", "wired"} + labels := []string{"name", "mac", "site_name", "gw_name", "sw_name", "vlan", + "ip", "oui", "network", "sw_port", "ap_name", "wired"} labelW := append([]string{"radio_name", "radio", "radio_proto", "channel", "essid", "bssid", "radio_desc"}, labels...) return &uclient{ @@ -64,25 +65,34 @@ func descClient(ns string) *uclient { TxPower: prometheus.NewDesc(ns+"radio_transmit_power_dbm", "Client Transmit Power", labelW, nil), TxRate: prometheus.NewDesc(ns+"radio_transmit_rate_bps", "Client Transmit Rate", labelW, nil), WifiTxAttempts: prometheus.NewDesc(ns+"wifi_attempts_transmit_total", "Client Wifi Transmit Attempts", labelW, nil), - Uptime: prometheus.NewDesc(ns+"uptime_seconds", "Client Uptime", labelW, nil), // XXX: re-purpose for info tags. + Uptime: prometheus.NewDesc(ns+"uptime_seconds", "Client Uptime", labelW, nil), /* needs more "looking into" - DpiStatsApp: prometheus.NewDesc(ns+"dpi_stats_app", "Client DPI Stats App", labels, nil), - DpiStatsCat: prometheus.NewDesc(ns+"dpi_stats_cat", "Client DPI Stats Cat", labels, nil), - DpiStatsRxBytes: prometheus.NewDesc(ns+"dpi_stats_receive_bytes_total", "Client DPI Stats Receive Bytes", labels, nil), - DpiStatsRxPackets: prometheus.NewDesc(ns+"dpi_stats_receive_packets_total", "Client DPI Stats Receive Packets", labels, nil), - DpiStatsTxBytes: prometheus.NewDesc(ns+"dpi_stats_transmit_bytes_total", "Client DPI Stats Transmit Bytes", labels, nil), - DpiStatsTxPackets: prometheus.NewDesc(ns+"dpi_stats_transmit_packets_total", "Client DPI Stats Transmit Packets", labels, nil), + DpiStatsApp: prometheus.NewDesc(ns+"dpi_stats_app", + "Client DPI Stats App", labels, nil), + DpiStatsCat: prometheus.NewDesc(ns+"dpi_stats_cat", + "Client DPI Stats Cat", labels, nil), + DpiStatsRxBytes: prometheus.NewDesc(ns+"dpi_stats_receive_bytes_total", + "Client DPI Stats Receive Bytes", labels, nil), + DpiStatsRxPackets: prometheus.NewDesc(ns+"dpi_stats_receive_packets_total", + "Client DPI Stats Receive Packets", labels, nil), + DpiStatsTxBytes: prometheus.NewDesc(ns+"dpi_stats_transmit_bytes_total", + "Client DPI Stats Transmit Bytes", labels, nil), + DpiStatsTxPackets: prometheus.NewDesc(ns+"dpi_stats_transmit_packets_total", + "Client DPI Stats Transmit Packets", labels, nil), */ } } func (u *promUnifi) exportClient(r report, c *unifi.Client) { - labels := []string{c.Name, c.Mac, c.SiteName, c.GwName, c.SwName, c.Vlan.Txt, c.IP, c.Oui, c.Network, c.SwPort.Txt, c.ApName, ""} - labelW := append([]string{c.RadioName, c.Radio, c.RadioProto, c.Channel.Txt, c.Essid, c.Bssid, c.RadioDescription}, labels...) + labels := []string{c.Name, c.Mac, c.SiteName, c.GwName, c.SwName, c.Vlan.Txt, + c.IP, c.Oui, c.Network, c.SwPort.Txt, c.ApName, ""} + labelW := append([]string{c.RadioName, c.Radio, c.RadioProto, c.Channel.Txt, + c.Essid, c.Bssid, c.RadioDescription}, labels...) if c.IsWired.Val { labels[len(labels)-1] = "true" labelW[len(labelW)-1] = "true" + r.send([]*metric{ {u.Client.RxBytes, counter, c.WiredRxBytes, labels}, {u.Client.RxBytesR, gauge, c.WiredRxBytesR, labels}, @@ -94,6 +104,7 @@ func (u *promUnifi) exportClient(r report, c *unifi.Client) { } else { labels[len(labels)-1] = "false" labelW[len(labelW)-1] = "false" + r.send([]*metric{ {u.Client.Anomalies, counter, c.Anomalies, labelW}, {u.Client.CCQ, gauge, float64(c.Ccq) / 1000.0, labelW}, @@ -118,12 +129,13 @@ func (u *promUnifi) exportClient(r report, c *unifi.Client) { } r.send([]*metric{{u.Client.Uptime, gauge, c.Uptime, labelW}}) - /* needs more "looking into" - {u.Client.DpiStatsApp, gauge, c.DpiStats.App, labels}, - {u.Client.DpiStatsCat, gauge, c.DpiStats.Cat, labels}, - {u.Client.DpiStatsRxBytes, counter, c.DpiStats.RxBytes, labels}, - {u.Client.DpiStatsRxPackets, counter, c.DpiStats.RxPackets, labels}, - {u.Client.DpiStatsTxBytes, counter, c.DpiStats.TxBytes, labels}, - {u.Client.DpiStatsTxPackets, counter, c.DpiStats.TxPackets, labels}, - */ } + +/* needs more "looking into" +{u.Client.DpiStatsApp, gauge, c.DpiStats.App, labels}, +{u.Client.DpiStatsCat, gauge, c.DpiStats.Cat, labels}, +{u.Client.DpiStatsRxBytes, counter, c.DpiStats.RxBytes, labels}, +{u.Client.DpiStatsRxPackets, counter, c.DpiStats.RxPackets, labels}, +{u.Client.DpiStatsTxBytes, counter, c.DpiStats.TxBytes, labels}, +{u.Client.DpiStatsTxPackets, counter, c.DpiStats.TxPackets, labels}, +*/ diff --git a/integrations/inputunifi/pkg/promunifi/collector.go b/integrations/inputunifi/pkg/promunifi/collector.go index e999ac3f..a4aad9f3 100644 --- a/integrations/inputunifi/pkg/promunifi/collector.go +++ b/integrations/inputunifi/pkg/promunifi/collector.go @@ -79,6 +79,7 @@ type Report struct { func init() { u := &promUnifi{Prometheus: &Prometheus{}} + poller.NewOutput(&poller.Output{ Name: "prometheus", Config: u.Prometheus, @@ -93,33 +94,27 @@ func (u *promUnifi) Run(c poller.Collect) error { return nil } + u.Config.Namespace = strings.Trim(strings.Replace(u.Config.Namespace, "-", "_", -1), "_") if u.Config.Namespace == "" { u.Config.Namespace = strings.Replace(poller.AppName, "-", "", -1) } - u.Config.Namespace = strings.Replace(u.Config.Namespace, "-", "_", -1) - if u.Config.HTTPListen == "" { u.Config.HTTPListen = defaultHTTPListen } prometheus.MustRegister(version.NewCollector(u.Config.Namespace)) - - if u.Config.Namespace = strings.Trim(u.Config.Namespace, "_") + "_"; u.Config.Namespace == "_" { - u.Config.Namespace = "" - } - prometheus.MustRegister(&promUnifi{ Collector: c, - Client: descClient(u.Config.Namespace + "client_"), - Device: descDevice(u.Config.Namespace + "device_"), // stats for all device types. - UAP: descUAP(u.Config.Namespace + "device_"), - USG: descUSG(u.Config.Namespace + "device_"), - USW: descUSW(u.Config.Namespace + "device_"), - Site: descSite(u.Config.Namespace + "site_"), + Client: descClient(u.Config.Namespace + "_client_"), + Device: descDevice(u.Config.Namespace + "_device_"), // stats for all device types. + UAP: descUAP(u.Config.Namespace + "_device_"), + USG: descUSG(u.Config.Namespace + "_device_"), + USW: descUSW(u.Config.Namespace + "_device_"), + Site: descSite(u.Config.Namespace + "_site_"), }) - - c.Logf("Exporting Measurements for Prometheus at https://%s/metrics, namespace: %s", u.Config.HTTPListen, u.Config.Namespace) + c.Logf("Exporting Measurements for Prometheus at https://%s/metrics, namespace: %s", + u.Config.HTTPListen, u.Config.Namespace) return http.ListenAndServe(u.Config.HTTPListen, nil) } @@ -152,6 +147,7 @@ func (u *promUnifi) Collect(ch chan<- prometheus.Metric) { r.error(ch, prometheus.NewInvalidDesc(fmt.Errorf("metric fetch failed")), err) return } + r.Fetch = time.Since(r.Start) if r.Metrics.Devices == nil { @@ -173,6 +169,7 @@ func (u *promUnifi) exportMetrics(r report, ch chan<- prometheus.Metric, ourChan for newMetrics := range ourChan { for _, m := range newMetrics { descs[m.Desc] = true + switch v := m.Value.(type) { case unifi.FlexInt: ch <- r.export(m, v.Val) @@ -195,48 +192,55 @@ func (u *promUnifi) loopExports(r report) { m := r.metrics() r.add() + r.add() + r.add() + r.add() + r.add() + r.add() + go func() { defer r.done() + for _, s := range m.Sites { u.exportSite(r, s) } }() - r.add() go func() { defer r.done() + for _, d := range m.UAPs { u.exportUAP(r, d) } }() - r.add() go func() { defer r.done() + for _, d := range m.UDMs { u.exportUDM(r, d) } }() - r.add() go func() { defer r.done() + for _, d := range m.USGs { u.exportUSG(r, d) } }() - r.add() go func() { defer r.done() + for _, d := range m.USWs { u.exportUSW(r, d) } }() - r.add() go func() { defer r.done() + for _, c := range m.Clients { u.exportClient(r, c) } diff --git a/integrations/inputunifi/pkg/promunifi/site.go b/integrations/inputunifi/pkg/promunifi/site.go index 1f8999e9..79c57b41 100644 --- a/integrations/inputunifi/pkg/promunifi/site.go +++ b/integrations/inputunifi/pkg/promunifi/site.go @@ -35,32 +35,34 @@ type site struct { func descSite(ns string) *site { labels := []string{"subsystem", "status", "site_name"} + nd := prometheus.NewDesc + return &site{ - NumUser: prometheus.NewDesc(ns+"users", "Number of Users", labels, nil), - NumGuest: prometheus.NewDesc(ns+"guests", "Number of Guests", labels, nil), - NumIot: prometheus.NewDesc(ns+"iots", "Number of IoT Devices", labels, nil), - TxBytesR: prometheus.NewDesc(ns+"transmit_rate_bytes", "Bytes Transmit Rate", labels, nil), - RxBytesR: prometheus.NewDesc(ns+"receive_rate_bytes", "Bytes Receive Rate", labels, nil), - NumAp: prometheus.NewDesc(ns+"aps", "Access Point Count", labels, nil), - NumAdopted: prometheus.NewDesc(ns+"adopted", "Adoption Count", labels, nil), - NumDisabled: prometheus.NewDesc(ns+"disabled", "Disabled Count", labels, nil), - NumDisconnected: prometheus.NewDesc(ns+"disconnected", "Disconnected Count", labels, nil), - NumPending: prometheus.NewDesc(ns+"pending", "Pending Count", labels, nil), - NumGw: prometheus.NewDesc(ns+"gateways", "Gateway Count", labels, nil), - NumSw: prometheus.NewDesc(ns+"switches", "Switch Count", labels, nil), - NumSta: prometheus.NewDesc(ns+"stations", "Station Count", labels, nil), - Latency: prometheus.NewDesc(ns+"latency_seconds", "Latency", labels, nil), - Uptime: prometheus.NewDesc(ns+"uptime_seconds", "Uptime", labels, nil), - Drops: prometheus.NewDesc(ns+"intenet_drops_total", "Internet (WAN) Disconnections", labels, nil), - XputUp: prometheus.NewDesc(ns+"xput_up_rate", "Speedtest Upload", labels, nil), - XputDown: prometheus.NewDesc(ns+"xput_down_rate", "Speedtest Download", labels, nil), - SpeedtestPing: prometheus.NewDesc(ns+"speedtest_ping", "Speedtest Ping", labels, nil), - RemoteUserNumActive: prometheus.NewDesc(ns+"remote_user_active", "Remote Users Active", labels, nil), - RemoteUserNumInactive: prometheus.NewDesc(ns+"remote_user_inactive", "Remote Users Inactive", labels, nil), - RemoteUserRxBytes: prometheus.NewDesc(ns+"remote_user_receive_bytes_total", "Remote Users Receive Bytes", labels, nil), - RemoteUserTxBytes: prometheus.NewDesc(ns+"remote_user_transmit_bytes_total", "Remote Users Transmit Bytes", labels, nil), - RemoteUserRxPackets: prometheus.NewDesc(ns+"remote_user_receive_packets_total", "Remote Users Receive Packets", labels, nil), - RemoteUserTxPackets: prometheus.NewDesc(ns+"remote_user_transmit_packets_total", "Remote Users Transmit Packets", labels, nil), + NumUser: nd(ns+"users", "Number of Users", labels, nil), + NumGuest: nd(ns+"guests", "Number of Guests", labels, nil), + NumIot: nd(ns+"iots", "Number of IoT Devices", labels, nil), + TxBytesR: nd(ns+"transmit_rate_bytes", "Bytes Transmit Rate", labels, nil), + RxBytesR: nd(ns+"receive_rate_bytes", "Bytes Receive Rate", labels, nil), + NumAp: nd(ns+"aps", "Access Point Count", labels, nil), + NumAdopted: nd(ns+"adopted", "Adoption Count", labels, nil), + NumDisabled: nd(ns+"disabled", "Disabled Count", labels, nil), + NumDisconnected: nd(ns+"disconnected", "Disconnected Count", labels, nil), + NumPending: nd(ns+"pending", "Pending Count", labels, nil), + NumGw: nd(ns+"gateways", "Gateway Count", labels, nil), + NumSw: nd(ns+"switches", "Switch Count", labels, nil), + NumSta: nd(ns+"stations", "Station Count", labels, nil), + Latency: nd(ns+"latency_seconds", "Latency", labels, nil), + Uptime: nd(ns+"uptime_seconds", "Uptime", labels, nil), + Drops: nd(ns+"intenet_drops_total", "Internet (WAN) Disconnections", labels, nil), + XputUp: nd(ns+"xput_up_rate", "Speedtest Upload", labels, nil), + XputDown: nd(ns+"xput_down_rate", "Speedtest Download", labels, nil), + SpeedtestPing: nd(ns+"speedtest_ping", "Speedtest Ping", labels, nil), + RemoteUserNumActive: nd(ns+"remote_user_active", "Remote Users Active", labels, nil), + RemoteUserNumInactive: nd(ns+"remote_user_inactive", "Remote Users Inactive", labels, nil), + RemoteUserRxBytes: nd(ns+"remote_user_receive_bytes_total", "Remote Users Receive Bytes", labels, nil), + RemoteUserTxBytes: nd(ns+"remote_user_transmit_bytes_total", "Remote Users Transmit Bytes", labels, nil), + RemoteUserRxPackets: nd(ns+"remote_user_receive_packets_total", "Remote Users Receive Packets", labels, nil), + RemoteUserTxPackets: nd(ns+"remote_user_transmit_packets_total", "Remote Users Transmit Packets", labels, nil), } } @@ -78,7 +80,6 @@ func (u *promUnifi) exportSite(r report, s *unifi.Site) { {u.Site.SpeedtestPing, gauge, h.SpeedtestPing, labels}, {u.Site.Drops, counter, h.Drops, labels}, }) - case "wlan": r.send([]*metric{ {u.Site.TxBytesR, gauge, h.TxBytesR, labels}, @@ -92,7 +93,6 @@ func (u *promUnifi) exportSite(r report, s *unifi.Site) { {u.Site.NumAp, gauge, h.NumAp, labels}, {u.Site.NumDisabled, gauge, h.NumDisabled, labels}, }) - case "wan": r.send([]*metric{ {u.Site.TxBytesR, gauge, h.TxBytesR, labels}, @@ -103,7 +103,6 @@ func (u *promUnifi) exportSite(r report, s *unifi.Site) { {u.Site.NumGw, gauge, h.NumGw, labels}, {u.Site.NumSta, gauge, h.NumSta, labels}, }) - case "lan": r.send([]*metric{ {u.Site.TxBytesR, gauge, h.TxBytesR, labels}, @@ -116,7 +115,6 @@ func (u *promUnifi) exportSite(r report, s *unifi.Site) { {u.Site.NumIot, gauge, h.NumIot, labels}, {u.Site.NumSw, gauge, h.NumSw, labels}, }) - case "vpn": r.send([]*metric{ {u.Site.RemoteUserNumActive, gauge, h.RemoteUserNumActive, labels}, diff --git a/integrations/inputunifi/pkg/promunifi/uap.go b/integrations/inputunifi/pkg/promunifi/uap.go index 53242159..0f3a05aa 100644 --- a/integrations/inputunifi/pkg/promunifi/uap.go +++ b/integrations/inputunifi/pkg/promunifi/uap.go @@ -83,79 +83,80 @@ func descUAP(ns string) *uap { labelA := []string{"stat", "site_name", "name"} // stat + labels[1:] labelV := []string{"vap_name", "bssid", "radio", "radio_name", "essid", "usage", "site_name", "name"} labelR := []string{"radio_name", "radio", "site_name", "name"} + nd := prometheus.NewDesc return &uap{ // 3x each - stat table: total, guest, user - ApWifiTxDropped: prometheus.NewDesc(ns+"stat_wifi_transmt_dropped_total", "Wifi Transmissions Dropped", labelA, nil), - ApRxErrors: prometheus.NewDesc(ns+"stat_receive_errors_total", "Receive Errors", labelA, nil), - ApRxDropped: prometheus.NewDesc(ns+"stat_receive_dropped_total", "Receive Dropped", labelA, nil), - ApRxFrags: prometheus.NewDesc(ns+"stat_receive_frags_total", "Received Frags", labelA, nil), - ApRxCrypts: prometheus.NewDesc(ns+"stat_receive_crypts_total", "Receive Crypts", labelA, nil), - ApTxPackets: prometheus.NewDesc(ns+"stat_transmit_packets_total", "Transmit Packets", labelA, nil), - ApTxBytes: prometheus.NewDesc(ns+"stat_transmit_bytes_total", "Transmit Bytes", labelA, nil), - ApTxErrors: prometheus.NewDesc(ns+"stat_transmit_errors_total", "Transmit Errors", labelA, nil), - ApTxDropped: prometheus.NewDesc(ns+"stat_transmit_dropped_total", "Transmit Dropped", labelA, nil), - ApTxRetries: prometheus.NewDesc(ns+"stat_retries_tx_total", "Transmit Retries", labelA, nil), - ApRxPackets: prometheus.NewDesc(ns+"stat_receive_packets_total", "Receive Packets", labelA, nil), - ApRxBytes: prometheus.NewDesc(ns+"stat_receive_bytes_total", "Receive Bytes", labelA, nil), - WifiTxAttempts: prometheus.NewDesc(ns+"stat_wifi_transmit_attempts_total", "Wifi Transmission Attempts", labelA, nil), - MacFilterRejections: prometheus.NewDesc(ns+"stat_mac_filter_rejects_total", "MAC Filter Rejections", labelA, nil), + ApWifiTxDropped: nd(ns+"stat_wifi_transmt_dropped_total", "Wifi Transmissions Dropped", labelA, nil), + ApRxErrors: nd(ns+"stat_receive_errors_total", "Receive Errors", labelA, nil), + ApRxDropped: nd(ns+"stat_receive_dropped_total", "Receive Dropped", labelA, nil), + ApRxFrags: nd(ns+"stat_receive_frags_total", "Received Frags", labelA, nil), + ApRxCrypts: nd(ns+"stat_receive_crypts_total", "Receive Crypts", labelA, nil), + ApTxPackets: nd(ns+"stat_transmit_packets_total", "Transmit Packets", labelA, nil), + ApTxBytes: nd(ns+"stat_transmit_bytes_total", "Transmit Bytes", labelA, nil), + ApTxErrors: nd(ns+"stat_transmit_errors_total", "Transmit Errors", labelA, nil), + ApTxDropped: nd(ns+"stat_transmit_dropped_total", "Transmit Dropped", labelA, nil), + ApTxRetries: nd(ns+"stat_retries_tx_total", "Transmit Retries", labelA, nil), + ApRxPackets: nd(ns+"stat_receive_packets_total", "Receive Packets", labelA, nil), + ApRxBytes: nd(ns+"stat_receive_bytes_total", "Receive Bytes", labelA, nil), + WifiTxAttempts: nd(ns+"stat_wifi_transmit_attempts_total", "Wifi Transmission Attempts", labelA, nil), + MacFilterRejections: nd(ns+"stat_mac_filter_rejects_total", "MAC Filter Rejections", labelA, nil), // N each - 1 per Virtual AP (VAP) - VAPCcq: prometheus.NewDesc(ns+"vap_ccq_ratio", "VAP Client Connection Quality", labelV, nil), - VAPMacFilterRejections: prometheus.NewDesc(ns+"vap_mac_filter_rejects_total", "VAP MAC Filter Rejections", labelV, nil), - VAPNumSatisfactionSta: prometheus.NewDesc(ns+"vap_satisfaction_stations", "VAP Number Satisifaction Stations", labelV, nil), - VAPAvgClientSignal: prometheus.NewDesc(ns+"vap_average_client_signal", "VAP Average Client Signal", labelV, nil), - VAPSatisfaction: prometheus.NewDesc(ns+"vap_satisfaction_ratio", "VAP Satisfaction", labelV, nil), - VAPSatisfactionNow: prometheus.NewDesc(ns+"vap_satisfaction_now_ratio", "VAP Satisfaction Now", labelV, nil), - VAPDNSAvgLatency: prometheus.NewDesc(ns+"vap_dns_latency_average_seconds", "VAP DNS Latency Average", labelV, nil), - VAPRxBytes: prometheus.NewDesc(ns+"vap_receive_bytes_total", "VAP Bytes Received", labelV, nil), - VAPRxCrypts: prometheus.NewDesc(ns+"vap_receive_crypts_total", "VAP Crypts Received", labelV, nil), - VAPRxDropped: prometheus.NewDesc(ns+"vap_receive_dropped_total", "VAP Dropped Received", labelV, nil), - VAPRxErrors: prometheus.NewDesc(ns+"vap_receive_errors_total", "VAP Errors Received", labelV, nil), - VAPRxFrags: prometheus.NewDesc(ns+"vap_receive_frags_total", "VAP Frags Received", labelV, nil), - VAPRxNwids: prometheus.NewDesc(ns+"vap_receive_nwids_total", "VAP Nwids Received", labelV, nil), - VAPRxPackets: prometheus.NewDesc(ns+"vap_receive_packets_total", "VAP Packets Received", labelV, nil), - VAPTxBytes: prometheus.NewDesc(ns+"vap_transmit_bytes_total", "VAP Bytes Transmitted", labelV, nil), - VAPTxDropped: prometheus.NewDesc(ns+"vap_transmit_dropped_total", "VAP Dropped Transmitted", labelV, nil), - VAPTxErrors: prometheus.NewDesc(ns+"vap_transmit_errors_total", "VAP Errors Transmitted", labelV, nil), - VAPTxPackets: prometheus.NewDesc(ns+"vap_transmit_packets_total", "VAP Packets Transmitted", labelV, nil), - VAPTxPower: prometheus.NewDesc(ns+"vap_transmit_power", "VAP Transmit Power", labelV, nil), - VAPTxRetries: prometheus.NewDesc(ns+"vap_transmit_retries_total", "VAP Retries Transmitted", labelV, nil), - VAPTxCombinedRetries: prometheus.NewDesc(ns+"vap_transmit_retries_combined_total", "VAP Retries Combined Transmitted", labelV, nil), - VAPTxDataMpduBytes: prometheus.NewDesc(ns+"vap_data_mpdu_transmit_bytes_total", "VAP Data MPDU Bytes Transmitted", labelV, nil), - VAPTxRtsRetries: prometheus.NewDesc(ns+"vap_transmit_rts_retries_total", "VAP RTS Retries Transmitted", labelV, nil), - VAPTxSuccess: prometheus.NewDesc(ns+"vap_transmit_success_total", "VAP Success Transmits", labelV, nil), - VAPTxTotal: prometheus.NewDesc(ns+"vap_transmit_total", "VAP Transmit Total", labelV, nil), - VAPTxGoodbytes: prometheus.NewDesc(ns+"vap_transmit_goodbyes", "VAP Goodbyes Transmitted", labelV, nil), - VAPTxLatAvg: prometheus.NewDesc(ns+"vap_transmit_latency_average_seconds", "VAP Latency Average Transmit", labelV, nil), - VAPTxLatMax: prometheus.NewDesc(ns+"vap_transmit_latency_maximum_seconds", "VAP Latency Maximum Transmit", labelV, nil), - VAPTxLatMin: prometheus.NewDesc(ns+"vap_transmit_latency_minimum_seconds", "VAP Latency Minimum Transmit", labelV, nil), - VAPRxGoodbytes: prometheus.NewDesc(ns+"vap_receive_goodbyes", "VAP Goodbyes Received", labelV, nil), - VAPRxLatAvg: prometheus.NewDesc(ns+"vap_receive_latency_average_seconds", "VAP Latency Average Receive", labelV, nil), - VAPRxLatMax: prometheus.NewDesc(ns+"vap_receive_latency_maximum_seconds", "VAP Latency Maximum Receive", labelV, nil), - VAPRxLatMin: prometheus.NewDesc(ns+"vap_receive_latency_minimum_seconds", "VAP Latency Minimum Receive", labelV, nil), - VAPWifiTxLatencyMovAvg: prometheus.NewDesc(ns+"vap_transmit_latency_moving_avg_seconds", "VAP Latency Moving Average Tramsit", labelV, nil), - VAPWifiTxLatencyMovMax: prometheus.NewDesc(ns+"vap_transmit_latency_moving_max_seconds", "VAP Latency Moving Maximum Tramsit", labelV, nil), - VAPWifiTxLatencyMovMin: prometheus.NewDesc(ns+"vap_transmit_latency_moving_min_seconds", "VAP Latency Moving Minimum Tramsit", labelV, nil), - VAPWifiTxLatencyMovTotal: prometheus.NewDesc(ns+"vap_transmit_latency_moving_total", "VAP Latency Moving Total Tramsit", labelV, nil), - VAPWifiTxLatencyMovCount: prometheus.NewDesc(ns+"vap_transmit_latency_moving_count", "VAP Latency Moving Count Tramsit", labelV, nil), + VAPCcq: nd(ns+"vap_ccq_ratio", "VAP Client Connection Quality", labelV, nil), + VAPMacFilterRejections: nd(ns+"vap_mac_filter_rejects_total", "VAP MAC Filter Rejections", labelV, nil), + VAPNumSatisfactionSta: nd(ns+"vap_satisfaction_stations", "VAP Number Satisifaction Stations", labelV, nil), + VAPAvgClientSignal: nd(ns+"vap_average_client_signal", "VAP Average Client Signal", labelV, nil), + VAPSatisfaction: nd(ns+"vap_satisfaction_ratio", "VAP Satisfaction", labelV, nil), + VAPSatisfactionNow: nd(ns+"vap_satisfaction_now_ratio", "VAP Satisfaction Now", labelV, nil), + VAPDNSAvgLatency: nd(ns+"vap_dns_latency_average_seconds", "VAP DNS Latency Average", labelV, nil), + VAPRxBytes: nd(ns+"vap_receive_bytes_total", "VAP Bytes Received", labelV, nil), + VAPRxCrypts: nd(ns+"vap_receive_crypts_total", "VAP Crypts Received", labelV, nil), + VAPRxDropped: nd(ns+"vap_receive_dropped_total", "VAP Dropped Received", labelV, nil), + VAPRxErrors: nd(ns+"vap_receive_errors_total", "VAP Errors Received", labelV, nil), + VAPRxFrags: nd(ns+"vap_receive_frags_total", "VAP Frags Received", labelV, nil), + VAPRxNwids: nd(ns+"vap_receive_nwids_total", "VAP Nwids Received", labelV, nil), + VAPRxPackets: nd(ns+"vap_receive_packets_total", "VAP Packets Received", labelV, nil), + VAPTxBytes: nd(ns+"vap_transmit_bytes_total", "VAP Bytes Transmitted", labelV, nil), + VAPTxDropped: nd(ns+"vap_transmit_dropped_total", "VAP Dropped Transmitted", labelV, nil), + VAPTxErrors: nd(ns+"vap_transmit_errors_total", "VAP Errors Transmitted", labelV, nil), + VAPTxPackets: nd(ns+"vap_transmit_packets_total", "VAP Packets Transmitted", labelV, nil), + VAPTxPower: nd(ns+"vap_transmit_power", "VAP Transmit Power", labelV, nil), + VAPTxRetries: nd(ns+"vap_transmit_retries_total", "VAP Retries Transmitted", labelV, nil), + VAPTxCombinedRetries: nd(ns+"vap_transmit_retries_combined_total", "VAP Retries Combined Tx", labelV, nil), + VAPTxDataMpduBytes: nd(ns+"vap_data_mpdu_transmit_bytes_total", "VAP Data MPDU Bytes Tx", labelV, nil), + VAPTxRtsRetries: nd(ns+"vap_transmit_rts_retries_total", "VAP RTS Retries Transmitted", labelV, nil), + VAPTxSuccess: nd(ns+"vap_transmit_success_total", "VAP Success Transmits", labelV, nil), + VAPTxTotal: nd(ns+"vap_transmit_total", "VAP Transmit Total", labelV, nil), + VAPTxGoodbytes: nd(ns+"vap_transmit_goodbyes", "VAP Goodbyes Transmitted", labelV, nil), + VAPTxLatAvg: nd(ns+"vap_transmit_latency_average_seconds", "VAP Latency Average Tx", labelV, nil), + VAPTxLatMax: nd(ns+"vap_transmit_latency_maximum_seconds", "VAP Latency Maximum Tx", labelV, nil), + VAPTxLatMin: nd(ns+"vap_transmit_latency_minimum_seconds", "VAP Latency Minimum Tx", labelV, nil), + VAPRxGoodbytes: nd(ns+"vap_receive_goodbyes", "VAP Goodbyes Received", labelV, nil), + VAPRxLatAvg: nd(ns+"vap_receive_latency_average_seconds", "VAP Latency Average Rx", labelV, nil), + VAPRxLatMax: nd(ns+"vap_receive_latency_maximum_seconds", "VAP Latency Maximum Rx", labelV, nil), + VAPRxLatMin: nd(ns+"vap_receive_latency_minimum_seconds", "VAP Latency Minimum Rx", labelV, nil), + VAPWifiTxLatencyMovAvg: nd(ns+"vap_transmit_latency_moving_avg_seconds", "VAP Latency Moving Avg Tx", labelV, nil), + VAPWifiTxLatencyMovMax: nd(ns+"vap_transmit_latency_moving_max_seconds", "VAP Latency Moving Min Tx", labelV, nil), + VAPWifiTxLatencyMovMin: nd(ns+"vap_transmit_latency_moving_min_seconds", "VAP Latency Moving Max Tx", labelV, nil), + VAPWifiTxLatencyMovTotal: nd(ns+"vap_transmit_latency_moving_total", "VAP Latency Moving Total Tramsit", labelV, nil), + VAPWifiTxLatencyMovCount: nd(ns+"vap_transmit_latency_moving_count", "VAP Latency Moving Count Tramsit", labelV, nil), // N each - 1 per Radio. 1-4 radios per AP usually - RadioCurrentAntennaGain: prometheus.NewDesc(ns+"radio_current_antenna_gain", "Radio Current Antenna Gain", labelR, nil), - RadioHt: prometheus.NewDesc(ns+"radio_ht", "Radio HT", labelR, nil), - RadioMaxTxpower: prometheus.NewDesc(ns+"radio_max_transmit_power", "Radio Maximum Transmit Power", labelR, nil), - RadioMinTxpower: prometheus.NewDesc(ns+"radio_min_transmit_power", "Radio Minimum Transmit Power", labelR, nil), - RadioNss: prometheus.NewDesc(ns+"radio_nss", "Radio Nss", labelR, nil), - RadioRadioCaps: prometheus.NewDesc(ns+"radio_caps", "Radio Capabilities", labelR, nil), - RadioTxPower: prometheus.NewDesc(ns+"radio_transmit_power", "Radio Transmit Power", labelR, nil), - RadioAstBeXmit: prometheus.NewDesc(ns+"radio_ast_be_xmit", "Radio AstBe Transmit", labelR, nil), - RadioChannel: prometheus.NewDesc(ns+"radio_channel", "Radio Channel", labelR, nil), - RadioCuSelfRx: prometheus.NewDesc(ns+"radio_channel_utilization_receive_ratio", "Radio Channel Utilization Receive", labelR, nil), - RadioCuSelfTx: prometheus.NewDesc(ns+"radio_channel_utilization_transmit_ratio", "Radio Channel Utilization Transmit", labelR, nil), - RadioExtchannel: prometheus.NewDesc(ns+"radio_ext_channel", "Radio Ext Channel", labelR, nil), - RadioGain: prometheus.NewDesc(ns+"radio_gain", "Radio Gain", labelR, nil), - RadioNumSta: prometheus.NewDesc(ns+"radio_stations", "Radio Total Station Count", append(labelR, "station_type"), nil), - RadioTxPackets: prometheus.NewDesc(ns+"radio_transmit_packets", "Radio Transmitted Packets", labelR, nil), - RadioTxRetries: prometheus.NewDesc(ns+"radio_transmit_retries", "Radio Transmit Retries", labelR, nil), + RadioCurrentAntennaGain: nd(ns+"radio_current_antenna_gain", "Radio Current Antenna Gain", labelR, nil), + RadioHt: nd(ns+"radio_ht", "Radio HT", labelR, nil), + RadioMaxTxpower: nd(ns+"radio_max_transmit_power", "Radio Maximum Transmit Power", labelR, nil), + RadioMinTxpower: nd(ns+"radio_min_transmit_power", "Radio Minimum Transmit Power", labelR, nil), + RadioNss: nd(ns+"radio_nss", "Radio Nss", labelR, nil), + RadioRadioCaps: nd(ns+"radio_caps", "Radio Capabilities", labelR, nil), + RadioTxPower: nd(ns+"radio_transmit_power", "Radio Transmit Power", labelR, nil), + RadioAstBeXmit: nd(ns+"radio_ast_be_xmit", "Radio AstBe Transmit", labelR, nil), + RadioChannel: nd(ns+"radio_channel", "Radio Channel", labelR, nil), + RadioCuSelfRx: nd(ns+"radio_channel_utilization_receive_ratio", "Channel Utilization Rx", labelR, nil), + RadioCuSelfTx: nd(ns+"radio_channel_utilization_transmit_ratio", "Channel Utilization Tx", labelR, nil), + RadioExtchannel: nd(ns+"radio_ext_channel", "Radio Ext Channel", labelR, nil), + RadioGain: nd(ns+"radio_gain", "Radio Gain", labelR, nil), + RadioNumSta: nd(ns+"radio_stations", "Radio Total Station Count", append(labelR, "station_type"), nil), + RadioTxPackets: nd(ns+"radio_transmit_packets", "Radio Transmitted Packets", labelR, nil), + RadioTxRetries: nd(ns+"radio_transmit_retries", "Radio Transmit Retries", labelR, nil), } } @@ -283,6 +284,7 @@ func (u *promUnifi) exportRADtable(r report, labels []string, rt unifi.RadioTabl labelR := []string{p.Name, p.Radio, labels[1], labels[2]} labelRUser := append(labelR, "user") labelRGuest := append(labelR, "guest") + r.send([]*metric{ {u.UAP.RadioCurrentAntennaGain, gauge, p.CurrentAntennaGain, labelR}, {u.UAP.RadioHt, gauge, p.Ht, labelR}, @@ -311,6 +313,7 @@ func (u *promUnifi) exportRADtable(r report, labels []string, rt unifi.RadioTabl {u.UAP.RadioTxPackets, gauge, t.TxPackets, labelR}, {u.UAP.RadioTxRetries, gauge, t.TxRetries, labelR}, }) + break } } diff --git a/integrations/inputunifi/pkg/promunifi/usg.go b/integrations/inputunifi/pkg/promunifi/usg.go index 631ef358..08ee3781 100644 --- a/integrations/inputunifi/pkg/promunifi/usg.go +++ b/integrations/inputunifi/pkg/promunifi/usg.go @@ -37,6 +37,7 @@ type usg struct { func descUSG(ns string) *usg { labels := []string{"port", "site_name", "name"} + return &usg{ WanRxPackets: prometheus.NewDesc(ns+"wan_receive_packets_total", "WAN Receive Packets Total", labels, nil), WanRxBytes: prometheus.NewDesc(ns+"wan_receive_bytes_total", "WAN Receive Bytes Total", labels, nil), @@ -75,6 +76,7 @@ func (u *promUnifi) exportUSG(r report, d *unifi.USG) { labels := []string{d.Type, d.SiteName, d.Name} infoLabels := []string{d.Version, d.Model, d.Serial, d.Mac, d.IP, d.ID, d.Bytes.Txt, d.Uptime.Txt} + // Gateway System Data. u.exportWANPorts(r, labels, d.Wan1, d.Wan2) u.exportBYTstats(r, labels, d.TxBytes, d.RxBytes) @@ -95,6 +97,7 @@ func (u *promUnifi) exportUSGstats(r report, labels []string, gw *unifi.Gw, st u labelLan := []string{"lan", labels[1], labels[2]} labelWan := []string{"all", labels[1], labels[2]} + r.send([]*metric{ {u.USG.LanRxPackets, counter, gw.LanRxPackets, labelLan}, {u.USG.LanRxBytes, counter, gw.LanRxBytes, labelLan}, @@ -119,6 +122,7 @@ func (u *promUnifi) exportWANPorts(r report, labels []string, wans ...unifi.Wan) } labelWan := []string{wan.Name, labels[1], labels[2]} + r.send([]*metric{ {u.USG.WanRxPackets, counter, wan.RxPackets, labelWan}, {u.USG.WanRxBytes, counter, wan.RxBytes, labelWan}, diff --git a/integrations/inputunifi/pkg/promunifi/usw.go b/integrations/inputunifi/pkg/promunifi/usw.go index e89506cb..d700583f 100644 --- a/integrations/inputunifi/pkg/promunifi/usw.go +++ b/integrations/inputunifi/pkg/promunifi/usw.go @@ -49,45 +49,46 @@ func descUSW(ns string) *usw { pns := ns + "port_" labelS := []string{"site_name", "name"} labelP := []string{"port_id", "port_num", "port_name", "port_mac", "port_ip", "site_name", "name"} + nd := prometheus.NewDesc return &usw{ // This data may be derivable by sum()ing the port data. - SwRxPackets: prometheus.NewDesc(ns+"switch_receive_packets_total", "Switch Packets Received Total", labelS, nil), - SwRxBytes: prometheus.NewDesc(ns+"switch_receive_bytes_total", "Switch Bytes Received Total", labelS, nil), - SwRxErrors: prometheus.NewDesc(ns+"switch_receive_errors_total", "Switch Errors Received Total", labelS, nil), - SwRxDropped: prometheus.NewDesc(ns+"switch_receive_dropped_total", "Switch Dropped Received Total", labelS, nil), - SwRxCrypts: prometheus.NewDesc(ns+"switch_receive_crypts_total", "Switch Crypts Received Total", labelS, nil), - SwRxFrags: prometheus.NewDesc(ns+"switch_receive_frags_total", "Switch Frags Received Total", labelS, nil), - SwTxPackets: prometheus.NewDesc(ns+"switch_transmit_packets_total", "Switch Packets Transmit Total", labelS, nil), - SwTxBytes: prometheus.NewDesc(ns+"switch_transmit_bytes_total", "Switch Bytes Transmit Total", labelS, nil), - SwTxErrors: prometheus.NewDesc(ns+"switch_transmit_errors_total", "Switch Errors Transmit Total", labelS, nil), - SwTxDropped: prometheus.NewDesc(ns+"switch_transmit_dropped_total", "Switch Dropped Transmit Total", labelS, nil), - SwTxRetries: prometheus.NewDesc(ns+"switch_transmit_retries_total", "Switch Retries Transmit Total", labelS, nil), - SwRxMulticast: prometheus.NewDesc(ns+"switch_receive_multicast_total", "Switch Multicast Receive Total", labelS, nil), - SwRxBroadcast: prometheus.NewDesc(ns+"switch_receive_broadcast_total", "Switch Broadcast Receive Total", labelS, nil), - SwTxMulticast: prometheus.NewDesc(ns+"switch_transmit_multicast_total", "Switch Multicast Transmit Total", labelS, nil), - SwTxBroadcast: prometheus.NewDesc(ns+"switch_transmit_broadcast_total", "Switch Broadcast Transmit Total", labelS, nil), - SwBytes: prometheus.NewDesc(ns+"switch_bytes_total", "Switch Bytes Transferred Total", labelS, nil), + SwRxPackets: nd(ns+"switch_receive_packets_total", "Switch Packets Received Total", labelS, nil), + SwRxBytes: nd(ns+"switch_receive_bytes_total", "Switch Bytes Received Total", labelS, nil), + SwRxErrors: nd(ns+"switch_receive_errors_total", "Switch Errors Received Total", labelS, nil), + SwRxDropped: nd(ns+"switch_receive_dropped_total", "Switch Dropped Received Total", labelS, nil), + SwRxCrypts: nd(ns+"switch_receive_crypts_total", "Switch Crypts Received Total", labelS, nil), + SwRxFrags: nd(ns+"switch_receive_frags_total", "Switch Frags Received Total", labelS, nil), + SwTxPackets: nd(ns+"switch_transmit_packets_total", "Switch Packets Transmit Total", labelS, nil), + SwTxBytes: nd(ns+"switch_transmit_bytes_total", "Switch Bytes Transmit Total", labelS, nil), + SwTxErrors: nd(ns+"switch_transmit_errors_total", "Switch Errors Transmit Total", labelS, nil), + SwTxDropped: nd(ns+"switch_transmit_dropped_total", "Switch Dropped Transmit Total", labelS, nil), + SwTxRetries: nd(ns+"switch_transmit_retries_total", "Switch Retries Transmit Total", labelS, nil), + SwRxMulticast: nd(ns+"switch_receive_multicast_total", "Switch Multicast Receive Total", labelS, nil), + SwRxBroadcast: nd(ns+"switch_receive_broadcast_total", "Switch Broadcast Receive Total", labelS, nil), + SwTxMulticast: nd(ns+"switch_transmit_multicast_total", "Switch Multicast Transmit Total", labelS, nil), + SwTxBroadcast: nd(ns+"switch_transmit_broadcast_total", "Switch Broadcast Transmit Total", labelS, nil), + SwBytes: nd(ns+"switch_bytes_total", "Switch Bytes Transferred Total", labelS, nil), // per-port data - PoeCurrent: prometheus.NewDesc(pns+"poe_amperes", "POE Current", labelP, nil), - PoePower: prometheus.NewDesc(pns+"poe_watts", "POE Power", labelP, nil), - PoeVoltage: prometheus.NewDesc(pns+"poe_volts", "POE Voltage", labelP, nil), - RxBroadcast: prometheus.NewDesc(pns+"receive_broadcast_total", "Receive Broadcast", labelP, nil), - RxBytes: prometheus.NewDesc(pns+"receive_bytes_total", "Total Receive Bytes", labelP, nil), - RxBytesR: prometheus.NewDesc(pns+"receive_rate_bytes", "Receive Bytes Rate", labelP, nil), - RxDropped: prometheus.NewDesc(pns+"receive_dropped_total", "Total Receive Dropped", labelP, nil), - RxErrors: prometheus.NewDesc(pns+"receive_errors_total", "Total Receive Errors", labelP, nil), - RxMulticast: prometheus.NewDesc(pns+"receive_multicast_total", "Total Receive Multicast", labelP, nil), - RxPackets: prometheus.NewDesc(pns+"receive_packets_total", "Total Receive Packets", labelP, nil), - Satisfaction: prometheus.NewDesc(pns+"satisfaction_ratio", "Satisfaction", labelP, nil), - Speed: prometheus.NewDesc(pns+"port_speed_bps", "Speed", labelP, nil), - TxBroadcast: prometheus.NewDesc(pns+"transmit_broadcast_total", "Total Transmit Broadcast", labelP, nil), - TxBytes: prometheus.NewDesc(pns+"transmit_bytes_total", "Total Transmit Bytes", labelP, nil), - TxBytesR: prometheus.NewDesc(pns+"transmit_rate_bytes", "Transmit Bytes Rate", labelP, nil), - TxDropped: prometheus.NewDesc(pns+"transmit_dropped_total", "Total Transmit Dropped", labelP, nil), - TxErrors: prometheus.NewDesc(pns+"transmit_errors_total", "Total Transmit Errors", labelP, nil), - TxMulticast: prometheus.NewDesc(pns+"transmit_multicast_total", "Total Tranmist Multicast", labelP, nil), - TxPackets: prometheus.NewDesc(pns+"transmit_packets_total", "Total Transmit Packets", labelP, nil), + PoeCurrent: nd(pns+"poe_amperes", "POE Current", labelP, nil), + PoePower: nd(pns+"poe_watts", "POE Power", labelP, nil), + PoeVoltage: nd(pns+"poe_volts", "POE Voltage", labelP, nil), + RxBroadcast: nd(pns+"receive_broadcast_total", "Receive Broadcast", labelP, nil), + RxBytes: nd(pns+"receive_bytes_total", "Total Receive Bytes", labelP, nil), + RxBytesR: nd(pns+"receive_rate_bytes", "Receive Bytes Rate", labelP, nil), + RxDropped: nd(pns+"receive_dropped_total", "Total Receive Dropped", labelP, nil), + RxErrors: nd(pns+"receive_errors_total", "Total Receive Errors", labelP, nil), + RxMulticast: nd(pns+"receive_multicast_total", "Total Receive Multicast", labelP, nil), + RxPackets: nd(pns+"receive_packets_total", "Total Receive Packets", labelP, nil), + Satisfaction: nd(pns+"satisfaction_ratio", "Satisfaction", labelP, nil), + Speed: nd(pns+"port_speed_bps", "Speed", labelP, nil), + TxBroadcast: nd(pns+"transmit_broadcast_total", "Total Transmit Broadcast", labelP, nil), + TxBytes: nd(pns+"transmit_bytes_total", "Total Transmit Bytes", labelP, nil), + TxBytesR: nd(pns+"transmit_rate_bytes", "Transmit Bytes Rate", labelP, nil), + TxDropped: nd(pns+"transmit_dropped_total", "Total Transmit Dropped", labelP, nil), + TxErrors: nd(pns+"transmit_errors_total", "Total Transmit Errors", labelP, nil), + TxMulticast: nd(pns+"transmit_multicast_total", "Total Tranmist Multicast", labelP, nil), + TxPackets: nd(pns+"transmit_packets_total", "Total Transmit Packets", labelP, nil), } } @@ -98,6 +99,7 @@ func (u *promUnifi) exportUSW(r report, d *unifi.USW) { labels := []string{d.Type, d.SiteName, d.Name} infoLabels := []string{d.Version, d.Model, d.Serial, d.Mac, d.IP, d.ID, d.Bytes.Txt, d.Uptime.Txt} + u.exportUSWstats(r, labels, d.Stat.Sw) u.exportPRTtable(r, labels, d.PortTable) u.exportBYTstats(r, labels, d.TxBytes, d.RxBytes) @@ -129,6 +131,7 @@ func (u *promUnifi) exportUSWstats(r report, labels []string, sw *unifi.Sw) { } labelS := labels[1:] + r.send([]*metric{ {u.USW.SwRxPackets, counter, sw.RxPackets, labelS}, {u.USW.SwRxBytes, counter, sw.RxBytes, labelS}, From 52ebf2266d6e527a0ece83d32b41326de062914d Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Mon, 16 Dec 2019 03:11:40 -0800 Subject: [PATCH 16/41] Add dynamic plugin support --- integrations/inputunifi/.gitignore | 1 + integrations/inputunifi/Makefile | 13 ++- integrations/inputunifi/examples/MANUAL.md | 6 +- .../inputunifi/examples/up.conf.example | 9 +- .../inputunifi/examples/up.json.example | 28 +++--- .../inputunifi/examples/up.xml.example | 26 +++--- .../inputunifi/examples/up.yaml.example | 22 ++--- .../inputunifi/pkg/inputunifi/input.go | 4 +- .../inputunifi/pkg/inputunifi/interface.go | 4 + .../inputunifi/pkg/poller/build_macos.go | 3 + .../inputunifi/pkg/poller/build_unix.go | 3 + .../inputunifi/pkg/poller/build_windows.go | 3 + integrations/inputunifi/pkg/poller/config.go | 88 ++++++++++++++----- .../inputunifi/plugins/mysql/README.md | 26 ++++++ integrations/inputunifi/plugins/mysql/main.go | 45 ++++++++++ 15 files changed, 217 insertions(+), 64 deletions(-) create mode 100644 integrations/inputunifi/plugins/mysql/README.md create mode 100644 integrations/inputunifi/plugins/mysql/main.go diff --git a/integrations/inputunifi/.gitignore b/integrations/inputunifi/.gitignore index 118e4d74..0ba43855 100644 --- a/integrations/inputunifi/.gitignore +++ b/integrations/inputunifi/.gitignore @@ -27,3 +27,4 @@ bitly_token github_deploy_key gpg.signing.key .secret-files.tar +*.so diff --git a/integrations/inputunifi/Makefile b/integrations/inputunifi/Makefile index 4cf96dc5..772d143f 100644 --- a/integrations/inputunifi/Makefile +++ b/integrations/inputunifi/Makefile @@ -8,6 +8,7 @@ IGNORED:=$(shell bash -c "source .metadata.sh ; env | sed 's/=/:=/;s/^/export /' # md2roff turns markdown into man files and html files. MD2ROFF_BIN=github.com/github/hub/md2roff-bin + # Travis CI passes the version in. Local builds get it from the current git tag. ifeq ($(VERSION),) include .metadata.make @@ -185,10 +186,11 @@ $(BINARY)_$(VERSION)-$(ITERATION)_armhf.deb: package_build_linux_armhf check_fpm # Build an environment that can be packaged for linux. package_build_linux: readme man linux # Building package environment for linux. - mkdir -p $@/usr/bin $@/etc/$(BINARY) $@/usr/share/man/man1 $@/usr/share/doc/$(BINARY) + mkdir -p $@/usr/bin $@/etc/$(BINARY) $@/usr/share/man/man1 $@/usr/share/doc/$(BINARY) $@/usr/lib/$(BINARY) # Copying the binary, config file, unit file, and man page into the env. cp $(BINARY).amd64.linux $@/usr/bin/$(BINARY) cp *.1.gz $@/usr/share/man/man1 + cp *.so $@/usr/lib/$(BINARY)/ cp examples/$(CONFIG_FILE).example $@/etc/$(BINARY)/ cp examples/$(CONFIG_FILE).example $@/etc/$(BINARY)/$(CONFIG_FILE) cp LICENSE *.html examples/*?.?* $@/usr/share/doc/$(BINARY)/ @@ -253,6 +255,12 @@ $(BINARY).rb: v$(VERSION).tar.gz.sha256 init/homebrew/$(FORMULA).rb.tmpl init/homebrew/$(FORMULA).rb.tmpl | tee $(BINARY).rb # That perl line turns hello-world into HelloWorld, etc. +# This is kind janky because it always builds the plugins, even if they are already built. +# Still needs to be made multi arch, which adds complications, especially when creating packages. +plugins: $(patsubst %.go,%.so,$(wildcard ./plugins/*/main.go)) +$(patsubst %.go,%.so,$(wildcard ./plugins/*/main.go)): + go build -o $(patsubst plugins/%/main.so,%.so,$@) -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./$(patsubst %main.so,%,$@) + # Extras # Run code tests and lint. @@ -285,8 +293,9 @@ install: man readme $(BINARY) @[ "$(PREFIX)" != "" ] || (echo "Unable to continue, PREFIX not set. Use: make install PREFIX=/usr/local ETC=/usr/local/etc" && false) @[ "$(ETC)" != "" ] || (echo "Unable to continue, ETC not set. Use: make install PREFIX=/usr/local ETC=/usr/local/etc" && false) # Copying the binary, config file, unit file, and man page into the env. - /usr/bin/install -m 0755 -d $(PREFIX)/bin $(PREFIX)/share/man/man1 $(ETC)/$(BINARY) $(PREFIX)/share/doc/$(BINARY) + /usr/bin/install -m 0755 -d $(PREFIX)/bin $(PREFIX)/share/man/man1 $(ETC)/$(BINARY) $(PREFIX)/share/doc/$(BINARY) $(PREFIX)/lib/$(BINARY) /usr/bin/install -m 0755 -cp $(BINARY) $(PREFIX)/bin/$(BINARY) + /usr/bin/install -m 0755 -cp *.so $(PREFIX)/lib/$(BINARY)/ /usr/bin/install -m 0644 -cp $(BINARY).1.gz $(PREFIX)/share/man/man1 /usr/bin/install -m 0644 -cp examples/$(CONFIG_FILE).example $(ETC)/$(BINARY)/ [ -f $(ETC)/$(BINARY)/$(CONFIG_FILE) ] || /usr/bin/install -m 0644 -cp examples/$(CONFIG_FILE).example $(ETC)/$(BINARY)/$(CONFIG_FILE) diff --git a/integrations/inputunifi/examples/MANUAL.md b/integrations/inputunifi/examples/MANUAL.md index 012cd343..cabf4db8 100644 --- a/integrations/inputunifi/examples/MANUAL.md +++ b/integrations/inputunifi/examples/MANUAL.md @@ -66,7 +66,7 @@ is provided so the application can be easily adapted to any environment. `Config File Parameters` Additional parameters are added by output packages. Parameters can also be set -using environment variables. See the GitHub wiki for more information! +using environment variables. See the GitHub wiki for more information! >>> POLLER FIELDS FOLLOW - you may have multiple controllers: @@ -79,7 +79,7 @@ using environment variables. See the GitHub wiki for more information! errors will be logged. Using this with debug=true adds line numbers to any error logs. - >>> CONTROLLER FIELDS FOLLOW - you may have multiple controllers: + >>> UNIFI CONTROLLER FIELDS FOLLOW - you may have multiple controllers: sites default: ["all"] This list of strings should represent the names of sites on the UniFi @@ -96,7 +96,7 @@ using environment variables. See the GitHub wiki for more information! Username used to authenticate with UniFi controller. This should be a special service account created on the control with read-only access. - user no default + pass no default Password used to authenticate with UniFi controller. This can also be set in an environment variable instead of a configuration file. diff --git a/integrations/inputunifi/examples/up.conf.example b/integrations/inputunifi/examples/up.conf.example index 75c12941..43aa9831 100644 --- a/integrations/inputunifi/examples/up.conf.example +++ b/integrations/inputunifi/examples/up.conf.example @@ -13,6 +13,8 @@ debug = false # Recommend enabling debug with this setting for better error logging. quiet = false +# Load dynamic plugins. Advanced use; only sample mysql plugin provided by default. +plugins = [] #### OUTPUTS @@ -40,9 +42,12 @@ interval = "30s" #### INPUTS +[unifi] +disable = false + # You may repeat the following section to poll additional controllers. -[[controller]] -# Friendly name used in dashboards. +[[unifi.controller]] +# Friendly name used in dashboards. Uses URL if left empty. name = "" url = "https://127.0.0.1:8443" diff --git a/integrations/inputunifi/examples/up.json.example b/integrations/inputunifi/examples/up.json.example index 65e4d27e..12ba4a4f 100644 --- a/integrations/inputunifi/examples/up.json.example +++ b/integrations/inputunifi/examples/up.json.example @@ -1,7 +1,8 @@ { "poller": { "debug": false, - "quiet": false + "quiet": false, + "plugins": [] }, "prometheus": { @@ -20,14 +21,19 @@ "interval": "30s" }, - "controller": [{ - "name": "", - "user": "influx", - "pass": "", - "url": "https://127.0.0.1:8443", - "sites": ["all"], - "save_ids": false, - "save_sites": true, - "verify_ssl": false - }] + "unifi": { + "disable": false, + "controllers": [ + { + "name": "", + "user": "influx", + "pass": "", + "url": "https://127.0.0.1:8443", + "sites": ["all"], + "save_ids": false, + "save_sites": true, + "verify_ssl": false + } + ] + } } diff --git a/integrations/inputunifi/examples/up.xml.example b/integrations/inputunifi/examples/up.xml.example index 710c01ba..ec98169d 100644 --- a/integrations/inputunifi/examples/up.xml.example +++ b/integrations/inputunifi/examples/up.xml.example @@ -4,8 +4,11 @@ # UniFi Poller primary configuration file. XML FORMAT # # provided values are defaults. See up.conf.example! # ####################################################### + + and are lists of strings and may be repeated. --> + 0.0.0.0:9130 @@ -21,15 +24,16 @@ false - - - all - influx - - https://127.0.0.1:8443 - false - false - true - - + + + + all + influx + + https://127.0.0.1:8443 + false + false + true + + diff --git a/integrations/inputunifi/examples/up.yaml.example b/integrations/inputunifi/examples/up.yaml.example index 611c5fa8..2be57012 100644 --- a/integrations/inputunifi/examples/up.yaml.example +++ b/integrations/inputunifi/examples/up.yaml.example @@ -7,6 +7,7 @@ poller: debug: false quiet: false + plugins: [] prometheus: disable: false @@ -22,13 +23,14 @@ influxdb: db: "unifi" verify_ssl: false -controller: - - name: "" - user: "influx" - pass: "" - url: "https://127.0.0.1:8443" - sites: - - all - verify_ssl: false - save_ids: false - save_sites: true +unifi: + controllers: + - name: "" + user: "influx" + pass: "" + url: "https://127.0.0.1:8443" + sites: + - all + verify_ssl: false + save_ids: false + save_sites: true diff --git a/integrations/inputunifi/pkg/inputunifi/input.go b/integrations/inputunifi/pkg/inputunifi/input.go index f797d35b..15a027dd 100644 --- a/integrations/inputunifi/pkg/inputunifi/input.go +++ b/integrations/inputunifi/pkg/inputunifi/input.go @@ -27,7 +27,7 @@ type Controller struct { User string `json:"user" toml:"user" xml:"user" yaml:"user"` Pass string `json:"pass" toml:"pass" xml:"pass" yaml:"pass"` URL string `json:"url" toml:"url" xml:"url" yaml:"url"` - Sites []string `json:"sites,omitempty" toml:"sites,omitempty" xml:"sites" yaml:"sites"` + Sites []string `json:"sites,omitempty" toml:"sites,omitempty" xml:"site" yaml:"sites"` Unifi *unifi.Unifi `json:"-" toml:"-" xml:"-" yaml:"-"` } @@ -35,7 +35,7 @@ type Controller struct { type Config struct { sync.RWMutex // locks the Unifi struct member when re-authing to unifi. Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` - Controllers []Controller `json:"controller" toml:"controller" xml:"controller" yaml:"controller"` + Controllers []Controller `json:"controllers" toml:"controller" xml:"controller" yaml:"controllers"` } func init() { diff --git a/integrations/inputunifi/pkg/inputunifi/interface.go b/integrations/inputunifi/pkg/inputunifi/interface.go index a2ef9765..1db32906 100644 --- a/integrations/inputunifi/pkg/inputunifi/interface.go +++ b/integrations/inputunifi/pkg/inputunifi/interface.go @@ -10,6 +10,10 @@ import ( // Metrics grabs all the measurements from a UniFi controller and returns them. func (u *InputUnifi) Metrics() (*poller.Metrics, error) { + if u.Config.Disable { + return nil, nil + } + errs := []string{} metrics := &poller.Metrics{} diff --git a/integrations/inputunifi/pkg/poller/build_macos.go b/integrations/inputunifi/pkg/poller/build_macos.go index b3f37dbf..9292f409 100644 --- a/integrations/inputunifi/pkg/poller/build_macos.go +++ b/integrations/inputunifi/pkg/poller/build_macos.go @@ -4,3 +4,6 @@ package poller // DefaultConfFile is where to find config if --config is not prvided. const DefaultConfFile = "/usr/local/etc/unifi-poller/up.conf" + +// DefaultObjPath is the path to look for shared object libraries (plugins). +const DefaultObjPath = "/usr/local/lib/unifi-poller" diff --git a/integrations/inputunifi/pkg/poller/build_unix.go b/integrations/inputunifi/pkg/poller/build_unix.go index c1f525a9..fd381e19 100644 --- a/integrations/inputunifi/pkg/poller/build_unix.go +++ b/integrations/inputunifi/pkg/poller/build_unix.go @@ -4,3 +4,6 @@ package poller // DefaultConfFile is where to find config if --config is not prvided. const DefaultConfFile = "/etc/unifi-poller/up.conf" + +// DefaultObjPath is the path to look for shared object libraries (plugins). +const DefaultObjPath = "/usr/lib/unifi-poller" diff --git a/integrations/inputunifi/pkg/poller/build_windows.go b/integrations/inputunifi/pkg/poller/build_windows.go index a74c76a8..69d964e8 100644 --- a/integrations/inputunifi/pkg/poller/build_windows.go +++ b/integrations/inputunifi/pkg/poller/build_windows.go @@ -4,3 +4,6 @@ package poller // DefaultConfFile is where to find config if --config is not prvided. const DefaultConfFile = `C:\ProgramData\unifi-poller\up.conf` + +// DefaultObjPath is useless in this context. Bummer. +const DefaultObjPath = "PLUGINS_DO_NOT_WORK_ON_WINDOWS_SOWWWWWY" diff --git a/integrations/inputunifi/pkg/poller/config.go b/integrations/inputunifi/pkg/poller/config.go index 03e69b81..a43caceb 100644 --- a/integrations/inputunifi/pkg/poller/config.go +++ b/integrations/inputunifi/pkg/poller/config.go @@ -9,6 +9,10 @@ package poller */ import ( + "os" + "path" + "plugin" + "strings" "time" "github.com/spf13/pflag" @@ -53,33 +57,24 @@ type Config struct { // Poller is the global config values. type Poller struct { - Debug bool `json:"debug" toml:"debug" xml:"debug,attr" yaml:"debug"` - Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet,attr" yaml:"quiet"` + Plugins []string `json:"plugins" toml:"plugins" xml:"plugin" yaml:"plugins"` + Debug bool `json:"debug" toml:"debug" xml:"debug,attr" yaml:"debug"` + Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet,attr" yaml:"quiet"` } -// ParseConfigs parses the poller config and the config for each registered output plugin. -func (u *UnifiPoller) ParseConfigs() error { - // Parse core config. - if err := u.ParseInterface(u.Config); err != nil { - return err - } +// LoadPlugins reads-in dynamic shared libraries. +// Not used very often, if at all. +func (u *UnifiPoller) LoadPlugins() error { + for _, p := range u.Plugins { + name := strings.TrimSuffix(p, ".so") + ".so" - // Parse output plugin configs. - outputSync.Lock() - defer outputSync.Unlock() - - for _, o := range outputs { - if err := u.ParseInterface(o.Config); err != nil { - return err + if _, err := os.Stat(name); os.IsNotExist(err) { + name = path.Join(DefaultObjPath, name) } - } - // Parse input plugin configs. - inputSync.Lock() - defer inputSync.Unlock() + u.Logf("Loading Dynamic Plugin: %s", name) - for _, i := range inputs { - if err := u.ParseInterface(i.Config); err != nil { + if _, err := plugin.Open(name); err != nil { return err } } @@ -87,8 +82,27 @@ func (u *UnifiPoller) ParseConfigs() error { return nil } -// ParseInterface parses the config file and environment variables into the provided interface. -func (u *UnifiPoller) ParseInterface(i interface{}) error { +// ParseConfigs parses the poller config and the config for each registered output plugin. +func (u *UnifiPoller) ParseConfigs() error { + // Parse core config. + if err := u.parseInterface(u.Config); err != nil { + return err + } + + // Load dynamic plugins. + if err := u.LoadPlugins(); err != nil { + return err + } + + if err := u.parseInputs(); err != nil { + return err + } + + return u.parseOutputs() +} + +// parseInterface parses the config file and environment variables into the provided interface. +func (u *UnifiPoller) parseInterface(i interface{}) error { // Parse config file into provided interface. if err := config.ParseFile(i, u.Flags.ConfigFile); err != nil { return err @@ -99,3 +113,31 @@ func (u *UnifiPoller) ParseInterface(i interface{}) error { return err } + +// Parse input plugin configs. +func (u *UnifiPoller) parseInputs() error { + inputSync.Lock() + defer inputSync.Unlock() + + for _, i := range inputs { + if err := u.parseInterface(i.Config); err != nil { + return err + } + } + + return nil +} + +// Parse output plugin configs. +func (u *UnifiPoller) parseOutputs() error { + outputSync.Lock() + defer outputSync.Unlock() + + for _, o := range outputs { + if err := u.parseInterface(o.Config); err != nil { + return err + } + } + + return nil +} diff --git a/integrations/inputunifi/plugins/mysql/README.md b/integrations/inputunifi/plugins/mysql/README.md new file mode 100644 index 00000000..9c32f7fe --- /dev/null +++ b/integrations/inputunifi/plugins/mysql/README.md @@ -0,0 +1,26 @@ +# MYSQL Output Plugin Example + +The code here, and the dynamic plugin provided shows an example of how you can +write your own output for unifi-poller. This plugin records some very basic +data about clients on a unifi network into a mysql database. + +You could write outputs that do... anything. An example: They could compare current +connected clients to a previous list (in a db, or stored in memory), and send a +notification if it changes. The possibilities are endless. + +You must compile your plugin using the unifi-poller source for the version you're +using. In other words, to build a plugin for version 2.0.1, do this: +``` +mkdir -p $GOPATH/src/github.com/davidnewhall +cd $GOPATH/src/github.com/davidnewhall + +git clone git@github.com:davidnewhall/unifi-poller.git +cd unifi-poller + +git checkout v2.0.1 +make vendor + +cp -r plugins/ +GOOS=linux make plugins +``` +The plugin you copy in *must* have a `main.go` file for `make plugins` to build it. diff --git a/integrations/inputunifi/plugins/mysql/main.go b/integrations/inputunifi/plugins/mysql/main.go new file mode 100644 index 00000000..e0c866b9 --- /dev/null +++ b/integrations/inputunifi/plugins/mysql/main.go @@ -0,0 +1,45 @@ +package main + +import ( + "fmt" + + "github.com/davidnewhall/unifi-poller/pkg/poller" + "golift.io/config" +) + +// mysqlConfig represents the data that is unmarshalled from the up.conf config file for this plugins. +type mysqlConfig struct { + Interval config.Duration `json:"interval" toml:"interval" xml:"interval" yaml:"interval"` + Host string `json:"host" toml:"host" xml:"host" yaml:"host"` + User string `json:"user" toml:"user" xml:"user" yaml:"user"` + Pass string `json:"pass" toml:"pass" xml:"pass" yaml:"pass"` + DB string `json:"db" toml:"db" xml:"db" yaml:"db"` + Table string `json:"table" toml:"table" xml:"table" yaml:"table"` + // Maps do not work with ENV VARIABLES yet, but may in the future. + Fields []string `json:"fields" toml:"fields" xml:"field" yaml:"fields"` +} + +// Pointers are ignored during ENV variable unmarshal, avoid pointers to your config. +// Only capital (exported) members are unmarshaled when passed into poller.NewOutput(). +type application struct { + Config mysqlConfig `json:"mysql" toml:"mysql" xml:"mysql" yaml:"mysql"` +} + +func init() { + u := &application{Config: mysqlConfig{}} + + poller.NewOutput(&poller.Output{ + Name: "mysql", + Config: u, // pass in the struct *above* your config (so it can see the struct tags). + Method: u.Run, + }) +} + +func main() { + fmt.Println("this is a unifi-poller plugin; not an application") +} + +func (a *application) Run(c poller.Collect) error { + c.Logf("mysql plugin is not finished") + return nil +} From 507ea95e1ed85e53c2f423e0f415b146ea8a0d41 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Mon, 16 Dec 2019 16:57:14 -0800 Subject: [PATCH 17/41] no idea if this will stick.. --- integrations/inputunifi/Makefile | 51 ++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/integrations/inputunifi/Makefile b/integrations/inputunifi/Makefile index 772d143f..2a3d0035 100644 --- a/integrations/inputunifi/Makefile +++ b/integrations/inputunifi/Makefile @@ -43,6 +43,8 @@ $(PACKAGE_SCRIPTS) \ --config-files "/etc/$(BINARY)/$(CONFIG_FILE)" endef +PLUGINS:=$(patsubst plugins/%/main.go,%,$(wildcard plugins/*/main.go)) + VERSION_LDFLAGS:= \ -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Branch=$(BRANCH) \ -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.BuildDate=$(DATE) \ @@ -184,13 +186,14 @@ $(BINARY)_$(VERSION)-$(ITERATION)_armhf.deb: package_build_linux_armhf check_fpm [ "$(SIGNING_KEY)" == "" ] || expect -c "spawn debsigs --default-key="$(SIGNING_KEY)" --sign=origin $(BINARY)_$(VERSION)-$(ITERATION)_armhf.deb; expect -exact \"Enter passphrase: \"; send \"$(PRIVATE_KEY)\r\"; expect eof" # Build an environment that can be packaged for linux. -package_build_linux: readme man linux +package_build_linux: readme man plugins_linux_amd64 linux # Building package environment for linux. mkdir -p $@/usr/bin $@/etc/$(BINARY) $@/usr/share/man/man1 $@/usr/share/doc/$(BINARY) $@/usr/lib/$(BINARY) # Copying the binary, config file, unit file, and man page into the env. cp $(BINARY).amd64.linux $@/usr/bin/$(BINARY) cp *.1.gz $@/usr/share/man/man1 - cp *.so $@/usr/lib/$(BINARY)/ + rm -f $@/usr/lib/$(BINARY)/*.so + cp *amd64.so $@/usr/lib/$(BINARY)/ cp examples/$(CONFIG_FILE).example $@/etc/$(BINARY)/ cp examples/$(CONFIG_FILE).example $@/etc/$(BINARY)/$(CONFIG_FILE) cp LICENSE *.html examples/*?.?* $@/usr/share/doc/$(BINARY)/ @@ -199,20 +202,26 @@ package_build_linux: readme man linux sed -e "s/{{BINARY}}/$(BINARY)/g" -e "s/{{DESC}}/$(DESC)/g" \ init/systemd/template.unit.service > $@/lib/systemd/system/$(BINARY).service -package_build_linux_386: package_build_linux linux386 +package_build_linux_386: package_build_linux plugins_linux_i386 linux386 mkdir -p $@ cp -r $ /dev/null || (echo "FPM missing. Install FPM: https://fpm.readthedocs.io/en/latest/installing.html" && false) @@ -255,11 +264,29 @@ $(BINARY).rb: v$(VERSION).tar.gz.sha256 init/homebrew/$(FORMULA).rb.tmpl init/homebrew/$(FORMULA).rb.tmpl | tee $(BINARY).rb # That perl line turns hello-world into HelloWorld, etc. -# This is kind janky because it always builds the plugins, even if they are already built. -# Still needs to be made multi arch, which adds complications, especially when creating packages. -plugins: $(patsubst %.go,%.so,$(wildcard ./plugins/*/main.go)) -$(patsubst %.go,%.so,$(wildcard ./plugins/*/main.go)): - go build -o $(patsubst plugins/%/main.so,%.so,$@) -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./$(patsubst %main.so,%,$@) +# This probably wont work for most people..... +plugins: linux_plugins plugins_darwin + +linux_plugins: plugins_linux_amd64 plugins_linux_i386 plugins_linux_arm64 plugins_linux_armhf +plugins_linux_amd64: $(patsubst %,%.linux_amd64.so,$(PLUGINS)) +$(patsubst %,%.linux_amd64.so,$(PLUGINS)): + GOOS=linux GOARCH=amd64 go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.linux_amd64.so,%,$@) + +plugins_linux_i386: $(patsubst %,%.linux_i386.so,$(PLUGINS)) +$(patsubst %,%.linux_i386.so,$(PLUGINS)): + GOOS=linux GOARCH=i386 go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.linux_i386.so,%,$@) + +plugins_linux_arm64: $(patsubst %,%.linux_arm64.so,$(PLUGINS)) +$(patsubst %,%.linux_arm64.so,$(PLUGINS)): + GOOS=linux GOARCH=arm64 go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.linux_arm64.so,%,$@) + +plugins_linux_armhf: $(patsubst %,%.linux_armhf.so,$(PLUGINS)) +$(patsubst %,%.linux_armhf.so,$(PLUGINS)): + GOOS=linux GOARCH=armhf go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.linux_armhf.so,%,$@) + +plugins_darwin: $(patsubst %,%.darwin.so,$(PLUGINS)) +$(patsubst %,%.darwin.so,$(PLUGINS)): + GOOS=darwin go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.darwin.so,%,$@) # Extras @@ -283,7 +310,7 @@ deps: # Homebrew stuff. macOS only. # Used for Homebrew only. Other distros can create packages. -install: man readme $(BINARY) +install: man readme $(BINARY) plugins_darwin @echo - Done Building! - @echo - Local installation with the Makefile is only supported on macOS. @echo If you wish to install the application manually on Linux, check out the wiki: https://$(SOURCE_URL)/wiki/Installation @@ -295,7 +322,7 @@ install: man readme $(BINARY) # Copying the binary, config file, unit file, and man page into the env. /usr/bin/install -m 0755 -d $(PREFIX)/bin $(PREFIX)/share/man/man1 $(ETC)/$(BINARY) $(PREFIX)/share/doc/$(BINARY) $(PREFIX)/lib/$(BINARY) /usr/bin/install -m 0755 -cp $(BINARY) $(PREFIX)/bin/$(BINARY) - /usr/bin/install -m 0755 -cp *.so $(PREFIX)/lib/$(BINARY)/ + /usr/bin/install -m 0755 -cp *darwin.so $(PREFIX)/lib/$(BINARY)/ /usr/bin/install -m 0644 -cp $(BINARY).1.gz $(PREFIX)/share/man/man1 /usr/bin/install -m 0644 -cp examples/$(CONFIG_FILE).example $(ETC)/$(BINARY)/ [ -f $(ETC)/$(BINARY)/$(CONFIG_FILE) ] || /usr/bin/install -m 0644 -cp examples/$(CONFIG_FILE).example $(ETC)/$(BINARY)/$(CONFIG_FILE) From 0168f314d3bc36951496c96b5fd97e2a73a8b027 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Mon, 16 Dec 2019 17:02:01 -0800 Subject: [PATCH 18/41] typo --- integrations/inputunifi/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/inputunifi/Makefile b/integrations/inputunifi/Makefile index 2a3d0035..30287041 100644 --- a/integrations/inputunifi/Makefile +++ b/integrations/inputunifi/Makefile @@ -274,7 +274,7 @@ $(patsubst %,%.linux_amd64.so,$(PLUGINS)): plugins_linux_i386: $(patsubst %,%.linux_i386.so,$(PLUGINS)) $(patsubst %,%.linux_i386.so,$(PLUGINS)): - GOOS=linux GOARCH=i386 go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.linux_i386.so,%,$@) + GOOS=linux GOARCH=386 go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.linux_i386.so,%,$@) plugins_linux_arm64: $(patsubst %,%.linux_arm64.so,$(PLUGINS)) $(patsubst %,%.linux_arm64.so,$(PLUGINS)): From 674f3f3130166079020af233337588961c9f3f97 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Mon, 16 Dec 2019 17:11:13 -0800 Subject: [PATCH 19/41] fixes --- integrations/inputunifi/.metadata.sh | 3 +-- integrations/inputunifi/Makefile | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/integrations/inputunifi/.metadata.sh b/integrations/inputunifi/.metadata.sh index 7a78171d..5c380efc 100755 --- a/integrations/inputunifi/.metadata.sh +++ b/integrations/inputunifi/.metadata.sh @@ -38,7 +38,6 @@ VERSION="$(echo $VVERSION | tr -d v | grep -E '^\S+$' || echo development)" # This produces a 0 in some envirnoments (like Homebrew), but it's only used for packages. ITERATION=$(git rev-list --count --all || echo 0) DATE="$(date -u +%Y-%m-%dT%H:%M:%SZ)" -BRANCH="$(git rev-parse --abbrev-ref HEAD || echo unknown)" COMMIT="$(git rev-parse --short HEAD || echo 0)" # Used by homebrew downloads. @@ -46,4 +45,4 @@ COMMIT="$(git rev-parse --short HEAD || echo 0)" # This is a custom download path for homebrew formula. SOURCE_PATH=https://golift.io/${BINARY}/archive/v${VERSION}.tar.gz -export IMPORT_PATH SOURCE_URL URL VVERSION VERSION ITERATION DATE BRANCH COMMIT SOURCE_PATH +export IMPORT_PATH SOURCE_URL URL VVERSION VERSION ITERATION DATE COMMIT SOURCE_PATH diff --git a/integrations/inputunifi/Makefile b/integrations/inputunifi/Makefile index 30287041..ee12f5b2 100644 --- a/integrations/inputunifi/Makefile +++ b/integrations/inputunifi/Makefile @@ -46,7 +46,7 @@ endef PLUGINS:=$(patsubst plugins/%/main.go,%,$(wildcard plugins/*/main.go)) VERSION_LDFLAGS:= \ - -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Branch=$(BRANCH) \ + -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Branch=$(TRAVIS_BRANCH) \ -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.BuildDate=$(DATE) \ -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Revision=$(COMMIT) \ -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Version=$(VERSION)-$(ITERATION) From 072a814a55aea7058f937d56ed54752d5b57bd2d Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Mon, 16 Dec 2019 17:18:45 -0800 Subject: [PATCH 20/41] haha --- integrations/inputunifi/.travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/integrations/inputunifi/.travis.yml b/integrations/inputunifi/.travis.yml index 6a71c421..43119458 100644 --- a/integrations/inputunifi/.travis.yml +++ b/integrations/inputunifi/.travis.yml @@ -15,8 +15,12 @@ addons: - debsigs - gnupg - expect + - musl-dev + - gcc + - libc6-dev-i386 + - build-base go: -- 1.12.x +- 1.13.x services: - docker install: From f7de48db0e8c27cc6f67e6ff78e2ac2a3cbe0da4 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Mon, 16 Dec 2019 17:30:37 -0800 Subject: [PATCH 21/41] doot --- integrations/inputunifi/.travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integrations/inputunifi/.travis.yml b/integrations/inputunifi/.travis.yml index 43119458..c607a2e9 100644 --- a/integrations/inputunifi/.travis.yml +++ b/integrations/inputunifi/.travis.yml @@ -17,7 +17,8 @@ addons: - expect - musl-dev - gcc - - libc6-dev-i386 + - gcc-multilib + - g++-multilib - build-base go: - 1.13.x From 628c8ddd5589007aca65ada86085133e0842d7de Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Mon, 16 Dec 2019 17:33:49 -0800 Subject: [PATCH 22/41] doot --- integrations/inputunifi/.travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/inputunifi/.travis.yml b/integrations/inputunifi/.travis.yml index c607a2e9..387d9da6 100644 --- a/integrations/inputunifi/.travis.yml +++ b/integrations/inputunifi/.travis.yml @@ -19,7 +19,7 @@ addons: - gcc - gcc-multilib - g++-multilib - - build-base + - libc6-dev-i386 go: - 1.13.x services: From c3519b8e23acfa289ef8ba612edd514e56bcf6b1 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Mon, 16 Dec 2019 20:24:41 -0800 Subject: [PATCH 23/41] giving in --- integrations/inputunifi/.travis.yml | 5 ----- integrations/inputunifi/Makefile | 29 ++++++----------------------- 2 files changed, 6 insertions(+), 28 deletions(-) diff --git a/integrations/inputunifi/.travis.yml b/integrations/inputunifi/.travis.yml index 387d9da6..39cdce39 100644 --- a/integrations/inputunifi/.travis.yml +++ b/integrations/inputunifi/.travis.yml @@ -15,11 +15,6 @@ addons: - debsigs - gnupg - expect - - musl-dev - - gcc - - gcc-multilib - - g++-multilib - - libc6-dev-i386 go: - 1.13.x services: diff --git a/integrations/inputunifi/Makefile b/integrations/inputunifi/Makefile index ee12f5b2..46fe2600 100644 --- a/integrations/inputunifi/Makefile +++ b/integrations/inputunifi/Makefile @@ -202,26 +202,20 @@ package_build_linux: readme man plugins_linux_amd64 linux sed -e "s/{{BINARY}}/$(BINARY)/g" -e "s/{{DESC}}/$(DESC)/g" \ init/systemd/template.unit.service > $@/lib/systemd/system/$(BINARY).service -package_build_linux_386: package_build_linux plugins_linux_i386 linux386 +package_build_linux_386: package_build_linux linux386 mkdir -p $@ cp -r $ /dev/null || (echo "FPM missing. Install FPM: https://fpm.readthedocs.io/en/latest/installing.html" && false) @@ -264,26 +258,15 @@ $(BINARY).rb: v$(VERSION).tar.gz.sha256 init/homebrew/$(FORMULA).rb.tmpl init/homebrew/$(FORMULA).rb.tmpl | tee $(BINARY).rb # That perl line turns hello-world into HelloWorld, etc. -# This probably wont work for most people..... -plugins: linux_plugins plugins_darwin +plugins: $(patsubst %,%.so,$(PLUGINS)) +$(patsubst %,%.so,$(PLUGINS)): + go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.so,%,$@) linux_plugins: plugins_linux_amd64 plugins_linux_i386 plugins_linux_arm64 plugins_linux_armhf plugins_linux_amd64: $(patsubst %,%.linux_amd64.so,$(PLUGINS)) $(patsubst %,%.linux_amd64.so,$(PLUGINS)): GOOS=linux GOARCH=amd64 go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.linux_amd64.so,%,$@) -plugins_linux_i386: $(patsubst %,%.linux_i386.so,$(PLUGINS)) -$(patsubst %,%.linux_i386.so,$(PLUGINS)): - GOOS=linux GOARCH=386 go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.linux_i386.so,%,$@) - -plugins_linux_arm64: $(patsubst %,%.linux_arm64.so,$(PLUGINS)) -$(patsubst %,%.linux_arm64.so,$(PLUGINS)): - GOOS=linux GOARCH=arm64 go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.linux_arm64.so,%,$@) - -plugins_linux_armhf: $(patsubst %,%.linux_armhf.so,$(PLUGINS)) -$(patsubst %,%.linux_armhf.so,$(PLUGINS)): - GOOS=linux GOARCH=armhf go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.linux_armhf.so,%,$@) - plugins_darwin: $(patsubst %,%.darwin.so,$(PLUGINS)) $(patsubst %,%.darwin.so,$(PLUGINS)): GOOS=darwin go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.darwin.so,%,$@) From a80adadf27a3a81961155086d19bf54ba78e26b6 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Tue, 17 Dec 2019 01:31:30 -0800 Subject: [PATCH 24/41] make dumper work --- integrations/inputunifi/Gopkg.lock | 4 +- .../inputunifi/examples/up.xml.example | 2 +- .../inputunifi/examples/up.yaml.example | 1 + .../inputunifi/pkg/influxunifi/influxdb.go | 2 +- .../inputunifi/pkg/inputunifi/collector.go | 16 +-- .../inputunifi/pkg/inputunifi/input.go | 83 +++++++++++- .../inputunifi/pkg/inputunifi/interface.go | 122 +++++++++--------- integrations/inputunifi/pkg/poller/dumper.go | 83 ++---------- integrations/inputunifi/pkg/poller/inputs.go | 7 + integrations/inputunifi/pkg/poller/logger.go | 4 +- integrations/inputunifi/pkg/poller/start.go | 6 +- 11 files changed, 178 insertions(+), 152 deletions(-) diff --git a/integrations/inputunifi/Gopkg.lock b/integrations/inputunifi/Gopkg.lock index 83212a0b..aa24b10e 100644 --- a/integrations/inputunifi/Gopkg.lock +++ b/integrations/inputunifi/Gopkg.lock @@ -107,11 +107,11 @@ [[projects]] branch = "master" - digest = "1:d54a8d89f95a4d2a5a24ce63cb1835ccdff337fde7776c87ceacb6fdbe4349ae" + digest = "1:7a90fad47972b5ae06013d4685eb2f3007e7c92609a1399d2adf59fe04cd9b63" name = "golift.io/config" packages = ["."] pruneopts = "UT" - revision = "fd8ffb02173aad2183e5555a03b1d1f909aca930" + revision = "fe642c8392dc00d72ddcc47f05a06096bd5d054b" [[projects]] digest = "1:2883cea734f2766f41ff9c9d4aefccccc53e3d44f5c8b08893b9c218cf666722" diff --git a/integrations/inputunifi/examples/up.xml.example b/integrations/inputunifi/examples/up.xml.example index ec98169d..14b2aa09 100644 --- a/integrations/inputunifi/examples/up.xml.example +++ b/integrations/inputunifi/examples/up.xml.example @@ -24,7 +24,7 @@ false - + all diff --git a/integrations/inputunifi/examples/up.yaml.example b/integrations/inputunifi/examples/up.yaml.example index 2be57012..bb8a4aa1 100644 --- a/integrations/inputunifi/examples/up.yaml.example +++ b/integrations/inputunifi/examples/up.yaml.example @@ -24,6 +24,7 @@ influxdb: verify_ssl: false unifi: + disable: false controllers: - name: "" user: "influx" diff --git a/integrations/inputunifi/pkg/influxunifi/influxdb.go b/integrations/inputunifi/pkg/influxunifi/influxdb.go index 199ea594..9bbe9c84 100644 --- a/integrations/inputunifi/pkg/influxunifi/influxdb.go +++ b/integrations/inputunifi/pkg/influxunifi/influxdb.go @@ -24,7 +24,7 @@ const ( // Config defines the data needed to store metrics in InfluxDB type Config struct { Interval config.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` - Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` + Disable bool `json:"disable" toml:"disable" xml:"disable,attr" yaml:"disable"` VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` URL string `json:"url,omitempty" toml:"url,omitempty" xml:"url" yaml:"url"` User string `json:"user,omitempty" toml:"user,omitempty" xml:"user" yaml:"user"` diff --git a/integrations/inputunifi/pkg/inputunifi/collector.go b/integrations/inputunifi/pkg/inputunifi/collector.go index 20aeafac..c9c67dcb 100644 --- a/integrations/inputunifi/pkg/inputunifi/collector.go +++ b/integrations/inputunifi/pkg/inputunifi/collector.go @@ -8,14 +8,14 @@ import ( "golift.io/unifi" ) -func (u *InputUnifi) isNill(c Controller) bool { +func (u *InputUnifi) isNill(c *Controller) bool { u.Config.RLock() defer u.Config.RUnlock() return c.Unifi == nil } -func (u *InputUnifi) collectController(c Controller) (*poller.Metrics, error) { +func (u *InputUnifi) collectController(c *Controller) (*poller.Metrics, error) { if u.isNill(c) { u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) @@ -32,7 +32,7 @@ func (u *InputUnifi) collectController(c Controller) (*poller.Metrics, error) { return u.pollController(c) } -func (u *InputUnifi) pollController(c Controller) (*poller.Metrics, error) { +func (u *InputUnifi) pollController(c *Controller) (*poller.Metrics, error) { var err error u.Config.RLock() @@ -67,7 +67,7 @@ func (u *InputUnifi) pollController(c Controller) (*poller.Metrics, error) { // augmentMetrics is our middleware layer between collecting metrics and writing them. // This is where we can manipuate the returned data or make arbitrary decisions. // This function currently adds parent device names to client metrics. -func (u *InputUnifi) augmentMetrics(c Controller, metrics *poller.Metrics) *poller.Metrics { +func (u *InputUnifi) augmentMetrics(c *Controller, metrics *poller.Metrics) *poller.Metrics { if metrics == nil || metrics.Devices == nil || metrics.Clients == nil { return metrics } @@ -113,22 +113,22 @@ func (u *InputUnifi) augmentMetrics(c Controller, metrics *poller.Metrics) *poll // getFilteredSites returns a list of sites to fetch data for. // Omits requested but unconfigured sites. Grabs the full list from the // controller and returns the sites provided in the config file. -func (u *InputUnifi) getFilteredSites(c Controller) (unifi.Sites, error) { +func (u *InputUnifi) getFilteredSites(c *Controller) (unifi.Sites, error) { u.Config.RLock() defer u.Config.RUnlock() sites, err := c.Unifi.GetSites() if err != nil { return nil, err - } else if len(c.Sites) < 1 || poller.StringInSlice("all", c.Sites) { + } else if len(c.Sites) < 1 || StringInSlice("all", c.Sites) { return sites, nil } - var i int + i := 0 for _, s := range sites { // Only include valid sites in the request filter. - if poller.StringInSlice(s.Name, c.Sites) { + if StringInSlice(s.Name, c.Sites) { sites[i] = s i++ } diff --git a/integrations/inputunifi/pkg/inputunifi/input.go b/integrations/inputunifi/pkg/inputunifi/input.go index 15a027dd..9a4877f9 100644 --- a/integrations/inputunifi/pkg/inputunifi/input.go +++ b/integrations/inputunifi/pkg/inputunifi/input.go @@ -4,6 +4,8 @@ package inputunifi import ( "fmt" + "os" + "strings" "sync" @@ -13,7 +15,7 @@ import ( // InputUnifi contains the running data. type InputUnifi struct { - Config Config `json:"unifi" toml:"unifi" xml:"unifi" yaml:"unifi"` + Config *Config `json:"unifi" toml:"unifi" xml:"unifi" yaml:"unifi"` poller.Logger } @@ -33,9 +35,9 @@ type Controller struct { // Config contains our configuration data type Config struct { - sync.RWMutex // locks the Unifi struct member when re-authing to unifi. - Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` - Controllers []Controller `json:"controllers" toml:"controller" xml:"controller" yaml:"controllers"` + sync.RWMutex // locks the Unifi struct member when re-authing to unifi. + Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` + Controllers []*Controller `json:"controllers" toml:"controller" xml:"controller" yaml:"controllers"` } func init() { @@ -48,7 +50,7 @@ func init() { } // getUnifi (re-)authenticates to a unifi controller. -func (u *InputUnifi) getUnifi(c Controller) error { +func (u *InputUnifi) getUnifi(c *Controller) error { var err error u.Config.Lock() @@ -76,3 +78,74 @@ func (u *InputUnifi) getUnifi(c Controller) error { return nil } + +// checkSites makes sure the list of provided sites exists on the controller. +// This only runs once during initialization. +func (u *InputUnifi) checkSites(c *Controller) error { + u.Config.RLock() + defer u.Config.RUnlock() + + if len(c.Sites) < 1 || c.Sites[0] == "" { + c.Sites = []string{"all"} + } + + u.LogDebugf("Checking Controller Sites List") + + sites, err := c.Unifi.GetSites() + if err != nil { + return err + } + + msg := []string{} + for _, site := range sites { + msg = append(msg, site.Name+" ("+site.Desc+")") + } + + u.Logf("Found %d site(s) on controller %s: %v", len(msg), c.Name, strings.Join(msg, ", ")) + + if StringInSlice("all", c.Sites) { + c.Sites = []string{"all"} + return nil + } + +FIRST: + for _, s := range c.Sites { + for _, site := range sites { + if s == site.Name { + continue FIRST + } + } + return fmt.Errorf("configured site not found on controller: %v", s) + } + + return nil +} + +func (u *InputUnifi) dumpSitesJSON(c *Controller, path, name string, sites unifi.Sites) ([]byte, error) { + allJSON := []byte{} + + for _, s := range sites { + apiPath := fmt.Sprintf(path, s.Name) + _, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping %s: '%s' JSON for site: %s (%s):\n", name, apiPath, s.Desc, s.Name) + + body, err := c.Unifi.GetJSON(apiPath) + if err != nil { + return allJSON, err + } + + allJSON = append(allJSON, body...) + } + + return allJSON, nil +} + +// StringInSlice returns true if a string is in a slice. +func StringInSlice(str string, slice []string) bool { + for _, s := range slice { + if strings.EqualFold(s, str) { + return true + } + } + + return false +} diff --git a/integrations/inputunifi/pkg/inputunifi/interface.go b/integrations/inputunifi/pkg/inputunifi/interface.go index 1db32906..f9b437de 100644 --- a/integrations/inputunifi/pkg/inputunifi/interface.go +++ b/integrations/inputunifi/pkg/inputunifi/interface.go @@ -1,13 +1,51 @@ package inputunifi +/* This file contains the three poller.Input interface methods. */ + import ( "fmt" + "os" "strings" "github.com/davidnewhall/unifi-poller/pkg/poller" "golift.io/unifi" ) +// Initialize gets called one time when starting up. +// Satisfies poller.Input interface. +func (u *InputUnifi) Initialize(l poller.Logger) error { + if u.Config.Disable { + l.Logf("unifi input disabled") + return nil + } + + if len(u.Config.Controllers) < 1 { + return fmt.Errorf("no unifi controllers defined for unifi input") + } + + u.Logger = l + + for i, c := range u.Config.Controllers { + if c.Name == "" { + u.Config.Controllers[i].Name = c.URL + } + + switch err := u.getUnifi(c); err { + case nil: + if err := u.checkSites(c); err != nil { + u.LogErrorf("checking sites on %s: %v", c.Name, err) + } + + u.Logf("Polling UniFi Controller at %s v%s as user %s. Sites: %v", + c.URL, c.Unifi.ServerVersion, c.User, c.Sites) + default: + u.LogErrorf("Controller Auth or Connection failed, but continuing to retry! %s: %v", c.Name, err) + } + } + + return nil +} + // Metrics grabs all the measurements from a UniFi controller and returns them. func (u *InputUnifi) Metrics() (*poller.Metrics, error) { if u.Config.Disable { @@ -52,75 +90,35 @@ func (u *InputUnifi) Metrics() (*poller.Metrics, error) { return metrics, nil } -// Initialize gets called one time when starting up. -// Satisfies poller.Input interface. -func (u *InputUnifi) Initialize(l poller.Logger) error { - if u.Config.Disable { - l.Logf("unifi input disabled") - return nil - } +// RawMetrics returns API output from the first configured unifi controller. +func (u *InputUnifi) RawMetrics(filter poller.Filter) ([]byte, error) { + c := u.Config.Controllers[0] // We could pull the controller number from the filter. + if u.isNill(c) { + u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) - if len(u.Config.Controllers) < 1 { - return fmt.Errorf("no unifi controllers defined for unifi input") - } - - u.Logger = l - - for i, c := range u.Config.Controllers { - if c.Name == "" { - u.Config.Controllers[i].Name = c.URL - } - - switch err := u.getUnifi(c); err { - case nil: - if err := u.checkSites(c); err != nil { - u.LogErrorf("checking sites on %s: %v", c.Name, err) - } - - u.Logf("Polling UniFi Controller at %s v%s as user %s. Sites: %v", - c.URL, c.Unifi.ServerVersion, c.User, c.Sites) - default: - u.LogErrorf("Controller Auth or Connection failed, but continuing to retry! %s: %v", c.Name, err) + if err := u.getUnifi(c); err != nil { + return nil, fmt.Errorf("re-authenticating to %s: %v", c.Name, err) } } - return nil -} + if err := u.checkSites(c); err != nil { + return nil, err + } -// checkSites makes sure the list of provided sites exists on the controller. -// This only runs once during initialization. -func (u *InputUnifi) checkSites(c Controller) error { - u.Config.RLock() - defer u.Config.RUnlock() - u.LogDebugf("Checking Controller Sites List") - - sites, err := c.Unifi.GetSites() + sites, err := u.getFilteredSites(c) if err != nil { - return err + return nil, err } - msg := []string{} - - for _, site := range sites { - msg = append(msg, site.Name+" ("+site.Desc+")") + switch filter.Type { + case "d", "device", "devices": + return u.dumpSitesJSON(c, unifi.APIDevicePath, "Devices", sites) + case "client", "clients", "c": + return u.dumpSitesJSON(c, unifi.APIClientPath, "Clients", sites) + case "other", "o": + _, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping Path '%s':\n", filter.Term) + return c.Unifi.GetJSON(filter.Term) + default: + return []byte{}, fmt.Errorf("must provide filter: devices, clients, other") } - - u.Logf("Found %d site(s) on controller: %v", len(msg), strings.Join(msg, ", ")) - - if poller.StringInSlice("all", c.Sites) { - c.Sites = []string{"all"} - return nil - } - -FIRST: - for _, s := range c.Sites { - for _, site := range sites { - if s == site.Name { - continue FIRST - } - } - return fmt.Errorf("configured site not found on controller: %v", s) - } - - return nil } diff --git a/integrations/inputunifi/pkg/poller/dumper.go b/integrations/inputunifi/pkg/poller/dumper.go index 18abe901..2a2b0dab 100644 --- a/integrations/inputunifi/pkg/poller/dumper.go +++ b/integrations/inputunifi/pkg/poller/dumper.go @@ -1,85 +1,28 @@ package poller import ( + "fmt" "strings" ) // DumpJSONPayload prints raw json from the UniFi Controller. // This only works with controller 0 (first one) in the config. func (u *UnifiPoller) DumpJSONPayload() (err error) { - if true { - return nil + u.Config.Quiet = true + + split := strings.SplitN(u.Flags.DumpJSON, " ", 2) + filter := Filter{Type: split[0]} + + if len(split) > 1 { + filter.Term = split[1] } - /* - u.Config.Quiet = true - config := u.Config.Controllers[0] - config.Unifi, err = unifi.NewUnifi(&unifi.Config{ - User: config.User, - Pass: config.Pass, - URL: config.URL, - VerifySSL: config.VerifySSL, - }) - if err != nil { - return err - } + m, err := inputs[0].RawMetrics(filter) + if err != nil { + return err + } - fmt.Fprintf(os.Stderr, "[INFO] Authenticated to UniFi Controller @ %v as user %v", config.URL, config.User) + fmt.Println(string(m)) - if err := u.CheckSites(config); err != nil { - return err - } - - config.Unifi.ErrorLog = func(m string, v ...interface{}) { - fmt.Fprintf(os.Stderr, "[ERROR] "+m, v...) - } // Log all errors to stderr. - - switch sites, err := u.GetFilteredSites(config); { - case err != nil: - return err - case StringInSlice(u.Flags.DumpJSON, []string{"d", "device", "devices"}): - return u.dumpSitesJSON(config, unifi.APIDevicePath, "Devices", sites) - case StringInSlice(u.Flags.DumpJSON, []string{"client", "clients", "c"}): - return u.dumpSitesJSON(config, unifi.APIClientPath, "Clients", sites) - case strings.HasPrefix(u.Flags.DumpJSON, "other "): - apiPath := strings.SplitN(u.Flags.DumpJSON, " ", 2)[1] - _, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping Path '%s':\n", apiPath) - return u.PrintRawAPIJSON(config, apiPath) - default: - return fmt.Errorf("must provide filter: devices, clients, other") - } - */ return nil } - -/* -func (u *UnifiPoller) dumpSitesJSON(c Controller, path, name string, sites unifi.Sites) error { - for _, s := range sites { - apiPath := fmt.Sprintf(path, s.Name) - _, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping %s: '%s' JSON for site: %s (%s):\n", - name, apiPath, s.Desc, s.Name) - if err := u.PrintRawAPIJSON(c, apiPath); err != nil { - return err - } - } - return nil -} - -// PrintRawAPIJSON prints the raw json for a user-provided path on a UniFi Controller. -func (u *UnifiPoller) PrintRawAPIJSON(c Controller, apiPath string) error { - body, err := c.Unifi.GetJSON(apiPath) - fmt.Println(string(body)) - return err -} -*/ - -// StringInSlice returns true if a string is in a slice. -func StringInSlice(str string, slice []string) bool { - for _, s := range slice { - if strings.EqualFold(s, str) { - return true - } - } - - return false -} diff --git a/integrations/inputunifi/pkg/poller/inputs.go b/integrations/inputunifi/pkg/poller/inputs.go index c2140b01..f2d64efb 100644 --- a/integrations/inputunifi/pkg/poller/inputs.go +++ b/integrations/inputunifi/pkg/poller/inputs.go @@ -17,6 +17,7 @@ var ( type Input interface { Initialize(Logger) error // Called once on startup to initialize the plugin. Metrics() (*Metrics, error) // Called every time new metrics are requested. + RawMetrics(Filter) ([]byte, error) } // InputPlugin describes an input plugin's consumable interface. @@ -25,6 +26,12 @@ type InputPlugin struct { Input } +// Filter is used for raw metrics filters. +type Filter struct { + Type string + Term string +} + // NewInput creates a metric input. This should be called by input plugins // init() functions. func NewInput(i *InputPlugin) { diff --git a/integrations/inputunifi/pkg/poller/logger.go b/integrations/inputunifi/pkg/poller/logger.go index b498a9b5..fa983e5f 100644 --- a/integrations/inputunifi/pkg/poller/logger.go +++ b/integrations/inputunifi/pkg/poller/logger.go @@ -16,14 +16,14 @@ type Logger interface { // Logf prints a log entry if quiet is false. func (u *UnifiPoller) Logf(m string, v ...interface{}) { - if !u.Config.Quiet { + if !u.Quiet { _ = log.Output(callDepth, fmt.Sprintf("[INFO] "+m, v...)) } } // LogDebugf prints a debug log entry if debug is true and quite is false func (u *UnifiPoller) LogDebugf(m string, v ...interface{}) { - if u.Config.Debug && !u.Config.Quiet { + if u.Debug && !u.Quiet { _ = log.Output(callDepth, fmt.Sprintf("[DEBUG] "+m, v...)) } } diff --git a/integrations/inputunifi/pkg/poller/start.go b/integrations/inputunifi/pkg/poller/start.go index 9f6bedc8..e88daf24 100644 --- a/integrations/inputunifi/pkg/poller/start.go +++ b/integrations/inputunifi/pkg/poller/start.go @@ -61,10 +61,14 @@ func (f *Flags) Parse(args []string) { // 3. Start a web server and wait for Prometheus to poll the application for metrics. func (u *UnifiPoller) Run() error { if u.Flags.DumpJSON != "" { + if err := u.InitializeInputs(); err != nil { + return err + } + return u.DumpJSONPayload() } - if u.Config.Debug { + if u.Debug { log.SetFlags(log.Lshortfile | log.Lmicroseconds | log.Ldate) u.LogDebugf("Debug Logging Enabled") } From 61233361b427757cf0cbefbcb1eb314cb45123ae Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Tue, 17 Dec 2019 02:09:30 -0800 Subject: [PATCH 25/41] fixes --- integrations/inputunifi/Gopkg.lock | 4 ++-- integrations/inputunifi/pkg/inputunifi/input.go | 9 ++++++++- integrations/inputunifi/pkg/poller/config.go | 3 ++- integrations/inputunifi/pkg/poller/start.go | 2 +- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/integrations/inputunifi/Gopkg.lock b/integrations/inputunifi/Gopkg.lock index aa24b10e..67310aa2 100644 --- a/integrations/inputunifi/Gopkg.lock +++ b/integrations/inputunifi/Gopkg.lock @@ -107,11 +107,11 @@ [[projects]] branch = "master" - digest = "1:7a90fad47972b5ae06013d4685eb2f3007e7c92609a1399d2adf59fe04cd9b63" + digest = "1:331d56d3a24e9b8d203883c28803313af060ca5c350be0a9c0552b43f355ecd8" name = "golift.io/config" packages = ["."] pruneopts = "UT" - revision = "fe642c8392dc00d72ddcc47f05a06096bd5d054b" + revision = "1861b4270bf42ec3ccaf86871aa1cf6742564d0f" [[projects]] digest = "1:2883cea734f2766f41ff9c9d4aefccccc53e3d44f5c8b08893b9c218cf666722" diff --git a/integrations/inputunifi/pkg/inputunifi/input.go b/integrations/inputunifi/pkg/inputunifi/input.go index 9a4877f9..2ddc1f4f 100644 --- a/integrations/inputunifi/pkg/inputunifi/input.go +++ b/integrations/inputunifi/pkg/inputunifi/input.go @@ -108,14 +108,21 @@ func (u *InputUnifi) checkSites(c *Controller) error { return nil } + keep := []string{} + FIRST: for _, s := range c.Sites { for _, site := range sites { if s == site.Name { + keep = append(keep, s) continue FIRST } } - return fmt.Errorf("configured site not found on controller: %v", s) + u.LogErrorf("Configured site not found on controller %s: %v", c.Name, s) + } + + if c.Sites = keep; len(keep) < 1 { + c.Sites = []string{"all"} } return nil diff --git a/integrations/inputunifi/pkg/poller/config.go b/integrations/inputunifi/pkg/poller/config.go index a43caceb..3e1217f1 100644 --- a/integrations/inputunifi/pkg/poller/config.go +++ b/integrations/inputunifi/pkg/poller/config.go @@ -52,7 +52,7 @@ type Metrics struct { // Config represents the core library input data. type Config struct { - Poller `json:"poller" toml:"poller" xml:"poller" yaml:"poller"` + *Poller `json:"poller" toml:"poller" xml:"poller" yaml:"poller"` } // Poller is the global config values. @@ -103,6 +103,7 @@ func (u *UnifiPoller) ParseConfigs() error { // parseInterface parses the config file and environment variables into the provided interface. func (u *UnifiPoller) parseInterface(i interface{}) error { + config.ENVTag = "xml" // xml tag is better formatted for slices. // Parse config file into provided interface. if err := config.ParseFile(i, u.Flags.ConfigFile); err != nil { return err diff --git a/integrations/inputunifi/pkg/poller/start.go b/integrations/inputunifi/pkg/poller/start.go index e88daf24..34cba9da 100644 --- a/integrations/inputunifi/pkg/poller/start.go +++ b/integrations/inputunifi/pkg/poller/start.go @@ -12,7 +12,7 @@ import ( // New returns a new poller struct. func New() *UnifiPoller { - return &UnifiPoller{Config: &Config{}, Flags: &Flags{}} + return &UnifiPoller{Config: &Config{Poller: &Poller{}}, Flags: &Flags{}} } // Start begins the application from a CLI. From 37d9b0f43942acec6f1c79099a7bb7c992f00050 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Tue, 17 Dec 2019 02:39:36 -0800 Subject: [PATCH 26/41] allow pulling specific sites through output plugins --- .../inputunifi/pkg/influxunifi/influxdb.go | 7 +- .../inputunifi/pkg/inputunifi/input.go | 1 + .../inputunifi/pkg/inputunifi/interface.go | 20 ++++-- integrations/inputunifi/pkg/poller/inputs.go | 64 +++++++++++++++++-- integrations/inputunifi/pkg/poller/outputs.go | 3 +- .../inputunifi/pkg/promunifi/collector.go | 9 ++- 6 files changed, 90 insertions(+), 14 deletions(-) diff --git a/integrations/inputunifi/pkg/influxunifi/influxdb.go b/integrations/inputunifi/pkg/influxunifi/influxdb.go index 9bbe9c84..c88f7fbd 100644 --- a/integrations/inputunifi/pkg/influxunifi/influxdb.go +++ b/integrations/inputunifi/pkg/influxunifi/influxdb.go @@ -69,10 +69,13 @@ func (u *InfluxUnifi) PollController() { log.Printf("[INFO] Everything checks out! Poller started, InfluxDB interval: %v", interval) for u.LastCheck = range ticker.C { - metrics, err := u.Collector.Metrics() + metrics, ok, err := u.Collector.Metrics() if err != nil { u.Collector.LogErrorf("%v", err) - continue + + if !ok { + continue + } } report, err := u.ReportMetrics(metrics) diff --git a/integrations/inputunifi/pkg/inputunifi/input.go b/integrations/inputunifi/pkg/inputunifi/input.go index 2ddc1f4f..982da1b4 100644 --- a/integrations/inputunifi/pkg/inputunifi/input.go +++ b/integrations/inputunifi/pkg/inputunifi/input.go @@ -44,6 +44,7 @@ func init() { u := &InputUnifi{} poller.NewInput(&poller.InputPlugin{ + Name: "unifi", Input: u, // this library implements poller.Input interface for Metrics(). Config: u, // Defines our config data interface. }) diff --git a/integrations/inputunifi/pkg/inputunifi/interface.go b/integrations/inputunifi/pkg/inputunifi/interface.go index f9b437de..99ee7f51 100644 --- a/integrations/inputunifi/pkg/inputunifi/interface.go +++ b/integrations/inputunifi/pkg/inputunifi/interface.go @@ -47,15 +47,25 @@ func (u *InputUnifi) Initialize(l poller.Logger) error { } // Metrics grabs all the measurements from a UniFi controller and returns them. -func (u *InputUnifi) Metrics() (*poller.Metrics, error) { +func (u *InputUnifi) Metrics() (*poller.Metrics, bool, error) { + return u.MetricsFrom(poller.Filter{}) +} + +// MetricsFrom grabs all the measurements from a UniFi controller and returns them. +func (u *InputUnifi) MetricsFrom(filter poller.Filter) (*poller.Metrics, bool, error) { if u.Config.Disable { - return nil, nil + return nil, false, nil } errs := []string{} metrics := &poller.Metrics{} + ok := false for _, c := range u.Config.Controllers { + if filter.Term != "" && c.Name != filter.Term { + continue + } + m, err := u.collectController(c) if err != nil { errs = append(errs, err.Error()) @@ -65,6 +75,8 @@ func (u *InputUnifi) Metrics() (*poller.Metrics, error) { continue } + ok = true + metrics.Sites = append(metrics.Sites, m.Sites...) metrics.Clients = append(metrics.Clients, m.Clients...) metrics.IDSList = append(metrics.IDSList, m.IDSList...) @@ -84,10 +96,10 @@ func (u *InputUnifi) Metrics() (*poller.Metrics, error) { } if len(errs) > 0 { - return metrics, fmt.Errorf(strings.Join(errs, ", ")) + return metrics, ok, fmt.Errorf(strings.Join(errs, ", ")) } - return metrics, nil + return metrics, ok, nil } // RawMetrics returns API output from the first configured unifi controller. diff --git a/integrations/inputunifi/pkg/poller/inputs.go b/integrations/inputunifi/pkg/poller/inputs.go index f2d64efb..5d361166 100644 --- a/integrations/inputunifi/pkg/poller/inputs.go +++ b/integrations/inputunifi/pkg/poller/inputs.go @@ -15,13 +15,15 @@ var ( // Input plugins must implement this interface. type Input interface { - Initialize(Logger) error // Called once on startup to initialize the plugin. - Metrics() (*Metrics, error) // Called every time new metrics are requested. + Initialize(Logger) error // Called once on startup to initialize the plugin. + Metrics() (*Metrics, bool, error) // Called every time new metrics are requested. + MetricsFrom(Filter) (*Metrics, bool, error) // Called every time new metrics are requested. RawMetrics(Filter) ([]byte, error) } // InputPlugin describes an input plugin's consumable interface. type InputPlugin struct { + Name string Config interface{} // Each config is passed into an unmarshaller later. Input } @@ -61,12 +63,13 @@ func (u *UnifiPoller) InitializeInputs() error { } // Metrics aggregates all the measurements from all configured inputs and returns them. -func (u *UnifiPoller) Metrics() (*Metrics, error) { +func (u *UnifiPoller) Metrics() (*Metrics, bool, error) { errs := []string{} metrics := &Metrics{} + ok := false for _, input := range inputs { - m, err := input.Metrics() + m, _, err := input.Metrics() if err != nil { errs = append(errs, err.Error()) } @@ -75,6 +78,8 @@ func (u *UnifiPoller) Metrics() (*Metrics, error) { continue } + ok = true + metrics.Sites = append(metrics.Sites, m.Sites...) metrics.Clients = append(metrics.Clients, m.Clients...) metrics.IDSList = append(metrics.IDSList, m.IDSList...) @@ -99,5 +104,54 @@ func (u *UnifiPoller) Metrics() (*Metrics, error) { err = fmt.Errorf(strings.Join(errs, ", ")) } - return metrics, err + return metrics, ok, err +} + +// MetricsFrom aggregates all the measurements from all configured inputs and returns them. +func (u *UnifiPoller) MetricsFrom(filter Filter) (*Metrics, bool, error) { + errs := []string{} + metrics := &Metrics{} + ok := false + + for _, input := range inputs { + if input.Name != filter.Type { + continue + } + + m, _, err := input.MetricsFrom(filter) + if err != nil { + errs = append(errs, err.Error()) + } + + if m == nil { + continue + } + + ok = true + + metrics.Sites = append(metrics.Sites, m.Sites...) + metrics.Clients = append(metrics.Clients, m.Clients...) + metrics.IDSList = append(metrics.IDSList, m.IDSList...) + + if m.Devices == nil { + continue + } + + if metrics.Devices == nil { + metrics.Devices = &unifi.Devices{} + } + + metrics.UAPs = append(metrics.UAPs, m.UAPs...) + metrics.USGs = append(metrics.USGs, m.USGs...) + metrics.USWs = append(metrics.USWs, m.USWs...) + metrics.UDMs = append(metrics.UDMs, m.UDMs...) + } + + var err error + + if len(errs) > 0 { + err = fmt.Errorf(strings.Join(errs, ", ")) + } + + return metrics, ok, err } diff --git a/integrations/inputunifi/pkg/poller/outputs.go b/integrations/inputunifi/pkg/poller/outputs.go index 3bbcb72d..0672d92b 100644 --- a/integrations/inputunifi/pkg/poller/outputs.go +++ b/integrations/inputunifi/pkg/poller/outputs.go @@ -13,7 +13,8 @@ var ( // Collect is passed into output packages so they may collect metrics to output. // Output packages must implement this interface. type Collect interface { - Metrics() (*Metrics, error) + Metrics() (*Metrics, bool, error) + MetricsFrom(Filter) (*Metrics, bool, error) Logger } diff --git a/integrations/inputunifi/pkg/promunifi/collector.go b/integrations/inputunifi/pkg/promunifi/collector.go index a4aad9f3..694bfd4c 100644 --- a/integrations/inputunifi/pkg/promunifi/collector.go +++ b/integrations/inputunifi/pkg/promunifi/collector.go @@ -140,12 +140,17 @@ func (u *promUnifi) Describe(ch chan<- *prometheus.Desc) { func (u *promUnifi) Collect(ch chan<- prometheus.Metric) { var err error + ok := false + r := &Report{Config: u.Config, ch: make(chan []*metric, buffer), Start: time.Now()} defer r.close() - if r.Metrics, err = u.Collector.Metrics(); err != nil { + if r.Metrics, ok, err = u.Collector.Metrics(); err != nil { r.error(ch, prometheus.NewInvalidDesc(fmt.Errorf("metric fetch failed")), err) - return + + if !ok { + return + } } r.Fetch = time.Since(r.Start) From e9c2e7c25764ca3348f982ba969886d1a3a5a28d Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Thu, 19 Dec 2019 00:29:56 -0800 Subject: [PATCH 27/41] Update deps --- integrations/inputunifi/Gopkg.lock | 33 +++++++++++++------- integrations/inputunifi/pkg/poller/config.go | 8 ++--- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/integrations/inputunifi/Gopkg.lock b/integrations/inputunifi/Gopkg.lock index 67310aa2..c5d41385 100644 --- a/integrations/inputunifi/Gopkg.lock +++ b/integrations/inputunifi/Gopkg.lock @@ -1,14 +1,6 @@ # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. -[[projects]] - digest = "1:9f3b30d9f8e0d7040f729b82dcbc8f0dead820a133b3147ce355fc451f32d761" - name = "github.com/BurntSushi/toml" - packages = ["."] - pruneopts = "UT" - revision = "3012a1dbe2e4bd1391d42b32f0577cb7bbc7f005" - version = "v0.3.1" - [[projects]] digest = "1:d6afaeed1502aa28e80a4ed0981d570ad91b2579193404256ce672ed0a609e0d" name = "github.com/beorn7/perks" @@ -45,6 +37,14 @@ revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" version = "v1.0.1" +[[projects]] + digest = "1:6eea828983c70075ca297bb915ffbcfd3e34c5a50affd94428a65df955c0ff9c" + name = "github.com/pelletier/go-toml" + packages = ["."] + pruneopts = "UT" + revision = "903d9455db9ff1d7ac1ab199062eca7266dd11a3" + version = "v1.6.0" + [[projects]] digest = "1:7097829edd12fd7211fca0d29496b44f94ef9e6d72f88fb64f3d7b06315818ad" name = "github.com/prometheus/client_golang" @@ -103,15 +103,23 @@ name = "golang.org/x/sys" packages = ["windows"] pruneopts = "UT" - revision = "ac6580df4449443a05718fd7858c1f91ad5f8d20" + revision = "4a24b406529242041050cb1dec3e0e4c46a5f1b6" [[projects]] - branch = "master" - digest = "1:331d56d3a24e9b8d203883c28803313af060ca5c350be0a9c0552b43f355ecd8" + digest = "1:54742bef5cb29f706614c9edcfdeb29fb5992f26090f26ca955f575dddf54f9e" + name = "golift.io/cnfg" + packages = ["."] + pruneopts = "UT" + revision = "961061d377655468e9da4a9333e71b9b77402470" + version = "v0.0.1" + +[[projects]] + digest = "1:54742bef5cb29f706614c9edcfdeb29fb5992f26090f26ca955f575dddf54f9e" name = "golift.io/config" packages = ["."] pruneopts = "UT" - revision = "1861b4270bf42ec3ccaf86871aa1cf6742564d0f" + revision = "961061d377655468e9da4a9333e71b9b77402470" + version = "v0.0.1" [[projects]] digest = "1:2883cea734f2766f41ff9c9d4aefccccc53e3d44f5c8b08893b9c218cf666722" @@ -137,6 +145,7 @@ "github.com/prometheus/client_golang/prometheus", "github.com/prometheus/common/version", "github.com/spf13/pflag", + "golift.io/cnfg", "golift.io/config", "golift.io/unifi", ] diff --git a/integrations/inputunifi/pkg/poller/config.go b/integrations/inputunifi/pkg/poller/config.go index 3e1217f1..006dc1fb 100644 --- a/integrations/inputunifi/pkg/poller/config.go +++ b/integrations/inputunifi/pkg/poller/config.go @@ -16,7 +16,7 @@ import ( "time" "github.com/spf13/pflag" - "golift.io/config" + "golift.io/cnfg" "golift.io/unifi" ) @@ -103,14 +103,14 @@ func (u *UnifiPoller) ParseConfigs() error { // parseInterface parses the config file and environment variables into the provided interface. func (u *UnifiPoller) parseInterface(i interface{}) error { - config.ENVTag = "xml" // xml tag is better formatted for slices. + cnfg.ENVTag = "xml" // xml tag is better formatted for slices. // Parse config file into provided interface. - if err := config.ParseFile(i, u.Flags.ConfigFile); err != nil { + if err := cnfg.ParseFile(i, u.Flags.ConfigFile); err != nil { return err } // Parse environment variables into provided interface. - _, err := config.ParseENV(i, ENVConfigPrefix) + _, err := cnfg.ParseENV(i, ENVConfigPrefix) return err } From 01b1bb2f29d2c199f0e270215ccc9b7d37f656ce Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Thu, 19 Dec 2019 19:59:51 -0800 Subject: [PATCH 28/41] allow dynamic controller scrapes --- integrations/inputunifi/Gopkg.lock | 17 ++---- .../inputunifi/pkg/influxunifi/influxdb.go | 22 +++---- .../inputunifi/pkg/inputunifi/collector.go | 43 ++++++++++++++ .../inputunifi/pkg/inputunifi/input.go | 31 ++++++++++ .../inputunifi/pkg/inputunifi/interface.go | 52 +++++++---------- integrations/inputunifi/pkg/poller/dumper.go | 2 +- integrations/inputunifi/pkg/poller/inputs.go | 10 ++-- integrations/inputunifi/pkg/poller/outputs.go | 2 +- .../inputunifi/pkg/promunifi/collector.go | 57 ++++++++++++++++++- integrations/inputunifi/plugins/mysql/main.go | 14 ++--- 10 files changed, 178 insertions(+), 72 deletions(-) diff --git a/integrations/inputunifi/Gopkg.lock b/integrations/inputunifi/Gopkg.lock index c5d41385..7cd32b0e 100644 --- a/integrations/inputunifi/Gopkg.lock +++ b/integrations/inputunifi/Gopkg.lock @@ -46,11 +46,12 @@ version = "v1.6.0" [[projects]] - digest = "1:7097829edd12fd7211fca0d29496b44f94ef9e6d72f88fb64f3d7b06315818ad" + digest = "1:eb04f69c8991e52eff33c428bd729e04208bf03235be88e4df0d88497c6861b9" name = "github.com/prometheus/client_golang" packages = [ "prometheus", "prometheus/internal", + "prometheus/promhttp", ] pruneopts = "UT" revision = "170205fb58decfd011f1550d4cfb737230d7ae4f" @@ -99,11 +100,11 @@ [[projects]] branch = "master" - digest = "1:68fe4216878f16dd6ef33413365fbbe8d2eb781177c7adab874cfc752ce96a7e" + digest = "1:07f0cb66f649e51f9ef23441f8dfc34a73e7d9bf0832417abcbad578f1d8c8d6" name = "golang.org/x/sys" packages = ["windows"] pruneopts = "UT" - revision = "4a24b406529242041050cb1dec3e0e4c46a5f1b6" + revision = "af0d71d358abe0ba3594483a5d519f429dbae3e9" [[projects]] digest = "1:54742bef5cb29f706614c9edcfdeb29fb5992f26090f26ca955f575dddf54f9e" @@ -113,14 +114,6 @@ revision = "961061d377655468e9da4a9333e71b9b77402470" version = "v0.0.1" -[[projects]] - digest = "1:54742bef5cb29f706614c9edcfdeb29fb5992f26090f26ca955f575dddf54f9e" - name = "golift.io/config" - packages = ["."] - pruneopts = "UT" - revision = "961061d377655468e9da4a9333e71b9b77402470" - version = "v0.0.1" - [[projects]] digest = "1:2883cea734f2766f41ff9c9d4aefccccc53e3d44f5c8b08893b9c218cf666722" name = "golift.io/unifi" @@ -143,10 +136,10 @@ input-imports = [ "github.com/influxdata/influxdb1-client/v2", "github.com/prometheus/client_golang/prometheus", + "github.com/prometheus/client_golang/prometheus/promhttp", "github.com/prometheus/common/version", "github.com/spf13/pflag", "golift.io/cnfg", - "golift.io/config", "golift.io/unifi", ] solver-name = "gps-cdcl" diff --git a/integrations/inputunifi/pkg/influxunifi/influxdb.go b/integrations/inputunifi/pkg/influxunifi/influxdb.go index c88f7fbd..280e94a9 100644 --- a/integrations/inputunifi/pkg/influxunifi/influxdb.go +++ b/integrations/inputunifi/pkg/influxunifi/influxdb.go @@ -10,7 +10,7 @@ import ( "github.com/davidnewhall/unifi-poller/pkg/poller" influx "github.com/influxdata/influxdb1-client/v2" - "golift.io/config" + "golift.io/cnfg" ) const ( @@ -23,13 +23,13 @@ const ( // Config defines the data needed to store metrics in InfluxDB type Config struct { - Interval config.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` - Disable bool `json:"disable" toml:"disable" xml:"disable,attr" yaml:"disable"` - VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` - URL string `json:"url,omitempty" toml:"url,omitempty" xml:"url" yaml:"url"` - User string `json:"user,omitempty" toml:"user,omitempty" xml:"user" yaml:"user"` - Pass string `json:"pass,omitempty" toml:"pass,omitempty" xml:"pass" yaml:"pass"` - DB string `json:"db,omitempty" toml:"db,omitempty" xml:"db" yaml:"db"` + Interval cnfg.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` + Disable bool `json:"disable" toml:"disable" xml:"disable,attr" yaml:"disable"` + VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` + URL string `json:"url,omitempty" toml:"url,omitempty" xml:"url" yaml:"url"` + User string `json:"user,omitempty" toml:"user,omitempty" xml:"user" yaml:"user"` + Pass string `json:"pass,omitempty" toml:"pass,omitempty" xml:"pass" yaml:"pass"` + DB string `json:"db,omitempty" toml:"db,omitempty" xml:"db" yaml:"db"` } // InfluxDB allows the data to be nested in the config file. @@ -133,12 +133,12 @@ func (u *InfluxUnifi) setConfigDefaults() { } if u.Config.Interval.Duration == 0 { - u.Config.Interval = config.Duration{Duration: defaultInterval} + u.Config.Interval = cnfg.Duration{Duration: defaultInterval} } else if u.Config.Interval.Duration < minimumInterval { - u.Config.Interval = config.Duration{Duration: minimumInterval} + u.Config.Interval = cnfg.Duration{Duration: minimumInterval} } - u.Config.Interval = config.Duration{Duration: u.Config.Interval.Duration.Round(time.Second)} + u.Config.Interval = cnfg.Duration{Duration: u.Config.Interval.Duration.Round(time.Second)} } // ReportMetrics batches all device and client data into influxdb data points. diff --git a/integrations/inputunifi/pkg/inputunifi/collector.go b/integrations/inputunifi/pkg/inputunifi/collector.go index c9c67dcb..1539f244 100644 --- a/integrations/inputunifi/pkg/inputunifi/collector.go +++ b/integrations/inputunifi/pkg/inputunifi/collector.go @@ -15,6 +15,49 @@ func (u *InputUnifi) isNill(c *Controller) bool { return c.Unifi == nil } +func (u *InputUnifi) dynamicController(url string) (*poller.Metrics, bool, error) { + c := u.Config.Default // copy defaults into new controller + c.Name = url + c.URL = url + + u.Logf("Authenticating to Dynamic UniFi Controller: %s", url) + + if err := u.getUnifi(&c); err != nil { + return nil, false, fmt.Errorf("authenticating to %s: %v", url, err) + } + + metrics := &poller.Metrics{} + ok, err := u.appendController(&c, metrics) + + return metrics, ok, err +} + +func (u *InputUnifi) appendController(c *Controller, metrics *poller.Metrics) (bool, error) { + m, err := u.collectController(c) + if err != nil || m == nil { + return false, err + } + + metrics.Sites = append(metrics.Sites, m.Sites...) + metrics.Clients = append(metrics.Clients, m.Clients...) + metrics.IDSList = append(metrics.IDSList, m.IDSList...) + + if m.Devices == nil { + return true, nil + } + + if metrics.Devices == nil { + metrics.Devices = &unifi.Devices{} + } + + metrics.UAPs = append(metrics.UAPs, m.UAPs...) + metrics.USGs = append(metrics.USGs, m.USGs...) + metrics.USWs = append(metrics.USWs, m.USWs...) + metrics.UDMs = append(metrics.UDMs, m.UDMs...) + + return true, nil +} + func (u *InputUnifi) collectController(c *Controller) (*poller.Metrics, error) { if u.isNill(c) { u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) diff --git a/integrations/inputunifi/pkg/inputunifi/input.go b/integrations/inputunifi/pkg/inputunifi/input.go index 982da1b4..4a2881aa 100644 --- a/integrations/inputunifi/pkg/inputunifi/input.go +++ b/integrations/inputunifi/pkg/inputunifi/input.go @@ -13,6 +13,13 @@ import ( "golift.io/unifi" ) +const ( + defaultURL = "https://127.0.0.1:8443" + defaultUser = "unifipoller" + defaultPass = "unifipollerp4$$w0rd" + defaultSite = "all" +) + // InputUnifi contains the running data. type InputUnifi struct { Config *Config `json:"unifi" toml:"unifi" xml:"unifi" yaml:"unifi"` @@ -36,7 +43,9 @@ type Controller struct { // Config contains our configuration data type Config struct { sync.RWMutex // locks the Unifi struct member when re-authing to unifi. + Default Controller `json:"defaults" toml:"defaults" xml:"default" yaml:"defaults"` Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` + Dynamic bool `json:"dynamic" toml:"dynamic" xml:"dynamic" yaml:"dynamic"` Controllers []*Controller `json:"controllers" toml:"controller" xml:"controller" yaml:"controllers"` } @@ -147,6 +156,28 @@ func (u *InputUnifi) dumpSitesJSON(c *Controller, path, name string, sites unifi return allJSON, nil } +func (u *InputUnifi) setDefaults(c *Controller) { + if c.URL == "" { + c.URL = defaultURL + } + + if c.Name == "" { + c.Name = c.URL + } + + if c.Pass == "" { + c.Pass = defaultPass + } + + if c.User == "" { + c.User = defaultUser + } + + if len(c.Sites) < 1 { + c.Sites = []string{defaultSite} + } +} + // StringInSlice returns true if a string is in a slice. func StringInSlice(str string, slice []string) bool { for _, s := range slice { diff --git a/integrations/inputunifi/pkg/inputunifi/interface.go b/integrations/inputunifi/pkg/inputunifi/interface.go index 99ee7f51..67f53cce 100644 --- a/integrations/inputunifi/pkg/inputunifi/interface.go +++ b/integrations/inputunifi/pkg/inputunifi/interface.go @@ -19,16 +19,15 @@ func (u *InputUnifi) Initialize(l poller.Logger) error { return nil } - if len(u.Config.Controllers) < 1 { - return fmt.Errorf("no unifi controllers defined for unifi input") + if u.setDefaults(&u.Config.Default); len(u.Config.Controllers) < 1 { + new := u.Config.Default // copy defaults. + u.Config.Controllers = []*Controller{&new} } u.Logger = l - for i, c := range u.Config.Controllers { - if c.Name == "" { - u.Config.Controllers[i].Name = c.URL - } + for _, c := range u.Config.Controllers { + u.setDefaults(c) switch err := u.getUnifi(c); err { case nil: @@ -48,12 +47,12 @@ func (u *InputUnifi) Initialize(l poller.Logger) error { // Metrics grabs all the measurements from a UniFi controller and returns them. func (u *InputUnifi) Metrics() (*poller.Metrics, bool, error) { - return u.MetricsFrom(poller.Filter{}) + return u.MetricsFrom(nil) } // MetricsFrom grabs all the measurements from a UniFi controller and returns them. -func (u *InputUnifi) MetricsFrom(filter poller.Filter) (*poller.Metrics, bool, error) { - if u.Config.Disable { +func (u *InputUnifi) MetricsFrom(filter *poller.Filter) (*poller.Metrics, bool, error) { + if u.Config.Disable || filter == nil || filter.Term == "" { return nil, false, nil } @@ -61,49 +60,36 @@ func (u *InputUnifi) MetricsFrom(filter poller.Filter) (*poller.Metrics, bool, e metrics := &poller.Metrics{} ok := false + // Check if the request is for an existing, configured controller. for _, c := range u.Config.Controllers { - if filter.Term != "" && c.Name != filter.Term { + if !strings.EqualFold(c.Name, filter.Term) { continue } - m, err := u.collectController(c) + exists, err := u.appendController(c, metrics) if err != nil { errs = append(errs, err.Error()) } - if m == nil { - continue + if exists { + ok = true } - - ok = true - - metrics.Sites = append(metrics.Sites, m.Sites...) - metrics.Clients = append(metrics.Clients, m.Clients...) - metrics.IDSList = append(metrics.IDSList, m.IDSList...) - - if m.Devices == nil { - continue - } - - if metrics.Devices == nil { - metrics.Devices = &unifi.Devices{} - } - - metrics.UAPs = append(metrics.UAPs, m.UAPs...) - metrics.USGs = append(metrics.USGs, m.USGs...) - metrics.USWs = append(metrics.USWs, m.USWs...) - metrics.UDMs = append(metrics.UDMs, m.UDMs...) } if len(errs) > 0 { return metrics, ok, fmt.Errorf(strings.Join(errs, ", ")) } + if u.Config.Dynamic && !ok && strings.HasPrefix(filter.Term, "http") { + // Attempt to a dynamic metrics fetch from an unconfigured controller. + return u.dynamicController(filter.Term) + } + return metrics, ok, nil } // RawMetrics returns API output from the first configured unifi controller. -func (u *InputUnifi) RawMetrics(filter poller.Filter) ([]byte, error) { +func (u *InputUnifi) RawMetrics(filter *poller.Filter) ([]byte, error) { c := u.Config.Controllers[0] // We could pull the controller number from the filter. if u.isNill(c) { u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) diff --git a/integrations/inputunifi/pkg/poller/dumper.go b/integrations/inputunifi/pkg/poller/dumper.go index 2a2b0dab..80c7f728 100644 --- a/integrations/inputunifi/pkg/poller/dumper.go +++ b/integrations/inputunifi/pkg/poller/dumper.go @@ -11,7 +11,7 @@ func (u *UnifiPoller) DumpJSONPayload() (err error) { u.Config.Quiet = true split := strings.SplitN(u.Flags.DumpJSON, " ", 2) - filter := Filter{Type: split[0]} + filter := &Filter{Type: split[0]} if len(split) > 1 { filter.Term = split[1] diff --git a/integrations/inputunifi/pkg/poller/inputs.go b/integrations/inputunifi/pkg/poller/inputs.go index 5d361166..8633ca2c 100644 --- a/integrations/inputunifi/pkg/poller/inputs.go +++ b/integrations/inputunifi/pkg/poller/inputs.go @@ -15,10 +15,10 @@ var ( // Input plugins must implement this interface. type Input interface { - Initialize(Logger) error // Called once on startup to initialize the plugin. - Metrics() (*Metrics, bool, error) // Called every time new metrics are requested. - MetricsFrom(Filter) (*Metrics, bool, error) // Called every time new metrics are requested. - RawMetrics(Filter) ([]byte, error) + Initialize(Logger) error // Called once on startup to initialize the plugin. + Metrics() (*Metrics, bool, error) // Called every time new metrics are requested. + MetricsFrom(*Filter) (*Metrics, bool, error) // Called every time new metrics are requested. + RawMetrics(*Filter) ([]byte, error) } // InputPlugin describes an input plugin's consumable interface. @@ -108,7 +108,7 @@ func (u *UnifiPoller) Metrics() (*Metrics, bool, error) { } // MetricsFrom aggregates all the measurements from all configured inputs and returns them. -func (u *UnifiPoller) MetricsFrom(filter Filter) (*Metrics, bool, error) { +func (u *UnifiPoller) MetricsFrom(filter *Filter) (*Metrics, bool, error) { errs := []string{} metrics := &Metrics{} ok := false diff --git a/integrations/inputunifi/pkg/poller/outputs.go b/integrations/inputunifi/pkg/poller/outputs.go index 0672d92b..0c73bfe7 100644 --- a/integrations/inputunifi/pkg/poller/outputs.go +++ b/integrations/inputunifi/pkg/poller/outputs.go @@ -14,7 +14,7 @@ var ( // Output packages must implement this interface. type Collect interface { Metrics() (*Metrics, bool, error) - MetricsFrom(Filter) (*Metrics, bool, error) + MetricsFrom(*Filter) (*Metrics, bool, error) Logger } diff --git a/integrations/inputunifi/pkg/promunifi/collector.go b/integrations/inputunifi/pkg/promunifi/collector.go index 694bfd4c..e4cf1e9c 100644 --- a/integrations/inputunifi/pkg/promunifi/collector.go +++ b/integrations/inputunifi/pkg/promunifi/collector.go @@ -11,6 +11,7 @@ import ( "github.com/davidnewhall/unifi-poller/pkg/poller" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/common/version" "golift.io/unifi" ) @@ -77,6 +78,11 @@ type Report struct { wg sync.WaitGroup } +type target struct { + *poller.Filter + *promUnifi +} + func init() { u := &promUnifi{Prometheus: &Prometheus{}} @@ -103,6 +109,8 @@ func (u *promUnifi) Run(c poller.Collect) error { u.Config.HTTPListen = defaultHTTPListen } + mux := http.NewServeMux() + prometheus.MustRegister(version.NewCollector(u.Config.Namespace)) prometheus.MustRegister(&promUnifi{ Collector: c, @@ -115,8 +123,43 @@ func (u *promUnifi) Run(c poller.Collect) error { }) c.Logf("Exporting Measurements for Prometheus at https://%s/metrics, namespace: %s", u.Config.HTTPListen, u.Config.Namespace) + mux.Handle("/metrics", promhttp.HandlerFor( + prometheus.DefaultGatherer, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}, + )) + mux.HandleFunc("/scrape", u.ScrapeHandler) - return http.ListenAndServe(u.Config.HTTPListen, nil) + return http.ListenAndServe(u.Config.HTTPListen, mux) +} + +// ScrapeHandler allows prometheus to scrape a single source, instead of all sources. +func (u *promUnifi) ScrapeHandler(w http.ResponseWriter, r *http.Request) { + t := &target{promUnifi: u, Filter: &poller.Filter{}} + if t.Filter.Type = r.URL.Query().Get("input"); t.Filter.Type == "" { + http.Error(w, `'input' parameter must be specified (try "unifi")`, 400) + return + } + + if t.Filter.Term = r.URL.Query().Get("target"); t.Filter.Term == "" { + http.Error(w, "'target' parameter must be specified, configured name, or unconfigured url", 400) + return + } + + registry := prometheus.NewRegistry() + + registry.MustRegister(t) + promhttp.HandlerFor( + registry, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}, + ).ServeHTTP(w, r) +} + +// Describe satisfies the prometheus Collector. This returns all of the +// metric descriptions that this packages produces. +func (t *target) Describe(ch chan<- *prometheus.Desc) { + t.promUnifi.Describe(ch) +} + +func (t *target) Collect(ch chan<- prometheus.Metric) { + t.promUnifi.collect(ch, t.Filter) } // Describe satisfies the prometheus Collector. This returns all of the @@ -138,6 +181,10 @@ func (u *promUnifi) Describe(ch chan<- *prometheus.Desc) { // Collect satisfies the prometheus Collector. This runs the input method to get // the current metrics (from another package) then exports them for prometheus. func (u *promUnifi) Collect(ch chan<- prometheus.Metric) { + u.collect(ch, nil) +} + +func (u *promUnifi) collect(ch chan<- prometheus.Metric, filter *poller.Filter) { var err error ok := false @@ -145,7 +192,13 @@ func (u *promUnifi) Collect(ch chan<- prometheus.Metric) { r := &Report{Config: u.Config, ch: make(chan []*metric, buffer), Start: time.Now()} defer r.close() - if r.Metrics, ok, err = u.Collector.Metrics(); err != nil { + if filter == nil { + r.Metrics, ok, err = u.Collector.Metrics() + } else { + r.Metrics, ok, err = u.Collector.MetricsFrom(filter) + } + + if err != nil { r.error(ch, prometheus.NewInvalidDesc(fmt.Errorf("metric fetch failed")), err) if !ok { diff --git a/integrations/inputunifi/plugins/mysql/main.go b/integrations/inputunifi/plugins/mysql/main.go index e0c866b9..361165a0 100644 --- a/integrations/inputunifi/plugins/mysql/main.go +++ b/integrations/inputunifi/plugins/mysql/main.go @@ -4,17 +4,17 @@ import ( "fmt" "github.com/davidnewhall/unifi-poller/pkg/poller" - "golift.io/config" + "golift.io/cnfg" ) // mysqlConfig represents the data that is unmarshalled from the up.conf config file for this plugins. type mysqlConfig struct { - Interval config.Duration `json:"interval" toml:"interval" xml:"interval" yaml:"interval"` - Host string `json:"host" toml:"host" xml:"host" yaml:"host"` - User string `json:"user" toml:"user" xml:"user" yaml:"user"` - Pass string `json:"pass" toml:"pass" xml:"pass" yaml:"pass"` - DB string `json:"db" toml:"db" xml:"db" yaml:"db"` - Table string `json:"table" toml:"table" xml:"table" yaml:"table"` + Interval cnfg.Duration `json:"interval" toml:"interval" xml:"interval" yaml:"interval"` + Host string `json:"host" toml:"host" xml:"host" yaml:"host"` + User string `json:"user" toml:"user" xml:"user" yaml:"user"` + Pass string `json:"pass" toml:"pass" xml:"pass" yaml:"pass"` + DB string `json:"db" toml:"db" xml:"db" yaml:"db"` + Table string `json:"table" toml:"table" xml:"table" yaml:"table"` // Maps do not work with ENV VARIABLES yet, but may in the future. Fields []string `json:"fields" toml:"fields" xml:"field" yaml:"fields"` } From 53a168716872e66e7a2211c859467452289137c8 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Fri, 20 Dec 2019 02:44:53 -0800 Subject: [PATCH 29/41] fixes --- integrations/inputunifi/examples/MANUAL.md | 52 +------- .../inputunifi/examples/up.conf.example | 122 ++++++++++-------- .../inputunifi/examples/up.json.example | 20 ++- .../inputunifi/examples/up.xml.example | 21 ++- .../inputunifi/examples/up.yaml.example | 21 ++- .../inputunifi/pkg/inputunifi/collector.go | 66 +++++----- .../inputunifi/pkg/inputunifi/input.go | 17 ++- .../inputunifi/pkg/inputunifi/interface.go | 50 ++++--- integrations/inputunifi/pkg/poller/dumper.go | 15 ++- integrations/inputunifi/pkg/poller/inputs.go | 62 ++++++--- .../inputunifi/pkg/promunifi/collector.go | 112 ++++++++-------- .../inputunifi/pkg/promunifi/report.go | 2 +- 12 files changed, 313 insertions(+), 247 deletions(-) diff --git a/integrations/inputunifi/examples/MANUAL.md b/integrations/inputunifi/examples/MANUAL.md index cabf4db8..d12c9ed3 100644 --- a/integrations/inputunifi/examples/MANUAL.md +++ b/integrations/inputunifi/examples/MANUAL.md @@ -65,56 +65,16 @@ is provided so the application can be easily adapted to any environment. `Config File Parameters` -Additional parameters are added by output packages. Parameters can also be set -using environment variables. See the GitHub wiki for more information! +Configuration file (up.conf) parameters are documented in the wiki. - >>> POLLER FIELDS FOLLOW - you may have multiple controllers: +* [https://github.com/davidnewhall/unifi-poller/wiki/Configuration](https://github.com/davidnewhall/unifi-poller/wiki/Configuration) - debug default: false - This turns on time stamps and line numbers in logs, outputs a few extra - lines of information while processing. +`Shell Environment Parameters` - quiet default: false - Setting this to true will turn off per-device and per-interval logs. Only - errors will be logged. Using this with debug=true adds line numbers to - any error logs. +This application can be fully configured using shell environment variables. +Find documentation for this feature on the Docker Wiki page. - >>> UNIFI CONTROLLER FIELDS FOLLOW - you may have multiple controllers: - - sites default: ["all"] - This list of strings should represent the names of sites on the UniFi - controller that will be polled for data. Pass `all` in the list to - poll all sites. On startup, the application prints out all site names - found in the controller; they're cryptic, but they have the human-name - next to them. The cryptic names go into the config file `sites` list. - The controller's first site is not cryptic and is named `default`. - - url default: https://127.0.0.1:8443 - This is the URL where the UniFi Controller is available. - - user default: influxdb - Username used to authenticate with UniFi controller. This should be a - special service account created on the control with read-only access. - - pass no default - Password used to authenticate with UniFi controller. This can also be - set in an environment variable instead of a configuration file. - - save_ids default: false - Setting this parameter to true will enable collection of Intrusion - Detection System data. IDS and IPS are the same data set. This is off - by default because most controllers do not have this enabled. It also - creates a lot of new metrics from controllers with a lot of IDS entries. - IDS data does not contain metrics, so this doesn't work with Prometheus. - - save_sites default: true - Setting this parameter to false will disable saving Network Site data. - This data populates the Sites dashboard, and this setting affects influx - and prometheus. - - verify_ssl default: false - If your UniFi controller has a valid SSL certificate, you can enable - this option to validate it. Otherwise, any SSL certificate is valid. +* [https://github.com/davidnewhall/unifi-poller/wiki/Docker](https://github.com/davidnewhall/unifi-poller/wiki/Docker) GO DURATION --- diff --git a/integrations/inputunifi/examples/up.conf.example b/integrations/inputunifi/examples/up.conf.example index 43aa9831..942c53cd 100644 --- a/integrations/inputunifi/examples/up.conf.example +++ b/integrations/inputunifi/examples/up.conf.example @@ -1,75 +1,95 @@ # UniFi Poller primary configuration file. TOML FORMAT # -# commented lines are defaults, uncomment to change. # ######################################################## - [poller] -# Turns on line numbers, microsecond logging, and a per-device log. -# The default is false, but I personally leave this on at home (four devices). -# This may be noisy if you have a lot of devices. It adds one line per device. -debug = false + # Turns on line numbers, microsecond logging, and a per-device log. + # The default is false, but I personally leave this on at home (four devices). + # This may be noisy if you have a lot of devices. It adds one line per device. + debug = false -# Turns off per-interval logs. Only startup and error logs will be emitted. -# Recommend enabling debug with this setting for better error logging. -quiet = false + # Turns off per-interval logs. Only startup and error logs will be emitted. + # Recommend enabling debug with this setting for better error logging. + quiet = false -# Load dynamic plugins. Advanced use; only sample mysql plugin provided by default. -plugins = [] + # Load dynamic plugins. Advanced use; only sample mysql plugin provided by default. + plugins = [] #### OUTPUTS + # If you don't use an output, you can disable it. + [prometheus] -disable = false -# This controls on which ip and port /metrics is exported when mode is "prometheus". -# This has no effect in other modes. Must contain a colon and port. -http_listen = "0.0.0.0:9130" -report_errors = false + disable = false + # This controls on which ip and port /metrics is exported when mode is "prometheus". + # This has no effect in other modes. Must contain a colon and port. + http_listen = "0.0.0.0:9130" + report_errors = false [influxdb] -disable = false -# InfluxDB does not require auth by default, so the user/password are probably unimportant. -url = "http://127.0.0.1:8086" -user = "unifi" -pass = "unifi" -# Be sure to create this database. -db = "unifi" -# If your InfluxDB uses a valid SSL cert, set this to true. -verify_ssl = false -# The UniFi Controller only updates traffic stats about every 30 seconds. -# Setting this to something lower may lead to "zeros" in your data. -# If you're getting zeros now, set this to "1m" -interval = "30s" + disable = false + # InfluxDB does not require auth by default, so the user/password are probably unimportant. + url = "http://127.0.0.1:8086" + user = "unifipoller" + pass = "unifipoller" + # Be sure to create this database. + db = "unifi" + # If your InfluxDB uses a valid SSL cert, set this to true. + verify_ssl = false + # The UniFi Controller only updates traffic stats about every 30 seconds. + # Setting this to something lower may lead to "zeros" in your data. + # If you're getting zeros now, set this to "1m" + interval = "30s" #### INPUTS [unifi] -disable = false + # Setting this to true and providing default credentials allows you to skip + # configuring controllers in this config file. Instead you configure them in + # your prometheus.yml config. Prometheus then sends the controller URL to + # unifi-poller when it performs the scrape. This is useful if you have many, + # or changing controllers. Most people can leave this off. See wiki for more. + dynamic = false + +# The following section contains the default credentials/configuration for any +# dynamic controller (see above section), or the primary controller if you do not +# provide one and dynamic is disabled. In other words, you can just add your +# controller here and delete the following section. Either works. +[unifi.defaults] + name = "https://127.0.0.1:8443" + url = "https://127.0.0.1:8443" + user = "unifipoller" + pass = "unifipoller" + sites = ["all"] + save_ids = false + save_sites = true + verify_ssl = false # You may repeat the following section to poll additional controllers. [[unifi.controller]] -# Friendly name used in dashboards. Uses URL if left empty. -name = "" + # Friendly name used in dashboards. Uses URL if left empty; which is fine. + # Avoid changing this later because it will live forever in your database. + name = "" -url = "https://127.0.0.1:8443" -# Make a read-only user in the UniFi Admin Settings. -user = "influx" -# You may also set env variable UNIFI_PASSWORD instead of putting this in the config. -pass = "4BB9345C-2341-48D7-99F5-E01B583FF77F" + url = "https://127.0.0.1:8443" + # Make a read-only user in the UniFi Admin Settings. + user = "unifipoller" + # You may also set env variable UNIFI_PASSWORD instead of putting this in the config. + pass = "4BB9345C-2341-48D7-99F5-E01B583FF77F" -# If the controller has more than one site, specify which sites to poll here. -# Set this to ["default"] to poll only the first site on the controller. -# A setting of ["all"] will poll all sites; this works if you only have 1 site too. -sites = ["all"] + # If the controller has more than one site, specify which sites to poll here. + # Set this to ["default"] to poll only the first site on the controller. + # A setting of ["all"] will poll all sites; this works if you only have 1 site too. + sites = ["all"] -# Enable collection of Intrusion Detection System Data (InfluxDB only). -# Only useful if IDS or IPS are enabled on one of the sites. -save_ids = false + # Enable collection of Intrusion Detection System Data (InfluxDB only). + # Only useful if IDS or IPS are enabled on one of the sites. + save_ids = false -# Enable collection of site data. This data powers the Network Sites dashboard. -# It's not valuable to everyone and setting this to false will save resources. -save_sites = true + # Enable collection of site data. This data powers the Network Sites dashboard. + # It's not valuable to everyone and setting this to false will save resources. + save_sites = true -# If your UniFi controller has a valid SSL certificate (like lets encrypt), -# you can enable this option to validate it. Otherwise, any SSL certificate is -# valid. If you don't know if you have a valid SSL cert, then you don't have one. -verify_ssl = false + # If your UniFi controller has a valid SSL certificate (like lets encrypt), + # you can enable this option to validate it. Otherwise, any SSL certificate is + # valid. If you don't know if you have a valid SSL cert, then you don't have one. + verify_ssl = false diff --git a/integrations/inputunifi/examples/up.json.example b/integrations/inputunifi/examples/up.json.example index 12ba4a4f..a4e0a401 100644 --- a/integrations/inputunifi/examples/up.json.example +++ b/integrations/inputunifi/examples/up.json.example @@ -14,20 +14,30 @@ "influxdb": { "disable": false, "url": "http://127.0.0.1:8086", - "user": "unifi", - "pass": "unifi", + "user": "unifipoller", + "pass": "unifipoller", "db": "unifi", "verify_ssl": false, "interval": "30s" }, "unifi": { - "disable": false, + "dynamic": false, + "defaults": { + "name": "https://127.0.0.1:8443", + "user": "unifipoller", + "pass": "unifipoller", + "url": "https://127.0.0.1:8443", + "sites": ["all"], + "save_ids": false, + "save_sites": true, + "verify_ssl": false + }, "controllers": [ { "name": "", - "user": "influx", - "pass": "", + "user": "unifipoller", + "pass": "unifipoller", "url": "https://127.0.0.1:8443", "sites": ["all"], "save_ids": false, diff --git a/integrations/inputunifi/examples/up.xml.example b/integrations/inputunifi/examples/up.xml.example index 14b2aa09..094f8347 100644 --- a/integrations/inputunifi/examples/up.xml.example +++ b/integrations/inputunifi/examples/up.xml.example @@ -18,22 +18,33 @@ 30s http://127.0.0.1:8086 - unifi - unifi + unifipoller + unifipoller unifi false - + + + all + unifipoller + unifipoller + https://127.0.0.1:8443 + false + false + true + + all - influx - + unifipoller + unifipoller https://127.0.0.1:8443 false false true + diff --git a/integrations/inputunifi/examples/up.yaml.example b/integrations/inputunifi/examples/up.yaml.example index bb8a4aa1..edf08506 100644 --- a/integrations/inputunifi/examples/up.yaml.example +++ b/integrations/inputunifi/examples/up.yaml.example @@ -18,17 +18,28 @@ influxdb: disable: false interval: "30s" url: "http://127.0.0.1:8086" - user: "unifi" - pass: "unifi" + user: "unifipoller" + pass: "unifipoller" db: "unifi" verify_ssl: false unifi: - disable: false + dynamic: false + defaults: + name: "https://127.0.0.1:8443" + user: "unifipoller" + pass: "unifipoller" + url: "https://127.0.0.1:8443" + sites: + - all + verify_ssl: false + save_ids: false + save_sites: true + controllers: - name: "" - user: "influx" - pass: "" + user: "unifipoller" + pass: "unifipoller" url: "https://127.0.0.1:8443" sites: - all diff --git a/integrations/inputunifi/pkg/inputunifi/collector.go b/integrations/inputunifi/pkg/inputunifi/collector.go index 1539f244..90f1feb4 100644 --- a/integrations/inputunifi/pkg/inputunifi/collector.go +++ b/integrations/inputunifi/pkg/inputunifi/collector.go @@ -2,6 +2,7 @@ package inputunifi import ( "fmt" + "strings" "time" "github.com/davidnewhall/unifi-poller/pkg/poller" @@ -15,47 +16,43 @@ func (u *InputUnifi) isNill(c *Controller) bool { return c.Unifi == nil } -func (u *InputUnifi) dynamicController(url string) (*poller.Metrics, bool, error) { - c := u.Config.Default // copy defaults into new controller +// newDynamicCntrlr creates and saves a controller (with auth cookie) for repeated use. +// This is called when an unconfigured controller is requested. +func (u *InputUnifi) newDynamicCntrlr(url string) (bool, *Controller) { + u.Lock() + defer u.Unlock() + + c := u.dynamic[url] + if c != nil { + // it already exists. + return false, c + } + + ccopy := u.Config.Default // copy defaults into new controller + c = &ccopy + u.dynamic[url] = c c.Name = url c.URL = url - u.Logf("Authenticating to Dynamic UniFi Controller: %s", url) - - if err := u.getUnifi(&c); err != nil { - return nil, false, fmt.Errorf("authenticating to %s: %v", url, err) - } - - metrics := &poller.Metrics{} - ok, err := u.appendController(&c, metrics) - - return metrics, ok, err + return true, c } -func (u *InputUnifi) appendController(c *Controller, metrics *poller.Metrics) (bool, error) { - m, err := u.collectController(c) - if err != nil || m == nil { - return false, err +func (u *InputUnifi) dynamicController(url string) (*poller.Metrics, error) { + if !strings.HasPrefix(url, "http") { + return nil, fmt.Errorf("scrape filter match failed, and filter is not http URL") } - metrics.Sites = append(metrics.Sites, m.Sites...) - metrics.Clients = append(metrics.Clients, m.Clients...) - metrics.IDSList = append(metrics.IDSList, m.IDSList...) + new, c := u.newDynamicCntrlr(url) - if m.Devices == nil { - return true, nil + if new { + u.Logf("Authenticating to Dynamic UniFi Controller: %s", url) + + if err := u.getUnifi(c); err != nil { + return nil, fmt.Errorf("authenticating to %s: %v", url, err) + } } - if metrics.Devices == nil { - metrics.Devices = &unifi.Devices{} - } - - metrics.UAPs = append(metrics.UAPs, m.UAPs...) - metrics.USGs = append(metrics.USGs, m.USGs...) - metrics.USWs = append(metrics.USWs, m.USWs...) - metrics.UDMs = append(metrics.UDMs, m.UDMs...) - - return true, nil + return u.collectController(c) } func (u *InputUnifi) collectController(c *Controller) (*poller.Metrics, error) { @@ -67,11 +64,6 @@ func (u *InputUnifi) collectController(c *Controller) (*poller.Metrics, error) { } } - m, err := u.pollController(c) - if err == nil { - return m, nil - } - return u.pollController(c) } @@ -146,7 +138,7 @@ func (u *InputUnifi) augmentMetrics(c *Controller, metrics *poller.Metrics) *pol metrics.Clients[i].RadioDescription = bssdIDs[metrics.Clients[i].Bssid] + metrics.Clients[i].RadioProto } - if !c.SaveSites { + if !*c.SaveSites { metrics.Sites = nil } diff --git a/integrations/inputunifi/pkg/inputunifi/input.go b/integrations/inputunifi/pkg/inputunifi/input.go index 4a2881aa..a540964d 100644 --- a/integrations/inputunifi/pkg/inputunifi/input.go +++ b/integrations/inputunifi/pkg/inputunifi/input.go @@ -16,13 +16,15 @@ import ( const ( defaultURL = "https://127.0.0.1:8443" defaultUser = "unifipoller" - defaultPass = "unifipollerp4$$w0rd" + defaultPass = "unifipoller" defaultSite = "all" ) // InputUnifi contains the running data. type InputUnifi struct { - Config *Config `json:"unifi" toml:"unifi" xml:"unifi" yaml:"unifi"` + Config *Config `json:"unifi" toml:"unifi" xml:"unifi" yaml:"unifi"` + dynamic map[string]*Controller + sync.Mutex // to lock the map above. poller.Logger } @@ -31,7 +33,7 @@ type InputUnifi struct { type Controller struct { VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` SaveIDS bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` - SaveSites bool `json:"save_sites" toml:"save_sites" xml:"save_sites" yaml:"save_sites"` + SaveSites *bool `json:"save_sites" toml:"save_sites" xml:"save_sites" yaml:"save_sites"` Name string `json:"name" toml:"name" xml:"name,attr" yaml:"name"` User string `json:"user" toml:"user" xml:"user" yaml:"user"` Pass string `json:"pass" toml:"pass" xml:"pass" yaml:"pass"` @@ -44,8 +46,8 @@ type Controller struct { type Config struct { sync.RWMutex // locks the Unifi struct member when re-authing to unifi. Default Controller `json:"defaults" toml:"defaults" xml:"default" yaml:"defaults"` - Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` - Dynamic bool `json:"dynamic" toml:"dynamic" xml:"dynamic" yaml:"dynamic"` + Disable bool `json:"disable" toml:"disable" xml:"disable,attr" yaml:"disable"` + Dynamic bool `json:"dynamic" toml:"dynamic" xml:"dynamic,attr" yaml:"dynamic"` Controllers []*Controller `json:"controllers" toml:"controller" xml:"controller" yaml:"controllers"` } @@ -157,6 +159,11 @@ func (u *InputUnifi) dumpSitesJSON(c *Controller, path, name string, sites unifi } func (u *InputUnifi) setDefaults(c *Controller) { + if c.SaveSites == nil { + t := true + c.SaveSites = &t + } + if c.URL == "" { c.URL = defaultURL } diff --git a/integrations/inputunifi/pkg/inputunifi/interface.go b/integrations/inputunifi/pkg/inputunifi/interface.go index 67f53cce..42745687 100644 --- a/integrations/inputunifi/pkg/inputunifi/interface.go +++ b/integrations/inputunifi/pkg/inputunifi/interface.go @@ -15,15 +15,20 @@ import ( // Satisfies poller.Input interface. func (u *InputUnifi) Initialize(l poller.Logger) error { if u.Config.Disable { - l.Logf("unifi input disabled") + l.Logf("UniFi input plugin disabled!") return nil } - if u.setDefaults(&u.Config.Default); len(u.Config.Controllers) < 1 { + if u.setDefaults(&u.Config.Default); len(u.Config.Controllers) < 1 && !u.Config.Dynamic { new := u.Config.Default // copy defaults. u.Config.Controllers = []*Controller{&new} } + if len(u.Config.Controllers) < 1 { + l.Logf("No controllers configured. Polling dynamic controllers only!") + } + + u.dynamic = make(map[string]*Controller) u.Logger = l for _, c := range u.Config.Controllers { @@ -35,7 +40,7 @@ func (u *InputUnifi) Initialize(l poller.Logger) error { u.LogErrorf("checking sites on %s: %v", c.Name, err) } - u.Logf("Polling UniFi Controller at %s v%s as user %s. Sites: %v", + u.Logf("Configured UniFi Controller at %s v%s as user %s. Sites: %v", c.URL, c.Unifi.ServerVersion, c.User, c.Sites) default: u.LogErrorf("Controller Auth or Connection failed, but continuing to retry! %s: %v", c.Name, err) @@ -52,7 +57,7 @@ func (u *InputUnifi) Metrics() (*poller.Metrics, bool, error) { // MetricsFrom grabs all the measurements from a UniFi controller and returns them. func (u *InputUnifi) MetricsFrom(filter *poller.Filter) (*poller.Metrics, bool, error) { - if u.Config.Disable || filter == nil || filter.Term == "" { + if u.Config.Disable { return nil, false, nil } @@ -62,35 +67,48 @@ func (u *InputUnifi) MetricsFrom(filter *poller.Filter) (*poller.Metrics, bool, // Check if the request is for an existing, configured controller. for _, c := range u.Config.Controllers { - if !strings.EqualFold(c.Name, filter.Term) { + if filter != nil && !strings.EqualFold(c.Name, filter.Term) { continue } - exists, err := u.appendController(c, metrics) + m, err := u.collectController(c) if err != nil { errs = append(errs, err.Error()) } - if exists { - ok = true + if m == nil { + continue } + + ok = true + metrics = poller.AppendMetrics(metrics, m) } if len(errs) > 0 { return metrics, ok, fmt.Errorf(strings.Join(errs, ", ")) } - if u.Config.Dynamic && !ok && strings.HasPrefix(filter.Term, "http") { - // Attempt to a dynamic metrics fetch from an unconfigured controller. - return u.dynamicController(filter.Term) + if ok { + return metrics, true, nil } - return metrics, ok, nil + if filter != nil && !u.Config.Dynamic { + return metrics, false, fmt.Errorf("scrape filter match failed and dynamic lookups disabled") + } + + // Attempt a dynamic metrics fetch from an unconfigured controller. + m, err := u.dynamicController(filter.Term) + + return m, err == nil && m != nil, err } // RawMetrics returns API output from the first configured unifi controller. func (u *InputUnifi) RawMetrics(filter *poller.Filter) ([]byte, error) { - c := u.Config.Controllers[0] // We could pull the controller number from the filter. + if l := len(u.Config.Controllers); filter.Unit >= l { + return nil, fmt.Errorf("control number %d not found, %d controller(s) configured (0 index)", filter.Unit, l) + } + + c := u.Config.Controllers[filter.Unit] if u.isNill(c) { u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) @@ -108,14 +126,14 @@ func (u *InputUnifi) RawMetrics(filter *poller.Filter) ([]byte, error) { return nil, err } - switch filter.Type { + switch filter.Kind { case "d", "device", "devices": return u.dumpSitesJSON(c, unifi.APIDevicePath, "Devices", sites) case "client", "clients", "c": return u.dumpSitesJSON(c, unifi.APIClientPath, "Clients", sites) case "other", "o": - _, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping Path '%s':\n", filter.Term) - return c.Unifi.GetJSON(filter.Term) + _, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping Path '%s':\n", filter.Path) + return c.Unifi.GetJSON(filter.Path) default: return []byte{}, fmt.Errorf("must provide filter: devices, clients, other") } diff --git a/integrations/inputunifi/pkg/poller/dumper.go b/integrations/inputunifi/pkg/poller/dumper.go index 80c7f728..c78edabb 100644 --- a/integrations/inputunifi/pkg/poller/dumper.go +++ b/integrations/inputunifi/pkg/poller/dumper.go @@ -2,19 +2,24 @@ package poller import ( "fmt" + "strconv" "strings" ) -// DumpJSONPayload prints raw json from the UniFi Controller. -// This only works with controller 0 (first one) in the config. +// DumpJSONPayload prints raw json from the UniFi Controller. This is currently +// tied into the -j CLI arg, and is probably not very useful outside that context. func (u *UnifiPoller) DumpJSONPayload() (err error) { u.Config.Quiet = true - split := strings.SplitN(u.Flags.DumpJSON, " ", 2) - filter := &Filter{Type: split[0]} + filter := &Filter{Kind: split[0]} + + if split2 := strings.Split(filter.Kind, ":"); len(split2) > 1 { + filter.Kind = split2[0] + filter.Unit, _ = strconv.Atoi(split2[1]) + } if len(split) > 1 { - filter.Term = split[1] + filter.Path = split[1] } m, err := inputs[0].RawMetrics(filter) diff --git a/integrations/inputunifi/pkg/poller/inputs.go b/integrations/inputunifi/pkg/poller/inputs.go index 8633ca2c..a637239c 100644 --- a/integrations/inputunifi/pkg/poller/inputs.go +++ b/integrations/inputunifi/pkg/poller/inputs.go @@ -28,10 +28,26 @@ type InputPlugin struct { Input } -// Filter is used for raw metrics filters. +// Filter is used for metrics filters. Many fields for lots of expansion. type Filter struct { Type string Term string + Name string + Tags string + Role string + Kind string + Path string + Area int + Item int + Unit int + Sign int64 + Mass int64 + Rate float64 + Cost float64 + Free bool + True bool + Done bool + Stop bool } // NewInput creates a metric input. This should be called by input plugins @@ -107,14 +123,14 @@ func (u *UnifiPoller) Metrics() (*Metrics, bool, error) { return metrics, ok, err } -// MetricsFrom aggregates all the measurements from all configured inputs and returns them. +// MetricsFrom aggregates all the measurements from filtered inputs and returns them. func (u *UnifiPoller) MetricsFrom(filter *Filter) (*Metrics, bool, error) { errs := []string{} metrics := &Metrics{} ok := false for _, input := range inputs { - if input.Name != filter.Type { + if !strings.EqualFold(input.Name, filter.Name) { continue } @@ -128,23 +144,7 @@ func (u *UnifiPoller) MetricsFrom(filter *Filter) (*Metrics, bool, error) { } ok = true - - metrics.Sites = append(metrics.Sites, m.Sites...) - metrics.Clients = append(metrics.Clients, m.Clients...) - metrics.IDSList = append(metrics.IDSList, m.IDSList...) - - if m.Devices == nil { - continue - } - - if metrics.Devices == nil { - metrics.Devices = &unifi.Devices{} - } - - metrics.UAPs = append(metrics.UAPs, m.UAPs...) - metrics.USGs = append(metrics.USGs, m.USGs...) - metrics.USWs = append(metrics.USWs, m.USWs...) - metrics.UDMs = append(metrics.UDMs, m.UDMs...) + metrics = AppendMetrics(metrics, m) } var err error @@ -155,3 +155,25 @@ func (u *UnifiPoller) MetricsFrom(filter *Filter) (*Metrics, bool, error) { return metrics, ok, err } + +// AppendMetrics combined the metrics from two sources. +func AppendMetrics(existing *Metrics, m *Metrics) *Metrics { + existing.Sites = append(existing.Sites, m.Sites...) + existing.Clients = append(existing.Clients, m.Clients...) + existing.IDSList = append(existing.IDSList, m.IDSList...) + + if m.Devices == nil { + return existing + } + + if existing.Devices == nil { + existing.Devices = &unifi.Devices{} + } + + existing.UAPs = append(existing.UAPs, m.UAPs...) + existing.USGs = append(existing.USGs, m.USGs...) + existing.USWs = append(existing.USWs, m.USWs...) + existing.UDMs = append(existing.UDMs, m.UDMs...) + + return existing +} diff --git a/integrations/inputunifi/pkg/promunifi/collector.go b/integrations/inputunifi/pkg/promunifi/collector.go index e4cf1e9c..bbfbda25 100644 --- a/integrations/inputunifi/pkg/promunifi/collector.go +++ b/integrations/inputunifi/pkg/promunifi/collector.go @@ -26,23 +26,18 @@ const ( ) type promUnifi struct { - *Prometheus - Client *uclient - Device *unifiDevice - UAP *uap - USG *usg - USW *usw - Site *site + *Config `json:"prometheus" toml:"prometheus" xml:"prometheus" yaml:"prometheus"` + Client *uclient + Device *unifiDevice + UAP *uap + USG *usg + USW *usw + Site *site // This interface is passed to the Collect() method. The Collect method uses // this interface to retrieve the latest UniFi measurements and export them. Collector poller.Collect } -// Prometheus allows the data to be nested in the config file. -type Prometheus struct { - Config Config `json:"prometheus" toml:"prometheus" xml:"prometheus" yaml:"prometheus"` -} - // Config is the input (config file) data used to initialize this output plugin. type Config struct { // If non-empty, each of the collected metrics is prefixed by the @@ -66,7 +61,7 @@ type metric struct { // Report accumulates counters that are printed to a log line. type Report struct { - Config + *Config Total int // Total count of metrics recorded. Errors int // Total count of errors recording metrics. Zeros int // Total count of metrics equal to zero. @@ -78,17 +73,18 @@ type Report struct { wg sync.WaitGroup } +// target is used for targeted (sometimes dynamic) metrics scrapes. type target struct { *poller.Filter - *promUnifi + u *promUnifi } func init() { - u := &promUnifi{Prometheus: &Prometheus{}} + u := &promUnifi{Config: &Config{}} poller.NewOutput(&poller.Output{ Name: "prometheus", - Config: u.Prometheus, + Config: u, Method: u.Run, }) } @@ -96,51 +92,55 @@ func init() { // Run creates the collectors and starts the web server up. // Should be run in a Go routine. Returns nil if not configured. func (u *promUnifi) Run(c poller.Collect) error { - if u.Config.Disable { + if u.Disable { return nil } - u.Config.Namespace = strings.Trim(strings.Replace(u.Config.Namespace, "-", "_", -1), "_") - if u.Config.Namespace == "" { - u.Config.Namespace = strings.Replace(poller.AppName, "-", "", -1) + u.Namespace = strings.Trim(strings.Replace(u.Namespace, "-", "_", -1), "_") + if u.Namespace == "" { + u.Namespace = strings.Replace(poller.AppName, "-", "", -1) } - if u.Config.HTTPListen == "" { - u.Config.HTTPListen = defaultHTTPListen + if u.HTTPListen == "" { + u.HTTPListen = defaultHTTPListen } + // Later can pass this in from poller by adding a method to the interface. + u.Collector = c + u.Client = descClient(u.Namespace + "_client_") + u.Device = descDevice(u.Namespace + "_device_") // stats for all device types. + u.UAP = descUAP(u.Namespace + "_device_") + u.USG = descUSG(u.Namespace + "_device_") + u.USW = descUSW(u.Namespace + "_device_") + u.Site = descSite(u.Namespace + "_site_") mux := http.NewServeMux() - prometheus.MustRegister(version.NewCollector(u.Config.Namespace)) - prometheus.MustRegister(&promUnifi{ - Collector: c, - Client: descClient(u.Config.Namespace + "_client_"), - Device: descDevice(u.Config.Namespace + "_device_"), // stats for all device types. - UAP: descUAP(u.Config.Namespace + "_device_"), - USG: descUSG(u.Config.Namespace + "_device_"), - USW: descUSW(u.Config.Namespace + "_device_"), - Site: descSite(u.Config.Namespace + "_site_"), - }) - c.Logf("Exporting Measurements for Prometheus at https://%s/metrics, namespace: %s", - u.Config.HTTPListen, u.Config.Namespace) - mux.Handle("/metrics", promhttp.HandlerFor( - prometheus.DefaultGatherer, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}, + prometheus.MustRegister(version.NewCollector(u.Namespace)) + prometheus.MustRegister(u) + c.Logf("Prometheus exported at https://%s/ - namespace: %s", u.HTTPListen, u.Namespace) + mux.Handle("/metrics", promhttp.HandlerFor(prometheus.DefaultGatherer, + promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}, )) mux.HandleFunc("/scrape", u.ScrapeHandler) + mux.HandleFunc("/", u.DefaultHandler) - return http.ListenAndServe(u.Config.HTTPListen, mux) + return http.ListenAndServe(u.HTTPListen, mux) } // ScrapeHandler allows prometheus to scrape a single source, instead of all sources. func (u *promUnifi) ScrapeHandler(w http.ResponseWriter, r *http.Request) { - t := &target{promUnifi: u, Filter: &poller.Filter{}} - if t.Filter.Type = r.URL.Query().Get("input"); t.Filter.Type == "" { + t := &target{u: u, Filter: &poller.Filter{}} + if t.Name = r.URL.Query().Get("input"); t.Name == "" { + u.Collector.LogErrorf("input parameter missing on scrape from %v", r.RemoteAddr) http.Error(w, `'input' parameter must be specified (try "unifi")`, 400) + return } - if t.Filter.Term = r.URL.Query().Get("target"); t.Filter.Term == "" { + if t.Term = r.URL.Query().Get("target"); t.Term == "" { + u.Collector.LogErrorf("target parameter missing on scrape from %v", r.RemoteAddr) http.Error(w, "'target' parameter must be specified, configured name, or unconfigured url", 400) + return } @@ -152,14 +152,15 @@ func (u *promUnifi) ScrapeHandler(w http.ResponseWriter, r *http.Request) { ).ServeHTTP(w, r) } +func (u *promUnifi) DefaultHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + _, _ = w.Write([]byte(poller.AppName + "\n")) +} + // Describe satisfies the prometheus Collector. This returns all of the // metric descriptions that this packages produces. func (t *target) Describe(ch chan<- *prometheus.Desc) { - t.promUnifi.Describe(ch) -} - -func (t *target) Collect(ch chan<- prometheus.Metric) { - t.promUnifi.collect(ch, t.Filter) + t.u.Describe(ch) } // Describe satisfies the prometheus Collector. This returns all of the @@ -178,6 +179,11 @@ func (u *promUnifi) Describe(ch chan<- *prometheus.Desc) { } } +// Collect satisfies the prometheus Collector. This runs for a single controller poll. +func (t *target) Collect(ch chan<- prometheus.Metric) { + t.u.collect(ch, t.Filter) +} + // Collect satisfies the prometheus Collector. This runs the input method to get // the current metrics (from another package) then exports them for prometheus. func (u *promUnifi) Collect(ch chan<- prometheus.Metric) { @@ -187,27 +193,31 @@ func (u *promUnifi) Collect(ch chan<- prometheus.Metric) { func (u *promUnifi) collect(ch chan<- prometheus.Metric, filter *poller.Filter) { var err error - ok := false - - r := &Report{Config: u.Config, ch: make(chan []*metric, buffer), Start: time.Now()} + r := &Report{ + Config: u.Config, + ch: make(chan []*metric, buffer), + Start: time.Now()} defer r.close() + ok := false + if filter == nil { r.Metrics, ok, err = u.Collector.Metrics() } else { r.Metrics, ok, err = u.Collector.MetricsFrom(filter) } + r.Fetch = time.Since(r.Start) + if err != nil { - r.error(ch, prometheus.NewInvalidDesc(fmt.Errorf("metric fetch failed")), err) + r.error(ch, prometheus.NewInvalidDesc(err), fmt.Errorf("metric fetch failed")) + u.Collector.LogErrorf("metric fetch failed: %v", err) if !ok { return } } - r.Fetch = time.Since(r.Start) - if r.Metrics.Devices == nil { r.Metrics.Devices = &unifi.Devices{} } diff --git a/integrations/inputunifi/pkg/promunifi/report.go b/integrations/inputunifi/pkg/promunifi/report.go index 9b6df74c..3eb66638 100644 --- a/integrations/inputunifi/pkg/promunifi/report.go +++ b/integrations/inputunifi/pkg/promunifi/report.go @@ -67,7 +67,7 @@ func (r *Report) export(m *metric, v float64) prometheus.Metric { func (r *Report) error(ch chan<- prometheus.Metric, d *prometheus.Desc, v interface{}) { r.Errors++ - if r.Config.ReportErrors { + if r.ReportErrors { ch <- prometheus.NewInvalidMetric(d, fmt.Errorf("error: %v", v)) } } From 382436c5b5c5e6e9e4da05ea2af30714af5f2713 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Fri, 20 Dec 2019 03:19:00 -0800 Subject: [PATCH 30/41] rename some things --- .../inputunifi/examples/up.conf.example | 5 +- .../inputunifi/examples/up.json.example | 4 +- .../inputunifi/examples/up.xml.example | 4 +- .../inputunifi/examples/up.yaml.example | 6 ++- .../inputunifi/pkg/inputunifi/collector.go | 18 +++---- .../inputunifi/pkg/inputunifi/input.go | 20 +++---- .../inputunifi/pkg/inputunifi/interface.go | 52 +++++++++---------- .../inputunifi/pkg/promunifi/collector.go | 14 +++-- 8 files changed, 65 insertions(+), 58 deletions(-) diff --git a/integrations/inputunifi/examples/up.conf.example b/integrations/inputunifi/examples/up.conf.example index 942c53cd..bb0f09dd 100644 --- a/integrations/inputunifi/examples/up.conf.example +++ b/integrations/inputunifi/examples/up.conf.example @@ -55,7 +55,7 @@ # provide one and dynamic is disabled. In other words, you can just add your # controller here and delete the following section. Either works. [unifi.defaults] - name = "https://127.0.0.1:8443" + role = "https://127.0.0.1:8443" url = "https://127.0.0.1:8443" user = "unifipoller" pass = "unifipoller" @@ -68,7 +68,8 @@ [[unifi.controller]] # Friendly name used in dashboards. Uses URL if left empty; which is fine. # Avoid changing this later because it will live forever in your database. - name = "" + # Multiple controllers may share a role. This allows grouping during scrapes. + role = "" url = "https://127.0.0.1:8443" # Make a read-only user in the UniFi Admin Settings. diff --git a/integrations/inputunifi/examples/up.json.example b/integrations/inputunifi/examples/up.json.example index a4e0a401..d523eee2 100644 --- a/integrations/inputunifi/examples/up.json.example +++ b/integrations/inputunifi/examples/up.json.example @@ -24,7 +24,7 @@ "unifi": { "dynamic": false, "defaults": { - "name": "https://127.0.0.1:8443", + "role": "https://127.0.0.1:8443", "user": "unifipoller", "pass": "unifipoller", "url": "https://127.0.0.1:8443", @@ -35,7 +35,7 @@ }, "controllers": [ { - "name": "", + "role": "", "user": "unifipoller", "pass": "unifipoller", "url": "https://127.0.0.1:8443", diff --git a/integrations/inputunifi/examples/up.xml.example b/integrations/inputunifi/examples/up.xml.example index 094f8347..d7caacc7 100644 --- a/integrations/inputunifi/examples/up.xml.example +++ b/integrations/inputunifi/examples/up.xml.example @@ -25,7 +25,7 @@ - + all unifipoller unifipoller @@ -36,7 +36,7 @@ - + all unifipoller unifipoller diff --git a/integrations/inputunifi/examples/up.yaml.example b/integrations/inputunifi/examples/up.yaml.example index edf08506..a675fd8e 100644 --- a/integrations/inputunifi/examples/up.yaml.example +++ b/integrations/inputunifi/examples/up.yaml.example @@ -26,7 +26,7 @@ influxdb: unifi: dynamic: false defaults: - name: "https://127.0.0.1:8443" + role: "https://127.0.0.1:8443" user: "unifipoller" pass: "unifipoller" url: "https://127.0.0.1:8443" @@ -36,8 +36,10 @@ unifi: save_ids: false save_sites: true + controllers: - - name: "" + # Repeat the following stanza to poll more controllers. + - role: "" user: "unifipoller" pass: "unifipoller" url: "https://127.0.0.1:8443" diff --git a/integrations/inputunifi/pkg/inputunifi/collector.go b/integrations/inputunifi/pkg/inputunifi/collector.go index 90f1feb4..85941fee 100644 --- a/integrations/inputunifi/pkg/inputunifi/collector.go +++ b/integrations/inputunifi/pkg/inputunifi/collector.go @@ -10,8 +10,8 @@ import ( ) func (u *InputUnifi) isNill(c *Controller) bool { - u.Config.RLock() - defer u.Config.RUnlock() + u.RLock() + defer u.RUnlock() return c.Unifi == nil } @@ -28,10 +28,10 @@ func (u *InputUnifi) newDynamicCntrlr(url string) (bool, *Controller) { return false, c } - ccopy := u.Config.Default // copy defaults into new controller + ccopy := u.Default // copy defaults into new controller c = &ccopy u.dynamic[url] = c - c.Name = url + c.Role = url c.URL = url return true, c @@ -60,7 +60,7 @@ func (u *InputUnifi) collectController(c *Controller) (*poller.Metrics, error) { u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) if err := u.getUnifi(c); err != nil { - return nil, fmt.Errorf("re-authenticating to %s: %v", c.Name, err) + return nil, fmt.Errorf("re-authenticating to %s: %v", c.Role, err) } } @@ -70,8 +70,8 @@ func (u *InputUnifi) collectController(c *Controller) (*poller.Metrics, error) { func (u *InputUnifi) pollController(c *Controller) (*poller.Metrics, error) { var err error - u.Config.RLock() - defer u.Config.RUnlock() + u.RLock() + defer u.RUnlock() m := &poller.Metrics{TS: time.Now()} // At this point, it's the Current Check. @@ -149,8 +149,8 @@ func (u *InputUnifi) augmentMetrics(c *Controller, metrics *poller.Metrics) *pol // Omits requested but unconfigured sites. Grabs the full list from the // controller and returns the sites provided in the config file. func (u *InputUnifi) getFilteredSites(c *Controller) (unifi.Sites, error) { - u.Config.RLock() - defer u.Config.RUnlock() + u.RLock() + defer u.RUnlock() sites, err := c.Unifi.GetSites() if err != nil { diff --git a/integrations/inputunifi/pkg/inputunifi/input.go b/integrations/inputunifi/pkg/inputunifi/input.go index a540964d..45d3e4e9 100644 --- a/integrations/inputunifi/pkg/inputunifi/input.go +++ b/integrations/inputunifi/pkg/inputunifi/input.go @@ -22,7 +22,7 @@ const ( // InputUnifi contains the running data. type InputUnifi struct { - Config *Config `json:"unifi" toml:"unifi" xml:"unifi" yaml:"unifi"` + *Config `json:"unifi" toml:"unifi" xml:"unifi" yaml:"unifi"` dynamic map[string]*Controller sync.Mutex // to lock the map above. poller.Logger @@ -34,7 +34,7 @@ type Controller struct { VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` SaveIDS bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` SaveSites *bool `json:"save_sites" toml:"save_sites" xml:"save_sites" yaml:"save_sites"` - Name string `json:"name" toml:"name" xml:"name,attr" yaml:"name"` + Role string `json:"role" toml:"role" xml:"role,attr" yaml:"role"` User string `json:"user" toml:"user" xml:"user" yaml:"user"` Pass string `json:"pass" toml:"pass" xml:"pass" yaml:"pass"` URL string `json:"url" toml:"url" xml:"url" yaml:"url"` @@ -65,8 +65,8 @@ func init() { func (u *InputUnifi) getUnifi(c *Controller) error { var err error - u.Config.Lock() - defer u.Config.Unlock() + u.Lock() + defer u.Unlock() if c.Unifi != nil { c.Unifi.CloseIdleConnections() @@ -94,8 +94,8 @@ func (u *InputUnifi) getUnifi(c *Controller) error { // checkSites makes sure the list of provided sites exists on the controller. // This only runs once during initialization. func (u *InputUnifi) checkSites(c *Controller) error { - u.Config.RLock() - defer u.Config.RUnlock() + u.RLock() + defer u.RUnlock() if len(c.Sites) < 1 || c.Sites[0] == "" { c.Sites = []string{"all"} @@ -113,7 +113,7 @@ func (u *InputUnifi) checkSites(c *Controller) error { msg = append(msg, site.Name+" ("+site.Desc+")") } - u.Logf("Found %d site(s) on controller %s: %v", len(msg), c.Name, strings.Join(msg, ", ")) + u.Logf("Found %d site(s) on controller %s: %v", len(msg), c.Role, strings.Join(msg, ", ")) if StringInSlice("all", c.Sites) { c.Sites = []string{"all"} @@ -130,7 +130,7 @@ FIRST: continue FIRST } } - u.LogErrorf("Configured site not found on controller %s: %v", c.Name, s) + u.LogErrorf("Configured site not found on controller %s: %v", c.Role, s) } if c.Sites = keep; len(keep) < 1 { @@ -168,8 +168,8 @@ func (u *InputUnifi) setDefaults(c *Controller) { c.URL = defaultURL } - if c.Name == "" { - c.Name = c.URL + if c.Role == "" { + c.Role = c.URL } if c.Pass == "" { diff --git a/integrations/inputunifi/pkg/inputunifi/interface.go b/integrations/inputunifi/pkg/inputunifi/interface.go index 42745687..daf21ae7 100644 --- a/integrations/inputunifi/pkg/inputunifi/interface.go +++ b/integrations/inputunifi/pkg/inputunifi/interface.go @@ -14,36 +14,36 @@ import ( // Initialize gets called one time when starting up. // Satisfies poller.Input interface. func (u *InputUnifi) Initialize(l poller.Logger) error { - if u.Config.Disable { + if u.Disable { l.Logf("UniFi input plugin disabled!") return nil } - if u.setDefaults(&u.Config.Default); len(u.Config.Controllers) < 1 && !u.Config.Dynamic { - new := u.Config.Default // copy defaults. - u.Config.Controllers = []*Controller{&new} + if u.setDefaults(&u.Default); len(u.Controllers) < 1 && !u.Dynamic { + new := u.Default // copy defaults. + u.Controllers = []*Controller{&new} } - if len(u.Config.Controllers) < 1 { + if len(u.Controllers) < 1 { l.Logf("No controllers configured. Polling dynamic controllers only!") } u.dynamic = make(map[string]*Controller) u.Logger = l - for _, c := range u.Config.Controllers { + for _, c := range u.Controllers { u.setDefaults(c) switch err := u.getUnifi(c); err { case nil: if err := u.checkSites(c); err != nil { - u.LogErrorf("checking sites on %s: %v", c.Name, err) + u.LogErrorf("checking sites on %s: %v", c.Role, err) } u.Logf("Configured UniFi Controller at %s v%s as user %s. Sites: %v", c.URL, c.Unifi.ServerVersion, c.User, c.Sites) default: - u.LogErrorf("Controller Auth or Connection failed, but continuing to retry! %s: %v", c.Name, err) + u.LogErrorf("Controller Auth or Connection failed, but continuing to retry! %s: %v", c.Role, err) } } @@ -57,7 +57,7 @@ func (u *InputUnifi) Metrics() (*poller.Metrics, bool, error) { // MetricsFrom grabs all the measurements from a UniFi controller and returns them. func (u *InputUnifi) MetricsFrom(filter *poller.Filter) (*poller.Metrics, bool, error) { - if u.Config.Disable { + if u.Disable { return nil, false, nil } @@ -65,9 +65,20 @@ func (u *InputUnifi) MetricsFrom(filter *poller.Filter) (*poller.Metrics, bool, metrics := &poller.Metrics{} ok := false + if filter != nil && filter.Path != "" { + if !u.Dynamic { + return metrics, false, fmt.Errorf("filter path requested but dynamic lookups disabled") + } + + // Attempt a dynamic metrics fetch from an unconfigured controller. + m, err := u.dynamicController(filter.Path) + + return m, err == nil && m != nil, err + } + // Check if the request is for an existing, configured controller. - for _, c := range u.Config.Controllers { - if filter != nil && !strings.EqualFold(c.Name, filter.Term) { + for _, c := range u.Controllers { + if filter != nil && !strings.EqualFold(c.Role, filter.Role) { continue } @@ -88,32 +99,21 @@ func (u *InputUnifi) MetricsFrom(filter *poller.Filter) (*poller.Metrics, bool, return metrics, ok, fmt.Errorf(strings.Join(errs, ", ")) } - if ok { - return metrics, true, nil - } - - if filter != nil && !u.Config.Dynamic { - return metrics, false, fmt.Errorf("scrape filter match failed and dynamic lookups disabled") - } - - // Attempt a dynamic metrics fetch from an unconfigured controller. - m, err := u.dynamicController(filter.Term) - - return m, err == nil && m != nil, err + return metrics, ok, nil } // RawMetrics returns API output from the first configured unifi controller. func (u *InputUnifi) RawMetrics(filter *poller.Filter) ([]byte, error) { - if l := len(u.Config.Controllers); filter.Unit >= l { + if l := len(u.Controllers); filter.Unit >= l { return nil, fmt.Errorf("control number %d not found, %d controller(s) configured (0 index)", filter.Unit, l) } - c := u.Config.Controllers[filter.Unit] + c := u.Controllers[filter.Unit] if u.isNill(c) { u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) if err := u.getUnifi(c); err != nil { - return nil, fmt.Errorf("re-authenticating to %s: %v", c.Name, err) + return nil, fmt.Errorf("re-authenticating to %s: %v", c.Role, err) } } diff --git a/integrations/inputunifi/pkg/promunifi/collector.go b/integrations/inputunifi/pkg/promunifi/collector.go index bbfbda25..353bed01 100644 --- a/integrations/inputunifi/pkg/promunifi/collector.go +++ b/integrations/inputunifi/pkg/promunifi/collector.go @@ -129,17 +129,21 @@ func (u *promUnifi) Run(c poller.Collect) error { // ScrapeHandler allows prometheus to scrape a single source, instead of all sources. func (u *promUnifi) ScrapeHandler(w http.ResponseWriter, r *http.Request) { - t := &target{u: u, Filter: &poller.Filter{}} - if t.Name = r.URL.Query().Get("input"); t.Name == "" { + t := &target{u: u, Filter: &poller.Filter{ + Name: r.URL.Query().Get("input"), // "unifi" + Path: r.URL.Query().Get("path"), // url: "https://127.0.0.1:8443" + Role: r.URL.Query().Get("role"), // configured role in up.conf. + }} + if t.Name == "" { u.Collector.LogErrorf("input parameter missing on scrape from %v", r.RemoteAddr) http.Error(w, `'input' parameter must be specified (try "unifi")`, 400) return } - if t.Term = r.URL.Query().Get("target"); t.Term == "" { - u.Collector.LogErrorf("target parameter missing on scrape from %v", r.RemoteAddr) - http.Error(w, "'target' parameter must be specified, configured name, or unconfigured url", 400) + if t.Role == "" && t.Path == "" { + u.Collector.LogErrorf("role and path parameters missing on scrape from %v", r.RemoteAddr) + http.Error(w, "'role' OR 'path' parameter must be specified: configured role OR unconfigured url", 400) return } From f31b0ec7de19b9778316c890a2f8190d13664efa Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Fri, 20 Dec 2019 03:22:41 -0800 Subject: [PATCH 31/41] ocd --- integrations/inputunifi/examples/up.conf.example | 13 ++++++------- integrations/inputunifi/examples/up.json.example | 12 ++++++------ integrations/inputunifi/examples/up.yaml.example | 4 ++-- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/integrations/inputunifi/examples/up.conf.example b/integrations/inputunifi/examples/up.conf.example index bb0f09dd..06418489 100644 --- a/integrations/inputunifi/examples/up.conf.example +++ b/integrations/inputunifi/examples/up.conf.example @@ -55,12 +55,12 @@ # provide one and dynamic is disabled. In other words, you can just add your # controller here and delete the following section. Either works. [unifi.defaults] - role = "https://127.0.0.1:8443" - url = "https://127.0.0.1:8443" - user = "unifipoller" - pass = "unifipoller" - sites = ["all"] - save_ids = false + role = "https://127.0.0.1:8443" + url = "https://127.0.0.1:8443" + user = "unifipoller" + pass = "unifipoller" + sites = ["all"] + save_ids = false save_sites = true verify_ssl = false @@ -74,7 +74,6 @@ url = "https://127.0.0.1:8443" # Make a read-only user in the UniFi Admin Settings. user = "unifipoller" - # You may also set env variable UNIFI_PASSWORD instead of putting this in the config. pass = "4BB9345C-2341-48D7-99F5-E01B583FF77F" # If the controller has more than one site, specify which sites to poll here. diff --git a/integrations/inputunifi/examples/up.json.example b/integrations/inputunifi/examples/up.json.example index d523eee2..6e8c8c66 100644 --- a/integrations/inputunifi/examples/up.json.example +++ b/integrations/inputunifi/examples/up.json.example @@ -24,12 +24,12 @@ "unifi": { "dynamic": false, "defaults": { - "role": "https://127.0.0.1:8443", - "user": "unifipoller", - "pass": "unifipoller", - "url": "https://127.0.0.1:8443", - "sites": ["all"], - "save_ids": false, + "role": "https://127.0.0.1:8443", + "user": "unifipoller", + "pass": "unifipoller", + "url": "https://127.0.0.1:8443", + "sites": ["all"], + "save_ids": false, "save_sites": true, "verify_ssl": false }, diff --git a/integrations/inputunifi/examples/up.yaml.example b/integrations/inputunifi/examples/up.yaml.example index a675fd8e..33cc5088 100644 --- a/integrations/inputunifi/examples/up.yaml.example +++ b/integrations/inputunifi/examples/up.yaml.example @@ -29,11 +29,11 @@ unifi: role: "https://127.0.0.1:8443" user: "unifipoller" pass: "unifipoller" - url: "https://127.0.0.1:8443" + url: "https://127.0.0.1:8443" sites: - all verify_ssl: false - save_ids: false + save_ids: false save_sites: true From 21f24d13a64dd5618248168d57d948b8f8e55567 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Tue, 24 Dec 2019 23:43:49 -0800 Subject: [PATCH 32/41] Make a new docker image --- integrations/inputunifi/Gopkg.lock | 24 +++++++++++-------- integrations/inputunifi/Gopkg.toml | 5 ++++ .../inputunifi/examples/up.conf.example | 2 +- .../inputunifi/pkg/influxunifi/clients.go | 1 + .../inputunifi/pkg/influxunifi/ids.go | 2 ++ .../inputunifi/pkg/influxunifi/site.go | 1 + .../inputunifi/pkg/influxunifi/uap.go | 3 +++ .../inputunifi/pkg/influxunifi/udm.go | 4 ++++ .../inputunifi/pkg/influxunifi/usg.go | 3 +++ .../inputunifi/pkg/influxunifi/usw.go | 2 ++ integrations/inputunifi/pkg/poller/config.go | 6 ++--- .../inputunifi/pkg/promunifi/clients.go | 4 ++-- .../inputunifi/pkg/promunifi/collector.go | 1 + integrations/inputunifi/pkg/promunifi/site.go | 4 ++-- integrations/inputunifi/pkg/promunifi/uap.go | 16 ++++++------- integrations/inputunifi/pkg/promunifi/udm.go | 4 ++-- integrations/inputunifi/pkg/promunifi/usg.go | 10 ++++---- integrations/inputunifi/pkg/promunifi/usw.go | 8 +++---- integrations/inputunifi/plugins/mysql/main.go | 6 ++--- 19 files changed, 66 insertions(+), 40 deletions(-) diff --git a/integrations/inputunifi/Gopkg.lock b/integrations/inputunifi/Gopkg.lock index 7cd32b0e..ef55eb8f 100644 --- a/integrations/inputunifi/Gopkg.lock +++ b/integrations/inputunifi/Gopkg.lock @@ -58,12 +58,12 @@ version = "v1.1.0" [[projects]] - branch = "master" digest = "1:982be0b5396e16a663697899ce69cc7b1e71ddcae4153af157578d4dc9bc3f88" name = "github.com/prometheus/client_model" packages = ["go"] pruneopts = "UT" revision = "d1d2010b5beead3fa1c5f271a5cf626e40b3ad6e" + version = "v0.1.0" [[projects]] digest = "1:7dec9ab2db741c280b89b142b08ea142824152c5f40fb1f90c35b6ef7a694456" @@ -100,27 +100,30 @@ [[projects]] branch = "master" - digest = "1:07f0cb66f649e51f9ef23441f8dfc34a73e7d9bf0832417abcbad578f1d8c8d6" + digest = "1:0ca5ac8aedc2fd9cb63c90acbd71c0cba8ddb61dfcca58b96cf41550689bf56d" name = "golang.org/x/sys" packages = ["windows"] pruneopts = "UT" - revision = "af0d71d358abe0ba3594483a5d519f429dbae3e9" + revision = "c709ea063b76879dc9915358f55d4d77c16ab6d5" [[projects]] - digest = "1:54742bef5cb29f706614c9edcfdeb29fb5992f26090f26ca955f575dddf54f9e" + digest = "1:b914912b8cb13beebf85b913006d8692bbf20d8620122966a7a911bbfc6e0104" name = "golift.io/cnfg" - packages = ["."] + packages = [ + ".", + "cnfgfile", + ] pruneopts = "UT" - revision = "961061d377655468e9da4a9333e71b9b77402470" - version = "v0.0.1" + revision = "7d859f27a93d43faceb7101d02baeb8e69f16d4a" + version = "v0.0.4" [[projects]] - digest = "1:2883cea734f2766f41ff9c9d4aefccccc53e3d44f5c8b08893b9c218cf666722" + branch = "master" + digest = "1:13b9e827c1f958ffe31554d57940495010eef89928697f6c6beae78a750ee637" name = "golift.io/unifi" packages = ["."] pruneopts = "UT" - revision = "a607fe940c6a563c6994f2c945394b19d2183b1c" - version = "v4.1.6" + revision = "4d78625be4cd448ea2db35c63a40b451e7d0df99" [[projects]] digest = "1:b75b3deb2bce8bc079e16bb2aecfe01eb80098f5650f9e93e5643ca8b7b73737" @@ -140,6 +143,7 @@ "github.com/prometheus/common/version", "github.com/spf13/pflag", "golift.io/cnfg", + "golift.io/cnfg/cnfgfile", "golift.io/unifi", ] solver-name = "gps-cdcl" diff --git a/integrations/inputunifi/Gopkg.toml b/integrations/inputunifi/Gopkg.toml index 332001cc..044de96a 100644 --- a/integrations/inputunifi/Gopkg.toml +++ b/integrations/inputunifi/Gopkg.toml @@ -7,3 +7,8 @@ [prune] go-tests = true unused-packages = true + +# TODO: remove this!!! +[[constraint]] + name = "golift.io/unifi" + branch = "master" diff --git a/integrations/inputunifi/examples/up.conf.example b/integrations/inputunifi/examples/up.conf.example index 06418489..db3b14ad 100644 --- a/integrations/inputunifi/examples/up.conf.example +++ b/integrations/inputunifi/examples/up.conf.example @@ -28,7 +28,7 @@ [influxdb] disable = false # InfluxDB does not require auth by default, so the user/password are probably unimportant. - url = "http://127.0.0.1:8086" + url = "http://127.0.0.1:8086" user = "unifipoller" pass = "unifipoller" # Be sure to create this database. diff --git a/integrations/inputunifi/pkg/influxunifi/clients.go b/integrations/inputunifi/pkg/influxunifi/clients.go index b4c346dc..9e15ec19 100644 --- a/integrations/inputunifi/pkg/influxunifi/clients.go +++ b/integrations/inputunifi/pkg/influxunifi/clients.go @@ -10,6 +10,7 @@ func (u *InfluxUnifi) batchClient(r report, s *unifi.Client) { tags := map[string]string{ "mac": s.Mac, "site_name": s.SiteName, + "source": s.SourceName, "ap_name": s.ApName, "gw_name": s.GwName, "sw_name": s.SwName, diff --git a/integrations/inputunifi/pkg/influxunifi/ids.go b/integrations/inputunifi/pkg/influxunifi/ids.go index ad0b855a..fbc6e0dd 100644 --- a/integrations/inputunifi/pkg/influxunifi/ids.go +++ b/integrations/inputunifi/pkg/influxunifi/ids.go @@ -8,6 +8,8 @@ import ( // These points can be passed directly to influx. func (u *InfluxUnifi) batchIDS(r report, i *unifi.IDS) { tags := map[string]string{ + "site_name": i.SiteName, + "source": i.SourceName, "in_iface": i.InIface, "event_type": i.EventType, "proto": i.Proto, diff --git a/integrations/inputunifi/pkg/influxunifi/site.go b/integrations/inputunifi/pkg/influxunifi/site.go index 30e2ce37..49715e5a 100644 --- a/integrations/inputunifi/pkg/influxunifi/site.go +++ b/integrations/inputunifi/pkg/influxunifi/site.go @@ -11,6 +11,7 @@ func (u *InfluxUnifi) batchSite(r report, s *unifi.Site) { tags := map[string]string{ "name": s.Name, "site_name": s.SiteName, + "source": s.SourceName, "desc": s.Desc, "status": h.Status, "subsystem": h.Subsystem, diff --git a/integrations/inputunifi/pkg/influxunifi/uap.go b/integrations/inputunifi/pkg/influxunifi/uap.go index f47a11fc..3966cb4a 100644 --- a/integrations/inputunifi/pkg/influxunifi/uap.go +++ b/integrations/inputunifi/pkg/influxunifi/uap.go @@ -14,6 +14,7 @@ func (u *InfluxUnifi) batchUAP(r report, s *unifi.UAP) { tags := map[string]string{ "mac": s.Mac, "site_name": s.SiteName, + "source": s.SourceName, "name": s.Name, "version": s.Version, "model": s.Model, @@ -85,6 +86,7 @@ func (u *InfluxUnifi) processVAPTable(r report, t map[string]string, vt unifi.Va tags := map[string]string{ "device_name": t["name"], "site_name": t["site_name"], + "source": t["source"], "ap_mac": s.ApMac, "bssid": s.Bssid, "id": s.ID, @@ -148,6 +150,7 @@ func (u *InfluxUnifi) processRadTable(r report, t map[string]string, rt unifi.Ra tags := map[string]string{ "device_name": t["name"], "site_name": t["site_name"], + "source": t["source"], "channel": p.Channel.Txt, "radio": p.Radio, } diff --git a/integrations/inputunifi/pkg/influxunifi/udm.go b/integrations/inputunifi/pkg/influxunifi/udm.go index 20cea055..599bbbbe 100644 --- a/integrations/inputunifi/pkg/influxunifi/udm.go +++ b/integrations/inputunifi/pkg/influxunifi/udm.go @@ -40,6 +40,7 @@ func (u *InfluxUnifi) batchUDM(r report, s *unifi.UDM) { } tags := map[string]string{ + "source": s.SourceName, "mac": s.Mac, "site_name": s.SiteName, "name": s.Name, @@ -52,6 +53,7 @@ func (u *InfluxUnifi) batchUDM(r report, s *unifi.UDM) { u.batchUSGstat(s.SpeedtestStatus, s.Stat.Gw, s.Uplink), u.batchSysStats(s.SysStats, s.SystemStats), map[string]interface{}{ + "source": s.SourceName, "ip": s.IP, "bytes": s.Bytes.Val, "last_seen": s.LastSeen.Val, @@ -76,6 +78,7 @@ func (u *InfluxUnifi) batchUDM(r report, s *unifi.UDM) { tags = map[string]string{ "mac": s.Mac, "site_name": s.SiteName, + "source": s.SourceName, "name": s.Name, "version": s.Version, "model": s.Model, @@ -105,6 +108,7 @@ func (u *InfluxUnifi) batchUDM(r report, s *unifi.UDM) { tags = map[string]string{ "mac": s.Mac, "site_name": s.SiteName, + "source": s.SourceName, "name": s.Name, "version": s.Version, "model": s.Model, diff --git a/integrations/inputunifi/pkg/influxunifi/usg.go b/integrations/inputunifi/pkg/influxunifi/usg.go index 36c9cdbd..96306e88 100644 --- a/integrations/inputunifi/pkg/influxunifi/usg.go +++ b/integrations/inputunifi/pkg/influxunifi/usg.go @@ -14,6 +14,7 @@ func (u *InfluxUnifi) batchUSG(r report, s *unifi.USG) { tags := map[string]string{ "mac": s.Mac, "site_name": s.SiteName, + "source": s.SourceName, "name": s.Name, "version": s.Version, "model": s.Model, @@ -76,6 +77,7 @@ func (u *InfluxUnifi) batchUSGwans(r report, tags map[string]string, wans ...uni tags := map[string]string{ "device_name": tags["name"], "site_name": tags["site_name"], + "source": tags["source"], "ip": wan.IP, "purpose": wan.Name, "mac": wan.Mac, @@ -115,6 +117,7 @@ func (u *InfluxUnifi) batchNetTable(r report, tags map[string]string, nt unifi.N tags := map[string]string{ "device_name": tags["name"], "site_name": tags["site_name"], + "source": tags["source"], "up": p.Up.Txt, "enabled": p.Enabled.Txt, "ip": p.IP, diff --git a/integrations/inputunifi/pkg/influxunifi/usw.go b/integrations/inputunifi/pkg/influxunifi/usw.go index 0a91a506..db717c1a 100644 --- a/integrations/inputunifi/pkg/influxunifi/usw.go +++ b/integrations/inputunifi/pkg/influxunifi/usw.go @@ -14,6 +14,7 @@ func (u *InfluxUnifi) batchUSW(r report, s *unifi.USW) { tags := map[string]string{ "mac": s.Mac, "site_name": s.SiteName, + "source": s.SourceName, "name": s.Name, "version": s.Version, "model": s.Model, @@ -71,6 +72,7 @@ func (u *InfluxUnifi) batchPortTable(r report, t map[string]string, pt []unifi.P tags := map[string]string{ "site_name": t["site_name"], "device_name": t["name"], + "source": t["source"], "name": p.Name, "poe_mode": p.PoeMode, "port_poe": p.PortPoe.Txt, diff --git a/integrations/inputunifi/pkg/poller/config.go b/integrations/inputunifi/pkg/poller/config.go index 006dc1fb..de1edf07 100644 --- a/integrations/inputunifi/pkg/poller/config.go +++ b/integrations/inputunifi/pkg/poller/config.go @@ -17,6 +17,7 @@ import ( "github.com/spf13/pflag" "golift.io/cnfg" + "golift.io/cnfg/cnfgfile" "golift.io/unifi" ) @@ -103,14 +104,13 @@ func (u *UnifiPoller) ParseConfigs() error { // parseInterface parses the config file and environment variables into the provided interface. func (u *UnifiPoller) parseInterface(i interface{}) error { - cnfg.ENVTag = "xml" // xml tag is better formatted for slices. // Parse config file into provided interface. - if err := cnfg.ParseFile(i, u.Flags.ConfigFile); err != nil { + if err := cnfgfile.Unmarshal(i, u.Flags.ConfigFile); err != nil { return err } // Parse environment variables into provided interface. - _, err := cnfg.ParseENV(i, ENVConfigPrefix) + _, err := cnfg.UnmarshalENV(i, ENVConfigPrefix) return err } diff --git a/integrations/inputunifi/pkg/promunifi/clients.go b/integrations/inputunifi/pkg/promunifi/clients.go index 0ffaf54c..5e85825a 100644 --- a/integrations/inputunifi/pkg/promunifi/clients.go +++ b/integrations/inputunifi/pkg/promunifi/clients.go @@ -42,7 +42,7 @@ type uclient struct { func descClient(ns string) *uclient { labels := []string{"name", "mac", "site_name", "gw_name", "sw_name", "vlan", - "ip", "oui", "network", "sw_port", "ap_name", "wired"} + "ip", "oui", "network", "sw_port", "ap_name", "wired", "source"} labelW := append([]string{"radio_name", "radio", "radio_proto", "channel", "essid", "bssid", "radio_desc"}, labels...) return &uclient{ @@ -85,7 +85,7 @@ func descClient(ns string) *uclient { func (u *promUnifi) exportClient(r report, c *unifi.Client) { labels := []string{c.Name, c.Mac, c.SiteName, c.GwName, c.SwName, c.Vlan.Txt, - c.IP, c.Oui, c.Network, c.SwPort.Txt, c.ApName, ""} + c.IP, c.Oui, c.Network, c.SwPort.Txt, c.ApName, "", c.SourceName} labelW := append([]string{c.RadioName, c.Radio, c.RadioProto, c.Channel.Txt, c.Essid, c.Bssid, c.RadioDescription}, labels...) diff --git a/integrations/inputunifi/pkg/promunifi/collector.go b/integrations/inputunifi/pkg/promunifi/collector.go index 353bed01..61020625 100644 --- a/integrations/inputunifi/pkg/promunifi/collector.go +++ b/integrations/inputunifi/pkg/promunifi/collector.go @@ -134,6 +134,7 @@ func (u *promUnifi) ScrapeHandler(w http.ResponseWriter, r *http.Request) { Path: r.URL.Query().Get("path"), // url: "https://127.0.0.1:8443" Role: r.URL.Query().Get("role"), // configured role in up.conf. }} + if t.Name == "" { u.Collector.LogErrorf("input parameter missing on scrape from %v", r.RemoteAddr) http.Error(w, `'input' parameter must be specified (try "unifi")`, 400) diff --git a/integrations/inputunifi/pkg/promunifi/site.go b/integrations/inputunifi/pkg/promunifi/site.go index 79c57b41..cdae55c8 100644 --- a/integrations/inputunifi/pkg/promunifi/site.go +++ b/integrations/inputunifi/pkg/promunifi/site.go @@ -34,7 +34,7 @@ type site struct { } func descSite(ns string) *site { - labels := []string{"subsystem", "status", "site_name"} + labels := []string{"subsystem", "status", "site_name", "source"} nd := prometheus.NewDesc return &site{ @@ -68,7 +68,7 @@ func descSite(ns string) *site { func (u *promUnifi) exportSite(r report, s *unifi.Site) { for _, h := range s.Health { - switch labels := []string{h.Subsystem, h.Status, s.SiteName}; labels[0] { + switch labels := []string{h.Subsystem, h.Status, s.SiteName, s.SourceName}; labels[0] { case "www": r.send([]*metric{ {u.Site.TxBytesR, gauge, h.TxBytesR, labels}, diff --git a/integrations/inputunifi/pkg/promunifi/uap.go b/integrations/inputunifi/pkg/promunifi/uap.go index 0f3a05aa..71352062 100644 --- a/integrations/inputunifi/pkg/promunifi/uap.go +++ b/integrations/inputunifi/pkg/promunifi/uap.go @@ -80,9 +80,9 @@ type uap struct { } func descUAP(ns string) *uap { - labelA := []string{"stat", "site_name", "name"} // stat + labels[1:] - labelV := []string{"vap_name", "bssid", "radio", "radio_name", "essid", "usage", "site_name", "name"} - labelR := []string{"radio_name", "radio", "site_name", "name"} + labelA := []string{"stat", "site_name", "name", "source"} // stat + labels[1:] + labelV := []string{"vap_name", "bssid", "radio", "radio_name", "essid", "usage", "site_name", "name", "source"} + labelR := []string{"radio_name", "radio", "site_name", "name", "source"} nd := prometheus.NewDesc return &uap{ @@ -165,7 +165,7 @@ func (u *promUnifi) exportUAP(r report, d *unifi.UAP) { return } - labels := []string{d.Type, d.SiteName, d.Name} + labels := []string{d.Type, d.SiteName, d.Name, d.SourceName} infoLabels := []string{d.Version, d.Model, d.Serial, d.Mac, d.IP, d.ID, d.Bytes.Txt, d.Uptime.Txt} u.exportUAPstats(r, labels, d.Stat.Ap, d.BytesD, d.TxBytesD, d.RxBytesD, d.BytesR) u.exportVAPtable(r, labels, d.VapTable) @@ -185,8 +185,8 @@ func (u *promUnifi) exportUAPstats(r report, labels []string, ap *unifi.Ap, byte return } - labelU := []string{"user", labels[1], labels[2]} - labelG := []string{"guest", labels[1], labels[2]} + labelU := []string{"user", labels[1], labels[2], labels[3]} + labelG := []string{"guest", labels[1], labels[2], labels[3]} r.send([]*metric{ // ap only stuff. {u.Device.BytesD, counter, bytes[0], labels}, // not sure if these 3 Ds are counters or gauges. @@ -234,7 +234,7 @@ func (u *promUnifi) exportVAPtable(r report, labels []string, vt unifi.VapTable) continue } - labelV := []string{v.Name, v.Bssid, v.Radio, v.RadioName, v.Essid, v.Usage, labels[1], labels[2]} + labelV := []string{v.Name, v.Bssid, v.Radio, v.RadioName, v.Essid, v.Usage, labels[1], labels[2], labels[3]} r.send([]*metric{ {u.UAP.VAPCcq, gauge, float64(v.Ccq) / 1000.0, labelV}, {u.UAP.VAPMacFilterRejections, counter, v.MacFilterRejections, labelV}, @@ -281,7 +281,7 @@ func (u *promUnifi) exportVAPtable(r report, labels []string, vt unifi.VapTable) func (u *promUnifi) exportRADtable(r report, labels []string, rt unifi.RadioTable, rts unifi.RadioTableStats) { // radio table for _, p := range rt { - labelR := []string{p.Name, p.Radio, labels[1], labels[2]} + labelR := []string{p.Name, p.Radio, labels[1], labels[2], labels[3]} labelRUser := append(labelR, "user") labelRGuest := append(labelR, "guest") diff --git a/integrations/inputunifi/pkg/promunifi/udm.go b/integrations/inputunifi/pkg/promunifi/udm.go index 34d7e66d..b74e1b9e 100644 --- a/integrations/inputunifi/pkg/promunifi/udm.go +++ b/integrations/inputunifi/pkg/promunifi/udm.go @@ -31,7 +31,7 @@ type unifiDevice struct { } func descDevice(ns string) *unifiDevice { - labels := []string{"type", "site_name", "name"} + labels := []string{"type", "site_name", "name", "source"} infoLabels := []string{"version", "model", "serial", "mac", "ip", "id", "bytes", "uptime"} return &unifiDevice{ @@ -65,7 +65,7 @@ func (u *promUnifi) exportUDM(r report, d *unifi.UDM) { return } - labels := []string{d.Type, d.SiteName, d.Name} + labels := []string{d.Type, d.SiteName, d.Name, d.SourceName} infoLabels := []string{d.Version, d.Model, d.Serial, d.Mac, d.IP, d.ID, d.Bytes.Txt, d.Uptime.Txt} // Shared data (all devices do this). u.exportBYTstats(r, labels, d.TxBytes, d.RxBytes) diff --git a/integrations/inputunifi/pkg/promunifi/usg.go b/integrations/inputunifi/pkg/promunifi/usg.go index 08ee3781..83891cc6 100644 --- a/integrations/inputunifi/pkg/promunifi/usg.go +++ b/integrations/inputunifi/pkg/promunifi/usg.go @@ -36,7 +36,7 @@ type usg struct { } func descUSG(ns string) *usg { - labels := []string{"port", "site_name", "name"} + labels := []string{"port", "site_name", "name", "source"} return &usg{ WanRxPackets: prometheus.NewDesc(ns+"wan_receive_packets_total", "WAN Receive Packets Total", labels, nil), @@ -74,7 +74,7 @@ func (u *promUnifi) exportUSG(r report, d *unifi.USG) { return } - labels := []string{d.Type, d.SiteName, d.Name} + labels := []string{d.Type, d.SiteName, d.Name, d.SourceName} infoLabels := []string{d.Version, d.Model, d.Serial, d.Mac, d.IP, d.ID, d.Bytes.Txt, d.Uptime.Txt} // Gateway System Data. @@ -95,8 +95,8 @@ func (u *promUnifi) exportUSGstats(r report, labels []string, gw *unifi.Gw, st u return } - labelLan := []string{"lan", labels[1], labels[2]} - labelWan := []string{"all", labels[1], labels[2]} + labelLan := []string{"lan", labels[1], labels[2], labels[3]} + labelWan := []string{"all", labels[1], labels[2], labels[3]} r.send([]*metric{ {u.USG.LanRxPackets, counter, gw.LanRxPackets, labelLan}, @@ -121,7 +121,7 @@ func (u *promUnifi) exportWANPorts(r report, labels []string, wans ...unifi.Wan) continue // only record UP interfaces. } - labelWan := []string{wan.Name, labels[1], labels[2]} + labelWan := []string{wan.Name, labels[1], labels[2], labels[3]} r.send([]*metric{ {u.USG.WanRxPackets, counter, wan.RxPackets, labelWan}, diff --git a/integrations/inputunifi/pkg/promunifi/usw.go b/integrations/inputunifi/pkg/promunifi/usw.go index d700583f..1c736d05 100644 --- a/integrations/inputunifi/pkg/promunifi/usw.go +++ b/integrations/inputunifi/pkg/promunifi/usw.go @@ -47,8 +47,8 @@ type usw struct { func descUSW(ns string) *usw { pns := ns + "port_" - labelS := []string{"site_name", "name"} - labelP := []string{"port_id", "port_num", "port_name", "port_mac", "port_ip", "site_name", "name"} + labelS := []string{"site_name", "name", "source"} + labelP := []string{"port_id", "port_num", "port_name", "port_mac", "port_ip", "site_name", "name", "source"} nd := prometheus.NewDesc return &usw{ @@ -97,7 +97,7 @@ func (u *promUnifi) exportUSW(r report, d *unifi.USW) { return } - labels := []string{d.Type, d.SiteName, d.Name} + labels := []string{d.Type, d.SiteName, d.Name, d.SourceName} infoLabels := []string{d.Version, d.Model, d.Serial, d.Mac, d.IP, d.ID, d.Bytes.Txt, d.Uptime.Txt} u.exportUSWstats(r, labels, d.Stat.Sw) @@ -161,7 +161,7 @@ func (u *promUnifi) exportPRTtable(r report, labels []string, pt []unifi.Port) { } // Copy labels, and add four new ones. - labelP := []string{labels[2] + " Port " + p.PortIdx.Txt, p.PortIdx.Txt, p.Name, p.Mac, p.IP, labels[1], labels[2]} + labelP := []string{labels[2] + " Port " + p.PortIdx.Txt, p.PortIdx.Txt, p.Name, p.Mac, p.IP, labels[1], labels[2], labels[3]} if p.PoeEnable.Val && p.PortPoe.Val { r.send([]*metric{ diff --git a/integrations/inputunifi/plugins/mysql/main.go b/integrations/inputunifi/plugins/mysql/main.go index 361165a0..4776e0d7 100644 --- a/integrations/inputunifi/plugins/mysql/main.go +++ b/integrations/inputunifi/plugins/mysql/main.go @@ -21,12 +21,12 @@ type mysqlConfig struct { // Pointers are ignored during ENV variable unmarshal, avoid pointers to your config. // Only capital (exported) members are unmarshaled when passed into poller.NewOutput(). -type application struct { +type plugin struct { Config mysqlConfig `json:"mysql" toml:"mysql" xml:"mysql" yaml:"mysql"` } func init() { - u := &application{Config: mysqlConfig{}} + u := &plugin{Config: mysqlConfig{}} poller.NewOutput(&poller.Output{ Name: "mysql", @@ -39,7 +39,7 @@ func main() { fmt.Println("this is a unifi-poller plugin; not an application") } -func (a *application) Run(c poller.Collect) error { +func (a *plugin) Run(c poller.Collect) error { c.Logf("mysql plugin is not finished") return nil } From 24f1675d830706c53f5a2992b48726d270247327 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Tue, 24 Dec 2019 23:56:00 -0800 Subject: [PATCH 33/41] fix test --- integrations/inputunifi/pkg/promunifi/usw.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integrations/inputunifi/pkg/promunifi/usw.go b/integrations/inputunifi/pkg/promunifi/usw.go index 1c736d05..66d819d8 100644 --- a/integrations/inputunifi/pkg/promunifi/usw.go +++ b/integrations/inputunifi/pkg/promunifi/usw.go @@ -161,7 +161,8 @@ func (u *promUnifi) exportPRTtable(r report, labels []string, pt []unifi.Port) { } // Copy labels, and add four new ones. - labelP := []string{labels[2] + " Port " + p.PortIdx.Txt, p.PortIdx.Txt, p.Name, p.Mac, p.IP, labels[1], labels[2], labels[3]} + labelP := []string{labels[2] + " Port " + p.PortIdx.Txt, p.PortIdx.Txt, + p.Name, p.Mac, p.IP, labels[1], labels[2], labels[3]} if p.PoeEnable.Val && p.PortPoe.Val { r.send([]*metric{ From c3f81607af8a671ed379f6db3fc30ca9da168103 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Wed, 25 Dec 2019 00:19:40 -0800 Subject: [PATCH 34/41] export buffer, lock cnfg --- integrations/inputunifi/Gopkg.toml | 4 ++++ integrations/inputunifi/pkg/promunifi/collector.go | 11 +++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/integrations/inputunifi/Gopkg.toml b/integrations/inputunifi/Gopkg.toml index 044de96a..e3e5352f 100644 --- a/integrations/inputunifi/Gopkg.toml +++ b/integrations/inputunifi/Gopkg.toml @@ -12,3 +12,7 @@ [[constraint]] name = "golift.io/unifi" branch = "master" + +[[constraint]] + name = "golift.io/cnfg" + branch = "master" diff --git a/integrations/inputunifi/pkg/promunifi/collector.go b/integrations/inputunifi/pkg/promunifi/collector.go index 61020625..49be737a 100644 --- a/integrations/inputunifi/pkg/promunifi/collector.go +++ b/integrations/inputunifi/pkg/promunifi/collector.go @@ -18,7 +18,7 @@ import ( const ( // channel buffer, fits at least one batch. - buffer = 50 + defaultBuffer = 50 defaultHTTPListen = "0.0.0.0:9130" // simply fewer letters. counter = prometheus.CounterValue @@ -50,6 +50,9 @@ type Config struct { // will be collected at all. ReportErrors bool `json:"report_errors" toml:"report_errors" xml:"report_errors" yaml:"report_errors"` Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` + // Buffer is a channel buffer. + // Default is probably 50. Seems fast there; try 1 to see if CPU usage goes down? + Buffer int `json:"buffer" toml:"buffer" xml:"buffer" yaml:"buffer"` } type metric struct { @@ -105,6 +108,10 @@ func (u *promUnifi) Run(c poller.Collect) error { u.HTTPListen = defaultHTTPListen } + if u.Buffer == 0 { + u.Buffer = defaultBuffer + } + // Later can pass this in from poller by adding a method to the interface. u.Collector = c u.Client = descClient(u.Namespace + "_client_") @@ -200,7 +207,7 @@ func (u *promUnifi) collect(ch chan<- prometheus.Metric, filter *poller.Filter) r := &Report{ Config: u.Config, - ch: make(chan []*metric, buffer), + ch: make(chan []*metric, u.Config.Buffer), Start: time.Now()} defer r.close() From 37777a6266ebf927c798fa0efa8c80b95685f207 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Wed, 25 Dec 2019 01:02:36 -0800 Subject: [PATCH 35/41] deps --- integrations/inputunifi/Gopkg.lock | 43 +++++++++++++++---- .../inputunifi/examples/up.xml.example | 2 +- integrations/inputunifi/pkg/poller/config.go | 4 ++ 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/integrations/inputunifi/Gopkg.lock b/integrations/inputunifi/Gopkg.lock index ef55eb8f..65c3e1b6 100644 --- a/integrations/inputunifi/Gopkg.lock +++ b/integrations/inputunifi/Gopkg.lock @@ -1,6 +1,14 @@ # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. +[[projects]] + digest = "1:9f3b30d9f8e0d7040f729b82dcbc8f0dead820a133b3147ce355fc451f32d761" + name = "github.com/BurntSushi/toml" + packages = ["."] + pruneopts = "UT" + revision = "3012a1dbe2e4bd1391d42b32f0577cb7bbc7f005" + version = "v0.3.1" + [[projects]] digest = "1:d6afaeed1502aa28e80a4ed0981d570ad91b2579193404256ce672ed0a609e0d" name = "github.com/beorn7/perks" @@ -9,6 +17,14 @@ revision = "37c8de3658fcb183f997c4e13e8337516ab753e6" version = "v1.0.1" +[[projects]] + digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec" + name = "github.com/davecgh/go-spew" + packages = ["spew"] + pruneopts = "UT" + revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" + version = "v1.1.1" + [[projects]] digest = "1:573ca21d3669500ff845bdebee890eb7fc7f0f50c59f2132f2a0c6b03d85086a" name = "github.com/golang/protobuf" @@ -38,12 +54,12 @@ version = "v1.0.1" [[projects]] - digest = "1:6eea828983c70075ca297bb915ffbcfd3e34c5a50affd94428a65df955c0ff9c" - name = "github.com/pelletier/go-toml" - packages = ["."] + digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe" + name = "github.com/pmezard/go-difflib" + packages = ["difflib"] pruneopts = "UT" - revision = "903d9455db9ff1d7ac1ab199062eca7266dd11a3" - version = "v1.6.0" + revision = "792786c7400a136282c1664665ae0a8db921c6c2" + version = "v1.0.0" [[projects]] digest = "1:eb04f69c8991e52eff33c428bd729e04208bf03235be88e4df0d88497c6861b9" @@ -98,6 +114,14 @@ revision = "2e9d26c8c37aae03e3f9d4e90b7116f5accb7cab" version = "v1.0.5" +[[projects]] + digest = "1:8548c309c65a85933a625be5e7d52b6ac927ca30c56869fae58123b8a77a75e1" + name = "github.com/stretchr/testify" + packages = ["assert"] + pruneopts = "UT" + revision = "221dbe5ed46703ee255b1da0dec05086f5035f62" + version = "v1.4.0" + [[projects]] branch = "master" digest = "1:0ca5ac8aedc2fd9cb63c90acbd71c0cba8ddb61dfcca58b96cf41550689bf56d" @@ -107,15 +131,15 @@ revision = "c709ea063b76879dc9915358f55d4d77c16ab6d5" [[projects]] - digest = "1:b914912b8cb13beebf85b913006d8692bbf20d8620122966a7a911bbfc6e0104" + branch = "master" + digest = "1:89262a29b1c07290b1de8245b27ec0b8f24b1ca1205b1cd95e4a6b911caf11d7" name = "golift.io/cnfg" packages = [ ".", "cnfgfile", ] pruneopts = "UT" - revision = "7d859f27a93d43faceb7101d02baeb8e69f16d4a" - version = "v0.0.4" + revision = "bec8d8f51fc47101055ada9d952348522682ca16" [[projects]] branch = "master" @@ -137,14 +161,17 @@ analyzer-name = "dep" analyzer-version = 1 input-imports = [ + "github.com/BurntSushi/toml", "github.com/influxdata/influxdb1-client/v2", "github.com/prometheus/client_golang/prometheus", "github.com/prometheus/client_golang/prometheus/promhttp", "github.com/prometheus/common/version", "github.com/spf13/pflag", + "github.com/stretchr/testify/assert", "golift.io/cnfg", "golift.io/cnfg/cnfgfile", "golift.io/unifi", + "gopkg.in/yaml.v2", ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/integrations/inputunifi/examples/up.xml.example b/integrations/inputunifi/examples/up.xml.example index d7caacc7..d1a1be3b 100644 --- a/integrations/inputunifi/examples/up.xml.example +++ b/integrations/inputunifi/examples/up.xml.example @@ -8,7 +8,7 @@ and are lists of strings and may be repeated. --> - + 0.0.0.0:9130 diff --git a/integrations/inputunifi/pkg/poller/config.go b/integrations/inputunifi/pkg/poller/config.go index de1edf07..7d706060 100644 --- a/integrations/inputunifi/pkg/poller/config.go +++ b/integrations/inputunifi/pkg/poller/config.go @@ -69,6 +69,10 @@ func (u *UnifiPoller) LoadPlugins() error { for _, p := range u.Plugins { name := strings.TrimSuffix(p, ".so") + ".so" + if name == ".so" { + continue // Just ignore it. uhg. + } + if _, err := os.Stat(name); os.IsNotExist(err) { name = path.Join(DefaultObjPath, name) } From 5184a25df544f53d6d1cf732b53c4607639b7f4d Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Wed, 25 Dec 2019 13:52:30 -0800 Subject: [PATCH 36/41] oops --- integrations/inputunifi/pkg/promunifi/clients.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integrations/inputunifi/pkg/promunifi/clients.go b/integrations/inputunifi/pkg/promunifi/clients.go index 5e85825a..b24b2d62 100644 --- a/integrations/inputunifi/pkg/promunifi/clients.go +++ b/integrations/inputunifi/pkg/promunifi/clients.go @@ -42,7 +42,7 @@ type uclient struct { func descClient(ns string) *uclient { labels := []string{"name", "mac", "site_name", "gw_name", "sw_name", "vlan", - "ip", "oui", "network", "sw_port", "ap_name", "wired", "source"} + "ip", "oui", "network", "sw_port", "ap_name", "source", "wired"} labelW := append([]string{"radio_name", "radio", "radio_proto", "channel", "essid", "bssid", "radio_desc"}, labels...) return &uclient{ @@ -85,7 +85,7 @@ func descClient(ns string) *uclient { func (u *promUnifi) exportClient(r report, c *unifi.Client) { labels := []string{c.Name, c.Mac, c.SiteName, c.GwName, c.SwName, c.Vlan.Txt, - c.IP, c.Oui, c.Network, c.SwPort.Txt, c.ApName, "", c.SourceName} + c.IP, c.Oui, c.Network, c.SwPort.Txt, c.ApName, c.SourceName, ""} labelW := append([]string{c.RadioName, c.Radio, c.RadioProto, c.Channel.Txt, c.Essid, c.Bssid, c.RadioDescription}, labels...) From ac7795c8000edb1c03eac569b2bc3dc8c14a41b2 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Thu, 26 Dec 2019 20:32:16 -0800 Subject: [PATCH 37/41] Add go.mod --- integrations/inputunifi/Gopkg.lock | 177 ----------------------------- integrations/inputunifi/Gopkg.toml | 18 --- integrations/inputunifi/Makefile | 9 +- integrations/inputunifi/go.mod | 15 +++ integrations/inputunifi/go.sum | 107 +++++++++++++++++ 5 files changed, 127 insertions(+), 199 deletions(-) delete mode 100644 integrations/inputunifi/Gopkg.lock delete mode 100644 integrations/inputunifi/Gopkg.toml create mode 100644 integrations/inputunifi/go.mod create mode 100644 integrations/inputunifi/go.sum diff --git a/integrations/inputunifi/Gopkg.lock b/integrations/inputunifi/Gopkg.lock deleted file mode 100644 index 65c3e1b6..00000000 --- a/integrations/inputunifi/Gopkg.lock +++ /dev/null @@ -1,177 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[[projects]] - digest = "1:9f3b30d9f8e0d7040f729b82dcbc8f0dead820a133b3147ce355fc451f32d761" - name = "github.com/BurntSushi/toml" - packages = ["."] - pruneopts = "UT" - revision = "3012a1dbe2e4bd1391d42b32f0577cb7bbc7f005" - version = "v0.3.1" - -[[projects]] - digest = "1:d6afaeed1502aa28e80a4ed0981d570ad91b2579193404256ce672ed0a609e0d" - name = "github.com/beorn7/perks" - packages = ["quantile"] - pruneopts = "UT" - revision = "37c8de3658fcb183f997c4e13e8337516ab753e6" - version = "v1.0.1" - -[[projects]] - digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec" - name = "github.com/davecgh/go-spew" - packages = ["spew"] - pruneopts = "UT" - revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" - version = "v1.1.1" - -[[projects]] - digest = "1:573ca21d3669500ff845bdebee890eb7fc7f0f50c59f2132f2a0c6b03d85086a" - name = "github.com/golang/protobuf" - packages = ["proto"] - pruneopts = "UT" - revision = "6c65a5562fc06764971b7c5d05c76c75e84bdbf7" - version = "v1.3.2" - -[[projects]] - branch = "master" - digest = "1:00e5ad58045d6d2a6c9e65d1809ff2594bc396e911712ae892a93976fdece115" - name = "github.com/influxdata/influxdb1-client" - packages = [ - "models", - "pkg/escape", - "v2", - ] - pruneopts = "UT" - revision = "8bf82d3c094dc06be9da8e5bf9d3589b6ea032ae" - -[[projects]] - digest = "1:ff5ebae34cfbf047d505ee150de27e60570e8c394b3b8fdbb720ff6ac71985fc" - name = "github.com/matttproud/golang_protobuf_extensions" - packages = ["pbutil"] - pruneopts = "UT" - revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" - version = "v1.0.1" - -[[projects]] - digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe" - name = "github.com/pmezard/go-difflib" - packages = ["difflib"] - pruneopts = "UT" - revision = "792786c7400a136282c1664665ae0a8db921c6c2" - version = "v1.0.0" - -[[projects]] - digest = "1:eb04f69c8991e52eff33c428bd729e04208bf03235be88e4df0d88497c6861b9" - name = "github.com/prometheus/client_golang" - packages = [ - "prometheus", - "prometheus/internal", - "prometheus/promhttp", - ] - pruneopts = "UT" - revision = "170205fb58decfd011f1550d4cfb737230d7ae4f" - version = "v1.1.0" - -[[projects]] - digest = "1:982be0b5396e16a663697899ce69cc7b1e71ddcae4153af157578d4dc9bc3f88" - name = "github.com/prometheus/client_model" - packages = ["go"] - pruneopts = "UT" - revision = "d1d2010b5beead3fa1c5f271a5cf626e40b3ad6e" - version = "v0.1.0" - -[[projects]] - digest = "1:7dec9ab2db741c280b89b142b08ea142824152c5f40fb1f90c35b6ef7a694456" - name = "github.com/prometheus/common" - packages = [ - "expfmt", - "internal/bitbucket.org/ww/goautoneg", - "model", - "version", - ] - pruneopts = "UT" - revision = "287d3e634a1e550c9e463dd7e5a75a422c614505" - version = "v0.7.0" - -[[projects]] - digest = "1:ec0ff4bd619a67065e34d6477711ed0117e335f99059a4c508e0fe21cfe7b304" - name = "github.com/prometheus/procfs" - packages = [ - ".", - "internal/fs", - "internal/util", - ] - pruneopts = "UT" - revision = "6d489fc7f1d9cd890a250f3ea3431b1744b9623f" - version = "v0.0.8" - -[[projects]] - digest = "1:524b71991fc7d9246cc7dc2d9e0886ccb97648091c63e30eef619e6862c955dd" - name = "github.com/spf13/pflag" - packages = ["."] - pruneopts = "UT" - revision = "2e9d26c8c37aae03e3f9d4e90b7116f5accb7cab" - version = "v1.0.5" - -[[projects]] - digest = "1:8548c309c65a85933a625be5e7d52b6ac927ca30c56869fae58123b8a77a75e1" - name = "github.com/stretchr/testify" - packages = ["assert"] - pruneopts = "UT" - revision = "221dbe5ed46703ee255b1da0dec05086f5035f62" - version = "v1.4.0" - -[[projects]] - branch = "master" - digest = "1:0ca5ac8aedc2fd9cb63c90acbd71c0cba8ddb61dfcca58b96cf41550689bf56d" - name = "golang.org/x/sys" - packages = ["windows"] - pruneopts = "UT" - revision = "c709ea063b76879dc9915358f55d4d77c16ab6d5" - -[[projects]] - branch = "master" - digest = "1:89262a29b1c07290b1de8245b27ec0b8f24b1ca1205b1cd95e4a6b911caf11d7" - name = "golift.io/cnfg" - packages = [ - ".", - "cnfgfile", - ] - pruneopts = "UT" - revision = "bec8d8f51fc47101055ada9d952348522682ca16" - -[[projects]] - branch = "master" - digest = "1:13b9e827c1f958ffe31554d57940495010eef89928697f6c6beae78a750ee637" - name = "golift.io/unifi" - packages = ["."] - pruneopts = "UT" - revision = "4d78625be4cd448ea2db35c63a40b451e7d0df99" - -[[projects]] - digest = "1:b75b3deb2bce8bc079e16bb2aecfe01eb80098f5650f9e93e5643ca8b7b73737" - name = "gopkg.in/yaml.v2" - packages = ["."] - pruneopts = "UT" - revision = "1f64d6156d11335c3f22d9330b0ad14fc1e789ce" - version = "v2.2.7" - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 - input-imports = [ - "github.com/BurntSushi/toml", - "github.com/influxdata/influxdb1-client/v2", - "github.com/prometheus/client_golang/prometheus", - "github.com/prometheus/client_golang/prometheus/promhttp", - "github.com/prometheus/common/version", - "github.com/spf13/pflag", - "github.com/stretchr/testify/assert", - "golift.io/cnfg", - "golift.io/cnfg/cnfgfile", - "golift.io/unifi", - "gopkg.in/yaml.v2", - ] - solver-name = "gps-cdcl" - solver-version = 1 diff --git a/integrations/inputunifi/Gopkg.toml b/integrations/inputunifi/Gopkg.toml deleted file mode 100644 index e3e5352f..00000000 --- a/integrations/inputunifi/Gopkg.toml +++ /dev/null @@ -1,18 +0,0 @@ -# dep configuration file -# -# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html -# for detailed Gopkg.toml documentation. -# - -[prune] - go-tests = true - unused-packages = true - -# TODO: remove this!!! -[[constraint]] - name = "golift.io/unifi" - branch = "master" - -[[constraint]] - name = "golift.io/cnfg" - branch = "master" diff --git a/integrations/inputunifi/Makefile b/integrations/inputunifi/Makefile index 46fe2600..f47e8c89 100644 --- a/integrations/inputunifi/Makefile +++ b/integrations/inputunifi/Makefile @@ -283,12 +283,13 @@ lint: # This is safe; recommended even. dep: vendor -vendor: Gopkg.* - dep ensure --vendor-only +vendor: go.mod go.sum + go mod vendor # Don't run this unless you're ready to debug untested vendored dependencies. -deps: - dep ensure --update +deps: update vendor +update: + go get -u -d # Homebrew stuff. macOS only. diff --git a/integrations/inputunifi/go.mod b/integrations/inputunifi/go.mod new file mode 100644 index 00000000..ffc8b47f --- /dev/null +++ b/integrations/inputunifi/go.mod @@ -0,0 +1,15 @@ +module github.com/davidnewhall/unifi-poller + +go 1.13 + +require ( + github.com/BurntSushi/toml v0.3.1 // indirect + github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d + github.com/prometheus/client_golang v1.3.0 + github.com/prometheus/common v0.7.0 + github.com/spf13/pflag v1.0.5 + golang.org/x/sys v0.0.0-20191224085550-c709ea063b76 // indirect + golift.io/cnfg v0.0.5 + golift.io/unifi v4.1.7-0.20191215214413-4d78625be4cd+incompatible + gopkg.in/yaml.v2 v2.2.7 // indirect +) diff --git a/integrations/inputunifi/go.sum b/integrations/inputunifi/go.sum new file mode 100644 index 00000000..b2fbc010 --- /dev/null +++ b/integrations/inputunifi/go.sum @@ -0,0 +1,107 @@ +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d h1:/WZQPMZNsjZ7IlCpsLGdQBINg5bxKQ1K1sh6awxLtkA= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= +github.com/prometheus/client_golang v1.3.0 h1:miYCvYqFXtl/J9FIy8eNpBfYthAEFg+Ys0XyUVEcDsc= +github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f h1:68K/z8GLUxV76xGSqwTWw2gyk/jwn79LUL43rES2g8o= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191224085550-c709ea063b76 h1:Dho5nD6R3PcW2SH1or8vS0dszDaXRxIw55lBX7XiE5g= +golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golift.io/cnfg v0.0.0-20191225081851-bec8d8f51fc4 h1:lCMyrYNLbSBxOiGDnn0ZKWmafS0OEBvaiu7qUJZFrwA= +golift.io/cnfg v0.0.0-20191225081851-bec8d8f51fc4/go.mod h1:ScFDIJg/rJGHbRaed/i7g1lBhywEjB0JiP2uZr3xC3A= +golift.io/cnfg v0.0.5 h1:HnMU8Z9C/igKvir1dqaHx5BPuNGZrp99FCtdJyP2Z4I= +golift.io/cnfg v0.0.5/go.mod h1:ScFDIJg/rJGHbRaed/i7g1lBhywEjB0JiP2uZr3xC3A= +golift.io/unifi v0.0.0-20191215214413-4d78625be4cd h1:d4wPilxOdtcVqwv7WhAXsrSp9/gvMc0ff4xYBTxcqSs= +golift.io/unifi v0.0.0-20191215214413-4d78625be4cd/go.mod h1:Zjw57ZAzTzCMw784pE8CdCFgkYSzVZzmJ++WUttbjto= +golift.io/unifi v4.1.6+incompatible h1:Yhb/+obX2vT9i6PElGislSuQ1WUtOf+l+sRjVxlY6nM= +golift.io/unifi v4.1.6+incompatible/go.mod h1:Zjw57ZAzTzCMw784pE8CdCFgkYSzVZzmJ++WUttbjto= +golift.io/unifi v4.1.7-0.20191215214413-4d78625be4cd+incompatible h1:PBa6XT7PYrYUULBON83Owjv8cUhyu1e13lgl+zAxwHo= +golift.io/unifi v4.1.7-0.20191215214413-4d78625be4cd+incompatible/go.mod h1:Zjw57ZAzTzCMw784pE8CdCFgkYSzVZzmJ++WUttbjto= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From a205c9e4da0a6082b6130f9b6b0ff2a839a999ef Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Thu, 26 Dec 2019 20:38:55 -0800 Subject: [PATCH 38/41] fix dock and version --- integrations/inputunifi/Makefile | 8 ++++---- integrations/inputunifi/init/docker/Dockerfile | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/integrations/inputunifi/Makefile b/integrations/inputunifi/Makefile index f47e8c89..609141b6 100644 --- a/integrations/inputunifi/Makefile +++ b/integrations/inputunifi/Makefile @@ -46,10 +46,10 @@ endef PLUGINS:=$(patsubst plugins/%/main.go,%,$(wildcard plugins/*/main.go)) VERSION_LDFLAGS:= \ - -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Branch=$(TRAVIS_BRANCH) \ - -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.BuildDate=$(DATE) \ - -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Revision=$(COMMIT) \ - -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Version=$(VERSION)-$(ITERATION) + -X github.com/prometheus/common/version.Branch=$(TRAVIS_BRANCH) \ + -X github.com/prometheus/common/version.BuildDate=$(DATE) \ + -X github.com/prometheus/common/version.Revision=$(COMMIT) \ + -X github.com/prometheus/common/version.Version=$(VERSION)-$(ITERATION) # Makefile targets follow. diff --git a/integrations/inputunifi/init/docker/Dockerfile b/integrations/inputunifi/init/docker/Dockerfile index c1e21799..cdfeef05 100644 --- a/integrations/inputunifi/init/docker/Dockerfile +++ b/integrations/inputunifi/init/docker/Dockerfile @@ -25,7 +25,7 @@ RUN apt-get update \ COPY . $GOPATH/src/${IMPORT_PATH} WORKDIR $GOPATH/src/${IMPORT_PATH} -RUN dep ensure --vendor-only \ +RUN go mod vendor \ && CGO_ENABLED=0 make ${BINARY}.${ARCH}.${OS} FROM scratch From c2fbd8bbef62b96fe9e833b0a8265d5084a54e96 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Thu, 26 Dec 2019 20:57:01 -0800 Subject: [PATCH 39/41] go all the way --- integrations/inputunifi/.metadata.sh | 8 ++------ integrations/inputunifi/Makefile | 2 -- integrations/inputunifi/init/docker/Dockerfile | 17 +++++------------ integrations/inputunifi/init/docker/hooks/build | 1 - .../inputunifi/init/homebrew/service.rb.tmpl | 10 +++------- 5 files changed, 10 insertions(+), 28 deletions(-) diff --git a/integrations/inputunifi/.metadata.sh b/integrations/inputunifi/.metadata.sh index 5c380efc..42b73a6c 100755 --- a/integrations/inputunifi/.metadata.sh +++ b/integrations/inputunifi/.metadata.sh @@ -25,10 +25,8 @@ export BINARY GHUSER HBREPO MAINT VENDOR DESC GOLANGCI_LINT_ARGS CONFIG_FILE LIC # Fix the repo if it doesn't match the binary name. # Provide a better URL if one exists. -# Used as go import path in docker and homebrew builds. -IMPORT_PATH="github.com/${GHUSER}/${BINARY}" # Used for source links and wiki links. -SOURCE_URL="https://${IMPORT_PATH}" +SOURCE_URL="https://github.com/${GHUSER}/${BINARY}" # Used for documentation links. URL="${SOURCE_URL}" @@ -40,9 +38,7 @@ ITERATION=$(git rev-list --count --all || echo 0) DATE="$(date -u +%Y-%m-%dT%H:%M:%SZ)" COMMIT="$(git rev-parse --short HEAD || echo 0)" -# Used by homebrew downloads. -#SOURCE_PATH=https://codeload.${IMPORT_PATH}/tar.gz/v${VERSION} # This is a custom download path for homebrew formula. SOURCE_PATH=https://golift.io/${BINARY}/archive/v${VERSION}.tar.gz -export IMPORT_PATH SOURCE_URL URL VVERSION VERSION ITERATION DATE COMMIT SOURCE_PATH +export SOURCE_URL URL VVERSION VERSION ITERATION DATE COMMIT SOURCE_PATH diff --git a/integrations/inputunifi/Makefile b/integrations/inputunifi/Makefile index 609141b6..0214ea0d 100644 --- a/integrations/inputunifi/Makefile +++ b/integrations/inputunifi/Makefile @@ -231,7 +231,6 @@ docker: --build-arg "VENDOR=$(VENDOR)" \ --build-arg "AUTHOR=$(MAINT)" \ --build-arg "BINARY=$(BINARY)" \ - --build-arg "IMPORT_PATH=$(IMPORT_PATH)" \ --build-arg "SOURCE_URL=$(SOURCE_URL)" \ --build-arg "CONFIG_FILE=$(CONFIG_FILE)" \ --tag $(BINARY) . @@ -250,7 +249,6 @@ $(BINARY).rb: v$(VERSION).tar.gz.sha256 init/homebrew/$(FORMULA).rb.tmpl -e "s/{{SHA256}}/$(shell head -c64 $<)/g" \ -e "s/{{Desc}}/$(DESC)/g" \ -e "s%{{URL}}%$(URL)%g" \ - -e "s%{{IMPORT_PATH}}%$(IMPORT_PATH)%g" \ -e "s%{{SOURCE_PATH}}%$(SOURCE_PATH)%g" \ -e "s%{{SOURCE_URL}}%$(SOURCE_URL)%g" \ -e "s%{{CONFIG_FILE}}%$(CONFIG_FILE)%g" \ diff --git a/integrations/inputunifi/init/docker/Dockerfile b/integrations/inputunifi/init/docker/Dockerfile index cdfeef05..2b74c41e 100644 --- a/integrations/inputunifi/init/docker/Dockerfile +++ b/integrations/inputunifi/init/docker/Dockerfile @@ -9,21 +9,15 @@ ARG BUILD_DATE=0 ARG COMMIT=0 ARG VERSION=unknown ARG BINARY=application-builder -ARG IMPORT_PATH=github.com/golift/application-builder FROM golang:stretch as builder ARG ARCH ARG OS ARG BINARY -ARG IMPORT_PATH -RUN mkdir -p $GOPATH/pkg/mod $GOPATH/bin $GOPATH/src/${IMPORT_PATH} -RUN apt-get update \ - && apt-get install -y curl \ - && curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - -COPY . $GOPATH/src/${IMPORT_PATH} -WORKDIR $GOPATH/src/${IMPORT_PATH} +RUN mkdir -p $GOPATH/pkg/mod $GOPATH/bin $GOPATH/src /${BINARY} +COPY . /${BINARY} +WORKDIR /${BINARY} RUN go mod vendor \ && CGO_ENABLED=0 make ${BINARY}.${ARCH}.${OS} @@ -36,7 +30,6 @@ ARG COMMIT ARG VERSION ARG LICENSE=MIT ARG BINARY -ARG IMPORT_PATH ARG SOURCE_URL=http://github.com/golift/application-builder ARG URL=http://github.com/golift/application-builder ARG DESC=application-builder @@ -58,8 +51,8 @@ LABEL org.opencontainers.image.created="${BUILD_DATE}" \ org.opencontainers.image.licenses="${LICENSE}" \ org.opencontainers.image.version="${VERSION}" -COPY --from=builder /go/src/${IMPORT_PATH}/${BINARY}.${ARCH}.${OS} /image -COPY --from=builder /go/src/${IMPORT_PATH}/examples/${CONFIG_FILE}.example /etc/${BINARY}/${CONFIG_FILE} +COPY --from=builder /${BINARY}/${BINARY}.${ARCH}.${OS} /image +COPY --from=builder /${BINARY}/examples/${CONFIG_FILE}.example /etc/${BINARY}/${CONFIG_FILE} COPY --from=builder /etc/ssl /etc/ssl VOLUME [ "/etc/${BINARY}" ] diff --git a/integrations/inputunifi/init/docker/hooks/build b/integrations/inputunifi/init/docker/hooks/build index 3eb4e50e..c8f2796f 100755 --- a/integrations/inputunifi/init/docker/hooks/build +++ b/integrations/inputunifi/init/docker/hooks/build @@ -28,7 +28,6 @@ for build in $BUILDS; do --build-arg "VENDOR=${VENDOR}" \ --build-arg "AUTHOR=${MAINT}" \ --build-arg "BINARY=${BINARY}" \ - --build-arg "IMPORT_PATH=${IMPORT_PATH}" \ --build-arg "SOURCE_URL=${SOURCE_URL}" \ --build-arg "CONFIG_FILE=${CONFIG_FILE}" \ --tag "${IMAGE_NAME}_${os}_${name}" \ diff --git a/integrations/inputunifi/init/homebrew/service.rb.tmpl b/integrations/inputunifi/init/homebrew/service.rb.tmpl index ff8802d9..cb10dcb2 100644 --- a/integrations/inputunifi/init/homebrew/service.rb.tmpl +++ b/integrations/inputunifi/init/homebrew/service.rb.tmpl @@ -1,7 +1,6 @@ # Homebrew Formula Template. Built by Makefile: `make fomula` # This is part of Application Builder. # https://github.com/golift/application-builder -# This file is used when FORMULA is set to 'service'. class {{Class}} < Formula desc "{{Desc}}" homepage "{{URL}}" @@ -13,14 +12,11 @@ class {{Class}} < Formula depends_on "dep" def install - ENV["GOPATH"] = buildpath - - bin_path = buildpath/"src/{{IMPORT_PATH}}" - # Copy all files from their current location (GOPATH root) - # to $GOPATH/src/{{IMPORT_PATH}} + bin_path = buildpath/"#{name}" + # Copy all files from their current location to buildpath/#{name} bin_path.install Dir["*",".??*"] cd bin_path do - system "dep", "ensure", "--vendor-only" + system "make" "vendor" system "make", "install", "VERSION=#{version}", "ITERATION={{Iter}}", "PREFIX=#{prefix}", "ETC=#{etc}" # If this fails, the user gets a nice big warning about write permissions on their # #{var}/log folder. The alternative could be letting the app silently fail From 94887efefd8c63f2769b6e0ebe9e6ed88d91a3eb Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Fri, 27 Dec 2019 23:59:16 -0800 Subject: [PATCH 40/41] Add DPI collection --- .../inputunifi/examples/up.conf.example | 9 ++ .../inputunifi/examples/up.json.example | 2 + .../inputunifi/examples/up.xml.example | 2 + .../inputunifi/examples/up.yaml.example | 6 +- integrations/inputunifi/go.mod | 4 +- integrations/inputunifi/go.sum | 20 +--- .../inputunifi/pkg/influxunifi/clients.go | 22 ++++ .../inputunifi/pkg/influxunifi/influxdb.go | 25 ++++- .../inputunifi/pkg/influxunifi/site.go | 20 ++++ .../inputunifi/pkg/inputunifi/collector.go | 21 ++++ .../inputunifi/pkg/inputunifi/input.go | 1 + integrations/inputunifi/pkg/poller/config.go | 2 + integrations/inputunifi/pkg/poller/inputs.go | 20 +--- .../inputunifi/pkg/promunifi/clients.go | 105 +++++++++--------- .../inputunifi/pkg/promunifi/collector.go | 34 ++++-- integrations/inputunifi/pkg/promunifi/site.go | 23 ++++ 16 files changed, 209 insertions(+), 107 deletions(-) diff --git a/integrations/inputunifi/examples/up.conf.example b/integrations/inputunifi/examples/up.conf.example index db3b14ad..fb397501 100644 --- a/integrations/inputunifi/examples/up.conf.example +++ b/integrations/inputunifi/examples/up.conf.example @@ -61,6 +61,7 @@ pass = "unifipoller" sites = ["all"] save_ids = false + save_dpi = false save_sites = true verify_ssl = false @@ -85,6 +86,14 @@ # Only useful if IDS or IPS are enabled on one of the sites. save_ids = false + # Enable collection of Deep Packet Inspection data. This data breaks down traffic + # types for each client and site, it powers a dedicated DPI dashboard. + # Enabling this adds roughly 150 data points per client. That's 6000 metrics for + # 40 clients. This adds a little bit of poller run time per interval and causes + # more API requests to your controller(s). Don't let these "cons" sway you: + # it's cool data. Please provide feedback on your experience with this feature. + save_dpi = false + # Enable collection of site data. This data powers the Network Sites dashboard. # It's not valuable to everyone and setting this to false will save resources. save_sites = true diff --git a/integrations/inputunifi/examples/up.json.example b/integrations/inputunifi/examples/up.json.example index 6e8c8c66..4675bf61 100644 --- a/integrations/inputunifi/examples/up.json.example +++ b/integrations/inputunifi/examples/up.json.example @@ -30,6 +30,7 @@ "url": "https://127.0.0.1:8443", "sites": ["all"], "save_ids": false, + "save_dpi": false, "save_sites": true, "verify_ssl": false }, @@ -40,6 +41,7 @@ "pass": "unifipoller", "url": "https://127.0.0.1:8443", "sites": ["all"], + "save_dpi": false, "save_ids": false, "save_sites": true, "verify_ssl": false diff --git a/integrations/inputunifi/examples/up.xml.example b/integrations/inputunifi/examples/up.xml.example index d1a1be3b..2ff1eecf 100644 --- a/integrations/inputunifi/examples/up.xml.example +++ b/integrations/inputunifi/examples/up.xml.example @@ -32,6 +32,7 @@ https://127.0.0.1:8443 false false + false true @@ -43,6 +44,7 @@ https://127.0.0.1:8443 false false + false true diff --git a/integrations/inputunifi/examples/up.yaml.example b/integrations/inputunifi/examples/up.yaml.example index 33cc5088..4ed27193 100644 --- a/integrations/inputunifi/examples/up.yaml.example +++ b/integrations/inputunifi/examples/up.yaml.example @@ -34,6 +34,7 @@ unifi: - all verify_ssl: false save_ids: false + save_dpi: false save_sites: true @@ -42,9 +43,10 @@ unifi: - role: "" user: "unifipoller" pass: "unifipoller" - url: "https://127.0.0.1:8443" + url: "https://127.0.0.1:8443" sites: - all verify_ssl: false - save_ids: false + save_ids: false + save_dpi: false save_sites: true diff --git a/integrations/inputunifi/go.mod b/integrations/inputunifi/go.mod index ffc8b47f..b5cb3c39 100644 --- a/integrations/inputunifi/go.mod +++ b/integrations/inputunifi/go.mod @@ -8,8 +8,6 @@ require ( github.com/prometheus/client_golang v1.3.0 github.com/prometheus/common v0.7.0 github.com/spf13/pflag v1.0.5 - golang.org/x/sys v0.0.0-20191224085550-c709ea063b76 // indirect golift.io/cnfg v0.0.5 - golift.io/unifi v4.1.7-0.20191215214413-4d78625be4cd+incompatible - gopkg.in/yaml.v2 v2.2.7 // indirect + golift.io/unifi v0.0.400 ) diff --git a/integrations/inputunifi/go.sum b/integrations/inputunifi/go.sum index b2fbc010..f9e33d08 100644 --- a/integrations/inputunifi/go.sum +++ b/integrations/inputunifi/go.sum @@ -22,13 +22,11 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d h1:/WZQPMZNsjZ7IlCpsLGdQBINg5bxKQ1K1sh6awxLtkA= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -45,8 +43,6 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8= -github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.3.0 h1:miYCvYqFXtl/J9FIy8eNpBfYthAEFg+Ys0XyUVEcDsc= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -54,12 +50,10 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -82,26 +76,16 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220142924-d4481acd189f h1:68K/z8GLUxV76xGSqwTWw2gyk/jwn79LUL43rES2g8o= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191224085550-c709ea063b76 h1:Dho5nD6R3PcW2SH1or8vS0dszDaXRxIw55lBX7XiE5g= -golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golift.io/cnfg v0.0.0-20191225081851-bec8d8f51fc4 h1:lCMyrYNLbSBxOiGDnn0ZKWmafS0OEBvaiu7qUJZFrwA= -golift.io/cnfg v0.0.0-20191225081851-bec8d8f51fc4/go.mod h1:ScFDIJg/rJGHbRaed/i7g1lBhywEjB0JiP2uZr3xC3A= golift.io/cnfg v0.0.5 h1:HnMU8Z9C/igKvir1dqaHx5BPuNGZrp99FCtdJyP2Z4I= golift.io/cnfg v0.0.5/go.mod h1:ScFDIJg/rJGHbRaed/i7g1lBhywEjB0JiP2uZr3xC3A= -golift.io/unifi v0.0.0-20191215214413-4d78625be4cd h1:d4wPilxOdtcVqwv7WhAXsrSp9/gvMc0ff4xYBTxcqSs= -golift.io/unifi v0.0.0-20191215214413-4d78625be4cd/go.mod h1:Zjw57ZAzTzCMw784pE8CdCFgkYSzVZzmJ++WUttbjto= +golift.io/unifi v0.0.400 h1:r8FlE+p+zmm8jnQdT367H2aGVMTgxZTrHSwbsHBcayA= +golift.io/unifi v0.0.400/go.mod h1:4BjegFlwA3am3mPlY0qHAnSKli4eexLQV42QKaRx9OY= golift.io/unifi v4.1.6+incompatible h1:Yhb/+obX2vT9i6PElGislSuQ1WUtOf+l+sRjVxlY6nM= golift.io/unifi v4.1.6+incompatible/go.mod h1:Zjw57ZAzTzCMw784pE8CdCFgkYSzVZzmJ++WUttbjto= -golift.io/unifi v4.1.7-0.20191215214413-4d78625be4cd+incompatible h1:PBa6XT7PYrYUULBON83Owjv8cUhyu1e13lgl+zAxwHo= -golift.io/unifi v4.1.7-0.20191215214413-4d78625be4cd+incompatible/go.mod h1:Zjw57ZAzTzCMw784pE8CdCFgkYSzVZzmJ++WUttbjto= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= -gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/integrations/inputunifi/pkg/influxunifi/clients.go b/integrations/inputunifi/pkg/influxunifi/clients.go index 9e15ec19..8ac00a4d 100644 --- a/integrations/inputunifi/pkg/influxunifi/clients.go +++ b/integrations/inputunifi/pkg/influxunifi/clients.go @@ -71,3 +71,25 @@ func (u *InfluxUnifi) batchClient(r report, s *unifi.Client) { r.send(&metric{Table: "clients", Tags: tags, Fields: fields}) } + +func (u *InfluxUnifi) batchClientDPI(r report, s *unifi.DPITable) { + for _, dpi := range s.ByApp { + r.send(&metric{ + Table: "clientdpi", + Tags: map[string]string{ + "category": unifi.DPICats.Get(dpi.Cat), + "application": unifi.DPIApps.GetApp(dpi.Cat, dpi.App), + "name": s.Name, + "mac": s.MAC, + "site_name": s.SiteName, + "source": s.SourceName, + }, + Fields: map[string]interface{}{ + "tx_packets": dpi.TxPackets, + "rx_packets": dpi.RxPackets, + "tx_bytes": dpi.TxBytes, + "rx_bytes": dpi.RxBytes, + }}, + ) + } +} diff --git a/integrations/inputunifi/pkg/influxunifi/influxdb.go b/integrations/inputunifi/pkg/influxunifi/influxdb.go index 280e94a9..61d63ce1 100644 --- a/integrations/inputunifi/pkg/influxunifi/influxdb.go +++ b/integrations/inputunifi/pkg/influxunifi/influxdb.go @@ -194,6 +194,16 @@ func (u *InfluxUnifi) loopPoints(r report) { r.add() r.add() r.add() + r.add() + r.add() + + go func() { + defer r.done() + + for _, s := range m.SitesDPI { + u.batchSiteDPI(r, s) + } + }() go func() { defer r.done() @@ -203,6 +213,14 @@ func (u *InfluxUnifi) loopPoints(r report) { } }() + go func() { + defer r.done() + + for _, s := range m.ClientsDPI { + u.batchClientDPI(r, s) + } + }() + go func() { defer r.done() @@ -219,15 +237,14 @@ func (u *InfluxUnifi) loopPoints(r report) { } }() - if m.Devices == nil { - return - } - u.loopDevicePoints(r) } func (u *InfluxUnifi) loopDevicePoints(r report) { m := r.metrics() + if m.Devices == nil { + return + } r.add() r.add() diff --git a/integrations/inputunifi/pkg/influxunifi/site.go b/integrations/inputunifi/pkg/influxunifi/site.go index 49715e5a..1ae313c4 100644 --- a/integrations/inputunifi/pkg/influxunifi/site.go +++ b/integrations/inputunifi/pkg/influxunifi/site.go @@ -56,3 +56,23 @@ func (u *InfluxUnifi) batchSite(r report, s *unifi.Site) { r.send(&metric{Table: "subsystems", Tags: tags, Fields: fields}) } } + +func (u *InfluxUnifi) batchSiteDPI(r report, s *unifi.DPITable) { + for _, dpi := range s.ByApp { + r.send(&metric{ + Table: "sitedpi", + Tags: map[string]string{ + "category": unifi.DPICats.Get(dpi.Cat), + "application": unifi.DPIApps.GetApp(dpi.Cat, dpi.App), + "site_name": s.SiteName, + "source": s.SourceName, + }, + Fields: map[string]interface{}{ + "tx_packets": dpi.TxPackets, + "rx_packets": dpi.RxPackets, + "tx_bytes": dpi.TxBytes, + "rx_bytes": dpi.RxBytes, + }}, + ) + } +} diff --git a/integrations/inputunifi/pkg/inputunifi/collector.go b/integrations/inputunifi/pkg/inputunifi/collector.go index 85941fee..f044401f 100644 --- a/integrations/inputunifi/pkg/inputunifi/collector.go +++ b/integrations/inputunifi/pkg/inputunifi/collector.go @@ -80,6 +80,16 @@ func (u *InputUnifi) pollController(c *Controller) (*poller.Metrics, error) { return m, fmt.Errorf("unifi.GetSites(%v): %v", c.URL, err) } + if c.SaveDPI { + if m.SitesDPI, err = c.Unifi.GetSiteDPI(m.Sites); err != nil { + return m, fmt.Errorf("unifi.GetSiteDPI(%v): %v", c.URL, err) + } + + if m.ClientsDPI, err = c.Unifi.GetClientsDPI(m.Sites); err != nil { + return m, fmt.Errorf("unifi.GetClientsDPI(%v): %v", c.URL, err) + } + } + if c.SaveIDS { m.IDSList, err = c.Unifi.GetIDS(m.Sites, time.Now().Add(2*time.Minute), time.Now()) if err != nil { @@ -132,12 +142,23 @@ func (u *InputUnifi) augmentMetrics(c *Controller, metrics *poller.Metrics) *pol // These come blank, so set them here. for i, c := range metrics.Clients { + if devices[c.Mac] = c.Name; c.Name == "" { + devices[c.Mac] = c.Hostname + } metrics.Clients[i].SwName = devices[c.SwMac] metrics.Clients[i].ApName = devices[c.ApMac] metrics.Clients[i].GwName = devices[c.GwMac] metrics.Clients[i].RadioDescription = bssdIDs[metrics.Clients[i].Bssid] + metrics.Clients[i].RadioProto } + for i := range metrics.ClientsDPI { + // Name on Client DPI data also comes blank, find it based on MAC address. + metrics.ClientsDPI[i].Name = devices[metrics.ClientsDPI[i].MAC] + if metrics.ClientsDPI[i].Name == "" { + metrics.ClientsDPI[i].Name = metrics.ClientsDPI[i].MAC + } + } + if !*c.SaveSites { metrics.Sites = nil } diff --git a/integrations/inputunifi/pkg/inputunifi/input.go b/integrations/inputunifi/pkg/inputunifi/input.go index 45d3e4e9..d6540d83 100644 --- a/integrations/inputunifi/pkg/inputunifi/input.go +++ b/integrations/inputunifi/pkg/inputunifi/input.go @@ -33,6 +33,7 @@ type InputUnifi struct { type Controller struct { VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` SaveIDS bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` + SaveDPI bool `json:"save_dpi" toml:"save_dpi" xml:"save_dpi" yaml:"save_dpi"` SaveSites *bool `json:"save_sites" toml:"save_sites" xml:"save_sites" yaml:"save_sites"` Role string `json:"role" toml:"role" xml:"role,attr" yaml:"role"` User string `json:"user" toml:"user" xml:"user" yaml:"user"` diff --git a/integrations/inputunifi/pkg/poller/config.go b/integrations/inputunifi/pkg/poller/config.go index 7d706060..b40c8938 100644 --- a/integrations/inputunifi/pkg/poller/config.go +++ b/integrations/inputunifi/pkg/poller/config.go @@ -49,6 +49,8 @@ type Metrics struct { unifi.IDSList unifi.Clients *unifi.Devices + SitesDPI []*unifi.DPITable + ClientsDPI []*unifi.DPITable } // Config represents the core library input data. diff --git a/integrations/inputunifi/pkg/poller/inputs.go b/integrations/inputunifi/pkg/poller/inputs.go index a637239c..897a8d0d 100644 --- a/integrations/inputunifi/pkg/poller/inputs.go +++ b/integrations/inputunifi/pkg/poller/inputs.go @@ -95,23 +95,7 @@ func (u *UnifiPoller) Metrics() (*Metrics, bool, error) { } ok = true - - metrics.Sites = append(metrics.Sites, m.Sites...) - metrics.Clients = append(metrics.Clients, m.Clients...) - metrics.IDSList = append(metrics.IDSList, m.IDSList...) - - if m.Devices == nil { - continue - } - - if metrics.Devices == nil { - metrics.Devices = &unifi.Devices{} - } - - metrics.UAPs = append(metrics.UAPs, m.UAPs...) - metrics.USGs = append(metrics.USGs, m.USGs...) - metrics.USWs = append(metrics.USWs, m.USWs...) - metrics.UDMs = append(metrics.UDMs, m.UDMs...) + metrics = AppendMetrics(metrics, m) } var err error @@ -158,7 +142,9 @@ func (u *UnifiPoller) MetricsFrom(filter *Filter) (*Metrics, bool, error) { // AppendMetrics combined the metrics from two sources. func AppendMetrics(existing *Metrics, m *Metrics) *Metrics { + existing.SitesDPI = append(existing.SitesDPI, m.SitesDPI...) existing.Sites = append(existing.Sites, m.Sites...) + existing.ClientsDPI = append(existing.ClientsDPI, m.ClientsDPI...) existing.Clients = append(existing.Clients, m.Clients...) existing.IDSList = append(existing.IDSList, m.IDSList...) diff --git a/integrations/inputunifi/pkg/promunifi/clients.go b/integrations/inputunifi/pkg/promunifi/clients.go index b24b2d62..844d899d 100644 --- a/integrations/inputunifi/pkg/promunifi/clients.go +++ b/integrations/inputunifi/pkg/promunifi/clients.go @@ -6,44 +6,43 @@ import ( ) type uclient struct { - Anomalies *prometheus.Desc - BytesR *prometheus.Desc - CCQ *prometheus.Desc - Satisfaction *prometheus.Desc - Noise *prometheus.Desc - RoamCount *prometheus.Desc - RSSI *prometheus.Desc - RxBytes *prometheus.Desc - RxBytesR *prometheus.Desc - RxPackets *prometheus.Desc - RxRate *prometheus.Desc - Signal *prometheus.Desc - TxBytes *prometheus.Desc - TxBytesR *prometheus.Desc - TxPackets *prometheus.Desc - TxRetries *prometheus.Desc - TxPower *prometheus.Desc - TxRate *prometheus.Desc - Uptime *prometheus.Desc - WifiTxAttempts *prometheus.Desc - WiredRxBytes *prometheus.Desc - WiredRxBytesR *prometheus.Desc - WiredRxPackets *prometheus.Desc - WiredTxBytes *prometheus.Desc - WiredTxBytesR *prometheus.Desc - WiredTxPackets *prometheus.Desc - DpiStatsApp *prometheus.Desc - DpiStatsCat *prometheus.Desc - DpiStatsRxBytes *prometheus.Desc - DpiStatsRxPackets *prometheus.Desc - DpiStatsTxBytes *prometheus.Desc - DpiStatsTxPackets *prometheus.Desc + Anomalies *prometheus.Desc + BytesR *prometheus.Desc + CCQ *prometheus.Desc + Satisfaction *prometheus.Desc + Noise *prometheus.Desc + RoamCount *prometheus.Desc + RSSI *prometheus.Desc + RxBytes *prometheus.Desc + RxBytesR *prometheus.Desc + RxPackets *prometheus.Desc + RxRate *prometheus.Desc + Signal *prometheus.Desc + TxBytes *prometheus.Desc + TxBytesR *prometheus.Desc + TxPackets *prometheus.Desc + TxRetries *prometheus.Desc + TxPower *prometheus.Desc + TxRate *prometheus.Desc + Uptime *prometheus.Desc + WifiTxAttempts *prometheus.Desc + WiredRxBytes *prometheus.Desc + WiredRxBytesR *prometheus.Desc + WiredRxPackets *prometheus.Desc + WiredTxBytes *prometheus.Desc + WiredTxBytesR *prometheus.Desc + WiredTxPackets *prometheus.Desc + DPITxPackets *prometheus.Desc + DPIRxPackets *prometheus.Desc + DPITxBytes *prometheus.Desc + DPIRxBytes *prometheus.Desc } func descClient(ns string) *uclient { labels := []string{"name", "mac", "site_name", "gw_name", "sw_name", "vlan", "ip", "oui", "network", "sw_port", "ap_name", "source", "wired"} labelW := append([]string{"radio_name", "radio", "radio_proto", "channel", "essid", "bssid", "radio_desc"}, labels...) + labelDPI := []string{"name", "mac", "site_name", "source", "category", "application"} return &uclient{ Anomalies: prometheus.NewDesc(ns+"anomalies", "Client Anomalies", labelW, nil), @@ -66,20 +65,25 @@ func descClient(ns string) *uclient { TxRate: prometheus.NewDesc(ns+"radio_transmit_rate_bps", "Client Transmit Rate", labelW, nil), WifiTxAttempts: prometheus.NewDesc(ns+"wifi_attempts_transmit_total", "Client Wifi Transmit Attempts", labelW, nil), Uptime: prometheus.NewDesc(ns+"uptime_seconds", "Client Uptime", labelW, nil), - /* needs more "looking into" - DpiStatsApp: prometheus.NewDesc(ns+"dpi_stats_app", - "Client DPI Stats App", labels, nil), - DpiStatsCat: prometheus.NewDesc(ns+"dpi_stats_cat", - "Client DPI Stats Cat", labels, nil), - DpiStatsRxBytes: prometheus.NewDesc(ns+"dpi_stats_receive_bytes_total", - "Client DPI Stats Receive Bytes", labels, nil), - DpiStatsRxPackets: prometheus.NewDesc(ns+"dpi_stats_receive_packets_total", - "Client DPI Stats Receive Packets", labels, nil), - DpiStatsTxBytes: prometheus.NewDesc(ns+"dpi_stats_transmit_bytes_total", - "Client DPI Stats Transmit Bytes", labels, nil), - DpiStatsTxPackets: prometheus.NewDesc(ns+"dpi_stats_transmit_packets_total", - "Client DPI Stats Transmit Packets", labels, nil), - */ + DPITxPackets: prometheus.NewDesc(ns+"dpi_transmit_packets", "Client DPI Transmit Packets", labelDPI, nil), + DPIRxPackets: prometheus.NewDesc(ns+"dpi_receive_packets", "Client DPI Receive Packets", labelDPI, nil), + DPITxBytes: prometheus.NewDesc(ns+"dpi_transmit_bytes", "Client DPI Transmit Bytes", labelDPI, nil), + DPIRxBytes: prometheus.NewDesc(ns+"dpi_receive_bytes", "Client DPI Receive Bytes", labelDPI, nil), + } +} + +func (u *promUnifi) exportClientDPI(r report, s *unifi.DPITable) { + for _, dpi := range s.ByApp { + labelDPI := []string{s.Name, s.MAC, s.SiteName, s.SourceName, + unifi.DPICats.Get(dpi.Cat), unifi.DPIApps.GetApp(dpi.Cat, dpi.App)} + + // log.Println(labelDPI, dpi.Cat, dpi.App, dpi.TxBytes, dpi.RxBytes, dpi.TxPackets, dpi.RxPackets) + r.send([]*metric{ + {u.Client.DPITxPackets, gauge, dpi.TxPackets, labelDPI}, + {u.Client.DPIRxPackets, gauge, dpi.RxPackets, labelDPI}, + {u.Client.DPITxBytes, gauge, dpi.TxBytes, labelDPI}, + {u.Client.DPIRxBytes, gauge, dpi.RxBytes, labelDPI}, + }) } } @@ -130,12 +134,3 @@ func (u *promUnifi) exportClient(r report, c *unifi.Client) { r.send([]*metric{{u.Client.Uptime, gauge, c.Uptime, labelW}}) } - -/* needs more "looking into" -{u.Client.DpiStatsApp, gauge, c.DpiStats.App, labels}, -{u.Client.DpiStatsCat, gauge, c.DpiStats.Cat, labels}, -{u.Client.DpiStatsRxBytes, counter, c.DpiStats.RxBytes, labels}, -{u.Client.DpiStatsRxPackets, counter, c.DpiStats.RxPackets, labels}, -{u.Client.DpiStatsTxBytes, counter, c.DpiStats.TxBytes, labels}, -{u.Client.DpiStatsTxPackets, counter, c.DpiStats.TxPackets, labels}, -*/ diff --git a/integrations/inputunifi/pkg/promunifi/collector.go b/integrations/inputunifi/pkg/promunifi/collector.go index 49be737a..7fae2ccc 100644 --- a/integrations/inputunifi/pkg/promunifi/collector.go +++ b/integrations/inputunifi/pkg/promunifi/collector.go @@ -277,6 +277,8 @@ func (u *promUnifi) loopExports(r report) { r.add() r.add() r.add() + r.add() + r.add() go func() { defer r.done() @@ -286,6 +288,30 @@ func (u *promUnifi) loopExports(r report) { } }() + go func() { + defer r.done() + + for _, s := range m.SitesDPI { + u.exportSiteDPI(r, s) + } + }() + + go func() { + defer r.done() + + for _, c := range m.Clients { + u.exportClient(r, c) + } + }() + + go func() { + defer r.done() + + for _, c := range m.ClientsDPI { + u.exportClientDPI(r, c) + } + }() + go func() { defer r.done() @@ -317,12 +343,4 @@ func (u *promUnifi) loopExports(r report) { u.exportUSW(r, d) } }() - - go func() { - defer r.done() - - for _, c := range m.Clients { - u.exportClient(r, c) - } - }() } diff --git a/integrations/inputunifi/pkg/promunifi/site.go b/integrations/inputunifi/pkg/promunifi/site.go index cdae55c8..c515b8a4 100644 --- a/integrations/inputunifi/pkg/promunifi/site.go +++ b/integrations/inputunifi/pkg/promunifi/site.go @@ -31,10 +31,15 @@ type site struct { RemoteUserTxBytes *prometheus.Desc RemoteUserRxPackets *prometheus.Desc RemoteUserTxPackets *prometheus.Desc + DPITxPackets *prometheus.Desc + DPIRxPackets *prometheus.Desc + DPITxBytes *prometheus.Desc + DPIRxBytes *prometheus.Desc } func descSite(ns string) *site { labels := []string{"subsystem", "status", "site_name", "source"} + labelDPI := []string{"category", "application", "site_name", "source"} nd := prometheus.NewDesc return &site{ @@ -63,6 +68,24 @@ func descSite(ns string) *site { RemoteUserTxBytes: nd(ns+"remote_user_transmit_bytes_total", "Remote Users Transmit Bytes", labels, nil), RemoteUserRxPackets: nd(ns+"remote_user_receive_packets_total", "Remote Users Receive Packets", labels, nil), RemoteUserTxPackets: nd(ns+"remote_user_transmit_packets_total", "Remote Users Transmit Packets", labels, nil), + DPITxPackets: nd(ns+"dpi_transmit_packets", "Site DPI Transmit Packets", labelDPI, nil), + DPIRxPackets: nd(ns+"dpi_receive_packets", "Site DPI Receive Packets", labelDPI, nil), + DPITxBytes: nd(ns+"dpi_transmit_bytes", "Site DPI Transmit Bytes", labelDPI, nil), + DPIRxBytes: nd(ns+"dpi_receive_bytes", "Site DPI Receive Bytes", labelDPI, nil), + } +} + +func (u *promUnifi) exportSiteDPI(r report, s *unifi.DPITable) { + for _, dpi := range s.ByApp { + labelDPI := []string{unifi.DPICats.Get(dpi.Cat), unifi.DPIApps.GetApp(dpi.Cat, dpi.App), s.SiteName, s.SourceName} + + // log.Println(labelsDPI, dpi.Cat, dpi.App, dpi.TxBytes, dpi.RxBytes, dpi.TxPackets, dpi.RxPackets) + r.send([]*metric{ + {u.Site.DPITxPackets, gauge, dpi.TxPackets, labelDPI}, + {u.Site.DPIRxPackets, gauge, dpi.RxPackets, labelDPI}, + {u.Site.DPITxBytes, gauge, dpi.TxBytes, labelDPI}, + {u.Site.DPIRxBytes, gauge, dpi.RxBytes, labelDPI}, + }) } } From 3d2d96bd3120ad8de6d7eeb1f90775f653299893 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sat, 28 Dec 2019 00:07:16 -0800 Subject: [PATCH 41/41] fix lint --- integrations/inputunifi/pkg/inputunifi/collector.go | 1 + 1 file changed, 1 insertion(+) diff --git a/integrations/inputunifi/pkg/inputunifi/collector.go b/integrations/inputunifi/pkg/inputunifi/collector.go index f044401f..195b2d51 100644 --- a/integrations/inputunifi/pkg/inputunifi/collector.go +++ b/integrations/inputunifi/pkg/inputunifi/collector.go @@ -145,6 +145,7 @@ func (u *InputUnifi) augmentMetrics(c *Controller, metrics *poller.Metrics) *pol if devices[c.Mac] = c.Name; c.Name == "" { devices[c.Mac] = c.Hostname } + metrics.Clients[i].SwName = devices[c.SwMac] metrics.Clients[i].ApName = devices[c.ApMac] metrics.Clients[i].GwName = devices[c.GwMac]