From 17e7c8edb3858c4f96b64e66b45a2ead5bef2077 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Thu, 19 Dec 2019 19:59:51 -0800 Subject: [PATCH] allow dynamic controller scrapes --- Gopkg.lock | 17 ++++------- pkg/influxunifi/influxdb.go | 22 +++++++------- pkg/inputunifi/collector.go | 43 ++++++++++++++++++++++++++++ pkg/inputunifi/input.go | 31 ++++++++++++++++++++ pkg/inputunifi/interface.go | 52 +++++++++++++-------------------- pkg/poller/dumper.go | 2 +- pkg/poller/inputs.go | 10 +++---- pkg/poller/outputs.go | 2 +- pkg/promunifi/collector.go | 57 +++++++++++++++++++++++++++++++++++-- plugins/mysql/main.go | 14 ++++----- 10 files changed, 178 insertions(+), 72 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index c5d41385..7cd32b0e 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -46,11 +46,12 @@ version = "v1.6.0" [[projects]] - digest = "1:7097829edd12fd7211fca0d29496b44f94ef9e6d72f88fb64f3d7b06315818ad" + digest = "1:eb04f69c8991e52eff33c428bd729e04208bf03235be88e4df0d88497c6861b9" name = "github.com/prometheus/client_golang" packages = [ "prometheus", "prometheus/internal", + "prometheus/promhttp", ] pruneopts = "UT" revision = "170205fb58decfd011f1550d4cfb737230d7ae4f" @@ -99,11 +100,11 @@ [[projects]] branch = "master" - digest = "1:68fe4216878f16dd6ef33413365fbbe8d2eb781177c7adab874cfc752ce96a7e" + digest = "1:07f0cb66f649e51f9ef23441f8dfc34a73e7d9bf0832417abcbad578f1d8c8d6" name = "golang.org/x/sys" packages = ["windows"] pruneopts = "UT" - revision = "4a24b406529242041050cb1dec3e0e4c46a5f1b6" + revision = "af0d71d358abe0ba3594483a5d519f429dbae3e9" [[projects]] digest = "1:54742bef5cb29f706614c9edcfdeb29fb5992f26090f26ca955f575dddf54f9e" @@ -113,14 +114,6 @@ revision = "961061d377655468e9da4a9333e71b9b77402470" version = "v0.0.1" -[[projects]] - digest = "1:54742bef5cb29f706614c9edcfdeb29fb5992f26090f26ca955f575dddf54f9e" - name = "golift.io/config" - packages = ["."] - pruneopts = "UT" - revision = "961061d377655468e9da4a9333e71b9b77402470" - version = "v0.0.1" - [[projects]] digest = "1:2883cea734f2766f41ff9c9d4aefccccc53e3d44f5c8b08893b9c218cf666722" name = "golift.io/unifi" @@ -143,10 +136,10 @@ input-imports = [ "github.com/influxdata/influxdb1-client/v2", "github.com/prometheus/client_golang/prometheus", + "github.com/prometheus/client_golang/prometheus/promhttp", "github.com/prometheus/common/version", "github.com/spf13/pflag", "golift.io/cnfg", - "golift.io/config", "golift.io/unifi", ] solver-name = "gps-cdcl" diff --git a/pkg/influxunifi/influxdb.go b/pkg/influxunifi/influxdb.go index c88f7fbd..280e94a9 100644 --- a/pkg/influxunifi/influxdb.go +++ b/pkg/influxunifi/influxdb.go @@ -10,7 +10,7 @@ import ( "github.com/davidnewhall/unifi-poller/pkg/poller" influx "github.com/influxdata/influxdb1-client/v2" - "golift.io/config" + "golift.io/cnfg" ) const ( @@ -23,13 +23,13 @@ const ( // Config defines the data needed to store metrics in InfluxDB type Config struct { - Interval config.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` - Disable bool `json:"disable" toml:"disable" xml:"disable,attr" yaml:"disable"` - VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` - URL string `json:"url,omitempty" toml:"url,omitempty" xml:"url" yaml:"url"` - User string `json:"user,omitempty" toml:"user,omitempty" xml:"user" yaml:"user"` - Pass string `json:"pass,omitempty" toml:"pass,omitempty" xml:"pass" yaml:"pass"` - DB string `json:"db,omitempty" toml:"db,omitempty" xml:"db" yaml:"db"` + Interval cnfg.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` + Disable bool `json:"disable" toml:"disable" xml:"disable,attr" yaml:"disable"` + VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` + URL string `json:"url,omitempty" toml:"url,omitempty" xml:"url" yaml:"url"` + User string `json:"user,omitempty" toml:"user,omitempty" xml:"user" yaml:"user"` + Pass string `json:"pass,omitempty" toml:"pass,omitempty" xml:"pass" yaml:"pass"` + DB string `json:"db,omitempty" toml:"db,omitempty" xml:"db" yaml:"db"` } // InfluxDB allows the data to be nested in the config file. @@ -133,12 +133,12 @@ func (u *InfluxUnifi) setConfigDefaults() { } if u.Config.Interval.Duration == 0 { - u.Config.Interval = config.Duration{Duration: defaultInterval} + u.Config.Interval = cnfg.Duration{Duration: defaultInterval} } else if u.Config.Interval.Duration < minimumInterval { - u.Config.Interval = config.Duration{Duration: minimumInterval} + u.Config.Interval = cnfg.Duration{Duration: minimumInterval} } - u.Config.Interval = config.Duration{Duration: u.Config.Interval.Duration.Round(time.Second)} + u.Config.Interval = cnfg.Duration{Duration: u.Config.Interval.Duration.Round(time.Second)} } // ReportMetrics batches all device and client data into influxdb data points. diff --git a/pkg/inputunifi/collector.go b/pkg/inputunifi/collector.go index c9c67dcb..1539f244 100644 --- a/pkg/inputunifi/collector.go +++ b/pkg/inputunifi/collector.go @@ -15,6 +15,49 @@ func (u *InputUnifi) isNill(c *Controller) bool { return c.Unifi == nil } +func (u *InputUnifi) dynamicController(url string) (*poller.Metrics, bool, error) { + c := u.Config.Default // copy defaults into new controller + c.Name = url + c.URL = url + + u.Logf("Authenticating to Dynamic UniFi Controller: %s", url) + + if err := u.getUnifi(&c); err != nil { + return nil, false, fmt.Errorf("authenticating to %s: %v", url, err) + } + + metrics := &poller.Metrics{} + ok, err := u.appendController(&c, metrics) + + return metrics, ok, err +} + +func (u *InputUnifi) appendController(c *Controller, metrics *poller.Metrics) (bool, error) { + m, err := u.collectController(c) + if err != nil || m == nil { + return false, err + } + + metrics.Sites = append(metrics.Sites, m.Sites...) + metrics.Clients = append(metrics.Clients, m.Clients...) + metrics.IDSList = append(metrics.IDSList, m.IDSList...) + + if m.Devices == nil { + return true, nil + } + + if metrics.Devices == nil { + metrics.Devices = &unifi.Devices{} + } + + metrics.UAPs = append(metrics.UAPs, m.UAPs...) + metrics.USGs = append(metrics.USGs, m.USGs...) + metrics.USWs = append(metrics.USWs, m.USWs...) + metrics.UDMs = append(metrics.UDMs, m.UDMs...) + + return true, nil +} + func (u *InputUnifi) collectController(c *Controller) (*poller.Metrics, error) { if u.isNill(c) { u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) diff --git a/pkg/inputunifi/input.go b/pkg/inputunifi/input.go index 982da1b4..4a2881aa 100644 --- a/pkg/inputunifi/input.go +++ b/pkg/inputunifi/input.go @@ -13,6 +13,13 @@ import ( "golift.io/unifi" ) +const ( + defaultURL = "https://127.0.0.1:8443" + defaultUser = "unifipoller" + defaultPass = "unifipollerp4$$w0rd" + defaultSite = "all" +) + // InputUnifi contains the running data. type InputUnifi struct { Config *Config `json:"unifi" toml:"unifi" xml:"unifi" yaml:"unifi"` @@ -36,7 +43,9 @@ type Controller struct { // Config contains our configuration data type Config struct { sync.RWMutex // locks the Unifi struct member when re-authing to unifi. + Default Controller `json:"defaults" toml:"defaults" xml:"default" yaml:"defaults"` Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` + Dynamic bool `json:"dynamic" toml:"dynamic" xml:"dynamic" yaml:"dynamic"` Controllers []*Controller `json:"controllers" toml:"controller" xml:"controller" yaml:"controllers"` } @@ -147,6 +156,28 @@ func (u *InputUnifi) dumpSitesJSON(c *Controller, path, name string, sites unifi return allJSON, nil } +func (u *InputUnifi) setDefaults(c *Controller) { + if c.URL == "" { + c.URL = defaultURL + } + + if c.Name == "" { + c.Name = c.URL + } + + if c.Pass == "" { + c.Pass = defaultPass + } + + if c.User == "" { + c.User = defaultUser + } + + if len(c.Sites) < 1 { + c.Sites = []string{defaultSite} + } +} + // StringInSlice returns true if a string is in a slice. func StringInSlice(str string, slice []string) bool { for _, s := range slice { diff --git a/pkg/inputunifi/interface.go b/pkg/inputunifi/interface.go index 99ee7f51..67f53cce 100644 --- a/pkg/inputunifi/interface.go +++ b/pkg/inputunifi/interface.go @@ -19,16 +19,15 @@ func (u *InputUnifi) Initialize(l poller.Logger) error { return nil } - if len(u.Config.Controllers) < 1 { - return fmt.Errorf("no unifi controllers defined for unifi input") + if u.setDefaults(&u.Config.Default); len(u.Config.Controllers) < 1 { + new := u.Config.Default // copy defaults. + u.Config.Controllers = []*Controller{&new} } u.Logger = l - for i, c := range u.Config.Controllers { - if c.Name == "" { - u.Config.Controllers[i].Name = c.URL - } + for _, c := range u.Config.Controllers { + u.setDefaults(c) switch err := u.getUnifi(c); err { case nil: @@ -48,12 +47,12 @@ func (u *InputUnifi) Initialize(l poller.Logger) error { // Metrics grabs all the measurements from a UniFi controller and returns them. func (u *InputUnifi) Metrics() (*poller.Metrics, bool, error) { - return u.MetricsFrom(poller.Filter{}) + return u.MetricsFrom(nil) } // MetricsFrom grabs all the measurements from a UniFi controller and returns them. -func (u *InputUnifi) MetricsFrom(filter poller.Filter) (*poller.Metrics, bool, error) { - if u.Config.Disable { +func (u *InputUnifi) MetricsFrom(filter *poller.Filter) (*poller.Metrics, bool, error) { + if u.Config.Disable || filter == nil || filter.Term == "" { return nil, false, nil } @@ -61,49 +60,36 @@ func (u *InputUnifi) MetricsFrom(filter poller.Filter) (*poller.Metrics, bool, e metrics := &poller.Metrics{} ok := false + // Check if the request is for an existing, configured controller. for _, c := range u.Config.Controllers { - if filter.Term != "" && c.Name != filter.Term { + if !strings.EqualFold(c.Name, filter.Term) { continue } - m, err := u.collectController(c) + exists, err := u.appendController(c, metrics) if err != nil { errs = append(errs, err.Error()) } - if m == nil { - continue + if exists { + ok = true } - - ok = true - - metrics.Sites = append(metrics.Sites, m.Sites...) - metrics.Clients = append(metrics.Clients, m.Clients...) - metrics.IDSList = append(metrics.IDSList, m.IDSList...) - - if m.Devices == nil { - continue - } - - if metrics.Devices == nil { - metrics.Devices = &unifi.Devices{} - } - - metrics.UAPs = append(metrics.UAPs, m.UAPs...) - metrics.USGs = append(metrics.USGs, m.USGs...) - metrics.USWs = append(metrics.USWs, m.USWs...) - metrics.UDMs = append(metrics.UDMs, m.UDMs...) } if len(errs) > 0 { return metrics, ok, fmt.Errorf(strings.Join(errs, ", ")) } + if u.Config.Dynamic && !ok && strings.HasPrefix(filter.Term, "http") { + // Attempt to a dynamic metrics fetch from an unconfigured controller. + return u.dynamicController(filter.Term) + } + return metrics, ok, nil } // RawMetrics returns API output from the first configured unifi controller. -func (u *InputUnifi) RawMetrics(filter poller.Filter) ([]byte, error) { +func (u *InputUnifi) RawMetrics(filter *poller.Filter) ([]byte, error) { c := u.Config.Controllers[0] // We could pull the controller number from the filter. if u.isNill(c) { u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) diff --git a/pkg/poller/dumper.go b/pkg/poller/dumper.go index 2a2b0dab..80c7f728 100644 --- a/pkg/poller/dumper.go +++ b/pkg/poller/dumper.go @@ -11,7 +11,7 @@ func (u *UnifiPoller) DumpJSONPayload() (err error) { u.Config.Quiet = true split := strings.SplitN(u.Flags.DumpJSON, " ", 2) - filter := Filter{Type: split[0]} + filter := &Filter{Type: split[0]} if len(split) > 1 { filter.Term = split[1] diff --git a/pkg/poller/inputs.go b/pkg/poller/inputs.go index 5d361166..8633ca2c 100644 --- a/pkg/poller/inputs.go +++ b/pkg/poller/inputs.go @@ -15,10 +15,10 @@ var ( // Input plugins must implement this interface. type Input interface { - Initialize(Logger) error // Called once on startup to initialize the plugin. - Metrics() (*Metrics, bool, error) // Called every time new metrics are requested. - MetricsFrom(Filter) (*Metrics, bool, error) // Called every time new metrics are requested. - RawMetrics(Filter) ([]byte, error) + Initialize(Logger) error // Called once on startup to initialize the plugin. + Metrics() (*Metrics, bool, error) // Called every time new metrics are requested. + MetricsFrom(*Filter) (*Metrics, bool, error) // Called every time new metrics are requested. + RawMetrics(*Filter) ([]byte, error) } // InputPlugin describes an input plugin's consumable interface. @@ -108,7 +108,7 @@ func (u *UnifiPoller) Metrics() (*Metrics, bool, error) { } // MetricsFrom aggregates all the measurements from all configured inputs and returns them. -func (u *UnifiPoller) MetricsFrom(filter Filter) (*Metrics, bool, error) { +func (u *UnifiPoller) MetricsFrom(filter *Filter) (*Metrics, bool, error) { errs := []string{} metrics := &Metrics{} ok := false diff --git a/pkg/poller/outputs.go b/pkg/poller/outputs.go index 0672d92b..0c73bfe7 100644 --- a/pkg/poller/outputs.go +++ b/pkg/poller/outputs.go @@ -14,7 +14,7 @@ var ( // Output packages must implement this interface. type Collect interface { Metrics() (*Metrics, bool, error) - MetricsFrom(Filter) (*Metrics, bool, error) + MetricsFrom(*Filter) (*Metrics, bool, error) Logger } diff --git a/pkg/promunifi/collector.go b/pkg/promunifi/collector.go index 694bfd4c..e4cf1e9c 100644 --- a/pkg/promunifi/collector.go +++ b/pkg/promunifi/collector.go @@ -11,6 +11,7 @@ import ( "github.com/davidnewhall/unifi-poller/pkg/poller" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/common/version" "golift.io/unifi" ) @@ -77,6 +78,11 @@ type Report struct { wg sync.WaitGroup } +type target struct { + *poller.Filter + *promUnifi +} + func init() { u := &promUnifi{Prometheus: &Prometheus{}} @@ -103,6 +109,8 @@ func (u *promUnifi) Run(c poller.Collect) error { u.Config.HTTPListen = defaultHTTPListen } + mux := http.NewServeMux() + prometheus.MustRegister(version.NewCollector(u.Config.Namespace)) prometheus.MustRegister(&promUnifi{ Collector: c, @@ -115,8 +123,43 @@ func (u *promUnifi) Run(c poller.Collect) error { }) c.Logf("Exporting Measurements for Prometheus at https://%s/metrics, namespace: %s", u.Config.HTTPListen, u.Config.Namespace) + mux.Handle("/metrics", promhttp.HandlerFor( + prometheus.DefaultGatherer, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}, + )) + mux.HandleFunc("/scrape", u.ScrapeHandler) - return http.ListenAndServe(u.Config.HTTPListen, nil) + return http.ListenAndServe(u.Config.HTTPListen, mux) +} + +// ScrapeHandler allows prometheus to scrape a single source, instead of all sources. +func (u *promUnifi) ScrapeHandler(w http.ResponseWriter, r *http.Request) { + t := &target{promUnifi: u, Filter: &poller.Filter{}} + if t.Filter.Type = r.URL.Query().Get("input"); t.Filter.Type == "" { + http.Error(w, `'input' parameter must be specified (try "unifi")`, 400) + return + } + + if t.Filter.Term = r.URL.Query().Get("target"); t.Filter.Term == "" { + http.Error(w, "'target' parameter must be specified, configured name, or unconfigured url", 400) + return + } + + registry := prometheus.NewRegistry() + + registry.MustRegister(t) + promhttp.HandlerFor( + registry, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}, + ).ServeHTTP(w, r) +} + +// Describe satisfies the prometheus Collector. This returns all of the +// metric descriptions that this packages produces. +func (t *target) Describe(ch chan<- *prometheus.Desc) { + t.promUnifi.Describe(ch) +} + +func (t *target) Collect(ch chan<- prometheus.Metric) { + t.promUnifi.collect(ch, t.Filter) } // Describe satisfies the prometheus Collector. This returns all of the @@ -138,6 +181,10 @@ func (u *promUnifi) Describe(ch chan<- *prometheus.Desc) { // Collect satisfies the prometheus Collector. This runs the input method to get // the current metrics (from another package) then exports them for prometheus. func (u *promUnifi) Collect(ch chan<- prometheus.Metric) { + u.collect(ch, nil) +} + +func (u *promUnifi) collect(ch chan<- prometheus.Metric, filter *poller.Filter) { var err error ok := false @@ -145,7 +192,13 @@ func (u *promUnifi) Collect(ch chan<- prometheus.Metric) { r := &Report{Config: u.Config, ch: make(chan []*metric, buffer), Start: time.Now()} defer r.close() - if r.Metrics, ok, err = u.Collector.Metrics(); err != nil { + if filter == nil { + r.Metrics, ok, err = u.Collector.Metrics() + } else { + r.Metrics, ok, err = u.Collector.MetricsFrom(filter) + } + + if err != nil { r.error(ch, prometheus.NewInvalidDesc(fmt.Errorf("metric fetch failed")), err) if !ok { diff --git a/plugins/mysql/main.go b/plugins/mysql/main.go index e0c866b9..361165a0 100644 --- a/plugins/mysql/main.go +++ b/plugins/mysql/main.go @@ -4,17 +4,17 @@ import ( "fmt" "github.com/davidnewhall/unifi-poller/pkg/poller" - "golift.io/config" + "golift.io/cnfg" ) // mysqlConfig represents the data that is unmarshalled from the up.conf config file for this plugins. type mysqlConfig struct { - Interval config.Duration `json:"interval" toml:"interval" xml:"interval" yaml:"interval"` - Host string `json:"host" toml:"host" xml:"host" yaml:"host"` - User string `json:"user" toml:"user" xml:"user" yaml:"user"` - Pass string `json:"pass" toml:"pass" xml:"pass" yaml:"pass"` - DB string `json:"db" toml:"db" xml:"db" yaml:"db"` - Table string `json:"table" toml:"table" xml:"table" yaml:"table"` + Interval cnfg.Duration `json:"interval" toml:"interval" xml:"interval" yaml:"interval"` + Host string `json:"host" toml:"host" xml:"host" yaml:"host"` + User string `json:"user" toml:"user" xml:"user" yaml:"user"` + Pass string `json:"pass" toml:"pass" xml:"pass" yaml:"pass"` + DB string `json:"db" toml:"db" xml:"db" yaml:"db"` + Table string `json:"table" toml:"table" xml:"table" yaml:"table"` // Maps do not work with ENV VARIABLES yet, but may in the future. Fields []string `json:"fields" toml:"fields" xml:"field" yaml:"fields"` }