From ac39d1727f3b06ea6fe6ef94114cbac4ec69be04 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sun, 15 Dec 2019 02:52:43 -0800 Subject: [PATCH 01/32] make output plugins call in to initialize --- .metadata.sh | 7 +- Makefile | 2 +- examples/MANUAL.md | 59 ++-------------- examples/up.conf.example | 48 ++++++------- examples/up.json.example | 49 ++++++++----- examples/up.xml.example | 29 ++++---- examples/up.yaml.example | 25 ++++--- main.go | 3 + pkg/influxunifi/metrics.go | 141 ++++++++++++++++++++++++++++++++----- pkg/influxunifi/report.go | 8 +-- pkg/metrics/metrics.go | 16 ----- pkg/poller/config.go | 81 +++++++++++++-------- pkg/poller/influx.go | 79 --------------------- pkg/poller/outputs.go | 72 +++++++++++++++++++ pkg/poller/prometheus.go | 53 -------------- pkg/poller/start.go | 48 +++---------- pkg/poller/unifi.go | 20 +++--- pkg/promunifi/collector.go | 140 ++++++++++++++++++++++-------------- pkg/promunifi/report.go | 27 +++---- 19 files changed, 461 insertions(+), 446 deletions(-) delete mode 100644 pkg/metrics/metrics.go delete mode 100644 pkg/poller/influx.go create mode 100644 pkg/poller/outputs.go delete mode 100644 pkg/poller/prometheus.go diff --git a/.metadata.sh b/.metadata.sh index 37506a59..11ba0838 100755 --- a/.metadata.sh +++ b/.metadata.sh @@ -32,11 +32,6 @@ SOURCE_URL="https://${IMPORT_PATH}" # Used for documentation links. URL="${SOURCE_URL}" -# This parameter is passed in as -X to go build. Used to override the Version variable in a package. -# This makes a path like github.com/user/hello-world/helloworld.Version=1.3.3 -# Name the Version-containing library the same as the github repo, without dashes. -VERSION_PATH="${IMPORT_PATH}/pkg/poller.Version" - # Dynamic. Recommend not changing. VVERSION=$(git describe --abbrev=0 --tags $(git rev-list --tags --max-count=1)) VERSION="$(echo $VVERSION | tr -d v | grep -E '^\S+$' || echo development)" @@ -51,4 +46,4 @@ COMMIT="$(git rev-parse --short HEAD || echo 0)" # This is a custom download path for homebrew formula. SOURCE_PATH=https://golift.io/${BINARY}/archive/v${VERSION}.tar.gz -export IMPORT_PATH SOURCE_URL URL VERSION_PATH VVERSION VERSION ITERATION DATE BRANCH COMMIT SOURCE_PATH +export IMPORT_PATH SOURCE_URL URL VVERSION VERSION ITERATION DATE BRANCH COMMIT SOURCE_PATH diff --git a/Makefile b/Makefile index 063fdafa..4cf96dc5 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ VERSION_LDFLAGS:= \ -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Branch=$(BRANCH) \ -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.BuildDate=$(DATE) \ -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Revision=$(COMMIT) \ - -X $(VERSION_PATH)=$(VERSION)-$(ITERATION) + -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Version=$(VERSION)-$(ITERATION) # Makefile targets follow. diff --git a/examples/MANUAL.md b/examples/MANUAL.md index 0b1a8860..012cd343 100644 --- a/examples/MANUAL.md +++ b/examples/MANUAL.md @@ -65,10 +65,10 @@ is provided so the application can be easily adapted to any environment. `Config File Parameters` - interval default: 30s - How often to poll the controller for updated client and device data. - The UniFi Controller only updates traffic stats about every 30-60 seconds. - Only works if "mode" (below) is "influx" - other modes do not use interval. +Additional parameters are added by output packages. Parameters can also be set +using environment variables. See the GitHub wiki for more information! + + >>> POLLER FIELDS FOLLOW - you may have multiple controllers: debug default: false This turns on time stamps and line numbers in logs, outputs a few extra @@ -79,56 +79,7 @@ is provided so the application can be easily adapted to any environment. errors will be logged. Using this with debug=true adds line numbers to any error logs. - mode default: "influx" - * Value: influx - This default mode runs this application as a daemon. It will poll - the controller at the configured interval and report measurements to - InfluxDB. Providing an invalid value will run in this default mode. - - * Value: influxlambda - Setting this value will invoke a run-once mode where the application - immediately polls the controller and reports the metrics to InfluxDB. - Then it exits. This mode is useful in an AWS Lambda or a crontab where - the execution timings are controlled. This mode may also be adapted - to run in other collector scripts and apps like telegraf or diamond. - This mode can also be combined with a "test database" in InfluxDB to - give yourself a "test config file" you may run ad-hoc to test changes. - - * Value: prometheus - In this mode the application opens an http interface and exports the - measurements at /metrics for collection by prometheus. Enabling this - mode disables InfluxDB usage entirely. - - * Value: both - Setting the mode to "both" will cause the InfluxDB poller routine to run - along with the Prometheus exporter. You can run both at the same time. - - http_listen default: 0.0.0.0:9130 - This option controls the IP and port the http listener uses when the - mode is set to prometheus. This setting has no effect when other modes - are in use. Metrics become available at the /metrics URI. - - influx_url default: http://127.0.0.1:8086 - This is the URL where the Influx web server is available. - - influx_user default: unifi - Username used to authenticate with InfluxDB. - - influx_pass default: unifi - Password used to authenticate with InfluxDB. - - influx_db default: unifi - Custom database created in InfluxDB to use with this application. - On first setup, log into InfluxDB and create access: - $ influx -host localhost -port 8086 - CREATE DATABASE unifi - CREATE USER unifi WITH PASSWORD 'unifi' WITH ALL PRIVILEGES - GRANT ALL ON unifi TO unifi - - influx_insecure_ssl default: false - Setting this to true will allow use of InfluxDB with an invalid SSL certificate. - - >>> CONTROLLER FIELDS FOLLOW - you may have multiple controllers: + >>> CONTROLLER FIELDS FOLLOW - you may have multiple controllers: sites default: ["all"] This list of strings should represent the names of sites on the UniFi diff --git a/examples/up.conf.example b/examples/up.conf.example index 31713a84..75c12941 100644 --- a/examples/up.conf.example +++ b/examples/up.conf.example @@ -3,11 +3,7 @@ ######################################################## -# The UniFi Controller only updates traffic stats about every 30 seconds. -# Setting this to something lower may lead to "zeros" in your data. -# If you're getting zeros now, set this to "1m" -interval = "30s" - +[poller] # Turns on line numbers, microsecond logging, and a per-device log. # The default is false, but I personally leave this on at home (four devices). # This may be noisy if you have a lot of devices. It adds one line per device. @@ -17,38 +13,34 @@ debug = false # Recommend enabling debug with this setting for better error logging. quiet = false -# Which mode to run this application in. The default mode is "influx". Providing -# an invalid mode will also result in "influx". In this default mode the application -# runs as a daemon and polls the controller at the configured interval. -# -# Other options: "influxlambda", "prometheus", "both" -# -# Mode "influxlambda" makes the application exit after collecting and reporting metrics -# to InfluxDB one time. This mode requires an external process like an AWS Lambda -# or a simple crontab to keep the timings accurate on UniFi Poller run intervals. -# -# Mode "prometheus" opens an HTTP server on port 9130 and exports the metrics at -# /metrics for polling collection by a prometheus server. This disables influxdb. -# -# Mode "both" runs the Prometheus HTTP server and InfluxDB poller interval at -# the same time. -mode = "influx" +#### OUTPUTS + +[prometheus] +disable = false # This controls on which ip and port /metrics is exported when mode is "prometheus". # This has no effect in other modes. Must contain a colon and port. http_listen = "0.0.0.0:9130" +report_errors = false +[influxdb] +disable = false # InfluxDB does not require auth by default, so the user/password are probably unimportant. -influx_url = "http://127.0.0.1:8086" -influx_user = "unifi" -influx_pass = "unifi" +url = "http://127.0.0.1:8086" +user = "unifi" +pass = "unifi" # Be sure to create this database. -influx_db = "unifi" -# If your InfluxDB uses an invalid SSL cert, set this to true. -influx_insecure_ssl = false +db = "unifi" +# If your InfluxDB uses a valid SSL cert, set this to true. +verify_ssl = false +# The UniFi Controller only updates traffic stats about every 30 seconds. +# Setting this to something lower may lead to "zeros" in your data. +# If you're getting zeros now, set this to "1m" +interval = "30s" + +#### INPUTS # You may repeat the following section to poll additional controllers. - [[controller]] # Friendly name used in dashboards. name = "" diff --git a/examples/up.json.example b/examples/up.json.example index 07ecd509..65e4d27e 100644 --- a/examples/up.json.example +++ b/examples/up.json.example @@ -1,22 +1,33 @@ { - "interval": "30s", - "debug": false, - "quiet": false, - "mode": "influx", - "http_listen": "0.0.0.0:9130", - "influx_url": "http://127.0.0.1:8086", - "influx_user": "unifi", - "influx_pass": "unifi", - "influx_db": "unifi", - "influx_insecure_ssl": false, - "controller": [{ - "name": "", - "user": "influx", - "pass": "", - "url": "https://127.0.0.1:8443", - "sites": ["all"], - "save_ids": false, - "save_sites": true, - "verify_ssl": false + "poller": { + "debug": false, + "quiet": false + }, + + "prometheus": { + "disable": false, + "http_listen": "0.0.0.0:9130", + "report_errors": false + }, + + "influxdb": { + "disable": false, + "url": "http://127.0.0.1:8086", + "user": "unifi", + "pass": "unifi", + "db": "unifi", + "verify_ssl": false, + "interval": "30s" + }, + + "controller": [{ + "name": "", + "user": "influx", + "pass": "", + "url": "https://127.0.0.1:8443", + "sites": ["all"], + "save_ids": false, + "save_sites": true, + "verify_ssl": false }] } diff --git a/examples/up.xml.example b/examples/up.xml.example index 551ff8b7..710c01ba 100644 --- a/examples/up.xml.example +++ b/examples/up.xml.example @@ -5,21 +5,23 @@ # provided values are defaults. See up.conf.example! # ####################################################### --> - + - 60s + + 0.0.0.0:9130 + false + - false - false + + 30s + http://127.0.0.1:8086 + unifi + unifi + unifi + false + - influx - 0.0.0.0:9130 - - unifi - unifi - http://127.0.0.1:8086 - unifi - false + all influx @@ -29,4 +31,5 @@ false true - + + diff --git a/examples/up.yaml.example b/examples/up.yaml.example index b076feb4..611c5fa8 100644 --- a/examples/up.yaml.example +++ b/examples/up.yaml.example @@ -3,19 +3,24 @@ # provided values are defaults. See up.conf.example! # ######################################################## --- -interval: "30s" -debug: false -quiet: false +poller: + debug: false + quiet: false -mode: "influx" -http_listen: "0.0.0.0:9130" +prometheus: + disable: false + http_listen: "0.0.0.0:9130" + report_errors: false -influx_url: "http://127.0.0.1:8086" -influx_user: "unifi" -influx_pass: "unifi" -influx_db: "unifi" -influx_insecure_ssl: false +influxdb: + disable: false + interval: "30s" + url: "http://127.0.0.1:8086" + user: "unifi" + pass: "unifi" + db: "unifi" + verify_ssl: false controller: - name: "" diff --git a/main.go b/main.go index 332bfd5d..be0fe13b 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,9 @@ import ( "log" "github.com/davidnewhall/unifi-poller/pkg/poller" + // Enable output plugins! + _ "github.com/davidnewhall/unifi-poller/pkg/influxunifi" + _ "github.com/davidnewhall/unifi-poller/pkg/promunifi" ) // Keep it simple. diff --git a/pkg/influxunifi/metrics.go b/pkg/influxunifi/metrics.go index 90a95a31..6775ac06 100644 --- a/pkg/influxunifi/metrics.go +++ b/pkg/influxunifi/metrics.go @@ -1,29 +1,47 @@ -// Package influx provides the methods to turn UniFi measurements into influx +// Package influxunifi provides the methods to turn UniFi measurements into influx // data-points with appropriate tags and fields. package influxunifi import ( "crypto/tls" "fmt" + "log" "time" - "github.com/davidnewhall/unifi-poller/pkg/metrics" + "github.com/davidnewhall/unifi-poller/pkg/poller" influx "github.com/influxdata/influxdb1-client/v2" + conf "golift.io/config" +) + +const ( + defaultInterval = 30 * time.Second + defaultInfluxDB = "unifi" + defaultInfluxUser = "unifi" + defaultInfluxURL = "http://127.0.0.1:8086" ) // Config defines the data needed to store metrics in InfluxDB type Config struct { - Database string - URL string - User string - Pass string - BadSSL bool + Interval conf.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` + Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` + VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` + URL string `json:"url,omitempty" toml:"url,omitempty" xml:"url" yaml:"url"` + User string `json:"user,omitempty" toml:"user,omitempty" xml:"user" yaml:"user"` + Pass string `json:"pass,omitempty" toml:"pass,omitempty" xml:"pass" yaml:"pass"` + DB string `json:"db,omitempty" toml:"db,omitempty" xml:"db" yaml:"db"` +} + +// InfluxDB allows the data to be nested in the config file. +type InfluxDB struct { + Config Config `json:"influxdb" toml:"influxdb" xml:"influxdb" yaml:"influxdb"` } // InfluxUnifi is returned by New() after you provide a Config. type InfluxUnifi struct { - cf *Config - influx influx.Client + Collector poller.Collect + influx influx.Client + LastCheck time.Time + *InfluxDB } type metric struct { @@ -32,26 +50,101 @@ type metric struct { Fields map[string]interface{} } -// New returns an InfluxDB interface. -func New(c *Config) (*InfluxUnifi, error) { - i, err := influx.NewHTTPClient(influx.HTTPConfig{ - Addr: c.URL, - Username: c.User, - Password: c.Pass, - TLSConfig: &tls.Config{InsecureSkipVerify: c.BadSSL}, +func init() { + u := &InfluxUnifi{InfluxDB: &InfluxDB{}, LastCheck: time.Now()} + poller.NewOutput(&poller.Output{ + Name: "influxdb", + Config: u.InfluxDB, + Method: u.Run, }) - return &InfluxUnifi{cf: c, influx: i}, err +} + +// PollController runs forever, polling UniFi and pushing to InfluxDB +// This is started by Run() or RunBoth() after everything checks out. +func (u *InfluxUnifi) PollController() { + interval := u.Config.Interval.Round(time.Second) + log.Printf("[INFO] Everything checks out! Poller started, InfluxDB interval: %v", interval) + + ticker := time.NewTicker(interval) + for u.LastCheck = range ticker.C { + metrics, err := u.Collector.Metrics() + if err != nil { + u.Collector.LogErrorf("%v", err) + continue + } + + report, err := u.ReportMetrics(metrics) + if err != nil { + // XXX: reset and re-auth? not sure.. + u.Collector.LogErrorf("%v", err) + continue + } + + u.LogInfluxReport(report) + } +} + +// Run runs a ticker to poll the unifi server and update influxdb. +func (u *InfluxUnifi) Run(c poller.Collect) error { + var err error + + if u.Config.Disable { + return nil + } + + u.Collector = c + u.setConfigDefaults() + + u.influx, err = influx.NewHTTPClient(influx.HTTPConfig{ + Addr: u.Config.URL, + Username: u.Config.User, + Password: u.Config.Pass, + TLSConfig: &tls.Config{InsecureSkipVerify: !u.Config.VerifySSL}, + }) + if err != nil { + return err + } + + u.PollController() + + return nil +} + +func (u *InfluxUnifi) setConfigDefaults() { + if u.Config.URL == "" { + u.Config.URL = defaultInfluxURL + } + + if u.Config.User == "" { + u.Config.User = defaultInfluxUser + } + + if u.Config.Pass == "" { + u.Config.Pass = defaultInfluxUser + } + + if u.Config.DB == "" { + u.Config.DB = defaultInfluxDB + } + + if u.Config.Interval.Duration == 0 { + u.Config.Interval = conf.Duration{Duration: defaultInterval} + } else if u.Config.Interval.Duration < defaultInterval/2 { + u.Config.Interval = conf.Duration{Duration: defaultInterval / 2} + } + + u.Config.Interval = conf.Duration{Duration: u.Config.Interval.Duration.Round(time.Second)} } // ReportMetrics batches all device and client data into influxdb data points. // Call this after you've collected all the data you care about. // Returns an error if influxdb calls fail, otherwise returns a report. -func (u *InfluxUnifi) ReportMetrics(m *metrics.Metrics) (*Report, error) { +func (u *InfluxUnifi) ReportMetrics(m *poller.Metrics) (*Report, error) { r := &Report{Metrics: m, ch: make(chan *metric), Start: time.Now()} defer close(r.ch) // Make a new Influx Points Batcher. var err error - r.bp, err = influx.NewBatchPoints(influx.BatchPointsConfig{Database: u.cf.Database}) + r.bp, err = influx.NewBatchPoints(influx.BatchPointsConfig{Database: u.Config.DB}) if err != nil { return nil, fmt.Errorf("influx.NewBatchPoints: %v", err) } @@ -140,3 +233,13 @@ func (u *InfluxUnifi) loopPoints(r report) { } }() } + +// LogInfluxReport writes a log message after exporting to influxdb. +func (u *InfluxUnifi) LogInfluxReport(r *Report) { + idsMsg := fmt.Sprintf("IDS Events: %d, ", len(r.Metrics.IDSList)) + u.Collector.Logf("UniFi Metrics Recorded. Sites: %d, Clients: %d, "+ + "UAP: %d, USG/UDM: %d, USW: %d, %sPoints: %d, Fields: %d, Errs: %d, Elapsed: %v", + len(r.Metrics.Sites), len(r.Metrics.Clients), len(r.Metrics.UAPs), + len(r.Metrics.UDMs)+len(r.Metrics.USGs), len(r.Metrics.USWs), idsMsg, r.Total, + r.Fields, len(r.Errors), r.Elapsed.Round(time.Millisecond)) +} diff --git a/pkg/influxunifi/report.go b/pkg/influxunifi/report.go index 5d1e760d..3fdf77a9 100644 --- a/pkg/influxunifi/report.go +++ b/pkg/influxunifi/report.go @@ -4,13 +4,13 @@ import ( "sync" "time" - "github.com/davidnewhall/unifi-poller/pkg/metrics" + "github.com/davidnewhall/unifi-poller/pkg/poller" influx "github.com/influxdata/influxdb1-client/v2" ) // Report is returned to the calling procedure after everything is processed. type Report struct { - Metrics *metrics.Metrics + Metrics *poller.Metrics Errors []error Total int Fields int @@ -28,10 +28,10 @@ type report interface { send(m *metric) error(err error) batch(m *metric, pt *influx.Point) - metrics() *metrics.Metrics + metrics() *poller.Metrics } -func (r *Report) metrics() *metrics.Metrics { +func (r *Report) metrics() *poller.Metrics { return r.Metrics } diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go deleted file mode 100644 index 2d38f54e..00000000 --- a/pkg/metrics/metrics.go +++ /dev/null @@ -1,16 +0,0 @@ -package metrics - -import ( - "time" - - "golift.io/unifi" -) - -// Metrics is a type shared by the exporting and reporting packages. -type Metrics struct { - TS time.Time - unifi.Sites - unifi.IDSList - unifi.Clients - *unifi.Devices -} diff --git a/pkg/poller/config.go b/pkg/poller/config.go index 04d3f1a4..0d6fe126 100644 --- a/pkg/poller/config.go +++ b/pkg/poller/config.go @@ -12,26 +12,17 @@ import ( "sync" "time" - "github.com/davidnewhall/unifi-poller/pkg/influxunifi" "github.com/spf13/pflag" "golift.io/config" "golift.io/unifi" ) -// Version is injected by the Makefile -var Version = "development" - +// App defaults in case they're missing from the config. const ( - // App defaults in case they're missing from the config. - appName = "unifi-poller" - defaultInterval = 30 * time.Second - defaultInfluxDB = "unifi" - defaultInfluxUser = "unifi" - defaultInfluxPass = "unifi" - defaultInfluxURL = "http://127.0.0.1:8086" - defaultUnifiUser = "influx" - defaultUnifiURL = "https://127.0.0.1:8443" - defaultHTTPListen = "0.0.0.0:9130" + // AppName is the name of the application. + AppName = "unifi-poller" + defaultUnifiUser = "influx" + defaultUnifiURL = "https://127.0.0.1:8443" ) // ENVConfigPrefix is the prefix appended to an env variable tag @@ -40,10 +31,8 @@ const ENVConfigPrefix = "UP" // UnifiPoller contains the application startup data, and auth info for UniFi & Influx. type UnifiPoller struct { - Influx *influxunifi.InfluxUnifi Flag *Flag Config *Config - LastCheck time.Time sync.Mutex // locks the Unifi struct member when re-authing to unifi. } @@ -55,6 +44,15 @@ type Flag struct { *pflag.FlagSet } +// Metrics is a type shared by the exporting and reporting packages. +type Metrics struct { + TS time.Time + unifi.Sites + unifi.IDSList + unifi.Clients + *unifi.Devices +} + // Controller represents the configuration for a UniFi Controller. // Each polled controller may have its own configuration. type Controller struct { @@ -73,16 +71,43 @@ type Controller struct { // This is all of the data stored in the config file. // Any with explicit defaults have omitempty on json and toml tags. type Config struct { - Interval config.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` - Debug bool `json:"debug" toml:"debug" xml:"debug" yaml:"debug"` - Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet" yaml:"quiet"` - InfxBadSSL bool `json:"influx_insecure_ssl" toml:"influx_insecure_ssl" xml:"influx_insecure_ssl" yaml:"influx_insecure_ssl"` - Mode string `json:"mode" toml:"mode" xml:"mode" yaml:"mode"` - HTTPListen string `json:"http_listen" toml:"http_listen" xml:"http_listen" yaml:"http_listen"` - Namespace string `json:"namespace" toml:"namespace" xml:"namespace" yaml:"namespace"` - InfluxURL string `json:"influx_url,omitempty" toml:"influx_url,omitempty" xml:"influx_url" yaml:"influx_url"` - InfluxUser string `json:"influx_user,omitempty" toml:"influx_user,omitempty" xml:"influx_user" yaml:"influx_user"` - InfluxPass string `json:"influx_pass,omitempty" toml:"influx_pass,omitempty" xml:"influx_pass" yaml:"influx_pass"` - InfluxDB string `json:"influx_db,omitempty" toml:"influx_db,omitempty" xml:"influx_db" yaml:"influx_db"` - Controllers []Controller `json:"controller,omitempty" toml:"controller,omitempty" xml:"controller" yaml:"controller"` + Poller `json:"poller" toml:"poller" xml:"poller" yaml:"poller"` + Controllers []Controller `json:"controller,omitempty" toml:"controller,omitempty" xml:"controller" yaml:"controller"` +} + +// Poller is the global config values. +type Poller struct { + Debug bool `json:"debug" toml:"debug" xml:"debug,attr" yaml:"debug"` + Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet,attr" yaml:"quiet"` +} + +// ParseConfigs parses the poller config and the config for each registered output plugin. +func (u *UnifiPoller) ParseConfigs() error { + // Parse config file. + if err := config.ParseFile(u.Config, u.Flag.ConfigFile); err != nil { + u.Flag.Usage() + return err + } + + // Update Config with ENV variable overrides. + if _, err := config.ParseENV(u.Config, ENVConfigPrefix); err != nil { + return err + } + + outputSync.Lock() + defer outputSync.Unlock() + + for _, o := range outputs { + // Parse config file for each output plugin. + if err := config.ParseFile(o.Config, u.Flag.ConfigFile); err != nil { + return err + } + + // Update Config for each output with ENV variable overrides. + if _, err := config.ParseENV(o.Config, ENVConfigPrefix); err != nil { + return err + } + } + + return nil } diff --git a/pkg/poller/influx.go b/pkg/poller/influx.go deleted file mode 100644 index 52690c13..00000000 --- a/pkg/poller/influx.go +++ /dev/null @@ -1,79 +0,0 @@ -package poller - -import ( - "fmt" - "log" - "time" - - "github.com/davidnewhall/unifi-poller/pkg/influxunifi" -) - -// GetInfluxDB returns an InfluxDB interface. -func (u *UnifiPoller) GetInfluxDB() (err error) { - if u.Influx != nil { - return nil - } - - u.Influx, err = influxunifi.New(&influxunifi.Config{ - Database: u.Config.InfluxDB, - User: u.Config.InfluxUser, - Pass: u.Config.InfluxPass, - BadSSL: u.Config.InfxBadSSL, - URL: u.Config.InfluxURL, - }) - if err != nil { - return fmt.Errorf("influxdb: %v", err) - } - - u.Logf("Logging Measurements to InfluxDB at %s as user %s", u.Config.InfluxURL, u.Config.InfluxUser) - - return nil -} - -// PollController runs forever, polling UniFi and pushing to InfluxDB -// This is started by Run() or RunBoth() after everything checks out. -func (u *UnifiPoller) PollController() { - interval := u.Config.Interval.Round(time.Second) - log.Printf("[INFO] Everything checks out! Poller started, InfluxDB interval: %v", interval) - - ticker := time.NewTicker(interval) - for u.LastCheck = range ticker.C { - if err := u.CollectAndProcess(); err != nil { - u.LogErrorf("%v", err) - } - } -} - -// CollectAndProcess collects measurements and then reports them to InfluxDB -// Can be called once or in a ticker loop. This function and all the ones below -// handle their own logging. An error is returned so the calling function may -// determine if there was a read or write error and act on it. This is currently -// called in two places in this library. One returns an error, one does not. -func (u *UnifiPoller) CollectAndProcess() error { - if err := u.GetInfluxDB(); err != nil { - return err - } - - metrics, err := u.CollectMetrics() - if err != nil { - return err - } - - report, err := u.Influx.ReportMetrics(metrics) - if err != nil { - return err - } - - u.LogInfluxReport(report) - return nil -} - -// LogInfluxReport writes a log message after exporting to influxdb. -func (u *UnifiPoller) LogInfluxReport(r *influxunifi.Report) { - idsMsg := fmt.Sprintf("IDS Events: %d, ", len(r.Metrics.IDSList)) - u.Logf("UniFi Metrics Recorded. Sites: %d, Clients: %d, "+ - "UAP: %d, USG/UDM: %d, USW: %d, %sPoints: %d, Fields: %d, Errs: %d, Elapsed: %v", - len(r.Metrics.Sites), len(r.Metrics.Clients), len(r.Metrics.UAPs), - len(r.Metrics.UDMs)+len(r.Metrics.USGs), len(r.Metrics.USWs), idsMsg, r.Total, - r.Fields, len(r.Errors), r.Elapsed.Round(time.Millisecond)) -} diff --git a/pkg/poller/outputs.go b/pkg/poller/outputs.go new file mode 100644 index 00000000..37f8fbb2 --- /dev/null +++ b/pkg/poller/outputs.go @@ -0,0 +1,72 @@ +package poller + +import ( + "fmt" + "sync" +) + +var ( + outputs []*Output + outputSync sync.Mutex +) + +// Collect is passed into output packages so they may collect metrics to output. +// Output packages must implement this interface. +type Collect interface { + Metrics() (*Metrics, error) + Logf(m string, v ...interface{}) + LogErrorf(m string, v ...interface{}) + LogDebugf(m string, v ...interface{}) +} + +// Output defines the output data for a metric exporter like influx or prometheus. +// Output packages should call NewOutput with this struct in init(). +type Output struct { + Name string + Config interface{} // Each config is passed into an unmarshaller later. + Method func(Collect) error // Called on startup for each configured output. +} + +// NewOutput should be called by each output package's init function. +func NewOutput(o *Output) { + outputSync.Lock() + defer outputSync.Unlock() + + if o == nil || o.Method == nil { + panic("nil output or method passed to poller.NewOutput") + } + + outputs = append(outputs, o) +} + +// InitializeOutputs runs all the configured output plugins. +// If none exist, or they all exit an error is returned. +func (u *UnifiPoller) InitializeOutputs() error { + v := make(chan error) + defer close(v) + + var count int + + for _, o := range outputs { + count++ + go func(o *Output) { + v <- o.Method(u) + }(o) + } + + if count < 1 { + return fmt.Errorf("no output plugins configured") + } + + for err := range v { + if err != nil { + return err + } + + if count--; count < 1 { + return fmt.Errorf("all output plugins have stopped") + } + } + + return nil +} diff --git a/pkg/poller/prometheus.go b/pkg/poller/prometheus.go deleted file mode 100644 index 13e4de49..00000000 --- a/pkg/poller/prometheus.go +++ /dev/null @@ -1,53 +0,0 @@ -package poller - -import ( - "net/http" - "strings" - "time" - - "github.com/davidnewhall/unifi-poller/pkg/metrics" - "github.com/davidnewhall/unifi-poller/pkg/promunifi" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/prometheus/common/version" -) - -const oneDecimalPoint = 10 - -// RunPrometheus starts the web server and registers the collector. -func (u *UnifiPoller) RunPrometheus() error { - u.Logf("Exporting Measurements for Prometheus at https://%s/metrics", u.Config.HTTPListen) - http.Handle("/metrics", promhttp.Handler()) - ns := strings.Replace(u.Config.Namespace, "-", "", -1) - prometheus.MustRegister(promunifi.NewUnifiCollector(promunifi.UnifiCollectorCnfg{ - Namespace: ns, - CollectFn: u.ExportMetrics, - LoggingFn: u.LogExportReport, - ReportErrors: true, // XXX: Does this need to be configurable? - })) - - version.Version = Version - prometheus.MustRegister(version.NewCollector(ns)) - - return http.ListenAndServe(u.Config.HTTPListen, nil) -} - -// ExportMetrics updates the internal metrics provided via -// HTTP at /metrics for prometheus collection. -// This is run by Prometheus as CollectFn. -func (u *UnifiPoller) ExportMetrics() (*metrics.Metrics, error) { - return u.CollectMetrics() -} - -// LogExportReport is called after prometheus exports metrics. -// This is run by Prometheus as LoggingFn -func (u *UnifiPoller) LogExportReport(report *promunifi.Report) { - m := report.Metrics - u.Logf("UniFi Measurements Exported. Site: %d, Client: %d, "+ - "UAP: %d, USG/UDM: %d, USW: %d, Descs: %d, "+ - "Metrics: %d, Errs: %d, 0s: %d, Reqs/Total: %v / %v", - len(m.Sites), len(m.Clients), len(m.UAPs), len(m.UDMs)+len(m.USGs), len(m.USWs), - report.Descs, report.Total, report.Errors, report.Zeros, - report.Fetch.Round(time.Millisecond/oneDecimalPoint), - report.Elapsed.Round(time.Millisecond/oneDecimalPoint)) -} diff --git a/pkg/poller/start.go b/pkg/poller/start.go index 2a81ac03..fa4213f5 100644 --- a/pkg/poller/start.go +++ b/pkg/poller/start.go @@ -5,26 +5,16 @@ import ( "fmt" "log" "os" - "strings" - "time" + "github.com/github/hub/version" "github.com/spf13/pflag" - "golift.io/config" ) // New returns a new poller struct preloaded with default values. // No need to call this if you call Start.c func New() *UnifiPoller { return &UnifiPoller{ - Config: &Config{ - InfluxURL: defaultInfluxURL, - InfluxUser: defaultInfluxUser, - InfluxPass: defaultInfluxPass, - InfluxDB: defaultInfluxDB, - Interval: config.Duration{Duration: defaultInterval}, - HTTPListen: defaultHTTPListen, - Namespace: appName, - }, + Config: &Config{}, Flag: &Flag{ ConfigFile: DefaultConfFile, }, @@ -40,7 +30,7 @@ func (u *UnifiPoller) Start() error { u.Flag.Parse(os.Args[1:]) if u.Flag.ShowVer { - fmt.Printf("%s v%s\n", appName, Version) + fmt.Printf("%s v%s\n", AppName, version.Version) return nil // don't run anything else w/ version request. } @@ -48,14 +38,8 @@ func (u *UnifiPoller) Start() error { u.Logf("Loading Configuration File: %s", u.Flag.ConfigFile) } - // Parse config file. - if err := config.ParseFile(u.Config, u.Flag.ConfigFile); err != nil { - u.Flag.Usage() - return err - } - - // Update Config with ENV variable overrides. - if _, err := config.ParseENV(u.Config, ENVConfigPrefix); err != nil { + // Parse config file and ENV variables. + if err := u.ParseConfigs(); err != nil { return err } @@ -78,16 +62,14 @@ func (u *UnifiPoller) Start() error { u.LogDebugf("Debug Logging Enabled") } - log.Printf("[INFO] UniFi Poller v%v Starting Up! PID: %d", Version, os.Getpid()) - return u.Run() } // Parse turns CLI arguments into data structures. Called by Start() on startup. func (f *Flag) Parse(args []string) { - f.FlagSet = pflag.NewFlagSet(appName, pflag.ExitOnError) + f.FlagSet = pflag.NewFlagSet(AppName, pflag.ExitOnError) f.Usage = func() { - fmt.Printf("Usage: %s [--config=/path/to/up.conf] [--version]", appName) + fmt.Printf("Usage: %s [--config=/path/to/up.conf] [--version]", AppName) f.PrintDefaults() } @@ -103,6 +85,8 @@ func (f *Flag) Parse(args []string) { // 2. Run the collector one time and report the metrics to influxdb. (lambda) // 3. Start a web server and wait for Prometheus to poll the application for metrics. func (u *UnifiPoller) Run() error { + log.Printf("[INFO] UniFi Poller v%v Starting Up! PID: %d", version.Version, os.Getpid()) + for i, c := range u.Config.Controllers { if c.Name == "" { u.Config.Controllers[i].Name = c.URL @@ -117,17 +101,5 @@ func (u *UnifiPoller) Run() error { } } - switch strings.ToLower(u.Config.Mode) { - default: - u.PollController() - return nil - case "influxlambda", "lambdainflux", "lambda_influx", "influx_lambda": - u.LastCheck = time.Now() - return u.CollectAndProcess() - case "both": - go u.PollController() - fallthrough - case "prometheus", "exporter": - return u.RunPrometheus() - } + return u.InitializeOutputs() } diff --git a/pkg/poller/unifi.go b/pkg/poller/unifi.go index 0e598716..516310a3 100644 --- a/pkg/poller/unifi.go +++ b/pkg/poller/unifi.go @@ -5,7 +5,6 @@ import ( "strings" "time" - "github.com/davidnewhall/unifi-poller/pkg/metrics" "golift.io/unifi" ) @@ -42,10 +41,6 @@ func (u *UnifiPoller) GetUnifi(c Controller) error { // CheckSites makes sure the list of provided sites exists on the controller. // This does not run in Lambda (run-once) mode. func (u *UnifiPoller) CheckSites(c Controller) error { - if strings.Contains(strings.ToLower(u.Config.Mode), "lambda") { - return nil // Skip this in lambda mode. - } - u.LogDebugf("Checking Controller Sites List") sites, err := c.Unifi.GetSites() @@ -58,6 +53,7 @@ func (u *UnifiPoller) CheckSites(c Controller) error { for _, site := range sites { msg = append(msg, site.Name+" ("+site.Desc+")") } + u.Logf("Found %d site(s) on controller: %v", len(msg), strings.Join(msg, ", ")) if StringInSlice("all", c.Sites) { @@ -78,10 +74,10 @@ FIRST: return nil } -// CollectMetrics grabs all the measurements from a UniFi controller and returns them. -func (u *UnifiPoller) CollectMetrics() (*metrics.Metrics, error) { +// Metrics grabs all the measurements from a UniFi controller and returns them. +func (u *UnifiPoller) Metrics() (*Metrics, error) { errs := []string{} - metrics := &metrics.Metrics{} + metrics := &Metrics{} for _, c := range u.Config.Controllers { m, err := u.checkAndPollController(c) @@ -120,7 +116,7 @@ func (u *UnifiPoller) CollectMetrics() (*metrics.Metrics, error) { return metrics, err } -func (u *UnifiPoller) checkAndPollController(c Controller) (*metrics.Metrics, error) { +func (u *UnifiPoller) checkAndPollController(c Controller) (*Metrics, error) { if c.Unifi == nil { u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) @@ -146,10 +142,10 @@ func (u *UnifiPoller) checkAndPollController(c Controller) (*metrics.Metrics, er return u.collectController(c) } -func (u *UnifiPoller) collectController(c Controller) (*metrics.Metrics, error) { +func (u *UnifiPoller) collectController(c Controller) (*Metrics, error) { var err error - m := &metrics.Metrics{TS: u.LastCheck} // At this point, it's the Current Check. + m := &Metrics{TS: time.Now()} // At this point, it's the Current Check. // Get the sites we care about. if m.Sites, err = u.GetFilteredSites(c); err != nil { @@ -178,7 +174,7 @@ func (u *UnifiPoller) collectController(c Controller) (*metrics.Metrics, error) // augmentMetrics is our middleware layer between collecting metrics and writing them. // This is where we can manipuate the returned data or make arbitrary decisions. // This function currently adds parent device names to client metrics. -func (u *UnifiPoller) augmentMetrics(c Controller, metrics *metrics.Metrics) *metrics.Metrics { +func (u *UnifiPoller) augmentMetrics(c Controller, metrics *Metrics) *Metrics { if metrics == nil || metrics.Devices == nil || metrics.Clients == nil { return metrics } diff --git a/pkg/promunifi/collector.go b/pkg/promunifi/collector.go index 5911d005..6da6db38 100644 --- a/pkg/promunifi/collector.go +++ b/pkg/promunifi/collector.go @@ -1,50 +1,59 @@ -// Package promunifi provides the bridge between unifi metrics and prometheus. +// Package promunifi provides the bridge between unifi-poller metrics and prometheus. package promunifi import ( "fmt" + "net/http" "reflect" "strings" "sync" "time" - "github.com/davidnewhall/unifi-poller/pkg/metrics" + "github.com/davidnewhall/unifi-poller/pkg/poller" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/version" "golift.io/unifi" ) -// channel buffer, fits at least one batch. -const buffer = 50 - -// simply fewer letters. -const counter = prometheus.CounterValue -const gauge = prometheus.GaugeValue - -// UnifiCollectorCnfg defines the data needed to collect and report UniFi Metrics. -type UnifiCollectorCnfg struct { - // If non-empty, each of the collected metrics is prefixed by the - // provided string and an underscore ("_"). - Namespace string - // If true, any error encountered during collection is reported as an - // invalid metric (see NewInvalidMetric). Otherwise, errors are ignored - // and the collected metrics will be incomplete. Possibly, no metrics - // will be collected at all. - ReportErrors bool - // This function is passed to the Collect() method. The Collect method runs - // this function to retrieve the latest UniFi measurements and export them. - CollectFn func() (*metrics.Metrics, error) - // Provide a logger function if you want to run a routine *after* prometheus checks in. - LoggingFn func(*Report) -} +const ( + // channel buffer, fits at least one batch. + buffer = 50 + defaultHTTPListen = "0.0.0.0:9130" + // simply fewer letters. + counter = prometheus.CounterValue + gauge = prometheus.GaugeValue +) type promUnifi struct { - Config UnifiCollectorCnfg + *Prometheus Client *uclient Device *unifiDevice UAP *uap USG *usg USW *usw Site *site + // This interface is passed to the Collect() method. The Collect method uses + // this interface to retrieve the latest UniFi measurements and export them. + Collector poller.Collect +} + +// Prometheus allows the data to be nested in the config file. +type Prometheus struct { + Config Config `json:"prometheus" toml:"prometheus" xml:"prometheus" yaml:"prometheus"` +} + +// Config is the input (config file) data used to initialize this output plugin. +type Config struct { + Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` + // If non-empty, each of the collected metrics is prefixed by the + // provided string and an underscore ("_"). + Namespace string `json:"namespace" toml:"namespace" xml:"namespace" yaml:"namespace"` + // If true, any error encountered during collection is reported as an + // invalid metric (see NewInvalidMetric). Otherwise, errors are ignored + // and the collected metrics will be incomplete. Possibly, no metrics + // will be collected at all. + ReportErrors bool `json:"report_errors" toml:"report_errors" xml:"report_errors" yaml:"report_errors"` + HTTPListen string `json:"http_listen" toml:"http_listen" xml:"http_listen" yaml:"http_listen"` } type metric struct { @@ -54,41 +63,64 @@ type metric struct { Labels []string } -// Report is passed into LoggingFn to log the export metrics to stdout (outside this package). +// Report accumulates counters that are printed to a log line. type Report struct { - Total int // Total count of metrics recorded. - Errors int // Total count of errors recording metrics. - Zeros int // Total count of metrics equal to zero. - Descs int // Total count of unique metrics descriptions. - Metrics *metrics.Metrics // Metrics collected and recorded. - Elapsed time.Duration // Duration elapsed collecting and exporting. - Fetch time.Duration // Duration elapsed making controller requests. - Start time.Time // Time collection began. + Total int // Total count of metrics recorded. + Errors int // Total count of errors recording metrics. + Zeros int // Total count of metrics equal to zero. + Metrics *poller.Metrics // Metrics collected and recorded. + Elapsed time.Duration // Duration elapsed collecting and exporting. + Fetch time.Duration // Duration elapsed making controller requests. + Start time.Time // Time collection began. ch chan []*metric wg sync.WaitGroup - cf UnifiCollectorCnfg + Config } -// NewUnifiCollector returns a prometheus collector that will export any available -// UniFi metrics. You must provide a collection function in the opts. -func NewUnifiCollector(opts UnifiCollectorCnfg) prometheus.Collector { - if opts.CollectFn == nil { - panic("nil collector function") +func init() { + u := &promUnifi{Prometheus: &Prometheus{}} + poller.NewOutput(&poller.Output{ + Name: "prometheus", + Config: u.Prometheus, + Method: u.Run, + }) +} + +// Run creates the collectors and starts the web server up. +// Should be run in a Go routine. Returns nil if not configured. +func (u *promUnifi) Run(c poller.Collect) error { + if u.Config.Disable { + return nil } - if opts.Namespace = strings.Trim(opts.Namespace, "_") + "_"; opts.Namespace == "_" { - opts.Namespace = "" + if u.Config.Namespace == "" { + u.Config.Namespace = strings.Replace(poller.AppName, "-", "", -1) } - return &promUnifi{ - Config: opts, - Client: descClient(opts.Namespace + "client_"), - Device: descDevice(opts.Namespace + "device_"), // stats for all device types. - UAP: descUAP(opts.Namespace + "device_"), - USG: descUSG(opts.Namespace + "device_"), - USW: descUSW(opts.Namespace + "device_"), - Site: descSite(opts.Namespace + "site_"), + if u.Config.HTTPListen == "" { + u.Config.HTTPListen = defaultHTTPListen } + + name := strings.Replace(u.Config.Namespace, "-", "_", -1) + + ns := name + if ns = strings.Trim(ns, "_") + "_"; ns == "_" { + ns = "" + } + + prometheus.MustRegister(version.NewCollector(name)) + prometheus.MustRegister(&promUnifi{ + Collector: c, + Client: descClient(ns + "client_"), + Device: descDevice(ns + "device_"), // stats for all device types. + UAP: descUAP(ns + "device_"), + USG: descUSG(ns + "device_"), + USW: descUSW(ns + "device_"), + Site: descSite(ns + "site_"), + }) + c.Logf("Exporting Measurements for Prometheus at https://%s/metrics, namespace: %s", u.Config.HTTPListen, u.Config.Namespace) + + return http.ListenAndServe(u.Config.HTTPListen, nil) } // Describe satisfies the prometheus Collector. This returns all of the @@ -112,10 +144,10 @@ func (u *promUnifi) Describe(ch chan<- *prometheus.Desc) { func (u *promUnifi) Collect(ch chan<- prometheus.Metric) { var err error - r := &Report{cf: u.Config, ch: make(chan []*metric, buffer), Start: time.Now()} + r := &Report{Config: u.Config, ch: make(chan []*metric, buffer), Start: time.Now()} defer r.close() - if r.Metrics, err = r.cf.CollectFn(); err != nil { + if r.Metrics, err = u.Collector.Metrics(); err != nil { r.error(ch, prometheus.NewInvalidDesc(fmt.Errorf("metric fetch failed")), err) return } @@ -135,7 +167,7 @@ func (u *promUnifi) Collect(ch chan<- prometheus.Metric) { // This is where our channels connects to the prometheus channel. func (u *promUnifi) exportMetrics(r report, ch chan<- prometheus.Metric, ourChan chan []*metric) { descs := make(map[*prometheus.Desc]bool) // used as a counter - defer r.report(descs) + defer r.report(u.Collector, descs) for newMetrics := range ourChan { for _, m := range newMetrics { diff --git a/pkg/promunifi/report.go b/pkg/promunifi/report.go index 0ddf29d3..9b6df74c 100644 --- a/pkg/promunifi/report.go +++ b/pkg/promunifi/report.go @@ -4,7 +4,7 @@ import ( "fmt" "time" - "github.com/davidnewhall/unifi-poller/pkg/metrics" + "github.com/davidnewhall/unifi-poller/pkg/poller" "github.com/prometheus/client_golang/prometheus" ) @@ -16,14 +16,15 @@ type report interface { add() done() send([]*metric) - metrics() *metrics.Metrics - report(descs map[*prometheus.Desc]bool) + metrics() *poller.Metrics + report(c poller.Collect, descs map[*prometheus.Desc]bool) export(m *metric, v float64) prometheus.Metric error(ch chan<- prometheus.Metric, d *prometheus.Desc, v interface{}) } // satisfy gomnd const one = 1 +const oneDecimalPoint = 10.0 func (r *Report) add() { r.wg.Add(one) @@ -38,17 +39,19 @@ func (r *Report) send(m []*metric) { r.ch <- m } -func (r *Report) metrics() *metrics.Metrics { +func (r *Report) metrics() *poller.Metrics { return r.Metrics } -func (r *Report) report(descs map[*prometheus.Desc]bool) { - if r.cf.LoggingFn == nil { - return - } - - r.Descs = len(descs) - r.cf.LoggingFn(r) +func (r *Report) report(c poller.Collect, descs map[*prometheus.Desc]bool) { + m := r.Metrics + c.Logf("UniFi Measurements Exported. Site: %d, Client: %d, "+ + "UAP: %d, USG/UDM: %d, USW: %d, Descs: %d, "+ + "Metrics: %d, Errs: %d, 0s: %d, Reqs/Total: %v / %v", + len(m.Sites), len(m.Clients), len(m.UAPs), len(m.UDMs)+len(m.USGs), len(m.USWs), + len(descs), r.Total, r.Errors, r.Zeros, + r.Fetch.Round(time.Millisecond/oneDecimalPoint), + r.Elapsed.Round(time.Millisecond/oneDecimalPoint)) } func (r *Report) export(m *metric, v float64) prometheus.Metric { @@ -64,7 +67,7 @@ func (r *Report) export(m *metric, v float64) prometheus.Metric { func (r *Report) error(ch chan<- prometheus.Metric, d *prometheus.Desc, v interface{}) { r.Errors++ - if r.cf.ReportErrors { + if r.Config.ReportErrors { ch <- prometheus.NewInvalidMetric(d, fmt.Errorf("error: %v", v)) } } From 072d0689d0da43fc70e238e120efb3cf2c69a1a6 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sun, 15 Dec 2019 02:57:08 -0800 Subject: [PATCH 02/32] fix import --- pkg/poller/start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/poller/start.go b/pkg/poller/start.go index fa4213f5..06069583 100644 --- a/pkg/poller/start.go +++ b/pkg/poller/start.go @@ -6,7 +6,7 @@ import ( "log" "os" - "github.com/github/hub/version" + "github.com/prometheus/common/version" "github.com/spf13/pflag" ) From a277df309690dd92fd48a282ab2cc0e4fe4a2bc2 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sun, 15 Dec 2019 02:57:53 -0800 Subject: [PATCH 03/32] update vendors --- Gopkg.lock | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 238a1b81..83212a0b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -46,12 +46,11 @@ version = "v1.0.1" [[projects]] - digest = "1:eb04f69c8991e52eff33c428bd729e04208bf03235be88e4df0d88497c6861b9" + digest = "1:7097829edd12fd7211fca0d29496b44f94ef9e6d72f88fb64f3d7b06315818ad" name = "github.com/prometheus/client_golang" packages = [ "prometheus", "prometheus/internal", - "prometheus/promhttp", ] pruneopts = "UT" revision = "170205fb58decfd011f1550d4cfb737230d7ae4f" @@ -136,7 +135,6 @@ input-imports = [ "github.com/influxdata/influxdb1-client/v2", "github.com/prometheus/client_golang/prometheus", - "github.com/prometheus/client_golang/prometheus/promhttp", "github.com/prometheus/common/version", "github.com/spf13/pflag", "golift.io/config", From 1def43a4bee3dfc33ff226679478d9e45997219d Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sun, 15 Dec 2019 03:27:34 -0800 Subject: [PATCH 04/32] trimming --- pkg/influxunifi/metrics.go | 34 +++++++++++++++++++++------------- pkg/poller/config.go | 12 ++++++------ pkg/poller/dumper.go | 8 ++++---- pkg/poller/outputs.go | 4 ++-- pkg/poller/start.go | 22 ++++++++-------------- pkg/promunifi/collector.go | 27 +++++++++++++++------------ 6 files changed, 56 insertions(+), 51 deletions(-) diff --git a/pkg/influxunifi/metrics.go b/pkg/influxunifi/metrics.go index 6775ac06..e983edce 100644 --- a/pkg/influxunifi/metrics.go +++ b/pkg/influxunifi/metrics.go @@ -10,11 +10,12 @@ import ( "github.com/davidnewhall/unifi-poller/pkg/poller" influx "github.com/influxdata/influxdb1-client/v2" - conf "golift.io/config" + "golift.io/config" ) const ( defaultInterval = 30 * time.Second + minimumInterval = 10 * time.Second defaultInfluxDB = "unifi" defaultInfluxUser = "unifi" defaultInfluxURL = "http://127.0.0.1:8086" @@ -22,13 +23,13 @@ const ( // Config defines the data needed to store metrics in InfluxDB type Config struct { - Interval conf.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` - Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` - VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` - URL string `json:"url,omitempty" toml:"url,omitempty" xml:"url" yaml:"url"` - User string `json:"user,omitempty" toml:"user,omitempty" xml:"user" yaml:"user"` - Pass string `json:"pass,omitempty" toml:"pass,omitempty" xml:"pass" yaml:"pass"` - DB string `json:"db,omitempty" toml:"db,omitempty" xml:"db" yaml:"db"` + Interval config.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` + Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` + VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` + URL string `json:"url,omitempty" toml:"url,omitempty" xml:"url" yaml:"url"` + User string `json:"user,omitempty" toml:"user,omitempty" xml:"user" yaml:"user"` + Pass string `json:"pass,omitempty" toml:"pass,omitempty" xml:"pass" yaml:"pass"` + DB string `json:"db,omitempty" toml:"db,omitempty" xml:"db" yaml:"db"` } // InfluxDB allows the data to be nested in the config file. @@ -63,9 +64,9 @@ func init() { // This is started by Run() or RunBoth() after everything checks out. func (u *InfluxUnifi) PollController() { interval := u.Config.Interval.Round(time.Second) + ticker := time.NewTicker(interval) log.Printf("[INFO] Everything checks out! Poller started, InfluxDB interval: %v", interval) - ticker := time.NewTicker(interval) for u.LastCheck = range ticker.C { metrics, err := u.Collector.Metrics() if err != nil { @@ -128,12 +129,12 @@ func (u *InfluxUnifi) setConfigDefaults() { } if u.Config.Interval.Duration == 0 { - u.Config.Interval = conf.Duration{Duration: defaultInterval} - } else if u.Config.Interval.Duration < defaultInterval/2 { - u.Config.Interval = conf.Duration{Duration: defaultInterval / 2} + u.Config.Interval = config.Duration{Duration: defaultInterval} + } else if u.Config.Interval.Duration < minimumInterval { + u.Config.Interval = config.Duration{Duration: minimumInterval} } - u.Config.Interval = conf.Duration{Duration: u.Config.Interval.Duration.Round(time.Second)} + u.Config.Interval = config.Duration{Duration: u.Config.Interval.Duration.Round(time.Second)} } // ReportMetrics batches all device and client data into influxdb data points. @@ -179,6 +180,7 @@ func (u *InfluxUnifi) collect(r report, ch chan *metric) { // to the collect routine through the metric channel. func (u *InfluxUnifi) loopPoints(r report) { m := r.metrics() + r.add() go func() { defer r.done() @@ -186,6 +188,7 @@ func (u *InfluxUnifi) loopPoints(r report) { u.batchSite(r, s) } }() + r.add() go func() { defer r.done() @@ -193,6 +196,7 @@ func (u *InfluxUnifi) loopPoints(r report) { u.batchClient(r, s) } }() + r.add() go func() { defer r.done() @@ -200,6 +204,7 @@ func (u *InfluxUnifi) loopPoints(r report) { u.batchIDS(r, s) } }() + if m.Devices == nil { return } @@ -211,6 +216,7 @@ func (u *InfluxUnifi) loopPoints(r report) { u.batchUAP(r, s) } }() + r.add() go func() { defer r.done() @@ -218,6 +224,7 @@ func (u *InfluxUnifi) loopPoints(r report) { u.batchUSG(r, s) } }() + r.add() go func() { defer r.done() @@ -225,6 +232,7 @@ func (u *InfluxUnifi) loopPoints(r report) { u.batchUSW(r, s) } }() + r.add() go func() { defer r.done() diff --git a/pkg/poller/config.go b/pkg/poller/config.go index 0d6fe126..e77db20b 100644 --- a/pkg/poller/config.go +++ b/pkg/poller/config.go @@ -31,13 +31,13 @@ const ENVConfigPrefix = "UP" // UnifiPoller contains the application startup data, and auth info for UniFi & Influx. type UnifiPoller struct { - Flag *Flag + Flags *Flags Config *Config sync.Mutex // locks the Unifi struct member when re-authing to unifi. } -// Flag represents the CLI args available and their settings. -type Flag struct { +// Flags represents the CLI args available and their settings. +type Flags struct { ConfigFile string DumpJSON string ShowVer bool @@ -84,8 +84,8 @@ type Poller struct { // ParseConfigs parses the poller config and the config for each registered output plugin. func (u *UnifiPoller) ParseConfigs() error { // Parse config file. - if err := config.ParseFile(u.Config, u.Flag.ConfigFile); err != nil { - u.Flag.Usage() + if err := config.ParseFile(u.Config, u.Flags.ConfigFile); err != nil { + u.Flags.Usage() return err } @@ -99,7 +99,7 @@ func (u *UnifiPoller) ParseConfigs() error { for _, o := range outputs { // Parse config file for each output plugin. - if err := config.ParseFile(o.Config, u.Flag.ConfigFile); err != nil { + if err := config.ParseFile(o.Config, u.Flags.ConfigFile); err != nil { return err } diff --git a/pkg/poller/dumper.go b/pkg/poller/dumper.go index 3151ae9c..5892488b 100644 --- a/pkg/poller/dumper.go +++ b/pkg/poller/dumper.go @@ -37,12 +37,12 @@ func (u *UnifiPoller) DumpJSONPayload() (err error) { switch sites, err := u.GetFilteredSites(config); { case err != nil: return err - case StringInSlice(u.Flag.DumpJSON, []string{"d", "device", "devices"}): + case StringInSlice(u.Flags.DumpJSON, []string{"d", "device", "devices"}): return u.dumpSitesJSON(config, unifi.APIDevicePath, "Devices", sites) - case StringInSlice(u.Flag.DumpJSON, []string{"client", "clients", "c"}): + case StringInSlice(u.Flags.DumpJSON, []string{"client", "clients", "c"}): return u.dumpSitesJSON(config, unifi.APIClientPath, "Clients", sites) - case strings.HasPrefix(u.Flag.DumpJSON, "other "): - apiPath := strings.SplitN(u.Flag.DumpJSON, " ", 2)[1] + case strings.HasPrefix(u.Flags.DumpJSON, "other "): + apiPath := strings.SplitN(u.Flags.DumpJSON, " ", 2)[1] _, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping Path '%s':\n", apiPath) return u.PrintRawAPIJSON(config, apiPath) default: diff --git a/pkg/poller/outputs.go b/pkg/poller/outputs.go index 37f8fbb2..aaa9a47d 100644 --- a/pkg/poller/outputs.go +++ b/pkg/poller/outputs.go @@ -55,7 +55,7 @@ func (u *UnifiPoller) InitializeOutputs() error { } if count < 1 { - return fmt.Errorf("no output plugins configured") + return fmt.Errorf("no output plugins imported") } for err := range v { @@ -64,7 +64,7 @@ func (u *UnifiPoller) InitializeOutputs() error { } if count--; count < 1 { - return fmt.Errorf("all output plugins have stopped") + return fmt.Errorf("all output plugins have stopped, or none enabled") } } diff --git a/pkg/poller/start.go b/pkg/poller/start.go index 06069583..7893f32f 100644 --- a/pkg/poller/start.go +++ b/pkg/poller/start.go @@ -10,15 +10,9 @@ import ( "github.com/spf13/pflag" ) -// New returns a new poller struct preloaded with default values. -// No need to call this if you call Start.c +// New returns a new poller struct. func New() *UnifiPoller { - return &UnifiPoller{ - Config: &Config{}, - Flag: &Flag{ - ConfigFile: DefaultConfFile, - }, - } + return &UnifiPoller{Config: &Config{}, Flags: &Flags{}} } // Start begins the application from a CLI. @@ -27,15 +21,15 @@ func New() *UnifiPoller { func (u *UnifiPoller) Start() error { log.SetOutput(os.Stdout) log.SetFlags(log.LstdFlags) - u.Flag.Parse(os.Args[1:]) + u.Flags.Parse(os.Args[1:]) - if u.Flag.ShowVer { + if u.Flags.ShowVer { fmt.Printf("%s v%s\n", AppName, version.Version) return nil // don't run anything else w/ version request. } - if u.Flag.DumpJSON == "" { // do not print this when dumping JSON. - u.Logf("Loading Configuration File: %s", u.Flag.ConfigFile) + if u.Flags.DumpJSON == "" { // do not print this when dumping JSON. + u.Logf("Loading Configuration File: %s", u.Flags.ConfigFile) } // Parse config file and ENV variables. @@ -53,7 +47,7 @@ func (u *UnifiPoller) Start() error { }} } - if u.Flag.DumpJSON != "" { + if u.Flags.DumpJSON != "" { return u.DumpJSONPayload() } @@ -66,7 +60,7 @@ func (u *UnifiPoller) Start() error { } // Parse turns CLI arguments into data structures. Called by Start() on startup. -func (f *Flag) Parse(args []string) { +func (f *Flags) Parse(args []string) { f.FlagSet = pflag.NewFlagSet(AppName, pflag.ExitOnError) f.Usage = func() { fmt.Printf("Usage: %s [--config=/path/to/up.conf] [--version]", AppName) diff --git a/pkg/promunifi/collector.go b/pkg/promunifi/collector.go index 6da6db38..61c8db81 100644 --- a/pkg/promunifi/collector.go +++ b/pkg/promunifi/collector.go @@ -65,6 +65,7 @@ type metric struct { // Report accumulates counters that are printed to a log line. type Report struct { + Config Total int // Total count of metrics recorded. Errors int // Total count of errors recording metrics. Zeros int // Total count of metrics equal to zero. @@ -74,7 +75,6 @@ type Report struct { Start time.Time // Time collection began. ch chan []*metric wg sync.WaitGroup - Config } func init() { @@ -97,27 +97,28 @@ func (u *promUnifi) Run(c poller.Collect) error { u.Config.Namespace = strings.Replace(poller.AppName, "-", "", -1) } + u.Config.Namespace = strings.Replace(u.Config.Namespace, "-", "_", -1) + if u.Config.HTTPListen == "" { u.Config.HTTPListen = defaultHTTPListen } - name := strings.Replace(u.Config.Namespace, "-", "_", -1) + prometheus.MustRegister(version.NewCollector(u.Config.Namespace)) - ns := name - if ns = strings.Trim(ns, "_") + "_"; ns == "_" { - ns = "" + if u.Config.Namespace = strings.Trim(u.Config.Namespace, "_") + "_"; u.Config.Namespace == "_" { + u.Config.Namespace = "" } - prometheus.MustRegister(version.NewCollector(name)) prometheus.MustRegister(&promUnifi{ Collector: c, - Client: descClient(ns + "client_"), - Device: descDevice(ns + "device_"), // stats for all device types. - UAP: descUAP(ns + "device_"), - USG: descUSG(ns + "device_"), - USW: descUSW(ns + "device_"), - Site: descSite(ns + "site_"), + Client: descClient(u.Config.Namespace + "client_"), + Device: descDevice(u.Config.Namespace + "device_"), // stats for all device types. + UAP: descUAP(u.Config.Namespace + "device_"), + USG: descUSG(u.Config.Namespace + "device_"), + USW: descUSW(u.Config.Namespace + "device_"), + Site: descSite(u.Config.Namespace + "site_"), }) + c.Logf("Exporting Measurements for Prometheus at https://%s/metrics, namespace: %s", u.Config.HTTPListen, u.Config.Namespace) return http.ListenAndServe(u.Config.HTTPListen, nil) @@ -185,12 +186,14 @@ func (u *promUnifi) exportMetrics(r report, ch chan<- prometheus.Metric, ourChan r.error(ch, m.Desc, fmt.Sprintf("not a number: %v", m.Value)) } } + r.done() } } func (u *promUnifi) loopExports(r report) { m := r.metrics() + r.add() go func() { defer r.done() From 4e48247f5443c30f28f6a860075341ca6bb1abdb Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sun, 15 Dec 2019 13:15:00 -0800 Subject: [PATCH 05/32] fix tests --- .metadata.sh | 2 +- pkg/poller/start.go | 2 +- pkg/promunifi/collector.go | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.metadata.sh b/.metadata.sh index 11ba0838..c4b30be1 100755 --- a/.metadata.sh +++ b/.metadata.sh @@ -11,7 +11,7 @@ HBREPO="golift/homebrew-mugs" MAINT="David Newhall II " VENDOR="Go Lift " DESC="Polls a UniFi controller, exports metrics to InfluxDB and Prometheus" -GOLANGCI_LINT_ARGS="--enable-all -D gochecknoglobals -D dupl -D lll -D funlen -D wsl -e G402" +GOLANGCI_LINT_ARGS="--enable-all -D gochecknoglobals -D dupl -D lll -D funlen -D wsl -e G402 -D gochecknoinits" # Example must exist at examples/$CONFIG_FILE.example CONFIG_FILE="up.conf" LICENSE="MIT" diff --git a/pkg/poller/start.go b/pkg/poller/start.go index 7893f32f..cd293b2f 100644 --- a/pkg/poller/start.go +++ b/pkg/poller/start.go @@ -91,7 +91,7 @@ func (u *UnifiPoller) Run() error { u.Logf("Polling UniFi Controller at %s v%s as user %s. Sites: %v", c.URL, c.Unifi.ServerVersion, c.User, c.Sites) default: - u.LogErrorf("Controller Auth or Connection failed, but continuing to retry! %s: %v", c.URL, err) + u.LogErrorf("Controller Auth or Connection failed, but continuing to retry! %s: %v", c.Name, err) } } diff --git a/pkg/promunifi/collector.go b/pkg/promunifi/collector.go index 61c8db81..e999ac3f 100644 --- a/pkg/promunifi/collector.go +++ b/pkg/promunifi/collector.go @@ -44,16 +44,16 @@ type Prometheus struct { // Config is the input (config file) data used to initialize this output plugin. type Config struct { - Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` // If non-empty, each of the collected metrics is prefixed by the // provided string and an underscore ("_"). - Namespace string `json:"namespace" toml:"namespace" xml:"namespace" yaml:"namespace"` + Namespace string `json:"namespace" toml:"namespace" xml:"namespace" yaml:"namespace"` + HTTPListen string `json:"http_listen" toml:"http_listen" xml:"http_listen" yaml:"http_listen"` // If true, any error encountered during collection is reported as an // invalid metric (see NewInvalidMetric). Otherwise, errors are ignored // and the collected metrics will be incomplete. Possibly, no metrics // will be collected at all. - ReportErrors bool `json:"report_errors" toml:"report_errors" xml:"report_errors" yaml:"report_errors"` - HTTPListen string `json:"http_listen" toml:"http_listen" xml:"http_listen" yaml:"http_listen"` + ReportErrors bool `json:"report_errors" toml:"report_errors" xml:"report_errors" yaml:"report_errors"` + Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` } type metric struct { From 0b8473657e4c1983dd642323a7108e546aeedd50 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sun, 15 Dec 2019 20:56:42 -0800 Subject: [PATCH 06/32] convert input to plugin --- .metadata.sh | 2 +- main.go | 4 +- pkg/influxunifi/clients.go | 9 +- pkg/influxunifi/ids.go | 1 + pkg/influxunifi/{metrics.go => influxdb.go} | 36 ++- pkg/influxunifi/site.go | 1 + pkg/influxunifi/uap.go | 7 + pkg/influxunifi/udm.go | 12 +- pkg/influxunifi/usg.go | 38 +-- pkg/influxunifi/usw.go | 6 + pkg/inputunifi/collector.go | 138 +++++++++++ pkg/inputunifi/input.go | 78 +++++++ pkg/inputunifi/interface.go | 122 ++++++++++ pkg/poller/build_macos.go | 2 +- pkg/poller/build_unix.go | 2 +- pkg/poller/build_windows.go | 2 +- pkg/poller/config.go | 74 +++--- pkg/poller/dumper.go | 87 ++++--- pkg/poller/inputs.go | 96 ++++++++ pkg/poller/{helpers.go => logger.go} | 14 +- pkg/poller/outputs.go | 5 +- pkg/poller/start.go | 42 +--- pkg/poller/unifi.go | 241 -------------------- pkg/promunifi/clients.go | 48 ++-- pkg/promunifi/collector.go | 44 ++-- pkg/promunifi/site.go | 56 +++-- pkg/promunifi/uap.go | 139 +++++------ pkg/promunifi/usg.go | 4 + pkg/promunifi/usw.go | 73 +++--- 29 files changed, 797 insertions(+), 586 deletions(-) rename pkg/influxunifi/{metrics.go => influxdb.go} (97%) create mode 100644 pkg/inputunifi/collector.go create mode 100644 pkg/inputunifi/input.go create mode 100644 pkg/inputunifi/interface.go create mode 100644 pkg/poller/inputs.go rename pkg/poller/{helpers.go => logger.go} (75%) delete mode 100644 pkg/poller/unifi.go diff --git a/.metadata.sh b/.metadata.sh index c4b30be1..7a78171d 100755 --- a/.metadata.sh +++ b/.metadata.sh @@ -11,7 +11,7 @@ HBREPO="golift/homebrew-mugs" MAINT="David Newhall II " VENDOR="Go Lift " DESC="Polls a UniFi controller, exports metrics to InfluxDB and Prometheus" -GOLANGCI_LINT_ARGS="--enable-all -D gochecknoglobals -D dupl -D lll -D funlen -D wsl -e G402 -D gochecknoinits" +GOLANGCI_LINT_ARGS="--enable-all -D gochecknoglobals -D funlen -e G402 -D gochecknoinits" # Example must exist at examples/$CONFIG_FILE.example CONFIG_FILE="up.conf" LICENSE="MIT" diff --git a/main.go b/main.go index be0fe13b..24315b5d 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,9 @@ import ( "log" "github.com/davidnewhall/unifi-poller/pkg/poller" - // Enable output plugins! + // Load input plugins! + _ "github.com/davidnewhall/unifi-poller/pkg/inputunifi" + // Load output plugins! _ "github.com/davidnewhall/unifi-poller/pkg/influxunifi" _ "github.com/davidnewhall/unifi-poller/pkg/promunifi" ) diff --git a/pkg/influxunifi/clients.go b/pkg/influxunifi/clients.go index 127ab7e2..b4c346dc 100644 --- a/pkg/influxunifi/clients.go +++ b/pkg/influxunifi/clients.go @@ -66,14 +66,7 @@ func (u *InfluxUnifi) batchClient(r report, s *unifi.Client) { "wired-tx_bytes": s.WiredTxBytes, "wired-tx_bytes-r": s.WiredTxBytesR, "wired-tx_packets": s.WiredTxPackets, - /* - "dpi_app": c.DpiStats.App.Val, - "dpi_cat": c.DpiStats.Cat.Val, - "dpi_rx_bytes": c.DpiStats.RxBytes.Val, - "dpi_rx_packets": c.DpiStats.RxPackets.Val, - "dpi_tx_bytes": c.DpiStats.TxBytes.Val, - "dpi_tx_packets": c.DpiStats.TxPackets.Val, - */ } + r.send(&metric{Table: "clients", Tags: tags, Fields: fields}) } diff --git a/pkg/influxunifi/ids.go b/pkg/influxunifi/ids.go index c7b8edba..ad0b855a 100644 --- a/pkg/influxunifi/ids.go +++ b/pkg/influxunifi/ids.go @@ -35,5 +35,6 @@ func (u *InfluxUnifi) batchIDS(r report, i *unifi.IDS) { "srcipASN": i.SrcipASN, "usgipASN": i.UsgipASN, } + r.send(&metric{Table: "intrusion_detect", Tags: tags, Fields: fields}) } diff --git a/pkg/influxunifi/metrics.go b/pkg/influxunifi/influxdb.go similarity index 97% rename from pkg/influxunifi/metrics.go rename to pkg/influxunifi/influxdb.go index e983edce..199ea594 100644 --- a/pkg/influxunifi/metrics.go +++ b/pkg/influxunifi/influxdb.go @@ -17,7 +17,7 @@ const ( defaultInterval = 30 * time.Second minimumInterval = 10 * time.Second defaultInfluxDB = "unifi" - defaultInfluxUser = "unifi" + defaultInfluxUser = "unifipoller" defaultInfluxURL = "http://127.0.0.1:8086" ) @@ -53,6 +53,7 @@ type metric struct { func init() { u := &InfluxUnifi{InfluxDB: &InfluxDB{}, LastCheck: time.Now()} + poller.NewOutput(&poller.Output{ Name: "influxdb", Config: u.InfluxDB, @@ -143,9 +144,12 @@ func (u *InfluxUnifi) setConfigDefaults() { func (u *InfluxUnifi) ReportMetrics(m *poller.Metrics) (*Report, error) { r := &Report{Metrics: m, ch: make(chan *metric), Start: time.Now()} defer close(r.ch) - // Make a new Influx Points Batcher. + var err error + + // Make a new Influx Points Batcher. r.bp, err = influx.NewBatchPoints(influx.BatchPointsConfig{Database: u.Config.DB}) + if err != nil { return nil, fmt.Errorf("influx.NewBatchPoints: %v", err) } @@ -159,7 +163,9 @@ func (u *InfluxUnifi) ReportMetrics(m *poller.Metrics) (*Report, error) { if err = u.influx.Write(r.bp); err != nil { return nil, fmt.Errorf("influxdb.Write(points): %v", err) } + r.Elapsed = time.Since(r.Start) + return r, nil } @@ -172,6 +178,7 @@ func (u *InfluxUnifi) collect(r report, ch chan *metric) { } else { r.batch(m, pt) } + r.done() } } @@ -182,24 +189,28 @@ func (u *InfluxUnifi) loopPoints(r report) { m := r.metrics() r.add() + r.add() + r.add() + go func() { defer r.done() + for _, s := range m.Sites { u.batchSite(r, s) } }() - r.add() go func() { defer r.done() + for _, s := range m.Clients { u.batchClient(r, s) } }() - r.add() go func() { defer r.done() + for _, s := range m.IDSList { u.batchIDS(r, s) } @@ -209,33 +220,44 @@ func (u *InfluxUnifi) loopPoints(r report) { return } + u.loopDevicePoints(r) +} + +func (u *InfluxUnifi) loopDevicePoints(r report) { + m := r.metrics() + r.add() + r.add() + r.add() + r.add() + go func() { defer r.done() + for _, s := range m.UAPs { u.batchUAP(r, s) } }() - r.add() go func() { defer r.done() + for _, s := range m.USGs { u.batchUSG(r, s) } }() - r.add() go func() { defer r.done() + for _, s := range m.USWs { u.batchUSW(r, s) } }() - r.add() go func() { defer r.done() + for _, s := range m.UDMs { u.batchUDM(r, s) } diff --git a/pkg/influxunifi/site.go b/pkg/influxunifi/site.go index 243d2acc..30e2ce37 100644 --- a/pkg/influxunifi/site.go +++ b/pkg/influxunifi/site.go @@ -51,6 +51,7 @@ func (u *InfluxUnifi) batchSite(r report, s *unifi.Site) { "remote_user_tx_packets": h.RemoteUserTxPackets.Val, "num_new_alarms": s.NumNewAlarms.Val, } + r.send(&metric{Table: "subsystems", Tags: tags, Fields: fields}) } } diff --git a/pkg/influxunifi/uap.go b/pkg/influxunifi/uap.go index 46c25c32..f47a11fc 100644 --- a/pkg/influxunifi/uap.go +++ b/pkg/influxunifi/uap.go @@ -10,6 +10,7 @@ func (u *InfluxUnifi) batchUAP(r report, s *unifi.UAP) { if !s.Adopted.Val || s.Locating.Val { return } + tags := map[string]string{ "mac": s.Mac, "site_name": s.SiteName, @@ -30,6 +31,7 @@ func (u *InfluxUnifi) batchUAP(r report, s *unifi.UAP) { fields["user-num_sta"] = int(s.UserNumSta.Val) fields["guest-num_sta"] = int(s.GuestNumSta.Val) fields["num_sta"] = s.NumSta.Val + r.send(&metric{Table: "uap", Tags: tags, Fields: fields}) u.processRadTable(r, tags, s.RadioTable, s.RadioTableStats) u.processVAPTable(r, tags, s.VapTable) @@ -39,6 +41,7 @@ func (u *InfluxUnifi) processUAPstats(ap *unifi.Ap) map[string]interface{} { if ap == nil { return map[string]interface{}{} } + // Accumulative Statistics. return map[string]interface{}{ "stat_user-rx_packets": ap.UserRxPackets.Val, @@ -135,6 +138,7 @@ func (u *InfluxUnifi) processVAPTable(r report, t map[string]string, vt unifi.Va "wifi_tx_latency_mov_total": s.WifiTxLatencyMov.Total.Val, "wifi_tx_latency_mov_cuont": s.WifiTxLatencyMov.TotalCount.Val, } + r.send(&metric{Table: "uap_vaps", Tags: tags, Fields: fields}) } } @@ -155,6 +159,7 @@ func (u *InfluxUnifi) processRadTable(r report, t map[string]string, rt unifi.Ra "nss": p.Nss.Val, "radio_caps": p.RadioCaps.Val, } + for _, t := range rts { if t.Name == p.Name { fields["ast_be_xmit"] = t.AstBeXmit.Val @@ -171,9 +176,11 @@ func (u *InfluxUnifi) processRadTable(r report, t map[string]string, rt unifi.Ra fields["tx_power"] = t.TxPower.Val fields["tx_retries"] = t.TxRetries.Val fields["user-num_sta"] = t.UserNumSta.Val + break } } + r.send(&metric{Table: "uap_radios", Tags: tags, Fields: fields}) } } diff --git a/pkg/influxunifi/udm.go b/pkg/influxunifi/udm.go index af42e5d0..20cea055 100644 --- a/pkg/influxunifi/udm.go +++ b/pkg/influxunifi/udm.go @@ -4,14 +4,16 @@ import ( "golift.io/unifi" ) -// Combines concatenates N maps. This will delete things if not used with caution. +// Combine concatenates N maps. This will delete things if not used with caution. func Combine(in ...map[string]interface{}) map[string]interface{} { out := make(map[string]interface{}) + for i := range in { for k := range in[i] { out[k] = in[i][k] } } + return out } @@ -36,6 +38,7 @@ func (u *InfluxUnifi) batchUDM(r report, s *unifi.UDM) { if !s.Adopted.Val || s.Locating.Val { return } + tags := map[string]string{ "mac": s.Mac, "site_name": s.SiteName, @@ -65,6 +68,7 @@ func (u *InfluxUnifi) batchUDM(r report, s *unifi.UDM) { "num_mobile": s.NumMobile.Val, }, ) + r.send(&metric{Table: "usg", Tags: tags, Fields: fields}) u.batchNetTable(r, tags, s.NetworkTable) u.batchUSGwans(r, tags, s.Wan1, s.Wan2) @@ -90,13 +94,14 @@ func (u *InfluxUnifi) batchUDM(r report, s *unifi.UDM) { "uptime": s.Uptime.Val, "state": s.State.Val, }) + r.send(&metric{Table: "usw", Tags: tags, Fields: fields}) u.batchPortTable(r, tags, s.PortTable) if s.Stat.Ap == nil { - return - // we're done now. the following code process UDM (non-pro) UAP data. + return // we're done now. the following code process UDM (non-pro) UAP data. } + tags = map[string]string{ "mac": s.Mac, "site_name": s.SiteName, @@ -117,6 +122,7 @@ func (u *InfluxUnifi) batchUDM(r report, s *unifi.UDM) { fields["user-num_sta"] = int(s.UserNumSta.Val) fields["guest-num_sta"] = int(s.GuestNumSta.Val) fields["num_sta"] = s.NumSta.Val + r.send(&metric{Table: "uap", Tags: tags, Fields: fields}) u.processRadTable(r, tags, *s.RadioTable, *s.RadioTableStats) u.processVAPTable(r, tags, *s.VapTable) diff --git a/pkg/influxunifi/usg.go b/pkg/influxunifi/usg.go index 221e0e40..36c9cdbd 100644 --- a/pkg/influxunifi/usg.go +++ b/pkg/influxunifi/usg.go @@ -10,6 +10,7 @@ func (u *InfluxUnifi) batchUSG(r report, s *unifi.USG) { if !s.Adopted.Val || s.Locating.Val { return } + tags := map[string]string{ "mac": s.Mac, "site_name": s.SiteName, @@ -39,44 +40,17 @@ func (u *InfluxUnifi) batchUSG(r report, s *unifi.USG) { "num_mobile": s.NumMobile.Val, }, ) + r.send(&metric{Table: "usg", Tags: tags, Fields: fields}) u.batchNetTable(r, tags, s.NetworkTable) u.batchUSGwans(r, tags, s.Wan1, s.Wan2) - - /* - for _, p := range s.PortTable { - t := map[string]string{ - "device_name": tags["name"], - "site_name": tags["site_name"], - "name": p.Name, - "ifname": p.Ifname, - "ip": p.IP, - "mac": p.Mac, - "up": p.Up.Txt, - "speed": p.Speed.Txt, - "full_duplex": p.FullDuplex.Txt, - "enable": p.Enable.Txt, - } - f := map[string]interface{}{ - "rx_bytes": p.RxBytes.Val, - "rx_dropped": p.RxDropped.Val, - "rx_errors": p.RxErrors.Val, - "rx_packets": p.RxBytes.Val, - "tx_bytes": p.TxBytes.Val, - "tx_dropped": p.TxDropped.Val, - "tx_errors": p.TxErrors.Val, - "tx_packets": p.TxPackets.Val, - "rx_multicast": p.RxMulticast.Val, - "dns_servers": strings.Join(p.DNS, ","), - } - r.send(&metric{Table: "usg_ports", Tags: t, Fields: f}) - } - */ } + func (u *InfluxUnifi) batchUSGstat(ss unifi.SpeedtestStatus, gw *unifi.Gw, ul unifi.Uplink) map[string]interface{} { if gw == nil { return map[string]interface{}{} } + return map[string]interface{}{ "uplink_latency": ul.Latency.Val, "uplink_speed": ul.Speed.Val, @@ -92,11 +66,13 @@ func (u *InfluxUnifi) batchUSGstat(ss unifi.SpeedtestStatus, gw *unifi.Gw, ul un "lan-rx_dropped": gw.LanRxDropped.Val, } } + func (u *InfluxUnifi) batchUSGwans(r report, tags map[string]string, wans ...unifi.Wan) { for _, wan := range wans { if !wan.Up.Val { continue } + tags := map[string]string{ "device_name": tags["name"], "site_name": tags["site_name"], @@ -129,6 +105,7 @@ func (u *InfluxUnifi) batchUSGwans(r report, tags map[string]string, wans ...uni "tx_broadcast": wan.TxBroadcast.Val, "tx_multicast": wan.TxMulticast.Val, } + r.send(&metric{Table: "usg_wan_ports", Tags: tags, Fields: fields}) } } @@ -154,6 +131,7 @@ func (u *InfluxUnifi) batchNetTable(r report, tags map[string]string, nt unifi.N "tx_bytes": p.TxBytes.Val, "tx_packets": p.TxPackets.Val, } + r.send(&metric{Table: "usg_networks", Tags: tags, Fields: fields}) } } diff --git a/pkg/influxunifi/usw.go b/pkg/influxunifi/usw.go index 7bc31c37..0a91a506 100644 --- a/pkg/influxunifi/usw.go +++ b/pkg/influxunifi/usw.go @@ -36,6 +36,7 @@ func (u *InfluxUnifi) batchUSW(r report, s *unifi.USW) { "state": s.State.Val, "user-num_sta": s.UserNumSta.Val, }) + r.send(&metric{Table: "usw", Tags: tags, Fields: fields}) u.batchPortTable(r, tags, s.PortTable) } @@ -44,6 +45,7 @@ func (u *InfluxUnifi) batchUSWstat(sw *unifi.Sw) map[string]interface{} { if sw == nil { return map[string]interface{}{} } + return map[string]interface{}{ "stat_bytes": sw.Bytes.Val, "stat_rx_bytes": sw.RxBytes.Val, @@ -59,11 +61,13 @@ func (u *InfluxUnifi) batchUSWstat(sw *unifi.Sw) map[string]interface{} { "stat_tx_retries": sw.TxRetries.Val, } } + func (u *InfluxUnifi) batchPortTable(r report, t map[string]string, pt []unifi.Port) { for _, p := range pt { if !p.Up.Val || !p.Enable.Val { continue // only record UP ports. } + tags := map[string]string{ "site_name": t["site_name"], "device_name": t["name"], @@ -96,11 +100,13 @@ func (u *InfluxUnifi) batchPortTable(r report, t map[string]string, pt []unifi.P "tx_multicast": p.TxMulticast.Val, "tx_packets": p.TxPackets.Val, } + if p.PoeEnable.Val && p.PortPoe.Val { fields["poe_current"] = p.PoeCurrent.Val fields["poe_power"] = p.PoePower.Val fields["poe_voltage"] = p.PoeVoltage.Val } + r.send(&metric{Table: "usw_ports", Tags: tags, Fields: fields}) } } diff --git a/pkg/inputunifi/collector.go b/pkg/inputunifi/collector.go new file mode 100644 index 00000000..20aeafac --- /dev/null +++ b/pkg/inputunifi/collector.go @@ -0,0 +1,138 @@ +package inputunifi + +import ( + "fmt" + "time" + + "github.com/davidnewhall/unifi-poller/pkg/poller" + "golift.io/unifi" +) + +func (u *InputUnifi) isNill(c Controller) bool { + u.Config.RLock() + defer u.Config.RUnlock() + + return c.Unifi == nil +} + +func (u *InputUnifi) collectController(c Controller) (*poller.Metrics, error) { + if u.isNill(c) { + u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) + + if err := u.getUnifi(c); err != nil { + return nil, fmt.Errorf("re-authenticating to %s: %v", c.Name, err) + } + } + + m, err := u.pollController(c) + if err == nil { + return m, nil + } + + return u.pollController(c) +} + +func (u *InputUnifi) pollController(c Controller) (*poller.Metrics, error) { + var err error + + u.Config.RLock() + defer u.Config.RUnlock() + + m := &poller.Metrics{TS: time.Now()} // At this point, it's the Current Check. + + // Get the sites we care about. + if m.Sites, err = u.getFilteredSites(c); err != nil { + return m, fmt.Errorf("unifi.GetSites(%v): %v", c.URL, err) + } + + if c.SaveIDS { + m.IDSList, err = c.Unifi.GetIDS(m.Sites, time.Now().Add(2*time.Minute), time.Now()) + if err != nil { + return m, fmt.Errorf("unifi.GetIDS(%v): %v", c.URL, err) + } + } + + // Get all the points. + if m.Clients, err = c.Unifi.GetClients(m.Sites); err != nil { + return m, fmt.Errorf("unifi.GetClients(%v): %v", c.URL, err) + } + + if m.Devices, err = c.Unifi.GetDevices(m.Sites); err != nil { + return m, fmt.Errorf("unifi.GetDevices(%v): %v", c.URL, err) + } + + return u.augmentMetrics(c, m), nil +} + +// augmentMetrics is our middleware layer between collecting metrics and writing them. +// This is where we can manipuate the returned data or make arbitrary decisions. +// This function currently adds parent device names to client metrics. +func (u *InputUnifi) augmentMetrics(c Controller, metrics *poller.Metrics) *poller.Metrics { + if metrics == nil || metrics.Devices == nil || metrics.Clients == nil { + return metrics + } + + devices := make(map[string]string) + bssdIDs := make(map[string]string) + + for _, r := range metrics.UAPs { + devices[r.Mac] = r.Name + + for _, v := range r.VapTable { + bssdIDs[v.Bssid] = fmt.Sprintf("%s %s %s:", r.Name, v.Radio, v.RadioName) + } + } + + for _, r := range metrics.USGs { + devices[r.Mac] = r.Name + } + + for _, r := range metrics.USWs { + devices[r.Mac] = r.Name + } + + for _, r := range metrics.UDMs { + devices[r.Mac] = r.Name + } + + // These come blank, so set them here. + for i, c := range metrics.Clients { + metrics.Clients[i].SwName = devices[c.SwMac] + metrics.Clients[i].ApName = devices[c.ApMac] + metrics.Clients[i].GwName = devices[c.GwMac] + metrics.Clients[i].RadioDescription = bssdIDs[metrics.Clients[i].Bssid] + metrics.Clients[i].RadioProto + } + + if !c.SaveSites { + metrics.Sites = nil + } + + return metrics +} + +// getFilteredSites returns a list of sites to fetch data for. +// Omits requested but unconfigured sites. Grabs the full list from the +// controller and returns the sites provided in the config file. +func (u *InputUnifi) getFilteredSites(c Controller) (unifi.Sites, error) { + u.Config.RLock() + defer u.Config.RUnlock() + + sites, err := c.Unifi.GetSites() + if err != nil { + return nil, err + } else if len(c.Sites) < 1 || poller.StringInSlice("all", c.Sites) { + return sites, nil + } + + var i int + + for _, s := range sites { + // Only include valid sites in the request filter. + if poller.StringInSlice(s.Name, c.Sites) { + sites[i] = s + i++ + } + } + + return sites[:i], nil +} diff --git a/pkg/inputunifi/input.go b/pkg/inputunifi/input.go new file mode 100644 index 00000000..f797d35b --- /dev/null +++ b/pkg/inputunifi/input.go @@ -0,0 +1,78 @@ +// Package inputunifi implements the poller.Input interface and bridges the gap between +// metrics from the unifi library, and the augments required to pump them into unifi-poller. +package inputunifi + +import ( + "fmt" + + "sync" + + "github.com/davidnewhall/unifi-poller/pkg/poller" + "golift.io/unifi" +) + +// InputUnifi contains the running data. +type InputUnifi struct { + Config Config `json:"unifi" toml:"unifi" xml:"unifi" yaml:"unifi"` + poller.Logger +} + +// Controller represents the configuration for a UniFi Controller. +// Each polled controller may have its own configuration. +type Controller struct { + VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` + SaveIDS bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` + SaveSites bool `json:"save_sites" toml:"save_sites" xml:"save_sites" yaml:"save_sites"` + Name string `json:"name" toml:"name" xml:"name,attr" yaml:"name"` + User string `json:"user" toml:"user" xml:"user" yaml:"user"` + Pass string `json:"pass" toml:"pass" xml:"pass" yaml:"pass"` + URL string `json:"url" toml:"url" xml:"url" yaml:"url"` + Sites []string `json:"sites,omitempty" toml:"sites,omitempty" xml:"sites" yaml:"sites"` + Unifi *unifi.Unifi `json:"-" toml:"-" xml:"-" yaml:"-"` +} + +// Config contains our configuration data +type Config struct { + sync.RWMutex // locks the Unifi struct member when re-authing to unifi. + Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` + Controllers []Controller `json:"controller" toml:"controller" xml:"controller" yaml:"controller"` +} + +func init() { + u := &InputUnifi{} + + poller.NewInput(&poller.InputPlugin{ + Input: u, // this library implements poller.Input interface for Metrics(). + Config: u, // Defines our config data interface. + }) +} + +// getUnifi (re-)authenticates to a unifi controller. +func (u *InputUnifi) getUnifi(c Controller) error { + var err error + + u.Config.Lock() + defer u.Config.Unlock() + + if c.Unifi != nil { + c.Unifi.CloseIdleConnections() + } + + // Create an authenticated session to the Unifi Controller. + c.Unifi, err = unifi.NewUnifi(&unifi.Config{ + User: c.User, + Pass: c.Pass, + URL: c.URL, + VerifySSL: c.VerifySSL, + ErrorLog: u.LogErrorf, // Log all errors. + DebugLog: u.LogDebugf, // Log debug messages. + }) + if err != nil { + c.Unifi = nil + return fmt.Errorf("unifi controller: %v", err) + } + + u.LogDebugf("Authenticated with controller successfully, %s", c.URL) + + return nil +} diff --git a/pkg/inputunifi/interface.go b/pkg/inputunifi/interface.go new file mode 100644 index 00000000..a2ef9765 --- /dev/null +++ b/pkg/inputunifi/interface.go @@ -0,0 +1,122 @@ +package inputunifi + +import ( + "fmt" + "strings" + + "github.com/davidnewhall/unifi-poller/pkg/poller" + "golift.io/unifi" +) + +// Metrics grabs all the measurements from a UniFi controller and returns them. +func (u *InputUnifi) Metrics() (*poller.Metrics, error) { + errs := []string{} + metrics := &poller.Metrics{} + + for _, c := range u.Config.Controllers { + m, err := u.collectController(c) + if err != nil { + errs = append(errs, err.Error()) + } + + if m == nil { + continue + } + + metrics.Sites = append(metrics.Sites, m.Sites...) + metrics.Clients = append(metrics.Clients, m.Clients...) + metrics.IDSList = append(metrics.IDSList, m.IDSList...) + + if m.Devices == nil { + continue + } + + if metrics.Devices == nil { + metrics.Devices = &unifi.Devices{} + } + + metrics.UAPs = append(metrics.UAPs, m.UAPs...) + metrics.USGs = append(metrics.USGs, m.USGs...) + metrics.USWs = append(metrics.USWs, m.USWs...) + metrics.UDMs = append(metrics.UDMs, m.UDMs...) + } + + if len(errs) > 0 { + return metrics, fmt.Errorf(strings.Join(errs, ", ")) + } + + return metrics, nil +} + +// Initialize gets called one time when starting up. +// Satisfies poller.Input interface. +func (u *InputUnifi) Initialize(l poller.Logger) error { + if u.Config.Disable { + l.Logf("unifi input disabled") + return nil + } + + if len(u.Config.Controllers) < 1 { + return fmt.Errorf("no unifi controllers defined for unifi input") + } + + u.Logger = l + + for i, c := range u.Config.Controllers { + if c.Name == "" { + u.Config.Controllers[i].Name = c.URL + } + + switch err := u.getUnifi(c); err { + case nil: + if err := u.checkSites(c); err != nil { + u.LogErrorf("checking sites on %s: %v", c.Name, err) + } + + u.Logf("Polling UniFi Controller at %s v%s as user %s. Sites: %v", + c.URL, c.Unifi.ServerVersion, c.User, c.Sites) + default: + u.LogErrorf("Controller Auth or Connection failed, but continuing to retry! %s: %v", c.Name, err) + } + } + + return nil +} + +// checkSites makes sure the list of provided sites exists on the controller. +// This only runs once during initialization. +func (u *InputUnifi) checkSites(c Controller) error { + u.Config.RLock() + defer u.Config.RUnlock() + u.LogDebugf("Checking Controller Sites List") + + sites, err := c.Unifi.GetSites() + if err != nil { + return err + } + + msg := []string{} + + for _, site := range sites { + msg = append(msg, site.Name+" ("+site.Desc+")") + } + + u.Logf("Found %d site(s) on controller: %v", len(msg), strings.Join(msg, ", ")) + + if poller.StringInSlice("all", c.Sites) { + c.Sites = []string{"all"} + return nil + } + +FIRST: + for _, s := range c.Sites { + for _, site := range sites { + if s == site.Name { + continue FIRST + } + } + return fmt.Errorf("configured site not found on controller: %v", s) + } + + return nil +} diff --git a/pkg/poller/build_macos.go b/pkg/poller/build_macos.go index 1ab32471..b3f37dbf 100644 --- a/pkg/poller/build_macos.go +++ b/pkg/poller/build_macos.go @@ -2,5 +2,5 @@ package poller -// DefaultConfFile is where to find config is --config is not prvided. +// DefaultConfFile is where to find config if --config is not prvided. const DefaultConfFile = "/usr/local/etc/unifi-poller/up.conf" diff --git a/pkg/poller/build_unix.go b/pkg/poller/build_unix.go index c1001ac9..c1f525a9 100644 --- a/pkg/poller/build_unix.go +++ b/pkg/poller/build_unix.go @@ -2,5 +2,5 @@ package poller -// DefaultConfFile is where to find config is --config is not prvided. +// DefaultConfFile is where to find config if --config is not prvided. const DefaultConfFile = "/etc/unifi-poller/up.conf" diff --git a/pkg/poller/build_windows.go b/pkg/poller/build_windows.go index 5c31504f..a74c76a8 100644 --- a/pkg/poller/build_windows.go +++ b/pkg/poller/build_windows.go @@ -2,5 +2,5 @@ package poller -// DefaultConfFile is where to find config is --config is not prvided. +// DefaultConfFile is where to find config if --config is not prvided. const DefaultConfFile = `C:\ProgramData\unifi-poller\up.conf` diff --git a/pkg/poller/config.go b/pkg/poller/config.go index e77db20b..03e69b81 100644 --- a/pkg/poller/config.go +++ b/pkg/poller/config.go @@ -9,7 +9,6 @@ package poller */ import ( - "sync" "time" "github.com/spf13/pflag" @@ -17,23 +16,17 @@ import ( "golift.io/unifi" ) -// App defaults in case they're missing from the config. const ( // AppName is the name of the application. - AppName = "unifi-poller" - defaultUnifiUser = "influx" - defaultUnifiURL = "https://127.0.0.1:8443" + AppName = "unifi-poller" + // ENVConfigPrefix is the prefix appended to an env variable tag name. + ENVConfigPrefix = "UP" ) -// ENVConfigPrefix is the prefix appended to an env variable tag -// name before retrieving the value from the OS. -const ENVConfigPrefix = "UP" - // UnifiPoller contains the application startup data, and auth info for UniFi & Influx. type UnifiPoller struct { - Flags *Flags - Config *Config - sync.Mutex // locks the Unifi struct member when re-authing to unifi. + Flags *Flags + *Config } // Flags represents the CLI args available and their settings. @@ -53,26 +46,9 @@ type Metrics struct { *unifi.Devices } -// Controller represents the configuration for a UniFi Controller. -// Each polled controller may have its own configuration. -type Controller struct { - VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` - SaveIDS bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` - SaveSites bool `json:"save_sites,omitempty" toml:"save_sites,omitempty" xml:"save_sites" yaml:"save_sites"` - Name string `json:"name" toml:"name" xml:"name,attr" yaml:"name"` - User string `json:"user,omitempty" toml:"user,omitempty" xml:"user" yaml:"user"` - Pass string `json:"pass,omitempty" toml:"pass,omitempty" xml:"pass" yaml:"pass"` - URL string `json:"url,omitempty" toml:"url,omitempty" xml:"url" yaml:"url"` - Sites []string `json:"sites,omitempty" toml:"sites,omitempty" xml:"sites" yaml:"sites"` - Unifi *unifi.Unifi `json:"-" toml:"-" xml:"-" yaml:"-"` -} - -// Config represents the data needed to poll a controller and report to influxdb. -// This is all of the data stored in the config file. -// Any with explicit defaults have omitempty on json and toml tags. +// Config represents the core library input data. type Config struct { - Poller `json:"poller" toml:"poller" xml:"poller" yaml:"poller"` - Controllers []Controller `json:"controller,omitempty" toml:"controller,omitempty" xml:"controller" yaml:"controller"` + Poller `json:"poller" toml:"poller" xml:"poller" yaml:"poller"` } // Poller is the global config values. @@ -83,31 +59,43 @@ type Poller struct { // ParseConfigs parses the poller config and the config for each registered output plugin. func (u *UnifiPoller) ParseConfigs() error { - // Parse config file. - if err := config.ParseFile(u.Config, u.Flags.ConfigFile); err != nil { - u.Flags.Usage() - return err - } - - // Update Config with ENV variable overrides. - if _, err := config.ParseENV(u.Config, ENVConfigPrefix); err != nil { + // Parse core config. + if err := u.ParseInterface(u.Config); err != nil { return err } + // Parse output plugin configs. outputSync.Lock() defer outputSync.Unlock() for _, o := range outputs { - // Parse config file for each output plugin. - if err := config.ParseFile(o.Config, u.Flags.ConfigFile); err != nil { + if err := u.ParseInterface(o.Config); err != nil { return err } + } - // Update Config for each output with ENV variable overrides. - if _, err := config.ParseENV(o.Config, ENVConfigPrefix); err != nil { + // Parse input plugin configs. + inputSync.Lock() + defer inputSync.Unlock() + + for _, i := range inputs { + if err := u.ParseInterface(i.Config); err != nil { return err } } return nil } + +// ParseInterface parses the config file and environment variables into the provided interface. +func (u *UnifiPoller) ParseInterface(i interface{}) error { + // Parse config file into provided interface. + if err := config.ParseFile(i, u.Flags.ConfigFile); err != nil { + return err + } + + // Parse environment variables into provided interface. + _, err := config.ParseENV(i, ENVConfigPrefix) + + return err +} diff --git a/pkg/poller/dumper.go b/pkg/poller/dumper.go index 5892488b..18abe901 100644 --- a/pkg/poller/dumper.go +++ b/pkg/poller/dumper.go @@ -1,55 +1,58 @@ package poller import ( - "fmt" - "os" "strings" - - "golift.io/unifi" ) // DumpJSONPayload prints raw json from the UniFi Controller. // This only works with controller 0 (first one) in the config. func (u *UnifiPoller) DumpJSONPayload() (err error) { - u.Config.Quiet = true - config := u.Config.Controllers[0] - - config.Unifi, err = unifi.NewUnifi(&unifi.Config{ - User: config.User, - Pass: config.Pass, - URL: config.URL, - VerifySSL: config.VerifySSL, - }) - if err != nil { - return err + if true { + return nil } + /* + u.Config.Quiet = true + config := u.Config.Controllers[0] - fmt.Fprintf(os.Stderr, "[INFO] Authenticated to UniFi Controller @ %v as user %v", config.URL, config.User) + config.Unifi, err = unifi.NewUnifi(&unifi.Config{ + User: config.User, + Pass: config.Pass, + URL: config.URL, + VerifySSL: config.VerifySSL, + }) + if err != nil { + return err + } - if err := u.CheckSites(config); err != nil { - return err - } + fmt.Fprintf(os.Stderr, "[INFO] Authenticated to UniFi Controller @ %v as user %v", config.URL, config.User) - config.Unifi.ErrorLog = func(m string, v ...interface{}) { - fmt.Fprintf(os.Stderr, "[ERROR] "+m, v...) - } // Log all errors to stderr. + if err := u.CheckSites(config); err != nil { + return err + } - switch sites, err := u.GetFilteredSites(config); { - case err != nil: - return err - case StringInSlice(u.Flags.DumpJSON, []string{"d", "device", "devices"}): - return u.dumpSitesJSON(config, unifi.APIDevicePath, "Devices", sites) - case StringInSlice(u.Flags.DumpJSON, []string{"client", "clients", "c"}): - return u.dumpSitesJSON(config, unifi.APIClientPath, "Clients", sites) - case strings.HasPrefix(u.Flags.DumpJSON, "other "): - apiPath := strings.SplitN(u.Flags.DumpJSON, " ", 2)[1] - _, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping Path '%s':\n", apiPath) - return u.PrintRawAPIJSON(config, apiPath) - default: - return fmt.Errorf("must provide filter: devices, clients, other") - } + config.Unifi.ErrorLog = func(m string, v ...interface{}) { + fmt.Fprintf(os.Stderr, "[ERROR] "+m, v...) + } // Log all errors to stderr. + + switch sites, err := u.GetFilteredSites(config); { + case err != nil: + return err + case StringInSlice(u.Flags.DumpJSON, []string{"d", "device", "devices"}): + return u.dumpSitesJSON(config, unifi.APIDevicePath, "Devices", sites) + case StringInSlice(u.Flags.DumpJSON, []string{"client", "clients", "c"}): + return u.dumpSitesJSON(config, unifi.APIClientPath, "Clients", sites) + case strings.HasPrefix(u.Flags.DumpJSON, "other "): + apiPath := strings.SplitN(u.Flags.DumpJSON, " ", 2)[1] + _, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping Path '%s':\n", apiPath) + return u.PrintRawAPIJSON(config, apiPath) + default: + return fmt.Errorf("must provide filter: devices, clients, other") + } + */ + return nil } +/* func (u *UnifiPoller) dumpSitesJSON(c Controller, path, name string, sites unifi.Sites) error { for _, s := range sites { apiPath := fmt.Sprintf(path, s.Name) @@ -68,3 +71,15 @@ func (u *UnifiPoller) PrintRawAPIJSON(c Controller, apiPath string) error { fmt.Println(string(body)) return err } +*/ + +// StringInSlice returns true if a string is in a slice. +func StringInSlice(str string, slice []string) bool { + for _, s := range slice { + if strings.EqualFold(s, str) { + return true + } + } + + return false +} diff --git a/pkg/poller/inputs.go b/pkg/poller/inputs.go new file mode 100644 index 00000000..c2140b01 --- /dev/null +++ b/pkg/poller/inputs.go @@ -0,0 +1,96 @@ +package poller + +import ( + "fmt" + "strings" + "sync" + + "golift.io/unifi" +) + +var ( + inputs []*InputPlugin + inputSync sync.Mutex +) + +// Input plugins must implement this interface. +type Input interface { + Initialize(Logger) error // Called once on startup to initialize the plugin. + Metrics() (*Metrics, error) // Called every time new metrics are requested. +} + +// InputPlugin describes an input plugin's consumable interface. +type InputPlugin struct { + Config interface{} // Each config is passed into an unmarshaller later. + Input +} + +// NewInput creates a metric input. This should be called by input plugins +// init() functions. +func NewInput(i *InputPlugin) { + inputSync.Lock() + defer inputSync.Unlock() + + if i == nil || i.Input == nil { + panic("nil output or method passed to poller.NewOutput") + } + + inputs = append(inputs, i) +} + +// InitializeInputs runs the passed-in initializer method for each input plugin. +func (u *UnifiPoller) InitializeInputs() error { + inputSync.Lock() + defer inputSync.Unlock() + + for _, input := range inputs { + // This must return, or the app locks up here. + if err := input.Initialize(u); err != nil { + return err + } + } + + return nil +} + +// Metrics aggregates all the measurements from all configured inputs and returns them. +func (u *UnifiPoller) Metrics() (*Metrics, error) { + errs := []string{} + metrics := &Metrics{} + + for _, input := range inputs { + m, err := input.Metrics() + if err != nil { + errs = append(errs, err.Error()) + } + + if m == nil { + continue + } + + metrics.Sites = append(metrics.Sites, m.Sites...) + metrics.Clients = append(metrics.Clients, m.Clients...) + metrics.IDSList = append(metrics.IDSList, m.IDSList...) + + if m.Devices == nil { + continue + } + + if metrics.Devices == nil { + metrics.Devices = &unifi.Devices{} + } + + metrics.UAPs = append(metrics.UAPs, m.UAPs...) + metrics.USGs = append(metrics.USGs, m.USGs...) + metrics.USWs = append(metrics.USWs, m.USWs...) + metrics.UDMs = append(metrics.UDMs, m.UDMs...) + } + + var err error + + if len(errs) > 0 { + err = fmt.Errorf(strings.Join(errs, ", ")) + } + + return metrics, err +} diff --git a/pkg/poller/helpers.go b/pkg/poller/logger.go similarity index 75% rename from pkg/poller/helpers.go rename to pkg/poller/logger.go index 92acd223..b498a9b5 100644 --- a/pkg/poller/helpers.go +++ b/pkg/poller/logger.go @@ -3,19 +3,15 @@ package poller import ( "fmt" "log" - "strings" ) const callDepth = 2 -// StringInSlice returns true if a string is in a slice. -func StringInSlice(str string, slice []string) bool { - for _, s := range slice { - if strings.EqualFold(s, str) { - return true - } - } - return false +// Logger is passed into input packages so they may write logs. +type Logger interface { + Logf(m string, v ...interface{}) + LogErrorf(m string, v ...interface{}) + LogDebugf(m string, v ...interface{}) } // Logf prints a log entry if quiet is false. diff --git a/pkg/poller/outputs.go b/pkg/poller/outputs.go index aaa9a47d..3bbcb72d 100644 --- a/pkg/poller/outputs.go +++ b/pkg/poller/outputs.go @@ -14,9 +14,7 @@ var ( // Output packages must implement this interface. type Collect interface { Metrics() (*Metrics, error) - Logf(m string, v ...interface{}) - LogErrorf(m string, v ...interface{}) - LogDebugf(m string, v ...interface{}) + Logger } // Output defines the output data for a metric exporter like influx or prometheus. @@ -49,6 +47,7 @@ func (u *UnifiPoller) InitializeOutputs() error { for _, o := range outputs { count++ + go func(o *Output) { v <- o.Method(u) }(o) diff --git a/pkg/poller/start.go b/pkg/poller/start.go index cd293b2f..9f6bedc8 100644 --- a/pkg/poller/start.go +++ b/pkg/poller/start.go @@ -37,25 +37,6 @@ func (u *UnifiPoller) Start() error { return err } - if len(u.Config.Controllers) < 1 { - u.Config.Controllers = []Controller{{ - Sites: []string{"all"}, - User: defaultUnifiUser, - Pass: "", - URL: defaultUnifiURL, - SaveSites: true, - }} - } - - if u.Flags.DumpJSON != "" { - return u.DumpJSONPayload() - } - - if u.Config.Debug { - log.SetFlags(log.Lshortfile | log.Lmicroseconds | log.Ldate) - u.LogDebugf("Debug Logging Enabled") - } - return u.Run() } @@ -79,20 +60,19 @@ func (f *Flags) Parse(args []string) { // 2. Run the collector one time and report the metrics to influxdb. (lambda) // 3. Start a web server and wait for Prometheus to poll the application for metrics. func (u *UnifiPoller) Run() error { + if u.Flags.DumpJSON != "" { + return u.DumpJSONPayload() + } + + if u.Config.Debug { + log.SetFlags(log.Lshortfile | log.Lmicroseconds | log.Ldate) + u.LogDebugf("Debug Logging Enabled") + } + log.Printf("[INFO] UniFi Poller v%v Starting Up! PID: %d", version.Version, os.Getpid()) - for i, c := range u.Config.Controllers { - if c.Name == "" { - u.Config.Controllers[i].Name = c.URL - } - - switch err := u.GetUnifi(c); err { - case nil: - u.Logf("Polling UniFi Controller at %s v%s as user %s. Sites: %v", - c.URL, c.Unifi.ServerVersion, c.User, c.Sites) - default: - u.LogErrorf("Controller Auth or Connection failed, but continuing to retry! %s: %v", c.Name, err) - } + if err := u.InitializeInputs(); err != nil { + return err } return u.InitializeOutputs() diff --git a/pkg/poller/unifi.go b/pkg/poller/unifi.go deleted file mode 100644 index 516310a3..00000000 --- a/pkg/poller/unifi.go +++ /dev/null @@ -1,241 +0,0 @@ -package poller - -import ( - "fmt" - "strings" - "time" - - "golift.io/unifi" -) - -// GetUnifi returns a UniFi controller interface. -func (u *UnifiPoller) GetUnifi(c Controller) error { - var err error - - u.Lock() - defer u.Unlock() - - if c.Unifi != nil { - c.Unifi.CloseIdleConnections() - } - // Create an authenticated session to the Unifi Controller. - c.Unifi, err = unifi.NewUnifi(&unifi.Config{ - User: c.User, - Pass: c.Pass, - URL: c.URL, - VerifySSL: c.VerifySSL, - ErrorLog: u.LogErrorf, // Log all errors. - DebugLog: u.LogDebugf, // Log debug messages. - }) - - if err != nil { - c.Unifi = nil - return fmt.Errorf("unifi controller: %v", err) - } - - u.LogDebugf("Authenticated with controller successfully, %s", c.URL) - - return u.CheckSites(c) -} - -// CheckSites makes sure the list of provided sites exists on the controller. -// This does not run in Lambda (run-once) mode. -func (u *UnifiPoller) CheckSites(c Controller) error { - u.LogDebugf("Checking Controller Sites List") - - sites, err := c.Unifi.GetSites() - if err != nil { - return err - } - - msg := []string{} - - for _, site := range sites { - msg = append(msg, site.Name+" ("+site.Desc+")") - } - - u.Logf("Found %d site(s) on controller: %v", len(msg), strings.Join(msg, ", ")) - - if StringInSlice("all", c.Sites) { - c.Sites = []string{"all"} - return nil - } - -FIRST: - for _, s := range c.Sites { - for _, site := range sites { - if s == site.Name { - continue FIRST - } - } - return fmt.Errorf("configured site not found on controller: %v", s) - } - - return nil -} - -// Metrics grabs all the measurements from a UniFi controller and returns them. -func (u *UnifiPoller) Metrics() (*Metrics, error) { - errs := []string{} - metrics := &Metrics{} - - for _, c := range u.Config.Controllers { - m, err := u.checkAndPollController(c) - if err != nil { - errs = append(errs, err.Error()) - } - - if m == nil { - continue - } - - metrics.Sites = append(metrics.Sites, m.Sites...) - metrics.Clients = append(metrics.Clients, m.Clients...) - metrics.IDSList = append(metrics.IDSList, m.IDSList...) - - if m.Devices == nil { - continue - } - - if metrics.Devices == nil { - metrics.Devices = &unifi.Devices{} - } - - metrics.UAPs = append(metrics.UAPs, m.UAPs...) - metrics.USGs = append(metrics.USGs, m.USGs...) - metrics.USWs = append(metrics.USWs, m.USWs...) - metrics.UDMs = append(metrics.UDMs, m.UDMs...) - } - - var err error - - if len(errs) > 0 { - err = fmt.Errorf(strings.Join(errs, ", ")) - } - - return metrics, err -} - -func (u *UnifiPoller) checkAndPollController(c Controller) (*Metrics, error) { - if c.Unifi == nil { - u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) - - if err := u.GetUnifi(c); err != nil { - u.LogErrorf("re-authenticating to %s: %v", c.URL, err) - return nil, err - } - } - - m, err := u.collectController(c) - if err == nil { - return m, nil - } - - u.LogErrorf("collecting metrics %v", err) - u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) - - if err := u.GetUnifi(c); err != nil { - u.LogErrorf("re-authenticating to %s: %v", c.URL, err) - return nil, err - } - - return u.collectController(c) -} - -func (u *UnifiPoller) collectController(c Controller) (*Metrics, error) { - var err error - - m := &Metrics{TS: time.Now()} // At this point, it's the Current Check. - - // Get the sites we care about. - if m.Sites, err = u.GetFilteredSites(c); err != nil { - return m, fmt.Errorf("unifi.GetSites(%v): %v", c.URL, err) - } - - if c.SaveIDS { - m.IDSList, err = c.Unifi.GetIDS(m.Sites, time.Now().Add(2*time.Minute), time.Now()) - if err != nil { - return m, fmt.Errorf("unifi.GetIDS(%v): %v", c.URL, err) - } - } - - // Get all the points. - if m.Clients, err = c.Unifi.GetClients(m.Sites); err != nil { - return m, fmt.Errorf("unifi.GetClients(%v): %v", c.URL, err) - } - - if m.Devices, err = c.Unifi.GetDevices(m.Sites); err != nil { - return m, fmt.Errorf("unifi.GetDevices(%v): %v", c.URL, err) - } - - return u.augmentMetrics(c, m), nil -} - -// augmentMetrics is our middleware layer between collecting metrics and writing them. -// This is where we can manipuate the returned data or make arbitrary decisions. -// This function currently adds parent device names to client metrics. -func (u *UnifiPoller) augmentMetrics(c Controller, metrics *Metrics) *Metrics { - if metrics == nil || metrics.Devices == nil || metrics.Clients == nil { - return metrics - } - - devices := make(map[string]string) - bssdIDs := make(map[string]string) - - for _, r := range metrics.UAPs { - devices[r.Mac] = r.Name - for _, v := range r.VapTable { - bssdIDs[v.Bssid] = fmt.Sprintf("%s %s %s:", r.Name, v.Radio, v.RadioName) - } - } - - for _, r := range metrics.USGs { - devices[r.Mac] = r.Name - } - - for _, r := range metrics.USWs { - devices[r.Mac] = r.Name - } - - for _, r := range metrics.UDMs { - devices[r.Mac] = r.Name - } - - // These come blank, so set them here. - for i, c := range metrics.Clients { - metrics.Clients[i].SwName = devices[c.SwMac] - metrics.Clients[i].ApName = devices[c.ApMac] - metrics.Clients[i].GwName = devices[c.GwMac] - metrics.Clients[i].RadioDescription = bssdIDs[metrics.Clients[i].Bssid] + metrics.Clients[i].RadioProto - } - - if !c.SaveSites { - metrics.Sites = nil - } - - return metrics -} - -// GetFilteredSites returns a list of sites to fetch data for. -// Omits requested but unconfigured sites. Grabs the full list from the -// controller and returns the sites provided in the config file. -func (u *UnifiPoller) GetFilteredSites(c Controller) (unifi.Sites, error) { - var i int - - sites, err := c.Unifi.GetSites() - if err != nil { - return nil, err - } else if len(c.Sites) < 1 || StringInSlice("all", c.Sites) { - return sites, nil - } - - for _, s := range sites { - // Only include valid sites in the request filter. - if StringInSlice(s.Name, c.Sites) { - sites[i] = s - i++ - } - } - - return sites[:i], nil -} diff --git a/pkg/promunifi/clients.go b/pkg/promunifi/clients.go index fff79857..0ffaf54c 100644 --- a/pkg/promunifi/clients.go +++ b/pkg/promunifi/clients.go @@ -41,7 +41,8 @@ type uclient struct { } func descClient(ns string) *uclient { - labels := []string{"name", "mac", "site_name", "gw_name", "sw_name", "vlan", "ip", "oui", "network", "sw_port", "ap_name", "wired"} + labels := []string{"name", "mac", "site_name", "gw_name", "sw_name", "vlan", + "ip", "oui", "network", "sw_port", "ap_name", "wired"} labelW := append([]string{"radio_name", "radio", "radio_proto", "channel", "essid", "bssid", "radio_desc"}, labels...) return &uclient{ @@ -64,25 +65,34 @@ func descClient(ns string) *uclient { TxPower: prometheus.NewDesc(ns+"radio_transmit_power_dbm", "Client Transmit Power", labelW, nil), TxRate: prometheus.NewDesc(ns+"radio_transmit_rate_bps", "Client Transmit Rate", labelW, nil), WifiTxAttempts: prometheus.NewDesc(ns+"wifi_attempts_transmit_total", "Client Wifi Transmit Attempts", labelW, nil), - Uptime: prometheus.NewDesc(ns+"uptime_seconds", "Client Uptime", labelW, nil), // XXX: re-purpose for info tags. + Uptime: prometheus.NewDesc(ns+"uptime_seconds", "Client Uptime", labelW, nil), /* needs more "looking into" - DpiStatsApp: prometheus.NewDesc(ns+"dpi_stats_app", "Client DPI Stats App", labels, nil), - DpiStatsCat: prometheus.NewDesc(ns+"dpi_stats_cat", "Client DPI Stats Cat", labels, nil), - DpiStatsRxBytes: prometheus.NewDesc(ns+"dpi_stats_receive_bytes_total", "Client DPI Stats Receive Bytes", labels, nil), - DpiStatsRxPackets: prometheus.NewDesc(ns+"dpi_stats_receive_packets_total", "Client DPI Stats Receive Packets", labels, nil), - DpiStatsTxBytes: prometheus.NewDesc(ns+"dpi_stats_transmit_bytes_total", "Client DPI Stats Transmit Bytes", labels, nil), - DpiStatsTxPackets: prometheus.NewDesc(ns+"dpi_stats_transmit_packets_total", "Client DPI Stats Transmit Packets", labels, nil), + DpiStatsApp: prometheus.NewDesc(ns+"dpi_stats_app", + "Client DPI Stats App", labels, nil), + DpiStatsCat: prometheus.NewDesc(ns+"dpi_stats_cat", + "Client DPI Stats Cat", labels, nil), + DpiStatsRxBytes: prometheus.NewDesc(ns+"dpi_stats_receive_bytes_total", + "Client DPI Stats Receive Bytes", labels, nil), + DpiStatsRxPackets: prometheus.NewDesc(ns+"dpi_stats_receive_packets_total", + "Client DPI Stats Receive Packets", labels, nil), + DpiStatsTxBytes: prometheus.NewDesc(ns+"dpi_stats_transmit_bytes_total", + "Client DPI Stats Transmit Bytes", labels, nil), + DpiStatsTxPackets: prometheus.NewDesc(ns+"dpi_stats_transmit_packets_total", + "Client DPI Stats Transmit Packets", labels, nil), */ } } func (u *promUnifi) exportClient(r report, c *unifi.Client) { - labels := []string{c.Name, c.Mac, c.SiteName, c.GwName, c.SwName, c.Vlan.Txt, c.IP, c.Oui, c.Network, c.SwPort.Txt, c.ApName, ""} - labelW := append([]string{c.RadioName, c.Radio, c.RadioProto, c.Channel.Txt, c.Essid, c.Bssid, c.RadioDescription}, labels...) + labels := []string{c.Name, c.Mac, c.SiteName, c.GwName, c.SwName, c.Vlan.Txt, + c.IP, c.Oui, c.Network, c.SwPort.Txt, c.ApName, ""} + labelW := append([]string{c.RadioName, c.Radio, c.RadioProto, c.Channel.Txt, + c.Essid, c.Bssid, c.RadioDescription}, labels...) if c.IsWired.Val { labels[len(labels)-1] = "true" labelW[len(labelW)-1] = "true" + r.send([]*metric{ {u.Client.RxBytes, counter, c.WiredRxBytes, labels}, {u.Client.RxBytesR, gauge, c.WiredRxBytesR, labels}, @@ -94,6 +104,7 @@ func (u *promUnifi) exportClient(r report, c *unifi.Client) { } else { labels[len(labels)-1] = "false" labelW[len(labelW)-1] = "false" + r.send([]*metric{ {u.Client.Anomalies, counter, c.Anomalies, labelW}, {u.Client.CCQ, gauge, float64(c.Ccq) / 1000.0, labelW}, @@ -118,12 +129,13 @@ func (u *promUnifi) exportClient(r report, c *unifi.Client) { } r.send([]*metric{{u.Client.Uptime, gauge, c.Uptime, labelW}}) - /* needs more "looking into" - {u.Client.DpiStatsApp, gauge, c.DpiStats.App, labels}, - {u.Client.DpiStatsCat, gauge, c.DpiStats.Cat, labels}, - {u.Client.DpiStatsRxBytes, counter, c.DpiStats.RxBytes, labels}, - {u.Client.DpiStatsRxPackets, counter, c.DpiStats.RxPackets, labels}, - {u.Client.DpiStatsTxBytes, counter, c.DpiStats.TxBytes, labels}, - {u.Client.DpiStatsTxPackets, counter, c.DpiStats.TxPackets, labels}, - */ } + +/* needs more "looking into" +{u.Client.DpiStatsApp, gauge, c.DpiStats.App, labels}, +{u.Client.DpiStatsCat, gauge, c.DpiStats.Cat, labels}, +{u.Client.DpiStatsRxBytes, counter, c.DpiStats.RxBytes, labels}, +{u.Client.DpiStatsRxPackets, counter, c.DpiStats.RxPackets, labels}, +{u.Client.DpiStatsTxBytes, counter, c.DpiStats.TxBytes, labels}, +{u.Client.DpiStatsTxPackets, counter, c.DpiStats.TxPackets, labels}, +*/ diff --git a/pkg/promunifi/collector.go b/pkg/promunifi/collector.go index e999ac3f..a4aad9f3 100644 --- a/pkg/promunifi/collector.go +++ b/pkg/promunifi/collector.go @@ -79,6 +79,7 @@ type Report struct { func init() { u := &promUnifi{Prometheus: &Prometheus{}} + poller.NewOutput(&poller.Output{ Name: "prometheus", Config: u.Prometheus, @@ -93,33 +94,27 @@ func (u *promUnifi) Run(c poller.Collect) error { return nil } + u.Config.Namespace = strings.Trim(strings.Replace(u.Config.Namespace, "-", "_", -1), "_") if u.Config.Namespace == "" { u.Config.Namespace = strings.Replace(poller.AppName, "-", "", -1) } - u.Config.Namespace = strings.Replace(u.Config.Namespace, "-", "_", -1) - if u.Config.HTTPListen == "" { u.Config.HTTPListen = defaultHTTPListen } prometheus.MustRegister(version.NewCollector(u.Config.Namespace)) - - if u.Config.Namespace = strings.Trim(u.Config.Namespace, "_") + "_"; u.Config.Namespace == "_" { - u.Config.Namespace = "" - } - prometheus.MustRegister(&promUnifi{ Collector: c, - Client: descClient(u.Config.Namespace + "client_"), - Device: descDevice(u.Config.Namespace + "device_"), // stats for all device types. - UAP: descUAP(u.Config.Namespace + "device_"), - USG: descUSG(u.Config.Namespace + "device_"), - USW: descUSW(u.Config.Namespace + "device_"), - Site: descSite(u.Config.Namespace + "site_"), + Client: descClient(u.Config.Namespace + "_client_"), + Device: descDevice(u.Config.Namespace + "_device_"), // stats for all device types. + UAP: descUAP(u.Config.Namespace + "_device_"), + USG: descUSG(u.Config.Namespace + "_device_"), + USW: descUSW(u.Config.Namespace + "_device_"), + Site: descSite(u.Config.Namespace + "_site_"), }) - - c.Logf("Exporting Measurements for Prometheus at https://%s/metrics, namespace: %s", u.Config.HTTPListen, u.Config.Namespace) + c.Logf("Exporting Measurements for Prometheus at https://%s/metrics, namespace: %s", + u.Config.HTTPListen, u.Config.Namespace) return http.ListenAndServe(u.Config.HTTPListen, nil) } @@ -152,6 +147,7 @@ func (u *promUnifi) Collect(ch chan<- prometheus.Metric) { r.error(ch, prometheus.NewInvalidDesc(fmt.Errorf("metric fetch failed")), err) return } + r.Fetch = time.Since(r.Start) if r.Metrics.Devices == nil { @@ -173,6 +169,7 @@ func (u *promUnifi) exportMetrics(r report, ch chan<- prometheus.Metric, ourChan for newMetrics := range ourChan { for _, m := range newMetrics { descs[m.Desc] = true + switch v := m.Value.(type) { case unifi.FlexInt: ch <- r.export(m, v.Val) @@ -195,48 +192,55 @@ func (u *promUnifi) loopExports(r report) { m := r.metrics() r.add() + r.add() + r.add() + r.add() + r.add() + r.add() + go func() { defer r.done() + for _, s := range m.Sites { u.exportSite(r, s) } }() - r.add() go func() { defer r.done() + for _, d := range m.UAPs { u.exportUAP(r, d) } }() - r.add() go func() { defer r.done() + for _, d := range m.UDMs { u.exportUDM(r, d) } }() - r.add() go func() { defer r.done() + for _, d := range m.USGs { u.exportUSG(r, d) } }() - r.add() go func() { defer r.done() + for _, d := range m.USWs { u.exportUSW(r, d) } }() - r.add() go func() { defer r.done() + for _, c := range m.Clients { u.exportClient(r, c) } diff --git a/pkg/promunifi/site.go b/pkg/promunifi/site.go index 1f8999e9..79c57b41 100644 --- a/pkg/promunifi/site.go +++ b/pkg/promunifi/site.go @@ -35,32 +35,34 @@ type site struct { func descSite(ns string) *site { labels := []string{"subsystem", "status", "site_name"} + nd := prometheus.NewDesc + return &site{ - NumUser: prometheus.NewDesc(ns+"users", "Number of Users", labels, nil), - NumGuest: prometheus.NewDesc(ns+"guests", "Number of Guests", labels, nil), - NumIot: prometheus.NewDesc(ns+"iots", "Number of IoT Devices", labels, nil), - TxBytesR: prometheus.NewDesc(ns+"transmit_rate_bytes", "Bytes Transmit Rate", labels, nil), - RxBytesR: prometheus.NewDesc(ns+"receive_rate_bytes", "Bytes Receive Rate", labels, nil), - NumAp: prometheus.NewDesc(ns+"aps", "Access Point Count", labels, nil), - NumAdopted: prometheus.NewDesc(ns+"adopted", "Adoption Count", labels, nil), - NumDisabled: prometheus.NewDesc(ns+"disabled", "Disabled Count", labels, nil), - NumDisconnected: prometheus.NewDesc(ns+"disconnected", "Disconnected Count", labels, nil), - NumPending: prometheus.NewDesc(ns+"pending", "Pending Count", labels, nil), - NumGw: prometheus.NewDesc(ns+"gateways", "Gateway Count", labels, nil), - NumSw: prometheus.NewDesc(ns+"switches", "Switch Count", labels, nil), - NumSta: prometheus.NewDesc(ns+"stations", "Station Count", labels, nil), - Latency: prometheus.NewDesc(ns+"latency_seconds", "Latency", labels, nil), - Uptime: prometheus.NewDesc(ns+"uptime_seconds", "Uptime", labels, nil), - Drops: prometheus.NewDesc(ns+"intenet_drops_total", "Internet (WAN) Disconnections", labels, nil), - XputUp: prometheus.NewDesc(ns+"xput_up_rate", "Speedtest Upload", labels, nil), - XputDown: prometheus.NewDesc(ns+"xput_down_rate", "Speedtest Download", labels, nil), - SpeedtestPing: prometheus.NewDesc(ns+"speedtest_ping", "Speedtest Ping", labels, nil), - RemoteUserNumActive: prometheus.NewDesc(ns+"remote_user_active", "Remote Users Active", labels, nil), - RemoteUserNumInactive: prometheus.NewDesc(ns+"remote_user_inactive", "Remote Users Inactive", labels, nil), - RemoteUserRxBytes: prometheus.NewDesc(ns+"remote_user_receive_bytes_total", "Remote Users Receive Bytes", labels, nil), - RemoteUserTxBytes: prometheus.NewDesc(ns+"remote_user_transmit_bytes_total", "Remote Users Transmit Bytes", labels, nil), - RemoteUserRxPackets: prometheus.NewDesc(ns+"remote_user_receive_packets_total", "Remote Users Receive Packets", labels, nil), - RemoteUserTxPackets: prometheus.NewDesc(ns+"remote_user_transmit_packets_total", "Remote Users Transmit Packets", labels, nil), + NumUser: nd(ns+"users", "Number of Users", labels, nil), + NumGuest: nd(ns+"guests", "Number of Guests", labels, nil), + NumIot: nd(ns+"iots", "Number of IoT Devices", labels, nil), + TxBytesR: nd(ns+"transmit_rate_bytes", "Bytes Transmit Rate", labels, nil), + RxBytesR: nd(ns+"receive_rate_bytes", "Bytes Receive Rate", labels, nil), + NumAp: nd(ns+"aps", "Access Point Count", labels, nil), + NumAdopted: nd(ns+"adopted", "Adoption Count", labels, nil), + NumDisabled: nd(ns+"disabled", "Disabled Count", labels, nil), + NumDisconnected: nd(ns+"disconnected", "Disconnected Count", labels, nil), + NumPending: nd(ns+"pending", "Pending Count", labels, nil), + NumGw: nd(ns+"gateways", "Gateway Count", labels, nil), + NumSw: nd(ns+"switches", "Switch Count", labels, nil), + NumSta: nd(ns+"stations", "Station Count", labels, nil), + Latency: nd(ns+"latency_seconds", "Latency", labels, nil), + Uptime: nd(ns+"uptime_seconds", "Uptime", labels, nil), + Drops: nd(ns+"intenet_drops_total", "Internet (WAN) Disconnections", labels, nil), + XputUp: nd(ns+"xput_up_rate", "Speedtest Upload", labels, nil), + XputDown: nd(ns+"xput_down_rate", "Speedtest Download", labels, nil), + SpeedtestPing: nd(ns+"speedtest_ping", "Speedtest Ping", labels, nil), + RemoteUserNumActive: nd(ns+"remote_user_active", "Remote Users Active", labels, nil), + RemoteUserNumInactive: nd(ns+"remote_user_inactive", "Remote Users Inactive", labels, nil), + RemoteUserRxBytes: nd(ns+"remote_user_receive_bytes_total", "Remote Users Receive Bytes", labels, nil), + RemoteUserTxBytes: nd(ns+"remote_user_transmit_bytes_total", "Remote Users Transmit Bytes", labels, nil), + RemoteUserRxPackets: nd(ns+"remote_user_receive_packets_total", "Remote Users Receive Packets", labels, nil), + RemoteUserTxPackets: nd(ns+"remote_user_transmit_packets_total", "Remote Users Transmit Packets", labels, nil), } } @@ -78,7 +80,6 @@ func (u *promUnifi) exportSite(r report, s *unifi.Site) { {u.Site.SpeedtestPing, gauge, h.SpeedtestPing, labels}, {u.Site.Drops, counter, h.Drops, labels}, }) - case "wlan": r.send([]*metric{ {u.Site.TxBytesR, gauge, h.TxBytesR, labels}, @@ -92,7 +93,6 @@ func (u *promUnifi) exportSite(r report, s *unifi.Site) { {u.Site.NumAp, gauge, h.NumAp, labels}, {u.Site.NumDisabled, gauge, h.NumDisabled, labels}, }) - case "wan": r.send([]*metric{ {u.Site.TxBytesR, gauge, h.TxBytesR, labels}, @@ -103,7 +103,6 @@ func (u *promUnifi) exportSite(r report, s *unifi.Site) { {u.Site.NumGw, gauge, h.NumGw, labels}, {u.Site.NumSta, gauge, h.NumSta, labels}, }) - case "lan": r.send([]*metric{ {u.Site.TxBytesR, gauge, h.TxBytesR, labels}, @@ -116,7 +115,6 @@ func (u *promUnifi) exportSite(r report, s *unifi.Site) { {u.Site.NumIot, gauge, h.NumIot, labels}, {u.Site.NumSw, gauge, h.NumSw, labels}, }) - case "vpn": r.send([]*metric{ {u.Site.RemoteUserNumActive, gauge, h.RemoteUserNumActive, labels}, diff --git a/pkg/promunifi/uap.go b/pkg/promunifi/uap.go index 53242159..0f3a05aa 100644 --- a/pkg/promunifi/uap.go +++ b/pkg/promunifi/uap.go @@ -83,79 +83,80 @@ func descUAP(ns string) *uap { labelA := []string{"stat", "site_name", "name"} // stat + labels[1:] labelV := []string{"vap_name", "bssid", "radio", "radio_name", "essid", "usage", "site_name", "name"} labelR := []string{"radio_name", "radio", "site_name", "name"} + nd := prometheus.NewDesc return &uap{ // 3x each - stat table: total, guest, user - ApWifiTxDropped: prometheus.NewDesc(ns+"stat_wifi_transmt_dropped_total", "Wifi Transmissions Dropped", labelA, nil), - ApRxErrors: prometheus.NewDesc(ns+"stat_receive_errors_total", "Receive Errors", labelA, nil), - ApRxDropped: prometheus.NewDesc(ns+"stat_receive_dropped_total", "Receive Dropped", labelA, nil), - ApRxFrags: prometheus.NewDesc(ns+"stat_receive_frags_total", "Received Frags", labelA, nil), - ApRxCrypts: prometheus.NewDesc(ns+"stat_receive_crypts_total", "Receive Crypts", labelA, nil), - ApTxPackets: prometheus.NewDesc(ns+"stat_transmit_packets_total", "Transmit Packets", labelA, nil), - ApTxBytes: prometheus.NewDesc(ns+"stat_transmit_bytes_total", "Transmit Bytes", labelA, nil), - ApTxErrors: prometheus.NewDesc(ns+"stat_transmit_errors_total", "Transmit Errors", labelA, nil), - ApTxDropped: prometheus.NewDesc(ns+"stat_transmit_dropped_total", "Transmit Dropped", labelA, nil), - ApTxRetries: prometheus.NewDesc(ns+"stat_retries_tx_total", "Transmit Retries", labelA, nil), - ApRxPackets: prometheus.NewDesc(ns+"stat_receive_packets_total", "Receive Packets", labelA, nil), - ApRxBytes: prometheus.NewDesc(ns+"stat_receive_bytes_total", "Receive Bytes", labelA, nil), - WifiTxAttempts: prometheus.NewDesc(ns+"stat_wifi_transmit_attempts_total", "Wifi Transmission Attempts", labelA, nil), - MacFilterRejections: prometheus.NewDesc(ns+"stat_mac_filter_rejects_total", "MAC Filter Rejections", labelA, nil), + ApWifiTxDropped: nd(ns+"stat_wifi_transmt_dropped_total", "Wifi Transmissions Dropped", labelA, nil), + ApRxErrors: nd(ns+"stat_receive_errors_total", "Receive Errors", labelA, nil), + ApRxDropped: nd(ns+"stat_receive_dropped_total", "Receive Dropped", labelA, nil), + ApRxFrags: nd(ns+"stat_receive_frags_total", "Received Frags", labelA, nil), + ApRxCrypts: nd(ns+"stat_receive_crypts_total", "Receive Crypts", labelA, nil), + ApTxPackets: nd(ns+"stat_transmit_packets_total", "Transmit Packets", labelA, nil), + ApTxBytes: nd(ns+"stat_transmit_bytes_total", "Transmit Bytes", labelA, nil), + ApTxErrors: nd(ns+"stat_transmit_errors_total", "Transmit Errors", labelA, nil), + ApTxDropped: nd(ns+"stat_transmit_dropped_total", "Transmit Dropped", labelA, nil), + ApTxRetries: nd(ns+"stat_retries_tx_total", "Transmit Retries", labelA, nil), + ApRxPackets: nd(ns+"stat_receive_packets_total", "Receive Packets", labelA, nil), + ApRxBytes: nd(ns+"stat_receive_bytes_total", "Receive Bytes", labelA, nil), + WifiTxAttempts: nd(ns+"stat_wifi_transmit_attempts_total", "Wifi Transmission Attempts", labelA, nil), + MacFilterRejections: nd(ns+"stat_mac_filter_rejects_total", "MAC Filter Rejections", labelA, nil), // N each - 1 per Virtual AP (VAP) - VAPCcq: prometheus.NewDesc(ns+"vap_ccq_ratio", "VAP Client Connection Quality", labelV, nil), - VAPMacFilterRejections: prometheus.NewDesc(ns+"vap_mac_filter_rejects_total", "VAP MAC Filter Rejections", labelV, nil), - VAPNumSatisfactionSta: prometheus.NewDesc(ns+"vap_satisfaction_stations", "VAP Number Satisifaction Stations", labelV, nil), - VAPAvgClientSignal: prometheus.NewDesc(ns+"vap_average_client_signal", "VAP Average Client Signal", labelV, nil), - VAPSatisfaction: prometheus.NewDesc(ns+"vap_satisfaction_ratio", "VAP Satisfaction", labelV, nil), - VAPSatisfactionNow: prometheus.NewDesc(ns+"vap_satisfaction_now_ratio", "VAP Satisfaction Now", labelV, nil), - VAPDNSAvgLatency: prometheus.NewDesc(ns+"vap_dns_latency_average_seconds", "VAP DNS Latency Average", labelV, nil), - VAPRxBytes: prometheus.NewDesc(ns+"vap_receive_bytes_total", "VAP Bytes Received", labelV, nil), - VAPRxCrypts: prometheus.NewDesc(ns+"vap_receive_crypts_total", "VAP Crypts Received", labelV, nil), - VAPRxDropped: prometheus.NewDesc(ns+"vap_receive_dropped_total", "VAP Dropped Received", labelV, nil), - VAPRxErrors: prometheus.NewDesc(ns+"vap_receive_errors_total", "VAP Errors Received", labelV, nil), - VAPRxFrags: prometheus.NewDesc(ns+"vap_receive_frags_total", "VAP Frags Received", labelV, nil), - VAPRxNwids: prometheus.NewDesc(ns+"vap_receive_nwids_total", "VAP Nwids Received", labelV, nil), - VAPRxPackets: prometheus.NewDesc(ns+"vap_receive_packets_total", "VAP Packets Received", labelV, nil), - VAPTxBytes: prometheus.NewDesc(ns+"vap_transmit_bytes_total", "VAP Bytes Transmitted", labelV, nil), - VAPTxDropped: prometheus.NewDesc(ns+"vap_transmit_dropped_total", "VAP Dropped Transmitted", labelV, nil), - VAPTxErrors: prometheus.NewDesc(ns+"vap_transmit_errors_total", "VAP Errors Transmitted", labelV, nil), - VAPTxPackets: prometheus.NewDesc(ns+"vap_transmit_packets_total", "VAP Packets Transmitted", labelV, nil), - VAPTxPower: prometheus.NewDesc(ns+"vap_transmit_power", "VAP Transmit Power", labelV, nil), - VAPTxRetries: prometheus.NewDesc(ns+"vap_transmit_retries_total", "VAP Retries Transmitted", labelV, nil), - VAPTxCombinedRetries: prometheus.NewDesc(ns+"vap_transmit_retries_combined_total", "VAP Retries Combined Transmitted", labelV, nil), - VAPTxDataMpduBytes: prometheus.NewDesc(ns+"vap_data_mpdu_transmit_bytes_total", "VAP Data MPDU Bytes Transmitted", labelV, nil), - VAPTxRtsRetries: prometheus.NewDesc(ns+"vap_transmit_rts_retries_total", "VAP RTS Retries Transmitted", labelV, nil), - VAPTxSuccess: prometheus.NewDesc(ns+"vap_transmit_success_total", "VAP Success Transmits", labelV, nil), - VAPTxTotal: prometheus.NewDesc(ns+"vap_transmit_total", "VAP Transmit Total", labelV, nil), - VAPTxGoodbytes: prometheus.NewDesc(ns+"vap_transmit_goodbyes", "VAP Goodbyes Transmitted", labelV, nil), - VAPTxLatAvg: prometheus.NewDesc(ns+"vap_transmit_latency_average_seconds", "VAP Latency Average Transmit", labelV, nil), - VAPTxLatMax: prometheus.NewDesc(ns+"vap_transmit_latency_maximum_seconds", "VAP Latency Maximum Transmit", labelV, nil), - VAPTxLatMin: prometheus.NewDesc(ns+"vap_transmit_latency_minimum_seconds", "VAP Latency Minimum Transmit", labelV, nil), - VAPRxGoodbytes: prometheus.NewDesc(ns+"vap_receive_goodbyes", "VAP Goodbyes Received", labelV, nil), - VAPRxLatAvg: prometheus.NewDesc(ns+"vap_receive_latency_average_seconds", "VAP Latency Average Receive", labelV, nil), - VAPRxLatMax: prometheus.NewDesc(ns+"vap_receive_latency_maximum_seconds", "VAP Latency Maximum Receive", labelV, nil), - VAPRxLatMin: prometheus.NewDesc(ns+"vap_receive_latency_minimum_seconds", "VAP Latency Minimum Receive", labelV, nil), - VAPWifiTxLatencyMovAvg: prometheus.NewDesc(ns+"vap_transmit_latency_moving_avg_seconds", "VAP Latency Moving Average Tramsit", labelV, nil), - VAPWifiTxLatencyMovMax: prometheus.NewDesc(ns+"vap_transmit_latency_moving_max_seconds", "VAP Latency Moving Maximum Tramsit", labelV, nil), - VAPWifiTxLatencyMovMin: prometheus.NewDesc(ns+"vap_transmit_latency_moving_min_seconds", "VAP Latency Moving Minimum Tramsit", labelV, nil), - VAPWifiTxLatencyMovTotal: prometheus.NewDesc(ns+"vap_transmit_latency_moving_total", "VAP Latency Moving Total Tramsit", labelV, nil), - VAPWifiTxLatencyMovCount: prometheus.NewDesc(ns+"vap_transmit_latency_moving_count", "VAP Latency Moving Count Tramsit", labelV, nil), + VAPCcq: nd(ns+"vap_ccq_ratio", "VAP Client Connection Quality", labelV, nil), + VAPMacFilterRejections: nd(ns+"vap_mac_filter_rejects_total", "VAP MAC Filter Rejections", labelV, nil), + VAPNumSatisfactionSta: nd(ns+"vap_satisfaction_stations", "VAP Number Satisifaction Stations", labelV, nil), + VAPAvgClientSignal: nd(ns+"vap_average_client_signal", "VAP Average Client Signal", labelV, nil), + VAPSatisfaction: nd(ns+"vap_satisfaction_ratio", "VAP Satisfaction", labelV, nil), + VAPSatisfactionNow: nd(ns+"vap_satisfaction_now_ratio", "VAP Satisfaction Now", labelV, nil), + VAPDNSAvgLatency: nd(ns+"vap_dns_latency_average_seconds", "VAP DNS Latency Average", labelV, nil), + VAPRxBytes: nd(ns+"vap_receive_bytes_total", "VAP Bytes Received", labelV, nil), + VAPRxCrypts: nd(ns+"vap_receive_crypts_total", "VAP Crypts Received", labelV, nil), + VAPRxDropped: nd(ns+"vap_receive_dropped_total", "VAP Dropped Received", labelV, nil), + VAPRxErrors: nd(ns+"vap_receive_errors_total", "VAP Errors Received", labelV, nil), + VAPRxFrags: nd(ns+"vap_receive_frags_total", "VAP Frags Received", labelV, nil), + VAPRxNwids: nd(ns+"vap_receive_nwids_total", "VAP Nwids Received", labelV, nil), + VAPRxPackets: nd(ns+"vap_receive_packets_total", "VAP Packets Received", labelV, nil), + VAPTxBytes: nd(ns+"vap_transmit_bytes_total", "VAP Bytes Transmitted", labelV, nil), + VAPTxDropped: nd(ns+"vap_transmit_dropped_total", "VAP Dropped Transmitted", labelV, nil), + VAPTxErrors: nd(ns+"vap_transmit_errors_total", "VAP Errors Transmitted", labelV, nil), + VAPTxPackets: nd(ns+"vap_transmit_packets_total", "VAP Packets Transmitted", labelV, nil), + VAPTxPower: nd(ns+"vap_transmit_power", "VAP Transmit Power", labelV, nil), + VAPTxRetries: nd(ns+"vap_transmit_retries_total", "VAP Retries Transmitted", labelV, nil), + VAPTxCombinedRetries: nd(ns+"vap_transmit_retries_combined_total", "VAP Retries Combined Tx", labelV, nil), + VAPTxDataMpduBytes: nd(ns+"vap_data_mpdu_transmit_bytes_total", "VAP Data MPDU Bytes Tx", labelV, nil), + VAPTxRtsRetries: nd(ns+"vap_transmit_rts_retries_total", "VAP RTS Retries Transmitted", labelV, nil), + VAPTxSuccess: nd(ns+"vap_transmit_success_total", "VAP Success Transmits", labelV, nil), + VAPTxTotal: nd(ns+"vap_transmit_total", "VAP Transmit Total", labelV, nil), + VAPTxGoodbytes: nd(ns+"vap_transmit_goodbyes", "VAP Goodbyes Transmitted", labelV, nil), + VAPTxLatAvg: nd(ns+"vap_transmit_latency_average_seconds", "VAP Latency Average Tx", labelV, nil), + VAPTxLatMax: nd(ns+"vap_transmit_latency_maximum_seconds", "VAP Latency Maximum Tx", labelV, nil), + VAPTxLatMin: nd(ns+"vap_transmit_latency_minimum_seconds", "VAP Latency Minimum Tx", labelV, nil), + VAPRxGoodbytes: nd(ns+"vap_receive_goodbyes", "VAP Goodbyes Received", labelV, nil), + VAPRxLatAvg: nd(ns+"vap_receive_latency_average_seconds", "VAP Latency Average Rx", labelV, nil), + VAPRxLatMax: nd(ns+"vap_receive_latency_maximum_seconds", "VAP Latency Maximum Rx", labelV, nil), + VAPRxLatMin: nd(ns+"vap_receive_latency_minimum_seconds", "VAP Latency Minimum Rx", labelV, nil), + VAPWifiTxLatencyMovAvg: nd(ns+"vap_transmit_latency_moving_avg_seconds", "VAP Latency Moving Avg Tx", labelV, nil), + VAPWifiTxLatencyMovMax: nd(ns+"vap_transmit_latency_moving_max_seconds", "VAP Latency Moving Min Tx", labelV, nil), + VAPWifiTxLatencyMovMin: nd(ns+"vap_transmit_latency_moving_min_seconds", "VAP Latency Moving Max Tx", labelV, nil), + VAPWifiTxLatencyMovTotal: nd(ns+"vap_transmit_latency_moving_total", "VAP Latency Moving Total Tramsit", labelV, nil), + VAPWifiTxLatencyMovCount: nd(ns+"vap_transmit_latency_moving_count", "VAP Latency Moving Count Tramsit", labelV, nil), // N each - 1 per Radio. 1-4 radios per AP usually - RadioCurrentAntennaGain: prometheus.NewDesc(ns+"radio_current_antenna_gain", "Radio Current Antenna Gain", labelR, nil), - RadioHt: prometheus.NewDesc(ns+"radio_ht", "Radio HT", labelR, nil), - RadioMaxTxpower: prometheus.NewDesc(ns+"radio_max_transmit_power", "Radio Maximum Transmit Power", labelR, nil), - RadioMinTxpower: prometheus.NewDesc(ns+"radio_min_transmit_power", "Radio Minimum Transmit Power", labelR, nil), - RadioNss: prometheus.NewDesc(ns+"radio_nss", "Radio Nss", labelR, nil), - RadioRadioCaps: prometheus.NewDesc(ns+"radio_caps", "Radio Capabilities", labelR, nil), - RadioTxPower: prometheus.NewDesc(ns+"radio_transmit_power", "Radio Transmit Power", labelR, nil), - RadioAstBeXmit: prometheus.NewDesc(ns+"radio_ast_be_xmit", "Radio AstBe Transmit", labelR, nil), - RadioChannel: prometheus.NewDesc(ns+"radio_channel", "Radio Channel", labelR, nil), - RadioCuSelfRx: prometheus.NewDesc(ns+"radio_channel_utilization_receive_ratio", "Radio Channel Utilization Receive", labelR, nil), - RadioCuSelfTx: prometheus.NewDesc(ns+"radio_channel_utilization_transmit_ratio", "Radio Channel Utilization Transmit", labelR, nil), - RadioExtchannel: prometheus.NewDesc(ns+"radio_ext_channel", "Radio Ext Channel", labelR, nil), - RadioGain: prometheus.NewDesc(ns+"radio_gain", "Radio Gain", labelR, nil), - RadioNumSta: prometheus.NewDesc(ns+"radio_stations", "Radio Total Station Count", append(labelR, "station_type"), nil), - RadioTxPackets: prometheus.NewDesc(ns+"radio_transmit_packets", "Radio Transmitted Packets", labelR, nil), - RadioTxRetries: prometheus.NewDesc(ns+"radio_transmit_retries", "Radio Transmit Retries", labelR, nil), + RadioCurrentAntennaGain: nd(ns+"radio_current_antenna_gain", "Radio Current Antenna Gain", labelR, nil), + RadioHt: nd(ns+"radio_ht", "Radio HT", labelR, nil), + RadioMaxTxpower: nd(ns+"radio_max_transmit_power", "Radio Maximum Transmit Power", labelR, nil), + RadioMinTxpower: nd(ns+"radio_min_transmit_power", "Radio Minimum Transmit Power", labelR, nil), + RadioNss: nd(ns+"radio_nss", "Radio Nss", labelR, nil), + RadioRadioCaps: nd(ns+"radio_caps", "Radio Capabilities", labelR, nil), + RadioTxPower: nd(ns+"radio_transmit_power", "Radio Transmit Power", labelR, nil), + RadioAstBeXmit: nd(ns+"radio_ast_be_xmit", "Radio AstBe Transmit", labelR, nil), + RadioChannel: nd(ns+"radio_channel", "Radio Channel", labelR, nil), + RadioCuSelfRx: nd(ns+"radio_channel_utilization_receive_ratio", "Channel Utilization Rx", labelR, nil), + RadioCuSelfTx: nd(ns+"radio_channel_utilization_transmit_ratio", "Channel Utilization Tx", labelR, nil), + RadioExtchannel: nd(ns+"radio_ext_channel", "Radio Ext Channel", labelR, nil), + RadioGain: nd(ns+"radio_gain", "Radio Gain", labelR, nil), + RadioNumSta: nd(ns+"radio_stations", "Radio Total Station Count", append(labelR, "station_type"), nil), + RadioTxPackets: nd(ns+"radio_transmit_packets", "Radio Transmitted Packets", labelR, nil), + RadioTxRetries: nd(ns+"radio_transmit_retries", "Radio Transmit Retries", labelR, nil), } } @@ -283,6 +284,7 @@ func (u *promUnifi) exportRADtable(r report, labels []string, rt unifi.RadioTabl labelR := []string{p.Name, p.Radio, labels[1], labels[2]} labelRUser := append(labelR, "user") labelRGuest := append(labelR, "guest") + r.send([]*metric{ {u.UAP.RadioCurrentAntennaGain, gauge, p.CurrentAntennaGain, labelR}, {u.UAP.RadioHt, gauge, p.Ht, labelR}, @@ -311,6 +313,7 @@ func (u *promUnifi) exportRADtable(r report, labels []string, rt unifi.RadioTabl {u.UAP.RadioTxPackets, gauge, t.TxPackets, labelR}, {u.UAP.RadioTxRetries, gauge, t.TxRetries, labelR}, }) + break } } diff --git a/pkg/promunifi/usg.go b/pkg/promunifi/usg.go index 631ef358..08ee3781 100644 --- a/pkg/promunifi/usg.go +++ b/pkg/promunifi/usg.go @@ -37,6 +37,7 @@ type usg struct { func descUSG(ns string) *usg { labels := []string{"port", "site_name", "name"} + return &usg{ WanRxPackets: prometheus.NewDesc(ns+"wan_receive_packets_total", "WAN Receive Packets Total", labels, nil), WanRxBytes: prometheus.NewDesc(ns+"wan_receive_bytes_total", "WAN Receive Bytes Total", labels, nil), @@ -75,6 +76,7 @@ func (u *promUnifi) exportUSG(r report, d *unifi.USG) { labels := []string{d.Type, d.SiteName, d.Name} infoLabels := []string{d.Version, d.Model, d.Serial, d.Mac, d.IP, d.ID, d.Bytes.Txt, d.Uptime.Txt} + // Gateway System Data. u.exportWANPorts(r, labels, d.Wan1, d.Wan2) u.exportBYTstats(r, labels, d.TxBytes, d.RxBytes) @@ -95,6 +97,7 @@ func (u *promUnifi) exportUSGstats(r report, labels []string, gw *unifi.Gw, st u labelLan := []string{"lan", labels[1], labels[2]} labelWan := []string{"all", labels[1], labels[2]} + r.send([]*metric{ {u.USG.LanRxPackets, counter, gw.LanRxPackets, labelLan}, {u.USG.LanRxBytes, counter, gw.LanRxBytes, labelLan}, @@ -119,6 +122,7 @@ func (u *promUnifi) exportWANPorts(r report, labels []string, wans ...unifi.Wan) } labelWan := []string{wan.Name, labels[1], labels[2]} + r.send([]*metric{ {u.USG.WanRxPackets, counter, wan.RxPackets, labelWan}, {u.USG.WanRxBytes, counter, wan.RxBytes, labelWan}, diff --git a/pkg/promunifi/usw.go b/pkg/promunifi/usw.go index e89506cb..d700583f 100644 --- a/pkg/promunifi/usw.go +++ b/pkg/promunifi/usw.go @@ -49,45 +49,46 @@ func descUSW(ns string) *usw { pns := ns + "port_" labelS := []string{"site_name", "name"} labelP := []string{"port_id", "port_num", "port_name", "port_mac", "port_ip", "site_name", "name"} + nd := prometheus.NewDesc return &usw{ // This data may be derivable by sum()ing the port data. - SwRxPackets: prometheus.NewDesc(ns+"switch_receive_packets_total", "Switch Packets Received Total", labelS, nil), - SwRxBytes: prometheus.NewDesc(ns+"switch_receive_bytes_total", "Switch Bytes Received Total", labelS, nil), - SwRxErrors: prometheus.NewDesc(ns+"switch_receive_errors_total", "Switch Errors Received Total", labelS, nil), - SwRxDropped: prometheus.NewDesc(ns+"switch_receive_dropped_total", "Switch Dropped Received Total", labelS, nil), - SwRxCrypts: prometheus.NewDesc(ns+"switch_receive_crypts_total", "Switch Crypts Received Total", labelS, nil), - SwRxFrags: prometheus.NewDesc(ns+"switch_receive_frags_total", "Switch Frags Received Total", labelS, nil), - SwTxPackets: prometheus.NewDesc(ns+"switch_transmit_packets_total", "Switch Packets Transmit Total", labelS, nil), - SwTxBytes: prometheus.NewDesc(ns+"switch_transmit_bytes_total", "Switch Bytes Transmit Total", labelS, nil), - SwTxErrors: prometheus.NewDesc(ns+"switch_transmit_errors_total", "Switch Errors Transmit Total", labelS, nil), - SwTxDropped: prometheus.NewDesc(ns+"switch_transmit_dropped_total", "Switch Dropped Transmit Total", labelS, nil), - SwTxRetries: prometheus.NewDesc(ns+"switch_transmit_retries_total", "Switch Retries Transmit Total", labelS, nil), - SwRxMulticast: prometheus.NewDesc(ns+"switch_receive_multicast_total", "Switch Multicast Receive Total", labelS, nil), - SwRxBroadcast: prometheus.NewDesc(ns+"switch_receive_broadcast_total", "Switch Broadcast Receive Total", labelS, nil), - SwTxMulticast: prometheus.NewDesc(ns+"switch_transmit_multicast_total", "Switch Multicast Transmit Total", labelS, nil), - SwTxBroadcast: prometheus.NewDesc(ns+"switch_transmit_broadcast_total", "Switch Broadcast Transmit Total", labelS, nil), - SwBytes: prometheus.NewDesc(ns+"switch_bytes_total", "Switch Bytes Transferred Total", labelS, nil), + SwRxPackets: nd(ns+"switch_receive_packets_total", "Switch Packets Received Total", labelS, nil), + SwRxBytes: nd(ns+"switch_receive_bytes_total", "Switch Bytes Received Total", labelS, nil), + SwRxErrors: nd(ns+"switch_receive_errors_total", "Switch Errors Received Total", labelS, nil), + SwRxDropped: nd(ns+"switch_receive_dropped_total", "Switch Dropped Received Total", labelS, nil), + SwRxCrypts: nd(ns+"switch_receive_crypts_total", "Switch Crypts Received Total", labelS, nil), + SwRxFrags: nd(ns+"switch_receive_frags_total", "Switch Frags Received Total", labelS, nil), + SwTxPackets: nd(ns+"switch_transmit_packets_total", "Switch Packets Transmit Total", labelS, nil), + SwTxBytes: nd(ns+"switch_transmit_bytes_total", "Switch Bytes Transmit Total", labelS, nil), + SwTxErrors: nd(ns+"switch_transmit_errors_total", "Switch Errors Transmit Total", labelS, nil), + SwTxDropped: nd(ns+"switch_transmit_dropped_total", "Switch Dropped Transmit Total", labelS, nil), + SwTxRetries: nd(ns+"switch_transmit_retries_total", "Switch Retries Transmit Total", labelS, nil), + SwRxMulticast: nd(ns+"switch_receive_multicast_total", "Switch Multicast Receive Total", labelS, nil), + SwRxBroadcast: nd(ns+"switch_receive_broadcast_total", "Switch Broadcast Receive Total", labelS, nil), + SwTxMulticast: nd(ns+"switch_transmit_multicast_total", "Switch Multicast Transmit Total", labelS, nil), + SwTxBroadcast: nd(ns+"switch_transmit_broadcast_total", "Switch Broadcast Transmit Total", labelS, nil), + SwBytes: nd(ns+"switch_bytes_total", "Switch Bytes Transferred Total", labelS, nil), // per-port data - PoeCurrent: prometheus.NewDesc(pns+"poe_amperes", "POE Current", labelP, nil), - PoePower: prometheus.NewDesc(pns+"poe_watts", "POE Power", labelP, nil), - PoeVoltage: prometheus.NewDesc(pns+"poe_volts", "POE Voltage", labelP, nil), - RxBroadcast: prometheus.NewDesc(pns+"receive_broadcast_total", "Receive Broadcast", labelP, nil), - RxBytes: prometheus.NewDesc(pns+"receive_bytes_total", "Total Receive Bytes", labelP, nil), - RxBytesR: prometheus.NewDesc(pns+"receive_rate_bytes", "Receive Bytes Rate", labelP, nil), - RxDropped: prometheus.NewDesc(pns+"receive_dropped_total", "Total Receive Dropped", labelP, nil), - RxErrors: prometheus.NewDesc(pns+"receive_errors_total", "Total Receive Errors", labelP, nil), - RxMulticast: prometheus.NewDesc(pns+"receive_multicast_total", "Total Receive Multicast", labelP, nil), - RxPackets: prometheus.NewDesc(pns+"receive_packets_total", "Total Receive Packets", labelP, nil), - Satisfaction: prometheus.NewDesc(pns+"satisfaction_ratio", "Satisfaction", labelP, nil), - Speed: prometheus.NewDesc(pns+"port_speed_bps", "Speed", labelP, nil), - TxBroadcast: prometheus.NewDesc(pns+"transmit_broadcast_total", "Total Transmit Broadcast", labelP, nil), - TxBytes: prometheus.NewDesc(pns+"transmit_bytes_total", "Total Transmit Bytes", labelP, nil), - TxBytesR: prometheus.NewDesc(pns+"transmit_rate_bytes", "Transmit Bytes Rate", labelP, nil), - TxDropped: prometheus.NewDesc(pns+"transmit_dropped_total", "Total Transmit Dropped", labelP, nil), - TxErrors: prometheus.NewDesc(pns+"transmit_errors_total", "Total Transmit Errors", labelP, nil), - TxMulticast: prometheus.NewDesc(pns+"transmit_multicast_total", "Total Tranmist Multicast", labelP, nil), - TxPackets: prometheus.NewDesc(pns+"transmit_packets_total", "Total Transmit Packets", labelP, nil), + PoeCurrent: nd(pns+"poe_amperes", "POE Current", labelP, nil), + PoePower: nd(pns+"poe_watts", "POE Power", labelP, nil), + PoeVoltage: nd(pns+"poe_volts", "POE Voltage", labelP, nil), + RxBroadcast: nd(pns+"receive_broadcast_total", "Receive Broadcast", labelP, nil), + RxBytes: nd(pns+"receive_bytes_total", "Total Receive Bytes", labelP, nil), + RxBytesR: nd(pns+"receive_rate_bytes", "Receive Bytes Rate", labelP, nil), + RxDropped: nd(pns+"receive_dropped_total", "Total Receive Dropped", labelP, nil), + RxErrors: nd(pns+"receive_errors_total", "Total Receive Errors", labelP, nil), + RxMulticast: nd(pns+"receive_multicast_total", "Total Receive Multicast", labelP, nil), + RxPackets: nd(pns+"receive_packets_total", "Total Receive Packets", labelP, nil), + Satisfaction: nd(pns+"satisfaction_ratio", "Satisfaction", labelP, nil), + Speed: nd(pns+"port_speed_bps", "Speed", labelP, nil), + TxBroadcast: nd(pns+"transmit_broadcast_total", "Total Transmit Broadcast", labelP, nil), + TxBytes: nd(pns+"transmit_bytes_total", "Total Transmit Bytes", labelP, nil), + TxBytesR: nd(pns+"transmit_rate_bytes", "Transmit Bytes Rate", labelP, nil), + TxDropped: nd(pns+"transmit_dropped_total", "Total Transmit Dropped", labelP, nil), + TxErrors: nd(pns+"transmit_errors_total", "Total Transmit Errors", labelP, nil), + TxMulticast: nd(pns+"transmit_multicast_total", "Total Tranmist Multicast", labelP, nil), + TxPackets: nd(pns+"transmit_packets_total", "Total Transmit Packets", labelP, nil), } } @@ -98,6 +99,7 @@ func (u *promUnifi) exportUSW(r report, d *unifi.USW) { labels := []string{d.Type, d.SiteName, d.Name} infoLabels := []string{d.Version, d.Model, d.Serial, d.Mac, d.IP, d.ID, d.Bytes.Txt, d.Uptime.Txt} + u.exportUSWstats(r, labels, d.Stat.Sw) u.exportPRTtable(r, labels, d.PortTable) u.exportBYTstats(r, labels, d.TxBytes, d.RxBytes) @@ -129,6 +131,7 @@ func (u *promUnifi) exportUSWstats(r report, labels []string, sw *unifi.Sw) { } labelS := labels[1:] + r.send([]*metric{ {u.USW.SwRxPackets, counter, sw.RxPackets, labelS}, {u.USW.SwRxBytes, counter, sw.RxBytes, labelS}, From 44c544d8e152c89ed1a24c455badd64b395c263f Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Mon, 16 Dec 2019 03:11:40 -0800 Subject: [PATCH 07/32] Add dynamic plugin support --- .gitignore | 1 + Makefile | 13 +++++- examples/MANUAL.md | 6 +-- examples/up.conf.example | 9 +++- examples/up.json.example | 28 +++++++----- examples/up.xml.example | 26 ++++++----- examples/up.yaml.example | 22 +++++----- pkg/inputunifi/input.go | 4 +- pkg/inputunifi/interface.go | 4 ++ pkg/poller/build_macos.go | 3 ++ pkg/poller/build_unix.go | 3 ++ pkg/poller/build_windows.go | 3 ++ pkg/poller/config.go | 88 +++++++++++++++++++++++++++---------- plugins/mysql/README.md | 26 +++++++++++ plugins/mysql/main.go | 45 +++++++++++++++++++ 15 files changed, 217 insertions(+), 64 deletions(-) create mode 100644 plugins/mysql/README.md create mode 100644 plugins/mysql/main.go diff --git a/.gitignore b/.gitignore index 118e4d74..0ba43855 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ bitly_token github_deploy_key gpg.signing.key .secret-files.tar +*.so diff --git a/Makefile b/Makefile index 4cf96dc5..772d143f 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ IGNORED:=$(shell bash -c "source .metadata.sh ; env | sed 's/=/:=/;s/^/export /' # md2roff turns markdown into man files and html files. MD2ROFF_BIN=github.com/github/hub/md2roff-bin + # Travis CI passes the version in. Local builds get it from the current git tag. ifeq ($(VERSION),) include .metadata.make @@ -185,10 +186,11 @@ $(BINARY)_$(VERSION)-$(ITERATION)_armhf.deb: package_build_linux_armhf check_fpm # Build an environment that can be packaged for linux. package_build_linux: readme man linux # Building package environment for linux. - mkdir -p $@/usr/bin $@/etc/$(BINARY) $@/usr/share/man/man1 $@/usr/share/doc/$(BINARY) + mkdir -p $@/usr/bin $@/etc/$(BINARY) $@/usr/share/man/man1 $@/usr/share/doc/$(BINARY) $@/usr/lib/$(BINARY) # Copying the binary, config file, unit file, and man page into the env. cp $(BINARY).amd64.linux $@/usr/bin/$(BINARY) cp *.1.gz $@/usr/share/man/man1 + cp *.so $@/usr/lib/$(BINARY)/ cp examples/$(CONFIG_FILE).example $@/etc/$(BINARY)/ cp examples/$(CONFIG_FILE).example $@/etc/$(BINARY)/$(CONFIG_FILE) cp LICENSE *.html examples/*?.?* $@/usr/share/doc/$(BINARY)/ @@ -253,6 +255,12 @@ $(BINARY).rb: v$(VERSION).tar.gz.sha256 init/homebrew/$(FORMULA).rb.tmpl init/homebrew/$(FORMULA).rb.tmpl | tee $(BINARY).rb # That perl line turns hello-world into HelloWorld, etc. +# This is kind janky because it always builds the plugins, even if they are already built. +# Still needs to be made multi arch, which adds complications, especially when creating packages. +plugins: $(patsubst %.go,%.so,$(wildcard ./plugins/*/main.go)) +$(patsubst %.go,%.so,$(wildcard ./plugins/*/main.go)): + go build -o $(patsubst plugins/%/main.so,%.so,$@) -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./$(patsubst %main.so,%,$@) + # Extras # Run code tests and lint. @@ -285,8 +293,9 @@ install: man readme $(BINARY) @[ "$(PREFIX)" != "" ] || (echo "Unable to continue, PREFIX not set. Use: make install PREFIX=/usr/local ETC=/usr/local/etc" && false) @[ "$(ETC)" != "" ] || (echo "Unable to continue, ETC not set. Use: make install PREFIX=/usr/local ETC=/usr/local/etc" && false) # Copying the binary, config file, unit file, and man page into the env. - /usr/bin/install -m 0755 -d $(PREFIX)/bin $(PREFIX)/share/man/man1 $(ETC)/$(BINARY) $(PREFIX)/share/doc/$(BINARY) + /usr/bin/install -m 0755 -d $(PREFIX)/bin $(PREFIX)/share/man/man1 $(ETC)/$(BINARY) $(PREFIX)/share/doc/$(BINARY) $(PREFIX)/lib/$(BINARY) /usr/bin/install -m 0755 -cp $(BINARY) $(PREFIX)/bin/$(BINARY) + /usr/bin/install -m 0755 -cp *.so $(PREFIX)/lib/$(BINARY)/ /usr/bin/install -m 0644 -cp $(BINARY).1.gz $(PREFIX)/share/man/man1 /usr/bin/install -m 0644 -cp examples/$(CONFIG_FILE).example $(ETC)/$(BINARY)/ [ -f $(ETC)/$(BINARY)/$(CONFIG_FILE) ] || /usr/bin/install -m 0644 -cp examples/$(CONFIG_FILE).example $(ETC)/$(BINARY)/$(CONFIG_FILE) diff --git a/examples/MANUAL.md b/examples/MANUAL.md index 012cd343..cabf4db8 100644 --- a/examples/MANUAL.md +++ b/examples/MANUAL.md @@ -66,7 +66,7 @@ is provided so the application can be easily adapted to any environment. `Config File Parameters` Additional parameters are added by output packages. Parameters can also be set -using environment variables. See the GitHub wiki for more information! +using environment variables. See the GitHub wiki for more information! >>> POLLER FIELDS FOLLOW - you may have multiple controllers: @@ -79,7 +79,7 @@ using environment variables. See the GitHub wiki for more information! errors will be logged. Using this with debug=true adds line numbers to any error logs. - >>> CONTROLLER FIELDS FOLLOW - you may have multiple controllers: + >>> UNIFI CONTROLLER FIELDS FOLLOW - you may have multiple controllers: sites default: ["all"] This list of strings should represent the names of sites on the UniFi @@ -96,7 +96,7 @@ using environment variables. See the GitHub wiki for more information! Username used to authenticate with UniFi controller. This should be a special service account created on the control with read-only access. - user no default + pass no default Password used to authenticate with UniFi controller. This can also be set in an environment variable instead of a configuration file. diff --git a/examples/up.conf.example b/examples/up.conf.example index 75c12941..43aa9831 100644 --- a/examples/up.conf.example +++ b/examples/up.conf.example @@ -13,6 +13,8 @@ debug = false # Recommend enabling debug with this setting for better error logging. quiet = false +# Load dynamic plugins. Advanced use; only sample mysql plugin provided by default. +plugins = [] #### OUTPUTS @@ -40,9 +42,12 @@ interval = "30s" #### INPUTS +[unifi] +disable = false + # You may repeat the following section to poll additional controllers. -[[controller]] -# Friendly name used in dashboards. +[[unifi.controller]] +# Friendly name used in dashboards. Uses URL if left empty. name = "" url = "https://127.0.0.1:8443" diff --git a/examples/up.json.example b/examples/up.json.example index 65e4d27e..12ba4a4f 100644 --- a/examples/up.json.example +++ b/examples/up.json.example @@ -1,7 +1,8 @@ { "poller": { "debug": false, - "quiet": false + "quiet": false, + "plugins": [] }, "prometheus": { @@ -20,14 +21,19 @@ "interval": "30s" }, - "controller": [{ - "name": "", - "user": "influx", - "pass": "", - "url": "https://127.0.0.1:8443", - "sites": ["all"], - "save_ids": false, - "save_sites": true, - "verify_ssl": false - }] + "unifi": { + "disable": false, + "controllers": [ + { + "name": "", + "user": "influx", + "pass": "", + "url": "https://127.0.0.1:8443", + "sites": ["all"], + "save_ids": false, + "save_sites": true, + "verify_ssl": false + } + ] + } } diff --git a/examples/up.xml.example b/examples/up.xml.example index 710c01ba..ec98169d 100644 --- a/examples/up.xml.example +++ b/examples/up.xml.example @@ -4,8 +4,11 @@ # UniFi Poller primary configuration file. XML FORMAT # # provided values are defaults. See up.conf.example! # ####################################################### + + and are lists of strings and may be repeated. --> + 0.0.0.0:9130 @@ -21,15 +24,16 @@ false - - - all - influx - - https://127.0.0.1:8443 - false - false - true - - + + + + all + influx + + https://127.0.0.1:8443 + false + false + true + + diff --git a/examples/up.yaml.example b/examples/up.yaml.example index 611c5fa8..2be57012 100644 --- a/examples/up.yaml.example +++ b/examples/up.yaml.example @@ -7,6 +7,7 @@ poller: debug: false quiet: false + plugins: [] prometheus: disable: false @@ -22,13 +23,14 @@ influxdb: db: "unifi" verify_ssl: false -controller: - - name: "" - user: "influx" - pass: "" - url: "https://127.0.0.1:8443" - sites: - - all - verify_ssl: false - save_ids: false - save_sites: true +unifi: + controllers: + - name: "" + user: "influx" + pass: "" + url: "https://127.0.0.1:8443" + sites: + - all + verify_ssl: false + save_ids: false + save_sites: true diff --git a/pkg/inputunifi/input.go b/pkg/inputunifi/input.go index f797d35b..15a027dd 100644 --- a/pkg/inputunifi/input.go +++ b/pkg/inputunifi/input.go @@ -27,7 +27,7 @@ type Controller struct { User string `json:"user" toml:"user" xml:"user" yaml:"user"` Pass string `json:"pass" toml:"pass" xml:"pass" yaml:"pass"` URL string `json:"url" toml:"url" xml:"url" yaml:"url"` - Sites []string `json:"sites,omitempty" toml:"sites,omitempty" xml:"sites" yaml:"sites"` + Sites []string `json:"sites,omitempty" toml:"sites,omitempty" xml:"site" yaml:"sites"` Unifi *unifi.Unifi `json:"-" toml:"-" xml:"-" yaml:"-"` } @@ -35,7 +35,7 @@ type Controller struct { type Config struct { sync.RWMutex // locks the Unifi struct member when re-authing to unifi. Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` - Controllers []Controller `json:"controller" toml:"controller" xml:"controller" yaml:"controller"` + Controllers []Controller `json:"controllers" toml:"controller" xml:"controller" yaml:"controllers"` } func init() { diff --git a/pkg/inputunifi/interface.go b/pkg/inputunifi/interface.go index a2ef9765..1db32906 100644 --- a/pkg/inputunifi/interface.go +++ b/pkg/inputunifi/interface.go @@ -10,6 +10,10 @@ import ( // Metrics grabs all the measurements from a UniFi controller and returns them. func (u *InputUnifi) Metrics() (*poller.Metrics, error) { + if u.Config.Disable { + return nil, nil + } + errs := []string{} metrics := &poller.Metrics{} diff --git a/pkg/poller/build_macos.go b/pkg/poller/build_macos.go index b3f37dbf..9292f409 100644 --- a/pkg/poller/build_macos.go +++ b/pkg/poller/build_macos.go @@ -4,3 +4,6 @@ package poller // DefaultConfFile is where to find config if --config is not prvided. const DefaultConfFile = "/usr/local/etc/unifi-poller/up.conf" + +// DefaultObjPath is the path to look for shared object libraries (plugins). +const DefaultObjPath = "/usr/local/lib/unifi-poller" diff --git a/pkg/poller/build_unix.go b/pkg/poller/build_unix.go index c1f525a9..fd381e19 100644 --- a/pkg/poller/build_unix.go +++ b/pkg/poller/build_unix.go @@ -4,3 +4,6 @@ package poller // DefaultConfFile is where to find config if --config is not prvided. const DefaultConfFile = "/etc/unifi-poller/up.conf" + +// DefaultObjPath is the path to look for shared object libraries (plugins). +const DefaultObjPath = "/usr/lib/unifi-poller" diff --git a/pkg/poller/build_windows.go b/pkg/poller/build_windows.go index a74c76a8..69d964e8 100644 --- a/pkg/poller/build_windows.go +++ b/pkg/poller/build_windows.go @@ -4,3 +4,6 @@ package poller // DefaultConfFile is where to find config if --config is not prvided. const DefaultConfFile = `C:\ProgramData\unifi-poller\up.conf` + +// DefaultObjPath is useless in this context. Bummer. +const DefaultObjPath = "PLUGINS_DO_NOT_WORK_ON_WINDOWS_SOWWWWWY" diff --git a/pkg/poller/config.go b/pkg/poller/config.go index 03e69b81..a43caceb 100644 --- a/pkg/poller/config.go +++ b/pkg/poller/config.go @@ -9,6 +9,10 @@ package poller */ import ( + "os" + "path" + "plugin" + "strings" "time" "github.com/spf13/pflag" @@ -53,33 +57,24 @@ type Config struct { // Poller is the global config values. type Poller struct { - Debug bool `json:"debug" toml:"debug" xml:"debug,attr" yaml:"debug"` - Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet,attr" yaml:"quiet"` + Plugins []string `json:"plugins" toml:"plugins" xml:"plugin" yaml:"plugins"` + Debug bool `json:"debug" toml:"debug" xml:"debug,attr" yaml:"debug"` + Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet,attr" yaml:"quiet"` } -// ParseConfigs parses the poller config and the config for each registered output plugin. -func (u *UnifiPoller) ParseConfigs() error { - // Parse core config. - if err := u.ParseInterface(u.Config); err != nil { - return err - } +// LoadPlugins reads-in dynamic shared libraries. +// Not used very often, if at all. +func (u *UnifiPoller) LoadPlugins() error { + for _, p := range u.Plugins { + name := strings.TrimSuffix(p, ".so") + ".so" - // Parse output plugin configs. - outputSync.Lock() - defer outputSync.Unlock() - - for _, o := range outputs { - if err := u.ParseInterface(o.Config); err != nil { - return err + if _, err := os.Stat(name); os.IsNotExist(err) { + name = path.Join(DefaultObjPath, name) } - } - // Parse input plugin configs. - inputSync.Lock() - defer inputSync.Unlock() + u.Logf("Loading Dynamic Plugin: %s", name) - for _, i := range inputs { - if err := u.ParseInterface(i.Config); err != nil { + if _, err := plugin.Open(name); err != nil { return err } } @@ -87,8 +82,27 @@ func (u *UnifiPoller) ParseConfigs() error { return nil } -// ParseInterface parses the config file and environment variables into the provided interface. -func (u *UnifiPoller) ParseInterface(i interface{}) error { +// ParseConfigs parses the poller config and the config for each registered output plugin. +func (u *UnifiPoller) ParseConfigs() error { + // Parse core config. + if err := u.parseInterface(u.Config); err != nil { + return err + } + + // Load dynamic plugins. + if err := u.LoadPlugins(); err != nil { + return err + } + + if err := u.parseInputs(); err != nil { + return err + } + + return u.parseOutputs() +} + +// parseInterface parses the config file and environment variables into the provided interface. +func (u *UnifiPoller) parseInterface(i interface{}) error { // Parse config file into provided interface. if err := config.ParseFile(i, u.Flags.ConfigFile); err != nil { return err @@ -99,3 +113,31 @@ func (u *UnifiPoller) ParseInterface(i interface{}) error { return err } + +// Parse input plugin configs. +func (u *UnifiPoller) parseInputs() error { + inputSync.Lock() + defer inputSync.Unlock() + + for _, i := range inputs { + if err := u.parseInterface(i.Config); err != nil { + return err + } + } + + return nil +} + +// Parse output plugin configs. +func (u *UnifiPoller) parseOutputs() error { + outputSync.Lock() + defer outputSync.Unlock() + + for _, o := range outputs { + if err := u.parseInterface(o.Config); err != nil { + return err + } + } + + return nil +} diff --git a/plugins/mysql/README.md b/plugins/mysql/README.md new file mode 100644 index 00000000..9c32f7fe --- /dev/null +++ b/plugins/mysql/README.md @@ -0,0 +1,26 @@ +# MYSQL Output Plugin Example + +The code here, and the dynamic plugin provided shows an example of how you can +write your own output for unifi-poller. This plugin records some very basic +data about clients on a unifi network into a mysql database. + +You could write outputs that do... anything. An example: They could compare current +connected clients to a previous list (in a db, or stored in memory), and send a +notification if it changes. The possibilities are endless. + +You must compile your plugin using the unifi-poller source for the version you're +using. In other words, to build a plugin for version 2.0.1, do this: +``` +mkdir -p $GOPATH/src/github.com/davidnewhall +cd $GOPATH/src/github.com/davidnewhall + +git clone git@github.com:davidnewhall/unifi-poller.git +cd unifi-poller + +git checkout v2.0.1 +make vendor + +cp -r plugins/ +GOOS=linux make plugins +``` +The plugin you copy in *must* have a `main.go` file for `make plugins` to build it. diff --git a/plugins/mysql/main.go b/plugins/mysql/main.go new file mode 100644 index 00000000..e0c866b9 --- /dev/null +++ b/plugins/mysql/main.go @@ -0,0 +1,45 @@ +package main + +import ( + "fmt" + + "github.com/davidnewhall/unifi-poller/pkg/poller" + "golift.io/config" +) + +// mysqlConfig represents the data that is unmarshalled from the up.conf config file for this plugins. +type mysqlConfig struct { + Interval config.Duration `json:"interval" toml:"interval" xml:"interval" yaml:"interval"` + Host string `json:"host" toml:"host" xml:"host" yaml:"host"` + User string `json:"user" toml:"user" xml:"user" yaml:"user"` + Pass string `json:"pass" toml:"pass" xml:"pass" yaml:"pass"` + DB string `json:"db" toml:"db" xml:"db" yaml:"db"` + Table string `json:"table" toml:"table" xml:"table" yaml:"table"` + // Maps do not work with ENV VARIABLES yet, but may in the future. + Fields []string `json:"fields" toml:"fields" xml:"field" yaml:"fields"` +} + +// Pointers are ignored during ENV variable unmarshal, avoid pointers to your config. +// Only capital (exported) members are unmarshaled when passed into poller.NewOutput(). +type application struct { + Config mysqlConfig `json:"mysql" toml:"mysql" xml:"mysql" yaml:"mysql"` +} + +func init() { + u := &application{Config: mysqlConfig{}} + + poller.NewOutput(&poller.Output{ + Name: "mysql", + Config: u, // pass in the struct *above* your config (so it can see the struct tags). + Method: u.Run, + }) +} + +func main() { + fmt.Println("this is a unifi-poller plugin; not an application") +} + +func (a *application) Run(c poller.Collect) error { + c.Logf("mysql plugin is not finished") + return nil +} From 141f6bea556f7c948e1c68858a84a046d620c543 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Mon, 16 Dec 2019 16:57:14 -0800 Subject: [PATCH 08/32] no idea if this will stick.. --- Makefile | 51 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 772d143f..2a3d0035 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,8 @@ $(PACKAGE_SCRIPTS) \ --config-files "/etc/$(BINARY)/$(CONFIG_FILE)" endef +PLUGINS:=$(patsubst plugins/%/main.go,%,$(wildcard plugins/*/main.go)) + VERSION_LDFLAGS:= \ -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Branch=$(BRANCH) \ -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.BuildDate=$(DATE) \ @@ -184,13 +186,14 @@ $(BINARY)_$(VERSION)-$(ITERATION)_armhf.deb: package_build_linux_armhf check_fpm [ "$(SIGNING_KEY)" == "" ] || expect -c "spawn debsigs --default-key="$(SIGNING_KEY)" --sign=origin $(BINARY)_$(VERSION)-$(ITERATION)_armhf.deb; expect -exact \"Enter passphrase: \"; send \"$(PRIVATE_KEY)\r\"; expect eof" # Build an environment that can be packaged for linux. -package_build_linux: readme man linux +package_build_linux: readme man plugins_linux_amd64 linux # Building package environment for linux. mkdir -p $@/usr/bin $@/etc/$(BINARY) $@/usr/share/man/man1 $@/usr/share/doc/$(BINARY) $@/usr/lib/$(BINARY) # Copying the binary, config file, unit file, and man page into the env. cp $(BINARY).amd64.linux $@/usr/bin/$(BINARY) cp *.1.gz $@/usr/share/man/man1 - cp *.so $@/usr/lib/$(BINARY)/ + rm -f $@/usr/lib/$(BINARY)/*.so + cp *amd64.so $@/usr/lib/$(BINARY)/ cp examples/$(CONFIG_FILE).example $@/etc/$(BINARY)/ cp examples/$(CONFIG_FILE).example $@/etc/$(BINARY)/$(CONFIG_FILE) cp LICENSE *.html examples/*?.?* $@/usr/share/doc/$(BINARY)/ @@ -199,20 +202,26 @@ package_build_linux: readme man linux sed -e "s/{{BINARY}}/$(BINARY)/g" -e "s/{{DESC}}/$(DESC)/g" \ init/systemd/template.unit.service > $@/lib/systemd/system/$(BINARY).service -package_build_linux_386: package_build_linux linux386 +package_build_linux_386: package_build_linux plugins_linux_i386 linux386 mkdir -p $@ cp -r $ /dev/null || (echo "FPM missing. Install FPM: https://fpm.readthedocs.io/en/latest/installing.html" && false) @@ -255,11 +264,29 @@ $(BINARY).rb: v$(VERSION).tar.gz.sha256 init/homebrew/$(FORMULA).rb.tmpl init/homebrew/$(FORMULA).rb.tmpl | tee $(BINARY).rb # That perl line turns hello-world into HelloWorld, etc. -# This is kind janky because it always builds the plugins, even if they are already built. -# Still needs to be made multi arch, which adds complications, especially when creating packages. -plugins: $(patsubst %.go,%.so,$(wildcard ./plugins/*/main.go)) -$(patsubst %.go,%.so,$(wildcard ./plugins/*/main.go)): - go build -o $(patsubst plugins/%/main.so,%.so,$@) -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./$(patsubst %main.so,%,$@) +# This probably wont work for most people..... +plugins: linux_plugins plugins_darwin + +linux_plugins: plugins_linux_amd64 plugins_linux_i386 plugins_linux_arm64 plugins_linux_armhf +plugins_linux_amd64: $(patsubst %,%.linux_amd64.so,$(PLUGINS)) +$(patsubst %,%.linux_amd64.so,$(PLUGINS)): + GOOS=linux GOARCH=amd64 go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.linux_amd64.so,%,$@) + +plugins_linux_i386: $(patsubst %,%.linux_i386.so,$(PLUGINS)) +$(patsubst %,%.linux_i386.so,$(PLUGINS)): + GOOS=linux GOARCH=i386 go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.linux_i386.so,%,$@) + +plugins_linux_arm64: $(patsubst %,%.linux_arm64.so,$(PLUGINS)) +$(patsubst %,%.linux_arm64.so,$(PLUGINS)): + GOOS=linux GOARCH=arm64 go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.linux_arm64.so,%,$@) + +plugins_linux_armhf: $(patsubst %,%.linux_armhf.so,$(PLUGINS)) +$(patsubst %,%.linux_armhf.so,$(PLUGINS)): + GOOS=linux GOARCH=armhf go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.linux_armhf.so,%,$@) + +plugins_darwin: $(patsubst %,%.darwin.so,$(PLUGINS)) +$(patsubst %,%.darwin.so,$(PLUGINS)): + GOOS=darwin go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.darwin.so,%,$@) # Extras @@ -283,7 +310,7 @@ deps: # Homebrew stuff. macOS only. # Used for Homebrew only. Other distros can create packages. -install: man readme $(BINARY) +install: man readme $(BINARY) plugins_darwin @echo - Done Building! - @echo - Local installation with the Makefile is only supported on macOS. @echo If you wish to install the application manually on Linux, check out the wiki: https://$(SOURCE_URL)/wiki/Installation @@ -295,7 +322,7 @@ install: man readme $(BINARY) # Copying the binary, config file, unit file, and man page into the env. /usr/bin/install -m 0755 -d $(PREFIX)/bin $(PREFIX)/share/man/man1 $(ETC)/$(BINARY) $(PREFIX)/share/doc/$(BINARY) $(PREFIX)/lib/$(BINARY) /usr/bin/install -m 0755 -cp $(BINARY) $(PREFIX)/bin/$(BINARY) - /usr/bin/install -m 0755 -cp *.so $(PREFIX)/lib/$(BINARY)/ + /usr/bin/install -m 0755 -cp *darwin.so $(PREFIX)/lib/$(BINARY)/ /usr/bin/install -m 0644 -cp $(BINARY).1.gz $(PREFIX)/share/man/man1 /usr/bin/install -m 0644 -cp examples/$(CONFIG_FILE).example $(ETC)/$(BINARY)/ [ -f $(ETC)/$(BINARY)/$(CONFIG_FILE) ] || /usr/bin/install -m 0644 -cp examples/$(CONFIG_FILE).example $(ETC)/$(BINARY)/$(CONFIG_FILE) From 9e1cadd40aa9bf7549ea0dcd8960698008dc9370 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Mon, 16 Dec 2019 17:02:01 -0800 Subject: [PATCH 09/32] typo --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2a3d0035..30287041 100644 --- a/Makefile +++ b/Makefile @@ -274,7 +274,7 @@ $(patsubst %,%.linux_amd64.so,$(PLUGINS)): plugins_linux_i386: $(patsubst %,%.linux_i386.so,$(PLUGINS)) $(patsubst %,%.linux_i386.so,$(PLUGINS)): - GOOS=linux GOARCH=i386 go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.linux_i386.so,%,$@) + GOOS=linux GOARCH=386 go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.linux_i386.so,%,$@) plugins_linux_arm64: $(patsubst %,%.linux_arm64.so,$(PLUGINS)) $(patsubst %,%.linux_arm64.so,$(PLUGINS)): From 7497f681d524ecd18ce334cdc035b86dc6c0a0ee Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Mon, 16 Dec 2019 17:11:13 -0800 Subject: [PATCH 10/32] fixes --- .metadata.sh | 3 +-- Makefile | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.metadata.sh b/.metadata.sh index 7a78171d..5c380efc 100755 --- a/.metadata.sh +++ b/.metadata.sh @@ -38,7 +38,6 @@ VERSION="$(echo $VVERSION | tr -d v | grep -E '^\S+$' || echo development)" # This produces a 0 in some envirnoments (like Homebrew), but it's only used for packages. ITERATION=$(git rev-list --count --all || echo 0) DATE="$(date -u +%Y-%m-%dT%H:%M:%SZ)" -BRANCH="$(git rev-parse --abbrev-ref HEAD || echo unknown)" COMMIT="$(git rev-parse --short HEAD || echo 0)" # Used by homebrew downloads. @@ -46,4 +45,4 @@ COMMIT="$(git rev-parse --short HEAD || echo 0)" # This is a custom download path for homebrew formula. SOURCE_PATH=https://golift.io/${BINARY}/archive/v${VERSION}.tar.gz -export IMPORT_PATH SOURCE_URL URL VVERSION VERSION ITERATION DATE BRANCH COMMIT SOURCE_PATH +export IMPORT_PATH SOURCE_URL URL VVERSION VERSION ITERATION DATE COMMIT SOURCE_PATH diff --git a/Makefile b/Makefile index 30287041..ee12f5b2 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ endef PLUGINS:=$(patsubst plugins/%/main.go,%,$(wildcard plugins/*/main.go)) VERSION_LDFLAGS:= \ - -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Branch=$(BRANCH) \ + -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Branch=$(TRAVIS_BRANCH) \ -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.BuildDate=$(DATE) \ -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Revision=$(COMMIT) \ -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Version=$(VERSION)-$(ITERATION) From d8443a5ae5020c3600ac4476e2ce2ec2bf592297 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Mon, 16 Dec 2019 17:18:45 -0800 Subject: [PATCH 11/32] haha --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6a71c421..43119458 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,8 +15,12 @@ addons: - debsigs - gnupg - expect + - musl-dev + - gcc + - libc6-dev-i386 + - build-base go: -- 1.12.x +- 1.13.x services: - docker install: From 3dbc7bfe71c499ceb6271676fb35130764d77f2c Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Mon, 16 Dec 2019 17:30:37 -0800 Subject: [PATCH 12/32] doot --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 43119458..c607a2e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,8 @@ addons: - expect - musl-dev - gcc - - libc6-dev-i386 + - gcc-multilib + - g++-multilib - build-base go: - 1.13.x From ed74c8cf8527ae61487517d5e996e3e0527c32b8 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Mon, 16 Dec 2019 17:33:49 -0800 Subject: [PATCH 13/32] doot --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c607a2e9..387d9da6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ addons: - gcc - gcc-multilib - g++-multilib - - build-base + - libc6-dev-i386 go: - 1.13.x services: From aec4b12d3c19b89019858bb15a61b7d16dc7a2d0 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Mon, 16 Dec 2019 20:24:41 -0800 Subject: [PATCH 14/32] giving in --- .travis.yml | 5 ----- Makefile | 29 ++++++----------------------- 2 files changed, 6 insertions(+), 28 deletions(-) diff --git a/.travis.yml b/.travis.yml index 387d9da6..39cdce39 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,11 +15,6 @@ addons: - debsigs - gnupg - expect - - musl-dev - - gcc - - gcc-multilib - - g++-multilib - - libc6-dev-i386 go: - 1.13.x services: diff --git a/Makefile b/Makefile index ee12f5b2..46fe2600 100644 --- a/Makefile +++ b/Makefile @@ -202,26 +202,20 @@ package_build_linux: readme man plugins_linux_amd64 linux sed -e "s/{{BINARY}}/$(BINARY)/g" -e "s/{{DESC}}/$(DESC)/g" \ init/systemd/template.unit.service > $@/lib/systemd/system/$(BINARY).service -package_build_linux_386: package_build_linux plugins_linux_i386 linux386 +package_build_linux_386: package_build_linux linux386 mkdir -p $@ cp -r $ /dev/null || (echo "FPM missing. Install FPM: https://fpm.readthedocs.io/en/latest/installing.html" && false) @@ -264,26 +258,15 @@ $(BINARY).rb: v$(VERSION).tar.gz.sha256 init/homebrew/$(FORMULA).rb.tmpl init/homebrew/$(FORMULA).rb.tmpl | tee $(BINARY).rb # That perl line turns hello-world into HelloWorld, etc. -# This probably wont work for most people..... -plugins: linux_plugins plugins_darwin +plugins: $(patsubst %,%.so,$(PLUGINS)) +$(patsubst %,%.so,$(PLUGINS)): + go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.so,%,$@) linux_plugins: plugins_linux_amd64 plugins_linux_i386 plugins_linux_arm64 plugins_linux_armhf plugins_linux_amd64: $(patsubst %,%.linux_amd64.so,$(PLUGINS)) $(patsubst %,%.linux_amd64.so,$(PLUGINS)): GOOS=linux GOARCH=amd64 go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.linux_amd64.so,%,$@) -plugins_linux_i386: $(patsubst %,%.linux_i386.so,$(PLUGINS)) -$(patsubst %,%.linux_i386.so,$(PLUGINS)): - GOOS=linux GOARCH=386 go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.linux_i386.so,%,$@) - -plugins_linux_arm64: $(patsubst %,%.linux_arm64.so,$(PLUGINS)) -$(patsubst %,%.linux_arm64.so,$(PLUGINS)): - GOOS=linux GOARCH=arm64 go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.linux_arm64.so,%,$@) - -plugins_linux_armhf: $(patsubst %,%.linux_armhf.so,$(PLUGINS)) -$(patsubst %,%.linux_armhf.so,$(PLUGINS)): - GOOS=linux GOARCH=armhf go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.linux_armhf.so,%,$@) - plugins_darwin: $(patsubst %,%.darwin.so,$(PLUGINS)) $(patsubst %,%.darwin.so,$(PLUGINS)): GOOS=darwin go build -o $@ -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./plugins/$(patsubst %.darwin.so,%,$@) From d3d420597e936dc6d592f8d4325fe72d50d9a629 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Tue, 17 Dec 2019 01:31:30 -0800 Subject: [PATCH 15/32] make dumper work --- Gopkg.lock | 4 +- examples/up.xml.example | 2 +- examples/up.yaml.example | 1 + pkg/influxunifi/influxdb.go | 2 +- pkg/inputunifi/collector.go | 16 ++--- pkg/inputunifi/input.go | 83 ++++++++++++++++++++++-- pkg/inputunifi/interface.go | 122 ++++++++++++++++++------------------ pkg/poller/dumper.go | 83 ++++-------------------- pkg/poller/inputs.go | 7 +++ pkg/poller/logger.go | 4 +- pkg/poller/start.go | 6 +- 11 files changed, 178 insertions(+), 152 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 83212a0b..aa24b10e 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -107,11 +107,11 @@ [[projects]] branch = "master" - digest = "1:d54a8d89f95a4d2a5a24ce63cb1835ccdff337fde7776c87ceacb6fdbe4349ae" + digest = "1:7a90fad47972b5ae06013d4685eb2f3007e7c92609a1399d2adf59fe04cd9b63" name = "golift.io/config" packages = ["."] pruneopts = "UT" - revision = "fd8ffb02173aad2183e5555a03b1d1f909aca930" + revision = "fe642c8392dc00d72ddcc47f05a06096bd5d054b" [[projects]] digest = "1:2883cea734f2766f41ff9c9d4aefccccc53e3d44f5c8b08893b9c218cf666722" diff --git a/examples/up.xml.example b/examples/up.xml.example index ec98169d..14b2aa09 100644 --- a/examples/up.xml.example +++ b/examples/up.xml.example @@ -24,7 +24,7 @@ false - + all diff --git a/examples/up.yaml.example b/examples/up.yaml.example index 2be57012..bb8a4aa1 100644 --- a/examples/up.yaml.example +++ b/examples/up.yaml.example @@ -24,6 +24,7 @@ influxdb: verify_ssl: false unifi: + disable: false controllers: - name: "" user: "influx" diff --git a/pkg/influxunifi/influxdb.go b/pkg/influxunifi/influxdb.go index 199ea594..9bbe9c84 100644 --- a/pkg/influxunifi/influxdb.go +++ b/pkg/influxunifi/influxdb.go @@ -24,7 +24,7 @@ const ( // Config defines the data needed to store metrics in InfluxDB type Config struct { Interval config.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` - Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` + Disable bool `json:"disable" toml:"disable" xml:"disable,attr" yaml:"disable"` VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` URL string `json:"url,omitempty" toml:"url,omitempty" xml:"url" yaml:"url"` User string `json:"user,omitempty" toml:"user,omitempty" xml:"user" yaml:"user"` diff --git a/pkg/inputunifi/collector.go b/pkg/inputunifi/collector.go index 20aeafac..c9c67dcb 100644 --- a/pkg/inputunifi/collector.go +++ b/pkg/inputunifi/collector.go @@ -8,14 +8,14 @@ import ( "golift.io/unifi" ) -func (u *InputUnifi) isNill(c Controller) bool { +func (u *InputUnifi) isNill(c *Controller) bool { u.Config.RLock() defer u.Config.RUnlock() return c.Unifi == nil } -func (u *InputUnifi) collectController(c Controller) (*poller.Metrics, error) { +func (u *InputUnifi) collectController(c *Controller) (*poller.Metrics, error) { if u.isNill(c) { u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) @@ -32,7 +32,7 @@ func (u *InputUnifi) collectController(c Controller) (*poller.Metrics, error) { return u.pollController(c) } -func (u *InputUnifi) pollController(c Controller) (*poller.Metrics, error) { +func (u *InputUnifi) pollController(c *Controller) (*poller.Metrics, error) { var err error u.Config.RLock() @@ -67,7 +67,7 @@ func (u *InputUnifi) pollController(c Controller) (*poller.Metrics, error) { // augmentMetrics is our middleware layer between collecting metrics and writing them. // This is where we can manipuate the returned data or make arbitrary decisions. // This function currently adds parent device names to client metrics. -func (u *InputUnifi) augmentMetrics(c Controller, metrics *poller.Metrics) *poller.Metrics { +func (u *InputUnifi) augmentMetrics(c *Controller, metrics *poller.Metrics) *poller.Metrics { if metrics == nil || metrics.Devices == nil || metrics.Clients == nil { return metrics } @@ -113,22 +113,22 @@ func (u *InputUnifi) augmentMetrics(c Controller, metrics *poller.Metrics) *poll // getFilteredSites returns a list of sites to fetch data for. // Omits requested but unconfigured sites. Grabs the full list from the // controller and returns the sites provided in the config file. -func (u *InputUnifi) getFilteredSites(c Controller) (unifi.Sites, error) { +func (u *InputUnifi) getFilteredSites(c *Controller) (unifi.Sites, error) { u.Config.RLock() defer u.Config.RUnlock() sites, err := c.Unifi.GetSites() if err != nil { return nil, err - } else if len(c.Sites) < 1 || poller.StringInSlice("all", c.Sites) { + } else if len(c.Sites) < 1 || StringInSlice("all", c.Sites) { return sites, nil } - var i int + i := 0 for _, s := range sites { // Only include valid sites in the request filter. - if poller.StringInSlice(s.Name, c.Sites) { + if StringInSlice(s.Name, c.Sites) { sites[i] = s i++ } diff --git a/pkg/inputunifi/input.go b/pkg/inputunifi/input.go index 15a027dd..9a4877f9 100644 --- a/pkg/inputunifi/input.go +++ b/pkg/inputunifi/input.go @@ -4,6 +4,8 @@ package inputunifi import ( "fmt" + "os" + "strings" "sync" @@ -13,7 +15,7 @@ import ( // InputUnifi contains the running data. type InputUnifi struct { - Config Config `json:"unifi" toml:"unifi" xml:"unifi" yaml:"unifi"` + Config *Config `json:"unifi" toml:"unifi" xml:"unifi" yaml:"unifi"` poller.Logger } @@ -33,9 +35,9 @@ type Controller struct { // Config contains our configuration data type Config struct { - sync.RWMutex // locks the Unifi struct member when re-authing to unifi. - Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` - Controllers []Controller `json:"controllers" toml:"controller" xml:"controller" yaml:"controllers"` + sync.RWMutex // locks the Unifi struct member when re-authing to unifi. + Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` + Controllers []*Controller `json:"controllers" toml:"controller" xml:"controller" yaml:"controllers"` } func init() { @@ -48,7 +50,7 @@ func init() { } // getUnifi (re-)authenticates to a unifi controller. -func (u *InputUnifi) getUnifi(c Controller) error { +func (u *InputUnifi) getUnifi(c *Controller) error { var err error u.Config.Lock() @@ -76,3 +78,74 @@ func (u *InputUnifi) getUnifi(c Controller) error { return nil } + +// checkSites makes sure the list of provided sites exists on the controller. +// This only runs once during initialization. +func (u *InputUnifi) checkSites(c *Controller) error { + u.Config.RLock() + defer u.Config.RUnlock() + + if len(c.Sites) < 1 || c.Sites[0] == "" { + c.Sites = []string{"all"} + } + + u.LogDebugf("Checking Controller Sites List") + + sites, err := c.Unifi.GetSites() + if err != nil { + return err + } + + msg := []string{} + for _, site := range sites { + msg = append(msg, site.Name+" ("+site.Desc+")") + } + + u.Logf("Found %d site(s) on controller %s: %v", len(msg), c.Name, strings.Join(msg, ", ")) + + if StringInSlice("all", c.Sites) { + c.Sites = []string{"all"} + return nil + } + +FIRST: + for _, s := range c.Sites { + for _, site := range sites { + if s == site.Name { + continue FIRST + } + } + return fmt.Errorf("configured site not found on controller: %v", s) + } + + return nil +} + +func (u *InputUnifi) dumpSitesJSON(c *Controller, path, name string, sites unifi.Sites) ([]byte, error) { + allJSON := []byte{} + + for _, s := range sites { + apiPath := fmt.Sprintf(path, s.Name) + _, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping %s: '%s' JSON for site: %s (%s):\n", name, apiPath, s.Desc, s.Name) + + body, err := c.Unifi.GetJSON(apiPath) + if err != nil { + return allJSON, err + } + + allJSON = append(allJSON, body...) + } + + return allJSON, nil +} + +// StringInSlice returns true if a string is in a slice. +func StringInSlice(str string, slice []string) bool { + for _, s := range slice { + if strings.EqualFold(s, str) { + return true + } + } + + return false +} diff --git a/pkg/inputunifi/interface.go b/pkg/inputunifi/interface.go index 1db32906..f9b437de 100644 --- a/pkg/inputunifi/interface.go +++ b/pkg/inputunifi/interface.go @@ -1,13 +1,51 @@ package inputunifi +/* This file contains the three poller.Input interface methods. */ + import ( "fmt" + "os" "strings" "github.com/davidnewhall/unifi-poller/pkg/poller" "golift.io/unifi" ) +// Initialize gets called one time when starting up. +// Satisfies poller.Input interface. +func (u *InputUnifi) Initialize(l poller.Logger) error { + if u.Config.Disable { + l.Logf("unifi input disabled") + return nil + } + + if len(u.Config.Controllers) < 1 { + return fmt.Errorf("no unifi controllers defined for unifi input") + } + + u.Logger = l + + for i, c := range u.Config.Controllers { + if c.Name == "" { + u.Config.Controllers[i].Name = c.URL + } + + switch err := u.getUnifi(c); err { + case nil: + if err := u.checkSites(c); err != nil { + u.LogErrorf("checking sites on %s: %v", c.Name, err) + } + + u.Logf("Polling UniFi Controller at %s v%s as user %s. Sites: %v", + c.URL, c.Unifi.ServerVersion, c.User, c.Sites) + default: + u.LogErrorf("Controller Auth or Connection failed, but continuing to retry! %s: %v", c.Name, err) + } + } + + return nil +} + // Metrics grabs all the measurements from a UniFi controller and returns them. func (u *InputUnifi) Metrics() (*poller.Metrics, error) { if u.Config.Disable { @@ -52,75 +90,35 @@ func (u *InputUnifi) Metrics() (*poller.Metrics, error) { return metrics, nil } -// Initialize gets called one time when starting up. -// Satisfies poller.Input interface. -func (u *InputUnifi) Initialize(l poller.Logger) error { - if u.Config.Disable { - l.Logf("unifi input disabled") - return nil - } +// RawMetrics returns API output from the first configured unifi controller. +func (u *InputUnifi) RawMetrics(filter poller.Filter) ([]byte, error) { + c := u.Config.Controllers[0] // We could pull the controller number from the filter. + if u.isNill(c) { + u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) - if len(u.Config.Controllers) < 1 { - return fmt.Errorf("no unifi controllers defined for unifi input") - } - - u.Logger = l - - for i, c := range u.Config.Controllers { - if c.Name == "" { - u.Config.Controllers[i].Name = c.URL - } - - switch err := u.getUnifi(c); err { - case nil: - if err := u.checkSites(c); err != nil { - u.LogErrorf("checking sites on %s: %v", c.Name, err) - } - - u.Logf("Polling UniFi Controller at %s v%s as user %s. Sites: %v", - c.URL, c.Unifi.ServerVersion, c.User, c.Sites) - default: - u.LogErrorf("Controller Auth or Connection failed, but continuing to retry! %s: %v", c.Name, err) + if err := u.getUnifi(c); err != nil { + return nil, fmt.Errorf("re-authenticating to %s: %v", c.Name, err) } } - return nil -} + if err := u.checkSites(c); err != nil { + return nil, err + } -// checkSites makes sure the list of provided sites exists on the controller. -// This only runs once during initialization. -func (u *InputUnifi) checkSites(c Controller) error { - u.Config.RLock() - defer u.Config.RUnlock() - u.LogDebugf("Checking Controller Sites List") - - sites, err := c.Unifi.GetSites() + sites, err := u.getFilteredSites(c) if err != nil { - return err + return nil, err } - msg := []string{} - - for _, site := range sites { - msg = append(msg, site.Name+" ("+site.Desc+")") + switch filter.Type { + case "d", "device", "devices": + return u.dumpSitesJSON(c, unifi.APIDevicePath, "Devices", sites) + case "client", "clients", "c": + return u.dumpSitesJSON(c, unifi.APIClientPath, "Clients", sites) + case "other", "o": + _, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping Path '%s':\n", filter.Term) + return c.Unifi.GetJSON(filter.Term) + default: + return []byte{}, fmt.Errorf("must provide filter: devices, clients, other") } - - u.Logf("Found %d site(s) on controller: %v", len(msg), strings.Join(msg, ", ")) - - if poller.StringInSlice("all", c.Sites) { - c.Sites = []string{"all"} - return nil - } - -FIRST: - for _, s := range c.Sites { - for _, site := range sites { - if s == site.Name { - continue FIRST - } - } - return fmt.Errorf("configured site not found on controller: %v", s) - } - - return nil } diff --git a/pkg/poller/dumper.go b/pkg/poller/dumper.go index 18abe901..2a2b0dab 100644 --- a/pkg/poller/dumper.go +++ b/pkg/poller/dumper.go @@ -1,85 +1,28 @@ package poller import ( + "fmt" "strings" ) // DumpJSONPayload prints raw json from the UniFi Controller. // This only works with controller 0 (first one) in the config. func (u *UnifiPoller) DumpJSONPayload() (err error) { - if true { - return nil + u.Config.Quiet = true + + split := strings.SplitN(u.Flags.DumpJSON, " ", 2) + filter := Filter{Type: split[0]} + + if len(split) > 1 { + filter.Term = split[1] } - /* - u.Config.Quiet = true - config := u.Config.Controllers[0] - config.Unifi, err = unifi.NewUnifi(&unifi.Config{ - User: config.User, - Pass: config.Pass, - URL: config.URL, - VerifySSL: config.VerifySSL, - }) - if err != nil { - return err - } + m, err := inputs[0].RawMetrics(filter) + if err != nil { + return err + } - fmt.Fprintf(os.Stderr, "[INFO] Authenticated to UniFi Controller @ %v as user %v", config.URL, config.User) + fmt.Println(string(m)) - if err := u.CheckSites(config); err != nil { - return err - } - - config.Unifi.ErrorLog = func(m string, v ...interface{}) { - fmt.Fprintf(os.Stderr, "[ERROR] "+m, v...) - } // Log all errors to stderr. - - switch sites, err := u.GetFilteredSites(config); { - case err != nil: - return err - case StringInSlice(u.Flags.DumpJSON, []string{"d", "device", "devices"}): - return u.dumpSitesJSON(config, unifi.APIDevicePath, "Devices", sites) - case StringInSlice(u.Flags.DumpJSON, []string{"client", "clients", "c"}): - return u.dumpSitesJSON(config, unifi.APIClientPath, "Clients", sites) - case strings.HasPrefix(u.Flags.DumpJSON, "other "): - apiPath := strings.SplitN(u.Flags.DumpJSON, " ", 2)[1] - _, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping Path '%s':\n", apiPath) - return u.PrintRawAPIJSON(config, apiPath) - default: - return fmt.Errorf("must provide filter: devices, clients, other") - } - */ return nil } - -/* -func (u *UnifiPoller) dumpSitesJSON(c Controller, path, name string, sites unifi.Sites) error { - for _, s := range sites { - apiPath := fmt.Sprintf(path, s.Name) - _, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping %s: '%s' JSON for site: %s (%s):\n", - name, apiPath, s.Desc, s.Name) - if err := u.PrintRawAPIJSON(c, apiPath); err != nil { - return err - } - } - return nil -} - -// PrintRawAPIJSON prints the raw json for a user-provided path on a UniFi Controller. -func (u *UnifiPoller) PrintRawAPIJSON(c Controller, apiPath string) error { - body, err := c.Unifi.GetJSON(apiPath) - fmt.Println(string(body)) - return err -} -*/ - -// StringInSlice returns true if a string is in a slice. -func StringInSlice(str string, slice []string) bool { - for _, s := range slice { - if strings.EqualFold(s, str) { - return true - } - } - - return false -} diff --git a/pkg/poller/inputs.go b/pkg/poller/inputs.go index c2140b01..f2d64efb 100644 --- a/pkg/poller/inputs.go +++ b/pkg/poller/inputs.go @@ -17,6 +17,7 @@ var ( type Input interface { Initialize(Logger) error // Called once on startup to initialize the plugin. Metrics() (*Metrics, error) // Called every time new metrics are requested. + RawMetrics(Filter) ([]byte, error) } // InputPlugin describes an input plugin's consumable interface. @@ -25,6 +26,12 @@ type InputPlugin struct { Input } +// Filter is used for raw metrics filters. +type Filter struct { + Type string + Term string +} + // NewInput creates a metric input. This should be called by input plugins // init() functions. func NewInput(i *InputPlugin) { diff --git a/pkg/poller/logger.go b/pkg/poller/logger.go index b498a9b5..fa983e5f 100644 --- a/pkg/poller/logger.go +++ b/pkg/poller/logger.go @@ -16,14 +16,14 @@ type Logger interface { // Logf prints a log entry if quiet is false. func (u *UnifiPoller) Logf(m string, v ...interface{}) { - if !u.Config.Quiet { + if !u.Quiet { _ = log.Output(callDepth, fmt.Sprintf("[INFO] "+m, v...)) } } // LogDebugf prints a debug log entry if debug is true and quite is false func (u *UnifiPoller) LogDebugf(m string, v ...interface{}) { - if u.Config.Debug && !u.Config.Quiet { + if u.Debug && !u.Quiet { _ = log.Output(callDepth, fmt.Sprintf("[DEBUG] "+m, v...)) } } diff --git a/pkg/poller/start.go b/pkg/poller/start.go index 9f6bedc8..e88daf24 100644 --- a/pkg/poller/start.go +++ b/pkg/poller/start.go @@ -61,10 +61,14 @@ func (f *Flags) Parse(args []string) { // 3. Start a web server and wait for Prometheus to poll the application for metrics. func (u *UnifiPoller) Run() error { if u.Flags.DumpJSON != "" { + if err := u.InitializeInputs(); err != nil { + return err + } + return u.DumpJSONPayload() } - if u.Config.Debug { + if u.Debug { log.SetFlags(log.Lshortfile | log.Lmicroseconds | log.Ldate) u.LogDebugf("Debug Logging Enabled") } From da75406310cf107efe530cc0111874106eacd93a Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Tue, 17 Dec 2019 02:09:30 -0800 Subject: [PATCH 16/32] fixes --- Gopkg.lock | 4 ++-- pkg/inputunifi/input.go | 9 ++++++++- pkg/poller/config.go | 3 ++- pkg/poller/start.go | 2 +- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index aa24b10e..67310aa2 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -107,11 +107,11 @@ [[projects]] branch = "master" - digest = "1:7a90fad47972b5ae06013d4685eb2f3007e7c92609a1399d2adf59fe04cd9b63" + digest = "1:331d56d3a24e9b8d203883c28803313af060ca5c350be0a9c0552b43f355ecd8" name = "golift.io/config" packages = ["."] pruneopts = "UT" - revision = "fe642c8392dc00d72ddcc47f05a06096bd5d054b" + revision = "1861b4270bf42ec3ccaf86871aa1cf6742564d0f" [[projects]] digest = "1:2883cea734f2766f41ff9c9d4aefccccc53e3d44f5c8b08893b9c218cf666722" diff --git a/pkg/inputunifi/input.go b/pkg/inputunifi/input.go index 9a4877f9..2ddc1f4f 100644 --- a/pkg/inputunifi/input.go +++ b/pkg/inputunifi/input.go @@ -108,14 +108,21 @@ func (u *InputUnifi) checkSites(c *Controller) error { return nil } + keep := []string{} + FIRST: for _, s := range c.Sites { for _, site := range sites { if s == site.Name { + keep = append(keep, s) continue FIRST } } - return fmt.Errorf("configured site not found on controller: %v", s) + u.LogErrorf("Configured site not found on controller %s: %v", c.Name, s) + } + + if c.Sites = keep; len(keep) < 1 { + c.Sites = []string{"all"} } return nil diff --git a/pkg/poller/config.go b/pkg/poller/config.go index a43caceb..3e1217f1 100644 --- a/pkg/poller/config.go +++ b/pkg/poller/config.go @@ -52,7 +52,7 @@ type Metrics struct { // Config represents the core library input data. type Config struct { - Poller `json:"poller" toml:"poller" xml:"poller" yaml:"poller"` + *Poller `json:"poller" toml:"poller" xml:"poller" yaml:"poller"` } // Poller is the global config values. @@ -103,6 +103,7 @@ func (u *UnifiPoller) ParseConfigs() error { // parseInterface parses the config file and environment variables into the provided interface. func (u *UnifiPoller) parseInterface(i interface{}) error { + config.ENVTag = "xml" // xml tag is better formatted for slices. // Parse config file into provided interface. if err := config.ParseFile(i, u.Flags.ConfigFile); err != nil { return err diff --git a/pkg/poller/start.go b/pkg/poller/start.go index e88daf24..34cba9da 100644 --- a/pkg/poller/start.go +++ b/pkg/poller/start.go @@ -12,7 +12,7 @@ import ( // New returns a new poller struct. func New() *UnifiPoller { - return &UnifiPoller{Config: &Config{}, Flags: &Flags{}} + return &UnifiPoller{Config: &Config{Poller: &Poller{}}, Flags: &Flags{}} } // Start begins the application from a CLI. From 9022ab288ddd2b7fead18a674c524aab1a5bcfa5 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Tue, 17 Dec 2019 02:39:36 -0800 Subject: [PATCH 17/32] allow pulling specific sites through output plugins --- pkg/influxunifi/influxdb.go | 7 ++-- pkg/inputunifi/input.go | 1 + pkg/inputunifi/interface.go | 20 +++++++++--- pkg/poller/inputs.go | 64 ++++++++++++++++++++++++++++++++++--- pkg/poller/outputs.go | 3 +- pkg/promunifi/collector.go | 9 ++++-- 6 files changed, 90 insertions(+), 14 deletions(-) diff --git a/pkg/influxunifi/influxdb.go b/pkg/influxunifi/influxdb.go index 9bbe9c84..c88f7fbd 100644 --- a/pkg/influxunifi/influxdb.go +++ b/pkg/influxunifi/influxdb.go @@ -69,10 +69,13 @@ func (u *InfluxUnifi) PollController() { log.Printf("[INFO] Everything checks out! Poller started, InfluxDB interval: %v", interval) for u.LastCheck = range ticker.C { - metrics, err := u.Collector.Metrics() + metrics, ok, err := u.Collector.Metrics() if err != nil { u.Collector.LogErrorf("%v", err) - continue + + if !ok { + continue + } } report, err := u.ReportMetrics(metrics) diff --git a/pkg/inputunifi/input.go b/pkg/inputunifi/input.go index 2ddc1f4f..982da1b4 100644 --- a/pkg/inputunifi/input.go +++ b/pkg/inputunifi/input.go @@ -44,6 +44,7 @@ func init() { u := &InputUnifi{} poller.NewInput(&poller.InputPlugin{ + Name: "unifi", Input: u, // this library implements poller.Input interface for Metrics(). Config: u, // Defines our config data interface. }) diff --git a/pkg/inputunifi/interface.go b/pkg/inputunifi/interface.go index f9b437de..99ee7f51 100644 --- a/pkg/inputunifi/interface.go +++ b/pkg/inputunifi/interface.go @@ -47,15 +47,25 @@ func (u *InputUnifi) Initialize(l poller.Logger) error { } // Metrics grabs all the measurements from a UniFi controller and returns them. -func (u *InputUnifi) Metrics() (*poller.Metrics, error) { +func (u *InputUnifi) Metrics() (*poller.Metrics, bool, error) { + return u.MetricsFrom(poller.Filter{}) +} + +// MetricsFrom grabs all the measurements from a UniFi controller and returns them. +func (u *InputUnifi) MetricsFrom(filter poller.Filter) (*poller.Metrics, bool, error) { if u.Config.Disable { - return nil, nil + return nil, false, nil } errs := []string{} metrics := &poller.Metrics{} + ok := false for _, c := range u.Config.Controllers { + if filter.Term != "" && c.Name != filter.Term { + continue + } + m, err := u.collectController(c) if err != nil { errs = append(errs, err.Error()) @@ -65,6 +75,8 @@ func (u *InputUnifi) Metrics() (*poller.Metrics, error) { continue } + ok = true + metrics.Sites = append(metrics.Sites, m.Sites...) metrics.Clients = append(metrics.Clients, m.Clients...) metrics.IDSList = append(metrics.IDSList, m.IDSList...) @@ -84,10 +96,10 @@ func (u *InputUnifi) Metrics() (*poller.Metrics, error) { } if len(errs) > 0 { - return metrics, fmt.Errorf(strings.Join(errs, ", ")) + return metrics, ok, fmt.Errorf(strings.Join(errs, ", ")) } - return metrics, nil + return metrics, ok, nil } // RawMetrics returns API output from the first configured unifi controller. diff --git a/pkg/poller/inputs.go b/pkg/poller/inputs.go index f2d64efb..5d361166 100644 --- a/pkg/poller/inputs.go +++ b/pkg/poller/inputs.go @@ -15,13 +15,15 @@ var ( // Input plugins must implement this interface. type Input interface { - Initialize(Logger) error // Called once on startup to initialize the plugin. - Metrics() (*Metrics, error) // Called every time new metrics are requested. + Initialize(Logger) error // Called once on startup to initialize the plugin. + Metrics() (*Metrics, bool, error) // Called every time new metrics are requested. + MetricsFrom(Filter) (*Metrics, bool, error) // Called every time new metrics are requested. RawMetrics(Filter) ([]byte, error) } // InputPlugin describes an input plugin's consumable interface. type InputPlugin struct { + Name string Config interface{} // Each config is passed into an unmarshaller later. Input } @@ -61,12 +63,13 @@ func (u *UnifiPoller) InitializeInputs() error { } // Metrics aggregates all the measurements from all configured inputs and returns them. -func (u *UnifiPoller) Metrics() (*Metrics, error) { +func (u *UnifiPoller) Metrics() (*Metrics, bool, error) { errs := []string{} metrics := &Metrics{} + ok := false for _, input := range inputs { - m, err := input.Metrics() + m, _, err := input.Metrics() if err != nil { errs = append(errs, err.Error()) } @@ -75,6 +78,8 @@ func (u *UnifiPoller) Metrics() (*Metrics, error) { continue } + ok = true + metrics.Sites = append(metrics.Sites, m.Sites...) metrics.Clients = append(metrics.Clients, m.Clients...) metrics.IDSList = append(metrics.IDSList, m.IDSList...) @@ -99,5 +104,54 @@ func (u *UnifiPoller) Metrics() (*Metrics, error) { err = fmt.Errorf(strings.Join(errs, ", ")) } - return metrics, err + return metrics, ok, err +} + +// MetricsFrom aggregates all the measurements from all configured inputs and returns them. +func (u *UnifiPoller) MetricsFrom(filter Filter) (*Metrics, bool, error) { + errs := []string{} + metrics := &Metrics{} + ok := false + + for _, input := range inputs { + if input.Name != filter.Type { + continue + } + + m, _, err := input.MetricsFrom(filter) + if err != nil { + errs = append(errs, err.Error()) + } + + if m == nil { + continue + } + + ok = true + + metrics.Sites = append(metrics.Sites, m.Sites...) + metrics.Clients = append(metrics.Clients, m.Clients...) + metrics.IDSList = append(metrics.IDSList, m.IDSList...) + + if m.Devices == nil { + continue + } + + if metrics.Devices == nil { + metrics.Devices = &unifi.Devices{} + } + + metrics.UAPs = append(metrics.UAPs, m.UAPs...) + metrics.USGs = append(metrics.USGs, m.USGs...) + metrics.USWs = append(metrics.USWs, m.USWs...) + metrics.UDMs = append(metrics.UDMs, m.UDMs...) + } + + var err error + + if len(errs) > 0 { + err = fmt.Errorf(strings.Join(errs, ", ")) + } + + return metrics, ok, err } diff --git a/pkg/poller/outputs.go b/pkg/poller/outputs.go index 3bbcb72d..0672d92b 100644 --- a/pkg/poller/outputs.go +++ b/pkg/poller/outputs.go @@ -13,7 +13,8 @@ var ( // Collect is passed into output packages so they may collect metrics to output. // Output packages must implement this interface. type Collect interface { - Metrics() (*Metrics, error) + Metrics() (*Metrics, bool, error) + MetricsFrom(Filter) (*Metrics, bool, error) Logger } diff --git a/pkg/promunifi/collector.go b/pkg/promunifi/collector.go index a4aad9f3..694bfd4c 100644 --- a/pkg/promunifi/collector.go +++ b/pkg/promunifi/collector.go @@ -140,12 +140,17 @@ func (u *promUnifi) Describe(ch chan<- *prometheus.Desc) { func (u *promUnifi) Collect(ch chan<- prometheus.Metric) { var err error + ok := false + r := &Report{Config: u.Config, ch: make(chan []*metric, buffer), Start: time.Now()} defer r.close() - if r.Metrics, err = u.Collector.Metrics(); err != nil { + if r.Metrics, ok, err = u.Collector.Metrics(); err != nil { r.error(ch, prometheus.NewInvalidDesc(fmt.Errorf("metric fetch failed")), err) - return + + if !ok { + return + } } r.Fetch = time.Since(r.Start) From 1dd5b4761c4f20d5ba81aae7720b1ca80e073865 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Thu, 19 Dec 2019 00:29:56 -0800 Subject: [PATCH 18/32] Update deps --- Gopkg.lock | 33 +++++++++++++++++++++------------ pkg/poller/config.go | 8 ++++---- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 67310aa2..c5d41385 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1,14 +1,6 @@ # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. -[[projects]] - digest = "1:9f3b30d9f8e0d7040f729b82dcbc8f0dead820a133b3147ce355fc451f32d761" - name = "github.com/BurntSushi/toml" - packages = ["."] - pruneopts = "UT" - revision = "3012a1dbe2e4bd1391d42b32f0577cb7bbc7f005" - version = "v0.3.1" - [[projects]] digest = "1:d6afaeed1502aa28e80a4ed0981d570ad91b2579193404256ce672ed0a609e0d" name = "github.com/beorn7/perks" @@ -45,6 +37,14 @@ revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" version = "v1.0.1" +[[projects]] + digest = "1:6eea828983c70075ca297bb915ffbcfd3e34c5a50affd94428a65df955c0ff9c" + name = "github.com/pelletier/go-toml" + packages = ["."] + pruneopts = "UT" + revision = "903d9455db9ff1d7ac1ab199062eca7266dd11a3" + version = "v1.6.0" + [[projects]] digest = "1:7097829edd12fd7211fca0d29496b44f94ef9e6d72f88fb64f3d7b06315818ad" name = "github.com/prometheus/client_golang" @@ -103,15 +103,23 @@ name = "golang.org/x/sys" packages = ["windows"] pruneopts = "UT" - revision = "ac6580df4449443a05718fd7858c1f91ad5f8d20" + revision = "4a24b406529242041050cb1dec3e0e4c46a5f1b6" [[projects]] - branch = "master" - digest = "1:331d56d3a24e9b8d203883c28803313af060ca5c350be0a9c0552b43f355ecd8" + digest = "1:54742bef5cb29f706614c9edcfdeb29fb5992f26090f26ca955f575dddf54f9e" + name = "golift.io/cnfg" + packages = ["."] + pruneopts = "UT" + revision = "961061d377655468e9da4a9333e71b9b77402470" + version = "v0.0.1" + +[[projects]] + digest = "1:54742bef5cb29f706614c9edcfdeb29fb5992f26090f26ca955f575dddf54f9e" name = "golift.io/config" packages = ["."] pruneopts = "UT" - revision = "1861b4270bf42ec3ccaf86871aa1cf6742564d0f" + revision = "961061d377655468e9da4a9333e71b9b77402470" + version = "v0.0.1" [[projects]] digest = "1:2883cea734f2766f41ff9c9d4aefccccc53e3d44f5c8b08893b9c218cf666722" @@ -137,6 +145,7 @@ "github.com/prometheus/client_golang/prometheus", "github.com/prometheus/common/version", "github.com/spf13/pflag", + "golift.io/cnfg", "golift.io/config", "golift.io/unifi", ] diff --git a/pkg/poller/config.go b/pkg/poller/config.go index 3e1217f1..006dc1fb 100644 --- a/pkg/poller/config.go +++ b/pkg/poller/config.go @@ -16,7 +16,7 @@ import ( "time" "github.com/spf13/pflag" - "golift.io/config" + "golift.io/cnfg" "golift.io/unifi" ) @@ -103,14 +103,14 @@ func (u *UnifiPoller) ParseConfigs() error { // parseInterface parses the config file and environment variables into the provided interface. func (u *UnifiPoller) parseInterface(i interface{}) error { - config.ENVTag = "xml" // xml tag is better formatted for slices. + cnfg.ENVTag = "xml" // xml tag is better formatted for slices. // Parse config file into provided interface. - if err := config.ParseFile(i, u.Flags.ConfigFile); err != nil { + if err := cnfg.ParseFile(i, u.Flags.ConfigFile); err != nil { return err } // Parse environment variables into provided interface. - _, err := config.ParseENV(i, ENVConfigPrefix) + _, err := cnfg.ParseENV(i, ENVConfigPrefix) return err } From 17e7c8edb3858c4f96b64e66b45a2ead5bef2077 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Thu, 19 Dec 2019 19:59:51 -0800 Subject: [PATCH 19/32] allow dynamic controller scrapes --- Gopkg.lock | 17 ++++------- pkg/influxunifi/influxdb.go | 22 +++++++------- pkg/inputunifi/collector.go | 43 ++++++++++++++++++++++++++++ pkg/inputunifi/input.go | 31 ++++++++++++++++++++ pkg/inputunifi/interface.go | 52 +++++++++++++-------------------- pkg/poller/dumper.go | 2 +- pkg/poller/inputs.go | 10 +++---- pkg/poller/outputs.go | 2 +- pkg/promunifi/collector.go | 57 +++++++++++++++++++++++++++++++++++-- plugins/mysql/main.go | 14 ++++----- 10 files changed, 178 insertions(+), 72 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index c5d41385..7cd32b0e 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -46,11 +46,12 @@ version = "v1.6.0" [[projects]] - digest = "1:7097829edd12fd7211fca0d29496b44f94ef9e6d72f88fb64f3d7b06315818ad" + digest = "1:eb04f69c8991e52eff33c428bd729e04208bf03235be88e4df0d88497c6861b9" name = "github.com/prometheus/client_golang" packages = [ "prometheus", "prometheus/internal", + "prometheus/promhttp", ] pruneopts = "UT" revision = "170205fb58decfd011f1550d4cfb737230d7ae4f" @@ -99,11 +100,11 @@ [[projects]] branch = "master" - digest = "1:68fe4216878f16dd6ef33413365fbbe8d2eb781177c7adab874cfc752ce96a7e" + digest = "1:07f0cb66f649e51f9ef23441f8dfc34a73e7d9bf0832417abcbad578f1d8c8d6" name = "golang.org/x/sys" packages = ["windows"] pruneopts = "UT" - revision = "4a24b406529242041050cb1dec3e0e4c46a5f1b6" + revision = "af0d71d358abe0ba3594483a5d519f429dbae3e9" [[projects]] digest = "1:54742bef5cb29f706614c9edcfdeb29fb5992f26090f26ca955f575dddf54f9e" @@ -113,14 +114,6 @@ revision = "961061d377655468e9da4a9333e71b9b77402470" version = "v0.0.1" -[[projects]] - digest = "1:54742bef5cb29f706614c9edcfdeb29fb5992f26090f26ca955f575dddf54f9e" - name = "golift.io/config" - packages = ["."] - pruneopts = "UT" - revision = "961061d377655468e9da4a9333e71b9b77402470" - version = "v0.0.1" - [[projects]] digest = "1:2883cea734f2766f41ff9c9d4aefccccc53e3d44f5c8b08893b9c218cf666722" name = "golift.io/unifi" @@ -143,10 +136,10 @@ input-imports = [ "github.com/influxdata/influxdb1-client/v2", "github.com/prometheus/client_golang/prometheus", + "github.com/prometheus/client_golang/prometheus/promhttp", "github.com/prometheus/common/version", "github.com/spf13/pflag", "golift.io/cnfg", - "golift.io/config", "golift.io/unifi", ] solver-name = "gps-cdcl" diff --git a/pkg/influxunifi/influxdb.go b/pkg/influxunifi/influxdb.go index c88f7fbd..280e94a9 100644 --- a/pkg/influxunifi/influxdb.go +++ b/pkg/influxunifi/influxdb.go @@ -10,7 +10,7 @@ import ( "github.com/davidnewhall/unifi-poller/pkg/poller" influx "github.com/influxdata/influxdb1-client/v2" - "golift.io/config" + "golift.io/cnfg" ) const ( @@ -23,13 +23,13 @@ const ( // Config defines the data needed to store metrics in InfluxDB type Config struct { - Interval config.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` - Disable bool `json:"disable" toml:"disable" xml:"disable,attr" yaml:"disable"` - VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` - URL string `json:"url,omitempty" toml:"url,omitempty" xml:"url" yaml:"url"` - User string `json:"user,omitempty" toml:"user,omitempty" xml:"user" yaml:"user"` - Pass string `json:"pass,omitempty" toml:"pass,omitempty" xml:"pass" yaml:"pass"` - DB string `json:"db,omitempty" toml:"db,omitempty" xml:"db" yaml:"db"` + Interval cnfg.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` + Disable bool `json:"disable" toml:"disable" xml:"disable,attr" yaml:"disable"` + VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` + URL string `json:"url,omitempty" toml:"url,omitempty" xml:"url" yaml:"url"` + User string `json:"user,omitempty" toml:"user,omitempty" xml:"user" yaml:"user"` + Pass string `json:"pass,omitempty" toml:"pass,omitempty" xml:"pass" yaml:"pass"` + DB string `json:"db,omitempty" toml:"db,omitempty" xml:"db" yaml:"db"` } // InfluxDB allows the data to be nested in the config file. @@ -133,12 +133,12 @@ func (u *InfluxUnifi) setConfigDefaults() { } if u.Config.Interval.Duration == 0 { - u.Config.Interval = config.Duration{Duration: defaultInterval} + u.Config.Interval = cnfg.Duration{Duration: defaultInterval} } else if u.Config.Interval.Duration < minimumInterval { - u.Config.Interval = config.Duration{Duration: minimumInterval} + u.Config.Interval = cnfg.Duration{Duration: minimumInterval} } - u.Config.Interval = config.Duration{Duration: u.Config.Interval.Duration.Round(time.Second)} + u.Config.Interval = cnfg.Duration{Duration: u.Config.Interval.Duration.Round(time.Second)} } // ReportMetrics batches all device and client data into influxdb data points. diff --git a/pkg/inputunifi/collector.go b/pkg/inputunifi/collector.go index c9c67dcb..1539f244 100644 --- a/pkg/inputunifi/collector.go +++ b/pkg/inputunifi/collector.go @@ -15,6 +15,49 @@ func (u *InputUnifi) isNill(c *Controller) bool { return c.Unifi == nil } +func (u *InputUnifi) dynamicController(url string) (*poller.Metrics, bool, error) { + c := u.Config.Default // copy defaults into new controller + c.Name = url + c.URL = url + + u.Logf("Authenticating to Dynamic UniFi Controller: %s", url) + + if err := u.getUnifi(&c); err != nil { + return nil, false, fmt.Errorf("authenticating to %s: %v", url, err) + } + + metrics := &poller.Metrics{} + ok, err := u.appendController(&c, metrics) + + return metrics, ok, err +} + +func (u *InputUnifi) appendController(c *Controller, metrics *poller.Metrics) (bool, error) { + m, err := u.collectController(c) + if err != nil || m == nil { + return false, err + } + + metrics.Sites = append(metrics.Sites, m.Sites...) + metrics.Clients = append(metrics.Clients, m.Clients...) + metrics.IDSList = append(metrics.IDSList, m.IDSList...) + + if m.Devices == nil { + return true, nil + } + + if metrics.Devices == nil { + metrics.Devices = &unifi.Devices{} + } + + metrics.UAPs = append(metrics.UAPs, m.UAPs...) + metrics.USGs = append(metrics.USGs, m.USGs...) + metrics.USWs = append(metrics.USWs, m.USWs...) + metrics.UDMs = append(metrics.UDMs, m.UDMs...) + + return true, nil +} + func (u *InputUnifi) collectController(c *Controller) (*poller.Metrics, error) { if u.isNill(c) { u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) diff --git a/pkg/inputunifi/input.go b/pkg/inputunifi/input.go index 982da1b4..4a2881aa 100644 --- a/pkg/inputunifi/input.go +++ b/pkg/inputunifi/input.go @@ -13,6 +13,13 @@ import ( "golift.io/unifi" ) +const ( + defaultURL = "https://127.0.0.1:8443" + defaultUser = "unifipoller" + defaultPass = "unifipollerp4$$w0rd" + defaultSite = "all" +) + // InputUnifi contains the running data. type InputUnifi struct { Config *Config `json:"unifi" toml:"unifi" xml:"unifi" yaml:"unifi"` @@ -36,7 +43,9 @@ type Controller struct { // Config contains our configuration data type Config struct { sync.RWMutex // locks the Unifi struct member when re-authing to unifi. + Default Controller `json:"defaults" toml:"defaults" xml:"default" yaml:"defaults"` Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` + Dynamic bool `json:"dynamic" toml:"dynamic" xml:"dynamic" yaml:"dynamic"` Controllers []*Controller `json:"controllers" toml:"controller" xml:"controller" yaml:"controllers"` } @@ -147,6 +156,28 @@ func (u *InputUnifi) dumpSitesJSON(c *Controller, path, name string, sites unifi return allJSON, nil } +func (u *InputUnifi) setDefaults(c *Controller) { + if c.URL == "" { + c.URL = defaultURL + } + + if c.Name == "" { + c.Name = c.URL + } + + if c.Pass == "" { + c.Pass = defaultPass + } + + if c.User == "" { + c.User = defaultUser + } + + if len(c.Sites) < 1 { + c.Sites = []string{defaultSite} + } +} + // StringInSlice returns true if a string is in a slice. func StringInSlice(str string, slice []string) bool { for _, s := range slice { diff --git a/pkg/inputunifi/interface.go b/pkg/inputunifi/interface.go index 99ee7f51..67f53cce 100644 --- a/pkg/inputunifi/interface.go +++ b/pkg/inputunifi/interface.go @@ -19,16 +19,15 @@ func (u *InputUnifi) Initialize(l poller.Logger) error { return nil } - if len(u.Config.Controllers) < 1 { - return fmt.Errorf("no unifi controllers defined for unifi input") + if u.setDefaults(&u.Config.Default); len(u.Config.Controllers) < 1 { + new := u.Config.Default // copy defaults. + u.Config.Controllers = []*Controller{&new} } u.Logger = l - for i, c := range u.Config.Controllers { - if c.Name == "" { - u.Config.Controllers[i].Name = c.URL - } + for _, c := range u.Config.Controllers { + u.setDefaults(c) switch err := u.getUnifi(c); err { case nil: @@ -48,12 +47,12 @@ func (u *InputUnifi) Initialize(l poller.Logger) error { // Metrics grabs all the measurements from a UniFi controller and returns them. func (u *InputUnifi) Metrics() (*poller.Metrics, bool, error) { - return u.MetricsFrom(poller.Filter{}) + return u.MetricsFrom(nil) } // MetricsFrom grabs all the measurements from a UniFi controller and returns them. -func (u *InputUnifi) MetricsFrom(filter poller.Filter) (*poller.Metrics, bool, error) { - if u.Config.Disable { +func (u *InputUnifi) MetricsFrom(filter *poller.Filter) (*poller.Metrics, bool, error) { + if u.Config.Disable || filter == nil || filter.Term == "" { return nil, false, nil } @@ -61,49 +60,36 @@ func (u *InputUnifi) MetricsFrom(filter poller.Filter) (*poller.Metrics, bool, e metrics := &poller.Metrics{} ok := false + // Check if the request is for an existing, configured controller. for _, c := range u.Config.Controllers { - if filter.Term != "" && c.Name != filter.Term { + if !strings.EqualFold(c.Name, filter.Term) { continue } - m, err := u.collectController(c) + exists, err := u.appendController(c, metrics) if err != nil { errs = append(errs, err.Error()) } - if m == nil { - continue + if exists { + ok = true } - - ok = true - - metrics.Sites = append(metrics.Sites, m.Sites...) - metrics.Clients = append(metrics.Clients, m.Clients...) - metrics.IDSList = append(metrics.IDSList, m.IDSList...) - - if m.Devices == nil { - continue - } - - if metrics.Devices == nil { - metrics.Devices = &unifi.Devices{} - } - - metrics.UAPs = append(metrics.UAPs, m.UAPs...) - metrics.USGs = append(metrics.USGs, m.USGs...) - metrics.USWs = append(metrics.USWs, m.USWs...) - metrics.UDMs = append(metrics.UDMs, m.UDMs...) } if len(errs) > 0 { return metrics, ok, fmt.Errorf(strings.Join(errs, ", ")) } + if u.Config.Dynamic && !ok && strings.HasPrefix(filter.Term, "http") { + // Attempt to a dynamic metrics fetch from an unconfigured controller. + return u.dynamicController(filter.Term) + } + return metrics, ok, nil } // RawMetrics returns API output from the first configured unifi controller. -func (u *InputUnifi) RawMetrics(filter poller.Filter) ([]byte, error) { +func (u *InputUnifi) RawMetrics(filter *poller.Filter) ([]byte, error) { c := u.Config.Controllers[0] // We could pull the controller number from the filter. if u.isNill(c) { u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) diff --git a/pkg/poller/dumper.go b/pkg/poller/dumper.go index 2a2b0dab..80c7f728 100644 --- a/pkg/poller/dumper.go +++ b/pkg/poller/dumper.go @@ -11,7 +11,7 @@ func (u *UnifiPoller) DumpJSONPayload() (err error) { u.Config.Quiet = true split := strings.SplitN(u.Flags.DumpJSON, " ", 2) - filter := Filter{Type: split[0]} + filter := &Filter{Type: split[0]} if len(split) > 1 { filter.Term = split[1] diff --git a/pkg/poller/inputs.go b/pkg/poller/inputs.go index 5d361166..8633ca2c 100644 --- a/pkg/poller/inputs.go +++ b/pkg/poller/inputs.go @@ -15,10 +15,10 @@ var ( // Input plugins must implement this interface. type Input interface { - Initialize(Logger) error // Called once on startup to initialize the plugin. - Metrics() (*Metrics, bool, error) // Called every time new metrics are requested. - MetricsFrom(Filter) (*Metrics, bool, error) // Called every time new metrics are requested. - RawMetrics(Filter) ([]byte, error) + Initialize(Logger) error // Called once on startup to initialize the plugin. + Metrics() (*Metrics, bool, error) // Called every time new metrics are requested. + MetricsFrom(*Filter) (*Metrics, bool, error) // Called every time new metrics are requested. + RawMetrics(*Filter) ([]byte, error) } // InputPlugin describes an input plugin's consumable interface. @@ -108,7 +108,7 @@ func (u *UnifiPoller) Metrics() (*Metrics, bool, error) { } // MetricsFrom aggregates all the measurements from all configured inputs and returns them. -func (u *UnifiPoller) MetricsFrom(filter Filter) (*Metrics, bool, error) { +func (u *UnifiPoller) MetricsFrom(filter *Filter) (*Metrics, bool, error) { errs := []string{} metrics := &Metrics{} ok := false diff --git a/pkg/poller/outputs.go b/pkg/poller/outputs.go index 0672d92b..0c73bfe7 100644 --- a/pkg/poller/outputs.go +++ b/pkg/poller/outputs.go @@ -14,7 +14,7 @@ var ( // Output packages must implement this interface. type Collect interface { Metrics() (*Metrics, bool, error) - MetricsFrom(Filter) (*Metrics, bool, error) + MetricsFrom(*Filter) (*Metrics, bool, error) Logger } diff --git a/pkg/promunifi/collector.go b/pkg/promunifi/collector.go index 694bfd4c..e4cf1e9c 100644 --- a/pkg/promunifi/collector.go +++ b/pkg/promunifi/collector.go @@ -11,6 +11,7 @@ import ( "github.com/davidnewhall/unifi-poller/pkg/poller" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/common/version" "golift.io/unifi" ) @@ -77,6 +78,11 @@ type Report struct { wg sync.WaitGroup } +type target struct { + *poller.Filter + *promUnifi +} + func init() { u := &promUnifi{Prometheus: &Prometheus{}} @@ -103,6 +109,8 @@ func (u *promUnifi) Run(c poller.Collect) error { u.Config.HTTPListen = defaultHTTPListen } + mux := http.NewServeMux() + prometheus.MustRegister(version.NewCollector(u.Config.Namespace)) prometheus.MustRegister(&promUnifi{ Collector: c, @@ -115,8 +123,43 @@ func (u *promUnifi) Run(c poller.Collect) error { }) c.Logf("Exporting Measurements for Prometheus at https://%s/metrics, namespace: %s", u.Config.HTTPListen, u.Config.Namespace) + mux.Handle("/metrics", promhttp.HandlerFor( + prometheus.DefaultGatherer, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}, + )) + mux.HandleFunc("/scrape", u.ScrapeHandler) - return http.ListenAndServe(u.Config.HTTPListen, nil) + return http.ListenAndServe(u.Config.HTTPListen, mux) +} + +// ScrapeHandler allows prometheus to scrape a single source, instead of all sources. +func (u *promUnifi) ScrapeHandler(w http.ResponseWriter, r *http.Request) { + t := &target{promUnifi: u, Filter: &poller.Filter{}} + if t.Filter.Type = r.URL.Query().Get("input"); t.Filter.Type == "" { + http.Error(w, `'input' parameter must be specified (try "unifi")`, 400) + return + } + + if t.Filter.Term = r.URL.Query().Get("target"); t.Filter.Term == "" { + http.Error(w, "'target' parameter must be specified, configured name, or unconfigured url", 400) + return + } + + registry := prometheus.NewRegistry() + + registry.MustRegister(t) + promhttp.HandlerFor( + registry, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}, + ).ServeHTTP(w, r) +} + +// Describe satisfies the prometheus Collector. This returns all of the +// metric descriptions that this packages produces. +func (t *target) Describe(ch chan<- *prometheus.Desc) { + t.promUnifi.Describe(ch) +} + +func (t *target) Collect(ch chan<- prometheus.Metric) { + t.promUnifi.collect(ch, t.Filter) } // Describe satisfies the prometheus Collector. This returns all of the @@ -138,6 +181,10 @@ func (u *promUnifi) Describe(ch chan<- *prometheus.Desc) { // Collect satisfies the prometheus Collector. This runs the input method to get // the current metrics (from another package) then exports them for prometheus. func (u *promUnifi) Collect(ch chan<- prometheus.Metric) { + u.collect(ch, nil) +} + +func (u *promUnifi) collect(ch chan<- prometheus.Metric, filter *poller.Filter) { var err error ok := false @@ -145,7 +192,13 @@ func (u *promUnifi) Collect(ch chan<- prometheus.Metric) { r := &Report{Config: u.Config, ch: make(chan []*metric, buffer), Start: time.Now()} defer r.close() - if r.Metrics, ok, err = u.Collector.Metrics(); err != nil { + if filter == nil { + r.Metrics, ok, err = u.Collector.Metrics() + } else { + r.Metrics, ok, err = u.Collector.MetricsFrom(filter) + } + + if err != nil { r.error(ch, prometheus.NewInvalidDesc(fmt.Errorf("metric fetch failed")), err) if !ok { diff --git a/plugins/mysql/main.go b/plugins/mysql/main.go index e0c866b9..361165a0 100644 --- a/plugins/mysql/main.go +++ b/plugins/mysql/main.go @@ -4,17 +4,17 @@ import ( "fmt" "github.com/davidnewhall/unifi-poller/pkg/poller" - "golift.io/config" + "golift.io/cnfg" ) // mysqlConfig represents the data that is unmarshalled from the up.conf config file for this plugins. type mysqlConfig struct { - Interval config.Duration `json:"interval" toml:"interval" xml:"interval" yaml:"interval"` - Host string `json:"host" toml:"host" xml:"host" yaml:"host"` - User string `json:"user" toml:"user" xml:"user" yaml:"user"` - Pass string `json:"pass" toml:"pass" xml:"pass" yaml:"pass"` - DB string `json:"db" toml:"db" xml:"db" yaml:"db"` - Table string `json:"table" toml:"table" xml:"table" yaml:"table"` + Interval cnfg.Duration `json:"interval" toml:"interval" xml:"interval" yaml:"interval"` + Host string `json:"host" toml:"host" xml:"host" yaml:"host"` + User string `json:"user" toml:"user" xml:"user" yaml:"user"` + Pass string `json:"pass" toml:"pass" xml:"pass" yaml:"pass"` + DB string `json:"db" toml:"db" xml:"db" yaml:"db"` + Table string `json:"table" toml:"table" xml:"table" yaml:"table"` // Maps do not work with ENV VARIABLES yet, but may in the future. Fields []string `json:"fields" toml:"fields" xml:"field" yaml:"fields"` } From de16ba2399be6c807b66da309cf439d1ca843600 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Fri, 20 Dec 2019 02:44:53 -0800 Subject: [PATCH 20/32] fixes --- examples/MANUAL.md | 52 ++------------- examples/up.conf.example | 122 +++++++++++++++++++++--------------- examples/up.json.example | 20 ++++-- examples/up.xml.example | 21 +++++-- examples/up.yaml.example | 21 +++++-- pkg/inputunifi/collector.go | 66 +++++++++---------- pkg/inputunifi/input.go | 17 +++-- pkg/inputunifi/interface.go | 50 ++++++++++----- pkg/poller/dumper.go | 15 +++-- pkg/poller/inputs.go | 62 ++++++++++++------ pkg/promunifi/collector.go | 112 ++++++++++++++++++--------------- pkg/promunifi/report.go | 2 +- 12 files changed, 313 insertions(+), 247 deletions(-) diff --git a/examples/MANUAL.md b/examples/MANUAL.md index cabf4db8..d12c9ed3 100644 --- a/examples/MANUAL.md +++ b/examples/MANUAL.md @@ -65,56 +65,16 @@ is provided so the application can be easily adapted to any environment. `Config File Parameters` -Additional parameters are added by output packages. Parameters can also be set -using environment variables. See the GitHub wiki for more information! +Configuration file (up.conf) parameters are documented in the wiki. - >>> POLLER FIELDS FOLLOW - you may have multiple controllers: +* [https://github.com/davidnewhall/unifi-poller/wiki/Configuration](https://github.com/davidnewhall/unifi-poller/wiki/Configuration) - debug default: false - This turns on time stamps and line numbers in logs, outputs a few extra - lines of information while processing. +`Shell Environment Parameters` - quiet default: false - Setting this to true will turn off per-device and per-interval logs. Only - errors will be logged. Using this with debug=true adds line numbers to - any error logs. +This application can be fully configured using shell environment variables. +Find documentation for this feature on the Docker Wiki page. - >>> UNIFI CONTROLLER FIELDS FOLLOW - you may have multiple controllers: - - sites default: ["all"] - This list of strings should represent the names of sites on the UniFi - controller that will be polled for data. Pass `all` in the list to - poll all sites. On startup, the application prints out all site names - found in the controller; they're cryptic, but they have the human-name - next to them. The cryptic names go into the config file `sites` list. - The controller's first site is not cryptic and is named `default`. - - url default: https://127.0.0.1:8443 - This is the URL where the UniFi Controller is available. - - user default: influxdb - Username used to authenticate with UniFi controller. This should be a - special service account created on the control with read-only access. - - pass no default - Password used to authenticate with UniFi controller. This can also be - set in an environment variable instead of a configuration file. - - save_ids default: false - Setting this parameter to true will enable collection of Intrusion - Detection System data. IDS and IPS are the same data set. This is off - by default because most controllers do not have this enabled. It also - creates a lot of new metrics from controllers with a lot of IDS entries. - IDS data does not contain metrics, so this doesn't work with Prometheus. - - save_sites default: true - Setting this parameter to false will disable saving Network Site data. - This data populates the Sites dashboard, and this setting affects influx - and prometheus. - - verify_ssl default: false - If your UniFi controller has a valid SSL certificate, you can enable - this option to validate it. Otherwise, any SSL certificate is valid. +* [https://github.com/davidnewhall/unifi-poller/wiki/Docker](https://github.com/davidnewhall/unifi-poller/wiki/Docker) GO DURATION --- diff --git a/examples/up.conf.example b/examples/up.conf.example index 43aa9831..942c53cd 100644 --- a/examples/up.conf.example +++ b/examples/up.conf.example @@ -1,75 +1,95 @@ # UniFi Poller primary configuration file. TOML FORMAT # -# commented lines are defaults, uncomment to change. # ######################################################## - [poller] -# Turns on line numbers, microsecond logging, and a per-device log. -# The default is false, but I personally leave this on at home (four devices). -# This may be noisy if you have a lot of devices. It adds one line per device. -debug = false + # Turns on line numbers, microsecond logging, and a per-device log. + # The default is false, but I personally leave this on at home (four devices). + # This may be noisy if you have a lot of devices. It adds one line per device. + debug = false -# Turns off per-interval logs. Only startup and error logs will be emitted. -# Recommend enabling debug with this setting for better error logging. -quiet = false + # Turns off per-interval logs. Only startup and error logs will be emitted. + # Recommend enabling debug with this setting for better error logging. + quiet = false -# Load dynamic plugins. Advanced use; only sample mysql plugin provided by default. -plugins = [] + # Load dynamic plugins. Advanced use; only sample mysql plugin provided by default. + plugins = [] #### OUTPUTS + # If you don't use an output, you can disable it. + [prometheus] -disable = false -# This controls on which ip and port /metrics is exported when mode is "prometheus". -# This has no effect in other modes. Must contain a colon and port. -http_listen = "0.0.0.0:9130" -report_errors = false + disable = false + # This controls on which ip and port /metrics is exported when mode is "prometheus". + # This has no effect in other modes. Must contain a colon and port. + http_listen = "0.0.0.0:9130" + report_errors = false [influxdb] -disable = false -# InfluxDB does not require auth by default, so the user/password are probably unimportant. -url = "http://127.0.0.1:8086" -user = "unifi" -pass = "unifi" -# Be sure to create this database. -db = "unifi" -# If your InfluxDB uses a valid SSL cert, set this to true. -verify_ssl = false -# The UniFi Controller only updates traffic stats about every 30 seconds. -# Setting this to something lower may lead to "zeros" in your data. -# If you're getting zeros now, set this to "1m" -interval = "30s" + disable = false + # InfluxDB does not require auth by default, so the user/password are probably unimportant. + url = "http://127.0.0.1:8086" + user = "unifipoller" + pass = "unifipoller" + # Be sure to create this database. + db = "unifi" + # If your InfluxDB uses a valid SSL cert, set this to true. + verify_ssl = false + # The UniFi Controller only updates traffic stats about every 30 seconds. + # Setting this to something lower may lead to "zeros" in your data. + # If you're getting zeros now, set this to "1m" + interval = "30s" #### INPUTS [unifi] -disable = false + # Setting this to true and providing default credentials allows you to skip + # configuring controllers in this config file. Instead you configure them in + # your prometheus.yml config. Prometheus then sends the controller URL to + # unifi-poller when it performs the scrape. This is useful if you have many, + # or changing controllers. Most people can leave this off. See wiki for more. + dynamic = false + +# The following section contains the default credentials/configuration for any +# dynamic controller (see above section), or the primary controller if you do not +# provide one and dynamic is disabled. In other words, you can just add your +# controller here and delete the following section. Either works. +[unifi.defaults] + name = "https://127.0.0.1:8443" + url = "https://127.0.0.1:8443" + user = "unifipoller" + pass = "unifipoller" + sites = ["all"] + save_ids = false + save_sites = true + verify_ssl = false # You may repeat the following section to poll additional controllers. [[unifi.controller]] -# Friendly name used in dashboards. Uses URL if left empty. -name = "" + # Friendly name used in dashboards. Uses URL if left empty; which is fine. + # Avoid changing this later because it will live forever in your database. + name = "" -url = "https://127.0.0.1:8443" -# Make a read-only user in the UniFi Admin Settings. -user = "influx" -# You may also set env variable UNIFI_PASSWORD instead of putting this in the config. -pass = "4BB9345C-2341-48D7-99F5-E01B583FF77F" + url = "https://127.0.0.1:8443" + # Make a read-only user in the UniFi Admin Settings. + user = "unifipoller" + # You may also set env variable UNIFI_PASSWORD instead of putting this in the config. + pass = "4BB9345C-2341-48D7-99F5-E01B583FF77F" -# If the controller has more than one site, specify which sites to poll here. -# Set this to ["default"] to poll only the first site on the controller. -# A setting of ["all"] will poll all sites; this works if you only have 1 site too. -sites = ["all"] + # If the controller has more than one site, specify which sites to poll here. + # Set this to ["default"] to poll only the first site on the controller. + # A setting of ["all"] will poll all sites; this works if you only have 1 site too. + sites = ["all"] -# Enable collection of Intrusion Detection System Data (InfluxDB only). -# Only useful if IDS or IPS are enabled on one of the sites. -save_ids = false + # Enable collection of Intrusion Detection System Data (InfluxDB only). + # Only useful if IDS or IPS are enabled on one of the sites. + save_ids = false -# Enable collection of site data. This data powers the Network Sites dashboard. -# It's not valuable to everyone and setting this to false will save resources. -save_sites = true + # Enable collection of site data. This data powers the Network Sites dashboard. + # It's not valuable to everyone and setting this to false will save resources. + save_sites = true -# If your UniFi controller has a valid SSL certificate (like lets encrypt), -# you can enable this option to validate it. Otherwise, any SSL certificate is -# valid. If you don't know if you have a valid SSL cert, then you don't have one. -verify_ssl = false + # If your UniFi controller has a valid SSL certificate (like lets encrypt), + # you can enable this option to validate it. Otherwise, any SSL certificate is + # valid. If you don't know if you have a valid SSL cert, then you don't have one. + verify_ssl = false diff --git a/examples/up.json.example b/examples/up.json.example index 12ba4a4f..a4e0a401 100644 --- a/examples/up.json.example +++ b/examples/up.json.example @@ -14,20 +14,30 @@ "influxdb": { "disable": false, "url": "http://127.0.0.1:8086", - "user": "unifi", - "pass": "unifi", + "user": "unifipoller", + "pass": "unifipoller", "db": "unifi", "verify_ssl": false, "interval": "30s" }, "unifi": { - "disable": false, + "dynamic": false, + "defaults": { + "name": "https://127.0.0.1:8443", + "user": "unifipoller", + "pass": "unifipoller", + "url": "https://127.0.0.1:8443", + "sites": ["all"], + "save_ids": false, + "save_sites": true, + "verify_ssl": false + }, "controllers": [ { "name": "", - "user": "influx", - "pass": "", + "user": "unifipoller", + "pass": "unifipoller", "url": "https://127.0.0.1:8443", "sites": ["all"], "save_ids": false, diff --git a/examples/up.xml.example b/examples/up.xml.example index 14b2aa09..094f8347 100644 --- a/examples/up.xml.example +++ b/examples/up.xml.example @@ -18,22 +18,33 @@ 30s http://127.0.0.1:8086 - unifi - unifi + unifipoller + unifipoller unifi false - + + + all + unifipoller + unifipoller + https://127.0.0.1:8443 + false + false + true + + all - influx - + unifipoller + unifipoller https://127.0.0.1:8443 false false true + diff --git a/examples/up.yaml.example b/examples/up.yaml.example index bb8a4aa1..edf08506 100644 --- a/examples/up.yaml.example +++ b/examples/up.yaml.example @@ -18,17 +18,28 @@ influxdb: disable: false interval: "30s" url: "http://127.0.0.1:8086" - user: "unifi" - pass: "unifi" + user: "unifipoller" + pass: "unifipoller" db: "unifi" verify_ssl: false unifi: - disable: false + dynamic: false + defaults: + name: "https://127.0.0.1:8443" + user: "unifipoller" + pass: "unifipoller" + url: "https://127.0.0.1:8443" + sites: + - all + verify_ssl: false + save_ids: false + save_sites: true + controllers: - name: "" - user: "influx" - pass: "" + user: "unifipoller" + pass: "unifipoller" url: "https://127.0.0.1:8443" sites: - all diff --git a/pkg/inputunifi/collector.go b/pkg/inputunifi/collector.go index 1539f244..90f1feb4 100644 --- a/pkg/inputunifi/collector.go +++ b/pkg/inputunifi/collector.go @@ -2,6 +2,7 @@ package inputunifi import ( "fmt" + "strings" "time" "github.com/davidnewhall/unifi-poller/pkg/poller" @@ -15,47 +16,43 @@ func (u *InputUnifi) isNill(c *Controller) bool { return c.Unifi == nil } -func (u *InputUnifi) dynamicController(url string) (*poller.Metrics, bool, error) { - c := u.Config.Default // copy defaults into new controller +// newDynamicCntrlr creates and saves a controller (with auth cookie) for repeated use. +// This is called when an unconfigured controller is requested. +func (u *InputUnifi) newDynamicCntrlr(url string) (bool, *Controller) { + u.Lock() + defer u.Unlock() + + c := u.dynamic[url] + if c != nil { + // it already exists. + return false, c + } + + ccopy := u.Config.Default // copy defaults into new controller + c = &ccopy + u.dynamic[url] = c c.Name = url c.URL = url - u.Logf("Authenticating to Dynamic UniFi Controller: %s", url) - - if err := u.getUnifi(&c); err != nil { - return nil, false, fmt.Errorf("authenticating to %s: %v", url, err) - } - - metrics := &poller.Metrics{} - ok, err := u.appendController(&c, metrics) - - return metrics, ok, err + return true, c } -func (u *InputUnifi) appendController(c *Controller, metrics *poller.Metrics) (bool, error) { - m, err := u.collectController(c) - if err != nil || m == nil { - return false, err +func (u *InputUnifi) dynamicController(url string) (*poller.Metrics, error) { + if !strings.HasPrefix(url, "http") { + return nil, fmt.Errorf("scrape filter match failed, and filter is not http URL") } - metrics.Sites = append(metrics.Sites, m.Sites...) - metrics.Clients = append(metrics.Clients, m.Clients...) - metrics.IDSList = append(metrics.IDSList, m.IDSList...) + new, c := u.newDynamicCntrlr(url) - if m.Devices == nil { - return true, nil + if new { + u.Logf("Authenticating to Dynamic UniFi Controller: %s", url) + + if err := u.getUnifi(c); err != nil { + return nil, fmt.Errorf("authenticating to %s: %v", url, err) + } } - if metrics.Devices == nil { - metrics.Devices = &unifi.Devices{} - } - - metrics.UAPs = append(metrics.UAPs, m.UAPs...) - metrics.USGs = append(metrics.USGs, m.USGs...) - metrics.USWs = append(metrics.USWs, m.USWs...) - metrics.UDMs = append(metrics.UDMs, m.UDMs...) - - return true, nil + return u.collectController(c) } func (u *InputUnifi) collectController(c *Controller) (*poller.Metrics, error) { @@ -67,11 +64,6 @@ func (u *InputUnifi) collectController(c *Controller) (*poller.Metrics, error) { } } - m, err := u.pollController(c) - if err == nil { - return m, nil - } - return u.pollController(c) } @@ -146,7 +138,7 @@ func (u *InputUnifi) augmentMetrics(c *Controller, metrics *poller.Metrics) *pol metrics.Clients[i].RadioDescription = bssdIDs[metrics.Clients[i].Bssid] + metrics.Clients[i].RadioProto } - if !c.SaveSites { + if !*c.SaveSites { metrics.Sites = nil } diff --git a/pkg/inputunifi/input.go b/pkg/inputunifi/input.go index 4a2881aa..a540964d 100644 --- a/pkg/inputunifi/input.go +++ b/pkg/inputunifi/input.go @@ -16,13 +16,15 @@ import ( const ( defaultURL = "https://127.0.0.1:8443" defaultUser = "unifipoller" - defaultPass = "unifipollerp4$$w0rd" + defaultPass = "unifipoller" defaultSite = "all" ) // InputUnifi contains the running data. type InputUnifi struct { - Config *Config `json:"unifi" toml:"unifi" xml:"unifi" yaml:"unifi"` + Config *Config `json:"unifi" toml:"unifi" xml:"unifi" yaml:"unifi"` + dynamic map[string]*Controller + sync.Mutex // to lock the map above. poller.Logger } @@ -31,7 +33,7 @@ type InputUnifi struct { type Controller struct { VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` SaveIDS bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` - SaveSites bool `json:"save_sites" toml:"save_sites" xml:"save_sites" yaml:"save_sites"` + SaveSites *bool `json:"save_sites" toml:"save_sites" xml:"save_sites" yaml:"save_sites"` Name string `json:"name" toml:"name" xml:"name,attr" yaml:"name"` User string `json:"user" toml:"user" xml:"user" yaml:"user"` Pass string `json:"pass" toml:"pass" xml:"pass" yaml:"pass"` @@ -44,8 +46,8 @@ type Controller struct { type Config struct { sync.RWMutex // locks the Unifi struct member when re-authing to unifi. Default Controller `json:"defaults" toml:"defaults" xml:"default" yaml:"defaults"` - Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` - Dynamic bool `json:"dynamic" toml:"dynamic" xml:"dynamic" yaml:"dynamic"` + Disable bool `json:"disable" toml:"disable" xml:"disable,attr" yaml:"disable"` + Dynamic bool `json:"dynamic" toml:"dynamic" xml:"dynamic,attr" yaml:"dynamic"` Controllers []*Controller `json:"controllers" toml:"controller" xml:"controller" yaml:"controllers"` } @@ -157,6 +159,11 @@ func (u *InputUnifi) dumpSitesJSON(c *Controller, path, name string, sites unifi } func (u *InputUnifi) setDefaults(c *Controller) { + if c.SaveSites == nil { + t := true + c.SaveSites = &t + } + if c.URL == "" { c.URL = defaultURL } diff --git a/pkg/inputunifi/interface.go b/pkg/inputunifi/interface.go index 67f53cce..42745687 100644 --- a/pkg/inputunifi/interface.go +++ b/pkg/inputunifi/interface.go @@ -15,15 +15,20 @@ import ( // Satisfies poller.Input interface. func (u *InputUnifi) Initialize(l poller.Logger) error { if u.Config.Disable { - l.Logf("unifi input disabled") + l.Logf("UniFi input plugin disabled!") return nil } - if u.setDefaults(&u.Config.Default); len(u.Config.Controllers) < 1 { + if u.setDefaults(&u.Config.Default); len(u.Config.Controllers) < 1 && !u.Config.Dynamic { new := u.Config.Default // copy defaults. u.Config.Controllers = []*Controller{&new} } + if len(u.Config.Controllers) < 1 { + l.Logf("No controllers configured. Polling dynamic controllers only!") + } + + u.dynamic = make(map[string]*Controller) u.Logger = l for _, c := range u.Config.Controllers { @@ -35,7 +40,7 @@ func (u *InputUnifi) Initialize(l poller.Logger) error { u.LogErrorf("checking sites on %s: %v", c.Name, err) } - u.Logf("Polling UniFi Controller at %s v%s as user %s. Sites: %v", + u.Logf("Configured UniFi Controller at %s v%s as user %s. Sites: %v", c.URL, c.Unifi.ServerVersion, c.User, c.Sites) default: u.LogErrorf("Controller Auth or Connection failed, but continuing to retry! %s: %v", c.Name, err) @@ -52,7 +57,7 @@ func (u *InputUnifi) Metrics() (*poller.Metrics, bool, error) { // MetricsFrom grabs all the measurements from a UniFi controller and returns them. func (u *InputUnifi) MetricsFrom(filter *poller.Filter) (*poller.Metrics, bool, error) { - if u.Config.Disable || filter == nil || filter.Term == "" { + if u.Config.Disable { return nil, false, nil } @@ -62,35 +67,48 @@ func (u *InputUnifi) MetricsFrom(filter *poller.Filter) (*poller.Metrics, bool, // Check if the request is for an existing, configured controller. for _, c := range u.Config.Controllers { - if !strings.EqualFold(c.Name, filter.Term) { + if filter != nil && !strings.EqualFold(c.Name, filter.Term) { continue } - exists, err := u.appendController(c, metrics) + m, err := u.collectController(c) if err != nil { errs = append(errs, err.Error()) } - if exists { - ok = true + if m == nil { + continue } + + ok = true + metrics = poller.AppendMetrics(metrics, m) } if len(errs) > 0 { return metrics, ok, fmt.Errorf(strings.Join(errs, ", ")) } - if u.Config.Dynamic && !ok && strings.HasPrefix(filter.Term, "http") { - // Attempt to a dynamic metrics fetch from an unconfigured controller. - return u.dynamicController(filter.Term) + if ok { + return metrics, true, nil } - return metrics, ok, nil + if filter != nil && !u.Config.Dynamic { + return metrics, false, fmt.Errorf("scrape filter match failed and dynamic lookups disabled") + } + + // Attempt a dynamic metrics fetch from an unconfigured controller. + m, err := u.dynamicController(filter.Term) + + return m, err == nil && m != nil, err } // RawMetrics returns API output from the first configured unifi controller. func (u *InputUnifi) RawMetrics(filter *poller.Filter) ([]byte, error) { - c := u.Config.Controllers[0] // We could pull the controller number from the filter. + if l := len(u.Config.Controllers); filter.Unit >= l { + return nil, fmt.Errorf("control number %d not found, %d controller(s) configured (0 index)", filter.Unit, l) + } + + c := u.Config.Controllers[filter.Unit] if u.isNill(c) { u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) @@ -108,14 +126,14 @@ func (u *InputUnifi) RawMetrics(filter *poller.Filter) ([]byte, error) { return nil, err } - switch filter.Type { + switch filter.Kind { case "d", "device", "devices": return u.dumpSitesJSON(c, unifi.APIDevicePath, "Devices", sites) case "client", "clients", "c": return u.dumpSitesJSON(c, unifi.APIClientPath, "Clients", sites) case "other", "o": - _, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping Path '%s':\n", filter.Term) - return c.Unifi.GetJSON(filter.Term) + _, _ = fmt.Fprintf(os.Stderr, "[INFO] Dumping Path '%s':\n", filter.Path) + return c.Unifi.GetJSON(filter.Path) default: return []byte{}, fmt.Errorf("must provide filter: devices, clients, other") } diff --git a/pkg/poller/dumper.go b/pkg/poller/dumper.go index 80c7f728..c78edabb 100644 --- a/pkg/poller/dumper.go +++ b/pkg/poller/dumper.go @@ -2,19 +2,24 @@ package poller import ( "fmt" + "strconv" "strings" ) -// DumpJSONPayload prints raw json from the UniFi Controller. -// This only works with controller 0 (first one) in the config. +// DumpJSONPayload prints raw json from the UniFi Controller. This is currently +// tied into the -j CLI arg, and is probably not very useful outside that context. func (u *UnifiPoller) DumpJSONPayload() (err error) { u.Config.Quiet = true - split := strings.SplitN(u.Flags.DumpJSON, " ", 2) - filter := &Filter{Type: split[0]} + filter := &Filter{Kind: split[0]} + + if split2 := strings.Split(filter.Kind, ":"); len(split2) > 1 { + filter.Kind = split2[0] + filter.Unit, _ = strconv.Atoi(split2[1]) + } if len(split) > 1 { - filter.Term = split[1] + filter.Path = split[1] } m, err := inputs[0].RawMetrics(filter) diff --git a/pkg/poller/inputs.go b/pkg/poller/inputs.go index 8633ca2c..a637239c 100644 --- a/pkg/poller/inputs.go +++ b/pkg/poller/inputs.go @@ -28,10 +28,26 @@ type InputPlugin struct { Input } -// Filter is used for raw metrics filters. +// Filter is used for metrics filters. Many fields for lots of expansion. type Filter struct { Type string Term string + Name string + Tags string + Role string + Kind string + Path string + Area int + Item int + Unit int + Sign int64 + Mass int64 + Rate float64 + Cost float64 + Free bool + True bool + Done bool + Stop bool } // NewInput creates a metric input. This should be called by input plugins @@ -107,14 +123,14 @@ func (u *UnifiPoller) Metrics() (*Metrics, bool, error) { return metrics, ok, err } -// MetricsFrom aggregates all the measurements from all configured inputs and returns them. +// MetricsFrom aggregates all the measurements from filtered inputs and returns them. func (u *UnifiPoller) MetricsFrom(filter *Filter) (*Metrics, bool, error) { errs := []string{} metrics := &Metrics{} ok := false for _, input := range inputs { - if input.Name != filter.Type { + if !strings.EqualFold(input.Name, filter.Name) { continue } @@ -128,23 +144,7 @@ func (u *UnifiPoller) MetricsFrom(filter *Filter) (*Metrics, bool, error) { } ok = true - - metrics.Sites = append(metrics.Sites, m.Sites...) - metrics.Clients = append(metrics.Clients, m.Clients...) - metrics.IDSList = append(metrics.IDSList, m.IDSList...) - - if m.Devices == nil { - continue - } - - if metrics.Devices == nil { - metrics.Devices = &unifi.Devices{} - } - - metrics.UAPs = append(metrics.UAPs, m.UAPs...) - metrics.USGs = append(metrics.USGs, m.USGs...) - metrics.USWs = append(metrics.USWs, m.USWs...) - metrics.UDMs = append(metrics.UDMs, m.UDMs...) + metrics = AppendMetrics(metrics, m) } var err error @@ -155,3 +155,25 @@ func (u *UnifiPoller) MetricsFrom(filter *Filter) (*Metrics, bool, error) { return metrics, ok, err } + +// AppendMetrics combined the metrics from two sources. +func AppendMetrics(existing *Metrics, m *Metrics) *Metrics { + existing.Sites = append(existing.Sites, m.Sites...) + existing.Clients = append(existing.Clients, m.Clients...) + existing.IDSList = append(existing.IDSList, m.IDSList...) + + if m.Devices == nil { + return existing + } + + if existing.Devices == nil { + existing.Devices = &unifi.Devices{} + } + + existing.UAPs = append(existing.UAPs, m.UAPs...) + existing.USGs = append(existing.USGs, m.USGs...) + existing.USWs = append(existing.USWs, m.USWs...) + existing.UDMs = append(existing.UDMs, m.UDMs...) + + return existing +} diff --git a/pkg/promunifi/collector.go b/pkg/promunifi/collector.go index e4cf1e9c..bbfbda25 100644 --- a/pkg/promunifi/collector.go +++ b/pkg/promunifi/collector.go @@ -26,23 +26,18 @@ const ( ) type promUnifi struct { - *Prometheus - Client *uclient - Device *unifiDevice - UAP *uap - USG *usg - USW *usw - Site *site + *Config `json:"prometheus" toml:"prometheus" xml:"prometheus" yaml:"prometheus"` + Client *uclient + Device *unifiDevice + UAP *uap + USG *usg + USW *usw + Site *site // This interface is passed to the Collect() method. The Collect method uses // this interface to retrieve the latest UniFi measurements and export them. Collector poller.Collect } -// Prometheus allows the data to be nested in the config file. -type Prometheus struct { - Config Config `json:"prometheus" toml:"prometheus" xml:"prometheus" yaml:"prometheus"` -} - // Config is the input (config file) data used to initialize this output plugin. type Config struct { // If non-empty, each of the collected metrics is prefixed by the @@ -66,7 +61,7 @@ type metric struct { // Report accumulates counters that are printed to a log line. type Report struct { - Config + *Config Total int // Total count of metrics recorded. Errors int // Total count of errors recording metrics. Zeros int // Total count of metrics equal to zero. @@ -78,17 +73,18 @@ type Report struct { wg sync.WaitGroup } +// target is used for targeted (sometimes dynamic) metrics scrapes. type target struct { *poller.Filter - *promUnifi + u *promUnifi } func init() { - u := &promUnifi{Prometheus: &Prometheus{}} + u := &promUnifi{Config: &Config{}} poller.NewOutput(&poller.Output{ Name: "prometheus", - Config: u.Prometheus, + Config: u, Method: u.Run, }) } @@ -96,51 +92,55 @@ func init() { // Run creates the collectors and starts the web server up. // Should be run in a Go routine. Returns nil if not configured. func (u *promUnifi) Run(c poller.Collect) error { - if u.Config.Disable { + if u.Disable { return nil } - u.Config.Namespace = strings.Trim(strings.Replace(u.Config.Namespace, "-", "_", -1), "_") - if u.Config.Namespace == "" { - u.Config.Namespace = strings.Replace(poller.AppName, "-", "", -1) + u.Namespace = strings.Trim(strings.Replace(u.Namespace, "-", "_", -1), "_") + if u.Namespace == "" { + u.Namespace = strings.Replace(poller.AppName, "-", "", -1) } - if u.Config.HTTPListen == "" { - u.Config.HTTPListen = defaultHTTPListen + if u.HTTPListen == "" { + u.HTTPListen = defaultHTTPListen } + // Later can pass this in from poller by adding a method to the interface. + u.Collector = c + u.Client = descClient(u.Namespace + "_client_") + u.Device = descDevice(u.Namespace + "_device_") // stats for all device types. + u.UAP = descUAP(u.Namespace + "_device_") + u.USG = descUSG(u.Namespace + "_device_") + u.USW = descUSW(u.Namespace + "_device_") + u.Site = descSite(u.Namespace + "_site_") mux := http.NewServeMux() - prometheus.MustRegister(version.NewCollector(u.Config.Namespace)) - prometheus.MustRegister(&promUnifi{ - Collector: c, - Client: descClient(u.Config.Namespace + "_client_"), - Device: descDevice(u.Config.Namespace + "_device_"), // stats for all device types. - UAP: descUAP(u.Config.Namespace + "_device_"), - USG: descUSG(u.Config.Namespace + "_device_"), - USW: descUSW(u.Config.Namespace + "_device_"), - Site: descSite(u.Config.Namespace + "_site_"), - }) - c.Logf("Exporting Measurements for Prometheus at https://%s/metrics, namespace: %s", - u.Config.HTTPListen, u.Config.Namespace) - mux.Handle("/metrics", promhttp.HandlerFor( - prometheus.DefaultGatherer, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}, + prometheus.MustRegister(version.NewCollector(u.Namespace)) + prometheus.MustRegister(u) + c.Logf("Prometheus exported at https://%s/ - namespace: %s", u.HTTPListen, u.Namespace) + mux.Handle("/metrics", promhttp.HandlerFor(prometheus.DefaultGatherer, + promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}, )) mux.HandleFunc("/scrape", u.ScrapeHandler) + mux.HandleFunc("/", u.DefaultHandler) - return http.ListenAndServe(u.Config.HTTPListen, mux) + return http.ListenAndServe(u.HTTPListen, mux) } // ScrapeHandler allows prometheus to scrape a single source, instead of all sources. func (u *promUnifi) ScrapeHandler(w http.ResponseWriter, r *http.Request) { - t := &target{promUnifi: u, Filter: &poller.Filter{}} - if t.Filter.Type = r.URL.Query().Get("input"); t.Filter.Type == "" { + t := &target{u: u, Filter: &poller.Filter{}} + if t.Name = r.URL.Query().Get("input"); t.Name == "" { + u.Collector.LogErrorf("input parameter missing on scrape from %v", r.RemoteAddr) http.Error(w, `'input' parameter must be specified (try "unifi")`, 400) + return } - if t.Filter.Term = r.URL.Query().Get("target"); t.Filter.Term == "" { + if t.Term = r.URL.Query().Get("target"); t.Term == "" { + u.Collector.LogErrorf("target parameter missing on scrape from %v", r.RemoteAddr) http.Error(w, "'target' parameter must be specified, configured name, or unconfigured url", 400) + return } @@ -152,14 +152,15 @@ func (u *promUnifi) ScrapeHandler(w http.ResponseWriter, r *http.Request) { ).ServeHTTP(w, r) } +func (u *promUnifi) DefaultHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + _, _ = w.Write([]byte(poller.AppName + "\n")) +} + // Describe satisfies the prometheus Collector. This returns all of the // metric descriptions that this packages produces. func (t *target) Describe(ch chan<- *prometheus.Desc) { - t.promUnifi.Describe(ch) -} - -func (t *target) Collect(ch chan<- prometheus.Metric) { - t.promUnifi.collect(ch, t.Filter) + t.u.Describe(ch) } // Describe satisfies the prometheus Collector. This returns all of the @@ -178,6 +179,11 @@ func (u *promUnifi) Describe(ch chan<- *prometheus.Desc) { } } +// Collect satisfies the prometheus Collector. This runs for a single controller poll. +func (t *target) Collect(ch chan<- prometheus.Metric) { + t.u.collect(ch, t.Filter) +} + // Collect satisfies the prometheus Collector. This runs the input method to get // the current metrics (from another package) then exports them for prometheus. func (u *promUnifi) Collect(ch chan<- prometheus.Metric) { @@ -187,27 +193,31 @@ func (u *promUnifi) Collect(ch chan<- prometheus.Metric) { func (u *promUnifi) collect(ch chan<- prometheus.Metric, filter *poller.Filter) { var err error - ok := false - - r := &Report{Config: u.Config, ch: make(chan []*metric, buffer), Start: time.Now()} + r := &Report{ + Config: u.Config, + ch: make(chan []*metric, buffer), + Start: time.Now()} defer r.close() + ok := false + if filter == nil { r.Metrics, ok, err = u.Collector.Metrics() } else { r.Metrics, ok, err = u.Collector.MetricsFrom(filter) } + r.Fetch = time.Since(r.Start) + if err != nil { - r.error(ch, prometheus.NewInvalidDesc(fmt.Errorf("metric fetch failed")), err) + r.error(ch, prometheus.NewInvalidDesc(err), fmt.Errorf("metric fetch failed")) + u.Collector.LogErrorf("metric fetch failed: %v", err) if !ok { return } } - r.Fetch = time.Since(r.Start) - if r.Metrics.Devices == nil { r.Metrics.Devices = &unifi.Devices{} } diff --git a/pkg/promunifi/report.go b/pkg/promunifi/report.go index 9b6df74c..3eb66638 100644 --- a/pkg/promunifi/report.go +++ b/pkg/promunifi/report.go @@ -67,7 +67,7 @@ func (r *Report) export(m *metric, v float64) prometheus.Metric { func (r *Report) error(ch chan<- prometheus.Metric, d *prometheus.Desc, v interface{}) { r.Errors++ - if r.Config.ReportErrors { + if r.ReportErrors { ch <- prometheus.NewInvalidMetric(d, fmt.Errorf("error: %v", v)) } } From ed3bed4c7bac1ea10237d2bcc30198de2f8076c1 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Fri, 20 Dec 2019 03:19:00 -0800 Subject: [PATCH 21/32] rename some things --- examples/up.conf.example | 5 ++-- examples/up.json.example | 4 +-- examples/up.xml.example | 4 +-- examples/up.yaml.example | 6 +++-- pkg/inputunifi/collector.go | 18 ++++++------- pkg/inputunifi/input.go | 20 +++++++------- pkg/inputunifi/interface.go | 52 ++++++++++++++++++------------------- pkg/promunifi/collector.go | 14 ++++++---- 8 files changed, 65 insertions(+), 58 deletions(-) diff --git a/examples/up.conf.example b/examples/up.conf.example index 942c53cd..bb0f09dd 100644 --- a/examples/up.conf.example +++ b/examples/up.conf.example @@ -55,7 +55,7 @@ # provide one and dynamic is disabled. In other words, you can just add your # controller here and delete the following section. Either works. [unifi.defaults] - name = "https://127.0.0.1:8443" + role = "https://127.0.0.1:8443" url = "https://127.0.0.1:8443" user = "unifipoller" pass = "unifipoller" @@ -68,7 +68,8 @@ [[unifi.controller]] # Friendly name used in dashboards. Uses URL if left empty; which is fine. # Avoid changing this later because it will live forever in your database. - name = "" + # Multiple controllers may share a role. This allows grouping during scrapes. + role = "" url = "https://127.0.0.1:8443" # Make a read-only user in the UniFi Admin Settings. diff --git a/examples/up.json.example b/examples/up.json.example index a4e0a401..d523eee2 100644 --- a/examples/up.json.example +++ b/examples/up.json.example @@ -24,7 +24,7 @@ "unifi": { "dynamic": false, "defaults": { - "name": "https://127.0.0.1:8443", + "role": "https://127.0.0.1:8443", "user": "unifipoller", "pass": "unifipoller", "url": "https://127.0.0.1:8443", @@ -35,7 +35,7 @@ }, "controllers": [ { - "name": "", + "role": "", "user": "unifipoller", "pass": "unifipoller", "url": "https://127.0.0.1:8443", diff --git a/examples/up.xml.example b/examples/up.xml.example index 094f8347..d7caacc7 100644 --- a/examples/up.xml.example +++ b/examples/up.xml.example @@ -25,7 +25,7 @@ - + all unifipoller unifipoller @@ -36,7 +36,7 @@ - + all unifipoller unifipoller diff --git a/examples/up.yaml.example b/examples/up.yaml.example index edf08506..a675fd8e 100644 --- a/examples/up.yaml.example +++ b/examples/up.yaml.example @@ -26,7 +26,7 @@ influxdb: unifi: dynamic: false defaults: - name: "https://127.0.0.1:8443" + role: "https://127.0.0.1:8443" user: "unifipoller" pass: "unifipoller" url: "https://127.0.0.1:8443" @@ -36,8 +36,10 @@ unifi: save_ids: false save_sites: true + controllers: - - name: "" + # Repeat the following stanza to poll more controllers. + - role: "" user: "unifipoller" pass: "unifipoller" url: "https://127.0.0.1:8443" diff --git a/pkg/inputunifi/collector.go b/pkg/inputunifi/collector.go index 90f1feb4..85941fee 100644 --- a/pkg/inputunifi/collector.go +++ b/pkg/inputunifi/collector.go @@ -10,8 +10,8 @@ import ( ) func (u *InputUnifi) isNill(c *Controller) bool { - u.Config.RLock() - defer u.Config.RUnlock() + u.RLock() + defer u.RUnlock() return c.Unifi == nil } @@ -28,10 +28,10 @@ func (u *InputUnifi) newDynamicCntrlr(url string) (bool, *Controller) { return false, c } - ccopy := u.Config.Default // copy defaults into new controller + ccopy := u.Default // copy defaults into new controller c = &ccopy u.dynamic[url] = c - c.Name = url + c.Role = url c.URL = url return true, c @@ -60,7 +60,7 @@ func (u *InputUnifi) collectController(c *Controller) (*poller.Metrics, error) { u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) if err := u.getUnifi(c); err != nil { - return nil, fmt.Errorf("re-authenticating to %s: %v", c.Name, err) + return nil, fmt.Errorf("re-authenticating to %s: %v", c.Role, err) } } @@ -70,8 +70,8 @@ func (u *InputUnifi) collectController(c *Controller) (*poller.Metrics, error) { func (u *InputUnifi) pollController(c *Controller) (*poller.Metrics, error) { var err error - u.Config.RLock() - defer u.Config.RUnlock() + u.RLock() + defer u.RUnlock() m := &poller.Metrics{TS: time.Now()} // At this point, it's the Current Check. @@ -149,8 +149,8 @@ func (u *InputUnifi) augmentMetrics(c *Controller, metrics *poller.Metrics) *pol // Omits requested but unconfigured sites. Grabs the full list from the // controller and returns the sites provided in the config file. func (u *InputUnifi) getFilteredSites(c *Controller) (unifi.Sites, error) { - u.Config.RLock() - defer u.Config.RUnlock() + u.RLock() + defer u.RUnlock() sites, err := c.Unifi.GetSites() if err != nil { diff --git a/pkg/inputunifi/input.go b/pkg/inputunifi/input.go index a540964d..45d3e4e9 100644 --- a/pkg/inputunifi/input.go +++ b/pkg/inputunifi/input.go @@ -22,7 +22,7 @@ const ( // InputUnifi contains the running data. type InputUnifi struct { - Config *Config `json:"unifi" toml:"unifi" xml:"unifi" yaml:"unifi"` + *Config `json:"unifi" toml:"unifi" xml:"unifi" yaml:"unifi"` dynamic map[string]*Controller sync.Mutex // to lock the map above. poller.Logger @@ -34,7 +34,7 @@ type Controller struct { VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` SaveIDS bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` SaveSites *bool `json:"save_sites" toml:"save_sites" xml:"save_sites" yaml:"save_sites"` - Name string `json:"name" toml:"name" xml:"name,attr" yaml:"name"` + Role string `json:"role" toml:"role" xml:"role,attr" yaml:"role"` User string `json:"user" toml:"user" xml:"user" yaml:"user"` Pass string `json:"pass" toml:"pass" xml:"pass" yaml:"pass"` URL string `json:"url" toml:"url" xml:"url" yaml:"url"` @@ -65,8 +65,8 @@ func init() { func (u *InputUnifi) getUnifi(c *Controller) error { var err error - u.Config.Lock() - defer u.Config.Unlock() + u.Lock() + defer u.Unlock() if c.Unifi != nil { c.Unifi.CloseIdleConnections() @@ -94,8 +94,8 @@ func (u *InputUnifi) getUnifi(c *Controller) error { // checkSites makes sure the list of provided sites exists on the controller. // This only runs once during initialization. func (u *InputUnifi) checkSites(c *Controller) error { - u.Config.RLock() - defer u.Config.RUnlock() + u.RLock() + defer u.RUnlock() if len(c.Sites) < 1 || c.Sites[0] == "" { c.Sites = []string{"all"} @@ -113,7 +113,7 @@ func (u *InputUnifi) checkSites(c *Controller) error { msg = append(msg, site.Name+" ("+site.Desc+")") } - u.Logf("Found %d site(s) on controller %s: %v", len(msg), c.Name, strings.Join(msg, ", ")) + u.Logf("Found %d site(s) on controller %s: %v", len(msg), c.Role, strings.Join(msg, ", ")) if StringInSlice("all", c.Sites) { c.Sites = []string{"all"} @@ -130,7 +130,7 @@ FIRST: continue FIRST } } - u.LogErrorf("Configured site not found on controller %s: %v", c.Name, s) + u.LogErrorf("Configured site not found on controller %s: %v", c.Role, s) } if c.Sites = keep; len(keep) < 1 { @@ -168,8 +168,8 @@ func (u *InputUnifi) setDefaults(c *Controller) { c.URL = defaultURL } - if c.Name == "" { - c.Name = c.URL + if c.Role == "" { + c.Role = c.URL } if c.Pass == "" { diff --git a/pkg/inputunifi/interface.go b/pkg/inputunifi/interface.go index 42745687..daf21ae7 100644 --- a/pkg/inputunifi/interface.go +++ b/pkg/inputunifi/interface.go @@ -14,36 +14,36 @@ import ( // Initialize gets called one time when starting up. // Satisfies poller.Input interface. func (u *InputUnifi) Initialize(l poller.Logger) error { - if u.Config.Disable { + if u.Disable { l.Logf("UniFi input plugin disabled!") return nil } - if u.setDefaults(&u.Config.Default); len(u.Config.Controllers) < 1 && !u.Config.Dynamic { - new := u.Config.Default // copy defaults. - u.Config.Controllers = []*Controller{&new} + if u.setDefaults(&u.Default); len(u.Controllers) < 1 && !u.Dynamic { + new := u.Default // copy defaults. + u.Controllers = []*Controller{&new} } - if len(u.Config.Controllers) < 1 { + if len(u.Controllers) < 1 { l.Logf("No controllers configured. Polling dynamic controllers only!") } u.dynamic = make(map[string]*Controller) u.Logger = l - for _, c := range u.Config.Controllers { + for _, c := range u.Controllers { u.setDefaults(c) switch err := u.getUnifi(c); err { case nil: if err := u.checkSites(c); err != nil { - u.LogErrorf("checking sites on %s: %v", c.Name, err) + u.LogErrorf("checking sites on %s: %v", c.Role, err) } u.Logf("Configured UniFi Controller at %s v%s as user %s. Sites: %v", c.URL, c.Unifi.ServerVersion, c.User, c.Sites) default: - u.LogErrorf("Controller Auth or Connection failed, but continuing to retry! %s: %v", c.Name, err) + u.LogErrorf("Controller Auth or Connection failed, but continuing to retry! %s: %v", c.Role, err) } } @@ -57,7 +57,7 @@ func (u *InputUnifi) Metrics() (*poller.Metrics, bool, error) { // MetricsFrom grabs all the measurements from a UniFi controller and returns them. func (u *InputUnifi) MetricsFrom(filter *poller.Filter) (*poller.Metrics, bool, error) { - if u.Config.Disable { + if u.Disable { return nil, false, nil } @@ -65,9 +65,20 @@ func (u *InputUnifi) MetricsFrom(filter *poller.Filter) (*poller.Metrics, bool, metrics := &poller.Metrics{} ok := false + if filter != nil && filter.Path != "" { + if !u.Dynamic { + return metrics, false, fmt.Errorf("filter path requested but dynamic lookups disabled") + } + + // Attempt a dynamic metrics fetch from an unconfigured controller. + m, err := u.dynamicController(filter.Path) + + return m, err == nil && m != nil, err + } + // Check if the request is for an existing, configured controller. - for _, c := range u.Config.Controllers { - if filter != nil && !strings.EqualFold(c.Name, filter.Term) { + for _, c := range u.Controllers { + if filter != nil && !strings.EqualFold(c.Role, filter.Role) { continue } @@ -88,32 +99,21 @@ func (u *InputUnifi) MetricsFrom(filter *poller.Filter) (*poller.Metrics, bool, return metrics, ok, fmt.Errorf(strings.Join(errs, ", ")) } - if ok { - return metrics, true, nil - } - - if filter != nil && !u.Config.Dynamic { - return metrics, false, fmt.Errorf("scrape filter match failed and dynamic lookups disabled") - } - - // Attempt a dynamic metrics fetch from an unconfigured controller. - m, err := u.dynamicController(filter.Term) - - return m, err == nil && m != nil, err + return metrics, ok, nil } // RawMetrics returns API output from the first configured unifi controller. func (u *InputUnifi) RawMetrics(filter *poller.Filter) ([]byte, error) { - if l := len(u.Config.Controllers); filter.Unit >= l { + if l := len(u.Controllers); filter.Unit >= l { return nil, fmt.Errorf("control number %d not found, %d controller(s) configured (0 index)", filter.Unit, l) } - c := u.Config.Controllers[filter.Unit] + c := u.Controllers[filter.Unit] if u.isNill(c) { u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) if err := u.getUnifi(c); err != nil { - return nil, fmt.Errorf("re-authenticating to %s: %v", c.Name, err) + return nil, fmt.Errorf("re-authenticating to %s: %v", c.Role, err) } } diff --git a/pkg/promunifi/collector.go b/pkg/promunifi/collector.go index bbfbda25..353bed01 100644 --- a/pkg/promunifi/collector.go +++ b/pkg/promunifi/collector.go @@ -129,17 +129,21 @@ func (u *promUnifi) Run(c poller.Collect) error { // ScrapeHandler allows prometheus to scrape a single source, instead of all sources. func (u *promUnifi) ScrapeHandler(w http.ResponseWriter, r *http.Request) { - t := &target{u: u, Filter: &poller.Filter{}} - if t.Name = r.URL.Query().Get("input"); t.Name == "" { + t := &target{u: u, Filter: &poller.Filter{ + Name: r.URL.Query().Get("input"), // "unifi" + Path: r.URL.Query().Get("path"), // url: "https://127.0.0.1:8443" + Role: r.URL.Query().Get("role"), // configured role in up.conf. + }} + if t.Name == "" { u.Collector.LogErrorf("input parameter missing on scrape from %v", r.RemoteAddr) http.Error(w, `'input' parameter must be specified (try "unifi")`, 400) return } - if t.Term = r.URL.Query().Get("target"); t.Term == "" { - u.Collector.LogErrorf("target parameter missing on scrape from %v", r.RemoteAddr) - http.Error(w, "'target' parameter must be specified, configured name, or unconfigured url", 400) + if t.Role == "" && t.Path == "" { + u.Collector.LogErrorf("role and path parameters missing on scrape from %v", r.RemoteAddr) + http.Error(w, "'role' OR 'path' parameter must be specified: configured role OR unconfigured url", 400) return } From 3f6a3e9721d6baa353251b08b05904385e4f8154 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Fri, 20 Dec 2019 03:22:41 -0800 Subject: [PATCH 22/32] ocd --- examples/up.conf.example | 13 ++++++------- examples/up.json.example | 12 ++++++------ examples/up.yaml.example | 4 ++-- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/examples/up.conf.example b/examples/up.conf.example index bb0f09dd..06418489 100644 --- a/examples/up.conf.example +++ b/examples/up.conf.example @@ -55,12 +55,12 @@ # provide one and dynamic is disabled. In other words, you can just add your # controller here and delete the following section. Either works. [unifi.defaults] - role = "https://127.0.0.1:8443" - url = "https://127.0.0.1:8443" - user = "unifipoller" - pass = "unifipoller" - sites = ["all"] - save_ids = false + role = "https://127.0.0.1:8443" + url = "https://127.0.0.1:8443" + user = "unifipoller" + pass = "unifipoller" + sites = ["all"] + save_ids = false save_sites = true verify_ssl = false @@ -74,7 +74,6 @@ url = "https://127.0.0.1:8443" # Make a read-only user in the UniFi Admin Settings. user = "unifipoller" - # You may also set env variable UNIFI_PASSWORD instead of putting this in the config. pass = "4BB9345C-2341-48D7-99F5-E01B583FF77F" # If the controller has more than one site, specify which sites to poll here. diff --git a/examples/up.json.example b/examples/up.json.example index d523eee2..6e8c8c66 100644 --- a/examples/up.json.example +++ b/examples/up.json.example @@ -24,12 +24,12 @@ "unifi": { "dynamic": false, "defaults": { - "role": "https://127.0.0.1:8443", - "user": "unifipoller", - "pass": "unifipoller", - "url": "https://127.0.0.1:8443", - "sites": ["all"], - "save_ids": false, + "role": "https://127.0.0.1:8443", + "user": "unifipoller", + "pass": "unifipoller", + "url": "https://127.0.0.1:8443", + "sites": ["all"], + "save_ids": false, "save_sites": true, "verify_ssl": false }, diff --git a/examples/up.yaml.example b/examples/up.yaml.example index a675fd8e..33cc5088 100644 --- a/examples/up.yaml.example +++ b/examples/up.yaml.example @@ -29,11 +29,11 @@ unifi: role: "https://127.0.0.1:8443" user: "unifipoller" pass: "unifipoller" - url: "https://127.0.0.1:8443" + url: "https://127.0.0.1:8443" sites: - all verify_ssl: false - save_ids: false + save_ids: false save_sites: true From 84edadaa94a418055248789ee5a8144cb5c7e1dd Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Tue, 24 Dec 2019 23:43:49 -0800 Subject: [PATCH 23/32] Make a new docker image --- Gopkg.lock | 24 ++++++++++++++---------- Gopkg.toml | 5 +++++ examples/up.conf.example | 2 +- pkg/influxunifi/clients.go | 1 + pkg/influxunifi/ids.go | 2 ++ pkg/influxunifi/site.go | 1 + pkg/influxunifi/uap.go | 3 +++ pkg/influxunifi/udm.go | 4 ++++ pkg/influxunifi/usg.go | 3 +++ pkg/influxunifi/usw.go | 2 ++ pkg/poller/config.go | 6 +++--- pkg/promunifi/clients.go | 4 ++-- pkg/promunifi/collector.go | 1 + pkg/promunifi/site.go | 4 ++-- pkg/promunifi/uap.go | 16 ++++++++-------- pkg/promunifi/udm.go | 4 ++-- pkg/promunifi/usg.go | 10 +++++----- pkg/promunifi/usw.go | 8 ++++---- plugins/mysql/main.go | 6 +++--- 19 files changed, 66 insertions(+), 40 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 7cd32b0e..ef55eb8f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -58,12 +58,12 @@ version = "v1.1.0" [[projects]] - branch = "master" digest = "1:982be0b5396e16a663697899ce69cc7b1e71ddcae4153af157578d4dc9bc3f88" name = "github.com/prometheus/client_model" packages = ["go"] pruneopts = "UT" revision = "d1d2010b5beead3fa1c5f271a5cf626e40b3ad6e" + version = "v0.1.0" [[projects]] digest = "1:7dec9ab2db741c280b89b142b08ea142824152c5f40fb1f90c35b6ef7a694456" @@ -100,27 +100,30 @@ [[projects]] branch = "master" - digest = "1:07f0cb66f649e51f9ef23441f8dfc34a73e7d9bf0832417abcbad578f1d8c8d6" + digest = "1:0ca5ac8aedc2fd9cb63c90acbd71c0cba8ddb61dfcca58b96cf41550689bf56d" name = "golang.org/x/sys" packages = ["windows"] pruneopts = "UT" - revision = "af0d71d358abe0ba3594483a5d519f429dbae3e9" + revision = "c709ea063b76879dc9915358f55d4d77c16ab6d5" [[projects]] - digest = "1:54742bef5cb29f706614c9edcfdeb29fb5992f26090f26ca955f575dddf54f9e" + digest = "1:b914912b8cb13beebf85b913006d8692bbf20d8620122966a7a911bbfc6e0104" name = "golift.io/cnfg" - packages = ["."] + packages = [ + ".", + "cnfgfile", + ] pruneopts = "UT" - revision = "961061d377655468e9da4a9333e71b9b77402470" - version = "v0.0.1" + revision = "7d859f27a93d43faceb7101d02baeb8e69f16d4a" + version = "v0.0.4" [[projects]] - digest = "1:2883cea734f2766f41ff9c9d4aefccccc53e3d44f5c8b08893b9c218cf666722" + branch = "master" + digest = "1:13b9e827c1f958ffe31554d57940495010eef89928697f6c6beae78a750ee637" name = "golift.io/unifi" packages = ["."] pruneopts = "UT" - revision = "a607fe940c6a563c6994f2c945394b19d2183b1c" - version = "v4.1.6" + revision = "4d78625be4cd448ea2db35c63a40b451e7d0df99" [[projects]] digest = "1:b75b3deb2bce8bc079e16bb2aecfe01eb80098f5650f9e93e5643ca8b7b73737" @@ -140,6 +143,7 @@ "github.com/prometheus/common/version", "github.com/spf13/pflag", "golift.io/cnfg", + "golift.io/cnfg/cnfgfile", "golift.io/unifi", ] solver-name = "gps-cdcl" diff --git a/Gopkg.toml b/Gopkg.toml index 332001cc..044de96a 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -7,3 +7,8 @@ [prune] go-tests = true unused-packages = true + +# TODO: remove this!!! +[[constraint]] + name = "golift.io/unifi" + branch = "master" diff --git a/examples/up.conf.example b/examples/up.conf.example index 06418489..db3b14ad 100644 --- a/examples/up.conf.example +++ b/examples/up.conf.example @@ -28,7 +28,7 @@ [influxdb] disable = false # InfluxDB does not require auth by default, so the user/password are probably unimportant. - url = "http://127.0.0.1:8086" + url = "http://127.0.0.1:8086" user = "unifipoller" pass = "unifipoller" # Be sure to create this database. diff --git a/pkg/influxunifi/clients.go b/pkg/influxunifi/clients.go index b4c346dc..9e15ec19 100644 --- a/pkg/influxunifi/clients.go +++ b/pkg/influxunifi/clients.go @@ -10,6 +10,7 @@ func (u *InfluxUnifi) batchClient(r report, s *unifi.Client) { tags := map[string]string{ "mac": s.Mac, "site_name": s.SiteName, + "source": s.SourceName, "ap_name": s.ApName, "gw_name": s.GwName, "sw_name": s.SwName, diff --git a/pkg/influxunifi/ids.go b/pkg/influxunifi/ids.go index ad0b855a..fbc6e0dd 100644 --- a/pkg/influxunifi/ids.go +++ b/pkg/influxunifi/ids.go @@ -8,6 +8,8 @@ import ( // These points can be passed directly to influx. func (u *InfluxUnifi) batchIDS(r report, i *unifi.IDS) { tags := map[string]string{ + "site_name": i.SiteName, + "source": i.SourceName, "in_iface": i.InIface, "event_type": i.EventType, "proto": i.Proto, diff --git a/pkg/influxunifi/site.go b/pkg/influxunifi/site.go index 30e2ce37..49715e5a 100644 --- a/pkg/influxunifi/site.go +++ b/pkg/influxunifi/site.go @@ -11,6 +11,7 @@ func (u *InfluxUnifi) batchSite(r report, s *unifi.Site) { tags := map[string]string{ "name": s.Name, "site_name": s.SiteName, + "source": s.SourceName, "desc": s.Desc, "status": h.Status, "subsystem": h.Subsystem, diff --git a/pkg/influxunifi/uap.go b/pkg/influxunifi/uap.go index f47a11fc..3966cb4a 100644 --- a/pkg/influxunifi/uap.go +++ b/pkg/influxunifi/uap.go @@ -14,6 +14,7 @@ func (u *InfluxUnifi) batchUAP(r report, s *unifi.UAP) { tags := map[string]string{ "mac": s.Mac, "site_name": s.SiteName, + "source": s.SourceName, "name": s.Name, "version": s.Version, "model": s.Model, @@ -85,6 +86,7 @@ func (u *InfluxUnifi) processVAPTable(r report, t map[string]string, vt unifi.Va tags := map[string]string{ "device_name": t["name"], "site_name": t["site_name"], + "source": t["source"], "ap_mac": s.ApMac, "bssid": s.Bssid, "id": s.ID, @@ -148,6 +150,7 @@ func (u *InfluxUnifi) processRadTable(r report, t map[string]string, rt unifi.Ra tags := map[string]string{ "device_name": t["name"], "site_name": t["site_name"], + "source": t["source"], "channel": p.Channel.Txt, "radio": p.Radio, } diff --git a/pkg/influxunifi/udm.go b/pkg/influxunifi/udm.go index 20cea055..599bbbbe 100644 --- a/pkg/influxunifi/udm.go +++ b/pkg/influxunifi/udm.go @@ -40,6 +40,7 @@ func (u *InfluxUnifi) batchUDM(r report, s *unifi.UDM) { } tags := map[string]string{ + "source": s.SourceName, "mac": s.Mac, "site_name": s.SiteName, "name": s.Name, @@ -52,6 +53,7 @@ func (u *InfluxUnifi) batchUDM(r report, s *unifi.UDM) { u.batchUSGstat(s.SpeedtestStatus, s.Stat.Gw, s.Uplink), u.batchSysStats(s.SysStats, s.SystemStats), map[string]interface{}{ + "source": s.SourceName, "ip": s.IP, "bytes": s.Bytes.Val, "last_seen": s.LastSeen.Val, @@ -76,6 +78,7 @@ func (u *InfluxUnifi) batchUDM(r report, s *unifi.UDM) { tags = map[string]string{ "mac": s.Mac, "site_name": s.SiteName, + "source": s.SourceName, "name": s.Name, "version": s.Version, "model": s.Model, @@ -105,6 +108,7 @@ func (u *InfluxUnifi) batchUDM(r report, s *unifi.UDM) { tags = map[string]string{ "mac": s.Mac, "site_name": s.SiteName, + "source": s.SourceName, "name": s.Name, "version": s.Version, "model": s.Model, diff --git a/pkg/influxunifi/usg.go b/pkg/influxunifi/usg.go index 36c9cdbd..96306e88 100644 --- a/pkg/influxunifi/usg.go +++ b/pkg/influxunifi/usg.go @@ -14,6 +14,7 @@ func (u *InfluxUnifi) batchUSG(r report, s *unifi.USG) { tags := map[string]string{ "mac": s.Mac, "site_name": s.SiteName, + "source": s.SourceName, "name": s.Name, "version": s.Version, "model": s.Model, @@ -76,6 +77,7 @@ func (u *InfluxUnifi) batchUSGwans(r report, tags map[string]string, wans ...uni tags := map[string]string{ "device_name": tags["name"], "site_name": tags["site_name"], + "source": tags["source"], "ip": wan.IP, "purpose": wan.Name, "mac": wan.Mac, @@ -115,6 +117,7 @@ func (u *InfluxUnifi) batchNetTable(r report, tags map[string]string, nt unifi.N tags := map[string]string{ "device_name": tags["name"], "site_name": tags["site_name"], + "source": tags["source"], "up": p.Up.Txt, "enabled": p.Enabled.Txt, "ip": p.IP, diff --git a/pkg/influxunifi/usw.go b/pkg/influxunifi/usw.go index 0a91a506..db717c1a 100644 --- a/pkg/influxunifi/usw.go +++ b/pkg/influxunifi/usw.go @@ -14,6 +14,7 @@ func (u *InfluxUnifi) batchUSW(r report, s *unifi.USW) { tags := map[string]string{ "mac": s.Mac, "site_name": s.SiteName, + "source": s.SourceName, "name": s.Name, "version": s.Version, "model": s.Model, @@ -71,6 +72,7 @@ func (u *InfluxUnifi) batchPortTable(r report, t map[string]string, pt []unifi.P tags := map[string]string{ "site_name": t["site_name"], "device_name": t["name"], + "source": t["source"], "name": p.Name, "poe_mode": p.PoeMode, "port_poe": p.PortPoe.Txt, diff --git a/pkg/poller/config.go b/pkg/poller/config.go index 006dc1fb..de1edf07 100644 --- a/pkg/poller/config.go +++ b/pkg/poller/config.go @@ -17,6 +17,7 @@ import ( "github.com/spf13/pflag" "golift.io/cnfg" + "golift.io/cnfg/cnfgfile" "golift.io/unifi" ) @@ -103,14 +104,13 @@ func (u *UnifiPoller) ParseConfigs() error { // parseInterface parses the config file and environment variables into the provided interface. func (u *UnifiPoller) parseInterface(i interface{}) error { - cnfg.ENVTag = "xml" // xml tag is better formatted for slices. // Parse config file into provided interface. - if err := cnfg.ParseFile(i, u.Flags.ConfigFile); err != nil { + if err := cnfgfile.Unmarshal(i, u.Flags.ConfigFile); err != nil { return err } // Parse environment variables into provided interface. - _, err := cnfg.ParseENV(i, ENVConfigPrefix) + _, err := cnfg.UnmarshalENV(i, ENVConfigPrefix) return err } diff --git a/pkg/promunifi/clients.go b/pkg/promunifi/clients.go index 0ffaf54c..5e85825a 100644 --- a/pkg/promunifi/clients.go +++ b/pkg/promunifi/clients.go @@ -42,7 +42,7 @@ type uclient struct { func descClient(ns string) *uclient { labels := []string{"name", "mac", "site_name", "gw_name", "sw_name", "vlan", - "ip", "oui", "network", "sw_port", "ap_name", "wired"} + "ip", "oui", "network", "sw_port", "ap_name", "wired", "source"} labelW := append([]string{"radio_name", "radio", "radio_proto", "channel", "essid", "bssid", "radio_desc"}, labels...) return &uclient{ @@ -85,7 +85,7 @@ func descClient(ns string) *uclient { func (u *promUnifi) exportClient(r report, c *unifi.Client) { labels := []string{c.Name, c.Mac, c.SiteName, c.GwName, c.SwName, c.Vlan.Txt, - c.IP, c.Oui, c.Network, c.SwPort.Txt, c.ApName, ""} + c.IP, c.Oui, c.Network, c.SwPort.Txt, c.ApName, "", c.SourceName} labelW := append([]string{c.RadioName, c.Radio, c.RadioProto, c.Channel.Txt, c.Essid, c.Bssid, c.RadioDescription}, labels...) diff --git a/pkg/promunifi/collector.go b/pkg/promunifi/collector.go index 353bed01..61020625 100644 --- a/pkg/promunifi/collector.go +++ b/pkg/promunifi/collector.go @@ -134,6 +134,7 @@ func (u *promUnifi) ScrapeHandler(w http.ResponseWriter, r *http.Request) { Path: r.URL.Query().Get("path"), // url: "https://127.0.0.1:8443" Role: r.URL.Query().Get("role"), // configured role in up.conf. }} + if t.Name == "" { u.Collector.LogErrorf("input parameter missing on scrape from %v", r.RemoteAddr) http.Error(w, `'input' parameter must be specified (try "unifi")`, 400) diff --git a/pkg/promunifi/site.go b/pkg/promunifi/site.go index 79c57b41..cdae55c8 100644 --- a/pkg/promunifi/site.go +++ b/pkg/promunifi/site.go @@ -34,7 +34,7 @@ type site struct { } func descSite(ns string) *site { - labels := []string{"subsystem", "status", "site_name"} + labels := []string{"subsystem", "status", "site_name", "source"} nd := prometheus.NewDesc return &site{ @@ -68,7 +68,7 @@ func descSite(ns string) *site { func (u *promUnifi) exportSite(r report, s *unifi.Site) { for _, h := range s.Health { - switch labels := []string{h.Subsystem, h.Status, s.SiteName}; labels[0] { + switch labels := []string{h.Subsystem, h.Status, s.SiteName, s.SourceName}; labels[0] { case "www": r.send([]*metric{ {u.Site.TxBytesR, gauge, h.TxBytesR, labels}, diff --git a/pkg/promunifi/uap.go b/pkg/promunifi/uap.go index 0f3a05aa..71352062 100644 --- a/pkg/promunifi/uap.go +++ b/pkg/promunifi/uap.go @@ -80,9 +80,9 @@ type uap struct { } func descUAP(ns string) *uap { - labelA := []string{"stat", "site_name", "name"} // stat + labels[1:] - labelV := []string{"vap_name", "bssid", "radio", "radio_name", "essid", "usage", "site_name", "name"} - labelR := []string{"radio_name", "radio", "site_name", "name"} + labelA := []string{"stat", "site_name", "name", "source"} // stat + labels[1:] + labelV := []string{"vap_name", "bssid", "radio", "radio_name", "essid", "usage", "site_name", "name", "source"} + labelR := []string{"radio_name", "radio", "site_name", "name", "source"} nd := prometheus.NewDesc return &uap{ @@ -165,7 +165,7 @@ func (u *promUnifi) exportUAP(r report, d *unifi.UAP) { return } - labels := []string{d.Type, d.SiteName, d.Name} + labels := []string{d.Type, d.SiteName, d.Name, d.SourceName} infoLabels := []string{d.Version, d.Model, d.Serial, d.Mac, d.IP, d.ID, d.Bytes.Txt, d.Uptime.Txt} u.exportUAPstats(r, labels, d.Stat.Ap, d.BytesD, d.TxBytesD, d.RxBytesD, d.BytesR) u.exportVAPtable(r, labels, d.VapTable) @@ -185,8 +185,8 @@ func (u *promUnifi) exportUAPstats(r report, labels []string, ap *unifi.Ap, byte return } - labelU := []string{"user", labels[1], labels[2]} - labelG := []string{"guest", labels[1], labels[2]} + labelU := []string{"user", labels[1], labels[2], labels[3]} + labelG := []string{"guest", labels[1], labels[2], labels[3]} r.send([]*metric{ // ap only stuff. {u.Device.BytesD, counter, bytes[0], labels}, // not sure if these 3 Ds are counters or gauges. @@ -234,7 +234,7 @@ func (u *promUnifi) exportVAPtable(r report, labels []string, vt unifi.VapTable) continue } - labelV := []string{v.Name, v.Bssid, v.Radio, v.RadioName, v.Essid, v.Usage, labels[1], labels[2]} + labelV := []string{v.Name, v.Bssid, v.Radio, v.RadioName, v.Essid, v.Usage, labels[1], labels[2], labels[3]} r.send([]*metric{ {u.UAP.VAPCcq, gauge, float64(v.Ccq) / 1000.0, labelV}, {u.UAP.VAPMacFilterRejections, counter, v.MacFilterRejections, labelV}, @@ -281,7 +281,7 @@ func (u *promUnifi) exportVAPtable(r report, labels []string, vt unifi.VapTable) func (u *promUnifi) exportRADtable(r report, labels []string, rt unifi.RadioTable, rts unifi.RadioTableStats) { // radio table for _, p := range rt { - labelR := []string{p.Name, p.Radio, labels[1], labels[2]} + labelR := []string{p.Name, p.Radio, labels[1], labels[2], labels[3]} labelRUser := append(labelR, "user") labelRGuest := append(labelR, "guest") diff --git a/pkg/promunifi/udm.go b/pkg/promunifi/udm.go index 34d7e66d..b74e1b9e 100644 --- a/pkg/promunifi/udm.go +++ b/pkg/promunifi/udm.go @@ -31,7 +31,7 @@ type unifiDevice struct { } func descDevice(ns string) *unifiDevice { - labels := []string{"type", "site_name", "name"} + labels := []string{"type", "site_name", "name", "source"} infoLabels := []string{"version", "model", "serial", "mac", "ip", "id", "bytes", "uptime"} return &unifiDevice{ @@ -65,7 +65,7 @@ func (u *promUnifi) exportUDM(r report, d *unifi.UDM) { return } - labels := []string{d.Type, d.SiteName, d.Name} + labels := []string{d.Type, d.SiteName, d.Name, d.SourceName} infoLabels := []string{d.Version, d.Model, d.Serial, d.Mac, d.IP, d.ID, d.Bytes.Txt, d.Uptime.Txt} // Shared data (all devices do this). u.exportBYTstats(r, labels, d.TxBytes, d.RxBytes) diff --git a/pkg/promunifi/usg.go b/pkg/promunifi/usg.go index 08ee3781..83891cc6 100644 --- a/pkg/promunifi/usg.go +++ b/pkg/promunifi/usg.go @@ -36,7 +36,7 @@ type usg struct { } func descUSG(ns string) *usg { - labels := []string{"port", "site_name", "name"} + labels := []string{"port", "site_name", "name", "source"} return &usg{ WanRxPackets: prometheus.NewDesc(ns+"wan_receive_packets_total", "WAN Receive Packets Total", labels, nil), @@ -74,7 +74,7 @@ func (u *promUnifi) exportUSG(r report, d *unifi.USG) { return } - labels := []string{d.Type, d.SiteName, d.Name} + labels := []string{d.Type, d.SiteName, d.Name, d.SourceName} infoLabels := []string{d.Version, d.Model, d.Serial, d.Mac, d.IP, d.ID, d.Bytes.Txt, d.Uptime.Txt} // Gateway System Data. @@ -95,8 +95,8 @@ func (u *promUnifi) exportUSGstats(r report, labels []string, gw *unifi.Gw, st u return } - labelLan := []string{"lan", labels[1], labels[2]} - labelWan := []string{"all", labels[1], labels[2]} + labelLan := []string{"lan", labels[1], labels[2], labels[3]} + labelWan := []string{"all", labels[1], labels[2], labels[3]} r.send([]*metric{ {u.USG.LanRxPackets, counter, gw.LanRxPackets, labelLan}, @@ -121,7 +121,7 @@ func (u *promUnifi) exportWANPorts(r report, labels []string, wans ...unifi.Wan) continue // only record UP interfaces. } - labelWan := []string{wan.Name, labels[1], labels[2]} + labelWan := []string{wan.Name, labels[1], labels[2], labels[3]} r.send([]*metric{ {u.USG.WanRxPackets, counter, wan.RxPackets, labelWan}, diff --git a/pkg/promunifi/usw.go b/pkg/promunifi/usw.go index d700583f..1c736d05 100644 --- a/pkg/promunifi/usw.go +++ b/pkg/promunifi/usw.go @@ -47,8 +47,8 @@ type usw struct { func descUSW(ns string) *usw { pns := ns + "port_" - labelS := []string{"site_name", "name"} - labelP := []string{"port_id", "port_num", "port_name", "port_mac", "port_ip", "site_name", "name"} + labelS := []string{"site_name", "name", "source"} + labelP := []string{"port_id", "port_num", "port_name", "port_mac", "port_ip", "site_name", "name", "source"} nd := prometheus.NewDesc return &usw{ @@ -97,7 +97,7 @@ func (u *promUnifi) exportUSW(r report, d *unifi.USW) { return } - labels := []string{d.Type, d.SiteName, d.Name} + labels := []string{d.Type, d.SiteName, d.Name, d.SourceName} infoLabels := []string{d.Version, d.Model, d.Serial, d.Mac, d.IP, d.ID, d.Bytes.Txt, d.Uptime.Txt} u.exportUSWstats(r, labels, d.Stat.Sw) @@ -161,7 +161,7 @@ func (u *promUnifi) exportPRTtable(r report, labels []string, pt []unifi.Port) { } // Copy labels, and add four new ones. - labelP := []string{labels[2] + " Port " + p.PortIdx.Txt, p.PortIdx.Txt, p.Name, p.Mac, p.IP, labels[1], labels[2]} + labelP := []string{labels[2] + " Port " + p.PortIdx.Txt, p.PortIdx.Txt, p.Name, p.Mac, p.IP, labels[1], labels[2], labels[3]} if p.PoeEnable.Val && p.PortPoe.Val { r.send([]*metric{ diff --git a/plugins/mysql/main.go b/plugins/mysql/main.go index 361165a0..4776e0d7 100644 --- a/plugins/mysql/main.go +++ b/plugins/mysql/main.go @@ -21,12 +21,12 @@ type mysqlConfig struct { // Pointers are ignored during ENV variable unmarshal, avoid pointers to your config. // Only capital (exported) members are unmarshaled when passed into poller.NewOutput(). -type application struct { +type plugin struct { Config mysqlConfig `json:"mysql" toml:"mysql" xml:"mysql" yaml:"mysql"` } func init() { - u := &application{Config: mysqlConfig{}} + u := &plugin{Config: mysqlConfig{}} poller.NewOutput(&poller.Output{ Name: "mysql", @@ -39,7 +39,7 @@ func main() { fmt.Println("this is a unifi-poller plugin; not an application") } -func (a *application) Run(c poller.Collect) error { +func (a *plugin) Run(c poller.Collect) error { c.Logf("mysql plugin is not finished") return nil } From 4a354647aa8d9ebda11a392bbc1120df9dd41008 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Tue, 24 Dec 2019 23:56:00 -0800 Subject: [PATCH 24/32] fix test --- pkg/promunifi/usw.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/promunifi/usw.go b/pkg/promunifi/usw.go index 1c736d05..66d819d8 100644 --- a/pkg/promunifi/usw.go +++ b/pkg/promunifi/usw.go @@ -161,7 +161,8 @@ func (u *promUnifi) exportPRTtable(r report, labels []string, pt []unifi.Port) { } // Copy labels, and add four new ones. - labelP := []string{labels[2] + " Port " + p.PortIdx.Txt, p.PortIdx.Txt, p.Name, p.Mac, p.IP, labels[1], labels[2], labels[3]} + labelP := []string{labels[2] + " Port " + p.PortIdx.Txt, p.PortIdx.Txt, + p.Name, p.Mac, p.IP, labels[1], labels[2], labels[3]} if p.PoeEnable.Val && p.PortPoe.Val { r.send([]*metric{ From ad562e597601c427f3621851d5b0abe6dc5fbd6f Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Wed, 25 Dec 2019 00:19:40 -0800 Subject: [PATCH 25/32] export buffer, lock cnfg --- Gopkg.toml | 4 ++++ pkg/promunifi/collector.go | 11 +++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Gopkg.toml b/Gopkg.toml index 044de96a..e3e5352f 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -12,3 +12,7 @@ [[constraint]] name = "golift.io/unifi" branch = "master" + +[[constraint]] + name = "golift.io/cnfg" + branch = "master" diff --git a/pkg/promunifi/collector.go b/pkg/promunifi/collector.go index 61020625..49be737a 100644 --- a/pkg/promunifi/collector.go +++ b/pkg/promunifi/collector.go @@ -18,7 +18,7 @@ import ( const ( // channel buffer, fits at least one batch. - buffer = 50 + defaultBuffer = 50 defaultHTTPListen = "0.0.0.0:9130" // simply fewer letters. counter = prometheus.CounterValue @@ -50,6 +50,9 @@ type Config struct { // will be collected at all. ReportErrors bool `json:"report_errors" toml:"report_errors" xml:"report_errors" yaml:"report_errors"` Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` + // Buffer is a channel buffer. + // Default is probably 50. Seems fast there; try 1 to see if CPU usage goes down? + Buffer int `json:"buffer" toml:"buffer" xml:"buffer" yaml:"buffer"` } type metric struct { @@ -105,6 +108,10 @@ func (u *promUnifi) Run(c poller.Collect) error { u.HTTPListen = defaultHTTPListen } + if u.Buffer == 0 { + u.Buffer = defaultBuffer + } + // Later can pass this in from poller by adding a method to the interface. u.Collector = c u.Client = descClient(u.Namespace + "_client_") @@ -200,7 +207,7 @@ func (u *promUnifi) collect(ch chan<- prometheus.Metric, filter *poller.Filter) r := &Report{ Config: u.Config, - ch: make(chan []*metric, buffer), + ch: make(chan []*metric, u.Config.Buffer), Start: time.Now()} defer r.close() From 6facff133736e55c8d41eac0b126d6e0d966998f Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Wed, 25 Dec 2019 01:02:36 -0800 Subject: [PATCH 26/32] deps --- Gopkg.lock | 43 +++++++++++++++++++++++++++++++++-------- examples/up.xml.example | 2 +- pkg/poller/config.go | 4 ++++ 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index ef55eb8f..65c3e1b6 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1,6 +1,14 @@ # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. +[[projects]] + digest = "1:9f3b30d9f8e0d7040f729b82dcbc8f0dead820a133b3147ce355fc451f32d761" + name = "github.com/BurntSushi/toml" + packages = ["."] + pruneopts = "UT" + revision = "3012a1dbe2e4bd1391d42b32f0577cb7bbc7f005" + version = "v0.3.1" + [[projects]] digest = "1:d6afaeed1502aa28e80a4ed0981d570ad91b2579193404256ce672ed0a609e0d" name = "github.com/beorn7/perks" @@ -9,6 +17,14 @@ revision = "37c8de3658fcb183f997c4e13e8337516ab753e6" version = "v1.0.1" +[[projects]] + digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec" + name = "github.com/davecgh/go-spew" + packages = ["spew"] + pruneopts = "UT" + revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" + version = "v1.1.1" + [[projects]] digest = "1:573ca21d3669500ff845bdebee890eb7fc7f0f50c59f2132f2a0c6b03d85086a" name = "github.com/golang/protobuf" @@ -38,12 +54,12 @@ version = "v1.0.1" [[projects]] - digest = "1:6eea828983c70075ca297bb915ffbcfd3e34c5a50affd94428a65df955c0ff9c" - name = "github.com/pelletier/go-toml" - packages = ["."] + digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe" + name = "github.com/pmezard/go-difflib" + packages = ["difflib"] pruneopts = "UT" - revision = "903d9455db9ff1d7ac1ab199062eca7266dd11a3" - version = "v1.6.0" + revision = "792786c7400a136282c1664665ae0a8db921c6c2" + version = "v1.0.0" [[projects]] digest = "1:eb04f69c8991e52eff33c428bd729e04208bf03235be88e4df0d88497c6861b9" @@ -98,6 +114,14 @@ revision = "2e9d26c8c37aae03e3f9d4e90b7116f5accb7cab" version = "v1.0.5" +[[projects]] + digest = "1:8548c309c65a85933a625be5e7d52b6ac927ca30c56869fae58123b8a77a75e1" + name = "github.com/stretchr/testify" + packages = ["assert"] + pruneopts = "UT" + revision = "221dbe5ed46703ee255b1da0dec05086f5035f62" + version = "v1.4.0" + [[projects]] branch = "master" digest = "1:0ca5ac8aedc2fd9cb63c90acbd71c0cba8ddb61dfcca58b96cf41550689bf56d" @@ -107,15 +131,15 @@ revision = "c709ea063b76879dc9915358f55d4d77c16ab6d5" [[projects]] - digest = "1:b914912b8cb13beebf85b913006d8692bbf20d8620122966a7a911bbfc6e0104" + branch = "master" + digest = "1:89262a29b1c07290b1de8245b27ec0b8f24b1ca1205b1cd95e4a6b911caf11d7" name = "golift.io/cnfg" packages = [ ".", "cnfgfile", ] pruneopts = "UT" - revision = "7d859f27a93d43faceb7101d02baeb8e69f16d4a" - version = "v0.0.4" + revision = "bec8d8f51fc47101055ada9d952348522682ca16" [[projects]] branch = "master" @@ -137,14 +161,17 @@ analyzer-name = "dep" analyzer-version = 1 input-imports = [ + "github.com/BurntSushi/toml", "github.com/influxdata/influxdb1-client/v2", "github.com/prometheus/client_golang/prometheus", "github.com/prometheus/client_golang/prometheus/promhttp", "github.com/prometheus/common/version", "github.com/spf13/pflag", + "github.com/stretchr/testify/assert", "golift.io/cnfg", "golift.io/cnfg/cnfgfile", "golift.io/unifi", + "gopkg.in/yaml.v2", ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/examples/up.xml.example b/examples/up.xml.example index d7caacc7..d1a1be3b 100644 --- a/examples/up.xml.example +++ b/examples/up.xml.example @@ -8,7 +8,7 @@ and are lists of strings and may be repeated. --> - + 0.0.0.0:9130 diff --git a/pkg/poller/config.go b/pkg/poller/config.go index de1edf07..7d706060 100644 --- a/pkg/poller/config.go +++ b/pkg/poller/config.go @@ -69,6 +69,10 @@ func (u *UnifiPoller) LoadPlugins() error { for _, p := range u.Plugins { name := strings.TrimSuffix(p, ".so") + ".so" + if name == ".so" { + continue // Just ignore it. uhg. + } + if _, err := os.Stat(name); os.IsNotExist(err) { name = path.Join(DefaultObjPath, name) } From b812708b1a36280905c1ea4c6fc0af75689e7ef7 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Wed, 25 Dec 2019 13:52:30 -0800 Subject: [PATCH 27/32] oops --- pkg/promunifi/clients.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/promunifi/clients.go b/pkg/promunifi/clients.go index 5e85825a..b24b2d62 100644 --- a/pkg/promunifi/clients.go +++ b/pkg/promunifi/clients.go @@ -42,7 +42,7 @@ type uclient struct { func descClient(ns string) *uclient { labels := []string{"name", "mac", "site_name", "gw_name", "sw_name", "vlan", - "ip", "oui", "network", "sw_port", "ap_name", "wired", "source"} + "ip", "oui", "network", "sw_port", "ap_name", "source", "wired"} labelW := append([]string{"radio_name", "radio", "radio_proto", "channel", "essid", "bssid", "radio_desc"}, labels...) return &uclient{ @@ -85,7 +85,7 @@ func descClient(ns string) *uclient { func (u *promUnifi) exportClient(r report, c *unifi.Client) { labels := []string{c.Name, c.Mac, c.SiteName, c.GwName, c.SwName, c.Vlan.Txt, - c.IP, c.Oui, c.Network, c.SwPort.Txt, c.ApName, "", c.SourceName} + c.IP, c.Oui, c.Network, c.SwPort.Txt, c.ApName, c.SourceName, ""} labelW := append([]string{c.RadioName, c.Radio, c.RadioProto, c.Channel.Txt, c.Essid, c.Bssid, c.RadioDescription}, labels...) From ad91081e862e558b620ab2e4a0a95e5aad5f70c8 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Thu, 26 Dec 2019 20:32:16 -0800 Subject: [PATCH 28/32] Add go.mod --- Gopkg.lock | 177 ----------------------------------------------------- Gopkg.toml | 18 ------ Makefile | 9 +-- go.mod | 15 +++++ go.sum | 107 ++++++++++++++++++++++++++++++++ 5 files changed, 127 insertions(+), 199 deletions(-) delete mode 100644 Gopkg.lock delete mode 100644 Gopkg.toml create mode 100644 go.mod create mode 100644 go.sum diff --git a/Gopkg.lock b/Gopkg.lock deleted file mode 100644 index 65c3e1b6..00000000 --- a/Gopkg.lock +++ /dev/null @@ -1,177 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[[projects]] - digest = "1:9f3b30d9f8e0d7040f729b82dcbc8f0dead820a133b3147ce355fc451f32d761" - name = "github.com/BurntSushi/toml" - packages = ["."] - pruneopts = "UT" - revision = "3012a1dbe2e4bd1391d42b32f0577cb7bbc7f005" - version = "v0.3.1" - -[[projects]] - digest = "1:d6afaeed1502aa28e80a4ed0981d570ad91b2579193404256ce672ed0a609e0d" - name = "github.com/beorn7/perks" - packages = ["quantile"] - pruneopts = "UT" - revision = "37c8de3658fcb183f997c4e13e8337516ab753e6" - version = "v1.0.1" - -[[projects]] - digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec" - name = "github.com/davecgh/go-spew" - packages = ["spew"] - pruneopts = "UT" - revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" - version = "v1.1.1" - -[[projects]] - digest = "1:573ca21d3669500ff845bdebee890eb7fc7f0f50c59f2132f2a0c6b03d85086a" - name = "github.com/golang/protobuf" - packages = ["proto"] - pruneopts = "UT" - revision = "6c65a5562fc06764971b7c5d05c76c75e84bdbf7" - version = "v1.3.2" - -[[projects]] - branch = "master" - digest = "1:00e5ad58045d6d2a6c9e65d1809ff2594bc396e911712ae892a93976fdece115" - name = "github.com/influxdata/influxdb1-client" - packages = [ - "models", - "pkg/escape", - "v2", - ] - pruneopts = "UT" - revision = "8bf82d3c094dc06be9da8e5bf9d3589b6ea032ae" - -[[projects]] - digest = "1:ff5ebae34cfbf047d505ee150de27e60570e8c394b3b8fdbb720ff6ac71985fc" - name = "github.com/matttproud/golang_protobuf_extensions" - packages = ["pbutil"] - pruneopts = "UT" - revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" - version = "v1.0.1" - -[[projects]] - digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe" - name = "github.com/pmezard/go-difflib" - packages = ["difflib"] - pruneopts = "UT" - revision = "792786c7400a136282c1664665ae0a8db921c6c2" - version = "v1.0.0" - -[[projects]] - digest = "1:eb04f69c8991e52eff33c428bd729e04208bf03235be88e4df0d88497c6861b9" - name = "github.com/prometheus/client_golang" - packages = [ - "prometheus", - "prometheus/internal", - "prometheus/promhttp", - ] - pruneopts = "UT" - revision = "170205fb58decfd011f1550d4cfb737230d7ae4f" - version = "v1.1.0" - -[[projects]] - digest = "1:982be0b5396e16a663697899ce69cc7b1e71ddcae4153af157578d4dc9bc3f88" - name = "github.com/prometheus/client_model" - packages = ["go"] - pruneopts = "UT" - revision = "d1d2010b5beead3fa1c5f271a5cf626e40b3ad6e" - version = "v0.1.0" - -[[projects]] - digest = "1:7dec9ab2db741c280b89b142b08ea142824152c5f40fb1f90c35b6ef7a694456" - name = "github.com/prometheus/common" - packages = [ - "expfmt", - "internal/bitbucket.org/ww/goautoneg", - "model", - "version", - ] - pruneopts = "UT" - revision = "287d3e634a1e550c9e463dd7e5a75a422c614505" - version = "v0.7.0" - -[[projects]] - digest = "1:ec0ff4bd619a67065e34d6477711ed0117e335f99059a4c508e0fe21cfe7b304" - name = "github.com/prometheus/procfs" - packages = [ - ".", - "internal/fs", - "internal/util", - ] - pruneopts = "UT" - revision = "6d489fc7f1d9cd890a250f3ea3431b1744b9623f" - version = "v0.0.8" - -[[projects]] - digest = "1:524b71991fc7d9246cc7dc2d9e0886ccb97648091c63e30eef619e6862c955dd" - name = "github.com/spf13/pflag" - packages = ["."] - pruneopts = "UT" - revision = "2e9d26c8c37aae03e3f9d4e90b7116f5accb7cab" - version = "v1.0.5" - -[[projects]] - digest = "1:8548c309c65a85933a625be5e7d52b6ac927ca30c56869fae58123b8a77a75e1" - name = "github.com/stretchr/testify" - packages = ["assert"] - pruneopts = "UT" - revision = "221dbe5ed46703ee255b1da0dec05086f5035f62" - version = "v1.4.0" - -[[projects]] - branch = "master" - digest = "1:0ca5ac8aedc2fd9cb63c90acbd71c0cba8ddb61dfcca58b96cf41550689bf56d" - name = "golang.org/x/sys" - packages = ["windows"] - pruneopts = "UT" - revision = "c709ea063b76879dc9915358f55d4d77c16ab6d5" - -[[projects]] - branch = "master" - digest = "1:89262a29b1c07290b1de8245b27ec0b8f24b1ca1205b1cd95e4a6b911caf11d7" - name = "golift.io/cnfg" - packages = [ - ".", - "cnfgfile", - ] - pruneopts = "UT" - revision = "bec8d8f51fc47101055ada9d952348522682ca16" - -[[projects]] - branch = "master" - digest = "1:13b9e827c1f958ffe31554d57940495010eef89928697f6c6beae78a750ee637" - name = "golift.io/unifi" - packages = ["."] - pruneopts = "UT" - revision = "4d78625be4cd448ea2db35c63a40b451e7d0df99" - -[[projects]] - digest = "1:b75b3deb2bce8bc079e16bb2aecfe01eb80098f5650f9e93e5643ca8b7b73737" - name = "gopkg.in/yaml.v2" - packages = ["."] - pruneopts = "UT" - revision = "1f64d6156d11335c3f22d9330b0ad14fc1e789ce" - version = "v2.2.7" - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 - input-imports = [ - "github.com/BurntSushi/toml", - "github.com/influxdata/influxdb1-client/v2", - "github.com/prometheus/client_golang/prometheus", - "github.com/prometheus/client_golang/prometheus/promhttp", - "github.com/prometheus/common/version", - "github.com/spf13/pflag", - "github.com/stretchr/testify/assert", - "golift.io/cnfg", - "golift.io/cnfg/cnfgfile", - "golift.io/unifi", - "gopkg.in/yaml.v2", - ] - solver-name = "gps-cdcl" - solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml deleted file mode 100644 index e3e5352f..00000000 --- a/Gopkg.toml +++ /dev/null @@ -1,18 +0,0 @@ -# dep configuration file -# -# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html -# for detailed Gopkg.toml documentation. -# - -[prune] - go-tests = true - unused-packages = true - -# TODO: remove this!!! -[[constraint]] - name = "golift.io/unifi" - branch = "master" - -[[constraint]] - name = "golift.io/cnfg" - branch = "master" diff --git a/Makefile b/Makefile index 46fe2600..f47e8c89 100644 --- a/Makefile +++ b/Makefile @@ -283,12 +283,13 @@ lint: # This is safe; recommended even. dep: vendor -vendor: Gopkg.* - dep ensure --vendor-only +vendor: go.mod go.sum + go mod vendor # Don't run this unless you're ready to debug untested vendored dependencies. -deps: - dep ensure --update +deps: update vendor +update: + go get -u -d # Homebrew stuff. macOS only. diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..ffc8b47f --- /dev/null +++ b/go.mod @@ -0,0 +1,15 @@ +module github.com/davidnewhall/unifi-poller + +go 1.13 + +require ( + github.com/BurntSushi/toml v0.3.1 // indirect + github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d + github.com/prometheus/client_golang v1.3.0 + github.com/prometheus/common v0.7.0 + github.com/spf13/pflag v1.0.5 + golang.org/x/sys v0.0.0-20191224085550-c709ea063b76 // indirect + golift.io/cnfg v0.0.5 + golift.io/unifi v4.1.7-0.20191215214413-4d78625be4cd+incompatible + gopkg.in/yaml.v2 v2.2.7 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..b2fbc010 --- /dev/null +++ b/go.sum @@ -0,0 +1,107 @@ +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d h1:/WZQPMZNsjZ7IlCpsLGdQBINg5bxKQ1K1sh6awxLtkA= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= +github.com/prometheus/client_golang v1.3.0 h1:miYCvYqFXtl/J9FIy8eNpBfYthAEFg+Ys0XyUVEcDsc= +github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f h1:68K/z8GLUxV76xGSqwTWw2gyk/jwn79LUL43rES2g8o= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191224085550-c709ea063b76 h1:Dho5nD6R3PcW2SH1or8vS0dszDaXRxIw55lBX7XiE5g= +golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golift.io/cnfg v0.0.0-20191225081851-bec8d8f51fc4 h1:lCMyrYNLbSBxOiGDnn0ZKWmafS0OEBvaiu7qUJZFrwA= +golift.io/cnfg v0.0.0-20191225081851-bec8d8f51fc4/go.mod h1:ScFDIJg/rJGHbRaed/i7g1lBhywEjB0JiP2uZr3xC3A= +golift.io/cnfg v0.0.5 h1:HnMU8Z9C/igKvir1dqaHx5BPuNGZrp99FCtdJyP2Z4I= +golift.io/cnfg v0.0.5/go.mod h1:ScFDIJg/rJGHbRaed/i7g1lBhywEjB0JiP2uZr3xC3A= +golift.io/unifi v0.0.0-20191215214413-4d78625be4cd h1:d4wPilxOdtcVqwv7WhAXsrSp9/gvMc0ff4xYBTxcqSs= +golift.io/unifi v0.0.0-20191215214413-4d78625be4cd/go.mod h1:Zjw57ZAzTzCMw784pE8CdCFgkYSzVZzmJ++WUttbjto= +golift.io/unifi v4.1.6+incompatible h1:Yhb/+obX2vT9i6PElGislSuQ1WUtOf+l+sRjVxlY6nM= +golift.io/unifi v4.1.6+incompatible/go.mod h1:Zjw57ZAzTzCMw784pE8CdCFgkYSzVZzmJ++WUttbjto= +golift.io/unifi v4.1.7-0.20191215214413-4d78625be4cd+incompatible h1:PBa6XT7PYrYUULBON83Owjv8cUhyu1e13lgl+zAxwHo= +golift.io/unifi v4.1.7-0.20191215214413-4d78625be4cd+incompatible/go.mod h1:Zjw57ZAzTzCMw784pE8CdCFgkYSzVZzmJ++WUttbjto= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 3cf7d89363e8e3b9b073e71f9408a07b59567e27 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Thu, 26 Dec 2019 20:38:55 -0800 Subject: [PATCH 29/32] fix dock and version --- Makefile | 8 ++++---- init/docker/Dockerfile | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index f47e8c89..609141b6 100644 --- a/Makefile +++ b/Makefile @@ -46,10 +46,10 @@ endef PLUGINS:=$(patsubst plugins/%/main.go,%,$(wildcard plugins/*/main.go)) VERSION_LDFLAGS:= \ - -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Branch=$(TRAVIS_BRANCH) \ - -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.BuildDate=$(DATE) \ - -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Revision=$(COMMIT) \ - -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Version=$(VERSION)-$(ITERATION) + -X github.com/prometheus/common/version.Branch=$(TRAVIS_BRANCH) \ + -X github.com/prometheus/common/version.BuildDate=$(DATE) \ + -X github.com/prometheus/common/version.Revision=$(COMMIT) \ + -X github.com/prometheus/common/version.Version=$(VERSION)-$(ITERATION) # Makefile targets follow. diff --git a/init/docker/Dockerfile b/init/docker/Dockerfile index c1e21799..cdfeef05 100644 --- a/init/docker/Dockerfile +++ b/init/docker/Dockerfile @@ -25,7 +25,7 @@ RUN apt-get update \ COPY . $GOPATH/src/${IMPORT_PATH} WORKDIR $GOPATH/src/${IMPORT_PATH} -RUN dep ensure --vendor-only \ +RUN go mod vendor \ && CGO_ENABLED=0 make ${BINARY}.${ARCH}.${OS} FROM scratch From 4ddff5bb251df9426682f5378fc4444e3cfbc322 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Thu, 26 Dec 2019 20:57:01 -0800 Subject: [PATCH 30/32] go all the way --- .metadata.sh | 8 ++------ Makefile | 2 -- init/docker/Dockerfile | 17 +++++------------ init/docker/hooks/build | 1 - init/homebrew/service.rb.tmpl | 10 +++------- 5 files changed, 10 insertions(+), 28 deletions(-) diff --git a/.metadata.sh b/.metadata.sh index 5c380efc..42b73a6c 100755 --- a/.metadata.sh +++ b/.metadata.sh @@ -25,10 +25,8 @@ export BINARY GHUSER HBREPO MAINT VENDOR DESC GOLANGCI_LINT_ARGS CONFIG_FILE LIC # Fix the repo if it doesn't match the binary name. # Provide a better URL if one exists. -# Used as go import path in docker and homebrew builds. -IMPORT_PATH="github.com/${GHUSER}/${BINARY}" # Used for source links and wiki links. -SOURCE_URL="https://${IMPORT_PATH}" +SOURCE_URL="https://github.com/${GHUSER}/${BINARY}" # Used for documentation links. URL="${SOURCE_URL}" @@ -40,9 +38,7 @@ ITERATION=$(git rev-list --count --all || echo 0) DATE="$(date -u +%Y-%m-%dT%H:%M:%SZ)" COMMIT="$(git rev-parse --short HEAD || echo 0)" -# Used by homebrew downloads. -#SOURCE_PATH=https://codeload.${IMPORT_PATH}/tar.gz/v${VERSION} # This is a custom download path for homebrew formula. SOURCE_PATH=https://golift.io/${BINARY}/archive/v${VERSION}.tar.gz -export IMPORT_PATH SOURCE_URL URL VVERSION VERSION ITERATION DATE COMMIT SOURCE_PATH +export SOURCE_URL URL VVERSION VERSION ITERATION DATE COMMIT SOURCE_PATH diff --git a/Makefile b/Makefile index 609141b6..0214ea0d 100644 --- a/Makefile +++ b/Makefile @@ -231,7 +231,6 @@ docker: --build-arg "VENDOR=$(VENDOR)" \ --build-arg "AUTHOR=$(MAINT)" \ --build-arg "BINARY=$(BINARY)" \ - --build-arg "IMPORT_PATH=$(IMPORT_PATH)" \ --build-arg "SOURCE_URL=$(SOURCE_URL)" \ --build-arg "CONFIG_FILE=$(CONFIG_FILE)" \ --tag $(BINARY) . @@ -250,7 +249,6 @@ $(BINARY).rb: v$(VERSION).tar.gz.sha256 init/homebrew/$(FORMULA).rb.tmpl -e "s/{{SHA256}}/$(shell head -c64 $<)/g" \ -e "s/{{Desc}}/$(DESC)/g" \ -e "s%{{URL}}%$(URL)%g" \ - -e "s%{{IMPORT_PATH}}%$(IMPORT_PATH)%g" \ -e "s%{{SOURCE_PATH}}%$(SOURCE_PATH)%g" \ -e "s%{{SOURCE_URL}}%$(SOURCE_URL)%g" \ -e "s%{{CONFIG_FILE}}%$(CONFIG_FILE)%g" \ diff --git a/init/docker/Dockerfile b/init/docker/Dockerfile index cdfeef05..2b74c41e 100644 --- a/init/docker/Dockerfile +++ b/init/docker/Dockerfile @@ -9,21 +9,15 @@ ARG BUILD_DATE=0 ARG COMMIT=0 ARG VERSION=unknown ARG BINARY=application-builder -ARG IMPORT_PATH=github.com/golift/application-builder FROM golang:stretch as builder ARG ARCH ARG OS ARG BINARY -ARG IMPORT_PATH -RUN mkdir -p $GOPATH/pkg/mod $GOPATH/bin $GOPATH/src/${IMPORT_PATH} -RUN apt-get update \ - && apt-get install -y curl \ - && curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - -COPY . $GOPATH/src/${IMPORT_PATH} -WORKDIR $GOPATH/src/${IMPORT_PATH} +RUN mkdir -p $GOPATH/pkg/mod $GOPATH/bin $GOPATH/src /${BINARY} +COPY . /${BINARY} +WORKDIR /${BINARY} RUN go mod vendor \ && CGO_ENABLED=0 make ${BINARY}.${ARCH}.${OS} @@ -36,7 +30,6 @@ ARG COMMIT ARG VERSION ARG LICENSE=MIT ARG BINARY -ARG IMPORT_PATH ARG SOURCE_URL=http://github.com/golift/application-builder ARG URL=http://github.com/golift/application-builder ARG DESC=application-builder @@ -58,8 +51,8 @@ LABEL org.opencontainers.image.created="${BUILD_DATE}" \ org.opencontainers.image.licenses="${LICENSE}" \ org.opencontainers.image.version="${VERSION}" -COPY --from=builder /go/src/${IMPORT_PATH}/${BINARY}.${ARCH}.${OS} /image -COPY --from=builder /go/src/${IMPORT_PATH}/examples/${CONFIG_FILE}.example /etc/${BINARY}/${CONFIG_FILE} +COPY --from=builder /${BINARY}/${BINARY}.${ARCH}.${OS} /image +COPY --from=builder /${BINARY}/examples/${CONFIG_FILE}.example /etc/${BINARY}/${CONFIG_FILE} COPY --from=builder /etc/ssl /etc/ssl VOLUME [ "/etc/${BINARY}" ] diff --git a/init/docker/hooks/build b/init/docker/hooks/build index 3eb4e50e..c8f2796f 100755 --- a/init/docker/hooks/build +++ b/init/docker/hooks/build @@ -28,7 +28,6 @@ for build in $BUILDS; do --build-arg "VENDOR=${VENDOR}" \ --build-arg "AUTHOR=${MAINT}" \ --build-arg "BINARY=${BINARY}" \ - --build-arg "IMPORT_PATH=${IMPORT_PATH}" \ --build-arg "SOURCE_URL=${SOURCE_URL}" \ --build-arg "CONFIG_FILE=${CONFIG_FILE}" \ --tag "${IMAGE_NAME}_${os}_${name}" \ diff --git a/init/homebrew/service.rb.tmpl b/init/homebrew/service.rb.tmpl index ff8802d9..cb10dcb2 100644 --- a/init/homebrew/service.rb.tmpl +++ b/init/homebrew/service.rb.tmpl @@ -1,7 +1,6 @@ # Homebrew Formula Template. Built by Makefile: `make fomula` # This is part of Application Builder. # https://github.com/golift/application-builder -# This file is used when FORMULA is set to 'service'. class {{Class}} < Formula desc "{{Desc}}" homepage "{{URL}}" @@ -13,14 +12,11 @@ class {{Class}} < Formula depends_on "dep" def install - ENV["GOPATH"] = buildpath - - bin_path = buildpath/"src/{{IMPORT_PATH}}" - # Copy all files from their current location (GOPATH root) - # to $GOPATH/src/{{IMPORT_PATH}} + bin_path = buildpath/"#{name}" + # Copy all files from their current location to buildpath/#{name} bin_path.install Dir["*",".??*"] cd bin_path do - system "dep", "ensure", "--vendor-only" + system "make" "vendor" system "make", "install", "VERSION=#{version}", "ITERATION={{Iter}}", "PREFIX=#{prefix}", "ETC=#{etc}" # If this fails, the user gets a nice big warning about write permissions on their # #{var}/log folder. The alternative could be letting the app silently fail From bd0680b92acacbeafc17fb79c13b6a79567c62a5 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Fri, 27 Dec 2019 23:59:16 -0800 Subject: [PATCH 31/32] Add DPI collection --- examples/up.conf.example | 9 ++++ examples/up.json.example | 2 + examples/up.xml.example | 2 + examples/up.yaml.example | 6 ++- go.mod | 4 +- go.sum | 20 +------ pkg/influxunifi/clients.go | 22 ++++++++ pkg/influxunifi/influxdb.go | 25 +++++++-- pkg/influxunifi/site.go | 20 +++++++ pkg/inputunifi/collector.go | 21 ++++++++ pkg/inputunifi/input.go | 1 + pkg/poller/config.go | 2 + pkg/poller/inputs.go | 20 ++----- pkg/promunifi/clients.go | 105 +++++++++++++++++------------------- pkg/promunifi/collector.go | 34 +++++++++--- pkg/promunifi/site.go | 23 ++++++++ 16 files changed, 209 insertions(+), 107 deletions(-) diff --git a/examples/up.conf.example b/examples/up.conf.example index db3b14ad..fb397501 100644 --- a/examples/up.conf.example +++ b/examples/up.conf.example @@ -61,6 +61,7 @@ pass = "unifipoller" sites = ["all"] save_ids = false + save_dpi = false save_sites = true verify_ssl = false @@ -85,6 +86,14 @@ # Only useful if IDS or IPS are enabled on one of the sites. save_ids = false + # Enable collection of Deep Packet Inspection data. This data breaks down traffic + # types for each client and site, it powers a dedicated DPI dashboard. + # Enabling this adds roughly 150 data points per client. That's 6000 metrics for + # 40 clients. This adds a little bit of poller run time per interval and causes + # more API requests to your controller(s). Don't let these "cons" sway you: + # it's cool data. Please provide feedback on your experience with this feature. + save_dpi = false + # Enable collection of site data. This data powers the Network Sites dashboard. # It's not valuable to everyone and setting this to false will save resources. save_sites = true diff --git a/examples/up.json.example b/examples/up.json.example index 6e8c8c66..4675bf61 100644 --- a/examples/up.json.example +++ b/examples/up.json.example @@ -30,6 +30,7 @@ "url": "https://127.0.0.1:8443", "sites": ["all"], "save_ids": false, + "save_dpi": false, "save_sites": true, "verify_ssl": false }, @@ -40,6 +41,7 @@ "pass": "unifipoller", "url": "https://127.0.0.1:8443", "sites": ["all"], + "save_dpi": false, "save_ids": false, "save_sites": true, "verify_ssl": false diff --git a/examples/up.xml.example b/examples/up.xml.example index d1a1be3b..2ff1eecf 100644 --- a/examples/up.xml.example +++ b/examples/up.xml.example @@ -32,6 +32,7 @@ https://127.0.0.1:8443 false false + false true @@ -43,6 +44,7 @@ https://127.0.0.1:8443 false false + false true diff --git a/examples/up.yaml.example b/examples/up.yaml.example index 33cc5088..4ed27193 100644 --- a/examples/up.yaml.example +++ b/examples/up.yaml.example @@ -34,6 +34,7 @@ unifi: - all verify_ssl: false save_ids: false + save_dpi: false save_sites: true @@ -42,9 +43,10 @@ unifi: - role: "" user: "unifipoller" pass: "unifipoller" - url: "https://127.0.0.1:8443" + url: "https://127.0.0.1:8443" sites: - all verify_ssl: false - save_ids: false + save_ids: false + save_dpi: false save_sites: true diff --git a/go.mod b/go.mod index ffc8b47f..b5cb3c39 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,6 @@ require ( github.com/prometheus/client_golang v1.3.0 github.com/prometheus/common v0.7.0 github.com/spf13/pflag v1.0.5 - golang.org/x/sys v0.0.0-20191224085550-c709ea063b76 // indirect golift.io/cnfg v0.0.5 - golift.io/unifi v4.1.7-0.20191215214413-4d78625be4cd+incompatible - gopkg.in/yaml.v2 v2.2.7 // indirect + golift.io/unifi v0.0.400 ) diff --git a/go.sum b/go.sum index b2fbc010..f9e33d08 100644 --- a/go.sum +++ b/go.sum @@ -22,13 +22,11 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d h1:/WZQPMZNsjZ7IlCpsLGdQBINg5bxKQ1K1sh6awxLtkA= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -45,8 +43,6 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8= -github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.3.0 h1:miYCvYqFXtl/J9FIy8eNpBfYthAEFg+Ys0XyUVEcDsc= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -54,12 +50,10 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -82,26 +76,16 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220142924-d4481acd189f h1:68K/z8GLUxV76xGSqwTWw2gyk/jwn79LUL43rES2g8o= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191224085550-c709ea063b76 h1:Dho5nD6R3PcW2SH1or8vS0dszDaXRxIw55lBX7XiE5g= -golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golift.io/cnfg v0.0.0-20191225081851-bec8d8f51fc4 h1:lCMyrYNLbSBxOiGDnn0ZKWmafS0OEBvaiu7qUJZFrwA= -golift.io/cnfg v0.0.0-20191225081851-bec8d8f51fc4/go.mod h1:ScFDIJg/rJGHbRaed/i7g1lBhywEjB0JiP2uZr3xC3A= golift.io/cnfg v0.0.5 h1:HnMU8Z9C/igKvir1dqaHx5BPuNGZrp99FCtdJyP2Z4I= golift.io/cnfg v0.0.5/go.mod h1:ScFDIJg/rJGHbRaed/i7g1lBhywEjB0JiP2uZr3xC3A= -golift.io/unifi v0.0.0-20191215214413-4d78625be4cd h1:d4wPilxOdtcVqwv7WhAXsrSp9/gvMc0ff4xYBTxcqSs= -golift.io/unifi v0.0.0-20191215214413-4d78625be4cd/go.mod h1:Zjw57ZAzTzCMw784pE8CdCFgkYSzVZzmJ++WUttbjto= +golift.io/unifi v0.0.400 h1:r8FlE+p+zmm8jnQdT367H2aGVMTgxZTrHSwbsHBcayA= +golift.io/unifi v0.0.400/go.mod h1:4BjegFlwA3am3mPlY0qHAnSKli4eexLQV42QKaRx9OY= golift.io/unifi v4.1.6+incompatible h1:Yhb/+obX2vT9i6PElGislSuQ1WUtOf+l+sRjVxlY6nM= golift.io/unifi v4.1.6+incompatible/go.mod h1:Zjw57ZAzTzCMw784pE8CdCFgkYSzVZzmJ++WUttbjto= -golift.io/unifi v4.1.7-0.20191215214413-4d78625be4cd+incompatible h1:PBa6XT7PYrYUULBON83Owjv8cUhyu1e13lgl+zAxwHo= -golift.io/unifi v4.1.7-0.20191215214413-4d78625be4cd+incompatible/go.mod h1:Zjw57ZAzTzCMw784pE8CdCFgkYSzVZzmJ++WUttbjto= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= -gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/influxunifi/clients.go b/pkg/influxunifi/clients.go index 9e15ec19..8ac00a4d 100644 --- a/pkg/influxunifi/clients.go +++ b/pkg/influxunifi/clients.go @@ -71,3 +71,25 @@ func (u *InfluxUnifi) batchClient(r report, s *unifi.Client) { r.send(&metric{Table: "clients", Tags: tags, Fields: fields}) } + +func (u *InfluxUnifi) batchClientDPI(r report, s *unifi.DPITable) { + for _, dpi := range s.ByApp { + r.send(&metric{ + Table: "clientdpi", + Tags: map[string]string{ + "category": unifi.DPICats.Get(dpi.Cat), + "application": unifi.DPIApps.GetApp(dpi.Cat, dpi.App), + "name": s.Name, + "mac": s.MAC, + "site_name": s.SiteName, + "source": s.SourceName, + }, + Fields: map[string]interface{}{ + "tx_packets": dpi.TxPackets, + "rx_packets": dpi.RxPackets, + "tx_bytes": dpi.TxBytes, + "rx_bytes": dpi.RxBytes, + }}, + ) + } +} diff --git a/pkg/influxunifi/influxdb.go b/pkg/influxunifi/influxdb.go index 280e94a9..61d63ce1 100644 --- a/pkg/influxunifi/influxdb.go +++ b/pkg/influxunifi/influxdb.go @@ -194,6 +194,16 @@ func (u *InfluxUnifi) loopPoints(r report) { r.add() r.add() r.add() + r.add() + r.add() + + go func() { + defer r.done() + + for _, s := range m.SitesDPI { + u.batchSiteDPI(r, s) + } + }() go func() { defer r.done() @@ -203,6 +213,14 @@ func (u *InfluxUnifi) loopPoints(r report) { } }() + go func() { + defer r.done() + + for _, s := range m.ClientsDPI { + u.batchClientDPI(r, s) + } + }() + go func() { defer r.done() @@ -219,15 +237,14 @@ func (u *InfluxUnifi) loopPoints(r report) { } }() - if m.Devices == nil { - return - } - u.loopDevicePoints(r) } func (u *InfluxUnifi) loopDevicePoints(r report) { m := r.metrics() + if m.Devices == nil { + return + } r.add() r.add() diff --git a/pkg/influxunifi/site.go b/pkg/influxunifi/site.go index 49715e5a..1ae313c4 100644 --- a/pkg/influxunifi/site.go +++ b/pkg/influxunifi/site.go @@ -56,3 +56,23 @@ func (u *InfluxUnifi) batchSite(r report, s *unifi.Site) { r.send(&metric{Table: "subsystems", Tags: tags, Fields: fields}) } } + +func (u *InfluxUnifi) batchSiteDPI(r report, s *unifi.DPITable) { + for _, dpi := range s.ByApp { + r.send(&metric{ + Table: "sitedpi", + Tags: map[string]string{ + "category": unifi.DPICats.Get(dpi.Cat), + "application": unifi.DPIApps.GetApp(dpi.Cat, dpi.App), + "site_name": s.SiteName, + "source": s.SourceName, + }, + Fields: map[string]interface{}{ + "tx_packets": dpi.TxPackets, + "rx_packets": dpi.RxPackets, + "tx_bytes": dpi.TxBytes, + "rx_bytes": dpi.RxBytes, + }}, + ) + } +} diff --git a/pkg/inputunifi/collector.go b/pkg/inputunifi/collector.go index 85941fee..f044401f 100644 --- a/pkg/inputunifi/collector.go +++ b/pkg/inputunifi/collector.go @@ -80,6 +80,16 @@ func (u *InputUnifi) pollController(c *Controller) (*poller.Metrics, error) { return m, fmt.Errorf("unifi.GetSites(%v): %v", c.URL, err) } + if c.SaveDPI { + if m.SitesDPI, err = c.Unifi.GetSiteDPI(m.Sites); err != nil { + return m, fmt.Errorf("unifi.GetSiteDPI(%v): %v", c.URL, err) + } + + if m.ClientsDPI, err = c.Unifi.GetClientsDPI(m.Sites); err != nil { + return m, fmt.Errorf("unifi.GetClientsDPI(%v): %v", c.URL, err) + } + } + if c.SaveIDS { m.IDSList, err = c.Unifi.GetIDS(m.Sites, time.Now().Add(2*time.Minute), time.Now()) if err != nil { @@ -132,12 +142,23 @@ func (u *InputUnifi) augmentMetrics(c *Controller, metrics *poller.Metrics) *pol // These come blank, so set them here. for i, c := range metrics.Clients { + if devices[c.Mac] = c.Name; c.Name == "" { + devices[c.Mac] = c.Hostname + } metrics.Clients[i].SwName = devices[c.SwMac] metrics.Clients[i].ApName = devices[c.ApMac] metrics.Clients[i].GwName = devices[c.GwMac] metrics.Clients[i].RadioDescription = bssdIDs[metrics.Clients[i].Bssid] + metrics.Clients[i].RadioProto } + for i := range metrics.ClientsDPI { + // Name on Client DPI data also comes blank, find it based on MAC address. + metrics.ClientsDPI[i].Name = devices[metrics.ClientsDPI[i].MAC] + if metrics.ClientsDPI[i].Name == "" { + metrics.ClientsDPI[i].Name = metrics.ClientsDPI[i].MAC + } + } + if !*c.SaveSites { metrics.Sites = nil } diff --git a/pkg/inputunifi/input.go b/pkg/inputunifi/input.go index 45d3e4e9..d6540d83 100644 --- a/pkg/inputunifi/input.go +++ b/pkg/inputunifi/input.go @@ -33,6 +33,7 @@ type InputUnifi struct { type Controller struct { VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` SaveIDS bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` + SaveDPI bool `json:"save_dpi" toml:"save_dpi" xml:"save_dpi" yaml:"save_dpi"` SaveSites *bool `json:"save_sites" toml:"save_sites" xml:"save_sites" yaml:"save_sites"` Role string `json:"role" toml:"role" xml:"role,attr" yaml:"role"` User string `json:"user" toml:"user" xml:"user" yaml:"user"` diff --git a/pkg/poller/config.go b/pkg/poller/config.go index 7d706060..b40c8938 100644 --- a/pkg/poller/config.go +++ b/pkg/poller/config.go @@ -49,6 +49,8 @@ type Metrics struct { unifi.IDSList unifi.Clients *unifi.Devices + SitesDPI []*unifi.DPITable + ClientsDPI []*unifi.DPITable } // Config represents the core library input data. diff --git a/pkg/poller/inputs.go b/pkg/poller/inputs.go index a637239c..897a8d0d 100644 --- a/pkg/poller/inputs.go +++ b/pkg/poller/inputs.go @@ -95,23 +95,7 @@ func (u *UnifiPoller) Metrics() (*Metrics, bool, error) { } ok = true - - metrics.Sites = append(metrics.Sites, m.Sites...) - metrics.Clients = append(metrics.Clients, m.Clients...) - metrics.IDSList = append(metrics.IDSList, m.IDSList...) - - if m.Devices == nil { - continue - } - - if metrics.Devices == nil { - metrics.Devices = &unifi.Devices{} - } - - metrics.UAPs = append(metrics.UAPs, m.UAPs...) - metrics.USGs = append(metrics.USGs, m.USGs...) - metrics.USWs = append(metrics.USWs, m.USWs...) - metrics.UDMs = append(metrics.UDMs, m.UDMs...) + metrics = AppendMetrics(metrics, m) } var err error @@ -158,7 +142,9 @@ func (u *UnifiPoller) MetricsFrom(filter *Filter) (*Metrics, bool, error) { // AppendMetrics combined the metrics from two sources. func AppendMetrics(existing *Metrics, m *Metrics) *Metrics { + existing.SitesDPI = append(existing.SitesDPI, m.SitesDPI...) existing.Sites = append(existing.Sites, m.Sites...) + existing.ClientsDPI = append(existing.ClientsDPI, m.ClientsDPI...) existing.Clients = append(existing.Clients, m.Clients...) existing.IDSList = append(existing.IDSList, m.IDSList...) diff --git a/pkg/promunifi/clients.go b/pkg/promunifi/clients.go index b24b2d62..844d899d 100644 --- a/pkg/promunifi/clients.go +++ b/pkg/promunifi/clients.go @@ -6,44 +6,43 @@ import ( ) type uclient struct { - Anomalies *prometheus.Desc - BytesR *prometheus.Desc - CCQ *prometheus.Desc - Satisfaction *prometheus.Desc - Noise *prometheus.Desc - RoamCount *prometheus.Desc - RSSI *prometheus.Desc - RxBytes *prometheus.Desc - RxBytesR *prometheus.Desc - RxPackets *prometheus.Desc - RxRate *prometheus.Desc - Signal *prometheus.Desc - TxBytes *prometheus.Desc - TxBytesR *prometheus.Desc - TxPackets *prometheus.Desc - TxRetries *prometheus.Desc - TxPower *prometheus.Desc - TxRate *prometheus.Desc - Uptime *prometheus.Desc - WifiTxAttempts *prometheus.Desc - WiredRxBytes *prometheus.Desc - WiredRxBytesR *prometheus.Desc - WiredRxPackets *prometheus.Desc - WiredTxBytes *prometheus.Desc - WiredTxBytesR *prometheus.Desc - WiredTxPackets *prometheus.Desc - DpiStatsApp *prometheus.Desc - DpiStatsCat *prometheus.Desc - DpiStatsRxBytes *prometheus.Desc - DpiStatsRxPackets *prometheus.Desc - DpiStatsTxBytes *prometheus.Desc - DpiStatsTxPackets *prometheus.Desc + Anomalies *prometheus.Desc + BytesR *prometheus.Desc + CCQ *prometheus.Desc + Satisfaction *prometheus.Desc + Noise *prometheus.Desc + RoamCount *prometheus.Desc + RSSI *prometheus.Desc + RxBytes *prometheus.Desc + RxBytesR *prometheus.Desc + RxPackets *prometheus.Desc + RxRate *prometheus.Desc + Signal *prometheus.Desc + TxBytes *prometheus.Desc + TxBytesR *prometheus.Desc + TxPackets *prometheus.Desc + TxRetries *prometheus.Desc + TxPower *prometheus.Desc + TxRate *prometheus.Desc + Uptime *prometheus.Desc + WifiTxAttempts *prometheus.Desc + WiredRxBytes *prometheus.Desc + WiredRxBytesR *prometheus.Desc + WiredRxPackets *prometheus.Desc + WiredTxBytes *prometheus.Desc + WiredTxBytesR *prometheus.Desc + WiredTxPackets *prometheus.Desc + DPITxPackets *prometheus.Desc + DPIRxPackets *prometheus.Desc + DPITxBytes *prometheus.Desc + DPIRxBytes *prometheus.Desc } func descClient(ns string) *uclient { labels := []string{"name", "mac", "site_name", "gw_name", "sw_name", "vlan", "ip", "oui", "network", "sw_port", "ap_name", "source", "wired"} labelW := append([]string{"radio_name", "radio", "radio_proto", "channel", "essid", "bssid", "radio_desc"}, labels...) + labelDPI := []string{"name", "mac", "site_name", "source", "category", "application"} return &uclient{ Anomalies: prometheus.NewDesc(ns+"anomalies", "Client Anomalies", labelW, nil), @@ -66,20 +65,25 @@ func descClient(ns string) *uclient { TxRate: prometheus.NewDesc(ns+"radio_transmit_rate_bps", "Client Transmit Rate", labelW, nil), WifiTxAttempts: prometheus.NewDesc(ns+"wifi_attempts_transmit_total", "Client Wifi Transmit Attempts", labelW, nil), Uptime: prometheus.NewDesc(ns+"uptime_seconds", "Client Uptime", labelW, nil), - /* needs more "looking into" - DpiStatsApp: prometheus.NewDesc(ns+"dpi_stats_app", - "Client DPI Stats App", labels, nil), - DpiStatsCat: prometheus.NewDesc(ns+"dpi_stats_cat", - "Client DPI Stats Cat", labels, nil), - DpiStatsRxBytes: prometheus.NewDesc(ns+"dpi_stats_receive_bytes_total", - "Client DPI Stats Receive Bytes", labels, nil), - DpiStatsRxPackets: prometheus.NewDesc(ns+"dpi_stats_receive_packets_total", - "Client DPI Stats Receive Packets", labels, nil), - DpiStatsTxBytes: prometheus.NewDesc(ns+"dpi_stats_transmit_bytes_total", - "Client DPI Stats Transmit Bytes", labels, nil), - DpiStatsTxPackets: prometheus.NewDesc(ns+"dpi_stats_transmit_packets_total", - "Client DPI Stats Transmit Packets", labels, nil), - */ + DPITxPackets: prometheus.NewDesc(ns+"dpi_transmit_packets", "Client DPI Transmit Packets", labelDPI, nil), + DPIRxPackets: prometheus.NewDesc(ns+"dpi_receive_packets", "Client DPI Receive Packets", labelDPI, nil), + DPITxBytes: prometheus.NewDesc(ns+"dpi_transmit_bytes", "Client DPI Transmit Bytes", labelDPI, nil), + DPIRxBytes: prometheus.NewDesc(ns+"dpi_receive_bytes", "Client DPI Receive Bytes", labelDPI, nil), + } +} + +func (u *promUnifi) exportClientDPI(r report, s *unifi.DPITable) { + for _, dpi := range s.ByApp { + labelDPI := []string{s.Name, s.MAC, s.SiteName, s.SourceName, + unifi.DPICats.Get(dpi.Cat), unifi.DPIApps.GetApp(dpi.Cat, dpi.App)} + + // log.Println(labelDPI, dpi.Cat, dpi.App, dpi.TxBytes, dpi.RxBytes, dpi.TxPackets, dpi.RxPackets) + r.send([]*metric{ + {u.Client.DPITxPackets, gauge, dpi.TxPackets, labelDPI}, + {u.Client.DPIRxPackets, gauge, dpi.RxPackets, labelDPI}, + {u.Client.DPITxBytes, gauge, dpi.TxBytes, labelDPI}, + {u.Client.DPIRxBytes, gauge, dpi.RxBytes, labelDPI}, + }) } } @@ -130,12 +134,3 @@ func (u *promUnifi) exportClient(r report, c *unifi.Client) { r.send([]*metric{{u.Client.Uptime, gauge, c.Uptime, labelW}}) } - -/* needs more "looking into" -{u.Client.DpiStatsApp, gauge, c.DpiStats.App, labels}, -{u.Client.DpiStatsCat, gauge, c.DpiStats.Cat, labels}, -{u.Client.DpiStatsRxBytes, counter, c.DpiStats.RxBytes, labels}, -{u.Client.DpiStatsRxPackets, counter, c.DpiStats.RxPackets, labels}, -{u.Client.DpiStatsTxBytes, counter, c.DpiStats.TxBytes, labels}, -{u.Client.DpiStatsTxPackets, counter, c.DpiStats.TxPackets, labels}, -*/ diff --git a/pkg/promunifi/collector.go b/pkg/promunifi/collector.go index 49be737a..7fae2ccc 100644 --- a/pkg/promunifi/collector.go +++ b/pkg/promunifi/collector.go @@ -277,6 +277,8 @@ func (u *promUnifi) loopExports(r report) { r.add() r.add() r.add() + r.add() + r.add() go func() { defer r.done() @@ -286,6 +288,30 @@ func (u *promUnifi) loopExports(r report) { } }() + go func() { + defer r.done() + + for _, s := range m.SitesDPI { + u.exportSiteDPI(r, s) + } + }() + + go func() { + defer r.done() + + for _, c := range m.Clients { + u.exportClient(r, c) + } + }() + + go func() { + defer r.done() + + for _, c := range m.ClientsDPI { + u.exportClientDPI(r, c) + } + }() + go func() { defer r.done() @@ -317,12 +343,4 @@ func (u *promUnifi) loopExports(r report) { u.exportUSW(r, d) } }() - - go func() { - defer r.done() - - for _, c := range m.Clients { - u.exportClient(r, c) - } - }() } diff --git a/pkg/promunifi/site.go b/pkg/promunifi/site.go index cdae55c8..c515b8a4 100644 --- a/pkg/promunifi/site.go +++ b/pkg/promunifi/site.go @@ -31,10 +31,15 @@ type site struct { RemoteUserTxBytes *prometheus.Desc RemoteUserRxPackets *prometheus.Desc RemoteUserTxPackets *prometheus.Desc + DPITxPackets *prometheus.Desc + DPIRxPackets *prometheus.Desc + DPITxBytes *prometheus.Desc + DPIRxBytes *prometheus.Desc } func descSite(ns string) *site { labels := []string{"subsystem", "status", "site_name", "source"} + labelDPI := []string{"category", "application", "site_name", "source"} nd := prometheus.NewDesc return &site{ @@ -63,6 +68,24 @@ func descSite(ns string) *site { RemoteUserTxBytes: nd(ns+"remote_user_transmit_bytes_total", "Remote Users Transmit Bytes", labels, nil), RemoteUserRxPackets: nd(ns+"remote_user_receive_packets_total", "Remote Users Receive Packets", labels, nil), RemoteUserTxPackets: nd(ns+"remote_user_transmit_packets_total", "Remote Users Transmit Packets", labels, nil), + DPITxPackets: nd(ns+"dpi_transmit_packets", "Site DPI Transmit Packets", labelDPI, nil), + DPIRxPackets: nd(ns+"dpi_receive_packets", "Site DPI Receive Packets", labelDPI, nil), + DPITxBytes: nd(ns+"dpi_transmit_bytes", "Site DPI Transmit Bytes", labelDPI, nil), + DPIRxBytes: nd(ns+"dpi_receive_bytes", "Site DPI Receive Bytes", labelDPI, nil), + } +} + +func (u *promUnifi) exportSiteDPI(r report, s *unifi.DPITable) { + for _, dpi := range s.ByApp { + labelDPI := []string{unifi.DPICats.Get(dpi.Cat), unifi.DPIApps.GetApp(dpi.Cat, dpi.App), s.SiteName, s.SourceName} + + // log.Println(labelsDPI, dpi.Cat, dpi.App, dpi.TxBytes, dpi.RxBytes, dpi.TxPackets, dpi.RxPackets) + r.send([]*metric{ + {u.Site.DPITxPackets, gauge, dpi.TxPackets, labelDPI}, + {u.Site.DPIRxPackets, gauge, dpi.RxPackets, labelDPI}, + {u.Site.DPITxBytes, gauge, dpi.TxBytes, labelDPI}, + {u.Site.DPIRxBytes, gauge, dpi.RxBytes, labelDPI}, + }) } } From 1d923fd7c54a4b527484546b73d8a2b20f468d18 Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Sat, 28 Dec 2019 00:07:16 -0800 Subject: [PATCH 32/32] fix lint --- pkg/inputunifi/collector.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/inputunifi/collector.go b/pkg/inputunifi/collector.go index f044401f..195b2d51 100644 --- a/pkg/inputunifi/collector.go +++ b/pkg/inputunifi/collector.go @@ -145,6 +145,7 @@ func (u *InputUnifi) augmentMetrics(c *Controller, metrics *poller.Metrics) *pol if devices[c.Mac] = c.Name; c.Name == "" { devices[c.Mac] = c.Hostname } + metrics.Clients[i].SwName = devices[c.SwMac] metrics.Clients[i].ApName = devices[c.ApMac] metrics.Clients[i].GwName = devices[c.GwMac]