diff --git a/.metadata.sh b/.metadata.sh index 37506a59..11ba0838 100755 --- a/.metadata.sh +++ b/.metadata.sh @@ -32,11 +32,6 @@ SOURCE_URL="https://${IMPORT_PATH}" # Used for documentation links. URL="${SOURCE_URL}" -# This parameter is passed in as -X to go build. Used to override the Version variable in a package. -# This makes a path like github.com/user/hello-world/helloworld.Version=1.3.3 -# Name the Version-containing library the same as the github repo, without dashes. -VERSION_PATH="${IMPORT_PATH}/pkg/poller.Version" - # Dynamic. Recommend not changing. VVERSION=$(git describe --abbrev=0 --tags $(git rev-list --tags --max-count=1)) VERSION="$(echo $VVERSION | tr -d v | grep -E '^\S+$' || echo development)" @@ -51,4 +46,4 @@ COMMIT="$(git rev-parse --short HEAD || echo 0)" # This is a custom download path for homebrew formula. SOURCE_PATH=https://golift.io/${BINARY}/archive/v${VERSION}.tar.gz -export IMPORT_PATH SOURCE_URL URL VERSION_PATH VVERSION VERSION ITERATION DATE BRANCH COMMIT SOURCE_PATH +export IMPORT_PATH SOURCE_URL URL VVERSION VERSION ITERATION DATE BRANCH COMMIT SOURCE_PATH diff --git a/Makefile b/Makefile index 063fdafa..4cf96dc5 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ VERSION_LDFLAGS:= \ -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Branch=$(BRANCH) \ -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.BuildDate=$(DATE) \ -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Revision=$(COMMIT) \ - -X $(VERSION_PATH)=$(VERSION)-$(ITERATION) + -X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Version=$(VERSION)-$(ITERATION) # Makefile targets follow. diff --git a/examples/MANUAL.md b/examples/MANUAL.md index 0b1a8860..012cd343 100644 --- a/examples/MANUAL.md +++ b/examples/MANUAL.md @@ -65,10 +65,10 @@ is provided so the application can be easily adapted to any environment. `Config File Parameters` - interval default: 30s - How often to poll the controller for updated client and device data. - The UniFi Controller only updates traffic stats about every 30-60 seconds. - Only works if "mode" (below) is "influx" - other modes do not use interval. +Additional parameters are added by output packages. Parameters can also be set +using environment variables. See the GitHub wiki for more information! + + >>> POLLER FIELDS FOLLOW - you may have multiple controllers: debug default: false This turns on time stamps and line numbers in logs, outputs a few extra @@ -79,56 +79,7 @@ is provided so the application can be easily adapted to any environment. errors will be logged. Using this with debug=true adds line numbers to any error logs. - mode default: "influx" - * Value: influx - This default mode runs this application as a daemon. It will poll - the controller at the configured interval and report measurements to - InfluxDB. Providing an invalid value will run in this default mode. - - * Value: influxlambda - Setting this value will invoke a run-once mode where the application - immediately polls the controller and reports the metrics to InfluxDB. - Then it exits. This mode is useful in an AWS Lambda or a crontab where - the execution timings are controlled. This mode may also be adapted - to run in other collector scripts and apps like telegraf or diamond. - This mode can also be combined with a "test database" in InfluxDB to - give yourself a "test config file" you may run ad-hoc to test changes. - - * Value: prometheus - In this mode the application opens an http interface and exports the - measurements at /metrics for collection by prometheus. Enabling this - mode disables InfluxDB usage entirely. - - * Value: both - Setting the mode to "both" will cause the InfluxDB poller routine to run - along with the Prometheus exporter. You can run both at the same time. - - http_listen default: 0.0.0.0:9130 - This option controls the IP and port the http listener uses when the - mode is set to prometheus. This setting has no effect when other modes - are in use. Metrics become available at the /metrics URI. - - influx_url default: http://127.0.0.1:8086 - This is the URL where the Influx web server is available. - - influx_user default: unifi - Username used to authenticate with InfluxDB. - - influx_pass default: unifi - Password used to authenticate with InfluxDB. - - influx_db default: unifi - Custom database created in InfluxDB to use with this application. - On first setup, log into InfluxDB and create access: - $ influx -host localhost -port 8086 - CREATE DATABASE unifi - CREATE USER unifi WITH PASSWORD 'unifi' WITH ALL PRIVILEGES - GRANT ALL ON unifi TO unifi - - influx_insecure_ssl default: false - Setting this to true will allow use of InfluxDB with an invalid SSL certificate. - - >>> CONTROLLER FIELDS FOLLOW - you may have multiple controllers: + >>> CONTROLLER FIELDS FOLLOW - you may have multiple controllers: sites default: ["all"] This list of strings should represent the names of sites on the UniFi diff --git a/examples/up.conf.example b/examples/up.conf.example index 31713a84..75c12941 100644 --- a/examples/up.conf.example +++ b/examples/up.conf.example @@ -3,11 +3,7 @@ ######################################################## -# The UniFi Controller only updates traffic stats about every 30 seconds. -# Setting this to something lower may lead to "zeros" in your data. -# If you're getting zeros now, set this to "1m" -interval = "30s" - +[poller] # Turns on line numbers, microsecond logging, and a per-device log. # The default is false, but I personally leave this on at home (four devices). # This may be noisy if you have a lot of devices. It adds one line per device. @@ -17,38 +13,34 @@ debug = false # Recommend enabling debug with this setting for better error logging. quiet = false -# Which mode to run this application in. The default mode is "influx". Providing -# an invalid mode will also result in "influx". In this default mode the application -# runs as a daemon and polls the controller at the configured interval. -# -# Other options: "influxlambda", "prometheus", "both" -# -# Mode "influxlambda" makes the application exit after collecting and reporting metrics -# to InfluxDB one time. This mode requires an external process like an AWS Lambda -# or a simple crontab to keep the timings accurate on UniFi Poller run intervals. -# -# Mode "prometheus" opens an HTTP server on port 9130 and exports the metrics at -# /metrics for polling collection by a prometheus server. This disables influxdb. -# -# Mode "both" runs the Prometheus HTTP server and InfluxDB poller interval at -# the same time. -mode = "influx" +#### OUTPUTS + +[prometheus] +disable = false # This controls on which ip and port /metrics is exported when mode is "prometheus". # This has no effect in other modes. Must contain a colon and port. http_listen = "0.0.0.0:9130" +report_errors = false +[influxdb] +disable = false # InfluxDB does not require auth by default, so the user/password are probably unimportant. -influx_url = "http://127.0.0.1:8086" -influx_user = "unifi" -influx_pass = "unifi" +url = "http://127.0.0.1:8086" +user = "unifi" +pass = "unifi" # Be sure to create this database. -influx_db = "unifi" -# If your InfluxDB uses an invalid SSL cert, set this to true. -influx_insecure_ssl = false +db = "unifi" +# If your InfluxDB uses a valid SSL cert, set this to true. +verify_ssl = false +# The UniFi Controller only updates traffic stats about every 30 seconds. +# Setting this to something lower may lead to "zeros" in your data. +# If you're getting zeros now, set this to "1m" +interval = "30s" + +#### INPUTS # You may repeat the following section to poll additional controllers. - [[controller]] # Friendly name used in dashboards. name = "" diff --git a/examples/up.json.example b/examples/up.json.example index 07ecd509..65e4d27e 100644 --- a/examples/up.json.example +++ b/examples/up.json.example @@ -1,22 +1,33 @@ { - "interval": "30s", - "debug": false, - "quiet": false, - "mode": "influx", - "http_listen": "0.0.0.0:9130", - "influx_url": "http://127.0.0.1:8086", - "influx_user": "unifi", - "influx_pass": "unifi", - "influx_db": "unifi", - "influx_insecure_ssl": false, - "controller": [{ - "name": "", - "user": "influx", - "pass": "", - "url": "https://127.0.0.1:8443", - "sites": ["all"], - "save_ids": false, - "save_sites": true, - "verify_ssl": false + "poller": { + "debug": false, + "quiet": false + }, + + "prometheus": { + "disable": false, + "http_listen": "0.0.0.0:9130", + "report_errors": false + }, + + "influxdb": { + "disable": false, + "url": "http://127.0.0.1:8086", + "user": "unifi", + "pass": "unifi", + "db": "unifi", + "verify_ssl": false, + "interval": "30s" + }, + + "controller": [{ + "name": "", + "user": "influx", + "pass": "", + "url": "https://127.0.0.1:8443", + "sites": ["all"], + "save_ids": false, + "save_sites": true, + "verify_ssl": false }] } diff --git a/examples/up.xml.example b/examples/up.xml.example index 551ff8b7..710c01ba 100644 --- a/examples/up.xml.example +++ b/examples/up.xml.example @@ -5,21 +5,23 @@ # provided values are defaults. See up.conf.example! # ####################################################### --> - + - 60s + + 0.0.0.0:9130 + false + - false - false + + 30s + http://127.0.0.1:8086 + unifi + unifi + unifi + false + - influx - 0.0.0.0:9130 - - unifi - unifi - http://127.0.0.1:8086 - unifi - false + all influx @@ -29,4 +31,5 @@ false true - + + diff --git a/examples/up.yaml.example b/examples/up.yaml.example index b076feb4..611c5fa8 100644 --- a/examples/up.yaml.example +++ b/examples/up.yaml.example @@ -3,19 +3,24 @@ # provided values are defaults. See up.conf.example! # ######################################################## --- -interval: "30s" -debug: false -quiet: false +poller: + debug: false + quiet: false -mode: "influx" -http_listen: "0.0.0.0:9130" +prometheus: + disable: false + http_listen: "0.0.0.0:9130" + report_errors: false -influx_url: "http://127.0.0.1:8086" -influx_user: "unifi" -influx_pass: "unifi" -influx_db: "unifi" -influx_insecure_ssl: false +influxdb: + disable: false + interval: "30s" + url: "http://127.0.0.1:8086" + user: "unifi" + pass: "unifi" + db: "unifi" + verify_ssl: false controller: - name: "" diff --git a/main.go b/main.go index 332bfd5d..be0fe13b 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,9 @@ import ( "log" "github.com/davidnewhall/unifi-poller/pkg/poller" + // Enable output plugins! + _ "github.com/davidnewhall/unifi-poller/pkg/influxunifi" + _ "github.com/davidnewhall/unifi-poller/pkg/promunifi" ) // Keep it simple. diff --git a/pkg/influxunifi/metrics.go b/pkg/influxunifi/metrics.go index 90a95a31..6775ac06 100644 --- a/pkg/influxunifi/metrics.go +++ b/pkg/influxunifi/metrics.go @@ -1,29 +1,47 @@ -// Package influx provides the methods to turn UniFi measurements into influx +// Package influxunifi provides the methods to turn UniFi measurements into influx // data-points with appropriate tags and fields. package influxunifi import ( "crypto/tls" "fmt" + "log" "time" - "github.com/davidnewhall/unifi-poller/pkg/metrics" + "github.com/davidnewhall/unifi-poller/pkg/poller" influx "github.com/influxdata/influxdb1-client/v2" + conf "golift.io/config" +) + +const ( + defaultInterval = 30 * time.Second + defaultInfluxDB = "unifi" + defaultInfluxUser = "unifi" + defaultInfluxURL = "http://127.0.0.1:8086" ) // Config defines the data needed to store metrics in InfluxDB type Config struct { - Database string - URL string - User string - Pass string - BadSSL bool + Interval conf.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` + Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` + VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"` + URL string `json:"url,omitempty" toml:"url,omitempty" xml:"url" yaml:"url"` + User string `json:"user,omitempty" toml:"user,omitempty" xml:"user" yaml:"user"` + Pass string `json:"pass,omitempty" toml:"pass,omitempty" xml:"pass" yaml:"pass"` + DB string `json:"db,omitempty" toml:"db,omitempty" xml:"db" yaml:"db"` +} + +// InfluxDB allows the data to be nested in the config file. +type InfluxDB struct { + Config Config `json:"influxdb" toml:"influxdb" xml:"influxdb" yaml:"influxdb"` } // InfluxUnifi is returned by New() after you provide a Config. type InfluxUnifi struct { - cf *Config - influx influx.Client + Collector poller.Collect + influx influx.Client + LastCheck time.Time + *InfluxDB } type metric struct { @@ -32,26 +50,101 @@ type metric struct { Fields map[string]interface{} } -// New returns an InfluxDB interface. -func New(c *Config) (*InfluxUnifi, error) { - i, err := influx.NewHTTPClient(influx.HTTPConfig{ - Addr: c.URL, - Username: c.User, - Password: c.Pass, - TLSConfig: &tls.Config{InsecureSkipVerify: c.BadSSL}, +func init() { + u := &InfluxUnifi{InfluxDB: &InfluxDB{}, LastCheck: time.Now()} + poller.NewOutput(&poller.Output{ + Name: "influxdb", + Config: u.InfluxDB, + Method: u.Run, }) - return &InfluxUnifi{cf: c, influx: i}, err +} + +// PollController runs forever, polling UniFi and pushing to InfluxDB +// This is started by Run() or RunBoth() after everything checks out. +func (u *InfluxUnifi) PollController() { + interval := u.Config.Interval.Round(time.Second) + log.Printf("[INFO] Everything checks out! Poller started, InfluxDB interval: %v", interval) + + ticker := time.NewTicker(interval) + for u.LastCheck = range ticker.C { + metrics, err := u.Collector.Metrics() + if err != nil { + u.Collector.LogErrorf("%v", err) + continue + } + + report, err := u.ReportMetrics(metrics) + if err != nil { + // XXX: reset and re-auth? not sure.. + u.Collector.LogErrorf("%v", err) + continue + } + + u.LogInfluxReport(report) + } +} + +// Run runs a ticker to poll the unifi server and update influxdb. +func (u *InfluxUnifi) Run(c poller.Collect) error { + var err error + + if u.Config.Disable { + return nil + } + + u.Collector = c + u.setConfigDefaults() + + u.influx, err = influx.NewHTTPClient(influx.HTTPConfig{ + Addr: u.Config.URL, + Username: u.Config.User, + Password: u.Config.Pass, + TLSConfig: &tls.Config{InsecureSkipVerify: !u.Config.VerifySSL}, + }) + if err != nil { + return err + } + + u.PollController() + + return nil +} + +func (u *InfluxUnifi) setConfigDefaults() { + if u.Config.URL == "" { + u.Config.URL = defaultInfluxURL + } + + if u.Config.User == "" { + u.Config.User = defaultInfluxUser + } + + if u.Config.Pass == "" { + u.Config.Pass = defaultInfluxUser + } + + if u.Config.DB == "" { + u.Config.DB = defaultInfluxDB + } + + if u.Config.Interval.Duration == 0 { + u.Config.Interval = conf.Duration{Duration: defaultInterval} + } else if u.Config.Interval.Duration < defaultInterval/2 { + u.Config.Interval = conf.Duration{Duration: defaultInterval / 2} + } + + u.Config.Interval = conf.Duration{Duration: u.Config.Interval.Duration.Round(time.Second)} } // ReportMetrics batches all device and client data into influxdb data points. // Call this after you've collected all the data you care about. // Returns an error if influxdb calls fail, otherwise returns a report. -func (u *InfluxUnifi) ReportMetrics(m *metrics.Metrics) (*Report, error) { +func (u *InfluxUnifi) ReportMetrics(m *poller.Metrics) (*Report, error) { r := &Report{Metrics: m, ch: make(chan *metric), Start: time.Now()} defer close(r.ch) // Make a new Influx Points Batcher. var err error - r.bp, err = influx.NewBatchPoints(influx.BatchPointsConfig{Database: u.cf.Database}) + r.bp, err = influx.NewBatchPoints(influx.BatchPointsConfig{Database: u.Config.DB}) if err != nil { return nil, fmt.Errorf("influx.NewBatchPoints: %v", err) } @@ -140,3 +233,13 @@ func (u *InfluxUnifi) loopPoints(r report) { } }() } + +// LogInfluxReport writes a log message after exporting to influxdb. +func (u *InfluxUnifi) LogInfluxReport(r *Report) { + idsMsg := fmt.Sprintf("IDS Events: %d, ", len(r.Metrics.IDSList)) + u.Collector.Logf("UniFi Metrics Recorded. Sites: %d, Clients: %d, "+ + "UAP: %d, USG/UDM: %d, USW: %d, %sPoints: %d, Fields: %d, Errs: %d, Elapsed: %v", + len(r.Metrics.Sites), len(r.Metrics.Clients), len(r.Metrics.UAPs), + len(r.Metrics.UDMs)+len(r.Metrics.USGs), len(r.Metrics.USWs), idsMsg, r.Total, + r.Fields, len(r.Errors), r.Elapsed.Round(time.Millisecond)) +} diff --git a/pkg/influxunifi/report.go b/pkg/influxunifi/report.go index 5d1e760d..3fdf77a9 100644 --- a/pkg/influxunifi/report.go +++ b/pkg/influxunifi/report.go @@ -4,13 +4,13 @@ import ( "sync" "time" - "github.com/davidnewhall/unifi-poller/pkg/metrics" + "github.com/davidnewhall/unifi-poller/pkg/poller" influx "github.com/influxdata/influxdb1-client/v2" ) // Report is returned to the calling procedure after everything is processed. type Report struct { - Metrics *metrics.Metrics + Metrics *poller.Metrics Errors []error Total int Fields int @@ -28,10 +28,10 @@ type report interface { send(m *metric) error(err error) batch(m *metric, pt *influx.Point) - metrics() *metrics.Metrics + metrics() *poller.Metrics } -func (r *Report) metrics() *metrics.Metrics { +func (r *Report) metrics() *poller.Metrics { return r.Metrics } diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go deleted file mode 100644 index 2d38f54e..00000000 --- a/pkg/metrics/metrics.go +++ /dev/null @@ -1,16 +0,0 @@ -package metrics - -import ( - "time" - - "golift.io/unifi" -) - -// Metrics is a type shared by the exporting and reporting packages. -type Metrics struct { - TS time.Time - unifi.Sites - unifi.IDSList - unifi.Clients - *unifi.Devices -} diff --git a/pkg/poller/config.go b/pkg/poller/config.go index 04d3f1a4..0d6fe126 100644 --- a/pkg/poller/config.go +++ b/pkg/poller/config.go @@ -12,26 +12,17 @@ import ( "sync" "time" - "github.com/davidnewhall/unifi-poller/pkg/influxunifi" "github.com/spf13/pflag" "golift.io/config" "golift.io/unifi" ) -// Version is injected by the Makefile -var Version = "development" - +// App defaults in case they're missing from the config. const ( - // App defaults in case they're missing from the config. - appName = "unifi-poller" - defaultInterval = 30 * time.Second - defaultInfluxDB = "unifi" - defaultInfluxUser = "unifi" - defaultInfluxPass = "unifi" - defaultInfluxURL = "http://127.0.0.1:8086" - defaultUnifiUser = "influx" - defaultUnifiURL = "https://127.0.0.1:8443" - defaultHTTPListen = "0.0.0.0:9130" + // AppName is the name of the application. + AppName = "unifi-poller" + defaultUnifiUser = "influx" + defaultUnifiURL = "https://127.0.0.1:8443" ) // ENVConfigPrefix is the prefix appended to an env variable tag @@ -40,10 +31,8 @@ const ENVConfigPrefix = "UP" // UnifiPoller contains the application startup data, and auth info for UniFi & Influx. type UnifiPoller struct { - Influx *influxunifi.InfluxUnifi Flag *Flag Config *Config - LastCheck time.Time sync.Mutex // locks the Unifi struct member when re-authing to unifi. } @@ -55,6 +44,15 @@ type Flag struct { *pflag.FlagSet } +// Metrics is a type shared by the exporting and reporting packages. +type Metrics struct { + TS time.Time + unifi.Sites + unifi.IDSList + unifi.Clients + *unifi.Devices +} + // Controller represents the configuration for a UniFi Controller. // Each polled controller may have its own configuration. type Controller struct { @@ -73,16 +71,43 @@ type Controller struct { // This is all of the data stored in the config file. // Any with explicit defaults have omitempty on json and toml tags. type Config struct { - Interval config.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"` - Debug bool `json:"debug" toml:"debug" xml:"debug" yaml:"debug"` - Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet" yaml:"quiet"` - InfxBadSSL bool `json:"influx_insecure_ssl" toml:"influx_insecure_ssl" xml:"influx_insecure_ssl" yaml:"influx_insecure_ssl"` - Mode string `json:"mode" toml:"mode" xml:"mode" yaml:"mode"` - HTTPListen string `json:"http_listen" toml:"http_listen" xml:"http_listen" yaml:"http_listen"` - Namespace string `json:"namespace" toml:"namespace" xml:"namespace" yaml:"namespace"` - InfluxURL string `json:"influx_url,omitempty" toml:"influx_url,omitempty" xml:"influx_url" yaml:"influx_url"` - InfluxUser string `json:"influx_user,omitempty" toml:"influx_user,omitempty" xml:"influx_user" yaml:"influx_user"` - InfluxPass string `json:"influx_pass,omitempty" toml:"influx_pass,omitempty" xml:"influx_pass" yaml:"influx_pass"` - InfluxDB string `json:"influx_db,omitempty" toml:"influx_db,omitempty" xml:"influx_db" yaml:"influx_db"` - Controllers []Controller `json:"controller,omitempty" toml:"controller,omitempty" xml:"controller" yaml:"controller"` + Poller `json:"poller" toml:"poller" xml:"poller" yaml:"poller"` + Controllers []Controller `json:"controller,omitempty" toml:"controller,omitempty" xml:"controller" yaml:"controller"` +} + +// Poller is the global config values. +type Poller struct { + Debug bool `json:"debug" toml:"debug" xml:"debug,attr" yaml:"debug"` + Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet,attr" yaml:"quiet"` +} + +// ParseConfigs parses the poller config and the config for each registered output plugin. +func (u *UnifiPoller) ParseConfigs() error { + // Parse config file. + if err := config.ParseFile(u.Config, u.Flag.ConfigFile); err != nil { + u.Flag.Usage() + return err + } + + // Update Config with ENV variable overrides. + if _, err := config.ParseENV(u.Config, ENVConfigPrefix); err != nil { + return err + } + + outputSync.Lock() + defer outputSync.Unlock() + + for _, o := range outputs { + // Parse config file for each output plugin. + if err := config.ParseFile(o.Config, u.Flag.ConfigFile); err != nil { + return err + } + + // Update Config for each output with ENV variable overrides. + if _, err := config.ParseENV(o.Config, ENVConfigPrefix); err != nil { + return err + } + } + + return nil } diff --git a/pkg/poller/influx.go b/pkg/poller/influx.go deleted file mode 100644 index 52690c13..00000000 --- a/pkg/poller/influx.go +++ /dev/null @@ -1,79 +0,0 @@ -package poller - -import ( - "fmt" - "log" - "time" - - "github.com/davidnewhall/unifi-poller/pkg/influxunifi" -) - -// GetInfluxDB returns an InfluxDB interface. -func (u *UnifiPoller) GetInfluxDB() (err error) { - if u.Influx != nil { - return nil - } - - u.Influx, err = influxunifi.New(&influxunifi.Config{ - Database: u.Config.InfluxDB, - User: u.Config.InfluxUser, - Pass: u.Config.InfluxPass, - BadSSL: u.Config.InfxBadSSL, - URL: u.Config.InfluxURL, - }) - if err != nil { - return fmt.Errorf("influxdb: %v", err) - } - - u.Logf("Logging Measurements to InfluxDB at %s as user %s", u.Config.InfluxURL, u.Config.InfluxUser) - - return nil -} - -// PollController runs forever, polling UniFi and pushing to InfluxDB -// This is started by Run() or RunBoth() after everything checks out. -func (u *UnifiPoller) PollController() { - interval := u.Config.Interval.Round(time.Second) - log.Printf("[INFO] Everything checks out! Poller started, InfluxDB interval: %v", interval) - - ticker := time.NewTicker(interval) - for u.LastCheck = range ticker.C { - if err := u.CollectAndProcess(); err != nil { - u.LogErrorf("%v", err) - } - } -} - -// CollectAndProcess collects measurements and then reports them to InfluxDB -// Can be called once or in a ticker loop. This function and all the ones below -// handle their own logging. An error is returned so the calling function may -// determine if there was a read or write error and act on it. This is currently -// called in two places in this library. One returns an error, one does not. -func (u *UnifiPoller) CollectAndProcess() error { - if err := u.GetInfluxDB(); err != nil { - return err - } - - metrics, err := u.CollectMetrics() - if err != nil { - return err - } - - report, err := u.Influx.ReportMetrics(metrics) - if err != nil { - return err - } - - u.LogInfluxReport(report) - return nil -} - -// LogInfluxReport writes a log message after exporting to influxdb. -func (u *UnifiPoller) LogInfluxReport(r *influxunifi.Report) { - idsMsg := fmt.Sprintf("IDS Events: %d, ", len(r.Metrics.IDSList)) - u.Logf("UniFi Metrics Recorded. Sites: %d, Clients: %d, "+ - "UAP: %d, USG/UDM: %d, USW: %d, %sPoints: %d, Fields: %d, Errs: %d, Elapsed: %v", - len(r.Metrics.Sites), len(r.Metrics.Clients), len(r.Metrics.UAPs), - len(r.Metrics.UDMs)+len(r.Metrics.USGs), len(r.Metrics.USWs), idsMsg, r.Total, - r.Fields, len(r.Errors), r.Elapsed.Round(time.Millisecond)) -} diff --git a/pkg/poller/outputs.go b/pkg/poller/outputs.go new file mode 100644 index 00000000..37f8fbb2 --- /dev/null +++ b/pkg/poller/outputs.go @@ -0,0 +1,72 @@ +package poller + +import ( + "fmt" + "sync" +) + +var ( + outputs []*Output + outputSync sync.Mutex +) + +// Collect is passed into output packages so they may collect metrics to output. +// Output packages must implement this interface. +type Collect interface { + Metrics() (*Metrics, error) + Logf(m string, v ...interface{}) + LogErrorf(m string, v ...interface{}) + LogDebugf(m string, v ...interface{}) +} + +// Output defines the output data for a metric exporter like influx or prometheus. +// Output packages should call NewOutput with this struct in init(). +type Output struct { + Name string + Config interface{} // Each config is passed into an unmarshaller later. + Method func(Collect) error // Called on startup for each configured output. +} + +// NewOutput should be called by each output package's init function. +func NewOutput(o *Output) { + outputSync.Lock() + defer outputSync.Unlock() + + if o == nil || o.Method == nil { + panic("nil output or method passed to poller.NewOutput") + } + + outputs = append(outputs, o) +} + +// InitializeOutputs runs all the configured output plugins. +// If none exist, or they all exit an error is returned. +func (u *UnifiPoller) InitializeOutputs() error { + v := make(chan error) + defer close(v) + + var count int + + for _, o := range outputs { + count++ + go func(o *Output) { + v <- o.Method(u) + }(o) + } + + if count < 1 { + return fmt.Errorf("no output plugins configured") + } + + for err := range v { + if err != nil { + return err + } + + if count--; count < 1 { + return fmt.Errorf("all output plugins have stopped") + } + } + + return nil +} diff --git a/pkg/poller/prometheus.go b/pkg/poller/prometheus.go deleted file mode 100644 index 13e4de49..00000000 --- a/pkg/poller/prometheus.go +++ /dev/null @@ -1,53 +0,0 @@ -package poller - -import ( - "net/http" - "strings" - "time" - - "github.com/davidnewhall/unifi-poller/pkg/metrics" - "github.com/davidnewhall/unifi-poller/pkg/promunifi" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/prometheus/common/version" -) - -const oneDecimalPoint = 10 - -// RunPrometheus starts the web server and registers the collector. -func (u *UnifiPoller) RunPrometheus() error { - u.Logf("Exporting Measurements for Prometheus at https://%s/metrics", u.Config.HTTPListen) - http.Handle("/metrics", promhttp.Handler()) - ns := strings.Replace(u.Config.Namespace, "-", "", -1) - prometheus.MustRegister(promunifi.NewUnifiCollector(promunifi.UnifiCollectorCnfg{ - Namespace: ns, - CollectFn: u.ExportMetrics, - LoggingFn: u.LogExportReport, - ReportErrors: true, // XXX: Does this need to be configurable? - })) - - version.Version = Version - prometheus.MustRegister(version.NewCollector(ns)) - - return http.ListenAndServe(u.Config.HTTPListen, nil) -} - -// ExportMetrics updates the internal metrics provided via -// HTTP at /metrics for prometheus collection. -// This is run by Prometheus as CollectFn. -func (u *UnifiPoller) ExportMetrics() (*metrics.Metrics, error) { - return u.CollectMetrics() -} - -// LogExportReport is called after prometheus exports metrics. -// This is run by Prometheus as LoggingFn -func (u *UnifiPoller) LogExportReport(report *promunifi.Report) { - m := report.Metrics - u.Logf("UniFi Measurements Exported. Site: %d, Client: %d, "+ - "UAP: %d, USG/UDM: %d, USW: %d, Descs: %d, "+ - "Metrics: %d, Errs: %d, 0s: %d, Reqs/Total: %v / %v", - len(m.Sites), len(m.Clients), len(m.UAPs), len(m.UDMs)+len(m.USGs), len(m.USWs), - report.Descs, report.Total, report.Errors, report.Zeros, - report.Fetch.Round(time.Millisecond/oneDecimalPoint), - report.Elapsed.Round(time.Millisecond/oneDecimalPoint)) -} diff --git a/pkg/poller/start.go b/pkg/poller/start.go index 2a81ac03..fa4213f5 100644 --- a/pkg/poller/start.go +++ b/pkg/poller/start.go @@ -5,26 +5,16 @@ import ( "fmt" "log" "os" - "strings" - "time" + "github.com/github/hub/version" "github.com/spf13/pflag" - "golift.io/config" ) // New returns a new poller struct preloaded with default values. // No need to call this if you call Start.c func New() *UnifiPoller { return &UnifiPoller{ - Config: &Config{ - InfluxURL: defaultInfluxURL, - InfluxUser: defaultInfluxUser, - InfluxPass: defaultInfluxPass, - InfluxDB: defaultInfluxDB, - Interval: config.Duration{Duration: defaultInterval}, - HTTPListen: defaultHTTPListen, - Namespace: appName, - }, + Config: &Config{}, Flag: &Flag{ ConfigFile: DefaultConfFile, }, @@ -40,7 +30,7 @@ func (u *UnifiPoller) Start() error { u.Flag.Parse(os.Args[1:]) if u.Flag.ShowVer { - fmt.Printf("%s v%s\n", appName, Version) + fmt.Printf("%s v%s\n", AppName, version.Version) return nil // don't run anything else w/ version request. } @@ -48,14 +38,8 @@ func (u *UnifiPoller) Start() error { u.Logf("Loading Configuration File: %s", u.Flag.ConfigFile) } - // Parse config file. - if err := config.ParseFile(u.Config, u.Flag.ConfigFile); err != nil { - u.Flag.Usage() - return err - } - - // Update Config with ENV variable overrides. - if _, err := config.ParseENV(u.Config, ENVConfigPrefix); err != nil { + // Parse config file and ENV variables. + if err := u.ParseConfigs(); err != nil { return err } @@ -78,16 +62,14 @@ func (u *UnifiPoller) Start() error { u.LogDebugf("Debug Logging Enabled") } - log.Printf("[INFO] UniFi Poller v%v Starting Up! PID: %d", Version, os.Getpid()) - return u.Run() } // Parse turns CLI arguments into data structures. Called by Start() on startup. func (f *Flag) Parse(args []string) { - f.FlagSet = pflag.NewFlagSet(appName, pflag.ExitOnError) + f.FlagSet = pflag.NewFlagSet(AppName, pflag.ExitOnError) f.Usage = func() { - fmt.Printf("Usage: %s [--config=/path/to/up.conf] [--version]", appName) + fmt.Printf("Usage: %s [--config=/path/to/up.conf] [--version]", AppName) f.PrintDefaults() } @@ -103,6 +85,8 @@ func (f *Flag) Parse(args []string) { // 2. Run the collector one time and report the metrics to influxdb. (lambda) // 3. Start a web server and wait for Prometheus to poll the application for metrics. func (u *UnifiPoller) Run() error { + log.Printf("[INFO] UniFi Poller v%v Starting Up! PID: %d", version.Version, os.Getpid()) + for i, c := range u.Config.Controllers { if c.Name == "" { u.Config.Controllers[i].Name = c.URL @@ -117,17 +101,5 @@ func (u *UnifiPoller) Run() error { } } - switch strings.ToLower(u.Config.Mode) { - default: - u.PollController() - return nil - case "influxlambda", "lambdainflux", "lambda_influx", "influx_lambda": - u.LastCheck = time.Now() - return u.CollectAndProcess() - case "both": - go u.PollController() - fallthrough - case "prometheus", "exporter": - return u.RunPrometheus() - } + return u.InitializeOutputs() } diff --git a/pkg/poller/unifi.go b/pkg/poller/unifi.go index 0e598716..516310a3 100644 --- a/pkg/poller/unifi.go +++ b/pkg/poller/unifi.go @@ -5,7 +5,6 @@ import ( "strings" "time" - "github.com/davidnewhall/unifi-poller/pkg/metrics" "golift.io/unifi" ) @@ -42,10 +41,6 @@ func (u *UnifiPoller) GetUnifi(c Controller) error { // CheckSites makes sure the list of provided sites exists on the controller. // This does not run in Lambda (run-once) mode. func (u *UnifiPoller) CheckSites(c Controller) error { - if strings.Contains(strings.ToLower(u.Config.Mode), "lambda") { - return nil // Skip this in lambda mode. - } - u.LogDebugf("Checking Controller Sites List") sites, err := c.Unifi.GetSites() @@ -58,6 +53,7 @@ func (u *UnifiPoller) CheckSites(c Controller) error { for _, site := range sites { msg = append(msg, site.Name+" ("+site.Desc+")") } + u.Logf("Found %d site(s) on controller: %v", len(msg), strings.Join(msg, ", ")) if StringInSlice("all", c.Sites) { @@ -78,10 +74,10 @@ FIRST: return nil } -// CollectMetrics grabs all the measurements from a UniFi controller and returns them. -func (u *UnifiPoller) CollectMetrics() (*metrics.Metrics, error) { +// Metrics grabs all the measurements from a UniFi controller and returns them. +func (u *UnifiPoller) Metrics() (*Metrics, error) { errs := []string{} - metrics := &metrics.Metrics{} + metrics := &Metrics{} for _, c := range u.Config.Controllers { m, err := u.checkAndPollController(c) @@ -120,7 +116,7 @@ func (u *UnifiPoller) CollectMetrics() (*metrics.Metrics, error) { return metrics, err } -func (u *UnifiPoller) checkAndPollController(c Controller) (*metrics.Metrics, error) { +func (u *UnifiPoller) checkAndPollController(c Controller) (*Metrics, error) { if c.Unifi == nil { u.Logf("Re-authenticating to UniFi Controller: %s", c.URL) @@ -146,10 +142,10 @@ func (u *UnifiPoller) checkAndPollController(c Controller) (*metrics.Metrics, er return u.collectController(c) } -func (u *UnifiPoller) collectController(c Controller) (*metrics.Metrics, error) { +func (u *UnifiPoller) collectController(c Controller) (*Metrics, error) { var err error - m := &metrics.Metrics{TS: u.LastCheck} // At this point, it's the Current Check. + m := &Metrics{TS: time.Now()} // At this point, it's the Current Check. // Get the sites we care about. if m.Sites, err = u.GetFilteredSites(c); err != nil { @@ -178,7 +174,7 @@ func (u *UnifiPoller) collectController(c Controller) (*metrics.Metrics, error) // augmentMetrics is our middleware layer between collecting metrics and writing them. // This is where we can manipuate the returned data or make arbitrary decisions. // This function currently adds parent device names to client metrics. -func (u *UnifiPoller) augmentMetrics(c Controller, metrics *metrics.Metrics) *metrics.Metrics { +func (u *UnifiPoller) augmentMetrics(c Controller, metrics *Metrics) *Metrics { if metrics == nil || metrics.Devices == nil || metrics.Clients == nil { return metrics } diff --git a/pkg/promunifi/collector.go b/pkg/promunifi/collector.go index 5911d005..6da6db38 100644 --- a/pkg/promunifi/collector.go +++ b/pkg/promunifi/collector.go @@ -1,50 +1,59 @@ -// Package promunifi provides the bridge between unifi metrics and prometheus. +// Package promunifi provides the bridge between unifi-poller metrics and prometheus. package promunifi import ( "fmt" + "net/http" "reflect" "strings" "sync" "time" - "github.com/davidnewhall/unifi-poller/pkg/metrics" + "github.com/davidnewhall/unifi-poller/pkg/poller" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/version" "golift.io/unifi" ) -// channel buffer, fits at least one batch. -const buffer = 50 - -// simply fewer letters. -const counter = prometheus.CounterValue -const gauge = prometheus.GaugeValue - -// UnifiCollectorCnfg defines the data needed to collect and report UniFi Metrics. -type UnifiCollectorCnfg struct { - // If non-empty, each of the collected metrics is prefixed by the - // provided string and an underscore ("_"). - Namespace string - // If true, any error encountered during collection is reported as an - // invalid metric (see NewInvalidMetric). Otherwise, errors are ignored - // and the collected metrics will be incomplete. Possibly, no metrics - // will be collected at all. - ReportErrors bool - // This function is passed to the Collect() method. The Collect method runs - // this function to retrieve the latest UniFi measurements and export them. - CollectFn func() (*metrics.Metrics, error) - // Provide a logger function if you want to run a routine *after* prometheus checks in. - LoggingFn func(*Report) -} +const ( + // channel buffer, fits at least one batch. + buffer = 50 + defaultHTTPListen = "0.0.0.0:9130" + // simply fewer letters. + counter = prometheus.CounterValue + gauge = prometheus.GaugeValue +) type promUnifi struct { - Config UnifiCollectorCnfg + *Prometheus Client *uclient Device *unifiDevice UAP *uap USG *usg USW *usw Site *site + // This interface is passed to the Collect() method. The Collect method uses + // this interface to retrieve the latest UniFi measurements and export them. + Collector poller.Collect +} + +// Prometheus allows the data to be nested in the config file. +type Prometheus struct { + Config Config `json:"prometheus" toml:"prometheus" xml:"prometheus" yaml:"prometheus"` +} + +// Config is the input (config file) data used to initialize this output plugin. +type Config struct { + Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` + // If non-empty, each of the collected metrics is prefixed by the + // provided string and an underscore ("_"). + Namespace string `json:"namespace" toml:"namespace" xml:"namespace" yaml:"namespace"` + // If true, any error encountered during collection is reported as an + // invalid metric (see NewInvalidMetric). Otherwise, errors are ignored + // and the collected metrics will be incomplete. Possibly, no metrics + // will be collected at all. + ReportErrors bool `json:"report_errors" toml:"report_errors" xml:"report_errors" yaml:"report_errors"` + HTTPListen string `json:"http_listen" toml:"http_listen" xml:"http_listen" yaml:"http_listen"` } type metric struct { @@ -54,41 +63,64 @@ type metric struct { Labels []string } -// Report is passed into LoggingFn to log the export metrics to stdout (outside this package). +// Report accumulates counters that are printed to a log line. type Report struct { - Total int // Total count of metrics recorded. - Errors int // Total count of errors recording metrics. - Zeros int // Total count of metrics equal to zero. - Descs int // Total count of unique metrics descriptions. - Metrics *metrics.Metrics // Metrics collected and recorded. - Elapsed time.Duration // Duration elapsed collecting and exporting. - Fetch time.Duration // Duration elapsed making controller requests. - Start time.Time // Time collection began. + Total int // Total count of metrics recorded. + Errors int // Total count of errors recording metrics. + Zeros int // Total count of metrics equal to zero. + Metrics *poller.Metrics // Metrics collected and recorded. + Elapsed time.Duration // Duration elapsed collecting and exporting. + Fetch time.Duration // Duration elapsed making controller requests. + Start time.Time // Time collection began. ch chan []*metric wg sync.WaitGroup - cf UnifiCollectorCnfg + Config } -// NewUnifiCollector returns a prometheus collector that will export any available -// UniFi metrics. You must provide a collection function in the opts. -func NewUnifiCollector(opts UnifiCollectorCnfg) prometheus.Collector { - if opts.CollectFn == nil { - panic("nil collector function") +func init() { + u := &promUnifi{Prometheus: &Prometheus{}} + poller.NewOutput(&poller.Output{ + Name: "prometheus", + Config: u.Prometheus, + Method: u.Run, + }) +} + +// Run creates the collectors and starts the web server up. +// Should be run in a Go routine. Returns nil if not configured. +func (u *promUnifi) Run(c poller.Collect) error { + if u.Config.Disable { + return nil } - if opts.Namespace = strings.Trim(opts.Namespace, "_") + "_"; opts.Namespace == "_" { - opts.Namespace = "" + if u.Config.Namespace == "" { + u.Config.Namespace = strings.Replace(poller.AppName, "-", "", -1) } - return &promUnifi{ - Config: opts, - Client: descClient(opts.Namespace + "client_"), - Device: descDevice(opts.Namespace + "device_"), // stats for all device types. - UAP: descUAP(opts.Namespace + "device_"), - USG: descUSG(opts.Namespace + "device_"), - USW: descUSW(opts.Namespace + "device_"), - Site: descSite(opts.Namespace + "site_"), + if u.Config.HTTPListen == "" { + u.Config.HTTPListen = defaultHTTPListen } + + name := strings.Replace(u.Config.Namespace, "-", "_", -1) + + ns := name + if ns = strings.Trim(ns, "_") + "_"; ns == "_" { + ns = "" + } + + prometheus.MustRegister(version.NewCollector(name)) + prometheus.MustRegister(&promUnifi{ + Collector: c, + Client: descClient(ns + "client_"), + Device: descDevice(ns + "device_"), // stats for all device types. + UAP: descUAP(ns + "device_"), + USG: descUSG(ns + "device_"), + USW: descUSW(ns + "device_"), + Site: descSite(ns + "site_"), + }) + c.Logf("Exporting Measurements for Prometheus at https://%s/metrics, namespace: %s", u.Config.HTTPListen, u.Config.Namespace) + + return http.ListenAndServe(u.Config.HTTPListen, nil) } // Describe satisfies the prometheus Collector. This returns all of the @@ -112,10 +144,10 @@ func (u *promUnifi) Describe(ch chan<- *prometheus.Desc) { func (u *promUnifi) Collect(ch chan<- prometheus.Metric) { var err error - r := &Report{cf: u.Config, ch: make(chan []*metric, buffer), Start: time.Now()} + r := &Report{Config: u.Config, ch: make(chan []*metric, buffer), Start: time.Now()} defer r.close() - if r.Metrics, err = r.cf.CollectFn(); err != nil { + if r.Metrics, err = u.Collector.Metrics(); err != nil { r.error(ch, prometheus.NewInvalidDesc(fmt.Errorf("metric fetch failed")), err) return } @@ -135,7 +167,7 @@ func (u *promUnifi) Collect(ch chan<- prometheus.Metric) { // This is where our channels connects to the prometheus channel. func (u *promUnifi) exportMetrics(r report, ch chan<- prometheus.Metric, ourChan chan []*metric) { descs := make(map[*prometheus.Desc]bool) // used as a counter - defer r.report(descs) + defer r.report(u.Collector, descs) for newMetrics := range ourChan { for _, m := range newMetrics { diff --git a/pkg/promunifi/report.go b/pkg/promunifi/report.go index 0ddf29d3..9b6df74c 100644 --- a/pkg/promunifi/report.go +++ b/pkg/promunifi/report.go @@ -4,7 +4,7 @@ import ( "fmt" "time" - "github.com/davidnewhall/unifi-poller/pkg/metrics" + "github.com/davidnewhall/unifi-poller/pkg/poller" "github.com/prometheus/client_golang/prometheus" ) @@ -16,14 +16,15 @@ type report interface { add() done() send([]*metric) - metrics() *metrics.Metrics - report(descs map[*prometheus.Desc]bool) + metrics() *poller.Metrics + report(c poller.Collect, descs map[*prometheus.Desc]bool) export(m *metric, v float64) prometheus.Metric error(ch chan<- prometheus.Metric, d *prometheus.Desc, v interface{}) } // satisfy gomnd const one = 1 +const oneDecimalPoint = 10.0 func (r *Report) add() { r.wg.Add(one) @@ -38,17 +39,19 @@ func (r *Report) send(m []*metric) { r.ch <- m } -func (r *Report) metrics() *metrics.Metrics { +func (r *Report) metrics() *poller.Metrics { return r.Metrics } -func (r *Report) report(descs map[*prometheus.Desc]bool) { - if r.cf.LoggingFn == nil { - return - } - - r.Descs = len(descs) - r.cf.LoggingFn(r) +func (r *Report) report(c poller.Collect, descs map[*prometheus.Desc]bool) { + m := r.Metrics + c.Logf("UniFi Measurements Exported. Site: %d, Client: %d, "+ + "UAP: %d, USG/UDM: %d, USW: %d, Descs: %d, "+ + "Metrics: %d, Errs: %d, 0s: %d, Reqs/Total: %v / %v", + len(m.Sites), len(m.Clients), len(m.UAPs), len(m.UDMs)+len(m.USGs), len(m.USWs), + len(descs), r.Total, r.Errors, r.Zeros, + r.Fetch.Round(time.Millisecond/oneDecimalPoint), + r.Elapsed.Round(time.Millisecond/oneDecimalPoint)) } func (r *Report) export(m *metric, v float64) prometheus.Metric { @@ -64,7 +67,7 @@ func (r *Report) export(m *metric, v float64) prometheus.Metric { func (r *Report) error(ch chan<- prometheus.Metric, d *prometheus.Desc, v interface{}) { r.Errors++ - if r.cf.ReportErrors { + if r.Config.ReportErrors { ch <- prometheus.NewInvalidMetric(d, fmt.Errorf("error: %v", v)) } }