public_git
/
unpoller_unpoller
mirror of https://github.com/unpoller/unpoller.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

make output plugins call in to initialize

This commit is contained in:
davidnewhall2 2019-12-15 02:52:43 -08:00
parent 60d645c1a7
commit ac39d1727f
19 changed files with 461 additions and 446 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.metadata.sh
View File

@ -32,11 +32,6 @@ SOURCE_URL="https://${IMPORT_PATH}"
# Used for documentation links.
URL="${SOURCE_URL}"
# This parameter is passed in as -X to go build. Used to override the Version variable in a package.
# This makes a path like github.com/user/hello-world/helloworld.Version=1.3.3
# Name the Version-containing library the same as the github repo, without dashes.
VERSION_PATH="${IMPORT_PATH}/pkg/poller.Version"
# Dynamic. Recommend not changing.
VVERSION=$(git describe --abbrev=0 --tags $(git rev-list --tags --max-count=1))
VERSION="$(echo $VVERSION | tr -d v | grep -E '^\S+$' || echo development)"
@ -51,4 +46,4 @@ COMMIT="$(git rev-parse --short HEAD || echo 0)"
# This is a custom download path for homebrew formula.
SOURCE_PATH=https://golift.io/${BINARY}/archive/v${VERSION}.tar.gz
export IMPORT_PATH SOURCE_URL URL VERSION_PATH VVERSION VERSION ITERATION DATE BRANCH COMMIT SOURCE_PATH
export IMPORT_PATH SOURCE_URL URL VVERSION VERSION ITERATION DATE BRANCH COMMIT SOURCE_PATH

2
Makefile
View File

@ -46,7 +46,7 @@ VERSION_LDFLAGS:= \
-X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Branch=$(BRANCH) \
-X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.BuildDate=$(DATE) \
-X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Revision=$(COMMIT) \
-X $(VERSION_PATH)=$(VERSION)-$(ITERATION)
-X $(IMPORT_PATH)/vendor/github.com/prometheus/common/version.Version=$(VERSION)-$(ITERATION)
# Makefile targets follow.

59
examples/MANUAL.md
View File

@ -65,10 +65,10 @@ is provided so the application can be easily adapted to any environment.
`Config File Parameters`
interval default: 30s
How often to poll the controller for updated client and device data.
The UniFi Controller only updates traffic stats about every 30-60 seconds.
Only works if "mode" (below) is "influx" - other modes do not use interval.
Additional parameters are added by output packages. Parameters can also be set
using environment variables. See the GitHub wiki for more information!
>>> POLLER FIELDS FOLLOW - you may have multiple controllers:
debug default: false
This turns on time stamps and line numbers in logs, outputs a few extra
@ -79,56 +79,7 @@ is provided so the application can be easily adapted to any environment.
errors will be logged. Using this with debug=true adds line numbers to
any error logs.
mode default: "influx"
* Value: influx
This default mode runs this application as a daemon. It will poll
the controller at the configured interval and report measurements to
InfluxDB. Providing an invalid value will run in this default mode.
* Value: influxlambda
Setting this value will invoke a run-once mode where the application
immediately polls the controller and reports the metrics to InfluxDB.
Then it exits. This mode is useful in an AWS Lambda or a crontab where
the execution timings are controlled. This mode may also be adapted
to run in other collector scripts and apps like telegraf or diamond.
This mode can also be combined with a "test database" in InfluxDB to
give yourself a "test config file" you may run ad-hoc to test changes.
* Value: prometheus
In this mode the application opens an http interface and exports the
measurements at /metrics for collection by prometheus. Enabling this
mode disables InfluxDB usage entirely.
* Value: both
Setting the mode to "both" will cause the InfluxDB poller routine to run
along with the Prometheus exporter. You can run both at the same time.
http_listen default: 0.0.0.0:9130
This option controls the IP and port the http listener uses when the
mode is set to prometheus. This setting has no effect when other modes
are in use. Metrics become available at the /metrics URI.
influx_url default: http://127.0.0.1:8086
This is the URL where the Influx web server is available.
influx_user default: unifi
Username used to authenticate with InfluxDB.
influx_pass default: unifi
Password used to authenticate with InfluxDB.
influx_db default: unifi
Custom database created in InfluxDB to use with this application.
On first setup, log into InfluxDB and create access:
$ influx -host localhost -port 8086
CREATE DATABASE unifi
CREATE USER unifi WITH PASSWORD 'unifi' WITH ALL PRIVILEGES
GRANT ALL ON unifi TO unifi
influx_insecure_ssl default: false
Setting this to true will allow use of InfluxDB with an invalid SSL certificate.
>>> CONTROLLER FIELDS FOLLOW - you may have multiple controllers:
>>> CONTROLLER FIELDS FOLLOW - you may have multiple controllers:
sites default: ["all"]
This list of strings should represent the names of sites on the UniFi

48
examples/up.conf.example
View File

@ -3,11 +3,7 @@
########################################################
# The UniFi Controller only updates traffic stats about every 30 seconds.
# Setting this to something lower may lead to "zeros" in your data.
# If you're getting zeros now, set this to "1m"
interval = "30s"
[poller]
# Turns on line numbers, microsecond logging, and a per-device log.
# The default is false, but I personally leave this on at home (four devices).
# This may be noisy if you have a lot of devices. It adds one line per device.
@ -17,38 +13,34 @@ debug = false
# Recommend enabling debug with this setting for better error logging.
quiet = false
# Which mode to run this application in. The default mode is "influx". Providing
# an invalid mode will also result in "influx". In this default mode the application
# runs as a daemon and polls the controller at the configured interval.
#
# Other options: "influxlambda", "prometheus", "both"
#
# Mode "influxlambda" makes the application exit after collecting and reporting metrics
# to InfluxDB one time. This mode requires an external process like an AWS Lambda
# or a simple crontab to keep the timings accurate on UniFi Poller run intervals.
#
# Mode "prometheus" opens an HTTP server on port 9130 and exports the metrics at
# /metrics for polling collection by a prometheus server. This disables influxdb.
#
# Mode "both" runs the Prometheus HTTP server and InfluxDB poller interval at
# the same time.
mode = "influx"
#### OUTPUTS
[prometheus]
disable = false
# This controls on which ip and port /metrics is exported when mode is "prometheus".
# This has no effect in other modes. Must contain a colon and port.
http_listen = "0.0.0.0:9130"
report_errors = false
[influxdb]
disable = false
# InfluxDB does not require auth by default, so the user/password are probably unimportant.
influx_url = "http://127.0.0.1:8086"
influx_user = "unifi"
influx_pass = "unifi"
url = "http://127.0.0.1:8086"
user = "unifi"
pass = "unifi"
# Be sure to create this database.
influx_db = "unifi"
# If your InfluxDB uses an invalid SSL cert, set this to true.
influx_insecure_ssl = false
db = "unifi"
# If your InfluxDB uses a valid SSL cert, set this to true.
verify_ssl = false
# The UniFi Controller only updates traffic stats about every 30 seconds.
# Setting this to something lower may lead to "zeros" in your data.
# If you're getting zeros now, set this to "1m"
interval = "30s"
#### INPUTS
# You may repeat the following section to poll additional controllers.
[[controller]]
# Friendly name used in dashboards.
name = ""

49
examples/up.json.example
View File

@ -1,22 +1,33 @@
{
"interval": "30s",
"debug": false,
"quiet": false,
"mode": "influx",
"http_listen": "0.0.0.0:9130",
"influx_url": "http://127.0.0.1:8086",
"influx_user": "unifi",
"influx_pass": "unifi",
"influx_db": "unifi",
"influx_insecure_ssl": false,
"controller": [{
"name": "",
"user": "influx",
"pass": "",
"url": "https://127.0.0.1:8443",
"sites": ["all"],
"save_ids": false,
"save_sites": true,
"verify_ssl": false
"poller": {
"debug": false,
"quiet": false
},
"prometheus": {
"disable": false,
"http_listen": "0.0.0.0:9130",
"report_errors": false
},
"influxdb": {
"disable": false,
"url": "http://127.0.0.1:8086",
"user": "unifi",
"pass": "unifi",
"db": "unifi",
"verify_ssl": false,
"interval": "30s"
},
"controller": [{
"name": "",
"user": "influx",
"pass": "",
"url": "https://127.0.0.1:8443",
"sites": ["all"],
"save_ids": false,
"save_sites": true,
"verify_ssl": false
}]
}

29
examples/up.xml.example
View File

@ -5,21 +5,23 @@
# provided values are defaults. See up.conf.example! #
#######################################################
-->
<unifi-poller>
<poller debug="false" quiet="false">
<interval>60s</interval>
<prometheus disable="false">
<http_listen>0.0.0.0:9130</http_listen>
<report_errors>false</report_errors>
</prometheus>
<debug>false</debug>
<quiet>false</quiet>
<influxdb disable="false">
<interval>30s</interval>
<url>http://127.0.0.1:8086</url>
<user>unifi</user>
<pass>unifi</pass>
<db>unifi</db>
<verify_ssl>false</verify_ssl>
</influxdb>
<mode>influx</mode>
<http_listen>0.0.0.0:9130</http_listen>
<influx_db>unifi</influx_db>
<influx_pass>unifi</influx_pass>
<influx_url>http://127.0.0.1:8086</influx_url>
<influx_user>unifi</influx_user>
<influx_insecure_ssl>false</influx_insecure_ssl>
<!-- Repeat this stanza to poll additional controllers. -->
<controller name="">
<sites>all</sites>
<user>influx</user>
@ -29,4 +31,5 @@
<save_ids>false</save_ids>
<save_sites>true</save_sites>
</controller>
</unifi-poller>
</poller>

25
examples/up.yaml.example
View File

@ -3,19 +3,24 @@
# provided values are defaults. See up.conf.example! #
########################################################
---
interval: "30s"
debug: false
quiet: false
poller:
debug: false
quiet: false
mode: "influx"
http_listen: "0.0.0.0:9130"
prometheus:
disable: false
http_listen: "0.0.0.0:9130"
report_errors: false
influx_url: "http://127.0.0.1:8086"
influx_user: "unifi"
influx_pass: "unifi"
influx_db: "unifi"
influx_insecure_ssl: false
influxdb:
disable: false
interval: "30s"
url: "http://127.0.0.1:8086"
user: "unifi"
pass: "unifi"
db: "unifi"
verify_ssl: false
controller:
- name: ""

3
main.go
View File

@ -4,6 +4,9 @@ import (
"log"
"github.com/davidnewhall/unifi-poller/pkg/poller"
// Enable output plugins!
_ "github.com/davidnewhall/unifi-poller/pkg/influxunifi"
_ "github.com/davidnewhall/unifi-poller/pkg/promunifi"
)
// Keep it simple.

141
pkg/influxunifi/metrics.go
View File

@ -1,29 +1,47 @@
// Package influx provides the methods to turn UniFi measurements into influx
// Package influxunifi provides the methods to turn UniFi measurements into influx
// data-points with appropriate tags and fields.
package influxunifi
import (
"crypto/tls"
"fmt"
"log"
"time"
"github.com/davidnewhall/unifi-poller/pkg/metrics"
"github.com/davidnewhall/unifi-poller/pkg/poller"
influx "github.com/influxdata/influxdb1-client/v2"
conf "golift.io/config"
)
const (
defaultInterval = 30 * time.Second
defaultInfluxDB = "unifi"
defaultInfluxUser = "unifi"
defaultInfluxURL = "http://127.0.0.1:8086"
)
// Config defines the data needed to store metrics in InfluxDB
type Config struct {
Database string
URL string
User string
Pass string
BadSSL bool
Interval conf.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"`
Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"`
VerifySSL bool `json:"verify_ssl" toml:"verify_ssl" xml:"verify_ssl" yaml:"verify_ssl"`
URL string `json:"url,omitempty" toml:"url,omitempty" xml:"url" yaml:"url"`
User string `json:"user,omitempty" toml:"user,omitempty" xml:"user" yaml:"user"`
Pass string `json:"pass,omitempty" toml:"pass,omitempty" xml:"pass" yaml:"pass"`
DB string `json:"db,omitempty" toml:"db,omitempty" xml:"db" yaml:"db"`
}
// InfluxDB allows the data to be nested in the config file.
type InfluxDB struct {
Config Config `json:"influxdb" toml:"influxdb" xml:"influxdb" yaml:"influxdb"`
}
// InfluxUnifi is returned by New() after you provide a Config.
type InfluxUnifi struct {
cf *Config
influx influx.Client
Collector poller.Collect
influx influx.Client
LastCheck time.Time
*InfluxDB
}
type metric struct {
@ -32,26 +50,101 @@ type metric struct {
Fields map[string]interface{}
}
// New returns an InfluxDB interface.
func New(c *Config) (*InfluxUnifi, error) {
i, err := influx.NewHTTPClient(influx.HTTPConfig{
Addr: c.URL,
Username: c.User,
Password: c.Pass,
TLSConfig: &tls.Config{InsecureSkipVerify: c.BadSSL},
func init() {
u := &InfluxUnifi{InfluxDB: &InfluxDB{}, LastCheck: time.Now()}
poller.NewOutput(&poller.Output{
Name: "influxdb",
Config: u.InfluxDB,
Method: u.Run,
})
return &InfluxUnifi{cf: c, influx: i}, err
}
// PollController runs forever, polling UniFi and pushing to InfluxDB
// This is started by Run() or RunBoth() after everything checks out.
func (u *InfluxUnifi) PollController() {
interval := u.Config.Interval.Round(time.Second)
log.Printf("[INFO] Everything checks out! Poller started, InfluxDB interval: %v", interval)
ticker := time.NewTicker(interval)
for u.LastCheck = range ticker.C {
metrics, err := u.Collector.Metrics()
if err != nil {
u.Collector.LogErrorf("%v", err)
continue
}
report, err := u.ReportMetrics(metrics)
if err != nil {
// XXX: reset and re-auth? not sure..
u.Collector.LogErrorf("%v", err)
continue
}
u.LogInfluxReport(report)
}
}
// Run runs a ticker to poll the unifi server and update influxdb.
func (u *InfluxUnifi) Run(c poller.Collect) error {
var err error
if u.Config.Disable {
return nil
}
u.Collector = c
u.setConfigDefaults()
u.influx, err = influx.NewHTTPClient(influx.HTTPConfig{
Addr: u.Config.URL,
Username: u.Config.User,
Password: u.Config.Pass,
TLSConfig: &tls.Config{InsecureSkipVerify: !u.Config.VerifySSL},
})
if err != nil {
return err
}
u.PollController()
return nil
}
func (u *InfluxUnifi) setConfigDefaults() {
if u.Config.URL == "" {
u.Config.URL = defaultInfluxURL
}
if u.Config.User == "" {
u.Config.User = defaultInfluxUser
}
if u.Config.Pass == "" {
u.Config.Pass = defaultInfluxUser
}
if u.Config.DB == "" {
u.Config.DB = defaultInfluxDB
}
if u.Config.Interval.Duration == 0 {
u.Config.Interval = conf.Duration{Duration: defaultInterval}
} else if u.Config.Interval.Duration < defaultInterval/2 {
u.Config.Interval = conf.Duration{Duration: defaultInterval / 2}
}
u.Config.Interval = conf.Duration{Duration: u.Config.Interval.Duration.Round(time.Second)}
}
// ReportMetrics batches all device and client data into influxdb data points.
// Call this after you've collected all the data you care about.
// Returns an error if influxdb calls fail, otherwise returns a report.
func (u *InfluxUnifi) ReportMetrics(m *metrics.Metrics) (*Report, error) {
func (u *InfluxUnifi) ReportMetrics(m *poller.Metrics) (*Report, error) {
r := &Report{Metrics: m, ch: make(chan *metric), Start: time.Now()}
defer close(r.ch)
// Make a new Influx Points Batcher.
var err error
r.bp, err = influx.NewBatchPoints(influx.BatchPointsConfig{Database: u.cf.Database})
r.bp, err = influx.NewBatchPoints(influx.BatchPointsConfig{Database: u.Config.DB})
if err != nil {
return nil, fmt.Errorf("influx.NewBatchPoints: %v", err)
}
@ -140,3 +233,13 @@ func (u *InfluxUnifi) loopPoints(r report) {
}
}()
}
// LogInfluxReport writes a log message after exporting to influxdb.
func (u *InfluxUnifi) LogInfluxReport(r *Report) {
idsMsg := fmt.Sprintf("IDS Events: %d, ", len(r.Metrics.IDSList))
u.Collector.Logf("UniFi Metrics Recorded. Sites: %d, Clients: %d, "+
"UAP: %d, USG/UDM: %d, USW: %d, %sPoints: %d, Fields: %d, Errs: %d, Elapsed: %v",
len(r.Metrics.Sites), len(r.Metrics.Clients), len(r.Metrics.UAPs),
len(r.Metrics.UDMs)+len(r.Metrics.USGs), len(r.Metrics.USWs), idsMsg, r.Total,
r.Fields, len(r.Errors), r.Elapsed.Round(time.Millisecond))
}

8
pkg/influxunifi/report.go
View File

@ -4,13 +4,13 @@ import (
"sync"
"time"
"github.com/davidnewhall/unifi-poller/pkg/metrics"
"github.com/davidnewhall/unifi-poller/pkg/poller"
influx "github.com/influxdata/influxdb1-client/v2"
)
// Report is returned to the calling procedure after everything is processed.
type Report struct {
Metrics *metrics.Metrics
Metrics *poller.Metrics
Errors []error
Total int
Fields int
@ -28,10 +28,10 @@ type report interface {
send(m *metric)
error(err error)
batch(m *metric, pt *influx.Point)
metrics() *metrics.Metrics
metrics() *poller.Metrics
}
func (r *Report) metrics() *metrics.Metrics {
func (r *Report) metrics() *poller.Metrics {
return r.Metrics
}

16
pkg/metrics/metrics.go
View File

@ -1,16 +0,0 @@
package metrics
import (
"time"
"golift.io/unifi"
)
// Metrics is a type shared by the exporting and reporting packages.
type Metrics struct {
TS time.Time
unifi.Sites
unifi.IDSList
unifi.Clients
*unifi.Devices
}

81
pkg/poller/config.go
View File

@ -12,26 +12,17 @@ import (
"sync"
"time"
"github.com/davidnewhall/unifi-poller/pkg/influxunifi"
"github.com/spf13/pflag"
"golift.io/config"
"golift.io/unifi"
)
// Version is injected by the Makefile
var Version = "development"
// App defaults in case they're missing from the config.
const (
// App defaults in case they're missing from the config.
appName = "unifi-poller"
defaultInterval = 30 * time.Second
defaultInfluxDB = "unifi"
defaultInfluxUser = "unifi"
defaultInfluxPass = "unifi"
defaultInfluxURL = "http://127.0.0.1:8086"
defaultUnifiUser = "influx"
defaultUnifiURL = "https://127.0.0.1:8443"
defaultHTTPListen = "0.0.0.0:9130"
// AppName is the name of the application.
AppName = "unifi-poller"
defaultUnifiUser = "influx"
defaultUnifiURL = "https://127.0.0.1:8443"
)
// ENVConfigPrefix is the prefix appended to an env variable tag
@ -40,10 +31,8 @@ const ENVConfigPrefix = "UP"
// UnifiPoller contains the application startup data, and auth info for UniFi & Influx.
type UnifiPoller struct {
Influx *influxunifi.InfluxUnifi
Flag *Flag
Config *Config
LastCheck time.Time
sync.Mutex // locks the Unifi struct member when re-authing to unifi.
}
@ -55,6 +44,15 @@ type Flag struct {
*pflag.FlagSet
}
// Metrics is a type shared by the exporting and reporting packages.
type Metrics struct {
TS time.Time
unifi.Sites
unifi.IDSList
unifi.Clients
*unifi.Devices
}
// Controller represents the configuration for a UniFi Controller.
// Each polled controller may have its own configuration.
type Controller struct {
@ -73,16 +71,43 @@ type Controller struct {
// This is all of the data stored in the config file.
// Any with explicit defaults have omitempty on json and toml tags.
type Config struct {
Interval config.Duration `json:"interval,omitempty" toml:"interval,omitempty" xml:"interval" yaml:"interval"`
Debug bool `json:"debug" toml:"debug" xml:"debug" yaml:"debug"`
Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet" yaml:"quiet"`
InfxBadSSL bool `json:"influx_insecure_ssl" toml:"influx_insecure_ssl" xml:"influx_insecure_ssl" yaml:"influx_insecure_ssl"`
Mode string `json:"mode" toml:"mode" xml:"mode" yaml:"mode"`
HTTPListen string `json:"http_listen" toml:"http_listen" xml:"http_listen" yaml:"http_listen"`
Namespace string `json:"namespace" toml:"namespace" xml:"namespace" yaml:"namespace"`
InfluxURL string `json:"influx_url,omitempty" toml:"influx_url,omitempty" xml:"influx_url" yaml:"influx_url"`
InfluxUser string `json:"influx_user,omitempty" toml:"influx_user,omitempty" xml:"influx_user" yaml:"influx_user"`
InfluxPass string `json:"influx_pass,omitempty" toml:"influx_pass,omitempty" xml:"influx_pass" yaml:"influx_pass"`
InfluxDB string `json:"influx_db,omitempty" toml:"influx_db,omitempty" xml:"influx_db" yaml:"influx_db"`
Controllers []Controller `json:"controller,omitempty" toml:"controller,omitempty" xml:"controller" yaml:"controller"`
Poller `json:"poller" toml:"poller" xml:"poller" yaml:"poller"`
Controllers []Controller `json:"controller,omitempty" toml:"controller,omitempty" xml:"controller" yaml:"controller"`
}
// Poller is the global config values.
type Poller struct {
Debug bool `json:"debug" toml:"debug" xml:"debug,attr" yaml:"debug"`
Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet,attr" yaml:"quiet"`
}
// ParseConfigs parses the poller config and the config for each registered output plugin.
func (u *UnifiPoller) ParseConfigs() error {
// Parse config file.
if err := config.ParseFile(u.Config, u.Flag.ConfigFile); err != nil {
u.Flag.Usage()
return err
}
// Update Config with ENV variable overrides.
if _, err := config.ParseENV(u.Config, ENVConfigPrefix); err != nil {
return err
}
outputSync.Lock()
defer outputSync.Unlock()
for _, o := range outputs {
// Parse config file for each output plugin.
if err := config.ParseFile(o.Config, u.Flag.ConfigFile); err != nil {
return err
}
// Update Config for each output with ENV variable overrides.
if _, err := config.ParseENV(o.Config, ENVConfigPrefix); err != nil {
return err
}
}
return nil
}

79
pkg/poller/influx.go
View File

@ -1,79 +0,0 @@
package poller
import (
"fmt"
"log"
"time"
"github.com/davidnewhall/unifi-poller/pkg/influxunifi"
)
// GetInfluxDB returns an InfluxDB interface.
func (u *UnifiPoller) GetInfluxDB() (err error) {
if u.Influx != nil {
return nil
}
u.Influx, err = influxunifi.New(&influxunifi.Config{
Database: u.Config.InfluxDB,
User: u.Config.InfluxUser,
Pass: u.Config.InfluxPass,
BadSSL: u.Config.InfxBadSSL,
URL: u.Config.InfluxURL,
})
if err != nil {
return fmt.Errorf("influxdb: %v", err)
}
u.Logf("Logging Measurements to InfluxDB at %s as user %s", u.Config.InfluxURL, u.Config.InfluxUser)
return nil
}
// PollController runs forever, polling UniFi and pushing to InfluxDB
// This is started by Run() or RunBoth() after everything checks out.
func (u *UnifiPoller) PollController() {
interval := u.Config.Interval.Round(time.Second)
log.Printf("[INFO] Everything checks out! Poller started, InfluxDB interval: %v", interval)
ticker := time.NewTicker(interval)
for u.LastCheck = range ticker.C {
if err := u.CollectAndProcess(); err != nil {
u.LogErrorf("%v", err)
}
}
}
// CollectAndProcess collects measurements and then reports them to InfluxDB
// Can be called once or in a ticker loop. This function and all the ones below
// handle their own logging. An error is returned so the calling function may
// determine if there was a read or write error and act on it. This is currently
// called in two places in this library. One returns an error, one does not.
func (u *UnifiPoller) CollectAndProcess() error {
if err := u.GetInfluxDB(); err != nil {
return err
}
metrics, err := u.CollectMetrics()
if err != nil {
return err
}
report, err := u.Influx.ReportMetrics(metrics)
if err != nil {
return err
}
u.LogInfluxReport(report)
return nil
}
// LogInfluxReport writes a log message after exporting to influxdb.
func (u *UnifiPoller) LogInfluxReport(r *influxunifi.Report) {
idsMsg := fmt.Sprintf("IDS Events: %d, ", len(r.Metrics.IDSList))
u.Logf("UniFi Metrics Recorded. Sites: %d, Clients: %d, "+
"UAP: %d, USG/UDM: %d, USW: %d, %sPoints: %d, Fields: %d, Errs: %d, Elapsed: %v",
len(r.Metrics.Sites), len(r.Metrics.Clients), len(r.Metrics.UAPs),
len(r.Metrics.UDMs)+len(r.Metrics.USGs), len(r.Metrics.USWs), idsMsg, r.Total,
r.Fields, len(r.Errors), r.Elapsed.Round(time.Millisecond))
}

72
pkg/poller/outputs.go Normal file
View File

@ -0,0 +1,72 @@
package poller
import (
"fmt"
"sync"
)
var (
outputs []*Output
outputSync sync.Mutex
)
// Collect is passed into output packages so they may collect metrics to output.
// Output packages must implement this interface.
type Collect interface {
Metrics() (*Metrics, error)
Logf(m string, v ...interface{})
LogErrorf(m string, v ...interface{})
LogDebugf(m string, v ...interface{})
}
// Output defines the output data for a metric exporter like influx or prometheus.
// Output packages should call NewOutput with this struct in init().
type Output struct {
Name string
Config interface{} // Each config is passed into an unmarshaller later.
Method func(Collect) error // Called on startup for each configured output.
}
// NewOutput should be called by each output package's init function.
func NewOutput(o *Output) {
outputSync.Lock()
defer outputSync.Unlock()
if o == nil || o.Method == nil {
panic("nil output or method passed to poller.NewOutput")
}
outputs = append(outputs, o)
}
// InitializeOutputs runs all the configured output plugins.
// If none exist, or they all exit an error is returned.
func (u *UnifiPoller) InitializeOutputs() error {
v := make(chan error)
defer close(v)
var count int
for _, o := range outputs {
count++
go func(o *Output) {
v <- o.Method(u)
}(o)
}
if count < 1 {
return fmt.Errorf("no output plugins configured")
}
for err := range v {
if err != nil {
return err
}
if count--; count < 1 {
return fmt.Errorf("all output plugins have stopped")
}
}
return nil
}

53
pkg/poller/prometheus.go
View File

@ -1,53 +0,0 @@
package poller
import (
"net/http"
"strings"
"time"
"github.com/davidnewhall/unifi-poller/pkg/metrics"
"github.com/davidnewhall/unifi-poller/pkg/promunifi"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/prometheus/common/version"
)
const oneDecimalPoint = 10
// RunPrometheus starts the web server and registers the collector.
func (u *UnifiPoller) RunPrometheus() error {
u.Logf("Exporting Measurements for Prometheus at https://%s/metrics", u.Config.HTTPListen)
http.Handle("/metrics", promhttp.Handler())
ns := strings.Replace(u.Config.Namespace, "-", "", -1)
prometheus.MustRegister(promunifi.NewUnifiCollector(promunifi.UnifiCollectorCnfg{
Namespace: ns,
CollectFn: u.ExportMetrics,
LoggingFn: u.LogExportReport,
ReportErrors: true, // XXX: Does this need to be configurable?
}))
version.Version = Version
prometheus.MustRegister(version.NewCollector(ns))
return http.ListenAndServe(u.Config.HTTPListen, nil)
}
// ExportMetrics updates the internal metrics provided via
// HTTP at /metrics for prometheus collection.
// This is run by Prometheus as CollectFn.
func (u *UnifiPoller) ExportMetrics() (*metrics.Metrics, error) {
return u.CollectMetrics()
}
// LogExportReport is called after prometheus exports metrics.
// This is run by Prometheus as LoggingFn
func (u *UnifiPoller) LogExportReport(report *promunifi.Report) {
m := report.Metrics
u.Logf("UniFi Measurements Exported. Site: %d, Client: %d, "+
"UAP: %d, USG/UDM: %d, USW: %d, Descs: %d, "+
"Metrics: %d, Errs: %d, 0s: %d, Reqs/Total: %v / %v",
len(m.Sites), len(m.Clients), len(m.UAPs), len(m.UDMs)+len(m.USGs), len(m.USWs),
report.Descs, report.Total, report.Errors, report.Zeros,
report.Fetch.Round(time.Millisecond/oneDecimalPoint),
report.Elapsed.Round(time.Millisecond/oneDecimalPoint))
}

48
pkg/poller/start.go
View File

@ -5,26 +5,16 @@ import (
"fmt"
"log"
"os"
"strings"
"time"
"github.com/github/hub/version"
"github.com/spf13/pflag"
"golift.io/config"
)
// New returns a new poller struct preloaded with default values.
// No need to call this if you call Start.c
func New() *UnifiPoller {
return &UnifiPoller{
Config: &Config{
InfluxURL: defaultInfluxURL,
InfluxUser: defaultInfluxUser,
InfluxPass: defaultInfluxPass,
InfluxDB: defaultInfluxDB,
Interval: config.Duration{Duration: defaultInterval},
HTTPListen: defaultHTTPListen,
Namespace: appName,
},
Config: &Config{},
Flag: &Flag{
ConfigFile: DefaultConfFile,
},
@ -40,7 +30,7 @@ func (u *UnifiPoller) Start() error {
u.Flag.Parse(os.Args[1:])
if u.Flag.ShowVer {
fmt.Printf("%s v%s\n", appName, Version)
fmt.Printf("%s v%s\n", AppName, version.Version)
return nil // don't run anything else w/ version request.
}
@ -48,14 +38,8 @@ func (u *UnifiPoller) Start() error {
u.Logf("Loading Configuration File: %s", u.Flag.ConfigFile)
}
// Parse config file.
if err := config.ParseFile(u.Config, u.Flag.ConfigFile); err != nil {
u.Flag.Usage()
return err
}
// Update Config with ENV variable overrides.
if _, err := config.ParseENV(u.Config, ENVConfigPrefix); err != nil {
// Parse config file and ENV variables.
if err := u.ParseConfigs(); err != nil {
return err
}
@ -78,16 +62,14 @@ func (u *UnifiPoller) Start() error {
u.LogDebugf("Debug Logging Enabled")
}
log.Printf("[INFO] UniFi Poller v%v Starting Up! PID: %d", Version, os.Getpid())
return u.Run()
}
// Parse turns CLI arguments into data structures. Called by Start() on startup.
func (f *Flag) Parse(args []string) {
f.FlagSet = pflag.NewFlagSet(appName, pflag.ExitOnError)
f.FlagSet = pflag.NewFlagSet(AppName, pflag.ExitOnError)
f.Usage = func() {
fmt.Printf("Usage: %s [--config=/path/to/up.conf] [--version]", appName)
fmt.Printf("Usage: %s [--config=/path/to/up.conf] [--version]", AppName)
f.PrintDefaults()
}
@ -103,6 +85,8 @@ func (f *Flag) Parse(args []string) {
// 2. Run the collector one time and report the metrics to influxdb. (lambda)
// 3. Start a web server and wait for Prometheus to poll the application for metrics.
func (u *UnifiPoller) Run() error {
log.Printf("[INFO] UniFi Poller v%v Starting Up! PID: %d", version.Version, os.Getpid())
for i, c := range u.Config.Controllers {
if c.Name == "" {
u.Config.Controllers[i].Name = c.URL
@ -117,17 +101,5 @@ func (u *UnifiPoller) Run() error {
}
}
switch strings.ToLower(u.Config.Mode) {
default:
u.PollController()
return nil
case "influxlambda", "lambdainflux", "lambda_influx", "influx_lambda":
u.LastCheck = time.Now()
return u.CollectAndProcess()
case "both":
go u.PollController()
fallthrough
case "prometheus", "exporter":
return u.RunPrometheus()
}
return u.InitializeOutputs()
}

20
pkg/poller/unifi.go
View File

@ -5,7 +5,6 @@ import (
"strings"
"time"
"github.com/davidnewhall/unifi-poller/pkg/metrics"
"golift.io/unifi"
)
@ -42,10 +41,6 @@ func (u *UnifiPoller) GetUnifi(c Controller) error {
// CheckSites makes sure the list of provided sites exists on the controller.
// This does not run in Lambda (run-once) mode.
func (u *UnifiPoller) CheckSites(c Controller) error {
if strings.Contains(strings.ToLower(u.Config.Mode), "lambda") {
return nil // Skip this in lambda mode.
}
u.LogDebugf("Checking Controller Sites List")
sites, err := c.Unifi.GetSites()
@ -58,6 +53,7 @@ func (u *UnifiPoller) CheckSites(c Controller) error {
for _, site := range sites {
msg = append(msg, site.Name+" ("+site.Desc+")")
}
u.Logf("Found %d site(s) on controller: %v", len(msg), strings.Join(msg, ", "))
if StringInSlice("all", c.Sites) {
@ -78,10 +74,10 @@ FIRST:
return nil
}
// CollectMetrics grabs all the measurements from a UniFi controller and returns them.
func (u *UnifiPoller) CollectMetrics() (*metrics.Metrics, error) {
// Metrics grabs all the measurements from a UniFi controller and returns them.
func (u *UnifiPoller) Metrics() (*Metrics, error) {
errs := []string{}
metrics := &metrics.Metrics{}
metrics := &Metrics{}
for _, c := range u.Config.Controllers {
m, err := u.checkAndPollController(c)
@ -120,7 +116,7 @@ func (u *UnifiPoller) CollectMetrics() (*metrics.Metrics, error) {
return metrics, err
}
func (u *UnifiPoller) checkAndPollController(c Controller) (*metrics.Metrics, error) {
func (u *UnifiPoller) checkAndPollController(c Controller) (*Metrics, error) {
if c.Unifi == nil {
u.Logf("Re-authenticating to UniFi Controller: %s", c.URL)
@ -146,10 +142,10 @@ func (u *UnifiPoller) checkAndPollController(c Controller) (*metrics.Metrics, er
return u.collectController(c)
}
func (u *UnifiPoller) collectController(c Controller) (*metrics.Metrics, error) {
func (u *UnifiPoller) collectController(c Controller) (*Metrics, error) {
var err error
m := &metrics.Metrics{TS: u.LastCheck} // At this point, it's the Current Check.
m := &Metrics{TS: time.Now()} // At this point, it's the Current Check.
// Get the sites we care about.
if m.Sites, err = u.GetFilteredSites(c); err != nil {
@ -178,7 +174,7 @@ func (u *UnifiPoller) collectController(c Controller) (*metrics.Metrics, error)
// augmentMetrics is our middleware layer between collecting metrics and writing them.
// This is where we can manipuate the returned data or make arbitrary decisions.
// This function currently adds parent device names to client metrics.
func (u *UnifiPoller) augmentMetrics(c Controller, metrics *metrics.Metrics) *metrics.Metrics {
func (u *UnifiPoller) augmentMetrics(c Controller, metrics *Metrics) *Metrics {
if metrics == nil || metrics.Devices == nil || metrics.Clients == nil {
return metrics
}

140
pkg/promunifi/collector.go
View File

@ -1,50 +1,59 @@
// Package promunifi provides the bridge between unifi metrics and prometheus.
// Package promunifi provides the bridge between unifi-poller metrics and prometheus.
package promunifi
import (
"fmt"
"net/http"
"reflect"
"strings"
"sync"
"time"
"github.com/davidnewhall/unifi-poller/pkg/metrics"
"github.com/davidnewhall/unifi-poller/pkg/poller"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/version"
"golift.io/unifi"
)
// channel buffer, fits at least one batch.
const buffer = 50
// simply fewer letters.
const counter = prometheus.CounterValue
const gauge = prometheus.GaugeValue
// UnifiCollectorCnfg defines the data needed to collect and report UniFi Metrics.
type UnifiCollectorCnfg struct {
// If non-empty, each of the collected metrics is prefixed by the
// provided string and an underscore ("_").
Namespace string
// If true, any error encountered during collection is reported as an
// invalid metric (see NewInvalidMetric). Otherwise, errors are ignored
// and the collected metrics will be incomplete. Possibly, no metrics
// will be collected at all.
ReportErrors bool
// This function is passed to the Collect() method. The Collect method runs
// this function to retrieve the latest UniFi measurements and export them.
CollectFn func() (*metrics.Metrics, error)
// Provide a logger function if you want to run a routine *after* prometheus checks in.
LoggingFn func(*Report)
}
const (
// channel buffer, fits at least one batch.
buffer = 50
defaultHTTPListen = "0.0.0.0:9130"
// simply fewer letters.
counter = prometheus.CounterValue
gauge = prometheus.GaugeValue
)
type promUnifi struct {
Config UnifiCollectorCnfg
*Prometheus
Client *uclient
Device *unifiDevice
UAP *uap
USG *usg
USW *usw
Site *site
// This interface is passed to the Collect() method. The Collect method uses
// this interface to retrieve the latest UniFi measurements and export them.
Collector poller.Collect
}
// Prometheus allows the data to be nested in the config file.
type Prometheus struct {
Config Config `json:"prometheus" toml:"prometheus" xml:"prometheus" yaml:"prometheus"`
}
// Config is the input (config file) data used to initialize this output plugin.
type Config struct {
Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"`
// If non-empty, each of the collected metrics is prefixed by the
// provided string and an underscore ("_").
Namespace string `json:"namespace" toml:"namespace" xml:"namespace" yaml:"namespace"`
// If true, any error encountered during collection is reported as an
// invalid metric (see NewInvalidMetric). Otherwise, errors are ignored
// and the collected metrics will be incomplete. Possibly, no metrics
// will be collected at all.
ReportErrors bool `json:"report_errors" toml:"report_errors" xml:"report_errors" yaml:"report_errors"`
HTTPListen string `json:"http_listen" toml:"http_listen" xml:"http_listen" yaml:"http_listen"`
}
type metric struct {
@ -54,41 +63,64 @@ type metric struct {
Labels []string
}
// Report is passed into LoggingFn to log the export metrics to stdout (outside this package).
// Report accumulates counters that are printed to a log line.
type Report struct {
Total int // Total count of metrics recorded.
Errors int // Total count of errors recording metrics.
Zeros int // Total count of metrics equal to zero.
Descs int // Total count of unique metrics descriptions.
Metrics *metrics.Metrics // Metrics collected and recorded.
Elapsed time.Duration // Duration elapsed collecting and exporting.
Fetch time.Duration // Duration elapsed making controller requests.
Start time.Time // Time collection began.
Total int // Total count of metrics recorded.
Errors int // Total count of errors recording metrics.
Zeros int // Total count of metrics equal to zero.
Metrics *poller.Metrics // Metrics collected and recorded.
Elapsed time.Duration // Duration elapsed collecting and exporting.
Fetch time.Duration // Duration elapsed making controller requests.
Start time.Time // Time collection began.
ch chan []*metric
wg sync.WaitGroup
cf UnifiCollectorCnfg
Config
}
// NewUnifiCollector returns a prometheus collector that will export any available
// UniFi metrics. You must provide a collection function in the opts.
func NewUnifiCollector(opts UnifiCollectorCnfg) prometheus.Collector {
if opts.CollectFn == nil {
panic("nil collector function")
func init() {
u := &promUnifi{Prometheus: &Prometheus{}}
poller.NewOutput(&poller.Output{
Name: "prometheus",
Config: u.Prometheus,
Method: u.Run,
})
}
// Run creates the collectors and starts the web server up.
// Should be run in a Go routine. Returns nil if not configured.
func (u *promUnifi) Run(c poller.Collect) error {
if u.Config.Disable {
return nil
}
if opts.Namespace = strings.Trim(opts.Namespace, "_") + "_"; opts.Namespace == "_" {
opts.Namespace = ""
if u.Config.Namespace == "" {
u.Config.Namespace = strings.Replace(poller.AppName, "-", "", -1)
}
return &promUnifi{
Config: opts,
Client: descClient(opts.Namespace + "client_"),
Device: descDevice(opts.Namespace + "device_"), // stats for all device types.
UAP: descUAP(opts.Namespace + "device_"),
USG: descUSG(opts.Namespace + "device_"),
USW: descUSW(opts.Namespace + "device_"),
Site: descSite(opts.Namespace + "site_"),
if u.Config.HTTPListen == "" {
u.Config.HTTPListen = defaultHTTPListen
}
name := strings.Replace(u.Config.Namespace, "-", "_", -1)
ns := name
if ns = strings.Trim(ns, "_") + "_"; ns == "_" {
ns = ""
}
prometheus.MustRegister(version.NewCollector(name))
prometheus.MustRegister(&promUnifi{
Collector: c,
Client: descClient(ns + "client_"),
Device: descDevice(ns + "device_"), // stats for all device types.
UAP: descUAP(ns + "device_"),
USG: descUSG(ns + "device_"),
USW: descUSW(ns + "device_"),
Site: descSite(ns + "site_"),
})
c.Logf("Exporting Measurements for Prometheus at https://%s/metrics, namespace: %s", u.Config.HTTPListen, u.Config.Namespace)
return http.ListenAndServe(u.Config.HTTPListen, nil)
}
// Describe satisfies the prometheus Collector. This returns all of the
@ -112,10 +144,10 @@ func (u *promUnifi) Describe(ch chan<- *prometheus.Desc) {
func (u *promUnifi) Collect(ch chan<- prometheus.Metric) {
var err error
r := &Report{cf: u.Config, ch: make(chan []*metric, buffer), Start: time.Now()}
r := &Report{Config: u.Config, ch: make(chan []*metric, buffer), Start: time.Now()}
defer r.close()
if r.Metrics, err = r.cf.CollectFn(); err != nil {
if r.Metrics, err = u.Collector.Metrics(); err != nil {
r.error(ch, prometheus.NewInvalidDesc(fmt.Errorf("metric fetch failed")), err)
return
}
@ -135,7 +167,7 @@ func (u *promUnifi) Collect(ch chan<- prometheus.Metric) {
// This is where our channels connects to the prometheus channel.
func (u *promUnifi) exportMetrics(r report, ch chan<- prometheus.Metric, ourChan chan []*metric) {
descs := make(map[*prometheus.Desc]bool) // used as a counter
defer r.report(descs)
defer r.report(u.Collector, descs)
for newMetrics := range ourChan {
for _, m := range newMetrics {

27
pkg/promunifi/report.go
View File

@ -4,7 +4,7 @@ import (
"fmt"
"time"
"github.com/davidnewhall/unifi-poller/pkg/metrics"
"github.com/davidnewhall/unifi-poller/pkg/poller"
"github.com/prometheus/client_golang/prometheus"
)
@ -16,14 +16,15 @@ type report interface {
add()
done()
send([]*metric)
metrics() *metrics.Metrics
report(descs map[*prometheus.Desc]bool)
metrics() *poller.Metrics
report(c poller.Collect, descs map[*prometheus.Desc]bool)
export(m *metric, v float64) prometheus.Metric
error(ch chan<- prometheus.Metric, d *prometheus.Desc, v interface{})
}
// satisfy gomnd
const one = 1
const oneDecimalPoint = 10.0
func (r *Report) add() {
r.wg.Add(one)
@ -38,17 +39,19 @@ func (r *Report) send(m []*metric) {
r.ch <- m
}
func (r *Report) metrics() *metrics.Metrics {
func (r *Report) metrics() *poller.Metrics {
return r.Metrics
}
func (r *Report) report(descs map[*prometheus.Desc]bool) {
if r.cf.LoggingFn == nil {
return
}
r.Descs = len(descs)
r.cf.LoggingFn(r)
func (r *Report) report(c poller.Collect, descs map[*prometheus.Desc]bool) {
m := r.Metrics
c.Logf("UniFi Measurements Exported. Site: %d, Client: %d, "+
"UAP: %d, USG/UDM: %d, USW: %d, Descs: %d, "+
"Metrics: %d, Errs: %d, 0s: %d, Reqs/Total: %v / %v",
len(m.Sites), len(m.Clients), len(m.UAPs), len(m.UDMs)+len(m.USGs), len(m.USWs),
len(descs), r.Total, r.Errors, r.Zeros,
r.Fetch.Round(time.Millisecond/oneDecimalPoint),
r.Elapsed.Round(time.Millisecond/oneDecimalPoint))
}
func (r *Report) export(m *metric, v float64) prometheus.Metric {
@ -64,7 +67,7 @@ func (r *Report) export(m *metric, v float64) prometheus.Metric {
func (r *Report) error(ch chan<- prometheus.Metric, d *prometheus.Desc, v interface{}) {
r.Errors++
if r.cf.ReportErrors {
if r.Config.ReportErrors {
ch <- prometheus.NewInvalidMetric(d, fmt.Errorf("error: %v", v))
}
}