From 44c544d8e152c89ed1a24c455badd64b395c263f Mon Sep 17 00:00:00 2001 From: davidnewhall2 Date: Mon, 16 Dec 2019 03:11:40 -0800 Subject: [PATCH] Add dynamic plugin support --- .gitignore | 1 + Makefile | 13 +++++- examples/MANUAL.md | 6 +-- examples/up.conf.example | 9 +++- examples/up.json.example | 28 +++++++----- examples/up.xml.example | 26 ++++++----- examples/up.yaml.example | 22 +++++----- pkg/inputunifi/input.go | 4 +- pkg/inputunifi/interface.go | 4 ++ pkg/poller/build_macos.go | 3 ++ pkg/poller/build_unix.go | 3 ++ pkg/poller/build_windows.go | 3 ++ pkg/poller/config.go | 88 +++++++++++++++++++++++++++---------- plugins/mysql/README.md | 26 +++++++++++ plugins/mysql/main.go | 45 +++++++++++++++++++ 15 files changed, 217 insertions(+), 64 deletions(-) create mode 100644 plugins/mysql/README.md create mode 100644 plugins/mysql/main.go diff --git a/.gitignore b/.gitignore index 118e4d74..0ba43855 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ bitly_token github_deploy_key gpg.signing.key .secret-files.tar +*.so diff --git a/Makefile b/Makefile index 4cf96dc5..772d143f 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ IGNORED:=$(shell bash -c "source .metadata.sh ; env | sed 's/=/:=/;s/^/export /' # md2roff turns markdown into man files and html files. MD2ROFF_BIN=github.com/github/hub/md2roff-bin + # Travis CI passes the version in. Local builds get it from the current git tag. ifeq ($(VERSION),) include .metadata.make @@ -185,10 +186,11 @@ $(BINARY)_$(VERSION)-$(ITERATION)_armhf.deb: package_build_linux_armhf check_fpm # Build an environment that can be packaged for linux. package_build_linux: readme man linux # Building package environment for linux. - mkdir -p $@/usr/bin $@/etc/$(BINARY) $@/usr/share/man/man1 $@/usr/share/doc/$(BINARY) + mkdir -p $@/usr/bin $@/etc/$(BINARY) $@/usr/share/man/man1 $@/usr/share/doc/$(BINARY) $@/usr/lib/$(BINARY) # Copying the binary, config file, unit file, and man page into the env. cp $(BINARY).amd64.linux $@/usr/bin/$(BINARY) cp *.1.gz $@/usr/share/man/man1 + cp *.so $@/usr/lib/$(BINARY)/ cp examples/$(CONFIG_FILE).example $@/etc/$(BINARY)/ cp examples/$(CONFIG_FILE).example $@/etc/$(BINARY)/$(CONFIG_FILE) cp LICENSE *.html examples/*?.?* $@/usr/share/doc/$(BINARY)/ @@ -253,6 +255,12 @@ $(BINARY).rb: v$(VERSION).tar.gz.sha256 init/homebrew/$(FORMULA).rb.tmpl init/homebrew/$(FORMULA).rb.tmpl | tee $(BINARY).rb # That perl line turns hello-world into HelloWorld, etc. +# This is kind janky because it always builds the plugins, even if they are already built. +# Still needs to be made multi arch, which adds complications, especially when creating packages. +plugins: $(patsubst %.go,%.so,$(wildcard ./plugins/*/main.go)) +$(patsubst %.go,%.so,$(wildcard ./plugins/*/main.go)): + go build -o $(patsubst plugins/%/main.so,%.so,$@) -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./$(patsubst %main.so,%,$@) + # Extras # Run code tests and lint. @@ -285,8 +293,9 @@ install: man readme $(BINARY) @[ "$(PREFIX)" != "" ] || (echo "Unable to continue, PREFIX not set. Use: make install PREFIX=/usr/local ETC=/usr/local/etc" && false) @[ "$(ETC)" != "" ] || (echo "Unable to continue, ETC not set. Use: make install PREFIX=/usr/local ETC=/usr/local/etc" && false) # Copying the binary, config file, unit file, and man page into the env. - /usr/bin/install -m 0755 -d $(PREFIX)/bin $(PREFIX)/share/man/man1 $(ETC)/$(BINARY) $(PREFIX)/share/doc/$(BINARY) + /usr/bin/install -m 0755 -d $(PREFIX)/bin $(PREFIX)/share/man/man1 $(ETC)/$(BINARY) $(PREFIX)/share/doc/$(BINARY) $(PREFIX)/lib/$(BINARY) /usr/bin/install -m 0755 -cp $(BINARY) $(PREFIX)/bin/$(BINARY) + /usr/bin/install -m 0755 -cp *.so $(PREFIX)/lib/$(BINARY)/ /usr/bin/install -m 0644 -cp $(BINARY).1.gz $(PREFIX)/share/man/man1 /usr/bin/install -m 0644 -cp examples/$(CONFIG_FILE).example $(ETC)/$(BINARY)/ [ -f $(ETC)/$(BINARY)/$(CONFIG_FILE) ] || /usr/bin/install -m 0644 -cp examples/$(CONFIG_FILE).example $(ETC)/$(BINARY)/$(CONFIG_FILE) diff --git a/examples/MANUAL.md b/examples/MANUAL.md index 012cd343..cabf4db8 100644 --- a/examples/MANUAL.md +++ b/examples/MANUAL.md @@ -66,7 +66,7 @@ is provided so the application can be easily adapted to any environment. `Config File Parameters` Additional parameters are added by output packages. Parameters can also be set -using environment variables. See the GitHub wiki for more information! +using environment variables. See the GitHub wiki for more information! >>> POLLER FIELDS FOLLOW - you may have multiple controllers: @@ -79,7 +79,7 @@ using environment variables. See the GitHub wiki for more information! errors will be logged. Using this with debug=true adds line numbers to any error logs. - >>> CONTROLLER FIELDS FOLLOW - you may have multiple controllers: + >>> UNIFI CONTROLLER FIELDS FOLLOW - you may have multiple controllers: sites default: ["all"] This list of strings should represent the names of sites on the UniFi @@ -96,7 +96,7 @@ using environment variables. See the GitHub wiki for more information! Username used to authenticate with UniFi controller. This should be a special service account created on the control with read-only access. - user no default + pass no default Password used to authenticate with UniFi controller. This can also be set in an environment variable instead of a configuration file. diff --git a/examples/up.conf.example b/examples/up.conf.example index 75c12941..43aa9831 100644 --- a/examples/up.conf.example +++ b/examples/up.conf.example @@ -13,6 +13,8 @@ debug = false # Recommend enabling debug with this setting for better error logging. quiet = false +# Load dynamic plugins. Advanced use; only sample mysql plugin provided by default. +plugins = [] #### OUTPUTS @@ -40,9 +42,12 @@ interval = "30s" #### INPUTS +[unifi] +disable = false + # You may repeat the following section to poll additional controllers. -[[controller]] -# Friendly name used in dashboards. +[[unifi.controller]] +# Friendly name used in dashboards. Uses URL if left empty. name = "" url = "https://127.0.0.1:8443" diff --git a/examples/up.json.example b/examples/up.json.example index 65e4d27e..12ba4a4f 100644 --- a/examples/up.json.example +++ b/examples/up.json.example @@ -1,7 +1,8 @@ { "poller": { "debug": false, - "quiet": false + "quiet": false, + "plugins": [] }, "prometheus": { @@ -20,14 +21,19 @@ "interval": "30s" }, - "controller": [{ - "name": "", - "user": "influx", - "pass": "", - "url": "https://127.0.0.1:8443", - "sites": ["all"], - "save_ids": false, - "save_sites": true, - "verify_ssl": false - }] + "unifi": { + "disable": false, + "controllers": [ + { + "name": "", + "user": "influx", + "pass": "", + "url": "https://127.0.0.1:8443", + "sites": ["all"], + "save_ids": false, + "save_sites": true, + "verify_ssl": false + } + ] + } } diff --git a/examples/up.xml.example b/examples/up.xml.example index 710c01ba..ec98169d 100644 --- a/examples/up.xml.example +++ b/examples/up.xml.example @@ -4,8 +4,11 @@ # UniFi Poller primary configuration file. XML FORMAT # # provided values are defaults. See up.conf.example! # ####################################################### + + and are lists of strings and may be repeated. --> + 0.0.0.0:9130 @@ -21,15 +24,16 @@ false - - - all - influx - - https://127.0.0.1:8443 - false - false - true - - + + + + all + influx + + https://127.0.0.1:8443 + false + false + true + + diff --git a/examples/up.yaml.example b/examples/up.yaml.example index 611c5fa8..2be57012 100644 --- a/examples/up.yaml.example +++ b/examples/up.yaml.example @@ -7,6 +7,7 @@ poller: debug: false quiet: false + plugins: [] prometheus: disable: false @@ -22,13 +23,14 @@ influxdb: db: "unifi" verify_ssl: false -controller: - - name: "" - user: "influx" - pass: "" - url: "https://127.0.0.1:8443" - sites: - - all - verify_ssl: false - save_ids: false - save_sites: true +unifi: + controllers: + - name: "" + user: "influx" + pass: "" + url: "https://127.0.0.1:8443" + sites: + - all + verify_ssl: false + save_ids: false + save_sites: true diff --git a/pkg/inputunifi/input.go b/pkg/inputunifi/input.go index f797d35b..15a027dd 100644 --- a/pkg/inputunifi/input.go +++ b/pkg/inputunifi/input.go @@ -27,7 +27,7 @@ type Controller struct { User string `json:"user" toml:"user" xml:"user" yaml:"user"` Pass string `json:"pass" toml:"pass" xml:"pass" yaml:"pass"` URL string `json:"url" toml:"url" xml:"url" yaml:"url"` - Sites []string `json:"sites,omitempty" toml:"sites,omitempty" xml:"sites" yaml:"sites"` + Sites []string `json:"sites,omitempty" toml:"sites,omitempty" xml:"site" yaml:"sites"` Unifi *unifi.Unifi `json:"-" toml:"-" xml:"-" yaml:"-"` } @@ -35,7 +35,7 @@ type Controller struct { type Config struct { sync.RWMutex // locks the Unifi struct member when re-authing to unifi. Disable bool `json:"disable" toml:"disable" xml:"disable" yaml:"disable"` - Controllers []Controller `json:"controller" toml:"controller" xml:"controller" yaml:"controller"` + Controllers []Controller `json:"controllers" toml:"controller" xml:"controller" yaml:"controllers"` } func init() { diff --git a/pkg/inputunifi/interface.go b/pkg/inputunifi/interface.go index a2ef9765..1db32906 100644 --- a/pkg/inputunifi/interface.go +++ b/pkg/inputunifi/interface.go @@ -10,6 +10,10 @@ import ( // Metrics grabs all the measurements from a UniFi controller and returns them. func (u *InputUnifi) Metrics() (*poller.Metrics, error) { + if u.Config.Disable { + return nil, nil + } + errs := []string{} metrics := &poller.Metrics{} diff --git a/pkg/poller/build_macos.go b/pkg/poller/build_macos.go index b3f37dbf..9292f409 100644 --- a/pkg/poller/build_macos.go +++ b/pkg/poller/build_macos.go @@ -4,3 +4,6 @@ package poller // DefaultConfFile is where to find config if --config is not prvided. const DefaultConfFile = "/usr/local/etc/unifi-poller/up.conf" + +// DefaultObjPath is the path to look for shared object libraries (plugins). +const DefaultObjPath = "/usr/local/lib/unifi-poller" diff --git a/pkg/poller/build_unix.go b/pkg/poller/build_unix.go index c1f525a9..fd381e19 100644 --- a/pkg/poller/build_unix.go +++ b/pkg/poller/build_unix.go @@ -4,3 +4,6 @@ package poller // DefaultConfFile is where to find config if --config is not prvided. const DefaultConfFile = "/etc/unifi-poller/up.conf" + +// DefaultObjPath is the path to look for shared object libraries (plugins). +const DefaultObjPath = "/usr/lib/unifi-poller" diff --git a/pkg/poller/build_windows.go b/pkg/poller/build_windows.go index a74c76a8..69d964e8 100644 --- a/pkg/poller/build_windows.go +++ b/pkg/poller/build_windows.go @@ -4,3 +4,6 @@ package poller // DefaultConfFile is where to find config if --config is not prvided. const DefaultConfFile = `C:\ProgramData\unifi-poller\up.conf` + +// DefaultObjPath is useless in this context. Bummer. +const DefaultObjPath = "PLUGINS_DO_NOT_WORK_ON_WINDOWS_SOWWWWWY" diff --git a/pkg/poller/config.go b/pkg/poller/config.go index 03e69b81..a43caceb 100644 --- a/pkg/poller/config.go +++ b/pkg/poller/config.go @@ -9,6 +9,10 @@ package poller */ import ( + "os" + "path" + "plugin" + "strings" "time" "github.com/spf13/pflag" @@ -53,33 +57,24 @@ type Config struct { // Poller is the global config values. type Poller struct { - Debug bool `json:"debug" toml:"debug" xml:"debug,attr" yaml:"debug"` - Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet,attr" yaml:"quiet"` + Plugins []string `json:"plugins" toml:"plugins" xml:"plugin" yaml:"plugins"` + Debug bool `json:"debug" toml:"debug" xml:"debug,attr" yaml:"debug"` + Quiet bool `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet,attr" yaml:"quiet"` } -// ParseConfigs parses the poller config and the config for each registered output plugin. -func (u *UnifiPoller) ParseConfigs() error { - // Parse core config. - if err := u.ParseInterface(u.Config); err != nil { - return err - } +// LoadPlugins reads-in dynamic shared libraries. +// Not used very often, if at all. +func (u *UnifiPoller) LoadPlugins() error { + for _, p := range u.Plugins { + name := strings.TrimSuffix(p, ".so") + ".so" - // Parse output plugin configs. - outputSync.Lock() - defer outputSync.Unlock() - - for _, o := range outputs { - if err := u.ParseInterface(o.Config); err != nil { - return err + if _, err := os.Stat(name); os.IsNotExist(err) { + name = path.Join(DefaultObjPath, name) } - } - // Parse input plugin configs. - inputSync.Lock() - defer inputSync.Unlock() + u.Logf("Loading Dynamic Plugin: %s", name) - for _, i := range inputs { - if err := u.ParseInterface(i.Config); err != nil { + if _, err := plugin.Open(name); err != nil { return err } } @@ -87,8 +82,27 @@ func (u *UnifiPoller) ParseConfigs() error { return nil } -// ParseInterface parses the config file and environment variables into the provided interface. -func (u *UnifiPoller) ParseInterface(i interface{}) error { +// ParseConfigs parses the poller config and the config for each registered output plugin. +func (u *UnifiPoller) ParseConfigs() error { + // Parse core config. + if err := u.parseInterface(u.Config); err != nil { + return err + } + + // Load dynamic plugins. + if err := u.LoadPlugins(); err != nil { + return err + } + + if err := u.parseInputs(); err != nil { + return err + } + + return u.parseOutputs() +} + +// parseInterface parses the config file and environment variables into the provided interface. +func (u *UnifiPoller) parseInterface(i interface{}) error { // Parse config file into provided interface. if err := config.ParseFile(i, u.Flags.ConfigFile); err != nil { return err @@ -99,3 +113,31 @@ func (u *UnifiPoller) ParseInterface(i interface{}) error { return err } + +// Parse input plugin configs. +func (u *UnifiPoller) parseInputs() error { + inputSync.Lock() + defer inputSync.Unlock() + + for _, i := range inputs { + if err := u.parseInterface(i.Config); err != nil { + return err + } + } + + return nil +} + +// Parse output plugin configs. +func (u *UnifiPoller) parseOutputs() error { + outputSync.Lock() + defer outputSync.Unlock() + + for _, o := range outputs { + if err := u.parseInterface(o.Config); err != nil { + return err + } + } + + return nil +} diff --git a/plugins/mysql/README.md b/plugins/mysql/README.md new file mode 100644 index 00000000..9c32f7fe --- /dev/null +++ b/plugins/mysql/README.md @@ -0,0 +1,26 @@ +# MYSQL Output Plugin Example + +The code here, and the dynamic plugin provided shows an example of how you can +write your own output for unifi-poller. This plugin records some very basic +data about clients on a unifi network into a mysql database. + +You could write outputs that do... anything. An example: They could compare current +connected clients to a previous list (in a db, or stored in memory), and send a +notification if it changes. The possibilities are endless. + +You must compile your plugin using the unifi-poller source for the version you're +using. In other words, to build a plugin for version 2.0.1, do this: +``` +mkdir -p $GOPATH/src/github.com/davidnewhall +cd $GOPATH/src/github.com/davidnewhall + +git clone git@github.com:davidnewhall/unifi-poller.git +cd unifi-poller + +git checkout v2.0.1 +make vendor + +cp -r plugins/ +GOOS=linux make plugins +``` +The plugin you copy in *must* have a `main.go` file for `make plugins` to build it. diff --git a/plugins/mysql/main.go b/plugins/mysql/main.go new file mode 100644 index 00000000..e0c866b9 --- /dev/null +++ b/plugins/mysql/main.go @@ -0,0 +1,45 @@ +package main + +import ( + "fmt" + + "github.com/davidnewhall/unifi-poller/pkg/poller" + "golift.io/config" +) + +// mysqlConfig represents the data that is unmarshalled from the up.conf config file for this plugins. +type mysqlConfig struct { + Interval config.Duration `json:"interval" toml:"interval" xml:"interval" yaml:"interval"` + Host string `json:"host" toml:"host" xml:"host" yaml:"host"` + User string `json:"user" toml:"user" xml:"user" yaml:"user"` + Pass string `json:"pass" toml:"pass" xml:"pass" yaml:"pass"` + DB string `json:"db" toml:"db" xml:"db" yaml:"db"` + Table string `json:"table" toml:"table" xml:"table" yaml:"table"` + // Maps do not work with ENV VARIABLES yet, but may in the future. + Fields []string `json:"fields" toml:"fields" xml:"field" yaml:"fields"` +} + +// Pointers are ignored during ENV variable unmarshal, avoid pointers to your config. +// Only capital (exported) members are unmarshaled when passed into poller.NewOutput(). +type application struct { + Config mysqlConfig `json:"mysql" toml:"mysql" xml:"mysql" yaml:"mysql"` +} + +func init() { + u := &application{Config: mysqlConfig{}} + + poller.NewOutput(&poller.Output{ + Name: "mysql", + Config: u, // pass in the struct *above* your config (so it can see the struct tags). + Method: u.Run, + }) +} + +func main() { + fmt.Println("this is a unifi-poller plugin; not an application") +} + +func (a *application) Run(c poller.Collect) error { + c.Logf("mysql plugin is not finished") + return nil +}