Add dynamic plugin support

.gitignore

@@ -27,3 +27,4 @@ bitly_token
 github_deploy_key
 gpg.signing.key
 .secret-files.tar
+*.so

Makefile

@@ -8,6 +8,7 @@ IGNORED:=$(shell bash -c "source .metadata.sh ; env | sed 's/=/:=/;s/^/export /'
 # md2roff turns markdown into man files and html files.
 MD2ROFF_BIN=github.com/github/hub/md2roff-bin
 
+
 # Travis CI passes the version in. Local builds get it from the current git tag.
 ifeq ($(VERSION),)
 	include .metadata.make
@@ -185,10 +186,11 @@ $(BINARY)_$(VERSION)-$(ITERATION)_armhf.deb: package_build_linux_armhf check_fpm
 # Build an environment that can be packaged for linux.
 package_build_linux: readme man linux
 	# Building package environment for linux.
-	mkdir -p $@/usr/bin $@/etc/$(BINARY) $@/usr/share/man/man1 $@/usr/share/doc/$(BINARY)
+	mkdir -p $@/usr/bin $@/etc/$(BINARY) $@/usr/share/man/man1 $@/usr/share/doc/$(BINARY) $@/usr/lib/$(BINARY)
 	# Copying the binary, config file, unit file, and man page into the env.
 	cp $(BINARY).amd64.linux $@/usr/bin/$(BINARY)
 	cp *.1.gz $@/usr/share/man/man1
+	cp *.so $@/usr/lib/$(BINARY)/
 	cp examples/$(CONFIG_FILE).example $@/etc/$(BINARY)/
 	cp examples/$(CONFIG_FILE).example $@/etc/$(BINARY)/$(CONFIG_FILE)
 	cp LICENSE *.html examples/*?.?* $@/usr/share/doc/$(BINARY)/
@@ -253,6 +255,12 @@ $(BINARY).rb: v$(VERSION).tar.gz.sha256 init/homebrew/$(FORMULA).rb.tmpl
 		init/homebrew/$(FORMULA).rb.tmpl | tee $(BINARY).rb
 		# That perl line turns hello-world into HelloWorld, etc.
 
+# This is kind janky because it always builds the plugins, even if they are already built.
+# Still needs to be made multi arch, which adds complications, especially when creating packages.
+plugins: $(patsubst %.go,%.so,$(wildcard ./plugins/*/main.go))
+$(patsubst %.go,%.so,$(wildcard ./plugins/*/main.go)):
+	go build -o $(patsubst plugins/%/main.so,%.so,$@) -ldflags "$(VERSION_LDFLAGS)" -buildmode=plugin ./$(patsubst %main.so,%,$@)
+
 # Extras
 
 # Run code tests and lint.
@@ -285,8 +293,9 @@ install: man readme $(BINARY)
 	@[ "$(PREFIX)" != "" ] || (echo "Unable to continue, PREFIX not set. Use: make install PREFIX=/usr/local ETC=/usr/local/etc" && false)
 	@[ "$(ETC)" != "" ] || (echo "Unable to continue, ETC not set. Use: make install PREFIX=/usr/local ETC=/usr/local/etc" && false)
 	# Copying the binary, config file, unit file, and man page into the env.
-	/usr/bin/install -m 0755 -d $(PREFIX)/bin $(PREFIX)/share/man/man1 $(ETC)/$(BINARY) $(PREFIX)/share/doc/$(BINARY)
+	/usr/bin/install -m 0755 -d $(PREFIX)/bin $(PREFIX)/share/man/man1 $(ETC)/$(BINARY) $(PREFIX)/share/doc/$(BINARY) $(PREFIX)/lib/$(BINARY)
 	/usr/bin/install -m 0755 -cp $(BINARY) $(PREFIX)/bin/$(BINARY)
+	/usr/bin/install -m 0755 -cp *.so $(PREFIX)/lib/$(BINARY)/
 	/usr/bin/install -m 0644 -cp $(BINARY).1.gz $(PREFIX)/share/man/man1
 	/usr/bin/install -m 0644 -cp examples/$(CONFIG_FILE).example $(ETC)/$(BINARY)/
 	[ -f $(ETC)/$(BINARY)/$(CONFIG_FILE) ] || /usr/bin/install -m 0644 -cp  examples/$(CONFIG_FILE).example $(ETC)/$(BINARY)/$(CONFIG_FILE)

examples/MANUAL.md

@@ -79,7 +79,7 @@ using environment variables. See the GitHub wiki for more information!
         errors will be logged. Using this with debug=true adds line numbers to
         any error logs.
 
-    >>> CONTROLLER FIELDS FOLLOW - you may have multiple controllers:
+    >>> UNIFI CONTROLLER FIELDS FOLLOW - you may have multiple controllers:
 
     sites                  default: ["all"]
         This list of strings should represent the names of sites on the UniFi
@@ -96,7 +96,7 @@ using environment variables. See the GitHub wiki for more information!
         Username used to authenticate with UniFi controller. This should be a
         special service account created on the control with read-only access.
 
-    user                   no default
+    pass                   no default
         Password used to authenticate with UniFi controller. This can also be
         set in an environment variable instead of a configuration file.
 

examples/up.conf.example

@@ -13,6 +13,8 @@ debug = false
 # Recommend enabling debug with this setting for better error logging.
 quiet = false
 
+# Load dynamic plugins. Advanced use; only sample mysql plugin provided by default.
+plugins = []
 
 #### OUTPUTS
 
@@ -40,9 +42,12 @@ interval = "30s"
 
 #### INPUTS
 
+[unifi]
+disable = false
+
 # You may repeat the following section to poll additional controllers.
-[[controller]]
+[[unifi.controller]]
-# Friendly name used in dashboards.
+# Friendly name used in dashboards. Uses URL if left empty.
 name = ""
 
 url = "https://127.0.0.1:8443"

examples/up.json.example

@@ -1,7 +1,8 @@
 {
   "poller": {
     "debug": false,
-    "quiet": false
+    "quiet": false,
+    "plugins": []
   },
 
   "prometheus": {
@@ -20,7 +21,10 @@
      "interval": "30s"
   },
 
-  "controller": [{
+  "unifi": {
+    "disable": false,
+    "controllers": [
+      {
        "name": "",
        "user": "influx",
        "pass": "",
@@ -29,5 +33,7 @@
        "save_ids": false,
        "save_sites": true,
        "verify_ssl": false
-  }]
+      }
+    ]
+  }
 }

examples/up.xml.example

@@ -4,8 +4,11 @@
 # UniFi Poller primary configuration file. XML FORMAT #
 # provided values are defaults. See up.conf.example!  #
 #######################################################
+
+<plugin> and <site> are lists of strings and may be repeated.
 -->
 <poller debug="false" quiet="false">
+  <plugin></plugin>
 
   <prometheus disable="false">
     <http_listen>0.0.0.0:9130</http_listen>
@@ -21,9 +24,10 @@
     <verify_ssl>false</verify_ssl>
   </influxdb>
 
+  <unifi>
     <!-- Repeat this stanza to poll additional controllers. -->
     <controller name="">
-    <sites>all</sites>
+      <site>all</site>
       <user>influx</user>
       <pass></pass>
       <url>https://127.0.0.1:8443</url>
@@ -31,5 +35,5 @@
       <save_ids>false</save_ids>
       <save_sites>true</save_sites>
     </controller>
-
+  </unifi>
 </poller>

examples/up.yaml.example

@@ -7,6 +7,7 @@
 poller:
   debug: false
   quiet: false
+  plugins: []
 
 prometheus:
   disable: false
@@ -22,7 +23,8 @@ influxdb:
   db:   "unifi"
   verify_ssl: false
 
-controller:
+unifi:
+  controllers:
     - name: ""
       user: "influx"
       pass: ""

pkg/inputunifi/input.go

@@ -27,7 +27,7 @@ type Controller struct {
 	User      string       `json:"user" toml:"user" xml:"user" yaml:"user"`
 	Pass      string       `json:"pass" toml:"pass" xml:"pass" yaml:"pass"`
 	URL       string       `json:"url" toml:"url" xml:"url" yaml:"url"`
-	Sites     []string     `json:"sites,omitempty" toml:"sites,omitempty" xml:"sites" yaml:"sites"`
+	Sites     []string     `json:"sites,omitempty" toml:"sites,omitempty" xml:"site" yaml:"sites"`
 	Unifi     *unifi.Unifi `json:"-" toml:"-" xml:"-" yaml:"-"`
 }
 
@@ -35,7 +35,7 @@ type Controller struct {
 type Config struct {
 	sync.RWMutex              // locks the Unifi struct member when re-authing to unifi.
 	Disable      bool         `json:"disable" toml:"disable" xml:"disable" yaml:"disable"`
-	Controllers  []Controller `json:"controller" toml:"controller" xml:"controller" yaml:"controller"`
+	Controllers  []Controller `json:"controllers" toml:"controller" xml:"controller" yaml:"controllers"`
 }
 
 func init() {

pkg/inputunifi/interface.go

@@ -10,6 +10,10 @@ import (
 
 // Metrics grabs all the measurements from a UniFi controller and returns them.
 func (u *InputUnifi) Metrics() (*poller.Metrics, error) {
+	if u.Config.Disable {
+		return nil, nil
+	}
+
 	errs := []string{}
 	metrics := &poller.Metrics{}
 

pkg/poller/build_macos.go

@@ -4,3 +4,6 @@ package poller
 
 // DefaultConfFile is where to find config if --config is not prvided.
 const DefaultConfFile = "/usr/local/etc/unifi-poller/up.conf"
+
+// DefaultObjPath is the path to look for shared object libraries (plugins).
+const DefaultObjPath = "/usr/local/lib/unifi-poller"

pkg/poller/build_unix.go

@@ -4,3 +4,6 @@ package poller
 
 // DefaultConfFile is where to find config if --config is not prvided.
 const DefaultConfFile = "/etc/unifi-poller/up.conf"
+
+// DefaultObjPath is the path to look for shared object libraries (plugins).
+const DefaultObjPath = "/usr/lib/unifi-poller"

pkg/poller/build_windows.go

@@ -4,3 +4,6 @@ package poller
 
 // DefaultConfFile is where to find config if --config is not prvided.
 const DefaultConfFile = `C:\ProgramData\unifi-poller\up.conf`
+
+// DefaultObjPath is useless in this context. Bummer.
+const DefaultObjPath = "PLUGINS_DO_NOT_WORK_ON_WINDOWS_SOWWWWWY"

pkg/poller/config.go

@@ -9,6 +9,10 @@ package poller
 */
 
 import (
+	"os"
+	"path"
+	"plugin"
+	"strings"
 	"time"
 
 	"github.com/spf13/pflag"
@@ -53,33 +57,24 @@ type Config struct {
 
 // Poller is the global config values.
 type Poller struct {
+	Plugins []string `json:"plugins" toml:"plugins" xml:"plugin" yaml:"plugins"`
 	Debug   bool     `json:"debug" toml:"debug" xml:"debug,attr" yaml:"debug"`
 	Quiet   bool     `json:"quiet,omitempty" toml:"quiet,omitempty" xml:"quiet,attr" yaml:"quiet"`
 }
 
-// ParseConfigs parses the poller config and the config for each registered output plugin.
+// LoadPlugins reads-in dynamic shared libraries.
-func (u *UnifiPoller) ParseConfigs() error {
+// Not used very often, if at all.
-	// Parse core config.
+func (u *UnifiPoller) LoadPlugins() error {
-	if err := u.ParseInterface(u.Config); err != nil {
+	for _, p := range u.Plugins {
-		return err
+		name := strings.TrimSuffix(p, ".so") + ".so"
+
+		if _, err := os.Stat(name); os.IsNotExist(err) {
+			name = path.Join(DefaultObjPath, name)
 		}
 
-	// Parse output plugin configs.
+		u.Logf("Loading Dynamic Plugin: %s", name)
-	outputSync.Lock()
-	defer outputSync.Unlock()
 
-	for _, o := range outputs {
+		if _, err := plugin.Open(name); err != nil {
-		if err := u.ParseInterface(o.Config); err != nil {
-			return err
-		}
-	}
-
-	// Parse input plugin configs.
-	inputSync.Lock()
-	defer inputSync.Unlock()
-
-	for _, i := range inputs {
-		if err := u.ParseInterface(i.Config); err != nil {
 			return err
 		}
 	}
@@ -87,8 +82,27 @@ func (u *UnifiPoller) ParseConfigs() error {
 	return nil
 }
 
-// ParseInterface parses the config file and environment variables into the provided interface.
+// ParseConfigs parses the poller config and the config for each registered output plugin.
-func (u *UnifiPoller) ParseInterface(i interface{}) error {
+func (u *UnifiPoller) ParseConfigs() error {
+	// Parse core config.
+	if err := u.parseInterface(u.Config); err != nil {
+		return err
+	}
+
+	// Load dynamic plugins.
+	if err := u.LoadPlugins(); err != nil {
+		return err
+	}
+
+	if err := u.parseInputs(); err != nil {
+		return err
+	}
+
+	return u.parseOutputs()
+}
+
+// parseInterface parses the config file and environment variables into the provided interface.
+func (u *UnifiPoller) parseInterface(i interface{}) error {
 	// Parse config file into provided interface.
 	if err := config.ParseFile(i, u.Flags.ConfigFile); err != nil {
 		return err
@@ -99,3 +113,31 @@ func (u *UnifiPoller) ParseInterface(i interface{}) error {
 
 	return err
 }
+
+// Parse input plugin configs.
+func (u *UnifiPoller) parseInputs() error {
+	inputSync.Lock()
+	defer inputSync.Unlock()
+
+	for _, i := range inputs {
+		if err := u.parseInterface(i.Config); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Parse output plugin configs.
+func (u *UnifiPoller) parseOutputs() error {
+	outputSync.Lock()
+	defer outputSync.Unlock()
+
+	for _, o := range outputs {
+		if err := u.parseInterface(o.Config); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

plugins/mysql/README.md

@@ -0,0 +1,26 @@
+# MYSQL Output Plugin Example
+
+The code here, and the dynamic plugin provided shows an example of how you can
+write your own output for unifi-poller. This plugin records some very basic
+data about clients on a unifi network into a mysql database.
+
+You could write outputs that do... anything. An example: They could compare current
+connected clients to a previous list (in a db, or stored in memory), and send a
+notification if it changes. The possibilities are endless.
+
+You must compile your plugin using the unifi-poller source for the version you're
+using. In other words, to build a plugin for version 2.0.1, do this:
+```
+mkdir -p $GOPATH/src/github.com/davidnewhall
+cd $GOPATH/src/github.com/davidnewhall
+
+git clone git@github.com:davidnewhall/unifi-poller.git
+cd unifi-poller
+
+git checkout v2.0.1
+make vendor
+
+cp -r <your plugin> plugins/
+GOOS=linux make plugins
+```
+The plugin you copy in *must* have a `main.go` file for `make plugins` to build it.

plugins/mysql/main.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/davidnewhall/unifi-poller/pkg/poller"
+	"golift.io/config"
+)
+
+// mysqlConfig represents the data that is unmarshalled from the up.conf config file for this plugins.
+type mysqlConfig struct {
+	Interval config.Duration `json:"interval" toml:"interval" xml:"interval" yaml:"interval"`
+	Host     string          `json:"host" toml:"host" xml:"host" yaml:"host"`
+	User     string          `json:"user" toml:"user" xml:"user" yaml:"user"`
+	Pass     string          `json:"pass" toml:"pass" xml:"pass" yaml:"pass"`
+	DB       string          `json:"db" toml:"db" xml:"db" yaml:"db"`
+	Table    string          `json:"table" toml:"table" xml:"table" yaml:"table"`
+	// Maps do not work with ENV VARIABLES yet, but may in the future.
+	Fields []string `json:"fields" toml:"fields" xml:"field" yaml:"fields"`
+}
+
+// Pointers are ignored during ENV variable unmarshal, avoid pointers to your config.
+// Only capital (exported) members are unmarshaled when passed into poller.NewOutput().
+type application struct {
+	Config mysqlConfig `json:"mysql" toml:"mysql" xml:"mysql" yaml:"mysql"`
+}
+
+func init() {
+	u := &application{Config: mysqlConfig{}}
+
+	poller.NewOutput(&poller.Output{
+		Name:   "mysql",
+		Config: u, // pass in the struct *above* your config (so it can see the struct tags).
+		Method: u.Run,
+	})
+}
+
+func main() {
+	fmt.Println("this is a unifi-poller plugin; not an application")
+}
+
+func (a *application) Run(c poller.Collect) error {
+	c.Logf("mysql plugin is not finished")
+	return nil
+}