Merge pull request #479 from unpoller/drop_pii_support

Add support to completely remove PII

pkg/inputunifi/collectevents.go

@@ -99,7 +99,7 @@ func (u *InputUnifi) collectEvents(logs []any, sites []*unifi.Site, c *Controlle
 			}
 
 			for _, e := range events {
-				e := redactEvent(e, c.HashPII)
+				e := redactEvent(e, c.HashPII, c.DropPII)
 				logs = append(logs, e)
 
 				webserver.NewInputEvent(PluginName, s.ID+"_events", &webserver.Event{
@@ -141,18 +141,26 @@ func (u *InputUnifi) collectIDS(logs []any, sites []*unifi.Site, c *Controller)
 
 // redactEvent attempts to mask personally identying information from log messages.
 // This currently misses the "msg" value entirely and leaks PII information.
-func redactEvent(e *unifi.Event, hash *bool) *unifi.Event {
-	if !*hash {
+func redactEvent(e *unifi.Event, hash *bool, dropPII *bool) *unifi.Event {
+	if !*hash && !*dropPII {
 		return e
 	}
 
 	// metrics.Events[i].Msg <-- not sure what to do here.
 	e.DestIPGeo = unifi.IPGeo{}
 	e.SourceIPGeo = unifi.IPGeo{}
-	e.Host = RedactNamePII(e.Host, hash)
-	e.Hostname = RedactNamePII(e.Hostname, hash)
-	e.DstMAC = RedactMacPII(e.DstMAC, hash)
-	e.SrcMAC = RedactMacPII(e.SrcMAC, hash)
+	if *dropPII {
+		e.Host = ""
+		e.Hostname = ""
+		e.DstMAC = ""
+		e.SrcMAC = ""
+	} else {
+		// hash it
+		e.Host = RedactNamePII(e.Host, hash, dropPII)
+		e.Hostname = RedactNamePII(e.Hostname, hash, dropPII)
+		e.DstMAC = RedactMacPII(e.DstMAC, hash, dropPII)
+		e.SrcMAC = RedactMacPII(e.SrcMAC, hash, dropPII)
+	}
 
 	return e
 }

pkg/inputunifi/collector.go

@@ -139,9 +139,9 @@ func (u *InputUnifi) augmentMetrics(c *Controller, metrics *Metrics) *poller.Met
 			devices[client.Mac] = client.Hostname
 		}
 
-		client.Mac = RedactMacPII(client.Mac, c.HashPII)
-		client.Name = RedactNamePII(client.Name, c.HashPII)
-		client.Hostname = RedactNamePII(client.Hostname, c.HashPII)
+		client.Mac = RedactMacPII(client.Mac, c.HashPII, c.DropPII)
+		client.Name = RedactNamePII(client.Name, c.HashPII, c.DropPII)
+		client.Hostname = RedactNamePII(client.Hostname, c.HashPII, c.DropPII)
 		client.SwName = devices[client.SwMac]
 		client.ApName = devices[client.ApMac]
 		client.GwName = devices[client.GwMac]
@@ -156,8 +156,8 @@ func (u *InputUnifi) augmentMetrics(c *Controller, metrics *Metrics) *poller.Met
 			client.Name = client.MAC
 		}
 
-		client.Name = RedactNamePII(client.Name, c.HashPII)
-		client.MAC = RedactMacPII(client.MAC, c.HashPII)
+		client.Name = RedactNamePII(client.Name, c.HashPII, c.DropPII)
+		client.MAC = RedactMacPII(client.MAC, c.HashPII, c.DropPII)
 		m.ClientsDPI = append(m.ClientsDPI, client)
 	}
 
@@ -219,7 +219,11 @@ func extractDevices(metrics *Metrics) (*poller.Metrics, map[string]string, map[s
 
 // RedactNamePII converts a name string to an md5 hash (first 24 chars only).
 // Useful for maskiing out personally identifying information.
-func RedactNamePII(pii string, hash *bool) string {
+func RedactNamePII(pii string, hash *bool, dropPII *bool) string {
+	if dropPII != nil && *dropPII {
+		return ""
+	}
+
 	if hash == nil || !*hash || pii == "" {
 		return pii
 	}
@@ -231,7 +235,11 @@ func RedactNamePII(pii string, hash *bool) string {
 
 // RedactMacPII converts a MAC address to an md5 hashed version (first 14 chars only).
 // Useful for maskiing out personally identifying information.
-func RedactMacPII(pii string, hash *bool) (output string) {
+func RedactMacPII(pii string, hash *bool, dropPII *bool) (output string) {
+	if dropPII != nil && *dropPII {
+		return ""
+	}
+
 	if hash == nil || !*hash || pii == "" {
 		return pii
 	}

pkg/inputunifi/input.go

@@ -42,6 +42,7 @@ type Controller struct {
 	SaveDPI    *bool        `json:"save_dpi" toml:"save_dpi" xml:"save_dpi" yaml:"save_dpi"`
 	SaveRogue  *bool        `json:"save_rogue" toml:"save_rogue" xml:"save_rogue" yaml:"save_rogue"`
 	HashPII    *bool        `json:"hash_pii" toml:"hash_pii" xml:"hash_pii" yaml:"hash_pii"`
+	DropPII    *bool        `json:"drop_pii" toml:"drop_pii" xml:"drop_pii" yaml:"drop_pii"`
 	SaveSites  *bool        `json:"save_sites" toml:"save_sites" xml:"save_sites" yaml:"save_sites"`
 	CertPaths  []string     `json:"ssl_cert_paths" toml:"ssl_cert_paths" xml:"ssl_cert_path" yaml:"ssl_cert_paths"`
 	User       string       `json:"user" toml:"user" xml:"user" yaml:"user"`
@@ -215,6 +216,10 @@ func (u *InputUnifi) setDefaults(c *Controller) { //nolint:cyclop
 		c.HashPII = &f
 	}
 
+	if c.DropPII == nil {
+		c.DropPII = &f
+	}
+
 	if c.SaveDPI == nil {
 		c.SaveDPI = &f
 	}
@@ -280,6 +285,10 @@ func (u *InputUnifi) setControllerDefaults(c *Controller) *Controller { //nolint
 		c.HashPII = u.Default.HashPII
 	}
 
+	if c.DropPII == nil {
+		c.DropPII = u.Default.DropPII
+	}
+
 	if c.SaveDPI == nil {
 		c.SaveDPI = u.Default.SaveDPI
 	}

pkg/inputunifi/interface.go

@@ -69,10 +69,10 @@ func (u *InputUnifi) logController(c *Controller) {
 	}
 
 	u.Logf("   => Username: %s (has password: %v)", c.User, c.Pass != "")
-	u.Logf("   => Hash PII / Poll Sites: %v / %s", *c.HashPII, strings.Join(c.Sites, ", "))
-	u.Logf("   => Save Sites / Save DPI: %v / %v (metrics)", *c.SaveSites, *c.SaveDPI)
-	u.Logf("   => Save Events / Save IDS: %v / %v (logs)", *c.SaveEvents, *c.SaveIDS)
-	u.Logf("   => Save Alarms / Anomalies: %v / %v (logs)", *c.SaveAlarms, *c.SaveAnomal)
+	u.Logf("   => Hash PII %v / Drop PII %v / Poll Sites: %s", *c.HashPII, *c.DropPII, strings.Join(c.Sites, ", "))
+	u.Logf("   => Save Sites %v / Save DPI %v (metrics)", *c.SaveSites, *c.SaveDPI)
+	u.Logf("   => Save Events %v / Save IDS %v (logs)", *c.SaveEvents, *c.SaveIDS)
+	u.Logf("   => Save Alarms %v / Anomalies %v (logs)", *c.SaveAlarms, *c.SaveAnomal)
 	u.Logf("   => Save Rogue APs: %v", *c.SaveRogue)
 }
 

pkg/inputunifi/updateweb.go

@@ -47,6 +47,7 @@ func formatControllers(controllers []*Controller) []*Controller {
 			SaveIDS:    c.SaveIDS,
 			SaveDPI:    c.SaveDPI,
 			HashPII:    c.HashPII,
+			DropPII:    c.DropPII,
 			SaveSites:  c.SaveSites,
 			User:       c.User,
 			Pass:       strconv.FormatBool(c.Pass != ""),