From 07e1e5bc4d381361a5c25d321b2df14356c4258d Mon Sep 17 00:00:00 2001 From: Sven Grossmann Date: Mon, 22 Dec 2025 22:55:30 +0100 Subject: [PATCH] feat: add UniFi Protect logs support with Loki integration - Add SaveProtectLogs config option to enable Protect log collection - Add ProtectThumbnails config option to fetch event thumbnails - Add collectProtectLogs function with 24h default fetch window - Add ProtectLogEvent for Loki reporting with separate thumbnail log lines - Add PII redaction for Protect log entries - Filter thumbnail fetching to camera events only (motion, smartDetect*, etc.) - Update log output to show Protect logs status --- pkg/inputunifi/collectevents.go | 85 ++++++++++++++++++++++++++++++++- pkg/inputunifi/input.go | 18 +++++++ pkg/inputunifi/interface.go | 2 +- pkg/inputunifi/updateweb.go | 34 ++++++------- pkg/lokiunifi/report.go | 14 ++++-- pkg/lokiunifi/report_protect.go | 69 ++++++++++++++++++++++++++ 6 files changed, 201 insertions(+), 21 deletions(-) create mode 100644 pkg/lokiunifi/report_protect.go diff --git a/pkg/inputunifi/collectevents.go b/pkg/inputunifi/collectevents.go index 0a0f4676..7c4475c0 100644 --- a/pkg/inputunifi/collectevents.go +++ b/pkg/inputunifi/collectevents.go @@ -1,6 +1,7 @@ package inputunifi import ( + "encoding/base64" "fmt" "time" @@ -34,7 +35,7 @@ func (u *InputUnifi) collectControllerEvents(c *Controller) ([]any, error) { type caller func([]any, []*unifi.Site, *Controller) ([]any, error) - for _, call := range []caller{u.collectIDs, u.collectAnomalies, u.collectAlarms, u.collectEvents, u.collectSyslog} { + for _, call := range []caller{u.collectIDs, u.collectAnomalies, u.collectAlarms, u.collectEvents, u.collectSyslog, u.collectProtectLogs} { if newLogs, err = call(logs, sites, c); err != nil { return logs, err } @@ -152,6 +153,53 @@ func (u *InputUnifi) collectSyslog(logs []any, sites []*unifi.Site, c *Controlle return logs, nil } +func (u *InputUnifi) collectProtectLogs(logs []any, _ []*unifi.Site, c *Controller) ([]any, error) { + if *c.SaveProtectLogs { + u.LogDebugf("Collecting Protect logs: %s (%s)", c.URL, c.ID) + + req := unifi.DefaultProtectLogRequest(0) // Uses default 24-hour window + entries, err := c.Unifi.GetProtectLogs(req) + if err != nil { + return logs, fmt.Errorf("unifi.GetProtectLogs(): %w", err) + } + + for _, e := range entries { + e := redactProtectLogEntry(e, c.HashPII, c.DropPII) + + // Fetch thumbnail if enabled and event has a camera (only camera events have real thumbnails) + // Skip access/adminActivity events - they don't have actual camera thumbnails + if *c.ProtectThumbnails && e.Thumbnail != "" && e.Camera != "" && hasProtectThumbnail(e.Type) { + // Thumbnail field is like "e-69499de2037add03e4015fa8" - strip "e-" prefix + thumbID := e.Thumbnail + if len(thumbID) > 2 && thumbID[:2] == "e-" { + thumbID = thumbID[2:] + } + if thumbData, err := c.Unifi.GetProtectEventThumbnail(thumbID); err == nil { + e.ThumbnailBase64 = base64.StdEncoding.EncodeToString(thumbData) + } else { + u.LogDebugf("Failed to fetch thumbnail for event %s (thumb: %s): %v", e.ID, thumbID, err) + } + } + + logs = append(logs, e) + + webserver.NewInputEvent(PluginName, "protect_logs", &webserver.Event{ + Msg: e.Msg(), Ts: e.Datetime(), Tags: map[string]string{ + "type": "protect_log", + "event_type": e.GetEventType(), + "category": e.GetCategory(), + "subcategory": e.GetSubCategory(), + "severity": e.GetSeverity(), + "camera": e.Camera, + "source": e.SourceName, + }, + }) + } + } + + return logs, nil +} + func (u *InputUnifi) collectIDs(logs []any, sites []*unifi.Site, c *Controller) ([]any, error) { if *c.SaveIDs { u.LogDebugf("Collecting controller IDs data: %s (%s)", c.URL, c.ID) @@ -251,3 +299,38 @@ func redactSystemLogEntry(e *unifi.SystemLogEntry, hash *bool, dropPII *bool) *u return e } + +// redactProtectLogEntry attempts to mask personally identifying information from Protect log entries. +func redactProtectLogEntry(e *unifi.ProtectLogEntry, hash *bool, dropPII *bool) *unifi.ProtectLogEntry { + if !*hash && !*dropPII { + return e + } + + // Redact user names from message keys + if e.Description != nil { + for i, mk := range e.Description.MessageKeys { + if mk.Key == "userLink" || mk.Action == "viewUsers" { + if *dropPII { + e.Description.MessageKeys[i].Text = "" + } else { + e.Description.MessageKeys[i].Text = RedactNamePII(mk.Text, hash, dropPII) + } + } + } + } + + return e +} + +// hasProtectThumbnail returns true if the event type has actual camera thumbnails. +// Access and adminActivity events don't have real thumbnails (they're user activity logs). +func hasProtectThumbnail(eventType string) bool { + switch eventType { + case "motion", "smartDetectZone", "smartDetectLine", "ring", "sensorMotion", + "sensorContact", "sensorAlarm", "doorbell", "package", "person", "vehicle", + "animal", "face", "licensePlate": + return true + default: + return false + } +} diff --git a/pkg/inputunifi/input.go b/pkg/inputunifi/input.go index 7aa2147d..91f20c84 100644 --- a/pkg/inputunifi/input.go +++ b/pkg/inputunifi/input.go @@ -39,6 +39,8 @@ type Controller struct { SaveAlarms *bool `json:"save_alarms" toml:"save_alarms" xml:"save_alarms" yaml:"save_alarms"` SaveEvents *bool `json:"save_events" toml:"save_events" xml:"save_events" yaml:"save_events"` SaveSyslog *bool `json:"save_syslog" toml:"save_syslog" xml:"save_syslog" yaml:"save_syslog"` + SaveProtectLogs *bool `json:"save_protect_logs" toml:"save_protect_logs" xml:"save_protect_logs" yaml:"save_protect_logs"` + ProtectThumbnails *bool `json:"protect_thumbnails" toml:"protect_thumbnails" xml:"protect_thumbnails" yaml:"protect_thumbnails"` SaveIDs *bool `json:"save_ids" toml:"save_ids" xml:"save_ids" yaml:"save_ids"` SaveDPI *bool `json:"save_dpi" toml:"save_dpi" xml:"save_dpi" yaml:"save_dpi"` SaveRogue *bool `json:"save_rogue" toml:"save_rogue" xml:"save_rogue" yaml:"save_rogue"` @@ -249,6 +251,14 @@ func (u *InputUnifi) setDefaults(c *Controller) { //nolint:cyclop c.SaveSyslog = &f } + if c.SaveProtectLogs == nil { + c.SaveProtectLogs = &f + } + + if c.ProtectThumbnails == nil { + c.ProtectThumbnails = &f + } + if c.SaveAlarms == nil { c.SaveAlarms = &f } @@ -332,6 +342,14 @@ func (u *InputUnifi) setControllerDefaults(c *Controller) *Controller { //nolint c.SaveSyslog = u.Default.SaveSyslog } + if c.SaveProtectLogs == nil { + c.SaveProtectLogs = u.Default.SaveProtectLogs + } + + if c.ProtectThumbnails == nil { + c.ProtectThumbnails = u.Default.ProtectThumbnails + } + if c.SaveAlarms == nil { c.SaveAlarms = u.Default.SaveAlarms } diff --git a/pkg/inputunifi/interface.go b/pkg/inputunifi/interface.go index 0ee63fe6..4dd13181 100644 --- a/pkg/inputunifi/interface.go +++ b/pkg/inputunifi/interface.go @@ -128,7 +128,7 @@ func (u *InputUnifi) logController(c *Controller) { u.Logf(" => Hash PII %v / Drop PII %v / Poll Sites: %s", *c.HashPII, *c.DropPII, strings.Join(c.Sites, ", ")) u.Logf(" => Save Sites %v / Save DPI %v (metrics)", *c.SaveSites, *c.SaveDPI) u.Logf(" => Save Events %v / Save Syslog %v / Save IDs %v (logs)", *c.SaveEvents, *c.SaveSyslog, *c.SaveIDs) - u.Logf(" => Save Alarms %v / Anomalies %v (logs)", *c.SaveAlarms, *c.SaveAnomal) + u.Logf(" => Save Alarms %v / Anomalies %v / Protect Logs %v (thumbnails: %v)", *c.SaveAlarms, *c.SaveAnomal, *c.SaveProtectLogs, *c.ProtectThumbnails) u.Logf(" => Save Rogue APs: %v", *c.SaveRogue) } diff --git a/pkg/inputunifi/updateweb.go b/pkg/inputunifi/updateweb.go index 3f07e81d..bed179ec 100644 --- a/pkg/inputunifi/updateweb.go +++ b/pkg/inputunifi/updateweb.go @@ -39,22 +39,24 @@ func formatControllers(controllers []*Controller) []*Controller { } fixed = append(fixed, &Controller{ - VerifySSL: c.VerifySSL, - SaveAnomal: c.SaveAnomal, - SaveAlarms: c.SaveAlarms, - SaveRogue: c.SaveRogue, - SaveEvents: c.SaveEvents, - SaveIDs: c.SaveIDs, - SaveDPI: c.SaveDPI, - HashPII: c.HashPII, - DropPII: c.DropPII, - SaveSites: c.SaveSites, - User: c.User, - Pass: strconv.FormatBool(c.Pass != ""), - APIKey: strconv.FormatBool(c.APIKey != ""), - URL: c.URL, - Sites: c.Sites, - ID: id, + VerifySSL: c.VerifySSL, + SaveAnomal: c.SaveAnomal, + SaveAlarms: c.SaveAlarms, + SaveRogue: c.SaveRogue, + SaveEvents: c.SaveEvents, + SaveSyslog: c.SaveSyslog, + SaveProtectLogs: c.SaveProtectLogs, + SaveIDs: c.SaveIDs, + SaveDPI: c.SaveDPI, + HashPII: c.HashPII, + DropPII: c.DropPII, + SaveSites: c.SaveSites, + User: c.User, + Pass: strconv.FormatBool(c.Pass != ""), + APIKey: strconv.FormatBool(c.APIKey != ""), + URL: c.URL, + Sites: c.Sites, + ID: id, }) } diff --git a/pkg/lokiunifi/report.go b/pkg/lokiunifi/report.go index 9011892f..f622e5e9 100644 --- a/pkg/lokiunifi/report.go +++ b/pkg/lokiunifi/report.go @@ -57,6 +57,8 @@ func (r *Report) ProcessEventLogs(events *poller.Events) *Logs { r.Anomaly(event, logs) case *unifi.SystemLogEntry: r.SystemLogEvent(event, logs) + case *unifi.ProtectLogEntry: + r.ProtectLogEvent(event, logs) default: // unlikely. r.LogErrorf("unknown event type: %T", e) } @@ -66,11 +68,17 @@ func (r *Report) ProcessEventLogs(events *poller.Events) *Logs { } func (r *Report) String() string { - return fmt.Sprintf("%s: %d, %s: %d, %s: %d, %s: %d, %s: %d, Dur: %v", + s := fmt.Sprintf("%s: %d, %s: %d, %s: %d, %s: %d, %s: %d, %s: %d", typeEvent, r.Counts[typeEvent], typeIDs, r.Counts[typeIDs], typeAlarm, r.Counts[typeAlarm], typeAnomaly, r.Counts[typeAnomaly], - typeSystemLog, r.Counts[typeSystemLog], - time.Since(r.Start).Round(time.Millisecond)) + typeSystemLog, r.Counts[typeSystemLog], typeProtectLog, r.Counts[typeProtectLog]) + + if r.Counts[typeProtectThumbnail] > 0 { + s += fmt.Sprintf(" (thumbs: %d)", r.Counts[typeProtectThumbnail]) + } + + s += fmt.Sprintf(", Dur: %v", time.Since(r.Start).Round(time.Millisecond)) + return s } // CleanLabels removes any tag that is empty. diff --git a/pkg/lokiunifi/report_protect.go b/pkg/lokiunifi/report_protect.go new file mode 100644 index 00000000..50ddd673 --- /dev/null +++ b/pkg/lokiunifi/report_protect.go @@ -0,0 +1,69 @@ +package lokiunifi + +import ( + "encoding/json" + "strconv" + + "github.com/unpoller/unifi/v5" +) + +const typeProtectLog = "ProtectLog" +const typeProtectThumbnail = "ProtectThumbnail" + +// ProtectLogEvent stores a structured UniFi Protect Log Entry for batch sending to Loki. +// Logs the raw JSON for parsing with Loki's `| json` pipeline. +// If the event has a thumbnail, it's sent as a separate log line. +func (r *Report) ProtectLogEvent(event *unifi.ProtectLogEntry, logs *Logs) { + if event.Datetime().Before(r.Oldest) { + return + } + + r.Counts[typeProtectLog]++ // increase counter and append new log line. + + // Store thumbnail separately before marshaling (it's excluded from JSON by default now) + thumbnailBase64 := event.ThumbnailBase64 + + // Marshal event to JSON for the log line (without thumbnail to keep it small) + event.ThumbnailBase64 = "" // Temporarily clear for marshaling + msg, err := json.Marshal(event) + if err != nil { + msg = []byte(event.Msg()) + } + event.ThumbnailBase64 = thumbnailBase64 // Restore + + // Add event log line + logs.Streams = append(logs.Streams, LogStream{ + Entries: [][]string{{strconv.FormatInt(event.Datetime().UnixNano(), 10), string(msg)}}, + Labels: CleanLabels(map[string]string{ + "application": "unifi_protect_log", + "source": event.SourceName, + "event_type": event.GetEventType(), + "category": event.GetCategory(), + "severity": event.GetSeverity(), + "camera": event.Camera, + }), + }) + + // Add thumbnail as separate log line if present + if thumbnailBase64 != "" { + r.Counts[typeProtectThumbnail]++ + + thumbnailJSON, _ := json.Marshal(map[string]string{ + "event_id": event.ID, + "thumbnail_base64": thumbnailBase64, + "mime_type": "image/jpeg", + }) + + // Use timestamp + 1 nanosecond to ensure ordering (thumbnail after event) + logs.Streams = append(logs.Streams, LogStream{ + Entries: [][]string{{strconv.FormatInt(event.Datetime().UnixNano()+1, 10), string(thumbnailJSON)}}, + Labels: CleanLabels(map[string]string{ + "application": "unifi_protect_thumbnail", + "source": event.SourceName, + "event_id": event.ID, + "camera": event.Camera, + }), + }) + } +} +