diff --git a/.gitignore b/.gitignore index e1bbf848..39c9980e 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ github_deploy_key* dist/ .vscode/ .idea/ -# Endpoint discovery tool -tools/endpoint-discovery/__pycache__ -tools/endpoint-discovery/.venv +# Local test config (contains credentials) +up.discover-test.json +# Generated discovery report (re-run with --discover to recreate) +api_endpoints_discovery.md diff --git a/docs/PR_936_REPLACEMENT.md b/docs/PR_936_REPLACEMENT.md new file mode 100644 index 00000000..601b0d67 --- /dev/null +++ b/docs/PR_936_REPLACEMENT.md @@ -0,0 +1,66 @@ +# Endpoint discovery via `--discover` (Go) + +The [unpoller](https://github.com/unpoller/unpoller) repo has a Python/Playwright endpoint-discovery tool in `tools/endpoint-discovery/` (browser-based; discovers unknown endpoints; more support currently). This doc describes the Go-based discovery: it lives in [unpoller/unifi](https://github.com/unpoller/unifi) and unpoller, probes a fixed list of known API paths, and discovers/confirms known endpoints only. + +--- + +## Summary + +- **Feature:** `--discover` flag on unpoller that probes known API endpoints on the controller and writes a shareable markdown report. +- **Credentials:** Uses the same config file (and first unifi controller) as normal unpoller runs. +- **Output:** Markdown file (default `api_endpoints_discovery.md`) with method, path, and HTTP status for each endpoint. Users can share this when reporting API/404 issues (e.g. [issue #935](https://github.com/unpoller/unpoller/issues/935)). +- **Dependency:** Requires [unpoller/unifi](https://github.com/unpoller/unifi) with `DiscoverEndpoints` and `Probe` merged. + +--- + +## Changes in unpoller (this PR) + +| File | Change | +|------|--------| +| `pkg/poller/config.go` | Add `Discover bool`, `DiscoverOutput string` to `Flags`. | +| `pkg/poller/start.go` | Register `--discover` and `--discover-output`; when `--discover`, call `RunDiscover()` and exit. | +| `pkg/poller/commands.go` | Add `RunDiscover()`: load config, init inputs, find input implementing `Discoverer`, call `Discover(outputPath)`. | +| `pkg/poller/inputs.go` | Add optional interface `Discoverer` with `Discover(outputPath string) error`. | +| `pkg/inputunifi/discover.go` | **New file.** Implement `Discoverer`: first controller, authenticate, get sites, call `c.Unifi.DiscoverEndpoints(site, outputPath)`. | +| `.gitignore` | Add `up.discover-test.json`, `api_endpoints_discovery.md`. | + +--- + +## Usage + +```bash +# Same config as normal unpoller (first unifi controller is used) +unpoller --discover --config /path/to/up.conf --discover-output api_endpoints_discovery.md +``` + +If config is in the default search path: + +```bash +unpoller --discover --discover-output api_endpoints_discovery.md +``` + +--- + +## PR title + +**Add `--discover` to probe API endpoints and write shareable report** + +--- + +## PR description (suggested) + +Go-based endpoint discovery: probes known API paths on the controller and writes a shareable report. Uses the [unifi](https://github.com/unpoller/unifi) library; same config as normal polling. The Python tool in `tools/endpoint-discovery/` remains for browser-based discovery (more coverage). + +**What it does** +- `unpoller --discover` uses the first unifi controller from your config, authenticates, and probes a set of known API paths. +- Writes a markdown report (default `api_endpoints_discovery.md`) with method, path, and HTTP status for each endpoint. +- Same credentials as normal polling; users can share the file when reporting API/404 issues (e.g. #935). + +**Usage** +```bash +unpoller --discover --config /path/to/up.conf --discover-output api_endpoints_discovery.md +``` + +**Requires** [unpoller/unifi](https://github.com/unpoller/unifi) with `DiscoverEndpoints` (and `Probe`) merged. + +**CI:** Merge unifi first, then this PR (or update go.mod to require the new unifi release). For local testing, use `replace github.com/unpoller/unifi/v5 => ../unifi` with the unifi repo checked out. diff --git a/go.mod b/go.mod index 5794e193..4ee34dd5 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/prometheus/common v0.67.5 github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.11.1 - github.com/unpoller/unifi/v5 v5.12.0 + github.com/unpoller/unifi/v5 v5.13.0 golang.org/x/crypto v0.47.0 golang.org/x/term v0.39.0 golift.io/cnfg v0.2.3 @@ -46,5 +46,3 @@ require ( golang.org/x/sys v0.40.0 // indirect google.golang.org/protobuf v1.36.11 // indirect ) - -// replace github.com/unpoller/unifi/v5 => ../unifi diff --git a/go.sum b/go.sum index f2c2fc72..b7a03e77 100644 --- a/go.sum +++ b/go.sum @@ -77,8 +77,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/unpoller/unifi/v5 v5.12.0 h1:wdC7HpASWXUL7iEHkmaAeKMFpZ8UazWDIpuijY43saE= -github.com/unpoller/unifi/v5 v5.12.0/go.mod h1:vSIXIclPG9dpKxUp+pavfgENHWaTZXvDg7F036R1YCo= +github.com/unpoller/unifi/v5 v5.13.0 h1:0pdMckXzGoH0Lg8+WGV4p6ZJeMQsmqmfkdZLtumdIao= +github.com/unpoller/unifi/v5 v5.13.0/go.mod h1:vSIXIclPG9dpKxUp+pavfgENHWaTZXvDg7F036R1YCo= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= diff --git a/pkg/inputunifi/discover.go b/pkg/inputunifi/discover.go new file mode 100644 index 00000000..7b2e07cb --- /dev/null +++ b/pkg/inputunifi/discover.go @@ -0,0 +1,49 @@ +package inputunifi + +import ( + "fmt" +) + +// Discover implements poller.Discoverer. It uses the first configured controller +// to probe known API endpoints and write a shareable report to outputPath. +// Uses the same credentials as normal polling (from config file). +func (u *InputUnifi) Discover(outputPath string) error { + if u.Config == nil || u.Disable { + return fmt.Errorf("unifi input disabled or not configured") + } + + u.setDefaults(&u.Default) + + if len(u.Controllers) == 0 && !u.Dynamic { + u.Controllers = []*Controller{&u.Default} + } + + if len(u.Controllers) == 0 { + return fmt.Errorf("no unifi controller configured") + } + + c := u.setControllerDefaults(u.Controllers[0]) + if c.URL == "" { + return fmt.Errorf("first controller has no URL") + } + + if err := u.getUnifi(c); err != nil { + return fmt.Errorf("authenticating to controller: %w", err) + } + + sites, err := c.Unifi.GetSites() + if err != nil { + return fmt.Errorf("getting sites: %w", err) + } + + site := "default" + if len(sites) > 0 && sites[0].Name != "" { + site = sites[0].Name + } + + if err := c.Unifi.DiscoverEndpoints(site, outputPath); err != nil { + return fmt.Errorf("writing discovery report: %w", err) + } + + return nil +} diff --git a/pkg/poller/commands.go b/pkg/poller/commands.go index 5f8fe8b1..53f15346 100644 --- a/pkg/poller/commands.go +++ b/pkg/poller/commands.go @@ -191,3 +191,45 @@ func (u *UnifiPoller) HealthCheck() error { // All checks passed, application is healthy. return nil } + +// RunDiscover loads config, initializes inputs, finds an input that implements +// Discoverer, and runs Discover(outputPath). Uses the same config as normal polling. +func (u *UnifiPoller) RunDiscover() error { + cfile, err := getFirstFile(strings.Split(u.Flags.ConfigFile, ",")) + if err != nil { + return fmt.Errorf("discover: config file not found: %w", err) + } + + u.Flags.ConfigFile = cfile + u.Logf("Loading Configuration File: %s", u.Flags.ConfigFile) + + if err := u.ParseConfigs(); err != nil { + return fmt.Errorf("discover: parse config: %w", err) + } + + if err := u.InitializeInputs(); err != nil { + return fmt.Errorf("discover: initialize inputs: %w", err) + } + + outputPath := u.Flags.DiscoverOutput + if outputPath == "" { + outputPath = "api_endpoints_discovery.md" + } + + inputSync.RLock() + defer inputSync.RUnlock() + + for _, input := range inputs { + if d, ok := input.Input.(Discoverer); ok { + if err := d.Discover(outputPath); err != nil { + return fmt.Errorf("discover: %w", err) + } + + u.Logf("Discovery report written to %s (share with maintainers for API/404 issues).", outputPath) + + return nil + } + } + + return fmt.Errorf("discover: no input plugin supports discovery (unifi input required)") +} diff --git a/pkg/poller/config.go b/pkg/poller/config.go index cb09f31c..033abc99 100644 --- a/pkg/poller/config.go +++ b/pkg/poller/config.go @@ -68,12 +68,14 @@ type UnifiPoller struct { // Flags represents the CLI args available and their settings. type Flags struct { - ConfigFile string - DumpJSON string - HashPW string - ShowVer bool - DebugIO bool - Health bool + ConfigFile string + DumpJSON string + HashPW string + ShowVer bool + DebugIO bool + Health bool + Discover bool + DiscoverOutput string *pflag.FlagSet } diff --git a/pkg/poller/inputs.go b/pkg/poller/inputs.go index a5e848f2..f61deb1d 100644 --- a/pkg/poller/inputs.go +++ b/pkg/poller/inputs.go @@ -24,6 +24,12 @@ type Input interface { DebugInput() (bool, error) } +// Discoverer is an optional interface for inputs that can discover API endpoints +// on a controller and write a shareable report (e.g. for support/debugging). +type Discoverer interface { + Discover(outputPath string) error +} + // InputPlugin describes an input plugin's consumable interface. type InputPlugin struct { Name string diff --git a/pkg/poller/start.go b/pkg/poller/start.go index 45a85f7f..6f1a0be8 100644 --- a/pkg/poller/start.go +++ b/pkg/poller/start.go @@ -38,6 +38,10 @@ func (u *UnifiPoller) Start() error { return u.HealthCheck() } + if u.Flags.Discover { + return u.RunDiscover() + } + cfile, err := getFirstFile(strings.Split(u.Flags.ConfigFile, ",")) if err != nil { return err @@ -81,6 +85,9 @@ func (f *Flags) Parse(args []string) { "This debug option prints a json payload and exits. See man page for more info.") f.BoolVarP(&f.DebugIO, "debugio", "d", false, "Debug the Inputs and Outputs configured and exit.") f.BoolVarP(&f.Health, "health", "", false, "Run health check and exit with status 0 (healthy) or 1 (unhealthy).") + f.BoolVarP(&f.Discover, "discover", "", false, "Discover API endpoints on the controller and write a shareable report, then exit.") + f.StringVarP(&f.DiscoverOutput, "discover-output", "", "api_endpoints_discovery.md", + "Path for the discovery report when using --discover.") f.StringVarP(&f.ConfigFile, "config", "c", DefaultConfFile(), "Poller config file path. Separating multiple paths with a comma will load the first config file found.") f.BoolVarP(&f.ShowVer, "version", "v", false, "Print the version and exit.") diff --git a/tools/endpoint-discovery/.env.example b/tools/endpoint-discovery/.env.example deleted file mode 100644 index 925e6534..00000000 --- a/tools/endpoint-discovery/.env.example +++ /dev/null @@ -1,10 +0,0 @@ -# UniFi controller URL (same URL you use for unpoller) -UNIFI_URL=https://192.168.1.1 -UNIFI_USER=admin -UNIFI_PASS=your-password - -# Optional: run with visible browser -# HEADLESS=false - -# Optional: custom output directory (default: this directory) -# OUTPUT_DIR=. diff --git a/tools/endpoint-discovery/README.md b/tools/endpoint-discovery/README.md deleted file mode 100644 index 615cccf6..00000000 --- a/tools/endpoint-discovery/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# UniFi endpoint discovery (headless) - -Runs a headless browser against your UniFi controller, logs in, and records all API requests the UI makes. Use this to see which endpoints your controller exposes (e.g. when debugging 404s like [device-tags #935](https://github.com/unpoller/unpoller/issues/935)). - -## What it does - -1. Launches Chromium (headless by default). -2. Navigates to your UniFi controller URL. -3. If it sees a login form, fills username/password and submits. -4. Visits common UI paths to trigger API calls. -5. Captures every XHR/fetch request to `/api` or `/proxy/` (same origin). -6. Writes a markdown file: `API_ENDPOINTS_HEADLESS_YYYY-MM-DD.md` in this directory. - -## Quick start - -```bash -cd tools/endpoint-discovery -python3 -m venv .venv -.venv/bin/pip install playwright -.venv/bin/playwright install chromium -UNIFI_URL=https://YOUR_CONTROLLER UNIFI_USER=admin UNIFI_PASS=yourpassword .venv/bin/python discover.py -``` - -The script writes `API_ENDPOINTS_HEADLESS_.md` in the same directory. Paste that file (or its contents) in an issue or comment when reporting which endpoints your controller supports. - -## Options - -- **Headed (see the browser):** `HEADLESS=false` before the command. -- **Output elsewhere:** `OUTPUT_DIR=/path/to/dir` before the command. -- **Use .env:** Copy `.env.example` to `.env`, set your values, then run `python discover.py` (install `python-dotenv` for .env support). - -## Requirements - -- Python 3.8+ -- Direct controller URL (e.g. `https://192.168.1.1`) works best; same-origin requests are captured. diff --git a/tools/endpoint-discovery/discover.py b/tools/endpoint-discovery/discover.py deleted file mode 100644 index d72f8c8b..00000000 --- a/tools/endpoint-discovery/discover.py +++ /dev/null @@ -1,198 +0,0 @@ -#!/usr/bin/env python3 -""" -Headless endpoint discovery: navigate the UniFi controller UI and capture -all XHR/fetch requests (method, URL, optional headers), similar to manually -clicking around in Chrome and logging network traffic. - -Usage: - UNIFI_URL=https://192.168.1.1 UNIFI_USER=admin UNIFI_PASS=... python discover.py - Or copy .env.example to .env and run (with python-dotenv): python discover.py - -Output: API_ENDPOINTS_HEADLESS_.md in current dir (or set OUTPUT_DIR) - -Requires: pip install playwright && playwright install chromium -Optional: pip install python-dotenv (to load .env) -""" - -import os -from datetime import date -from pathlib import Path -from urllib.parse import urlparse - -try: - from dotenv import load_dotenv - load_dotenv(Path(__file__).resolve().parent / ".env") -except ImportError: - pass - -from playwright.sync_api import sync_playwright - -SCRIPT_DIR = Path(__file__).resolve().parent -# Default: write output in the same directory as the script (easy for users) -OUTPUT_DIR = Path(os.environ.get("OUTPUT_DIR", str(SCRIPT_DIR))) -HEADLESS = os.environ.get("HEADLESS", "true").lower() != "false" - -GROUP_ORDER = ["api", "proxy-network-api", "proxy-network-v2", "proxy-users", "proxy-other", "other"] -GROUP_TITLES = { - "api": "API (legacy)", - "proxy-network-api": "Proxy /network API (v1)", - "proxy-network-v2": "Proxy /network v2 API", - "proxy-users": "Proxy /users API", - "proxy-other": "Proxy (other)", - "other": "Other", -} - - -def is_api_like(pathname: str) -> bool: - return "/api" in pathname or "/proxy/" in pathname - - -def normalize_url(url_str: str) -> str: - try: - u = urlparse(url_str) - return f"{u.scheme}://{u.netloc}{u.path or ''}{u.query and '?' + u.query or ''}" - except Exception: - return url_str - - -def run_group(pathname: str) -> str: - if pathname.startswith("/api/"): - return "api" - if pathname.startswith("/proxy/network/api/"): - return "proxy-network-api" - if pathname.startswith("/proxy/network/v2/"): - return "proxy-network-v2" - if pathname.startswith("/proxy/users/"): - return "proxy-users" - if pathname.startswith("/proxy/"): - return "proxy-other" - return "other" - - -def main() -> None: - base_url = (os.environ.get("UNIFI_URL") or "").rstrip("/") - user = os.environ.get("UNIFI_USER") or "" - password = os.environ.get("UNIFI_PASS") or "" - - if not base_url or not user or not password: - print("Set UNIFI_URL, UNIFI_USER, and UNIFI_PASS (env or .env).", file=__import__("sys").stderr) - raise SystemExit(1) - - try: - origin = urlparse(base_url) - our_origin = f"{origin.scheme}://{origin.netloc}" - except Exception: - our_origin = "" - - captured: dict[str, dict] = {} - - def on_request(request): - if request.resource_type not in ("xhr", "fetch"): - return - url = request.url - try: - u = urlparse(url) - req_origin = f"{u.scheme}://{u.netloc}" - if req_origin != our_origin: - return - pathname = u.path or "" - if not is_api_like(pathname): - return - key = f"{request.method} {normalize_url(url)}" - if key not in captured: - captured[key] = { - "method": request.method, - "url": normalize_url(url), - "pathname": pathname, - "request_headers": request.headers, - } - except Exception: - pass - - with sync_playwright() as p: - browser = p.chromium.launch( - headless=HEADLESS, - args=["--ignore-certificate-errors"], - ) - context = browser.new_context( - ignore_https_errors=True, - user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", - ) - page = context.new_page() - page.on("request", on_request) - - print("Navigating to", base_url) - try: - page.goto(base_url, wait_until="networkidle", timeout=30000) - except Exception: - pass - - # Login - if page.locator('input[type="password"]').count() > 0: - print("Login form detected, submitting credentials...") - page.locator('input[name="username"], input[name="email"], input[type="text"]').first.fill(user) - page.locator('input[type="password"]').fill(password) - page.locator('button[type="submit"], input[type="submit"], button:has-text("Login"), button:has-text("Sign in")').first.click() - try: - page.wait_for_load_state("networkidle") - except Exception: - pass - page.wait_for_timeout(3000) - - # Visit common paths to trigger more API calls - for path in ["/", "/devices", "/clients", "/settings", "/insights", "/topology", "/dashboard"]: - try: - page.goto(base_url + path, wait_until="domcontentloaded", timeout=10000) - page.wait_for_timeout(2000) - except Exception: - pass - - browser.close() - - # Build markdown - today = date.today().isoformat() - entries = sorted(captured.values(), key=lambda e: e["url"]) - by_group: dict[str, list] = {} - for e in entries: - g = run_group(e["pathname"]) - by_group.setdefault(g, []).append(e) - - lines = [ - "# API Endpoints (headless discovery)", - "", - f"- **Date**: {today}", - f"- **Controller**: {base_url}", - f"- **Total unique requests**: {len(entries)}", - "", - "---", - "", - ] - for g in GROUP_ORDER: - list_ = by_group.get(g) - if not list_: - continue - lines.append(f"## {GROUP_TITLES.get(g, g)}") - lines.append("") - for e in list_: - u = urlparse(e["url"]) - path_only = (u.path or "") + (("?" + u.query) if u.query else "") - lines.append(f"- `{e['method']} {path_only}`") - lines.append("") - - lines.extend(["---", "", "## Sample request headers (first request)", ""]) - if entries and entries[0].get("request_headers"): - lines.append("```") - for k, v in entries[0]["request_headers"].items(): - kl = k.lower() - if kl.startswith("x-") or kl in ("accept", "authorization"): - lines.append(f"{k}: {v}") - lines.append("```") - - out_path = OUTPUT_DIR / f"API_ENDPOINTS_HEADLESS_{today}.md" - OUTPUT_DIR.mkdir(parents=True, exist_ok=True) - out_path.write_text("\n".join(lines) + "\n", encoding="utf-8") - print("Wrote", out_path, f"({len(entries)} unique endpoints)") - - -if __name__ == "__main__": - main() diff --git a/tools/endpoint-discovery/requirements.txt b/tools/endpoint-discovery/requirements.txt deleted file mode 100644 index 49711953..00000000 --- a/tools/endpoint-discovery/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -playwright>=1.49.0 -# Optional: for .env support -# python-dotenv>=1.0.0