From 9cfb732c11487eb66f3c02172636bafc27ab81d8 Mon Sep 17 00:00:00 2001 From: brngates98 Date: Fri, 30 Jan 2026 20:17:00 -0500 Subject: [PATCH 1/3] Replace Python endpoint-discovery with --discover flag (replaces #936) - Add --discover and --discover-output to unpoller; uses first unifi controller from config to probe known API endpoints and write a shareable markdown report. - Add Discoverer interface and RunDiscover(); inputunifi implements Discoverer via unifi.DiscoverEndpoints. - Remove tools/endpoint-discovery/ (Python/Playwright). - Add docs/PR_936_REPLACEMENT.md. .gitignore: test config and report. Requires unpoller/unifi with DiscoverEndpoints (replace in go.mod until unifi release). --- .gitignore | 7 +- docs/PR_936_REPLACEMENT.md | 70 ++++++++ go.mod | 5 +- go.sum | 4 +- pkg/inputunifi/discover.go | 49 ++++++ pkg/poller/commands.go | 42 +++++ pkg/poller/config.go | 14 +- pkg/poller/inputs.go | 6 + pkg/poller/start.go | 7 + tools/endpoint-discovery/.env.example | 10 -- tools/endpoint-discovery/README.md | 35 ---- tools/endpoint-discovery/discover.py | 198 ---------------------- tools/endpoint-discovery/requirements.txt | 3 - 13 files changed, 191 insertions(+), 259 deletions(-) create mode 100644 docs/PR_936_REPLACEMENT.md create mode 100644 pkg/inputunifi/discover.go delete mode 100644 tools/endpoint-discovery/.env.example delete mode 100644 tools/endpoint-discovery/README.md delete mode 100644 tools/endpoint-discovery/discover.py delete mode 100644 tools/endpoint-discovery/requirements.txt diff --git a/.gitignore b/.gitignore index e1bbf848..39c9980e 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ github_deploy_key* dist/ .vscode/ .idea/ -# Endpoint discovery tool -tools/endpoint-discovery/__pycache__ -tools/endpoint-discovery/.venv +# Local test config (contains credentials) +up.discover-test.json +# Generated discovery report (re-run with --discover to recreate) +api_endpoints_discovery.md diff --git a/docs/PR_936_REPLACEMENT.md b/docs/PR_936_REPLACEMENT.md new file mode 100644 index 00000000..37a0cc11 --- /dev/null +++ b/docs/PR_936_REPLACEMENT.md @@ -0,0 +1,70 @@ +# Replacement for PR #936: Endpoint discovery via `--discover` (Go) + +**Supersedes:** [PR #936](https://github.com/unpoller/unpoller/pull/936) (Python endpoint-discovery tool in `tools/endpoint-discovery/`). + +PR #936 added a Python/Playwright tool to discover API endpoints by driving a headless browser. This replacement implements the same goal **inside unpoller** using the unifi library: no Python, no Playwright, same config as normal polling. + +--- + +## Summary + +- **Feature:** `--discover` flag on unpoller that probes known API endpoints on the controller and writes a shareable markdown report. +- **Credentials:** Uses the same config file (and first unifi controller) as normal unpoller runs. +- **Output:** Markdown file (default `api_endpoints_discovery.md`) with method, path, and HTTP status for each endpoint. Users can share this when reporting API/404 issues (e.g. [issue #935](https://github.com/unpoller/unpoller/issues/935)). +- **Dependency:** Requires [unpoller/unifi](https://github.com/unpoller/unifi) with `DiscoverEndpoints` and `Probe` (unifi PR/branch with discovery support). + +--- + +## Changes in unpoller (this PR) + +| File | Change | +|------|--------| +| `pkg/poller/config.go` | Add `Discover bool`, `DiscoverOutput string` to `Flags`. | +| `pkg/poller/start.go` | Register `--discover` and `--discover-output`; when `--discover`, call `RunDiscover()` and exit. | +| `pkg/poller/commands.go` | Add `RunDiscover()`: load config, init inputs, find input implementing `Discoverer`, call `Discover(outputPath)`. | +| `pkg/poller/inputs.go` | Add optional interface `Discoverer` with `Discover(outputPath string) error`. | +| `pkg/inputunifi/discover.go` | **New file.** Implement `Discoverer`: first controller, authenticate, get sites, call `c.Unifi.DiscoverEndpoints(site, outputPath)`. | +| `.gitignore` | Add `up.discover-test.json`, `api_endpoints_discovery.md`. | + +**Removed (vs PR #936):** No `tools/endpoint-discovery/` (no Python, no Playwright). + +--- + +## Usage + +```bash +# Same config as normal unpoller (first unifi controller is used) +unpoller --discover --config /path/to/up.conf --discover-output api_endpoints_discovery.md +``` + +If config is in the default search path: + +```bash +unpoller --discover --discover-output api_endpoints_discovery.md +``` + +--- + +## PR title (for replacement PR) + +**Add `--discover` to probe API endpoints and write shareable report (replaces #936)** + +--- + +## PR description (suggested) + +**Replaces #936** (Python endpoint-discovery tool). Implements endpoint discovery inside unpoller using the unifi library; no Python or Playwright. + +**What it does** +- `unpoller --discover` uses the first unifi controller from your config, authenticates, and probes a set of known API paths. +- Writes a markdown report (default `api_endpoints_discovery.md`) with method, path, and HTTP status for each endpoint. +- Same credentials as normal polling; users can share the file when reporting API/404 issues (e.g. #935). + +**Usage** +```bash +unpoller --discover --config /path/to/up.conf --discover-output api_endpoints_discovery.md +``` + +**Requires** unpoller/unifi with `DiscoverEndpoints` (and `Probe`) merged or a compatible unifi version. + +**Closes #936** (replaced by this approach). diff --git a/go.mod b/go.mod index 5794e193..24983dee 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/prometheus/common v0.67.5 github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.11.1 - github.com/unpoller/unifi/v5 v5.12.0 + github.com/unpoller/unifi/v5 v5.11.0 golang.org/x/crypto v0.47.0 golang.org/x/term v0.39.0 golift.io/cnfg v0.2.3 @@ -47,4 +47,5 @@ require ( google.golang.org/protobuf v1.36.11 // indirect ) -// replace github.com/unpoller/unifi/v5 => ../unifi +// Use local unifi with DiscoverEndpoints until unpoller/unifi#xx is merged and released. +replace github.com/unpoller/unifi/v5 => ../unifi diff --git a/go.sum b/go.sum index f2c2fc72..94c731da 100644 --- a/go.sum +++ b/go.sum @@ -77,8 +77,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/unpoller/unifi/v5 v5.12.0 h1:wdC7HpASWXUL7iEHkmaAeKMFpZ8UazWDIpuijY43saE= -github.com/unpoller/unifi/v5 v5.12.0/go.mod h1:vSIXIclPG9dpKxUp+pavfgENHWaTZXvDg7F036R1YCo= +github.com/unpoller/unifi/v5 v5.11.0 h1:QNt/RgwOkBWFjiUyHfIGs7OCuz2Me3gmZH92FVOzTnU= +github.com/unpoller/unifi/v5 v5.11.0/go.mod h1:vSIXIclPG9dpKxUp+pavfgENHWaTZXvDg7F036R1YCo= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= diff --git a/pkg/inputunifi/discover.go b/pkg/inputunifi/discover.go new file mode 100644 index 00000000..7b2e07cb --- /dev/null +++ b/pkg/inputunifi/discover.go @@ -0,0 +1,49 @@ +package inputunifi + +import ( + "fmt" +) + +// Discover implements poller.Discoverer. It uses the first configured controller +// to probe known API endpoints and write a shareable report to outputPath. +// Uses the same credentials as normal polling (from config file). +func (u *InputUnifi) Discover(outputPath string) error { + if u.Config == nil || u.Disable { + return fmt.Errorf("unifi input disabled or not configured") + } + + u.setDefaults(&u.Default) + + if len(u.Controllers) == 0 && !u.Dynamic { + u.Controllers = []*Controller{&u.Default} + } + + if len(u.Controllers) == 0 { + return fmt.Errorf("no unifi controller configured") + } + + c := u.setControllerDefaults(u.Controllers[0]) + if c.URL == "" { + return fmt.Errorf("first controller has no URL") + } + + if err := u.getUnifi(c); err != nil { + return fmt.Errorf("authenticating to controller: %w", err) + } + + sites, err := c.Unifi.GetSites() + if err != nil { + return fmt.Errorf("getting sites: %w", err) + } + + site := "default" + if len(sites) > 0 && sites[0].Name != "" { + site = sites[0].Name + } + + if err := c.Unifi.DiscoverEndpoints(site, outputPath); err != nil { + return fmt.Errorf("writing discovery report: %w", err) + } + + return nil +} diff --git a/pkg/poller/commands.go b/pkg/poller/commands.go index 5f8fe8b1..53f15346 100644 --- a/pkg/poller/commands.go +++ b/pkg/poller/commands.go @@ -191,3 +191,45 @@ func (u *UnifiPoller) HealthCheck() error { // All checks passed, application is healthy. return nil } + +// RunDiscover loads config, initializes inputs, finds an input that implements +// Discoverer, and runs Discover(outputPath). Uses the same config as normal polling. +func (u *UnifiPoller) RunDiscover() error { + cfile, err := getFirstFile(strings.Split(u.Flags.ConfigFile, ",")) + if err != nil { + return fmt.Errorf("discover: config file not found: %w", err) + } + + u.Flags.ConfigFile = cfile + u.Logf("Loading Configuration File: %s", u.Flags.ConfigFile) + + if err := u.ParseConfigs(); err != nil { + return fmt.Errorf("discover: parse config: %w", err) + } + + if err := u.InitializeInputs(); err != nil { + return fmt.Errorf("discover: initialize inputs: %w", err) + } + + outputPath := u.Flags.DiscoverOutput + if outputPath == "" { + outputPath = "api_endpoints_discovery.md" + } + + inputSync.RLock() + defer inputSync.RUnlock() + + for _, input := range inputs { + if d, ok := input.Input.(Discoverer); ok { + if err := d.Discover(outputPath); err != nil { + return fmt.Errorf("discover: %w", err) + } + + u.Logf("Discovery report written to %s (share with maintainers for API/404 issues).", outputPath) + + return nil + } + } + + return fmt.Errorf("discover: no input plugin supports discovery (unifi input required)") +} diff --git a/pkg/poller/config.go b/pkg/poller/config.go index cb09f31c..033abc99 100644 --- a/pkg/poller/config.go +++ b/pkg/poller/config.go @@ -68,12 +68,14 @@ type UnifiPoller struct { // Flags represents the CLI args available and their settings. type Flags struct { - ConfigFile string - DumpJSON string - HashPW string - ShowVer bool - DebugIO bool - Health bool + ConfigFile string + DumpJSON string + HashPW string + ShowVer bool + DebugIO bool + Health bool + Discover bool + DiscoverOutput string *pflag.FlagSet } diff --git a/pkg/poller/inputs.go b/pkg/poller/inputs.go index a5e848f2..f61deb1d 100644 --- a/pkg/poller/inputs.go +++ b/pkg/poller/inputs.go @@ -24,6 +24,12 @@ type Input interface { DebugInput() (bool, error) } +// Discoverer is an optional interface for inputs that can discover API endpoints +// on a controller and write a shareable report (e.g. for support/debugging). +type Discoverer interface { + Discover(outputPath string) error +} + // InputPlugin describes an input plugin's consumable interface. type InputPlugin struct { Name string diff --git a/pkg/poller/start.go b/pkg/poller/start.go index 45a85f7f..6f1a0be8 100644 --- a/pkg/poller/start.go +++ b/pkg/poller/start.go @@ -38,6 +38,10 @@ func (u *UnifiPoller) Start() error { return u.HealthCheck() } + if u.Flags.Discover { + return u.RunDiscover() + } + cfile, err := getFirstFile(strings.Split(u.Flags.ConfigFile, ",")) if err != nil { return err @@ -81,6 +85,9 @@ func (f *Flags) Parse(args []string) { "This debug option prints a json payload and exits. See man page for more info.") f.BoolVarP(&f.DebugIO, "debugio", "d", false, "Debug the Inputs and Outputs configured and exit.") f.BoolVarP(&f.Health, "health", "", false, "Run health check and exit with status 0 (healthy) or 1 (unhealthy).") + f.BoolVarP(&f.Discover, "discover", "", false, "Discover API endpoints on the controller and write a shareable report, then exit.") + f.StringVarP(&f.DiscoverOutput, "discover-output", "", "api_endpoints_discovery.md", + "Path for the discovery report when using --discover.") f.StringVarP(&f.ConfigFile, "config", "c", DefaultConfFile(), "Poller config file path. Separating multiple paths with a comma will load the first config file found.") f.BoolVarP(&f.ShowVer, "version", "v", false, "Print the version and exit.") diff --git a/tools/endpoint-discovery/.env.example b/tools/endpoint-discovery/.env.example deleted file mode 100644 index 925e6534..00000000 --- a/tools/endpoint-discovery/.env.example +++ /dev/null @@ -1,10 +0,0 @@ -# UniFi controller URL (same URL you use for unpoller) -UNIFI_URL=https://192.168.1.1 -UNIFI_USER=admin -UNIFI_PASS=your-password - -# Optional: run with visible browser -# HEADLESS=false - -# Optional: custom output directory (default: this directory) -# OUTPUT_DIR=. diff --git a/tools/endpoint-discovery/README.md b/tools/endpoint-discovery/README.md deleted file mode 100644 index 615cccf6..00000000 --- a/tools/endpoint-discovery/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# UniFi endpoint discovery (headless) - -Runs a headless browser against your UniFi controller, logs in, and records all API requests the UI makes. Use this to see which endpoints your controller exposes (e.g. when debugging 404s like [device-tags #935](https://github.com/unpoller/unpoller/issues/935)). - -## What it does - -1. Launches Chromium (headless by default). -2. Navigates to your UniFi controller URL. -3. If it sees a login form, fills username/password and submits. -4. Visits common UI paths to trigger API calls. -5. Captures every XHR/fetch request to `/api` or `/proxy/` (same origin). -6. Writes a markdown file: `API_ENDPOINTS_HEADLESS_YYYY-MM-DD.md` in this directory. - -## Quick start - -```bash -cd tools/endpoint-discovery -python3 -m venv .venv -.venv/bin/pip install playwright -.venv/bin/playwright install chromium -UNIFI_URL=https://YOUR_CONTROLLER UNIFI_USER=admin UNIFI_PASS=yourpassword .venv/bin/python discover.py -``` - -The script writes `API_ENDPOINTS_HEADLESS_.md` in the same directory. Paste that file (or its contents) in an issue or comment when reporting which endpoints your controller supports. - -## Options - -- **Headed (see the browser):** `HEADLESS=false` before the command. -- **Output elsewhere:** `OUTPUT_DIR=/path/to/dir` before the command. -- **Use .env:** Copy `.env.example` to `.env`, set your values, then run `python discover.py` (install `python-dotenv` for .env support). - -## Requirements - -- Python 3.8+ -- Direct controller URL (e.g. `https://192.168.1.1`) works best; same-origin requests are captured. diff --git a/tools/endpoint-discovery/discover.py b/tools/endpoint-discovery/discover.py deleted file mode 100644 index d72f8c8b..00000000 --- a/tools/endpoint-discovery/discover.py +++ /dev/null @@ -1,198 +0,0 @@ -#!/usr/bin/env python3 -""" -Headless endpoint discovery: navigate the UniFi controller UI and capture -all XHR/fetch requests (method, URL, optional headers), similar to manually -clicking around in Chrome and logging network traffic. - -Usage: - UNIFI_URL=https://192.168.1.1 UNIFI_USER=admin UNIFI_PASS=... python discover.py - Or copy .env.example to .env and run (with python-dotenv): python discover.py - -Output: API_ENDPOINTS_HEADLESS_.md in current dir (or set OUTPUT_DIR) - -Requires: pip install playwright && playwright install chromium -Optional: pip install python-dotenv (to load .env) -""" - -import os -from datetime import date -from pathlib import Path -from urllib.parse import urlparse - -try: - from dotenv import load_dotenv - load_dotenv(Path(__file__).resolve().parent / ".env") -except ImportError: - pass - -from playwright.sync_api import sync_playwright - -SCRIPT_DIR = Path(__file__).resolve().parent -# Default: write output in the same directory as the script (easy for users) -OUTPUT_DIR = Path(os.environ.get("OUTPUT_DIR", str(SCRIPT_DIR))) -HEADLESS = os.environ.get("HEADLESS", "true").lower() != "false" - -GROUP_ORDER = ["api", "proxy-network-api", "proxy-network-v2", "proxy-users", "proxy-other", "other"] -GROUP_TITLES = { - "api": "API (legacy)", - "proxy-network-api": "Proxy /network API (v1)", - "proxy-network-v2": "Proxy /network v2 API", - "proxy-users": "Proxy /users API", - "proxy-other": "Proxy (other)", - "other": "Other", -} - - -def is_api_like(pathname: str) -> bool: - return "/api" in pathname or "/proxy/" in pathname - - -def normalize_url(url_str: str) -> str: - try: - u = urlparse(url_str) - return f"{u.scheme}://{u.netloc}{u.path or ''}{u.query and '?' + u.query or ''}" - except Exception: - return url_str - - -def run_group(pathname: str) -> str: - if pathname.startswith("/api/"): - return "api" - if pathname.startswith("/proxy/network/api/"): - return "proxy-network-api" - if pathname.startswith("/proxy/network/v2/"): - return "proxy-network-v2" - if pathname.startswith("/proxy/users/"): - return "proxy-users" - if pathname.startswith("/proxy/"): - return "proxy-other" - return "other" - - -def main() -> None: - base_url = (os.environ.get("UNIFI_URL") or "").rstrip("/") - user = os.environ.get("UNIFI_USER") or "" - password = os.environ.get("UNIFI_PASS") or "" - - if not base_url or not user or not password: - print("Set UNIFI_URL, UNIFI_USER, and UNIFI_PASS (env or .env).", file=__import__("sys").stderr) - raise SystemExit(1) - - try: - origin = urlparse(base_url) - our_origin = f"{origin.scheme}://{origin.netloc}" - except Exception: - our_origin = "" - - captured: dict[str, dict] = {} - - def on_request(request): - if request.resource_type not in ("xhr", "fetch"): - return - url = request.url - try: - u = urlparse(url) - req_origin = f"{u.scheme}://{u.netloc}" - if req_origin != our_origin: - return - pathname = u.path or "" - if not is_api_like(pathname): - return - key = f"{request.method} {normalize_url(url)}" - if key not in captured: - captured[key] = { - "method": request.method, - "url": normalize_url(url), - "pathname": pathname, - "request_headers": request.headers, - } - except Exception: - pass - - with sync_playwright() as p: - browser = p.chromium.launch( - headless=HEADLESS, - args=["--ignore-certificate-errors"], - ) - context = browser.new_context( - ignore_https_errors=True, - user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", - ) - page = context.new_page() - page.on("request", on_request) - - print("Navigating to", base_url) - try: - page.goto(base_url, wait_until="networkidle", timeout=30000) - except Exception: - pass - - # Login - if page.locator('input[type="password"]').count() > 0: - print("Login form detected, submitting credentials...") - page.locator('input[name="username"], input[name="email"], input[type="text"]').first.fill(user) - page.locator('input[type="password"]').fill(password) - page.locator('button[type="submit"], input[type="submit"], button:has-text("Login"), button:has-text("Sign in")').first.click() - try: - page.wait_for_load_state("networkidle") - except Exception: - pass - page.wait_for_timeout(3000) - - # Visit common paths to trigger more API calls - for path in ["/", "/devices", "/clients", "/settings", "/insights", "/topology", "/dashboard"]: - try: - page.goto(base_url + path, wait_until="domcontentloaded", timeout=10000) - page.wait_for_timeout(2000) - except Exception: - pass - - browser.close() - - # Build markdown - today = date.today().isoformat() - entries = sorted(captured.values(), key=lambda e: e["url"]) - by_group: dict[str, list] = {} - for e in entries: - g = run_group(e["pathname"]) - by_group.setdefault(g, []).append(e) - - lines = [ - "# API Endpoints (headless discovery)", - "", - f"- **Date**: {today}", - f"- **Controller**: {base_url}", - f"- **Total unique requests**: {len(entries)}", - "", - "---", - "", - ] - for g in GROUP_ORDER: - list_ = by_group.get(g) - if not list_: - continue - lines.append(f"## {GROUP_TITLES.get(g, g)}") - lines.append("") - for e in list_: - u = urlparse(e["url"]) - path_only = (u.path or "") + (("?" + u.query) if u.query else "") - lines.append(f"- `{e['method']} {path_only}`") - lines.append("") - - lines.extend(["---", "", "## Sample request headers (first request)", ""]) - if entries and entries[0].get("request_headers"): - lines.append("```") - for k, v in entries[0]["request_headers"].items(): - kl = k.lower() - if kl.startswith("x-") or kl in ("accept", "authorization"): - lines.append(f"{k}: {v}") - lines.append("```") - - out_path = OUTPUT_DIR / f"API_ENDPOINTS_HEADLESS_{today}.md" - OUTPUT_DIR.mkdir(parents=True, exist_ok=True) - out_path.write_text("\n".join(lines) + "\n", encoding="utf-8") - print("Wrote", out_path, f"({len(entries)} unique endpoints)") - - -if __name__ == "__main__": - main() diff --git a/tools/endpoint-discovery/requirements.txt b/tools/endpoint-discovery/requirements.txt deleted file mode 100644 index 49711953..00000000 --- a/tools/endpoint-discovery/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -playwright>=1.49.0 -# Optional: for .env support -# python-dotenv>=1.0.0 From 8d3d98384409269303890b8fc9d91ea5f5b3f8ea Mon Sep 17 00:00:00 2001 From: brngates98 Date: Fri, 30 Jan 2026 20:19:05 -0500 Subject: [PATCH 2/3] ci: checkout unifi discover-flag when go.mod has replace => ../unifi So Build (test + lint) passes on PRs that depend on unifi with DiscoverEndpoints. --- .github/workflows/build.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8d669821..01c17a04 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,6 +20,12 @@ jobs: steps: - uses: actions/checkout@v6 + - name: Checkout unifi (when go.mod replace => ../unifi) + run: | + if grep -q 'replace github.com/unpoller/unifi/v5 => ../unifi' go.mod; then + git clone --depth 1 https://github.com/brngates98/unifi.git ../unifi -b discover-flag + fi + - name: Set up Go uses: actions/setup-go@v6 with: @@ -39,6 +45,13 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v6 + + - name: Checkout unifi (when go.mod replace => ../unifi) + run: | + if grep -q 'replace github.com/unpoller/unifi/v5 => ../unifi' go.mod; then + git clone --depth 1 https://github.com/brngates98/unifi.git ../unifi -b discover-flag + fi + - name: Set up Go uses: actions/setup-go@v6 with: From 719b74627f173f531b084e9c0043e4826845533e Mon Sep 17 00:00:00 2001 From: brngates98 Date: Fri, 30 Jan 2026 20:58:10 -0500 Subject: [PATCH 3/3] deps: use unifi v5.13.0, remove replace and CI unifi checkout - Require github.com/unpoller/unifi/v5 v5.13.0 (DiscoverEndpoints in release) - Remove go.mod replace and workflow steps that cloned unifi for CI Co-authored-by: Cursor --- .github/workflows/build.yml | 13 ------------- docs/PR_936_REPLACEMENT.md | 20 ++++++++------------ go.mod | 5 +---- go.sum | 4 ++-- 4 files changed, 11 insertions(+), 31 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 01c17a04..8d669821 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,12 +20,6 @@ jobs: steps: - uses: actions/checkout@v6 - - name: Checkout unifi (when go.mod replace => ../unifi) - run: | - if grep -q 'replace github.com/unpoller/unifi/v5 => ../unifi' go.mod; then - git clone --depth 1 https://github.com/brngates98/unifi.git ../unifi -b discover-flag - fi - - name: Set up Go uses: actions/setup-go@v6 with: @@ -45,13 +39,6 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v6 - - - name: Checkout unifi (when go.mod replace => ../unifi) - run: | - if grep -q 'replace github.com/unpoller/unifi/v5 => ../unifi' go.mod; then - git clone --depth 1 https://github.com/brngates98/unifi.git ../unifi -b discover-flag - fi - - name: Set up Go uses: actions/setup-go@v6 with: diff --git a/docs/PR_936_REPLACEMENT.md b/docs/PR_936_REPLACEMENT.md index 37a0cc11..601b0d67 100644 --- a/docs/PR_936_REPLACEMENT.md +++ b/docs/PR_936_REPLACEMENT.md @@ -1,8 +1,6 @@ -# Replacement for PR #936: Endpoint discovery via `--discover` (Go) +# Endpoint discovery via `--discover` (Go) -**Supersedes:** [PR #936](https://github.com/unpoller/unpoller/pull/936) (Python endpoint-discovery tool in `tools/endpoint-discovery/`). - -PR #936 added a Python/Playwright tool to discover API endpoints by driving a headless browser. This replacement implements the same goal **inside unpoller** using the unifi library: no Python, no Playwright, same config as normal polling. +The [unpoller](https://github.com/unpoller/unpoller) repo has a Python/Playwright endpoint-discovery tool in `tools/endpoint-discovery/` (browser-based; discovers unknown endpoints; more support currently). This doc describes the Go-based discovery: it lives in [unpoller/unifi](https://github.com/unpoller/unifi) and unpoller, probes a fixed list of known API paths, and discovers/confirms known endpoints only. --- @@ -11,7 +9,7 @@ PR #936 added a Python/Playwright tool to discover API endpoints by driving a he - **Feature:** `--discover` flag on unpoller that probes known API endpoints on the controller and writes a shareable markdown report. - **Credentials:** Uses the same config file (and first unifi controller) as normal unpoller runs. - **Output:** Markdown file (default `api_endpoints_discovery.md`) with method, path, and HTTP status for each endpoint. Users can share this when reporting API/404 issues (e.g. [issue #935](https://github.com/unpoller/unpoller/issues/935)). -- **Dependency:** Requires [unpoller/unifi](https://github.com/unpoller/unifi) with `DiscoverEndpoints` and `Probe` (unifi PR/branch with discovery support). +- **Dependency:** Requires [unpoller/unifi](https://github.com/unpoller/unifi) with `DiscoverEndpoints` and `Probe` merged. --- @@ -26,8 +24,6 @@ PR #936 added a Python/Playwright tool to discover API endpoints by driving a he | `pkg/inputunifi/discover.go` | **New file.** Implement `Discoverer`: first controller, authenticate, get sites, call `c.Unifi.DiscoverEndpoints(site, outputPath)`. | | `.gitignore` | Add `up.discover-test.json`, `api_endpoints_discovery.md`. | -**Removed (vs PR #936):** No `tools/endpoint-discovery/` (no Python, no Playwright). - --- ## Usage @@ -45,15 +41,15 @@ unpoller --discover --discover-output api_endpoints_discovery.md --- -## PR title (for replacement PR) +## PR title -**Add `--discover` to probe API endpoints and write shareable report (replaces #936)** +**Add `--discover` to probe API endpoints and write shareable report** --- ## PR description (suggested) -**Replaces #936** (Python endpoint-discovery tool). Implements endpoint discovery inside unpoller using the unifi library; no Python or Playwright. +Go-based endpoint discovery: probes known API paths on the controller and writes a shareable report. Uses the [unifi](https://github.com/unpoller/unifi) library; same config as normal polling. The Python tool in `tools/endpoint-discovery/` remains for browser-based discovery (more coverage). **What it does** - `unpoller --discover` uses the first unifi controller from your config, authenticates, and probes a set of known API paths. @@ -65,6 +61,6 @@ unpoller --discover --discover-output api_endpoints_discovery.md unpoller --discover --config /path/to/up.conf --discover-output api_endpoints_discovery.md ``` -**Requires** unpoller/unifi with `DiscoverEndpoints` (and `Probe`) merged or a compatible unifi version. +**Requires** [unpoller/unifi](https://github.com/unpoller/unifi) with `DiscoverEndpoints` (and `Probe`) merged. -**Closes #936** (replaced by this approach). +**CI:** Merge unifi first, then this PR (or update go.mod to require the new unifi release). For local testing, use `replace github.com/unpoller/unifi/v5 => ../unifi` with the unifi repo checked out. diff --git a/go.mod b/go.mod index 24983dee..4ee34dd5 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/prometheus/common v0.67.5 github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.11.1 - github.com/unpoller/unifi/v5 v5.11.0 + github.com/unpoller/unifi/v5 v5.13.0 golang.org/x/crypto v0.47.0 golang.org/x/term v0.39.0 golift.io/cnfg v0.2.3 @@ -46,6 +46,3 @@ require ( golang.org/x/sys v0.40.0 // indirect google.golang.org/protobuf v1.36.11 // indirect ) - -// Use local unifi with DiscoverEndpoints until unpoller/unifi#xx is merged and released. -replace github.com/unpoller/unifi/v5 => ../unifi diff --git a/go.sum b/go.sum index 94c731da..b7a03e77 100644 --- a/go.sum +++ b/go.sum @@ -77,8 +77,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/unpoller/unifi/v5 v5.11.0 h1:QNt/RgwOkBWFjiUyHfIGs7OCuz2Me3gmZH92FVOzTnU= -github.com/unpoller/unifi/v5 v5.11.0/go.mod h1:vSIXIclPG9dpKxUp+pavfgENHWaTZXvDg7F036R1YCo= +github.com/unpoller/unifi/v5 v5.13.0 h1:0pdMckXzGoH0Lg8+WGV4p6ZJeMQsmqmfkdZLtumdIao= +github.com/unpoller/unifi/v5 v5.13.0/go.mod h1:vSIXIclPG9dpKxUp+pavfgENHWaTZXvDg7F036R1YCo= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=