diff --git a/.gitignore b/.gitignore index 15b7129..f543932 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,6 @@ knx-workspace/ .idea/ .vscode/ *.swp + +# local corpus map: real project paths, never committed +tools/corpus_map.json diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 74d9a81..760735f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,6 +61,31 @@ python tests/test_pipeline.py CI runs the smoke test on Python 3.10–3.12 for every push and PR. +### Local corpus guard + +Public CI only sees the synthetic fixtures in `tests/`. Most real regressions show up on real +ETS projects, which are confidential and never leave the maintainer's machine. `tools/corpus_check.py` +closes that gap locally: it runs every project of a private corpus through the full pipeline +(parse -> checks -> HA YAML -> entity suggestions) and compares the resulting counts against a +recorded baseline. + +```bash +cp tools/corpus_map.example.json tools/corpus_map.json # once: point it at local .knxproj files +python tools/corpus_check.py # exits 1 and prints every metric that moved +python tools/corpus_check.py --tolerance 2 # allow 2% drift per metric +python tools/corpus_check.py --update # re-record the baseline after an intended change +``` + +`tools/corpus_map.json` is gitignored, because real project paths carry client names. The corpus +root defaults to a sibling `demo-home-for-friend/` directory and can be overridden with +`NICKOL_KNX_CORPUS_DIR`. Without a map the script exits 2 and does nothing. + +`tools/corpus_baseline.json` is committed, but it holds numbers only: counts, severity and code +histograms, and a short digest of each source file. No project names, no paths, no addresses. +The corpus itself stays out of the repository. Contributors without the corpus can skip this +step; the maintainer runs it before every release, and `--update` belongs in the same commit as +the change that moved the numbers, so the diff shows what moved and why. + ## Code of conduct Be kind and constructive. This is a hobby/community project; assume good faith. diff --git a/docs/RELEASE.md b/docs/RELEASE.md index bf657e0..b856335 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -32,6 +32,16 @@ uvx twine check dist/* Both artifacts must say PASSED. +Run the local corpus guard before the build — it is the only check that sees real ETS projects: + +```bash +python tools/corpus_check.py +``` + +It must exit 0. If it reports drift, either the change is a regression, or the new numbers are +intended and `tools/corpus_check.py --update` belongs in a commit of its own before the release. +Do not release on unexplained drift. + ## 3. Upload to PyPI (owner, needs the API token) ```bash diff --git a/tools/corpus_baseline.json b/tools/corpus_baseline.json new file mode 100644 index 0000000..840c87a --- /dev/null +++ b/tools/corpus_baseline.json @@ -0,0 +1,357 @@ +{ + "projects": { + "demo": { + "categories": { + "diagnostics": 5, + "energy": 4, + "hvac": 78, + "lighting": 91, + "scene": 4, + "sensor": 30, + "shutter": 24, + "unknown": 2 + }, + "devices": 0, + "file_digest": "77273cacf3f9", + "findings_by_code": { + "central_macro_no_status": 2, + "duplicate_name": 1, + "inconsistent_dpt": 1, + "missing_dpt": 1, + "missing_status_address": 2, + "missing_value_status": 2, + "scene_no_status": 4, + "subdpt_suspect": 1 + }, + "findings_by_severity": { + "error": 1, + "info": 6, + "warning": 7 + }, + "group_addresses": 238, + "ha_entities": { + "binary_sensor": 13, + "climate": 6, + "cover": 6, + "light": 12, + "sensor": 41, + "switch": 19 + }, + "ha_review_items": 57, + "suggestion_hints": { + "channels": 0, + "diagnostics_skipped": 0, + "duplicates_skipped": 0, + "fallback": 0, + "pseudo_channels": 0, + "review": 0, + "sensors": 0, + "skipped_fb_covered": 0, + "structural": 0, + "subunits": 0, + "unwired_flagged": 0 + }, + "suggestions_by_platform": {} + }, + "hdl-ets6": { + "categories": { + "diagnostics": 5, + "hvac": 206, + "lighting": 425, + "sensor": 34, + "shutter": 135, + "unknown": 54 + }, + "devices": 53, + "file_digest": "a360d74c65ef", + "findings_by_code": { + "duplicate_name": 156, + "inconsistent_dpt": 15, + "line_without_coupler": 1, + "missing_dpt": 107, + "missing_status_address": 92, + "missing_value_status": 264, + "name_too_short": 1, + "relative_only_dimming": 20, + "reserve_without_dpt": 81, + "status_pairing_summary": 1 + }, + "findings_by_severity": { + "error": 107, + "info": 83, + "warning": 548 + }, + "group_addresses": 859, + "ha_entities": { + "binary_sensor": 66, + "climate": 3, + "cover": 1, + "expose": 1, + "light": 142, + "sensor": 55, + "switch": 85 + }, + "ha_review_items": 408, + "suggestion_hints": { + "channels": 229, + "diagnostics_skipped": 2, + "duplicates_skipped": 4, + "fallback": 0, + "pseudo_channels": 151, + "review": 41, + "sensors": 37, + "skipped_fb_covered": 0, + "structural": 80, + "subunits": 0, + "unwired_flagged": 29 + }, + "suggestions_by_platform": { + "binary_sensor": 14, + "light": 79, + "sensor": 23, + "switch": 1 + } + }, + "mixed-small": { + "categories": { + "hvac": 58, + "lighting": 95, + "sensor": 2, + "unknown": 12 + }, + "devices": 14, + "file_digest": "bfaf0f366105", + "findings_by_code": { + "central_macro_no_status": 5, + "line_without_coupler": 1, + "missing_dpt": 21, + "missing_status_address": 4, + "missing_value_status": 2, + "status_pairing_summary": 1 + }, + "findings_by_severity": { + "error": 21, + "info": 7, + "warning": 6 + }, + "group_addresses": 167, + "ha_entities": { + "expose": 1, + "light": 12, + "sensor": 19, + "switch": 20 + }, + "ha_review_items": 69, + "suggestion_hints": { + "channels": 52, + "diagnostics_skipped": 0, + "duplicates_skipped": 0, + "fallback": 0, + "pseudo_channels": 23, + "review": 7, + "sensors": 23, + "skipped_fb_covered": 0, + "structural": 7, + "subunits": 0, + "unwired_flagged": 1 + }, + "suggestions_by_platform": { + "binary_sensor": 8, + "climate": 6, + "light": 1, + "sensor": 15 + } + }, + "zennio-flat": { + "categories": { + "diagnostics": 47, + "energy": 39, + "hvac": 201, + "lighting": 179, + "scene": 4, + "sensor": 70, + "shutter": 49, + "unknown": 96 + }, + "devices": 41, + "file_digest": "d980991b0f9e", + "findings_by_code": { + "central_macro_no_status": 1, + "dpt_mismatch_co": 11, + "dpt_on_logic_object": 1, + "duplicate_name": 6, + "inconsistent_dpt": 2, + "line_without_coupler": 1, + "missing_dpt": 6, + "missing_status_address": 11, + "missing_value_status": 16, + "reserve_without_dpt": 23, + "safety_input_no_status": 22, + "status_pairing_summary": 1, + "subdpt_suspect": 7 + }, + "findings_by_severity": { + "error": 6, + "info": 49, + "warning": 53 + }, + "group_addresses": 685, + "ha_entities": { + "binary_sensor": 49, + "climate": 7, + "cover": 7, + "expose": 1, + "light": 15, + "sensor": 111, + "switch": 141 + }, + "ha_review_items": 324, + "suggestion_hints": { + "channels": 179, + "diagnostics_skipped": 21, + "duplicates_skipped": 4, + "fallback": 0, + "pseudo_channels": 6, + "review": 57, + "sensors": 70, + "skipped_fb_covered": 0, + "structural": 79, + "subunits": 71, + "unwired_flagged": 0 + }, + "suggestions_by_platform": { + "binary_sensor": 35, + "climate": 13, + "cover": 4, + "light": 56, + "sensor": 35, + "switch": 6 + } + }, + "zennio-large": { + "categories": { + "diagnostics": 150, + "energy": 1, + "hvac": 802, + "lighting": 1748, + "scene": 2, + "sensor": 225, + "shutter": 357, + "unknown": 361 + }, + "devices": 275, + "file_digest": "d305fb55d52e", + "findings_by_code": { + "central_macro_no_status": 26, + "dpt_mismatch_co": 2, + "dpt_on_logic_object": 3, + "duplicate_name": 25, + "line_without_coupler": 1, + "missing_dpt": 32, + "missing_status_address": 58, + "missing_value_status": 76, + "reserve_without_dpt": 169, + "safety_input_no_status": 47, + "scene_no_status": 2, + "status_pairing_summary": 1, + "subdpt_suspect": 1, + "topology_segment_limit": 1 + }, + "findings_by_severity": { + "error": 32, + "info": 250, + "warning": 162 + }, + "group_addresses": 3646, + "ha_entities": { + "binary_sensor": 89, + "climate": 23, + "cover": 40, + "expose": 1, + "light": 250, + "sensor": 343, + "switch": 524 + }, + "ha_review_items": 1545, + "suggestion_hints": { + "channels": 983, + "diagnostics_skipped": 53, + "duplicates_skipped": 157, + "fallback": 0, + "pseudo_channels": 328, + "review": 220, + "sensors": 367, + "skipped_fb_covered": 0, + "structural": 513, + "subunits": 309, + "unwired_flagged": 20 + }, + "suggestions_by_platform": { + "binary_sensor": 254, + "climate": 28, + "cover": 92, + "light": 356, + "sensor": 113, + "switch": 37 + } + }, + "shutters-medium": { + "categories": { + "diagnostics": 9, + "hvac": 162, + "lighting": 93, + "sensor": 9, + "shutter": 36, + "unknown": 40 + }, + "devices": 34, + "file_digest": "8ed1f5a53c9c", + "findings_by_code": { + "central_macro_no_status": 2, + "dpt_mismatch_co": 1, + "duplicate_name": 55, + "inconsistent_dpt": 2, + "missing_dpt": 42, + "missing_status_address": 64, + "missing_value_status": 2, + "relative_only_dimming": 4, + "status_pairing_summary": 1 + }, + "findings_by_severity": { + "error": 42, + "info": 3, + "warning": 128 + }, + "group_addresses": 349, + "ha_entities": { + "binary_sensor": 5, + "cover": 8, + "expose": 1, + "light": 1, + "sensor": 87, + "switch": 58 + }, + "ha_review_items": 233, + "suggestion_hints": { + "channels": 135, + "diagnostics_skipped": 8, + "duplicates_skipped": 4, + "fallback": 0, + "pseudo_channels": 106, + "review": 61, + "sensors": 38, + "skipped_fb_covered": 0, + "structural": 65, + "subunits": 0, + "unwired_flagged": 2 + }, + "suggestions_by_platform": { + "binary_sensor": 13, + "climate": 2, + "cover": 4, + "light": 59, + "sensor": 25 + } + } + } +} diff --git a/tools/corpus_check.py b/tools/corpus_check.py new file mode 100644 index 0000000..5be5423 --- /dev/null +++ b/tools/corpus_check.py @@ -0,0 +1,185 @@ +"""Corpus guard — the regression gate the public CI cannot be. + +Every real bug this project has had (#11 cover pairing, #12 RM/Rückmeldung, #13 policy +profile) came from a real ETS file. Those files are confidential friends' projects and must +never reach a GitHub runner, so the public CI only ever sees synthetic fixtures. This script +closes that gap locally: it runs the whole pipeline over the real corpus, records a small set +of NUMBERS, and fails when they drift from the recorded baseline. + +What leaves the machine: nothing. What goes into git: only the numbers in +``tools/corpus_baseline.json`` — counts and percentages, no addresses, no names, no paths +beyond a short label the owner chooses. + + python tools/corpus_check.py # check against the baseline + python tools/corpus_check.py --update # re-record the baseline (after a deliberate change) + python tools/corpus_check.py --tolerance 2 # allow ±2 % drift per metric (default 0 for counts) + +Exit code 1 on drift, so it can be wired into a pre-commit hook or a nightly job. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import sys +from pathlib import Path +from typing import Any + +import yaml +from xknxproject import XKNXProj + +from nickol_knx_mcp.project import build_loaded_from_raw +from nickol_knx_mcp.analyze import (validate_naming, detect_missing_status, detect_dpt_issues, + detect_topology_issues, detect_role_completeness) +from nickol_knx_mcp.generate_ha import generate_ha_yaml +from nickol_knx_mcp.suggest import suggest_entities + +# Corpus lives outside the package and outside git. Labels are deliberate: a label never +# reveals a client, and the baseline file carries labels only. +CORPUS_DIR = Path(__file__).resolve().parents[2] / "demo-home-for-friend" +# The corpus map lives in tools/corpus_map.json, which is gitignored: real project paths carry +# client names and must never enter the repository. See tools/corpus_map.example.json. +# Override the corpus root with NICKOL_KNX_CORPUS_DIR. +CORPUS_MAP = Path(__file__).resolve().parent / "corpus_map.json" + + +def _load_corpus() -> list[tuple[str, str]]: + env_dir = os.environ.get("NICKOL_KNX_CORPUS_DIR") + if env_dir: + globals()["CORPUS_DIR"] = Path(env_dir).expanduser() + if not CORPUS_MAP.exists(): + print("no corpus map: copy tools/corpus_map.example.json -> tools/corpus_map.json " + "and point it at local .knxproj files") + raise SystemExit(2) + data = json.loads(CORPUS_MAP.read_text()) + return [(e["label"], e["path"]) for e in data["projects"]] + + +CORPUS: list[tuple[str, str]] = _load_corpus() +BASELINE = Path(__file__).resolve().parent / "corpus_baseline.json" + + +def _metrics(path: Path) -> dict[str, Any]: + raw = XKNXProj(str(path)).parse() + proj = build_loaded_from_raw(raw, str(path)) + + findings: list[dict[str, Any]] = [] + for check in (validate_naming, detect_missing_status, detect_dpt_issues, + detect_topology_issues, detect_role_completeness): + findings.extend(check(proj) or []) + sev: dict[str, int] = {} + codes: dict[str, int] = {} + for f in findings: + sev[f.get("severity", "?")] = sev.get(f.get("severity", "?"), 0) + 1 + codes[f.get("code", "?")] = codes.get(f.get("code", "?"), 0) + 1 + + ha = generate_ha_yaml(proj) + doc = yaml.safe_load(ha["yaml"]) or {} + knx = doc.get("knx") or {} + entities = {k: len(v) for k, v in knx.items() if isinstance(v, list)} + + sug = suggest_entities(raw, fallback=False) + plat: dict[str, int] = {} + for s in sug["suggestions"]: + p = s["platform_options"][0] + plat[p] = plat.get(p, 0) + 1 + + cats: dict[str, int] = {} + for ga in proj.gas.values(): + cats[ga.category] = cats.get(ga.category, 0) + 1 + + return { + "group_addresses": len(proj.gas), + "devices": len(proj.devices), + "categories": dict(sorted(cats.items())), + "findings_by_severity": dict(sorted(sev.items())), + "findings_by_code": dict(sorted(codes.items())), + "ha_entities": dict(sorted(entities.items())), + "ha_review_items": len(ha.get("review") or []), + "suggestions_by_platform": dict(sorted(plat.items())), + "suggestion_hints": {k: v for k, v in sug["hints"].items() if k != "state"}, + } + + +def _flat(d: dict[str, Any], prefix: str = "") -> dict[str, int]: + out: dict[str, int] = {} + for k, v in d.items(): + key = f"{prefix}{k}" + if isinstance(v, dict): + out.update(_flat(v, key + ".")) + elif isinstance(v, (int, float)): + out[key] = v + return out + + +def _compare(label: str, base: dict, now: dict, tol: float) -> list[str]: + a, b = _flat(base), _flat(now) + drift = [] + for key in sorted(set(a) | set(b)): + old, new = a.get(key), b.get(key) + if old == new: + continue + if old is None: + drift.append(f"{label}: NEW {key} = {new}") + elif new is None: + drift.append(f"{label}: GONE {key} (was {old})") + else: + delta = abs(new - old) + allowed = max(0.0, old * tol / 100.0) + if delta > allowed: + drift.append(f"{label}: {key} {old} -> {new} ({new - old:+d})") + return drift + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--update", action="store_true", help="re-record the baseline") + ap.add_argument("--tolerance", type=float, default=0.0, help="allowed drift per metric, %%") + args = ap.parse_args() + + baseline = json.loads(BASELINE.read_text()) if BASELINE.exists() else {"projects": {}} + result: dict[str, Any] = {"projects": {}} + drift: list[str] = [] + missing: list[str] = [] + + for label, rel in CORPUS: + path = CORPUS_DIR / rel + if not path.exists(): + missing.append(label) + continue + m = _metrics(path) + # a digest of the file identifies "the same file", without storing its name or content + m["file_digest"] = hashlib.sha256(path.read_bytes()).hexdigest()[:12] + result["projects"][label] = m + old = baseline["projects"].get(label) + if old is None: + drift.append(f"{label}: no baseline yet") + continue + if old.get("file_digest") != m["file_digest"]: + drift.append(f"{label}: the .knxproj itself changed — re-record with --update if intended") + continue + drift.extend(_compare(label, {k: v for k, v in old.items() if k != "file_digest"}, + {k: v for k, v in m.items() if k != "file_digest"}, args.tolerance)) + + for label in missing: + print(f"skip {label}: file not present on this machine") + + if args.update: + BASELINE.write_text(json.dumps(result, indent=2, ensure_ascii=False, sort_keys=True) + "\n") + print(f"baseline recorded for {len(result['projects'])} project(s) -> {BASELINE.name}") + return 0 + + if drift: + print(f"\nCORPUS DRIFT — {len(drift)} metric(s) moved:\n") + for line in drift: + print(" " + line) + print("\nIf the change is deliberate, re-record: python tools/corpus_check.py --update") + return 1 + + print(f"corpus OK — {len(result['projects'])} project(s), no drift against the baseline") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/corpus_map.example.json b/tools/corpus_map.example.json new file mode 100644 index 0000000..75c8f14 --- /dev/null +++ b/tools/corpus_map.example.json @@ -0,0 +1,17 @@ +{ + "_comment": "Copy to corpus_map.json and point each path at a real .knxproj. corpus_map.json is gitignored: paths and client names never enter the repo. Labels are what the baseline stores, so keep them neutral.", + "projects": [ + { + "label": "vendor-a-large", + "path": "some-folder/big-project.knxproj" + }, + { + "label": "vendor-b-flat", + "path": "some-folder/flat.knxproj" + }, + { + "label": "demo", + "path": "demo-home/demo-home.knxproj" + } + ] +}