feat(support): record process memory, threads and children in bundles (#2734)

A bundle described everything except the process it runs in. So a report of
memory climbing over days until the OOM killer fires arrives with no way to
act on it: the numbers that name the mechanism only exist while it is
happening, and by the time anyone asks, the container has been restarted.

The new `process` section carries what actually separates the candidates.
Resident against virtual memory: 650MB RSS with 12.9GB VMS is address
space — thread stacks or allocator arenas — not a heap full of live data,
and that reading is the opposite of the one the reporter drew from the same
figures. Thread count and child-process count then split those two apart,
and a census of live objects by type names what a growing heap is filling
up with. Open files, sockets and uptime round it out.

Three constraints worth keeping:

The heap census is skipped above 2GB. gc.get_objects() materialises every
tracked object, so it costs most on exactly the process that can least
afford it — a bundle generated to diagnose runaway memory must not be the
allocation that tips the host over. Everything else is still collected, and
the skip is recorded with its reason rather than silently omitted.

Children are recorded by executable name only. An ffmpeg command line
carries the camera URL, and with it the camera's password.

Collection runs off the event loop and every metric is independently
best-effort. psutil raises on hardened kernels and in restricted
containers, and the bundle is how someone reports a problem in the first
place — it has to be produced even when half the numbers are unavailable.

This does not fix #2734, and nothing here should be read as having found
its cause. The bundle's own evidence contradicts both proposed causes: the
orphan janitor ran 7 times in 26 days over 725 stream-ends and killed no
orphaned ffmpeg, which is not the #776 signature; and the 5 "database is
locked" errors all fall between two OOM kills, making them a symptom of the
memory pressure rather than a source of it.
This commit is contained in:
maziggy
2026-07-31 15:32:08 +02:00
parent ce3e59884a
commit c457cf54bf
3 changed files with 218 additions and 0 deletions
+101
View File
@@ -1472,3 +1472,104 @@ class TestSanitizePushStatusValues:
assert raw == before, "input was mutated"
assert out["tag_uid"] == "[SERIAL]" # and the copy really was redacted
class TestProcessInfo:
"""Bambuddy's own footprint in the bundle (#2734).
Bundles carried nothing about the process itself, so "memory climbs over
days until the OOM killer fires" could not be triaged from a bundle — the
reporter had to run shell commands by hand, and the numbers that would have
named the mechanism were unrecoverable afterwards.
"""
def test_reports_the_figures_that_separate_the_mechanisms(self):
"""RSS vs VMS, threads and children distinguish a heap that is growing
from address space, a thread leak, and a child-process leak."""
from backend.app.api.routes.support import _collect_process_info
info = _collect_process_info()
assert info["available"] is True
for key in ("rss_bytes", "vms_bytes", "num_threads", "children_total"):
assert isinstance(info[key], int), key
def test_children_are_named_but_never_quoted(self):
"""An ffmpeg argv carries the camera URL, and with it its password. The
count per executable is what identifies a leak; the arguments are not
needed and must not travel."""
from backend.app.api.routes.support import _collect_process_info
info = _collect_process_info()
for name in info.get("children_by_name", {}):
assert " " not in name, f"looks like a command line, not a name: {name!r}"
assert "://" not in name
def test_heap_census_is_skipped_on_a_large_process(self):
"""gc.get_objects() materialises every tracked object, so the census
costs most on the process that can least afford it. A bundle generated
to diagnose runaway memory must not be the allocation that tips the
host over."""
from unittest.mock import MagicMock, patch
import backend.app.api.routes.support as support_module
fake = MagicMock()
fake.memory_info.return_value = MagicMock(rss=8 * 1024**3, vms=12 * 1024**3)
fake.num_threads.return_value = 40
fake.create_time.return_value = 0.0
fake.open_files.return_value = []
fake.net_connections.return_value = []
fake.children.return_value = []
with patch("psutil.Process", return_value=fake):
info = support_module._collect_process_info()
assert "gc_top_types" not in info
assert "skipped" in info["gc_census"]
# The discriminating numbers still come through — those are the point.
assert info["rss_bytes"] == 8 * 1024**3
assert info["num_threads"] == 40
def test_heap_census_runs_on_a_normal_process(self):
from backend.app.api.routes.support import _collect_process_info
info = _collect_process_info()
assert info["gc_tracked_objects"] > 0
assert len(info["gc_top_types"]) <= 15
def test_survives_a_hostile_psutil(self):
"""psutil raises on hardened kernels and in restricted containers. A
support bundle must still be produced when it does — the bundle is how
someone reports the problem in the first place."""
from unittest.mock import patch
import backend.app.api.routes.support as support_module
with patch("psutil.Process", side_effect=RuntimeError("no /proc for you")):
info = support_module._collect_process_info()
assert info == {"available": False}
def test_partial_failures_do_not_lose_the_rest(self):
"""One inaccessible metric must not cost the others."""
from unittest.mock import MagicMock, patch
import backend.app.api.routes.support as support_module
fake = MagicMock()
fake.memory_info.return_value = MagicMock(rss=100, vms=200)
fake.num_threads.side_effect = PermissionError("denied")
fake.create_time.return_value = 0.0
fake.open_files.side_effect = PermissionError("denied")
fake.net_connections.side_effect = PermissionError("denied")
fake.children.return_value = []
with patch("psutil.Process", return_value=fake):
info = support_module._collect_process_info()
assert info["rss_bytes"] == 100
assert "num_threads" not in info
assert info["children_total"] == 0