public_git/bambuddy
1
0
Fork 0
mirror of https://github.com/maziggy/bambuddy.git synced 2026-10-19 15:54:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(camera): bound the post-kill wait on ffmpeg cleanup (#2580)

Browse Source 
After an RTSP read timeout the stream cleanup killed the stalled ffmpeg
and then awaited process.wait() unbounded. A SIGKILLed ffmpeg stuck in
uninterruptible I/O on a dead RTSP socket can take arbitrarily long to
be reaped, so the fan-out stream coroutine sat parked in that wait (12
hours in the reported case) while every new viewer attached to the
stalled broadcaster and received no frames.

Bound the post-kill wait to 2s in all three places it existed: the
stream generator's _terminate_ffmpeg (the reported hang), the camera
stop endpoint (which would hang the recovery request itself; now uses
the shared helper instead of an inline copy), and the orphan-cleanup
janitor (whose hang would disable the safety net). On timeout the
zombie is abandoned; the janitor's /proc scan reaps it next pass and
the stream proceeds to its normal reconnect.
This commit is contained in:
maziggy
2026-07-17 07:03:10 +02:00
parent e97413edc7
commit 75b0175e3d
3 changed files with 248 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/api/routes/camera.py
+38 -16
View File
@@ -45,6 +45,14 @@ from backend.app.services.camera_profiles import get_camera_profile
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/printers", tags=["camera"])
# Upper bound on waiting for a SIGKILLed ffmpeg to be reaped (#2580). A killed
# ffmpeg stuck in uninterruptible I/O on a dead RTSP socket can take arbitrarily
# long to exit — an unbounded post-kill wait() parked the fan-out stream
# coroutine for 12 hours on a P2S, leaving every viewer attached to a stalled
# broadcaster. Abandoning the wait is safe: cleanup_orphaned_streams' /proc scan
# reaps any Bambu ffmpeg not attached to an active stream on its next pass.
_FFMPEG_KILL_TIMEOUT = 2.0
# Track active ffmpeg processes for cleanup
_active_streams: dict[str, asyncio.subprocess.Process] = {}
@@ -243,7 +251,17 @@ async def _terminate_ffmpeg(process: asyncio.subprocess.Process, stream_id: str
except TimeoutError:
logger.warning("ffmpeg didn't terminate gracefully, killing (stream_id=%s)", stream_id)
process.kill()
await process.wait()
try:
await asyncio.wait_for(process.wait(), timeout=_FFMPEG_KILL_TIMEOUT)
except TimeoutError:
# Do NOT keep waiting (#2580): the caller is the stream
# generator, and blocking here pins the fan-out pump forever.
# The orphan janitor reaps the process later.
logger.error(
"ffmpeg did not exit within %.1fs of SIGKILL; abandoning wait (stream_id=%s)",
_FFMPEG_KILL_TIMEOUT,
stream_id,
)
except ProcessLookupError:
pass # Already dead
except OSError as e:
@@ -829,20 +847,13 @@ async def stop_camera_stream(
if event:
event.set()
if process.returncode is None:
try:
process.terminate()
try:
await asyncio.wait_for(process.wait(), timeout=2.0)
except TimeoutError:
logger.warning("ffmpeg didn't terminate gracefully, killing (stream_id=%s)", stream_id)
process.kill()
await process.wait()
stopped += 1
logger.info("Terminated ffmpeg process for stream %s", stream_id)
except ProcessLookupError:
pass # Process already dead
except OSError as e:
logger.warning("Error stopping stream %s: %s", stream_id, e)
# Shared helper, not an inline copy: it bounds the post-kill
# wait (#2580) — a killed-but-unreaped ffmpeg used to hang this
# request forever, exactly when the user hit Stop to recover a
# stuck stream.
await _terminate_ffmpeg(process, stream_id)
stopped += 1
logger.info("Terminated ffmpeg process for stream %s", stream_id)
_spawned_ffmpeg_pids.pop(process.pid, None)
for stream_id in to_remove:
@@ -1615,9 +1626,20 @@ async def cleanup_orphaned_streams():
event.set()
try:
proc.kill()
await proc.wait()
# Bounded (#2580): an unreaped SIGKILLed ffmpeg must not hang
# the periodic cleanup loop — this janitor is the safety net
# that recovers stalled streams, so it can least afford to
# block. The /proc scan above retries the kill next pass.
await asyncio.wait_for(proc.wait(), timeout=_FFMPEG_KILL_TIMEOUT)
except (ProcessLookupError, OSError):
pass
except TimeoutError:
logger.error(
"ffmpeg (pid=%d) did not exit within %.1fs of SIGKILL; abandoning wait (stream_id=%s)",
proc.pid,
_FFMPEG_KILL_TIMEOUT,
sid,
)
_active_streams.pop(sid, None)
_disconnect_events.pop(sid, None)
_stream_last_frame_times.pop(sid, None)
backend/tests/unit/test_camera_ffmpeg_termination.py
+209
View File
@@ -0,0 +1,209 @@
"""Bounded post-kill ffmpeg cleanup (#2580, fix shape from PR #2581 by @ronaldheft).
A SIGKILLed ffmpeg stuck in uninterruptible I/O on a dead RTSP socket can take
arbitrarily long to be reaped. The cleanup paths used to ``await process.wait()``
unbounded after ``kill()`` — on a P2S RTSP read timeout this parked the fan-out
stream coroutine for 12 hours, leaving every viewer attached to a stalled
broadcaster while snapshots/diagnostics (fresh connections) kept working.
The same unbounded wait existed in THREE places, all bounded now:
1. ``_terminate_ffmpeg`` — the stream generator's cleanup (the reported hang).
2. ``stop_camera`` — hung the very request a user makes to recover.
3. ``cleanup_orphaned_streams`` — hung the janitor that is the safety net.
"""
from __future__ import annotations
import asyncio
import time
from contextlib import suppress
import pytest
from backend.app.api.routes import camera
pytestmark = pytest.mark.asyncio
class _FakeServer:
def close(self) -> None:
pass
async def wait_closed(self) -> None:
pass
class _TimeoutReader:
"""stdout that immediately reports a read timeout (stalled RTSP)."""
async def read(self, _size: int = -1) -> bytes:
raise TimeoutError
class _SingleFrameReader:
"""stdout that yields one complete JPEG then EOF."""
def __init__(self) -> None:
self._sent = False
async def read(self, _size: int = -1) -> bytes:
if self._sent:
return b""
self._sent = True
return b"\xff\xd8fresh-frame\xff\xd9"
class _StuckPostKillProcess:
"""ffmpeg whose post-kill wait() never completes unless cancelled."""
def __init__(self, pid: int = 41001) -> None:
self.pid = pid
self.returncode = None
self.stdout = _TimeoutReader()
self.stderr = None
self.wait_calls = 0
self.killed = False
self.post_kill_wait_cancelled = asyncio.Event()
self._release = asyncio.Event()
def terminate(self) -> None:
pass
def kill(self) -> None:
self.killed = True
async def wait(self) -> int:
self.wait_calls += 1
if self.wait_calls == 1:
# Graceful-terminate window: simulate "didn't exit in time".
raise TimeoutError
try:
await self._release.wait()
except asyncio.CancelledError:
self.post_kill_wait_cancelled.set()
raise
self.returncode = -9
return self.returncode
class _FrameProcess:
"""Healthy replacement ffmpeg delivering one frame."""
def __init__(self, pid: int = 41002) -> None:
self.pid = pid
self.returncode = None
self.stdout = _SingleFrameReader()
self.stderr = None
def terminate(self) -> None:
pass
def kill(self) -> None:
pass
async def wait(self) -> int:
self.returncode = 0
return self.returncode
# ---------------------------------------------------------------------------
# 1. _terminate_ffmpeg — the helper itself is bounded
# ---------------------------------------------------------------------------
async def test_terminate_ffmpeg_abandons_unreaped_kill(monkeypatch):
monkeypatch.setattr(camera, "_FFMPEG_KILL_TIMEOUT", 0.05)
proc = _StuckPostKillProcess()
# Must return promptly instead of hanging on the post-kill wait.
await asyncio.wait_for(camera._terminate_ffmpeg(proc, "test"), timeout=1.0)
assert proc.killed is True
assert proc.post_kill_wait_cancelled.is_set()
assert proc.pid not in camera._spawned_ffmpeg_pids
# ---------------------------------------------------------------------------
# 2. Stream generator — reconnects instead of pinning the fan-out pump
# (regression scenario from PR #2581)
# ---------------------------------------------------------------------------
async def test_rtsp_stream_reconnects_past_unreaped_ffmpeg(monkeypatch):
"""RTSP read timeout → kill hangs → generator must still spawn a fresh
ffmpeg and deliver a frame, not block in cleanup forever."""
stalled = _StuckPostKillProcess()
recovered = _FrameProcess()
processes = iter((stalled, recovered))
spawned: list[object] = []
async def fake_create_subprocess_exec(*_args, **_kwargs):
process = next(processes)
spawned.append(process)
return process
async def fake_create_tls_proxy(_ip_address: str, _port: int):
return 48521, _FakeServer()
monkeypatch.setattr(camera, "get_ffmpeg_path", lambda: "/fake/ffmpeg")
monkeypatch.setattr(camera, "create_tls_proxy", fake_create_tls_proxy)
monkeypatch.setattr(camera.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
monkeypatch.setattr(camera, "_FFMPEG_KILL_TIMEOUT", 0.01)
stream = camera.generate_rtsp_mjpeg_stream(
ip_address="192.0.2.17",
access_code="test-code",
model="P2S",
fps=15,
stream_id="9999-fanout",
disconnect_event=asyncio.Event(),
printer_id=9999,
)
try:
chunk = await asyncio.wait_for(anext(stream), timeout=5.0)
assert b"fresh-frame" in chunk
assert stalled.killed is True
assert stalled.post_kill_wait_cancelled.is_set()
assert len(spawned) == 2, "expected a replacement ffmpeg to be spawned"
finally:
stalled._release.set()
with suppress(Exception):
await asyncio.wait_for(stream.aclose(), timeout=2.0)
# ---------------------------------------------------------------------------
# 3. Janitor — cleanup_orphaned_streams must not hang on an unreaped process
# ---------------------------------------------------------------------------
async def test_cleanup_orphaned_streams_bounded_on_unreaped_process(monkeypatch):
monkeypatch.setattr(camera, "_FFMPEG_KILL_TIMEOUT", 0.05)
monkeypatch.setattr(camera, "_scan_bambu_ffmpeg_pids", lambda: [])
import os
# Real pid: janitor layer 2 prunes _spawned_ffmpeg_pids entries whose pid
# doesn't exist (os.kill(pid, 0)), which would reset the spawn age and
# skip the stale-stream kill below.
proc = _StuckPostKillProcess(pid=os.getpid())
proc.wait_calls = 1 # skip the graceful-terminate branch; janitor kills directly
sid = "9998-fanout"
now = time.time()
camera._active_streams[sid] = proc
camera._spawned_ffmpeg_pids[proc.pid] = now - 120 # spawned long ago
camera._stream_last_frame_times[sid] = now - 60 # stale: no frames >30s
try:
# Must complete despite proc.wait() never returning.
await asyncio.wait_for(camera.cleanup_orphaned_streams(), timeout=2.0)
assert proc.killed is True
assert sid not in camera._active_streams
assert proc.pid not in camera._spawned_ffmpeg_pids
finally:
proc._release.set()
camera._active_streams.pop(sid, None)
camera._spawned_ffmpeg_pids.pop(proc.pid, None)
camera._stream_last_frame_times.pop(sid, None)
camera._disconnect_events.pop(sid, None)