Add opt-in FFmpeg continuity slates with per-viewer localization

experiments/run-continuity-e2e.py

@@ -0,0 +1,71 @@
+"""Run the isolated synthetic C# harness and measure its own source FFmpeg only.
+
+No Jellyfin configuration changes, restarts, API keys or receiver connections.
+"""
+import argparse
+import json
+import os
+from pathlib import Path
+import signal
+import subprocess
+import time
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--harness', required=True)
+    parser.add_argument('--ffmpeg', default='/usr/lib/jellyfin-ffmpeg/ffmpeg')
+    parser.add_argument('--output', required=True)
+    parser.add_argument('--long', action='store_true')
+    args = parser.parse_args()
+    root = Path(args.output).resolve()
+    if root.exists():
+        raise ValueError('Choose a new output directory')
+    samples, previous = [], {}
+    with Path(str(root) + '.log').open('x') as log:
+        process = subprocess.Popen([args.harness, '--long-continuity-e2e' if args.long else '--continuity-e2e', args.ffmpeg, str(root)], stdout=log,
+                                   stderr=subprocess.STDOUT, start_new_session=True)
+        deadline = time.monotonic() + 240
+        try:
+            while process.poll() is None:
+                if time.monotonic() > deadline:
+                    raise TimeoutError('Synthetic harness deadline exceeded')
+                now = time.monotonic()
+                for proc in Path('/proc').glob('[0-9]*'):
+                    try:
+                        if not str((proc / 'cwd').resolve()).startswith(str(root / 'cache' / 'source-')):
+                            continue
+                        value = (proc / 'stat').read_text()
+                        fields = value[value.rfind(')') + 2:].split()
+                        cpu = (int(fields[11]) + int(fields[12])) / os.sysconf('SC_CLK_TCK')
+                        rss = int(fields[21]) * os.sysconf('SC_PAGE_SIZE') / 1048576
+                        if proc.name in previous:
+                            before, at = previous[proc.name]
+                            samples.append({'one_core_percent': round((cpu-before)/(now-at)*100, 1),
+                                            'rss_mib': round(rss, 1)})
+                        previous[proc.name] = (cpu, now)
+                    except OSError:
+                        pass
+                time.sleep(1)
+        finally:
+            if process.poll() is None:
+                os.killpg(process.pid, signal.SIGTERM)
+                try:
+                    process.wait(timeout=5)
+                except subprocess.TimeoutExpired:
+                    os.killpg(process.pid, signal.SIGKILL)
+                    process.wait()
+    report = {'harness_exit_code': process.returncode, 'samples': samples,
+              'scope': 'Only synthetic plugin source encoding; excludes fixture and player decoders'}
+    (root / 'cpu-sample.json').write_text(json.dumps(report, indent=2))
+    if process.returncode == 0 and args.long:
+        subprocess.run(['python3', str(Path(__file__).with_name('verify-continuity-recording.py')), '--output', str(root), '--ffmpeg', args.ffmpeg], check=True)
+    print(Path(str(root) + '.log').read_text()[-3500:])
+    print(json.dumps({'exit_code': process.returncode, 'cpu_samples': len(samples),
+                      'maximum_one_core_percent': max((s['one_core_percent'] for s in samples), default=None),
+                      'maximum_rss_mib': max((s['rss_mib'] for s in samples), default=None)}))
+    raise SystemExit(0 if process.returncode == 0 else 1)
+
+
+if __name__ == '__main__':
+    main()

experiments/verify-continuity-recording.py

@@ -0,0 +1,58 @@
+"""Verify actual rendered stages in the synthetic long-trial recordings.
+
+Copies of H.264 packets alone are insufficient evidence that clients saw a
+retry slate or a recovered channel. Compare decoded text pixels, not file names.
+"""
+import argparse
+import json
+from pathlib import Path
+import subprocess
+
+
+def pixels(ffmpeg, path, second=0):
+    command = [ffmpeg, '-hide_banner', '-loglevel', 'error', '-nostdin', '-threads', '1', '-filter_threads', '1',
+               '-ss', str(second), '-i', str(path), '-frames:v', '1', '-vf',
+               'crop=1920:260:0:430,scale=640:86:flags=area', '-pix_fmt', 'gray', '-f', 'rawvideo', 'pipe:1']
+    output = subprocess.run(command, capture_output=True, check=True, timeout=20).stdout
+    if len(output) != 640 * 86:
+        raise RuntimeError('Incomplete video frame')
+    return output
+
+
+def difference(first, second):
+    return sum(abs(a-b) for a, b in zip(first, second)) / len(first)
+
+
+def verify(root, ffmpeg):
+    result = json.loads((root / 'result.json').read_text())
+    if result['FailureSeconds'] != 75:
+        raise ValueError('Use the long trial with the documented stage times')
+    errors = {}
+    for language in ('pl', 'en'):
+        waiting = pixels(ffmpeg, root / (language + '-waiting-slate.ts'))
+        retrying = pixels(ffmpeg, root / (language + '-retrying-slate.ts'))
+        observed_waiting = pixels(ffmpeg, root / (language + '.mkv'), 1)
+        observed_retrying = pixels(ffmpeg, root / (language + '.mkv'), 70)
+        live = pixels(ffmpeg, root / (language + '.mkv'), 39)
+        recovered = pixels(ffmpeg, root / (language + '.mkv'), 140)
+        metrics = {'waiting_error': difference(waiting, observed_waiting),
+                   'retrying_error': difference(retrying, observed_retrying),
+                   'live_difference': difference(retrying, live),
+                   'recovered_difference': difference(retrying, recovered)}
+        assert metrics['waiting_error'] < 0.5 and metrics['retrying_error'] < 0.5, metrics
+        assert metrics['live_difference'] > 15 and metrics['recovered_difference'] > 15, metrics
+        other = 'en' if language == 'pl' else 'pl'
+        assert difference(observed_retrying, pixels(ffmpeg, root / (other + '-retrying-slate.ts'))) > 2
+        errors[language] = {key: round(value, 4) for key, value in metrics.items()}
+    result.update(RenderedWaitingVerified=True, RenderedRetryingVerified=True, RenderedLiveRecoveryVerified=True,
+                  RenderedPerViewerLanguageVerified=True, VisualErrorMetrics=errors)
+    (root / 'result.json').write_text(json.dumps(result, indent=2) + '\n')
+    print(json.dumps({'rendered_stages_verified': True, 'metrics': errors}))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--output', type=Path, required=True)
+    parser.add_argument('--ffmpeg', default='/usr/lib/jellyfin-ffmpeg/ffmpeg')
+    args = parser.parse_args()
+    verify(args.output, args.ffmpeg)