"""Verify actual rendered stages in the synthetic long-trial recordings. Copies of H.264 packets alone are insufficient evidence that clients saw a retry slate or a recovered channel. Compare decoded text pixels, not file names. """ import argparse import json from pathlib import Path import subprocess def pixels(ffmpeg, path, second=0): command = [ffmpeg, '-hide_banner', '-loglevel', 'error', '-nostdin', '-threads', '1', '-filter_threads', '1', '-ss', str(second), '-i', str(path), '-frames:v', '1', '-vf', 'crop=1920:260:0:430,scale=640:86:flags=area', '-pix_fmt', 'gray', '-f', 'rawvideo', 'pipe:1'] output = subprocess.run(command, capture_output=True, check=True, timeout=20).stdout if len(output) != 640 * 86: raise RuntimeError('Incomplete video frame') return output def difference(first, second): return sum(abs(a-b) for a, b in zip(first, second)) / len(first) def verify(root, ffmpeg): result = json.loads((root / 'result.json').read_text()) if result['FailureSeconds'] != 75: raise ValueError('Use the long trial with the documented stage times') errors = {} for language in ('pl', 'en'): waiting = pixels(ffmpeg, root / (language + '-waiting-slate.ts')) retrying = pixels(ffmpeg, root / (language + '-retrying-slate.ts')) observed_waiting = pixels(ffmpeg, root / (language + '.mkv'), 1) observed_retrying = pixels(ffmpeg, root / (language + '.mkv'), 70) live = pixels(ffmpeg, root / (language + '.mkv'), 39) recovered = pixels(ffmpeg, root / (language + '.mkv'), 140) metrics = {'waiting_error': difference(waiting, observed_waiting), 'retrying_error': difference(retrying, observed_retrying), 'live_difference': difference(retrying, live), 'recovered_difference': difference(retrying, recovered)} assert metrics['waiting_error'] < 0.5 and metrics['retrying_error'] < 0.5, metrics assert metrics['live_difference'] > 15 and metrics['recovered_difference'] > 15, metrics other = 'en' if language == 'pl' else 'pl' assert difference(observed_retrying, pixels(ffmpeg, root / (other + '-retrying-slate.ts'))) > 2 errors[language] = {key: round(value, 4) for key, value in metrics.items()} result.update(RenderedWaitingVerified=True, RenderedRetryingVerified=True, RenderedLiveRecoveryVerified=True, RenderedPerViewerLanguageVerified=True, VisualErrorMetrics=errors) (root / 'result.json').write_text(json.dumps(result, indent=2) + '\n') print(json.dumps({'rendered_stages_verified': True, 'metrics': errors})) if __name__ == '__main__': parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--output', type=Path, required=True) parser.add_argument('--ffmpeg', default='/usr/lib/jellyfin-ffmpeg/ffmpeg') args = parser.parse_args() verify(args.output, args.ffmpeg)