Add opt-in FFmpeg continuity slates with per-viewer localization

experiments/verify-continuity-recording.py

@@ -0,0 +1,58 @@
+"""Verify actual rendered stages in the synthetic long-trial recordings.
+
+Copies of H.264 packets alone are insufficient evidence that clients saw a
+retry slate or a recovered channel. Compare decoded text pixels, not file names.
+"""
+import argparse
+import json
+from pathlib import Path
+import subprocess
+
+
+def pixels(ffmpeg, path, second=0):
+    command = [ffmpeg, '-hide_banner', '-loglevel', 'error', '-nostdin', '-threads', '1', '-filter_threads', '1',
+               '-ss', str(second), '-i', str(path), '-frames:v', '1', '-vf',
+               'crop=1920:260:0:430,scale=640:86:flags=area', '-pix_fmt', 'gray', '-f', 'rawvideo', 'pipe:1']
+    output = subprocess.run(command, capture_output=True, check=True, timeout=20).stdout
+    if len(output) != 640 * 86:
+        raise RuntimeError('Incomplete video frame')
+    return output
+
+
+def difference(first, second):
+    return sum(abs(a-b) for a, b in zip(first, second)) / len(first)
+
+
+def verify(root, ffmpeg):
+    result = json.loads((root / 'result.json').read_text())
+    if result['FailureSeconds'] != 75:
+        raise ValueError('Use the long trial with the documented stage times')
+    errors = {}
+    for language in ('pl', 'en'):
+        waiting = pixels(ffmpeg, root / (language + '-waiting-slate.ts'))
+        retrying = pixels(ffmpeg, root / (language + '-retrying-slate.ts'))
+        observed_waiting = pixels(ffmpeg, root / (language + '.mkv'), 1)
+        observed_retrying = pixels(ffmpeg, root / (language + '.mkv'), 70)
+        live = pixels(ffmpeg, root / (language + '.mkv'), 39)
+        recovered = pixels(ffmpeg, root / (language + '.mkv'), 140)
+        metrics = {'waiting_error': difference(waiting, observed_waiting),
+                   'retrying_error': difference(retrying, observed_retrying),
+                   'live_difference': difference(retrying, live),
+                   'recovered_difference': difference(retrying, recovered)}
+        assert metrics['waiting_error'] < 0.5 and metrics['retrying_error'] < 0.5, metrics
+        assert metrics['live_difference'] > 15 and metrics['recovered_difference'] > 15, metrics
+        other = 'en' if language == 'pl' else 'pl'
+        assert difference(observed_retrying, pixels(ffmpeg, root / (other + '-retrying-slate.ts'))) > 2
+        errors[language] = {key: round(value, 4) for key, value in metrics.items()}
+    result.update(RenderedWaitingVerified=True, RenderedRetryingVerified=True, RenderedLiveRecoveryVerified=True,
+                  RenderedPerViewerLanguageVerified=True, VisualErrorMetrics=errors)
+    (root / 'result.json').write_text(json.dumps(result, indent=2) + '\n')
+    print(json.dumps({'rendered_stages_verified': True, 'metrics': errors}))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--output', type=Path, required=True)
+    parser.add_argument('--ffmpeg', default='/usr/lib/jellyfin-ffmpeg/ffmpeg')
+    args = parser.parse_args()
+    verify(args.output, args.ffmpeg)