59 lines
3.1 KiB
Python
59 lines
3.1 KiB
Python
"""Verify actual rendered stages in the synthetic long-trial recordings.
|
|
|
|
Copies of H.264 packets alone are insufficient evidence that clients saw a
|
|
retry slate or a recovered channel. Compare decoded text pixels, not file names.
|
|
"""
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
|
|
|
|
def pixels(ffmpeg, path, second=0):
|
|
command = [ffmpeg, '-hide_banner', '-loglevel', 'error', '-nostdin', '-threads', '1', '-filter_threads', '1',
|
|
'-ss', str(second), '-i', str(path), '-frames:v', '1', '-vf',
|
|
'scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,crop=1920:260:0:430,scale=640:86:flags=area', '-pix_fmt', 'gray', '-f', 'rawvideo', 'pipe:1']
|
|
output = subprocess.run(command, capture_output=True, check=True, timeout=20).stdout
|
|
if len(output) != 640 * 86:
|
|
raise RuntimeError('Incomplete video frame')
|
|
return output
|
|
|
|
|
|
def difference(first, second):
|
|
return sum(abs(a-b) for a, b in zip(first, second)) / len(first)
|
|
|
|
|
|
def verify(root, ffmpeg):
|
|
result = json.loads((root / 'result.json').read_text())
|
|
if result['FailureSeconds'] != 75:
|
|
raise ValueError('Use the long trial with the documented stage times')
|
|
errors = {}
|
|
for language in ('pl', 'en'):
|
|
waiting = pixels(ffmpeg, root / (language + '-waiting-slate.ts'))
|
|
retrying = pixels(ffmpeg, root / (language + '-retrying-slate.ts'))
|
|
observed_waiting = pixels(ffmpeg, root / (language + '.mkv'), 1)
|
|
observed_retrying = pixels(ffmpeg, root / (language + '.mkv'), 70)
|
|
live = max((pixels(ffmpeg, root / (language + '.mkv'), at) for at in (18,22,26,30,34,38,42,46)), key=lambda frame: difference(retrying,frame))
|
|
recovered = pixels(ffmpeg, root / (language + '.mkv'), 140)
|
|
metrics = {'waiting_error': difference(waiting, observed_waiting),
|
|
'retrying_error': difference(retrying, observed_retrying),
|
|
'live_difference': difference(retrying, live),
|
|
'recovered_difference': difference(retrying, recovered)}
|
|
assert metrics['waiting_error'] < 0.5 and metrics['retrying_error'] < 0.5, metrics
|
|
assert metrics['live_difference'] > 15 and metrics['recovered_difference'] > 15, metrics
|
|
other = 'en' if language == 'pl' else 'pl'
|
|
assert difference(observed_retrying, pixels(ffmpeg, root / (other + '-retrying-slate.ts'))) > 2
|
|
errors[language] = {key: round(value, 4) for key, value in metrics.items()}
|
|
result.update(RenderedWaitingVerified=True, RenderedRetryingVerified=True, RenderedLiveRecoveryVerified=True,
|
|
RenderedPerViewerLanguageVerified=True, VisualErrorMetrics=errors)
|
|
(root / 'result.json').write_text(json.dumps(result, indent=2) + '\n')
|
|
print(json.dumps({'rendered_stages_verified': True, 'metrics': errors}))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
parser.add_argument('--ffmpeg', default='/usr/lib/jellyfin-ffmpeg/ffmpeg')
|
|
args = parser.parse_args()
|
|
verify(args.output, args.ffmpeg)
|