Add opt-in FFmpeg continuity slates with per-viewer localization
This commit is contained in:
@@ -0,0 +1,123 @@
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Jellyfin.Plugin.Enigma2;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
/// <summary>Real FFmpeg HTTP/HLS failover; synthetic source only, no decoder or Jellyfin configuration.</summary>
|
||||
public static class ContinuityE2E
|
||||
{
|
||||
private static string ffmpeg;
|
||||
private static volatile bool enabled;
|
||||
private static int connections, active, maximum;
|
||||
public static async Task Run(string encoder, string output, bool longTest = false)
|
||||
{
|
||||
ffmpeg = encoder; output = Path.GetFullPath(output);
|
||||
if (Directory.Exists(output)) throw new Exception("Choose a new output directory"); Directory.CreateDirectory(output);
|
||||
using var listener = new TcpListener(IPAddress.Loopback, 0); listener.Start(); var port = ((IPEndPoint)listener.LocalEndpoint).Port; listener.Stop();
|
||||
var builder = WebApplication.CreateBuilder(); builder.Logging.ClearProviders(); builder.WebHost.UseUrls($"http://127.0.0.1:{port}");
|
||||
using var streams = new ContinuityStreams(encoder, Path.Combine(output, "cache"), port, new StartupMessages(), NullLogger.Instance);
|
||||
using var relay = new LiveStreamRelay(new System.Net.Http.HttpClient(), port, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(5));
|
||||
builder.Services.AddSingleton(relay); builder.Services.AddSingleton(streams); builder.Services.AddControllers().AddApplicationPart(typeof(ContinuityController).Assembly);
|
||||
await using var app = builder.Build(); app.MapControllers(); app.MapGet("/fixture.ts", Fixture); await app.StartAsync();
|
||||
Process first = null, second = null;
|
||||
try
|
||||
{
|
||||
var source = relay.Register($"http://127.0.0.1:{port}/fixture.ts");
|
||||
var pl = await streams.Open(source, "pl", CancellationToken.None);
|
||||
var en = await streams.Open(source, "en", CancellationToken.None);
|
||||
Require(JsonSerializer.Serialize(streams.Status()).Contains("\"Sources\":1"), "two languages share one source worker");
|
||||
Require(!streams.Segment(pl.Id, 0).SequenceEqual(streams.Segment(en.Id, 0)), "same timeline uses different language slates");
|
||||
await File.WriteAllBytesAsync(Path.Combine(output, "pl-slate.ts"), streams.Segment(pl.Id, 0));
|
||||
await File.WriteAllBytesAsync(Path.Combine(output, "en-slate.ts"), streams.Segment(en.Id, 0));
|
||||
var denied = new ContinuityController(streams) { ControllerContext = new Microsoft.AspNetCore.Mvc.ControllerContext { HttpContext = new DefaultHttpContext() } };
|
||||
denied.HttpContext.Connection.RemoteIpAddress = IPAddress.Parse("192.0.2.1");
|
||||
Require(denied.Playlist(pl.Id) is Microsoft.AspNetCore.Mvc.StatusCodeResult { StatusCode: 403 }, "private continuity endpoint denies remote access");
|
||||
denied.HttpContext.Connection.RemoteIpAddress = IPAddress.Loopback; denied.Request.Headers["X-Forwarded-For"] = "192.0.2.1";
|
||||
Require(denied.Playlist(pl.Id) is Microsoft.AspNetCore.Mvc.StatusCodeResult { StatusCode: 403 }, "private continuity endpoint denies forwarded requests");
|
||||
first = Start(new[] { "-xerror", "-i", pl.Path, "-t", longTest ? "150" : "65", "-map", "0:v:0", "-map", "0:a:0", "-c", "copy", Path.Combine(output, "pl.mkv") }, output, "pl-client");
|
||||
second = Start(new[] { "-xerror", "-i", en.Path, "-t", longTest ? "150" : "65", "-map", "0:v:0", "-map", "0:a:0", "-c", "copy", Path.Combine(output, "en.mkv") }, output, "en-client");
|
||||
await Task.Delay(8000); enabled = true; Console.WriteLine("E2E source enabled after initial failure");
|
||||
await Task.Delay(20000);
|
||||
var beforeLoss = JsonSerializer.SerializeToElement(streams.Status()).GetProperty("LiveSegmentsProduced").GetInt64();
|
||||
Console.WriteLine("E2E status before loss " + JsonSerializer.Serialize(streams.Status()));
|
||||
Require(beforeLoss > 0, "real synthetic video reached clients before induced loss"); enabled = false; Console.WriteLine("E2E source disconnected");
|
||||
await Task.Delay(longTest ? 75000 : 15000); enabled = true; Console.WriteLine("E2E source recovered");
|
||||
await Task.WhenAll(first.WaitForExitAsync(), second.WaitForExitAsync()).WaitAsync(TimeSpan.FromSeconds(65));
|
||||
Require(first.ExitCode == 0 && second.ExitCode == 0, "both HLS clients survive waiting, source loss and recovery without reopening");
|
||||
foreach (var language in new[] { "pl", "en" })
|
||||
{
|
||||
using var decode = Start(new[] { "-xerror", "-i", Path.Combine(output, language + ".mkv"), "-map", "0:v:0", "-map", "0:a:0", "-f", "null", "-" }, output, language + "-decode");
|
||||
await decode.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(40)); Require(decode.ExitCode == 0, "strict decode across all switches: " + language);
|
||||
foreach (var (name, secondAt) in new[] { ("waiting", 1), ("live", longTest ? 39 : 22), ("retrying", longTest ? 70 : 39), ("recovered", longTest ? 140 : 60) })
|
||||
{
|
||||
using var image = Start(new[] { "-ss", secondAt.ToString(), "-i", Path.Combine(output, language + ".mkv"), "-frames:v", "1", "-update", "1", Path.Combine(output, language + "-" + name + ".png") }, output, language + "-" + name);
|
||||
await image.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(20)); Require(image.ExitCode == 0, "frame extraction: " + language + " " + name);
|
||||
}
|
||||
}
|
||||
Console.WriteLine("E2E status after recovery "+JsonSerializer.Serialize(streams.Status()));
|
||||
Require(JsonSerializer.SerializeToElement(streams.Status()).GetProperty("LiveSegmentsProduced").GetInt64() > beforeLoss, "new live segments arrive after recovery");
|
||||
streams.Close(pl.Id); Require(JsonSerializer.Serialize(streams.Status()).Contains("\"Viewers\":1"), "one viewer stopping leaves the other active");
|
||||
streams.Close(en.Id); await Task.Delay(2000);
|
||||
Require(active == 0 && maximum <= 1 && connections >= 2, "one upstream at a time, retries and final source cleanup");
|
||||
foreach (var language in new[] { "pl", "en", "de", "fr", "es", "uk" })
|
||||
foreach (var phase in new[] { "waiting", "retrying" })
|
||||
{
|
||||
var path=Path.Combine(output,language+"-"+phase+"-slate.ts");
|
||||
await File.WriteAllBytesAsync(path,await streams.SlatePreview(language,phase,CancellationToken.None));
|
||||
using var image=Start(new[]{"-i",path,"-frames:v","1","-update","1",Path.Combine(output,language+"-"+phase+"-slate.png")},output,language+"-"+phase+"-slate");
|
||||
await image.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(20));Require(image.ExitCode==0,"localized FFmpeg slate: "+language+" "+phase);
|
||||
}
|
||||
var report = new { Passed = true, SyntheticOnly = true, ReceiverConnectionsOpened = 0, ViewerLanguages = new[] { "pl", "en" }, SourceConnections = connections, FailureSeconds = longTest ? 75 : 15,
|
||||
MaximumConcurrentSourceConnections = maximum, RemainingConnections = active, WaitingLossRecoverySamePlayer = true,
|
||||
StrictVideoAudioDecode = true, Width = ContinuityStreams.Width, Height = ContinuityStreams.Height, FrameRate = ContinuityStreams.FrameRate,
|
||||
PhysicalAndroidTvTested = false, ActualJellyfinPlayerTested = false };
|
||||
await File.WriteAllTextAsync(Path.Combine(output, "result.json"), JsonSerializer.Serialize(report, new JsonSerializerOptions { WriteIndented = true })); Console.WriteLine(JsonSerializer.Serialize(report));
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (first is { HasExited: false }) first.Kill(true); if (second is { HasExited: false }) second.Kill(true);
|
||||
first?.Dispose(); second?.Dispose(); streams.Dispose(); await app.StopAsync();
|
||||
}
|
||||
}
|
||||
private static void Require(bool condition, string message) { if (!condition) throw new Exception(message); Console.WriteLine("PASS " + message); }
|
||||
private static Process Start(string[] args, string output, string name)
|
||||
{
|
||||
var info = new ProcessStartInfo(ffmpeg) { UseShellExecute = false, RedirectStandardError = true, RedirectStandardOutput = true };
|
||||
foreach (var arg in new[] { "-hide_banner", "-loglevel", "warning", "-nostdin", "-threads", "1", "-filter_threads", "1" }.Concat(args)) info.ArgumentList.Add(arg);
|
||||
var process = Process.Start(info); process.ErrorDataReceived += (_, e) => { if (e.Data != null) lock (typeof(ContinuityE2E)) File.AppendAllText(Path.Combine(output, name + ".log"), e.Data + "\n"); };
|
||||
process.BeginErrorReadLine(); return process;
|
||||
}
|
||||
private static async Task Fixture(HttpContext context)
|
||||
{
|
||||
Interlocked.Increment(ref connections);
|
||||
if (!enabled) { context.Response.StatusCode = 503; return; }
|
||||
var current = Interlocked.Increment(ref active); maximum = Math.Max(maximum, current);
|
||||
var info = new ProcessStartInfo(ffmpeg) { UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardError = true };
|
||||
foreach (var arg in new[] { "-hide_banner", "-loglevel", "error", "-nostdin", "-threads", "1", "-filter_threads", "1", "-re", "-f", "lavfi", "-i", "testsrc2=size=1280x720:rate=25", "-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000", "-c:v", "libx264", "-threads", "1", "-preset", "ultrafast", "-g", "25", "-c:a", "aac", "-f", "mpegts", "pipe:1" }) info.ArgumentList.Add(arg);
|
||||
using var process = Process.Start(info); process.ErrorDataReceived += (_, _) => { }; process.BeginErrorReadLine(); context.Response.ContentType = "video/mp2t";
|
||||
try
|
||||
{
|
||||
var buffer = new byte[64 * 1024];
|
||||
while (enabled && !context.RequestAborted.IsCancellationRequested)
|
||||
{
|
||||
var count = await process.StandardOutput.BaseStream.ReadAsync(buffer, context.RequestAborted); if (count == 0) break;
|
||||
await context.Response.Body.WriteAsync(buffer.AsMemory(0, count), context.RequestAborted); await context.Response.Body.FlushAsync(context.RequestAborted);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) { }
|
||||
catch (IOException) { }
|
||||
finally { if (!process.HasExited) process.Kill(true); await process.WaitForExitAsync(); Interlocked.Decrement(ref active); context.Abort(); }
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user