using System; using System.Diagnostics; using System.IO; using System.Linq; using System.Net; using System.Net.Sockets; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Jellyfin.Plugin.Enigma2; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; /// Real FFmpeg HTTP/HLS failover; synthetic source only, no decoder or Jellyfin configuration. public static class ContinuityE2E { private static string ffmpeg; private static volatile bool enabled; private static int connections, active, maximum; public static async Task Run(string encoder, string output, bool longTest = false) { ffmpeg = encoder; output = Path.GetFullPath(output); if (Directory.Exists(output)) throw new Exception("Choose a new output directory"); Directory.CreateDirectory(output); using var listener = new TcpListener(IPAddress.Loopback, 0); listener.Start(); var port = ((IPEndPoint)listener.LocalEndpoint).Port; listener.Stop(); var builder = WebApplication.CreateBuilder(); builder.Logging.ClearProviders(); builder.WebHost.UseUrls($"http://127.0.0.1:{port}"); using var streams = new ContinuityStreams(encoder, Path.Combine(output, "cache"), port, new StartupMessages(), NullLogger.Instance); using var relay = new LiveStreamRelay(new System.Net.Http.HttpClient(), port, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(5)); builder.Services.AddSingleton(relay); builder.Services.AddSingleton(streams); builder.Services.AddControllers().AddApplicationPart(typeof(ContinuityController).Assembly); await using var app = builder.Build(); app.MapControllers(); app.MapGet("/fixture.ts", Fixture); await app.StartAsync(); Process first = null, second = null; try { var source = relay.Register($"http://127.0.0.1:{port}/fixture.ts"); var pl = await streams.Open(source, "pl", CancellationToken.None); var en = await streams.Open(source, "en", CancellationToken.None); Require(JsonSerializer.Serialize(streams.Status()).Contains("\"Sources\":1"), "two languages share one source worker"); Require(!streams.Segment(pl.Id, 0).SequenceEqual(streams.Segment(en.Id, 0)), "same timeline uses different language slates"); await File.WriteAllBytesAsync(Path.Combine(output, "pl-slate.ts"), streams.Segment(pl.Id, 0)); await File.WriteAllBytesAsync(Path.Combine(output, "en-slate.ts"), streams.Segment(en.Id, 0)); var denied = new ContinuityController(streams) { ControllerContext = new Microsoft.AspNetCore.Mvc.ControllerContext { HttpContext = new DefaultHttpContext() } }; denied.HttpContext.Connection.RemoteIpAddress = IPAddress.Parse("192.0.2.1"); Require(denied.Playlist(pl.Id) is Microsoft.AspNetCore.Mvc.StatusCodeResult { StatusCode: 403 }, "private continuity endpoint denies remote access"); denied.HttpContext.Connection.RemoteIpAddress = IPAddress.Loopback; denied.Request.Headers["X-Forwarded-For"] = "192.0.2.1"; Require(denied.Playlist(pl.Id) is Microsoft.AspNetCore.Mvc.StatusCodeResult { StatusCode: 403 }, "private continuity endpoint denies forwarded requests"); first = Start(new[] { "-xerror", "-i", pl.Path, "-t", longTest ? "150" : "65", "-map", "0:v:0", "-map", "0:a:0", "-c", "copy", Path.Combine(output, "pl.mkv") }, output, "pl-client"); second = Start(new[] { "-xerror", "-i", en.Path, "-t", longTest ? "150" : "65", "-map", "0:v:0", "-map", "0:a:0", "-c", "copy", Path.Combine(output, "en.mkv") }, output, "en-client"); await Task.Delay(8000); enabled = true; Console.WriteLine("E2E source enabled after initial failure"); await Task.Delay(20000); var beforeLoss = JsonSerializer.SerializeToElement(streams.Status()).GetProperty("LiveSegmentsProduced").GetInt64(); Console.WriteLine("E2E status before loss " + JsonSerializer.Serialize(streams.Status())); Require(beforeLoss > 0, "real synthetic video reached clients before induced loss"); enabled = false; Console.WriteLine("E2E source disconnected"); await Task.Delay(longTest ? 75000 : 15000); enabled = true; Console.WriteLine("E2E source recovered"); await Task.WhenAll(first.WaitForExitAsync(), second.WaitForExitAsync()).WaitAsync(TimeSpan.FromSeconds(65)); Require(first.ExitCode == 0 && second.ExitCode == 0, "both HLS clients survive waiting, source loss and recovery without reopening"); foreach (var language in new[] { "pl", "en" }) { using var decode = Start(new[] { "-xerror", "-i", Path.Combine(output, language + ".mkv"), "-map", "0:v:0", "-map", "0:a:0", "-f", "null", "-" }, output, language + "-decode"); await decode.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(40)); Require(decode.ExitCode == 0, "strict decode across all switches: " + language); foreach (var (name, secondAt) in new[] { ("waiting", 1), ("live", longTest ? 39 : 22), ("retrying", longTest ? 70 : 39), ("recovered", longTest ? 140 : 60) }) { using var image = Start(new[] { "-ss", secondAt.ToString(), "-i", Path.Combine(output, language + ".mkv"), "-frames:v", "1", "-update", "1", Path.Combine(output, language + "-" + name + ".png") }, output, language + "-" + name); await image.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(20)); Require(image.ExitCode == 0, "frame extraction: " + language + " " + name); } } Console.WriteLine("E2E status after recovery "+JsonSerializer.Serialize(streams.Status())); Require(JsonSerializer.SerializeToElement(streams.Status()).GetProperty("LiveSegmentsProduced").GetInt64() > beforeLoss, "new live segments arrive after recovery"); streams.Close(pl.Id); Require(JsonSerializer.Serialize(streams.Status()).Contains("\"Viewers\":1"), "one viewer stopping leaves the other active"); streams.Close(en.Id); await Task.Delay(2000); Require(active == 0 && maximum <= 1 && connections >= 2, "one upstream at a time, retries and final source cleanup"); foreach (var language in new[] { "pl", "en", "de", "fr", "es", "uk" }) foreach (var phase in new[] { "waiting", "retrying" }) { var path=Path.Combine(output,language+"-"+phase+"-slate.ts"); await File.WriteAllBytesAsync(path,await streams.SlatePreview(language,phase,CancellationToken.None)); using var image=Start(new[]{"-i",path,"-frames:v","1","-update","1",Path.Combine(output,language+"-"+phase+"-slate.png")},output,language+"-"+phase+"-slate"); await image.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(20));Require(image.ExitCode==0,"localized FFmpeg slate: "+language+" "+phase); } var report = new { Passed = true, SyntheticOnly = true, ReceiverConnectionsOpened = 0, ViewerLanguages = new[] { "pl", "en" }, SourceConnections = connections, FailureSeconds = longTest ? 75 : 15, MaximumConcurrentSourceConnections = maximum, RemainingConnections = active, WaitingLossRecoverySamePlayer = true, StrictVideoAudioDecode = true, Width = ContinuityStreams.Width, Height = ContinuityStreams.Height, FrameRate = ContinuityStreams.FrameRate, PhysicalAndroidTvTested = false, ActualJellyfinPlayerTested = false }; await File.WriteAllTextAsync(Path.Combine(output, "result.json"), JsonSerializer.Serialize(report, new JsonSerializerOptions { WriteIndented = true })); Console.WriteLine(JsonSerializer.Serialize(report)); } finally { if (first is { HasExited: false }) first.Kill(true); if (second is { HasExited: false }) second.Kill(true); first?.Dispose(); second?.Dispose(); streams.Dispose(); await app.StopAsync(); } } private static void Require(bool condition, string message) { if (!condition) throw new Exception(message); Console.WriteLine("PASS " + message); } private static Process Start(string[] args, string output, string name) { var info = new ProcessStartInfo(ffmpeg) { UseShellExecute = false, RedirectStandardError = true, RedirectStandardOutput = true }; foreach (var arg in new[] { "-hide_banner", "-loglevel", "warning", "-nostdin", "-threads", "1", "-filter_threads", "1" }.Concat(args)) info.ArgumentList.Add(arg); var process = Process.Start(info); process.ErrorDataReceived += (_, e) => { if (e.Data != null) lock (typeof(ContinuityE2E)) File.AppendAllText(Path.Combine(output, name + ".log"), e.Data + "\n"); }; process.BeginErrorReadLine(); return process; } private static async Task Fixture(HttpContext context) { Interlocked.Increment(ref connections); if (!enabled) { context.Response.StatusCode = 503; return; } var current = Interlocked.Increment(ref active); maximum = Math.Max(maximum, current); var info = new ProcessStartInfo(ffmpeg) { UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardError = true }; foreach (var arg in new[] { "-hide_banner", "-loglevel", "error", "-nostdin", "-threads", "1", "-filter_threads", "1", "-re", "-f", "lavfi", "-i", "testsrc2=size=1280x720:rate=25", "-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000", "-c:v", "libx264", "-threads", "1", "-preset", "ultrafast", "-g", "25", "-c:a", "aac", "-f", "mpegts", "pipe:1" }) info.ArgumentList.Add(arg); using var process = Process.Start(info); process.ErrorDataReceived += (_, _) => { }; process.BeginErrorReadLine(); context.Response.ContentType = "video/mp2t"; try { var buffer = new byte[64 * 1024]; while (enabled && !context.RequestAborted.IsCancellationRequested) { var count = await process.StandardOutput.BaseStream.ReadAsync(buffer, context.RequestAborted); if (count == 0) break; await context.Response.Body.WriteAsync(buffer.AsMemory(0, count), context.RequestAborted); await context.Response.Body.FlushAsync(context.RequestAborted); } } catch (OperationCanceledException) { } catch (IOException) { } finally { if (!process.HasExited) process.Kill(true); await process.WaitForExitAsync(); Interlocked.Decrement(ref active); context.Abort(); } } }