diff --git a/src/ArmRipper.Core/Configuration/ArmSettings.cs b/src/ArmRipper.Core/Configuration/ArmSettings.cs index a0f0f9f..2d3f61f 100644 --- a/src/ArmRipper.Core/Configuration/ArmSettings.cs +++ b/src/ArmRipper.Core/Configuration/ArmSettings.cs @@ -107,6 +107,21 @@ public class ArmSettings // public string? DiscDbApiBaseUrl { get; set; } = "https://api.thediscdb.com/graphql"; public string? DiscDbApiBaseUrl { get; set; } = "https://thediscdb.com/graphql"; + // ── MakeMKV I/O Watchdog ── + /// + /// Maximum bytes a makemkvcon process can read from disc before the I/O watchdog + /// cancels the job. This prevents runaway read retry storms on scratched/damaged discs. + /// Set to 0 to disable the watchdog entirely. + /// Default: 150 GiB (approximately 3× a BDXL disc, well above any legitimate single rip). + /// + public long MakemkvMaxReadBytes { get; set; } = 150L * 1024 * 1024 * 1024; + + /// + /// Polling interval in seconds for the MakeMKV I/O watchdog. Default: 30. + /// Minimum effective value is 10 seconds. + /// + public int MakemkvIoWatchdogIntervalSeconds { get; set; } = 30; + // ── OVID Integration ── /// Whether to submit OVID fingerprints to the community OVID database. public bool OvidSubmitEnabled { get; set; } = true; diff --git a/src/ArmRipper.Core/Configuration/ArmYamlConfigLoader.cs b/src/ArmRipper.Core/Configuration/ArmYamlConfigLoader.cs index 476a703..af88edc 100644 --- a/src/ArmRipper.Core/Configuration/ArmYamlConfigLoader.cs +++ b/src/ArmRipper.Core/Configuration/ArmYamlConfigLoader.cs @@ -68,6 +68,9 @@ public static class ArmYamlConfigLoader // ── Disc polling (ARM-Sharp specific) ── ["DISC_POLLING_ENABLED"] = "Arm:DiscPollingEnabled", ["DISC_POLL_INTERVAL"] = "Arm:DiscPollIntervalSeconds", + // ── MakeMKV I/O watchdog (ARM-Sharp specific) ── + ["MAKEMKV_MAX_READ_BYTES"] = "Arm:MakemkvMaxReadBytes", + ["MAKEMKV_IO_WATCHDOG_INTERVAL"] = "Arm:MakemkvIoWatchdogIntervalSeconds", }; public static Dictionary LoadYamlValues(string yamlPath) diff --git a/src/ArmRipper.Core/Infrastructure/MakemkvIoWatchdog.cs b/src/ArmRipper.Core/Infrastructure/MakemkvIoWatchdog.cs new file mode 100644 index 0000000..32ca9d2 --- /dev/null +++ b/src/ArmRipper.Core/Infrastructure/MakemkvIoWatchdog.cs @@ -0,0 +1,319 @@ +using ArmRipper.Core.Configuration; +using ArmRipper.Core.Infrastructure.Data; +using ArmRipper.Core.Models; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace ArmRipper.Core.Infrastructure; + +/// +/// Background service that monitors makemkvcon processes' I/O consumption +/// and cancels jobs whose read I/O exceeds a configured threshold. +/// +/// When a scratched or damaged disc causes MakeMKV to enter a read retry storm, +/// I/O can balloon to hundreds of gigabytes. This watchdog detects that condition +/// and terminates the offending process to prevent resource starvation of the +/// container and the ASP.NET request pipeline. +/// +public sealed class MakemkvIoWatchdog : BackgroundService +{ + private readonly IServiceScopeFactory _scopeFactory; + private readonly IBackgroundRipService _backgroundRipService; + private readonly IOptions _settings; + private readonly ILogger _logger; + + public MakemkvIoWatchdog( + IServiceScopeFactory scopeFactory, + IBackgroundRipService backgroundRipService, + IOptions settings, + ILogger logger) + { + _scopeFactory = scopeFactory; + _backgroundRipService = backgroundRipService; + _settings = settings; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + if (!OperatingSystem.IsLinux()) + { + _logger.LogWarning("MakemkvIoWatchdog requires Linux — I/O monitoring disabled"); + return; + } + + var maxReadBytes = _settings.Value.MakemkvMaxReadBytes; + if (maxReadBytes <= 0) + { + _logger.LogInformation( + "MakemkvIoWatchdog disabled (MakemkvMaxReadBytes = {Value})", maxReadBytes); + return; + } + + var intervalSec = Math.Max(10, _settings.Value.MakemkvIoWatchdogIntervalSeconds); + _logger.LogInformation( + "MakemkvIoWatchdog started (maxReadBytes={MaxBytes}, interval={Interval}s)", + maxReadBytes, intervalSec); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + await Task.Delay(TimeSpan.FromSeconds(intervalSec), stoppingToken); + await CheckAsync(stoppingToken); + } + catch (OperationCanceledException) + { + break; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "MakemkvIoWatchdog iteration failed"); + } + } + } + + private async Task CheckAsync(CancellationToken ct) + { + // Skip scanning when there are no active rips — nothing to monitor. + if (_backgroundRipService.ActiveCount == 0) + return; + + // 1. Find all makemkvcon processes currently running. + var makemkvPids = FindMakemkvProcesses(); + if (makemkvPids.Count == 0) + return; + + // 2. Read I/O stats for each process. + var pidStats = new Dictionary(); + foreach (var pid in makemkvPids) + { + var readBytes = ReadProcessIoBytes(pid); + if (readBytes.HasValue) + pidStats[pid] = readBytes.Value; + } + + if (pidStats.Count == 0) + return; + + var threshold = _settings.Value.MakemkvMaxReadBytes; + + // 3. Check each process against the threshold. + foreach (var (pid, readBytes) in pidStats) + { + if (readBytes <= threshold) + { + _logger.LogDebug( + "makemkvcon PID {Pid}: {ReadBytes:N0} bytes read (threshold: {Threshold:N0})", + pid, readBytes, threshold); + continue; + } + + _logger.LogWarning( + "makemkvcon PID {Pid} exceeded I/O threshold: {ReadBytes:N0} bytes read (limit: {Threshold:N0})", + pid, readBytes, threshold); + + // 4. Find the associated job and cancel it. + await CancelJobForProcessAsync(pid, readBytes, ct); + } + } + + /// + /// Enumerates all processes on the system whose comm (process name) + /// is makemkvcon. + /// + private static List FindMakemkvProcesses() + { + var pids = new List(); + try + { + foreach (var procDir in Directory.EnumerateDirectories("/proc")) + { + var dirName = Path.GetFileName(procDir); + if (!int.TryParse(dirName, out var pid)) + continue; + + try + { + var commPath = Path.Combine(procDir, "comm"); + if (!File.Exists(commPath)) + continue; + + var comm = File.ReadAllText(commPath).Trim(); + if (string.Equals(comm, "makemkvcon", StringComparison.OrdinalIgnoreCase)) + pids.Add(pid); + } + catch + { + // Process may have exited between enumeration and read — skip. + } + } + } + catch (Exception ex) + { + // /proc not accessible (extremely unlikely on Linux, but handle gracefully). + System.Diagnostics.Debug.WriteLine($"Failed to enumerate /proc: {ex.Message}"); + } + + return pids; + } + + /// + /// Reads the read_bytes counter from /proc/[pid]/io. + /// This counter tracks the number of bytes this process has actually read + /// from block storage (physical I/O, not just page cache hits). + /// Returns null if the process no longer exists or the file is unreadable. + /// + private static long? ReadProcessIoBytes(int pid) + { + try + { + var ioPath = Path.Combine("/proc", pid.ToString(), "io"); + if (!File.Exists(ioPath)) + return null; + + var lines = File.ReadAllLines(ioPath); + foreach (var line in lines) + { + // Format: "read_bytes: 123456789" + if (line.StartsWith("read_bytes:", StringComparison.Ordinal)) + { + var valueStr = line.AsSpan("read_bytes:".Length).Trim(); + if (long.TryParse(valueStr, out var bytes)) + return bytes; + } + } + } + catch + { + // Process likely exited between enumeration and read. + } + + return null; + } + + /// + /// Extracts the optical device path (e.g., /dev/sr0) from a + /// makemkvcon process's command line by looking for an argument + /// starting with dev:. + /// + private static string? GetDevicePathFromProcess(int pid) + { + try + { + var cmdlinePath = Path.Combine("/proc", pid.ToString(), "cmdline"); + if (!File.Exists(cmdlinePath)) + return null; + + var cmdline = File.ReadAllText(cmdlinePath); + // Arguments are separated by null bytes in /proc/[pid]/cmdline. + var args = cmdline.Split('\0', StringSplitOptions.RemoveEmptyEntries); + + foreach (var arg in args) + { + if (arg.StartsWith("dev:", StringComparison.Ordinal)) + return arg[4..]; // Extract "/dev/sr0" from "dev:/dev/sr0" + } + } + catch + { + // Process may have exited. + } + + return null; + } + + /// + /// Finds the active job associated with the given makemkvcon PID and + /// cancels it via . + /// Falls back to directly killing the process if no matching job is found. + /// + private async Task CancelJobForProcessAsync(int pid, long readBytes, CancellationToken ct) + { + // Parse the process command line to find which optical drive it's using. + var devPath = GetDevicePathFromProcess(pid); + + using var scope = _scopeFactory.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + if (devPath is not null) + { + // Look up the active job on this device. + var job = await db.Jobs + .Where(j => j.DevPath == devPath + && j.Status != JobState.Success + && j.Status != JobState.Failure + && j.Status != JobState.Cancelled + && j.Status != JobState.Stopping + && j.Stage == RipStage.Rip) + .FirstOrDefaultAsync(ct); + + if (job is not null) + { + var threshold = _settings.Value.MakemkvMaxReadBytes; + _logger.LogWarning( + "Cancelling job {JobId} ({Title}) on {DevPath} — " + + "makemkvcon PID {Pid} exceeded I/O threshold: {ReadBytes:N0} bytes (limit: {Threshold:N0})", + job.Id, job.Title, job.DevPath, pid, readBytes, threshold); + + // Save the error reason before cancellation so it survives + // the Conductor's catch block (which sets Status/ProgressMessage). + job.Errors = $"Cancelled by I/O watchdog: makemkvcon PID {pid} " + + $"read {readBytes:N0} bytes (limit: {threshold:N0})"; + job.ProgressMessage = "Cancelled — makemkvcon I/O exceeded threshold"; + await db.SaveChangesAsync(ct); + + // Cancel via BackgroundRipService. This triggers the CTS token + // for this devPath, which cascades to the Conductor's + // OperationCanceledException handler and ultimately kills + // the makemkvcon process via CliProcessRunner's ct.Register callback. + _backgroundRipService.CancelRip(devPath); + return; + } + + _logger.LogWarning( + "No active rip job found for device {DevPath} (makemkvcon PID {Pid})", + devPath, pid); + } + else + { + _logger.LogWarning( + "Could not determine device path from makemkvcon PID {Pid} command line", pid); + } + + // Fallback: kill the process directly. Since we can't associate it with + // a job, this at least frees the I/O bandwidth. + _logger.LogWarning( + "Directly killing makemkvcon PID {Pid} — no matching job found", pid); + KillProcess(pid); + } + + /// Sends SIGTERM to the given process. + private static void KillProcess(int pid) + { + try + { + using var proc = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "kill", + Arguments = $"-TERM {pid}", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + proc.Start(); + proc.WaitForExit(5000); + } + catch (Exception ex) + { + System.Diagnostics.Debug.WriteLine($"Failed to kill PID {pid}: {ex.Message}"); + } + } +} diff --git a/src/ArmRipper.WebUi/Program.cs b/src/ArmRipper.WebUi/Program.cs index b9e862b..445fc13 100644 --- a/src/ArmRipper.WebUi/Program.cs +++ b/src/ArmRipper.WebUi/Program.cs @@ -90,6 +90,7 @@ builder.Services.AddHostedService(sp => sp.GetRequiredService()); builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddHostedService(); +builder.Services.AddHostedService(); // ── ArmMedia TV series identification pipeline ── builder.Services.AddHttpClient("Tmdb", client =>