Add black frame analyzer

Author: ConfusedPolarBear

ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestBlackFrames.cs

@@ -2,36 +2,50 @@ namespace ConfusedPolarBear.Plugin.IntroSkipper.Tests;
 
 using System;
 using System.Collections.Generic;
+using Microsoft.Extensions.Logging;
 using Xunit;
 
 public class TestBlackFrames
 {
     [FactSkipFFmpegTests]
     public void TestBlackFrameDetection()
     {
+        var range = 1e-5;
+
         var expected = new List<BlackFrame>();
         expected.AddRange(CreateFrameSequence(2, 3));
         expected.AddRange(CreateFrameSequence(5, 6));
         expected.AddRange(CreateFrameSequence(8, 9.96));
 
-        var actual = FFmpegWrapper.DetectBlackFrames(
-            queueFile("rainbow.mp4"),
-            new TimeRange(0, 10)
-        );
+        var actual = FFmpegWrapper.DetectBlackFrames(queueFile("rainbow.mp4"), new(0, 10), 85);
 
         for (var i = 0; i < expected.Count; i++)
         {
             var (e, a) = (expected[i], actual[i]);
             Assert.Equal(e.Percentage, a.Percentage);
-            Assert.True(Math.Abs(e.Time - a.Time) <= 0.005);
+            Assert.InRange(a.Time, e.Time - range, e.Time + range);
         }
     }
 
+    [FactSkipFFmpegTests]
+    public void TestEndCreditDetection()
+    {
+        var analyzer = CreateBlackFrameAnalyzer();
+
+        var episode = queueFile("credits.mp4");
+        episode.Duration = (int)new TimeSpan(0, 5, 30).TotalSeconds;
+
+        var result = analyzer.AnalyzeMediaFile(episode, AnalysisMode.Credits, 85);
+        Assert.NotNull(result);
+        Assert.Equal(300, result.IntroStart);
+    }
+
     private QueuedEpisode queueFile(string path)
     {
         return new()
         {
             EpisodeId = Guid.NewGuid(),
+            Name = path,
             Path = "../../../video/" + path
         };
     }
@@ -47,4 +61,10 @@ private BlackFrame[] CreateFrameSequence(double start, double end)
 
         return frames.ToArray();
     }
+
+    private BlackFrameAnalyzer CreateBlackFrameAnalyzer()
+    {
+        var logger = new LoggerFactory().CreateLogger<BlackFrameAnalyzer>();
+        return new(logger);
+    }
 }

ConfusedPolarBear.Plugin.IntroSkipper.Tests/video/credits.mp4

@@ -0,0 +1,125 @@
+namespace ConfusedPolarBear.Plugin.IntroSkipper;
+
+using System;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.Linq;
+using System.Threading;
+using Microsoft.Extensions.Logging;
+
+/// <summary>
+/// Media file analyzer used to detect end credits that consist of text overlaid on a black background.
+/// Bisects the end of the video file to perform an efficient search.
+/// </summary>
+public class BlackFrameAnalyzer : IMediaFileAnalyzer
+{
+    private readonly TimeSpan _maximumError = new(0, 0, 4);
+
+    private readonly ILogger<BlackFrameAnalyzer> _logger;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="BlackFrameAnalyzer"/> class.
+    /// </summary>
+    /// <param name="logger">Logger.</param>
+    public BlackFrameAnalyzer(ILogger<BlackFrameAnalyzer> logger)
+    {
+        _logger = logger;
+    }
+
+    /// <inheritdoc />
+    public ReadOnlyCollection<QueuedEpisode> AnalyzeMediaFiles(
+        ReadOnlyCollection<QueuedEpisode> analysisQueue,
+        AnalysisMode mode,
+        CancellationToken cancellationToken)
+    {
+        if (mode != AnalysisMode.Credits)
+        {
+            throw new NotImplementedException("mode must equal Credits");
+        }
+
+        var creditTimes = new Dictionary<Guid, Intro>();
+
+        foreach (var episode in analysisQueue)
+        {
+            if (cancellationToken.IsCancellationRequested)
+            {
+                break;
+            }
+
+            var intro = AnalyzeMediaFile(
+                episode,
+                mode,
+                Plugin.Instance!.Configuration.BlackFrameMinimumPercentage);
+
+            if (intro is null)
+            {
+                continue;
+            }
+
+            creditTimes[episode.EpisodeId] = intro;
+        }
+
+        Plugin.Instance!.UpdateTimestamps(creditTimes, mode);
+
+        return analysisQueue
+            .Where(x => !creditTimes.ContainsKey(x.EpisodeId))
+            .ToList()
+            .AsReadOnly();
+    }
+
+    /// <summary>
+    /// Analyzes an individual media file. Only public because of unit tests.
+    /// </summary>
+    /// <param name="episode">Media file to analyze.</param>
+    /// <param name="mode">Analysis mode.</param>
+    /// <param name="minimum">Percentage of the frame that must be black.</param>
+    /// <returns>Credits timestamp.</returns>
+    public Intro? AnalyzeMediaFile(QueuedEpisode episode, AnalysisMode mode, int minimum)
+    {
+        // Start by analyzing the last four minutes of the file.
+        var start = TimeSpan.FromMinutes(4);
+        var end = TimeSpan.Zero;
+        var firstFrameTime = 0.0;
+
+        // Continue bisecting the end of the file until the range that contains the first black
+        // frame is smaller than the maximum permitted error.
+        while (start - end > _maximumError)
+        {
+            // Analyze the middle two seconds from the current bisected range
+            var midpoint = (start + end) / 2;
+            var scanTime = episode.Duration - midpoint.TotalSeconds;
+            var tr = new TimeRange(scanTime, scanTime + 2);
+
+            _logger.LogTrace(
+                "{Episode}, dur {Duration}, bisect [{BStart}, {BEnd}], time [{Start}, {End}]",
+                episode.Name,
+                episode.Duration,
+                start,
+                end,
+                tr.Start,
+                tr.End);
+
+            var frames = FFmpegWrapper.DetectBlackFrames(episode, tr, minimum);
+            _logger.LogTrace("{Episode}, black frames: {Count}", episode.Name, frames.Length);
+
+            if (frames.Length == 0)
+            {
+                // Since no black frames were found, slide the range closer to the end
+                start = midpoint;
+            }
+            else
+            {
+                // Some black frames were found, slide the range closer to the start
+                end = midpoint;
+                firstFrameTime = frames[0].Time + scanTime;
+            }
+        }
+
+        if (firstFrameTime > 0)
+        {
+            return new(episode.EpisodeId, new TimeRange(firstFrameTime, episode.Duration));
+        }
+
+        return null;
+    }
+}

ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/BlackFrameAnalyzer.cs

@@ -222,8 +222,12 @@ public static TimeRange[] DetectSilence(QueuedEpisode episode, int limit)
     /// </summary>
     /// <param name="episode">Media file to analyze.</param>
     /// <param name="range">Time range to search.</param>
-    /// <returns>Array of frames that are at least 50% black.</returns>
-    public static BlackFrame[] DetectBlackFrames(QueuedEpisode episode, TimeRange range)
+    /// <param name="minimum">Percentage of the frame that must be black.</param>
+    /// <returns>Array of frames that are mostly black.</returns>
+    public static BlackFrame[] DetectBlackFrames(
+        QueuedEpisode episode,
+        TimeRange range,
+        int minimum)
     {
         // Seek to the start of the time range and find frames that are at least 50% black.
         var args = string.Format(
@@ -233,10 +237,10 @@ public static BlackFrame[] DetectBlackFrames(QueuedEpisode episode, TimeRange ra
             episode.Path,
             range.End - range.Start);
 
-        // Cache the results to GUID-blackframes-v1-START-END.
+        // Cache the results to GUID-blackframes-START-END-v1.
         var cacheKey = string.Format(
             CultureInfo.InvariantCulture,
-            "{0}-blackframes-v1-{1}-{2}",
+            "{0}-blackframes-{1}-{2}-v1",
             episode.EpisodeId.ToString("N"),
             range.Start,
             range.End);
@@ -263,10 +267,14 @@ public static BlackFrame[] DetectBlackFrames(QueuedEpisode episode, TimeRange ra
                 matches[1].Value.Split(':')[1]
             );
 
-            blackFrames.Add(new(
+            var bf = new BlackFrame(
                 Convert.ToInt32(strPercent, CultureInfo.InvariantCulture),
-                Convert.ToDouble(strTime, CultureInfo.InvariantCulture)
-            ));
+                Convert.ToDouble(strTime, CultureInfo.InvariantCulture));
+
+            if (bf.Percentage > minimum)
+            {
+                blackFrames.Add(bf);
+            }
         }
 
         return blackFrames.ToArray();

ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs

@@ -122,10 +122,8 @@ public Task ExecuteAsync(IProgress<double> progress, CancellationToken cancellat
                     return;
                 }
 
-                // Increment totalProcessed by the number of episodes in this season that were actually analyzed
-                // (instead of just using the number of episodes in the current season).
-                var analyzed = AnalyzeSeason(episodes, cancellationToken);
-                Interlocked.Add(ref totalProcessed, analyzed);
+                AnalyzeSeason(episodes, cancellationToken);
+                Interlocked.Add(ref totalProcessed, episodes.Count);
             }
             catch (FingerprintException ex)
             {
@@ -151,39 +149,49 @@ public Task ExecuteAsync(IProgress<double> progress, CancellationToken cancellat
     }
 
     /// <summary>
-    /// Fingerprints all episodes in the provided season and stores the timestamps of all introductions.
+    /// Analyzes all episodes in the season for end credits.
     /// </summary>
     /// <param name="episodes">Episodes in this season.</param>
     /// <param name="cancellationToken">Cancellation token provided by the scheduled task.</param>
-    /// <returns>Number of episodes from the provided season that were analyzed.</returns>
-    private int AnalyzeSeason(
+    private void AnalyzeSeason(
         ReadOnlyCollection<QueuedEpisode> episodes,
         CancellationToken cancellationToken)
     {
-        // Skip seasons with an insufficient number of episodes.
-        if (episodes.Count <= 1)
+        // Only analyze specials (season 0) if the user has opted in.
+        if (episodes[0].SeasonNumber == 0 && !Plugin.Instance!.Configuration.AnalyzeSeasonZero)
         {
-            return episodes.Count;
+            return;
         }
 
-        // Only analyze specials (season 0) if the user has opted in.
-        var first = episodes[0];
-        if (first.SeasonNumber == 0 && !Plugin.Instance!.Configuration.AnalyzeSeasonZero)
+        // Analyze with Chromaprint first and fall back to the black frame detector
+        var analyzers = new IMediaFileAnalyzer[]
+        {
+            // TODO: FIXME: new ChromaprintAnalyzer(_loggerFactory.CreateLogger<ChromaprintAnalyzer>()),
+            new BlackFrameAnalyzer(_loggerFactory.CreateLogger<BlackFrameAnalyzer>())
+        };
+
+        // Use each analyzer to find credits in all media files, removing successfully analyzed files
+        // from the queue.
+        var remaining = new ReadOnlyCollection<QueuedEpisode>(episodes);
+        foreach (var analyzer in analyzers)
         {
-            return 0;
+            remaining = AnalyzeFiles(remaining, analyzer, cancellationToken);
         }
+    }
 
+    private ReadOnlyCollection<QueuedEpisode> AnalyzeFiles(
+        ReadOnlyCollection<QueuedEpisode> episodes,
+        IMediaFileAnalyzer analyzer,
+        CancellationToken cancellationToken)
+    {
         _logger.LogInformation(
-            "Analyzing {Count} episodes from {Name} season {Season}",
+            "Analyzing {Count} episodes from {Name} season {Season} with {Analyzer}",
             episodes.Count,
-            first.SeriesName,
-            first.SeasonNumber);
-
-        // Analyze the season with Chromaprint
-        var chromaprint = new ChromaprintAnalyzer(_loggerFactory.CreateLogger<ChromaprintAnalyzer>());
-        chromaprint.AnalyzeMediaFiles(episodes, AnalysisMode.Credits, cancellationToken);
+            episodes[0].SeriesName,
+            episodes[0].SeasonNumber,
+            analyzer.GetType().Name);
 
-        return episodes.Count;
+        return analyzer.AnalyzeMediaFiles(episodes, AnalysisMode.Credits, cancellationToken);
     }
 
     /// <summary>