From 8ee400f1f1b0e6e1a263787579f32285376e18d6 Mon Sep 17 00:00:00 2001
From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com>
Date: Tue, 1 Nov 2022 00:53:56 -0500
Subject: [PATCH] Add blackframe API

---
 .../TestBlackFrames.cs                        |  50 +++++++++++
 .../video/rainbow.mp4                         | Bin 0 -> 15787 bytes
 .../Configuration/PluginConfiguration.cs      |   5 ++
 .../Data/BlackFrame.cs                        |  28 ++++++
 .../FFmpegWrapper.cs                          |  80 ++++++++++++++++--
 5 files changed, 154 insertions(+), 9 deletions(-)
 create mode 100644 ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestBlackFrames.cs
 create mode 100644 ConfusedPolarBear.Plugin.IntroSkipper.Tests/video/rainbow.mp4
 create mode 100644 ConfusedPolarBear.Plugin.IntroSkipper/Data/BlackFrame.cs

diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestBlackFrames.cs b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestBlackFrames.cs
new file mode 100644
index 0000000..794ecd1
--- /dev/null
+++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestBlackFrames.cs
@@ -0,0 +1,50 @@
+namespace ConfusedPolarBear.Plugin.IntroSkipper.Tests;
+
+using System;
+using System.Collections.Generic;
+using Xunit;
+
+public class TestBlackFrames
+{
+    [FactSkipFFmpegTests]
+    public void TestBlackFrameDetection()
+    {
+        var expected = new List<BlackFrame>();
+        expected.AddRange(CreateFrameSequence(2, 3));
+        expected.AddRange(CreateFrameSequence(5, 6));
+        expected.AddRange(CreateFrameSequence(8, 9.96));
+
+        var actual = FFmpegWrapper.DetectBlackFrames(
+            queueFile("rainbow.mp4"),
+            new TimeRange(0, 10)
+        );
+
+        for (var i = 0; i < expected.Count; i++)
+        {
+            var (e, a) = (expected[i], actual[i]);
+            Assert.Equal(e.Percentage, a.Percentage);
+            Assert.True(Math.Abs(e.Time - a.Time) <= 0.005);
+        }
+    }
+
+    private QueuedEpisode queueFile(string path)
+    {
+        return new()
+        {
+            EpisodeId = Guid.NewGuid(),
+            Path = "../../../video/" + path
+        };
+    }
+
+    private BlackFrame[] CreateFrameSequence(double start, double end)
+    {
+        var frames = new List<BlackFrame>();
+
+        for (var i = start; i < end; i += 0.04)
+        {
+            frames.Add(new(100, i));
+        }
+
+        return frames.ToArray();
+    }
+}
diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/video/rainbow.mp4 b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/video/rainbow.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..8e01cce620da0f4d72f647169d5f42f3a9b67c08
GIT binary patch
literal 15787
zcmeHOeQXp(6rbbxN<*<gm7>KgT2xGX-MxMnHuh+$MT-LALsUHGcK3QqcW<}5)Aovy
z)Cj~sBqkb5C`lU<u%bjzRKg#om7tNYQ2tP8LX1jcKoUi0v_ZY|&F;0mo<#qV=0axD
zck|xNo8NqG=ia=1hY;ezq}^0)Lq}*5qB)Qsw@QL;mI?@=>0wJz5b|ldj0xy`Fm*~I
z@!rL{E1&&1SbwCm@B%mZ;LpFr{N<%w5hodz!U<(%oaL_)DmWfjmdaI?P*Lv#h2rHa
z>YgYn<pOJKK~q*FsHio}c1;Nr&d>8DMSfnO+7Uv`>f++Iwl-g@Dl3MDqdvoGET+Bq
zB1G3f$1sU%MD1!$!XYe)0%s{<v4oSAkY-3tqEO9O^Bj(1t=(2cK2}oA#{{8-(-kqI
z#5lVp1Y8AY+U?L3K7*Dl3O*hh!3U?SF+~niL;+d^EgWrBM4_CMB9@`!AT$*?Vkw%Y
z+Q3!DDrJcPBQ@(H4@1E6W+SSIexXnhxG=U!(6pOWlWMpC&1TRDhi!$3MShM%ENEfF
zXf(rU!VwUHBV__-YpMie87bhosO9t_sX8Wf9BPy(mWDwI%0pU<h1-LYp_`b1EWr{H
z3#(Cp07VN^ov?*<#fHTS1<iKgRax`{k7aDqS%iWi72C8E)l!^!wJB<2goL2ZFqLSq
z(J-ObEi|ENlhO{@qQ8t!vO!&qg2>sD5>=!YB9`)Wm=>l>YAJREnp;wEbh@<Ol0?Z0
zb9xBYl}-x;(O=@@xn?>Uk@uAYYf@k*$1$<I3JPqZm|`iXny^XmK)@aX7H)?9r@M(d
z;0d1BD5LBHgMq*wo*aa{h<<pvUzhIM2$fSKS1+A<pdYw^^4eCkI1#_}{&4QOgL}@l
z?Oj=Ma7cRQZ0=6&`kadVgYjuEmu*CaxN~UHmkn<}KHw=tXm<aZ?R?k%6H9L1T(=TH
zvIAWOT5Vr^e<wl)Ma>FyFO6+WAoS`hRPuCVva}DOPdi*m+T=*IZcd;pvh3YJWLLxV
zwuB4tITX(bbZ@VlRal$?vCoA_0`|HP6c8W|0ECtT(eFZ>th;+0#FcTQxH=BvS`xxY
z#Cx|Q^eydYMxbZrc!-{&)KT;lk3q~1>@H10G$!KRImjD!5mu!kG^Y-t#~z0eOCqdG
z#4Br1_A(cta|EFS<Sb9mC*#I&#6^JdvQvtkQg{9-b>^o<1Mz3)Y}rWIDaB4HW=k<!
z3QWh0u%#9!ZK>a%?whoG^|432Z?4Pw?d0Bv)*oHf<hlM1LN}gVzo+k|Q!ie=Hf1_O
z+VR8l&n?(j`T83bwba^RRuov4%!*=nJ$BclcfQ+u*E>JhMH)_<agJG0?5@W=Qp_Vo
z9dH?8MWuVB@ag3rtvI*iu_dcE9R8}|qSZG%f9UHD^x^jB2YToE*W6b#`4B=+dCT*T
zZR#z!arMfE0JEhWM;o)HnBScF&2N+6oZa=hn8%#G*K=MBviEwK{a)|Fiq)jyi$~(y
zK4u<s<}qg;bLKJ6#Jk=%)_{FJ{+8xr(Fe>U#XM5XBb6~8DR$RmcRd!v$znJ&HHMR&
zPj|xkbk|(nFk0c=g{HSgWXMyJ_cB5c=O7QFe`9H!0L5u?&6wt`xJd}j3_y5t6I8xH
znw&npDT+xn=r=v3^>eF!JvGqin$xgAk%<jlzM|O>OYbB=ptzHhW6nOR%PNK%PM6c?
z?gyPAwC1?et|%gFmJ5LpjglYXX}*?5&DZ0otSM9{TZXu48ibqMsym>m!!D5Bx_lXe
zvuR^Uw0=g5rExBwU0{=t27JFw?2!$jI)~^F1t6y>71J%KfoazRONpj`Xt>2GpLq!g
zb4KNWTI+$J?Zo&061?phiQWEdc$h-unAkQUr@c_hg-G(`A}`EGh!V=PiL(K~IyyJ$
zj~*aZ!o29l(3Ay4+usXR0@MIR7k3NLPM||TCxCuHXxewu7#?8JcCe97PGM4IDpJ|$
z5AsZ}XY<Ic{bKvU_QN@s{^xUqJy-0xV$aps_cs0a8vUNm2fILWN}Muhos!@4-anyk
zbFz+RQ0~1S@>xJLQ)n*mVBn3G{g-@Cntqx*A${b*81965OvHY$wHkZ2)1Fn7CVMAh
zttOxklxq{ofd3Ji=v{#a3bn`tL;jquI61IrLFy$9E_N0+O?r<SO+~7E6I{5T7!+_m
W{K%SuP*Bu>Tf=2lKB3$v@ciHQ!_Ged

literal 0
HcmV?d00001

diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs
index ce79fdd..f4789db 100644
--- a/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs
+++ b/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs
@@ -77,6 +77,11 @@ public class PluginConfiguration : BasePluginConfiguration
     /// </summary>
     public int MaximumEpisodeCreditsDuration { get; set; } = 4;
 
+    /// <summary>
+    /// Gets or sets the minimum percentage of a frame that must consist of black pixels before it is considered a black frame.
+    /// </summary>
+    public int BlackFrameMinimumPercentage { get; set; } = 85;
+
     // ===== Playback settings =====
 
     /// <summary>
diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Data/BlackFrame.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Data/BlackFrame.cs
new file mode 100644
index 0000000..df3a957
--- /dev/null
+++ b/ConfusedPolarBear.Plugin.IntroSkipper/Data/BlackFrame.cs
@@ -0,0 +1,28 @@
+namespace ConfusedPolarBear.Plugin.IntroSkipper;
+
+/// <summary>
+/// A frame of video that partially (or entirely) consists of black pixels.
+/// </summary>
+public class BlackFrame
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="BlackFrame"/> class.
+    /// </summary>
+    /// <param name="percent">Percentage of the frame that is black.</param>
+    /// <param name="time">Time this frame appears at.</param>
+    public BlackFrame(int percent, double time)
+    {
+        Percentage = percent;
+        Time = time;
+    }
+
+    /// <summary>
+    /// Gets or sets the percentage of the frame that is black.
+    /// </summary>
+    public int Percentage { get; set; }
+
+    /// <summary>
+    /// Gets or sets the time (in seconds) this frame appeared at.
+    /// </summary>
+    public double Time { get; set; }
+}
diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs b/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs
index 66fc68b..e8ba464 100644
--- a/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs
+++ b/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs
@@ -16,16 +16,17 @@ public static class FFmpegWrapper
 {
     private static readonly object InvertedIndexCacheLock = new();
 
-    // FFmpeg logs lines similar to the following:
-    // [silencedetect @ 0x000000000000] silence_start: 12.34
-    // [silencedetect @ 0x000000000000] silence_end: 56.123 | silence_duration: 43.783
-
     /// <summary>
     /// Used with FFmpeg's silencedetect filter to extract the start and end times of silence.
     /// </summary>
     private static readonly Regex SilenceDetectionExpression = new(
         "silence_(?<type>start|end): (?<time>[0-9\\.]+)");
 
+    /// <summary>
+    /// Used with FFmpeg's blackframe filter to extract the time and percentage of black pixels.
+    /// </summary>
+    private static readonly Regex BlackFrameRegex = new("(pblack|t):[0-9.]+");
+
     /// <summary>
     /// Gets or sets the logger.
     /// </summary>
@@ -190,7 +191,12 @@ public static class FFmpegWrapper
         var currentRange = new TimeRange();
         var silenceRanges = new List<TimeRange>();
 
-        // Each match will have a type (either "start" or "end") and a timecode (a double).
+        /* Each match will have a type (either "start" or "end") and a timecode (a double).
+         *
+         * Sample output:
+         * [silencedetect @ 0x000000000000] silence_start: 12.34
+         * [silencedetect @ 0x000000000000] silence_end: 56.123 | silence_duration: 43.783
+        */
         var raw = Encoding.UTF8.GetString(GetOutput(args, cacheKey, true));
         foreach (Match match in SilenceDetectionExpression.Matches(raw))
         {
@@ -211,6 +217,61 @@ public static class FFmpegWrapper
         return silenceRanges.ToArray();
     }
 
+    /// <summary>
+    /// Finds the location of all black frames in a media file within a time range.
+    /// </summary>
+    /// <param name="episode">Media file to analyze.</param>
+    /// <param name="range">Time range to search.</param>
+    /// <returns>Array of frames that are at least 50% black.</returns>
+    public static BlackFrame[] DetectBlackFrames(QueuedEpisode episode, TimeRange range)
+    {
+        // Seek to the start of the time range and find frames that are at least 50% black.
+        var args = string.Format(
+            CultureInfo.InvariantCulture,
+            "-ss {0} -i \"{1}\" -to {2} -an -dn -sn -vf \"blackframe=amount=50\" -f null -",
+            range.Start,
+            episode.Path,
+            range.End - range.Start);
+
+        // Cache the results to GUID-blackframes-v1-START-END.
+        var cacheKey = string.Format(
+            CultureInfo.InvariantCulture,
+            "{0}-blackframes-v1-{1}-{2}",
+            episode.EpisodeId.ToString("N"),
+            range.Start,
+            range.End);
+
+        var blackFrames = new List<BlackFrame>();
+
+        /* Run the blackframe filter.
+         *
+         * Sample output:
+         * [Parsed_blackframe_0 @ 0x0000000] frame:1 pblack:99 pts:43 t:0.043000 type:B last_keyframe:0
+         * [Parsed_blackframe_0 @ 0x0000000] frame:2 pblack:99 pts:85 t:0.085000 type:B last_keyframe:0
+         */
+        var raw = Encoding.UTF8.GetString(GetOutput(args, cacheKey, true));
+        foreach (var line in raw.Split('\n'))
+        {
+            var matches = BlackFrameRegex.Matches(line);
+            if (matches.Count != 2)
+            {
+                continue;
+            }
+
+            var (strPercent, strTime) = (
+                matches[0].Value.Split(':')[1],
+                matches[1].Value.Split(':')[1]
+            );
+
+            blackFrames.Add(new(
+                Convert.ToInt32(strPercent, CultureInfo.InvariantCulture),
+                Convert.ToDouble(strTime, CultureInfo.InvariantCulture)
+            ));
+        }
+
+        return blackFrames.ToArray();
+    }
+
     /// <summary>
     /// Gets Chromaprint debugging logs.
     /// </summary>
@@ -296,10 +357,11 @@ public static class FFmpegWrapper
     {
         var ffmpegPath = Plugin.Instance?.FFmpegPath ?? "ffmpeg";
 
-        // The silencedetect filter outputs silence timestamps at the info log level.
-        var logLevel = args.Contains("silencedetect", StringComparison.OrdinalIgnoreCase) ?
-            "info" :
-            "warning";
+        // The silencedetect and blackframe filters output data at the info log level.
+        var useInfoLevel = args.Contains("silencedetect", StringComparison.OrdinalIgnoreCase) ||
+            args.Contains("blackframe", StringComparison.OrdinalIgnoreCase);
+
+        var logLevel = useInfoLevel ? "info" : "warning";
 
         var cacheOutput =
             (Plugin.Instance?.Configuration.CacheFingerprints ?? false) &&