Enhance Chromaprint Accuracy Using Chapters (#203)

* Use Helper Class and Limit silence scanning to the relevant time range only

* Include the timerange in the filename

* Update AnalyzerHelper.cs

* Update AnalyzerHelper.cs

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: rlauu <46294892+rlauu@users.noreply.github.com>

ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs

@@ -132,7 +132,8 @@ public class TestAudioFingerprinting
             new TimeRange(54.9294, 55.2590),
         };
 
-        var actual = FFmpegWrapper.DetectSilence(clip, 60);
+        var range = new TimeRange(0, 60);
+        var actual = FFmpegWrapper.DetectSilence(clip, range);
 
         Assert.Equal(expected, actual);
     }

ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/AnalyzerHelper.cs

@@ -0,0 +1,132 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using ConfusedPolarBear.Plugin.IntroSkipper.Configuration;
+using ConfusedPolarBear.Plugin.IntroSkipper.Data;
+using MediaBrowser.Model.Entities;
+using Microsoft.Extensions.Logging;
+
+namespace ConfusedPolarBear.Plugin.IntroSkipper;
+
+/// <summary>
+/// Analyzer Helper.
+/// </summary>
+public class AnalyzerHelper
+{
+    private readonly ILogger _logger;
+    private readonly double silenceDetectionMinimumDuration;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="AnalyzerHelper"/> class.
+    /// </summary>
+    /// <param name="logger">Logger.</param>
+    public AnalyzerHelper(ILogger logger)
+    {
+        var config = Plugin.Instance?.Configuration ?? new PluginConfiguration();
+        silenceDetectionMinimumDuration = config.SilenceDetectionMinimumDuration;
+        _logger = logger;
+    }
+
+    /// <summary>
+    /// Adjusts the end timestamps of all intros so that they end at silence.
+    /// </summary>
+    /// <param name="episodes">QueuedEpisodes to adjust.</param>
+    /// <param name="originalIntros">Original introductions.</param>
+    /// <param name="mode">Analysis mode.</param>
+    /// <returns>Modified Intro Timestamps.</returns>
+    public Dictionary<Guid, Intro> AdjustIntroTimes(
+            ReadOnlyCollection<QueuedEpisode> episodes,
+            Dictionary<Guid, Intro> originalIntros,
+            AnalysisMode mode)
+        {
+            var modifiedIntros = new Dictionary<Guid, Intro>();
+
+            foreach (var episode in episodes)
+            {
+                _logger.LogTrace("Adjusting introduction end time for {Name} ({Id})", episode.Name, episode.EpisodeId);
+
+                if (!originalIntros.TryGetValue(episode.EpisodeId, out var originalIntro))
+                {
+                    _logger.LogTrace("{Name} does not have an intro", episode.Name);
+                    continue;
+                }
+
+                var adjustedIntro = AdjustIntroForEpisode(episode, originalIntro, mode);
+                modifiedIntros[episode.EpisodeId] = adjustedIntro;
+            }
+
+            return modifiedIntros;
+        }
+
+    private Intro AdjustIntroForEpisode(QueuedEpisode episode, Intro originalIntro, AnalysisMode mode)
+    {
+        var chapters = GetChaptersWithVirtualEnd(episode);
+        var adjustedIntro = new Intro(originalIntro);
+
+        var originalIntroStart = new TimeRange(Math.Max(0, (int)originalIntro.IntroStart - 5), (int)originalIntro.IntroStart + 10);
+        var originalIntroEnd = new TimeRange((int)originalIntro.IntroEnd - 10, Math.Min(episode.Duration, (int)originalIntro.IntroEnd + 5));
+
+        _logger.LogTrace("{Name} original intro: {Start} - {End}", episode.Name, originalIntro.IntroStart, originalIntro.IntroEnd);
+
+        if (!AdjustIntroBasedOnChapters(episode, chapters, adjustedIntro, originalIntroStart, originalIntroEnd)
+            && mode == AnalysisMode.Introduction)
+        {
+            AdjustIntroBasedOnSilence(episode, adjustedIntro, originalIntroEnd);
+        }
+
+        return adjustedIntro;
+    }
+
+    private List<ChapterInfo> GetChaptersWithVirtualEnd(QueuedEpisode episode)
+    {
+        var chapters = Plugin.Instance?.GetChapters(episode.EpisodeId) ?? new List<ChapterInfo>();
+        chapters.Add(new ChapterInfo { StartPositionTicks = TimeSpan.FromSeconds(episode.Duration).Ticks });
+        return chapters;
+    }
+
+    private bool AdjustIntroBasedOnChapters(QueuedEpisode episode, List<ChapterInfo> chapters, Intro adjustedIntro, TimeRange originalIntroStart, TimeRange originalIntroEnd)
+    {
+        foreach (var chapter in chapters)
+        {
+            var chapterStartSeconds = TimeSpan.FromTicks(chapter.StartPositionTicks).TotalSeconds;
+
+            if (originalIntroStart.Start < chapterStartSeconds && chapterStartSeconds < originalIntroStart.End)
+            {
+                adjustedIntro.IntroStart = chapterStartSeconds;
+                _logger.LogTrace("{Name} chapter found close to intro start: {Start}", episode.Name, chapterStartSeconds);
+            }
+
+            if (originalIntroEnd.Start < chapterStartSeconds && chapterStartSeconds < originalIntroEnd.End)
+            {
+                adjustedIntro.IntroEnd = chapterStartSeconds;
+                _logger.LogTrace("{Name} chapter found close to intro end: {End}", episode.Name, chapterStartSeconds);
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    private void AdjustIntroBasedOnSilence(QueuedEpisode episode, Intro adjustedIntro, TimeRange originalIntroEnd)
+    {
+        var silence = FFmpegWrapper.DetectSilence(episode, originalIntroEnd);
+
+        foreach (var currentRange in silence)
+        {
+            _logger.LogTrace("{Name} silence: {Start} - {End}", episode.Name, currentRange.Start, currentRange.End);
+
+            if (IsValidSilenceForIntroAdjustment(currentRange, originalIntroEnd, adjustedIntro))
+            {
+                adjustedIntro.IntroEnd = currentRange.Start;
+                break;
+            }
+        }
+    }
+
+    private bool IsValidSilenceForIntroAdjustment(TimeRange silenceRange, TimeRange originalIntroEnd, Intro adjustedIntro)
+    {
+        return originalIntroEnd.Intersects(silenceRange) &&
+                silenceRange.Duration >= silenceDetectionMinimumDuration &&
+                silenceRange.Start >= adjustedIntro.IntroStart;
+    }
+}

ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/BlackFrameAnalyzer.cs

@@ -119,6 +119,9 @@ public class BlackFrameAnalyzer : IMediaFileAnalyzer
             episode.State.SetAnalyzed(mode, true);
         }
 
+        var analyzerHelper = new AnalyzerHelper(_logger);
+        creditTimes = analyzerHelper.AdjustIntroTimes(analysisQueue, creditTimes, mode);
+
         Plugin.Instance!.UpdateTimestamps(creditTimes, mode);
 
         return episodeAnalysisQueue.AsReadOnly();

ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChromaprintAnalyzer.cs

@@ -30,8 +30,6 @@ public class ChromaprintAnalyzer : IMediaFileAnalyzer
 
     private double maximumTimeSkip;
 
-    private double silenceDetectionMinimumDuration;
-
     private ILogger<ChromaprintAnalyzer> _logger;
 
     private AnalysisMode _analysisMode;
@@ -46,7 +44,6 @@ public class ChromaprintAnalyzer : IMediaFileAnalyzer
         maximumDifferences = config.MaximumFingerprintPointDifferences;
         invertedIndexShift = config.InvertedIndexShift;
         maximumTimeSkip = config.MaximumTimeSkip;
-        silenceDetectionMinimumDuration = config.SilenceDetectionMinimumDuration;
         minimumIntroDuration = config.MinimumIntroDuration;
 
         _logger = logger;
@@ -204,11 +201,9 @@ public class ChromaprintAnalyzer : IMediaFileAnalyzer
             return analysisQueue;
         }
 
-        if (_analysisMode == AnalysisMode.Introduction)
+        // Adjust all introduction times.
-        {
+        var analyzerHelper = new AnalyzerHelper(_logger);
-            // Adjust all introduction end times so that they end at silence.
+        seasonIntros = analyzerHelper.AdjustIntroTimes(analysisQueue, seasonIntros, this._analysisMode);
-            seasonIntros = AdjustIntroEndTimes(analysisQueue, seasonIntros);
-        }
 
         Plugin.Instance!.UpdateTimestamps(seasonIntros, _analysisMode);
 
@@ -402,103 +397,9 @@ public class ChromaprintAnalyzer : IMediaFileAnalyzer
 
         // Since LHS had a contiguous time range, RHS must have one also.
         var rContiguous = TimeRangeHelpers.FindContiguous(rhsTimes.ToArray(), maximumTimeSkip)!;
-
-        if (_analysisMode == AnalysisMode.Introduction)
-        {
-            // Tweak the end timestamps just a bit to ensure as little content as possible is skipped over.
-            // TODO: remove this
-            if (lContiguous.Duration >= 90)
-            {
-                lContiguous.End -= 2 * maximumTimeSkip;
-                rContiguous.End -= 2 * maximumTimeSkip;
-            }
-            else if (lContiguous.Duration >= 30)
-            {
-                lContiguous.End -= maximumTimeSkip;
-                rContiguous.End -= maximumTimeSkip;
-            }
-        }
-
         return (lContiguous, rContiguous);
     }
 
-    /// <summary>
-    /// Adjusts the end timestamps of all intros so that they end at silence.
-    /// </summary>
-    /// <param name="episodes">QueuedEpisodes to adjust.</param>
-    /// <param name="originalIntros">Original introductions.</param>
-    private Dictionary<Guid, Intro> AdjustIntroEndTimes(
-        ReadOnlyCollection<QueuedEpisode> episodes,
-        Dictionary<Guid, Intro> originalIntros)
-    {
-        Dictionary<Guid, Intro> modifiedIntros = new();
-
-        // For all episodes
-        foreach (var episode in episodes)
-        {
-            _logger.LogTrace(
-                "Adjusting introduction end time for {Name} ({Id})",
-                episode.Name,
-                episode.EpisodeId);
-
-            // If no intro was found for this episode, skip it.
-            if (!originalIntros.TryGetValue(episode.EpisodeId, out var originalIntro))
-            {
-                _logger.LogTrace("{Name} does not have an intro", episode.Name);
-                continue;
-            }
-
-            // Only adjust the end timestamp of the intro
-            var originalIntroEnd = new TimeRange(originalIntro.IntroEnd - 15, originalIntro.IntroEnd);
-
-            _logger.LogTrace(
-                "{Name} original intro: {Start} - {End}",
-                episode.Name,
-                originalIntro.IntroStart,
-                originalIntro.IntroEnd);
-
-            // Detect silence in the media file up to the end of the intro.
-            var silence = FFmpegWrapper.DetectSilence(episode, (int)originalIntro.IntroEnd + 2);
-
-            // For all periods of silence
-            foreach (var currentRange in silence)
-            {
-                _logger.LogTrace(
-                    "{Name} silence: {Start} - {End}",
-                    episode.Name,
-                    currentRange.Start,
-                    currentRange.End);
-
-                // Ignore any silence that:
-                // * doesn't intersect the ending of the intro, or
-                // * is shorter than the user defined minimum duration, or
-                // * starts before the introduction does
-                if (
-                    !originalIntroEnd.Intersects(currentRange) ||
-                    currentRange.Duration < silenceDetectionMinimumDuration ||
-                    currentRange.Start < originalIntro.IntroStart)
-                {
-                    continue;
-                }
-
-                // Adjust the end timestamp of the intro to match the start of the silence region.
-                originalIntro.IntroEnd = currentRange.Start;
-                break;
-            }
-
-            _logger.LogTrace(
-                "{Name} adjusted intro: {Start} - {End}",
-                episode.Name,
-                originalIntro.IntroStart,
-                originalIntro.IntroEnd);
-
-            // Add the (potentially) modified intro back.
-            modifiedIntros[episode.EpisodeId] = originalIntro;
-        }
-
-        return modifiedIntros;
-    }
-
     /// <summary>
     /// Count the number of bits that are set in the provided number.
     /// </summary>

ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs

@@ -166,27 +166,34 @@ public static class FFmpegWrapper
     /// Detect ranges of silence in the provided episode.
     /// </summary>
     /// <param name="episode">Queued episode.</param>
-    /// <param name="limit">Maximum amount of audio (in seconds) to detect silence in.</param>
+    /// <param name="range">Time range to search.</param>
     /// <returns>Array of TimeRange objects that are silent in the queued episode.</returns>
-    public static TimeRange[] DetectSilence(QueuedEpisode episode, int limit)
+    public static TimeRange[] DetectSilence(QueuedEpisode episode, TimeRange range)
     {
         Logger?.LogTrace(
-            "Detecting silence in \"{File}\" (limit {Limit}, id {Id})",
+            "Detecting silence in \"{File}\" (range {Start}-{End}, id {Id})",
             episode.Path,
-            limit,
+            range.Start,
+            range.End,
             episode.EpisodeId);
 
         // -vn, -sn, -dn: ignore video, subtitle, and data tracks
         var args = string.Format(
             CultureInfo.InvariantCulture,
             "-vn -sn -dn " +
-                "-i \"{0}\" -to {1} -af \"silencedetect=noise={2}dB:duration=0.1\" -f null -",
+                "-ss {0} -i \"{1}\" -to {2} -af \"silencedetect=noise={3}dB:duration=0.1\" -f null -",
+            range.Start,
             episode.Path,
-            limit,
+            range.End - range.Start,
             Plugin.Instance?.Configuration.SilenceDetectionMaximumNoise ?? -50);
 
-        // Cache the output of this command to "GUID-intro-silence-v1"
+        // Cache the output of this command to "GUID-intro-silence-v2"
-        var cacheKey = episode.EpisodeId.ToString("N") + "-intro-silence-v1";
+        var cacheKey = string.Format(
+            CultureInfo.InvariantCulture,
+            "{0}-silence-{1}-{2}-v2",
+            episode.EpisodeId.ToString("N"),
+            range.Start,
+            range.End);
 
         var currentRange = new TimeRange();
         var silenceRanges = new List<TimeRange>();
@@ -205,11 +212,11 @@ public static class FFmpegWrapper
 
             if (isStart)
             {
-                currentRange.Start = time;
+                currentRange.Start = time + range.Start;
             }
             else
             {
-                currentRange.End = time;
+                currentRange.End = time + range.Start;
                 silenceRanges.Add(new TimeRange(currentRange));
             }
         }