From c4e890a9ac090397e29bf5982ffcb9d24536e1ee Mon Sep 17 00:00:00 2001 From: rlauuzo <46294892+rlauuzo@users.noreply.github.com> Date: Sat, 31 Aug 2024 19:32:37 +0200 Subject: [PATCH] Enhance Chromaprint Accuracy Using Chapters (#203) * Use Helper Class and Limit silence scanning to the relevant time range only * Include the timerange in the filename * Update AnalyzerHelper.cs * Update AnalyzerHelper.cs --------- Co-authored-by: github-actions[bot] Co-authored-by: rlauu <46294892+rlauu@users.noreply.github.com> --- .../TestAudioFingerprinting.cs | 3 +- .../Analyzers/AnalyzerHelper.cs | 132 ++++++++++++++++++ .../Analyzers/BlackFrameAnalyzer.cs | 3 + .../Analyzers/ChromaprintAnalyzer.cs | 105 +------------- .../FFmpegWrapper.cs | 27 ++-- 5 files changed, 157 insertions(+), 113 deletions(-) create mode 100644 ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/AnalyzerHelper.cs diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs index 4bf7025..afe614d 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs @@ -132,7 +132,8 @@ public class TestAudioFingerprinting new TimeRange(54.9294, 55.2590), }; - var actual = FFmpegWrapper.DetectSilence(clip, 60); + var range = new TimeRange(0, 60); + var actual = FFmpegWrapper.DetectSilence(clip, range); Assert.Equal(expected, actual); } diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/AnalyzerHelper.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/AnalyzerHelper.cs new file mode 100644 index 0000000..ae236fb --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/AnalyzerHelper.cs @@ -0,0 +1,132 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using ConfusedPolarBear.Plugin.IntroSkipper.Configuration; +using ConfusedPolarBear.Plugin.IntroSkipper.Data; +using MediaBrowser.Model.Entities; +using Microsoft.Extensions.Logging; + +namespace ConfusedPolarBear.Plugin.IntroSkipper; + +/// +/// Analyzer Helper. +/// +public class AnalyzerHelper +{ + private readonly ILogger _logger; + private readonly double silenceDetectionMinimumDuration; + + /// + /// Initializes a new instance of the class. + /// + /// Logger. + public AnalyzerHelper(ILogger logger) + { + var config = Plugin.Instance?.Configuration ?? new PluginConfiguration(); + silenceDetectionMinimumDuration = config.SilenceDetectionMinimumDuration; + _logger = logger; + } + + /// + /// Adjusts the end timestamps of all intros so that they end at silence. + /// + /// QueuedEpisodes to adjust. + /// Original introductions. + /// Analysis mode. + /// Modified Intro Timestamps. + public Dictionary AdjustIntroTimes( + ReadOnlyCollection episodes, + Dictionary originalIntros, + AnalysisMode mode) + { + var modifiedIntros = new Dictionary(); + + foreach (var episode in episodes) + { + _logger.LogTrace("Adjusting introduction end time for {Name} ({Id})", episode.Name, episode.EpisodeId); + + if (!originalIntros.TryGetValue(episode.EpisodeId, out var originalIntro)) + { + _logger.LogTrace("{Name} does not have an intro", episode.Name); + continue; + } + + var adjustedIntro = AdjustIntroForEpisode(episode, originalIntro, mode); + modifiedIntros[episode.EpisodeId] = adjustedIntro; + } + + return modifiedIntros; + } + + private Intro AdjustIntroForEpisode(QueuedEpisode episode, Intro originalIntro, AnalysisMode mode) + { + var chapters = GetChaptersWithVirtualEnd(episode); + var adjustedIntro = new Intro(originalIntro); + + var originalIntroStart = new TimeRange(Math.Max(0, (int)originalIntro.IntroStart - 5), (int)originalIntro.IntroStart + 10); + var originalIntroEnd = new TimeRange((int)originalIntro.IntroEnd - 10, Math.Min(episode.Duration, (int)originalIntro.IntroEnd + 5)); + + _logger.LogTrace("{Name} original intro: {Start} - {End}", episode.Name, originalIntro.IntroStart, originalIntro.IntroEnd); + + if (!AdjustIntroBasedOnChapters(episode, chapters, adjustedIntro, originalIntroStart, originalIntroEnd) + && mode == AnalysisMode.Introduction) + { + AdjustIntroBasedOnSilence(episode, adjustedIntro, originalIntroEnd); + } + + return adjustedIntro; + } + + private List GetChaptersWithVirtualEnd(QueuedEpisode episode) + { + var chapters = Plugin.Instance?.GetChapters(episode.EpisodeId) ?? new List(); + chapters.Add(new ChapterInfo { StartPositionTicks = TimeSpan.FromSeconds(episode.Duration).Ticks }); + return chapters; + } + + private bool AdjustIntroBasedOnChapters(QueuedEpisode episode, List chapters, Intro adjustedIntro, TimeRange originalIntroStart, TimeRange originalIntroEnd) + { + foreach (var chapter in chapters) + { + var chapterStartSeconds = TimeSpan.FromTicks(chapter.StartPositionTicks).TotalSeconds; + + if (originalIntroStart.Start < chapterStartSeconds && chapterStartSeconds < originalIntroStart.End) + { + adjustedIntro.IntroStart = chapterStartSeconds; + _logger.LogTrace("{Name} chapter found close to intro start: {Start}", episode.Name, chapterStartSeconds); + } + + if (originalIntroEnd.Start < chapterStartSeconds && chapterStartSeconds < originalIntroEnd.End) + { + adjustedIntro.IntroEnd = chapterStartSeconds; + _logger.LogTrace("{Name} chapter found close to intro end: {End}", episode.Name, chapterStartSeconds); + return true; + } + } + + return false; + } + + private void AdjustIntroBasedOnSilence(QueuedEpisode episode, Intro adjustedIntro, TimeRange originalIntroEnd) + { + var silence = FFmpegWrapper.DetectSilence(episode, originalIntroEnd); + + foreach (var currentRange in silence) + { + _logger.LogTrace("{Name} silence: {Start} - {End}", episode.Name, currentRange.Start, currentRange.End); + + if (IsValidSilenceForIntroAdjustment(currentRange, originalIntroEnd, adjustedIntro)) + { + adjustedIntro.IntroEnd = currentRange.Start; + break; + } + } + } + + private bool IsValidSilenceForIntroAdjustment(TimeRange silenceRange, TimeRange originalIntroEnd, Intro adjustedIntro) + { + return originalIntroEnd.Intersects(silenceRange) && + silenceRange.Duration >= silenceDetectionMinimumDuration && + silenceRange.Start >= adjustedIntro.IntroStart; + } +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/BlackFrameAnalyzer.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/BlackFrameAnalyzer.cs index b5464de..ad79d67 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/BlackFrameAnalyzer.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/BlackFrameAnalyzer.cs @@ -119,6 +119,9 @@ public class BlackFrameAnalyzer : IMediaFileAnalyzer episode.State.SetAnalyzed(mode, true); } + var analyzerHelper = new AnalyzerHelper(_logger); + creditTimes = analyzerHelper.AdjustIntroTimes(analysisQueue, creditTimes, mode); + Plugin.Instance!.UpdateTimestamps(creditTimes, mode); return episodeAnalysisQueue.AsReadOnly(); diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChromaprintAnalyzer.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChromaprintAnalyzer.cs index 187a520..7b5144c 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChromaprintAnalyzer.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChromaprintAnalyzer.cs @@ -30,8 +30,6 @@ public class ChromaprintAnalyzer : IMediaFileAnalyzer private double maximumTimeSkip; - private double silenceDetectionMinimumDuration; - private ILogger _logger; private AnalysisMode _analysisMode; @@ -46,7 +44,6 @@ public class ChromaprintAnalyzer : IMediaFileAnalyzer maximumDifferences = config.MaximumFingerprintPointDifferences; invertedIndexShift = config.InvertedIndexShift; maximumTimeSkip = config.MaximumTimeSkip; - silenceDetectionMinimumDuration = config.SilenceDetectionMinimumDuration; minimumIntroDuration = config.MinimumIntroDuration; _logger = logger; @@ -204,11 +201,9 @@ public class ChromaprintAnalyzer : IMediaFileAnalyzer return analysisQueue; } - if (_analysisMode == AnalysisMode.Introduction) - { - // Adjust all introduction end times so that they end at silence. - seasonIntros = AdjustIntroEndTimes(analysisQueue, seasonIntros); - } + // Adjust all introduction times. + var analyzerHelper = new AnalyzerHelper(_logger); + seasonIntros = analyzerHelper.AdjustIntroTimes(analysisQueue, seasonIntros, this._analysisMode); Plugin.Instance!.UpdateTimestamps(seasonIntros, _analysisMode); @@ -402,103 +397,9 @@ public class ChromaprintAnalyzer : IMediaFileAnalyzer // Since LHS had a contiguous time range, RHS must have one also. var rContiguous = TimeRangeHelpers.FindContiguous(rhsTimes.ToArray(), maximumTimeSkip)!; - - if (_analysisMode == AnalysisMode.Introduction) - { - // Tweak the end timestamps just a bit to ensure as little content as possible is skipped over. - // TODO: remove this - if (lContiguous.Duration >= 90) - { - lContiguous.End -= 2 * maximumTimeSkip; - rContiguous.End -= 2 * maximumTimeSkip; - } - else if (lContiguous.Duration >= 30) - { - lContiguous.End -= maximumTimeSkip; - rContiguous.End -= maximumTimeSkip; - } - } - return (lContiguous, rContiguous); } - /// - /// Adjusts the end timestamps of all intros so that they end at silence. - /// - /// QueuedEpisodes to adjust. - /// Original introductions. - private Dictionary AdjustIntroEndTimes( - ReadOnlyCollection episodes, - Dictionary originalIntros) - { - Dictionary modifiedIntros = new(); - - // For all episodes - foreach (var episode in episodes) - { - _logger.LogTrace( - "Adjusting introduction end time for {Name} ({Id})", - episode.Name, - episode.EpisodeId); - - // If no intro was found for this episode, skip it. - if (!originalIntros.TryGetValue(episode.EpisodeId, out var originalIntro)) - { - _logger.LogTrace("{Name} does not have an intro", episode.Name); - continue; - } - - // Only adjust the end timestamp of the intro - var originalIntroEnd = new TimeRange(originalIntro.IntroEnd - 15, originalIntro.IntroEnd); - - _logger.LogTrace( - "{Name} original intro: {Start} - {End}", - episode.Name, - originalIntro.IntroStart, - originalIntro.IntroEnd); - - // Detect silence in the media file up to the end of the intro. - var silence = FFmpegWrapper.DetectSilence(episode, (int)originalIntro.IntroEnd + 2); - - // For all periods of silence - foreach (var currentRange in silence) - { - _logger.LogTrace( - "{Name} silence: {Start} - {End}", - episode.Name, - currentRange.Start, - currentRange.End); - - // Ignore any silence that: - // * doesn't intersect the ending of the intro, or - // * is shorter than the user defined minimum duration, or - // * starts before the introduction does - if ( - !originalIntroEnd.Intersects(currentRange) || - currentRange.Duration < silenceDetectionMinimumDuration || - currentRange.Start < originalIntro.IntroStart) - { - continue; - } - - // Adjust the end timestamp of the intro to match the start of the silence region. - originalIntro.IntroEnd = currentRange.Start; - break; - } - - _logger.LogTrace( - "{Name} adjusted intro: {Start} - {End}", - episode.Name, - originalIntro.IntroStart, - originalIntro.IntroEnd); - - // Add the (potentially) modified intro back. - modifiedIntros[episode.EpisodeId] = originalIntro; - } - - return modifiedIntros; - } - /// /// Count the number of bits that are set in the provided number. /// diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs b/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs index abe142f..ca7f3b6 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs @@ -166,27 +166,34 @@ public static class FFmpegWrapper /// Detect ranges of silence in the provided episode. /// /// Queued episode. - /// Maximum amount of audio (in seconds) to detect silence in. + /// Time range to search. /// Array of TimeRange objects that are silent in the queued episode. - public static TimeRange[] DetectSilence(QueuedEpisode episode, int limit) + public static TimeRange[] DetectSilence(QueuedEpisode episode, TimeRange range) { Logger?.LogTrace( - "Detecting silence in \"{File}\" (limit {Limit}, id {Id})", + "Detecting silence in \"{File}\" (range {Start}-{End}, id {Id})", episode.Path, - limit, + range.Start, + range.End, episode.EpisodeId); // -vn, -sn, -dn: ignore video, subtitle, and data tracks var args = string.Format( CultureInfo.InvariantCulture, "-vn -sn -dn " + - "-i \"{0}\" -to {1} -af \"silencedetect=noise={2}dB:duration=0.1\" -f null -", + "-ss {0} -i \"{1}\" -to {2} -af \"silencedetect=noise={3}dB:duration=0.1\" -f null -", + range.Start, episode.Path, - limit, + range.End - range.Start, Plugin.Instance?.Configuration.SilenceDetectionMaximumNoise ?? -50); - // Cache the output of this command to "GUID-intro-silence-v1" - var cacheKey = episode.EpisodeId.ToString("N") + "-intro-silence-v1"; + // Cache the output of this command to "GUID-intro-silence-v2" + var cacheKey = string.Format( + CultureInfo.InvariantCulture, + "{0}-silence-{1}-{2}-v2", + episode.EpisodeId.ToString("N"), + range.Start, + range.End); var currentRange = new TimeRange(); var silenceRanges = new List(); @@ -205,11 +212,11 @@ public static class FFmpegWrapper if (isStart) { - currentRange.Start = time; + currentRange.Start = time + range.Start; } else { - currentRange.End = time; + currentRange.End = time + range.Start; silenceRanges.Add(new TimeRange(currentRange)); } }