From ce52a0b97994ba3c9f739b8275519b8e0070e03d Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Fri, 28 Oct 2022 02:25:57 -0500 Subject: [PATCH] Relocate Chromaprint analysis code --- .../TestAudioFingerprinting.cs | 13 +- .../Analyzers/Chromaprint.cs | 445 +++++++++++++++++ .../Analyzers/IMediaFileAnalyzer.cs | 22 + .../Data/AnalysisMode.cs | 17 + .../Plugin.cs | 16 +- .../ScheduledTasks/AnalyzeEpisodesTask.cs | 461 +----------------- 6 files changed, 518 insertions(+), 456 deletions(-) create mode 100644 ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/Chromaprint.cs create mode 100644 ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/IMediaFileAnalyzer.cs create mode 100644 ConfusedPolarBear.Plugin.IntroSkipper/Data/AnalysisMode.cs diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs index c0da162..5470b4f 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs @@ -26,7 +26,8 @@ public class TestAudioFingerprinting [InlineData(19, 2_465_585_877)] public void TestBitCounting(int expectedBits, uint number) { - Assert.Equal(expectedBits, AnalyzeEpisodesTask.CountBits(number)); + var chromaprint = CreateChromaprintAnalyzer(); + Assert.Equal(expectedBits, chromaprint.CountBits(number)); } [FactSkipFFmpegTests] @@ -86,14 +87,14 @@ public class TestAudioFingerprinting [FactSkipFFmpegTests] public void TestIntroDetection() { - var task = new AnalyzeEpisodesTask(new LoggerFactory()); + var chromaprint = CreateChromaprintAnalyzer(); var lhsEpisode = queueEpisode("audio/big_buck_bunny_intro.mp3"); var rhsEpisode = queueEpisode("audio/big_buck_bunny_clip.mp3"); var lhsFingerprint = FFmpegWrapper.Fingerprint(lhsEpisode); var rhsFingerprint = FFmpegWrapper.Fingerprint(rhsEpisode); - var (lhs, rhs) = task.CompareEpisodes( + var (lhs, rhs) = chromaprint.CompareEpisodes( lhsEpisode.EpisodeId, lhsFingerprint, rhsEpisode.EpisodeId, @@ -140,6 +141,12 @@ public class TestAudioFingerprinting FingerprintDuration = 60 }; } + + private ChromaprintAnalyzer CreateChromaprintAnalyzer() + { + var logger = new LoggerFactory().CreateLogger(); + return new(logger); + } } public class FactSkipFFmpegTests : FactAttribute diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/Chromaprint.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/Chromaprint.cs new file mode 100644 index 0000000..319d678 --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/Chromaprint.cs @@ -0,0 +1,445 @@ +namespace ConfusedPolarBear.Plugin.IntroSkipper; + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Numerics; +using System.Threading; +using Microsoft.Extensions.Logging; + +/// +/// Chromaprint audio analyzer. +/// +public class ChromaprintAnalyzer : IMediaFileAnalyzer +{ + /// + /// Seconds of audio in one fingerprint point. + /// This value is defined by the Chromaprint library and should not be changed. + /// + private const double SamplesToSeconds = 0.128; + + private int minimumIntroDuration; + + private int maximumDifferences; + + private int invertedIndexShift; + + private double maximumTimeSkip; + + private double silenceDetectionMinimumDuration; + + private ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// Logger. + public ChromaprintAnalyzer(ILogger logger) + { + var config = Plugin.Instance?.Configuration ?? new Configuration.PluginConfiguration(); + maximumDifferences = config.MaximumFingerprintPointDifferences; + invertedIndexShift = config.InvertedIndexShift; + maximumTimeSkip = config.MaximumTimeSkip; + silenceDetectionMinimumDuration = config.SilenceDetectionMinimumDuration; + minimumIntroDuration = config.MinimumIntroDuration; + + _logger = logger; + } + + /// + public ReadOnlyCollection AnalyzeMediaFiles( + ReadOnlyCollection analysisQueue, + AnalysisMode mode, + CancellationToken cancellationToken) + { + // All intros for this season. + var seasonIntros = new Dictionary(); + + // Cache of all fingerprints for this season. + var fingerprintCache = new Dictionary(); + + // Episode analysis queue. + var episodeAnalysisQueue = new List(analysisQueue); + + // Episodes that were analyzed and do not have an introduction. + var episodesWithoutIntros = new List(); + + // Compute fingerprints for all episodes in the season + foreach (var episode in episodeAnalysisQueue) + { + try + { + fingerprintCache[episode.EpisodeId] = FFmpegWrapper.Fingerprint(episode); + + if (cancellationToken.IsCancellationRequested) + { + return analysisQueue; + } + } + catch (FingerprintException ex) + { + _logger.LogWarning("Caught fingerprint error: {Ex}", ex); + + // Fallback to an empty fingerprint on any error + fingerprintCache[episode.EpisodeId] = Array.Empty(); + } + } + + // While there are still episodes in the queue + while (episodeAnalysisQueue.Count > 0) + { + // Pop the first episode from the queue + var currentEpisode = episodeAnalysisQueue[0]; + episodeAnalysisQueue.RemoveAt(0); + + // Search through all remaining episodes. + foreach (var remainingEpisode in episodeAnalysisQueue) + { + // Compare the current episode to all remaining episodes in the queue. + var (currentIntro, remainingIntro) = CompareEpisodes( + currentEpisode.EpisodeId, + fingerprintCache[currentEpisode.EpisodeId], + remainingEpisode.EpisodeId, + fingerprintCache[remainingEpisode.EpisodeId]); + + // Ignore this comparison result if: + // - one of the intros isn't valid, or + // - the introduction exceeds the configured limit + if ( + !remainingIntro.Valid || + remainingIntro.Duration > Plugin.Instance!.Configuration.MaximumIntroDuration) + { + continue; + } + + // Only save the discovered intro if it is: + // - the first intro discovered for this episode + // - longer than the previously discovered intro + if ( + !seasonIntros.TryGetValue(currentIntro.EpisodeId, out var savedCurrentIntro) || + currentIntro.Duration > savedCurrentIntro.Duration) + { + seasonIntros[currentIntro.EpisodeId] = currentIntro; + } + + if ( + !seasonIntros.TryGetValue(remainingIntro.EpisodeId, out var savedRemainingIntro) || + remainingIntro.Duration > savedRemainingIntro.Duration) + { + seasonIntros[remainingIntro.EpisodeId] = remainingIntro; + } + + break; + } + + // If no intro is found at this point, the popped episode is not reinserted into the queue. + episodesWithoutIntros.Add(currentEpisode); + } + + // If cancellation was requested, report that no episodes were analyzed. + if (cancellationToken.IsCancellationRequested) + { + return analysisQueue; + } + + // Adjust all introduction end times so that they end at silence. + seasonIntros = AdjustIntroEndTimes(analysisQueue, seasonIntros); + + Plugin.Instance!.UpdateTimestamps(seasonIntros); + + return episodesWithoutIntros.AsReadOnly(); + } + + /// + /// Analyze two episodes to find an introduction sequence shared between them. + /// + /// First episode id. + /// First episode fingerprint points. + /// Second episode id. + /// Second episode fingerprint points. + /// Intros for the first and second episodes. + public (Intro Lhs, Intro Rhs) CompareEpisodes( + Guid lhsId, + uint[] lhsPoints, + Guid rhsId, + uint[] rhsPoints) + { + // Creates an inverted fingerprint point index for both episodes. + // For every point which is a 100% match, search for an introduction at that point. + var (lhsRanges, rhsRanges) = SearchInvertedIndex(lhsId, lhsPoints, rhsId, rhsPoints); + + if (lhsRanges.Count > 0) + { + _logger.LogTrace("Index search successful"); + + return GetLongestTimeRange(lhsId, lhsRanges, rhsId, rhsRanges); + } + + _logger.LogTrace( + "Unable to find a shared introduction sequence between {LHS} and {RHS}", + lhsId, + rhsId); + + return (new Intro(lhsId), new Intro(rhsId)); + } + + /// + /// Locates the longest range of similar audio and returns an Intro class for each range. + /// + /// First episode id. + /// First episode shared timecodes. + /// Second episode id. + /// Second episode shared timecodes. + /// Intros for the first and second episodes. + private (Intro Lhs, Intro Rhs) GetLongestTimeRange( + Guid lhsId, + List lhsRanges, + Guid rhsId, + List rhsRanges) + { + // Store the longest time range as the introduction. + lhsRanges.Sort(); + rhsRanges.Sort(); + + var lhsIntro = lhsRanges[0]; + var rhsIntro = rhsRanges[0]; + + // If the intro starts early in the episode, move it to the beginning. + if (lhsIntro.Start <= 5) + { + lhsIntro.Start = 0; + } + + if (rhsIntro.Start <= 5) + { + rhsIntro.Start = 0; + } + + // Create Intro classes for each time range. + return (new Intro(lhsId, lhsIntro), new Intro(rhsId, rhsIntro)); + } + + /// + /// Search for a shared introduction sequence using inverted indexes. + /// + /// LHS ID. + /// Left episode fingerprint points. + /// RHS ID. + /// Right episode fingerprint points. + /// List of shared TimeRanges between the left and right episodes. + private (List Lhs, List Rhs) SearchInvertedIndex( + Guid lhsId, + uint[] lhsPoints, + Guid rhsId, + uint[] rhsPoints) + { + var lhsRanges = new List(); + var rhsRanges = new List(); + + // Generate inverted indexes for the left and right episodes. + var lhsIndex = FFmpegWrapper.CreateInvertedIndex(lhsId, lhsPoints); + var rhsIndex = FFmpegWrapper.CreateInvertedIndex(rhsId, rhsPoints); + var indexShifts = new HashSet(); + + // For all audio points in the left episode, check if the right episode has a point which matches exactly. + // If an exact match is found, calculate the shift that must be used to align the points. + foreach (var kvp in lhsIndex) + { + var originalPoint = kvp.Key; + + for (var i = -1 * invertedIndexShift; i <= invertedIndexShift; i++) + { + var modifiedPoint = (uint)(originalPoint + i); + + if (rhsIndex.ContainsKey(modifiedPoint)) + { + var lhsFirst = (int)lhsIndex[originalPoint]; + var rhsFirst = (int)rhsIndex[modifiedPoint]; + indexShifts.Add(rhsFirst - lhsFirst); + } + } + } + + // Use all discovered shifts to compare the episodes. + foreach (var shift in indexShifts) + { + var (lhsIndexContiguous, rhsIndexContiguous) = FindContiguous(lhsPoints, rhsPoints, shift); + if (lhsIndexContiguous.End > 0 && rhsIndexContiguous.End > 0) + { + lhsRanges.Add(lhsIndexContiguous); + rhsRanges.Add(rhsIndexContiguous); + } + } + + return (lhsRanges, rhsRanges); + } + + /// + /// Finds the longest contiguous region of similar audio between two fingerprints using the provided shift amount. + /// + /// First fingerprint to compare. + /// Second fingerprint to compare. + /// Amount to shift one fingerprint by. + private (TimeRange Lhs, TimeRange Rhs) FindContiguous( + uint[] lhs, + uint[] rhs, + int shiftAmount) + { + var leftOffset = 0; + var rightOffset = 0; + + // Calculate the offsets for the left and right hand sides. + if (shiftAmount < 0) + { + leftOffset -= shiftAmount; + } + else + { + rightOffset += shiftAmount; + } + + // Store similar times for both LHS and RHS. + var lhsTimes = new List(); + var rhsTimes = new List(); + var upperLimit = Math.Min(lhs.Length, rhs.Length) - Math.Abs(shiftAmount); + + // XOR all elements in LHS and RHS, using the shift amount from above. + for (var i = 0; i < upperLimit; i++) + { + // XOR both samples at the current position. + var lhsPosition = i + leftOffset; + var rhsPosition = i + rightOffset; + var diff = lhs[lhsPosition] ^ rhs[rhsPosition]; + + // If the difference between the samples is small, flag both times as similar. + if (CountBits(diff) > maximumDifferences) + { + continue; + } + + var lhsTime = lhsPosition * SamplesToSeconds; + var rhsTime = rhsPosition * SamplesToSeconds; + + lhsTimes.Add(lhsTime); + rhsTimes.Add(rhsTime); + } + + // Ensure the last timestamp is checked + lhsTimes.Add(double.MaxValue); + rhsTimes.Add(double.MaxValue); + + // Now that both fingerprints have been compared at this shift, see if there's a contiguous time range. + var lContiguous = TimeRangeHelpers.FindContiguous(lhsTimes.ToArray(), maximumTimeSkip); + if (lContiguous is null || lContiguous.Duration < minimumIntroDuration) + { + return (new TimeRange(), new TimeRange()); + } + + // Since LHS had a contiguous time range, RHS must have one also. + var rContiguous = TimeRangeHelpers.FindContiguous(rhsTimes.ToArray(), maximumTimeSkip)!; + + // Tweak the end timestamps just a bit to ensure as little content as possible is skipped over. + if (lContiguous.Duration >= 90) + { + lContiguous.End -= 2 * maximumTimeSkip; + rContiguous.End -= 2 * maximumTimeSkip; + } + else if (lContiguous.Duration >= 30) + { + lContiguous.End -= maximumTimeSkip; + rContiguous.End -= maximumTimeSkip; + } + + return (lContiguous, rContiguous); + } + + /// + /// Adjusts the end timestamps of all intros so that they end at silence. + /// + /// QueuedEpisodes to adjust. + /// Original introductions. + private Dictionary AdjustIntroEndTimes( + ReadOnlyCollection episodes, + Dictionary originalIntros) + { + // The minimum duration of audio that must be silent before adjusting the intro's end. + var minimumSilence = Plugin.Instance!.Configuration.SilenceDetectionMinimumDuration; + + Dictionary modifiedIntros = new(); + + // For all episodes + foreach (var episode in episodes) + { + _logger.LogTrace( + "Adjusting introduction end time for {Name} ({Id})", + episode.Name, + episode.EpisodeId); + + // If no intro was found for this episode, skip it. + if (!originalIntros.TryGetValue(episode.EpisodeId, out var originalIntro)) + { + _logger.LogTrace("{Name} does not have an intro", episode.Name); + continue; + } + + // Only adjust the end timestamp of the intro + var originalIntroEnd = new TimeRange(originalIntro.IntroEnd - 15, originalIntro.IntroEnd); + + _logger.LogTrace( + "{Name} original intro: {Start} - {End}", + episode.Name, + originalIntro.IntroStart, + originalIntro.IntroEnd); + + // Detect silence in the media file up to the end of the intro. + var silence = FFmpegWrapper.DetectSilence(episode, (int)originalIntro.IntroEnd + 2); + + // For all periods of silence + foreach (var currentRange in silence) + { + _logger.LogTrace( + "{Name} silence: {Start} - {End}", + episode.Name, + currentRange.Start, + currentRange.End); + + // Ignore any silence that: + // * doesn't intersect the ending of the intro, or + // * is shorter than the user defined minimum duration, or + // * starts before the introduction does + if ( + !originalIntroEnd.Intersects(currentRange) || + currentRange.Duration < silenceDetectionMinimumDuration || + currentRange.Start < originalIntro.IntroStart) + { + continue; + } + + // Adjust the end timestamp of the intro to match the start of the silence region. + originalIntro.IntroEnd = currentRange.Start; + break; + } + + _logger.LogTrace( + "{Name} adjusted intro: {Start} - {End}", + episode.Name, + originalIntro.IntroStart, + originalIntro.IntroEnd); + + // Add the (potentially) modified intro back. + modifiedIntros[episode.EpisodeId] = originalIntro; + } + + return modifiedIntros; + } + + /// + /// Count the number of bits that are set in the provided number. + /// + /// Number to count bits in. + /// Number of bits that are equal to 1. + public int CountBits(uint number) + { + return BitOperations.PopCount(number); + } +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/IMediaFileAnalyzer.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/IMediaFileAnalyzer.cs new file mode 100644 index 0000000..7721a55 --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/IMediaFileAnalyzer.cs @@ -0,0 +1,22 @@ +namespace ConfusedPolarBear.Plugin.IntroSkipper; + +using System.Collections.ObjectModel; +using System.Threading; + +/// +/// Media file analyzer interface. +/// +public interface IMediaFileAnalyzer +{ + /// + /// Analyze media files for shared introductions or credits, returning all media files that were **not successfully analyzed**. + /// + /// Collection of unanalyzed media files. + /// Analysis mode. + /// Cancellation token from scheduled task. + /// Collection of media files that were **unsuccessfully analyzed**. + public ReadOnlyCollection AnalyzeMediaFiles( + ReadOnlyCollection analysisQueue, + AnalysisMode mode, + CancellationToken cancellationToken); +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Data/AnalysisMode.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Data/AnalysisMode.cs new file mode 100644 index 0000000..7ed0367 --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Data/AnalysisMode.cs @@ -0,0 +1,17 @@ +namespace ConfusedPolarBear.Plugin.IntroSkipper; + +/// +/// Type of media file analysis to perform. +/// +public enum AnalysisMode +{ + /// + /// Detect introduction sequences. + /// + Introduction, + + /// + /// Detect credits. + /// + Credits, +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Plugin.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Plugin.cs index 80522ca..94604d9 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Plugin.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Plugin.cs @@ -18,7 +18,8 @@ namespace ConfusedPolarBear.Plugin.IntroSkipper; /// public class Plugin : BasePlugin, IHasWebPages { - private readonly object _serializationLock = new object(); + private readonly object _serializationLock = new(); + private readonly object _introsLock = new(); private IXmlSerializer _xmlSerializer; private ILibraryManager _libraryManager; private ILogger _logger; @@ -162,6 +163,19 @@ public class Plugin : BasePlugin, IHasWebPages return GetItem(id).Path; } + internal void UpdateTimestamps(Dictionary newIntros) + { + lock (_introsLock) + { + foreach (var intro in newIntros) + { + Plugin.Instance!.Intros[intro.Key] = intro.Value; + } + + Plugin.Instance!.SaveTimestamps(); + } + } + /// public IEnumerable GetPages() { diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs index 467b278..3b1c531 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs @@ -2,7 +2,6 @@ using System; using System.Collections.Generic; using System.Collections.ObjectModel; using System.IO; -using System.Numerics; using System.Threading; using System.Threading.Tasks; using MediaBrowser.Controller.Library; @@ -16,36 +15,12 @@ namespace ConfusedPolarBear.Plugin.IntroSkipper; /// public class AnalyzeEpisodesTask : IScheduledTask { - /// - /// Seconds of audio in one fingerprint point. - /// This value is defined by the Chromaprint library and should not be changed. - /// - private const double SamplesToSeconds = 0.128; - private readonly ILogger _logger; - private readonly ILogger _queueLogger; + private readonly ILoggerFactory _loggerFactory; private readonly ILibraryManager? _libraryManager; - /// - /// Lock which guards the shared dictionary of intros. - /// - private readonly object _introsLock = new object(); - - /// - /// Minimum duration of similar audio that will be considered an introduction. - /// - private static int minimumIntroDuration = 15; - - private static int maximumDifferences = 6; - - private static int invertedIndexShift = 2; - - private static double maximumTimeSkip = 3.5; - - private static double silenceDetectionMinimumDuration = 0.33; - /// /// Initializes a new instance of the class. /// @@ -65,7 +40,7 @@ public class AnalyzeEpisodesTask : IScheduledTask public AnalyzeEpisodesTask(ILoggerFactory loggerFactory) { _logger = loggerFactory.CreateLogger(); - _queueLogger = loggerFactory.CreateLogger(); + _loggerFactory = loggerFactory; EdlManager.Initialize(_logger); } @@ -104,7 +79,10 @@ public class AnalyzeEpisodesTask : IScheduledTask } // Make sure the analysis queue matches what's currently in Jellyfin. - var queueManager = new QueueManager(_queueLogger, _libraryManager); + var queueManager = new QueueManager( + _loggerFactory.CreateLogger(), + _libraryManager); + queueManager.EnqueueAllEpisodes(); var queue = Plugin.Instance!.AnalysisQueue; @@ -115,13 +93,6 @@ public class AnalyzeEpisodesTask : IScheduledTask "No episodes to analyze. If you are limiting the list of libraries to analyze, check that all library names have been spelled correctly."); } - // Load analysis settings from configuration - var config = Plugin.Instance?.Configuration ?? new Configuration.PluginConfiguration(); - maximumDifferences = config.MaximumFingerprintPointDifferences; - invertedIndexShift = config.InvertedIndexShift; - maximumTimeSkip = config.MaximumTimeSkip; - silenceDetectionMinimumDuration = config.SilenceDetectionMinimumDuration; - // Log EDL settings EdlManager.LogConfiguration(); @@ -131,8 +102,6 @@ public class AnalyzeEpisodesTask : IScheduledTask MaxDegreeOfParallelism = Plugin.Instance!.Configuration.MaxParallelism }; - minimumIntroDuration = Plugin.Instance!.Configuration.MinimumIntroDuration; - // TODO: if the queue is modified while the task is running, the task will fail. // clone the queue before running the task to prevent this. @@ -251,30 +220,6 @@ public class AnalyzeEpisodesTask : IScheduledTask return (verified.AsReadOnly(), unanalyzed); } - /// - /// Count the number of previously processed episodes to ensure the reported progress is correct. - /// - /// Number of previously processed episodes. - private int CountProcessedEpisodes() - { - var previous = 0; - - foreach (var season in Plugin.Instance!.AnalysisQueue) - { - foreach (var episode in season.Value) - { - if (!Plugin.Instance!.Intros.TryGetValue(episode.EpisodeId, out var intro) || !intro.Valid) - { - continue; - } - - previous++; - } - } - - return previous; - } - /// /// Fingerprints all episodes in the provided season and stores the timestamps of all introductions. /// @@ -285,15 +230,6 @@ public class AnalyzeEpisodesTask : IScheduledTask ReadOnlyCollection episodes, CancellationToken cancellationToken) { - // All intros for this season. - var seasonIntros = new Dictionary(); - - // Cache of all fingerprints for this season. - var fingerprintCache = new Dictionary(); - - // Episode analysis queue. - var episodeAnalysisQueue = new List(episodes); - // Skip seasons with an insufficient number of episodes. if (episodes.Count <= 1) { @@ -313,392 +249,13 @@ public class AnalyzeEpisodesTask : IScheduledTask first.SeriesName, first.SeasonNumber); - // Compute fingerprints for all episodes in the season - foreach (var episode in episodeAnalysisQueue) - { - try - { - fingerprintCache[episode.EpisodeId] = FFmpegWrapper.Fingerprint(episode); - - if (cancellationToken.IsCancellationRequested) - { - return episodes.Count; - } - } - catch (FingerprintException ex) - { - _logger.LogWarning("Caught fingerprint error: {Ex}", ex); - - // Fallback to an empty fingerprint on any error - fingerprintCache[episode.EpisodeId] = Array.Empty(); - } - } - - // While there are still episodes in the queue - while (episodeAnalysisQueue.Count > 0) - { - // Pop the first episode from the queue - var currentEpisode = episodeAnalysisQueue[0]; - episodeAnalysisQueue.RemoveAt(0); - - // Search through all remaining episodes. - foreach (var remainingEpisode in episodeAnalysisQueue) - { - // Compare the current episode to all remaining episodes in the queue. - var (currentIntro, remainingIntro) = CompareEpisodes( - currentEpisode.EpisodeId, - fingerprintCache[currentEpisode.EpisodeId], - remainingEpisode.EpisodeId, - fingerprintCache[remainingEpisode.EpisodeId]); - - // Ignore this comparison result if: - // - one of the intros isn't valid, or - // - the introduction exceeds the configured limit - if ( - !remainingIntro.Valid || - remainingIntro.Duration > Plugin.Instance!.Configuration.MaximumIntroDuration) - { - continue; - } - - // Only save the discovered intro if it is: - // - the first intro discovered for this episode - // - longer than the previously discovered intro - if ( - !seasonIntros.TryGetValue(currentIntro.EpisodeId, out var savedCurrentIntro) || - currentIntro.Duration > savedCurrentIntro.Duration) - { - seasonIntros[currentIntro.EpisodeId] = currentIntro; - } - - if ( - !seasonIntros.TryGetValue(remainingIntro.EpisodeId, out var savedRemainingIntro) || - remainingIntro.Duration > savedRemainingIntro.Duration) - { - seasonIntros[remainingIntro.EpisodeId] = remainingIntro; - } - - break; - } - - // If no intro is found at this point, the popped episode is not reinserted into the queue. - } - - if (cancellationToken.IsCancellationRequested) - { - return episodes.Count; - } - - // Adjust all introduction end times so that they end at silence. - seasonIntros = AdjustIntroEndTimes(episodes, seasonIntros); - - // Ensure only one thread at a time can update the shared intro dictionary. - lock (_introsLock) - { - foreach (var intro in seasonIntros) - { - Plugin.Instance!.Intros[intro.Key] = intro.Value; - } - - Plugin.Instance!.SaveTimestamps(); - } + // Analyze the season with Chromaprint + var chromaprint = new ChromaprintAnalyzer(_loggerFactory.CreateLogger()); + chromaprint.AnalyzeMediaFiles(episodes, AnalysisMode.Introduction, cancellationToken); return episodes.Count; } - /// - /// Analyze two episodes to find an introduction sequence shared between them. - /// - /// First episode id. - /// First episode fingerprint points. - /// Second episode id. - /// Second episode fingerprint points. - /// Intros for the first and second episodes. - public (Intro Lhs, Intro Rhs) CompareEpisodes( - Guid lhsId, - uint[] lhsPoints, - Guid rhsId, - uint[] rhsPoints) - { - // Creates an inverted fingerprint point index for both episodes. - // For every point which is a 100% match, search for an introduction at that point. - var (lhsRanges, rhsRanges) = SearchInvertedIndex(lhsId, lhsPoints, rhsId, rhsPoints); - - if (lhsRanges.Count > 0) - { - _logger.LogTrace("Index search successful"); - - return GetLongestTimeRange(lhsId, lhsRanges, rhsId, rhsRanges); - } - - _logger.LogTrace( - "Unable to find a shared introduction sequence between {LHS} and {RHS}", - lhsId, - rhsId); - - return (new Intro(lhsId), new Intro(rhsId)); - } - - /// - /// Locates the longest range of similar audio and returns an Intro class for each range. - /// - /// First episode id. - /// First episode shared timecodes. - /// Second episode id. - /// Second episode shared timecodes. - /// Intros for the first and second episodes. - private (Intro Lhs, Intro Rhs) GetLongestTimeRange( - Guid lhsId, - List lhsRanges, - Guid rhsId, - List rhsRanges) - { - // Store the longest time range as the introduction. - lhsRanges.Sort(); - rhsRanges.Sort(); - - var lhsIntro = lhsRanges[0]; - var rhsIntro = rhsRanges[0]; - - // If the intro starts early in the episode, move it to the beginning. - if (lhsIntro.Start <= 5) - { - lhsIntro.Start = 0; - } - - if (rhsIntro.Start <= 5) - { - rhsIntro.Start = 0; - } - - // Create Intro classes for each time range. - return (new Intro(lhsId, lhsIntro), new Intro(rhsId, rhsIntro)); - } - - /// - /// Search for a shared introduction sequence using inverted indexes. - /// - /// LHS ID. - /// Left episode fingerprint points. - /// RHS ID. - /// Right episode fingerprint points. - /// List of shared TimeRanges between the left and right episodes. - private (List Lhs, List Rhs) SearchInvertedIndex( - Guid lhsId, - uint[] lhsPoints, - Guid rhsId, - uint[] rhsPoints) - { - var lhsRanges = new List(); - var rhsRanges = new List(); - - // Generate inverted indexes for the left and right episodes. - var lhsIndex = FFmpegWrapper.CreateInvertedIndex(lhsId, lhsPoints); - var rhsIndex = FFmpegWrapper.CreateInvertedIndex(rhsId, rhsPoints); - var indexShifts = new HashSet(); - - // For all audio points in the left episode, check if the right episode has a point which matches exactly. - // If an exact match is found, calculate the shift that must be used to align the points. - foreach (var kvp in lhsIndex) - { - var originalPoint = kvp.Key; - - for (var i = -1 * invertedIndexShift; i <= invertedIndexShift; i++) - { - var modifiedPoint = (uint)(originalPoint + i); - - if (rhsIndex.ContainsKey(modifiedPoint)) - { - var lhsFirst = (int)lhsIndex[originalPoint]; - var rhsFirst = (int)rhsIndex[modifiedPoint]; - indexShifts.Add(rhsFirst - lhsFirst); - } - } - } - - // Use all discovered shifts to compare the episodes. - foreach (var shift in indexShifts) - { - var (lhsIndexContiguous, rhsIndexContiguous) = FindContiguous(lhsPoints, rhsPoints, shift); - if (lhsIndexContiguous.End > 0 && rhsIndexContiguous.End > 0) - { - lhsRanges.Add(lhsIndexContiguous); - rhsRanges.Add(rhsIndexContiguous); - } - } - - return (lhsRanges, rhsRanges); - } - - /// - /// Finds the longest contiguous region of similar audio between two fingerprints using the provided shift amount. - /// - /// First fingerprint to compare. - /// Second fingerprint to compare. - /// Amount to shift one fingerprint by. - private static (TimeRange Lhs, TimeRange Rhs) FindContiguous( - uint[] lhs, - uint[] rhs, - int shiftAmount) - { - var leftOffset = 0; - var rightOffset = 0; - - // Calculate the offsets for the left and right hand sides. - if (shiftAmount < 0) - { - leftOffset -= shiftAmount; - } - else - { - rightOffset += shiftAmount; - } - - // Store similar times for both LHS and RHS. - var lhsTimes = new List(); - var rhsTimes = new List(); - var upperLimit = Math.Min(lhs.Length, rhs.Length) - Math.Abs(shiftAmount); - - // XOR all elements in LHS and RHS, using the shift amount from above. - for (var i = 0; i < upperLimit; i++) - { - // XOR both samples at the current position. - var lhsPosition = i + leftOffset; - var rhsPosition = i + rightOffset; - var diff = lhs[lhsPosition] ^ rhs[rhsPosition]; - - // If the difference between the samples is small, flag both times as similar. - if (CountBits(diff) > maximumDifferences) - { - continue; - } - - var lhsTime = lhsPosition * SamplesToSeconds; - var rhsTime = rhsPosition * SamplesToSeconds; - - lhsTimes.Add(lhsTime); - rhsTimes.Add(rhsTime); - } - - // Ensure the last timestamp is checked - lhsTimes.Add(double.MaxValue); - rhsTimes.Add(double.MaxValue); - - // Now that both fingerprints have been compared at this shift, see if there's a contiguous time range. - var lContiguous = TimeRangeHelpers.FindContiguous(lhsTimes.ToArray(), maximumTimeSkip); - if (lContiguous is null || lContiguous.Duration < minimumIntroDuration) - { - return (new TimeRange(), new TimeRange()); - } - - // Since LHS had a contiguous time range, RHS must have one also. - var rContiguous = TimeRangeHelpers.FindContiguous(rhsTimes.ToArray(), maximumTimeSkip)!; - - // Tweak the end timestamps just a bit to ensure as little content as possible is skipped over. - if (lContiguous.Duration >= 90) - { - lContiguous.End -= 2 * maximumTimeSkip; - rContiguous.End -= 2 * maximumTimeSkip; - } - else if (lContiguous.Duration >= 30) - { - lContiguous.End -= maximumTimeSkip; - rContiguous.End -= maximumTimeSkip; - } - - return (lContiguous, rContiguous); - } - - /// - /// Adjusts the end timestamps of all intros so that they end at silence. - /// - /// QueuedEpisodes to adjust. - /// Original introductions. - private Dictionary AdjustIntroEndTimes( - ReadOnlyCollection episodes, - Dictionary originalIntros) - { - // The minimum duration of audio that must be silent before adjusting the intro's end. - var minimumSilence = Plugin.Instance!.Configuration.SilenceDetectionMinimumDuration; - - Dictionary modifiedIntros = new(); - - // For all episodes - foreach (var episode in episodes) - { - _logger.LogTrace( - "Adjusting introduction end time for {Name} ({Id})", - episode.Name, - episode.EpisodeId); - - // If no intro was found for this episode, skip it. - if (!originalIntros.TryGetValue(episode.EpisodeId, out var originalIntro)) - { - _logger.LogTrace("{Name} does not have an intro", episode.Name); - continue; - } - - // Only adjust the end timestamp of the intro - var originalIntroEnd = new TimeRange(originalIntro.IntroEnd - 15, originalIntro.IntroEnd); - - _logger.LogTrace( - "{Name} original intro: {Start} - {End}", - episode.Name, - originalIntro.IntroStart, - originalIntro.IntroEnd); - - // Detect silence in the media file up to the end of the intro. - var silence = FFmpegWrapper.DetectSilence(episode, (int)originalIntro.IntroEnd + 2); - - // For all periods of silence - foreach (var currentRange in silence) - { - _logger.LogTrace( - "{Name} silence: {Start} - {End}", - episode.Name, - currentRange.Start, - currentRange.End); - - // Ignore any silence that: - // * doesn't intersect the ending of the intro, or - // * is shorter than the user defined minimum duration, or - // * starts before the introduction does - if ( - !originalIntroEnd.Intersects(currentRange) || - currentRange.Duration < silenceDetectionMinimumDuration || - currentRange.Start < originalIntro.IntroStart) - { - continue; - } - - // Adjust the end timestamp of the intro to match the start of the silence region. - originalIntro.IntroEnd = currentRange.Start; - break; - } - - _logger.LogTrace( - "{Name} adjusted intro: {Start} - {End}", - episode.Name, - originalIntro.IntroStart, - originalIntro.IntroEnd); - - // Add the (potentially) modified intro back. - modifiedIntros[episode.EpisodeId] = originalIntro; - } - - return modifiedIntros; - } - - /// - /// Count the number of bits that are set in the provided number. - /// - /// Number to count bits in. - /// Number of bits that are equal to 1. - public static int CountBits(uint number) - { - return BitOperations.PopCount(number); - } - /// /// Get task triggers. ///