From 4558b59e9578584e58fca98f090ef41ca0573327 Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Thu, 25 Aug 2022 01:28:41 -0500 Subject: [PATCH 01/29] Only use inverted indexes to search --- .../Data/AnalysisStatistics.cs | 19 +- .../ScheduledTasks/AnalyzeEpisodesTask.cs | 233 +----------------- 2 files changed, 4 insertions(+), 248 deletions(-) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Data/AnalysisStatistics.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Data/AnalysisStatistics.cs index d0f1e97..522dbdf 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Data/AnalysisStatistics.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Data/AnalysisStatistics.cs @@ -25,30 +25,15 @@ public class AnalysisStatistics /// public ThreadSafeInteger IndexSearches { get; } = new ThreadSafeInteger(); - /// - /// Gets the number of times a quick scan successfully located a pair of introductions. - /// - public ThreadSafeInteger QuickScans { get; } = new ThreadSafeInteger(); - - /// - /// Gets the number of times a full scan successfully located a pair of introductions. - /// - public ThreadSafeInteger FullScans { get; } = new ThreadSafeInteger(); - /// /// Gets the total CPU time spent waiting for audio fingerprints to be generated. /// public ThreadSafeInteger FingerprintCPUTime { get; } = new ThreadSafeInteger(); /// - /// Gets the total CPU time spent analyzing fingerprints in the initial pass. + /// Gets the total CPU time spent analyzing fingerprints. /// - public ThreadSafeInteger FirstPassCPUTime { get; } = new ThreadSafeInteger(); - - /// - /// Gets the total CPU time spent analyzing fingerprints in the second pass. - /// - public ThreadSafeInteger SecondPassCPUTime { get; } = new ThreadSafeInteger(); + public ThreadSafeInteger AnalysisCPUTime { get; } = new ThreadSafeInteger(); /// /// Gets the total task runtime across all threads. diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs index 47033e4..0d4507f 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs @@ -30,16 +30,6 @@ public class AnalyzeEpisodesTask : IScheduledTask /// private const double SamplesToSeconds = 0.128; - /// - /// Bucket size used in the reanalysis histogram. - /// - private const int ReanalysisBucketWidth = 5; - - /// - /// Maximum time (in seconds) that an intro's duration can be different from a typical intro's duration before marking it for reanalysis. - /// - private const double ReanalysisTolerance = ReanalysisBucketWidth * 1.5; - private readonly ILogger _logger; private readonly ILogger _queueLogger; @@ -375,17 +365,6 @@ public class AnalyzeEpisodesTask : IScheduledTask } } - // Only run the second pass if the user hasn't requested cancellation and we found an intro - if (!cancellationToken.IsCancellationRequested && everFoundIntro) - { - var start = DateTime.Now; - - // Run a second pass over this season to remove outliers and fix episodes that failed in the first pass. - RunSecondPass(season.Value); - - analysisStatistics.SecondPassCPUTime.AddDuration(start); - } - lock (_introsLock) { Plugin.Instance!.SaveTimestamps(); @@ -441,7 +420,6 @@ public class AnalyzeEpisodesTask : IScheduledTask // If this isn't running as part of the first analysis pass, don't count this CPU time as first pass time. var start = isFirstPass ? DateTime.Now : DateTime.MinValue; - // ===== Method 1: Inverted indexes ===== // Creates an inverted fingerprint point index for both episodes. // For every point which is a 100% match, search for an introduction at that point. var (lhsRanges, rhsRanges) = SearchInvertedIndex(lhsPoints, rhsPoints); @@ -450,46 +428,17 @@ public class AnalyzeEpisodesTask : IScheduledTask { _logger.LogTrace("Index search successful"); analysisStatistics.IndexSearches.Increment(); - analysisStatistics.FirstPassCPUTime.AddDuration(start); + analysisStatistics.AnalysisCPUTime.AddDuration(start); return GetLongestTimeRange(lhsId, lhsRanges, rhsId, rhsRanges); } - // ===== Method 2: Quick scan ===== - // Tests if an intro can be found within the first 5 seconds of the episodes. ±5/0.128 = ±40 samples. - (lhsRanges, rhsRanges) = ShiftEpisodes(lhsPoints, rhsPoints, -40, 40); - - if (lhsRanges.Count > 0) - { - _logger.LogTrace("Quick scan successful"); - analysisStatistics.QuickScans.Increment(); - analysisStatistics.FirstPassCPUTime.AddDuration(start); - - return GetLongestTimeRange(lhsId, lhsRanges, rhsId, rhsRanges); - } - - // ===== Method 3: Full scan ===== - // Compares all elements of the shortest fingerprint to the other fingerprint. - var limit = Math.Min(lhsPoints.Length, rhsPoints.Length); - (lhsRanges, rhsRanges) = ShiftEpisodes(lhsPoints, rhsPoints, -1 * limit, limit); - - if (lhsRanges.Count > 0) - { - _logger.LogTrace("Full scan successful"); - analysisStatistics.FullScans.Increment(); - analysisStatistics.FirstPassCPUTime.AddDuration(start); - - return GetLongestTimeRange(lhsId, lhsRanges, rhsId, rhsRanges); - } - - // No method was able to find an introduction, return nothing. - _logger.LogTrace( "Unable to find a shared introduction sequence between {LHS} and {RHS}", lhsId, rhsId); - analysisStatistics.FirstPassCPUTime.AddDuration(start); + analysisStatistics.AnalysisCPUTime.AddDuration(start); return (new Intro(lhsId), new Intro(rhsId)); } @@ -694,184 +643,6 @@ public class AnalyzeEpisodesTask : IScheduledTask return BitOperations.PopCount(number); } - /// - /// Reanalyze the most recently analyzed season. - /// Looks for and fixes intro durations that were either not found or are statistical outliers. - /// - /// List of episodes that was just analyzed. - private void RunSecondPass(List episodes) - { - // First, assert that at least half of the episodes in this season have an intro. - var validCount = 0; - var totalCount = episodes.Count; - - foreach (var episode in episodes) - { - if (Plugin.Instance!.Intros.TryGetValue(episode.EpisodeId, out var intro) && intro.Valid) - { - validCount++; - } - } - - var percentValid = (validCount * 100) / totalCount; - _logger.LogTrace("Found intros in {Valid}/{Total} ({Percent}%) of episodes", validCount, totalCount, percentValid); - if (percentValid < 50) - { - return; - } - - // Create a histogram of all episode durations - var histogram = new Dictionary(); - foreach (var episode in episodes) - { - var id = episode.EpisodeId; - var duration = GetIntroDuration(id); - - if (duration < minimumIntroDuration) - { - continue; - } - - // Bucket the duration into equally sized groups - var bucket = Convert.ToInt32(Math.Floor(duration / ReanalysisBucketWidth)) * ReanalysisBucketWidth; - - // TryAdd returns true when the key was successfully added (i.e. for newly created buckets). - // Newly created buckets are initialized with the provided episode ID, so nothing else needs to be done for them. - if (histogram.TryAdd(bucket, new SeasonHistogram(id))) - { - continue; - } - - histogram[bucket].Episodes.Add(id); - } - - // Find the bucket that was seen most often, as this is likely to be the true intro length. - var maxDuration = 0; - var maxBucket = new SeasonHistogram(Guid.Empty); - foreach (var entry in histogram) - { - if (entry.Value.Count > maxBucket.Count) - { - maxDuration = entry.Key; - maxBucket = entry.Value; - } - } - - // Ensure that the most frequently seen bucket has a majority - percentValid = (maxBucket.Count * 100) / validCount; - _logger.LogTrace( - "Intro duration {Duration} appeared {Frequency} times ({Percent}%)", - maxDuration, - maxBucket.Count, - percentValid); - - if (percentValid < 50 || maxBucket.Episodes[0].Equals(Guid.Empty)) - { - return; - } - - _logger.LogTrace("Second pass is processing {Count} episodes", totalCount - maxBucket.Count); - - // Calculate a range of intro durations that are most likely to be correct. - var maxEpisode = episodes.Find(x => x.EpisodeId == maxBucket.Episodes[0]); - if (maxEpisode is null) - { - _logger.LogError("Second pass failed to get episode from bucket"); - return; - } - - var lhsDuration = GetIntroDuration(maxEpisode.EpisodeId); - var (lowTargetDuration, highTargetDuration) = ( - lhsDuration - ReanalysisTolerance, - lhsDuration + ReanalysisTolerance); - - // TODO: add limit and make it customizable - var count = maxBucket.Episodes.Count - 1; - var goodFingerprints = new List(); - foreach (var id in maxBucket.Episodes) - { - if (!_fingerprintCache.TryGetValue(id, out var fp)) - { - _logger.LogTrace("Second pass: max bucket episode {Id} not in cache, skipping", id); - continue; - } - - goodFingerprints.Add(fp); - } - - foreach (var episode in episodes) - { - // Don't reanalyze episodes from the max bucket - if (maxBucket.Episodes.Contains(episode.EpisodeId)) - { - continue; - } - - var oldDuration = GetIntroDuration(episode.EpisodeId); - - // If the episode's intro duration is close enough to the targeted bucket, leave it alone. - if (Math.Abs(lhsDuration - oldDuration) <= ReanalysisTolerance) - { - _logger.LogTrace( - "Not reanalyzing episode {Path} (intro is {Initial}, target is {Max})", - episode.Path, - Math.Round(oldDuration, 2), - maxDuration); - - continue; - } - - _logger.LogTrace( - "Reanalyzing episode {Path} (intro is {Initial}, target is {Max})", - episode.Path, - Math.Round(oldDuration, 2), - maxDuration); - - // Analyze the episode again, ignoring whatever is returned for the known good episode. - foreach (var lhsFingerprint in goodFingerprints) - { - if (!_fingerprintCache.TryGetValue(episode.EpisodeId, out var fp)) - { - _logger.LogTrace("Unable to get cached fingerprint for {Id}, skipping", episode.EpisodeId); - continue; - } - - var (_, newRhs) = FingerprintEpisodes( - maxEpisode.EpisodeId, - lhsFingerprint, - episode.EpisodeId, - fp, - false); - - // Ensure that the new intro duration is within the targeted bucket and longer than what was found previously. - var newDuration = Math.Round(newRhs.IntroEnd - newRhs.IntroStart, 2); - if (newDuration < oldDuration || newDuration < lowTargetDuration || newDuration > highTargetDuration) - { - _logger.LogTrace( - "Ignoring reanalysis for {Path} (was {Initial}, now is {New})", - episode.Path, - oldDuration, - newDuration); - - continue; - } - - _logger.LogTrace( - "Reanalysis succeeded for {Path} (was {Initial}, now is {New})", - episode.Path, - oldDuration, - newDuration); - - lock (_introsLock) - { - Plugin.Instance!.Intros[episode.EpisodeId] = newRhs; - } - - break; - } - } - } - private double GetIntroDuration(Guid id) { if (!Plugin.Instance!.Intros.TryGetValue(id, out var episode)) From 1e1db330d7b49541073ed7618c05c5d7a4e5ca2e Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Thu, 25 Aug 2022 01:37:57 -0500 Subject: [PATCH 02/29] Always analyze episodes --- .../ScheduledTasks/AnalyzeEpisodesTask.cs | 64 ++----------------- 1 file changed, 5 insertions(+), 59 deletions(-) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs index 0d4507f..c66b5e9 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs @@ -264,35 +264,11 @@ public class AnalyzeEpisodesTask : IScheduledTask return episodes.Count; } - var unanalyzed = false; - - // Only log an analysis message if there are unanalyzed episodes in this season. - foreach (var episode in episodes) - { - if (!Plugin.Instance!.Intros.ContainsKey(episode.EpisodeId)) - { - unanalyzed = true; - break; - } - } - - if (unanalyzed) - { - _logger.LogInformation( - "Analyzing {Count} episodes from {Name} season {Season}", - season.Value.Count, - first.SeriesName, - first.SeasonNumber); - } - else - { - _logger.LogDebug( - "All episodes from {Name} season {Season} have already been analyzed", - first.SeriesName, - first.SeasonNumber); - - return 0; - } + _logger.LogInformation( + "Analyzing {Count} episodes from {Name} season {Season}", + season.Value.Count, + first.SeriesName, + first.SeasonNumber); // Ensure there are an even number of episodes if (episodes.Count % 2 != 0) @@ -301,8 +277,6 @@ public class AnalyzeEpisodesTask : IScheduledTask } // Analyze each pair of episodes in the current season - var everFoundIntro = false; - var failures = 0; for (var i = 0; i < episodes.Count; i += 2) { if (cancellationToken.IsCancellationRequested) @@ -313,26 +287,6 @@ public class AnalyzeEpisodesTask : IScheduledTask var lhs = episodes[i]; var rhs = episodes[i + 1]; - if (!everFoundIntro && failures >= 20) - { - _logger.LogWarning( - "Failed to find an introduction in {Series} season {Season}", - lhs.SeriesName, - lhs.SeasonNumber); - - break; - } - - if (Plugin.Instance!.Intros.ContainsKey(lhs.EpisodeId) && Plugin.Instance!.Intros.ContainsKey(rhs.EpisodeId)) - { - _logger.LogTrace( - "Episodes {LHS} and {RHS} have both already been fingerprinted", - lhs.EpisodeId, - rhs.EpisodeId); - - continue; - } - try { _logger.LogTrace("Analyzing {LHS} and {RHS}", lhs.Path, rhs.Path); @@ -341,14 +295,6 @@ public class AnalyzeEpisodesTask : IScheduledTask seasonIntros[lhsIntro.EpisodeId] = lhsIntro; seasonIntros[rhsIntro.EpisodeId] = rhsIntro; analysisStatistics.TotalAnalyzedEpisodes.Add(2); - - if (!lhsIntro.Valid) - { - failures += 2; - continue; - } - - everFoundIntro = true; } catch (FingerprintException ex) { From 5867bb378f5f4078c7f86f71897d1fb193146e51 Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Thu, 25 Aug 2022 22:11:39 -0500 Subject: [PATCH 03/29] Implement new search algorithm --- .../TestAudioFingerprinting.cs | 2 +- .../Data/Intro.cs | 7 ++ .../ScheduledTasks/AnalyzeEpisodesTask.cs | 106 +++++++++++------- 3 files changed, 76 insertions(+), 39 deletions(-) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs index c6e2570..dbd4fab 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs @@ -90,7 +90,7 @@ public class TestAudioFingerprinting var lhsEpisode = queueEpisode("audio/big_buck_bunny_intro.mp3"); var rhsEpisode = queueEpisode("audio/big_buck_bunny_clip.mp3"); - var (lhs, rhs) = task.FingerprintEpisodes(lhsEpisode, rhsEpisode); + var (lhs, rhs) = task.CompareEpisodes(lhsEpisode, rhsEpisode); Assert.True(lhs.Valid); Assert.Equal(0, lhs.IntroStart); diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Data/Intro.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Data/Intro.cs index 0875b80..4cd88e5 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Data/Intro.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Data/Intro.cs @@ -1,4 +1,5 @@ using System; +using System.Text.Json.Serialization; namespace ConfusedPolarBear.Plugin.IntroSkipper; @@ -49,6 +50,12 @@ public class Intro /// public bool Valid => IntroEnd > 0; + /// + /// Gets the duration of this intro. + /// + [JsonIgnore] + public double Duration => IntroEnd - IntroStart; + /// /// Gets or sets the introduction sequence start time. /// diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs index c66b5e9..d66cae7 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs @@ -161,7 +161,7 @@ public class AnalyzeEpisodesTask : IScheduledTask { // Increment totalProcessed by the number of episodes in this season that were actually analyzed // (instead of just using the number of episodes in the current season). - var analyzed = AnalyzeSeason(season, cancellationToken); + var analyzed = AnalyzeSeason(season.Value, cancellationToken); Interlocked.Add(ref totalProcessed, analyzed); writeEdl = analyzed > 0 || Plugin.Instance!.Configuration.RegenerateEdlFiles; } @@ -241,66 +241,94 @@ public class AnalyzeEpisodesTask : IScheduledTask return previous; } + // TODO: restore warning +#pragma warning disable CA1002 + /// /// Fingerprints all episodes in the provided season and stores the timestamps of all introductions. /// - /// Pairing of season GUID to a list of QueuedEpisode objects. + /// Episodes in this season. /// Cancellation token provided by the scheduled task. /// Number of episodes from the provided season that were analyzed. private int AnalyzeSeason( - KeyValuePair> season, + List episodes, CancellationToken cancellationToken) { var seasonIntros = new Dictionary(); - var episodes = season.Value; - var first = episodes[0]; /* Don't analyze specials or seasons with an insufficient number of episodes. * A season with only 1 episode can't be analyzed as it would compare the episode to itself, * which would result in the entire episode being marked as an introduction, as the audio is identical. */ - if (season.Value.Count < 2 || first.SeasonNumber == 0) + if (episodes.Count < 2 || episodes[0].SeasonNumber == 0) { return episodes.Count; } + var first = episodes[0]; + _logger.LogInformation( "Analyzing {Count} episodes from {Name} season {Season}", - season.Value.Count, + episodes.Count, first.SeriesName, first.SeasonNumber); - // Ensure there are an even number of episodes - if (episodes.Count % 2 != 0) + // TODO: cache fingerprints and inverted indexes + + // TODO: implementing bucketing + + // For all episodes + foreach (var outer in episodes) { - episodes.Add(episodes[episodes.Count - 2]); + // Compare the outer episode to all other episodes + foreach (var inner in episodes) + { + // Don't compare the episode to itself + if (outer.EpisodeId == inner.EpisodeId) + { + continue; + } + + // Fingerprint both episodes + Intro outerIntro; + Intro innerIntro; + + try + { + (outerIntro, innerIntro) = CompareEpisodes(outer, inner); + } + catch (FingerprintException ex) + { + // TODO: remove the episode that threw the error from additional processing + _logger.LogWarning("Caught fingerprint error: {Ex}", ex); + continue; + } + + if (!outerIntro.Valid) + { + continue; + } + + // Save this intro if: + // - it is the first one we've seen for this episode + // - OR it is longer than the previous one + if ( + !seasonIntros.TryGetValue(outer.EpisodeId, out var currentOuterIntro) || + outerIntro.Duration > currentOuterIntro.Duration) + { + seasonIntros[outer.EpisodeId] = outerIntro; + } + + if ( + !seasonIntros.TryGetValue(inner.EpisodeId, out var currentInnerIntro) || + innerIntro.Duration > currentInnerIntro.Duration) + { + seasonIntros[inner.EpisodeId] = innerIntro; + } + } } - // Analyze each pair of episodes in the current season - for (var i = 0; i < episodes.Count; i += 2) - { - if (cancellationToken.IsCancellationRequested) - { - break; - } - - var lhs = episodes[i]; - var rhs = episodes[i + 1]; - - try - { - _logger.LogTrace("Analyzing {LHS} and {RHS}", lhs.Path, rhs.Path); - - var (lhsIntro, rhsIntro) = FingerprintEpisodes(lhs, rhs); - seasonIntros[lhsIntro.EpisodeId] = lhsIntro; - seasonIntros[rhsIntro.EpisodeId] = rhsIntro; - analysisStatistics.TotalAnalyzedEpisodes.Add(2); - } - catch (FingerprintException ex) - { - _logger.LogError("Caught fingerprint error: {Ex}", ex); - } - } + // TODO: analysisStatistics.TotalAnalyzedEpisodes.Add(2); // Ensure only one thread at a time can update the shared intro dictionary. lock (_introsLock) @@ -319,13 +347,15 @@ public class AnalyzeEpisodesTask : IScheduledTask return episodes.Count; } +#pragma warning restore CA1002 + /// /// Analyze two episodes to find an introduction sequence shared between them. /// /// First episode to analyze. /// Second episode to analyze. /// Intros for the first and second episodes. - public (Intro Lhs, Intro Rhs) FingerprintEpisodes(QueuedEpisode lhsEpisode, QueuedEpisode rhsEpisode) + public (Intro Lhs, Intro Rhs) CompareEpisodes(QueuedEpisode lhsEpisode, QueuedEpisode rhsEpisode) { var start = DateTime.Now; var lhsFingerprint = Chromaprint.Fingerprint(lhsEpisode); @@ -339,7 +369,7 @@ public class AnalyzeEpisodesTask : IScheduledTask _fingerprintCache[rhsEpisode.EpisodeId] = rhsFingerprint; } - return FingerprintEpisodes( + return CompareEpisodes( lhsEpisode.EpisodeId, lhsFingerprint, rhsEpisode.EpisodeId, @@ -356,7 +386,7 @@ public class AnalyzeEpisodesTask : IScheduledTask /// Second episode fingerprint points. /// If this was called as part of the first analysis pass, add the elapsed time to the statistics. /// Intros for the first and second episodes. - public (Intro Lhs, Intro Rhs) FingerprintEpisodes( + public (Intro Lhs, Intro Rhs) CompareEpisodes( Guid lhsId, uint[] lhsPoints, Guid rhsId, From e4da9342c838eee35f4f3ada7acc3e610f188f27 Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Thu, 25 Aug 2022 23:01:46 -0500 Subject: [PATCH 04/29] Change fingerprint cache policy --- .../TestAudioFingerprinting.cs | 15 ++- .../ScheduledTasks/AnalyzeEpisodesTask.cs | 91 +++++-------------- 2 files changed, 36 insertions(+), 70 deletions(-) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs index dbd4fab..a67e858 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs @@ -89,8 +89,14 @@ public class TestAudioFingerprinting var lhsEpisode = queueEpisode("audio/big_buck_bunny_intro.mp3"); var rhsEpisode = queueEpisode("audio/big_buck_bunny_clip.mp3"); + var lhsFingerprint = Chromaprint.Fingerprint(lhsEpisode); + var rhsFingerprint = Chromaprint.Fingerprint(rhsEpisode); - var (lhs, rhs) = task.CompareEpisodes(lhsEpisode, rhsEpisode); + var (lhs, rhs) = task.CompareEpisodes( + lhsEpisode.EpisodeId, + lhsFingerprint, + rhsEpisode.EpisodeId, + rhsFingerprint); Assert.True(lhs.Valid); Assert.Equal(0, lhs.IntroStart); @@ -111,10 +117,11 @@ public class TestAudioFingerprinting } } -public class FactSkipFFmpegTests : FactAttribute { - #if SKIP_FFMPEG_TESTS +public class FactSkipFFmpegTests : FactAttribute +{ +#if SKIP_FFMPEG_TESTS public FactSkipFFmpegTests() { Skip = "SKIP_FFMPEG_TESTS defined, skipping unit tests that require FFmpeg to be installed"; } - #endif +#endif } diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs index d66cae7..941f8b0 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs @@ -36,22 +36,11 @@ public class AnalyzeEpisodesTask : IScheduledTask private readonly ILibraryManager? _libraryManager; - /// - /// Lock which guards the fingerprint cache dictionary. - /// - private readonly object _fingerprintCacheLock = new object(); - /// /// Lock which guards the shared dictionary of intros. /// private readonly object _introsLock = new object(); - /// - /// Temporary fingerprint cache to speed up reanalysis. - /// Fingerprints are removed from this after a season is analyzed. - /// - private Dictionary _fingerprintCache; - /// /// Statistics for the currently running analysis task. /// @@ -81,8 +70,6 @@ public class AnalyzeEpisodesTask : IScheduledTask _logger = loggerFactory.CreateLogger(); _queueLogger = loggerFactory.CreateLogger(); - _fingerprintCache = new Dictionary(); - EdlManager.Initialize(_logger); } @@ -182,15 +169,6 @@ public class AnalyzeEpisodesTask : IScheduledTask ex); } - // Clear this season's episodes from the temporary fingerprint cache. - lock (_fingerprintCacheLock) - { - foreach (var ep in season.Value) - { - _fingerprintCache.Remove(ep.EpisodeId); - } - } - if (writeEdl && Plugin.Instance!.Configuration.EdlAction != EdlAction.None) { EdlManager.UpdateEDLFiles(season.Value.AsReadOnly()); @@ -255,6 +233,7 @@ public class AnalyzeEpisodesTask : IScheduledTask CancellationToken cancellationToken) { var seasonIntros = new Dictionary(); + var fingerprintCache = new Dictionary(); /* Don't analyze specials or seasons with an insufficient number of episodes. * A season with only 1 episode can't be analyzed as it would compare the episode to itself, @@ -273,7 +252,23 @@ public class AnalyzeEpisodesTask : IScheduledTask first.SeriesName, first.SeasonNumber); - // TODO: cache fingerprints and inverted indexes + // Cache all fingerprints + foreach (var episode in episodes) + { + try + { + fingerprintCache[episode.EpisodeId] = Chromaprint.Fingerprint(episode); + } + catch (FingerprintException ex) + { + _logger.LogWarning("Caught fingerprint error: {Ex}", ex); + + // fallback to an empty fingerprint + fingerprintCache[episode.EpisodeId] = Array.Empty(); + } + } + + // TODO: cache inverted indexes // TODO: implementing bucketing @@ -293,16 +288,11 @@ public class AnalyzeEpisodesTask : IScheduledTask Intro outerIntro; Intro innerIntro; - try - { - (outerIntro, innerIntro) = CompareEpisodes(outer, inner); - } - catch (FingerprintException ex) - { - // TODO: remove the episode that threw the error from additional processing - _logger.LogWarning("Caught fingerprint error: {Ex}", ex); - continue; - } + (outerIntro, innerIntro) = CompareEpisodes( + outer.EpisodeId, + fingerprintCache[outer.EpisodeId], + inner.EpisodeId, + fingerprintCache[inner.EpisodeId]); if (!outerIntro.Valid) { @@ -349,34 +339,6 @@ public class AnalyzeEpisodesTask : IScheduledTask #pragma warning restore CA1002 - /// - /// Analyze two episodes to find an introduction sequence shared between them. - /// - /// First episode to analyze. - /// Second episode to analyze. - /// Intros for the first and second episodes. - public (Intro Lhs, Intro Rhs) CompareEpisodes(QueuedEpisode lhsEpisode, QueuedEpisode rhsEpisode) - { - var start = DateTime.Now; - var lhsFingerprint = Chromaprint.Fingerprint(lhsEpisode); - var rhsFingerprint = Chromaprint.Fingerprint(rhsEpisode); - analysisStatistics.FingerprintCPUTime.AddDuration(start); - - // Cache the fingerprints for quicker recall in the second pass (if one is needed). - lock (_fingerprintCacheLock) - { - _fingerprintCache[lhsEpisode.EpisodeId] = lhsFingerprint; - _fingerprintCache[rhsEpisode.EpisodeId] = rhsFingerprint; - } - - return CompareEpisodes( - lhsEpisode.EpisodeId, - lhsFingerprint, - rhsEpisode.EpisodeId, - rhsFingerprint, - true); - } - /// /// Analyze two episodes to find an introduction sequence shared between them. /// @@ -384,17 +346,14 @@ public class AnalyzeEpisodesTask : IScheduledTask /// First episode fingerprint points. /// Second episode id. /// Second episode fingerprint points. - /// If this was called as part of the first analysis pass, add the elapsed time to the statistics. /// Intros for the first and second episodes. public (Intro Lhs, Intro Rhs) CompareEpisodes( Guid lhsId, uint[] lhsPoints, Guid rhsId, - uint[] rhsPoints, - bool isFirstPass) + uint[] rhsPoints) { - // If this isn't running as part of the first analysis pass, don't count this CPU time as first pass time. - var start = isFirstPass ? DateTime.Now : DateTime.MinValue; + var start = DateTime.Now; // Creates an inverted fingerprint point index for both episodes. // For every point which is a 100% match, search for an introduction at that point. From f92eea20b30d0b456f9c38b1addfa53c32a1b142 Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Fri, 26 Aug 2022 00:50:45 -0500 Subject: [PATCH 05/29] Cache inverted indexes --- .../TestAudioFingerprinting.cs | 4 +++- ConfusedPolarBear.Plugin.IntroSkipper/Chromaprint.cs | 12 +++++++++++- .../ScheduledTasks/AnalyzeEpisodesTask.cs | 12 +++++++----- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs index a67e858..140ebd7 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs @@ -2,6 +2,7 @@ * which supports both chromaprint and the "-fp_format raw" flag. */ +using System; using System.Collections.Generic; using Xunit; using Microsoft.Extensions.Logging; @@ -77,7 +78,7 @@ public class TestAudioFingerprinting {77, 5}, }; - var actual = Chromaprint.CreateInvertedIndex(fpr); + var actual = Chromaprint.CreateInvertedIndex(Guid.NewGuid(), fpr); Assert.Equal(expected, actual); } @@ -111,6 +112,7 @@ public class TestAudioFingerprinting { return new QueuedEpisode() { + EpisodeId = Guid.NewGuid(), Path = "../../../" + path, FingerprintDuration = 60 }; diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Chromaprint.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Chromaprint.cs index 68fd809..ffe3602 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Chromaprint.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Chromaprint.cs @@ -20,6 +20,8 @@ public static class Chromaprint private static Dictionary ChromaprintLogs { get; set; } = new(); + private static Dictionary> InvertedIndexCache { get; set; } = new(); + /// /// Check that the installed version of ffmpeg supports chromaprint. /// @@ -124,10 +126,16 @@ public static class Chromaprint /// /// Transforms a Chromaprint into an inverted index of fingerprint points to the last index it appeared at. /// + /// Episode ID. /// Chromaprint fingerprint. /// Inverted index. - public static Dictionary CreateInvertedIndex(uint[] fingerprint) + public static Dictionary CreateInvertedIndex(Guid id, uint[] fingerprint) { + if (InvertedIndexCache.TryGetValue(id, out var cached)) + { + return cached; + } + var invIndex = new Dictionary(); for (int i = 0; i < fingerprint.Length; i++) @@ -139,6 +147,8 @@ public static class Chromaprint invIndex[point] = i; } + InvertedIndexCache[id] = invIndex; + return invIndex; } diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs index 941f8b0..e2a0f78 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs @@ -268,8 +268,6 @@ public class AnalyzeEpisodesTask : IScheduledTask } } - // TODO: cache inverted indexes - // TODO: implementing bucketing // For all episodes @@ -357,7 +355,7 @@ public class AnalyzeEpisodesTask : IScheduledTask // Creates an inverted fingerprint point index for both episodes. // For every point which is a 100% match, search for an introduction at that point. - var (lhsRanges, rhsRanges) = SearchInvertedIndex(lhsPoints, rhsPoints); + var (lhsRanges, rhsRanges) = SearchInvertedIndex(lhsId, lhsPoints, rhsId, rhsPoints); if (lhsRanges.Count > 0) { @@ -417,19 +415,23 @@ public class AnalyzeEpisodesTask : IScheduledTask /// /// Search for a shared introduction sequence using inverted indexes. /// + /// LHS ID. /// Left episode fingerprint points. + /// RHS ID. /// Right episode fingerprint points. /// List of shared TimeRanges between the left and right episodes. private (List Lhs, List Rhs) SearchInvertedIndex( + Guid lhsId, uint[] lhsPoints, + Guid rhsId, uint[] rhsPoints) { var lhsRanges = new List(); var rhsRanges = new List(); // Generate inverted indexes for the left and right episodes. - var lhsIndex = Chromaprint.CreateInvertedIndex(lhsPoints); - var rhsIndex = Chromaprint.CreateInvertedIndex(rhsPoints); + var lhsIndex = Chromaprint.CreateInvertedIndex(lhsId, lhsPoints); + var rhsIndex = Chromaprint.CreateInvertedIndex(rhsId, rhsPoints); var indexShifts = new HashSet(); // For all audio points in the left episode, check if the right episode has a point which matches exactly. From 874f77f94c0cf029529ae6e26c75b0c9a9bf743a Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Sat, 27 Aug 2022 01:58:46 -0500 Subject: [PATCH 06/29] Search episodes in O(n) instead of O(n^2) time --- .../Data/SeasonHistogram.cs | 31 ------ .../ScheduledTasks/AnalyzeEpisodesTask.cs | 101 +++++++++--------- 2 files changed, 49 insertions(+), 83 deletions(-) delete mode 100644 ConfusedPolarBear.Plugin.IntroSkipper/Data/SeasonHistogram.cs diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Data/SeasonHistogram.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Data/SeasonHistogram.cs deleted file mode 100644 index 4cd4a20..0000000 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Data/SeasonHistogram.cs +++ /dev/null @@ -1,31 +0,0 @@ -#pragma warning disable CA1815 // Override equals and operator equals on value types - -using System; -using System.Collections.ObjectModel; - -namespace ConfusedPolarBear.Plugin.IntroSkipper; - -/// -/// Histogram entry for episodes in a season. -/// -public struct SeasonHistogram -{ - /// - /// Initializes a new instance of the struct. - /// - /// First episode seen with this duration. - public SeasonHistogram(Guid firstEpisode) - { - Episodes.Add(firstEpisode); - } - - /// - /// Gets episodes with this duration. - /// - public Collection Episodes { get; } = new Collection(); - - /// - /// Gets the number of times an episode with an intro of this duration has been seen. - /// - public int Count => Episodes?.Count ?? 0; -} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs index e2a0f78..e97dee2 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Collections.ObjectModel; using System.Numerics; using System.Threading; using System.Threading.Tasks; @@ -146,9 +147,11 @@ public class AnalyzeEpisodesTask : IScheduledTask try { + var episodes = new Collection(season.Value); + // Increment totalProcessed by the number of episodes in this season that were actually analyzed // (instead of just using the number of episodes in the current season). - var analyzed = AnalyzeSeason(season.Value, cancellationToken); + var analyzed = AnalyzeSeason(episodes, cancellationToken); Interlocked.Add(ref totalProcessed, analyzed); writeEdl = analyzed > 0 || Plugin.Instance!.Configuration.RegenerateEdlFiles; } @@ -219,9 +222,6 @@ public class AnalyzeEpisodesTask : IScheduledTask return previous; } - // TODO: restore warning -#pragma warning disable CA1002 - /// /// Fingerprints all episodes in the provided season and stores the timestamps of all introductions. /// @@ -229,19 +229,26 @@ public class AnalyzeEpisodesTask : IScheduledTask /// Cancellation token provided by the scheduled task. /// Number of episodes from the provided season that were analyzed. private int AnalyzeSeason( - List episodes, + Collection episodes, CancellationToken cancellationToken) { + // All intros for this season. var seasonIntros = new Dictionary(); + + // Cache of all fingerprints for this season. var fingerprintCache = new Dictionary(); + // Total episodes in this season. Counted at the start of this function as episodes + // are popped from here during analysis. + var episodeCount = episodes.Count; + /* Don't analyze specials or seasons with an insufficient number of episodes. * A season with only 1 episode can't be analyzed as it would compare the episode to itself, * which would result in the entire episode being marked as an introduction, as the audio is identical. */ if (episodes.Count < 2 || episodes[0].SeasonNumber == 0) { - return episodes.Count; + return episodeCount; } var first = episodes[0]; @@ -252,7 +259,7 @@ public class AnalyzeEpisodesTask : IScheduledTask first.SeriesName, first.SeasonNumber); - // Cache all fingerprints + // Compute fingerprints for all episodes in the season foreach (var episode in episodes) { try @@ -263,60 +270,52 @@ public class AnalyzeEpisodesTask : IScheduledTask { _logger.LogWarning("Caught fingerprint error: {Ex}", ex); - // fallback to an empty fingerprint + // Fallback to an empty fingerprint on any error fingerprintCache[episode.EpisodeId] = Array.Empty(); } } - // TODO: implementing bucketing + /* Theory of operation: + * Episodes are analyzed in the same order that Jellyfin displays them in and are + * sorted into buckets based off of the intro sequence that the episode contains. + * + * Jellyfin's episode ordering is used because it is assumed that the introduction + * in each season of a show will likely either: + * - remain constant throughout the entire season + * - remain constant in subranges of the season (e.g. episodes 1 - 5 and 6 - 10 share intros) + * If the intros do not follow this pattern, the plugin should still find most + * of them. + */ - // For all episodes - foreach (var outer in episodes) + // While there are still episodes in the queue + while (episodes.Count > 0) { - // Compare the outer episode to all other episodes - foreach (var inner in episodes) + // Pop the first episode from the queue + var currentEpisode = episodes[0]; + episodes.RemoveAt(0); + + // Search through all remaining episodes. + foreach (var remainingEpisode in episodes) { - // Don't compare the episode to itself - if (outer.EpisodeId == inner.EpisodeId) + // Compare the current episode to all remaining episodes in the queue. + var (currentIntro, remainingIntro) = CompareEpisodes( + currentEpisode.EpisodeId, + fingerprintCache[currentEpisode.EpisodeId], + remainingEpisode.EpisodeId, + fingerprintCache[remainingEpisode.EpisodeId]); + + // If we found an intro, save it. + if (currentIntro.Valid) { - continue; - } + seasonIntros[currentIntro.EpisodeId] = currentIntro; + seasonIntros[remainingIntro.EpisodeId] = remainingIntro; - // Fingerprint both episodes - Intro outerIntro; - Intro innerIntro; - - (outerIntro, innerIntro) = CompareEpisodes( - outer.EpisodeId, - fingerprintCache[outer.EpisodeId], - inner.EpisodeId, - fingerprintCache[inner.EpisodeId]); - - if (!outerIntro.Valid) - { - continue; - } - - // Save this intro if: - // - it is the first one we've seen for this episode - // - OR it is longer than the previous one - if ( - !seasonIntros.TryGetValue(outer.EpisodeId, out var currentOuterIntro) || - outerIntro.Duration > currentOuterIntro.Duration) - { - seasonIntros[outer.EpisodeId] = outerIntro; - } - - if ( - !seasonIntros.TryGetValue(inner.EpisodeId, out var currentInnerIntro) || - innerIntro.Duration > currentInnerIntro.Duration) - { - seasonIntros[inner.EpisodeId] = innerIntro; + break; } } - } - // TODO: analysisStatistics.TotalAnalyzedEpisodes.Add(2); + // If no intro is found at this point, the popped episode is not reinserted into the queue. + } // Ensure only one thread at a time can update the shared intro dictionary. lock (_introsLock) @@ -332,11 +331,9 @@ public class AnalyzeEpisodesTask : IScheduledTask Plugin.Instance!.SaveTimestamps(); } - return episodes.Count; + return episodeCount; } -#pragma warning restore CA1002 - /// /// Analyze two episodes to find an introduction sequence shared between them. /// From bbed34b0e7dc7c7afc0a0bd72c77460adee3f940 Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Sun, 28 Aug 2022 22:25:27 -0500 Subject: [PATCH 07/29] Add inverted index shift amount --- .../Configuration/PluginConfiguration.cs | 6 ++ .../Controllers/SkipIntroController.cs | 1 + .../ScheduledTasks/AnalyzeEpisodesTask.cs | 58 ++++++++++++------- 3 files changed, 43 insertions(+), 22 deletions(-) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs index db86ce3..741c561 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs @@ -78,4 +78,10 @@ public class PluginConfiguration : BasePluginConfiguration /// Gets or sets the seconds after the intro starts to hide the skip prompt at. /// public int HidePromptAdjustment { get; set; } = 10; + + /// + /// Gets or sets the amount of intro to play (in seconds). + /// TODO: rename. + /// + public int AmountOfIntroToPlay { get; set; } = 5; } diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/SkipIntroController.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/SkipIntroController.cs index d4d07b7..4fcb8c6 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/SkipIntroController.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/SkipIntroController.cs @@ -44,6 +44,7 @@ public class SkipIntroController : ControllerBase var config = Plugin.Instance!.Configuration; intro.ShowSkipPromptAt = Math.Max(0, intro.IntroStart - config.ShowPromptAdjustment); intro.HideSkipPromptAt = intro.IntroStart + config.HidePromptAdjustment; + intro.IntroEnd -= config.AmountOfIntroToPlay; return intro; } diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs index e97dee2..f8d71be 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs @@ -26,6 +26,11 @@ public class AnalyzeEpisodesTask : IScheduledTask /// private const double MaximumDistance = 3.5; + /// + /// Amount to shift inverted index offsets by. + /// + private const int InvertedIndexShift = 2; + /// /// Seconds of audio in one fingerprint point. This value is defined by the Chromaprint library and should not be changed. /// @@ -275,18 +280,6 @@ public class AnalyzeEpisodesTask : IScheduledTask } } - /* Theory of operation: - * Episodes are analyzed in the same order that Jellyfin displays them in and are - * sorted into buckets based off of the intro sequence that the episode contains. - * - * Jellyfin's episode ordering is used because it is assumed that the introduction - * in each season of a show will likely either: - * - remain constant throughout the entire season - * - remain constant in subranges of the season (e.g. episodes 1 - 5 and 6 - 10 share intros) - * If the intros do not follow this pattern, the plugin should still find most - * of them. - */ - // While there are still episodes in the queue while (episodes.Count > 0) { @@ -304,14 +297,30 @@ public class AnalyzeEpisodesTask : IScheduledTask remainingEpisode.EpisodeId, fingerprintCache[remainingEpisode.EpisodeId]); - // If we found an intro, save it. - if (currentIntro.Valid) + // If one of the intros isn't valid, ignore this comparison result. + if (!currentIntro.Valid) + { + continue; + } + + // Only save the discovered intro if it is: + // - the first intro discovered for this episode + // - longer than the previously discovered intro + if ( + !seasonIntros.TryGetValue(currentIntro.EpisodeId, out var savedCurrentIntro) || + currentIntro.Duration > savedCurrentIntro.Duration) { seasonIntros[currentIntro.EpisodeId] = currentIntro; - seasonIntros[remainingIntro.EpisodeId] = remainingIntro; - - break; } + + if ( + !seasonIntros.TryGetValue(remainingIntro.EpisodeId, out var savedRemainingIntro) || + remainingIntro.Duration > savedRemainingIntro.Duration) + { + seasonIntros[remainingIntro.EpisodeId] = remainingIntro; + } + + break; } // If no intro is found at this point, the popped episode is not reinserted into the queue. @@ -435,13 +444,18 @@ public class AnalyzeEpisodesTask : IScheduledTask // If an exact match is found, calculate the shift that must be used to align the points. foreach (var kvp in lhsIndex) { - var point = kvp.Key; + var originalPoint = kvp.Key; - if (rhsIndex.ContainsKey(point)) + for (var i = -1 * InvertedIndexShift; i <= InvertedIndexShift; i++) { - var lhsFirst = (int)lhsIndex[point]; - var rhsFirst = (int)rhsIndex[point]; - indexShifts.Add(rhsFirst - lhsFirst); + var modifiedPoint = (uint)(originalPoint + i); + + if (rhsIndex.ContainsKey(modifiedPoint)) + { + var lhsFirst = (int)lhsIndex[originalPoint]; + var rhsFirst = (int)rhsIndex[modifiedPoint]; + indexShifts.Add(rhsFirst - lhsFirst); + } } } From fb6cd5c1d7a11295a2bb822f0f02d6d9331154ce Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Sun, 28 Aug 2022 22:35:43 -0500 Subject: [PATCH 08/29] Rename {Chromaprint,FFmpegWrapper}.cs --- .../TestAudioFingerprinting.cs | 10 +++++----- .../Controllers/TroubleshootingController.cs | 2 +- .../Controllers/VisualizationController.cs | 2 +- ConfusedPolarBear.Plugin.IntroSkipper/Entrypoint.cs | 2 +- .../{Chromaprint.cs => FFmpegWrapper.cs} | 4 ++-- ConfusedPolarBear.Plugin.IntroSkipper/QueueManager.cs | 2 +- .../ScheduledTasks/AnalyzeEpisodesTask.cs | 6 +++--- 7 files changed, 14 insertions(+), 14 deletions(-) rename ConfusedPolarBear.Plugin.IntroSkipper/{Chromaprint.cs => FFmpegWrapper.cs} (99%) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs index 140ebd7..a940067 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs @@ -14,7 +14,7 @@ public class TestAudioFingerprinting [FactSkipFFmpegTests] public void TestInstallationCheck() { - Assert.True(Chromaprint.CheckFFmpegVersion()); + Assert.True(FFmpegWrapper.CheckFFmpegVersion()); } [Theory] @@ -58,7 +58,7 @@ public class TestAudioFingerprinting 3472417825, 3395841056, 3458735136, 3341420624, 1076496560, 1076501168, 1076501136, 1076497024 }; - var actual = Chromaprint.Fingerprint(queueEpisode("audio/big_buck_bunny_intro.mp3")); + var actual = FFmpegWrapper.Fingerprint(queueEpisode("audio/big_buck_bunny_intro.mp3")); Assert.Equal(expected, actual); } @@ -78,7 +78,7 @@ public class TestAudioFingerprinting {77, 5}, }; - var actual = Chromaprint.CreateInvertedIndex(Guid.NewGuid(), fpr); + var actual = FFmpegWrapper.CreateInvertedIndex(Guid.NewGuid(), fpr); Assert.Equal(expected, actual); } @@ -90,8 +90,8 @@ public class TestAudioFingerprinting var lhsEpisode = queueEpisode("audio/big_buck_bunny_intro.mp3"); var rhsEpisode = queueEpisode("audio/big_buck_bunny_clip.mp3"); - var lhsFingerprint = Chromaprint.Fingerprint(lhsEpisode); - var rhsFingerprint = Chromaprint.Fingerprint(rhsEpisode); + var lhsFingerprint = FFmpegWrapper.Fingerprint(lhsEpisode); + var rhsFingerprint = FFmpegWrapper.Fingerprint(rhsEpisode); var (lhs, rhs) = task.CompareEpisodes( lhsEpisode.EpisodeId, diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/TroubleshootingController.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/TroubleshootingController.cs index 3554feb..8923daf 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/TroubleshootingController.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/TroubleshootingController.cs @@ -58,7 +58,7 @@ public class TroubleshootingController : ControllerBase bundle.Append(" seasons"); bundle.Append('\n'); - bundle.Append(Chromaprint.GetChromaprintLogs()); + bundle.Append(FFmpegWrapper.GetChromaprintLogs()); return bundle.ToString(); } diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/VisualizationController.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/VisualizationController.cs index 6d4d042..5cf7bc1 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/VisualizationController.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/VisualizationController.cs @@ -113,7 +113,7 @@ public class VisualizationController : ControllerBase { if (needle.EpisodeId == id) { - return Chromaprint.Fingerprint(needle); + return FFmpegWrapper.Fingerprint(needle); } } } diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Entrypoint.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Entrypoint.cs index 5113ef4..6a5dff2 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Entrypoint.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Entrypoint.cs @@ -46,7 +46,7 @@ public class Entrypoint : IServerEntryPoint /// Task. public Task RunAsync() { - Chromaprint.Logger = _logger; + FFmpegWrapper.Logger = _logger; #if DEBUG LogVersion(); diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Chromaprint.cs b/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs similarity index 99% rename from ConfusedPolarBear.Plugin.IntroSkipper/Chromaprint.cs rename to ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs index ffe3602..bda56ef 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Chromaprint.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs @@ -9,9 +9,9 @@ using Microsoft.Extensions.Logging; namespace ConfusedPolarBear.Plugin.IntroSkipper; /// -/// Wrapper for libchromaprint. +/// Wrapper for libchromaprint and the silencedetect filter. /// -public static class Chromaprint +public static class FFmpegWrapper { /// /// Gets or sets the logger. diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/QueueManager.cs b/ConfusedPolarBear.Plugin.IntroSkipper/QueueManager.cs index 6f07285..185cc60 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/QueueManager.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/QueueManager.cs @@ -40,7 +40,7 @@ public class QueueManager public void EnqueueAllEpisodes() { // Assert that ffmpeg with chromaprint is installed - if (!Chromaprint.CheckFFmpegVersion()) + if (!FFmpegWrapper.CheckFFmpegVersion()) { throw new FingerprintException( "ffmpeg with chromaprint is not installed on this system - episodes will not be analyzed. If Jellyfin is running natively, install jellyfin-ffmpeg5. If Jellyfin is running in a container, upgrade it to the latest version of 10.8.0."); diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs index f8d71be..42acf8e 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs @@ -269,7 +269,7 @@ public class AnalyzeEpisodesTask : IScheduledTask { try { - fingerprintCache[episode.EpisodeId] = Chromaprint.Fingerprint(episode); + fingerprintCache[episode.EpisodeId] = FFmpegWrapper.Fingerprint(episode); } catch (FingerprintException ex) { @@ -436,8 +436,8 @@ public class AnalyzeEpisodesTask : IScheduledTask var rhsRanges = new List(); // Generate inverted indexes for the left and right episodes. - var lhsIndex = Chromaprint.CreateInvertedIndex(lhsId, lhsPoints); - var rhsIndex = Chromaprint.CreateInvertedIndex(rhsId, rhsPoints); + var lhsIndex = FFmpegWrapper.CreateInvertedIndex(lhsId, lhsPoints); + var rhsIndex = FFmpegWrapper.CreateInvertedIndex(rhsId, rhsPoints); var indexShifts = new HashSet(); // For all audio points in the left episode, check if the right episode has a point which matches exactly. From cbd00b2675aa7667c7575e390febf7b2048d7ef7 Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Mon, 29 Aug 2022 23:56:13 -0500 Subject: [PATCH 09/29] Implement silence detection --- .../TestAudioFingerprinting.cs | 23 +++ .../TestContiguous.cs | 18 +++ .../AutoSkip.cs | 4 +- .../Configuration/PluginConfiguration.cs | 13 +- .../Data/TimeRange.cs | 12 ++ .../FFmpegWrapper.cs | 136 +++++++++++++++--- .../ScheduledTasks/AnalyzeEpisodesTask.cs | 91 +++++++++++- 7 files changed, 274 insertions(+), 23 deletions(-) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs index a940067..c0da162 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs @@ -108,6 +108,29 @@ public class TestAudioFingerprinting Assert.Equal(22.912, rhs.IntroEnd); } + /// + /// Test that the silencedetect wrapper is working. + /// + [FactSkipFFmpegTests] + public void TestSilenceDetection() + { + var clip = queueEpisode("audio/big_buck_bunny_clip.mp3"); + + var expected = new TimeRange[] + { + new TimeRange(44.6310, 44.8072), + new TimeRange(53.5905, 53.8070), + new TimeRange(53.8458, 54.2024), + new TimeRange(54.2611, 54.5935), + new TimeRange(54.7098, 54.9293), + new TimeRange(54.9294, 55.2590), + }; + + var actual = FFmpegWrapper.DetectSilence(clip, 60); + + Assert.Equal(expected, actual); + } + private QueuedEpisode queueEpisode(string path) { return new QueuedEpisode() diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestContiguous.cs b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestContiguous.cs index 286aa0c..50121f6 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestContiguous.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestContiguous.cs @@ -71,4 +71,22 @@ public class TestTimeRanges Assert.Equal(expected, actual); } + + /// + /// Tests that TimeRange intersections are detected correctly. + /// Tests each time range against a range of 5 to 10 seconds. + /// + [Theory] + [InlineData(1, 4, false)] // too early + [InlineData(4, 6, true)] // intersects on the left + [InlineData(7, 8, true)] // in the middle + [InlineData(9, 12, true)] // intersects on the right + [InlineData(13, 15, false)] // too late + public void TestTimeRangeIntersection(int start, int end, bool expected) + { + var large = new TimeRange(5, 10); + var testRange = new TimeRange(start, end); + + Assert.Equal(expected, large.Intersects(testRange)); + } } diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/AutoSkip.cs b/ConfusedPolarBear.Plugin.IntroSkipper/AutoSkip.cs index 3846bb9..1cac686 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/AutoSkip.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/AutoSkip.cs @@ -169,6 +169,8 @@ public class AutoSkip : IServerEntryPoint // Send the seek command _logger.LogDebug("Sending seek command to {Session}", deviceId); + var introEnd = (long)intro.IntroEnd - Plugin.Instance!.Configuration.AmountOfIntroToPlay; + _sessionManager.SendPlaystateCommand( session.Id, session.Id, @@ -176,7 +178,7 @@ public class AutoSkip : IServerEntryPoint { Command = PlaystateCommand.Seek, ControllingUserId = session.UserId.ToString("N"), - SeekPositionTicks = (long)intro.IntroEnd * TimeSpan.TicksPerSecond, + SeekPositionTicks = introEnd * TimeSpan.TicksPerSecond, }, CancellationToken.None); diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs index 741c561..d98554c 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs @@ -62,6 +62,17 @@ public class PluginConfiguration : BasePluginConfiguration /// public int MinimumIntroDuration { get; set; } = 15; + /// + /// Gets or sets the maximum amount of noise (in dB) that is considered silent. + /// Lowering this number will increase the filter's sensitivity to noise. + /// + public int SilenceDetectionMaximumNoise { get; set; } = -50; + + /// + /// Gets or sets the minimum duration of audio (in seconds) that is considered silent. + /// + public double SilenceDetectionMinimumDuration { get; set; } = 0.50; + // ===== Playback settings ===== /// @@ -83,5 +94,5 @@ public class PluginConfiguration : BasePluginConfiguration /// Gets or sets the amount of intro to play (in seconds). /// TODO: rename. /// - public int AmountOfIntroToPlay { get; set; } = 5; + public int AmountOfIntroToPlay { get; set; } = 2; } diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Data/TimeRange.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Data/TimeRange.cs index 7e7b51b..451df2f 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Data/TimeRange.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Data/TimeRange.cs @@ -69,6 +69,18 @@ public class TimeRange : IComparable return tr.Duration.CompareTo(Duration); } + + /// + /// Tests if this TimeRange object intersects the provided TimeRange. + /// + /// Second TimeRange object to test. + /// true if tr intersects the current TimeRange, false otherwise. + public bool Intersects(TimeRange tr) + { + return + (Start < tr.Start && tr.Start < End) || + (Start < tr.End && tr.End < End); + } } #pragma warning restore CA1036 diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs b/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs index bda56ef..7852b97 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.Globalization; using System.IO; using System.Text; +using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; namespace ConfusedPolarBear.Plugin.IntroSkipper; @@ -13,6 +14,16 @@ namespace ConfusedPolarBear.Plugin.IntroSkipper; /// public static class FFmpegWrapper { + // FFmpeg logs lines similar to the following: + // [silencedetect @ 0x000000000000] silence_start: 12.34 + // [silencedetect @ 0x000000000000] silence_end: 56.123 | silence_duration: 43.783 + + /// + /// Used with FFmpeg's silencedetect filter to extract the start and end times of silence. + /// + private static readonly Regex SilenceDetectionExpression = new( + "silence_(?start|end): (?