diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs index 877b8d7..b74784e 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs @@ -2,6 +2,8 @@ * which supports both chromaprint and the "-fp_format raw" flag. */ +using System.Collections.Generic; +using System.Collections.ObjectModel; using Xunit; using Microsoft.Extensions.Logging; @@ -61,6 +63,26 @@ public class TestAudioFingerprinting Assert.Equal(expected, actual); } + [Fact] + public void TestIndexGeneration() + { + // 0 1 2 3 4 5 6 7 + var fpr = new List(new uint[] { 1, 2, 3, 1, 5, 77, 42, 2 }).AsReadOnly(); + var expected = new Dictionary>() + { + {1, new Collection{ 0, 3 } }, + {2, new Collection{ 1, 7 } }, + {3, new Collection{ 2 } }, + {5, new Collection{ 4 } }, + {42, new Collection{ 6 } }, + {77, new Collection{ 5 } }, + }; + + var actual = Chromaprint.CreateInvertedIndex(fpr); + + Assert.Equal(expected, actual); + } + [Fact] public void TestIntroDetection() { @@ -76,8 +98,8 @@ public class TestAudioFingerprinting Assert.Equal(17.792, lhs.IntroEnd); Assert.True(rhs.Valid); - Assert.Equal(0, rhs.IntroStart); - Assert.Equal(22.656, rhs.IntroEnd); + Assert.Equal(5.12, rhs.IntroStart); + Assert.Equal(22.912, rhs.IntroEnd); } private QueuedEpisode queueEpisode(string path) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Chromaprint.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Chromaprint.cs index 950679a..e417dce 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Chromaprint.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Chromaprint.cs @@ -109,6 +109,30 @@ public static class Chromaprint return results.AsReadOnly(); } + /// + /// Transforms a Chromaprint into an inverted index of fingerprint points to the indexes they appeared at. + /// + /// Chromaprint fingerprint. + /// Inverted index. + public static Dictionary> CreateInvertedIndex(ReadOnlyCollection fingerprint) + { + var invIndex = new Dictionary>(); + + for (int i = 0; i < fingerprint.Count; i++) + { + // Get the current point. + var point = fingerprint[i]; + + // Create a new collection for points of this value if it doesn't exist already. + invIndex.TryAdd(point, new Collection()); + + // Append the current sample's timecode to the collection for this point. + invIndex[point].Add((uint)i); + } + + return invIndex; + } + /// /// Runs ffmpeg and returns standard output. /// diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/configPage.html b/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/configPage.html index 1666214..b8dadf7 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/configPage.html +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/configPage.html @@ -106,7 +106,7 @@
- Analysis Statistics + Analysis Statistics (experimental)
@@ -239,7 +239,7 @@ // Blank any old statistics const text = document.querySelector("pre#statisticsText"); - text.textContent = ""; + text.textContent = "All CPU times are displayed as seconds.\n\n"; Dashboard.showLoadingMsg(); @@ -247,14 +247,15 @@ let stats = await getJson("Intros/Statistics"); // Select which fields to print and label them with more friendly descriptions - let fields = "TotalCPUTime,FingerprintCPUTime,FirstPassCPUTime,SecondPassCPUTime,QuickScans,FullScans," + - "TotalQueuedEpisodes"; + let fields = "TotalCPUTime,FingerprintCPUTime,FirstPassCPUTime,SecondPassCPUTime,IndexSearches," + + "QuickScans,FullScans,TotalQueuedEpisodes"; let friendlyNames = { - TotalCPUTime: "Total CPU time (ms)", - FingerprintCPUTime: "Fingerprint time (ms)", - FirstPassCPUTime: "First pass (ms)", - SecondPassCPUTime: "Second pass (ms)", + TotalCPUTime: "Total CPU time", + FingerprintCPUTime: "Fingerprint CPU time", + FirstPassCPUTime: "First pass CPU time", + SecondPassCPUTime: "Second pass CPU time", + IndexSearches: "Index searches", QuickScans: "Quick scans", FullScans: "Full scans", TotalQueuedEpisodes: "Episodes queued", @@ -262,9 +263,25 @@ // Print all friendly names and data points for (var f of fields.split(",")) { - text.textContent += friendlyNames[f].padEnd(25) + stats[f] + "\n"; + let name = friendlyNames[f].padEnd(25); + let value = stats[f]; + + // If this statistic is a measure of CPU time, divide by 1,000 to turn milliseconds into seconds. + if (name.includes("time")) + { + value = Math.round(value / 1000); + } + + text.textContent += name + value + "\n"; } + // Calculate the percentage of time (to two decimal places) spent waiting for fingerprints + const percentWait = Math.round((stats.FingerprintCPUTime * 10_000) / stats.TotalCPUTime) / 100; + + // Breakdown CPU time by analysis component + text.textContent += "\nCPU time breakdown:\n"; + text.textContent += "Fingerprint generation " + percentWait + "%\n"; + Dashboard.hideLoadingMsg(); } diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Data/AnalysisStatistics.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Data/AnalysisStatistics.cs index 8ca0e27..d0f1e97 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Data/AnalysisStatistics.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Data/AnalysisStatistics.cs @@ -20,6 +20,11 @@ public class AnalysisStatistics /// public int TotalQueuedEpisodes { get; set; } + /// + /// Gets the number of times an index search successfully located a pair of introductions. + /// + public ThreadSafeInteger IndexSearches { get; } = new ThreadSafeInteger(); + /// /// Gets the number of times a quick scan successfully located a pair of introductions. /// @@ -89,6 +94,11 @@ public class ThreadSafeInteger /// Start time. public void AddDuration(DateTime start) { + if (start == DateTime.MinValue) + { + return; + } + var elapsed = DateTime.Now.Subtract(start); Add((int)elapsed.TotalMilliseconds); } diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/FingerprinterTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/FingerprinterTask.cs index 3598997..ed91580 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/FingerprinterTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/FingerprinterTask.cs @@ -392,7 +392,8 @@ public class FingerprinterTask : IScheduledTask lhsEpisode.EpisodeId, lhsFingerprint, rhsEpisode.EpisodeId, - rhsFingerprint); + rhsFingerprint, + true); } /// @@ -402,53 +403,86 @@ public class FingerprinterTask : IScheduledTask /// First episode fingerprint points. /// Second episode id. /// Second episode fingerprint points. + /// If this was called as part of the first analysis pass, add the elapsed time to the statistics. /// Intros for the first and second episodes. public (Intro Lhs, Intro Rhs) FingerprintEpisodes( Guid lhsId, ReadOnlyCollection lhsPoints, Guid rhsId, - ReadOnlyCollection rhsPoints) + ReadOnlyCollection rhsPoints, + bool isFirstPass) { - var start = DateTime.Now; - var lhsRanges = new List(); - var rhsRanges = new List(); + // If this isn't running as part of the first analysis pass, don't count this CPU time as first pass time. + var start = isFirstPass ? DateTime.Now : DateTime.MinValue; - // Compare all elements of the shortest fingerprint to the other fingerprint. - var limit = Math.Min(lhsPoints.Count, rhsPoints.Count); + // ===== Method 1: Inverted indexes ===== + // Creates an inverted fingerprint point index for both episodes. + // For every point which is a 100% match, search for an introduction at that point. + var (lhsRanges, rhsRanges) = SearchInvertedIndex(lhsPoints, rhsPoints); - // First, test if an intro can be found within the first 5 seconds of the episodes (±5/0.128 = ±40 samples). - var (lhsContiguous, rhsContiguous) = ShiftEpisodes(lhsPoints, rhsPoints, -40, 40); - lhsRanges.AddRange(lhsContiguous); - rhsRanges.AddRange(rhsContiguous); - - // If no valid ranges were found, re-analyze the episodes considering all possible shifts. - if (lhsRanges.Count == 0) + if (lhsRanges.Count > 0) { - _logger.LogTrace("quick scan unsuccessful, falling back to full scan (±{Limit})", limit); - analysisStatistics.FullScans.Increment(); - - (lhsContiguous, rhsContiguous) = ShiftEpisodes(lhsPoints, rhsPoints, -1 * limit, limit); - lhsRanges.AddRange(lhsContiguous); - rhsRanges.AddRange(rhsContiguous); - } - else - { - _logger.LogTrace("quick scan successful"); - analysisStatistics.QuickScans.Increment(); - } - - if (lhsRanges.Count == 0) - { - _logger.LogTrace( - "Unable to find a shared introduction sequence between {LHS} and {RHS}", - lhsId, - rhsId); - + _logger.LogTrace("Index search successful"); + analysisStatistics.IndexSearches.Increment(); analysisStatistics.FirstPassCPUTime.AddDuration(start); - return (new Intro(lhsId), new Intro(rhsId)); + + return GetLongestTimeRange(lhsId, lhsRanges, rhsId, rhsRanges); } - // After comparing both episodes at all possible shift positions, store the longest time range as the intro. + // ===== Method 2: Quick scan ===== + // Tests if an intro can be found within the first 5 seconds of the episodes. ±5/0.128 = ±40 samples. + (lhsRanges, rhsRanges) = ShiftEpisodes(lhsPoints, rhsPoints, -40, 40); + + if (lhsRanges.Count > 0) + { + _logger.LogTrace("Quick scan successful"); + analysisStatistics.QuickScans.Increment(); + analysisStatistics.FirstPassCPUTime.AddDuration(start); + + return GetLongestTimeRange(lhsId, lhsRanges, rhsId, rhsRanges); + } + + // ===== Method 3: Full scan ===== + // Compares all elements of the shortest fingerprint to the other fingerprint. + var limit = Math.Min(lhsPoints.Count, rhsPoints.Count); + (lhsRanges, rhsRanges) = ShiftEpisodes(lhsPoints, rhsPoints, -1 * limit, limit); + + if (lhsRanges.Count > 0) + { + _logger.LogTrace("Full scan successful"); + analysisStatistics.FullScans.Increment(); + analysisStatistics.FirstPassCPUTime.AddDuration(start); + + return GetLongestTimeRange(lhsId, lhsRanges, rhsId, rhsRanges); + } + + // No method was able to find an introduction, return nothing. + + _logger.LogTrace( + "Unable to find a shared introduction sequence between {LHS} and {RHS}", + lhsId, + rhsId); + + analysisStatistics.FirstPassCPUTime.AddDuration(start); + + return (new Intro(lhsId), new Intro(rhsId)); + } + + /// + /// Locates the longest range of similar audio and returns an Intro class for each range. + /// + /// First episode id. + /// First episode shared timecodes. + /// Second episode id. + /// Second episode shared timecodes. + /// Intros for the first and second episodes. + private (Intro Lhs, Intro Rhs) GetLongestTimeRange( + Guid lhsId, + List lhsRanges, + Guid rhsId, + List rhsRanges) + { + // Store the longest time range as the introduction. lhsRanges.Sort(); rhsRanges.Sort(); @@ -466,10 +500,54 @@ public class FingerprinterTask : IScheduledTask rhsIntro.Start = 0; } - analysisStatistics.FirstPassCPUTime.AddDuration(start); + // Create Intro classes for each time range. return (new Intro(lhsId, lhsIntro), new Intro(rhsId, rhsIntro)); } + /// + /// Search for a shared introduction sequence using inverted indexes. + /// + /// Left episode fingerprint points. + /// Right episode fingerprint points. + /// List of shared TimeRanges between the left and right episodes. + private (List Lhs, List Rhs) SearchInvertedIndex( + ReadOnlyCollection lhsPoints, + ReadOnlyCollection rhsPoints) + { + var lhsRanges = new List(); + var rhsRanges = new List(); + + // Generate inverted indexes for the left and right episodes. + var lhsIndex = Chromaprint.CreateInvertedIndex(lhsPoints); + var rhsIndex = Chromaprint.CreateInvertedIndex(rhsPoints); + var indexShifts = new HashSet(); + + // For all audio points in the left episode, check if the right episode has a point which matches exactly. + // If an exact match is found, calculate the shift that must be used to align the points. + foreach (var kvp in lhsIndex) + { + var point = kvp.Key; + + if (rhsIndex.ContainsKey(point)) + { + // TODO: consider all timecodes before falling back + var lhsFirst = (int)lhsIndex[point][0]; + var rhsFirst = (int)rhsIndex[point][0]; + indexShifts.Add(rhsFirst - lhsFirst); + } + } + + // Use all discovered shifts to compare the episodes. + foreach (var shift in indexShifts) + { + var (lhsIndexContiguous, rhsIndexContiguous) = ShiftEpisodes(lhsPoints, rhsPoints, shift, shift); + lhsRanges.AddRange(lhsIndexContiguous); + rhsRanges.AddRange(rhsIndexContiguous); + } + + return (lhsRanges, rhsRanges); + } + /// /// Shifts a pair of episodes through the range of provided shift amounts and returns discovered contiguous time ranges. ///