majomi/intro-skipper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement inverted indexes

Browse Source
This commit is contained in:
ConfusedPolarBear 2022-07-05 16:16:48 -05:00
parent 3a4e688376
commit 749ca9dcda
5 changed files with 199 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs
View File

@ -2,6 +2,8 @@
* which supports both chromaprint and the "-fp_format raw" flag.
*/
using System.Collections.Generic;
using System.Collections.ObjectModel;
using Xunit;
using Microsoft.Extensions.Logging;
@ -61,6 +63,26 @@ public class TestAudioFingerprinting
Assert.Equal(expected, actual);
}
[Fact]
public void TestIndexGeneration()
{
// 0 1 2 3 4 5 6 7
var fpr = new List<uint>(new uint[] { 1, 2, 3, 1, 5, 77, 42, 2 }).AsReadOnly();
var expected = new Dictionary<uint, Collection<uint>>()
{
{1, new Collection<uint>{ 0, 3 } },
{2, new Collection<uint>{ 1, 7 } },
{3, new Collection<uint>{ 2 } },
{5, new Collection<uint>{ 4 } },
{42, new Collection<uint>{ 6 } },
{77, new Collection<uint>{ 5 } },
};
var actual = Chromaprint.CreateInvertedIndex(fpr);
Assert.Equal(expected, actual);
}
[Fact]
public void TestIntroDetection()
{
@ -76,8 +98,8 @@ public class TestAudioFingerprinting
Assert.Equal(17.792, lhs.IntroEnd);
Assert.True(rhs.Valid);
Assert.Equal(0, rhs.IntroStart);
Assert.Equal(22.656, rhs.IntroEnd);
Assert.Equal(5.12, rhs.IntroStart);
Assert.Equal(22.912, rhs.IntroEnd);
}
private QueuedEpisode queueEpisode(string path)

24
ConfusedPolarBear.Plugin.IntroSkipper/Chromaprint.cs
View File

@ -109,6 +109,30 @@ public static class Chromaprint
return results.AsReadOnly();
}
/// <summary>
/// Transforms a Chromaprint into an inverted index of fingerprint points to the indexes they appeared at.
/// </summary>
/// <param name="fingerprint">Chromaprint fingerprint.</param>
/// <returns>Inverted index.</returns>
public static Dictionary<uint, Collection<uint>> CreateInvertedIndex(ReadOnlyCollection<uint> fingerprint)
{
var invIndex = new Dictionary<uint, Collection<uint>>();
for (int i = 0; i < fingerprint.Count; i++)
{
// Get the current point.
var point = fingerprint[i];
// Create a new collection for points of this value if it doesn't exist already.
invIndex.TryAdd(point, new Collection<uint>());
// Append the current sample's timecode to the collection for this point.
invIndex[point].Add((uint)i);
}
return invIndex;
}
/// <summary>
/// Runs ffmpeg and returns standard output.
/// </summary>

35
ConfusedPolarBear.Plugin.IntroSkipper/Configuration/configPage.html
View File

@ -106,7 +106,7 @@
</div>
<details id="statistics">
<summary>Analysis Statistics</summary>
<summary>Analysis Statistics (experimental)</summary>
<pre id="statisticsText" style="font-size: larger"></p>
</details>
@ -239,7 +239,7 @@
// Blank any old statistics
const text = document.querySelector("pre#statisticsText");
text.textContent = "";
text.textContent = "All CPU times are displayed as seconds.\n\n";
Dashboard.showLoadingMsg();
@ -247,14 +247,15 @@
let stats = await getJson("Intros/Statistics");
// Select which fields to print and label them with more friendly descriptions
let fields = "TotalCPUTime,FingerprintCPUTime,FirstPassCPUTime,SecondPassCPUTime,QuickScans,FullScans," +
"TotalQueuedEpisodes";
let fields = "TotalCPUTime,FingerprintCPUTime,FirstPassCPUTime,SecondPassCPUTime,IndexSearches," +
"QuickScans,FullScans,TotalQueuedEpisodes";
let friendlyNames = {
TotalCPUTime: "Total CPU time (ms)",
FingerprintCPUTime: "Fingerprint time (ms)",
FirstPassCPUTime: "First pass (ms)",
SecondPassCPUTime: "Second pass (ms)",
TotalCPUTime: "Total CPU time",
FingerprintCPUTime: "Fingerprint CPU time",
FirstPassCPUTime: "First pass CPU time",
SecondPassCPUTime: "Second pass CPU time",
IndexSearches: "Index searches",
QuickScans: "Quick scans",
FullScans: "Full scans",
TotalQueuedEpisodes: "Episodes queued",
@ -262,9 +263,25 @@
// Print all friendly names and data points
for (var f of fields.split(",")) {
text.textContent += friendlyNames[f].padEnd(25) + stats[f] + "\n";
let name = friendlyNames[f].padEnd(25);
let value = stats[f];
// If this statistic is a measure of CPU time, divide by 1,000 to turn milliseconds into seconds.
if (name.includes("time"))
{
value = Math.round(value / 1000);
}
text.textContent += name + value + "\n";
}
// Calculate the percentage of time (to two decimal places) spent waiting for fingerprints
const percentWait = Math.round((stats.FingerprintCPUTime * 10_000) / stats.TotalCPUTime) / 100;
// Breakdown CPU time by analysis component
text.textContent += "\nCPU time breakdown:\n";
text.textContent += "Fingerprint generation " + percentWait + "%\n";
Dashboard.hideLoadingMsg();
}

10
ConfusedPolarBear.Plugin.IntroSkipper/Data/AnalysisStatistics.cs
View File

@ -20,6 +20,11 @@ public class AnalysisStatistics
/// </summary>
public int TotalQueuedEpisodes { get; set; }
/// <summary>
/// Gets the number of times an index search successfully located a pair of introductions.
/// </summary>
public ThreadSafeInteger IndexSearches { get; } = new ThreadSafeInteger();
/// <summary>
/// Gets the number of times a quick scan successfully located a pair of introductions.
/// </summary>
@ -89,6 +94,11 @@ public class ThreadSafeInteger
/// <param name="start">Start time.</param>
public void AddDuration(DateTime start)
{
if (start == DateTime.MinValue)
{
return;
}
var elapsed = DateTime.Now.Subtract(start);
Add((int)elapsed.TotalMilliseconds);
}

152
ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/FingerprinterTask.cs
View File

@ -392,7 +392,8 @@ public class FingerprinterTask : IScheduledTask
lhsEpisode.EpisodeId,
lhsFingerprint,
rhsEpisode.EpisodeId,
rhsFingerprint);
rhsFingerprint,
true);
}
/// <summary>
@ -402,53 +403,86 @@ public class FingerprinterTask : IScheduledTask
/// <param name="lhsPoints">First episode fingerprint points.</param>
/// <param name="rhsId">Second episode id.</param>
/// <param name="rhsPoints">Second episode fingerprint points.</param>
/// <param name="isFirstPass">If this was called as part of the first analysis pass, add the elapsed time to the statistics.</param>
/// <returns>Intros for the first and second episodes.</returns>
public (Intro Lhs, Intro Rhs) FingerprintEpisodes(
Guid lhsId,
ReadOnlyCollection<uint> lhsPoints,
Guid rhsId,
ReadOnlyCollection<uint> rhsPoints)
ReadOnlyCollection<uint> rhsPoints,
bool isFirstPass)
{
var start = DateTime.Now;
var lhsRanges = new List<TimeRange>();
var rhsRanges = new List<TimeRange>();
// If this isn't running as part of the first analysis pass, don't count this CPU time as first pass time.
var start = isFirstPass ? DateTime.Now : DateTime.MinValue;
// Compare all elements of the shortest fingerprint to the other fingerprint.
var limit = Math.Min(lhsPoints.Count, rhsPoints.Count);
// ===== Method 1: Inverted indexes =====
// Creates an inverted fingerprint point index for both episodes.
// For every point which is a 100% match, search for an introduction at that point.
var (lhsRanges, rhsRanges) = SearchInvertedIndex(lhsPoints, rhsPoints);
// First, test if an intro can be found within the first 5 seconds of the episodes (±5/0.128 = ±40 samples).
var (lhsContiguous, rhsContiguous) = ShiftEpisodes(lhsPoints, rhsPoints, -40, 40);
lhsRanges.AddRange(lhsContiguous);
rhsRanges.AddRange(rhsContiguous);
// If no valid ranges were found, re-analyze the episodes considering all possible shifts.
if (lhsRanges.Count == 0)
if (lhsRanges.Count > 0)
{
_logger.LogTrace("quick scan unsuccessful, falling back to full scan (±{Limit})", limit);
analysisStatistics.FullScans.Increment();
(lhsContiguous, rhsContiguous) = ShiftEpisodes(lhsPoints, rhsPoints, -1 * limit, limit);
lhsRanges.AddRange(lhsContiguous);
rhsRanges.AddRange(rhsContiguous);
}
else
{
_logger.LogTrace("quick scan successful");
analysisStatistics.QuickScans.Increment();
}
if (lhsRanges.Count == 0)
{
_logger.LogTrace(
"Unable to find a shared introduction sequence between {LHS} and {RHS}",
lhsId,
rhsId);
_logger.LogTrace("Index search successful");
analysisStatistics.IndexSearches.Increment();
analysisStatistics.FirstPassCPUTime.AddDuration(start);
return (new Intro(lhsId), new Intro(rhsId));
return GetLongestTimeRange(lhsId, lhsRanges, rhsId, rhsRanges);
}
// After comparing both episodes at all possible shift positions, store the longest time range as the intro.
// ===== Method 2: Quick scan =====
// Tests if an intro can be found within the first 5 seconds of the episodes. ±5/0.128 = ±40 samples.
(lhsRanges, rhsRanges) = ShiftEpisodes(lhsPoints, rhsPoints, -40, 40);
if (lhsRanges.Count > 0)
{
_logger.LogTrace("Quick scan successful");
analysisStatistics.QuickScans.Increment();
analysisStatistics.FirstPassCPUTime.AddDuration(start);
return GetLongestTimeRange(lhsId, lhsRanges, rhsId, rhsRanges);
}
// ===== Method 3: Full scan =====
// Compares all elements of the shortest fingerprint to the other fingerprint.
var limit = Math.Min(lhsPoints.Count, rhsPoints.Count);
(lhsRanges, rhsRanges) = ShiftEpisodes(lhsPoints, rhsPoints, -1 * limit, limit);
if (lhsRanges.Count > 0)
{
_logger.LogTrace("Full scan successful");
analysisStatistics.FullScans.Increment();
analysisStatistics.FirstPassCPUTime.AddDuration(start);
return GetLongestTimeRange(lhsId, lhsRanges, rhsId, rhsRanges);
}
// No method was able to find an introduction, return nothing.
_logger.LogTrace(
"Unable to find a shared introduction sequence between {LHS} and {RHS}",
lhsId,
rhsId);
analysisStatistics.FirstPassCPUTime.AddDuration(start);
return (new Intro(lhsId), new Intro(rhsId));
}
/// <summary>
/// Locates the longest range of similar audio and returns an Intro class for each range.
/// </summary>
/// <param name="lhsId">First episode id.</param>
/// <param name="lhsRanges">First episode shared timecodes.</param>
/// <param name="rhsId">Second episode id.</param>
/// <param name="rhsRanges">Second episode shared timecodes.</param>
/// <returns>Intros for the first and second episodes.</returns>
private (Intro Lhs, Intro Rhs) GetLongestTimeRange(
Guid lhsId,
List<TimeRange> lhsRanges,
Guid rhsId,
List<TimeRange> rhsRanges)
{
// Store the longest time range as the introduction.
lhsRanges.Sort();
rhsRanges.Sort();
@ -466,10 +500,54 @@ public class FingerprinterTask : IScheduledTask
rhsIntro.Start = 0;
}
analysisStatistics.FirstPassCPUTime.AddDuration(start);
// Create Intro classes for each time range.
return (new Intro(lhsId, lhsIntro), new Intro(rhsId, rhsIntro));
}
/// <summary>
/// Search for a shared introduction sequence using inverted indexes.
/// </summary>
/// <param name="lhsPoints">Left episode fingerprint points.</param>
/// <param name="rhsPoints">Right episode fingerprint points.</param>
/// <returns>List of shared TimeRanges between the left and right episodes.</returns>
private (List<TimeRange> Lhs, List<TimeRange> Rhs) SearchInvertedIndex(
ReadOnlyCollection<uint> lhsPoints,
ReadOnlyCollection<uint> rhsPoints)
{
var lhsRanges = new List<TimeRange>();
var rhsRanges = new List<TimeRange>();
// Generate inverted indexes for the left and right episodes.
var lhsIndex = Chromaprint.CreateInvertedIndex(lhsPoints);
var rhsIndex = Chromaprint.CreateInvertedIndex(rhsPoints);
var indexShifts = new HashSet<int>();
// For all audio points in the left episode, check if the right episode has a point which matches exactly.
// If an exact match is found, calculate the shift that must be used to align the points.
foreach (var kvp in lhsIndex)
{
var point = kvp.Key;
if (rhsIndex.ContainsKey(point))
{
// TODO: consider all timecodes before falling back
var lhsFirst = (int)lhsIndex[point][0];
var rhsFirst = (int)rhsIndex[point][0];
indexShifts.Add(rhsFirst - lhsFirst);
}
}
// Use all discovered shifts to compare the episodes.
foreach (var shift in indexShifts)
{
var (lhsIndexContiguous, rhsIndexContiguous) = ShiftEpisodes(lhsPoints, rhsPoints, shift, shift);
lhsRanges.AddRange(lhsIndexContiguous);
rhsRanges.AddRange(rhsIndexContiguous);
}
return (lhsRanges, rhsRanges);
}
/// <summary>
/// Shifts a pair of episodes through the range of provided shift amounts and returns discovered contiguous time ranges.
/// </summary>