Switch to new fingerprint comparison algorithm

This commit is contained in:
ConfusedPolarBear 2022-05-31 16:12:11 -05:00
parent 872451cc7e
commit eee11e23bb
6 changed files with 234 additions and 138 deletions

View File

@ -70,11 +70,11 @@ public class TestFPCalc
Assert.True(lhs.Valid);
Assert.Equal(0, lhs.IntroStart);
Assert.Equal(17.792, lhs.IntroEnd);
Assert.Equal(17.92, lhs.IntroEnd);
Assert.True(rhs.Valid);
Assert.Equal(5.12, rhs.IntroStart);
Assert.Equal(22.912, rhs.IntroEnd);
Assert.Equal(0, rhs.IntroStart);
Assert.Equal(22.784, rhs.IntroEnd);
}
private QueuedEpisode queueEpisode(string path)

View File

@ -13,7 +13,7 @@ public class TestTimeRanges
};
var expected = new TimeRange(1, 4);
var actual = TimeRangeHelpers.FindContiguous(times, 3.25);
var actual = TimeRangeHelpers.FindContiguous(times, 2);
Assert.Equal(expected, actual);
}
@ -29,7 +29,45 @@ public class TestTimeRanges
};
var expected = new TimeRange(1, 5.3128);
var actual = TimeRangeHelpers.FindContiguous(times, 3.25);
var actual = TimeRangeHelpers.FindContiguous(times, 2);
Assert.Equal(expected, actual);
}
[Fact]
public void TestFuturama()
{
// These timestamps were manually extracted from Futurama S01E04 and S01E05.
var times = new double[]{
2.176, 8.32, 10.112, 11.264, 13.696, 16, 16.128, 16.64, 16.768, 16.896, 17.024, 17.152, 17.28,
17.408, 17.536, 17.664, 17.792, 17.92, 18.048, 18.176, 18.304, 18.432, 18.56, 18.688, 18.816,
18.944, 19.072, 19.2, 19.328, 19.456, 19.584, 19.712, 19.84, 19.968, 20.096, 20.224, 20.352,
20.48, 20.608, 20.736, 20.864, 20.992, 21.12, 21.248, 21.376, 21.504, 21.632, 21.76, 21.888,
22.016, 22.144, 22.272, 22.4, 22.528, 22.656, 22.784, 22.912, 23.04, 23.168, 23.296, 23.424,
23.552, 23.68, 23.808, 23.936, 24.064, 24.192, 24.32, 24.448, 24.576, 24.704, 24.832, 24.96,
25.088, 25.216, 25.344, 25.472, 25.6, 25.728, 25.856, 25.984, 26.112, 26.24, 26.368, 26.496,
26.624, 26.752, 26.88, 27.008, 27.136, 27.264, 27.392, 27.52, 27.648, 27.776, 27.904, 28.032,
28.16, 28.288, 28.416, 28.544, 28.672, 28.8, 28.928, 29.056, 29.184, 29.312, 29.44, 29.568,
29.696, 29.824, 29.952, 30.08, 30.208, 30.336, 30.464, 30.592, 30.72, 30.848, 30.976, 31.104,
31.232, 31.36, 31.488, 31.616, 31.744, 31.872, 32, 32.128, 32.256, 32.384, 32.512, 32.64,
32.768, 32.896, 33.024, 33.152, 33.28, 33.408, 33.536, 33.664, 33.792, 33.92, 34.048, 34.176,
34.304, 34.432, 34.56, 34.688, 34.816, 34.944, 35.072, 35.2, 35.328, 35.456, 35.584, 35.712,
35.84, 35.968, 36.096, 36.224, 36.352, 36.48, 36.608, 36.736, 36.864, 36.992, 37.12, 37.248,
37.376, 37.504, 37.632, 37.76, 37.888, 38.016, 38.144, 38.272, 38.4, 38.528, 38.656, 38.784,
38.912, 39.04, 39.168, 39.296, 39.424, 39.552, 39.68, 39.808, 39.936, 40.064, 40.192, 40.32,
40.448, 40.576, 40.704, 40.832, 40.96, 41.088, 41.216, 41.344, 41.472, 41.6, 41.728, 41.856,
41.984, 42.112, 42.24, 42.368, 42.496, 42.624, 42.752, 42.88, 43.008, 43.136, 43.264, 43.392,
43.52, 43.648, 43.776, 43.904, 44.032, 44.16, 44.288, 44.416, 44.544, 44.672, 44.8, 44.928,
45.056, 45.184, 57.344, 62.976, 68.864, 74.368, 81.92, 82.048, 86.528, 100.864, 102.656,
102.784, 102.912, 103.808, 110.976, 116.864, 125.696, 128.384, 133.248, 133.376, 136.064,
136.704, 142.976, 150.272, 152.064, 164.864, 164.992, 166.144, 166.272, 175.488, 190.08,
191.872, 192, 193.28, 193.536, 213.376, 213.504, 225.664, 225.792, 243.2, 243.84, 256,
264.448, 264.576, 264.704, 269.568, 274.816, 274.944, 276.096, 283.264, 294.784, 294.912,
295.04, 295.168, 313.984, 325.504, 333.568, 335.872, 336.384
};
var expected = new TimeRange(16, 45.184);
var actual = TimeRangeHelpers.FindContiguous(times, 2);
Assert.Equal(expected, actual);
}

View File

@ -49,9 +49,45 @@
</form>
</div>
<div>
<h3>Troubleshooter</h3>
<p>Compare the audio fingerprint of two episodes.</p>
<details>
<summary>Fingerprint Visualizer</summary>
<p>
Interactively compare the audio fingerprints of two episodes. <br />
The blue and red bar to the right of the fingerprint diff turns blue
when the corresponding fingerprint points are at least 75% similar.
</p>
<table>
<thead>
<td style="min-width: 100px; font-weight: bold">Key</td>
<td style="font-weight: bold">Function</td>
</thead>
<tbody>
<tr>
<td>Up arrow</td>
<td>
Shift the left episode up by 0.128 seconds.
Holding control will shift the episode by 10 seconds.
</td>
</tr>
<tr>
<td>Down arrow</td>
<td>
Shift the left episode down by 0.128 seconds.
Holding control will shift the episode by 10 seconds.
</td>
</tr>
<tr>
<td>Right arrow</td>
<td>Advance to the next pair of episodes.</td>
</tr>
<tr>
<td>Left arrow</td>
<td>Go back to the previous pair of episodes.</td>
</tr>
</tbody>
</table>
<br />
<select id="troubleshooterShow"></select>
<select id="troubleshooterSeason"></select>
@ -61,21 +97,23 @@
<select id="troubleshooterEpisode2"></select>
<br />
<span>Shift amount:</span>
<input type="number" min="-3000" max="3000" value="0" id="offset">
<br />
<br />
<canvas id="troubleshooter"></canvas>
<span id="timestamps"></span>
</div>
</details>
</div>
<script>
const pluginId = "c83d86bb-a1e0-4c35-a113-e2101cf4ee6b";
// first and second episodes to fingerprint & compare
var lhs = [];
var rhs = [];
var diffCount = []; // count of bits that are different
// fingerprint point comparison & miminum similarity threshold
var fprDiffs = [];
var fprDiffMinimum = 75.0;
// seasons grouped by show
var shows = {};
@ -92,8 +130,11 @@
async function onLoad() {
shows = await getJson("Intros/Shows");
// sort all show names & add to the select
for (var show of shows) {
var sorted = [];
for (var series in shows) { sorted.push(series); }
sorted.sort();
for (var show of sorted) {
addItem(selectShow, show, show);
}
@ -128,8 +169,11 @@
i++;
}
selectEpisode1.value = "";
selectEpisode2.value = "";
setTimeout(() => {
selectEpisode1.selectedIndex = 0;
selectEpisode2.selectedIndex = 1;
episodeChanged();
}, 100);
}
// episode changed, get fingerprints & calculate diff
@ -192,10 +236,67 @@
});
}
// key pressed
function keyDown(e) {
let episodeDelta = 0;
let offsetDelta = 0;
switch (e.key) {
case "ArrowDown":
// if the control key is pressed, shift LHS by 10s. Otherwise, shift by 1.
offsetDelta = e.ctrlKey ? 10 / 0.128 : 1;
break;
case "ArrowUp":
offsetDelta = e.ctrlKey ? -10 / 0.128 : -1;
break;
case "ArrowRight":
episodeDelta = 2;
break;
case "ArrowLeft":
episodeDelta = -2;
break;
default:
return;
}
if (offsetDelta != 0) {
txtOffset.value = Number(txtOffset.value) + Math.floor(offsetDelta);
}
if (episodeDelta != 0) {
// calculate the number of episodes remaining in the LHS and RHS episode pickers
const lhsRemaining = selectEpisode1.selectedIndex;
const rhsRemaining = selectEpisode2.length - selectEpisode2.selectedIndex - 1;
// if we're moving forward and the right episode picker is close to the end, don't move.
if (episodeDelta > 0 && rhsRemaining <= 1) {
return;
} else if (episodeDelta < 0 && lhsRemaining <= 1) {
return;
}
selectEpisode1.selectedIndex += episodeDelta;
selectEpisode2.selectedIndex += episodeDelta;
episodeChanged();
}
renderTroubleshooter();
e.preventDefault();
}
// converts seconds to a readable timestamp (i.e. 127 becomes "02:07").
function secondsToString(seconds) {
return new Date(seconds * 1000).toISOString().substr(14, 5);
}
document.querySelector('#TemplateConfigPage')
.addEventListener('pageshow', function () {
Dashboard.showLoadingMsg();
ApiClient.getPluginConfiguration(pluginId).then(function (config) {
ApiClient.getPluginConfiguration("c83d86bb-a1e0-4c35-a113-e2101cf4ee6b").then(function (config) {
document.querySelector('#CacheFingerprints').checked = config.CacheFingerprints;
document.querySelector('#ShowPromptAdjustment').value = config.ShowPromptAdjustment;
@ -208,13 +309,13 @@
document.querySelector('#FingerprintConfigForm')
.addEventListener('submit', function () {
Dashboard.showLoadingMsg();
ApiClient.getPluginConfiguration(pluginId).then(function (config) {
ApiClient.getPluginConfiguration("c83d86bb-a1e0-4c35-a113-e2101cf4ee6b").then(function (config) {
config.CacheFingerprints = document.querySelector('#CacheFingerprints').checked;
config.ShowPromptAdjustment = document.querySelector("#ShowPromptAdjustment").value;
config.HidePromptAdjustment = document.querySelector("#HidePromptAdjustment").value;
ApiClient.updatePluginConfiguration(pluginId, config).then(function (result) {
ApiClient.updatePluginConfiguration("c83d86bb-a1e0-4c35-a113-e2101cf4ee6b", config).then(function (result) {
Dashboard.processPluginConfigurationUpdateResult(result);
});
});
@ -227,6 +328,7 @@
selectSeason.addEventListener("change", seasonChanged);
selectEpisode1.addEventListener("change", episodeChanged);
selectEpisode2.addEventListener("change", episodeChanged);
document.addEventListener("keydown", keyDown); // TODO: remove document wide listener when the user exits the page
canvas.addEventListener("mousemove", (e) => {
const rect = e.currentTarget.getBoundingClientRect();
@ -244,13 +346,22 @@
diffPos = y - shift;
}
diffPos = Math.floor(diffPos);
lTime = Math.round(lTime * 100) / 100;
rTime = Math.round(rTime * 100) / 100;
const diff = fprDiffs[Math.floor(diffPos)];
const times = document.querySelector("span#timestamps");
times.textContent = lTime + ", " + rTime + " similarity is " + diffCount[diffPos];
if (!diff) {
times.style.display = "none";
return;
} else {
times.style.display = "unset";
}
// LHS timestamp, RHS timestamp, percent similarity
times.textContent =
secondsToString(lTime) + ", " +
secondsToString(rTime) + ", " +
Math.round(diff) + "%";
times.style.position = "relative";
times.style.left = "25px";
@ -262,7 +373,8 @@
</script>
<script>
// MIT licensed from https://github.com/dnknth/acoustid-match/blob/ffbf21d8c53c40d3b3b4c92238c35846545d3cd7/fingerprints/static/fingerprints/fputils.js
// Modified from https://github.com/dnknth/acoustid-match/blob/ffbf21d8c53c40d3b3b4c92238c35846545d3cd7/fingerprints/static/fingerprints/fputils.js
// Originally licensed as MIT.
function renderFingerprintData(ctx, fp, xor = false) {
const pixels = ctx.createImageData(32, fp.length);
let idx = 0;
@ -285,35 +397,29 @@
}
}
if (!xor) {
return pixels;
}
// if rendering the XOR of the fingerprints, count how many bits are different at each timecode
if (xor) {
diffCount = [];
fprDiffs = [];
for (let i = 0; i < fp.length; i++) {
let count = 0;
for (let i = 0; i < fp.length; i++) {
let count = 0;
for (let j = 0; j < 32; j++) {
if (fp[i] & (1 << j)) {
count++;
}
for (let j = 0; j < 32; j++) {
if (fp[i] & (1 << j)) {
count++;
}
// push the percentage similarity
diffCount[i] = 100 - (count * 100) / 32;
}
// push the percentage similarity
fprDiffs[i] = 100 - (count * 100) / 32;
}
return pixels;
}
function paintFingerprint(canvas, fp) {
const ctx = canvas.getContext('2d');
const pixels = renderFingerprintData(ctx, fp);
canvas.width = pixels.width;
canvas.height = pixels.height;
ctx.putImageData(pixels, 0, 0);
}
function paintFingerprintDiff(canvas, fp1, fp2, offset) {
let leftOffset = 0, rightOffset = 0;
if (offset < 0) {
@ -332,17 +438,40 @@
const pixels1 = renderFingerprintData(ctx, fp1);
const pixels2 = renderFingerprintData(ctx, fp2);
const pixelsDiff = renderFingerprintData(ctx, fpDiff, true);
const border = 4;
canvas.width = pixels1.width + border + // left fingerprint
pixels2.width + border + // right fingerprint
pixelsDiff.width + border // fingerprint diff
+ 4; // if diff[x] >= fprDiffMinimum
canvas.width = pixels1.width + 2 + pixels2.width + 2 + pixelsDiff.width;
canvas.height = Math.max(pixels1.height, pixels2.height) + Math.abs(offset);
ctx.rect(0, 0, canvas.width, canvas.height);
ctx.fillStyle = "#C5C5C5";
ctx.fill();
ctx.putImageData(pixels1, 0, rightOffset);
ctx.putImageData(pixels2, pixels1.width + 2, leftOffset);
ctx.putImageData(pixelsDiff, pixels1.width + 2 + pixels2.width + 2, Math.abs(offset));
// draw left fingerprint
let dx = 0;
ctx.putImageData(pixels1, dx, rightOffset);
dx += pixels1.width + border;
// draw right fingerprint
ctx.putImageData(pixels2, dx, leftOffset);
dx += pixels2.width + border;
// draw fingerprint diff
ctx.putImageData(pixelsDiff, dx, Math.abs(offset));
dx += pixelsDiff.width + border;
// draw the fingerprint diff similarity indicator
// https://davidmathlogic.com/colorblind/#%23EA3535-%232C92EF
for (var i in fprDiffs) {
const j = Number(i);
const y = Math.abs(offset) + j;
ctx.fillStyle = fprDiffs[j] >= fprDiffMinimum ? "#2C92EF" : "#EA3535";
ctx.fillRect(dx, y, 4, 1);
}
}
</script>
</div>

View File

@ -11,7 +11,7 @@ namespace ConfusedPolarBear.Plugin.IntroSkipper.Controllers;
/// <summary>
/// Intro skipper troubleshooting controller. Allows browsing fingerprints on a per episode basis.
/// </summary>
[Authorize]
[Authorize(Policy = "RequiresElevation")]
[ApiController]
[Produces(MediaTypeNames.Application.Json)]
[Route("Intros")]

View File

@ -3,6 +3,9 @@ using System.Collections.Generic;
namespace ConfusedPolarBear.Plugin.IntroSkipper;
// Supress CA1036: Override methods on comparable types.
#pragma warning disable CA1036
/// <summary>
/// Range of contiguous time.
/// </summary>
@ -54,98 +57,23 @@ public class TimeRange : IComparable
public double Duration => End - Start;
/// <summary>
/// Comparison operator.
/// Compare TimeRange durations.
/// </summary>
/// <param name="left">Left TimeRange.</param>
/// <param name="right">Right TimeRange.</param>
public static bool operator ==(TimeRange left, TimeRange right)
{
return left.Equals(right);
}
/// <summary>
/// Comparison operator.
/// </summary>
/// <param name="left">Left TimeRange.</param>
/// <param name="right">Right TimeRange.</param>
public static bool operator !=(TimeRange left, TimeRange right)
{
return !left.Equals(right);
}
/// <summary>
/// Comparison operator.
/// </summary>
/// <param name="left">Left TimeRange.</param>
/// <param name="right">Right TimeRange.</param>
public static bool operator <=(TimeRange left, TimeRange right)
{
return left.CompareTo(right) <= 0;
}
/// <summary>
/// Comparison operator.
/// </summary>
/// <param name="left">Left TimeRange.</param>
/// <param name="right">Right TimeRange.</param>
public static bool operator <(TimeRange left, TimeRange right)
{
return left.CompareTo(right) < 0;
}
/// <summary>
/// Comparison operator.
/// </summary>
/// <param name="left">Left TimeRange.</param>
/// <param name="right">Right TimeRange.</param>
public static bool operator >=(TimeRange left, TimeRange right)
{
return left.CompareTo(right) >= 0;
}
/// <summary>
/// Comparison operator.
/// </summary>
/// <param name="left">Left TimeRange.</param>
/// <param name="right">Right TimeRange.</param>
public static bool operator >(TimeRange left, TimeRange right)
{
return left.CompareTo(right) > 0;
}
/// <summary>
/// Compares this TimeRange to another TimeRange.
/// </summary>
/// <param name="obj">Other object to compare against.</param>
/// <returns>A signed integer that indicates whether this instance precedes, follows, or appears in the same position in the sort order as the obj parameter.</returns>
/// <param name="obj">Object to compare with.</param>
/// <returns>int.</returns>
public int CompareTo(object? obj)
{
if (obj is not TimeRange tr)
if (!(obj is TimeRange tr))
{
return 0;
throw new ArgumentException("obj must be a TimeRange");
}
return this.Duration.CompareTo(tr.Duration);
}
/// <inheritdoc/>
public override bool Equals(object? obj)
{
if (obj is null || obj is not TimeRange tr)
{
return false;
}
return this.Start == tr.Start && this.Duration == tr.Duration;
}
/// <inheritdoc/>
public override int GetHashCode()
{
return this.Start.GetHashCode() + this.Duration.GetHashCode();
return tr.Duration.CompareTo(Duration);
}
}
#pragma warning restore CA1036
/// <summary>
/// Time range helpers.
/// </summary>
@ -167,7 +95,7 @@ public static class TimeRangeHelpers
Array.Sort(times);
var ranges = new List<TimeRange>();
var currentRange = new TimeRange(times[0], 0);
var currentRange = new TimeRange(times[0], times[0]);
// For all provided timestamps, check if it is contiguous with its neighbor.
for (var i = 0; i < times.Length - 1; i++)
@ -182,7 +110,7 @@ public static class TimeRangeHelpers
}
ranges.Add(new TimeRange(currentRange));
currentRange.Start = next;
currentRange = new TimeRange(next, next);
}
// Find and return the longest contiguous range.

View File

@ -21,13 +21,14 @@ public class FingerprinterTask : IScheduledTask
/// <summary>
/// Maximum number of bits (out of 32 total) that can be different between segments before they are considered dissimilar.
/// 8 bits means the audio must be at least 75% similar (1 - 8 / 32).
/// </summary>
private const double MaximumDifferences = 3;
private const double MaximumDifferences = 8;
/// <summary>
/// Maximum time (in seconds) permitted between timestamps before they are considered non-contiguous.
/// </summary>
private const double MaximumDistance = 3.25;
private const double MaximumDistance = 2.5;
/// <summary>
/// Seconds of audio in one fingerprint point. This value is defined by the Chromaprint library and should not be changed.
@ -192,7 +193,7 @@ public class FingerprinterTask : IScheduledTask
var rhs = episodes[i + 1];
// TODO: make configurable
if (!everFoundIntro && failures >= 6)
if (!everFoundIntro && failures >= 20)
{
_logger.LogWarning(
"Failed to find an introduction in {Series} season {Season}",
@ -312,7 +313,7 @@ public class FingerprinterTask : IScheduledTask
// If no valid ranges were found, re-analyze the episodes considering all possible shifts.
if (lhsRanges.Count == 0)
{
_logger.LogDebug("quick scan unsuccessful, falling back to full scan");
_logger.LogDebug("quick scan unsuccessful, falling back to full scan (±{Limit})", limit);
(lhsContiguous, rhsContiguous) = ShiftEpisodes(lhsPoints, rhsPoints, -1 * limit, limit);
lhsRanges.AddRange(lhsContiguous);