majomi/intro-skipper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Relocate Chromaprint analysis code

Browse Source
This commit is contained in:
ConfusedPolarBear 2022-10-28 02:25:57 -05:00
parent c5925ae176
commit ce52a0b979
6 changed files with 518 additions and 456 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs
View File

@ -26,7 +26,8 @@ public class TestAudioFingerprinting
[InlineData(19, 2_465_585_877)]
public void TestBitCounting(int expectedBits, uint number)
{
Assert.Equal(expectedBits, AnalyzeEpisodesTask.CountBits(number));
var chromaprint = CreateChromaprintAnalyzer();
Assert.Equal(expectedBits, chromaprint.CountBits(number));
}
[FactSkipFFmpegTests]
@ -86,14 +87,14 @@ public class TestAudioFingerprinting
[FactSkipFFmpegTests]
public void TestIntroDetection()
{
var task = new AnalyzeEpisodesTask(new LoggerFactory());
var chromaprint = CreateChromaprintAnalyzer();
var lhsEpisode = queueEpisode("audio/big_buck_bunny_intro.mp3");
var rhsEpisode = queueEpisode("audio/big_buck_bunny_clip.mp3");
var lhsFingerprint = FFmpegWrapper.Fingerprint(lhsEpisode);
var rhsFingerprint = FFmpegWrapper.Fingerprint(rhsEpisode);
var (lhs, rhs) = task.CompareEpisodes(
var (lhs, rhs) = chromaprint.CompareEpisodes(
lhsEpisode.EpisodeId,
lhsFingerprint,
rhsEpisode.EpisodeId,
@ -140,6 +141,12 @@ public class TestAudioFingerprinting
FingerprintDuration = 60
};
}
private ChromaprintAnalyzer CreateChromaprintAnalyzer()
{
var logger = new LoggerFactory().CreateLogger<ChromaprintAnalyzer>();
return new(logger);
}
}
public class FactSkipFFmpegTests : FactAttribute

445
ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/Chromaprint.cs Normal file
View File

@ -0,0 +1,445 @@
namespace ConfusedPolarBear.Plugin.IntroSkipper;
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Numerics;
using System.Threading;
using Microsoft.Extensions.Logging;
/// <summary>
/// Chromaprint audio analyzer.
/// </summary>
public class ChromaprintAnalyzer : IMediaFileAnalyzer
{
/// <summary>
/// Seconds of audio in one fingerprint point.
/// This value is defined by the Chromaprint library and should not be changed.
/// </summary>
private const double SamplesToSeconds = 0.128;
private int minimumIntroDuration;
private int maximumDifferences;
private int invertedIndexShift;
private double maximumTimeSkip;
private double silenceDetectionMinimumDuration;
private ILogger<ChromaprintAnalyzer> _logger;
/// <summary>
/// Initializes a new instance of the <see cref="ChromaprintAnalyzer"/> class.
/// </summary>
/// <param name="logger">Logger.</param>
public ChromaprintAnalyzer(ILogger<ChromaprintAnalyzer> logger)
{
var config = Plugin.Instance?.Configuration ?? new Configuration.PluginConfiguration();
maximumDifferences = config.MaximumFingerprintPointDifferences;
invertedIndexShift = config.InvertedIndexShift;
maximumTimeSkip = config.MaximumTimeSkip;
silenceDetectionMinimumDuration = config.SilenceDetectionMinimumDuration;
minimumIntroDuration = config.MinimumIntroDuration;
_logger = logger;
}
/// <inheritdoc />
public ReadOnlyCollection<QueuedEpisode> AnalyzeMediaFiles(
ReadOnlyCollection<QueuedEpisode> analysisQueue,
AnalysisMode mode,
CancellationToken cancellationToken)
{
// All intros for this season.
var seasonIntros = new Dictionary<Guid, Intro>();
// Cache of all fingerprints for this season.
var fingerprintCache = new Dictionary<Guid, uint[]>();
// Episode analysis queue.
var episodeAnalysisQueue = new List<QueuedEpisode>(analysisQueue);
// Episodes that were analyzed and do not have an introduction.
var episodesWithoutIntros = new List<QueuedEpisode>();
// Compute fingerprints for all episodes in the season
foreach (var episode in episodeAnalysisQueue)
{
try
{
fingerprintCache[episode.EpisodeId] = FFmpegWrapper.Fingerprint(episode);
if (cancellationToken.IsCancellationRequested)
{
return analysisQueue;
}
}
catch (FingerprintException ex)
{
_logger.LogWarning("Caught fingerprint error: {Ex}", ex);
// Fallback to an empty fingerprint on any error
fingerprintCache[episode.EpisodeId] = Array.Empty<uint>();
}
}
// While there are still episodes in the queue
while (episodeAnalysisQueue.Count > 0)
{
// Pop the first episode from the queue
var currentEpisode = episodeAnalysisQueue[0];
episodeAnalysisQueue.RemoveAt(0);
// Search through all remaining episodes.
foreach (var remainingEpisode in episodeAnalysisQueue)
{
// Compare the current episode to all remaining episodes in the queue.
var (currentIntro, remainingIntro) = CompareEpisodes(
currentEpisode.EpisodeId,
fingerprintCache[currentEpisode.EpisodeId],
remainingEpisode.EpisodeId,
fingerprintCache[remainingEpisode.EpisodeId]);
// Ignore this comparison result if:
// - one of the intros isn't valid, or
// - the introduction exceeds the configured limit
if (
!remainingIntro.Valid ||
remainingIntro.Duration > Plugin.Instance!.Configuration.MaximumIntroDuration)
{
continue;
}
// Only save the discovered intro if it is:
// - the first intro discovered for this episode
// - longer than the previously discovered intro
if (
!seasonIntros.TryGetValue(currentIntro.EpisodeId, out var savedCurrentIntro) ||
currentIntro.Duration > savedCurrentIntro.Duration)
{
seasonIntros[currentIntro.EpisodeId] = currentIntro;
}
if (
!seasonIntros.TryGetValue(remainingIntro.EpisodeId, out var savedRemainingIntro) ||
remainingIntro.Duration > savedRemainingIntro.Duration)
{
seasonIntros[remainingIntro.EpisodeId] = remainingIntro;
}
break;
}
// If no intro is found at this point, the popped episode is not reinserted into the queue.
episodesWithoutIntros.Add(currentEpisode);
}
// If cancellation was requested, report that no episodes were analyzed.
if (cancellationToken.IsCancellationRequested)
{
return analysisQueue;
}
// Adjust all introduction end times so that they end at silence.
seasonIntros = AdjustIntroEndTimes(analysisQueue, seasonIntros);
Plugin.Instance!.UpdateTimestamps(seasonIntros);
return episodesWithoutIntros.AsReadOnly();
}
/// <summary>
/// Analyze two episodes to find an introduction sequence shared between them.
/// </summary>
/// <param name="lhsId">First episode id.</param>
/// <param name="lhsPoints">First episode fingerprint points.</param>
/// <param name="rhsId">Second episode id.</param>
/// <param name="rhsPoints">Second episode fingerprint points.</param>
/// <returns>Intros for the first and second episodes.</returns>
public (Intro Lhs, Intro Rhs) CompareEpisodes(
Guid lhsId,
uint[] lhsPoints,
Guid rhsId,
uint[] rhsPoints)
{
// Creates an inverted fingerprint point index for both episodes.
// For every point which is a 100% match, search for an introduction at that point.
var (lhsRanges, rhsRanges) = SearchInvertedIndex(lhsId, lhsPoints, rhsId, rhsPoints);
if (lhsRanges.Count > 0)
{
_logger.LogTrace("Index search successful");
return GetLongestTimeRange(lhsId, lhsRanges, rhsId, rhsRanges);
}
_logger.LogTrace(
"Unable to find a shared introduction sequence between {LHS} and {RHS}",
lhsId,
rhsId);
return (new Intro(lhsId), new Intro(rhsId));
}
/// <summary>
/// Locates the longest range of similar audio and returns an Intro class for each range.
/// </summary>
/// <param name="lhsId">First episode id.</param>
/// <param name="lhsRanges">First episode shared timecodes.</param>
/// <param name="rhsId">Second episode id.</param>
/// <param name="rhsRanges">Second episode shared timecodes.</param>
/// <returns>Intros for the first and second episodes.</returns>
private (Intro Lhs, Intro Rhs) GetLongestTimeRange(
Guid lhsId,
List<TimeRange> lhsRanges,
Guid rhsId,
List<TimeRange> rhsRanges)
{
// Store the longest time range as the introduction.
lhsRanges.Sort();
rhsRanges.Sort();
var lhsIntro = lhsRanges[0];
var rhsIntro = rhsRanges[0];
// If the intro starts early in the episode, move it to the beginning.
if (lhsIntro.Start <= 5)
{
lhsIntro.Start = 0;
}
if (rhsIntro.Start <= 5)
{
rhsIntro.Start = 0;
}
// Create Intro classes for each time range.
return (new Intro(lhsId, lhsIntro), new Intro(rhsId, rhsIntro));
}
/// <summary>
/// Search for a shared introduction sequence using inverted indexes.
/// </summary>
/// <param name="lhsId">LHS ID.</param>
/// <param name="lhsPoints">Left episode fingerprint points.</param>
/// <param name="rhsId">RHS ID.</param>
/// <param name="rhsPoints">Right episode fingerprint points.</param>
/// <returns>List of shared TimeRanges between the left and right episodes.</returns>
private (List<TimeRange> Lhs, List<TimeRange> Rhs) SearchInvertedIndex(
Guid lhsId,
uint[] lhsPoints,
Guid rhsId,
uint[] rhsPoints)
{
var lhsRanges = new List<TimeRange>();
var rhsRanges = new List<TimeRange>();
// Generate inverted indexes for the left and right episodes.
var lhsIndex = FFmpegWrapper.CreateInvertedIndex(lhsId, lhsPoints);
var rhsIndex = FFmpegWrapper.CreateInvertedIndex(rhsId, rhsPoints);
var indexShifts = new HashSet<int>();
// For all audio points in the left episode, check if the right episode has a point which matches exactly.
// If an exact match is found, calculate the shift that must be used to align the points.
foreach (var kvp in lhsIndex)
{
var originalPoint = kvp.Key;
for (var i = -1 * invertedIndexShift; i <= invertedIndexShift; i++)
{
var modifiedPoint = (uint)(originalPoint + i);
if (rhsIndex.ContainsKey(modifiedPoint))
{
var lhsFirst = (int)lhsIndex[originalPoint];
var rhsFirst = (int)rhsIndex[modifiedPoint];
indexShifts.Add(rhsFirst - lhsFirst);
}
}
}
// Use all discovered shifts to compare the episodes.
foreach (var shift in indexShifts)
{
var (lhsIndexContiguous, rhsIndexContiguous) = FindContiguous(lhsPoints, rhsPoints, shift);
if (lhsIndexContiguous.End > 0 && rhsIndexContiguous.End > 0)
{
lhsRanges.Add(lhsIndexContiguous);
rhsRanges.Add(rhsIndexContiguous);
}
}
return (lhsRanges, rhsRanges);
}
/// <summary>
/// Finds the longest contiguous region of similar audio between two fingerprints using the provided shift amount.
/// </summary>
/// <param name="lhs">First fingerprint to compare.</param>
/// <param name="rhs">Second fingerprint to compare.</param>
/// <param name="shiftAmount">Amount to shift one fingerprint by.</param>
private (TimeRange Lhs, TimeRange Rhs) FindContiguous(
uint[] lhs,
uint[] rhs,
int shiftAmount)
{
var leftOffset = 0;
var rightOffset = 0;
// Calculate the offsets for the left and right hand sides.
if (shiftAmount < 0)
{
leftOffset -= shiftAmount;
}
else
{
rightOffset += shiftAmount;
}
// Store similar times for both LHS and RHS.
var lhsTimes = new List<double>();
var rhsTimes = new List<double>();
var upperLimit = Math.Min(lhs.Length, rhs.Length) - Math.Abs(shiftAmount);
// XOR all elements in LHS and RHS, using the shift amount from above.
for (var i = 0; i < upperLimit; i++)
{
// XOR both samples at the current position.
var lhsPosition = i + leftOffset;
var rhsPosition = i + rightOffset;
var diff = lhs[lhsPosition] ^ rhs[rhsPosition];
// If the difference between the samples is small, flag both times as similar.
if (CountBits(diff) > maximumDifferences)
{
continue;
}
var lhsTime = lhsPosition * SamplesToSeconds;
var rhsTime = rhsPosition * SamplesToSeconds;
lhsTimes.Add(lhsTime);
rhsTimes.Add(rhsTime);
}
// Ensure the last timestamp is checked
lhsTimes.Add(double.MaxValue);
rhsTimes.Add(double.MaxValue);
// Now that both fingerprints have been compared at this shift, see if there's a contiguous time range.
var lContiguous = TimeRangeHelpers.FindContiguous(lhsTimes.ToArray(), maximumTimeSkip);
if (lContiguous is null || lContiguous.Duration < minimumIntroDuration)
{
return (new TimeRange(), new TimeRange());
}
// Since LHS had a contiguous time range, RHS must have one also.
var rContiguous = TimeRangeHelpers.FindContiguous(rhsTimes.ToArray(), maximumTimeSkip)!;
// Tweak the end timestamps just a bit to ensure as little content as possible is skipped over.
if (lContiguous.Duration >= 90)
{
lContiguous.End -= 2 * maximumTimeSkip;
rContiguous.End -= 2 * maximumTimeSkip;
}
else if (lContiguous.Duration >= 30)
{
lContiguous.End -= maximumTimeSkip;
rContiguous.End -= maximumTimeSkip;
}
return (lContiguous, rContiguous);
}
/// <summary>
/// Adjusts the end timestamps of all intros so that they end at silence.
/// </summary>
/// <param name="episodes">QueuedEpisodes to adjust.</param>
/// <param name="originalIntros">Original introductions.</param>
private Dictionary<Guid, Intro> AdjustIntroEndTimes(
ReadOnlyCollection<QueuedEpisode> episodes,
Dictionary<Guid, Intro> originalIntros)
{
// The minimum duration of audio that must be silent before adjusting the intro's end.
var minimumSilence = Plugin.Instance!.Configuration.SilenceDetectionMinimumDuration;
Dictionary<Guid, Intro> modifiedIntros = new();
// For all episodes
foreach (var episode in episodes)
{
_logger.LogTrace(
"Adjusting introduction end time for {Name} ({Id})",
episode.Name,
episode.EpisodeId);
// If no intro was found for this episode, skip it.
if (!originalIntros.TryGetValue(episode.EpisodeId, out var originalIntro))
{
_logger.LogTrace("{Name} does not have an intro", episode.Name);
continue;
}
// Only adjust the end timestamp of the intro
var originalIntroEnd = new TimeRange(originalIntro.IntroEnd - 15, originalIntro.IntroEnd);
_logger.LogTrace(
"{Name} original intro: {Start} - {End}",
episode.Name,
originalIntro.IntroStart,
originalIntro.IntroEnd);
// Detect silence in the media file up to the end of the intro.
var silence = FFmpegWrapper.DetectSilence(episode, (int)originalIntro.IntroEnd + 2);
// For all periods of silence
foreach (var currentRange in silence)
{
_logger.LogTrace(
"{Name} silence: {Start} - {End}",
episode.Name,
currentRange.Start,
currentRange.End);
// Ignore any silence that:
// * doesn't intersect the ending of the intro, or
// * is shorter than the user defined minimum duration, or
// * starts before the introduction does
if (
!originalIntroEnd.Intersects(currentRange) ||
currentRange.Duration < silenceDetectionMinimumDuration ||
currentRange.Start < originalIntro.IntroStart)
{
continue;
}
// Adjust the end timestamp of the intro to match the start of the silence region.
originalIntro.IntroEnd = currentRange.Start;
break;
}
_logger.LogTrace(
"{Name} adjusted intro: {Start} - {End}",
episode.Name,
originalIntro.IntroStart,
originalIntro.IntroEnd);
// Add the (potentially) modified intro back.
modifiedIntros[episode.EpisodeId] = originalIntro;
}
return modifiedIntros;
}
/// <summary>
/// Count the number of bits that are set in the provided number.
/// </summary>
/// <param name="number">Number to count bits in.</param>
/// <returns>Number of bits that are equal to 1.</returns>
public int CountBits(uint number)
{
return BitOperations.PopCount(number);
}
}

22
ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/IMediaFileAnalyzer.cs Normal file
View File

@ -0,0 +1,22 @@
namespace ConfusedPolarBear.Plugin.IntroSkipper;
using System.Collections.ObjectModel;
using System.Threading;
/// <summary>
/// Media file analyzer interface.
/// </summary>
public interface IMediaFileAnalyzer
{
/// <summary>
/// Analyze media files for shared introductions or credits, returning all media files that were **not successfully analyzed**.
/// </summary>
/// <param name="analysisQueue">Collection of unanalyzed media files.</param>
/// <param name="mode">Analysis mode.</param>
/// <param name="cancellationToken">Cancellation token from scheduled task.</param>
/// <returns>Collection of media files that were **unsuccessfully analyzed**.</returns>
public ReadOnlyCollection<QueuedEpisode> AnalyzeMediaFiles(
ReadOnlyCollection<QueuedEpisode> analysisQueue,
AnalysisMode mode,
CancellationToken cancellationToken);
}

17
ConfusedPolarBear.Plugin.IntroSkipper/Data/AnalysisMode.cs Normal file
View File

@ -0,0 +1,17 @@
namespace ConfusedPolarBear.Plugin.IntroSkipper;
/// <summary>
/// Type of media file analysis to perform.
/// </summary>
public enum AnalysisMode
{
/// <summary>
/// Detect introduction sequences.
/// </summary>
Introduction,
/// <summary>
/// Detect credits.
/// </summary>
Credits,
}

16
ConfusedPolarBear.Plugin.IntroSkipper/Plugin.cs
View File

@ -18,7 +18,8 @@ namespace ConfusedPolarBear.Plugin.IntroSkipper;
/// </summary>
public class Plugin : BasePlugin<PluginConfiguration>, IHasWebPages
{
private readonly object _serializationLock = new object();
private readonly object _serializationLock = new();
private readonly object _introsLock = new();
private IXmlSerializer _xmlSerializer;
private ILibraryManager _libraryManager;
private ILogger<Plugin> _logger;
@ -162,6 +163,19 @@ public class Plugin : BasePlugin<PluginConfiguration>, IHasWebPages
return GetItem(id).Path;
}
internal void UpdateTimestamps(Dictionary<Guid, Intro> newIntros)
{
lock (_introsLock)
{
foreach (var intro in newIntros)
{
Plugin.Instance!.Intros[intro.Key] = intro.Value;
}
Plugin.Instance!.SaveTimestamps();
}
}
/// <inheritdoc />
public IEnumerable<PluginPageInfo> GetPages()
{

461
ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs
View File

@ -2,7 +2,6 @@ using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.IO;
using System.Numerics;
using System.Threading;
using System.Threading.Tasks;
using MediaBrowser.Controller.Library;
@ -16,36 +15,12 @@ namespace ConfusedPolarBear.Plugin.IntroSkipper;
/// </summary>
public class AnalyzeEpisodesTask : IScheduledTask
{
/// <summary>
/// Seconds of audio in one fingerprint point.
/// This value is defined by the Chromaprint library and should not be changed.
/// </summary>
private const double SamplesToSeconds = 0.128;
private readonly ILogger<AnalyzeEpisodesTask> _logger;
private readonly ILogger<QueueManager> _queueLogger;
private readonly ILoggerFactory _loggerFactory;
private readonly ILibraryManager? _libraryManager;
/// <summary>
/// Lock which guards the shared dictionary of intros.
/// </summary>
private readonly object _introsLock = new object();
/// <summary>
/// Minimum duration of similar audio that will be considered an introduction.
/// </summary>
private static int minimumIntroDuration = 15;
private static int maximumDifferences = 6;
private static int invertedIndexShift = 2;
private static double maximumTimeSkip = 3.5;
private static double silenceDetectionMinimumDuration = 0.33;
/// <summary>
/// Initializes a new instance of the <see cref="AnalyzeEpisodesTask"/> class.
/// </summary>
@ -65,7 +40,7 @@ public class AnalyzeEpisodesTask : IScheduledTask
public AnalyzeEpisodesTask(ILoggerFactory loggerFactory)
{
_logger = loggerFactory.CreateLogger<AnalyzeEpisodesTask>();
_queueLogger = loggerFactory.CreateLogger<QueueManager>();
_loggerFactory = loggerFactory;
EdlManager.Initialize(_logger);
}
@ -104,7 +79,10 @@ public class AnalyzeEpisodesTask : IScheduledTask
}
// Make sure the analysis queue matches what's currently in Jellyfin.
var queueManager = new QueueManager(_queueLogger, _libraryManager);
var queueManager = new QueueManager(
_loggerFactory.CreateLogger<QueueManager>(),
_libraryManager);
queueManager.EnqueueAllEpisodes();
var queue = Plugin.Instance!.AnalysisQueue;
@ -115,13 +93,6 @@ public class AnalyzeEpisodesTask : IScheduledTask
"No episodes to analyze. If you are limiting the list of libraries to analyze, check that all library names have been spelled correctly.");
}
// Load analysis settings from configuration
var config = Plugin.Instance?.Configuration ?? new Configuration.PluginConfiguration();
maximumDifferences = config.MaximumFingerprintPointDifferences;
invertedIndexShift = config.InvertedIndexShift;
maximumTimeSkip = config.MaximumTimeSkip;
silenceDetectionMinimumDuration = config.SilenceDetectionMinimumDuration;
// Log EDL settings
EdlManager.LogConfiguration();
@ -131,8 +102,6 @@ public class AnalyzeEpisodesTask : IScheduledTask
MaxDegreeOfParallelism = Plugin.Instance!.Configuration.MaxParallelism
};
minimumIntroDuration = Plugin.Instance!.Configuration.MinimumIntroDuration;
// TODO: if the queue is modified while the task is running, the task will fail.
// clone the queue before running the task to prevent this.
@ -251,30 +220,6 @@ public class AnalyzeEpisodesTask : IScheduledTask
return (verified.AsReadOnly(), unanalyzed);
}
/// <summary>
/// Count the number of previously processed episodes to ensure the reported progress is correct.
/// </summary>
/// <returns>Number of previously processed episodes.</returns>
private int CountProcessedEpisodes()
{
var previous = 0;
foreach (var season in Plugin.Instance!.AnalysisQueue)
{
foreach (var episode in season.Value)
{
if (!Plugin.Instance!.Intros.TryGetValue(episode.EpisodeId, out var intro) || !intro.Valid)
{
continue;
}
previous++;
}
}
return previous;
}
/// <summary>
/// Fingerprints all episodes in the provided season and stores the timestamps of all introductions.
/// </summary>
@ -285,15 +230,6 @@ public class AnalyzeEpisodesTask : IScheduledTask
ReadOnlyCollection<QueuedEpisode> episodes,
CancellationToken cancellationToken)
{
// All intros for this season.
var seasonIntros = new Dictionary<Guid, Intro>();
// Cache of all fingerprints for this season.
var fingerprintCache = new Dictionary<Guid, uint[]>();
// Episode analysis queue.
var episodeAnalysisQueue = new List<QueuedEpisode>(episodes);
// Skip seasons with an insufficient number of episodes.
if (episodes.Count <= 1)
{
@ -313,392 +249,13 @@ public class AnalyzeEpisodesTask : IScheduledTask
first.SeriesName,
first.SeasonNumber);
// Compute fingerprints for all episodes in the season
foreach (var episode in episodeAnalysisQueue)
{
try
{
fingerprintCache[episode.EpisodeId] = FFmpegWrapper.Fingerprint(episode);
if (cancellationToken.IsCancellationRequested)
{
return episodes.Count;
}
}
catch (FingerprintException ex)
{
_logger.LogWarning("Caught fingerprint error: {Ex}", ex);
// Fallback to an empty fingerprint on any error
fingerprintCache[episode.EpisodeId] = Array.Empty<uint>();
}
}
// While there are still episodes in the queue
while (episodeAnalysisQueue.Count > 0)
{
// Pop the first episode from the queue
var currentEpisode = episodeAnalysisQueue[0];
episodeAnalysisQueue.RemoveAt(0);
// Search through all remaining episodes.
foreach (var remainingEpisode in episodeAnalysisQueue)
{
// Compare the current episode to all remaining episodes in the queue.
var (currentIntro, remainingIntro) = CompareEpisodes(
currentEpisode.EpisodeId,
fingerprintCache[currentEpisode.EpisodeId],
remainingEpisode.EpisodeId,
fingerprintCache[remainingEpisode.EpisodeId]);
// Ignore this comparison result if:
// - one of the intros isn't valid, or
// - the introduction exceeds the configured limit
if (
!remainingIntro.Valid ||
remainingIntro.Duration > Plugin.Instance!.Configuration.MaximumIntroDuration)
{
continue;
}
// Only save the discovered intro if it is:
// - the first intro discovered for this episode
// - longer than the previously discovered intro
if (
!seasonIntros.TryGetValue(currentIntro.EpisodeId, out var savedCurrentIntro) ||
currentIntro.Duration > savedCurrentIntro.Duration)
{
seasonIntros[currentIntro.EpisodeId] = currentIntro;
}
if (
!seasonIntros.TryGetValue(remainingIntro.EpisodeId, out var savedRemainingIntro) ||
remainingIntro.Duration > savedRemainingIntro.Duration)
{
seasonIntros[remainingIntro.EpisodeId] = remainingIntro;
}
break;
}
// If no intro is found at this point, the popped episode is not reinserted into the queue.
}
if (cancellationToken.IsCancellationRequested)
{
return episodes.Count;
}
// Adjust all introduction end times so that they end at silence.
seasonIntros = AdjustIntroEndTimes(episodes, seasonIntros);
// Ensure only one thread at a time can update the shared intro dictionary.
lock (_introsLock)
{
foreach (var intro in seasonIntros)
{
Plugin.Instance!.Intros[intro.Key] = intro.Value;
}
Plugin.Instance!.SaveTimestamps();
}
// Analyze the season with Chromaprint
var chromaprint = new ChromaprintAnalyzer(_loggerFactory.CreateLogger<ChromaprintAnalyzer>());
chromaprint.AnalyzeMediaFiles(episodes, AnalysisMode.Introduction, cancellationToken);
return episodes.Count;
}
/// <summary>
/// Analyze two episodes to find an introduction sequence shared between them.
/// </summary>
/// <param name="lhsId">First episode id.</param>
/// <param name="lhsPoints">First episode fingerprint points.</param>
/// <param name="rhsId">Second episode id.</param>
/// <param name="rhsPoints">Second episode fingerprint points.</param>
/// <returns>Intros for the first and second episodes.</returns>
public (Intro Lhs, Intro Rhs) CompareEpisodes(
Guid lhsId,
uint[] lhsPoints,
Guid rhsId,
uint[] rhsPoints)
{
// Creates an inverted fingerprint point index for both episodes.
// For every point which is a 100% match, search for an introduction at that point.
var (lhsRanges, rhsRanges) = SearchInvertedIndex(lhsId, lhsPoints, rhsId, rhsPoints);
if (lhsRanges.Count > 0)
{
_logger.LogTrace("Index search successful");
return GetLongestTimeRange(lhsId, lhsRanges, rhsId, rhsRanges);
}
_logger.LogTrace(
"Unable to find a shared introduction sequence between {LHS} and {RHS}",
lhsId,
rhsId);
return (new Intro(lhsId), new Intro(rhsId));
}
/// <summary>
/// Locates the longest range of similar audio and returns an Intro class for each range.
/// </summary>
/// <param name="lhsId">First episode id.</param>
/// <param name="lhsRanges">First episode shared timecodes.</param>
/// <param name="rhsId">Second episode id.</param>
/// <param name="rhsRanges">Second episode shared timecodes.</param>
/// <returns>Intros for the first and second episodes.</returns>
private (Intro Lhs, Intro Rhs) GetLongestTimeRange(
Guid lhsId,
List<TimeRange> lhsRanges,
Guid rhsId,
List<TimeRange> rhsRanges)
{
// Store the longest time range as the introduction.
lhsRanges.Sort();
rhsRanges.Sort();
var lhsIntro = lhsRanges[0];
var rhsIntro = rhsRanges[0];
// If the intro starts early in the episode, move it to the beginning.
if (lhsIntro.Start <= 5)
{
lhsIntro.Start = 0;
}
if (rhsIntro.Start <= 5)
{
rhsIntro.Start = 0;
}
// Create Intro classes for each time range.
return (new Intro(lhsId, lhsIntro), new Intro(rhsId, rhsIntro));
}
/// <summary>
/// Search for a shared introduction sequence using inverted indexes.
/// </summary>
/// <param name="lhsId">LHS ID.</param>
/// <param name="lhsPoints">Left episode fingerprint points.</param>
/// <param name="rhsId">RHS ID.</param>
/// <param name="rhsPoints">Right episode fingerprint points.</param>
/// <returns>List of shared TimeRanges between the left and right episodes.</returns>
private (List<TimeRange> Lhs, List<TimeRange> Rhs) SearchInvertedIndex(
Guid lhsId,
uint[] lhsPoints,
Guid rhsId,
uint[] rhsPoints)
{
var lhsRanges = new List<TimeRange>();
var rhsRanges = new List<TimeRange>();
// Generate inverted indexes for the left and right episodes.
var lhsIndex = FFmpegWrapper.CreateInvertedIndex(lhsId, lhsPoints);
var rhsIndex = FFmpegWrapper.CreateInvertedIndex(rhsId, rhsPoints);
var indexShifts = new HashSet<int>();
// For all audio points in the left episode, check if the right episode has a point which matches exactly.
// If an exact match is found, calculate the shift that must be used to align the points.
foreach (var kvp in lhsIndex)
{
var originalPoint = kvp.Key;
for (var i = -1 * invertedIndexShift; i <= invertedIndexShift; i++)
{
var modifiedPoint = (uint)(originalPoint + i);
if (rhsIndex.ContainsKey(modifiedPoint))
{
var lhsFirst = (int)lhsIndex[originalPoint];
var rhsFirst = (int)rhsIndex[modifiedPoint];
indexShifts.Add(rhsFirst - lhsFirst);
}
}
}
// Use all discovered shifts to compare the episodes.
foreach (var shift in indexShifts)
{
var (lhsIndexContiguous, rhsIndexContiguous) = FindContiguous(lhsPoints, rhsPoints, shift);
if (lhsIndexContiguous.End > 0 && rhsIndexContiguous.End > 0)
{
lhsRanges.Add(lhsIndexContiguous);
rhsRanges.Add(rhsIndexContiguous);
}
}
return (lhsRanges, rhsRanges);
}
/// <summary>
/// Finds the longest contiguous region of similar audio between two fingerprints using the provided shift amount.
/// </summary>
/// <param name="lhs">First fingerprint to compare.</param>
/// <param name="rhs">Second fingerprint to compare.</param>
/// <param name="shiftAmount">Amount to shift one fingerprint by.</param>
private static (TimeRange Lhs, TimeRange Rhs) FindContiguous(
uint[] lhs,
uint[] rhs,
int shiftAmount)
{
var leftOffset = 0;
var rightOffset = 0;
// Calculate the offsets for the left and right hand sides.
if (shiftAmount < 0)
{
leftOffset -= shiftAmount;
}
else
{
rightOffset += shiftAmount;
}
// Store similar times for both LHS and RHS.
var lhsTimes = new List<double>();
var rhsTimes = new List<double>();
var upperLimit = Math.Min(lhs.Length, rhs.Length) - Math.Abs(shiftAmount);
// XOR all elements in LHS and RHS, using the shift amount from above.
for (var i = 0; i < upperLimit; i++)
{
// XOR both samples at the current position.
var lhsPosition = i + leftOffset;
var rhsPosition = i + rightOffset;
var diff = lhs[lhsPosition] ^ rhs[rhsPosition];
// If the difference between the samples is small, flag both times as similar.
if (CountBits(diff) > maximumDifferences)
{
continue;
}
var lhsTime = lhsPosition * SamplesToSeconds;
var rhsTime = rhsPosition * SamplesToSeconds;
lhsTimes.Add(lhsTime);
rhsTimes.Add(rhsTime);
}
// Ensure the last timestamp is checked
lhsTimes.Add(double.MaxValue);
rhsTimes.Add(double.MaxValue);
// Now that both fingerprints have been compared at this shift, see if there's a contiguous time range.
var lContiguous = TimeRangeHelpers.FindContiguous(lhsTimes.ToArray(), maximumTimeSkip);
if (lContiguous is null || lContiguous.Duration < minimumIntroDuration)
{
return (new TimeRange(), new TimeRange());
}
// Since LHS had a contiguous time range, RHS must have one also.
var rContiguous = TimeRangeHelpers.FindContiguous(rhsTimes.ToArray(), maximumTimeSkip)!;
// Tweak the end timestamps just a bit to ensure as little content as possible is skipped over.
if (lContiguous.Duration >= 90)
{
lContiguous.End -= 2 * maximumTimeSkip;
rContiguous.End -= 2 * maximumTimeSkip;
}
else if (lContiguous.Duration >= 30)
{
lContiguous.End -= maximumTimeSkip;
rContiguous.End -= maximumTimeSkip;
}
return (lContiguous, rContiguous);
}
/// <summary>
/// Adjusts the end timestamps of all intros so that they end at silence.
/// </summary>
/// <param name="episodes">QueuedEpisodes to adjust.</param>
/// <param name="originalIntros">Original introductions.</param>
private Dictionary<Guid, Intro> AdjustIntroEndTimes(
ReadOnlyCollection<QueuedEpisode> episodes,
Dictionary<Guid, Intro> originalIntros)
{
// The minimum duration of audio that must be silent before adjusting the intro's end.
var minimumSilence = Plugin.Instance!.Configuration.SilenceDetectionMinimumDuration;
Dictionary<Guid, Intro> modifiedIntros = new();
// For all episodes
foreach (var episode in episodes)
{
_logger.LogTrace(
"Adjusting introduction end time for {Name} ({Id})",
episode.Name,
episode.EpisodeId);
// If no intro was found for this episode, skip it.
if (!originalIntros.TryGetValue(episode.EpisodeId, out var originalIntro))
{
_logger.LogTrace("{Name} does not have an intro", episode.Name);
continue;
}
// Only adjust the end timestamp of the intro
var originalIntroEnd = new TimeRange(originalIntro.IntroEnd - 15, originalIntro.IntroEnd);
_logger.LogTrace(
"{Name} original intro: {Start} - {End}",
episode.Name,
originalIntro.IntroStart,
originalIntro.IntroEnd);
// Detect silence in the media file up to the end of the intro.
var silence = FFmpegWrapper.DetectSilence(episode, (int)originalIntro.IntroEnd + 2);
// For all periods of silence
foreach (var currentRange in silence)
{
_logger.LogTrace(
"{Name} silence: {Start} - {End}",
episode.Name,
currentRange.Start,
currentRange.End);
// Ignore any silence that:
// * doesn't intersect the ending of the intro, or
// * is shorter than the user defined minimum duration, or
// * starts before the introduction does
if (
!originalIntroEnd.Intersects(currentRange) ||
currentRange.Duration < silenceDetectionMinimumDuration ||
currentRange.Start < originalIntro.IntroStart)
{
continue;
}
// Adjust the end timestamp of the intro to match the start of the silence region.
originalIntro.IntroEnd = currentRange.Start;
break;
}
_logger.LogTrace(
"{Name} adjusted intro: {Start} - {End}",
episode.Name,
originalIntro.IntroStart,
originalIntro.IntroEnd);
// Add the (potentially) modified intro back.
modifiedIntros[episode.EpisodeId] = originalIntro;
}
return modifiedIntros;
}
/// <summary>
/// Count the number of bits that are set in the provided number.
/// </summary>
/// <param name="number">Number to count bits in.</param>
/// <returns>Number of bits that are equal to 1.</returns>
public static int CountBits(uint number)
{
return BitOperations.PopCount(number);
}
/// <summary>
/// Get task triggers.
/// </summary>