using System; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Numerics; using System.Threading; using System.Threading.Tasks; using MediaBrowser.Controller.Library; using MediaBrowser.Model.Tasks; using Microsoft.Extensions.Logging; namespace ConfusedPolarBear.Plugin.IntroSkipper; /// /// Analyze all television episodes for introduction sequences. /// public class AnalyzeEpisodesTask : IScheduledTask { /// /// Maximum number of bits (out of 32 total) that can be different between segments before they are considered dissimilar. /// 6 bits means the audio must be at least 81% similar (1 - 6 / 32). /// private const double MaximumDifferences = 6; /// /// Maximum time (in seconds) permitted between timestamps before they are considered non-contiguous. /// private const double MaximumDistance = 3.5; /// /// Amount to shift inverted index offsets by. /// private const int InvertedIndexShift = 2; /// /// Seconds of audio in one fingerprint point. This value is defined by the Chromaprint library and should not be changed. /// private const double SamplesToSeconds = 0.128; private readonly ILogger _logger; private readonly ILogger _queueLogger; private readonly ILibraryManager? _libraryManager; /// /// Lock which guards the shared dictionary of intros. /// private readonly object _introsLock = new object(); /// /// Statistics for the currently running analysis task. /// private AnalysisStatistics analysisStatistics = new AnalysisStatistics(); /// /// Minimum duration of similar audio that will be considered an introduction. /// private static int minimumIntroDuration = 15; /// /// Initializes a new instance of the class. /// /// Logger factory. /// Library manager. public AnalyzeEpisodesTask(ILoggerFactory loggerFactory, ILibraryManager libraryManager) : this(loggerFactory) { _libraryManager = libraryManager; } /// /// Initializes a new instance of the class. /// /// Logger factory. public AnalyzeEpisodesTask(ILoggerFactory loggerFactory) { _logger = loggerFactory.CreateLogger(); _queueLogger = loggerFactory.CreateLogger(); EdlManager.Initialize(_logger); } /// /// Gets the task name. /// public string Name => "Detect Introductions"; /// /// Gets the task category. /// public string Category => "Intro Skipper"; /// /// Gets the task description. /// public string Description => "Analyzes the audio of all television episodes to find introduction sequences."; /// /// Gets the task key. /// public string Key => "CPBIntroSkipperDetectIntroductions"; /// /// Analyze all episodes in the queue. Only one instance of this task should be run at a time. /// /// Task progress. /// Cancellation token. /// Task. public Task ExecuteAsync(IProgress progress, CancellationToken cancellationToken) { if (_libraryManager is null) { throw new InvalidOperationException("Library manager must not be null"); } // Make sure the analysis queue matches what's currently in Jellyfin. var queueManager = new QueueManager(_queueLogger, _libraryManager); queueManager.EnqueueAllEpisodes(); var queue = Plugin.Instance!.AnalysisQueue; if (queue.Count == 0) { throw new FingerprintException( "No episodes to analyze. If you are limiting the list of libraries to analyze, check that all library names have been spelled correctly."); } // Log EDL settings EdlManager.LogConfiguration(); // Include the previously processed episodes in the percentage reported to the UI. var totalProcessed = CountProcessedEpisodes(); var options = new ParallelOptions() { MaxDegreeOfParallelism = Plugin.Instance!.Configuration.MaxParallelism }; var taskStart = DateTime.Now; analysisStatistics = new AnalysisStatistics(); analysisStatistics.TotalQueuedEpisodes = Plugin.Instance!.TotalQueued; minimumIntroDuration = Plugin.Instance!.Configuration.MinimumIntroDuration; // TODO: if the queue is modified while the task is running, the task will fail. // clone the queue before running the task to prevent this. // Analyze all episodes in the queue using the degrees of parallelism the user specified. Parallel.ForEach(queue, options, (season) => { var workerStart = DateTime.Now; var first = season.Value[0]; var writeEdl = false; try { var episodes = new Collection(season.Value); // Increment totalProcessed by the number of episodes in this season that were actually analyzed // (instead of just using the number of episodes in the current season). var analyzed = AnalyzeSeason(episodes, cancellationToken); Interlocked.Add(ref totalProcessed, analyzed); writeEdl = analyzed > 0 || Plugin.Instance!.Configuration.RegenerateEdlFiles; } catch (FingerprintException ex) { _logger.LogWarning( "Unable to analyze {Series} season {Season}: unable to fingerprint: {Ex}", first.SeriesName, first.SeasonNumber, ex); } catch (KeyNotFoundException ex) { _logger.LogWarning( "Unable to analyze {Series} season {Season}: cache miss: {Ex}", first.SeriesName, first.SeasonNumber, ex); } if (writeEdl && Plugin.Instance!.Configuration.EdlAction != EdlAction.None) { EdlManager.UpdateEDLFiles(season.Value.AsReadOnly()); } progress.Report((totalProcessed * 100) / Plugin.Instance!.TotalQueued); analysisStatistics.TotalCPUTime.AddDuration(workerStart); Plugin.Instance!.AnalysisStatistics = analysisStatistics; }); // Update analysis statistics analysisStatistics.TotalTaskTime.AddDuration(taskStart); Plugin.Instance!.AnalysisStatistics = analysisStatistics; // Turn the regenerate EDL flag off after the scan completes. if (Plugin.Instance!.Configuration.RegenerateEdlFiles) { _logger.LogInformation("Turning EDL file regeneration flag off"); Plugin.Instance!.Configuration.RegenerateEdlFiles = false; Plugin.Instance!.SaveConfiguration(); } return Task.CompletedTask; } /// /// Count the number of previously processed episodes to ensure the reported progress is correct. /// /// Number of previously processed episodes. private int CountProcessedEpisodes() { var previous = 0; foreach (var season in Plugin.Instance!.AnalysisQueue) { foreach (var episode in season.Value) { if (!Plugin.Instance!.Intros.TryGetValue(episode.EpisodeId, out var intro) || !intro.Valid) { continue; } previous++; } } return previous; } /// /// Fingerprints all episodes in the provided season and stores the timestamps of all introductions. /// /// Episodes in this season. /// Cancellation token provided by the scheduled task. /// Number of episodes from the provided season that were analyzed. private int AnalyzeSeason( Collection episodes, CancellationToken cancellationToken) { // All intros for this season. var seasonIntros = new Dictionary(); // Cache of all fingerprints for this season. var fingerprintCache = new Dictionary(); // Original episode queue. var originalEpisodes = new List(episodes); /* Don't analyze specials or seasons with an insufficient number of episodes. * A season with only 1 episode can't be analyzed as it would compare the episode to itself, * which would result in the entire episode being marked as an introduction, as the audio is identical. */ if (episodes.Count < 2 || episodes[0].SeasonNumber == 0) { return originalEpisodes.Count; } var first = episodes[0]; _logger.LogInformation( "Analyzing {Count} episodes from {Name} season {Season}", episodes.Count, first.SeriesName, first.SeasonNumber); // Compute fingerprints for all episodes in the season foreach (var episode in episodes) { try { fingerprintCache[episode.EpisodeId] = FFmpegWrapper.Fingerprint(episode); } catch (FingerprintException ex) { _logger.LogWarning("Caught fingerprint error: {Ex}", ex); // Fallback to an empty fingerprint on any error fingerprintCache[episode.EpisodeId] = Array.Empty(); } } // While there are still episodes in the queue while (episodes.Count > 0) { // Pop the first episode from the queue var currentEpisode = episodes[0]; episodes.RemoveAt(0); // Search through all remaining episodes. foreach (var remainingEpisode in episodes) { // Compare the current episode to all remaining episodes in the queue. var (currentIntro, remainingIntro) = CompareEpisodes( currentEpisode.EpisodeId, fingerprintCache[currentEpisode.EpisodeId], remainingEpisode.EpisodeId, fingerprintCache[remainingEpisode.EpisodeId]); // If one of the intros isn't valid, ignore this comparison result. if (!currentIntro.Valid) { continue; } // Only save the discovered intro if it is: // - the first intro discovered for this episode // - longer than the previously discovered intro if ( !seasonIntros.TryGetValue(currentIntro.EpisodeId, out var savedCurrentIntro) || currentIntro.Duration > savedCurrentIntro.Duration) { seasonIntros[currentIntro.EpisodeId] = currentIntro; } if ( !seasonIntros.TryGetValue(remainingIntro.EpisodeId, out var savedRemainingIntro) || remainingIntro.Duration > savedRemainingIntro.Duration) { seasonIntros[remainingIntro.EpisodeId] = remainingIntro; } break; } // If no intro is found at this point, the popped episode is not reinserted into the queue. } // Adjust all introduction end times so that they end at silence. seasonIntros = AdjustIntroEndTimes( new ReadOnlyCollection(originalEpisodes), seasonIntros); // Ensure only one thread at a time can update the shared intro dictionary. lock (_introsLock) { foreach (var intro in seasonIntros) { Plugin.Instance!.Intros[intro.Key] = intro.Value; } } lock (_introsLock) { Plugin.Instance!.SaveTimestamps(); } return originalEpisodes.Count; } /// /// Analyze two episodes to find an introduction sequence shared between them. /// /// First episode id. /// First episode fingerprint points. /// Second episode id. /// Second episode fingerprint points. /// Intros for the first and second episodes. public (Intro Lhs, Intro Rhs) CompareEpisodes( Guid lhsId, uint[] lhsPoints, Guid rhsId, uint[] rhsPoints) { var start = DateTime.Now; // Creates an inverted fingerprint point index for both episodes. // For every point which is a 100% match, search for an introduction at that point. var (lhsRanges, rhsRanges) = SearchInvertedIndex(lhsId, lhsPoints, rhsId, rhsPoints); if (lhsRanges.Count > 0) { _logger.LogTrace("Index search successful"); analysisStatistics.IndexSearches.Increment(); analysisStatistics.AnalysisCPUTime.AddDuration(start); return GetLongestTimeRange(lhsId, lhsRanges, rhsId, rhsRanges); } _logger.LogTrace( "Unable to find a shared introduction sequence between {LHS} and {RHS}", lhsId, rhsId); analysisStatistics.AnalysisCPUTime.AddDuration(start); return (new Intro(lhsId), new Intro(rhsId)); } /// /// Locates the longest range of similar audio and returns an Intro class for each range. /// /// First episode id. /// First episode shared timecodes. /// Second episode id. /// Second episode shared timecodes. /// Intros for the first and second episodes. private (Intro Lhs, Intro Rhs) GetLongestTimeRange( Guid lhsId, List lhsRanges, Guid rhsId, List rhsRanges) { // Store the longest time range as the introduction. lhsRanges.Sort(); rhsRanges.Sort(); var lhsIntro = lhsRanges[0]; var rhsIntro = rhsRanges[0]; // If the intro starts early in the episode, move it to the beginning. if (lhsIntro.Start <= 5) { lhsIntro.Start = 0; } if (rhsIntro.Start <= 5) { rhsIntro.Start = 0; } // Create Intro classes for each time range. return (new Intro(lhsId, lhsIntro), new Intro(rhsId, rhsIntro)); } /// /// Search for a shared introduction sequence using inverted indexes. /// /// LHS ID. /// Left episode fingerprint points. /// RHS ID. /// Right episode fingerprint points. /// List of shared TimeRanges between the left and right episodes. private (List Lhs, List Rhs) SearchInvertedIndex( Guid lhsId, uint[] lhsPoints, Guid rhsId, uint[] rhsPoints) { var lhsRanges = new List(); var rhsRanges = new List(); // Generate inverted indexes for the left and right episodes. var lhsIndex = FFmpegWrapper.CreateInvertedIndex(lhsId, lhsPoints); var rhsIndex = FFmpegWrapper.CreateInvertedIndex(rhsId, rhsPoints); var indexShifts = new HashSet(); // For all audio points in the left episode, check if the right episode has a point which matches exactly. // If an exact match is found, calculate the shift that must be used to align the points. foreach (var kvp in lhsIndex) { var originalPoint = kvp.Key; for (var i = -1 * InvertedIndexShift; i <= InvertedIndexShift; i++) { var modifiedPoint = (uint)(originalPoint + i); if (rhsIndex.ContainsKey(modifiedPoint)) { var lhsFirst = (int)lhsIndex[originalPoint]; var rhsFirst = (int)rhsIndex[modifiedPoint]; indexShifts.Add(rhsFirst - lhsFirst); } } } // Use all discovered shifts to compare the episodes. foreach (var shift in indexShifts) { var (lhsIndexContiguous, rhsIndexContiguous) = ShiftEpisodes(lhsPoints, rhsPoints, shift, shift); lhsRanges.AddRange(lhsIndexContiguous); rhsRanges.AddRange(rhsIndexContiguous); } return (lhsRanges, rhsRanges); } /// /// Shifts a pair of episodes through the range of provided shift amounts and returns discovered contiguous time ranges. /// /// First episode fingerprint. /// Second episode fingerprint. /// Lower end of the shift range. /// Upper end of the shift range. private static (List Lhs, List Rhs) ShiftEpisodes( uint[] lhs, uint[] rhs, int lower, int upper) { var lhsRanges = new List(); var rhsRanges = new List(); for (int amount = lower; amount <= upper; amount++) { var (lRange, rRange) = FindContiguous(lhs, rhs, amount); if (lRange.End == 0 && rRange.End == 0) { continue; } lhsRanges.Add(lRange); rhsRanges.Add(rRange); } return (lhsRanges, rhsRanges); } /// /// Finds the longest contiguous region of similar audio between two fingerprints using the provided shift amount. /// /// First fingerprint to compare. /// Second fingerprint to compare. /// Amount to shift one fingerprint by. private static (TimeRange Lhs, TimeRange Rhs) FindContiguous( uint[] lhs, uint[] rhs, int shiftAmount) { var leftOffset = 0; var rightOffset = 0; // Calculate the offsets for the left and right hand sides. if (shiftAmount < 0) { leftOffset -= shiftAmount; } else { rightOffset += shiftAmount; } // Store similar times for both LHS and RHS. var lhsTimes = new List(); var rhsTimes = new List(); var upperLimit = Math.Min(lhs.Length, rhs.Length) - Math.Abs(shiftAmount); // XOR all elements in LHS and RHS, using the shift amount from above. for (var i = 0; i < upperLimit; i++) { // XOR both samples at the current position. var lhsPosition = i + leftOffset; var rhsPosition = i + rightOffset; var diff = lhs[lhsPosition] ^ rhs[rhsPosition]; // If the difference between the samples is small, flag both times as similar. if (CountBits(diff) > MaximumDifferences) { continue; } var lhsTime = lhsPosition * SamplesToSeconds; var rhsTime = rhsPosition * SamplesToSeconds; lhsTimes.Add(lhsTime); rhsTimes.Add(rhsTime); } // Ensure the last timestamp is checked lhsTimes.Add(double.MaxValue); rhsTimes.Add(double.MaxValue); // Now that both fingerprints have been compared at this shift, see if there's a contiguous time range. var lContiguous = TimeRangeHelpers.FindContiguous(lhsTimes.ToArray(), MaximumDistance); if (lContiguous is null || lContiguous.Duration < minimumIntroDuration) { return (new TimeRange(), new TimeRange()); } // Since LHS had a contiguous time range, RHS must have one also. var rContiguous = TimeRangeHelpers.FindContiguous(rhsTimes.ToArray(), MaximumDistance)!; // Tweak the end timestamps just a bit to ensure as little content as possible is skipped over. if (lContiguous.Duration >= 90) { lContiguous.End -= 2 * MaximumDistance; rContiguous.End -= 2 * MaximumDistance; } else if (lContiguous.Duration >= 30) { lContiguous.End -= MaximumDistance; rContiguous.End -= MaximumDistance; } return (lContiguous, rContiguous); } /// /// Adjusts the end timestamps of all intros so that they end at silence. /// /// QueuedEpisodes to adjust. /// Original introductions. private Dictionary AdjustIntroEndTimes( ReadOnlyCollection episodes, Dictionary originalIntros) { // The minimum duration of audio that must be silent before adjusting the intro's end. var minimumSilence = Plugin.Instance!.Configuration.SilenceDetectionMinimumDuration; Dictionary modifiedIntros = new(); // For all episodes foreach (var episode in episodes) { _logger.LogTrace( "Adjusting introduction end time for {Name} ({Id})", episode.Name, episode.EpisodeId); // If no intro was found for this episode, skip it. if (!originalIntros.TryGetValue(episode.EpisodeId, out var originalIntro)) { _logger.LogTrace("{Name} does not have an intro", episode.Name); continue; } // Since we only want to adjust the end timestamp of the intro, create a new TimeRange // that covers the last few seconds. var originalIntroEnd = new TimeRange(originalIntro.IntroEnd - 10, originalIntro.IntroEnd); _logger.LogTrace( "{Name} original intro: {Start} - {End}", episode.Name, originalIntro.IntroStart, originalIntro.IntroEnd); // Detect silence in the media file up to the end of the intro. var silence = FFmpegWrapper.DetectSilence(episode, (int)originalIntro.IntroEnd + 2); // For all periods of silence foreach (var currentRange in silence) { _logger.LogTrace( "{Name} silence: {Start} - {End}", episode.Name, currentRange.Start, currentRange.End); // Ignore any silence that: // * doesn't intersect the ending of the intro, or // * is less than half a second long if (!originalIntroEnd.Intersects(currentRange) || currentRange.Duration < 0.5) { continue; } // Adjust the end timestamp of the intro to match the start of the silence region. originalIntro.IntroEnd = currentRange.Start; break; } _logger.LogTrace( "{Name} adjusted intro: {Start} - {End}", episode.Name, originalIntro.IntroStart, originalIntro.IntroEnd); // Add the (potentially) modified intro back. modifiedIntros[episode.EpisodeId] = originalIntro; } return modifiedIntros; } /// /// Count the number of bits that are set in the provided number. /// /// Number to count bits in. /// Number of bits that are equal to 1. public static int CountBits(uint number) { return BitOperations.PopCount(number); } private double GetIntroDuration(Guid id) { if (!Plugin.Instance!.Intros.TryGetValue(id, out var episode)) { return 0; } return episode.Valid ? Math.Round(episode.IntroEnd - episode.IntroStart, 2) : 0; } /// /// Get task triggers. /// /// Task triggers. public IEnumerable GetDefaultTriggers() { return new[] { new TaskTriggerInfo { Type = TaskTriggerInfo.TriggerDaily, TimeOfDayTicks = TimeSpan.FromHours(0).Ticks } }; } }