diff --git a/.gitignore b/.gitignore index 3722249..295855b 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,3 @@ BenchmarkDotNet.Artifacts/ # Ignore pre compiled web interface docker/dist - -# Ignore user provided end to end test results -ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/expected/*.json diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/.gitignore b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/.gitignore new file mode 100644 index 0000000..15c88d2 --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/.gitignore @@ -0,0 +1,14 @@ +# Binaries +/verifier/verifier +/run_tests +/plugin_binaries/ + +# Wrapper configuration and base configuration files +config.json +/config/ + +# Timestamp reports +/reports/ + +# Selenium screenshots +selenium/screenshots/ diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/README.md b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/README.md index 061f2b7..d0361be 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/README.md +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/README.md @@ -1,14 +1,52 @@ # End to end testing framework -This folder holds scripts used in performing end to end testing of the plugin. The script: +## wrapper -1. **Erases all currently discovered introduction timestamps** -2. Runs the Analyze episodes task -3. Waits for the analysis to complete -4. Checks that the current results are within one second of a previous result +The wrapper script (compiled as `run_tests`) runs multiple tests on Jellyfin servers to verify that the plugin works as intended. It tests: -## Usage +- Introduction timestamp accuracy (using `verifier`) +- Web interface functionality (using `selenium/main.py`) -1. Save the response returned by `/Intros/All?api_key=KEY` to a file somewhere. -2. Set the environment variable `JELLYFIN_TOKEN` to the access token of an administrator. -3. Run `python3 main.py -f FILENAME` +## verifier + +### Description + +This program is responsible for: +* Saving all discovered introduction timestamps into a report +* Comparing two reports against each other to find episodes that: + * Are missing introductions in both reports + * Have introductions in both reports, but with different timestamps + * Newly discovered introductions + * Introductions that were discovered previously, but not anymore +* Validating the schema of returned `Intro` objects from the `/IntroTimestamps` API endpoint + +### Usage examples +* Generate intro timestamp report from a local server: + * `./verifier -address http://127.0.0.1:8096 -key api_key` +* Generate intro timestamp report from a remote server, polling for task completion every 20 seconds: + * `./verifier -address https://example.com -key api_key -poll 20s -o example.json` +* Compare two previously generated reports: + * `./verifier -r1 v0.1.5.json -r2 v0.1.6.json` +* Validate the API schema for three episodes: + * `./verifier -address http://127.0.0.1:8096 -key api_key -validate id1,id2,id3` + +## Selenium web interface tests + +Selenium is used to verify that the plugin's web interface works as expected. It simulates a user: + +* Clicking the skip intro button + * Checks that clicking the button skips the intro and keeps playing the video +* Changing settings (will be added in the future) + * Maximum degree of parallelism + * Selecting libraries for analysis + * EDL settings + * Introduction requirements + * Auto skip + * Show/hide skip prompt +* Timestamp editor (will be added in the future) + * Displays timestamps + * Modifies timestamps + * Erases season timestamps +* Fingerprint visualizer (will be added in the future) + * Suggests shifts + * Visualizer canvas is drawn on diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/build.sh b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/build.sh new file mode 100755 index 0000000..8a29ab8 --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/build.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +echo "[+] Building timestamp verifier" +(cd verifier && go build -o verifier) || exit 1 + +echo "[+] Building test wrapper" +(cd wrapper && go test ./... && go build -o ../run_tests) || exit 1 + +echo +echo "[+] All programs built successfully" diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/config_sample.jsonc b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/config_sample.jsonc new file mode 100644 index 0000000..c2eedc8 --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/config_sample.jsonc @@ -0,0 +1,23 @@ +{ + "common": { + "library": "/full/path/to/test/library/on/host", + "episode": "Episode title to search for" + }, + "servers": [ + { + "comment": "Optional comment to identify this server", + "image": "ghcr.io/confusedpolarbear/jellyfin-intro-skipper:latest", + "username": "admin", + "password": "hunter2", + "base": "config/official", + "browsers": [ + "chrome", + "firefox" + ], // supported values are "chrome" and "firefox". + "tests": [ + "skip_button", // test skip intro button + "settings" // test plugin administration page + ] + } + ] +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/docker-compose.yml b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/docker-compose.yml new file mode 100644 index 0000000..aac5407 --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/docker-compose.yml @@ -0,0 +1,33 @@ +version: "3" +services: + chrome: + image: selenium/standalone-chrome:103.0 + shm_size: 2gb + ports: + - 4444:4444 + environment: + - SE_NODE_SESSION_TIMEOUT=10 + + firefox: + image: selenium/standalone-firefox:103.0 + shm_size: 2gb + ports: + - 4445:4444 + environment: + - SE_NODE_SESSION_TIMEOUT=10 + + chrome_video: + image: selenium/video + environment: + - DISPLAY_CONTAINER_NAME=chrome + - FILE_NAME=chrome_video.mp4 + volumes: + - /tmp/selenium/videos:/videos + + firefox_video: + image: selenium/video + environment: + - DISPLAY_CONTAINER_NAME=firefox + - FILE_NAME=firefox_video.mp4 + volumes: + - /tmp/selenium/videos:/videos diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/main.py b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/main.py deleted file mode 100644 index 4ed2d0f..0000000 --- a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/main.py +++ /dev/null @@ -1,228 +0,0 @@ -import argparse, json, os, random, time -import requests - -# Server address -addr = "" - -# Authentication token -token = "" - -# GUID of the analyze episodes scheduled task -taskId = "8863329048cc357f7dfebf080f2fe204" - -# Parse CLI arguments -def parse_args(): - parser = argparse.ArgumentParser() - - parser.add_argument( - "-s", - help="Server address (if different than http://127.0.0.1:8096)", - type=str, - dest="address", - default="http://127.0.0.1:8096", - metavar="ADDRESS", - ) - - parser.add_argument( - "-freq", - help="Interval to poll task completion state at (default is 10 seconds)", - type=int, - dest="frequency", - default=10, - ) - - parser.add_argument( - "-f", - help="Expected intro timestamps (as previously retrieved from /Intros/All)", - type=str, - dest="expected", - default="expected/dev.json", - metavar="FILENAME", - ) - - parser.add_argument( - "--skip-analysis", - help="Skip reanalyzing episodes and just validate timestamps and API versioning", - dest="skip", - action="store_true", - ) - - return parser.parse_args() - - -# Send an HTTP request and return the response -def send(url, method="GET", log=True): - global addr, token - - # Construct URL - r = None - url = addr + url - - # Log request - if log: - print(f"{method} {url} ", end="") - - # Send auth token - headers = {"Authorization": f"MediaBrowser Token={token}"} - - # Send the request - if method == "GET": - r = requests.get(url, headers=headers) - elif method == "POST": - r = requests.post(url, headers=headers) - else: - raise ValueError(f"Unknown method {method}") - - # Log status code - if log: - print(f"{r.status_code}\n") - - # Check status code - r.raise_for_status() - - return r - - -def close_enough(expected, actual): - # TODO: make customizable - return abs(expected - actual) <= 2 - - -# Validate that all episodes in expected have a similar entry in actual. -def validate(expected, actual): - good = 0 - bad = 0 - total = len(expected) - - for i in expected: - if i not in actual: - print(f"[!] Cound not find episode {i}") - bad += 1 - continue - - ex = expected[i] - ac = actual[i] - - start = close_enough(ex["IntroStart"], ac["IntroStart"]) - end = close_enough(ex["IntroEnd"], ac["IntroEnd"]) - - # If both the start and end times are close enough, keep going - if start and end: - good += 1 - continue - - # Oops - bad += 1 - - print(f"[!] Episode {i} is not correct") - print( - f"expected {ex['IntroStart']} => {ex['IntroEnd']} but found {ac['IntroStart']} => {ac['IntroEnd']}" - ) - - print() - - print("Statistics:") - print(f"Correct: {good} ({int((good * 100) / total)}%)") - print(f"Incorrect: {bad}") - print(f"Total: {total}") - - -def main(): - global addr, token - - # Validate arguments - args = parse_args() - addr = args.address - - # Validate token - token = os.environ.get("JELLYFIN_TOKEN") - if token is None: - print( - "Administrator access token is required, set environment variable JELLYFIN_TOKEN and try again" - ) - exit(1) - - # Validate expected timestamps - expected = [] - with open(args.expected, "r") as f: - expected = json.load(f) - - print(f"[+] Found {len(expected)} expected timestamps\n") - - if not args.skip: - # Erase old intro timestamps - print("[+] Erasing previously discovered introduction timestamps") - send("/Intros/EraseTimestamps", "POST") - - # Run analyze episodes task - print("[+] Starting episode analysis task") - send(f"/ScheduledTasks/Running/{taskId}", "POST") - else: - print("[+] Not running episode analysis") - args.frequency = 0 - - # Poll for completion - print("[+] Waiting for analysis task to complete") - - while True: - time.sleep(args.frequency) - task = send(f"/ScheduledTasks/{taskId}", "GET", False).json() - state = task["State"] - - # Calculate percentage analyzed - percent = 0 - - if state == "Idle": - percent = 100 - - elif state == "Running": - percent = 0 - if "CurrentProgressPercentage" in task: - percent = task["CurrentProgressPercentage"] - - # Print percentage analyzed - print(f"\r[+] Episodes analyzed: {percent}%", end="") - if percent == 100: - print("\n") - break - - # Download actual intro timestamps - print("[+] Getting actual timestamps") - intros = send("/Intros/All") - - actual = intros.json() - - # Store actual episodes to the filesystem - with open("/tmp/actual.json", "w") as f: - f.write(intros.text) - - # Verify timestamps - print(f"[+] Found {len(actual)} actual timestamps\n") - - validate(expected, actual) - - # Select some episodes to validate - keys = [] - for i in expected: - keys.append(i) - keys = random.choices(keys, k=min(len(keys), 10)) - - # Validate API version 1 (both implicitly and explicitly versioned) - for version in ["v1 (implicit)", "v1 (explicit)"]: - print() - print(f"[+] Validating API version: {version} with {len(keys)} episodes") - - if version.find("implicit") != -1: - version = "" - else: - version = "v1" - - for episode in keys: - ac = send( - f"/Episode/{episode}/IntroTimestamps/{version}", "GET", False - ).json() - - print(ac) - - -main() diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/expected/.keep b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/reports/.keep similarity index 100% rename from ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/expected/.keep rename to ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/reports/.keep diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/selenium/main.py b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/selenium/main.py new file mode 100644 index 0000000..e602cd8 --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/selenium/main.py @@ -0,0 +1,203 @@ +import argparse, os, time + +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.common.keys import Keys + + +# Driver function +def main(): + # Parse CLI arguments and store in a dictionary + parser = argparse.ArgumentParser() + parser.add_argument("-host", help="Jellyfin server address with protocol and port.") + parser.add_argument("-username", help="Username.") + parser.add_argument("-password", help="Password.") + parser.add_argument("-name", help="Name of episode to search for.") + parser.add_argument( + "--tests", help="Space separated list of Selenium tests to run.", type=str, nargs="+" + ) + parser.add_argument( + "--browsers", + help="Space separated list of browsers to run tests with.", + type=str, + nargs="+", + choices=["chrome", "firefox"], + ) + args = parser.parse_args() + + server = { + "host": args.host, + "username": args.username, + "password": args.password, + "episode": args.name, + "browsers": args.browsers, + "tests": args.tests, + } + + # Print the server info for debugging and run the test + print() + print(f"Browsers: {server['browsers']}") + print(f"Address: {server['host']}") + print(f"Username: {server['username']}") + print(f"Episode: \"{server['episode']}\"") + print(f"Tests: {server['tests']}") + print() + + # Setup the list of drivers to run tests with + if server["browsers"] is None: + print("[!] --browsers is required") + exit(1) + + drivers = [] + if "chrome" in server["browsers"]: + drivers = [("http://127.0.0.1:4444", "Chrome")] + if "firefox" in server["browsers"]: + drivers.append(("http://127.0.0.1:4445", "Firefox")) + + # Test with all selected drivers + for driver in drivers: + print(f"[!] Starting new test run using {driver[1]}") + test_server(server, driver[0], driver[1]) + print() + + +# Main server test function +def test_server(server, executor, driver_type): + # Configure Selenium to use a remote driver + print(f"[+] Configuring Selenium to use executor {executor} of type {driver_type}") + + opts = None + if driver_type == "Chrome": + opts = webdriver.ChromeOptions() + elif driver_type == "Firefox": + opts = webdriver.FirefoxOptions() + else: + raise ValueError(f"Unknown driver type {driver_type}") + + driver = webdriver.Remote(command_executor=executor, options=opts) + + try: + # Wait up to two seconds when finding an element before reporting failure + driver.implicitly_wait(2) + + # Login to Jellyfin + driver.get(make_url(server, "/")) + + print(f"[+] Authenticating as {server['username']}") + login(driver, server) + + if "skip_button" in server["tests"]: + # Play the user specified episode and verify skip intro button functionality. This episode is expected to: + # * already have been analyzed for an introduction + # * have an introduction at the beginning of the episode + print("[+] Testing skip intro button") + test_skip_button(driver, server) + + print("[+] All tests completed successfully") + finally: + # Unconditionally end the Selenium session + driver.quit() + + +def login(driver, server): + # Append the Enter key to the password to submit the form + us = server["username"] + pw = server["password"] + Keys.ENTER + + # Fill out and submit the login form + driver.find_element(By.ID, "txtManualName").send_keys(us) + driver.find_element(By.ID, "txtManualPassword").send_keys(pw) + + +def test_skip_button(driver, server): + print(f" [+] Searching for episode \"{server['episode']}\"") + + search = driver.find_element(By.CSS_SELECTOR, ".headerSearchButton span.search") + + if driver.capabilities["browserName"] == "firefox": + # Work around a FF bug where the search element isn't considered clickable right away + time.sleep(1) + + # Click the search button + search.click() + + # Type the episode name + driver.find_element(By.CSS_SELECTOR, ".searchfields-txtSearch").send_keys( + server["episode"] + ) + + # Click the first episode in the search results + driver.find_element( + By.CSS_SELECTOR, ".searchResults button[data-type='Episode']" + ).click() + + # Wait for the episode page to finish loading by searching for the episode description (overview) + driver.find_element(By.CSS_SELECTOR, ".overview") + + print(f" [+] Waiting for playback to start") + + # Click the play button in the toolbar + driver.find_element( + By.CSS_SELECTOR, "div.mainDetailButtons span.play_arrow" + ).click() + + # Wait for playback to start by searching for the lower OSD control bar + driver.find_element(By.CSS_SELECTOR, ".osdControls") + + # Let the video play a little bit so the position before clicking the button can be logged + print(" [+] Playing video") + time.sleep(2) + screenshot(driver, "skip_button_pre_skip") + assert_video_playing(driver) + + # Find the skip intro button and click it, logging the new video position after the seek is preformed + print(" [+] Clicking skip intro button") + driver.find_element(By.CSS_SELECTOR, "button.btnSkipIntro").click() + time.sleep(1) + screenshot(driver, "skip_button_post_skip") + assert_video_playing(driver) + + # Keep playing the video for a few seconds to ensure that: + # * the intro was successfully skipped + # * video playback continued automatically post button click + print(" [+] Verifying post skip position") + time.sleep(4) + + screenshot(driver, "skip_button_post_play") + assert_video_playing(driver) + + +# Utility functions +def make_url(server, url): + final = server["host"] + url + print(f"[+] Navigating to {final}") + return final + + +def screenshot(driver, filename): + dest = f"screenshots/{filename}.png" + driver.save_screenshot(dest) + + +# Returns the current video playback position and if the video is paused. +# Will raise an exception if playback is paused as the video shouldn't ever pause when using this plugin. +def assert_video_playing(driver): + ret = driver.execute_script( + """ + const video = document.querySelector("video"); + return { + "position": video.currentTime, + "paused": video.paused + }; + """ + ) + + if ret["paused"]: + raise Exception("Video should not be paused") + + print(f" [+] Video playback position: {ret['position']}") + + return ret + + +main() diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/selenium/requirements.txt b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/selenium/requirements.txt new file mode 100644 index 0000000..7978165 --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/selenium/requirements.txt @@ -0,0 +1 @@ +selenium >= 4.3.0 diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/selenium/screenshots/.keep b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/selenium/screenshots/.keep new file mode 100644 index 0000000..e69de29 diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/go.mod b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/go.mod new file mode 100644 index 0000000..0bc6481 --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/go.mod @@ -0,0 +1,3 @@ +module github.com/confusedpolarbear/intro_skipper_verifier + +go 1.17 diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/http.go b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/http.go new file mode 100644 index 0000000..e3178a8 --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/http.go @@ -0,0 +1,78 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/confusedpolarbear/intro_skipper_verifier/structs" +) + +// Gets the contents of the provided URL or panics. +func SendRequest(method, url, apiKey string) []byte { + http.DefaultClient.Timeout = 10 * time.Second + + // Construct the request + req, err := http.NewRequest(method, url, nil) + if err != nil { + panic(err) + } + + // Include the authorization token + req.Header.Set("Authorization", fmt.Sprintf(`MediaBrowser Token="%s"`, apiKey)) + + // Send the request + res, err := http.DefaultClient.Do(req) + + if !strings.Contains(url, "hideUrl") { + fmt.Printf("[+] %s %s: %d\n", method, url, res.StatusCode) + } + + // Panic if any error occurred + if err != nil { + panic(err) + } + + // Check for API key validity + if res.StatusCode == http.StatusUnauthorized { + panic("Server returned 401 (Unauthorized). Check API key validity and try again.") + } + + // Read and return the entire body + defer res.Body.Close() + body, err := io.ReadAll(res.Body) + if err != nil { + panic(err) + } + + return body +} + +func GetServerInfo(hostAddress, apiKey string) structs.PublicInfo { + var info structs.PublicInfo + + fmt.Println("[+] Getting server information") + rawInfo := SendRequest("GET", hostAddress+"/System/Info/Public", apiKey) + + if err := json.Unmarshal(rawInfo, &info); err != nil { + panic(err) + } + + return info +} + +func GetPluginConfiguration(hostAddress, apiKey string) structs.PluginConfiguration { + var config structs.PluginConfiguration + + fmt.Println("[+] Getting plugin configuration") + rawConfig := SendRequest("GET", hostAddress+"/Plugins/c83d86bb-a1e0-4c35-a113-e2101cf4ee6b/Configuration", apiKey) + + if err := json.Unmarshal(rawConfig, &config); err != nil { + panic(err) + } + + return config +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/main.go b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/main.go new file mode 100644 index 0000000..5e746ee --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/main.go @@ -0,0 +1,63 @@ +package main + +import ( + "flag" + "time" +) + +func flags() { + // Report generation + hostAddress := flag.String("address", "", "Address of Jellyfin server to extract intro information from.") + apiKey := flag.String("key", "", "Administrator API key to authenticate with.") + keepTimestamps := flag.Bool("keep", false, "Keep the current timestamps instead of erasing and reanalyzing.") + pollInterval := flag.Duration("poll", 10*time.Second, "Interval to poll task completion at.") + reportDestination := flag.String("o", "", "Report destination filename. Defaults to intros-ADDRESS-TIMESTAMP.json.") + + // Report comparison + report1 := flag.String("r1", "", "First report.") + report2 := flag.String("r2", "", "Second report.") + + // API schema validator + ids := flag.String("validate", "", "Comma separated item ids to validate the API schema for.") + + // Print usage examples + flag.CommandLine.Usage = func() { + flag.CommandLine.Output().Write([]byte("Flags:\n")) + flag.PrintDefaults() + + usage := "\nUsage:\n" + + "Generate intro timestamp report from a local server:\n" + + "./verifier -address http://127.0.0.1:8096 -key api_key\n\n" + + + "Generate intro timestamp report from a remote server, polling for task completion every 20 seconds:\n" + + "./verifier -address https://example.com -key api_key -poll 20s -o example.json\n\n" + + + "Compare two previously generated reports:\n" + + "./verifier -r1 v0.1.5.json -r2 v0.1.6.json\n\n" + + + "Validate the API schema for some item ids:\n" + + "./verifier -address http://127.0.0.1:8096 -key api_key -validate id1,id2,id3\n" + + flag.CommandLine.Output().Write([]byte(usage)) + } + + flag.Parse() + + if *hostAddress != "" && *apiKey != "" { + if *ids == "" { + generateReport(*hostAddress, *apiKey, *reportDestination, *keepTimestamps, *pollInterval) + } else { + validateApiSchema(*hostAddress, *apiKey, *ids) + } + + } else if *report1 != "" && *report2 != "" { + compareReports(*report1, *report2, *reportDestination) + + } else { + panic("Either (-address and -key) or (-r1 and -r2) are required.") + } +} + +func main() { + flags() +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/report.html b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/report.html new file mode 100644 index 0000000..75e5bb9 --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/report.html @@ -0,0 +1,389 @@ + + + + + + + + + + +
+

Intro Timestamp Differential

+ +
+

First report

+ + {{ block "ReportInfo" .OldReport }} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Path{{ .Path }}
Jellyfin{{ .ServerInfo.Version }} on {{ .ServerInfo.OperatingSystem }}
Analysis Settings{{ printAnalysisSettings .PluginConfig }}
Introduction Requirements{{ printIntroductionReqs .PluginConfig }}
Start time{{ printTime .StartedAt }}
End time{{ printTime .FinishedAt }}
Duration{{ printDuration .Runtime }}
+ {{ end }} +
+ +
+

Second report

+ + {{ template "ReportInfo" .NewReport }} +
+
+ +
+

Statistics

+ + + + + + + + + + + + + + + + + + + + + + + + +
Total episodes
Never found
Changed
Gains
Losses
+
+ +
+

Settings

+ +
+ +
+ + +
+ + +
+
+
+ + {{/* store a reference to the data before the range query */}} + {{ $p := . }} + + {{/* sort the show names and iterate over them */}} + {{ range $name := sortShows .OldReport.Shows }} +
+
+ {{/* get the unsorted seasons for this show */}} + {{ $seasons := index $p.OldReport.Shows $name }} + + {{/* log the show name and number of seasons */}} + + + {{ $name }} + + + + +
+ {{/* sort the seasons to ensure they display in numerical order */}} + {{ range $seasonNumber := (sortSeasons $seasons) }} +
+
+ + + Season {{ $seasonNumber }} + + + + + {{/* compare each episode in the old report to the same episode in the new report */}} + {{ range $episode := index $seasons $seasonNumber }} + + {{/* lookup and compare both episodes */}} + {{ $comparison := compareEpisodes $episode.EpisodeId $p }} + {{ $old := $comparison.Old }} + {{ $new := $comparison.New }} + + {{/* set attributes indicating if an intro was found in the old and new reports */}} +
+

{{ $episode.Title }}

+ +

+ Old: {{ $old.IntroStart }} - {{ $old.IntroEnd }} + ({{ $old.Duration }}) + (valid: {{ $old.Valid }})
+ + New: {{ $new.IntroStart }} - {{ $new.IntroEnd }} + ({{ $new.Duration }}) + (valid: {{ $new.Valid }})
+ + {{ if ne $comparison.WarningShort "okay" }} + Warning: {{ $comparison.Warning }} + {{ end }} +

+ +
+
+ {{ end }} +
+
+ {{ end }} +
+
+
+ {{ end }} + + + + + diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/report_comparison.go b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/report_comparison.go new file mode 100644 index 0000000..416876a --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/report_comparison.go @@ -0,0 +1,135 @@ +package main + +import ( + _ "embed" + "encoding/json" + "fmt" + "html/template" + "math" + "os" + "time" + + "github.com/confusedpolarbear/intro_skipper_verifier/structs" +) + +//go:embed report.html +var reportTemplate []byte + +func compareReports(oldReportPath, newReportPath, destination string) { + start := time.Now() + + // Populate the destination filename if none was provided + if destination == "" { + destination = fmt.Sprintf("report-%d.html", start.Unix()) + } + + // Open the report for writing + f, err := os.OpenFile(destination, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + panic(err) + } else { + defer f.Close() + } + + fmt.Printf("Started at: %s\n", start.Format(time.RFC1123)) + fmt.Printf("First report: %s\n", oldReportPath) + fmt.Printf("Second report: %s\n", newReportPath) + fmt.Printf("Destination: %s\n\n", destination) + + // Unmarshal both reports + oldReport, newReport := unmarshalReport(oldReportPath), unmarshalReport(newReportPath) + + fmt.Println("[+] Comparing reports") + + // Setup a function map with helper functions to use in the template + tmp := template.New("report") + + funcs := make(template.FuncMap) + + funcs["printTime"] = func(t time.Time) string { + return t.Format(time.RFC1123) + } + + funcs["printDuration"] = func(d time.Duration) string { + return d.Round(time.Second).String() + } + + funcs["printAnalysisSettings"] = func(pc structs.PluginConfiguration) string { + return pc.AnalysisSettings() + } + + funcs["printIntroductionReqs"] = func(pc structs.PluginConfiguration) string { + return pc.IntroductionRequirements() + } + + funcs["sortShows"] = templateSortShows + funcs["sortSeasons"] = templateSortSeason + funcs["compareEpisodes"] = templateCompareEpisodes + tmp.Funcs(funcs) + + // Load the template or panic + report := template.Must(tmp.Parse(string(reportTemplate))) + + err = report.Execute(f, + structs.TemplateReportData{ + OldReport: oldReport, + NewReport: newReport, + }) + + if err != nil { + panic(err) + } + + // Log success + fmt.Printf("[+] Reports successfully compared in %s\n", time.Since(start).Round(time.Millisecond)) +} + +func unmarshalReport(path string) structs.Report { + // Read the provided report + contents, err := os.ReadFile(path) + if err != nil { + panic(err) + } + + // Unmarshal + var report structs.Report + if err := json.Unmarshal(contents, &report); err != nil { + panic(err) + } + + // Setup maps and template data for later use + report.Path = path + report.Shows = make(map[string]structs.Seasons) + report.IntroMap = make(map[string]structs.Intro) + + // Sort episodes by show and season + for _, intro := range report.Intros { + // Round the duration to the nearest second to avoid showing 8 decimal places in the report + intro.Duration = float32(math.Round(float64(intro.Duration))) + + show, season := intro.Series, intro.Season + + // If this show hasn't been seen before, allocate space for it + if _, ok := report.Shows[show]; !ok { + report.Shows[show] = make(structs.Seasons) + } + + // Store this intro in the season of this show + episodes := report.Shows[show][season] + episodes = append(episodes, intro) + report.Shows[show][season] = episodes + + // Store a reference to this intro in a lookup table + report.IntroMap[intro.EpisodeId] = intro + } + + // Print report info + fmt.Printf("Report %s:\n", path) + fmt.Printf("Generated with Jellyfin %s running on %s\n", report.ServerInfo.Version, report.ServerInfo.OperatingSystem) + fmt.Printf("Analysis settings: %s\n", report.PluginConfig.AnalysisSettings()) + fmt.Printf("Introduction reqs: %s\n", report.PluginConfig.IntroductionRequirements()) + fmt.Printf("Episodes analyzed: %d\n", len(report.Intros)) + fmt.Println() + + return report +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/report_comparison_util.go b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/report_comparison_util.go new file mode 100644 index 0000000..ec51c6a --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/report_comparison_util.go @@ -0,0 +1,81 @@ +package main + +import ( + "fmt" + "math" + "sort" + + "github.com/confusedpolarbear/intro_skipper_verifier/structs" +) + +// report template helper functions + +// Sort show names alphabetically +func templateSortShows(shows map[string]structs.Seasons) []string { + var showNames []string + + for show := range shows { + showNames = append(showNames, show) + } + + sort.Strings(showNames) + + return showNames +} + +// Sort season numbers +func templateSortSeason(show structs.Seasons) []int { + var keys []int + + for season := range show { + keys = append(keys, season) + } + + sort.Ints(keys) + + return keys +} + +// Compare the episode with the provided ID in the old report to the episode in the new report. +func templateCompareEpisodes(id string, reports structs.TemplateReportData) structs.IntroPair { + var pair structs.IntroPair + var tolerance int = 5 + + // Locate both episodes + pair.Old = reports.OldReport.IntroMap[id] + pair.New = reports.NewReport.IntroMap[id] + + // Mark the timestamps as similar if they are within a few seconds of each other + similar := func(oldTime, newTime float32) bool { + diff := math.Abs(float64(newTime) - float64(oldTime)) + return diff <= float64(tolerance) + } + + if pair.Old.Valid && !pair.New.Valid { + // If an intro was found previously, but not now, flag it + pair.WarningShort = "only_previous" + pair.Warning = "Introduction found in previous report, but not the current one" + + } else if !pair.Old.Valid && pair.New.Valid { + // If an intro was not found previously, but found now, flag it + pair.WarningShort = "improvement" + pair.Warning = "New introduction discovered" + + } else if !pair.Old.Valid && !pair.New.Valid { + // If an intro has never been found for this episode + pair.WarningShort = "missing" + pair.Warning = "No introduction has ever been found for this episode" + + } else if !similar(pair.Old.IntroStart, pair.New.IntroStart) || !similar(pair.Old.IntroEnd, pair.New.IntroEnd) { + // If the intro timestamps are too different, flag it + pair.WarningShort = "different" + pair.Warning = fmt.Sprintf("Timestamps differ by more than %d seconds", tolerance) + + } else { + // No warning was generated + pair.WarningShort = "okay" + pair.Warning = "Okay" + } + + return pair +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/report_generator.go b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/report_generator.go new file mode 100644 index 0000000..f922e5c --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/report_generator.go @@ -0,0 +1,158 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "strings" + "time" + + "github.com/confusedpolarbear/intro_skipper_verifier/structs" +) + +var spinners []string +var spinnerIndex int + +func generateReport(hostAddress, apiKey, reportDestination string, keepTimestamps bool, pollInterval time.Duration) { + start := time.Now() + + // Setup the spinner + spinners = strings.Split("⣷⣯⣟⡿⢿⣻⣽⣾", "") + spinnerIndex = -1 // start the spinner on the first graphic + + // Setup the filename to save intros to + if reportDestination == "" { + reportDestination = fmt.Sprintf("intros-%s-%d.json", hostAddress, time.Now().Unix()) + reportDestination = strings.ReplaceAll(reportDestination, "http://", "") + reportDestination = strings.ReplaceAll(reportDestination, "https://", "") + } + + // Ensure the file is writable + if err := os.WriteFile(reportDestination, nil, 0600); err != nil { + panic(err) + } + + fmt.Printf("Started at: %s\n", start.Format(time.RFC1123)) + fmt.Printf("Address: %s\n", hostAddress) + fmt.Printf("Destination: %s\n", reportDestination) + fmt.Println() + + // Get Jellyfin server information and plugin configuration + info := GetServerInfo(hostAddress, apiKey) + config := GetPluginConfiguration(hostAddress, apiKey) + fmt.Println() + + fmt.Printf("Jellyfin OS: %s\n", info.OperatingSystem) + fmt.Printf("Jellyfin version: %s\n", info.Version) + fmt.Printf("Analysis settings: %s\n", config.AnalysisSettings()) + fmt.Printf("Introduction reqs: %s\n", config.IntroductionRequirements()) + fmt.Printf("Erase timestamps: %t\n", !keepTimestamps) + fmt.Println() + + // If not keeping timestamps, run the fingerprint task. + // Otherwise, log that the task isn't being run + if !keepTimestamps { + runAnalysisAndWait(hostAddress, apiKey, pollInterval) + } else { + fmt.Println("[+] Using previously discovered intros") + } + fmt.Println() + + // Save all intros from the server + fmt.Println("[+] Saving intros") + + var report structs.Report + rawIntros := SendRequest("GET", hostAddress+"/Intros/All", apiKey) + if err := json.Unmarshal(rawIntros, &report.Intros); err != nil { + panic(err) + } + + // Calculate the durations of all intros + for i := range report.Intros { + intro := report.Intros[i] + intro.Duration = intro.IntroEnd - intro.IntroStart + report.Intros[i] = intro + } + + fmt.Println() + fmt.Println("[+] Saving report") + + // TODO: also save analysis statistics + // Store timing data, server information, and plugin configuration + report.StartedAt = start + report.FinishedAt = time.Now() + report.Runtime = report.FinishedAt.Sub(report.StartedAt) + report.ServerInfo = info + report.PluginConfig = config + + // Marshal the report + marshalled, err := json.Marshal(report) + if err != nil { + panic(err) + } + + if err := os.WriteFile(reportDestination, marshalled, 0600); err != nil { + panic(err) + } + + fmt.Println("[+] Done") +} + +func runAnalysisAndWait(hostAddress, apiKey string, pollInterval time.Duration) { + type taskInfo struct { + State string + CurrentProgressPercentage int + } + + fmt.Println("[+] Erasing previously discovered intros") + SendRequest("POST", hostAddress+"/Intros/EraseTimestamps", apiKey) + fmt.Println() + + fmt.Println("[+] Starting analysis task") + SendRequest("POST", hostAddress+"/ScheduledTasks/Running/8863329048cc357f7dfebf080f2fe204", apiKey) + fmt.Println() + + fmt.Println("[+] Waiting for analysis task to complete") + fmt.Print("[+] Episodes analyzed: 0%") + + var info taskInfo // Last known scheduled task state + var lastQuery time.Time // Time the task info was last updated + + for { + time.Sleep(500 * time.Millisecond) + + // Update the spinner + if spinnerIndex++; spinnerIndex >= len(spinners) { + spinnerIndex = 0 + } + + fmt.Printf("\r[%s] Episodes analyzed: %d%%", spinners[spinnerIndex], info.CurrentProgressPercentage) + + if info.CurrentProgressPercentage == 100 { + fmt.Printf("\r[+]") // reset the spinner + fmt.Println() + break + } + + // Get the latest task state & unmarshal (only if enough time has passed since the last update) + if time.Since(lastQuery) <= pollInterval { + continue + } + + lastQuery = time.Now() + + raw := SendRequest("GET", hostAddress+"/ScheduledTasks/8863329048cc357f7dfebf080f2fe204?hideUrl=1", apiKey) + + if err := json.Unmarshal(raw, &info); err != nil { + fmt.Printf("[!] Unable to unmarshal response into taskInfo struct: %s\n", err) + fmt.Printf("%s\n", raw) + continue + } + + // Print the latest task state + switch info.State { + case "Idle": + info.CurrentProgressPercentage = 100 + } + } +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/schema_validation.go b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/schema_validation.go new file mode 100644 index 0000000..0ae77dd --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/schema_validation.go @@ -0,0 +1,116 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/confusedpolarbear/intro_skipper_verifier/structs" +) + +// Given a comma separated list of item IDs, validate the returned API schema. +func validateApiSchema(hostAddress, apiKey, rawIds string) { + // Iterate over the raw item IDs and validate the schema of API responses + ids := strings.Split(rawIds, ",") + + start := time.Now() + + fmt.Printf("Started at: %s\n", start.Format(time.RFC1123)) + fmt.Printf("Address: %s\n", hostAddress) + fmt.Println() + + // Get Jellyfin server information + info := GetServerInfo(hostAddress, apiKey) + fmt.Println() + + fmt.Printf("Jellyfin OS: %s\n", info.OperatingSystem) + fmt.Printf("Jellyfin version: %s\n", info.Version) + fmt.Println() + + for _, id := range ids { + fmt.Printf("[+] Validating item %s\n", id) + + fmt.Println(" [+] Validating API v1 (implicitly versioned)") + intro, schema := getTimestampsV1(hostAddress, apiKey, id, "") + validateV1Intro(id, intro, schema) + + fmt.Println(" [+] Validating API v1 (explicitly versioned)") + intro, schema = getTimestampsV1(hostAddress, apiKey, id, "v1") + validateV1Intro(id, intro, schema) + + fmt.Println() + } + + fmt.Printf("Validated %d items in %s\n", len(ids), time.Since(start).Round(time.Millisecond)) +} + +// Validates the returned intro object, panicking on any error. +func validateV1Intro(id string, intro structs.Intro, schema map[string]interface{}) { + // Validate the item ID + if intro.EpisodeId != id { + panic(fmt.Sprintf("Intro struct has incorrect item ID. Expected '%s', found '%s'", id, intro.EpisodeId)) + } + + // Validate the intro start and end times + if intro.IntroStart < 0 || intro.IntroEnd < 0 { + panic("Intro struct has a negative intro start or end time") + } + + if intro.ShowSkipPromptAt > intro.IntroStart { + panic("Intro struct show prompt time is after intro start") + } + + if intro.HideSkipPromptAt > intro.IntroEnd { + panic("Intro struct hide prompt time is after intro end") + } + + // Validate the intro duration + if duration := intro.IntroEnd - intro.IntroStart; duration < 15 { + panic(fmt.Sprintf("Intro struct has duration %0.2f but the minimum allowed is 15", duration)) + } + + // Ensure the intro is marked as valid. + if !intro.Valid { + panic("Intro struct is not marked as valid") + } + + // Check for any extraneous properties + allowedProperties := []string{"EpisodeId", "Valid", "IntroStart", "IntroEnd", "ShowSkipPromptAt", "HideSkipPromptAt"} + + for schemaKey := range schema { + okay := false + + for _, allowed := range allowedProperties { + if allowed == schemaKey { + okay = true + break + } + } + + if !okay { + panic(fmt.Sprintf("Intro object contains unknown key '%s'", schemaKey)) + } + } +} + +// Gets the timestamps for the provided item or panics. +func getTimestampsV1(hostAddress, apiKey, id, version string) (structs.Intro, map[string]interface{}) { + var rawResponse map[string]interface{} + var intro structs.Intro + + // Make an authenticated GET request to {Host}/Episode/{ItemId}/IntroTimestamps/{Version} + raw := SendRequest("GET", fmt.Sprintf("%s/Episode/%s/IntroTimestamps/%s?hideUrl=1", hostAddress, id, version), apiKey) + + // Unmarshal the response as a version 1 API response, ignoring any unknown fields. + if err := json.Unmarshal(raw, &intro); err != nil { + panic(err) + } + + // Second, unmarshal the response into a map so that any unknown fields can be detected and alerted on. + if err := json.Unmarshal(raw, &rawResponse); err != nil { + panic(err) + } + + return intro, rawResponse +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/structs/intro.go b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/structs/intro.go new file mode 100644 index 0000000..139effe --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/structs/intro.go @@ -0,0 +1,17 @@ +package structs + +type Intro struct { + EpisodeId string + + Series string + Season int + Title string + + IntroStart float32 + IntroEnd float32 + Duration float32 + Valid bool + + ShowSkipPromptAt float32 + HideSkipPromptAt float32 +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/structs/plugin_configuration.go b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/structs/plugin_configuration.go new file mode 100644 index 0000000..594e910 --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/structs/plugin_configuration.go @@ -0,0 +1,44 @@ +package structs + +import ( + "fmt" + "strings" +) + +type PluginConfiguration struct { + CacheFingerprints bool + MaxParallelism int + SelectedLibraries string + + AnalysisPercent int + AnalysisLengthLimit int + MinimumIntroDuration int +} + +func (c PluginConfiguration) AnalysisSettings() string { + // If no libraries have been selected, display a star. + // Otherwise, quote each library before displaying the slice. + var libs []string + if c.SelectedLibraries == "" { + libs = []string{"*"} + } else { + for _, tmp := range strings.Split(c.SelectedLibraries, ",") { + tmp = `"` + strings.TrimSpace(tmp) + `"` + libs = append(libs, tmp) + } + } + + return fmt.Sprintf( + "cfp=%t thr=%d lbs=%v", + c.CacheFingerprints, + c.MaxParallelism, + libs) +} + +func (c PluginConfiguration) IntroductionRequirements() string { + return fmt.Sprintf( + "per=%d%% max=%dm min=%ds", + c.AnalysisPercent, + c.AnalysisLengthLimit, + c.MinimumIntroDuration) +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/structs/public_info.go b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/structs/public_info.go new file mode 100644 index 0000000..901307d --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/structs/public_info.go @@ -0,0 +1,6 @@ +package structs + +type PublicInfo struct { + Version string + OperatingSystem string +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/structs/report.go b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/structs/report.go new file mode 100644 index 0000000..5e8a3b3 --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/structs/report.go @@ -0,0 +1,48 @@ +package structs + +import "time" + +type Seasons map[int][]Intro + +type Report struct { + Path string `json:"-"` + + StartedAt time.Time + FinishedAt time.Time + Runtime time.Duration + + ServerInfo PublicInfo + PluginConfig PluginConfiguration + + Intros []Intro + + // Intro lookup table. Only populated when loading a report. + IntroMap map[string]Intro `json:"-"` + + // Intros which have been sorted by show and season number. Only populated when loading a report. + Shows map[string]Seasons `json:"-"` +} + +// Data passed to the report template. +type TemplateReportData struct { + // First report. + OldReport Report + + // Second report. + NewReport Report +} + +// A pair of introductions from an old and new reports. +type IntroPair struct { + Old Intro + New Intro + + // Recognized warning types: + // * okay: no warning + // * different: timestamps are too dissimilar + // * only_previous: introduction found in old report but not new one + WarningShort string + + // If this pair of intros is not okay, a short description about the cause + Warning string +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/wrapper/exec.go b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/wrapper/exec.go new file mode 100644 index 0000000..488cd52 --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/wrapper/exec.go @@ -0,0 +1,64 @@ +package main + +import ( + "bufio" + "context" + "fmt" + "io" + "os/exec" + "regexp" + "strings" + "time" +) + +// Run an external program +func RunProgram(program string, args []string, timeout time.Duration) { + // Flag if we are starting or stopping a container + managingContainer := program == "docker" + + // Create context and command + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + cmd := exec.CommandContext(ctx, program, args...) + + // Stringify and censor the program's arguments + strArgs := redactString(strings.Join(args, " ")) + fmt.Printf(" [+] Running %s %s\n", program, strArgs) + + // Setup pipes + stdout, err := cmd.StdoutPipe() + if err != nil { + panic(err) + } + + stderr, err := cmd.StderrPipe() + if err != nil { + panic(err) + } + + // Start the command + if err := cmd.Start(); err != nil { + panic(err) + } + + // Stream any messages to the terminal + for _, r := range []io.Reader{stdout, stderr} { + // Don't log stdout from the container + if managingContainer && r == stdout { + continue + } + + scanner := bufio.NewScanner(r) + scanner.Split(bufio.ScanRunes) + + for scanner.Scan() { + fmt.Print(scanner.Text()) + } + } +} + +// Redacts sensitive command line arguments. +func redactString(raw string) string { + redactionRegex := regexp.MustCompilePOSIX(`-(user|pass|key) [^ ]+`) + return redactionRegex.ReplaceAllString(raw, "-$1 REDACTED") +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/wrapper/exec_test.go b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/wrapper/exec_test.go new file mode 100644 index 0000000..ba25cae --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/wrapper/exec_test.go @@ -0,0 +1,13 @@ +package main + +import "testing" + +func TestStringRedaction(t *testing.T) { + raw := "-key deadbeef -first second -user admin -third fourth -pass hunter2" + expected := "-key REDACTED -first second -user REDACTED -third fourth -pass REDACTED" + actual := redactString(raw) + + if expected != actual { + t.Errorf(`String was redacted incorrectly: "%s"`, actual) + } +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/wrapper/go.mod b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/wrapper/go.mod new file mode 100644 index 0000000..d3ab43e --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/wrapper/go.mod @@ -0,0 +1,3 @@ +module github.com/confusedpolarbear/intro_skipper_wrapper + +go 1.17 diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/wrapper/main.go b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/wrapper/main.go new file mode 100644 index 0000000..b22e339 --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/wrapper/main.go @@ -0,0 +1,334 @@ +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "os" + "path" + "strings" + "time" +) + +// IP address to use when connecting to local containers. +var containerAddress string + +// Path to compiled plugin DLL to install in local containers. +var pluginPath string + +func flags() { + flag.StringVar(&pluginPath, "dll", "", "Path to plugin DLL to install in container images.") + flag.StringVar(&containerAddress, "caddr", "", "IP address to use when connecting to local containers.") + flag.Parse() +} + +func main() { + flags() + + start := time.Now() + + fmt.Printf("[+] Start time: %s\n", start) + + // Load list of servers + fmt.Println("[+] Loading configuration") + config := loadConfiguration() + fmt.Println() + + // Start Selenium by bringing up the compose file in detatched mode + fmt.Println("[+] Starting Selenium") + RunProgram("docker-compose", []string{"up", "-d"}, 10*time.Second) + + // If any error occurs, bring Selenium down before exiting + defer func() { + fmt.Println("[+] Stopping Selenium") + RunProgram("docker-compose", []string{"down"}, 15*time.Second) + }() + + // Test all provided Jellyfin servers + for _, server := range config.Servers { + if server.Skip { + continue + } + + var configurationDirectory string + var apiKey string + var seleniumArgs []string + + // LSIO containers use some slighly different paths & permissions + lsioImage := strings.Contains(server.Image, "linuxserver") + + fmt.Println() + fmt.Printf("[+] Testing %s\n", server.Comment) + + if server.Docker { + var err error + + // Setup a temporary folder for the container's configuration + configurationDirectory, err = os.MkdirTemp("/dev/shm", "jf-e2e-*") + if err != nil { + panic(err) + } + + // Copy the contents of the base configuration directory to a temp folder for the container + src, dst := server.Base+"/.", configurationDirectory + fmt.Printf(" [+] Copying %s to %s\n", src, dst) + RunProgram("cp", []string{"-ar", src, dst}, 10*time.Second) + + // Create a folder to install the plugin into + pluginDirectory := path.Join(configurationDirectory, "plugins", "intro-skipper") + if lsioImage { + pluginDirectory = path.Join(configurationDirectory, "data", "plugins", "intro-skipper") + } + + fmt.Println(" [+] Creating plugin directory") + if err := os.MkdirAll(pluginDirectory, 0700); err != nil { + fmt.Printf(" [!] Failed to create plugin directory: %s\n", err) + goto cleanup + } + + // If this is an LSIO container, adjust the permissions on the plugin directory + if lsioImage { + if err := os.Chown(pluginDirectory, 911, 911); err != nil { + fmt.Printf(" [!] Failed to change plugin directory UID/GID: %s\n", err) + goto cleanup + } + } + + // Install the plugin + fmt.Printf(" [+] Copying plugin %s to %s\n", pluginPath, pluginDirectory) + RunProgram("cp", []string{pluginPath, pluginDirectory}, 2*time.Second) + + /* Start the container with the following settings: + * Name: jf-e2e + * Port: 8097 + * Media: Mounted to /media, read only + */ + containerArgs := []string{"run", "--name", "jf-e2e", "--rm", "-p", "8097:8096", + "-v", fmt.Sprintf("%s:%s:rw", configurationDirectory, "/config"), + "-v", fmt.Sprintf("%s:%s:ro", config.Common.Library, "/media"), + server.Image} + + fmt.Printf(" [+] Starting container %s\n", server.Image) + go RunProgram("docker", containerArgs, 60*time.Second) + + // Wait for the container to fully start + waitForServerStartup(server.Address) + } else { + fmt.Println("[+] Remote instance, assuming plugin is already installed") + } + + // Get an API key + apiKey = login(server) + + // Analyze episodes and save report + fmt.Println(" [+] Analyzing episodes") + fmt.Print("\033[37;1m") // change the color of the verifier's text + RunProgram( + "./verifier/verifier", + []string{ + "-address", server.Address, + "-key", apiKey, "-o", + fmt.Sprintf("reports/%s-%d.json", server.Comment, start.Unix())}, + 5*time.Minute) + fmt.Print("\033[39;0m") // reset terminal text color + + // Setup base Selenium arguments + seleniumArgs = []string{ + "-u", // force stdout to be unbuffered + "main.py", + "-host", server.Address, + "-user", server.Username, + "-pass", server.Password, + "-name", config.Common.Episode} + + // Append all requested Selenium tests + seleniumArgs = append(seleniumArgs, "--tests") + seleniumArgs = append(seleniumArgs, server.Tests...) + + // Append all requested browsers + seleniumArgs = append(seleniumArgs, "--browsers") + seleniumArgs = append(seleniumArgs, server.Browsers...) + + // Run Selenium + os.Chdir("selenium") + RunProgram("python3", seleniumArgs, time.Minute) + os.Chdir("..") + + cleanup: + if server.Docker { + // Stop the container + fmt.Println(" [+] Stopping and removing container") + RunProgram("docker", []string{"stop", "jf-e2e"}, 10*time.Second) + + // Cleanup the container's configuration + fmt.Printf(" [+] Deleting %s\n", configurationDirectory) + + if err := os.RemoveAll(configurationDirectory); err != nil { + panic(err) + } + } + } +} + +// Login to the specified Jellyfin server and return an API key +func login(server Server) string { + type AuthenticateUserByName struct { + AccessToken string + } + + fmt.Println(" [+] Sending authentication request") + + // Create request body + rawBody := fmt.Sprintf(`{"Username":"%s","Pw":"%s"}`, server.Username, server.Password) + body := bytes.NewBufferString(rawBody) + + // Create the request + req, err := http.NewRequest( + "POST", + fmt.Sprintf("%s/Users/AuthenticateByName", server.Address), + body) + + if err != nil { + panic(err) + } + + // Set headers + req.Header.Set("Content-Type", "application/json") + req.Header.Set( + "X-Emby-Authorization", + `MediaBrowser Client="JF E2E Tests", Version="0.0.1", DeviceId="E2E", Device="E2E"`) + + // Authenticate + res, err := http.DefaultClient.Do(req) + if err != nil { + panic(err) + } else if res.StatusCode != http.StatusOK { + panic(fmt.Sprintf("authentication returned code %d", res.StatusCode)) + } + + defer res.Body.Close() + + // Read body + fullBody, err := io.ReadAll(res.Body) + if err != nil { + panic(err) + } + + // Unmarshal body and return token + var token AuthenticateUserByName + if err := json.Unmarshal(fullBody, &token); err != nil { + panic(err) + } + + return token.AccessToken +} + +// Wait up to ten seconds for the provided Jellyfin server to fully startup +func waitForServerStartup(address string) { + attempts := 10 + fmt.Println(" [+] Waiting for server to finish starting") + + for { + // Sleep in between requests + time.Sleep(time.Second) + + // Ping the /System/Info/Public endpoint + res, err := http.Get(fmt.Sprintf("%s/System/Info/Public", address)) + + // If the server didn't return 200 OK, loop + if err != nil || res.StatusCode != http.StatusOK { + if attempts--; attempts <= 0 { + panic("server is taking too long to startup") + } + + continue + } + + // Assume startup has finished, break + break + } +} + +// Read configuration from config.json +func loadConfiguration() Configuration { + var config Configuration + + // Load the contents of the configuration file + raw, err := os.ReadFile("config.json") + if err != nil { + panic(err) + } + + // Unmarshal + if err := json.Unmarshal(raw, &config); err != nil { + panic(err) + } + + // Print debugging info + fmt.Printf("Library: %s\n", config.Common.Library) + fmt.Printf("Episode: \"%s\"\n", config.Common.Episode) + fmt.Println() + + // Check the validity of all entries + for i, server := range config.Servers { + // If this is an entry for a local container, ensure the server address is correct + if server.Image != "" { + // Ensure that values were provided for the host's IP address, base configuration directory, + // and a path to the compiled plugin DLL to install. + if containerAddress == "" { + panic("The -caddr argument is required.") + } + + if server.Base == "" { + panic("Original configuration directory is required") + } + + if pluginPath == "" { + panic("The -dll argument is required.") + } + + server.Address = fmt.Sprintf("http://%s:8097", containerAddress) + server.Docker = true + } + + // If no browsers were specified, default to Chrome (for speed) + if len(server.Browsers) == 0 { + server.Browsers = []string{"chrome"} + } + + // If no tests were specified, only test that the plugin settings page works + if len(server.Tests) == 0 { + server.Tests = []string{"settings"} + } + + // Verify that an address was provided + if len(server.Address) == 0 { + panic("Server address is required") + } + + fmt.Printf("===== Server: %s =====\n", server.Comment) + + if server.Skip { + fmt.Println("Skip: true") + } + + fmt.Printf("Docker: %t\n", server.Docker) + if server.Docker { + fmt.Printf("Image: %s\n", server.Image) + } + + fmt.Printf("Address: %s\n", server.Address) + fmt.Printf("Browsers: %v\n", server.Browsers) + fmt.Printf("Tests: %v\n", server.Tests) + fmt.Println() + + config.Servers[i] = server + } + + fmt.Println("=================") + + return config +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/wrapper/structs.go b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/wrapper/structs.go new file mode 100644 index 0000000..206c1f3 --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/wrapper/structs.go @@ -0,0 +1,26 @@ +package main + +type Configuration struct { + Common Common `json:"common"` + Servers []Server `json:"servers"` +} + +type Common struct { + Library string `json:"library"` + Episode string `json:"episode"` +} + +type Server struct { + Skip bool `json:"skip"` + Comment string `json:"comment"` + Address string `json:"address"` + Image string `json:"image"` + Username string `json:"username"` + Password string `json:"password"` + Base string `json:"base"` + Browsers []string `json:"browsers"` + Tests []string `json:"tests"` + + // These properties are set at runtime + Docker bool `json:"-"` +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/configPage.html b/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/configPage.html index 4a7970b..ec3401a 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/configPage.html +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/configPage.html @@ -48,7 +48,7 @@ -
+
EDL file generation
@@ -102,7 +102,7 @@
-
+
Modify introduction requirements
diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ConfusedPolarBear.Plugin.IntroSkipper.csproj b/ConfusedPolarBear.Plugin.IntroSkipper/ConfusedPolarBear.Plugin.IntroSkipper.csproj index 63ff668..664a3e1 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ConfusedPolarBear.Plugin.IntroSkipper.csproj +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ConfusedPolarBear.Plugin.IntroSkipper.csproj @@ -13,8 +13,8 @@ - - + + diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/SkipIntroController.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/SkipIntroController.cs index 4733c5e..d4d07b7 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/SkipIntroController.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/SkipIntroController.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Net.Mime; +using MediaBrowser.Controller.Entities.TV; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; @@ -73,11 +74,32 @@ public class SkipIntroController : ControllerBase /// Get all introductions. Only used by the end to end testing script. /// /// All introductions have been returned. - /// Dictionary of Intro objects. + /// List of IntroWithMetadata objects. [Authorize(Policy = "RequiresElevation")] [HttpGet("Intros/All")] - public ActionResult> GetAllIntros() + public ActionResult> GetAllIntros() { - return Plugin.Instance!.Intros; + List intros = new(); + + // Get metadata for all intros + foreach (var intro in Plugin.Instance!.Intros) + { + // Get the details of the item from Jellyfin + var rawItem = Plugin.Instance!.GetItem(intro.Key); + if (rawItem is not Episode episode) + { + throw new InvalidCastException("Unable to cast item id " + intro.Key + " to an Episode"); + } + + // Associate the metadata with the intro + intros.Add( + new IntroWithMetadata( + episode.SeriesName, + episode.AiredSeasonNumber ?? 0, + episode.Name, + intro.Value)); + } + + return intros; } } diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Data/Intro.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Data/Intro.cs index 3b174ee..0875b80 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Data/Intro.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Data/Intro.cs @@ -87,3 +87,42 @@ public class Intro return string.Format(System.Globalization.CultureInfo.InvariantCulture, "{0} {1} {2}", start, end, (int)action); } } + +/// +/// An Intro class with episode metadata. Only used in end to end testing programs. +/// +public class IntroWithMetadata : Intro +{ + /// + /// Initializes a new instance of the class. + /// + /// Series name. + /// Season number. + /// Episode title. + /// Intro timestamps. + public IntroWithMetadata(string series, int season, string title, Intro intro) + { + Series = series; + Season = season; + Title = title; + + EpisodeId = intro.EpisodeId; + IntroStart = intro.IntroStart; + IntroEnd = intro.IntroEnd; + } + + /// + /// Gets or sets the series name of the TV episode associated with this intro. + /// + public string Series { get; set; } + + /// + /// Gets or sets the season number of the TV episode associated with this intro. + /// + public int Season { get; set; } + + /// + /// Gets or sets the title of the TV episode associated with this intro. + /// + public string Title { get; set; } +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Plugin.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Plugin.cs index f14aad0..b53aec6 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Plugin.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Plugin.cs @@ -5,6 +5,7 @@ using ConfusedPolarBear.Plugin.IntroSkipper.Configuration; using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Plugins; using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Model.Plugins; using MediaBrowser.Model.Serialization; @@ -142,6 +143,11 @@ public class Plugin : BasePlugin, IHasWebPages } } + internal BaseItem GetItem(Guid id) + { + return _libraryManager.GetItemById(id); + } + /// /// Gets the full path for an item. /// @@ -149,7 +155,7 @@ public class Plugin : BasePlugin, IHasWebPages /// Full path to item. internal string GetItemPath(Guid id) { - return _libraryManager.GetItemById(id).Path; + return GetItem(id).Path; } /// diff --git a/docs/release.md b/docs/release.md index ac73c2a..7736337 100644 --- a/docs/release.md +++ b/docs/release.md @@ -7,18 +7,19 @@ ## Release plugin -1. Build release DLL with `dotnet build -c Release` -2. Zip release DLL -3. Update and commit latest changelog and manifest -4. Test plugin manifest +1. Run package plugin action and download bundle +2. Combine generated `manifest.json` with main plugin manifest +3. Test plugin manifest 1. Replace manifest URL with local IP address 2. Serve release ZIP and manifest with `python3 -m http.server` 3. Test updating plugin -5. Tag and push latest commit -6. Create release on GitHub with the following files: +4. Create release on GitHub with the following files: 1. Archived plugin DLL - 2. Latest web interface + 2. Link to the latest web interface ## Release container 1. Run publish container action +2. Update `latest` tag + 1. `docker tag ghcr.io/confusedpolarbear/jellyfin-intro-skipper:{COMMIT,latest}` + 2. `docker push ghcr.io/confusedpolarbear/jellyfin-intro-skipper:latest`