diff --git a/[help].bat b/[help].bat new file mode 100644 index 0000000..ed016c0 --- /dev/null +++ b/[help].bat @@ -0,0 +1,3 @@ +@echo off +poetry run wp dl -h +pause diff --git a/[install].bat b/[install].bat new file mode 100644 index 0000000..d64fe24 --- /dev/null +++ b/[install].bat @@ -0,0 +1,5 @@ +@echo off +python -m pip install poetry +poetry config virtualenvs.in-project true +poetry install +pause \ No newline at end of file diff --git a/[sync.db].bat b/[sync.db].bat new file mode 100644 index 0000000..56745cc --- /dev/null +++ b/[sync.db].bat @@ -0,0 +1,3 @@ +@echo off +poetry run python scripts/MergeKeyStores.py -i "H:\BDdownload\key_store.db" -o "H:\WPGSKD\wpgskd\key_store.db" +pause \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c08a6bb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,128 @@ +[build-system] +requires = ['poetry-core>=1.0.0'] +build-backend = 'poetry.core.masonry.api' + +[tool.poetry] +name = 'wpgskd' +version = '0.2.0' +description = 'Widevine, PlayReady, AES-128, and ClearKey downloader and decrypter' +authors = ["WPGSKD Contributors"] + +[tool.poetry.dependencies] +python = "^3.10" +subby = {path = "./scripts/subby", develop = true} +requests = {extras = ["socks"], version = "2.32.5"} +curl-cffi = "^0.6.0" +httpx = "^0.23.0" +lxml = "^5.3.0" +m3u8 = "^0.9.0" +isodate = "^0.6.1" +tldextract = "^3.1.0" +validators = "^0.18.2" +websocket-client = "^1.1.0" +pycryptodome = "^3.21.0" +pycryptodomex = "^3.4.3" +ecpy = "^1.2.5" +crccheck = "^1.0" +construct = "2.8.8" +protobuf = "^4.25.1" +base58 = "^2.1.1" +appdirs = "^1.4.4" +click = "^8.1.3" +coloredlogs = "^15.0" +rich = "^13.7.1" +pyyaml = "^6.0.1" +ruamel-yaml = "^0.18.10" +jsonpickle = "^2.0.0" +langcodes = {extras = ["data"], version = "^3.4.0"} +tqdm = "^4.67.1" +Unidecode = "^1.2.0" +pymediainfo = "^5.0.3" +defusedxml = "^0.7.1" +pproxy = "^2.7.7" +pysubs2 = "^1.6.1" +pycaption = "^2.1.1" +chardet = "^5.2.0" +ftfy = "^6.3.1" +pywidevine = "^1.8.0" +pyplayready = {git = "https://git.gay/ready-dl/pyplayready.git"} +numpy = "^1.26.0" +xmltodict = "^1.0.4" + +[tool.poetry.group.dev.dependencies] +flake8 = "^3.8.4" +isort = "^5.9.2" +pyinstaller = "^4.4" +ruff = "^0.6.0" +mypy = "^1.10.0" +bandit = "^1.7.9" +pre-commit = "^3.7.0" +pytest = "^8.0.0" +pytest-asyncio = "^0.23.0" +pytest-cov = "^5.0.0" +responses = "^0.25.0" +types-requests = "^2.31.0" +types-PyYAML = "^6.0.0" + +[tool.poetry.scripts] +wp = 'wpgskd.wpgskd:main' + +[tool.isort] +line_length = 120 +classes = ['CTV', 'FPS', 'IO', 'iTunes', 'MP4', 'TVNOW'] +extend_skip = ['scripts/pywidevine', 'scripts/subby', 'wpgskd/vendor'] + +[tool.ruff] +line-length = 120 +target-version = "py310" +force-exclude = true + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F", "W", "I", "UP", "B"] +ignore = ["E501"] + +[tool.ruff.lint.per-file-ignores] +"scripts/pywidevine/**" = ["ALL"] +"scripts/subby/**" = ["ALL"] +"wpgskd/vendor/**" = ["ALL"] +"tests/**" = ["B011"] + +[tool.ruff.format] +quote-style = "double" + +[tool.mypy] +python_version = "3.10" +ignore_missing_imports = true +follow_imports = "silent" +check_untyped_defs = false +disallow_untyped_defs = false +warn_unused_ignores = true +warn_redundant_casts = true +exclude = [ + "scripts/pywidevine/", + "scripts/subby/", + "wpgskd/vendor/", +] + +[tool.bandit] +exclude_dirs = ["tests", "scripts/pywidevine", "scripts/subby", "wpgskd/vendor"] +skips = [ + "B101", + "B324", + "B413", + "B314", + "B608", +] + +[tool.pytest.ini_options] +testpaths = ["tests"] +asyncio_mode = "auto" +addopts = "-ra --strict-markers" +markers = [ + "unit: fast, mocked tests (default)", + "slow: tests that may take >10s", + "live: end-to-end tests against real services (opt-in via --live)", +] +filterwarnings = [ + "ignore::DeprecationWarning", +] \ No newline at end of file diff --git a/scripts/AddKeysToKeyVault.py b/scripts/AddKeysToKeyVault.py new file mode 100644 index 0000000..a1bb07b --- /dev/null +++ b/scripts/AddKeysToKeyVault.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 + +import argparse +import re +import sqlite3 +import sys + +from wpgskd.utils.AtomicSQL import AtomicSQL + +""" +Add keys to key vault. File should have one KID:KEY per-line. +Optionally you can also put `:` at the end (after `KEY`). +""" + +parser = argparse.ArgumentParser( + "Key Vault DB batch adder/updater", + description="Simple script to add or update key information to a vinetrimmer key vault db" +) +parser.add_argument( + "-t", "--table", + help="table to store keys to. (e.g. amazon, netflix, disneyplus)", + required=True) +parser.add_argument( + "-i", "--input", + help="data used to parse from", + required=True) +parser.add_argument( + "-o", "--output", + help="key store db that will receive keys", + required=True) +parser.add_argument( + "-d", "--dry-run", + help="execute it, but never actually save/commit changes.", + action="store_true", required=False) +args = parser.parse_args() + +output_db = AtomicSQL() +output_db_id = output_db.load(sqlite3.connect(args.output)) + +# get all keys from input db +add_count = 0 +update_count = 0 +existed_count = 0 + +if args.input == "-": + input_ = sys.stdin.read() +else: + with open(args.input, encoding="utf-8") as fd: + input_ = fd.read() + +for line in input_.splitlines(keepends=False): + match = re.search(r"^(?P<kid>[0-9a-fA-F]{32}):(?P<key>[0-9a-fA-F]{32})(:(?P<title>[\w .:-]*))?$", line) + if not match: + continue + kid = match.group("kid").lower() + key = match.group("key").lower() + title = match.group("title") or None + + exists = output_db.safe_execute( + output_db_id, + lambda db, cursor: cursor.execute( + f"SELECT title FROM `{args.table}` WHERE `kid`=:kid", + {"kid": kid} + ) + ).fetchone() + + if exists: + if title and not exists[0]: + update_count += 1 + print(f"Updating {args.table} {kid}: {title}") + output_db.safe_execute( + output_db_id, + lambda db, cursor: cursor.execute( + f"UPDATE `{args.table}` SET `title`=:title", + {"title": title} + ) + ) + else: + existed_count += 1 + print(f"Key {args.table} {kid} already exists in the db with no differences, skipping...") + else: + add_count += 1 + print(f"Adding {args.table} {kid} ({title}): {key}") + output_db.safe_execute( + output_db_id, + lambda db, cursor: cursor.execute( + f"INSERT INTO `{args.table}` (kid, key_, title) VALUES (:kid, :key, :title)", + {"kid": kid, "key": key, "title": title} + ) + ) + +if args.dry_run: + print("--dry run enabled, have not commited any changes.") +else: + output_db.commit(output_db_id) + +print( + "Done!\n" + f"{add_count} added, {update_count} updated in some way, {existed_count} already existed (skipped)" +) diff --git a/scripts/ClientIDGen/ClientIDGen.py b/scripts/ClientIDGen/ClientIDGen.py new file mode 100644 index 0000000..375d520 --- /dev/null +++ b/scripts/ClientIDGen/ClientIDGen.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +import argparse +import base64 + +import yaml + +from pywidevine.protos.widevine_pb2 import ClientIdentificationRaw + +parser = argparse.ArgumentParser("Widevine Client ID building tool.") +parser.add_argument("-q", "--quiet", + help="do not print the generated client id", + action="store_true") +parser.add_argument("-c", "--config", + help="configuration yaml file", + default="config.yml") +parser.add_argument("-o", "--output", + default="device_client_id_blob", + help="output filename") +args = parser.parse_args() + +with open(args.config) as fd: + config = yaml.safe_load(fd) + +with open(config["token"], "rb") as fd: + token = fd.read() + +ci = ClientIdentificationRaw() +ci.Type = ClientIdentificationRaw.DEVICE_CERTIFICATE +ci.Token = token + +for name, value in config["client_info"].items(): + nv = ci.ClientInfo.add() + nv.Name = name + if name == "device_id": + value = base64.b64decode(value) + nv.Value = value + +capabilities = ClientIdentificationRaw.ClientCapabilities() +caps = config["capabilities"] +if "client_token" in caps: + capabilities.ClientToken = caps["client_token"] +if "session_token" in caps: + capabilities.SessionToken = caps["session_token"] +if "video_resolution_constraints" in caps: + capabilities.VideoResolutionConstraints = caps["video_resolution_constraints"] +if "max_hdcp_version" in caps: + max_hdcp_version = caps["max_hdcp_version"] + if str(max_hdcp_version).isdigit(): + max_hdcp_version = int(max_hdcp_version) + else: + max_hdcp_version = ClientIdentificationRaw.ClientCapabilities.HdcpVersion.Value(max_hdcp_version) + capabilities.MaxHdcpVersion = max_hdcp_version +if "oem_crypto_api_version" in caps: + capabilities.OemCryptoApiVersion = int(caps["oem_crypto_api_version"]) +# I have not seen any of the following in use: +if "anti_rollback_usage_table" in caps: + capabilities.AntiRollbackUsageTable = caps["anti_rollback_usage_table"] +if "srm_version" in caps: + capabilities.SrmVersion = int(caps["srm_version"]) +if "can_update_srm" in caps: + capabilities.ClientToken = caps["can_update_srm"] +# is it possible to refactor this? +if "supported_certificate_key_type" in caps: + supported_certificate_key_type = caps["supported_certificate_key_type"] + if str(supported_certificate_key_type).isdigit(): + supported_certificate_key_type = int(supported_certificate_key_type) + else: + supported_certificate_key_type = ClientIdentificationRaw.ClientCapabilities.CertificateKeyType.Value( + supported_certificate_key_type + ) + capabilities.SupportedCertificateKeyType.append(supported_certificate_key_type) +ci._ClientCapabilities.CopyFrom(capabilities) + +if not args.quiet: + print(ci) + +with open(args.output, "wb") as fd: + fd.write(ci.SerializeToString()) diff --git a/scripts/ClientIDGen/config.example.yml b/scripts/ClientIDGen/config.example.yml new file mode 100644 index 0000000..4376e70 --- /dev/null +++ b/scripts/ClientIDGen/config.example.yml @@ -0,0 +1,20 @@ +# NOTE! +# This client id gen script may use outdated ClientIdentification values. +# Just letting you know, do whatever you wish, but yeah + +token: 'token.bin' + +client_info: + company_name: 'motorola' + model_name: 'Nexus 6' + architecture_name: 'armeabi-v7a' + device_name: 'shamu' + product_name: 'shamu' + build_info: 'google/shamu/shamu:5.1.1/LMY48M/2167285:user/release-keys' + device_id: 'TU1JX0VGRkYwRkU2NUQ5OA==' + os_version: '5.1.12' + +capabilities: + session_token: 1 + max_hdcp_version: 'HDCP_V2_2' + oem_crypto_api_version: 11 diff --git a/scripts/GetVikiManifestFree.py b/scripts/GetVikiManifestFree.py new file mode 100644 index 0000000..32d5182 --- /dev/null +++ b/scripts/GetVikiManifestFree.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 + +import re +import sys + +import requests +from Cryptodome.Cipher import AES + +# create a session with a user agent +http = requests.Session() +http.headers.update({ + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0" +}) +# get player fragment page +fragment = http.get(sys.argv[1].replace("/videos/", "/player5_fragment/")).text +# get encrypted manifest urls for both hls and dash +encrypted_manifests = {k: bytes.fromhex(re.findall( + r'<source\s+type="application/' + v + r'"\s+src=".+?/e-stream-url\?stream=(.+?)"', + fragment +)[0][0]) for k, v in {"hls": "x-mpegURL", "dash": r"dash\+xml"}.items()} + +# decrypt all manifest urls in manifests +m = re.search(r"^\s*chabi:\s*'(.+?)'", fragment, re.MULTILINE) +if not m: + raise ValueError("Unable to get key") +key = m.group(1).encode() + +m = re.search(r"^\s*ecta:\s*'(.+?)'", fragment, re.MULTILINE) +if not m: + raise ValueError("Unable to get key") +iv = m.group(1).encode() + +manifests = {k: AES.new(key, AES.MODE_CBC, iv).decrypt(v).decode("utf-8") for k, v in encrypted_manifests.items()} +# print em out +print(manifests) diff --git a/scripts/MergeKeyStores.py b/scripts/MergeKeyStores.py new file mode 100644 index 0000000..0fea0de --- /dev/null +++ b/scripts/MergeKeyStores.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 + +import argparse +import sqlite3 +import os +import sys + +# Add path to import AtomicSQL +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from wpgskd.utils.AtomicSQL import AtomicSQL + +""" +Merge multiple Key Store DBs into one. +Correctly handles multi-table structure (one table per service). +""" + +parser = argparse.ArgumentParser( + "Key Store DB merger", + description="Script to merge one key store db into another" +) +parser.add_argument( + "-i", "--input", + help="key store db that will send keys (Source)", + required=True) +parser.add_argument( + "-o", "--output", + help="key store db that will receive keys (Target)", + required=True) +args = parser.parse_args() + +if not os.path.exists(args.input): + print(f"Input file not found: {args.input}") + sys.exit(1) + +# Ensure output dir exists +os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True) + +input_db = AtomicSQL() +input_id = input_db.load(sqlite3.connect(args.input)) + +output_db = AtomicSQL() +output_id = output_db.load(sqlite3.connect(args.output)) + +# 1. Get all table names from input DB +tables = input_db.safe_execute( + input_id, + lambda db, cursor: cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'") +).fetchall() + +tables = [t[0] for t in tables] +print(f"Found tables in input DB: {tables}") + +total_added = 0 +total_updated = 0 +total_skipped = 0 + +for table in tables: + print(f"\nProcessing table: {table}...") + + # 2. Ensure table exists in output DB + # We copy the schema from input if it doesn't exist in output + # But standard vault schema is: id, kid, key_, title + # To support 'type' column in future, we should check input columns + + # Get columns from input table + input_cols_info = input_db.safe_execute( + input_id, + lambda db, cursor: cursor.execute(f"PRAGMA table_info(`{table}`)") + ).fetchall() + input_cols = [col[1] for col in input_cols_info] + + # Check if table exists in output + out_table_exists = output_db.safe_execute( + output_id, + lambda db, cursor: cursor.execute("SELECT count(name) FROM sqlite_master WHERE type='table' AND name=?", [table]) + ).fetchone()[0] == 1 + + if not out_table_exists: + print(f" - Creating table {table} in output DB...") + # Standard creation from vaults.py, but let's try to be dynamic if we want 'type' support later + # For now, stick to standard schema to ensure compatibility with wpgskd + output_db.safe_execute( + output_id, + lambda db, cursor: cursor.execute( + f""" + CREATE TABLE IF NOT EXISTS `{table}` ( + "id" INTEGER NOT NULL UNIQUE, + "kid" TEXT NOT NULL COLLATE NOCASE, + "key_" TEXT NOT NULL COLLATE NOCASE, + "title" TEXT, + PRIMARY KEY("id" AUTOINCREMENT), + UNIQUE("kid", "key_") + ); + """ + ) + ) + # If input has 'type' column, we might want to add it? + # Let's handle Requirement 2 separately. + + # 3. Fetch all rows from input table + rows = input_db.safe_execute( + input_id, + lambda db, cursor: cursor.execute(f"SELECT kid, key_, title FROM `{table}`") + ).fetchall() + + for kid, key, title in rows: + # Check existence in output + exists = output_db.safe_execute( + output_id, + lambda db, cursor: cursor.execute( + f"SELECT title FROM `{table}` WHERE kid=? AND key_=?", + [kid, key] + ) + ).fetchone() + + if exists: + # Update title if missing + current_title = exists[0] + if title and not current_title: + output_db.safe_execute( + output_id, + lambda db, cursor: cursor.execute( + f"UPDATE `{table}` SET title=? WHERE kid=? AND key_=?", + (title, kid, key) + ) + ) + total_updated += 1 + # print(f" Updated {kid}") + else: + total_skipped += 1 + else: + # Insert + output_db.safe_execute( + output_id, + lambda db, cursor: cursor.execute( + f"INSERT INTO `{table}` (kid, key_, title) VALUES (?, ?, ?)", + (kid, key, title) + ) + ) + total_added += 1 + print(f" Added {kid}") + +output_db.commit(output_id) + +print("\n" + "="*30) +print(f"Merge Complete!") +print(f"Added: {total_added}") +print(f"Updated: {total_updated}") +print(f"Skipped: {total_skipped}") +print("="*30) \ No newline at end of file diff --git a/scripts/ParseClientID.py b/scripts/ParseClientID.py new file mode 100644 index 0000000..ee5169e --- /dev/null +++ b/scripts/ParseClientID.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 + +import argparse + +from pywidevine.device import LocalDevice +from pywidevine.protos.widevine_pb2 import ClientIdentification + +parser = argparse.ArgumentParser( + "Client identification parser", + description="Simple script to read a client id blob to see information about it" +) +parser.add_argument( + "input", + help="client id blob bin path or path to a wvd file", +) +args = parser.parse_args() + +client_id = ClientIdentification() +is_wvd = args.input.lower().endswith(".wvd") + +with open(args.input, "rb") as fd: + data = fd.read() + +if is_wvd: + client_id = LocalDevice.load(data).client_id +else: + client_id.ParseFromString(data) + +print(client_id) diff --git a/scripts/ParseKeybox.py b/scripts/ParseKeybox.py new file mode 100644 index 0000000..0bdbd7d --- /dev/null +++ b/scripts/ParseKeybox.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +import argparse + +from pywidevine.keybox import Keybox + +parser = argparse.ArgumentParser( + "Keybox parser", + description="Simple script to read a keybox to see information about it" +) +parser.add_argument( + "-k", "--keybox", + help="keybox path", + required=True) +args = parser.parse_args() + +keybox = Keybox.load(args.keybox) +print(repr(keybox)) diff --git a/scripts/ParsePSSH.py b/scripts/ParsePSSH.py new file mode 100644 index 0000000..4533af7 --- /dev/null +++ b/scripts/ParsePSSH.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +import argparse +import base64 + +from pywidevine.protos.widevine_pb2 import WidevineCencHeader +from wpgskd.vendor.pymp4.parser import Box + +parser = argparse.ArgumentParser( + "PSSH parser", + description="Simple script to read a PSSH to see information about it" +) +parser.add_argument( + "input", +) +args = parser.parse_args() + +args.input = base64.b64decode(args.input.encode("utf-8")) +box = Box.parse(args.input) +cenc_header = WidevineCencHeader() +cenc_header.ParseFromString(box.init_data) + +print("pssh box:") +print(box) + +print("init_data parsed as WidevineCencHeader:") +print(cenc_header) + +print("init_data's key_id as hex:") +print(cenc_header.key_id[0].hex()) diff --git a/scripts/TOMLtoYAML.py b/scripts/TOMLtoYAML.py new file mode 100644 index 0000000..f9d12da --- /dev/null +++ b/scripts/TOMLtoYAML.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 + +import argparse +import json +import os + +import toml +import yaml + +parser = argparse.ArgumentParser() +parser.add_argument("path", help="directory containing .toml files to convert") +args = parser.parse_args() + +for root, dirs, files in os.walk(args.path): + for f in files: + if f.endswith(".toml"): + data = toml.load(os.path.join(root, f)) + # Convert to a real dict instead of weird toml object that pyyaml can't handle + data = json.loads(json.dumps(data)) + with open(os.path.join(root, f"{os.path.splitext(f)[0]}.yml"), "w") as fd: + print(f"Writing {os.path.realpath(fd.name)}") + fd.write(yaml.safe_dump(data, sort_keys=False)) diff --git a/scripts/UpdateLocalKeyVault.py b/scripts/UpdateLocalKeyVault.py new file mode 100644 index 0000000..561184b --- /dev/null +++ b/scripts/UpdateLocalKeyVault.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 + +import argparse +import json +import sqlite3 + +from wpgskd.utils.AtomicSQL import AtomicSQL + + +class LocalVault: + def __init__(self, vault_path): + """ + Update local key vault to newer system. + This should ONLY be run if you have the old structure with keys in a table named `keys`. + It will move and update the structure of the items in `keys` to their respective new locations and structure. + :param vault_path: sqlite db path + """ + self.adb = AtomicSQL() + self.ticket = self.adb.load(sqlite3.connect(vault_path)) + if not self.table_exists("keys"): + return + rows = self.adb.safe_execute( + self.ticket, + lambda db, cursor: cursor.execute("SELECT `service`, `title`, `content_keys` FROM `keys`") + ).fetchall() + for service, title, content_keys in rows: + service = service.lower() + content_keys = json.loads(content_keys) + if not self.table_exists(service): + self.create_table(service) + for kid, key in [x.split(":") for x in content_keys]: + print(f"Inserting: {kid} {key} {title}") + existing_row, existing_title = self.row_exists(service, kid, key) + if existing_row: + if title and not existing_title: + print(" -- exists, but the title doesn't, so ill merge") + self.adb.safe_execute( + self.ticket, + lambda db, cursor: cursor.execute( + f"UPDATE `{service}` SET `title`=? WHERE `kid`=? AND `key_`=?", + (title, kid, key) + ) + ) + continue + print(" -- skipping (exists already)") + continue + self.adb.safe_execute( + self.ticket, + lambda db, cursor: cursor.execute( + f"INSERT INTO `{service}` (kid, key_, title) VALUES (?, ?, ?)", + (kid, key, title) + ) + ) + self.adb.commit(self.ticket) + + def row_exists(self, table, kid, key): + return self.adb.safe_execute( + self.ticket, + lambda db, cursor: cursor.execute( + f"SELECT count(id), title FROM `{table}` WHERE kid=? AND key_=?", + [kid, key] + ) + ).fetchone() + + def table_exists(self, name): + return self.adb.safe_execute( + self.ticket, + lambda db, cursor: cursor.execute( + "SELECT count(name) FROM sqlite_master WHERE type='table' AND name=?", + [name.lower()] + ) + ).fetchone()[0] == 1 + + def create_table(self, name): + self.adb.safe_execute( + self.ticket, + lambda db, cursor: cursor.execute( + """ + CREATE TABLE {} ( + "id" INTEGER NOT NULL UNIQUE, + "kid" TEXT NOT NULL COLLATE NOCASE, + "key_" TEXT NOT NULL COLLATE NOCASE, + "title" TEXT NULL, + PRIMARY KEY("id" AUTOINCREMENT), + UNIQUE("kid", "key_") + ); + """.format(name.lower()) + ) + ) + + +parser = argparse.ArgumentParser() +parser.add_argument( + "-i", "--input", + help="vault", + required=True) +args = parser.parse_args() + +LocalVault(args.input) diff --git a/scripts/VMPBlobGen/README.md b/scripts/VMPBlobGen/README.md new file mode 100644 index 0000000..1adc6a5 --- /dev/null +++ b/scripts/VMPBlobGen/README.md @@ -0,0 +1,8 @@ +# VMPBlobGen + +Notes on VMP: + +- Android doesn't require (or use!) a VMP blob (the oemcrypto hardware backs it and HDCP controls the path) +- Chrome and WidevineCDM both have signature files. The widevinecdm.dll and chrome.exe sign both the signature files, + then sign with the private key and inject to the license request in field 7, but you need a server cert to encrypt + the challenge otherwise. diff --git a/scripts/VMPBlobGen/VMPBlobGen.py b/scripts/VMPBlobGen/VMPBlobGen.py new file mode 100644 index 0000000..7151683 --- /dev/null +++ b/scripts/VMPBlobGen/VMPBlobGen.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 + +import os +import sys +from hashlib import sha512 + +from pywidevine.protos.widevine_pb2 import FileHashes +from pywidevine.vmp import WidevineSignatureReader + +""" +Script that generates a VMP blob for chromecdm +""" + +WIN32_FILES = [ + "chrome.exe", + "chrome.dll", + "chrome_child.dll", + "widevinecdmadapter.dll", + "widevinecdm.dll" +] + + +def sha512file(filename): + """Compute SHA-512 digest of file.""" + sha = sha512() + with open(filename, "rb") as fd: + for b in iter(lambda: fd.read(0x10000), b''): + sha.update(b) + return sha.digest() + + +def build_vmp_field(filenames): + """ + Create and fill out a FileHashes object. + + `filenames` is an array of pairs of filenames like (file, file_signature) + such as ("module.dll", "module.dll.sig"). This does not validate the signature + against the codesign root CA, or even the sha512 hash against the current signature+signer + """ + file_hashes = FileHashes() + + for basename, file, sig in filenames: + signature = WidevineSignatureReader.from_file(sig) + s = file_hashes.signatures.add() + s.filename = basename + s.test_signing = False # we can't check this without parsing signer + s.SHA512Hash = sha512file(file) + s.main_exe = signature.mainexe + s.signature = signature.signature + + file_hashes.signer = signature.signer + return file_hashes.SerializeToString() + + +def get_files_with_signatures(path, required_files=None, random_order=False, sig_ext="sig"): + """ + use on chrome dir (a given version). + random_order would put any files it found in the dir with sigs, + it's not the right way to do it and the browser does not do this. + this function can still fail (generate wrong output) in subtle ways if + the Chrome dir has copies of the exe/sigs, especially if those copies are modified in some way + """ + if not required_files: + required_files = WIN32_FILES + + all_files = [] + sig_files = [] + for dir_path, _, filenames in os.walk(path): + for filename in filenames: + full_path = os.path.join(dir_path, filename) + all_files.append(full_path) + if filename.endswith(sig_ext): + sig_files.append(full_path) + + base_names = [] + for path in sig_files: + orig_path = os.path.splitext(path)[0] + if orig_path not in all_files: + print("signature file {} lacks original file {}".format(path, orig_path)) + base_names.append(path.name) + + if not set(base_names).issuperset(set(required_files)): + # or should just make this warn as the next exception would be more specific + raise ValueError("Missing a binary/signature pair from {}".format(required_files)) + + files_to_hash = [] + if random_order: + for path in sig_files: + orig_path = os.path.splitext(path)[0] + files_to_hash.append((os.path.basename(orig_path), orig_path, path)) + else: + for basename in required_files: + found_file = False + for path in sig_files: + orig_path = os.path.splitext(path)[0] + if orig_path.endswith(basename): + files_to_hash.append((basename, orig_path, path)) + found_file = True + break + if not found_file: + raise Exception("Failed to locate a file sig/pair for {}".format(basename)) + + return files_to_hash + + +def make_vmp_buff(browser_dir, file_msg_out): + with open(file_msg_out, "wb") as fd: + fd.write(build_vmp_field(get_files_with_signatures(browser_dir))) + + +if len(sys.argv) < 3: + print("Usage: {} BrowserPathWithVersion OutputPBMessage.bin".format(sys.argv[0])) +else: + make_vmp_buff(sys.argv[1], sys.argv[2]) diff --git a/scripts/WVD/JsonWVDtoStructWVD.py b/scripts/WVD/JsonWVDtoStructWVD.py new file mode 100644 index 0000000..eb6a310 --- /dev/null +++ b/scripts/WVD/JsonWVDtoStructWVD.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 + +import argparse +import base64 +import json +import os + +from pywidevine.device import LocalDevice + +""" +Code to convert common folder/file structure to a vinetrimmer WVD. +""" + +parser = argparse.ArgumentParser( + "JsonWVDtoStructWVD", + description="Simple script to read cdm data from old wvd json and write it into a new WVD struct file." +) +parser.add_argument( + "-i", "--input", + help="path to wvd json file", + required=False) +parser.add_argument( + "-d", "--dir", + help="path to MULTIPLE wvd json files", + required=False) +args = parser.parse_args() + +files = [] +if args.dir: + files.extend(os.listdir(args.dir)) +elif args.input: + files.append(args.input) + +for file in files: + if not file.lower().endswith(".wvd") or os.path.splitext(file)[0].endswith(".struct"): + continue + + if not os.path.isfile(file): + raise ValueError("Not a file or doesn't exist...") + + print(f"Generating wvd struct file for {file}...") + + with open(file, encoding="utf-8") as fd: + wvd_json = json.load(fd) + + device = LocalDevice( + type=LocalDevice.Types[wvd_json["device_type"].upper()], + security_level=wvd_json["security_level"], + flags={ + "send_key_control_nonce": wvd_json["send_key_control_nonce"] + }, + private_key=base64.b64decode(wvd_json["device_private_key"]), + client_id=base64.b64decode(wvd_json["device_client_id_blob"]), + vmp=base64.b64decode(wvd_json["device_vmp_blob"]) if wvd_json.get("device_vmp_blob") else None + ) + + out = os.path.join(os.path.dirname(file), "structs", os.path.basename(file)) + os.makedirs(os.path.dirname(out), exist_ok=True) + + device.dump(out) + + print(device) + print(f"Done: {file}") + +print("Done") diff --git a/scripts/WVD/MakeWVD.py b/scripts/WVD/MakeWVD.py new file mode 100644 index 0000000..8e998a2 --- /dev/null +++ b/scripts/WVD/MakeWVD.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +import argparse +import json +import os +import re +import sys + +from pywidevine.device import LocalDevice + +""" +Code to convert common folder/file structure to a vinetrimmer WVD. +""" + +parser = argparse.ArgumentParser() +parser.add_argument("dirs", metavar="DIR", nargs="+", help="Directory containing device files") +args = parser.parse_args() + +configs = [] +for d in args.dirs: + for root, dirs, files in os.walk(d): + for f in files: + if f == "wv.json": + configs.append(os.path.join(root, f)) + +if not configs: + print("No wv.json file found in any of the specified directories.") + sys.exit(1) + +for f in configs: + d = os.path.dirname(f) + + print(f"Generating WVD struct file for {os.path.abspath(d)}...") + + with open(f, encoding="utf-8") as fd: + config = json.load(fd) + + device = LocalDevice.from_dir(d) + + # we cannot output to /data/CDM_Devices etc. as the CWD might not align up + # also best to keep the security level and system id definition on the filename for easy referencing + name = re.sub(r"_lvl\d$", "", config["name"]) + out_path = f"{name}_l{device.security_level}_{device.system_id}.wvd" + + device.dump(out_path) + + print(device) + + print(f"Done, saved to: {os.path.abspath(out_path)}") + print() diff --git a/scripts/migrate_credentials.py b/scripts/migrate_credentials.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/subby/LICENSE b/scripts/subby/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/scripts/subby/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + <program> Copyright (C) <year> <name of author> + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +<https://www.gnu.org/licenses/>. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +<https://www.gnu.org/licenses/why-not-lgpl.html>. diff --git a/scripts/subby/README.md b/scripts/subby/README.md new file mode 100644 index 0000000..1bb6fcb --- /dev/null +++ b/scripts/subby/README.md @@ -0,0 +1,137 @@ +# Subby +Advanced subtitle converter and processor. + +# Supported formats +WebVTT, DFXP/TTML/TTML2/SMPTE, SAMI, WVTT (WebVTT in MP4), STPP/ISMT (DFXP in MP4), JSON (Bilibili) + +# Functionality +- converts supported input format to SRT +- retains select formatting tags (italics, basic \an8 positioning) +- corrects often found flaws in subtitles +- opinionated timing and formatting improvements + +# Installation +``` +git clone https://github.com/vevv/subby +cd subby +pip install . +``` + +# Usage notes +`CommonIssuesFixer` should be ran both after conversion and SDH stripping +as it's designed to fix source issues, including ones which can cause playback problems. + +`CommonIssuesFixer` removes short gaps (2 frames) by default. +This can be disabled by setting `CommonIssuesFixer.remove_gaps` to `False` before running. + +`subby.SubRipFile` accepts similar methods to `pysrt.SubRipFile`, but isn't a fully compatible replacement. +Only `from_string`, `clean_indexes`, `export`, `save` are guaranteed to work. + +This object is otherwise just a list storing `srt.Subtitle` elements. + +## Language specific fixing + +As of 0.3.6, both `CommonIssuesFixer` and `SDHStripper` support a language parameter, +which accepts a BCP47 language code. + +This is currently used only for RTL tagging in CommonIssuesFixer. + +**It is highly recommended for every script to pass it for future use.** + +# Command line usage +``` +Usage: subby [OPTIONS] COMMAND [ARGS]... + + Subby—Advanced Subtitle Converter and Processor. + +Options: + -d, --debug Enable DEBUG level logs. + --help Show this message and exit. + +Commands: + convert Convert a Subtitle to SubRip (SRT). + process SubRip (SRT) post-processing. + version Print version information. +``` +Example + +``` +subby process /path/to/subs/subs.srt strip-sdh +``` + +# Library usage +## Converter +```py +from subby import WebVTTConverter +from pathlib import Path + +converter = WebVTTConverter() +file = Path('test.vtt') + +# All statements below are equivalent +srt = converter.from_file(file) +srt = converter.from_string(file.read_text()) +srt = converter.from_bytes(file.read_bytes()) + +# srt is subby.SubRipFile + +output = Path('file.srt') +srt.save(output) +# saved to file.srt +``` + +## Processor +Processor returns a bool indicating success - whether any changes were made, useful for determining if SDH subtitles should be saved. + +```py +from subby import CommonIssuesFixer +from pathlib import Path + +processor = CommonIssuesFixer() +file = Path('test.vtt') + +# All statements below are equivalent +srt, status = processor.from_file(file) +srt, status = processor.from_string(file.read_text()) +srt, status = processor.from_bytes(file.read_bytes()) + +# srt is subby.SubRipFile, status is bool + +output = Path('test_fixed.srt') +srt.save(output) +# saved to test_fixed.srt +``` + +## Chaining +The following example will convert a VTT file, attempt to strip SDH, and then save the result. + +```py +from subby import WebVTTConverter, CommonIssuesFixer, SDHStripper +from pathlib import Path + +converter = WebVTTConverter() +fixer = CommonIssuesFixer() +stripper = SDHStripper() + +file = Path('file.vtt') +file_sdh = Path('file_sdh.srt') +file_stripped = Path('file_stripped.srt') +srt, _ = fixer.from_srt(converter.from_file(file)) + +srt.save(file_sdh) +# saved to file_sdh.srt + +stripped, status = stripper.from_srt(srt) +if status is True: + print('stripping successful') + stripped.save(file_stripped) + # saved to file_stripped.srt +``` + +## Tests +To run tests, go to the "tests" directory and run `pytest`. + +## Contributors + +<a href="https://github.com/vevv"><img src="https://images.weserv.nl/?url=avatars.githubusercontent.com/u/68520787?v=4&h=25&w=25&fit=cover&mask=circle&maxage=7d" alt=""/></a> +<a href="https://github.com/rlaphoenix"><img src="https://images.weserv.nl/?url=avatars.githubusercontent.com/u/17136956?v=4&h=25&w=25&fit=cover&mask=circle&maxage=7d" alt=""/></a> diff --git a/scripts/subby/poetry.lock b/scripts/subby/poetry.lock new file mode 100644 index 0000000..56a17ed --- /dev/null +++ b/scripts/subby/poetry.lock @@ -0,0 +1,539 @@ +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. + +[[package]] +name = "beautifulsoup4" +version = "4.12.3" +description = "Screen-scraping library" +optional = false +python-versions = ">=3.6.0" +files = [ + {file = "beautifulsoup4-4.12.3-py3-none-any.whl", hash = "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"}, + {file = "beautifulsoup4-4.12.3.tar.gz", hash = "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051"}, +] + +[package.dependencies] +soupsieve = ">1.2" + +[package.extras] +cchardet = ["cchardet"] +chardet = ["chardet"] +charset-normalizer = ["charset-normalizer"] +html5lib = ["html5lib"] +lxml = ["lxml"] + +[[package]] +name = "click" +version = "8.1.7" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "construct" +version = "2.8.8" +description = "A powerful declarative parser/builder for binary data" +optional = false +python-versions = "*" +files = [ + {file = "construct-2.8.8.tar.gz", hash = "sha256:1b84b8147f6fd15bcf64b737c3e8ac5100811ad80c830cb4b2545140511c4157"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.2" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, + {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "langcodes" +version = "3.4.1" +description = "Tools for labeling human languages with IETF language tags" +optional = false +python-versions = ">=3.8" +files = [ + {file = "langcodes-3.4.1-py3-none-any.whl", hash = "sha256:68f686fc3d358f222674ecf697ddcee3ace3c2fe325083ecad2543fd28a20e77"}, + {file = "langcodes-3.4.1.tar.gz", hash = "sha256:a24879fed238013ac3af2424b9d1124e38b4a38b2044fd297c8ff38e5912e718"}, +] + +[package.dependencies] +language-data = ">=1.2" + +[package.extras] +build = ["build", "twine"] +test = ["pytest", "pytest-cov"] + +[[package]] +name = "language-data" +version = "1.3.0" +description = "Supplementary data about languages used by the langcodes module" +optional = false +python-versions = "*" +files = [ + {file = "language_data-1.3.0-py3-none-any.whl", hash = "sha256:e2ee943551b5ae5f89cd0e801d1fc3835bb0ef5b7e9c3a4e8e17b2b214548fbf"}, + {file = "language_data-1.3.0.tar.gz", hash = "sha256:7600ef8aa39555145d06c89f0c324bf7dab834ea0b0a439d8243762e3ebad7ec"}, +] + +[package.dependencies] +marisa-trie = ">=1.1.0" + +[package.extras] +build = ["build", "twine"] +test = ["pytest", "pytest-cov"] + +[[package]] +name = "lxml" +version = "5.3.0" +description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +optional = false +python-versions = ">=3.6" +files = [ + {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:dd36439be765e2dde7660212b5275641edbc813e7b24668831a5c8ac91180656"}, + {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ae5fe5c4b525aa82b8076c1a59d642c17b6e8739ecf852522c6321852178119d"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:501d0d7e26b4d261fca8132854d845e4988097611ba2531408ec91cf3fd9d20a"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb66442c2546446944437df74379e9cf9e9db353e61301d1a0e26482f43f0dd8"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9e41506fec7a7f9405b14aa2d5c8abbb4dbbd09d88f9496958b6d00cb4d45330"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f7d4a670107d75dfe5ad080bed6c341d18c4442f9378c9f58e5851e86eb79965"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41ce1f1e2c7755abfc7e759dc34d7d05fd221723ff822947132dc934d122fe22"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:44264ecae91b30e5633013fb66f6ddd05c006d3e0e884f75ce0b4755b3e3847b"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_ppc64le.whl", hash = "sha256:3c174dc350d3ec52deb77f2faf05c439331d6ed5e702fc247ccb4e6b62d884b7"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_s390x.whl", hash = "sha256:2dfab5fa6a28a0b60a20638dc48e6343c02ea9933e3279ccb132f555a62323d8"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b1c8c20847b9f34e98080da785bb2336ea982e7f913eed5809e5a3c872900f32"}, + {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2c86bf781b12ba417f64f3422cfc302523ac9cd1d8ae8c0f92a1c66e56ef2e86"}, + {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:c162b216070f280fa7da844531169be0baf9ccb17263cf5a8bf876fcd3117fa5"}, + {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:36aef61a1678cb778097b4a6eeae96a69875d51d1e8f4d4b491ab3cfb54b5a03"}, + {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f65e5120863c2b266dbcc927b306c5b78e502c71edf3295dfcb9501ec96e5fc7"}, + {file = "lxml-5.3.0-cp310-cp310-win32.whl", hash = "sha256:ef0c1fe22171dd7c7c27147f2e9c3e86f8bdf473fed75f16b0c2e84a5030ce80"}, + {file = "lxml-5.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:052d99051e77a4f3e8482c65014cf6372e61b0a6f4fe9edb98503bb5364cfee3"}, + {file = "lxml-5.3.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74bcb423462233bc5d6066e4e98b0264e7c1bed7541fff2f4e34fe6b21563c8b"}, + {file = "lxml-5.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a3d819eb6f9b8677f57f9664265d0a10dd6551d227afb4af2b9cd7bdc2ccbf18"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b8f5db71b28b8c404956ddf79575ea77aa8b1538e8b2ef9ec877945b3f46442"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c3406b63232fc7e9b8783ab0b765d7c59e7c59ff96759d8ef9632fca27c7ee4"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2ecdd78ab768f844c7a1d4a03595038c166b609f6395e25af9b0f3f26ae1230f"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:168f2dfcfdedf611eb285efac1516c8454c8c99caf271dccda8943576b67552e"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa617107a410245b8660028a7483b68e7914304a6d4882b5ff3d2d3eb5948d8c"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:69959bd3167b993e6e710b99051265654133a98f20cec1d9b493b931942e9c16"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:bd96517ef76c8654446fc3db9242d019a1bb5fe8b751ba414765d59f99210b79"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:ab6dd83b970dc97c2d10bc71aa925b84788c7c05de30241b9e96f9b6d9ea3080"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:eec1bb8cdbba2925bedc887bc0609a80e599c75b12d87ae42ac23fd199445654"}, + {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6a7095eeec6f89111d03dabfe5883a1fd54da319c94e0fb104ee8f23616b572d"}, + {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f651ebd0b21ec65dfca93aa629610a0dbc13dbc13554f19b0113da2e61a4763"}, + {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:f422a209d2455c56849442ae42f25dbaaba1c6c3f501d58761c619c7836642ec"}, + {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:62f7fdb0d1ed2065451f086519865b4c90aa19aed51081979ecd05a21eb4d1be"}, + {file = "lxml-5.3.0-cp311-cp311-win32.whl", hash = "sha256:c6379f35350b655fd817cd0d6cbeef7f265f3ae5fedb1caae2eb442bbeae9ab9"}, + {file = "lxml-5.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:9c52100e2c2dbb0649b90467935c4b0de5528833c76a35ea1a2691ec9f1ee7a1"}, + {file = "lxml-5.3.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:e99f5507401436fdcc85036a2e7dc2e28d962550afe1cbfc07c40e454256a859"}, + {file = "lxml-5.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:384aacddf2e5813a36495233b64cb96b1949da72bef933918ba5c84e06af8f0e"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:874a216bf6afaf97c263b56371434e47e2c652d215788396f60477540298218f"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65ab5685d56914b9a2a34d67dd5488b83213d680b0c5d10b47f81da5a16b0b0e"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aac0bbd3e8dd2d9c45ceb82249e8bdd3ac99131a32b4d35c8af3cc9db1657179"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b369d3db3c22ed14c75ccd5af429086f166a19627e84a8fdade3f8f31426e52a"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c24037349665434f375645fa9d1f5304800cec574d0310f618490c871fd902b3"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:62d172f358f33a26d6b41b28c170c63886742f5b6772a42b59b4f0fa10526cb1"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:c1f794c02903c2824fccce5b20c339a1a14b114e83b306ff11b597c5f71a1c8d"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:5d6a6972b93c426ace71e0be9a6f4b2cfae9b1baed2eed2006076a746692288c"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:3879cc6ce938ff4eb4900d901ed63555c778731a96365e53fadb36437a131a99"}, + {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:74068c601baff6ff021c70f0935b0c7bc528baa8ea210c202e03757c68c5a4ff"}, + {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ecd4ad8453ac17bc7ba3868371bffb46f628161ad0eefbd0a855d2c8c32dd81a"}, + {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:7e2f58095acc211eb9d8b5771bf04df9ff37d6b87618d1cbf85f92399c98dae8"}, + {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e63601ad5cd8f860aa99d109889b5ac34de571c7ee902d6812d5d9ddcc77fa7d"}, + {file = "lxml-5.3.0-cp312-cp312-win32.whl", hash = "sha256:17e8d968d04a37c50ad9c456a286b525d78c4a1c15dd53aa46c1d8e06bf6fa30"}, + {file = "lxml-5.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:c1a69e58a6bb2de65902051d57fde951febad631a20a64572677a1052690482f"}, + {file = "lxml-5.3.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8c72e9563347c7395910de6a3100a4840a75a6f60e05af5e58566868d5eb2d6a"}, + {file = "lxml-5.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e92ce66cd919d18d14b3856906a61d3f6b6a8500e0794142338da644260595cd"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d04f064bebdfef9240478f7a779e8c5dc32b8b7b0b2fc6a62e39b928d428e51"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c2fb570d7823c2bbaf8b419ba6e5662137f8166e364a8b2b91051a1fb40ab8b"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0c120f43553ec759f8de1fee2f4794452b0946773299d44c36bfe18e83caf002"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:562e7494778a69086f0312ec9689f6b6ac1c6b65670ed7d0267e49f57ffa08c4"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:423b121f7e6fa514ba0c7918e56955a1d4470ed35faa03e3d9f0e3baa4c7e492"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:c00f323cc00576df6165cc9d21a4c21285fa6b9989c5c39830c3903dc4303ef3"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:1fdc9fae8dd4c763e8a31e7630afef517eab9f5d5d31a278df087f307bf601f4"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:658f2aa69d31e09699705949b5fc4719cbecbd4a97f9656a232e7d6c7be1a367"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:1473427aff3d66a3fa2199004c3e601e6c4500ab86696edffdbc84954c72d832"}, + {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a87de7dd873bf9a792bf1e58b1c3887b9264036629a5bf2d2e6579fe8e73edff"}, + {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0d7b36afa46c97875303a94e8f3ad932bf78bace9e18e603f2085b652422edcd"}, + {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:cf120cce539453ae086eacc0130a324e7026113510efa83ab42ef3fcfccac7fb"}, + {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:df5c7333167b9674aa8ae1d4008fa4bc17a313cc490b2cca27838bbdcc6bb15b"}, + {file = "lxml-5.3.0-cp313-cp313-win32.whl", hash = "sha256:c802e1c2ed9f0c06a65bc4ed0189d000ada8049312cfeab6ca635e39c9608957"}, + {file = "lxml-5.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:406246b96d552e0503e17a1006fd27edac678b3fcc9f1be71a2f94b4ff61528d"}, + {file = "lxml-5.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8f0de2d390af441fe8b2c12626d103540b5d850d585b18fcada58d972b74a74e"}, + {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1afe0a8c353746e610bd9031a630a95bcfb1a720684c3f2b36c4710a0a96528f"}, + {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56b9861a71575f5795bde89256e7467ece3d339c9b43141dbdd54544566b3b94"}, + {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:9fb81d2824dff4f2e297a276297e9031f46d2682cafc484f49de182aa5e5df99"}, + {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:2c226a06ecb8cdef28845ae976da407917542c5e6e75dcac7cc33eb04aaeb237"}, + {file = "lxml-5.3.0-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:7d3d1ca42870cdb6d0d29939630dbe48fa511c203724820fc0fd507b2fb46577"}, + {file = "lxml-5.3.0-cp36-cp36m-win32.whl", hash = "sha256:094cb601ba9f55296774c2d57ad68730daa0b13dc260e1f941b4d13678239e70"}, + {file = "lxml-5.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:eafa2c8658f4e560b098fe9fc54539f86528651f61849b22111a9b107d18910c"}, + {file = "lxml-5.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cb83f8a875b3d9b458cada4f880fa498646874ba4011dc974e071a0a84a1b033"}, + {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:25f1b69d41656b05885aa185f5fdf822cb01a586d1b32739633679699f220391"}, + {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23e0553b8055600b3bf4a00b255ec5c92e1e4aebf8c2c09334f8368e8bd174d6"}, + {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ada35dd21dc6c039259596b358caab6b13f4db4d4a7f8665764d616daf9cc1d"}, + {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:81b4e48da4c69313192d8c8d4311e5d818b8be1afe68ee20f6385d0e96fc9512"}, + {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:2bc9fd5ca4729af796f9f59cd8ff160fe06a474da40aca03fcc79655ddee1a8b"}, + {file = "lxml-5.3.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:07da23d7ee08577760f0a71d67a861019103e4812c87e2fab26b039054594cc5"}, + {file = "lxml-5.3.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:ea2e2f6f801696ad7de8aec061044d6c8c0dd4037608c7cab38a9a4d316bfb11"}, + {file = "lxml-5.3.0-cp37-cp37m-win32.whl", hash = "sha256:5c54afdcbb0182d06836cc3d1be921e540be3ebdf8b8a51ee3ef987537455f84"}, + {file = "lxml-5.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:f2901429da1e645ce548bf9171784c0f74f0718c3f6150ce166be39e4dd66c3e"}, + {file = "lxml-5.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c56a1d43b2f9ee4786e4658c7903f05da35b923fb53c11025712562d5cc02753"}, + {file = "lxml-5.3.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ee8c39582d2652dcd516d1b879451500f8db3fe3607ce45d7c5957ab2596040"}, + {file = "lxml-5.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fdf3a3059611f7585a78ee10399a15566356116a4288380921a4b598d807a22"}, + {file = "lxml-5.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:146173654d79eb1fc97498b4280c1d3e1e5d58c398fa530905c9ea50ea849b22"}, + {file = "lxml-5.3.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:0a7056921edbdd7560746f4221dca89bb7a3fe457d3d74267995253f46343f15"}, + {file = "lxml-5.3.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:9e4b47ac0f5e749cfc618efdf4726269441014ae1d5583e047b452a32e221920"}, + {file = "lxml-5.3.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:f914c03e6a31deb632e2daa881fe198461f4d06e57ac3d0e05bbcab8eae01945"}, + {file = "lxml-5.3.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:213261f168c5e1d9b7535a67e68b1f59f92398dd17a56d934550837143f79c42"}, + {file = "lxml-5.3.0-cp38-cp38-win32.whl", hash = "sha256:218c1b2e17a710e363855594230f44060e2025b05c80d1f0661258142b2add2e"}, + {file = "lxml-5.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:315f9542011b2c4e1d280e4a20ddcca1761993dda3afc7a73b01235f8641e903"}, + {file = "lxml-5.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:1ffc23010330c2ab67fac02781df60998ca8fe759e8efde6f8b756a20599c5de"}, + {file = "lxml-5.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2b3778cb38212f52fac9fe913017deea2fdf4eb1a4f8e4cfc6b009a13a6d3fcc"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b0c7a688944891086ba192e21c5229dea54382f4836a209ff8d0a660fac06be"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:747a3d3e98e24597981ca0be0fd922aebd471fa99d0043a3842d00cdcad7ad6a"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86a6b24b19eaebc448dc56b87c4865527855145d851f9fc3891673ff97950540"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b11a5d918a6216e521c715b02749240fb07ae5a1fefd4b7bf12f833bc8b4fe70"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68b87753c784d6acb8a25b05cb526c3406913c9d988d51f80adecc2b0775d6aa"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:109fa6fede314cc50eed29e6e56c540075e63d922455346f11e4d7a036d2b8cf"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_ppc64le.whl", hash = "sha256:02ced472497b8362c8e902ade23e3300479f4f43e45f4105c85ef43b8db85229"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_s390x.whl", hash = "sha256:6b038cc86b285e4f9fea2ba5ee76e89f21ed1ea898e287dc277a25884f3a7dfe"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:7437237c6a66b7ca341e868cda48be24b8701862757426852c9b3186de1da8a2"}, + {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7f41026c1d64043a36fda21d64c5026762d53a77043e73e94b71f0521939cc71"}, + {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:482c2f67761868f0108b1743098640fbb2a28a8e15bf3f47ada9fa59d9fe08c3"}, + {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:1483fd3358963cc5c1c9b122c80606a3a79ee0875bcac0204149fa09d6ff2727"}, + {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:2dec2d1130a9cda5b904696cec33b2cfb451304ba9081eeda7f90f724097300a"}, + {file = "lxml-5.3.0-cp39-cp39-win32.whl", hash = "sha256:a0eabd0a81625049c5df745209dc7fcef6e2aea7793e5f003ba363610aa0a3ff"}, + {file = "lxml-5.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:89e043f1d9d341c52bf2af6d02e6adde62e0a46e6755d5eb60dc6e4f0b8aeca2"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7b1cd427cb0d5f7393c31b7496419da594fe600e6fdc4b105a54f82405e6626c"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51806cfe0279e06ed8500ce19479d757db42a30fd509940b1701be9c86a5ff9a"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee70d08fd60c9565ba8190f41a46a54096afa0eeb8f76bd66f2c25d3b1b83005"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:8dc2c0395bea8254d8daebc76dcf8eb3a95ec2a46fa6fae5eaccee366bfe02ce"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:6ba0d3dcac281aad8a0e5b14c7ed6f9fa89c8612b47939fc94f80b16e2e9bc83"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:6e91cf736959057f7aac7adfc83481e03615a8e8dd5758aa1d95ea69e8931dba"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:94d6c3782907b5e40e21cadf94b13b0842ac421192f26b84c45f13f3c9d5dc27"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c300306673aa0f3ed5ed9372b21867690a17dba38c68c44b287437c362ce486b"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78d9b952e07aed35fe2e1a7ad26e929595412db48535921c5013edc8aa4a35ce"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:01220dca0d066d1349bd6a1726856a78f7929f3878f7e2ee83c296c69495309e"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:2d9b8d9177afaef80c53c0a9e30fa252ff3036fb1c6494d427c066a4ce6a282f"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:20094fc3f21ea0a8669dc4c61ed7fa8263bd37d97d93b90f28fc613371e7a875"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ace2c2326a319a0bb8a8b0e5b570c764962e95818de9f259ce814ee666603f19"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:92e67a0be1639c251d21e35fe74df6bcc40cba445c2cda7c4a967656733249e2"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd5350b55f9fecddc51385463a4f67a5da829bc741e38cf689f38ec9023f54ab"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:4c1fefd7e3d00921c44dc9ca80a775af49698bbfd92ea84498e56acffd4c5469"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:71a8dd38fbd2f2319136d4ae855a7078c69c9a38ae06e0c17c73fd70fc6caad8"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:97acf1e1fd66ab53dacd2c35b319d7e548380c2e9e8c54525c6e76d21b1ae3b1"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:68934b242c51eb02907c5b81d138cb977b2129a0a75a8f8b60b01cb8586c7b21"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b710bc2b8292966b23a6a0121f7a6c51d45d2347edcc75f016ac123b8054d3f2"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18feb4b93302091b1541221196a2155aa296c363fd233814fa11e181adebc52f"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:3eb44520c4724c2e1a57c0af33a379eee41792595023f367ba3952a2d96c2aab"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:609251a0ca4770e5a8768ff902aa02bf636339c5a93f9349b48eb1f606f7f3e9"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:516f491c834eb320d6c843156440fe7fc0d50b33e44387fcec5b02f0bc118a4c"}, + {file = "lxml-5.3.0.tar.gz", hash = "sha256:4e109ca30d1edec1ac60cdbe341905dc3b8f55b16855e03a54aaf59e51ec8c6f"}, +] + +[package.extras] +cssselect = ["cssselect (>=0.7)"] +html-clean = ["lxml-html-clean"] +html5 = ["html5lib"] +htmlsoup = ["BeautifulSoup4"] +source = ["Cython (>=3.0.11)"] + +[[package]] +name = "lxml-stubs" +version = "0.4.0" +description = "Type annotations for the lxml package" +optional = false +python-versions = "*" +files = [ + {file = "lxml-stubs-0.4.0.tar.gz", hash = "sha256:184877b42127256abc2b932ba8bd0ab5ea80bd0b0fee618d16daa40e0b71abee"}, + {file = "lxml_stubs-0.4.0-py3-none-any.whl", hash = "sha256:3b381e9e82397c64ea3cc4d6f79d1255d015f7b114806d4826218805c10ec003"}, +] + +[package.extras] +test = ["coverage[toml] (==5.2)", "pytest (>=6.0.0)", "pytest-mypy-plugins (==1.9.3)"] + +[[package]] +name = "marisa-trie" +version = "1.2.1" +description = "Static memory-efficient and fast Trie-like structures for Python." +optional = false +python-versions = ">=3.7" +files = [ + {file = "marisa_trie-1.2.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a2eb41d2f9114d8b7bd66772c237111e00d2bae2260824560eaa0a1e291ce9e8"}, + {file = "marisa_trie-1.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9e956e6a46f604b17d570901e66f5214fb6f658c21e5e7665deace236793cef6"}, + {file = "marisa_trie-1.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bd45142501300e7538b2e544905580918b67b1c82abed1275fe4c682c95635fa"}, + {file = "marisa_trie-1.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8443d116c612cfd1961fbf76769faf0561a46d8e317315dd13f9d9639ad500c"}, + {file = "marisa_trie-1.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:875a6248e60fbb48d947b574ffa4170f34981f9e579bde960d0f9a49ea393ecc"}, + {file = "marisa_trie-1.2.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:746a7c60a17fccd3cfcfd4326926f02ea4fcdfc25d513411a0c4fc8e4a1ca51f"}, + {file = "marisa_trie-1.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e70869737cc0e5bd903f620667da6c330d6737048d1f44db792a6af68a1d35be"}, + {file = "marisa_trie-1.2.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:06b099dd743676dbcd8abd8465ceac8f6d97d8bfaabe2c83b965495523b4cef2"}, + {file = "marisa_trie-1.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d2a82eb21afdaf22b50d9b996472305c05ca67fc4ff5a026a220320c9c961db6"}, + {file = "marisa_trie-1.2.1-cp310-cp310-win32.whl", hash = "sha256:8951e7ce5d3167fbd085703b4cbb3f47948ed66826bef9a2173c379508776cf5"}, + {file = "marisa_trie-1.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:5685a14b3099b1422c4f59fa38b0bf4b5342ee6cc38ae57df9666a0b28eeaad3"}, + {file = "marisa_trie-1.2.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ed3fb4ed7f2084597e862bcd56c56c5529e773729a426c083238682dba540e98"}, + {file = "marisa_trie-1.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0fe69fb9ffb2767746181f7b3b29bbd3454d1d24717b5958e030494f3d3cddf3"}, + {file = "marisa_trie-1.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4728ed3ae372d1ea2cdbd5eaa27b8f20a10e415d1f9d153314831e67d963f281"}, + {file = "marisa_trie-1.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8cf4f25cf895692b232f49aa5397af6aba78bb679fb917a05fce8d3cb1ee446d"}, + {file = "marisa_trie-1.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7cca7f96236ffdbf49be4b2e42c132e3df05968ac424544034767650913524de"}, + {file = "marisa_trie-1.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d7eb20bf0e8b55a58d2a9b518aabc4c18278787bdba476c551dd1c1ed109e509"}, + {file = "marisa_trie-1.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b1ec93f0d1ee6d7ab680a6d8ea1a08bf264636358e92692072170032dda652ba"}, + {file = "marisa_trie-1.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e2699255d7ac610dee26d4ae7bda5951d05c7d9123a22e1f7c6a6f1964e0a4e4"}, + {file = "marisa_trie-1.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c484410911182457a8a1a0249d0c09c01e2071b78a0a8538cd5f7fa45589b13a"}, + {file = "marisa_trie-1.2.1-cp311-cp311-win32.whl", hash = "sha256:ad548117744b2bcf0e3d97374608be0a92d18c2af13d98b728d37cd06248e571"}, + {file = "marisa_trie-1.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:436f62d27714970b9cdd3b3c41bdad046f260e62ebb0daa38125ef70536fc73b"}, + {file = "marisa_trie-1.2.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:638506eacf20ca503fff72221a7e66a6eadbf28d6a4a6f949fcf5b1701bb05ec"}, + {file = "marisa_trie-1.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:de1665eaafefa48a308e4753786519888021740501a15461c77bdfd57638e6b4"}, + {file = "marisa_trie-1.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f713af9b8aa66a34cd3a78c7d150a560a75734713abe818a69021fd269e927fa"}, + {file = "marisa_trie-1.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b2a7d00f53f4945320b551bccb826b3fb26948bde1a10d50bb9802fabb611b10"}, + {file = "marisa_trie-1.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98042040d1d6085792e8d0f74004fc0f5f9ca6091c298f593dd81a22a4643854"}, + {file = "marisa_trie-1.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6532615111eec2c79e711965ece0bc95adac1ff547a7fff5ffca525463116deb"}, + {file = "marisa_trie-1.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:20948e40ab2038e62b7000ca6b4a913bc16c91a2c2e6da501bd1f917eeb28d51"}, + {file = "marisa_trie-1.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:66b23e5b35dd547f85bf98db7c749bc0ffc57916ade2534a6bbc32db9a4abc44"}, + {file = "marisa_trie-1.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6704adf0247d2dda42e876b793be40775dff46624309ad99bc7537098bee106d"}, + {file = "marisa_trie-1.2.1-cp312-cp312-win32.whl", hash = "sha256:3ad356442c2fea4c2a6f514738ddf213d23930f942299a2b2c05df464a00848a"}, + {file = "marisa_trie-1.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:f2806f75817392cedcacb24ac5d80b0350dde8d3861d67d045c1d9b109764114"}, + {file = "marisa_trie-1.2.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:b5ea16e69bfda0ac028c921b58de1a4aaf83d43934892977368579cd3c0a2554"}, + {file = "marisa_trie-1.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9f627f4e41be710b6cb6ed54b0128b229ac9d50e2054d9cde3af0fef277c23cf"}, + {file = "marisa_trie-1.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5e649f3dc8ab5476732094f2828cc90cac3be7c79bc0c8318b6fda0c1d248db4"}, + {file = "marisa_trie-1.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:46e528ee71808c961baf8c3ce1c46a8337ec7a96cc55389d11baafe5b632f8e9"}, + {file = "marisa_trie-1.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36aa4401a1180615f74d575571a6550081d84fc6461e9aefc0bb7b2427af098e"}, + {file = "marisa_trie-1.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce59bcd2cda9bb52b0e90cc7f36413cd86c3d0ce7224143447424aafb9f4aa48"}, + {file = "marisa_trie-1.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f4cd800704a5fc57e53c39c3a6b0c9b1519ebdbcb644ede3ee67a06eb542697d"}, + {file = "marisa_trie-1.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2428b495003c189695fb91ceeb499f9fcced3a2dce853e17fa475519433c67ff"}, + {file = "marisa_trie-1.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:735c363d9aaac82eaf516a28f7c6b95084c2e176d8231c87328dc80e112a9afa"}, + {file = "marisa_trie-1.2.1-cp313-cp313-win32.whl", hash = "sha256:eba6ca45500ca1a042466a0684aacc9838e7f20fe2605521ee19f2853062798f"}, + {file = "marisa_trie-1.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:aa7cd17e1c690ce96c538b2f4aae003d9a498e65067dd433c52dd069009951d4"}, + {file = "marisa_trie-1.2.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5e43891a37b0d7f618819fea14bd951289a0a8e3dd0da50c596139ca83ebb9b1"}, + {file = "marisa_trie-1.2.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6946100a43f933fad6bc458c502a59926d80b321d5ac1ed2ff9c56605360496f"}, + {file = "marisa_trie-1.2.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a4177dc0bd1374e82be9b2ba4d0c2733b0a85b9d154ceeea83a5bee8c1e62fbf"}, + {file = "marisa_trie-1.2.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f35c2603a6be168088ed1db6ad1704b078aa8f39974c60888fbbced95dcadad4"}, + {file = "marisa_trie-1.2.1-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:d659fda873d8dcb2c14c2c331de1dee21f5a902d7f2de7978b62c6431a8850ef"}, + {file = "marisa_trie-1.2.1-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:b0ef26733d3c836be79e812071e1a431ce1f807955a27a981ebb7993d95f842b"}, + {file = "marisa_trie-1.2.1-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:536ea19ce6a2ce61c57fed4123ecd10d18d77a0db45cd2741afff2b8b68f15b3"}, + {file = "marisa_trie-1.2.1-cp37-cp37m-win32.whl", hash = "sha256:0ee6cf6a16d9c3d1c94e21c8e63c93d8b34bede170ca4e937e16e1c0700d399f"}, + {file = "marisa_trie-1.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7e7b1786e852e014d03e5f32dbd991f9a9eb223dd3fa9a2564108b807e4b7e1c"}, + {file = "marisa_trie-1.2.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:952af3a5859c3b20b15a00748c36e9eb8316eb2c70bd353ae1646da216322908"}, + {file = "marisa_trie-1.2.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24a81aa7566e4ec96fc4d934581fe26d62eac47fc02b35fa443a0bb718b471e8"}, + {file = "marisa_trie-1.2.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9c9b32b14651a6dcf9e8857d2df5d29d322a1ea8c0be5c8ffb88f9841c4ec62b"}, + {file = "marisa_trie-1.2.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ac170d20b97beb75059ba65d1ccad6b434d777c8992ab41ffabdade3b06dd74"}, + {file = "marisa_trie-1.2.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da4e4facb79614cc4653cfd859f398e4db4ca9ab26270ff12610e50ed7f1f6c6"}, + {file = "marisa_trie-1.2.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:25688f34cac3bec01b4f655ffdd6c599a01f0bd596b4a79cf56c6f01a7df3560"}, + {file = "marisa_trie-1.2.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:1db3213b451bf058d558f6e619bceff09d1d130214448a207c55e1526e2773a1"}, + {file = "marisa_trie-1.2.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:d5648c6dcc5dc9200297fb779b1663b8a4467bda034a3c69bd9c32d8afb33b1d"}, + {file = "marisa_trie-1.2.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:5bd39a4e1cc839a88acca2889d17ebc3f202a5039cd6059a13148ce75c8a6244"}, + {file = "marisa_trie-1.2.1-cp38-cp38-win32.whl", hash = "sha256:594f98491a96c7f1ffe13ce292cef1b4e63c028f0707effdea0f113364c1ae6c"}, + {file = "marisa_trie-1.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:5fe5a286f997848a410eebe1c28657506adaeb405220ee1e16cfcfd10deb37f2"}, + {file = "marisa_trie-1.2.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c0fe2ace0cb1806badbd1c551a8ec2f8d4cf97bf044313c082ef1acfe631ddca"}, + {file = "marisa_trie-1.2.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:67f0c2ec82c20a02c16fc9ba81dee2586ef20270127c470cb1054767aa8ba310"}, + {file = "marisa_trie-1.2.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a3c98613180cf1730e221933ff74b454008161b1a82597e41054127719964188"}, + {file = "marisa_trie-1.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:429858a0452a7bedcf67bc7bb34383d00f666c980cb75a31bcd31285fbdd4403"}, + {file = "marisa_trie-1.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2eacb84446543082ec50f2fb563f1a94c96804d4057b7da8ed815958d0cdfbe"}, + {file = "marisa_trie-1.2.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:852d7bcf14b0c63404de26e7c4c8d5d65ecaeca935e93794331bc4e2f213660b"}, + {file = "marisa_trie-1.2.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:e58788004adda24c401d1751331618ed20c507ffc23bfd28d7c0661a1cf0ad16"}, + {file = "marisa_trie-1.2.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:aefe0973cc4698e0907289dc0517ab0c7cdb13d588201932ff567d08a50b0e2e"}, + {file = "marisa_trie-1.2.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:6c50c861faad0a5c091bd763e0729f958c316e678dfa065d3984fbb9e4eacbcd"}, + {file = "marisa_trie-1.2.1-cp39-cp39-win32.whl", hash = "sha256:b1ce340da608530500ab4f963f12d6bfc8d8680900919a60dbdc9b78c02060a4"}, + {file = "marisa_trie-1.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:ce37d8ca462bb64cc13f529b9ed92f7b21fe8d1f1679b62e29f9cb7d0e888b49"}, + {file = "marisa_trie-1.2.1.tar.gz", hash = "sha256:3a27c408e2aefc03e0f1d25b2ff2afb85aac3568f6fa2ae2a53b57a2e87ce29d"}, +] + +[package.dependencies] +setuptools = "*" + +[package.extras] +test = ["hypothesis", "pytest", "readme-renderer"] + +[[package]] +name = "packaging" +version = "24.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, + {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, +] + +[[package]] +name = "pluggy" +version = "1.5.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pymp4" +version = "1.4.0" +description = "Python parser for MP4 boxes" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "pymp4-1.4.0-py3-none-any.whl", hash = "sha256:3401666c1e2a97ac94dffb18c5a5dcbd46d0a436da5272d378a6f9f6506dd12d"}, + {file = "pymp4-1.4.0.tar.gz", hash = "sha256:bc9e77732a8a143d34c38aa862a54180716246938e4bf3e07585d19252b77bb5"}, +] + +[package.dependencies] +construct = "2.8.8" + +[[package]] +name = "pytest" +version = "7.4.4" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, + {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "setuptools" +version = "75.3.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-75.3.0-py3-none-any.whl", hash = "sha256:f2504966861356aa38616760c0f66568e535562374995367b4e69c7143cf6bcd"}, + {file = "setuptools-75.3.0.tar.gz", hash = "sha256:fba5dd4d766e97be1b1681d98712680ae8f2f26d7881245f2ce9e40714f1a686"}, +] + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.5.2)"] +core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.collections", "jaraco.functools", "jaraco.text (>=3.7)", "more-itertools", "more-itertools (>=8.8)", "packaging", "packaging (>=24)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test (>=5.5)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib-metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.12.*)", "pytest-mypy"] + +[[package]] +name = "soupsieve" +version = "2.6" +description = "A modern CSS selector implementation for Beautiful Soup." +optional = false +python-versions = ">=3.8" +files = [ + {file = "soupsieve-2.6-py3-none-any.whl", hash = "sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9"}, + {file = "soupsieve-2.6.tar.gz", hash = "sha256:e2e68417777af359ec65daac1057404a3c8a5455bb8abc36f1a9866ab1a51abb"}, +] + +[[package]] +name = "srt" +version = "3.5.3" +description = "A tiny library for parsing, modifying, and composing SRT files." +optional = false +python-versions = ">=2.7" +files = [ + {file = "srt-3.5.3.tar.gz", hash = "sha256:4884315043a4f0740fd1f878ed6caa376ac06d70e135f306a6dc44632eed0cc0"}, +] + +[[package]] +name = "tinycss" +version = "0.4" +description = "tinycss is a complete yet simple CSS parser for Python." +optional = false +python-versions = "*" +files = [ + {file = "tinycss-0.4.tar.gz", hash = "sha256:12306fb50e5e9e7eaeef84b802ed877488ba80e35c672867f548c0924a76716e"}, +] + +[package.extras] +test = ["pytest-cov", "pytest-flake8", "pytest-isort", "pytest-runner"] + +[[package]] +name = "tomli" +version = "2.2.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, + {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8"}, + {file = "tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff"}, + {file = "tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b"}, + {file = "tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea"}, + {file = "tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e"}, + {file = "tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98"}, + {file = "tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4"}, + {file = "tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7"}, + {file = "tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744"}, + {file = "tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec"}, + {file = "tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69"}, + {file = "tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc"}, + {file = "tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.8" +content-hash = "d1dc09a1492c59373465886d52b572d761f7e370239ae93c63a7c88f4a69c47f" diff --git a/scripts/subby/pyproject.toml b/scripts/subby/pyproject.toml new file mode 100644 index 0000000..fac100f --- /dev/null +++ b/scripts/subby/pyproject.toml @@ -0,0 +1,33 @@ +[tool.poetry] +name = "subby" +version = "0.3.16" +description = "Advanced subtitle converter and processor" +authors = ["vevv"] +license = "GPL-3.0-or-later" +readme = "README.md" +repository = "https://github.com/vevv/subby" + +[tool.poetry.dependencies] +python = "^3.8" +pymp4 = "~1.4.0" +beautifulsoup4 = "^4.11.2" +tinycss = "^0.4" +click = "^8.1.3" +srt = "^3.5.3" +lxml = "^5.3.0" +langcodes = "^3.4.0" + +[tool.poetry.group.dev] +optional = true + +[tool.poetry.group.dev.dependencies] +pytest = "^7.4.3" +lxml-stubs = "^0.4.0" + + +[tool.poetry.scripts] +subby = "subby.cli:main" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/scripts/subby/setup.py b/scripts/subby/setup.py new file mode 100644 index 0000000..1abbd06 --- /dev/null +++ b/scripts/subby/setup.py @@ -0,0 +1,4 @@ +import setuptools + +if __name__ == "__main__": + setuptools.setup() diff --git a/scripts/subby/subby/__init__.py b/scripts/subby/subby/__init__.py new file mode 100644 index 0000000..aea9a6d --- /dev/null +++ b/scripts/subby/subby/__init__.py @@ -0,0 +1,25 @@ +from subby.converters.bilibili_json import BilibiliJSONConverter +from subby.converters.mp4 import ISMTConverter, WVTTConverter +from subby.converters.sami import SAMIConverter +from subby.converters.smpte import SMPTEConverter +from subby.converters.webvtt import WebVTTConverter +from subby.processors.common_issues import CommonIssuesFixer +from subby.processors.sdh import SDHStripper +from subby.subripfile import SubRipFile + +__version__ = '0.3.16' + +__all__ = [ + # Converters + 'SAMIConverter', + 'SMPTEConverter', 'ISMTConverter', + 'WebVTTConverter', 'WVTTConverter', + 'BilibiliJSONConverter', + # Processors + 'CommonIssuesFixer', + 'SDHStripper', + # Utility + 'SubRipFile', + # Version + '__version__' +] diff --git a/scripts/subby/subby/cli.py b/scripts/subby/subby/cli.py new file mode 100644 index 0000000..b70de80 --- /dev/null +++ b/scripts/subby/subby/cli.py @@ -0,0 +1,217 @@ +from __future__ import annotations + +import logging +from datetime import datetime +from pathlib import Path + +import click + +from subby import (BilibiliJSONConverter, CommonIssuesFixer, ISMTConverter, + SAMIConverter, SDHStripper, SMPTEConverter, WebVTTConverter, + WVTTConverter, __version__) + + +@click.group() +@click.option("-d", "--debug", is_flag=True, default=False, help="Enable debug level logs.") +def main(debug: bool) -> None: + """Subby—Advanced Subtitle Converter and Processor.""" + logging.basicConfig(level=logging.DEBUG if debug else logging.INFO) + logging.getLogger('srt').setLevel(logging.DEBUG if debug else logging.CRITICAL) + + +@main.command() +def version(): + """Print version information.""" + log = logging.getLogger(__name__) + + copyright_years = 2023 + current_year = datetime.now().year + if copyright_years != current_year: + copyright_years = f"{copyright_years}-{current_year}" + + log.info("Subby version %s Copyright (c) %s vevv", __version__, copyright_years) + log.info("https://github.com/vevv/subby") + + +@main.command() +@click.argument("file", type=Path) +@click.option("-o", "--out", type=Path, default=None, help="Output path.") +@click.option( + "-l", + "--language", + type=str, + default=None, + help="Subtitle language (used for language specific processing)" +) +@click.option( + "-e", + "--encoding", + type=str, + default="utf-8", + help="Character encoding (default: utf-8)." +) +@click.option( + "-n", + "--no-post-processing", + is_flag=True, + default=False, + help="Disable post-processing after conversion." +) +@click.option( + "-g", + "--keep-short-gaps", + is_flag=True, + help="Keep short gaps between lines (< 85 ms; only with post-processing enabled)" +) +def convert( + file: Path, + out: Path | None, + language: str, + encoding: str, + no_post_processing: bool, + keep_short_gaps: bool +): + """Convert a Subtitle to SubRip (SRT).""" + if not isinstance(file, Path): + raise click.ClickException(f"Expected file to be a {Path} not {file!r}") + if out and not isinstance(out, Path): + raise click.ClickException(f"Expected out to be a {Path} not {out!r}") + + if not out: + out = file.with_suffix(".srt") + + log = logging.getLogger("convert") + + data = file.read_bytes() + converter = None + + if b"mdat" in data and b"moof" in data: + if b"</tt>" in data: + log.info("Subtitle format: ISMT (DFXP in MP4)") + converter = ISMTConverter() + elif b"vttc" in data: + log.info("Subtitle format: WVTT (WebVTT in MP4)") + converter = WVTTConverter() + elif b"<SAMI>" in data: + log.info("Subtitle format: SAMI") + converter = SAMIConverter() + elif b"</tt>" in data or b"</tt:tt>" in data: + log.info("Subtitle format: DFXP/TTML/TTML2") + converter = SMPTEConverter() + elif b"WEBVTT" in data: + log.info("Subtitle format: WebVTT") + converter = WebVTTConverter() + elif data.startswith(b'{') and b'"Stroke"' in data and b'"background_color"' in data: + log.info("Subtitle format: JSON (Bilibili)") + converter = BilibiliJSONConverter() + + if not converter: + log.error("Subtitle format was unrecognized...") + return + + srt = converter.from_file(file) + log.info("Converted subtitle to SubRip (SRT)") + + if not no_post_processing: + processor = CommonIssuesFixer() + processor.remove_gaps = not keep_short_gaps + srt, status = processor.from_srt(srt, language=language) + log.info(f"Processed subtitle {['but no issues were found...', 'and repaired some issues!'][status]}") + + srt.save(out, encoding=encoding) + log.info(f"Saved to: {out}") + log.debug(f"Used character encoding {encoding}") + + +@main.group() +@click.argument("file", type=Path) +@click.option("-o", "--out", type=Path, default=None, help="Output path.") +@click.option( + "-l", + "--language", + type=str, + default=None, + help="Subtitle language (used for language specific processing)" +) +@click.option( + "-e", + "--encoding", + type=str, + default="utf-8", + help="Character encoding (default: utf-8)." +) +@click.option( + "-n", + "--no-post-processing", + is_flag=True, + default=False, + help="Disable post-processing after SDH stripping." +) +@click.option( + "-g", + "--keep-short-gaps", + is_flag=True, + help="Keep short gaps between lines (< 85 ms)" +) +def process(file: Path, out: Path | None, **__): + """SubRip (SRT) post-processing.""" + if not isinstance(file, Path): + raise click.ClickException(f"Expected file to be a {Path} not {file!r}") + if out and not isinstance(out, Path): + raise click.ClickException(f"Expected out to be a {Path} not {out!r}") + + +@process.command() +@click.pass_context +def mend(ctx: click.Context): + """Repair or Mend common issues in a Subtitle.""" + file = ctx.parent.params["file"] + + if not ctx.parent.params["out"]: + ctx.parent.params["out"] = file.with_stem(file.stem + "_mend") + + log = logging.getLogger("process.mend") + + processor = CommonIssuesFixer() + processor.remove_gaps = not ctx.parent.params["keep_short_gaps"] + processed_srt, status = processor.from_file(file, language=ctx.parent.params["language"]) + log.info(f"Processed subtitle {['but no issues were found...', 'and repaired some issues!'][status]}") + + return processed_srt, status + + +@process.command("strip-sdh") +@click.pass_context +def strip_sdh(ctx: click.Context): + """Remove Hard-of-hearing descriptions from Subtitles.""" + file = ctx.parent.params["file"] + + if not ctx.parent.params["out"]: + ctx.parent.params["out"] = file.with_stem(file.stem + "_sdh_stripped") + + log = logging.getLogger("process.strip_sdh") + + processor = SDHStripper() + processed_srt, status = processor.from_file(file, language=ctx.parent.params["language"]) + log.info(f"Processed subtitle {['but no SDH descriptions were found...', 'and removed SDH!'][status]}") + + if not ctx.parent.params["no_post_processing"]: + processor = CommonIssuesFixer() + processor.remove_gaps = not ctx.parent.params["keep_short_gaps"] + processed_srt, _ = processor.from_srt(processed_srt, language=ctx.parent.params["language"]) + log.info( + "Processed stripped subtitle " + + ['but no issues were found...', 'and repaired some issues!'][status] + ) + + return processed_srt, status + + +@process.result_callback() +def process_result(result, out, encoding, *_, **__): + log = logging.getLogger("process") + processed_srt, status = result + if status: + processed_srt.save(out, encoding=encoding) + log.info(f"Saved to: {out}") + log.debug(f"Used character encoding {encoding}") diff --git a/scripts/subby/subby/converters/__init__.py b/scripts/subby/subby/converters/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/subby/subby/converters/base.py b/scripts/subby/subby/converters/base.py new file mode 100644 index 0000000..4464f9c --- /dev/null +++ b/scripts/subby/subby/converters/base.py @@ -0,0 +1,27 @@ +from abc import ABC, abstractmethod +from io import BytesIO +from pathlib import Path +from typing import BinaryIO + +from subby.subripfile import SubRipFile + + +class BaseConverter(ABC): + """Base subtitle converter class""" + + def from_file(self, file: Path) -> SubRipFile: + """Reads a given file and converts it to srt""" + with file.open(mode='rb') as stream: + return self.parse(stream) + + def from_string(self, data: str) -> SubRipFile: + """Reads a given string and converts it to srt""" + return self.parse(BytesIO(data.encode('utf-8'))) + + def from_bytes(self, data: bytes) -> SubRipFile: + """Parses given data and converts it to srt""" + return self.parse(BytesIO(data)) + + @abstractmethod + def parse(self, stream: BinaryIO) -> SubRipFile: + """Parses data from a given stream and converts it to srt""" diff --git a/scripts/subby/subby/converters/bilibili_json.py b/scripts/subby/subby/converters/bilibili_json.py new file mode 100644 index 0000000..2559cc8 --- /dev/null +++ b/scripts/subby/subby/converters/bilibili_json.py @@ -0,0 +1,27 @@ +import datetime +import json + +from srt import Subtitle + +from subby.converters.base import BaseConverter +from subby.subripfile import SubRipFile + + +class BilibiliJSONConverter(BaseConverter): + """Bilibili JSON subtitle converter""" + + def parse(self, stream): + json_data = json.load(stream) + srt = SubRipFile() + for i, line in enumerate(json_data['body']): + if line['location'] != 2: + line['content'] = ('{\\an%s}' % line['location']) + line['content'] + + srt.append(Subtitle( + index=i, + start=datetime.timedelta(seconds=line['from']), + end=datetime.timedelta(seconds=line['to']), + content=line['content'] + )) + + return srt diff --git a/scripts/subby/subby/converters/mp4.py b/scripts/subby/subby/converters/mp4.py new file mode 100644 index 0000000..05fd110 --- /dev/null +++ b/scripts/subby/subby/converters/mp4.py @@ -0,0 +1,111 @@ +from collections import deque + +from pymp4.parser import MP4 +from pymp4.util import BoxUtil + +from subby.converters.base import BaseConverter +from subby.converters.smpte import SMPTEConverter +from subby.converters.webvtt import WebVTTConverter +from subby.subripfile import SubRipFile +from subby.utils.time import timestamp_from_ms + + +class ISMTConverter(BaseConverter): + """ISMT (DFXP in MP4) subtitle converter""" + + def parse(self, stream): + srt = SubRipFile([]) + for box in MP4.parse(stream.read()): + if box.type == b'mdat': + new = SMPTEConverter().from_bytes(box.data) + + # Offset timecodes if necessary + # https://github.com/SubtitleEdit/subtitleedit/blob/abd36e5/src/libse/SubtitleFormats/IsmtDfxp.cs#L85-L90 + if srt and new and srt[-1].start > new[0].start: + new.offset(srt[-1].end) + + srt.extend(new) + + return srt + + +class WVTTConverter(BaseConverter): + """WVTT (WebVTT in MP4) subtitle converter""" + + def parse(self, stream): # pylint: disable=too-many-locals, too-many-branches + sample_durations = deque() + vtt_lines = [] + timescale = 0 + + for box in MP4.parse(stream.read()): + if box.type == b'moov': + for mdhd in BoxUtil.find(box, b'mdhd'): + timescale = mdhd.timescale + break + + for stsd in BoxUtil.find(box, b'stsd'): + wvtt = stsd.entries[0] + header = [box.config for box in wvtt.children + if box.type == b'vttC'][0] + vtt_lines.append(f'{header}\n\n') + break + + if box.type == b'moof': + start_offset = 0 + duration = 0 + for tfdt in BoxUtil.find(box, b'tfdt'): + start_offset = tfdt.baseMediaDecodeTime + break + + for trun in BoxUtil.find(box, b'trun'): + for sample in trun.sample_info: + start_offset += sample.sample_composition_time_offsets or 0 + duration += sample.sample_duration or 0 + sample_durations.append({ + 'start_ms': (start_offset / timescale) * 1000, + 'end_ms': ((start_offset + duration) / timescale) * 1000 + }) + + if box.type == b'mdat': + vtt_boxes = MP4.parse(box.data) + new_start = None + for vtt_box in vtt_boxes: + settings = None + for sttg in BoxUtil.find(vtt_box, b'sttg'): + settings = sttg.settings + break + + cue_text = None + for payl in BoxUtil.find(vtt_box, b'payl'): + cue_text = payl.cue_text + break + + try: + sample_duration = sample_durations.popleft() + except IndexError: # broken line, no durations found + continue + + if vtt_box.type == b'vttc': + try: + start_ms = end_ms + except UnboundLocalError: + end_ms = sample_duration['end_ms'] + start_ms = end_ms + else: + start_ms = sample_duration['start_ms'] + + end_ms = sample_duration['end_ms'] + + if vtt_box.type == b'vtte': + new_start = end_ms + continue + + if new_start: + start_ms = new_start + new_start = None + + vtt_lines.append((f'{timestamp_from_ms(start_ms)} --> ' + f'{timestamp_from_ms(end_ms)} ' + f'{settings}\n{cue_text}\n\n')) + + return WebVTTConverter().from_string(''.join(vtt_lines)) diff --git a/scripts/subby/subby/converters/sami.py b/scripts/subby/subby/converters/sami.py new file mode 100644 index 0000000..ea261e4 --- /dev/null +++ b/scripts/subby/subby/converters/sami.py @@ -0,0 +1,90 @@ +from html.parser import HTMLParser + +from srt import Subtitle + +from subby.converters.base import BaseConverter +from subby.subripfile import SubRipFile +from subby.utils.time import timedelta_from_ms + + +class SAMIConverter(BaseConverter): + """SAMI subtitle converter""" + + def parse(self, stream): + return _SAMIConverter(stream.read().decode('utf-8-sig')).srt + + +# Internal converter class as we inherit from HTMLParser +class _SAMIConverter(HTMLParser): + def __init__(self, subtitle): + super().__init__() + self.lines = [] + self.tags = [] + + self.srt = SubRipFile([]) + self.line_list = [] + + self.feed(self._correct_tags(subtitle)) + self._convert() + + def handle_starttag(self, tag, attrs_org): + attrs = {} + for attr, val in attrs_org: + attrs[attr] = val + + if tag == 'sync': + data = {'text': ''} + data.update(attrs) + self.lines.append(data) + + self.tags.append({'name': tag, 'attrs': attrs}) + + def handle_data(self, data): + last_tag = self.tags[-1]['name'] + if last_tag == 'br': + self.lines[-1]['text'] += '\n' + return + + if last_tag == 'i' and data.strip(): + self.lines[-1]['text'] += f'<i>{data}</i>' + return + + if last_tag != 'sync' and self.lines: + self.lines[-1]['text'] += data + + def _convert(self): + for num, line in enumerate(self.lines): + # Use empty lines as the end of previous line + if not line.get('text', '').strip(): + end_time = float(line['start']) + self.line_list[-1]['end'] = end_time + continue + + if not line.get('end'): + # Arbitrarily set duration to 4s if end time not present + line['end'] = float(line['start']) + 4000 + + srt_line = { + 'start': float(line['start']), + 'end': float(line['end']), + 'content': line['text'].strip() + } + self.line_list.append(srt_line) + + for num, line in enumerate(self.line_list): + srt_line = Subtitle( + index=num, + start=timedelta_from_ms(line['start']), + end=timedelta_from_ms(line['end']), + content=line['content'] + ) + self.srt.append(srt_line) + + @staticmethod + def _correct_tags(data): + data = data.replace('<i/>', '<i>') + data = data.replace(';>', '>') + data = data.replace('<br>', '\n') + data = data.replace('<br/>', '\n') + data = data.replace('<br >', '\n') + return data diff --git a/scripts/subby/subby/converters/smpte.py b/scripts/subby/subby/converters/smpte.py new file mode 100644 index 0000000..e5d7f38 --- /dev/null +++ b/scripts/subby/subby/converters/smpte.py @@ -0,0 +1,168 @@ +import html +import logging +import re + +import bs4 +from srt import Subtitle + +from subby.converters.base import BaseConverter +from subby.subripfile import SubRipFile +from subby.utils.time import timedelta_from_timestamp, timestamp_from_ms + + +class SMPTEConverter(BaseConverter): + """DFXP/TTML/TTML2 subtitle converter""" + + def parse(self, stream): + data = stream.read().decode('utf-8-sig') + + if data.count('</tt>') == 1: + return _SMPTEConverter(data).srt + + # Support for multiple XML documents in a single file + smpte_subs = [s + '</tt>' for s in data.strip().split('</tt>') if s] + srt = SubRipFile([]) + + for sub in smpte_subs: + srt.extend(_SMPTEConverter(sub).srt) + + return srt + + +# Internal converter class as we need to handle multiple subs in one stream +class _SMPTEConverter: + def __init__(self, data): + self.logger = logging.getLogger(__name__) + self.root = bs4.BeautifulSoup(data, 'lxml-xml') + # Unescape only if necessary (parsing fails) + if not self.root: + self.root = bs4.BeautifulSoup(html.unescape(data), 'lxml-xml') + + self.srt = SubRipFile([]) + + self.tickrate = int(self.root.tt.get('ttp:tickRate', 0)) + self.frame_duration = 1 + if (rate := self.root.tt.get('ttp:frameRate')) is not None: + num, denom = map(int, self.root.tt.get('ttp:frameRateMultiplier', '1 1').split()) + framerate = (int(rate) * num) / denom + self.frame_duration = (1 / framerate) * 1000 # ms + + self.italics = {} + self.an8 = {} + self.all_span_italics = '<span tts:fontStyle="italic">' not in data + + self._parse_styles() + self._convert() + + def _convert(self): + try: + assert self.root.tt.body.div is not None + except (AttributeError, AssertionError): + return + + for num, line in enumerate(self.root.tt.body.div.find_all('p'), 1): + line_text = '' + + try: + for time in ('begin', 'end'): + if line[time].endswith('t'): + line[time] = self._convert_ticks(line[time]) + elif line[time].endswith('ms'): + line[time] = timestamp_from_ms(line[time][:-2]) + else: + line[time] = self._parse_timestamp(line[time]) + except (AttributeError, KeyError): + self.logger.warning( + 'Could not parse %s timestamp for line %02d, skipping', + time, num + ) + continue + + srt_line = Subtitle( + index=num, + start=timedelta_from_timestamp(line['begin']), + end=timedelta_from_timestamp(line['end']), + content='' + ) + + for element in line: + line_text += self._parse_element(element) + + if self._is_italic(line) and line_text.strip(): + line_text = line_text.replace('<i>', '') + line_text = line_text.replace('</i>', '') + line_text = '<i>%s</i>' % line_text.strip() + + if self._is_an8(line) and line_text.strip(): + line_text = '{\\an8}%s' % line_text.strip() + + srt_line.content = line_text.strip().strip('\n') + if srt_line.content: + self.srt.append(srt_line) + + def _parse_styles(self): + for style in self.root.find_all('style'): + if style.get('xml:id'): + self.italics[style['xml:id']] = self._is_italic(style) + for region in self.root.find_all('region'): + if region.get('xml:id'): + self.an8[region['xml:id']] = self._is_an8(region) + + def _parse_element(self, element): + element_text = '' + if isinstance(element, bs4.element.NavigableString): + element_text += element + elif isinstance(element, bs4.element.Tag): + subelement_text = '' + for subelement in element: + subelement_text += self._parse_element(subelement) + element_text += subelement_text + if element.name == 'br': + element_text += '\n' + + if self._is_italic(element) and element_text.strip(): + element_text = element_text.replace('<i>', '') + element_text = element_text.replace('</i>', '') + element_text = '<i>%s</i>' % element_text + + if self._is_an8(element) and element_text.strip(): + element_text = '{\\an8}%s' % element_text + + return element_text + + def _is_italic(self, element): + if element.get('tts:fontStyle'): + return element.get('tts:fontStyle') == 'italic' + elif element.get('style'): + return self.italics.get(element['style']) + elif element.name == 'span' and not element.attrs and self.all_span_italics: + return not self._is_italic(element.parent) + + return False + + def _is_an8(self, element): + if element.get('tts:displayAlign'): + return element.get('tts:displayAlign') == 'before' + elif element.get('region'): + return self.an8.get(element['region']) + + return False + + def _convert_ticks(self, ticks): + ticks = int(ticks[:-1]) + offset = 1.0 / self.tickrate + seconds = (offset * ticks) * 1000 + + return timestamp_from_ms(seconds) + + def _parse_timestamp(self, timestamp): + regex = r'([0-9]{2}):([0-9]{2}):([0-9]{2})[:\.,]?([0-9]{0,3})?' + parsed = re.search(regex, timestamp) + hours = int(parsed.group(1)) + minutes = int(parsed.group(2)) + seconds = int(parsed.group(3)) + miliseconds = 0 + if frames := parsed.group(4): + miliseconds = self.frame_duration * int(frames) + + return "%02d:%02d:%02d.%03d" % (hours, minutes, seconds, miliseconds) diff --git a/scripts/subby/subby/converters/webvtt.py b/scripts/subby/subby/converters/webvtt.py new file mode 100644 index 0000000..8421f88 --- /dev/null +++ b/scripts/subby/subby/converters/webvtt.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import html +import re +from functools import partial +from typing import Optional + +import tinycss +from srt import Subtitle + +from subby.converters.base import BaseConverter +from subby.subripfile import SubRipFile +from subby.utils.time import timedelta_from_timestamp + +HTML_TAG = re.compile(r'</?(?!/?i)[^>\s]+>') +STYLE_TAG_OPEN = re.compile(r'^<c.([a-zA-Z0-9]+)>([^<]+)') +STYLE_TAG = re.compile(r'<c.([a-zA-Z0-9]+)>([^<]+)<\/c>') +STYLE_TAG_CLOSE = re.compile(r'<\/c>$') +SKIP_WORDS = ('WEBVTT', 'NOTE', '/*', 'X-TIMESTAMP-MAP') +SPEAKER_TAG = re.compile(r'<v\s+[^>]+>') # Matches opening <v Name> tags, closing tags handled by STYLE_TAG_CLOSE + + +class WebVTTConverter(BaseConverter): + """WebVTT subtitle converter""" + + def parse(self, stream): + srt = SubRipFile() + looking_for_text = False + looking_for_style = False + text = [] + position = None + line_number = 1 + styles = {} + current_style = [] + + css_parser = tinycss.make_parser('page3') + + for line in stream: + # As our stream is bytes we have to deal with line breaks here + line = line.decode('utf-8').replace('\r\n', '\n').replace('\r', '\n').strip() + + # Skip processing any unnecessary lines + if any(line.startswith(word) for word in SKIP_WORDS): + continue + + # Empty line separates cues + if line == '': + # Parse current style + if looking_for_style: + stylesheet = css_parser.parse_stylesheet('\n'.join(current_style)) + for rule in stylesheet.rules: + ft = next((e for e in rule.selector if e.type == 'FUNCTION'), None) + if not ft: + continue + name = next((t for t in ft.content if t.type == 'IDENT'), None) + if not name: + continue + styles[name.value] = {} + for dec in rule.declarations: + styles[name.value][dec.name] = dec.value.as_css() + + looking_for_style = False + + # Keep looking for text if last line has none + # this will only happen if there's an unexpected line break + if not text: + continue + + srt[-1].content = '\n'.join(text) + text = [] + looking_for_text = False + + # Check for style start + elif 'STYLE' in line: + looking_for_style = True + + # Check for style content + elif looking_for_style: + current_style.append(line) + + # Check for time line + elif ' --> ' in line: + parts = line.strip().split() + position = self._get_position([p for p in parts[3:] if ':' in p]) + + start, _, end, *_ = parts + # Fix short timecodes (no hour) + if start.count(':') == 1: + start = f'00:{start}' + if end.count(':') == 1: + end = f'00:{end}' + + srt.append(Subtitle( + index=line_number, + start=timedelta_from_timestamp(start), + end=timedelta_from_timestamp(end), + content='' + )) + looking_for_text = True + line_number += 1 + + # Append text if we're inside a line + elif looking_for_text: + # Unescape html entities + line = html.unescape(line) + + # Remove speaker tags here + line = re.sub(SPEAKER_TAG, '', line) + + # Set \an8 tag if position is below 25 + # (value taken from SubtitleEdit) + if position is not None and position < 25: + line = '{\\an8}' + line + position = None + + text.append(line.strip()) + + # Add any leftover text to the last line + if text: + srt[-1].content += '\n'.join(text) + + for line in srt: + # Replace styles with italics tag when appropriate + # (replace instead of match, to handle nested) + line.content = re.sub( + STYLE_TAG, + partial(self._replace_italics, styles=styles), + line.content + ) + + # Strip non-italic tags + line.content = re.sub(HTML_TAG, '', line.content) + + return srt + + @staticmethod + def _get_position(cue_settings: list[str]) -> Optional[float]: + """ + Parses list of cue settings and extracts position offset as a float + Line number based offset and alignment strings are ignored + + https://www.w3.org/TR/webvtt1/#webvtt-line-cue-setting + """ + if not cue_settings or cue_settings == ['None']: + return None + + position = None + for key, val in (pos.split(':') for pos in cue_settings): + if key == 'line' and (val := val.split(',')[0])[-1] == '%': + position = float(val[:-1]) + break + + return position + + @staticmethod + def _replace_italics(match: re.Match, styles: dict[str, dict[str, str]]) -> str: + if (s := styles.get(match[1])) and s.get('font-style') == 'italic': + return f'<i>{match[2]}</i>' + return match[0] diff --git a/scripts/subby/subby/processors/__init__.py b/scripts/subby/subby/processors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/subby/subby/processors/base.py b/scripts/subby/subby/processors/base.py new file mode 100644 index 0000000..5e99e2d --- /dev/null +++ b/scripts/subby/subby/processors/base.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from pathlib import Path + +from subby.subripfile import SubRipFile + + +class BaseProcessor(ABC): + """Base subtitle processor class""" + + def from_srt(self, srt: SubRipFile, language: str | None = None) -> tuple[SubRipFile, bool]: + """Processes given SubRipFile""" + return self.process(srt, language) + + def from_file(self, file: Path, language: str | None = None) -> tuple[SubRipFile, bool]: + """Processes given srt file""" + with file.open(mode='r', encoding='utf-8') as stream: + return self.from_string(stream.read(), language) + + def from_string(self, data: str, language: str | None = None) -> tuple[SubRipFile, bool]: + """Processes given string with srt subtitles""" + return self.process(SubRipFile.from_string(data), language) + + @abstractmethod + def process(self, srt: SubRipFile, language: str | None = None) -> tuple[SubRipFile, bool]: + """ + Processes given SubRipFile + :return: Processed SubRipFile, success (whether any changes were made) + """ diff --git a/scripts/subby/subby/processors/common_issues.py b/scripts/subby/subby/processors/common_issues.py new file mode 100644 index 0000000..b7ab1bc --- /dev/null +++ b/scripts/subby/subby/processors/common_issues.py @@ -0,0 +1,278 @@ +import copy +import datetime +import html +import re +import unicodedata +from datetime import timedelta + +import langcodes + +from subby import regex as Regex +from subby.processors.base import BaseProcessor +from subby.processors.rtl import RTL_LANGUAGES, RTLFixer +from subby.subripfile import SubRipFile +from subby.utils.time import line_duration + + +class CommonIssuesFixer(BaseProcessor): + """Processor fixing common issues found in subtitles""" + + remove_gaps = True + + def process(self, srt, language=None): + fixed = self._fix_time_codes(copy.deepcopy(srt)) + corrected = self._correct_subtitles(fixed) + + if language and langcodes.get(language).language in RTL_LANGUAGES: + corrected, _ = RTLFixer().process(corrected, language=language) + + return corrected, corrected != srt + + def _correct_subtitles(self, srt: SubRipFile) -> SubRipFile: + def _fix_line(line): + # [GENERAL] - Affects other regexes + # Remove more than one space + line = re.sub(r' {2,}', ' ', line) + # Correct lines starting with space + line = re.sub(r'^\s*', '', line) + line = re.sub(r'\n\s*', '\n', line) + # + # [ENCODING FIXES, CHARACTER REPLACEMENTS] + # Fix musical notes garbled by encoding + # has to happen before normalization as that replaces the TM char + line = line.replace(r'♪', '♪') + # Normalize unicode characters + line = unicodedata.normalize('NFKC', line) + # Replace short hyphen with regular size + line = line.replace(r'‐', r'-') + # Replace double note with single note + line = line.replace(r'♫', r'♪') + # Replace hashes, asterisks at the start of a line with a musical note + line = re.sub( + r'^((?:{\\an8})?(?:<i>)?)(- ?)?[#\*]{1,}(?=\s+)', + r'\1\2♪', + line, + flags=re.M + ) + # Replace hashes, asterisks at the end of a line with a musical note + line = re.sub( + r'(?<=\s)(?<![#\*])(?:[#\*]{1,3}|[#\*]{1,3})(?![0-9A-Z])(</i>$|$)', + r'♪\1', + line, + flags=re.M + ) + line = re.sub(r'^[#\*]+$', r'♪', line, flags=re.M) + # Move notes into italics, if rest of the line is + line = re.sub(r'♪ <i>(.*)', r'<i>♪ \1', line) + line = re.sub(r'(♪.*)</i>\s*♪', r'\1 ♪</i>', line) + # Replace some pound signs with notes (Binge...) + # (Matches only start/end of a line with a space + # to avoid false positives) + line = re.sub(r'^£ ', r'♪ ', line) + line = re.sub(r' £$', r' ♪', line) + # Duplicated notes + line = re.sub(r'♪{1,}', r'♪', line) + # Add spaces between notes and text + line = re.sub(r'^♪([A-Za-z])', r'♪ \1', line) + line = re.sub(r'([A-Za-z])♪', r'\1 ♪', line) + # Replace \h (non-breaking space in ASS) with a regular space + # (result of ffmpeg extraction of mp4-embedded subtitles) + line = re.sub(r'(\\h)+', ' ', line).strip() + # Fix leftover amps (html unescape fixes those, but not when they're duped) + line = re.sub(r'&(amp;){1,}', r'&', line) + # Fix "it'`s" -> "it's" + line = re.sub(r"'[`’]", r"'", line) + + # [TAG STRIPPING AND CORRECTING] + # + # Replace ASS positioning tags with top only + line = re.sub(r'(\{\\an[0-9]\}){1,}', r'{\\an8}', line) + # Remove space after ASS positioning tags + line = re.sub(r'(\{\\an[0-9]\}) +(?=[A-Za-z-])', r'{\\an8}', line) + # Fix hanging tags + line = re.sub(r'^(<[a-z]>)\n', r'\1', line) + line = re.sub(r'</([a-z])>$\n<([a-z])>', r'\n', line, flags=re.M) + # Remove duplicated tags + line = re.sub(r'(<[a-z]>){1,}', r'\1', line) + line = re.sub(r'(</[a-z]>){1,}', r'\1', line) + # Remove an unnecessary space after italic tag open + line = re.sub(r'^(<[a-z]>) {1,}', r'\1', line) + line = re.sub(r'^ {1,}', '', line) + # Remove non-italic tags + line = re.sub(r'</?(?!i>)[a-z]+>', '', line) + # Remove spaces between tags + line = re.sub(r'(<[a-z]>|\{\\an8\}) (<[a-z]>|\{\\an8\})', r'\1\2', line) + # Move hanging opening tags onto separate lines + line = re.sub(r'(<[a-z]>)\n', r'\n\1', line) + # Move hanging closing tags onto separate lines + line = re.sub(r'\n(</[a-z]>)', r'\1\n', line) + # Move spaces outside italic tags + line = re.sub(r'(<[a-z]>) ', r' \1', line) + line = re.sub(r' (</[a-z]>)', r'\1 ', line) + # Remove needless spaces inside italic tags + line = re.sub(r'^(<[a-z]>) ', r'\1', line) + # Fix "</tag>space<tag>" + line = re.sub(r'(?:</[a-z]>)(\s*)(?:<[a-z]>)', r'\1', line, flags=re.M) + # Remove empty tags + line = re.sub(r'<[a-z]>\s*</[a-z]>', r'', line) + # Move "{\an8}" to the rest of the text if it's on a new line + line = re.sub(r'({\\an8\})\n', r'\1', line) + + # [REFORMATTING] + # + # Remove spaces inside brackets ("( TEXT )" -> "(TEXT)") + line = re.sub(r'\( (.*) \)', r'(\1)', line) + # Remove ">> " before text + line = re.sub(r'(^|\n)(</?[a-z]>|\{\\an8\})?>> ', r'\1\2', line) + # Remove lines consisting only of ">>" + line = re.sub(r'(^|\n)(</?[a-z]>|\{\\an8\})?>>($|\n)', r'', line) + # Replace any leftover <br> tags with a proper line break + line = re.sub(r'<br ?\/?>', '\n', line) + # Remove empty lines + line = re.sub(r'^\.?\s*$', '', line, flags=re.M) + line = re.sub(r'^-?\s*$', '', line, flags=re.M) + line = re.sub(r'^(</?i>|\{\\an8\})?\s*$', '', line, flags=re.M) + # Remove lines consisting only of a single character or digit + line = re.sub(r'^\[A-Za-z0-9]$', '', line) + # Adds missing spaces after "...", commas, and tags + line = re.sub(r'([a-z])(\.\.\.)([a-zA-Z][^.])', r'\1\2 \3', line) + line = re.sub(r'(</[a-z]>)(\w)', r'\1 \2', line) + line = re.sub(r'([a-z]),([a-zA-Z])', r'\1, \2', line) + line = re.sub(r',\n([a-z]+[\.\?])\s*$', r', \1', line) + # Correct front and end elypses + line = re.sub( + rf'({Regex.FRONT_OPTIONAL_TAGS_WITH_HYPHEN})' r'\.{1,}', + r'\1...', + line, flags=re.M + ) + line = re.sub(r'\.{2,}' rf'({Regex.TAGS})?' r'\s*$', r'...\1', line, flags=re.M) + # Add space after frontal speaker hyphen + line = re.sub(r"^(<i>|\{\\an8\})?-+(?='?[\w\"\[\(\<\{\.\$♪])", r'\1- ', line, flags=re.M) + # Remove unnecessary space before "--" + line = re.sub(r'\s*--(\s*)', r'--\1', line, flags=re.M) + # Move notes inside tags (</i> ♪ -> </i>) + line = re.sub(r'(</[a-z]>)(\s*♪{1,})$', r'\2\1', line, flags=re.M) + # Remove trailing spaces + line = re.sub(r' +$', r'', line, flags=re.M).strip() + + # [LINE SPLITS AND LINE BREAKS] + # + # Adds missing line splits (primarily present in Amazon subtitles) + line = re.sub(r'(.*)([^.][\]\)])([A-Z][^.])', r'\1\2\n\3', line) + line = re.sub( + r'(.*)([^\.\sA-Z][!\.;:?])(?<!(?:Mr|Ms)\.)(?<!Mrs\.)([A-Z][^.])', + r'- \1\2\n- \3', + line + ) + # Fix weird linebreaks (caused by stripping SDH or not) + line = re.sub(r'(^<[a-z]>|\n<[a-z]>)(\w+)\n', r'\1\2 ', line) + # Add missing hyphens + line = re.sub(r'^\s*(?!-)(.*)\n- ([A-Z][a-z]+)$', r'- \1\n- \2', line) + # Remove linebreaks inside lines + line = re.sub(r'\r\n{1,}', r'\r\n', line).strip() + line = re.sub(r'\n{1,}', r'\n', line).strip() + # Remove duplicate spaces around italics + line = re.sub(r' +</i> +', r'</i> ', line).strip() + # Remove italics from hyphen, when content immediately following is not italics + line = re.sub(r'<i>-</i>([^<]+)', r'-\1', line).strip() + + return line + + for line in srt: + # Unescape html entities (twice, because yes, double encoding happens...) + for _ in range(2): + line.content = html.unescape(line.content) + + # Run fix_line twice, as some of the fixes can introduce issues, e.g. double spaces + for _ in range(2): + line.content = _fix_line(line.content) + line.content = line.content.strip() + + # Remove remaining linebreaks + line.content = line.content.strip('\n') + + # Remove italics if every line is italicized, as this is almost certainly a mistake + # (using slices should be more performant than regex or startswith/endswith) + if len(srt) > 10 \ + and all(line.content[:3] == '<i>' and line.content[-4:] == '</i>' for line in srt): + for line in srt: + line.content = line.content[3:-4] + + combined = self._combine_timecodes(srt) + if self.remove_gaps: + return self._remove_gaps(combined) + + return combined + + def _combine_timecodes(self, srt: SubRipFile) -> SubRipFile: + """Combines lines with timecodes and same content""" + subs_copy = SubRipFile([]) + for line in srt: + if len(subs_copy) == 0: + subs_copy.append(line) + continue + if line_duration(subs_copy[-1]) == line_duration(line) \ + and subs_copy[-1].start == line.start \ + and subs_copy[-1].end == line.end: + if subs_copy[-1].content != line.content: + subs_copy[-1].content += '\n' + line.content + # Merge lines with the same text within 10 ms + elif self._subtract_ts(line.start, subs_copy[-1].end) < 10 \ + and line.content == subs_copy[-1].content: + subs_copy[-1].end = line.end + # Merge lines with less than 2 frames of gap and same text + # to avoid duplicating lines as we remove gaps later + elif 0 < self._subtract_ts(line.start, subs_copy[-1].end) <= 85 \ + and line.content.startswith(subs_copy[-1].content) \ + and self.remove_gaps: + subs_copy[-1].end = line.end + subs_copy[-1].content = line.content + # Fix overlapping times + elif self._subtract_ts(line.start, subs_copy[-1].end) == 0: + subs_copy[-1].end -= timedelta(milliseconds=1) + subs_copy.append(line) + elif line.content.strip(): + subs_copy.append(line) + + subs_copy = subs_copy or srt + subs_copy.clean_indexes() + return subs_copy + + def _remove_gaps(self, srt: SubRipFile) -> SubRipFile: + """Remove short gaps between lines""" + subs_copy = SubRipFile([]) + for line in srt: + if len(subs_copy) == 0: + subs_copy.append(line) + continue + # Remove 2-frame or smaller gaps (2 frames/83ms@24 is Netflix standard) + elif 1 < self._subtract_ts(line.start, subs_copy[-1].end) <= 85: + line.start = subs_copy[-1].end + subs_copy[-1].end -= timedelta(milliseconds=1) + subs_copy.append(line) + elif line.content.strip(): + subs_copy.append(line) + + subs_copy = subs_copy or srt + subs_copy.clean_indexes() + return subs_copy + + @staticmethod + def _fix_time_codes(srt: SubRipFile) -> SubRipFile: + """Fixes timecodes over 23:59, often present in live content""" + offset = 0 + for line in srt: + hours, _ = divmod(line.start.seconds, 3600) + hours += line.start.days * 24 + + if not offset and hours > 23: + offset = hours + if offset: + line.start -= datetime.timedelta(hours=offset) + line.end -= datetime.timedelta(hours=offset) + return srt + + @staticmethod + def _subtract_ts(ts1: datetime.timedelta, ts2: datetime.timedelta) -> int: + """Subtracts two timestamps and returns a difference as int of miliseconds""" + return round((ts1 - ts2).total_seconds() * 1000) diff --git a/scripts/subby/subby/processors/rtl.py b/scripts/subby/subby/processors/rtl.py new file mode 100644 index 0000000..0f324c8 --- /dev/null +++ b/scripts/subby/subby/processors/rtl.py @@ -0,0 +1,34 @@ +import logging + +import langcodes + +from subby.processors.base import BaseProcessor + +RTL_LANGUAGES = ('ar', 'fa', 'he', 'ps', 'syc', 'ug', 'ur') +RTL_CONTROL_CHARS = ('\u200e', '\u200f', '\u202a', '\u202b', '\u202c', '\u202d', '\u202e') +RTL_CHAR = '\u202b' + + +class RTLFixer(BaseProcessor): + """Processor fixing right-to-left language tagging""" + + def __init__(self): + self.logger = logging.getLogger(__name__) + + def process(self, srt, language=None): + if language and langcodes.get(language).language not in RTL_LANGUAGES: + self.logger.warning('RTL tagger running on an unexpected language (%s)', language) + + corrected = self._correct_subtitles(srt) + return srt, corrected != srt + + def _correct_subtitles(self, srt): + for line in srt: + # Remove previous RTL-related formatting + for char in RTL_CONTROL_CHARS: + line.content = line.content.replace(char, '') + + # Add RLM char at the start of every line + line.content = RTL_CHAR + line.content.replace("\n", f"\n{RTL_CHAR}") + + return srt diff --git a/scripts/subby/subby/processors/sdh.py b/scripts/subby/subby/processors/sdh.py new file mode 100644 index 0000000..353f348 --- /dev/null +++ b/scripts/subby/subby/processors/sdh.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import copy +import re + +from subby import regex as Regex +from subby.processors.base import BaseProcessor +from subby.subripfile import SubRipFile + + +class SDHStripper(BaseProcessor): + """Processor removing hard-of-hearing descriptions from subtitles""" + + def __init__(self, extra_regexes: list[str] | None = None): + self.extra_regexes = [ + re.compile(regex, re.MULTILINE) + for regex in extra_regexes or [] + ] + + def process(self, srt, language=None): + stripped = [line for line in copy.deepcopy(srt)] + stripped = self._clean_full_line_descriptions(stripped) + stripped = self._clean_new_line_descriptions(stripped) + stripped = self._clean_inline_descriptions(stripped) + stripped = self._clean_speaker_names(stripped) + stripped = self._strip_notes(stripped) + stripped = self._remove_extra_hyphens(stripped) + stripped = self._run_extra_regexes(stripped) + + stripped = SubRipFile([line for line in stripped if line.content]) + stripped.clean_indexes() + + return stripped, stripped != srt + + def _clean_full_line_descriptions(self, srt): + """Removes full line descriptions""" + for line in srt: + text = self._strip_tags(line.content) + for regex in (Regex.FULL_LINE_DESCIRPTION_BRACKET, Regex.FULL_LINE_DESCIRPTION_PARENTHESES): + text = re.sub(regex, r'', text, flags=re.S).strip() + + if not text: + continue + + yield line + + def _clean_new_line_descriptions(self, srt): + """Removes line descriptions taking up an entire line break""" + for line in srt: + position = re.match(Regex.POSITION_TAGS, line.content.strip()) + for regex in (Regex.NEW_LINE_DESCRIPTION_BRACKET, Regex.NEW_LINE_DESCRIPTION_PARENTHESES): + line.content = re.sub(regex, r'', line.content, flags=re.M).strip() + + # Restore position, if it has been removed with the description + if position and position[0] not in line.content: + line.content = position[0] + line.content + + yield line + + def _clean_inline_descriptions(self, srt): + """Removes inline""" + for line in srt: + line.content = re.sub(Regex.FRONT_DESCRIPTION_BRACKET, r'\10', line.content, flags=re.M) + line.content = re.sub(Regex.FRONT_DESCRIPTION_PARENTHESES, r'\1', line.content, flags=re.M) + for regex in ( + Regex.END_DESCRIPTION_BRACKET, + Regex.END_DESCRIPTION_PARENTHESES, + Regex.INLINE_DESCRIPTION + ): + line.content = re.sub(regex, r'', line.content, flags=re.M) + line.content = line.content.strip() + yield line + + def _clean_speaker_names(self, srt): + """Removes speaker names""" + for line in srt: + # Retain frontal tags/hyphens + for regex in (Regex.SPEAKER_PARENTHESES, Regex.SPEAKER): + line.content = re.sub(regex, r'\2\3', line.content, flags=re.M).strip() + yield line + + def _strip_notes(self, srt): + """Removes lines with just musical notes""" + for line in srt: + if re.match(r'^♪+$', re.sub(r'\s*', r'', self._strip_tags(line.content).strip())): + continue + + yield line + + def _run_extra_regexes(self, srt): + """Runs extra regexes provided by user""" + for line in srt: + for regex in self.extra_regexes: + line.content = re.sub(regex, r'', line.content) + yield line + + def _remove_extra_hyphens(self, srt): + """Remove speaker hyphens if there's only one line""" + for line in srt: + splits = len(re.findall(r'^(<i>|\{\\an8\})?-\s*', line.content, flags=re.M)) + if splits == 1: + line.content = re.sub(r'^(<i>|\{\\an8\})?-\s*', r'\1', line.content.strip()) + + yield line + + + @staticmethod + def _strip_tags(text: str) -> str: + return re.sub(Regex.TAGS, r'', text) diff --git a/scripts/subby/subby/regex.py b/scripts/subby/subby/regex.py new file mode 100644 index 0000000..894ddd0 --- /dev/null +++ b/scripts/subby/subby/regex.py @@ -0,0 +1,22 @@ +TAGS = r'[<{][/\\]?[a-z0-9.]+[}>]' +POSITION_TAGS = r'^{\\an[0-9]}' +FRONT_OPTIONAL_TAGS_WITH_HYPHEN = rf'^\s*({TAGS})?\s*(-)?\s*({TAGS})?\s*' +TIME_LOOKAHEAD = r'(?![0-9]{2})' + +SPEAKER = rf'({FRONT_OPTIONAL_TAGS_WITH_HYPHEN})\s*(Mc[A-Z][a-zA-Z]+|[A-Z0-9\&\[\]\.#\' ]+\s*|[A-Z][a-z]+):{TIME_LOOKAHEAD} ?' +SPEAKER_PARENTHESES = rf'({FRONT_OPTIONAL_TAGS_WITH_HYPHEN})\s*(?:[A-Z0-9\&\[\]\.#\' ]+\s*|[A-Z][a-z]+)(?: \([a-zA-Z ]+\)): ?' + +FRONT_NOTES = r'(?:♪+\s+)' +BACK_NOTES = r'(?:\s+♪+)' + +DESCRIPTION_BRACKET = r'\[(?:[^\]]|\s)*\]' +DESCRIPTION_PARENTHESES = r'\((?:[^\)]|\s)*\)' +FULL_LINE_DESCIRPTION_BRACKET = rf'^-?\s*{FRONT_NOTES}?\[[^\]]+\]{BACK_NOTES}?$' +NEW_LINE_DESCRIPTION_BRACKET = rf'^(?:{TAGS})?-?\s*{FRONT_NOTES}?{DESCRIPTION_BRACKET}(?:{TAGS})?{BACK_NOTES}?$' +FRONT_DESCRIPTION_BRACKET = rf'^(?:{SPEAKER}|{SPEAKER_PARENTHESES})?({FRONT_OPTIONAL_TAGS_WITH_HYPHEN}){DESCRIPTION_BRACKET}:?' +END_DESCRIPTION_BRACKET = rf'\s*{DESCRIPTION_BRACKET}\s*$' +FULL_LINE_DESCIRPTION_PARENTHESES = rf'^-?\s*{FRONT_NOTES}?\([^\)]+\){BACK_NOTES}?$' +NEW_LINE_DESCRIPTION_PARENTHESES = rf'^(?:{TAGS})?-?\s*{FRONT_NOTES}?{DESCRIPTION_PARENTHESES}{BACK_NOTES}?(?:{TAGS})?$' +FRONT_DESCRIPTION_PARENTHESES = rf'^({FRONT_OPTIONAL_TAGS_WITH_HYPHEN})(?:{SPEAKER}|{SPEAKER_PARENTHESES})?{DESCRIPTION_PARENTHESES}:?' +END_DESCRIPTION_PARENTHESES = rf'\s*{DESCRIPTION_PARENTHESES}:?\s*$' +INLINE_DESCRIPTION = r'(?:<[a-z]+>)?[\[(][A-Z]+[)\]](?:</[a-z]+>)?' diff --git a/scripts/subby/subby/subripfile.py b/scripts/subby/subby/subripfile.py new file mode 100644 index 0000000..7bfa8c0 --- /dev/null +++ b/scripts/subby/subby/subripfile.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from collections import UserList +from datetime import timedelta +from pathlib import Path + +import srt + + +class SubRipFile(UserList): + def __init__(self, data: list[srt.Subtitle] | None = None): + self.data: list[srt.Subtitle] = data or [] + + @classmethod + def from_string(cls, source: str): + return cls(list(srt.parse(source, ignore_errors=True))) + + def clean_indexes(self): + self.data = list(srt.sort_and_reindex(self.data)) + + def offset(self, offset: timedelta): + for line in self.data: + line.start += offset + line.end += offset + + def export(self, eol: str | None = None) -> str: + """Exports subtitle as text""" + return srt.compose(self.data, eol=eol) + + def save(self, path: Path, encoding: str = 'utf-8-sig', eol: str | None = None): + """Exports subtitle as text""" + with path.open(mode='wb') as fp: + fp.write(srt.compose(self.data, eol=eol).encode(encoding)) + + def __eq__(self, other): + if not isinstance(other, SubRipFile): + raise NotImplementedError + return self.export(eol='\n') == other.export(eol='\n') diff --git a/scripts/subby/subby/utils/time.py b/scripts/subby/subby/utils/time.py new file mode 100644 index 0000000..443623a --- /dev/null +++ b/scripts/subby/subby/utils/time.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import datetime +import re + +from srt import Subtitle + + +def timestamp_from_ms(duration: float | int) -> str: + """Returns a formatted timestamp from miliseconds""" + seconds, miliseconds = divmod(float(duration), 1000) + minutes, seconds = divmod(seconds, 60) + hours, minutes = divmod(minutes, 60) + return "%02d:%02d:%02d.%03d" % (hours, minutes, seconds, miliseconds) + + +def timestamp_from_seconds(duration: float | int) -> str: + """Returns a formatted timestamp from seconds""" + return timestamp_from_ms(duration * 1000) + + +def ms_from_timestamp(timestamp: str) -> int: + """Returns miliseconds from a timestamp""" + timestamp = re.sub(r'[;\.\,]', r':', timestamp.replace('T:', '')) + hours, minutes, seconds, miliseconds = map(int, timestamp.split(':')) + miliseconds += hours * 3600000 + miliseconds += minutes * 60000 + miliseconds += seconds * 1000 + return miliseconds + + +def timedelta_from_timestamp(timestamp: str) -> datetime.timedelta: + """Returns timedelta from a timestamp""" + return datetime.timedelta(seconds=ms_from_timestamp(timestamp) / 1000) + + +def timedelta_from_ms(duration: float | int) -> datetime.timedelta: + """Returns timedelta from miliseconds""" + return datetime.timedelta(seconds=duration / 1000) + + +def line_duration(line: Subtitle): + """Returns duration of a srt.Subtitle line""" + return abs(line.end - line.start) diff --git a/scripts/subby/tests/test_processing.py b/scripts/subby/tests/test_processing.py new file mode 100644 index 0000000..03e8168 --- /dev/null +++ b/scripts/subby/tests/test_processing.py @@ -0,0 +1,255 @@ +from datetime import time, timedelta + +from subby import CommonIssuesFixer + + +MUSICAL_NOTE_EXAMPLE = '''1 +00:01:00,000 --> 00:01:01,000 +#TestData + +2 +00:02:00,000 --> 00:02:01,000 +#TestData# + +3 +00:03:00,000 --> 00:03:01,000 +# #TestData # + +4 +00:04:00,000 --> 00:04:01,000 +# Song Lyrics # + +5 +00:05:00,000 --> 00:05:01,000 +We are #1! + +6 +00:06:00,000 --> 00:06:01,000 +# <i>Song Lyrics</i> + +7 +00:07:00,000 --> 00:07:01,000 +# Song Lyrics +On two separate lines # + +8 +00:08:00,000 --> 00:08:01,000 +#1 Radio Station + +9 +00:09:00,000 --> 00:09:01,000 +ABCD FM +#1 Radio Station + +10 +00:10:00,000 --> 00:10:01,000 +#One Radio Station + +11 +00:11:00,000 --> 00:11:01,000 +♪ <i>Fire</i>♪ + +12 +00:12:00,000 --> 00:12:01,000 +*Schnaub* + +13 +00:13:00,000 --> 00:13:01,000 +* Schnaub * + +14 +00:14:00,000 --> 00:14:01,000 +♫ Thunder''' + + +ADDING_LINE_BREAKS_EXAMPLE = '''1 +00:01:00,000 --> 00:01:01,000 +It's chocolate.Hmm? + +2 +00:02:00,000 --> 00:02:01,000 +We can't just leave him.He's already gone. + +3 +00:03:26,800 --> 00:03:31,200 +- Test. Mr.Teufel... +- Test...''' + + +ELIPSES_FIXING_EXAMPLE = '''1 +00:13:00,000 --> 00:13:01,000 +..noooooooooooooo.......... + +2 +00:14:00,000 --> 00:14:01,000 +<i>Stop this.................</i>''' + + +TAG_CORRECTIONS_EXAMPLE = '''1 +00:15:00,000 --> 00:15:01,000 +<i> Test</i> <i>line1</i> +<i>Test </i>line2 + +2 +00:16:00,000 --> 00:16:01,000 +{\\an3}{\\an8}{\\an8}<i><i>Test line1 +Test line2</i> + +3 +00:17:00,000 --> 00:17:01,000 +<b>test</b> + +4 +00:18:00,000 --> 00:18:01,000 +<i> +test +</i>''' + + +GAP_REMOVAL_EXAMPLE = '''1 +00:19:00,000 --> 00:19:00,100 +remove 2 frame gap between this + +2 +00:19:00,183 --> 00:19:01,000 +and that line''' + + +SPACE_REMOVAL_EXAMPLE = ''' +1 +00:22:00,000 --> 00:22:01,000 +<i>SOMETHING:</i> <i> +Synthetic test.</i> <i> +Definitely not real.</i> +''' + + +SPACES_AFTER_HYPHENS_EXAMPLE = '''1 +00:23:00,000 --> 00:23:01,000 +-Well. +-$5000?''' + +INVALID_TIMESTAMP_EXAMPLE = '''1 +27:27:00,000 --> 27:27:01,000 +Always. Run. Tests. + +2 +28:27:00,000 --> 28:27:01,000 +Really.''' + + +OVERLAPPING_TIME_EXAMPLE = '''1 +00:00:00,000 --> 00:00:00,105 +this line should end at 104 + +2 +00:00:00,105 --> 00:00:01,000 +and that line should end start at 105''' + + +def test_musical_notes(): + fixer = CommonIssuesFixer() + srt, _ = fixer.from_string(MUSICAL_NOTE_EXAMPLE) + # Test correct musical note conversion + assert srt[0].content == '#TestData' + assert srt[1].content == '#TestData#' + assert srt[2].content == '♪ #TestData ♪' + assert srt[3].content == '♪ Song Lyrics ♪' + assert srt[4].content == 'We are #1!' + assert srt[5].content == '<i>♪ Song Lyrics</i>' + assert srt[6].content == '♪ Song Lyrics\nOn two separate lines ♪' + assert srt[7].content == '#1 Radio Station' + assert srt[8].content == 'ABCD FM\n#1 Radio Station' + assert srt[9].content == '#One Radio Station' + assert srt[10].content == '<i>♪ Fire ♪</i>' + assert srt[11].content == '*Schnaub*' + assert srt[12].content == '♪ Schnaub ♪' + assert srt[13].content == '♪ Thunder' + + +# Test adding missing line breaks +def test_adding_line_breaks(): + fixer = CommonIssuesFixer() + srt, _ = fixer.from_string(ADDING_LINE_BREAKS_EXAMPLE) + assert srt[0].content == "- It's chocolate.\n- Hmm?" + assert srt[1].content == "- We can't just leave him.\n- He's already gone." + assert srt[2].content == "- Test. Mr.Teufel...\n- Test..." + + +# Test ellipses fixing +def test_elipses_fixing(): + fixer = CommonIssuesFixer() + srt, _ = fixer.from_string(ELIPSES_FIXING_EXAMPLE) + assert srt[0].content == "...noooooooooooooo..." + assert srt[1].content == "<i>Stop this...</i>" + + +# Test tag corrections +def test_tag_corrections(): + fixer = CommonIssuesFixer() + srt, _ = fixer.from_string(TAG_CORRECTIONS_EXAMPLE) + assert srt[0].content == "<i>Test line1\nTest</i> line2" + assert srt[1].content == "{\\an8}<i>Test line1\nTest line2</i>" + assert srt[2].content == "test" + assert srt[3].content == "<i>test</i>" + + +# Test 83 ms gap removal +def test_gap_removal(): + fixer = CommonIssuesFixer() + srt, _ = fixer.from_string(GAP_REMOVAL_EXAMPLE) + assert srt[0].end == timedelta(minutes=19, milliseconds=99) + assert srt[1].start == timedelta(minutes=19, milliseconds=100) + + fixer.remove_gaps = False + srt2, _ = fixer.from_string(GAP_REMOVAL_EXAMPLE) + assert srt2[0].end == timedelta(minutes=19, milliseconds=100) + assert srt2[1].start == timedelta(minutes=19, milliseconds=183) + + +# Test redundant space removal +def test_redundant_space_removal(): + fixer = CommonIssuesFixer() + srt, _ = fixer.from_string(SPACE_REMOVAL_EXAMPLE) + assert srt[0].content == "<i>SOMETHING:\nSynthetic test.\nDefinitely not real.</i>" + + +# Test adding spaces after frontal hyphens (dialogue) +def test_adding_spaces_after_frontal_hyphens(): + fixer = CommonIssuesFixer() + srt, _ = fixer.from_string(SPACES_AFTER_HYPHENS_EXAMPLE) + assert srt[0].content == "- Well.\n- $5000?" + + +# Test invalid timestamp fixing +def test_invalid_timestamp_fixing(): + fixer = CommonIssuesFixer() + srt, _ = fixer.from_string(INVALID_TIMESTAMP_EXAMPLE) + assert srt[0].start == timedelta(minutes=27) + assert srt[1].start == timedelta(hours=1, minutes=27) + + +# Test overlapping time fixing +def test_fix_overlapping_time(): + fixer = CommonIssuesFixer() + srt, _ = fixer.from_string(OVERLAPPING_TIME_EXAMPLE) + assert srt[0].end == timedelta(milliseconds=104) + assert srt[1].start == timedelta(milliseconds=105) + + fixer.remove_gaps = False + srt, _ = fixer.from_string(OVERLAPPING_TIME_EXAMPLE) + assert srt[0].end == timedelta(milliseconds=104) + assert srt[1].start == timedelta(milliseconds=105) + + + +if __name__ == "__main__": + test_musical_notes() + test_adding_line_breaks() + test_elipses_fixing() + test_tag_corrections() + test_gap_removal() + test_redundant_space_removal() + test_adding_spaces_after_frontal_hyphens() + test_invalid_timestamp_fixing() + test_fix_overlapping_time() diff --git a/scripts/subby/tests/test_sdh.py b/scripts/subby/tests/test_sdh.py new file mode 100644 index 0000000..c16ea48 --- /dev/null +++ b/scripts/subby/tests/test_sdh.py @@ -0,0 +1,66 @@ +from subby import SDHStripper, CommonIssuesFixer + +EXAMPLE_1 = '''1 +00:00:11,803 --> 00:00:13,346 +RADIO ANNOUNCER: +<i>"W" who?</i> + +2 +00:00:40,749 --> 00:00:42,375 +- ♪ Hey, boo ♪ +- ♪ Hey, boo ♪ + +3 +00:00:55,931 --> 00:00:58,134 +[ Maker's "Hold'em" playing ] + +4 +00:00:58,934 --> 00:01:06,567 +♪ + +5 +00:00:59,292 --> 00:01:01,561 +- [shouting] +- [continuous gunfire] + +6 +00:01:09,653 --> 00:01:11,822 +It's zoo time! +[ Kids cheering ] + +7 +00:01:29,881 --> 00:01:31,132 +{\\an8}(MYSTERIOUS MUSIC PLAYING) Spooky! + +8 +00:01:33,968 --> 00:01:35,387 +[John] Hmm? +<i>(Alice) Hello!</i> + +9 +00:01:40,016 --> 00:01:41,685 +- I did on magnets this summer. +- (ELECTRICITY ZAPS) + +10 +00:01:41,685 --> 00:01:42,769 +- Boo! +- STUDENT: No, thanks.''' + + +def test_sdh_stripping(): + stripper = SDHStripper() + fixer = CommonIssuesFixer() # Fixer is currently necessary to fix some of the issues from stripping + srt, _ = fixer.from_srt(stripper.from_string(EXAMPLE_1)[0]) + assert len(srt) == 7 + assert srt[0].content == '<i>"W" who?</i>' + assert srt[1].content == '- ♪ Hey, boo ♪\n- ♪ Hey, boo ♪' + assert srt[2].content == "It's zoo time!" + assert srt[3].content == '{\\an8}Spooky!' + assert srt[4].content == 'Hmm?\n<i>Hello!</i>' + assert srt[5].content == 'I did on magnets this summer.' + assert srt[6].content == '- Boo!\n- No, thanks.' + + +if __name__ == "__main__": + test_sdh_stripping() diff --git a/scripts/subby/tests/test_webvtt_conversion.py b/scripts/subby/tests/test_webvtt_conversion.py new file mode 100644 index 0000000..d274bad --- /dev/null +++ b/scripts/subby/tests/test_webvtt_conversion.py @@ -0,0 +1,19 @@ +from datetime import timedelta +from io import BytesIO + +from subby import WebVTTConverter + +SPEAKER_TAG_TEST = b'''1 +00:00:01.000 --> 00:00:03.000 +- <v ID>TESTY TESTERSON:</v> +<v Testerson>This is a test, if my name isn't Testy Testerson!</v> +''' + +def test_speaker_tag_stripping(): + converter = WebVTTConverter() + stream = BytesIO(SPEAKER_TAG_TEST) + srt = converter.parse(stream) + + # Verify that speaker tag is stripped + assert len(srt) == 1 + assert srt[0].content == "- TESTY TESTERSON:\nThis is a test, if my name isn't Testy Testerson!" diff --git a/wpgskd/commands/dl.py b/wpgskd/commands/dl.py new file mode 100644 index 0000000..516118e --- /dev/null +++ b/wpgskd/commands/dl.py @@ -0,0 +1,524 @@ +import logging +import math +import os +import random +import time +from pathlib import Path + +import click +import requests + +from wpgskd import servicookies as services +from wpgskd.config import config, directories, filenames +from wpgskd.core.cdm.loader import CdmProvider +from wpgskd.core.console import ConsoleUI +from wpgskd.core.decryptor import Decryptor +from wpgskd.core.downloader import Downloader +from wpgskd.core.events import EventManager, Events +from wpgskd.core.muxer import Muxer +from wpgskd.core.resolver import KeyResolver +from wpgskd.core.tracks.title import Title, Titles +from wpgskd.core.tracks.audio import AudioTrack +from wpgskd.core.tracks.tracks import TextTrack +from wpgskd.core.vault import LocalVault +from wpgskd.core.vaults import Vaults +from wpgskd.utils.click import (AliasedGroup, ContextData, acodec_param, + channels_param, language_param, quality_param, + range_param, vcodec_param, wanted_param) + +log = logging.getLogger("dl") + +@click.group(name="dl", short_help="Download from a service.", cls=AliasedGroup, context_settings=dict( + help_option_names=["-?", "-h", "--help"], + max_content_width=116, + default_map=config.arguments +)) +@click.option("--debug", is_flag=True, hidden=True) +@click.option("-p", "--profile", type=str, default=None, + help="Profile to use when multiple profiles are defined for a service.") +@click.option("-q", "--quality", callback=quality_param, default=None, + help="Download Resolution, defaults to best available.") +@click.option("-v", "--vcodec", callback=vcodec_param, default="H264", + help="Video Codec, defaults to H264.") +@click.option("-a", "--acodec", callback=acodec_param, default=None, + help="Audio Codec") +@click.option("-vb", "--vbitrate", "vbitrate", type=int, default=None, + help="Video Bitrate, defaults to Max.") +@click.option("-ab", "--abitrate", "abitrate", type=int, default=None, + help="Audio Bitrate, defaults to Max.") +@click.option("-aa", "--atmos", is_flag=True, default=False, + help="Prefer Atmos Audio") +@click.option("-ch", "--channels", callback=channels_param, default=None, + help="Audio Channels") +@click.option("-r", "--range", "range_", callback=range_param, default="SDR", + help="Video Color Range, defaults to SDR.") +@click.option("-w", "--wanted", callback=wanted_param, default=None, + help="Wanted episodes, e.g. `S01-S05,S07`, `S01E01-S02E03`, defaults to all.") +@click.option("-al", "--alang", callback=language_param, default="orig", + help="Language wanted for audio.") +@click.option("-sl", "--slang", callback=language_param, default="all", + help="Language wanted for subtitles.") +@click.option("--delay", type=int, default=None, + help="Delay between title processing") +@click.option("--proxy", type=str, default=None, + help="Proxy URI to use. If a 2-letter country is provided, it will try get a proxy from the config.") +@click.option("-A", "--audio-only", is_flag=True, default=False, help="Only download audio tracks.") +@click.option("-S", "--subs-only", is_flag=True, default=False, help="Only download subtitle tracks.") +@click.option("-C", "--chapters-only", is_flag=True, default=False, help="Only download chapters.") +@click.option("-ns", "--no-subs", is_flag=True, default=False, help="Do not download subtitle tracks.") +@click.option("-na", "--no-audio", is_flag=True, default=False, help="Do not download audio tracks.") +@click.option("-nv", "--no-video", is_flag=True, default=False, help="Do not download video tracks.") +@click.option("-nc", "--no-chapters", is_flag=True, default=False, help="Do not download chapters tracks.") +@click.option("-ad", "--audio-description", is_flag=True, default=False, help="Download audio description tracks.") +@click.option("--list", "list_", is_flag=True, default=False, help="List available tracks without downloading.") +@click.option("--selected", is_flag=True, default=False, help="List selected tracks without downloading.") +@click.option("--cdm", type=str, default=None, help="Override the CDM that will be used for decryption.") +@click.option("--export", "export_arg", is_flag=False, flag_value="", default=None, + help="Export track info and decryption keys to a JSON file. Can optionally specify file name.") +@click.option("--keys", is_flag=True, default=False, help="Skip downloading, retrieve keys and print them.") +@click.option("--cache", is_flag=True, default=False, help="Disable CDM use, only retrieve keys from Key Vaults.") +@click.option("--no-cache", is_flag=True, default=False, help="Disable Key Vaults use, only retrieve keys from CDM.") +@click.option("--no-proxy", is_flag=True, default=False, help="Force disable all proxy use.") +@click.option("--force-proxy", is_flag=True, default=False, help="Force using proxy even if current region matches.") +@click.option("-nm", "--no-mux", is_flag=True, default=False, help="Do not mux the downloaded and decrypted tracks.") +@click.option("--mux", is_flag=True, default=False, help="Force muxing when using --audio-only/--subs-only/--chapters-only.") +@click.option("--worst", is_flag=True, default=False, help="Choose the worst available video tracks rather than the best") +@click.option("--sync-vat", is_flag=True, default=False, help="Compress audio duration to match video duration before muxing.") +@click.option("-nys", "--no-sync-subs", is_flag=True, default=False, help="Do not merge/sync subtitle tracks during muxing.") +@click.pass_context +def dl(ctx, profile, cdm, *_, **__): + """Download from a specified service.""" + if ctx.params.get("debug"): + import coloredlogs + LOG_FORMAT = "{asctime} [{levelname[0]}] {name} : {message}" + LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" + LOG_STYLE = "{" + coloredlogs.install( + level=logging.DEBUG, + fmt=LOG_FORMAT, + datefmt=LOG_DATE_FORMAT, + style=LOG_STYLE, + handlers=[logging.StreamHandler()] + ) + + service_name = ctx.params.get("service_name") or services.get_service_key(ctx.invoked_subcommand) + if not service_name: + log.error(" - Unable to find service") + return + + profile = profile or config.profiles.get(service_name) or config.profiles.get("default") or "default" + + service_config = services.get_service_config(service_name) + + vaults_list = [] + for vault_cfg in config.key_vaults: + try: + vaults_list.append(Vaults.load_vault(vault_cfg)) + except Exception as e: + log.error(f" - Failed to load vault {vault_cfg.get('name')!r}: {e}") + + vaults_obj = Vaults(vaults_list, service=service_name) + local_count = sum(1 for v in vaults_obj.vaults if isinstance(v, LocalVault)) + remote_count = sum(1 for v in vaults_obj.vaults if not isinstance(v, LocalVault)) + log.info(f" + {local_count} Local, {remote_count} Remote Vault(s) loaded") + + cdm_cfg_dict = {k.lower(): v for k, v in config.cdm.items()} + cdm_name = cdm or cdm_cfg_dict.get(service_name.lower()) or cdm_cfg_dict.get("default") + + try: + cdm_prov = CdmProvider( + cdm_name=cdm_name, + device_dir=directories.devices, + cdm_api_config=config.cdm_api + ) + cdm_prov.log_info() + except Exception as e: + log.error(f" - CDM Init Error: {e}") + raise click.Abort() + return + + cookies = credentials_obj = None + needs_auth = service_config.get("needs_auth", True) + if profile: + cookies = services.get_cookie_jar(service_name, profile) + credentials_obj = services.get_credentials(service_name, profile) + if not cookies and not credentials_obj and needs_auth: + log.error(f" - Profile {profile!r} has no cookies or credentials") + return + + ctx.obj = ContextData( + config=service_config, + vaults=vaults_obj, + cdm=cdm_prov, + profile=profile, + cookies=cookies, + credentials=credentials_obj + ) + + +@dl.result_callback() +def result(service, quality, vcodec, acodec, range_, wanted, alang, slang, + audio_only, subs_only, chapters_only, audio_description, list_, keys, + cache, no_cache, no_subs, no_audio, no_video, no_chapters, atmos, + vbitrate: int, abitrate: int, channels, no_mux, worst, mux, delay, + selected, sync_vat, no_sync_subs, export_arg, *_, **__): + + log = service.log + service_name = service.__class__.__name__ + + log.info("Retrieving Titles") + try: + titles = Titles(service.get_titles()) + except requests.HTTPError as e: + log.error(f" - HTTP Error {e.response.status_code}: {e.response.reason}") + return + + if not titles: + log.error(" - No titles returned!") + return + + titles.order() + ConsoleUI.print_titles(titles) + + cdm_prov: CdmProvider = service.cdm + resolver = KeyResolver( + vaults=service.vaults, + cdm_provider=cdm_prov, + use_cache=not no_cache, + use_cdm=not cache + ) + downloader = Downloader(session=service.session) + + first = True + for title in titles.with_wanted(wanted): + if not first and delay: + jitter = random.randint(math.floor(-delay / 5), math.floor(delay / 5)) + d = delay + jitter + log.info(f"Delaying for {d}s before getting next title...") + time.sleep(d) + first = False + + _log_title(log, title) + + try: + title.tracks.add(service.get_tracks(title), warn_only=True) + chapters = service.get_chapters(title) + if chapters: + title.tracks.add(chapters) + except requests.HTTPError as e: + log.error(f" - HTTP Error getting tracks: {e.response.status_code}") + continue + + title.tracks.sort_videos() + title.tracks.sort_audios(by_language=alang) + title.tracks.sort_subtitles(by_language=slang) + title.tracks.sort_chapters() + + for track in title.tracks: + track.is_original_lang = track.language == title.original_lang + + if not list(title.tracks): + log.error(" - No tracks returned!") + continue + + if not selected: + log.info("> All Tracks:") + ConsoleUI.print_tracks(title.tracks, title) + + try: + if range_ == "DV+HDR": + title.tracks.select_videos_multi(["HDR10", "DV"], by_quality=quality, by_vbitrate=vbitrate) + else: + title.tracks.select_videos( + by_quality=quality, by_vbitrate=vbitrate, by_range=range_, + one_only=True, by_worst=worst, by_codec=vcodec + ) + title.tracks.select_audios( + by_language=alang, by_bitrate=abitrate, with_atmos=atmos, + with_descriptive=audio_description, by_channels=channels, by_codec=acodec + ) + title.tracks.select_subtitles(by_language=slang, with_forced=True) + except ValueError as e: + log.error(f" - {e}") + continue + + _apply_filters(title, no_video, no_audio, no_subs, no_chapters, audio_only, subs_only, chapters_only, mux) + + log.info("> Selected Tracks:") + ConsoleUI.print_tracks(title.tracks, title) + + if list_: + continue + + all_content_keys = {} + skip_title = False + + for track in title.tracks: + if track.encrypted and str(track.descriptor).split(".")[-1] == "M3U": + if not track.pssh and not track.pr_pssh: + track.get_pssh(service.session) + + enc_scheme = track.encryption_scheme.name if hasattr(track.encryption_scheme, 'name') else track.encryption_scheme + if not track.encrypted or enc_scheme in ["AES_128", "CLEARKEY"]: + continue + + log.info(f"Licensing: {str(track).replace('├─ ', '').replace('└─ ', '')}") + + if not track.pssh and not track.pr_pssh: + track.get_pssh(service.session) + + if not track.kid: + track.get_kid(service.session) + + cdm_type = cdm_prov.cdm_instance.cdm_type if cdm_prov else "widevine" + + if cdm_type == "playready": + if getattr(track, 'pr_pssh', None): + pssh_str = track.pr_pssh + if isinstance(pssh_str, bytes): + pssh_str = pssh_str.decode('utf-8', 'ignore') + log.info(f" + PR_PSSH: {pssh_str}") + else: # widevine + if getattr(track, 'pssh', None): + pssh_obj = track.pssh + try: + if hasattr(pssh_obj, 'dumps') and callable(pssh_obj.dumps): + dumped = pssh_obj.dumps() + if isinstance(dumped, bytes): + import base64 + log.info(f" + WV_PSSH: {base64.b64encode(dumped).decode('utf-8')}") + else: + log.info(f" + WV_PSSH: {dumped}") + else: + log.info(f" + WV_PSSH: {pssh_obj}") + except Exception: + log.info(f" + WV_PSSH: {pssh_obj}") + + if getattr(track, 'kid', None): + log.info(f" + KID: {track.kid}") + + try: + pk, akeys = resolver.resolve(track, title, service, service_name, service.session) + if cache and not pk: + skip_title = True + break + + if pk: + track.key = pk + all_content_keys.update(akeys) + log.info(f" + KEY: {pk[:32]}... (Resolved)") + + if export_arg is not None: + import click as click_mod + current_ctx = click_mod.get_current_context() + _export_keys( + directories.exports, service_name, title, track, akeys, + export_arg, + cli_title_id=current_ctx.parent.params.get("title", ""), + quality=quality, + vcodec=vcodec, + range_=range_ + ) + else: + log.error(" - No content key returned") + return + except Exception as e: + log.error(f" - Key Resolution Failed: {e}") + return + + if skip_title: + for track in title.tracks: + track.delete() + continue + + if keys: + continue + + EventManager.publish(Events.BEFORE_DOWNLOAD, title) + + for track in title.tracks: + log.info(f"\nDownloading: {track}") + + proxy = None + if track.needs_proxy: + proxy = next(iter(service.session.proxies.values()), None) + + try: + downloader.download(track, directories.temp, proxy=proxy, title_ref=title, all_keys=all_content_keys) + log.info(" + Downloaded") + EventManager.publish(Events.AFTER_DOWNLOAD, track) + except Exception as e: + log.error(f" - Download failed: {e}") + continue + + should_decrypt = track.encrypted and enc_scheme not in ["AES_128", "AES_128_ECB", "CLEARKEY"] + if should_decrypt: + log.info("Decrypting...") + dec_keys = {track.kid.lower().replace("-", ""): track.key.lower()} + dec_keys.update({k.lower(): v.lower() for k, v in all_content_keys.items()}) + + try: + dec_path = Decryptor.decrypt(track, dec_keys, config.decrypter, directories.temp) + if dec_path and track.swap(dec_path): + log.info(" + Decrypted") + EventManager.publish(Events.AFTER_DECRYPT, track) + + if track.needs_repack or config.decrypter == "mp4decrypt": + log.info("Repackaging stream with FFmpeg") + Decryptor.repackage(track.locate()) + log.info(" + Repackaged") + else: + log.warning(" - Decryption swap failed") + except Exception as e: + log.error(f" - Decryption failed: {e}") + + if range_ == "DV+HDR": + try: + if not any(v.dv and v.hdr10 for v in title.tracks.videos): + pass + except Exception as e: + log.warning(f" - Skipped DV+HDR: {e}") + + if not list(title.tracks) and not title.tracks.chapters: + continue + + EventManager.publish(Events.BEFORE_MUX, title) + + if no_mux: + _output_unmuxed(title, log) + else: + _output_muxed(title, log, audio_only, subs_only, service_name, sync_vat, no_sync_subs) + EventManager.publish(Events.AFTER_MUX, title) + + log.info("Processed all titles!") + +def _log_title(logger, title: Title): + if title.type == Title.Types.TV: + ep = f" - {title.episode_name}" if title.episode_name else "" + logger.info(f"Getting tracks for {title.name} S{title.season or 0:02}E{title.episode or 0:02}{ep} [{title.id}]") + else: + yr = f" ({title.year})" if title.year else "" + logger.info(f"Getting tracks for {title.name}{yr} [{title.id}]") + +def _apply_filters(title, nv, na, ns, nc, ao, so, co, mux): + if nv: title.tracks.videos.clear() + if na: title.tracks.audios.clear() + if ns: title.tracks.subtitles.clear() + if nc: title.tracks.chapters.clear() + if ao or so or co: + title.tracks.videos.clear() + if ao: + if not so: title.tracks.subtitles.clear() + if not co: title.tracks.chapters.clear() + elif so: + if not ao: title.tracks.audios.clear() + if not co: title.tracks.chapters.clear() + elif co: + if not ao: title.tracks.audios.clear() + if not so: title.tracks.subtitles.clear() + +def _output_unmuxed(title: Title, logger): + out_dir = Path(directories.downloads) + if title.type == Title.Types.TV: + out_dir = out_dir / title.parse_filename(folder=True) + out_dir.mkdir(parents=True, exist_ok=True) + + if title.tracks.chapters: + loc = out_dir / f"{title.filename}_chapters.txt" + title.tracks.export_chapters(str(loc)) + + for track in title.tracks: + if not track.locate(): continue + fn = title.parse_filename() + if isinstance(track, (AudioTrack, TextTrack)): + fn += f".{track.language}" + ext = track.codec if isinstance(track, TextTrack) else Path(track.locate()).suffix[1:] + if isinstance(track, AudioTrack) and ext == "mp4": ext = "m4a" + track.move(str(out_dir / f"{fn}.{track.id}.{ext}")) + +def _output_muxed(title: Title, logger, audio_only, subs_only, service_name, sync_vat, no_sync_subs): + try: + muxed_location, returncode = Muxer.mux(title, title.tracks, no_sync_subs=no_sync_subs) + + if returncode >= 2: + logger.error(" - Failed to mux tracks into MKV file") + return + + logger.info(" + Muxed") + + out_dir = Path(directories.downloads) + if title.type == Title.Types.TV: + out_dir = out_dir / title.parse_filename(folder=True) + out_dir.mkdir(parents=True, exist_ok=True) + + ext = "mka" if audio_only else "mks" if subs_only else "mkv" + target_path = out_dir / f"{title.parse_filename()}.{ext}" + + import shutil + shutil.move(muxed_location, str(target_path)) + logger.info(f" + Saved to: {target_path}") + + if sync_vat: + logger.info("Applying Audio-Video Sync (SyncVAT)...") + Muxer.apply_sync(str(target_path)) + + for track in title.tracks: + try: track.delete() + except: pass + if title.tracks.chapters: + try: os.unlink(filenames.chapters.format(filename=title.filename)) + except: pass + + except Exception as e: + logger.error(f" - Muxing failed: {e}") + +def _export_keys(export_dir, service_name, title, track, keys, export_name="", cli_title_id="", quality=None, vcodec=None, range_=None): + import json + export_dir = Path(export_dir) + export_dir.mkdir(parents=True, exist_ok=True) + + if export_name: + if not export_name.endswith(".json"): + export_name += ".json" + export_path = export_dir / export_name + else: + if isinstance(quality, int): + q_str = f"{quality}P" + elif quality: + q_str = str(quality).upper() + else: + q_str = "ALL" + + v_str = vcodec.upper() if vcodec else "ALL" + r_str = range_.upper() if range_ else "ALL" + + export_file = f"{service_name}_{cli_title_id}_{q_str}_{v_str}_{r_str}.json" + export_path = export_dir / export_file + + doc = {} + if export_path.is_file(): + try: + doc = json.loads(export_path.read_text(encoding="utf-8")) + except: pass + + titles_dict = doc.setdefault("titles", {}) + tinfo = titles_dict.setdefault(str(title.id), {}) + + tinfo["title_id"] = title.id + tinfo["title_name"] = title.name + tinfo["type"] = "TV" if title.type == Title.Types.TV else "MOVIE" + tinfo["year"] = title.year + if title.type == Title.Types.TV: + tinfo["season"] = title.season + tinfo["number"] = title.episode + + tinfo["cbr_manifest_url"] = getattr(title, 'cbr_manifest_url', None) + tinfo["cvbr_manifest_url"] = getattr(title, 'cvbr_manifest_url', None) + + tinfo["tracks"] = tinfo.get("tracks", {}) + track_data = tinfo["tracks"].setdefault(str(track), {}) + k_data = track_data.setdefault("keys", {}) + for kid, key in keys.items(): + k_data[kid] = key + + export_path.write_text(json.dumps(doc, indent=4, ensure_ascii=False), encoding="utf-8") \ No newline at end of file diff --git a/wpgskd/config/__init__.py b/wpgskd/config/__init__.py new file mode 100644 index 0000000..d5f7d78 --- /dev/null +++ b/wpgskd/config/__init__.py @@ -0,0 +1,123 @@ +import os +import sys +import logging +from types import SimpleNamespace +from pathlib import Path + +import yaml +from appdirs import AppDirs +from requests.utils import CaseInsensitiveDict + +from wpgskd.utils.collections import merge_dict + +class Directories: + def __init__(self): + self.app_dirs = AppDirs("wpgskd", False) + self.package_root = Path(__file__).resolve().parent.parent + self.project_root = self.package_root.parent + + self.configuration = self.project_root / "config" + self.user_configs = self.project_root + + self.service_configs = self.package_root / "servicookies" + + self.data = self.package_root + + self.downloads = self.project_root / "downloads" + self.temp = self.project_root / "temp" + self.cache = self.project_root / "cache" + self.logs = self.project_root / "logs" + self.exports = self.project_root / "exports" + + self.cookies = self.service_configs + + self.devices = self.project_root / "devices" + if not self.devices.exists(): + self.devices = self.package_root / "devices" + +class Filenames: + def __init__(self): + self.log = os.path.join(directories.logs, "wpgskd_{time}.log") + self.root_config = os.path.join(directories.package_root, "wpgskd.yml") + self.user_root_config = os.path.join(directories.user_configs, "wpgskd.yml") + + self.service_config = os.path.join(directories.configuration, "services", "{service}.yml") + self.user_service_config = os.path.join(directories.service_configs, "{service}.yml") + + self.subtitles = os.path.join(directories.temp, "TextTrack_{id}_{language_code}.srt") + self.chapters = os.path.join(directories.temp, "{filename}_chapters.txt") + +directories = Directories() +filenames = Filenames() + +os.makedirs(directories.logs, exist_ok=True) +os.makedirs(directories.temp, exist_ok=True) +os.makedirs(directories.downloads, exist_ok=True) +os.makedirs(directories.cache, exist_ok=True) +os.makedirs(directories.exports, exist_ok=True) + +config_data = {} +if os.path.exists(filenames.root_config): + try: + with open(filenames.root_config, encoding='utf-8') as fd: + loaded = yaml.safe_load(fd) + if loaded: config_data = loaded + except Exception as e: + print(f"Error loading config {filenames.root_config}: {e}") + +user_config_data = {} +if os.path.exists(filenames.user_root_config): + try: + with open(filenames.user_root_config, encoding='utf-8') as fd: + loaded = yaml.safe_load(fd) + if loaded: user_config_data = loaded + except Exception as e: + print(f"Error loading user config {filenames.user_root_config}: {e}") + +merge_dict(config_data, user_config_data) + +if not config_data: + print(f"Warning: No configuration loaded. Please ensure {filenames.root_config} exists.") + +config = SimpleNamespace(**config_data) + +credentials = getattr(config, 'credentials', {}) + +def setup_paths(): + if hasattr(config, 'directories'): + downloads_path = config.directories.get('downloads') + temp_path = config.directories.get('temp') + + if downloads_path: + p = Path(downloads_path) + if not p.is_absolute(): p = directories.project_root / p + directories.downloads = p + os.makedirs(directories.downloads, exist_ok=True) + + if temp_path: + p = Path(temp_path) + if not p.is_absolute(): p = directories.project_root / p + directories.temp = p + os.makedirs(directories.temp, exist_ok=True) + + filenames.subtitles = os.path.join(directories.temp, "TextTrack_{id}_{language_code}.srt") + filenames.chapters = os.path.join(directories.temp, "{filename}_chapters.txt") + +setup_paths() + +try: + from wpgskd.servicookies import SERVICE_MAP +except ImportError: + SERVICE_MAP = {} + +if not hasattr(config, 'arguments'): + config.arguments = {} + +if "range_" not in config.arguments: + config.arguments["range_"] = config.arguments.get("range") + +for service, aliases in SERVICE_MAP.items(): + for alias in aliases: + config.arguments[alias] = config.arguments.get(service) + +config.arguments = CaseInsensitiveDict(config.arguments) \ No newline at end of file diff --git a/wpgskd/config/wpgskd.yml b/wpgskd/config/wpgskd.yml new file mode 100644 index 0000000..6da17f7 --- /dev/null +++ b/wpgskd/config/wpgskd.yml @@ -0,0 +1,45 @@ +decrypter: 'packager' +tag: '' +tag_sd: '' + +arguments: {} + +aria2c: + file_allocation: 'prealloc' + +cdm: + default: '' + +credentials: {} + +directories: + temp: '' + downloads: '' + +headers: + User-Agent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36' + +key_vaults: + - type: 'local' + name: 'Local' + path: '{data_dir}/key_store.db' + +output_template: + movies: '{title}.{year}.{quality}.{source}.WEB-DL.{audio}.{video}-{tag}' + series: '{title}.{season_episode}.{episode_name}.{quality}.{source}.WEB-DL.{audio}.{video}-{tag}' + use_last_audio: false + +profiles: + default: 'default' + +default_proxy_service: null # Options: 'surfshark', 'nordvpn' + +proxies: {} + +nordvpn: + #username: '' + #password: '' + +surfshark: + #username: '' + #password: '' \ No newline at end of file diff --git a/wpgskd/constants.py b/wpgskd/constants.py new file mode 100644 index 0000000..12ea165 --- /dev/null +++ b/wpgskd/constants.py @@ -0,0 +1,56 @@ +from enum import Enum + +class EncryptionScheme(Enum): + NONE = "none" + WIDEVINE = "widevine" + PLAYREADY = "playready" + CLEARKEY = "clearkey" + AES_128 = "aes-128" + AES_128_ECB = "aes-128-ecb" + SAMPLE_AES = "SAMPLE-AES" + +LANGUAGE_MUX_MAP = { + "none": "und", + "nb": "nor", +} + +TERRITORY_MAP = { + "001": "", + "150": "European", + "419": "Latin American", + "AU": "Australian", + "BE": "Flemish", + "BR": "Brazilian", + "CA": "Canadian", + "CZ": "", + "CN": "Chinese Mainland", + "DK": "", + "EG": "Egyptian", + "ES": "European", + "FR": "European", + "GB": "British", + "GR": "", + "HK": "Hong Kong", + "IL": "", + "IN": "", + "JP": "Japan", + "KR": "", + "MY": "", + "NO": "", + "PH": "", + "PS": "Palestinian", + "PT": "European", + "SE": "", + "SY": "Syrian", + "TW": "Taiwan", + "US": "American", +} + +LANGUAGE_MAX_DISTANCE = 5 + +CODEC_MAP = { + "avc1": "H.264", "avc3": "H.264", "hev1": "H.265", "hvc1": "H.265", "dvh1": "H.265", "dvhe": "H.265", "av01": "AV1", + "aac": "AAC", "mp4a": "AAC", "stereo": "AAC", "HE": "HE-AAC", "ac3": "AC3", "ac-3": "AC3", "dd": "DD", + "eac": "E-AC3", "eac3": "E-AC3", "eac-3": "E-AC3", "ec-3": "DD+", "ddp": "DD+", "dd+": "DD+", "atmos": "DD+ Atmos", "ec3": "DD+", + "srt": "SRT", "vtt": "VTT", "wvtt": "WVTT", "dfxp": "TTML", "stpp": "TTML", "ttml": "TTML", "tt": "TTML", "ass": "ASS", "ssa": "SSA", +} \ No newline at end of file diff --git a/wpgskd/core/__init__.py b/wpgskd/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/wpgskd/core/cdm/__init__.py b/wpgskd/core/cdm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/wpgskd/core/cdm/base.py b/wpgskd/core/cdm/base.py new file mode 100644 index 0000000..3755247 --- /dev/null +++ b/wpgskd/core/cdm/base.py @@ -0,0 +1,40 @@ +from abc import ABC, abstractmethod +from typing import List, Optional, Union +from uuid import UUID + +class BaseCdm(ABC): + + @property + @abstractmethod + def security_level(self) -> int: + pass + + @property + @abstractmethod + def system_id(self) -> int: + pass + + @property + @abstractmethod + def cdm_type(self) -> str: + pass + + @abstractmethod + def open(self) -> bytes: + pass + + @abstractmethod + def close(self, session_id: bytes) -> None: + pass + + @abstractmethod + def get_license_challenge(self, session_id: bytes, pssh_data: Union[str, bytes], privacy_mode: bool = True) -> bytes: + pass + + @abstractmethod + def parse_license(self, session_id: bytes, license_message: Union[str, bytes]) -> None: + pass + + @abstractmethod + def get_keys(self, session_id: bytes, key_type: Optional[str] = None) -> List[dict]: + pass \ No newline at end of file diff --git a/wpgskd/core/cdm/custom_remote_cdm.py b/wpgskd/core/cdm/custom_remote_cdm.py new file mode 100644 index 0000000..d732606 --- /dev/null +++ b/wpgskd/core/cdm/custom_remote_cdm.py @@ -0,0 +1,67 @@ +import logging +import base64 +import requests +from typing import List, Optional, Union, Dict, Any +from wpgskd.core.cdm.base import BaseCdm + +log = logging.getLogger("RemoteCDM") + +class RemoteCdmAdapter(BaseCdm): + def __init__(self, api_config: Dict[str, Any]): + self.host = api_config["host"] + self.key = api_config["key"] + self.device_name = api_config["device"] + self._security_level = int(api_config.get("security_level", 3)) + self._system_id = int(api_config.get("system_id", 0)) + self._cdm_type = api_config.get("type", "widevine").lower() + self.session = requests.Session() + self.session.headers.update({"X-Secret-Key": self.key}) + + @property + def security_level(self): return self._security_level + @property + def system_id(self): return self._system_id + @property + def cdm_type(self): return self._cdm_type + + def open(self) -> bytes: + r = self.session.get(f"{self.host}/{self.device_name}/open").json() + if r['status'] != 200: raise ValueError(r['message']) + return bytes.fromhex(r["data"]["session_id"]) + + def close(self, session_id: bytes) -> None: + self.session.get(f"{self.host}/{self.device_name}/close/{session_id.hex()}") + + def get_license_challenge(self, session_id: bytes, pssh_data: Union[str, bytes], privacy_mode: bool = True) -> bytes: + if isinstance(pssh_data, bytes): + pssh_data = base64.b64encode(pssh_data).decode() + + payload = { + "session_id": session_id.hex(), + "init_data": pssh_data, + "privacy_mode": privacy_mode + } + r = self.session.post(f"{self.host}/{self.device_name}/get_license_challenge", json=payload).json() + if r['status'] != 200: raise ValueError(r['message']) + return base64.b64decode(r["data"]["challenge_b64"]) + + def parse_license(self, session_id: bytes, license_message: Union[str, bytes]) -> None: + if isinstance(license_message, bytes): + license_message = base64.b64encode(license_message).decode() + + payload = { + "session_id": session_id.hex(), + "license_message": license_message + } + r = self.session.post(f"{self.host}/{self.device_name}/parse_license", json=payload).json() + if r['status'] != 200: raise ValueError(r['message']) + + def get_keys(self, session_id: bytes, key_type: Optional[str] = None) -> List[dict]: + payload = {"session_id": session_id.hex()} + r = self.session.post(f"{self.host}/{self.device_name}/get_keys", json=payload).json() + if r['status'] != 200: raise ValueError(r['message']) + + keys = r["data"]["keys"] + if key_type: + return [k for k in keys if k.get("type") == key_type] + return keys \ No newline at end of file diff --git a/wpgskd/core/cdm/detect.py b/wpgskd/core/cdm/detect.py new file mode 100644 index 0000000..827d03e --- /dev/null +++ b/wpgskd/core/cdm/detect.py @@ -0,0 +1,19 @@ +from pathlib import Path +import logging + +log = logging.getLogger("CDMDetect") + +def detect_cdm_type(path: Path) -> str: + try: + with open(path, "rb") as f: + header = f.read(3) + + if header == b"WVD": + return "widevine" + elif header == b"PRD": + return "playready" + else: + raise ValueError(f"Unknown CDM file header: {header}") + except Exception as e: + log.error(f"Failed to detect CDM type for {path}: {e}") + raise \ No newline at end of file diff --git a/wpgskd/core/cdm/loader.py b/wpgskd/core/cdm/loader.py new file mode 100644 index 0000000..559f37f --- /dev/null +++ b/wpgskd/core/cdm/loader.py @@ -0,0 +1,261 @@ +import json +import time +import logging +from pathlib import Path +from typing import Optional, Dict, Any +from datetime import datetime, timedelta +from Crypto.Random import get_random_bytes + +from wpgskd.core.cdm.base import BaseCdm +from wpgskd.core.cdm.detect import detect_cdm_type + +log = logging.getLogger("CDMLoader") + +class CdmProvider: + def __init__(self, cdm_name: str, device_dir: Path, cdm_api_config: Optional[Dict[str, Any]] = None): + self.cdm_name = cdm_name + self.device_dir = device_dir + self.cdm_api_config = cdm_api_config or {} + self._cdm_instance: Optional[BaseCdm] = None + + @property + def cdm_instance(self) -> BaseCdm: + if self._cdm_instance is None: + self._cdm_instance = self._load_cdm() + return self._cdm_instance + + @property + def is_playready(self) -> bool: + return self.cdm_instance.cdm_type == "playready" + + @property + def is_widevine(self) -> bool: + return self.cdm_instance.cdm_type == "widevine" + + def _load_cdm(self) -> BaseCdm: + local_path = self.device_dir / self.cdm_name + if local_path.is_file(): + return self._load_local_cdm(local_path) + + for ext in ['.wvd', '.prd']: + path_with_ext = self.device_dir / f"{self.cdm_name}{ext}" + if path_with_ext.is_file(): + return self._load_local_cdm(path_with_ext) + + dev_dir = self.device_dir / self.cdm_name + if dev_dir.is_dir(): + if (dev_dir / 'zgpriv.dat').is_file() and (dev_dir / 'bgroupcert.dat').is_file(): + prd_path = self._create_playready_device(dev_dir) + if prd_path: + return self._load_local_cdm(prd_path) + + return self._load_local_dir(dev_dir) + + if self.cdm_name in self.cdm_api_config: + return self._load_remote_cdm(self.cdm_api_config[self.cdm_name]) + + raise ValueError(f"CDM '{self.cdm_name}' not found locally or in API config.") + + def _load_local_cdm(self, path: Path) -> BaseCdm: + cdm_type = detect_cdm_type(path) + + if cdm_type == "widevine": + try: + from pywidevine.cdm import Cdm as PyWidevineCdm + from pywidevine.device import Device as PyWidevineDevice + device = PyWidevineDevice.load(path) + return WidevineCdmAdapter(PyWidevineCdm.from_device(device)) + except ImportError: + log.warning("pywidevine not installed, falling back to built-in legacy widevine.") + from pywidevine.cdm import Cdm as LegacyWVCdm + from pywidevine.device import LocalDevice as LegacyWVDevice + device = LegacyWVDevice.load(path) + return WidevineCdmAdapter(LegacyWVCdm(device)) + + elif cdm_type == "playready": + try: + from pyplayready.cdm import Cdm as PyPlayReadyCdm + from pyplayready.device import Device as PyPlayReadyDevice + device = PyPlayReadyDevice.load(path) + return PlayReadyCdmAdapter(PyPlayReadyCdm.from_device(device)) + except ImportError: + log.warning("pyplayready not installed, falling back to built-in legacy playready.") + from pyplayready.cdm import Cdm as LegacyPRCdm + from pyplayready.device import Device as LegacyPRDevice + device = LegacyPRDevice.load(path) + return PlayReadyCdmAdapter(LegacyPRCdm.from_device(device)) + else: + raise ValueError(f"Unsupported CDM file format: {path.suffix}") + + def _load_local_dir(self, path: Path) -> BaseCdm: + if not path.is_dir(): + raise ValueError(f"CDM directory not found at: {path}") + + log.debug(f"Loading CDM from directory: {path}") + from pywidevine.device import LocalDevice as LegacyWVDevice + from pywidevine.cdm import Cdm as LegacyWVCdm + device = LegacyWVDevice.from_dir(str(path)) + return WidevineCdmAdapter(LegacyWVCdm(device)) + + def _create_playready_device(self, device_dir: Path) -> Optional[Path]: + try: + from pyplayready.crypto.ecc_key import ECCKey + from pyplayready.system.bcert import CertificateChain, Certificate + from pyplayready.device import Device as DevicePR + except ImportError: + log.error("Built-in pyplayready not available, cannot generate .prd from directory.") + return None + + group_key_path = device_dir / 'zgpriv.dat' + group_cert_path = device_dir / 'bgroupcert.dat' + infofile = device_dir / 'PR.json' + + if infofile.is_file(): + try: + with open(infofile, 'r') as f: + info = json.load(f) + if "expiry" in info and datetime.fromisoformat(info["expiry"]) > datetime.now(): + existing = device_dir / info["device"] + if existing.is_file(): + log.info(f" + Loading existing generated PlayReady device: {info['device']}") + return existing + except Exception: + pass + + log.info(" + Generating new PlayReady Device (.prd) from directory...") + try: + enc_key = ECCKey.generate() + sig_key = ECCKey.generate() + + gk_obj = ECCKey.load(group_key_path) + chain = CertificateChain.load(group_cert_path) + + new_cert = Certificate.new_leaf_cert( + cert_id=get_random_bytes(16), + security_level=chain.get_security_level(), + client_id=get_random_bytes(16), + signing_key=sig_key, + encryption_key=enc_key, + group_key=gk_obj, + parent=chain, + ) + chain.prepend(new_cert) + + device = DevicePR( + group_key=gk_obj.dumps(), + encryption_key=enc_key.dumps(), + signing_key=sig_key.dumps(), + group_certificate=chain.dumps(), + ) + + expiry = (datetime.now() + timedelta(days=3650)).isoformat() + raw = device.dumps() + out_path = device_dir / f"{device.get_name()}_{raw[:4].hex()}.prd" + + if out_path.exists(): + log.error(f"Device file already exists: {out_path}") + return None + + out_path.write_bytes(raw) + + with open(infofile, 'w') as f: + json.dump({ + "expiry": expiry, + "device": out_path.name, + "SecurityLevel": device.security_level, + "created": datetime.now().isoformat(), + }, f) + + log.info(f" + Created PlayReady Device: {out_path.name}") + return out_path + + except Exception as e: + log.error(f"Failed to generate PlayReady device: {e}") + return None + + def _load_remote_cdm(self, api_config: Dict[str, Any]) -> BaseCdm: + from wpgskd.core.cdm.custom_remote_cdm import RemoteCdmAdapter + log.info(f"Loading Remote CDM: {api_config.get('name')}") + return RemoteCdmAdapter(api_config) + + def log_info(self): + cdm = self.cdm_instance + log.info(f" + CDM Type: {cdm.cdm_type.upper()}") + log.info(f" + Security Level: L{cdm.security_level}") + if cdm.system_id: + log.info(f" + System ID: {cdm.system_id}") + +class WidevineCdmAdapter(BaseCdm): + def __init__(self, cdm_instance): + self._cdm = cdm_instance + + @property + def security_level(self): return self._cdm.security_level + @property + def system_id(self): return self._cdm.system_id + @property + def cdm_type(self): return "widevine" + + def open(self): return self._cdm.open() + def close(self, session_id): self._cdm.close(session_id) + + def get_license_challenge(self, session_id, pssh_data, privacy_mode=True): + try: + from pywidevine.pssh import PSSH + from wpgskd.vendor.pymp4.parser import Box as Pymp4Box + if hasattr(pssh_data, 'type') and hasattr(pssh_data, 'init_data') and not isinstance(pssh_data, PSSH): + pssh_bytes = Pymp4Box.build(pssh_data) + pssh_data = PSSH(pssh_bytes) + except Exception: + pass + + if hasattr(self._cdm, 'get_license_challenge'): + import inspect + sig = inspect.signature(self._cdm.get_license_challenge) + if 'service_name' in sig.parameters: + return self._cdm.get_license_challenge(session_id, pssh_data, service_name="default") + return self._cdm.get_license_challenge(session_id, pssh_data, privacy_mode=privacy_mode) + + def parse_license(self, session_id, license_message): + self._cdm.parse_license(session_id, license_message) + + def get_keys(self, session_id, key_type="CONTENT"): + keys = self._cdm.get_keys(session_id) + result = [] + for k in keys: + kid = k.kid.hex if hasattr(k.kid, 'hex') else str(k.kid).replace("-", "") + key = k.key.hex() if isinstance(k.key, bytes) else str(k.key) + result.append({"kid": kid, "key": key, "type": k.type}) + + if key_type: + return [k for k in result if k['type'] == key_type] + return result + +class PlayReadyCdmAdapter(BaseCdm): + def __init__(self, cdm_instance): + self._cdm = cdm_instance + + @property + def security_level(self): return self._cdm.security_level + @property + def system_id(self): return 1 + @property + def cdm_type(self): return "playready" + + def open(self): return self._cdm.open() + def close(self, session_id): self._cdm.close(session_id) + + def get_license_challenge(self, session_id, pssh_data, privacy_mode=True): + return self._cdm.get_license_challenge(session_id, pssh_data) + + def parse_license(self, session_id, license_message): + self._cdm.parse_license(session_id, license_message) + + def get_keys(self, session_id, key_type=None): + keys = self._cdm.get_keys(session_id) + result = [] + for k in keys: + kid = k.key_id.hex if hasattr(k.key_id, 'hex') else str(k.key_id).replace("-", "") + key = k.key.hex() if isinstance(k.key, bytes) else str(k.key) + result.append({"kid": kid, "key": key, "type": "CONTENT"}) + return result \ No newline at end of file diff --git a/wpgskd/core/config.py b/wpgskd/core/config.py new file mode 100644 index 0000000..7c9e355 --- /dev/null +++ b/wpgskd/core/config.py @@ -0,0 +1,51 @@ +import logging +from pathlib import Path +from typing import Any, Optional, List +from dataclasses import dataclass, field + +log = logging.getLogger("CoreConfig") + +@dataclass +class CoreConfig: + cdm_name: str = "default" + decrypter: str = "packager" + profile: str = "default" + + quality: Optional[Any] = None + vcodec: str = "H264" + acodec: Optional[str] = None + vbitrate: Optional[int] = None + abitrate: Optional[int] = None + atmos: bool = False + channels: Optional[str] = None + range_: str = "SDR" + wanted: Optional[List[str]] = None + alang: List[str] = field(default_factory=lambda: ["orig"]) + slang: List[str] = field(default_factory=lambda: ["all"]) + + audio_only: bool = False + subs_only: bool = False + chapters_only: bool = False + no_subs: bool = False + no_audio: bool = False + no_video: bool = False + no_chapters: bool = False + audio_description: bool = False + no_mux: bool = False + mux: bool = False + worst: bool = False + sync_vat: bool = False + no_sync_subs: bool = False + + use_cache: bool = True + use_cdm: bool = True + export: bool = False + keys_only: bool = False + + temp_dir: Path = None + out_dir: Path = None + + def apply_overrides(self, **kwargs): + for key, value in kwargs.items(): + if value is not None and hasattr(self, key): + setattr(self, key, value) \ No newline at end of file diff --git a/wpgskd/core/console.py b/wpgskd/core/console.py new file mode 100644 index 0000000..1c0be26 --- /dev/null +++ b/wpgskd/core/console.py @@ -0,0 +1,93 @@ +import logging +from typing import List, Any +from wpgskd.core.tracks.title import Title, Titles +from wpgskd.core.utilities import humanize_size, format_duration + +log = logging.getLogger("Console") + +class ConsoleUI: + + @staticmethod + def print_titles(titles: Titles): + if not titles: + return + + is_tv = any(x.type == Title.Types.TV for x in titles) + + if is_tv: + seasons = {} + for t in titles: + s = getattr(t, 'season', 0) + seasons.setdefault(s, []).append(t) + + breakdown = ", ".join(f"S{s}({len(seasons[s])})" for s in sorted(seasons.keys())) + log.info(f"{len(seasons)} seasons, {breakdown}") + else: + label = f"{len(titles)} Movie{['s', ''][len(titles) == 1]}" + log.info(label) + for m in titles: + name = getattr(m, 'name', str(m)) + year = getattr(m, 'year', None) + log.info(f" {name} ({year or '?'})") + + @staticmethod + def print_tracks(tracks: Any, title: Title = None): + if not tracks: + return + + for v in tracks.videos: + codec = v.get_codec_display() + + range_str = "SDR" + if getattr(v, 'dvhdr', False): range_str = "DV+HDR" + elif getattr(v, 'dv', False): range_str = "DV" + elif getattr(v, 'hdr10', False): range_str = "HDR10" + elif getattr(v, 'hlg', False): range_str = "HLG" + + res_str = f"{v.width}x{v.height}" + bitrate_str = f"{v.bitrate // 1000 if v.bitrate else '?'} kb/s" + fps_str = f"{v.fps:.3f} FPS" if v.fps else "N/A" + + dur_sec = v.duration_seconds() + size_bytes = v.size if v.size else v.computed_size_bytes() + size_str = humanize_size(size_bytes) if size_bytes else "N/A" + dur_str = format_duration(dur_sec) if dur_sec else "N/A" + + enc_str = "Encrypted" if v.encrypted else "Unencrypted" + + log.info(f"├─ VID | {codec} | {range_str} | {res_str} | {bitrate_str} | {fps_str} | {size_str} | {dur_str} | {enc_str}") + + for a in tracks.audios: + codec = a.get_codec_display() + ch_str = a.channels or "?" + bitrate_str = f"{a.bitrate // 1000 if a.bitrate else '?'} kb/s" + lang_str = str(a.language) + + desc_str = " (Descriptive)" if a.descriptive else "" + orig_str = " [Original]" if a.is_original_lang else "" + + dur_sec = a.duration_seconds() + size_bytes = a.size if a.size else a.computed_size_bytes() + size_str = humanize_size(size_bytes) if size_bytes else "N/A" + dur_str = format_duration(dur_sec) if dur_sec else "N/A" + + enc_str = "Encrypted" if a.encrypted else "Unencrypted" + + log.info(f"├─ AUD | {codec} | {ch_str} | {bitrate_str} | {lang_str}{orig_str}{desc_str} | {size_str} | {dur_str} | {enc_str}") + + for t in tracks.subtitles: + codec = t.codec or "vtt" + + flags = [] + if t.is_original_lang: flags.append("orig") + if t.forced: flags.append("Forced") + if t.sdh: flags.append("SDH") + if t.cc: flags.append("CC") + flag_str = " ".join(flags) + + lang_str = str(t.language) + + parts = ["├─ SUB", codec, lang_str] + if flag_str: + parts.append(flag_str) + log.info(" | ".join(parts)) \ No newline at end of file diff --git a/wpgskd/core/constants.py b/wpgskd/core/constants.py new file mode 100644 index 0000000..f4f42e0 --- /dev/null +++ b/wpgskd/core/constants.py @@ -0,0 +1,82 @@ +from enum import Enum + +class EncryptionScheme(Enum): + NONE = "none" + WIDEVINE = "widevine" + PLAYREADY = "playready" + CLEARKEY = "clearkey" + AES_128 = "aes-128" + AES_128_ECB = "aes-128-ecb" + SAMPLE_AES = "SAMPLE-AES" + +LANGUAGE_MUX_MAP = { + "none": "und", + "nb": "nor", +} + +TERRITORY_MAP = { + "001": "", + "150": "European", + "419": "Latin American", + "AU": "Australian", + "BE": "Flemish", + "BR": "Brazilian", + "CA": "Canadian", + "CZ": "", + "CN": "Chinese Mainland", + "DK": "", + "EG": "Egyptian", + "ES": "European", + "FR": "European", + "GB": "British", + "GR": "", + "HK": "Hong Kong", + "IL": "", + "IN": "", + "JP": "Japan", + "KR": "", + "MY": "", + "NO": "", + "PH": "", + "PS": "Palestinian", + "PT": "European", + "SE": "", + "SY": "Syrian", + "TW": "Taiwan", + "US": "American", +} + +LANGUAGE_MAX_DISTANCE = 5 + +CODEC_MAP = { + "avc1": "H.264", + "hev1": "H.265", + "hvc1": "H.265", + "dvh1": "H.265", + "dvhe": "H.265", + "av01": "AV1", + "aac": "AAC", + "mp4a": "AAC", + "stereo": "AAC", + "HE": "HE-AAC", + "ac3": "AC3", + "ac-3": "AC3", + "dd": "DD", + "eac": "E-AC3", + "eac3": "E-AC3", + "eac-3": "E-AC3", + "ec-3": "DD+", + "ddp": "DD+", + "dd+": "DD+", + "atmos": "DD+ Atmos", + "ec3": "DD+", + "srt": "SRT", + "vtt": "VTT", + "wvtt": "WVTT", + "dfxp": "TTML", + "stpp": "TTML", + "ttml": "TTML", + "tt": "TTML", + "ass": "ASS", + "ssa": "SSA", +} \ No newline at end of file diff --git a/wpgskd/core/credential.py b/wpgskd/core/credential.py new file mode 100644 index 0000000..440fa27 --- /dev/null +++ b/wpgskd/core/credential.py @@ -0,0 +1,59 @@ +import hashlib +import re +from typing import Optional + +import requests +import validators + +class Credential: + """Username (or Email) and Password Credential.""" + + def __init__(self, username: str, password: str, extra: Optional[str] = None): + self.username = username + self.password = password + self.extra = extra + self.sha1 = hashlib.sha1(self.dumps().encode()).hexdigest() + + def __bool__(self): + return bool(self.username) and bool(self.password) + + def __str__(self): + return self.dumps() + + def __repr__(self): + return "{name}({items})".format( + name=self.__class__.__name__, + items=", ".join([f"{k}={repr(v)}" for k, v in self.__dict__.items()]) + ) + + def dumps(self) -> str: + """Return credential data as a string.""" + return f"{self.username}:{self.password}" + (f":{self.extra}" if self.extra else "") + + def dump(self, path: str): + """Write credential data to a file.""" + with open(path, "w", encoding="utf-8") as fd: + fd.write(self.dumps()) + + @classmethod + def loads(cls, text: str) -> 'Credential': + """ + Load credential from a text string. + Format: {username}:{password}[:{extra}] + """ + text = "".join([x.strip() for x in text.splitlines(keepends=False)]).strip() + credential = re.fullmatch(r"^([^:]+?):([^:]+?)(?::(.+))?$", text) + if credential: + return cls(*credential.groups()) + raise ValueError("No credentials found in text string. Expecting the format `username:password`") + + @classmethod + def load(cls, uri: str, session: Optional[requests.Session] = None) -> 'Credential': + """ + Load Credential from a remote URL string or a local file path. + """ + if validators.url(uri): + return cls.loads((session or requests).get(uri).text) + else: + with open(uri, encoding="utf-8") as fd: + return cls.loads(fd.read()) \ No newline at end of file diff --git a/wpgskd/core/decryptor.py b/wpgskd/core/decryptor.py new file mode 100644 index 0000000..526bc9c --- /dev/null +++ b/wpgskd/core/decryptor.py @@ -0,0 +1,188 @@ +import os +import sys +import re +import shutil +import logging +import subprocess +from typing import Optional, Dict, Any +from io import TextIOWrapper + +from wpgskd.core.tracks.tracks import Track +from wpgskd.core.tracks.video import VideoTrack +from wpgskd.core.tracks.audio import AudioTrack + +log = logging.getLogger("Decryptor") + +class Decryptor: + + @staticmethod + def find_executable(name: str) -> Optional[str]: + if name == "packager": + plat = {"win32": "win", "darwin": "osx"}.get(sys.platform, sys.platform) + candidates = ["shaka-packager", "packager", f"packager-{plat}"] + for c in candidates: + path = shutil.which(c) + if path: return path + return None + if name == "mp4decrypt": + return shutil.which("mp4decrypt") + return shutil.which(name) + + @staticmethod + def decrypt(track: Track, keys: Dict[str, str], engine: str, temp_dir: str) -> Optional[str]: + src = track.locate() + if not src or not os.path.exists(src): + log.error(f"Source file not found for decryption: {src}") + return None + + dst = os.path.splitext(src)[0] + ".dec.mp4" + + if getattr(track, 'smooth', False) or getattr(track, 'encryption_scheme', None) == 'clearkey': + engine = "mp4decrypt" + + if engine == "packager": + dec = Decryptor._packager(track, keys, src, dst, temp_dir) + elif engine == "mp4decrypt": + dec = Decryptor._mp4decrypt(keys, src, dst) + else: + log.error(f"Unsupported decrypter engine: {engine}") + return None + + return dec + + @staticmethod + def _packager(track: Track, keys: Dict[str, str], src: str, dst: str, tmp: str) -> Optional[str]: + exe = Decryptor.find_executable("packager") + if not exe: + raise FileNotFoundError("shaka-packager executable not found") + + stream = track.__class__.__name__.lower().replace("track", "") + + pk = track.kid.lower().replace("-", "") + pv = keys.get(pk, next(iter(keys.values()), "")) if keys else "" + + if not pv: + log.error("No valid key provided for shaka-packager") + return None + + os.makedirs(tmp, exist_ok=True) + + cmd = [ + exe, + f"input={src},stream={stream},output={dst}", + "--enable_raw_key_decryption", "--keys", + f"label=0:key_id={pk}:key={pv.lower()}, " + f"label=1:key_id={'0' * 32}:key={pv.lower()}", + "--temp_dir", tmp, + ] + + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + last = "" + + for line in proc.stdout: + line = line.strip() + if not line: continue + + if re.match(r"^\d+/\d+$", line): + sys.stdout.write(f"\r + Decrypting: {line}") + sys.stdout.flush() + last = line + elif "Packaging completed successfully" in line: + msg = f"{last} - Complete" if last else "Complete" + sys.stdout.write(f"\r + Decrypting: {msg}\n") + sys.stdout.flush() + elif any(w in line.lower() for w in ("error", "fail", "warning")): + print(f"\n ! {line}") + elif line and not any(t in line for t in ("progress", "%", "[", "]")): + print(f"\n + {line}") + + proc.wait() + + if proc.returncode != 0: + raise subprocess.CalledProcessError(proc.returncode, proc.args) + + return dst + + @staticmethod + def _mp4decrypt(keys: Dict[str, str], src: str, dst: str) -> Optional[str]: + exe = Decryptor.find_executable("mp4decrypt") + if not exe: + raise FileNotFoundError("mp4decrypt executable not found") + + cmd = [exe, "--show-progress"] + + for kid, key in keys.items(): + cmd.extend(["--key", f"{kid}:{key.lower()}"]) + + cmd.extend([src, dst]) + + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + + for line in proc.stdout: + line = line.strip() + if not line: continue + + if re.search(r"\d+%", line) or re.search(r"\d+/\d+", line): + sys.stdout.write(f"\r + Decrypting: {line}") + sys.stdout.flush() + elif "Progress" in line: + continue + elif any(w in line.lower() for w in ("error", "fail")): + print(f"\n ! {line}") + else: + print(f" + {line}") + + proc.wait() + + if proc.returncode != 0: + raise subprocess.CalledProcessError(proc.returncode, proc.args) + + return dst + + @staticmethod + def repackage(path: str) -> bool: + if not shutil.which("ffmpeg"): + log.warning("FFmpeg not found, skipping repackage") + return False + + fixed = f"{path}_fixed.mkv" + try: + proc = subprocess.Popen([ + "ffmpeg", "-hide_banner", "-loglevel", "error", + "-i", path, "-map_metadata", "-1", + "-fflags", "bitexact", "-codec", "copy", fixed, + ], stderr=subprocess.PIPE, text=True) + + for line in proc.stderr: + line = line.strip() + if not line: continue + if re.search(r"frame=\s*\d+", line): + sys.stdout.write(f"\r + Repackaging: {line[:60]}") + sys.stdout.flush() + elif "Insufficient bits" in line: + sys.stdout.write(f"\n ! {line}\n + Continuing...") + sys.stdout.flush() + elif "error" in line.lower(): + print(f"\n ! {line}") + + proc.wait() + + if proc.returncode == 0 and os.path.exists(fixed): + sys.stdout.write("\r + Repackaging: Complete\n") + sys.stdout.flush() + os.unlink(path) + os.rename(fixed, path) + return True + + sys.stdout.write("\n") + log.warning(" - Repackage failed, keeping original file") + if os.path.exists(fixed): + os.unlink(fixed) + return False + + except Exception as e: + sys.stdout.write("\n") + log.warning(f" - Repackage failed: {e}") + if os.path.exists(fixed): + os.unlink(fixed) + return False \ No newline at end of file diff --git a/wpgskd/core/decryptors/__init__.py b/wpgskd/core/decryptors/__init__.py new file mode 100644 index 0000000..dfc9f14 --- /dev/null +++ b/wpgskd/core/decryptors/__init__.py @@ -0,0 +1,3 @@ +from wpgskd.core.decryptors.aes import AES128Decryptor + +__all__ = ['AES128Decryptor'] \ No newline at end of file diff --git a/wpgskd/core/decryptors/aes.py b/wpgskd/core/decryptors/aes.py new file mode 100644 index 0000000..0a4207e --- /dev/null +++ b/wpgskd/core/decryptors/aes.py @@ -0,0 +1,20 @@ +from Cryptodome.Cipher import AES +from wpgskd.core.decryptors.base import Decryptor + +class AES128Decryptor(Decryptor): + + def __init__(self, key: bytes, iv: bytes = None): + self.key = key + self.iv = iv + + def decrypt(self, data: bytes, sequence_number: int = 0, **kwargs) -> bytes: + current_iv = self.iv + if not current_iv: + current_iv = sequence_number.to_bytes(16, 'big') + + cipher = AES.new(self.key, AES.MODE_CBC, current_iv) + + try: + return cipher.decrypt(data) + except ValueError: + return data \ No newline at end of file diff --git a/wpgskd/core/decryptors/base.py b/wpgskd/core/decryptors/base.py new file mode 100644 index 0000000..c454887 --- /dev/null +++ b/wpgskd/core/decryptors/base.py @@ -0,0 +1,6 @@ +from abc import ABC, abstractmethod + +class Decryptor(ABC): + @abstractmethod + def decrypt(self, data: bytes, **kwargs) -> bytes: + pass \ No newline at end of file diff --git a/wpgskd/core/downloader.py b/wpgskd/core/downloader.py new file mode 100644 index 0000000..12d7f3d --- /dev/null +++ b/wpgskd/core/downloader.py @@ -0,0 +1,266 @@ +import os +import sys +import shutil +import logging +import asyncio +import subprocess +from pathlib import Path +from typing import Optional, Any + +import requests + +from wpgskd.constants import EncryptionScheme +from wpgskd.core.tracks.tracks import Track +from wpgskd.utils.io import aria2c, m3u8re + +log = logging.getLogger("Downloader") + +class Downloader: + + def __init__(self, session: requests.Session): + self.session = session + + def download(self, track: Track, out_dir: str, name: str = None, headers: dict = None, + proxy: str = None, title_ref: Any = None, all_keys: dict = None): + + if os.path.isfile(out_dir): + raise ValueError("Path must be to a directory and not to a file") + + os.makedirs(out_dir, exist_ok=True) + + merged_headers = {} + if headers: + merged_headers.update(headers) + if isinstance(getattr(track, 'extra', None), dict): + track_headers = track.extra.get("headers") + if isinstance(track_headers, dict): + merged_headers.update(track_headers) + headers = merged_headers or None + + re_name = (name or "{type}_{id}_{enc}").format( + type=track.__class__.__name__, + id=track.id, + enc="enc" if track.encrypted else "dec" + ) + + if track.source.lower() == "abematv": + self._download_abematv(track, out_dir, re_name) + return + + if getattr(track, 'manifest_url', None) and getattr(track, 'mpd_representation_id', None): + save_path = os.path.join(out_dir, self._get_filename(track, re_name)) + self._download_dash_manifest(track, save_path, headers, proxy) + track._location = save_path + return + + if track.descriptor == Track.Descriptor.ISM or getattr(track, 'smooth', False): + self._download_ism(track, out_dir, re_name, headers, proxy) + return + + if track.descriptor == Track.Descriptor.M3U and track.encryption_scheme == EncryptionScheme.AES_128: + save_path = os.path.join(out_dir, self._get_filename(track, re_name)) + self._download_m3u8(track, save_path, headers, proxy) + track._location = save_path + return + + first_url = track.url[0] if isinstance(track.url, list) else track.url + if track.descriptor == Track.Descriptor.M3U and isinstance(first_url, str) and ".m3u8" in first_url: + save_path = os.path.join(out_dir, self._get_filename(track, re_name)) + self._download_m3u8(track, save_path, headers, proxy) + track._location = save_path + return + + if isinstance(track.url, list) and ".m3u8" in first_url: + save_path = os.path.join(out_dir, self._get_filename(track, re_name)) + self._download_m3u8(track, save_path, headers, proxy) + track._location = save_path + return + + save_path = os.path.join(out_dir, self._get_filename(track, re_name)) + try: + req_headers = headers if track.source not in ["ATVP", "iT"] else {} + asyncio.run(aria2c( + track.url, save_path, + req_headers, + proxy if track.needs_proxy else None + )) + track._location = save_path + except (ValueError, subprocess.CalledProcessError) as e: + dash_url = getattr(title_ref, 'dash_manifest_url', None) if title_ref else None + if dash_url: + log.warning(f"aria2c download failed. Attempting fallback with N_m3u8DL-RE...") + try: + self._fallback_n_m3u8dl_re(track, dash_url, save_path, headers, proxy, all_keys) + track._location = save_path + except Exception as fallback_e: + log.error(f"Fallback download with N_m3u8DL-RE also failed: {fallback_e}") + raise e + else: + raise e + + def _get_filename(self, track: Track, re_name: str) -> str: + is_ass = hasattr(track, 'codec') and track.codec and track.codec.lower() in ['ass', 'ssa'] + is_wvtt = hasattr(track, 'codec') and track.codec and track.codec.lower() == 'wvtt' + + if is_ass: + return f"{re_name}.ass" + elif is_wvtt: + return f"{re_name}.vtt" + elif track.__class__.__name__ == "TextTrack": + return f"{re_name}.vtt" + elif track.__class__.__name__ == "AudioTrack" and track.source in ["iT", "ATVP", "TVer", "NHKPlus"]: + return f"{re_name}.m4a" + else: + return f"{re_name}.mp4" + + def _download_abematv(self, track: Track, out_dir: str, re_name: str): + base_name = "VideoTrack_master_enc" + muxed_location = os.path.join(out_dir, f"{base_name}.muxed.mkv") + + if os.path.exists(muxed_location): + log.info(f"AbemaTV: Found pre-muxed file at {muxed_location}") + track._location = muxed_location + return + + video_path = os.path.join(out_dir, f"{base_name}.mp4") + audio_path = os.path.join(out_dir, f"{base_name}.m4a") + + if os.path.exists(video_path) and os.path.exists(audio_path): + log.info("Muxing AbemaTV tracks (RE output)...") + cmd = [shutil.which("mkvmerge"), "-o", muxed_location, video_path, audio_path] + subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL) + try: + os.unlink(video_path) + os.unlink(audio_path) + except Exception: + pass + track._location = muxed_location + elif os.path.exists(video_path): + log.info("AbemaTV: Found single video file, using as muxed.") + os.rename(video_path, muxed_location) + track._location = muxed_location + else: + raise IOError("Missing RE output files for AbemaTV") + + def _download_m3u8(self, track: Track, save_path: str, headers: dict, proxy: str): + log.info(f"Downloading HLS stream using N_m3u8DL-RE...") + + key = None + if track.encryption_scheme == EncryptionScheme.AES_128: + pass + elif track.encryption_scheme == EncryptionScheme.CLEARKEY and track.key: + key = f"{track.kid}:{track.key}" + + try: + asyncio.run(m3u8re( + track.url[0] if isinstance(track.url, list) else track.url, + save_path, + headers, + proxy if track.needs_proxy else None, + key=key + )) + except Exception as e: + log.error(f"N_m3u8DL-RE failed: {e}") + raise + + def _download_dash_manifest(self, track: Track, save_path: str, headers: dict, proxy: str): + log.info(f"Downloading DASH manifest stream using N_m3u8DL-RE...") + executable = shutil.which("N_m3u8DL-RE") or shutil.which("m3u8re") + if not executable: + raise EnvironmentError("N_m3u8DL-RE executable not found...") + + mpd_url = getattr(track, 'manifest_url', None) + if not mpd_url: + raise RuntimeError("MPD manifest URL missing for track") + + out_dir = Path(save_path).parent + out_dir.mkdir(parents=True, exist_ok=True) + + cmd = [ + executable, + mpd_url, + "--save-name", Path(save_path).stem, + "--save-dir", str(out_dir), + "--tmp-dir", str(out_dir), + "--auto-subtitle-fix", "False", + "--log-level", "ERROR", + ] + + if hasattr(track, "mpd_representation_id") and track.mpd_representation_id: + cls_name = track.__class__.__name__ + if cls_name == "VideoTrack": + cmd += ["--select-video", f"id={track.mpd_representation_id}"] + elif cls_name == "AudioTrack": + cmd += ["--select-audio", f"id={track.mpd_representation_id}"] + elif cls_name == "TextTrack": + cmd += ["--select-subtitle", f"id={track.mpd_representation_id}"] + else: + if track.__class__.__name__ == "TextTrack": + cmd += ["--select-subtitle", f"lang={track.language}"] + + if track.needs_proxy and proxy: + cmd += ["--custom-proxy", proxy] + else: + cmd += ["--use-system-proxy", "False"] + + if track.encryption_scheme == EncryptionScheme.CLEARKEY and getattr(track, 'key', None) and getattr(track, 'kid', None): + cmd += ["--key", f"{track.kid}:{track.key}"] + + try: + subprocess.run(cmd, check=True) + except Exception as e: + raise e + + def _download_ism(self, track: Track, out_dir: str, re_name: str, headers: dict, proxy: str): + log.info(f"Downloading ISM stream using N_m3u8DL-RE...") + executable = shutil.which("N_m3u8DL-RE") or shutil.which("m3u8re") + if not executable: + raise EnvironmentError("N_m3u8DL-RE executable not found...") + + first_url = track.url[0] if isinstance(track.url, list) else track.url + ism_url = first_url.rsplit('/', 1)[0] + "/manifest" + ism_url = ism_url.split('?')[0] + + cmd = [ + executable, + ism_url, + "--save-name", re_name, + "--save-dir", out_dir, + "--tmp-dir", out_dir, + "--auto-subtitle-fix", "True", + "--log-level", "ERROR", + ] + + if track.needs_proxy and proxy: + cmd += ["--custom-proxy", proxy] + else: + cmd += ["--use-system-proxy", "False"] + + try: + subprocess.run(cmd, check=True) + files = list(Path(out_dir).glob(f"{re_name}*")) + if files: + track._location = str(files[0]) + else: + raise IOError("ISM download produced no file") + except Exception as e: + raise e + + def _fallback_n_m3u8dl_re(self, track: Track, dash_manifest_url: str, save_path: str, headers: dict, proxy: str, all_keys: dict): + executable = shutil.which("N_m3u8DL-RE") or shutil.which("m3u8re") + if not executable: + raise EnvironmentError("N_m3u8DL-RE executable not found...") + + cmd = [ + executable, + dash_manifest_url, + "--save-name", Path(save_path).stem, + "--save-dir", str(Path(save_path).parent), + "--tmp-dir", str(Path(save_path).parent), + "--log-level", "INFO", + ] + + if track.encrypted and all_keys and track.kid in all_keys: + cmd.extend(["--key", f"{track.kid}:{all_keys[track.kid]}"]) + + subprocess.run(cmd, check=True, capture_output=True, text=True) \ No newline at end of file diff --git a/wpgskd/core/drm/__init__.py b/wpgskd/core/drm/__init__.py new file mode 100644 index 0000000..42319b7 --- /dev/null +++ b/wpgskd/core/drm/__init__.py @@ -0,0 +1,6 @@ +from wpgskd.core.drm.base import BaseDRM +from wpgskd.core.drm.widevine import Widevine +from wpgskd.core.drm.playready import PlayReady +from wpgskd.core.drm.clearkey import ClearKey + +__all__ = ['BaseDRM', 'Widevine', 'PlayReady', 'ClearKey'] \ No newline at end of file diff --git a/wpgskd/core/drm/base.py b/wpgskd/core/drm/base.py new file mode 100644 index 0000000..6ddc158 --- /dev/null +++ b/wpgskd/core/drm/base.py @@ -0,0 +1,12 @@ +from abc import ABC, abstractmethod +from typing import Any, Dict, Optional, Tuple + +class BaseDRM(ABC): + + def __init__(self, cdm_provider: Any, service: Any): + self.cdm_provider = cdm_provider + self.service = service + + @abstractmethod + def get_keys(self, track: Any, title: Any, session: Any) -> Tuple[Optional[str], Dict[str, str]]: + pass \ No newline at end of file diff --git a/wpgskd/core/drm/clearkey.py b/wpgskd/core/drm/clearkey.py new file mode 100644 index 0000000..1ac2613 --- /dev/null +++ b/wpgskd/core/drm/clearkey.py @@ -0,0 +1,39 @@ +import logging +import base64 +import requests +from typing import Any, Dict, Tuple, Optional +from wpgskd.core.drm.base import BaseDRM +from wpgskd.core.resolver import KeyResolver + +log = logging.getLogger("DRM.ClearKey") + +class ClearKey(BaseDRM): + + def get_keys(self, track: Any, title: Any, session: Any) -> Tuple[Optional[str], Dict[str, str]]: + if getattr(track, 'key', None): + kid = KeyResolver._norm(track.kid) + return track.key, {kid: track.key} + + license_url = getattr(track, 'license_url', None) + if not license_url: + raise ValueError("Track missing license_url for ClearKey") + + kid_hex = track.kid.replace("-", "") + kid_b64 = base64.b64encode(bytes.fromhex(kid_hex)).decode() + + payload = {"kids": [kid_b64], "type": "temporary"} + + try: + res = session.post(license_url, json=payload) + res.raise_for_status() + data = res.json() + + k_b64 = data.get("keys", [{}])[0].get("k") + if not k_b64: + raise ValueError("No key returned from ClearKey server") + + key_hex = base64.b64decode(k_b64).hex() + return key_hex, {kid_hex: key_hex} + + except Exception as e: + raise ValueError(f"ClearKey request failed: {e}") \ No newline at end of file diff --git a/wpgskd/core/drm/playready.py b/wpgskd/core/drm/playready.py new file mode 100644 index 0000000..3c86851 --- /dev/null +++ b/wpgskd/core/drm/playready.py @@ -0,0 +1,62 @@ +import logging +import base64 +from typing import Any, Dict, Tuple, Optional +from wpgskd.core.drm.base import BaseDRM +from wpgskd.core.resolver import KeyResolver + +log = logging.getLogger("DRM.PlayReady") + +class PlayReady(BaseDRM): + + def get_keys(self, track: Any, title: Any, session: Any) -> Tuple[Optional[str], Dict[str, str]]: + cdm = self.cdm_provider.cdm_instance + + if cdm.cdm_type != "playready": + raise ValueError("CDM is not a PlayReady CDM") + + pr_pssh = getattr(track, 'pr_pssh', None) + if not pr_pssh: + raise ValueError("Track missing PR_PSSH for PlayReady CDM") + + try: + from pyplayready.system.pssh import PSSH as PRPSSH + wrm = PRPSSH(pr_pssh).wrm_headers[0] + except Exception: + raise ValueError("Failed to parse WRM Header from PR_PSSH") + + sid = cdm.open() + try: + challenge = cdm.get_license_challenge(sid, wrm).encode('utf-8') + license_res = self.service.license( + challenge=challenge, title=title, track=track, session_id=sid + ) + + if isinstance(license_res, bytes): + if b"<License>" in license_res: + license_res = base64.b64encode(license_res).decode() + else: + license_res = base64.b64decode(license_res).decode() + + cdm.parse_license(sid, license_res) + + keys_list = cdm.get_keys(sid) + result = {} + target_kid = KeyResolver._norm(track.kid) + + for k in keys_list: + kid = KeyResolver._norm(k.get('kid')) + key = k.get('key') + if kid and key and kid != "00" * 16: + result[kid] = key.lower() + + primary_key = result.get(target_kid) + if not primary_key and result: + primary_key = next(iter(result.values())) + log.warning(f"No exact KID match for {track.kid}, using fallback key.") + + return primary_key, result + + except Exception as e: + raise ValueError(f"PlayReady license request failed: {e}") + finally: + cdm.close(sid) \ No newline at end of file diff --git a/wpgskd/core/drm/widevine.py b/wpgskd/core/drm/widevine.py new file mode 100644 index 0000000..ee64a21 --- /dev/null +++ b/wpgskd/core/drm/widevine.py @@ -0,0 +1,48 @@ +import logging +from typing import Any, Dict, Tuple, Optional +from wpgskd.core.drm.base import BaseDRM +from wpgskd.core.resolver import KeyResolver + +log = logging.getLogger("DRM.Widevine") + +class Widevine(BaseDRM): + + def get_keys(self, track: Any, title: Any, session: Any) -> Tuple[Optional[str], Dict[str, str]]: + cdm = self.cdm_provider.cdm_instance + + if cdm.cdm_type != "widevine": + raise ValueError("CDM is not a Widevine CDM") + + pssh_data = getattr(track, 'pssh', None) + if not pssh_data: + raise ValueError("Track missing PSSH for Widevine CDM") + + sid = cdm.open() + try: + challenge = cdm.get_license_challenge(sid, pssh_data, privacy_mode=True) + license_res = self.service.license( + challenge=challenge, title=title, track=track, session_id=sid + ) + cdm.parse_license(sid, license_res) + + keys_list = cdm.get_keys(sid) + result = {} + target_kid = KeyResolver._norm(track.kid) + + for k in keys_list: + kid = KeyResolver._norm(k.get('kid')) + key = k.get('key') + if kid and key and kid != "00" * 16: + result[kid] = key.lower() + + primary_key = result.get(target_kid) + if not primary_key and result: + primary_key = next(iter(result.values())) + log.warning(f"No exact KID match for {track.kid}, using fallback key.") + + return primary_key, result + + except Exception as e: + raise ValueError(f"Widevine license request failed: {e}") + finally: + cdm.close(sid) \ No newline at end of file diff --git a/wpgskd/core/events.py b/wpgskd/core/events.py new file mode 100644 index 0000000..d8cd524 --- /dev/null +++ b/wpgskd/core/events.py @@ -0,0 +1,37 @@ +import logging +from typing import Callable, Dict, List, Any + +log = logging.getLogger("Events") + +class EventManager: + + _listeners: Dict[str, List[Callable]] = {} + + @classmethod + def subscribe(cls, event_name: str, callback: Callable): + if event_name not in cls._listeners: + cls._listeners[event_name] = [] + cls._listeners[event_name].append(callback) + log.debug(f"Subscribed to event '{event_name}': {callback.__name__}") + + @classmethod + def publish(cls, event_name: str, *args, **kwargs) -> Any: + if event_name not in cls._listeners: + return None + + for callback in cls._listeners[event_name]: + try: + result = callback(*args, **kwargs) + if result is not None: + return result + except Exception as e: + log.error(f"Error in event listener for '{event_name}': {e}", exc_info=True) + + return None + +class Events: + BEFORE_DOWNLOAD = "before_download" + AFTER_DOWNLOAD = "before_decrypt" + AFTER_DECRYPT = "after_decrypt" + BEFORE_MUX = "before_mux" + AFTER_MUX = "after_mux" \ No newline at end of file diff --git a/wpgskd/core/manifests/__init__.py b/wpgskd/core/manifests/__init__.py new file mode 100644 index 0000000..6517947 --- /dev/null +++ b/wpgskd/core/manifests/__init__.py @@ -0,0 +1,16 @@ +from wpgskd.core.manifests.dash import parse as parse_mpd +from wpgskd.core.manifests.hls import parse as parse_hls +from wpgskd.core.manifests.ism import parse as parse_ism +from wpgskd.core.manifests.map_init import extract_pssh_and_kid +from wpgskd.core.manifests.m3u8 import parse_media_playlist, fetch_pssh_and_kid_from_m3u8, fetch_aes_keys_from_m3u8 + +from wpgskd.core.manifests import hls as m3u8 +from wpgskd.core.manifests import dash as mpd +from wpgskd.core.manifests import ism + +__all__ = [ + "parse_mpd", "parse_hls", "parse_ism", + "extract_pssh_and_kid", + "parse_media_playlist", "fetch_pssh_and_kid_from_m3u8", "fetch_aes_keys_from_m3u8", + "m3u8", "mpd", "ism" +] \ No newline at end of file diff --git a/wpgskd/core/manifests/dash.py b/wpgskd/core/manifests/dash.py new file mode 100644 index 0000000..c6985dd --- /dev/null +++ b/wpgskd/core/manifests/dash.py @@ -0,0 +1,633 @@ +import xmltodict +import asyncio +import base64 +import json +import logging +import math +import os +import re +import urllib.parse +import uuid +from copy import copy +from hashlib import md5 +from typing import Optional + +import requests +from langcodes import Language +from langcodes.tag_parser import LanguageTagError + +from wpgskd.config import config, directories +from wpgskd.core.tracks import AudioTrack, TextTrack, Track, Tracks, VideoTrack +from wpgskd.utils import Cdm +from wpgskd.utils.io import aria2c +from wpgskd.utils.xml import load_xml +from wpgskd.vendor.pymp4.parser import Box + +log = logging.getLogger("MPD") + +def parse(*, url=None, data=None, source, session=None, downloader=None, multi_period=False): + if not data: + if not url: + raise ValueError("Neither a URL nor a document was provided to Tracks.from_mpd") + + if downloader is None: + data = (session or requests).get(url).text + elif downloader == "aria2c": + out = os.path.join(directories.temp, url.split("/")[-1]) + asyncio.run(aria2c(url, out)) + with open(out, encoding="utf-8") as fd: + data = fd.read() + try: + os.unlink(out) + except FileNotFoundError: + pass + else: + raise ValueError(f"Unsupported downloader: {downloader}") + + root = load_xml(data) + if root.tag != "MPD": + raise ValueError("Non-MPD document provided to Tracks.from_mpd") + + if multi_period: + log.info(f" + Using multi-period parser for {source}") + else: + log.debug(f" + Using single-period parser for {source}") + + return _parse_mpd(root, url, source, session) + +def _parse_mpd(root, url, source, session): + import re + import xml.etree.ElementTree as ET + + namespace_match = re.match(r'\{([^}]+)\}', root.tag) + namespace_uri = namespace_match.group(1) if namespace_match else "urn:mpeg:dash:schema:mpd:2011" + + if namespace_match: + periods = root.findall(f".//{{{namespace_uri}}}Period") + else: + periods = root.findall(".//Period") + + is_multi_period = len(periods) > 1 + period_tracks_list = [] + + root_base_url = root.findtext("BaseURL") + periods_to_process = periods if is_multi_period else [root] + + for period_idx, period_elem in enumerate(periods_to_process): + if is_multi_period: + log.debug(f" + Processing period {period_idx + 1}/{len(periods)}") + + period_str = ET.tostring(period_elem, encoding='unicode', method='xml') + virtual_mpd = f'''<?xml version="1.0" encoding="UTF-8"?> +<MPD xmlns="{namespace_uri}" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xmlns:cenc="urn:mpeg:cenc:2013" + xmlns:mspr="urn:microsoft:playready" + profiles="urn:mpeg:dash:profile:isoff-live:2011" + type="static" + minBufferTime="PT2S" + mediaPresentationDuration="{root.get('mediaPresentationDuration', 'PT0S')}"> + {period_str} +</MPD>''' + + virtual_mpd = virtual_mpd.replace('ns0:default_KID', 'cenc:default_KID') + virtual_mpd = virtual_mpd.replace('ns0:pssh', 'cenc:pssh') + virtual_mpd = virtual_mpd.replace('ns0:pro', 'mspr:pro') + virtual_mpd = virtual_mpd.replace('xmlns:ns0', 'xmlns:cenc') + virtual_mpd = re.sub(r'<!--.*?-->', '', virtual_mpd, flags=re.DOTALL) + + try: + period_root = load_xml(virtual_mpd) + except Exception as e: + log.warning(f" + Failed to parse period {period_idx + 1}: {e}") + new_root = ET.Element('MPD', attrib={ + 'xmlns': namespace_uri, + 'xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance', + 'xmlns:cenc': 'urn:mpeg:cenc:2013', + 'xmlns:mspr': 'urn:microsoft:playready', + 'profiles': 'urn:mpeg:dash:profile:isoff-live:2011', + 'type': 'static', + 'minBufferTime': 'PT2S', + 'mediaPresentationDuration': root.get('mediaPresentationDuration', 'PT0S') + }) + new_root.append(period_elem) + temp_mpd = ET.tostring(new_root, encoding='unicode', method='xml') + temp_mpd = temp_mpd.replace('ns0:default_KID', 'cenc:default_KID') + temp_mpd = temp_mpd.replace('ns0:pssh', 'cenc:pssh') + temp_mpd = temp_mpd.replace('ns0:pro', 'mspr:pro') + temp_mpd = temp_mpd.replace('xmlns:ns0', 'xmlns:cenc') + period_root = load_xml(temp_mpd) + else: + period_root = root + period_elem = root + + tracks = [] + + found_period = period_root.find(".//Period") + search_periods = (period_root.findall("Period") + if not is_multi_period + else [found_period if found_period is not None else period_root]) + + for period in search_periods: + if period is None: + period = period_root if period_root.tag == "Period" else period_root + + if source == "HULU" and next(iter(period.xpath("SegmentType/@value")), "content") != "content": + continue + + period_base_url = period.findtext("BaseURL") or root_base_url + if url and period_base_url and not re.match("^https?://", period_base_url.lower()): + period_base_url = period_base_url.replace('fly.eu.prd.media.max.com', 'akm.eu.prd.media.max.com') + period_base_url = period_base_url.replace('gcp.eu.prd.media.max.com', 'akm.eu.prd.media.max.com') + period_base_url = period_base_url.replace('fly.latam.prd.media.max.com', 'akm.latam.prd.media.max.com') + period_base_url = period_base_url.replace('gcp.latam.prd.media.max.com', 'akm.latam.prd.media.max.com') + period_duration = period.get("duration") + if period_duration: + period_duration = Track.pt_to_sec(period_duration) + mpd_duration = root.get("mediaPresentationDuration") + if mpd_duration: + mpd_duration = Track.pt_to_sec(mpd_duration) + + for adaptation_set in period.findall("AdaptationSet"): + if any(x.get("schemeIdUri") == "http://dashif.org/guidelines/trickmode" + for x in adaptation_set.findall("EssentialProperty") + + adaptation_set.findall("SupplementalProperty")): + continue + + for rep in adaptation_set.findall("Representation"): + try: + content_type = next(x for x in [ + rep.get("contentType"), + rep.get("mimeType"), + adaptation_set.get("contentType"), + adaptation_set.get("mimeType") + ] if bool(x)) + except StopIteration: + raise ValueError("No content type value could be found") + else: + content_type = content_type.split("/")[0] + if content_type.startswith("image"): + continue + + codecs = rep.get("codecs") or adaptation_set.get("codecs") + supplementalcodecs = (rep.get("{urn:scte:dash:scte214-extensions}supplementalCodecs") + or adaptation_set.get("{urn:scte:dash:scte214-extensions}supplementalCodecs")) + if content_type in ("text", "application"): + mime = adaptation_set.get("mimeType") + if mime and not mime.endswith("/mp4"): + codecs = mime.split("/")[1] + + track_lang = None + for lang in [rep.get("lang"), adaptation_set.get("lang")]: + lang = (lang or "").strip() + if not lang: + continue + try: + t = Language.get(lang.split("-")[0]) + if t == Language.get("und") or not t.is_valid(): + raise LanguageTagError() + except LanguageTagError: + continue + else: + track_lang = Language.get(lang) + break + + protections = rep.findall("ContentProtection") + adaptation_set.findall("ContentProtection") + encrypted = bool(protections) + pssh = None + pr_pssh = None + + kid = None + + for protection in adaptation_set.findall("ContentProtection"): + if protection.get("schemeIdUri") == "urn:mpeg:dash:mp4protection:2011": + kid_val = protection.get("{urn:mpeg:cenc:2013}default_KID") + if kid_val: + kid = uuid.UUID(kid_val).hex.lower() + log.debug(f" + KID from AdaptationSet {adaptation_set.get('id', '?')} " + f"for {content_type}: {kid}") + break + + if not kid: + for protection in rep.findall("ContentProtection"): + if protection.get("schemeIdUri") == "urn:mpeg:dash:mp4protection:2011": + kid_val = protection.get("{urn:mpeg:cenc:2013}default_KID") + if kid_val: + kid = uuid.UUID(kid_val).hex.lower() + log.debug(f" + KID from Representation for {content_type}: {kid}") + break + + if not kid and content_type == "audio": + # 从 Period 级别直接找第一个带 default_KID 的保护信息，避免 O(N²) 嵌套循环 + for protection in period.findall(".//ContentProtection"): + if protection.get("schemeIdUri") == "urn:mpeg:dash:mp4protection:2011": + kid_val = protection.get("{urn:mpeg:cenc:2013}default_KID") + if kid_val: + kid = uuid.UUID(kid_val).hex.lower() + log.debug(f" + Audio inheriting KID from video: {kid}") + break + + for protection in protections: + if "9a04f079-9840-4286-ab92-e65be0885f95" in protection.get("schemeIdUri", "").lower(): + pr_pssh = (protection.findtext("pro") + if source in ["STAN", "RKTN", "CR"] + else protection.findtext("pssh")) + if (protection.get("schemeIdUri") or "").lower() != Cdm.urn: + continue + pssh = protection.findtext("pssh") + if pssh: + pssh_bytes = base64.b64decode(pssh) + try: + from pywidevine.pssh import PSSH + pssh = PSSH(pssh_bytes) + except Exception: + try: + pssh = Box.parse(pssh_bytes) + except Exception: + pssh = Box.parse(Box.build(dict( + type=b"pssh", + version=0, + flags=0, + system_ID=Cdm.uuid, + init_data=pssh_bytes + ))) + + track_url = url + + seg_list_rep = rep.find("SegmentList") + seg_list_as = adaptation_set.find("SegmentList") + segment_list = seg_list_rep if seg_list_rep is not None else seg_list_as + + seg_tpl_rep = rep.find("SegmentTemplate") + seg_tpl_as = adaptation_set.find("SegmentTemplate") + segment_template = seg_tpl_rep if seg_tpl_rep is not None else seg_tpl_as + + if segment_list is None and segment_template is None: + rep_base_url = rep.findtext("BaseURL") + if rep_base_url: + if not re.match("^https?://", rep_base_url.lower()): + rep_base_url = urllib.parse.urljoin(period_base_url, rep_base_url) + query = urllib.parse.urlparse(url).query + if query and not urllib.parse.urlparse(rep_base_url).query: + rep_base_url += "?" + query + track_url = rep_base_url + + track_id = "{codec}-{lang}-{bitrate}-{extra}".format( + codec=codecs, + lang=track_lang, + bitrate=rep.get("bandwidth") or 0, + extra=(adaptation_set.get("audioTrackId") or "") + (rep.get("id") or ""), + ) + track_id = md5(track_id.encode()).hexdigest() + + def get_track_size(track_repr): + segment_list = track_repr.findall('SegmentList') + if segment_list: + file_size = sorted( + segment_list[0].findall('SegmentURL'), + key=lambda seg_url: int(seg_url.get('mediaRange').split('-')[1]), + reverse=True + ) + if file_size: + return int(file_size[0].get('mediaRange').split('-')[1]) + return None + + if content_type == "video": + fps_str = rep.get("frameRate") or adaptation_set.get("frameRate") + fps = None + if fps_str: + try: + if "/" in fps_str: + num, den = fps_str.split("/") + fps = float(num) / float(den) + else: + fps = float(fps_str) + except ValueError: + fps = None + + if not fps: + fps = _calculate_fps_from_timeline(rep, period) + if fps: + log.debug(f" + Calculated FPS {fps} for video track " + f"(codec: {codecs}, resolution: " + f"{rep.get('width')}x{rep.get('height')})") + + vt = VideoTrack( + id_=track_id, + source=source, + url=track_url, + codec=(codecs or "").split(".")[0], + language=track_lang, + bitrate=rep.get("bandwidth"), + width=int(rep.get("width") or 0) or adaptation_set.get("width"), + height=int(rep.get("height") or 0) or adaptation_set.get("height"), + fps=fps, + hdr10=any( + x.get("schemeIdUri") == "urn:mpeg:mpegB:cicp:TransferCharacteristics" + and x.get("value") == "16" + for x in adaptation_set.findall("SupplementalProperty") + adaptation_set.findall("EssentialProperty") + ) or any( + x.get("schemeIdUri") == "http://dashif.org/metadata/hdr" + and x.get("value") == "SMPTE2094-40" + for x in adaptation_set.findall("SupplementalProperty") + adaptation_set.findall("EssentialProperty") + ), + hlg=any( + x.get("schemeIdUri") == "urn:mpeg:mpegB:cicp:TransferCharacteristics" + and x.get("value") == "18" + for x in adaptation_set.findall("SupplementalProperty") + ), + dvhdr=( + (isinstance(codecs, str) and codecs.startswith(("dvhe.08", "dvh1.08"))) + or (isinstance(supplementalcodecs, str) and "dvh1.08" in supplementalcodecs) + ), + dv=codecs and codecs.startswith(("dvhe", "dvh1")), + descriptor=Track.Descriptor.MPD, + encrypted=encrypted, + pssh=pssh, + pr_pssh=pr_pssh, + kid=kid, + duration=mpd_duration or period_duration, + extra=(rep, adaptation_set) + ) + vt.manifest_url = url + vt.mpd_representation_id = rep.get("id") + tracks.append(vt) + + elif content_type == "audio": + at = AudioTrack( + id_=track_id, + source=source, + url=track_url, + codec=(codecs or "").split(".")[0], + language=track_lang, + bitrate=rep.get("bandwidth"), + channels=next(iter( + rep.xpath("AudioChannelConfiguration/@value") + or adaptation_set.xpath("AudioChannelConfiguration/@value") + ), None), + descriptive=any( + (x.get("schemeIdUri") == "urn:mpeg:dash:role:2011" + and x.get("value") == "description") + or (x.get("schemeIdUri") == "urn:tva:metadata:cs:AudioPurposeCS:2007" + and x.get("value") == "1") + for x in adaptation_set.findall("Accessibility") + ), + atmos=any( + prop.get("schemeIdUri") == "tag:dolby.com,2018:dash:EC3_ExtensionType:2018" + and prop.get("value") == "JOC" + for prop in rep.findall("SupplementalProperty") + ), + descriptor=Track.Descriptor.MPD, + encrypted=encrypted, + pssh=pssh, + pr_pssh=pr_pssh, + kid=kid, + duration=mpd_duration or period_duration, + extra=(rep, adaptation_set) + ) + at.manifest_url = url + at.mpd_representation_id = rep.get("id") + tracks.append(at) + + elif content_type in ("text", "application"): + role_elem = adaptation_set.find(".//{*}Role") + role = role_elem.get("value") if role_elem is not None else "" + + is_forced = (role == "forced-subtitle") + is_sdh = (role == "caption") or (role == "sdh") + is_normal = (role == "subtitle") or (role == "main") or (not role) + + adapt_set_id = adaptation_set.get("id", "") + if not is_forced and ("forced" in adapt_set_id.lower() + or "fn" in adapt_set_id.lower()): + is_forced = True + log.debug(f" + Detected forced subtitle by adaptation set ID: {adapt_set_id}") + + rep_id = rep.get("id", "") + if not is_forced and ("forced" in rep_id.lower() + or "fn" in rep_id.lower()): + is_forced = True + log.debug(f" + Detected forced subtitle by representation ID: {rep_id}") + + if not is_forced and codecs and ("forced" in codecs.lower() + or "fn" in codecs.lower()): + is_forced = True + log.debug(f" + Detected forced subtitle by codec pattern: {codecs}") + + if source == 'HMAX': + seg_tpl = rep.find("SegmentTemplate") + sub_path_url = rep.findtext("BaseURL") + if not sub_path_url: + sub_path_url = seg_tpl.get('media') if seg_tpl else None + + if not sub_path_url: + continue + + try: + path = re.search(r'(t\/.+?\/)t', sub_path_url).group(1) + except AttributeError: + path = 't/sub/' + + if is_normal: + track_url = period_base_url + path + adaptation_set.get('lang') + '_sub.vtt' + elif is_sdh: + track_url = period_base_url + path + adaptation_set.get('lang') + '_sdh.vtt' + elif is_forced: + track_url = period_base_url + path + adaptation_set.get('lang') + '_forced.vtt' + else: + track_url = period_base_url + path + adaptation_set.get('lang') + '_sub.vtt' + + if seg_tpl is not None and seg_tpl.get('media') and '$Number$' in seg_tpl.get('media'): + media_pattern = seg_tpl.get('media') + if not re.match("^https?://", media_pattern): + media_pattern = urllib.parse.urljoin(period_base_url, media_pattern) + track_url = media_pattern + + tt = TextTrack( + id_=track_id, + source=source, + url=track_url, + codec=(codecs or "").split(".")[0] if codecs else "vtt", + language=track_lang, + forced=is_forced, + sdh=is_sdh, + descriptor=Track.Descriptor.MPD, + extra=(rep, adaptation_set) + ) + tt.manifest_url = url + tt.mpd_representation_id = rep.get("id") + tracks.append(tt) + else: + extra_info = { + "role": role, + "adaptation_set_id": adapt_set_id, + "representation_id": rep_id, + "original_forced_detection": is_forced + } + + if track_url and isinstance(track_url, list): + for url_check in track_url: + if isinstance(url_check, str) and ( + "forced" in url_check.lower() + or "_fn" in url_check.lower() + or "forced-subtitle" in url_check.lower() + ): + is_forced = True + extra_info["detected_by_url"] = True + log.debug(f" + Detected forced subtitle by URL pattern: {url_check}") + break + + tt = TextTrack( + id_=track_id, + source=source, + url=track_url, + codec=(codecs or "").split(".")[0] if codecs else "vtt", + language=track_lang, + forced=is_forced, + sdh=is_sdh, + descriptor=Track.Descriptor.MPD, + encrypted=encrypted, + pssh=pssh, + pr_pssh=pr_pssh, + kid=kid, + extra=extra_info + ) + tt.manifest_url = url + tt.mpd_representation_id = rep.get("id") + tracks.append(tt) + + period_tracks_obj = Tracks() + period_tracks_obj.add(tracks, warn_only=True) + + if is_multi_period: + for track in (period_tracks_obj.videos + + period_tracks_obj.audios + + period_tracks_obj.subtitles): + if not isinstance(track.url, list): + track.url = [track.url] + period_tracks_list.append(period_tracks_obj) + else: + return period_tracks_obj + + if is_multi_period: + return _merge_periods(period_tracks_list, source, log) + + return Tracks() + +def _calculate_fps_from_timeline(rep, period, timescale_multiplier=1): + import statistics + segment_template = rep.find("SegmentTemplate") + if segment_template is None: + parent = rep.getparent() + if parent is not None: + segment_template = parent.find("SegmentTemplate") + + if segment_template is None: + return None + + timescale = int(segment_template.get("timescale", 24000)) + timescale = timescale * timescale_multiplier + + segment_timeline = segment_template.find("SegmentTimeline") + if segment_timeline is None: + return None + + durations = [] + for s in segment_timeline.findall("S"): + d = int(s.get("d", 0)) + if d > 0: + repeat = int(s.get("r", 0)) + if repeat > 0: + durations.extend([d] * (repeat + 1)) + else: + durations.append(d) + + if not durations: + return None + + if len(durations) > 1: + try: + variance = statistics.variance(durations) if len(durations) > 1 else 0 + if variance > 100: + log.debug(f" + Detected VFR content (variance={variance:.2f})") + avg_duration_ticks = statistics.median(durations) + else: + avg_duration_ticks = sum(durations) / len(durations) + except (statistics.StatisticsError, TypeError): + avg_duration_ticks = sum(durations) / len(durations) + else: + avg_duration_ticks = durations[0] + + avg_duration_sec = avg_duration_ticks / timescale + + if avg_duration_sec <= 0: + return None + + fps = 1.0 / avg_duration_sec + + for common_fps in [23.976, 24.0, 25.0, 29.97, 30.0, 50.0, 59.94, 60.0]: + if abs(fps - common_fps) < 0.01: + fps = common_fps + break + + log.debug(f" + Calculated FPS {fps:.3f} from {len(durations)} segments " + f"(timescale={timescale}, avg_duration_ticks={avg_duration_ticks:.2f})") + return round(fps, 3) + +def _merge_periods(period_tracks_list, source, log): + if not period_tracks_list: + return Tracks() + + if len(period_tracks_list) == 1: + return period_tracks_list[0] + + combined = Tracks() + + seen_videos = {} + seen_audios = {} + seen_subs = {} + + for period_tracks in period_tracks_list: + for video in period_tracks.videos: + key = getattr(video, 'mpd_representation_id', None) or (video.width, video.height, video.codec, video.bitrate, video.hdr10, video.dv) + if key not in seen_videos: + new_video = video + new_video.url = [] + seen_videos[key] = new_video + + if isinstance(video.url, list): + seen_videos[key].url.extend(video.url) + else: + seen_videos[key].url.append(video.url) + + for audio in period_tracks.audios: + lang = str(audio.language) if audio.language else "und" + if lang not in seen_audios: + new_audio = audio + new_audio.url = [] + seen_audios[lang] = new_audio + + if isinstance(audio.url, list): + seen_audios[lang].url.extend(audio.url) + else: + seen_audios[lang].url.append(audio.url) + + for sub in period_tracks.subtitles: + lang = str(sub.language) if sub.language else "und" + sub_type = "forced" if sub.forced else "sdh" if sub.sdh else "normal" + key = f"{lang}_{sub_type}" + + if key not in seen_subs: + seen_subs[key] = sub + if not isinstance(seen_subs[key].url, list): + seen_subs[key].url = [seen_subs[key].url] + + combined.videos = list(seen_videos.values()) + combined.audios = list(seen_audios.values()) + combined.subtitles = list(seen_subs.values()) + + log.debug(f" + Merged {len(period_tracks_list)} periods into: " + f"{len(combined.videos)} video, {len(combined.audios)} audio, " + f"{len(combined.subtitles)} subtitle tracks") + + return combined diff --git a/wpgskd/core/manifests/hls.py b/wpgskd/core/manifests/hls.py new file mode 100644 index 0000000..4cc3958 --- /dev/null +++ b/wpgskd/core/manifests/hls.py @@ -0,0 +1,279 @@ +import base64 +import re +import logging +from hashlib import md5 +import m3u8 + +from wpgskd.core.tracks import AudioTrack, TextTrack, Track, Tracks, VideoTrack +from wpgskd.constants import EncryptionScheme +from wpgskd.utils import Cdm +from wpgskd.vendor.pymp4.parser import Box + +log = logging.getLogger("HLSParser") + +def parse(master, source=None, session=None): + """ + Convert a Variant Playlist M3U8 document to a Tracks object with Video, Audio and + Subtitle Track objects. This is not an M3U8 parser, use https://github.com/globocom/m3u8 + to parse, and then feed the parsed M3U8 object. + + :param master: M3U8 object of the `m3u8` project: https://github.com/globocom/m3u8 + :param source: Source tag for the returned tracks. + """ + if not master.is_variant: + raise ValueError("Tracks.from_m3u8: Expected a Variant Playlist M3U8 document...") + + # Get PSSH if available + # Uses master.session_keys instead of master.keys as master.keys is ONLY EXT-X-KEYS and + # doesn't include EXT-X-SESSION-KEYS which is what's used for variant playlist M3U8. + widevine_urn = "urn:uuid:edef8ba9-79d6-4ace-a3c8-27dcd51d21ed" + widevine_keys = [x.uri for x in master.session_keys + if x.keyformat and x.keyformat.lower() == widevine_urn] + pssh = widevine_keys[0].split(",")[-1] if widevine_keys else None + + pr_keys = [x.uri for x in master.session_keys + if x.keyformat and "playready" in x.keyformat.lower()] + pr_pssh = pr_keys[0].split(",")[-1] if pr_keys else None + + if pssh: + pssh = base64.b64decode(pssh) + try: + pssh = Box.parse(pssh) + except Exception: + pssh = Box.parse(Box.build(dict( + type=b"pssh", + version=0, + flags=0, + system_ID=Cdm.uuid, + init_data=pssh + ))) + + # Also check top-level keys (non-session) as fallback + if not pssh and not pr_pssh: + widevine_top = [x.uri for x in master.keys + if x.keyformat and x.keyformat.lower() == widevine_urn] + if widevine_top: + pssh_raw = widevine_top[0].split(",")[-1] + pssh_raw = base64.b64decode(pssh_raw) + try: + pssh = Box.parse(pssh_raw) + except Exception: + pssh = Box.parse(Box.build(dict( + type=b"pssh", + version=0, + flags=0, + system_ID=Cdm.uuid, + init_data=pssh_raw + ))) + + pr_top = [x.uri for x in master.keys + if x.keyformat and "playready" in x.keyformat.lower()] + if pr_top: + pr_pssh = pr_top[0].split(",")[-1] + + # Determine default encryption scheme + default_scheme = EncryptionScheme.NONE + if pssh: + default_scheme = EncryptionScheme.WIDEVINE + elif pr_pssh: + default_scheme = EncryptionScheme.PLAYREADY + + # Check for AES-128 keys at master level + aes128_keys = [x for x in (master.keys + master.session_keys) + if x.method and x.method.upper() == "AES-128"] + if aes128_keys and not pssh and not pr_pssh: + default_scheme = EncryptionScheme.AES_128 + + has_encryption = bool(pssh or pr_pssh or aes128_keys or master.keys or master.session_keys) + + tracks_obj = Tracks() + + # ==================== VIDEO TRACKS ==================== + for x in master.playlists: + stream_info = x.stream_info + + codec_str = _safe_get_codec(stream_info) + resolution = _safe_get_resolution(stream_info) + + tracks_obj.add(VideoTrack( + id_=md5(str(x).encode()).hexdigest()[0:7], + source=source, + url=("" if re.match("^https?://", x.uri) else x.base_uri) + x.uri, + codec=codec_str, + language=None, # playlists don't state the language, fallback must be used + bitrate=_safe_get_bitrate(stream_info), + width=resolution[0], + height=resolution[1], + fps=_safe_get_frame_rate(stream_info), + hdr10=(not _is_dv(codec_str) and _safe_get_video_range(stream_info) != "SDR"), + hlg=False, + dv=_is_dv(codec_str), + descriptor=Track.Descriptor.M3U, + encryption_scheme=default_scheme, + encrypted=has_encryption, + extra={"original": x, "master_pssh": pssh, "master_pr_pssh": pr_pssh} + )) + + # ==================== AUDIO + SUBTITLE TRACKS ==================== + if hasattr(master, 'media') and master.media: + for x in master.media: + # === AUDIO === + if x.type == "AUDIO" and x.uri: + channels = x.channels if hasattr(x, 'channels') else None + characteristics = x.characteristics or "" if hasattr(x, 'characteristics') else "" + + tracks_obj.add(AudioTrack( + id_=md5(str(x).encode()).hexdigest()[0:6], + source=source, + url=("" if re.match("^https?://", x.uri) else x.base_uri) + x.uri, + codec=_safe_get_audio_codec(x), + language=x.language, + bitrate=0, + channels=channels, + atmos=(channels or "").endswith("/JOC"), + descriptive="public.accessibility.describes-video" in characteristics, + descriptor=Track.Descriptor.M3U, + encryption_scheme=default_scheme, + encrypted=has_encryption, + extra={"original": x, "master_pssh": pssh, "master_pr_pssh": pr_pssh} + )) + + # === SUBTITLES === + elif x.type == "SUBTITLES" and x.uri: + forced = x.forced == "YES" if hasattr(x, 'forced') else False + characteristics = x.characteristics or "" if hasattr(x, 'characteristics') else "" + + tracks_obj.add(TextTrack( + id_=md5(str(x).encode()).hexdigest()[0:6], + source=source, + url=("" if re.match("^https?://", x.uri) else x.base_uri) + x.uri, + codec="vtt", + language=x.language, + forced=forced, + sdh="public.accessibility.describes-music-and-sound" in characteristics, + descriptor=Track.Descriptor.M3U, + encryption_scheme=default_scheme, + encrypted=has_encryption, + extra={"original": x, "master_pssh": pssh, "master_pr_pssh": pr_pssh} + )) + + if tracks_obj.videos: + try: + from wpgskd.core.session import SessionBuilder + s = session or SessionBuilder.build() + except ImportError: + import requests as req_mod + s = session or req_mod.Session() + + first_video = tracks_obj.videos[0] + try: + sub_url = first_video.url + if isinstance(sub_url, list): + sub_url = sub_url[0] + res = s.get(sub_url, timeout=10) + res.raise_for_status() + sub_m3u8 = m3u8.loads(res.text, uri=sub_url) + + total_duration = sum(seg.duration for seg in sub_m3u8.segments if seg.duration) + fps = _infer_fps_from_segments(sub_m3u8.segments) + + if total_duration: + for v in tracks_obj.videos: + v.duration = total_duration + if fps: + v.fps = fps + if v.bitrate: + v.size = int((float(v.bitrate) * total_duration) / 8) + + for a in tracks_obj.audios: + a.duration = total_duration + if a.bitrate: + a.size = int((float(a.bitrate) * total_duration) / 8) + except Exception as e: + log.warning(f"Failed to probe HLS sub-manifest for duration/fps: {e}") + + return tracks_obj + +def _safe_get_codec(stream_info): + """Safely extract codec from stream_info, handling None values.""" + try: + if hasattr(stream_info, 'codecs') and stream_info.codecs: + return stream_info.codecs.split(",")[0].split(".")[0] + return "h264" + except (AttributeError, TypeError, IndexError): + return "h264" + +def _safe_get_resolution(stream_info): + """Safely extract resolution from stream_info, handling None values.""" + try: + if hasattr(stream_info, 'resolution') and stream_info.resolution: + return stream_info.resolution + return (0, 0) + except (TypeError, AttributeError): + return (0, 0) + +def _safe_get_frame_rate(stream_info): + """Safely extract frame rate from stream_info, handling None values.""" + try: + if hasattr(stream_info, 'frame_rate') and stream_info.frame_rate: + return stream_info.frame_rate + return None + except (TypeError, AttributeError): + return None + +def _safe_get_video_range(stream_info): + """Safely extract video range from stream_info, handling None values.""" + try: + if hasattr(stream_info, 'video_range') and stream_info.video_range: + return stream_info.video_range.strip('"') + return "SDR" + except (TypeError, AttributeError): + return "SDR" + +def _safe_get_bitrate(stream_info): + """Safely extract bitrate from stream_info, handling None values.""" + try: + if hasattr(stream_info, 'average_bandwidth') and stream_info.average_bandwidth: + return stream_info.average_bandwidth + if hasattr(stream_info, 'bandwidth') and stream_info.bandwidth: + return stream_info.bandwidth + return 0 + except (TypeError, AttributeError): + return 0 + +def _safe_get_audio_codec(media): + """Safely extract audio codec from media entry.""" + try: + if hasattr(media, 'codecs') and media.codecs: + return media.codecs.split(",")[0].split(".")[0] + if hasattr(media, 'group_id') and media.group_id: + return media.group_id.replace("audio-", "").split("-")[0].split(".")[0] + return "aac" + except (AttributeError, TypeError, IndexError): + return "aac" + +def _is_dv(codec_str): + """Check if codec indicates Dolby Vision.""" + try: + if codec_str: + return codec_str.split(".")[0] in ("dvhe", "dvh1") + return False + except (AttributeError, IndexError): + return False + +def _infer_fps_from_segments(segments): + if not segments: + return None + + durations = [seg.duration for seg in segments if seg.duration and seg.duration > 0] + if not durations: + return None + + avg_duration = sum(durations) / len(durations) + + for fps in [23.976, 24.0, 25.0, 29.97, 30.0, 50.0, 59.94, 60.0]: + frames = avg_duration * fps + if abs(frames - round(frames)) < 0.15: + return fps + + return None \ No newline at end of file diff --git a/wpgskd/core/manifests/ism.py b/wpgskd/core/manifests/ism.py new file mode 100644 index 0000000..08b5f64 --- /dev/null +++ b/wpgskd/core/manifests/ism.py @@ -0,0 +1,279 @@ +import asyncio +import hashlib +import logging +import urllib.parse +from typing import Optional + +import requests +from langcodes import Language +from langcodes.tag_parser import LanguageTagError + +from wpgskd.config import directories +from wpgskd.core.tracks import AudioTrack, TextTrack, Track, Tracks, VideoTrack +from wpgskd.utils.io import aria2c +from wpgskd.utils.xml import load_xml + +log = logging.getLogger("ISMParser") + +def _probe_ism_fps(url, session, timescale=10000000): + try: + s = session or requests + res = s.get(url, timeout=10) + res.raise_for_status() + data = res.content + + def find_box_pos(data, target, start=0): + pos = start + while pos < len(data) - 8: + size = int.from_bytes(data[pos:pos+4], 'big') + btype = data[pos+4:pos+8] + if size == 0: break + if btype == target: + return pos, size + if size < 8: break + pos += size + return -1, 0 + + moof_pos, moof_size = find_box_pos(data, b'moof') + if moof_pos == -1: return None + + traf_pos, traf_size = find_box_pos(data, b'traf', moof_pos + 8) + if traf_pos == -1: return None + + trun_pos, trun_size = find_box_pos(data, b'trun', traf_pos + 8) + if trun_pos == -1: return None + + version = data[trun_pos + 8] + flags = int.from_bytes(data[trun_pos + 9 : trun_pos + 12], 'big') + sample_count = int.from_bytes(data[trun_pos + 12 : trun_pos + 16], 'big') + + offset = trun_pos + 16 + if flags & 0x000001: # data_offset_present + offset += 4 + if flags & 0x000004: # first_sample_flags_present + offset += 4 + + if flags & 0x000100: # sample_duration_present + total_duration = 0 + for _ in range(sample_count): + total_duration += int.from_bytes(data[offset : offset + 4], 'big') + offset += 4 + if flags & 0x000200: # sample_size_present + offset += 4 + if flags & 0x000400: # sample_flags_present + offset += 4 + if flags & 0x000800: # sample_composition_time_present + offset += 4 if version == 0 else 8 + if total_duration > 0: + fps = (sample_count * timescale) / total_duration + return round(fps, 3) + + tfhd_pos, tfhd_size = find_box_pos(data, b'tfhd', traf_pos + 8) + if tfhd_pos != -1: + tfhd_flags = int.from_bytes(data[tfhd_pos + 9 : tfhd_pos + 12], 'big') + tfhd_offset = tfhd_pos + 16 + + if tfhd_flags & 0x000001: + tfhd_offset += 8 + if tfhd_flags & 0x000002: + tfhd_offset += 4 + + if tfhd_flags & 0x000008: + default_dur = int.from_bytes(data[tfhd_offset : tfhd_offset + 4], 'big') + if default_dur > 0: + return round(timescale / default_dur, 3) + + except Exception as e: + log.warning(f"Failed to probe ISM FPS via raw bytes: {e}") + return None + +def parse(url: str = None, data: str = None, source: str = None, session: requests.Session = None, downloader: str = None) -> Tracks: + if not data: + if downloader is None: + r = (session or requests).get(url) + url = r.url + data = r.content + elif downloader == "aria2c": + out = directories.temp / url.split("/")[-1] + asyncio.run(aria2c((url, out))) + data = out.read_bytes() + out.unlink(missing_ok=True) + else: + raise ValueError(f"Unsupported downloader: {downloader}") + + root = load_xml(data) + if root.tag != "SmoothStreamingMedia": + raise ValueError("Non-ISM document provided to ISM parser") + + tracks = [] + base_url = url + duration = int(root.attrib.get("Duration", 0)) + root_timescale = int(root.get("TimeScale", 10000000)) + duration_sec = duration / root_timescale if root_timescale else 0 + + if session is None: + session = requests.Session() + + for stream_index in root.findall("StreamIndex"): + stream_fps = None + fps_probed = False + for ql in stream_index.findall("QualityLevel"): + content_type = stream_index.get("Type") + if not content_type: + raise ValueError("No content type value could be found") + + codec = ql.get("FourCC") + if codec == "TTML": + codec = "STPP" + + track_lang = None + if lang := (stream_index.get("Language") or "").strip(): + try: + t = Language.get(lang.split("-")[0]) + if t == Language.get("und") or not t.is_valid(): + raise LanguageTagError() + except LanguageTagError: + pass + else: + track_lang = Language.get(lang) + + protections = root.xpath(".//ProtectionHeader") + pr_protections = [ + x for x in protections + if (x.get("SystemID") or "").lower() == "9a04f079-9840-4286-ab92-e65be0885f95" + ] + protections = pr_protections + encrypted = bool(protections) + pssh = None + pr_pssh = None + kid = None + + if pr_protections: + import base64 + import re + from uuid import UUID + for protection in pr_protections: + pr_pssh_text = "".join(protection.itertext()) + if pr_pssh_text: + pr_pssh = pr_pssh_text + try: + raw_bytes = base64.b64decode(pr_pssh_text) + clean_str = raw_bytes.replace(b'\x00', b'').decode('utf-8', errors='ignore') + kid_match = re.search(r'<KID>([a-zA-Z0-9+/=]+)</KID>', clean_str) + if kid_match: + kid_bytes = base64.b64decode(kid_match.group(1)) + if len(kid_bytes) == 16: + kid = UUID(bytes_le=kid_bytes).hex + except Exception: + pass + break + + track_url = [] + fragment_ctx = { + "time": 0, + } + stream_fragments = stream_index.findall("c") + for stream_fragment_index, stream_fragment in enumerate(stream_fragments): + fragment_ctx["time"] = int(stream_fragment.get("t", fragment_ctx["time"])) + fragment_repeat = int(stream_fragment.get("r", 1)) + fragment_ctx["duration"] = int(stream_fragment.get("d")) + + if not fragment_ctx["duration"]: + try: + next_fragment_time = int(stream_index[stream_fragment_index + 1].attrib["t"]) + except IndexError: + next_fragment_time = duration + fragment_ctx["duration"] = (next_fragment_time - fragment_ctx["time"]) / fragment_repeat + + for _ in range(fragment_repeat): + track_url.append( + urllib.parse.urljoin( + base_url, stream_index.get("Url").format_map({ + "bitrate": ql.get("Bitrate"), + "start time": str(fragment_ctx["time"]), + }), + ) + ) + fragment_ctx["time"] += fragment_ctx["duration"] + + if content_type == "video" and not fps_probed and track_url: + stream_name = stream_index.get("Name") or "video" + log.info(f" + Probing FPS from first fragment for stream: {stream_name}") + stream_fps = _probe_ism_fps(track_url[0], session, root_timescale) + fps_probed = True + if stream_fps: + log.info(f" + Detected FPS: {stream_fps}") + else: + log.warning(" + Could not detect FPS from fragment.") + + track_id = hashlib.md5( + f"{codec}-{track_lang}-{ql.get('Bitrate') or 0}-{ql.get('Index') or 0}".encode(), + ).hexdigest() + + if content_type == "video": + vt = VideoTrack( + id_=track_id, + source=source, + url=track_url, + codec=codec or "", + language=track_lang, + bitrate=ql.get("Bitrate"), + width=int(ql.get("MaxWidth") or 0) or stream_index.get("MaxWidth"), + height=int(ql.get("MaxHeight") or 0) or stream_index.get("MaxHeight"), + fps=stream_fps, + hdr10=False, + hlg=False, + dv=(codec and codec.lower() in ("dvhe", "dvh1")), + descriptor=Track.Descriptor.ISM, + encrypted=encrypted, + pr_pssh=pr_pssh, + pssh=pssh, + kid=kid, + duration=duration_sec, + extra=(ql, stream_index, root), + ) + vt.smooth = True + tracks.append(vt) + + elif content_type == "audio": + at = AudioTrack( + id_=track_id, + source=source, + url=track_url, + codec=codec or "", + language=track_lang, + bitrate=ql.get("Bitrate"), + channels=None, + descriptor=Track.Descriptor.ISM, + encrypted=encrypted, + pr_pssh=pr_pssh, + pssh=pssh, + kid=kid, + duration=duration_sec, + extra=(ql, stream_index, root), + ) + at.smooth = True + tracks.append(at) + + elif content_type == "text": + tt = TextTrack( + id_=track_id, + source=source, + url=track_url, + codec=codec or "ttml", + language=track_lang, + descriptor=Track.Descriptor.ISM, + encrypted=encrypted, + pr_pssh=pr_pssh, + pssh=pssh, + kid=kid, + duration=duration_sec, + extra=(ql, stream_index, root), + ) + tt.smooth = True + tracks.append(tt) + + tracks_obj = Tracks() + tracks_obj.add(tracks, warn_only=True) + + return tracks_obj \ No newline at end of file diff --git a/wpgskd/core/manifests/m3u8.py b/wpgskd/core/manifests/m3u8.py new file mode 100644 index 0000000..2691240 --- /dev/null +++ b/wpgskd/core/manifests/m3u8.py @@ -0,0 +1,118 @@ +import base64 +import logging +from typing import List, Optional, Tuple, Any +import requests +import m3u8 + +from wpgskd.utils import Cdm +from wpgskd.vendor.pymp4.parser import Box +from wpgskd.constants import EncryptionScheme + +log = logging.getLogger("M3U8Parser") + +def parse_media_playlist(url: str, session: requests.Session = None) -> dict: + if not session: + session = requests.Session() + + result = { + "pssh": None, + "pr_pssh": None, + "kid": None, + "aes_key_uri": None, + "aes_iv": None, + "init_url": None, + "segments": [] + } + + try: + res = session.get(url) + res.raise_for_status() + playlist = m3u8.loads(res.text, uri=url) + except Exception as e: + log.error(f"Failed to fetch/parse M3U8 playlist {url}: {e}") + return result + + if playlist.segment_map: + seg_map = playlist.segment_map + init_uri = None + if isinstance(seg_map, dict): + init_uri = seg_map.get("uri") + elif hasattr(seg_map, "uri"): + init_uri = seg_map.uri + + if init_uri: + result["init_url"] = init_uri if init_uri.startswith("http") else f"{playlist.base_uri}{init_uri}" + + for segment in playlist.segments: + result["segments"].append(segment.absolute_uri) + + keys = playlist.session_keys or playlist.keys + if not keys: + return result + + widevine_urn = "urn:uuid:edef8ba9-79d6-4ace-a3c8-27dcd51d21ed" + + for key in keys: + if not key or not key.method: + continue + + method = key.method.upper() + + if method in ("SAMPLE-AES", "SAMPLE-AES-CTR") and key.keyformat and key.keyformat.lower() == widevine_urn: + if key.uri and "base64," in key.uri: + pssh_b64 = key.uri.split("base64,")[-1] + try: + pssh_data = base64.b64decode(pssh_b64) + try: + result["pssh"] = Box.parse(pssh_data) + except Exception: + result["pssh"] = Box.parse(Box.build(dict( + type=b"pssh", version=0, flags=0, system_ID=Cdm.uuid, init_data=pssh_data + ))) + except Exception as e: + log.debug(f"Failed to decode Widevine PSSH from M3U8: {e}") + + + elif method in ("SAMPLE-AES", "SAMPLE-AES-CTR") and key.keyformat and "playready" in key.keyformat.lower(): + if key.uri and "base64," in key.uri: + result["pr_pssh"] = key.uri.split("base64,")[-1] + + elif method == "SAMPLE-AES" and key.keyformat and "apple" in key.keyformat.lower(): + result["aes_key_uri"] = key.uri + + elif method == "AES-128": + result["aes_key_uri"] = key.absolute_uri + if key.iv: + result["aes_iv"] = bytes.fromhex(key.iv.replace("0x", "")) + else: + pass + + return result + + +def fetch_pssh_and_kid_from_m3u8(url: str, session: requests.Session = None) -> Tuple[Optional[Any], Optional[str]]: + data = parse_media_playlist(url, session) + + pssh = data.get("pssh") + kid = data.get("kid") + + if (not pssh or not kid) and data.get("init_url"): + try: + from wpgskd.core.manifests.map_init import extract_pssh_and_kid + if not session: + session = requests.Session() + resp = session.get(data["init_url"], stream=True) + chunk = next(resp.iter_content(20000), b"") + pssh_list, kid_hex = extract_pssh_and_kid(chunk) + if not pssh and pssh_list: + pssh = pssh_list[0] + if not kid and kid_hex: + kid = kid_hex + except Exception as e: + log.debug(f"Failed to extract PSSH/KID from init.mp4 ({data['init_url']}): {e}") + + return pssh, kid + +def fetch_aes_keys_from_m3u8(url: str, session: requests.Session = None) -> Tuple[Optional[str], Optional[bytes]]: + data = parse_media_playlist(url, session) + return data.get("aes_key_uri"), data.get("aes_iv") \ No newline at end of file diff --git a/wpgskd/core/manifests/map_init.py b/wpgskd/core/manifests/map_init.py new file mode 100644 index 0000000..0909a5b --- /dev/null +++ b/wpgskd/core/manifests/map_init.py @@ -0,0 +1,82 @@ +import logging +import base64 +from typing import Optional, Tuple, List +from uuid import UUID + +from wpgskd.vendor.pymp4.parser import Box +from pywidevine.license_protocol_pb2 import WidevinePsshData + +log = logging.getLogger("MapInit") + +def extract_pssh_and_kid(data: bytes) -> Tuple[List[bytes], Optional[str]]: + pssh_list = [] + kid_hex = None + + try: + for box in _iterate_boxes(data, b"moov"): + for tenc in _find_boxes(box, b"tenc"): + if hasattr(tenc, 'key_ID') and tenc.key_ID: + kid_hex = tenc.key_ID.hex + break + + for pssh in _find_boxes(box, b"pssh"): + if hasattr(pssh, 'init_data') and pssh.init_data: + pssh_list.append(Box.build(pssh)) + + if kid_hex or pssh_list: + break + + except Exception as e: + log.debug(f"Failed to parse MP4 boxes for PSSH/KID: {e}") + + return pssh_list, kid_hex + +def parse_widevine_pssh(pssh_data: bytes) -> Tuple[Optional[bytes], Optional[str]]: + try: + box = Box.parse(pssh_data) + if hasattr(box, 'init_data') and box.init_data: + cenc_header = WidevinePsshData() + cenc_header.ParseFromString(box.init_data) + if cenc_header.key_id: + kid = cenc_header.key_id[0] + try: + int(kid, 16) + kid_hex = kid.decode().lower() + except ValueError: + kid_hex = kid.hex().lower() + return box, kid_hex + return box, None + except Exception: + pass + return None, None + +def _iterate_boxes(data: bytes, box_type: bytes): + offset = 0 + while offset + 8 <= len(data): + try: + size = int.from_bytes(data[offset:offset+4], "big") + btype = data[offset+4:offset+8] + if size < 8: break + + if btype == box_type: + yield data[offset:offset+size] + + offset += size + except Exception: + offset += 8 + +def _find_boxes(data: bytes, box_type: bytes): + offset = 0 + while offset + 8 <= len(data): + try: + size = int.from_bytes(data[offset:offset+4], "big") + btype = data[offset+4:offset+8] + if size < 8 or offset + size > len(data): break + + if btype == box_type: + box = Box.parse(data[offset:offset+size]) + yield box + + offset += size + except Exception: + offset += 8 \ No newline at end of file diff --git a/wpgskd/core/muxer.py b/wpgskd/core/muxer.py new file mode 100644 index 0000000..24c74c9 --- /dev/null +++ b/wpgskd/core/muxer.py @@ -0,0 +1,260 @@ +import os +import re +import sys +import json +import time +import shutil +import logging +import subprocess +from pathlib import Path +from typing import Tuple, List, Optional +from io import TextIOWrapper + +from wpgskd.config import directories, filenames +from wpgskd.core.tracks.tracks import Tracks, TextTrack +from wpgskd.core.tracks.title import Title +from wpgskd.utils import is_close_match +from wpgskd.constants import LANGUAGE_MUX_MAP + +log = logging.getLogger("Muxer") + +class Muxer: + + @staticmethod + def mux(title: Title, tracks: Tracks, no_sync_subs: bool = False) -> Tuple[str, int]: + if not shutil.which("mkvmerge"): + raise EnvironmentError("mkvmerge executable not found in PATH.") + + out_dir = Path(directories.downloads) + if title.type == Title.Types.TV: + out_dir = out_dir / title.parse_filename(folder=True) + out_dir.mkdir(parents=True, exist_ok=True) + + muxed_location = out_dir / f"{title.parse_filename()}.muxed.mkv" + + if muxed_location.exists(): + muxed_location.unlink() + + cl = ["mkvmerge", "--output", str(muxed_location)] + + for i, vt in enumerate(tracks.videos): + location = vt.locate() + if not location: + raise ValueError("A Video Track was not downloaded before muxing...") + cl.extend([ + "--language", "0:und", + "--disable-language-ietf", + "--default-track", f"0:{i == 0}", + "--compression", "0:none", + "(", location, ")" + ]) + + for i, at in enumerate(tracks.audios): + location = at.locate() + if not location: + raise ValueError("An Audio Track was not downloaded before muxing...") + + audio_display = at.get_codec_display() + if at.atmos and "Atmos" not in audio_display: + audio_display += " Atmos" + + cl.extend([ + "--track-name", f"0:{at.get_track_name() or audio_display}", + "--language", f"0:{LANGUAGE_MUX_MAP.get(str(at.language), at.language.to_alpha3())}", + "--disable-language-ietf", + "--default-track", f"0:{i == 0}", + "--compression", "0:none", + "(", location, ")" + ]) + + subtitles_to_mux = tracks.subtitles if not no_sync_subs else [] + for st in subtitles_to_mux: + location = st.locate() + if not location: + raise ValueError("A Text Track was not downloaded before muxing...") + + try: + if os.path.getsize(location) < 6: + continue + except Exception: + continue + + try: + with open(location, "r", encoding="utf-8", errors="ignore") as f: + head = f.read(512) + if re.match(r"CHAPTER\d+=", head.strip(), re.IGNORECASE): + continue + except Exception: + pass + + default = bool( + tracks.audios and is_close_match(st.language, [tracks.audios[0].language]) and st.forced + ) + + sub_cmd = [ + "--track-name", f"0:{st.get_track_name() or ''}", + "--language", f"0:{LANGUAGE_MUX_MAP.get(str(st.language), st.language.to_alpha3())}", + "--disable-language-ietf", + "--sub-charset", "0:UTF-8", + "--forced-track", f"0:{st.forced}", + "--default-track", f"0:{default}", + "--compression", "0:none", + ] + sub_cmd.extend(["(", location, ")"]) + cl.extend(sub_cmd) + + if tracks.chapters: + chapters_file = filenames.chapters.format(filename=title.filename) + tracks.export_chapters(chapters_file) + cl.extend(["--chapters", chapters_file]) + + log.info(f"Muxing tracks into {muxed_location.name}...") + p = subprocess.Popen(cl, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + in_progress = False + + for line in TextIOWrapper(p.stdout, encoding="utf-8"): + if re.search(r"Using the (?:demultiplexer|output module) for the format", line): + continue + if line.startswith("Progress:"): + in_progress = True + sys.stdout.write("\r" + line.rstrip('\n')) + else: + if in_progress: + in_progress = False + sys.stdout.write("\n") + sys.stdout.write(line) + + returncode = p.wait() + return str(muxed_location), returncode + + @staticmethod + def export_chapters(chapters: list, to_file: str = None) -> str: + data = "\n".join(map(repr, chapters)) + if to_file: + os.makedirs(os.path.dirname(to_file) or ".", exist_ok=True) + with open(to_file, "w", encoding="utf-8") as fd: + fd.write(data) + return data + + @staticmethod + def apply_sync(mkv_path: str): + sync_log = logging.getLogger("SyncVAT") + mkvmerge_path = shutil.which("mkvmerge") + ffprobe_path = shutil.which("ffprobe") + + if not os.path.exists(mkv_path): + sync_log.error(f"MKV file not found: {mkv_path}") + return + + sync_log.info("Waiting 2 seconds for file IO...") + time.sleep(2) + + output_path = os.path.splitext(mkv_path)[0] + ".synced.mkv" + + try: + result = subprocess.run( + [mkvmerge_path, "-J", mkv_path], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, errors="replace" + ) + mkv_info = json.loads(result.stdout) if result.returncode == 0 else {} + + video_duration = None + audio_tracks = [] + + for track in mkv_info.get("tracks", []): + track_type = track.get("type") + properties = track.get("properties", {}) + track_id = track["id"] + + duration_sec = None + if properties.get("tag_duration"): + try: + parts = properties["tag_duration"].split(":") + if len(parts) == 3: + duration_sec = float(parts[2]) + int(parts[1]) * 60 + int(parts[0]) * 3600 + except Exception: + pass + + if duration_sec is None and properties.get("duration"): + try: + duration_sec = float(properties["duration"]) / 1e9 + except Exception: + pass + + if track_type == "video": + if video_duration is None and duration_sec: + video_duration = duration_sec + elif track_type == "audio": + if duration_sec: + audio_tracks.append({"id": track_id, "duration": duration_sec}) + + if not video_duration and ffprobe_path: + sync_log.info("Video duration missing in mkvmerge, probing with ffprobe...") + try: + ff_cmd = [ + ffprobe_path, "-v", "error", + "-select_streams", "v:0", + "-show_entries", "format=duration:stream=duration", + "-of", "default=noprint_wrappers=1:nokey=1", + mkv_path + ] + ff_res = subprocess.run(ff_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + + valid_durations = [] + for line in ff_res.stdout.splitlines(): + line = line.strip() + if line and line != 'N/A': + try: + valid_durations.append(float(line)) + except ValueError: + pass + + if valid_durations: + video_duration = max(valid_durations) + except Exception as e: + sync_log.warning(f"FFprobe failed: {e}") + + if not video_duration: + sync_log.warning("Could not determine Video Duration. Skipping Sync.") + return + + sync_log.info(f"Video Duration: {video_duration:.4f}s") + + needs_sync = False + cmd = [mkvmerge_path, "-o", output_path] + + count = 0 + for audio in audio_tracks: + if audio["duration"] > video_duration + 0.1: + factor = video_duration / audio["duration"] + cmd.extend(["--sync", f"{audio['id']}:0,{factor:.9f}"]) + sync_log.info( + f"Syncing Audio {audio['id']}: {audio['duration']:.4f}s -> " + f"{video_duration:.4f}s (Factor: {factor:.6f})" + ) + needs_sync = True + count += 1 + + if not needs_sync: + sync_log.info("Audio duration matches video, no sync needed.") + return + + cmd.append(mkv_path) + + sync_log.info(f"Re-muxing to fix {count} audio tracks...") + subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + + time.sleep(1) + if os.path.exists(mkv_path): + os.unlink(mkv_path) + os.rename(output_path, mkv_path) + sync_log.info("Sync completed successfully on final file.") + + except Exception as e: + sync_log.error(f"SyncVAT Failed: {e}") + if os.path.exists(output_path): + try: + os.unlink(output_path) + except Exception: + pass \ No newline at end of file diff --git a/wpgskd/core/resolver.py b/wpgskd/core/resolver.py new file mode 100644 index 0000000..9e22452 --- /dev/null +++ b/wpgskd/core/resolver.py @@ -0,0 +1,235 @@ +import logging +import traceback +from typing import Optional, Tuple, Dict, Any +from uuid import UUID + +from wpgskd.core.cdm.loader import CdmProvider +from wpgskd.core.vaults import Vaults +from wpgskd.core.vault import InsertResult +from wpgskd.core.tracks.video import VideoTrack + +try: + from wpgskd.utils.monalisa import MonaLisa + MONALISA_AVAILABLE = True +except ImportError: + MONALISA_AVAILABLE = False + MonaLisa = None + +log = logging.getLogger("Resolver") + +class KeyResolver: + def __init__(self, vaults: Vaults, cdm_provider: CdmProvider, use_cache: bool = True, use_cdm: bool = True): + self.vaults = vaults + self.cdm_provider = cdm_provider + self.use_cache = use_cache + self.use_cdm = use_cdm + self._license_cache = {} + + def resolve(self, track: Any, title: Any, service: Any, service_name: str, session: Any = None) -> Tuple[Optional[str], Dict[str, str]]: + all_keys: Dict[str, str] = {} + + if MONALISA_AVAILABLE and getattr(track, 'monalisa', False) and getattr(track, 'key', None): + log.info(f" + KEY: {track.key} (From MonaLisa)") + if track.kid: + all_keys[self._norm(track.kid)] = track.key + return track.key, all_keys + + if self.use_cache and not getattr(track, 'key', None): + track.key, vault_used = self.vaults.get(track.kid, title.id) + if track.key: + log.debug(f" + KEY: {track.key} (From {vault_used.name} Vault)") + self._sync_vault(track.kid, track.key, title.id, service_name, vault_used) + all_keys[self._norm(track.kid)] = track.key + return track.key, all_keys + if self.use_cdm and not getattr(track, 'key', None): + try: + content_keys = self._license(track, title, service, service_name, session) + if content_keys: + all_keys.update(content_keys) + primary_key = self._match(track, content_keys, service_name) + if primary_key: + self._cache_all(content_keys, service_name, title.id) + return primary_key, all_keys + except Exception as e: + log.debug(traceback.format_exc()) + raise ValueError(f"CDM license error: {e}") + + return None, all_keys + + def _license(self, track: Any, title: Any, service: Any, service_name: str, session: Any) -> Optional[Dict[str, str]]: + cdm = self.cdm_provider.cdm_instance + + if self.cdm_provider.cdm_instance.cdm_type == "playready": + return self._pr_license(cdm, track, title, service) + + return self._wv_license(cdm, track, title, service, service_name) + + def _wv_license(self, cdm: Any, track: Any, title: Any, service: Any, service_name: str) -> Dict[str, str]: + pssh_data = getattr(track, 'pssh', None) + if not pssh_data: + raise ValueError("Track missing PSSH for Widevine CDM") + + pssh_key = str(pssh_data) + if pssh_key in self._license_cache: + return self._license_cache[pssh_key] + + sid = cdm.open() + try: + challenge = cdm.get_license_challenge(sid, pssh_data, privacy_mode=True) + license_res = service.license( + challenge=challenge, + title=title, + track=track, + session_id=sid, + drm_type="widevine" + ) + cdm.parse_license(sid, license_res) + + keys_list = cdm.get_keys(sid) + result = {} + for k in keys_list: + kid = self._norm(k.get('kid')) + key = k.get('key') + if kid and key and kid != "00" * 16: + result[kid] = key.lower() + + self._license_cache[pssh_key] = result + return result + + except Exception as e: + raise ValueError(f"Widevine license request failed: {e}") + finally: + cdm.close(sid) + + def _pr_license(self, cdm: Any, track: Any, title: Any, service: Any) -> Dict[str, str]: + pr_pssh = getattr(track, 'pr_pssh', None) + if not pr_pssh: + raise ValueError("Track missing PR_PSSH for PlayReady CDM") + + pssh_key = str(pr_pssh) + if pssh_key in self._license_cache: + return self._license_cache[pssh_key] + + try: + from pyplayready.system.pssh import PSSH as PRPSSH + wrm = PRPSSH(pr_pssh).wrm_headers[0] + except Exception: + raise ValueError("Failed to parse WRM Header from PR_PSSH") + + sid = cdm.open() + try: + challenge = cdm.get_license_challenge(sid, wrm).encode('utf-8') + license_res = service.license( + challenge=challenge, + title=title, + track=track, + session_id=sid, + drm_type="playready" + ) + + if isinstance(license_res, bytes): + license_res = license_res.decode('utf-8', errors='ignore') + + try: + cdm.parse_license(sid, license_res) + except Exception as parse_e: + log.error(f"Failed to parse PlayReady license. Response preview: {license_res[:500]}") + raise parse_e + + keys_list = cdm.get_keys(sid) + result = {} + for k in keys_list: + kid = self._norm(k.get('kid')) + key = k.get('key') + if kid and key and kid != "00" * 16: + result[kid] = key.lower() + + self._license_cache[pssh_key] = result + return result + + except Exception as e: + raise ValueError(f"PlayReady license request failed: {e}") + finally: + cdm.close(sid) + + def _match(self, track: Any, keys: Dict[str, str], service_name: str) -> Optional[str]: + target_kid = self._norm(track.kid) if track.kid else None + + filtered_keys = {k: v for k, v in keys.items() if k not in ("0" * 32, "b770d5b4bb6b594daf985845aae9aa5f")} + if not filtered_keys: + filtered_keys = keys + + if not filtered_keys: + return None + + for kid, key in filtered_keys.items(): + log.debug(f" + {kid}:{key}") + + if target_kid and target_kid in filtered_keys: + return filtered_keys[target_kid] + + if service_name == "YouTubeMovies" and isinstance(track, VideoTrack) and filtered_keys: + real_kid = next(iter(filtered_keys)) + log.info(f" + YouTube mapping: virtual {track.kid} -> real {real_kid}") + track.kid = real_kid + return filtered_keys[real_kid] + + if not target_kid: + log.warning(f" - Track has no KID, using fallback key") + else: + log.warning(f" - No exact KID match for {track.kid}") + log.warning(f" - Available: {list(filtered_keys.keys())}") + + if filtered_keys: + fallback_kid = next(iter(filtered_keys)) + log.info(f" + Using fallback key from KID: {fallback_kid[:8]}...") + if not track.kid: + track.kid = fallback_kid + return filtered_keys[fallback_kid] + + log.warning(f" - No exact KID match for {track.kid}") + log.warning(f" - Available: {list(filtered_keys.keys())}") + + if filtered_keys: + fallback_kid = next(iter(filtered_keys)) + log.info(f" + Using fallback key from KID: {fallback_kid[:8]}...") + return filtered_keys[fallback_kid] + + return None + + def _sync_vault(self, kid: str, key: str, title_id: str, service_name: str, source_vault: Any) -> None: + for v in self.vaults.vaults: + if v is source_vault: + continue + try: + res = v.insert_key(self.vaults.service, kid, key, title_id, commit=True) + if res == InsertResult.SUCCESS: + log.debug(f" + Cached to {v.name} vault") + except Exception: + pass + + def _cache_all(self, keys: Dict[str, str], service_name: str, title_id: str) -> None: + for v in self.vaults.vaults: + try: + added, existed = 0, 0 + for kid, key in keys.items(): + res = v.insert_key(self.vaults.service, kid, key, title_id, commit=False) + if res == InsertResult.SUCCESS: + added += 1 + elif res == InsertResult.ALREADY_EXISTS: + existed += 1 + v.commit() + if added > 0: + log.debug(f" + Cached {added}/{len(keys)} keys to {v.name}") + if existed > 0 and added == 0: + log.debug(f" + {existed}/{len(keys)} keys already existed in {v.name}") + except Exception: + pass + + @staticmethod + def _norm(kid: Any) -> str: + if hasattr(kid, 'hex'): + return kid.hex.lower() + if isinstance(kid, UUID): + return kid.hex.lower() + return str(kid).replace("-", "").replace("_", "").lower() \ No newline at end of file diff --git a/wpgskd/core/services.py b/wpgskd/core/services.py new file mode 100644 index 0000000..41c181a --- /dev/null +++ b/wpgskd/core/services.py @@ -0,0 +1,24 @@ +import re +from typing import List + +class Services: + ALIASES = { + "amzn": "amazon", + "atvp": "appletvplus", + "dsnp": "disneyplus", + "hmax": "hbomax", + "pmtp": "paramountplus", + "ytbe": "youtube" + } + + @classmethod + def get_tag(cls, service_name: str) -> str: + if not service_name: + return "unknown" + + tag = service_name.lower() + return cls.ALIASES.get(tag, tag) + + @classmethod + def get_tags(cls, service_names: List[str]) -> List[str]: + return [cls.get_tag(name) for name in service_names] \ No newline at end of file diff --git a/wpgskd/core/session.py b/wpgskd/core/session.py new file mode 100644 index 0000000..9762e92 --- /dev/null +++ b/wpgskd/core/session.py @@ -0,0 +1,96 @@ +import logging +from typing import Optional, Dict, Any +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +log = logging.getLogger("Session") + +class SessionBuilder: + + @staticmethod + def build( + headers: Optional[Dict[str, str]] = None, + retries: int = 5, + backoff_factor: int = 1, + status_forcelist: Optional[list] = None, + raise_on_error: bool = True + ) -> requests.Session: + if status_forcelist is None: + status_forcelist = [429, 500, 502, 503, 504] + + session = requests.Session() + + retry_strategy = Retry( + total=retries, + backoff_factor=backoff_factor, + status_forcelist=status_forcelist, + allowed_methods=["GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS"] + ) + + adapter = HTTPAdapter(max_retries=retry_strategy) + session.mount("https://", adapter) + session.mount("http://", adapter) + + if headers: + session.headers.update(headers) + + if raise_on_error: + session.hooks = { + "response": lambda r, *args, **kwargs: r.raise_for_status() + } + + return session + + @staticmethod + def build_hybrid( + headers: Optional[Dict[str, str]] = None, + impersonate: str = "chrome120", + verify: bool = False + ) -> Any: + try: + from curl_cffi import requests as curl_requests + import requests as std_requests + except ImportError: + log.warning("curl_cffi is not installed. Falling back to standard requests.") + return SessionBuilder.build(headers=headers) + + class HybridSession: + def __init__(self, hdrs, imp, vfy): + self.curl = curl_requests.Session(impersonate=imp, verify=vfy) + self.std = std_requests.Session() + + if hdrs: + self.curl.headers.update(hdrs) + self.std.headers.update(hdrs) + + self.headers = self.curl.headers + self.cookies = self.curl.cookies + self.proxies = {} + + def _sync_std_session(self): + for k, v in self.curl.headers.items(): + self.std.headers[k] = v + + def get(self, url, **kwargs): + if kwargs.get('stream'): + self._sync_std_session() + return self.std.get(url, **kwargs) + return self.curl.get(url, **kwargs) + + def post(self, url, **kwargs): + if kwargs.get('stream'): + self._sync_std_session() + return self.std.post(url, **kwargs) + return self.curl.post(url, **kwargs) + + def put(self, url, **kwargs): + return self.curl.put(url, **kwargs) + + def delete(self, url, **kwargs): + return self.curl.delete(url, **kwargs) + + def __getattr__(self, name): + return getattr(self.curl, name) + + return HybridSession(headers, impersonate, verify) \ No newline at end of file diff --git a/wpgskd/core/tracks/__init__.py b/wpgskd/core/tracks/__init__.py new file mode 100644 index 0000000..3d298fa --- /dev/null +++ b/wpgskd/core/tracks/__init__.py @@ -0,0 +1,6 @@ +from wpgskd.core.tracks.tracks import Track, TextTrack, Tracks +from wpgskd.core.tracks.video import VideoTrack +from wpgskd.core.tracks.audio import AudioTrack +from wpgskd.core.tracks.title import Title, Titles +from wpgskd.core.tracks.menu import MenuTrack +from wpgskd.core.tracks.hdgrange import DynamicRange, detect_dynamic_range \ No newline at end of file diff --git a/wpgskd/core/tracks/audio.py b/wpgskd/core/tracks/audio.py new file mode 100644 index 0000000..3e20612 --- /dev/null +++ b/wpgskd/core/tracks/audio.py @@ -0,0 +1,100 @@ +import math +from typing import Optional +from wpgskd.core.tracks.tracks import Track + +AUDIO_CODEC_MAP = { + "E-AC-3": "DD+", + "E-AC-3 JOC": "DD+ Atmos", + "AC-3": "DD", + "AAC": "AAC", + "AAC LC": "AAC", + "FLAC": "FLAC", + "Opus": "Opus", + "DTS": "DTS", + "DTS-HD": "DTS-HD", + "DTS-HD MA": "DTS-HD.MA", + "DTS XLL": "DTS-HD.MA", + "MLP FBA": "TrueHD", + "MLP FBA 16-ch": "TrueHD Atmos", +} + +class AudioTrack(Track): + def __init__(self, *args, bitrate: int, channels: Optional[str] = None, + descriptive: bool = False, atmos: bool = False, + mpd_representation_id: Optional[str] = None, **kwargs): + super().__init__(*args, **kwargs) + self.bitrate = int(math.ceil(float(bitrate))) if bitrate else None + self.channels = self.parse_channels(channels) if channels else None + self.descriptive = bool(descriptive) + self.atmos = bool(atmos) + self.mpd_representation_id = mpd_representation_id + + @staticmethod + def parse_channels(channels: str) -> str: + if channels in ["A000", "a000"]: return "2.0" + if channels in ["F801", "f801"]: return "5.1" + try: + ch = str(float(channels)) + if ch == "6.0": return "5.1" + return ch + except ValueError: + return str(channels) + + def get_codec_display(self) -> str: + if not self.codec: + return "Unknown" + + codec_str = str(self.codec) + codec_lower = codec_str.lower() + + display_name = codec_str + + if codec_str in AUDIO_CODEC_MAP: + display_name = AUDIO_CODEC_MAP[codec_str] + elif "ec-3" in codec_lower or "eac3" in codec_lower: + display_name = "DDP" + elif "ac-3" in codec_lower or "ac3" in codec_lower: + display_name = "DD" + elif "mp4a" in codec_lower or "aac" in codec_lower: + display_name = "AAC" + elif "opus" in codec_lower: + display_name = "Opus" + elif "flac" in codec_lower: + display_name = "FLAC" + elif "dts" in codec_lower: + display_name = "DTS" + + if self.atmos and "Atmos" not in display_name: + display_name += " Atmos" + + return display_name + + def get_track_name(self) -> Optional[str]: + track_name = super().get_track_name() or "" + flag = "Descriptive" if self.descriptive else "" + if flag: + if track_name: + flag = f" ({flag})" + track_name += flag + return track_name or None + + def __str__(self): + dur_sec = self.duration_seconds() + size_bytes = self.size if self.size else self.computed_size_bytes() + size_str = self.format_size_compact(size_bytes) if size_bytes else None + dur_str = self.format_hms(dur_sec) if dur_sec else None + + codec_display = self.get_codec_display() + if self.atmos and "Atmos" not in codec_display: + codec_display = f"{codec_display} Atmos" + + return " | ".join([x for x in [ + "├─ AUD", + codec_display, + f"{self.channels}" if self.channels else None, + f"{self.bitrate // 1000 if self.bitrate else '?'} kb/s", + f"{self.language}", + " ".join([self.get_track_name() or "", "[Original]" if self.is_original_lang else ""]).strip(), + size_str, + dur_str + ] if x]) \ No newline at end of file diff --git a/wpgskd/core/tracks/hdgrange.py b/wpgskd/core/tracks/hdgrange.py new file mode 100644 index 0000000..e840bdb --- /dev/null +++ b/wpgskd/core/tracks/hdgrange.py @@ -0,0 +1,22 @@ +from enum import Enum +from typing import Any + +class DynamicRange(Enum): + SDR = "SDR" + HDR10 = "HDR10" + HDR10PLUS = "HDR10+" + DV = "DV" + HLG = "HLG" + +def detect_dynamic_range(track: Any) -> DynamicRange: + if getattr(track, 'dv', False): + if getattr(track, 'hdr10', False): + if getattr(track, 'dvhdr', False): + return DynamicRange.HDR10 + return DynamicRange.DV + return DynamicRange.DV + if getattr(track, 'hdr10', False): + return DynamicRange.HDR10 + if getattr(track, 'hlg', False): + return DynamicRange.HLG + return DynamicRange.SDR \ No newline at end of file diff --git a/wpgskd/core/tracks/menu.py b/wpgskd/core/tracks/menu.py new file mode 100644 index 0000000..6b2498b --- /dev/null +++ b/wpgskd/core/tracks/menu.py @@ -0,0 +1,73 @@ +import re +from typing import Any, Optional + +class MenuTrack: + line_1 = re.compile(r"^CHAPTER(?P<number>\d+)=(?P<timecode>[\d\\.]+)$") + line_2 = re.compile(r"^CHAPTER(?P<number>\d+)NAME=(?P<title>[\d\\.]+)$") + + def __init__(self, number: int, title: str, timecode: str): + self.id = f"chapter-{number}" + self.number = number + self.title = title + if "." not in timecode: + timecode += ".000" + self.timecode = timecode + + def __bool__(self): + return bool(self.number and self.number >= 0 and self.title and self.timecode) + + def __repr__(self): + return "CHAPTER{num}={time}\nCHAPTER{num}NAME={name}".format( + num=f"{self.number:02}", time=self.timecode, name=self.title + ) + + def __str__(self): + return " | ".join([ + "├─ CHP", + f"[{self.number:02}]", + self.timecode, + self.title + ]) + + @classmethod + def loads(cls, data: str) -> 'MenuTrack': + lines = [x.strip() for x in data.strip().splitlines(keepends=False)] + if len(lines) > 2: + return MenuTrack.loads("\n".join(lines)) + one, two = lines + + one_m = cls.line_1.match(one) + two_m = cls.line_2.match(two) + if not one_m or not two_m: + raise SyntaxError(f"An unexpected syntax error near:\n{one}\n{two}") + + one_str, timecode = one_m.groups() + two_str, title = two_m.groups() + one_num, two_num = int(one_str.lstrip("0")), int(two_str.lstrip("0")) + + if one_num != two_num: + raise SyntaxError(f"The chapter numbers ({one_num},{two_num}) does not match.") + if not timecode: + raise SyntaxError("The timecode is missing.") + if not title: + raise SyntaxError("The title is missing.") + + return cls(number=one_num, title=title, timecode=timecode) + + @classmethod + def load(cls, path: str) -> 'MenuTrack': + with open(path, encoding="utf-8") as fd: + return cls.loads(fd.read()) + + def dumps(self) -> str: + return repr(self) + + def dump(self, path: str): + with open(path, "w", encoding="utf-8") as fd: + return fd.write(self.dumps()) + + @staticmethod + def format_duration(seconds: float) -> str: + minutes, seconds = divmod(seconds, 60) + hours, minutes = divmod(minutes, 60) + return f"{hours:02.0f}:{minutes:02.0f}:{seconds:06.3f}" \ No newline at end of file diff --git a/wpgskd/core/tracks/subtitles.py b/wpgskd/core/tracks/subtitles.py new file mode 100644 index 0000000..4f23cf1 --- /dev/null +++ b/wpgskd/core/tracks/subtitles.py @@ -0,0 +1,328 @@ +import os +import re +import logging +from pathlib import Path +from typing import Optional, List, Dict, Any +from io import BytesIO + +from wpgskd.vendor.pymp4.parser import Box + +log = logging.getLogger("Subtitles") + +try: + from subby import WebVTTConverter, SMPTEConverter, WVTTConverter, ISMTConverter, CommonIssuesFixer, SDHStripper + SUBBY_AVAILABLE = True +except ImportError: + SUBBY_AVAILABLE = False + log.warning("subby library not found. Subtitle conversion will be limited.") + +class SubtitleProcessor: + _SDH_PATTERNS = [ + r'\[[^\]]*\]', # [text] + r'\([^\)]*\)', # (text) + r'\{[^\}]*\}', # {text} + r'<[^>]*>', # <text> + r'♪[^♪]*♪', # ♪text♪ + r'[\*_][^\*_]+[\*_]', # *text* or _text_ + ] + + @staticmethod + def extract_mdat_text(data: bytes, codec: str) -> bytes: + codec_lower = codec.lower() + plain_text_codecs = {"vtt", "webvtt", "webvtt-lssdh-ios8", "ttml", "ttml2", "dfxp", "smpte"} + + if codec_lower in plain_text_codecs: + return data + + collected = [] + try: + for box_type, payload in SubtitleProcessor._iter_boxes(data): + if box_type != b"mdat": + continue + if codec_lower in ["wvtt", "stpp"]: + cues_data = SubtitleProcessor._extract_wvtt_text_from_mdat(payload) + if cues_data and len(cues_data) > 10: + collected.append(cues_data) + else: + clean = payload.lstrip(b"\x00").strip() + if clean and len(clean) > 10: + collected.append(clean) + else: + clean = payload.lstrip(b"\x00").strip() + if clean and len(clean) > 10: + collected.append(clean) + except Exception as e: + log.debug(f"Error extracting MDAT: {e}") + + if not collected: + return data + + result = b"\n".join(collected) + if b"WEBVTT" not in result and b"-->" in result: + lines = result.split(b'\n') + for i, line in enumerate(lines): + if b"-->" in line: + lines.insert(0, b"WEBVTT") + lines.insert(1, b"") + result = b'\n'.join(lines) + break + + return result.replace(b'\x00', b'') + + @staticmethod + def convert_to_srt(data: bytes, codec: str) -> Optional[Any]: + if not SUBBY_AVAILABLE: + return None + + codec_lower = codec.lower() + converter = None + + if codec_lower in ["dfxp", "ttml", "ttml2", "smpte"]: + converter = SMPTEConverter() + elif codec_lower in ["vtt", "webvtt", "webvtt-lssdh-ios8"]: + converter = WebVTTConverter() + elif codec_lower == "wvtt": + converter = WVTTConverter() + elif codec_lower in ("cmfc", "stpp", "dash", "ism-C", "timed text"): + converter = ISMTConverter() + else: + return None + + try: + if isinstance(data, bytes): + return converter.from_bytes(data) + return converter.from_string(data) + except Exception as e: + log.warning(f"Failed to convert {codec} to SRT: {e}") + return None + + @staticmethod + def convert_subtitle_to_srt(save_path: str, strip_sdh: bool = True) -> Optional[str]: + codec = "" + if save_path.endswith(".vtt"): codec = "vtt" + elif save_path.endswith(".ttml"): codec = "ttml" + else: codec = "unknown" + + if codec.lower() in ["ass", "ssa"]: + log.info(f" + ASS/SSA subtitle kept in original format") + return save_path + + with open(save_path, "rb") as fd: + raw = fd.read() + + if len(raw) < 10: + log.warning(f" - Subtitle file too small ({len(raw)} bytes), skipping conversion") + return save_path + + if codec.lower() in ["wvtt", "stpp"]: + log.info(f" + Extracting text from {codec.upper()} container...") + extracted = SubtitleProcessor.extract_mdat_text(raw, codec) + if extracted and len(extracted) > 10: + try: + vtt_content = extracted.decode('utf-8', errors='ignore') + srt_content = SubtitleProcessor.convert_vtt_to_srt(vtt_content, strip_sdh) + if srt_content and srt_content.strip(): + srt_path = os.path.splitext(save_path)[0] + '.srt' + with open(srt_path, 'w', encoding='utf-8') as f: + f.write(srt_content) + if os.path.exists(save_path): + os.unlink(save_path) + log.info(f" + Subtitle converted to SRT: {os.path.basename(srt_path)}") + return srt_path + except Exception as e: + log.warning(f" - Failed to decode extracted content: {e}") + + elif codec.lower() in ["vtt", "webvtt"]: + try: + if raw.startswith(b'\x00\x00\x00') or (len(raw) > 4 and raw[:4] == b'mdat'): + log.info(" + Detected binary VTT, extracting from MDAT...") + extracted = SubtitleProcessor.extract_mdat_text(raw, "vtt") + vtt_content = extracted.decode('utf-8', errors='ignore') + else: + vtt_content = raw.decode('utf-8', errors='ignore') + + srt_content = SubtitleProcessor.convert_vtt_to_srt(vtt_content, strip_sdh) + if srt_content and srt_content.strip(): + srt_path = os.path.splitext(save_path)[0] + '.srt' + with open(srt_path, 'w', encoding='utf-8') as f: + f.write(srt_content) + if os.path.exists(save_path) and save_path != srt_path: + try: os.unlink(save_path) + except: pass + log.info(f" + Subtitle converted to SRT: {os.path.basename(srt_path)}") + return srt_path + except Exception as e: + log.warning(f" - VTT conversion failed: {e}") + + return save_path + + @staticmethod + def convert_vtt_to_srt(vtt_content: str, strip_sdh: bool = True) -> str: + lines = [] + counter = 1 + vtt_content = vtt_content.replace('\r\n', '\n') + blocks = re.split(r'\n\s*\n', vtt_content) + + for block in blocks: + block_lines = block.strip().split('\n') + if not block_lines: continue + + first_line = block_lines[0].strip() if block_lines else '' + if first_line == 'WEBVTT' or first_line.startswith('WEBVTT'): continue + if first_line == 'STYLE': continue + + timestamp_line = None + text_lines = [] + settings = '' + + for line in block_lines: + line = line.strip() + if '-->' in line: + timestamp_line = line + if ' line:' in line or ' position:' in line or ' align:' in line: + settings = line[line.find('-->') + 3:].strip() + elif line and not line.isdigit() and '-->' not in line: + clean_line = re.sub(r'<(?!/?(?:i|b|u|s|ruby|rt|c))[^>]+>', '', line) + clean_line = re.sub(r' ', ' ', clean_line) + if clean_line.strip(): + if strip_sdh: + clean_line = SubtitleProcessor.strip_sdh_brackets(clean_line) + if clean_line.strip(): + text_lines.append(clean_line) + + if timestamp_line and text_lines: + ts_parts = timestamp_line.split('-->') + if len(ts_parts) == 2: + start = ts_parts[0].strip().replace('.', ',') + end = ts_parts[1].split()[0].strip().replace('.', ',') if ts_parts[1] else '' + + def normalize_timestamp(ts): + if not ts: return ts + parts = ts.replace(',', ':').split(':') + if len(parts) == 2: return f"00:{parts[0].zfill(2)}:{parts[1].zfill(2)}" + elif len(parts) == 3: return f"{parts[0].zfill(2)}:{parts[1].zfill(2)}:{parts[2].zfill(2)}" + return ts + + start = normalize_timestamp(start) + end = normalize_timestamp(end) + + if ',' in start: + main, ms = start.split(',') + ms = ms.ljust(3, '0')[:3] + start = f"{main},{ms}" + if ',' in end: + main, ms = end.split(',') + ms = ms.ljust(3, '0')[:3] + end = f"{main},{ms}" + + text = '\n'.join(text_lines) + if strip_sdh: + text = SubtitleProcessor.strip_sdh_brackets(text) + + if text.strip(): + lines.append(str(counter)) + lines.append(f"{start} --> {end}") + lines.append(text) + lines.append("") + counter += 1 + + return '\n'.join(lines) + + @staticmethod + def strip_sdh_brackets(text: str) -> str: + for pattern in SubtitleProcessor._SDH_PATTERNS: + text = re.sub(pattern, '', text) + text = re.sub(r'\s+', ' ', text) + return text.strip() + + @staticmethod + def _iter_boxes(data: bytes): + offset = 0 + while offset + 8 <= len(data): + box_size = int.from_bytes(data[offset:offset + 4], "big") + box_type = data[offset + 4:offset + 8] + if box_size < 8: break + payload = data[offset + 8: offset + box_size] + yield box_type, payload + offset += box_size + + @staticmethod + def _extract_wvtt_text_from_mdat(mdat_payload: bytes) -> bytes: + cues = [] + offset = 0 + while offset + 8 <= len(mdat_payload): + inner_size = int.from_bytes(mdat_payload[offset:offset + 4], "big") + inner_type = mdat_payload[offset + 4:offset + 8] + if inner_size < 8: break + + if inner_type == b"vttc": + inner_payload = mdat_payload[offset + 8: offset + inner_size] + cue_data = SubtitleProcessor._parse_vttc_box(inner_payload) + if cue_data: cues.append(cue_data) + offset += inner_size + + if not cues: return b"" + return SubtitleProcessor._reconstruct_vtt_from_cues(cues) + + @staticmethod + def _parse_vttc_box(data: bytes) -> Optional[Dict]: + result = {"start": None, "end": None, "text": [], "settings": ""} + offset = 0 + while offset + 8 <= len(data): + box_size = int.from_bytes(data[offset:offset + 4], "big") + box_type = data[offset + 4:offset + 8] + if box_size < 8: break + payload = data[offset + 8: offset + box_size] + + if box_type == b"payl": + text = payload.strip(b"\x00").strip() + if text: + try: + text_str = text.decode('utf-8', errors='ignore').replace('\x00', '') + if text_str.strip(): result["text"].append(text_str) + except: pass + elif box_type == b"sttg": + try: result["settings"] = payload.decode('utf-8', errors='ignore').strip() + except: pass + elif box_type == b"idnt": + try: + ident = payload.decode('utf-8', errors='ignore') + if '-->' in ident: + parts = ident.split('-->') + if len(parts) == 2: + result["start"] = parts[0].strip() + result["end"] = parts[1].strip() + except: pass + offset += box_size + return result if result["text"] else None + + @staticmethod + def _reconstruct_vtt_from_cues(cues: List[Dict]) -> bytes: + vtt_lines = ["WEBVTT", ""] + for i, cue in enumerate(cues): + if not cue["start"] or not cue["end"]: + start_sec = i * 4 + end_sec = start_sec + 4 + start = f"{start_sec // 3600:02d}:{(start_sec % 3600) // 60:02d}:{start_sec % 60:02d}.000" + end = f"{end_sec // 3600:02d}:{(end_sec % 3600) // 60:02d}:{end_sec % 60:02d}.000" + else: + start, end = cue["start"], cue["end"] + + timestamp_line = f"{start} --> {end}" + if cue["settings"]: timestamp_line += f" {cue['settings']}" + vtt_lines.append(timestamp_line) + + for text in cue["text"]: + text = re.sub(r'<c[^>]*>', '', text) + text = re.sub(r'</c>', '', text) + text = re.sub(r'<ruby>', '', text) + text = re.sub(r'</ruby>', '', text) + text = re.sub(r'<rt>', '', text) + text = re.sub(r'</rt>', '', text) + text = re.sub(r'<[0-9:]+>', '', text) + lines = text.split('\n') + for line in lines: + if line.strip(): vtt_lines.append(line.strip()) + vtt_lines.append("") + + return '\n'.join(vtt_lines).encode('utf-8') \ No newline at end of file diff --git a/wpgskd/core/tracks/title.py b/wpgskd/core/tracks/title.py new file mode 100644 index 0000000..ec5534f --- /dev/null +++ b/wpgskd/core/tracks/title.py @@ -0,0 +1,98 @@ +import re +import logging +from enum import Enum +from typing import Optional, List, Any, Iterator +from wpgskd.core.tracks.tracks import Tracks + +log = logging.getLogger("Title") + +class Title: + class Types(Enum): + MOVIE = 1 + TV = 2 + SONG = 3 + + def __init__(self, id_: Any, type_: Types, name: Optional[str] = None, year: Optional[int] = None, + season: Optional[int] = None, episode: Optional[Any] = None, episode_name: Optional[str] = None, + original_lang: Optional[str] = None, source: Optional[str] = None, + service_data: Optional[dict] = None, filename: Optional[str] = None): + + self.id = id_ + self.type = type_ + self.name = name or "" + self.year = year or 0 + self.season = season or 0 + self.episode = episode or 0 + self.episode_name = episode_name + self.original_lang = original_lang or "en" + self.source = source + self.service_data = service_data or {} + self.tracks = Tracks() + self.filename = filename or self._generate_filename() + + self.manifest_url: Optional[str] = None + self.dash_manifest_url: Optional[str] = None + self.cbr_manifest_url: Optional[str] = None + self.cvbr_manifest_url: Optional[str] = None + + def __eq__(self, other): + return isinstance(other, Title) and self.id == other.id + + def __str__(self): + if self.type == Title.Types.MOVIE: + return f"{self.name} ({self.year})" if self.year else self.name + elif self.type == Title.Types.TV: + ep_str = f"E{int(self.episode):02}" if isinstance(self.episode, int) else f"E{self.episode}" + s_str = f"S{int(self.season):02}" if isinstance(self.season, int) else f"S{self.season}" + return f"{self.name} {s_str}{ep_str}" + return self.name + + def _generate_filename(self) -> str: + if self.type == Title.Types.MOVIE: + base = self.name + if self.year: base += f" ({self.year})" + elif self.type == Title.Types.TV: + s_str = f"S{int(self.season):02}" if isinstance(self.season, int) else f"S{self.season}" + base = f"{self.name} {s_str}" + else: + base = self.name + + base = re.sub(r'[\\/:*?"<>|]', "", base) + return base.replace(" ", ".") + + def parse_filename(self, media_info=None, folder: bool = False) -> str: + if folder and self.type == Title.Types.TV: + s_str = f"S{int(self.season):02}" if isinstance(self.season, int) else f"S{self.season}" + return f"{self.name} {s_str}" + return self.filename + + +class Titles(list): + def __init__(self, *args, **kwargs): + items = args[0] if args else [] + if items and not isinstance(items, (list, tuple, set)): + items = [items] + super().__init__(items, **kwargs) + self.title_name = self[0].name if self else None + + def order(self): + self.sort(key=lambda t: int(getattr(t, 'year', 0) or 0)) + self.sort(key=lambda t: getattr(t, 'episode', 0) or 0) + self.sort(key=lambda t: int(getattr(t, 'season', 0) or 0)) + return self + + def with_wanted(self, wanted: Optional[List[str]]) -> Iterator[Title]: + for title in self: + if not wanted or (title.type == Title.Types.TV and f"{title.season}x{title.episode}" in wanted): + yield title + + def print(self): + if any(x.type == Title.Types.TV for x in self): + season_counts = {} + for x in self: + s = getattr(x, 'season', 0) + season_counts[s] = season_counts.get(s, 0) + 1 + info = ", ".join(f"S{s} ({c} eps)" for s, c in sorted(season_counts.items())) + log.info(f"Title: {self.title_name} | By Season: {info}") + else: + log.info(f"Title: {self.title_name}") \ No newline at end of file diff --git a/wpgskd/core/tracks/tracks.py b/wpgskd/core/tracks/tracks.py new file mode 100644 index 0000000..f502578 --- /dev/null +++ b/wpgskd/core/tracks/tracks.py @@ -0,0 +1,494 @@ +import logging +import os +import re +from enum import Enum +from typing import Optional, List, Any, Iterator +from langcodes import Language + +from wpgskd.utils import is_close_match, get_closest_match + +log = logging.getLogger("Tracks") + +class Track: + class Descriptor(Enum): + URL = 1 + M3U = 2 + MPD = 3 + ISM = 4 + DASH = 5 + HLS = 6 + + def __init__(self, id_: str, source: str, url: Any, codec: str, language: Any = None, + descriptor: Descriptor = Descriptor.URL, encrypted: bool = False, + pssh: Any = None, pr_pssh: Any = None, kid: str = None, key: str = None, + needs_proxy: bool = False, needs_repack: bool = False, + encryption_scheme: Any = None, **kwargs): + + self.id = id_ + self.source = source + self.url = url + self.codec = codec + self.language = Language.get(language or "und") + self.descriptor = descriptor + self.encrypted = encrypted + self.encryption_scheme = encryption_scheme + self.pssh = pssh + self.pr_pssh = pr_pssh + self.kid = kid + self.key = key + self.needs_proxy = needs_proxy + self.needs_repack = needs_repack + self.duration = kwargs.get("duration") + self.size = kwargs.get("size") + + self.is_original_lang = False + self._location: Optional[str] = None + self.extra = kwargs.get("extra", {}) + + def __repr__(self): + return f"{self.__class__.__name__}(id={self.id}, lang={self.language}, codec={self.codec})" + + def __eq__(self, other): + return isinstance(other, Track) and self.id == other.id + + def get_track_name(self) -> Optional[str]: + if self.language is None: + return None + return None + + def locate(self) -> Optional[str]: + return self._location + + def swap(self, target_path: str) -> bool: + if not os.path.exists(target_path) or not self._location: + return False + try: + os.unlink(self._location) + os.rename(target_path, self._location) + return True + except Exception: + return False + + def delete(self): + if self._location and os.path.exists(self._location): + try: + os.unlink(self._location) + except Exception: + pass + self._location = None + + def get_pssh(self, session=None) -> bool: + if self.descriptor == self.Descriptor.M3U and not getattr(self, '_sub_m3u8_parsed', False): + self._sub_m3u8_parsed = True + from wpgskd.core.manifests.m3u8 import parse_media_playlist + + data = parse_media_playlist(self.url, session) + + wv_pssh = data.get("pssh") + pr_pssh = data.get("pr_pssh") + kid = data.get("kid") + + if (not wv_pssh and not pr_pssh) and data.get("init_url"): + try: + from wpgskd.core.manifests.map_init import extract_pssh_and_kid + if not session: + session = requests.Session() + resp = session.get(data["init_url"], stream=True) + chunk = next(resp.iter_content(20000), b"") + pssh_list, kid_hex = extract_pssh_and_kid(chunk) + if pssh_list: + wv_pssh = pssh_list[0] + if kid_hex: + kid = kid_hex + except Exception: + pass + + if wv_pssh: + self.pssh = wv_pssh + if pr_pssh: + self.pr_pssh = pr_pssh + if kid and not self.kid: + self.kid = kid + + if not wv_pssh and not pr_pssh and not data.get("aes_key_uri"): + self.encrypted = False + self.encryption_scheme = None + return False + + if not self.pssh and isinstance(self.extra, dict) and self.extra.get("master_pssh"): + self.pssh = self.extra["master_pssh"] + if not self.pr_pssh and isinstance(self.extra, dict) and self.extra.get("master_pr_pssh"): + self.pr_pssh = self.extra["master_pr_pssh"] + + return bool(self.pssh or self.pr_pssh) + + def get_kid(self, session=None) -> bool: + if self.kid: + return True + return bool(self.kid) + + @staticmethod + def pt_to_sec(d): + if isinstance(d, (int, float)): + return float(d) + if not d: + return None + if d[0:2] == "P0": + d = d.replace("P0Y0M0DT", "PT") + if d[0:2] != "PT": + raise ValueError("Input data is not a valid time string.") + d = d[2:].upper() + m = re.findall(r"([\d.]+.)", d) + return sum( + float(x[0:-1]) * {"H": 60 * 60, "M": 60, "S": 1}[x[-1].upper()] + for x in m + ) + + def duration_seconds(self): + cand = getattr(self, "duration", None) + if cand is None: + return None + if isinstance(cand, (int, float)): + return float(cand) + try: + return float(cand) + except Exception: + pass + try: + return self.pt_to_sec(str(cand)) + except Exception: + return None + + def computed_size_bytes(self): + try: + bitrate = getattr(self, 'bitrate', None) + if not bitrate: + return None + dur = self.duration_seconds() + if not dur or dur <= 0: + return None + return int((float(bitrate) * float(dur)) / 8.0) + except Exception: + return None + + @staticmethod + def format_hms(seconds): + if seconds is None: + return None + try: + s = int(round(float(seconds))) + except Exception: + return None + h, rem = divmod(s, 3600) + m, s = divmod(rem, 60) + return f"{h:02}h{m:02}m{s:02}s" + + @staticmethod + def format_size_compact(num_bytes): + try: + size = float(num_bytes) + except Exception: + return "" + units = ["B", "KB", "MB", "GB", "TB"] + i = 0 + while size >= 1024 and i < len(units) - 1: + size /= 1024.0 + i += 1 + return f"{size:.2f} {units[i]}" + +class TextTrack(Track): + def __init__(self, *args, cc: bool = False, sdh: bool = False, forced: bool = False, **kwargs): + super().__init__(*args, **kwargs) + self.cc = cc + self.sdh = sdh + self.forced = forced + + def get_track_name(self) -> Optional[str]: + name = super().get_track_name() or "" + flag = "CC" if self.cc else "SDH" if self.sdh else "Forced" if self.forced else "" + if flag: + name += f" ({flag})" if name else flag + return name or None + + def convert_to_srt(self, strip_sdh: bool = True) -> Optional[str]: + from wpgskd.core.tracks.subtitles import SubtitleProcessor + if not self._location: + log.warning("Cannot convert subtitle, track not downloaded yet.") + return None + + if self.sdh and strip_sdh is None: + strip_sdh = True + + new_path = SubtitleProcessor.convert_subtitle_to_srt(self._location, strip_sdh) + if new_path and new_path != self._location: + self._location = new_path + self.codec = "srt" + return self._location + + +class Tracks: + def __init__(self, *tracks: Track): + self.videos: List[Any] = [] # VideoTrack + self.audios: List[Any] = [] # AudioTrack + self.subtitles: List[TextTrack] = [] + self.chapters: List[Any] = [] + + if tracks: + self.add(list(tracks)) + + def __iter__(self) -> Iterator[Track]: + return iter(self.videos + self.audios + self.subtitles) + + def add(self, tracks: Any, warn_only: bool = True): + if tracks is None: + return + + if isinstance(tracks, Tracks): + tracks = list(tracks) + tracks.chapters + elif isinstance(tracks, Track): + tracks = [tracks] + + existing_ids = {t.id for t in self} + + for track in tracks: + if track.id in existing_ids: + if not warn_only: + raise ValueError(f"Duplicate Track ID: {track.id}") + continue + + existing_ids.add(track.id) + + cls_name = track.__class__.__name__ + if cls_name == "VideoTrack": + self.videos.append(track) + elif cls_name == "AudioTrack": + self.audios.append(track) + elif cls_name == "TextTrack": + self.subtitles.append(track) + elif cls_name == "MenuTrack": + self.chapters.append(track) + + def sort_videos(self, by_language: Optional[List[str]] = None): + if not self.videos: return + def range_priority(x): + if getattr(x, 'dv', False): return 4 + if getattr(x, 'hdr10', False) or getattr(x, 'dvhdr', False): return 3 + if getattr(x, 'hlg', False): return 1 + return 2 # SDR + self.videos.sort(key=lambda x: (range_priority(x), float(x.bitrate or 0.0)), reverse=True) + + def sort_audios(self, by_language: Optional[List[str]] = None): + if not self.audios: return + self.audios.sort(key=lambda x: float(x.bitrate or 0.0), reverse=True) + self.audios.sort(key=lambda x: "" if x.descriptive else str(x.language)) + + if by_language: + for lang in reversed(by_language): + if str(lang) == "all": + lang = next((x.language for x in self.audios if x.is_original_lang), "") + if not lang: continue + self.audios.sort(key=lambda x: "" if is_close_match(lang, [x.language]) else str(x.language)) + + def sort_subtitles(self, by_language: Optional[List[str]] = None): + if not self.subtitles: return + self.subtitles.sort(key=lambda x: str(x.language) + ("-cc" if x.cc else "") + ("-sdh" if x.sdh else "")) + self.subtitles.sort(key=lambda x: not x.forced) + if by_language: + for lang in reversed(by_language): + if str(lang) == "all": + lang = next((x.language for x in self.subtitles if x.is_original_lang), "") + if not lang: continue + self.subtitles.sort(key=lambda x: "" if is_close_match(lang, [x.language]) else str(x.language)) + + def sort_chapters(self): + if not self.chapters: return + self.chapters.sort(key=lambda x: x.number) + + def select_videos(self, by_quality=None, by_vbitrate=None, by_range=None, one_only=True, by_worst=False, by_codec=None): + videos = self.videos + if by_quality: + q_videos = [x for x in videos if x.height == by_quality] + if not q_videos: q_videos = [x for x in videos if int(x.width * (9/16)) == by_quality] + if not q_videos and by_quality == "SD": q_videos = [x for x in videos if (x.width, x.height) < (1024, 576)] + if not q_videos and by_quality == "HD720": q_videos = [x for x in videos if (x.width, x.height) < (1482, 620)] + if not q_videos: raise ValueError(f"No {by_quality}p video track.") + videos = q_videos + + if by_vbitrate: + videos = [x for x in videos if int(x.bitrate or 0) <= int(by_vbitrate * 1001)] + if by_worst: + videos.sort(key=lambda x: float(x.bitrate or 0.0)) + if by_codec: + target = by_codec.upper() + c_videos = [] + for x in videos: + raw = (x.codec or "").lower() + if any(k in raw for k in ["hev", "hvc", "dvh"]): + std = "H265" + elif "avc" in raw: + std = "H264" + elif "av01" in raw or "dav1" in raw: + std = "AV1" + else: + std = raw.upper() + if std == target: c_videos.append(x) + if not c_videos: raise ValueError(f"No {by_codec} video tracks.") + videos = c_videos + if by_range: + target_range = by_range.upper() + if target_range == "DV+HDR": + videos = [x for x in videos if getattr(x, 'dv', False) and getattr(x, 'hdr10', False)] + elif target_range == "DV": + videos = [x for x in videos if getattr(x, 'dv', False)] + elif target_range == "HDR10": + videos = [x for x in videos if getattr(x, 'hdr10', False) and not getattr(x, 'dv', False)] + elif target_range == "HLG": + videos = [x for x in videos if getattr(x, 'hlg', False)] + elif target_range == "SDR": + videos = [x for x in videos if not x.hdr10 and not x.dv and not x.hlg and not getattr(x, 'dvhdr', False)] + else: + raise ValueError(f"Unsupported range: {by_range}") + + if not videos: raise ValueError(f"No {by_range} video track.") + + if one_only and videos: + self.videos = [videos[0]] + else: + self.videos = videos + + def select_videos_multi(self, ranges: list[str], by_quality=None, by_vbitrate=None, by_worst=False): + videos = self.videos + + for r in ranges: + r_upper = r.upper() + if r_upper == "DV": + videos = [x for x in videos if getattr(x, 'dv', False)] + elif r_upper == "HDR10": + videos = [x for x in videos if getattr(x, 'hdr10', False)] + elif r_upper == "HLG": + videos = [x for x in videos if getattr(x, 'hlg', False)] + elif r_upper == "DVHDR": + videos = [x for x in videos if getattr(x, 'dvhdr', False)] + + if not videos: + raise ValueError(f"No video tracks matching all ranges: {ranges}") + + if by_quality: + q_videos = [x for x in videos if x.height == by_quality] + if not q_videos: q_videos = [x for x in videos if int(x.width * (9/16)) == by_quality] + if not q_videos and by_quality == "SD": q_videos = [x for x in videos if (x.width, x.height) < (1024, 576)] + if not q_videos and by_quality == "HD720": q_videos = [x for x in videos if (x.width, x.height) < (1482, 620)] + if not q_videos: raise ValueError(f"No {by_quality}p video track in {ranges}.") + videos = q_videos + + if by_vbitrate: + videos = [x for x in videos if int(x.bitrate or 0) <= int(by_vbitrate * 1001)] + + if by_worst: + videos.sort(key=lambda x: float(x.bitrate or 0.0)) + else: + videos.sort(key=lambda x: float(x.bitrate or 0.0), reverse=True) + + if videos: + self.videos = [videos[0]] + else: + self.videos = videos + + def select_audios(self, by_language=None, by_bitrate=None, with_atmos=False, with_descriptive=True, by_channels=None, by_codec=None): + audios = self.audios + if not with_descriptive: + audios = [x for x in audios if not x.descriptive] + if by_codec: + target = by_codec.upper() + c_audios = [] + for x in audios: + raw = (x.codec or "").lower() + std = "EC3" if any(k in raw for k in ["ec-3", "eac3"]) else "AC3" if "ac-3" in raw else "AAC" if "aac" in raw else raw.upper() + if std == target: c_audios.append(x) + if c_audios: audios = c_audios + if with_atmos: + atmos = [x for x in audios if x.atmos] + if atmos: audios = atmos + if by_channels: + ch_audios = [x for x in audios if x.channels == by_channels] + if ch_audios: audios = ch_audios + if by_bitrate: + audios = [x for x in audios if int(x.bitrate or 0) <= int(by_bitrate * 1000)] + if by_language: + filtered = [] + for lang in by_language: + if str(lang) == "all": + filtered.extend(audios) + elif str(lang) == "orig": + orig_langs = [str(x.language).split("-")[0] for x in audios if x.is_original_lang] + if not orig_langs: + filtered.extend(audios) + else: + for x in audios: + if str(x.language).split("-")[0] in orig_langs: + filtered.append(x) + else: + base_lang = str(lang).split("-")[0] + for x in audios: + if str(x.language).split("-")[0] == base_lang: + filtered.append(x) + + seen_ids = set() + deduped = [] + for x in filtered: + if x.id not in seen_ids: + seen_ids.add(x.id) + deduped.append(x) + + best_per_lang = {} + for x in deduped: + bitrate = float(x.bitrate or 0.0) + lang_key = str(x.language).split("-")[0] + if lang_key not in best_per_lang or bitrate > best_per_lang[lang_key][1]: + best_per_lang[lang_key] = (x, bitrate) + + audios = [v[0] for v in best_per_lang.values()] + + self.audios = audios + + def select_subtitles(self, by_language=None, with_forced=None): + subs = self.subtitles + if by_language: + filtered = [] + for lang in by_language: + if str(lang) == "all": + filtered.extend(subs) + elif str(lang) == "orig": + filtered.extend([x for x in subs if x.is_original_lang]) + else: + match = get_closest_match(lang, [x.language for x in subs]) + if match: + filtered.extend([x for x in subs if x.language == match]) + + seen_ids = set() + deduped = [] + for x in filtered: + if x.id not in seen_ids: + seen_ids.add(x.id) + deduped.append(x) + subs = deduped + + if with_forced is False: + subs = [x for x in subs if not x.forced] + + self.subtitles = subs + + @staticmethod + def from_mpd(*args, **kwargs): + from wpgskd.core.manifests.dash import parse as parse_mpd + return parse_mpd(*args, **kwargs) + + @staticmethod + def from_m3u8(*args, **kwargs): + from wpgskd.core.manifests.hls import parse as parse_hls + return parse_hls(*args, **kwargs) + + @staticmethod + def from_ism(*args, **kwargs): + from wpgskd.core.manifests.ism import parse as parse_ism + return parse_ism(*args, **kwargs) \ No newline at end of file diff --git a/wpgskd/core/tracks/video.py b/wpgskd/core/tracks/video.py new file mode 100644 index 0000000..5fc9f01 --- /dev/null +++ b/wpgskd/core/tracks/video.py @@ -0,0 +1,80 @@ +import math +from typing import Optional +from wpgskd.core.tracks.tracks import Track + +VIDEO_CODEC_MAP = { + "AVC": "H.264", + "HEVC": "H.265", + "V_VC1": "VC-1", + "V_MPEGH/ISO/HEVC": "H.265", + "V_MPEG4/ISO/AVC": "H.264", + "AV1": "AV1", + "VP8": "VP8", + "VP9": "VP9", +} + +class VideoTrack(Track): + def __init__(self, *args, bitrate: int, width: int, height: int, fps: Optional[float] = None, + hdr10: bool = False, dvhdr: bool = False, hlg: bool = False, dv: bool = False, + needs_ccextractor: bool = False, mpd_representation_id: Optional[str] = None, **kwargs): + super().__init__(*args, **kwargs) + self.bitrate = int(math.ceil(float(bitrate))) if bitrate else None + self.width = int(width) + self.height = int(height) + self.fps = float(fps) if fps else None + + self.hdr10 = bool(hdr10) + self.dvhdr = bool(dvhdr) + self.hlg = bool(hlg) + self.dv = bool(dv) + + self.needs_ccextractor = needs_ccextractor + self.mpd_representation_id = mpd_representation_id + + def get_codec_display(self) -> str: + if not self.codec: + return "Unknown" + + codec_str = str(self.codec) + codec_lower = codec_str.lower() + + if codec_str in VIDEO_CODEC_MAP: + return VIDEO_CODEC_MAP[codec_str] + + if "avc" in codec_lower or "h264" in codec_lower: + return "H.264" + elif "hev" in codec_lower or "hvc" in codec_lower or "h265" in codec_lower or "dvh" in codec_lower: + return "H.265" + elif "av1" in codec_lower: + return "AV1" + elif "vp09" in codec_lower or "vp9" in codec_lower: + return "VP9" + elif "vp08" in codec_lower or "vp8" in codec_lower: + return "VP8" + elif "vc-1" in codec_lower or "vc1" in codec_lower: + return "VC-1" + return codec_str + + def __str__(self): + codec = self.get_codec_display() + range_str = "DV+HDR" if self.dvhdr else "HDR10" if self.hdr10 else "HLG" if self.hlg else "DV" if self.dv else "SDR" + fps_str = f"{self.fps:.3f} FPS" if self.fps else "Unknown FPS" + bitrate_str = f"{self.bitrate // 1000 if self.bitrate else '?'} kb/s" + enc_str = "Encrypted" if self.encrypted else "Unencrypted" + + dur_sec = self.duration_seconds() + size_bytes = self.size if self.size else self.computed_size_bytes() + size_str = self.format_size_compact(size_bytes) if size_bytes else None + dur_str = self.format_hms(dur_sec) if dur_sec else None + + return " | ".join([x for x in [ + "├─ VID", + codec, + range_str, + f"{self.width}x{self.height}", + bitrate_str, + fps_str, + enc_str, + size_str, + dur_str + ] if x]) \ No newline at end of file diff --git a/wpgskd/core/utilities.py b/wpgskd/core/utilities.py new file mode 100644 index 0000000..dd97cfe --- /dev/null +++ b/wpgskd/core/utilities.py @@ -0,0 +1,69 @@ +import re +import logging +from typing import Optional +from datetime import timedelta + +log = logging.getLogger("Utilities") + +def sanitize_filename(filename: str) -> str: + if not filename: + return "unknown" + + filename = filename.replace("/", " - ").replace("\\", " - ") + filename = filename.replace(":", " - ") + filename = re.sub(r'[\*\?\"\<\>\|]', "", filename) + filename = filename.replace("&", " and ") + filename = re.sub(r"[. ]{2,}", ".", filename) + filename = filename.strip(". ") + return filename + +def pt_to_sec(pt_str: str) -> Optional[float]: + if not pt_str: + return None + + pt_str = pt_str.strip() + if pt_str.startswith('P0Y0M0DT'): + pt_str = pt_str.replace('P0Y0M0DT', 'PT') + elif pt_str.startswith('P') and 'T' in pt_str: + pt_str = 'PT' + pt_str.split('T', 1)[1] + elif not pt_str.startswith('PT'): + return None + + match = re.match(r'PT(?:(\d+(?:\.\d+)?)H)?(?:(\d+(?:\.\d+)?)M)?(?:(\d+(?:\.\d+)?)S)?', pt_str) + if not match: + return None + + h = float(match.group(1) or 0) + m = float(match.group(2) or 0) + s = float(match.group(3) or 0) + return h * 3600 + m * 60 + s + +def format_duration(seconds: float) -> str: + if seconds is None: + return "N/A" + h = int(seconds // 3600) + m = int((seconds % 3600) // 60) + s = int(seconds % 60) + if h > 0: + return f"{h}h{m}m{s}s" + elif m > 0: + return f"{m}m{s}s" + else: + return f"{s}s" + +def get_track_size_estimate(bitrate: int, duration_sec: float) -> Optional[int]: + if not bitrate or not duration_sec or duration_sec <= 0: + return None + return int((float(bitrate) * float(duration_sec)) / 8.0) + +def humanize_size(num_bytes: int) -> str: + try: + size = float(num_bytes) + except TypeError: + return "N/A" + units = ["B", "KB", "MB", "GB", "TB"] + i = 0 + while size >= 1024 and i < len(units) - 1: + size /= 1024.0 + i += 1 + return f"{size:.2f} {units[i]}" \ No newline at end of file diff --git a/wpgskd/core/vault.py b/wpgskd/core/vault.py new file mode 100644 index 0000000..1f42d05 --- /dev/null +++ b/wpgskd/core/vault.py @@ -0,0 +1,131 @@ +import logging +import sqlite3 +import os +from abc import ABC, abstractmethod +from enum import Enum +from typing import Optional + +import requests + +from wpgskd.utils.AtomicSQL import AtomicSQL + +log = logging.getLogger("Vault") + + +class InsertResult(Enum): + FAILURE = 0 + SUCCESS = 1 + ALREADY_EXISTS = 2 + + +class BaseVault(ABC): + def __init__(self, name: str): + self.name = name + + @abstractmethod + def get_key(self, table: str, kid: str, title_id: str = "") -> Optional[str]: + pass + + @abstractmethod + def insert_key(self, table: str, kid: str, key: str, title: str = "", commit: bool = True) -> InsertResult: + pass + + def create_table(self, table: str): + pass + + def commit(self): + pass + + +class LocalVault(BaseVault): + def __init__(self, name: str, path: str, **kwargs): + super().__init__(name) + from wpgskd.config import directories + db_path = path.format(data_dir=directories.data) + os.makedirs(os.path.dirname(db_path), exist_ok=True) + + self.con = sqlite3.connect(db_path) + self.adb = AtomicSQL() + self.ticket = self.adb.load(self.con) + + def table_exists(self, table: str) -> bool: + r = self.adb.safe_execute(self.ticket, lambda db, cursor: cursor.execute( + "SELECT count(name) FROM sqlite_master WHERE type='table' AND name=?", [table] + )).fetchone() + return r[0] == 1 + + def create_table(self, table: str): + if not self.table_exists(table): + self.adb.safe_execute(self.ticket, lambda db, cursor: cursor.execute( + f"""CREATE TABLE `{table}` ( + "id" INTEGER NOT NULL UNIQUE, + "kid" TEXT NOT NULL COLLATE NOCASE, + "key_" TEXT NOT NULL COLLATE NOCASE, + "title" TEXT, + PRIMARY KEY("id" AUTOINCREMENT), + UNIQUE("kid", "key_") + );""" + )) + self.adb.commit(self.ticket) + + def get_key(self, table: str, kid: str, title_id: str = "") -> Optional[str]: + if not self.table_exists(table): + return None + r = self.adb.safe_execute(self.ticket, lambda db, cursor: cursor.execute( + f"SELECT `key_` FROM `{table}` WHERE `kid`=?", [kid] + )).fetchone() + return r[0] if r else None + + def insert_key(self, table: str, kid: str, key: str, title: str = "", commit: bool = True) -> InsertResult: + self.create_table(table) + exists = self.adb.safe_execute(self.ticket, lambda db, cursor: cursor.execute( + f"SELECT `id` FROM `{table}` WHERE `kid`=? AND `key_`=?", [kid, key] + )).fetchone() + if exists: + return InsertResult.ALREADY_EXISTS + + self.adb.safe_execute(self.ticket, lambda db, cursor: cursor.execute( + f"INSERT INTO `{table}` (kid, key_, title) VALUES (?, ?, ?)", (kid, key, title) + )) + if commit: + self.adb.commit(self.ticket) + return InsertResult.SUCCESS + + def commit(self): + self.adb.commit(self.ticket) + +class HTTPAPIVault(BaseVault): + def __init__(self, name: str, host: str, password: str, **kwargs): + super().__init__(name) + self.url = host if host.endswith('/') else host + '/' + self.password = password + + def get_key(self, table: str, kid: str, title_id: str = "") -> Optional[str]: + payload = { + "method": "GetKey", + "params": {"kid": kid, "service": table, "title": title_id}, + "token": self.password + } + try: + res = requests.post(self.url, json=payload).json() + keys = res.get("keys", []) + if keys: + return keys[0].get("key") + except Exception as e: + log.error(f"HTTPAPI Vault get failed: {e}") + return None + + def insert_key(self, table: str, kid: str, key: str, title: str = "", commit: bool = True) -> InsertResult: + payload = { + "method": "InsertKey", + "params": {"kid": kid, "key": key, "service": table, "title": title}, + "token": self.password + } + try: + res = requests.post(self.url, json=payload).json() + if res.get("inserted"): + return InsertResult.SUCCESS + return InsertResult.ALREADY_EXISTS + except Exception as e: + log.error(f"HTTPAPI Vault insert failed: {e}") + return InsertResult.FAILURE \ No newline at end of file diff --git a/wpgskd/core/vaults.py b/wpgskd/core/vaults.py new file mode 100644 index 0000000..a2479b4 --- /dev/null +++ b/wpgskd/core/vaults.py @@ -0,0 +1,72 @@ +import logging +from typing import Optional, Tuple, List, Type + +from wpgskd.core.vault import BaseVault, LocalVault, HTTPAPIVault + +log = logging.getLogger("Vaults") + +class Vaults: + VAULT_TYPES = { + "local": LocalVault, + "httpapi": HTTPAPIVault, + } + + def __init__(self, vaults_list: list, service: str): + self.vaults: List[BaseVault] = [] + self.service = service.lower() + + for v in vaults_list: + try: + if isinstance(v, BaseVault): + vault = v + else: + v_type = v.get("type", "").lower() + v_name = v.get("name", v_type) + vault_cls = self.VAULT_TYPES.get(v_type) + if not vault_cls: + log.warning(f"Unsupported vault type '{v_type}' for '{v_name}', skipping.") + continue + + cfg_copy = {k: val for k, val in v.items() if k not in ["type", "name"]} + vault = vault_cls(name=v_name, **cfg_copy) + + if isinstance(vault, LocalVault): + vault.create_table(self.service) + + self.vaults.append(vault) + except Exception as e: + v_name = v.name if isinstance(v, BaseVault) else v.get('name') + log.error(f"Failed to init vault {v_name}: {e}") + + self.vaults.sort(key=lambda v: 0 if isinstance(v, LocalVault) else 1) + + def get(self, kid: str, title_id: str = "") -> Tuple[Optional[str], Optional[BaseVault]]: + for v in self.vaults: + key = v.get_key(self.service, kid, title_id) + if key: + log.debug(f"Key {kid} found in vault {v.name}") + return key, v + return None, None + + def insert(self, kid: str, key: str, title_id: str = "") -> None: + for v in self.vaults: + try: + res = v.insert_key(self.service, kid, key, title_id) + if res.name == "SUCCESS": + log.debug(f"Inserted key to vault {v.name}") + except Exception as e: + log.warning(f"Failed to insert key to vault {v.name}: {e}") + + @staticmethod + def load_vault(vault_cfg: dict) -> BaseVault: + v_type = vault_cfg.get("type", "").lower() + v_name = vault_cfg.get("name", v_type) + no_push = vault_cfg.get("no_push", False) + + cfg_copy = {k: v for k, v in vault_cfg.items() if k not in ["type", "name", "no_push"]} + + vault_cls = Vaults.VAULT_TYPES.get(v_type) + if not vault_cls: + raise ValueError(f"Unknown vault type: {v_type}") + + return vault_cls(name=v_name, **cfg_copy) \ No newline at end of file